diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 87f7b8e8293e..61cb79645c10 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -1,684 +1,684 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif /* * To express interprocessor (as opposed to processor and device) memory * ordering constraints, use the atomic_*() functions with acquire and release * semantics rather than the *mb() functions. An architecture's memory * ordering (or memory consistency) model governs the order in which a * program's accesses to different locations may be performed by an * implementation of that architecture. In general, for memory regions * defined as writeback cacheable, the memory ordering implemented by amd64 * processors preserves the program ordering of a load followed by a load, a * load followed by a store, and a store followed by a store. Only a store * followed by a load to a different memory location may be reordered. * Therefore, except for special cases, like non-temporal memory accesses or * memory regions defined as write combining, the memory ordering effects * provided by the sfence instruction in the wmb() function and the lfence * instruction in the rmb() function are redundant. In contrast, the * atomic_*() functions with acquire and release semantics do not perform * redundant instructions for ordinary cases of interprocessor memory * ordering on any architecture. */ #define mb() __asm __volatile("mfence;" : : : "memory") #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") #ifdef _KERNEL /* * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ #define OFFSETOF_MONITORBUF 0x100 #endif #if defined(KCSAN) && !defined(KCSAN_RUNTIME) -#include +#include #else #include /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ /* * The above functions are expanded inline in the statically-linked * kernel. Lock prefixes are generated if an SMP kernel is being * built. * * Kernel modules call real functions which are built into the kernel. * This allows kernel modules to be portable between UP and SMP systems. */ #if !defined(__GNUCLIKE_ASM) #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); int atomic_fcmpset_short(volatile u_short *dst, u_short *expect, u_short src); int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandset_long(volatile u_long *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); int atomic_testandclear_long(volatile u_long *p, u_int v); void atomic_thread_fence_acq(void); void atomic_thread_fence_acq_rel(void); void atomic_thread_fence_rel(void); void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) #else /* !KLD_MODULE && __GNUCLIKE_ASM */ /* * For userland, always use lock prefixes so that the binaries will run * on both SMP and !SMP systems. */ #if defined(SMP) || !defined(_KERNEL) || defined(KLD_MODULE) #define MPLOCKED "lock ; " #else #define MPLOCKED #endif /* * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions. * * cmpset: * if (*dst == expect) * *dst = src * * fcmpset: * if (*dst == *expect) * *dst = src * else * *expect = *dst * * Returns 0 on failure, non-zero on success. */ #define ATOMIC_CMPSET(TYPE) \ static __inline int \ atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " " MPLOCKED " " \ " cmpxchg %3,%1 ; " \ "# atomic_cmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } \ \ static __inline int \ atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " " MPLOCKED " " \ " cmpxchg %3,%1 ; " \ "# atomic_fcmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (*expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } ATOMIC_CMPSET(char); ATOMIC_CMPSET(short); ATOMIC_CMPSET(int); ATOMIC_CMPSET(long); /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " " MPLOCKED " " " xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } /* * Atomically add the value of v to the long integer pointed to by p and return * the previous value of *p. */ static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { __asm __volatile( " " MPLOCKED " " " xaddq %0,%1 ; " "# atomic_fetchadd_long" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btsl %2,%1 ; " "# atomic_testandset_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btsq %2,%1 ; " "# atomic_testandset_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btrl %2,%1 ; " "# atomic_testandclear_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btrq %2,%1 ; " "# atomic_testandclear_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need a Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. To avoid false data dependencies, we use a * special address for "mem". In the kernel, we use a private per-cpu * cache line. In user space, we use a word in the stack's red zone * (-8(%rsp)). * * For UP kernels, however, the memory of the single processor is * always consistent, so we only need to stop the compiler from * reordering accesses in a way that violates the semantics of acquire * and release. */ #if defined(_KERNEL) #if defined(SMP) || defined(KLD_MODULE) static __inline void __storeload_barrier(void) { __asm __volatile("lock; addl $0,%%gs:%0" : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); } #else /* _KERNEL && UP */ static __inline void __storeload_barrier(void) { __compiler_membar(); } #endif /* SMP */ #else /* !_KERNEL */ static __inline void __storeload_barrier(void) { __asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc"); } #endif /* _KERNEL*/ #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orq %1,%0", "er", v); ATOMIC_ASM(clear, long, "andq %1,%0", "er", ~v); ATOMIC_ASM(add, long, "addq %1,%0", "er", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "er", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { __asm __volatile( " xchgq %1,%0 ; " "# atomic_swap_long" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } #else /* !__GNUCLIKE_ASM */ u_int atomic_swap_int(volatile u_int *p, u_int v); u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_cmpset_acq_char atomic_cmpset_char #define atomic_cmpset_rel_char atomic_cmpset_char #define atomic_fcmpset_acq_char atomic_fcmpset_char #define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_cmpset_acq_short atomic_cmpset_short #define atomic_cmpset_rel_short atomic_cmpset_short #define atomic_fcmpset_acq_short atomic_fcmpset_short #define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) #define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subtract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_testandset_64 atomic_testandset_long #define atomic_testandclear_64 atomic_testandclear_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ #endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/arm64/include/atomic.h b/sys/arm64/include/atomic.h index 9c5d6224f3e2..6c63357f85b9 100644 --- a/sys/arm64/include/atomic.h +++ b/sys/arm64/include/atomic.h @@ -1,609 +1,609 @@ /*- * Copyright (c) 2013 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #define isb() __asm __volatile("isb" : : : "memory") /* * Options for DMB and DSB: * oshld Outer Shareable, load * oshst Outer Shareable, store * osh Outer Shareable, all * nshld Non-shareable, load * nshst Non-shareable, store * nsh Non-shareable, all * ishld Inner Shareable, load * ishst Inner Shareable, store * ish Inner Shareable, all * ld Full system, load * st Full system, store * sy Full system, all */ #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") #define mb() dmb(sy) /* Full system memory barrier all */ #define wmb() dmb(st) /* Full system memory barrier store */ #define rmb() dmb(ld) /* Full system memory barrier load */ #if defined(KCSAN) && !defined(KCSAN_RUNTIME) -#include +#include #else #include #ifdef _KERNEL extern bool lse_supported; #ifdef LSE_ATOMICS #define _ATOMIC_LSE_SUPPORTED 1 #else #define _ATOMIC_LSE_SUPPORTED lse_supported #endif #else #define _ATOMIC_LSE_SUPPORTED 0 #endif #define _ATOMIC_OP_PROTO(t, op, bar, flav) \ static __inline void \ atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ _ATOMIC_OP_PROTO(t, op, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ \ pre; \ __asm __volatile( \ "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \ " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \ " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \ " cbnz %w1, 1b\n" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } \ \ _ATOMIC_OP_PROTO(t, op, bar, _lse) \ { \ uint##t##_t tmp; \ \ pre; \ __asm __volatile( \ ".arch_extension lse\n" \ "ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (tmp) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } \ \ _ATOMIC_OP_PROTO(t, op, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ atomic_##op##_##bar##t##_lse(p, val); \ else \ atomic_##op##_##bar##t##_llsc(p, val); \ } #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l) _ATOMIC_OP(add, add, add, ) _ATOMIC_OP(clear, bic, clr, ) _ATOMIC_OP(set, orr, set, ) _ATOMIC_OP(subtract, add, add, val = -val) #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \ static __inline int \ atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \ uint##t##_t cmpval, uint##t##_t newval) #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \ static __inline int \ atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \ uint##t##_t *cmpval, uint##t##_t newval) #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \ _ATOMIC_CMPSET_PROTO(t, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ \ __asm __volatile( \ "1: mov %w1, #1\n" \ " ld"#a"xr"#s" %"#w"0, [%2]\n" \ " cmp %"#w"0, %"#w"3\n" \ " b.ne 2f\n" \ " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ " cbnz %w1, 1b\n" \ "2:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (cmpval), "r" (newval) \ : "cc", "memory" \ ); \ \ return (!res); \ } \ \ _ATOMIC_CMPSET_PROTO(t, bar, _lse) \ { \ uint##t##_t oldval; \ int res; \ \ oldval = cmpval; \ __asm __volatile( \ ".arch_extension lse\n" \ "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ "cmp %"#w"1, %"#w"2\n" \ "cset %w0, eq\n" \ ".arch_extension nolse\n" \ : "=r" (res), "+&r" (cmpval) \ : "r" (oldval), "r" (p), "r" (newval) \ : "cc", "memory" \ ); \ \ return (res); \ } \ \ _ATOMIC_CMPSET_PROTO(t, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_cmpset_##bar##t##_lse(p, cmpval, \ newval)); \ else \ return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \ newval)); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ { \ uint##t##_t _cmpval, tmp; \ int res; \ \ _cmpval = *cmpval; \ __asm __volatile( \ " mov %w1, #1\n" \ " ld"#a"xr"#s" %"#w"0, [%2]\n" \ " cmp %"#w"0, %"#w"3\n" \ " b.ne 1f\n" \ " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ "1:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (_cmpval), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (!res); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \ { \ uint##t##_t _cmpval, tmp; \ int res; \ \ _cmpval = tmp = *cmpval; \ __asm __volatile( \ ".arch_extension lse\n" \ "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ "cmp %"#w"1, %"#w"2\n" \ "cset %w0, eq\n" \ ".arch_extension nolse\n" \ : "=r" (res), "+&r" (tmp) \ : "r" (_cmpval), "r" (p), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (res); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \ newval)); \ else \ return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \ newval)); \ } #define _ATOMIC_CMPSET(bar, a, l) \ _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \ _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \ _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \ _ATOMIC_CMPSET_IMPL(64, , , bar, a, l) #define atomic_cmpset_8 atomic_cmpset_8 #define atomic_fcmpset_8 atomic_fcmpset_8 #define atomic_cmpset_16 atomic_cmpset_16 #define atomic_fcmpset_16 atomic_fcmpset_16 _ATOMIC_CMPSET( , , ) _ATOMIC_CMPSET(acq_, a, ) _ATOMIC_CMPSET(rel_, ,l) #define _ATOMIC_FETCHADD_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_FETCHADD_IMPL(t, w) \ _ATOMIC_FETCHADD_PROTO(t, _llsc) \ { \ uint##t##_t ret, tmp; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"2, [%3]\n" \ " add %"#w"0, %"#w"2, %"#w"4\n" \ " stxr %w1, %"#w"0, [%3]\n" \ " cbnz %w1, 1b\n" \ : "=&r" (tmp), "=&r" (res), "=&r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_FETCHADD_PROTO(t, _lse) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ ".arch_extension lse\n" \ "ldadd %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_FETCHADD_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_fetchadd_##t##_lse(p, val)); \ else \ return (atomic_fetchadd_##t##_llsc(p, val)); \ } _ATOMIC_FETCHADD_IMPL(32, w) _ATOMIC_FETCHADD_IMPL(64, ) #define _ATOMIC_SWAP_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_READANDCLEAR_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_readandclear_##t##flav(volatile uint##t##_t *p) #define _ATOMIC_SWAP_IMPL(t, w, zreg) \ _ATOMIC_SWAP_PROTO(t, _llsc) \ { \ uint##t##_t ret; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"1, [%2]\n" \ " stxr %w0, %"#w"3, [%2]\n" \ " cbnz %w0, 1b\n" \ : "=&r" (res), "=&r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_SWAP_PROTO(t, _lse) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ ".arch_extension lse\n" \ "swp %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_SWAP_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_swap_##t##_lse(p, val)); \ else \ return (atomic_swap_##t##_llsc(p, val)); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ { \ uint##t##_t ret; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"1, [%2]\n" \ " stxr %w0, "#zreg", [%2]\n" \ " cbnz %w0, 1b\n" \ : "=&r" (res), "=&r" (ret) \ : "r" (p) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, _lse) \ { \ return (atomic_swap_##t##_lse(p, 0)); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_readandclear_##t##_lse(p)); \ else \ return (atomic_readandclear_##t##_llsc(p)); \ } _ATOMIC_SWAP_IMPL(32, w, wzr) _ATOMIC_SWAP_IMPL(64, , xzr) #define _ATOMIC_TEST_OP_PROTO(t, op, flav) \ static __inline int \ atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val) #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op) \ _ATOMIC_TEST_OP_PROTO(t, op, _llsc) \ { \ uint##t##_t mask, old, tmp; \ int res; \ \ mask = ((uint##t##_t)1) << (val & (t - 1)); \ __asm __volatile( \ "1: ldxr %"#w"2, [%3]\n" \ " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \ " stxr %w1, %"#w"0, [%3]\n" \ " cbnz %w1, 1b\n" \ : "=&r" (tmp), "=&r" (res), "=&r" (old) \ : "r" (p), "r" (mask) \ : "memory" \ ); \ \ return ((old & mask) != 0); \ } \ \ _ATOMIC_TEST_OP_PROTO(t, op, _lse) \ { \ uint##t##_t mask, old; \ \ mask = ((uint##t##_t)1) << (val & (t - 1)); \ __asm __volatile( \ ".arch_extension lse\n" \ "ld"#lse_asm_op" %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (old) \ : "r" (p), "r" (mask) \ : "memory" \ ); \ \ return ((old & mask) != 0); \ } \ \ _ATOMIC_TEST_OP_PROTO(t, op, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_testand##op##_##t##_lse(p, val)); \ else \ return (atomic_testand##op##_##t##_llsc(p, val)); \ } #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \ _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op) \ _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op) _ATOMIC_TEST_OP(clear, bic, clr) _ATOMIC_TEST_OP(set, orr, set) #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \ static __inline uint##t##_t \ atomic_load_acq_##t(volatile uint##t##_t *p) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ "ldar"#s" %"#w"0, [%1]\n" \ : "=&r" (ret) \ : "r" (p) \ : "memory"); \ \ return (ret); \ } #define atomic_load_acq_8 atomic_load_acq_8 #define atomic_load_acq_16 atomic_load_acq_16 _ATOMIC_LOAD_ACQ_IMPL(8, w, b) _ATOMIC_LOAD_ACQ_IMPL(16, w, h) _ATOMIC_LOAD_ACQ_IMPL(32, w, ) _ATOMIC_LOAD_ACQ_IMPL(64, , ) #define _ATOMIC_STORE_REL_IMPL(t, w, s) \ static __inline void \ atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \ { \ __asm __volatile( \ "stlr"#s" %"#w"0, [%1]\n" \ : \ : "r" (val), "r" (p) \ : "memory"); \ } _ATOMIC_STORE_REL_IMPL(8, w, b) _ATOMIC_STORE_REL_IMPL(16, w, h) _ATOMIC_STORE_REL_IMPL(32, w, ) _ATOMIC_STORE_REL_IMPL(64, , ) #define atomic_add_int atomic_add_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_swap_int atomic_swap_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_testandclear_int atomic_testandclear_32 #define atomic_testandset_int atomic_testandset_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_add_long atomic_add_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_swap_long atomic_swap_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_testandclear_long atomic_testandclear_64 #define atomic_testandset_long atomic_testandset_64 #define atomic_add_ptr atomic_add_64 #define atomic_fcmpset_ptr atomic_fcmpset_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_swap_ptr atomic_swap_64 #define atomic_subtract_ptr atomic_subtract_64 #define atomic_add_acq_long atomic_add_acq_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 #define atomic_clear_acq_ptr atomic_clear_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 static __inline void atomic_thread_fence_acq(void) { dmb(ld); } static __inline void atomic_thread_fence_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_acq_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_seq_cst(void) { dmb(sy); } #include #endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/arm64/include/bus.h b/sys/arm64/include/bus.h index 917b2618b275..417b918f595f 100644 --- a/sys/arm64/include/bus.h +++ b/sys/arm64/include/bus.h @@ -1,502 +1,502 @@ /* $NetBSD: bus.h,v 1.11 2003/07/28 17:35:54 thorpej Exp $ */ /*- * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1996 Charles M. Hannum. All rights reserved. * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Christopher G. Demetriou * for the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * From: sys/arm/include/bus.h * * $FreeBSD$ */ #ifndef _MACHINE_BUS_H_ #define _MACHINE_BUS_H_ #include #define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t) #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFFUL #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFFUL #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFUL #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFFUL #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFFUL #define BUS_SPACE_MAXSIZE_40BIT 0xFFFFFFFFFFUL #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFUL #define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFUL #define BUS_SPACE_MAP_CACHEABLE 0x01 #define BUS_SPACE_MAP_LINEAR 0x02 #define BUS_SPACE_MAP_PREFETCHABLE 0x04 #define BUS_SPACE_UNRESTRICTED (~0) #define BUS_SPACE_BARRIER_READ 0x01 #define BUS_SPACE_BARRIER_WRITE 0x02 #if defined(KCSAN) && !defined(KCSAN_RUNTIME) -#include +#include #else struct bus_space { /* cookie */ void *bs_cookie; /* mapping/unmapping */ int (*bs_map) (void *, bus_addr_t, bus_size_t, int, bus_space_handle_t *); void (*bs_unmap) (void *, bus_space_handle_t, bus_size_t); int (*bs_subregion) (void *, bus_space_handle_t, bus_size_t, bus_size_t, bus_space_handle_t *); /* allocation/deallocation */ int (*bs_alloc) (void *, bus_addr_t, bus_addr_t, bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, bus_space_handle_t *); void (*bs_free) (void *, bus_space_handle_t, bus_size_t); /* get kernel virtual address */ /* barrier */ void (*bs_barrier) (void *, bus_space_handle_t, bus_size_t, bus_size_t, int); /* read single */ u_int8_t (*bs_r_1) (void *, bus_space_handle_t, bus_size_t); u_int16_t (*bs_r_2) (void *, bus_space_handle_t, bus_size_t); u_int32_t (*bs_r_4) (void *, bus_space_handle_t, bus_size_t); u_int64_t (*bs_r_8) (void *, bus_space_handle_t, bus_size_t); /* read multiple */ void (*bs_rm_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rm_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rm_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rm_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* read region */ void (*bs_rr_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rr_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rr_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rr_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* write single */ void (*bs_w_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t); void (*bs_w_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t); void (*bs_w_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t); void (*bs_w_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t); /* write multiple */ void (*bs_wm_1) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wm_2) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wm_4) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wm_8) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* write region */ void (*bs_wr_1) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wr_2) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wr_4) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wr_8) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* set multiple */ void (*bs_sm_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t, bus_size_t); void (*bs_sm_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t, bus_size_t); void (*bs_sm_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t, bus_size_t); void (*bs_sm_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t, bus_size_t); /* set region */ void (*bs_sr_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t, bus_size_t); void (*bs_sr_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t, bus_size_t); void (*bs_sr_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t, bus_size_t); void (*bs_sr_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t, bus_size_t); /* copy */ void (*bs_c_1) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_2) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_4) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_8) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); /* read single stream */ u_int8_t (*bs_r_1_s) (void *, bus_space_handle_t, bus_size_t); u_int16_t (*bs_r_2_s) (void *, bus_space_handle_t, bus_size_t); u_int32_t (*bs_r_4_s) (void *, bus_space_handle_t, bus_size_t); u_int64_t (*bs_r_8_s) (void *, bus_space_handle_t, bus_size_t); /* read multiple stream */ void (*bs_rm_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rm_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rm_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rm_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* read region stream */ void (*bs_rr_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rr_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rr_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rr_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* write single stream */ void (*bs_w_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t); void (*bs_w_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t); void (*bs_w_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t); void (*bs_w_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t); /* write multiple stream */ void (*bs_wm_1_s) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wm_2_s) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wm_4_s) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wm_8_s) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* write region stream */ void (*bs_wr_1_s) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wr_2_s) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wr_4_s) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wr_8_s) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* peek */ int (*bs_peek_1)(void *, bus_space_handle_t, bus_size_t , uint8_t *); int (*bs_peek_2)(void *, bus_space_handle_t, bus_size_t , uint16_t *); int (*bs_peek_4)(void *, bus_space_handle_t, bus_size_t , uint32_t *); int (*bs_peek_8)(void *, bus_space_handle_t, bus_size_t , uint64_t *); /* poke */ int (*bs_poke_1)(void *, bus_space_handle_t, bus_size_t, uint8_t); int (*bs_poke_2)(void *, bus_space_handle_t, bus_size_t, uint16_t); int (*bs_poke_4)(void *, bus_space_handle_t, bus_size_t, uint32_t); int (*bs_poke_8)(void *, bus_space_handle_t, bus_size_t, uint64_t); }; /* * Utility macros; INTERNAL USE ONLY. */ #define __bs_c(a,b) __CONCAT(a,b) #define __bs_opname(op,size) __bs_c(__bs_c(__bs_c(bs_,op),_),size) #define __bs_rs(sz, t, h, o) \ (*(t)->__bs_opname(r,sz))((t)->bs_cookie, h, o) #define __bs_ws(sz, t, h, o, v) \ (*(t)->__bs_opname(w,sz))((t)->bs_cookie, h, o, v) #define __bs_nonsingle(type, sz, t, h, o, a, c) \ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c) #define __bs_set(type, sz, t, h, o, v, c) \ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, v, c) #define __bs_copy(sz, t, h1, o1, h2, o2, cnt) \ (*(t)->__bs_opname(c,sz))((t)->bs_cookie, h1, o1, h2, o2, cnt) #define __bs_opname_s(op,size) __bs_c(__bs_c(__bs_c(__bs_c(bs_,op),_),size),_s) #define __bs_rs_s(sz, t, h, o) \ (*(t)->__bs_opname_s(r,sz))((t)->bs_cookie, h, o) #define __bs_ws_s(sz, t, h, o, v) \ (*(t)->__bs_opname_s(w,sz))((t)->bs_cookie, h, o, v) #define __bs_peek(sz, t, h, o, vp) \ (*(t)->__bs_opname(peek, sz))((t)->bs_cookie, h, o, vp) #define __bs_poke(sz, t, h, o, v) \ (*(t)->__bs_opname(poke, sz))((t)->bs_cookie, h, o, v) #define __bs_nonsingle_s(type, sz, t, h, o, a, c) \ (*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c) /* * Mapping and unmapping operations. */ #define bus_space_map(t, a, s, c, hp) \ (*(t)->bs_map)((t)->bs_cookie, (a), (s), (c), (hp)) #define bus_space_unmap(t, h, s) \ (*(t)->bs_unmap)((t)->bs_cookie, (h), (s)) #define bus_space_subregion(t, h, o, s, hp) \ (*(t)->bs_subregion)((t)->bs_cookie, (h), (o), (s), (hp)) /* * Allocation and deallocation operations. */ #define bus_space_alloc(t, rs, re, s, a, b, c, ap, hp) \ (*(t)->bs_alloc)((t)->bs_cookie, (rs), (re), (s), (a), (b), \ (c), (ap), (hp)) #define bus_space_free(t, h, s) \ (*(t)->bs_free)((t)->bs_cookie, (h), (s)) /* * Bus barrier operations. */ #define bus_space_barrier(t, h, o, l, f) \ (*(t)->bs_barrier)((t)->bs_cookie, (h), (o), (l), (f)) /* * Bus read (single) operations. */ #define bus_space_read_1(t, h, o) __bs_rs(1,(t),(h),(o)) #define bus_space_read_2(t, h, o) __bs_rs(2,(t),(h),(o)) #define bus_space_read_4(t, h, o) __bs_rs(4,(t),(h),(o)) #define bus_space_read_8(t, h, o) __bs_rs(8,(t),(h),(o)) #define bus_space_read_stream_1(t, h, o) __bs_rs_s(1,(t), (h), (o)) #define bus_space_read_stream_2(t, h, o) __bs_rs_s(2,(t), (h), (o)) #define bus_space_read_stream_4(t, h, o) __bs_rs_s(4,(t), (h), (o)) #define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,(t), (h), (o)) /* * Bus read multiple operations. */ #define bus_space_read_multi_1(t, h, o, a, c) \ __bs_nonsingle(rm,1,(t),(h),(o),(a),(c)) #define bus_space_read_multi_2(t, h, o, a, c) \ __bs_nonsingle(rm,2,(t),(h),(o),(a),(c)) #define bus_space_read_multi_4(t, h, o, a, c) \ __bs_nonsingle(rm,4,(t),(h),(o),(a),(c)) #define bus_space_read_multi_8(t, h, o, a, c) \ __bs_nonsingle(rm,8,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(rm,1,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(rm,2,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(rm,4,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(rm,8,(t),(h),(o),(a),(c)) /* * Bus read region operations. */ #define bus_space_read_region_1(t, h, o, a, c) \ __bs_nonsingle(rr,1,(t),(h),(o),(a),(c)) #define bus_space_read_region_2(t, h, o, a, c) \ __bs_nonsingle(rr,2,(t),(h),(o),(a),(c)) #define bus_space_read_region_4(t, h, o, a, c) \ __bs_nonsingle(rr,4,(t),(h),(o),(a),(c)) #define bus_space_read_region_8(t, h, o, a, c) \ __bs_nonsingle(rr,8,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(rr,1,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(rr,2,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(rr,4,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(rr,8,(t),(h),(o),(a),(c)) /* * Bus write (single) operations. */ #define bus_space_write_1(t, h, o, v) __bs_ws(1,(t),(h),(o),(v)) #define bus_space_write_2(t, h, o, v) __bs_ws(2,(t),(h),(o),(v)) #define bus_space_write_4(t, h, o, v) __bs_ws(4,(t),(h),(o),(v)) #define bus_space_write_8(t, h, o, v) __bs_ws(8,(t),(h),(o),(v)) #define bus_space_write_stream_1(t, h, o, v) __bs_ws_s(1,(t),(h),(o),(v)) #define bus_space_write_stream_2(t, h, o, v) __bs_ws_s(2,(t),(h),(o),(v)) #define bus_space_write_stream_4(t, h, o, v) __bs_ws_s(4,(t),(h),(o),(v)) #define bus_space_write_stream_8(t, h, o, v) __bs_ws_s(8,(t),(h),(o),(v)) /* * Bus write multiple operations. */ #define bus_space_write_multi_1(t, h, o, a, c) \ __bs_nonsingle(wm,1,(t),(h),(o),(a),(c)) #define bus_space_write_multi_2(t, h, o, a, c) \ __bs_nonsingle(wm,2,(t),(h),(o),(a),(c)) #define bus_space_write_multi_4(t, h, o, a, c) \ __bs_nonsingle(wm,4,(t),(h),(o),(a),(c)) #define bus_space_write_multi_8(t, h, o, a, c) \ __bs_nonsingle(wm,8,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(wm,1,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(wm,2,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(wm,4,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(wm,8,(t),(h),(o),(a),(c)) /* * Bus write region operations. */ #define bus_space_write_region_1(t, h, o, a, c) \ __bs_nonsingle(wr,1,(t),(h),(o),(a),(c)) #define bus_space_write_region_2(t, h, o, a, c) \ __bs_nonsingle(wr,2,(t),(h),(o),(a),(c)) #define bus_space_write_region_4(t, h, o, a, c) \ __bs_nonsingle(wr,4,(t),(h),(o),(a),(c)) #define bus_space_write_region_8(t, h, o, a, c) \ __bs_nonsingle(wr,8,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(wr,1,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(wr,2,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(wr,4,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(wr,8,(t),(h),(o),(a),(c)) /* * Set multiple operations. */ #define bus_space_set_multi_1(t, h, o, v, c) \ __bs_set(sm,1,(t),(h),(o),(v),(c)) #define bus_space_set_multi_2(t, h, o, v, c) \ __bs_set(sm,2,(t),(h),(o),(v),(c)) #define bus_space_set_multi_4(t, h, o, v, c) \ __bs_set(sm,4,(t),(h),(o),(v),(c)) #define bus_space_set_multi_8(t, h, o, v, c) \ __bs_set(sm,8,(t),(h),(o),(v),(c)) /* * Set region operations. */ #define bus_space_set_region_1(t, h, o, v, c) \ __bs_set(sr,1,(t),(h),(o),(v),(c)) #define bus_space_set_region_2(t, h, o, v, c) \ __bs_set(sr,2,(t),(h),(o),(v),(c)) #define bus_space_set_region_4(t, h, o, v, c) \ __bs_set(sr,4,(t),(h),(o),(v),(c)) #define bus_space_set_region_8(t, h, o, v, c) \ __bs_set(sr,8,(t),(h),(o),(v),(c)) /* * Copy operations. */ #define bus_space_copy_region_1(t, h1, o1, h2, o2, c) \ __bs_copy(1, t, h1, o1, h2, o2, c) #define bus_space_copy_region_2(t, h1, o1, h2, o2, c) \ __bs_copy(2, t, h1, o1, h2, o2, c) #define bus_space_copy_region_4(t, h1, o1, h2, o2, c) \ __bs_copy(4, t, h1, o1, h2, o2, c) #define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \ __bs_copy(8, t, h1, o1, h2, o2, c) /* * Poke (checked write) operations. */ #define bus_space_poke_1(t, h, o, v) __bs_poke(1, (t), (h), (o), (v)) #define bus_space_poke_2(t, h, o, v) __bs_poke(2, (t), (h), (o), (v)) #define bus_space_poke_4(t, h, o, v) __bs_poke(4, (t), (h), (o), (v)) #define bus_space_poke_8(t, h, o, v) __bs_poke(8, (t), (h), (o), (v)) /* * Peek (checked read) operations. */ #define bus_space_peek_1(t, h, o, vp) __bs_peek(1, (t), (h), (o), (vp)) #define bus_space_peek_2(t, h, o, vp) __bs_peek(2, (t), (h), (o), (vp)) #define bus_space_peek_4(t, h, o, vp) __bs_peek(4, (t), (h), (o), (vp)) #define bus_space_peek_8(t, h, o, vp) __bs_peek(8, (t), (h), (o), (vp)) #endif #include #endif /* _MACHINE_BUS_H_ */ diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c index 3fa59ef2ea66..ec2fd23729b2 100644 --- a/sys/kern/subr_csan.c +++ b/sys/kern/subr_csan.c @@ -1,910 +1,910 @@ /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */ /* * Copyright (c) 2019 The NetBSD Foundation, Inc. * All rights reserved. * Copyright (c) 2019 Andrew Turner * * This code is derived from software contributed to The NetBSD Foundation * by Maxime Villard. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #define KCSAN_RUNTIME #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef KCSAN_PANIC #define REPORT panic #else #define REPORT printf #endif typedef struct { uintptr_t addr; uint32_t size; bool write:1; bool atomic:1; uintptr_t pc; } csan_cell_t; typedef struct { bool inited; uint32_t cnt; csan_cell_t cell; } csan_cpu_t; static csan_cpu_t kcsan_cpus[MAXCPU]; static bool kcsan_enabled __read_mostly; #define __RET_ADDR (uintptr_t)__builtin_return_address(0) #define KCSAN_NACCESSES 1024 #define KCSAN_DELAY 10 /* 10 microseconds */ /* -------------------------------------------------------------------------- */ /* The MD code. */ #include /* -------------------------------------------------------------------------- */ static void kcsan_enable(void *dummy __unused) { printf("Enabling KCSCAN, expect reduced performance.\n"); kcsan_enabled = true; } SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); void kcsan_cpu_init(u_int cpu) { kcsan_cpus[cpu].inited = true; } /* -------------------------------------------------------------------------- */ static inline void kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) { const char *newsym, *oldsym; #ifdef DDB c_db_sym_t sym; db_expr_t offset; sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &newsym, NULL); sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &oldsym, NULL); #else newsym = ""; oldsym = ""; #endif REPORT("CSan: Racy Access " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", newcpu, (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), (void *)new->addr, new->size, (void *)new->pc, newsym, oldcpu, (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), (void *)old->addr, old->size, (void *)old->pc, oldsym); kcsan_md_unwind(); } static inline bool kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) { if (new->write && !new->atomic) return false; if (old->write && !old->atomic) return false; return true; } static inline void kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) { csan_cell_t old, new; csan_cpu_t *cpu; uint64_t intr; size_t i; if (__predict_false(!kcsan_enabled)) return; if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr))) return; if (KERNEL_PANICKED()) return; new.addr = addr; new.size = size; new.write = write; new.atomic = atomic; new.pc = pc; CPU_FOREACH(i) { __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); if (old.addr + old.size <= new.addr) continue; if (new.addr + new.size <= old.addr) continue; if (__predict_true(!old.write && !new.write)) continue; if (__predict_true(kcsan_access_is_atomic(&new, &old))) continue; kcsan_report(&new, PCPU_GET(cpuid), &old, i); break; } if (__predict_false(!kcsan_md_is_avail())) return; kcsan_md_disable_intrs(&intr); cpu = &kcsan_cpus[PCPU_GET(cpuid)]; if (__predict_false(!cpu->inited)) goto out; cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; if (__predict_true(cpu->cnt != 0)) goto out; __builtin_memcpy(&cpu->cell, &new, sizeof(new)); kcsan_md_delay(KCSAN_DELAY); __builtin_memset(&cpu->cell, 0, sizeof(new)); out: kcsan_md_enable_intrs(&intr); } #define CSAN_READ(size) \ void __tsan_read##size(uintptr_t); \ void __tsan_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } \ void __tsan_unaligned_read##size(uintptr_t); \ void __tsan_unaligned_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } CSAN_READ(1) CSAN_READ(2) CSAN_READ(4) CSAN_READ(8) CSAN_READ(16) #define CSAN_WRITE(size) \ void __tsan_write##size(uintptr_t); \ void __tsan_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } \ void __tsan_unaligned_write##size(uintptr_t); \ void __tsan_unaligned_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } CSAN_WRITE(1) CSAN_WRITE(2) CSAN_WRITE(4) CSAN_WRITE(8) CSAN_WRITE(16) void __tsan_read_range(uintptr_t, size_t); void __tsan_write_range(uintptr_t, size_t); void __tsan_read_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, false, false, __RET_ADDR); } void __tsan_write_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, true, false, __RET_ADDR); } void __tsan_init(void); void __tsan_func_entry(void *); void __tsan_func_exit(void); void __tsan_init(void) { } void __tsan_func_entry(void *call_pc) { } void __tsan_func_exit(void) { } /* -------------------------------------------------------------------------- */ void * kcsan_memcpy(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memcpy(dst, src, len); } int kcsan_memcmp(const void *b1, const void *b2, size_t len) { kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); return __builtin_memcmp(b1, b2, len); } void * kcsan_memset(void *b, int c, size_t len) { kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); return __builtin_memset(b, c, len); } void * kcsan_memmove(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memmove(dst, src, len); } char * kcsan_strcpy(char *dst, const char *src) { char *save = dst; while (1) { kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); *dst = *src; if (*src == '\0') break; src++, dst++; } return save; } int kcsan_strcmp(const char *s1, const char *s2) { while (1) { kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); if (*s1 != *s2) break; if (*s1 == '\0') return 0; s1++, s2++; } return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } size_t kcsan_strlen(const char *str) { const char *s; s = str; while (1) { kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); if (*s == '\0') break; s++; } return (s - str); } #undef copyin #undef copyin_nofault #undef copyinstr #undef copyout #undef copyout_nofault int kcsan_copyin(const void *uaddr, void *kaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyin(uaddr, kaddr, len); } int kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyinstr(uaddr, kaddr, len, done); } int kcsan_copyout(const void *kaddr, void *uaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); return copyout(kaddr, uaddr, len); } /* -------------------------------------------------------------------------- */ #include -#include +#include #define _CSAN_ATOMIC_FUNC_ADD(name, type) \ void kcsan_atomic_add_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ type kcsan_atomic_load_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } #define CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kcsan_atomic_readandclear_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } #define _CSAN_ATOMIC_FUNC_SET(name, type) \ void kcsan_atomic_set_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _CSAN_ATOMIC_FUNC_STORE(name, type) \ void kcsan_atomic_store_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) #define CSAN_ATOMIC_FUNC_SWAP(name, type) \ type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_swap_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_testandclear_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, val)); \ } CSAN_ATOMIC_FUNC_ADD(8, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) CSAN_ATOMIC_FUNC_SET(8, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) _CSAN_ATOMIC_FUNC_STORE(8, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(16, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) CSAN_ATOMIC_FUNC_SET(16, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) _CSAN_ATOMIC_FUNC_STORE(16, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(32, uint32_t) CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_SET(32, uint32_t) CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) CSAN_ATOMIC_FUNC_STORE(32, uint32_t) CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) #endif CSAN_ATOMIC_FUNC_ADD(64, uint64_t) CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_SET(64, uint64_t) CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) CSAN_ATOMIC_FUNC_STORE(64, uint64_t) CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) #endif CSAN_ATOMIC_FUNC_ADD(char, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_LOAD(char, uint8_t) CSAN_ATOMIC_FUNC_SET(char, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t) _CSAN_ATOMIC_FUNC_STORE(char, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_SWAP(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(short, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_LOAD(short, uint16_t) CSAN_ATOMIC_FUNC_SET(short, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t) _CSAN_ATOMIC_FUNC_STORE(short, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_SWAP(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(int, u_int) CSAN_ATOMIC_FUNC_CLEAR(int, u_int) CSAN_ATOMIC_FUNC_CMPSET(int, u_int) CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) CSAN_ATOMIC_FUNC_LOAD(int, u_int) CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_SET(int, u_int) CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) CSAN_ATOMIC_FUNC_STORE(int, u_int) CSAN_ATOMIC_FUNC_SWAP(int, u_int) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) #endif CSAN_ATOMIC_FUNC_ADD(long, u_long) CSAN_ATOMIC_FUNC_CLEAR(long, u_long) CSAN_ATOMIC_FUNC_CMPSET(long, u_long) CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) CSAN_ATOMIC_FUNC_LOAD(long, u_long) CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_SET(long, u_long) CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) CSAN_ATOMIC_FUNC_STORE(long, u_long) CSAN_ATOMIC_FUNC_SWAP(long, u_long) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) #endif CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) #if !defined(__amd64__) CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) #endif CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) #if 0 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) #endif #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kcsan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) /* -------------------------------------------------------------------------- */ #include #include -#include +#include int kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } #if !defined(__amd64__) int kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *handlep) { return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, boundary, flags, addrp, handlep)); } #endif void kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #define CSAN_BUS_READ_FUNC(func, width, type) \ type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ return (bus_space_read##func##_##width(tag, hnd, \ offset)); \ } \ #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ false, false, __RET_ADDR); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_READ_FUNC(, 1, uint8_t) CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_READ_FUNC(, 2, uint16_t) CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_READ_FUNC(, 4, uint32_t) CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_READ_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_WRITE_FUNC(func, width, type) \ void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value); \ } \ #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ true, false, __RET_ADDR); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_WRITE_FUNC(, 1, uint8_t) CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_WRITE_FUNC(, 2, uint16_t) CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_WRITE_FUNC(, 4, uint32_t) CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_WRITE_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_SET_FUNC(func, width, type) \ void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } CSAN_BUS_SET_FUNC(multi, 1, uint8_t) CSAN_BUS_SET_FUNC(region, 1, uint8_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) #endif CSAN_BUS_SET_FUNC(multi, 2, uint16_t) CSAN_BUS_SET_FUNC(region, 2, uint16_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) #endif CSAN_BUS_SET_FUNC(multi, 4, uint32_t) CSAN_BUS_SET_FUNC(region, 4, uint32_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #endif #if !defined(__amd64__) CSAN_BUS_SET_FUNC(multi, 8, uint64_t) CSAN_BUS_SET_FUNC(region, 8, uint64_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) #endif #endif #define CSAN_BUS_PEEK_FUNC(width, type) \ int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ kcsan_access((uintptr_t)value, sizeof(type), true, false, \ __RET_ADDR); \ return (bus_space_peek_##width(tag, hnd, offset, value)); \ } CSAN_BUS_PEEK_FUNC(1, uint8_t) CSAN_BUS_PEEK_FUNC(2, uint16_t) CSAN_BUS_PEEK_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_PEEK_FUNC(8, uint64_t) #endif #define CSAN_BUS_POKE_FUNC(width, type) \ int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, value)); \ } CSAN_BUS_POKE_FUNC(1, uint8_t) CSAN_BUS_POKE_FUNC(2, uint16_t) CSAN_BUS_POKE_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_POKE_FUNC(8, uint64_t) #endif diff --git a/sys/sys/_cscan_atomic.h b/sys/sys/atomic_san.h similarity index 99% rename from sys/sys/_cscan_atomic.h rename to sys/sys/atomic_san.h index b458c24841bf..c2b963ae8b92 100644 --- a/sys/sys/_cscan_atomic.h +++ b/sys/sys/atomic_san.h @@ -1,380 +1,380 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Andrew Turner * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#ifndef _SYS__CSAN_ATOMIC_H_ -#define _SYS__CSAN_ATOMIC_H_ +#ifndef _SYS_ATOMIC_SAN_H_ +#define _SYS_ATOMIC_SAN_H_ #ifndef _MACHINE_ATOMIC_H_ #error do not include this header, use machine/atomic.h #endif #define KCSAN_ATOMIC_FUNC_1(op, name, type) \ void kcsan_atomic_##op##_##name(volatile type *, type); \ void kcsan_atomic_##op##_acq_##name(volatile type *, type); \ void kcsan_atomic_##op##_rel_##name(volatile type *, type) #define KCSAN_ATOMIC_CMPSET(name, type) \ int kcsan_atomic_cmpset_##name(volatile type *, type, type); \ int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \ int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type) #define KCSAN_ATOMIC_FCMPSET(name, type) \ int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \ int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \ int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type) #define KCSAN_ATOMIC_READ(op, name, type) \ type kcsan_atomic_##op##_##name(volatile type *, type) #define KCSAN_ATOMIC_READANDCLEAR(name, type) \ type kcsan_atomic_readandclear_##name(volatile type *) #define KCSAN_ATOMIC_LOAD(name, type) \ type kcsan_atomic_load_##name(volatile type *); \ type kcsan_atomic_load_acq_##name(volatile type *) #define KCSAN_ATOMIC_STORE(name, type) \ void kcsan_atomic_store_##name(volatile type *, type); \ void kcsan_atomic_store_rel_##name(volatile type *, type) #define KCSAN_ATOMIC_TEST(op, name, type) \ int kcsan_atomic_##op##_##name(volatile type *, u_int); \ int kcsan_atomic_##op##_acq_##name(volatile type *, u_int) #define KCSAN_ATOMIC_FUNCS(name, type) \ KCSAN_ATOMIC_FUNC_1(add, name, type); \ KCSAN_ATOMIC_FUNC_1(clear, name, type); \ KCSAN_ATOMIC_CMPSET(name, type); \ KCSAN_ATOMIC_FCMPSET(name, type); \ KCSAN_ATOMIC_READ(fetchadd, name, type); \ KCSAN_ATOMIC_LOAD(name, type); \ KCSAN_ATOMIC_READANDCLEAR(name, type); \ KCSAN_ATOMIC_FUNC_1(set, name, type); \ KCSAN_ATOMIC_FUNC_1(subtract, name, type); \ KCSAN_ATOMIC_STORE(name, type); \ KCSAN_ATOMIC_READ(swap, name, type); \ KCSAN_ATOMIC_TEST(testandclear, name, type); \ KCSAN_ATOMIC_TEST(testandset, name, type) KCSAN_ATOMIC_FUNCS(char, uint8_t); KCSAN_ATOMIC_FUNCS(short, uint16_t); KCSAN_ATOMIC_FUNCS(int, u_int); KCSAN_ATOMIC_FUNCS(long, u_long); KCSAN_ATOMIC_FUNCS(ptr, uintptr_t); KCSAN_ATOMIC_FUNCS(8, uint8_t); KCSAN_ATOMIC_FUNCS(16, uint16_t); KCSAN_ATOMIC_FUNCS(32, uint32_t); KCSAN_ATOMIC_FUNCS(64, uint64_t); void kcsan_atomic_thread_fence_acq(void); void kcsan_atomic_thread_fence_acq_rel(void); void kcsan_atomic_thread_fence_rel(void); void kcsan_atomic_thread_fence_seq_cst(void); #ifndef KCSAN_RUNTIME #define atomic_add_char kcsan_atomic_add_char #define atomic_add_acq_char kcsan_atomic_add_acq_char #define atomic_add_rel_char kcsan_atomic_add_rel_char #define atomic_clear_char kcsan_atomic_clear_char #define atomic_clear_acq_char kcsan_atomic_clear_acq_char #define atomic_clear_rel_char kcsan_atomic_clear_rel_char #define atomic_cmpset_char kcsan_atomic_cmpset_char #define atomic_cmpset_acq_char kcsan_atomic_cmpset_acq_char #define atomic_cmpset_rel_char kcsan_atomic_cmpset_rel_char #define atomic_fcmpset_char kcsan_atomic_fcmpset_char #define atomic_fcmpset_acq_char kcsan_atomic_fcmpset_acq_char #define atomic_fcmpset_rel_char kcsan_atomic_fcmpset_rel_char #define atomic_fetchadd_char kcsan_atomic_fetchadd_char #define atomic_load_char kcsan_atomic_load_char #define atomic_load_acq_char kcsan_atomic_load_acq_char #define atomic_readandclear_char kcsan_atomic_readandclear_char #define atomic_set_char kcsan_atomic_set_char #define atomic_set_acq_char kcsan_atomic_set_acq_char #define atomic_set_rel_char kcsan_atomic_set_rel_char #define atomic_subtract_char kcsan_atomic_subtract_char #define atomic_subtract_acq_char kcsan_atomic_subtract_acq_char #define atomic_subtract_rel_char kcsan_atomic_subtract_rel_char #define atomic_store_char kcsan_atomic_store_char #define atomic_store_rel_char kcsan_atomic_store_rel_char #define atomic_swap_char kcsan_atomic_swap_char #define atomic_testandclear_char kcsan_atomic_testandclear_char #define atomic_testandset_char kcsan_atomic_testandset_char #define atomic_add_short kcsan_atomic_add_short #define atomic_add_acq_short kcsan_atomic_add_acq_short #define atomic_add_rel_short kcsan_atomic_add_rel_short #define atomic_clear_short kcsan_atomic_clear_short #define atomic_clear_acq_short kcsan_atomic_clear_acq_short #define atomic_clear_rel_short kcsan_atomic_clear_rel_short #define atomic_cmpset_short kcsan_atomic_cmpset_short #define atomic_cmpset_acq_short kcsan_atomic_cmpset_acq_short #define atomic_cmpset_rel_short kcsan_atomic_cmpset_rel_short #define atomic_fcmpset_short kcsan_atomic_fcmpset_short #define atomic_fcmpset_acq_short kcsan_atomic_fcmpset_acq_short #define atomic_fcmpset_rel_short kcsan_atomic_fcmpset_rel_short #define atomic_fetchadd_short kcsan_atomic_fetchadd_short #define atomic_load_short kcsan_atomic_load_short #define atomic_load_acq_short kcsan_atomic_load_acq_short #define atomic_readandclear_short kcsan_atomic_readandclear_short #define atomic_set_short kcsan_atomic_set_short #define atomic_set_acq_short kcsan_atomic_set_acq_short #define atomic_set_rel_short kcsan_atomic_set_rel_short #define atomic_subtract_short kcsan_atomic_subtract_short #define atomic_subtract_acq_short kcsan_atomic_subtract_acq_short #define atomic_subtract_rel_short kcsan_atomic_subtract_rel_short #define atomic_store_short kcsan_atomic_store_short #define atomic_store_rel_short kcsan_atomic_store_rel_short #define atomic_swap_short kcsan_atomic_swap_short #define atomic_testandclear_short kcsan_atomic_testandclear_short #define atomic_testandset_short kcsan_atomic_testandset_short #define atomic_add_int kcsan_atomic_add_int #define atomic_add_acq_int kcsan_atomic_add_acq_int #define atomic_add_rel_int kcsan_atomic_add_rel_int #define atomic_clear_int kcsan_atomic_clear_int #define atomic_clear_acq_int kcsan_atomic_clear_acq_int #define atomic_clear_rel_int kcsan_atomic_clear_rel_int #define atomic_cmpset_int kcsan_atomic_cmpset_int #define atomic_cmpset_acq_int kcsan_atomic_cmpset_acq_int #define atomic_cmpset_rel_int kcsan_atomic_cmpset_rel_int #define atomic_fcmpset_int kcsan_atomic_fcmpset_int #define atomic_fcmpset_acq_int kcsan_atomic_fcmpset_acq_int #define atomic_fcmpset_rel_int kcsan_atomic_fcmpset_rel_int #define atomic_fetchadd_int kcsan_atomic_fetchadd_int #define atomic_load_int kcsan_atomic_load_int #define atomic_load_acq_int kcsan_atomic_load_acq_int #define atomic_readandclear_int kcsan_atomic_readandclear_int #define atomic_set_int kcsan_atomic_set_int #define atomic_set_acq_int kcsan_atomic_set_acq_int #define atomic_set_rel_int kcsan_atomic_set_rel_int #define atomic_subtract_int kcsan_atomic_subtract_int #define atomic_subtract_acq_int kcsan_atomic_subtract_acq_int #define atomic_subtract_rel_int kcsan_atomic_subtract_rel_int #define atomic_store_int kcsan_atomic_store_int #define atomic_store_rel_int kcsan_atomic_store_rel_int #define atomic_swap_int kcsan_atomic_swap_int #define atomic_testandclear_int kcsan_atomic_testandclear_int #define atomic_testandset_int kcsan_atomic_testandset_int #define atomic_add_long kcsan_atomic_add_long #define atomic_add_acq_long kcsan_atomic_add_acq_long #define atomic_add_rel_long kcsan_atomic_add_rel_long #define atomic_clear_long kcsan_atomic_clear_long #define atomic_clear_acq_long kcsan_atomic_clear_acq_long #define atomic_clear_rel_long kcsan_atomic_clear_rel_long #define atomic_cmpset_long kcsan_atomic_cmpset_long #define atomic_cmpset_acq_long kcsan_atomic_cmpset_acq_long #define atomic_cmpset_rel_long kcsan_atomic_cmpset_rel_long #define atomic_fcmpset_long kcsan_atomic_fcmpset_long #define atomic_fcmpset_acq_long kcsan_atomic_fcmpset_acq_long #define atomic_fcmpset_rel_long kcsan_atomic_fcmpset_rel_long #define atomic_fetchadd_long kcsan_atomic_fetchadd_long #define atomic_load_long kcsan_atomic_load_long #define atomic_load_acq_long kcsan_atomic_load_acq_long #define atomic_readandclear_long kcsan_atomic_readandclear_long #define atomic_set_long kcsan_atomic_set_long #define atomic_set_acq_long kcsan_atomic_set_acq_long #define atomic_set_rel_long kcsan_atomic_set_rel_long #define atomic_subtract_long kcsan_atomic_subtract_long #define atomic_subtract_acq_long kcsan_atomic_subtract_acq_long #define atomic_subtract_rel_long kcsan_atomic_subtract_rel_long #define atomic_store_long kcsan_atomic_store_long #define atomic_store_rel_long kcsan_atomic_store_rel_long #define atomic_swap_long kcsan_atomic_swap_long #define atomic_testandclear_long kcsan_atomic_testandclear_long #define atomic_testandset_long kcsan_atomic_testandset_long #define atomic_testandset_acq_long kcsan_atomic_testandset_acq_long #define atomic_add_ptr kcsan_atomic_add_ptr #define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr #define atomic_add_rel_ptr kcsan_atomic_add_rel_ptr #define atomic_clear_ptr kcsan_atomic_clear_ptr #define atomic_clear_acq_ptr kcsan_atomic_clear_acq_ptr #define atomic_clear_rel_ptr kcsan_atomic_clear_rel_ptr #define atomic_cmpset_ptr kcsan_atomic_cmpset_ptr #define atomic_cmpset_acq_ptr kcsan_atomic_cmpset_acq_ptr #define atomic_cmpset_rel_ptr kcsan_atomic_cmpset_rel_ptr #define atomic_fcmpset_ptr kcsan_atomic_fcmpset_ptr #define atomic_fcmpset_acq_ptr kcsan_atomic_fcmpset_acq_ptr #define atomic_fcmpset_rel_ptr kcsan_atomic_fcmpset_rel_ptr #define atomic_fetchadd_ptr kcsan_atomic_fetchadd_ptr #define atomic_load_ptr(x) ({ \ __typeof(*x) __retptr; \ __retptr = (void *)kcsan_atomic_load_ptr((volatile uintptr_t *)(x)); \ __retptr; \ }) #define atomic_load_acq_ptr kcsan_atomic_load_acq_ptr #define atomic_load_consume_ptr(x) ({ \ __typeof(*x) __retptr; \ __retptr = (void *)kcsan_atomic_load_acq_ptr((volatile uintptr_t *)(x));\ __retptr; \ }) #define atomic_readandclear_ptr kcsan_atomic_readandclear_ptr #define atomic_set_ptr kcsan_atomic_set_ptr #define atomic_set_acq_ptr kcsan_atomic_set_acq_ptr #define atomic_set_rel_ptr kcsan_atomic_set_rel_ptr #define atomic_subtract_ptr kcsan_atomic_subtract_ptr #define atomic_subtract_acq_ptr kcsan_atomic_subtract_acq_ptr #define atomic_subtract_rel_ptr kcsan_atomic_subtract_rel_ptr #define atomic_store_ptr(x, v) ({ \ __typeof(*x) __value = (v); \ kcsan_atomic_store_ptr((volatile uintptr_t *)(x), (uintptr_t)(__value));\ }) #define atomic_store_rel_ptr kcsan_atomic_store_rel_ptr #define atomic_swap_ptr kcsan_atomic_swap_ptr #define atomic_testandclear_ptr kcsan_atomic_testandclear_ptr #define atomic_testandset_ptr kcsan_atomic_testandset_ptr #define atomic_add_8 kcsan_atomic_add_8 #define atomic_add_acq_8 kcsan_atomic_add_acq_8 #define atomic_add_rel_8 kcsan_atomic_add_rel_8 #define atomic_clear_8 kcsan_atomic_clear_8 #define atomic_clear_acq_8 kcsan_atomic_clear_acq_8 #define atomic_clear_rel_8 kcsan_atomic_clear_rel_8 #define atomic_cmpset_8 kcsan_atomic_cmpset_8 #define atomic_cmpset_acq_8 kcsan_atomic_cmpset_acq_8 #define atomic_cmpset_rel_8 kcsan_atomic_cmpset_rel_8 #define atomic_fcmpset_8 kcsan_atomic_fcmpset_8 #define atomic_fcmpset_acq_8 kcsan_atomic_fcmpset_acq_8 #define atomic_fcmpset_rel_8 kcsan_atomic_fcmpset_rel_8 #define atomic_fetchadd_8 kcsan_atomic_fetchadd_8 #define atomic_load_8 kcsan_atomic_load_8 #define atomic_load_acq_8 kcsan_atomic_load_acq_8 #define atomic_readandclear_8 kcsan_atomic_readandclear_8 #define atomic_set_8 kcsan_atomic_set_8 #define atomic_set_acq_8 kcsan_atomic_set_acq_8 #define atomic_set_rel_8 kcsan_atomic_set_rel_8 #define atomic_subtract_8 kcsan_atomic_subtract_8 #define atomic_subtract_acq_8 kcsan_atomic_subtract_acq_8 #define atomic_subtract_rel_8 kcsan_atomic_subtract_rel_8 #define atomic_store_8 kcsan_atomic_store_8 #define atomic_store_rel_8 kcsan_atomic_store_rel_8 #define atomic_swap_8 kcsan_atomic_swap_8 #define atomic_testandclear_8 kcsan_atomic_testandclear_8 #define atomic_testandset_8 kcsan_atomic_testandset_8 #define atomic_add_16 kcsan_atomic_add_16 #define atomic_add_acq_16 kcsan_atomic_add_acq_16 #define atomic_add_rel_16 kcsan_atomic_add_rel_16 #define atomic_clear_16 kcsan_atomic_clear_16 #define atomic_clear_acq_16 kcsan_atomic_clear_acq_16 #define atomic_clear_rel_16 kcsan_atomic_clear_rel_16 #define atomic_cmpset_16 kcsan_atomic_cmpset_16 #define atomic_cmpset_acq_16 kcsan_atomic_cmpset_acq_16 #define atomic_cmpset_rel_16 kcsan_atomic_cmpset_rel_16 #define atomic_fcmpset_16 kcsan_atomic_fcmpset_16 #define atomic_fcmpset_acq_16 kcsan_atomic_fcmpset_acq_16 #define atomic_fcmpset_rel_16 kcsan_atomic_fcmpset_rel_16 #define atomic_fetchadd_16 kcsan_atomic_fetchadd_16 #define atomic_load_16 kcsan_atomic_load_16 #define atomic_load_acq_16 kcsan_atomic_load_acq_16 #define atomic_readandclear_16 kcsan_atomic_readandclear_16 #define atomic_set_16 kcsan_atomic_set_16 #define atomic_set_acq_16 kcsan_atomic_set_acq_16 #define atomic_set_rel_16 kcsan_atomic_set_rel_16 #define atomic_subtract_16 kcsan_atomic_subtract_16 #define atomic_subtract_acq_16 kcsan_atomic_subtract_acq_16 #define atomic_subtract_rel_16 kcsan_atomic_subtract_rel_16 #define atomic_store_16 kcsan_atomic_store_16 #define atomic_store_rel_16 kcsan_atomic_store_rel_16 #define atomic_swap_16 kcsan_atomic_swap_16 #define atomic_testandclear_16 kcsan_atomic_testandclear_16 #define atomic_testandset_16 kcsan_atomic_testandset_16 #define atomic_add_32 kcsan_atomic_add_32 #define atomic_add_acq_32 kcsan_atomic_add_acq_32 #define atomic_add_rel_32 kcsan_atomic_add_rel_32 #define atomic_clear_32 kcsan_atomic_clear_32 #define atomic_clear_acq_32 kcsan_atomic_clear_acq_32 #define atomic_clear_rel_32 kcsan_atomic_clear_rel_32 #define atomic_cmpset_32 kcsan_atomic_cmpset_32 #define atomic_cmpset_acq_32 kcsan_atomic_cmpset_acq_32 #define atomic_cmpset_rel_32 kcsan_atomic_cmpset_rel_32 #define atomic_fcmpset_32 kcsan_atomic_fcmpset_32 #define atomic_fcmpset_acq_32 kcsan_atomic_fcmpset_acq_32 #define atomic_fcmpset_rel_32 kcsan_atomic_fcmpset_rel_32 #define atomic_fetchadd_32 kcsan_atomic_fetchadd_32 #define atomic_load_32 kcsan_atomic_load_32 #define atomic_load_acq_32 kcsan_atomic_load_acq_32 #define atomic_readandclear_32 kcsan_atomic_readandclear_32 #define atomic_set_32 kcsan_atomic_set_32 #define atomic_set_acq_32 kcsan_atomic_set_acq_32 #define atomic_set_rel_32 kcsan_atomic_set_rel_32 #define atomic_subtract_32 kcsan_atomic_subtract_32 #define atomic_subtract_acq_32 kcsan_atomic_subtract_acq_32 #define atomic_subtract_rel_32 kcsan_atomic_subtract_rel_32 #define atomic_store_32 kcsan_atomic_store_32 #define atomic_store_rel_32 kcsan_atomic_store_rel_32 #define atomic_swap_32 kcsan_atomic_swap_32 #define atomic_testandclear_32 kcsan_atomic_testandclear_32 #define atomic_testandset_32 kcsan_atomic_testandset_32 #define atomic_add_64 kcsan_atomic_add_64 #define atomic_add_acq_64 kcsan_atomic_add_acq_64 #define atomic_add_rel_64 kcsan_atomic_add_rel_64 #define atomic_clear_64 kcsan_atomic_clear_64 #define atomic_clear_acq_64 kcsan_atomic_clear_acq_64 #define atomic_clear_rel_64 kcsan_atomic_clear_rel_64 #define atomic_cmpset_64 kcsan_atomic_cmpset_64 #define atomic_cmpset_acq_64 kcsan_atomic_cmpset_acq_64 #define atomic_cmpset_rel_64 kcsan_atomic_cmpset_rel_64 #define atomic_fcmpset_64 kcsan_atomic_fcmpset_64 #define atomic_fcmpset_acq_64 kcsan_atomic_fcmpset_acq_64 #define atomic_fcmpset_rel_64 kcsan_atomic_fcmpset_rel_64 #define atomic_fetchadd_64 kcsan_atomic_fetchadd_64 #define atomic_load_64 kcsan_atomic_load_64 #define atomic_load_acq_64 kcsan_atomic_load_acq_64 #define atomic_readandclear_64 kcsan_atomic_readandclear_64 #define atomic_set_64 kcsan_atomic_set_64 #define atomic_set_acq_64 kcsan_atomic_set_acq_64 #define atomic_set_rel_64 kcsan_atomic_set_rel_64 #define atomic_subtract_64 kcsan_atomic_subtract_64 #define atomic_subtract_acq_64 kcsan_atomic_subtract_acq_64 #define atomic_subtract_rel_64 kcsan_atomic_subtract_rel_64 #define atomic_store_64 kcsan_atomic_store_64 #define atomic_store_rel_64 kcsan_atomic_store_rel_64 #define atomic_swap_64 kcsan_atomic_swap_64 #define atomic_testandclear_64 kcsan_atomic_testandclear_64 #define atomic_testandset_64 kcsan_atomic_testandset_64 #define atomic_thread_fence_acq kcsan_atomic_thread_fence_acq #define atomic_thread_fence_acq_rel kcsan_atomic_thread_fence_acq_rel #define atomic_thread_fence_rel kcsan_atomic_thread_fence_rel #define atomic_thread_fence_seq_cst kcsan_atomic_thread_fence_seq_cst #define atomic_interrupt_fence __compiler_membar #endif /* !KCSAN_RUNTIME */ -#endif /* !_SYS__CSAN_ATOMIC_H_ */ +#endif /* !_SYS_ATOMIC_SAN_H_ */ diff --git a/sys/sys/_cscan_bus.h b/sys/sys/bus_san.h similarity index 99% rename from sys/sys/_cscan_bus.h rename to sys/sys/bus_san.h index b1fd4135261e..96b2441fbdbe 100644 --- a/sys/sys/_cscan_bus.h +++ b/sys/sys/bus_san.h @@ -1,209 +1,209 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Andrew Turner * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#ifndef _SYS__CSAN_BUS_H_ -#define _SYS__CSAN_BUS_H_ +#ifndef _SYS_BUS_SAN_H_ +#define _SYS_BUS_SAN_H_ #define KCSAN_BS_MULTI(rw, width, type) \ void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t) #define KCSAN_BS_READ(width, type) \ type kcsan_bus_space_read_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t); \ type kcsan_bus_space_read_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t); \ KCSAN_BS_MULTI(read, width, type) #define KCSAN_BS_WRITE(width, type) \ void kcsan_bus_space_write_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); \ void kcsan_bus_space_write_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); \ KCSAN_BS_MULTI(write, width, const type) #define KCSAN_BS_SET(width, type) \ void kcsan_bus_space_set_multi_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ void kcsan_bus_space_set_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t) #define KCSAN_BS_COPY(width, type) \ void kcsan_bus_space_copy_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, bus_space_handle_t, \ bus_size_t, bus_size_t); \ void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, bus_space_handle_t, \ bus_size_t, bus_size_t); #define KCSAN_BS_PEEK(width, type) \ int kcsan_bus_space_peek_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *); #define KCSAN_BS_POKE(width, type) \ int kcsan_bus_space_poke_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); #define KCSAN_BS(width, type) \ KCSAN_BS_READ(width, type); \ KCSAN_BS_WRITE(width, type); \ KCSAN_BS_SET(width, type); \ KCSAN_BS_COPY(width, type) \ KCSAN_BS_PEEK(width, type); \ KCSAN_BS_POKE(width, type); KCSAN_BS(1, uint8_t); KCSAN_BS(2, uint16_t); KCSAN_BS(4, uint32_t); KCSAN_BS(8, uint64_t); int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int, bus_space_handle_t *); void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t); int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t, bus_size_t, bus_space_handle_t *); int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t, bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, bus_space_handle_t *); void kcsan_bus_space_free(bus_space_tag_t, bus_space_handle_t, bus_size_t); void kcsan_bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t, bus_size_t, int); #ifndef KCSAN_RUNTIME #define bus_space_map kcsan_bus_space_map #define bus_space_unmap kcsan_bus_space_unmap #define bus_space_subregion kcsan_bus_space_subregion #define bus_space_alloc kcsan_bus_space_alloc #define bus_space_free kcsan_bus_space_free #define bus_space_barrier kcsan_bus_space_barrier #define bus_space_read_1 kcsan_bus_space_read_1 #define bus_space_read_stream_1 kcsan_bus_space_read_stream_1 #define bus_space_read_multi_1 kcsan_bus_space_read_multi_1 #define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1 #define bus_space_read_region_1 kcsan_bus_space_read_region_1 #define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1 #define bus_space_write_1 kcsan_bus_space_write_1 #define bus_space_write_stream_1 kcsan_bus_space_write_stream_1 #define bus_space_write_multi_1 kcsan_bus_space_write_multi_1 #define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1 #define bus_space_write_region_1 kcsan_bus_space_write_region_1 #define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1 #define bus_space_set_multi_1 kcsan_bus_space_set_multi_1 #define bus_space_set_multi_stream_1 kcsan_bus_space_set_multi_stream_1 #define bus_space_set_region_1 kcsan_bus_space_set_region_1 #define bus_space_set_region_stream_1 kcsan_bus_space_set_region_stream_1 #define bus_space_copy_multi_1 kcsan_bus_space_copy_multi_1 #define bus_space_copy_multi_stream_1 kcsan_bus_space_copy_multi_stream_1 #define bus_space_poke_1 kcsan_bus_space_poke_1 #define bus_space_peek_1 kcsan_bus_space_peek_1 #define bus_space_read_2 kcsan_bus_space_read_2 #define bus_space_read_stream_2 kcsan_bus_space_read_stream_2 #define bus_space_read_multi_2 kcsan_bus_space_read_multi_2 #define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2 #define bus_space_read_region_2 kcsan_bus_space_read_region_2 #define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2 #define bus_space_write_2 kcsan_bus_space_write_2 #define bus_space_write_stream_2 kcsan_bus_space_write_stream_2 #define bus_space_write_multi_2 kcsan_bus_space_write_multi_2 #define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2 #define bus_space_write_region_2 kcsan_bus_space_write_region_2 #define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2 #define bus_space_set_multi_2 kcsan_bus_space_set_multi_2 #define bus_space_set_multi_stream_2 kcsan_bus_space_set_multi_stream_2 #define bus_space_set_region_2 kcsan_bus_space_set_region_2 #define bus_space_set_region_stream_2 kcsan_bus_space_set_region_stream_2 #define bus_space_copy_multi_2 kcsan_bus_space_copy_multi_2 #define bus_space_copy_multi_stream_2 kcsan_bus_space_copy_multi_stream_2 #define bus_space_poke_2 kcsan_bus_space_poke_2 #define bus_space_peek_2 kcsan_bus_space_peek_2 #define bus_space_read_4 kcsan_bus_space_read_4 #define bus_space_read_stream_4 kcsan_bus_space_read_stream_4 #define bus_space_read_multi_4 kcsan_bus_space_read_multi_4 #define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4 #define bus_space_read_region_4 kcsan_bus_space_read_region_4 #define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4 #define bus_space_write_4 kcsan_bus_space_write_4 #define bus_space_write_stream_4 kcsan_bus_space_write_stream_4 #define bus_space_write_multi_4 kcsan_bus_space_write_multi_4 #define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4 #define bus_space_write_region_4 kcsan_bus_space_write_region_4 #define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4 #define bus_space_set_multi_4 kcsan_bus_space_set_multi_4 #define bus_space_set_multi_stream_4 kcsan_bus_space_set_multi_stream_4 #define bus_space_set_region_4 kcsan_bus_space_set_region_4 #define bus_space_set_region_stream_4 kcsan_bus_space_set_region_stream_4 #define bus_space_copy_multi_4 kcsan_bus_space_copy_multi_4 #define bus_space_copy_multi_stream_4 kcsan_bus_space_copy_multi_stream_4 #define bus_space_poke_4 kcsan_bus_space_poke_4 #define bus_space_peek_4 kcsan_bus_space_peek_4 #define bus_space_read_8 kcsan_bus_space_read_8 #define bus_space_read_stream_8 kcsan_bus_space_read_stream_8 #define bus_space_read_multi_8 kcsan_bus_space_read_multi_8 #define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8 #define bus_space_read_region_8 kcsan_bus_space_read_region_8 #define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8 #define bus_space_write_8 kcsan_bus_space_write_8 #define bus_space_write_stream_8 kcsan_bus_space_write_stream_8 #define bus_space_write_multi_8 kcsan_bus_space_write_multi_8 #define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8 #define bus_space_write_region_8 kcsan_bus_space_write_region_8 #define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8 #define bus_space_set_multi_8 kcsan_bus_space_set_multi_8 #define bus_space_set_multi_stream_8 kcsan_bus_space_set_multi_stream_8 #define bus_space_set_region_8 kcsan_bus_space_set_region_8 #define bus_space_set_region_stream_8 kcsan_bus_space_set_region_stream_8 #define bus_space_copy_multi_8 kcsan_bus_space_copy_multi_8 #define bus_space_copy_multi_stream_8 kcsan_bus_space_copy_multi_stream_8 #define bus_space_poke_8 kcsan_bus_space_poke_8 #define bus_space_peek_8 kcsan_bus_space_peek_8 #endif /* !KCSAN_RUNTIME */ -#endif /* !_SYS__CSAN_BUS_H_ */ +#endif /* !_SYS_BUS_SAN_H_ */ diff --git a/sys/x86/include/bus.h b/sys/x86/include/bus.h index 562445b81203..da2fe237ae99 100644 --- a/sys/x86/include/bus.h +++ b/sys/x86/include/bus.h @@ -1,1126 +1,1126 @@ /*- * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause-NetBSDE * * Copyright (c) KATO Takenori, 1999. * * All rights reserved. Unpublished rights reserved under the copyright * laws of Japan. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer as * the first lines of this file unmodified. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /* $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $ */ /*- * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1996 Charles M. Hannum. All rights reserved. * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Christopher G. Demetriou * for the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _X86_BUS_H_ #define _X86_BUS_H_ #include #include #include #ifndef __GNUCLIKE_ASM #error "no assembler code for your compiler" #endif /* * Values for the x86 bus space tag, not to be used directly by MI code. */ #define X86_BUS_SPACE_IO 0 /* space is i/o space */ #define X86_BUS_SPACE_MEM 1 /* space is mem space */ #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF #if defined(__amd64__) #define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFULL #else #define BUS_SPACE_MAXSIZE 0xFFFFFFFF #endif #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF #if defined(__amd64__) || defined(PAE) #define BUS_SPACE_MAXADDR_48BIT 0xFFFFFFFFFFFFULL #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL #else #define BUS_SPACE_MAXADDR 0xFFFFFFFF #endif #define BUS_SPACE_INVALID_DATA (~0) #define BUS_SPACE_UNRESTRICTED (~0) #define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ #define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ #if defined(KCSAN) && !defined(KCSAN_RUNTIME) -#include +#include #else /* * Map a region of device bus space into CPU virtual address space. */ int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, int flags, bus_space_handle_t *bshp); /* * Unmap a region of device bus space. */ void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size); /* * Get a new handle for a subregion of an already-mapped area of bus space. */ static __inline int bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp); static __inline int bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size __unused, bus_space_handle_t *nbshp) { *nbshp = bsh + offset; return (0); } /* * Allocate a region of memory that is accessible to devices in bus space. */ int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend, bus_size_t size, bus_size_t align, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *bshp); /* * Free a region of bus space accessible memory. */ static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size); static __inline void bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused, bus_size_t size __unused) { } /* * Read a 1, 2, 4, or 8 byte quantity from bus space * described by tag/handle/offset. */ static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); #ifdef __amd64__ static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); #endif static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inb(handle + offset)); return (*(volatile u_int8_t *)(handle + offset)); } static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inw(handle + offset)); return (*(volatile u_int16_t *)(handle + offset)); } static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inl(handle + offset)); return (*(volatile u_int32_t *)(handle + offset)); } #ifdef __amd64__ static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ return (BUS_SPACE_INVALID_DATA); return (*(volatile uint64_t *)(handle + offset)); } #endif /* * Read `count' 1, 2, 4, or 8 byte quantities from bus space * described by tag/handle/offset and copy into buffer provided. */ static __inline void bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count); static __inline void bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count); static __inline void bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count); static __inline void bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insb(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movb (%2),%%al \n\ stosb \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } static __inline void bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insw(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movw (%2),%%ax \n\ stosw \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } static __inline void bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insl(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movl (%2),%%eax \n\ stosl \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } #if 0 /* Cause a link error for bus_space_read_multi_8 */ #define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!! #endif /* * Read `count' 1, 2, 4, or 8 byte quantities from bus space * described by tag/handle and starting at `offset' and copy into * buffer provided. */ static __inline void bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count); static __inline void bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count); static __inline void bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count); static __inline void bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inb %w2,%%al \n\ stosb \n\ incl %2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsb" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } static __inline void bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inw %w2,%%ax \n\ stosw \n\ addl $2,%2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsw" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } static __inline void bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inl %w2,%%eax \n\ stosl \n\ addl $4,%2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsl" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_read_region_8 */ #define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!! #endif /* * Write the 1, 2, 4, or 8 byte value `value' to bus space * described by tag/handle/offset. */ static __inline void bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value); static __inline void bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value); static __inline void bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value); #ifdef __amd64__ static __inline void bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, uint64_t value); #endif static __inline void bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value) { if (tag == X86_BUS_SPACE_IO) outb(bsh + offset, value); else *(volatile u_int8_t *)(bsh + offset) = value; } static __inline void bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value) { if (tag == X86_BUS_SPACE_IO) outw(bsh + offset, value); else *(volatile u_int16_t *)(bsh + offset) = value; } static __inline void bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value) { if (tag == X86_BUS_SPACE_IO) outl(bsh + offset, value); else *(volatile u_int32_t *)(bsh + offset) = value; } #ifdef __amd64__ static __inline void bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, uint64_t value) { if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ return; else *(volatile uint64_t *)(bsh + offset) = value; } #endif /* * Write `count' 1, 2, 4, or 8 byte quantities from the buffer * provided to bus space described by tag/handle/offset. */ static __inline void bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count); static __inline void bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count); static __inline void bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count); static __inline void bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsb(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsb \n\ movb %%al,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } static __inline void bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsw(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsw \n\ movw %%ax,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } static __inline void bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsl(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsl \n\ movl %%eax,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_write_multi_8 */ #define bus_space_write_multi_8(t, h, o, a, c) \ !!! bus_space_write_multi_8 unimplemented !!! #endif /* * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided * to bus space described by tag/handle starting at `offset'. */ static __inline void bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count); static __inline void bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count); static __inline void bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count); static __inline void bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsb \n\ outb %%al,%w0 \n\ incl %0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsb" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } static __inline void bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsw \n\ outw %%ax,%w0 \n\ addl $2,%0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsw" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } static __inline void bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsl \n\ outl %%eax,%w0 \n\ addl $4,%0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsl" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_write_region_8 */ #define bus_space_write_region_8 \ !!! bus_space_write_region_8 unimplemented !!! #endif /* * Write the 1, 2, 4, or 8 byte value `val' to bus space described * by tag/handle/offset `count' times. */ static __inline void bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count); static __inline void bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count); static __inline void bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count); static __inline void bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outb(addr, value); else while (count--) *(volatile u_int8_t *)(addr) = value; } static __inline void bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outw(addr, value); else while (count--) *(volatile u_int16_t *)(addr) = value; } static __inline void bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outl(addr, value); else while (count--) *(volatile u_int32_t *)(addr) = value; } #if 0 /* Cause a link error for bus_space_set_multi_8 */ #define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!! #endif /* * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described * by tag/handle starting at `offset'. */ static __inline void bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count); static __inline void bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count); static __inline void bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count); static __inline void bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr++) outb(addr, value); else for (; count != 0; count--, addr++) *(volatile u_int8_t *)(addr) = value; } static __inline void bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr += 2) outw(addr, value); else for (; count != 0; count--, addr += 2) *(volatile u_int16_t *)(addr) = value; } static __inline void bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr += 4) outl(addr, value); else for (; count != 0; count--, addr += 4) *(volatile u_int32_t *)(addr) = value; } #if 0 /* Cause a link error for bus_space_set_region_8 */ #define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!! #endif /* * Copy `count' 1, 2, 4, or 8 byte values from bus space starting * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2. */ static __inline void bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1++, addr2++) outb(addr2, inb(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += (count - 1), addr2 += (count - 1); count != 0; count--, addr1--, addr2--) outb(addr2, inb(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1++, addr2++) *(volatile u_int8_t *)(addr2) = *(volatile u_int8_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += (count - 1), addr2 += (count - 1); count != 0; count--, addr1--, addr2--) *(volatile u_int8_t *)(addr2) = *(volatile u_int8_t *)(addr1); } } } static __inline void bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 2, addr2 += 2) outw(addr2, inw(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); count != 0; count--, addr1 -= 2, addr2 -= 2) outw(addr2, inw(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 2, addr2 += 2) *(volatile u_int16_t *)(addr2) = *(volatile u_int16_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); count != 0; count--, addr1 -= 2, addr2 -= 2) *(volatile u_int16_t *)(addr2) = *(volatile u_int16_t *)(addr1); } } } static __inline void bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 4, addr2 += 4) outl(addr2, inl(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); count != 0; count--, addr1 -= 4, addr2 -= 4) outl(addr2, inl(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 4, addr2 += 4) *(volatile u_int32_t *)(addr2) = *(volatile u_int32_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); count != 0; count--, addr1 -= 4, addr2 -= 4) *(volatile u_int32_t *)(addr2) = *(volatile u_int32_t *)(addr1); } } } #if 0 /* Cause a link error for bus_space_copy_8 */ #define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!! #endif /* * Bus read/write barrier methods. * * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, * bus_size_t offset, bus_size_t len, int flags); * * * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than * prevent reordering by the compiler; all Intel x86 processors currently * retire operations outside the CPU in program order. */ static __inline void bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, bus_size_t offset __unused, bus_size_t len __unused, int flags) { #ifdef __GNUCLIKE_ASM if (flags & BUS_SPACE_BARRIER_READ) #ifdef __amd64__ __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory"); #else __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory"); #endif else __compiler_membar(); #endif } #ifdef BUS_SPACE_NO_LEGACY #undef inb #undef outb #define inb(a) compiler_error #define inw(a) compiler_error #define inl(a) compiler_error #define outb(a, b) compiler_error #define outw(a, b) compiler_error #define outl(a, b) compiler_error #endif /* * Stream accesses are the same as normal accesses on x86; there are no * supported bus systems with an endianess different from the host one. */ #define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o)) #define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o)) #define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o)) #define bus_space_read_multi_stream_1(t, h, o, a, c) \ bus_space_read_multi_1((t), (h), (o), (a), (c)) #define bus_space_read_multi_stream_2(t, h, o, a, c) \ bus_space_read_multi_2((t), (h), (o), (a), (c)) #define bus_space_read_multi_stream_4(t, h, o, a, c) \ bus_space_read_multi_4((t), (h), (o), (a), (c)) #define bus_space_write_stream_1(t, h, o, v) \ bus_space_write_1((t), (h), (o), (v)) #define bus_space_write_stream_2(t, h, o, v) \ bus_space_write_2((t), (h), (o), (v)) #define bus_space_write_stream_4(t, h, o, v) \ bus_space_write_4((t), (h), (o), (v)) #define bus_space_write_multi_stream_1(t, h, o, a, c) \ bus_space_write_multi_1((t), (h), (o), (a), (c)) #define bus_space_write_multi_stream_2(t, h, o, a, c) \ bus_space_write_multi_2((t), (h), (o), (a), (c)) #define bus_space_write_multi_stream_4(t, h, o, a, c) \ bus_space_write_multi_4((t), (h), (o), (a), (c)) #define bus_space_set_multi_stream_1(t, h, o, v, c) \ bus_space_set_multi_1((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_2(t, h, o, v, c) \ bus_space_set_multi_2((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_4(t, h, o, v, c) \ bus_space_set_multi_4((t), (h), (o), (v), (c)) #define bus_space_read_region_stream_1(t, h, o, a, c) \ bus_space_read_region_1((t), (h), (o), (a), (c)) #define bus_space_read_region_stream_2(t, h, o, a, c) \ bus_space_read_region_2((t), (h), (o), (a), (c)) #define bus_space_read_region_stream_4(t, h, o, a, c) \ bus_space_read_region_4((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_1(t, h, o, a, c) \ bus_space_write_region_1((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_2(t, h, o, a, c) \ bus_space_write_region_2((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_4(t, h, o, a, c) \ bus_space_write_region_4((t), (h), (o), (a), (c)) #define bus_space_set_region_stream_1(t, h, o, v, c) \ bus_space_set_region_1((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_2(t, h, o, v, c) \ bus_space_set_region_2((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_4(t, h, o, v, c) \ bus_space_set_region_4((t), (h), (o), (v), (c)) #define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \ bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c)) #define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \ bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c)) #define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \ bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c)) #define BUS_PEEK_FUNC(width, type) \ static inline int \ bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ type tmp; \ tmp = bus_space_read_##width(tag, hnd, offset); \ *value = (type)tmp; \ return (0); \ } BUS_PEEK_FUNC(1, uint8_t) BUS_PEEK_FUNC(2, uint16_t) BUS_PEEK_FUNC(4, uint32_t) #ifdef __amd64__ BUS_PEEK_FUNC(8, uint64_t) #endif #define BUS_POKE_FUNC(width, type) \ static inline int \ bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write_##width(tag, hnd, offset, value); \ return (0); \ } BUS_POKE_FUNC(1, uint8_t) BUS_POKE_FUNC(2, uint16_t) BUS_POKE_FUNC(4, uint32_t) #ifdef __amd64__ BUS_POKE_FUNC(8, uint64_t) #endif #endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* _X86_BUS_H_ */