diff --git a/share/man/man9/atomic.9 b/share/man/man9/atomic.9 index 436cfb7ee225..581aac9878c8 100644 --- a/share/man/man9/atomic.9 +++ b/share/man/man9/atomic.9 @@ -1,619 +1,620 @@ .\" Copyright (c) 2000-2001 John H. Baldwin .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR .\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES .\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. .\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT, .\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT .\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .Dd November 19, 2024 .Dt ATOMIC 9 .Os .Sh NAME .Nm atomic_add , .Nm atomic_clear , .Nm atomic_cmpset , .Nm atomic_fcmpset , .Nm atomic_fetchadd , .Nm atomic_interrupt_fence , .Nm atomic_load , .Nm atomic_readandclear , .Nm atomic_set , .Nm atomic_subtract , .Nm atomic_store , .Nm atomic_thread_fence .Nd atomic operations .Sh SYNOPSIS .In machine/atomic.h .Ft void .Fn atomic_add_[acq_|rel_] "volatile *p" " v" .Ft void .Fn atomic_clear_[acq_|rel_] "volatile *p" " v" .Ft int .Fo atomic_cmpset_[acq_|rel_] .Fa "volatile *dst" .Fa " old" .Fa " new" .Fc .Ft int .Fo atomic_fcmpset_[acq_|rel_] .Fa "volatile *dst" .Fa " *old" .Fa " new" .Fc .Ft .Fn atomic_fetchadd_ "volatile *p" " v" .Ft void .Fn atomic_interrupt_fence "void" .Ft .Fn atomic_load_[acq_] "volatile *p" .Ft .Fn atomic_readandclear_ "volatile *p" .Ft void .Fn atomic_set_[acq_|rel_] "volatile *p" " v" .Ft void .Fn atomic_subtract_[acq_|rel_] "volatile *p" " v" .Ft void .Fn atomic_store_[rel_] "volatile *p" " v" .Ft .Fn atomic_swap_ "volatile *p" " v" .Ft int .Fn atomic_testandclear_ "volatile *p" "u_int v" .Ft int .Fn atomic_testandset_ "volatile *p" "u_int v" .Ft void .Fn atomic_thread_fence_[acq|acq_rel|rel|seq_cst] "void" .Sh DESCRIPTION Atomic operations are commonly used to implement reference counts and as building blocks for synchronization primitives, such as mutexes. .Pp All of these operations are performed .Em atomically across multiple threads and in the presence of interrupts, meaning that they are performed in an indivisible manner from the perspective of concurrently running threads and interrupt handlers. .Pp On all architectures supported by .Fx , ordinary loads and stores of integers in cache-coherent memory are inherently atomic if the integer is naturally aligned and its size does not exceed the processor's word size. However, such loads and stores may be elided from the program by the compiler, whereas atomic operations are always performed. .Pp When atomic operations are performed on cache-coherent memory, all operations on the same location are totally ordered. .Pp When an atomic load is performed on a location in cache-coherent memory, it reads the entire value that was defined by the last atomic store to each byte of the location. An atomic load will never return a value out of thin air. When an atomic store is performed on a location, no other thread or interrupt handler will observe a .Em torn write , or partial modification of the location. .Pp Except as noted below, the semantics of these operations are almost identical to the semantics of similarly named C11 atomic operations. .Ss Types Most atomic operations act upon a specific .Fa type . That type is indicated in the function name. In contrast to C11 atomic operations, .Fx Ns 's atomic operations are performed on ordinary integer types. The available types are: .Pp .Bl -tag -offset indent -width short -compact .It Li int unsigned integer .It Li long unsigned long integer .It Li ptr unsigned integer the size of a pointer .It Li 32 unsigned 32-bit integer .It Li 64 unsigned 64-bit integer .El .Pp For example, the function to atomically add two integers is called .Fn atomic_add_int . .Pp Certain architectures also provide operations for types smaller than .Dq Li int . .Pp .Bl -tag -offset indent -width short -compact .It Li char unsigned character .It Li short unsigned short integer .It Li 8 unsigned 8-bit integer .It Li 16 unsigned 16-bit integer .El .Pp These types must not be used in machine-independent code. .Ss Acquire and Release Operations By default, a thread's accesses to different memory locations might not be performed in .Em program order , that is, the order in which the accesses appear in the source code. To optimize the program's execution, both the compiler and processor might reorder the thread's accesses. However, both ensure that their reordering of the accesses is not visible to the thread. Otherwise, the traditional memory model that is expected by single-threaded programs would be violated. Nonetheless, other threads in a multithreaded program, such as the .Fx kernel, might observe the reordering. Moreover, in some cases, such as the implementation of synchronization between threads, arbitrary reordering might result in the incorrect execution of the program. To constrain the reordering that both the compiler and processor might perform on a thread's accesses, a programmer can use atomic operations with .Em acquire and .Em release semantics. .Pp Atomic operations on memory have up to three variants. The first, or .Em relaxed variant, performs the operation without imposing any ordering constraints on accesses to other memory locations. This variant is the default. The second variant has acquire semantics, and the third variant has release semantics. .Pp When an atomic operation has acquire semantics, the operation must have completed before any subsequent load or store (by program order) is performed. Conversely, acquire semantics do not require that prior loads or stores have completed before the atomic operation is performed. An atomic operation can only have acquire semantics if it performs a load from memory. To denote acquire semantics, the suffix .Dq Li _acq is inserted into the function name immediately prior to the .Dq Li _ Ns Aq Fa type suffix. For example, to subtract two integers ensuring that the subtraction is completed before any subsequent loads and stores are performed, use .Fn atomic_subtract_acq_int . .Pp When an atomic operation has release semantics, all prior loads or stores (by program order) must have completed before the operation is performed. Conversely, release semantics do not require that the atomic operation must have completed before any subsequent load or store is performed. An atomic operation can only have release semantics if it performs a store to memory. To denote release semantics, the suffix .Dq Li _rel is inserted into the function name immediately prior to the .Dq Li _ Ns Aq Fa type suffix. For example, to add two long integers ensuring that all prior loads and stores are completed before the addition is performed, use .Fn atomic_add_rel_long . .Pp When a release operation by one thread .Em synchronizes with an acquire operation by another thread, usually meaning that the acquire operation reads the value written by the release operation, then the effects of all prior stores by the releasing thread must become visible to subsequent loads by the acquiring thread. Moreover, the effects of all stores (by other threads) that were visible to the releasing thread must also become visible to the acquiring thread. These rules only apply to the synchronizing threads. Other threads might observe these stores in a different order. .Pp In effect, atomic operations with acquire and release semantics establish one-way barriers to reordering that enable the implementations of synchronization primitives to express their ordering requirements without also imposing unnecessary ordering. For example, for a critical section guarded by a mutex, an acquire operation when the mutex is locked and a release operation when the mutex is unlocked will prevent any loads or stores from moving outside of the critical section. However, they will not prevent the compiler or processor from moving loads or stores into the critical section, which does not violate the semantics of a mutex. .Ss Thread Fence Operations Alternatively, a programmer can use atomic thread fence operations to constrain the reordering of accesses. In contrast to other atomic operations, fences do not, themselves, access memory. .Pp When a fence has acquire semantics, all prior loads (by program order) must have completed before any subsequent load or store is performed. Thus, an acquire fence is a two-way barrier for load operations. To denote acquire semantics, the suffix .Dq Li _acq is appended to the function name, for example, .Fn atomic_thread_fence_acq . .Pp When a fence has release semantics, all prior loads or stores (by program order) must have completed before any subsequent store operation is performed. Thus, a release fence is a two-way barrier for store operations. To denote release semantics, the suffix .Dq Li _rel is appended to the function name, for example, .Fn atomic_thread_fence_rel . .Pp Although .Fn atomic_thread_fence_acq_rel implements both acquire and release semantics, it is not a full barrier. For example, a store prior to the fence (in program order) may be completed after a load subsequent to the fence. In contrast, .Fn atomic_thread_fence_seq_cst implements a full barrier. Neither loads nor stores may cross this barrier in either direction. .Pp In C11, a release fence by one thread synchronizes with an acquire fence by another thread when an atomic load that is prior to the acquire fence (by program order) reads the value written by an atomic store that is subsequent to the release fence. In constrast, in .Fx , because of the atomicity of ordinary, naturally aligned loads and stores, fences can also be synchronized by ordinary loads and stores. This simplifies the implementation and use of some synchronization primitives in .Fx . .Pp Since neither a compiler nor a processor can foresee which (atomic) load will read the value written by an (atomic) store, the ordering constraints imposed by fences must be more restrictive than acquire loads and release stores. Essentially, this is why fences are two-way barriers. .Pp Although fences impose more restrictive ordering than acquire loads and release stores, by separating access from ordering, they can sometimes facilitate more efficient implementations of synchronization primitives. For example, they can be used to avoid executing a memory barrier until a memory access shows that some condition is satisfied. .Ss Interrupt Fence Operations The .Fn atomic_interrupt_fence function establishes ordering between its call location and any interrupt handler executing on the same CPU. It is modeled after the similar C11 function .Fn atomic_signal_fence , and adapted for the kernel environment. .Ss Multiple Processors In multiprocessor systems, the atomicity of the atomic operations on memory depends on support for cache coherence in the underlying architecture. In general, cache coherence on the default memory type, .Dv VM_MEMATTR_DEFAULT , is guaranteed by all architectures that are supported by .Fx . For example, cache coherence is guaranteed on write-back memory by the .Tn amd64 and .Tn i386 architectures. However, on some architectures, cache coherence might not be enabled on all memory types. To determine if cache coherence is enabled for a non-default memory type, consult the architecture's documentation. .Ss Semantics This section describes the semantics of each operation using a C like notation. .Bl -hang .It Fn atomic_add p v .Bd -literal -compact *p += v; .Ed .It Fn atomic_clear p v .Bd -literal -compact *p &= ~v; .Ed .It Fn atomic_cmpset dst old new .Bd -literal -compact if (*dst == old) { *dst = new; return (1); } else return (0); .Ed .El .Pp Some architectures do not implement the .Fn atomic_cmpset functions for the types .Dq Li char , .Dq Li short , .Dq Li 8 , and .Dq Li 16 . .Bl -hang .It Fn atomic_fcmpset dst *old new .El .Pp On architectures implementing .Em Compare And Swap operation in hardware, the functionality can be described as .Bd -literal -offset indent -compact if (*dst == *old) { *dst = new; return (1); } else { *old = *dst; return (0); } .Ed On architectures which provide .Em Load Linked/Store Conditional primitive, the write to .Dv *dst might also fail for several reasons, most important of which is a parallel write to .Dv *dst cache line by other CPU. In this case .Fn atomic_fcmpset function also returns .Dv false , despite .Dl *old == *dst . .Pp Some architectures do not implement the .Fn atomic_fcmpset functions for the types .Dq Li char , .Dq Li short , .Dq Li 8 , and .Dq Li 16 . .Bl -hang .It Fn atomic_fetchadd p v .Bd -literal -compact tmp = *p; *p += v; return (tmp); .Ed .El .Pp The .Fn atomic_fetchadd functions are only implemented for the types .Dq Li int , .Dq Li long and .Dq Li 32 and do not have any variants with memory barriers at this time. .Bl -hang .It Fn atomic_load p .Bd -literal -compact return (*p); .Ed .It Fn atomic_readandclear p .Bd -literal -compact tmp = *p; *p = 0; return (tmp); .Ed .El .Pp The .Fn atomic_readandclear functions are not implemented for the types .Dq Li char , .Dq Li short , .Dq Li ptr , .Dq Li 8 , and .Dq Li 16 and do not have any variants with memory barriers at this time. .Bl -hang .It Fn atomic_set p v .Bd -literal -compact *p |= v; .Ed .It Fn atomic_subtract p v .Bd -literal -compact *p -= v; .Ed .It Fn atomic_store p v .Bd -literal -compact *p = v; .Ed .It Fn atomic_swap p v .Bd -literal -compact tmp = *p; *p = v; return (tmp); .Ed .El .Pp The .Fn atomic_swap functions are not implemented for the types .Dq Li char , .Dq Li short , .Dq Li ptr , .Dq Li 8 , and .Dq Li 16 and do not have any variants with memory barriers at this time. .Bl -hang .It Fn atomic_testandclear p v .Bd -literal -compact bit = 1 << (v % (sizeof(*p) * NBBY)); tmp = (*p & bit) != 0; *p &= ~bit; return (tmp); .Ed .El .Bl -hang .It Fn atomic_testandset p v .Bd -literal -compact bit = 1 << (v % (sizeof(*p) * NBBY)); tmp = (*p & bit) != 0; *p |= bit; return (tmp); .Ed .El .Pp The .Fn atomic_testandset and .Fn atomic_testandclear functions are only implemented for the types .Dq Li int , .Dq Li long , +.Dq ptr , .Dq Li 32 , and .Dq Li 64 and generally do not have any variants with memory barriers at this time except for .Fn atomic_testandset_acq_long . .Pp The type .Dq Li 64 is currently not implemented for some of the atomic operations on the .Tn arm , .Tn i386 , and .Tn powerpc architectures. .Sh RETURN VALUES The .Fn atomic_cmpset function returns the result of the compare operation. The .Fn atomic_fcmpset function returns .Dv true if the operation succeeded. Otherwise it returns .Dv false and sets .Va *old to the found value. The .Fn atomic_fetchadd , .Fn atomic_load , .Fn atomic_readandclear , and .Fn atomic_swap functions return the value at the specified address. The .Fn atomic_testandset and .Fn atomic_testandclear function returns the result of the test operation. .Sh EXAMPLES This example uses the .Fn atomic_cmpset_acq_ptr and .Fn atomic_set_ptr functions to obtain a sleep mutex and handle recursion. Since the .Va mtx_lock member of a .Vt "struct mtx" is a pointer, the .Dq Li ptr type is used. .Bd -literal /* Try to obtain mtx_lock once. */ #define _obtain_lock(mp, tid) \\ atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) /* Get a sleep lock, deal with recursion inline. */ #define _get_sleep_lock(mp, tid, opts, file, line) do { \\ uintptr_t _tid = (uintptr_t)(tid); \\ \\ if (!_obtain_lock(mp, tid)) { \\ if (((mp)->mtx_lock & MTX_FLAGMASK) != _tid) \\ _mtx_lock_sleep((mp), _tid, (opts), (file), (line));\\ else { \\ atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSE); \\ (mp)->mtx_recurse++; \\ } \\ } \\ } while (0) .Ed .Sh HISTORY The .Fn atomic_add , .Fn atomic_clear , .Fn atomic_set , and .Fn atomic_subtract operations were introduced in .Fx 3.0 . Initially, these operations were defined on the types .Dq Li char , .Dq Li short , .Dq Li int , and .Dq Li long . .Pp The .Fn atomic_cmpset , .Fn atomic_load_acq , .Fn atomic_readandclear , and .Fn atomic_store_rel operations were added in .Fx 5.0 . Simultaneously, the acquire and release variants were introduced, and support was added for operation on the types .Dq Li 8 , .Dq Li 16 , .Dq Li 32 , .Dq Li 64 , and .Dq Li ptr . .Pp The .Fn atomic_fetchadd operation was added in .Fx 6.0 . .Pp The .Fn atomic_swap and .Fn atomic_testandset operations were added in .Fx 10.0 . .Pp The .Fn atomic_testandclear and .Fn atomic_thread_fence operations were added in .Fx 11.0 . .Pp The relaxed variants of .Fn atomic_load and .Fn atomic_store were added in .Fx 12.0 . .Pp The .Fn atomic_interrupt_fence operation was added in .Fx 13.0 . diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 75a88e03290e..a8faedd58123 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -1,599 +1,601 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifdef __i386__ #include #else /* !__i386__ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ /* * To express interprocessor (as opposed to processor and device) memory * ordering constraints, use the atomic_*() functions with acquire and release * semantics rather than the *mb() functions. An architecture's memory * ordering (or memory consistency) model governs the order in which a * program's accesses to different locations may be performed by an * implementation of that architecture. In general, for memory regions * defined as writeback cacheable, the memory ordering implemented by amd64 * processors preserves the program ordering of a load followed by a load, a * load followed by a store, and a store followed by a store. Only a store * followed by a load to a different memory location may be reordered. * Therefore, except for special cases, like non-temporal memory accesses or * memory regions defined as write combining, the memory ordering effects * provided by the sfence instruction in the wmb() function and the lfence * instruction in the rmb() function are redundant. In contrast, the * atomic_*() functions with acquire and release semantics do not perform * redundant instructions for ordinary cases of interprocessor memory * ordering on any architecture. */ #define mb() __asm __volatile("mfence;" : : : "memory") #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") #ifdef _KERNEL /* * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ #define OFFSETOF_MONITORBUF 0x100 #endif #if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) #include #else #include /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ /* * Always use lock prefixes. The result is slightly less optimal for * UP systems, but it matters less now, and sometimes UP is emulated * over SMP. * * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions. * * cmpset: * if (*dst == expect) * *dst = src * * fcmpset: * if (*dst == *expect) * *dst = src * else * *expect = *dst * * Returns 0 on failure, non-zero on success. */ #define ATOMIC_CMPSET(TYPE) \ static __inline int \ atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ "# atomic_cmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } \ \ static __inline int \ atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ "# atomic_fcmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (*expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } ATOMIC_CMPSET(char); ATOMIC_CMPSET(short); ATOMIC_CMPSET(int); ATOMIC_CMPSET(long); /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " lock; xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } /* * Atomically add the value of v to the long integer pointed to by p and return * the previous value of *p. */ static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { __asm __volatile( " lock; xaddq %0,%1 ; " "# atomic_fetchadd_long" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btsl %2,%1 ; " "# atomic_testandset_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " lock; btsq %2,%1 ; " "# atomic_testandset_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btrl %2,%1 ; " "# atomic_testandclear_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " lock; btrq %2,%1 ; " "# atomic_testandclear_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need a Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. To avoid false data dependencies, we use a * special address for "mem". In the kernel, we use a private per-cpu * cache line. In user space, we use a word in the stack's red zone * (-8(%rsp)). */ static __inline void __storeload_barrier(void) { #if defined(_KERNEL) __asm __volatile("lock; addl $0,%%gs:%c0" : : "i" (OFFSETOF_MONITORBUF) : "memory", "cc"); #else /* !_KERNEL */ __asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc"); #endif /* _KERNEL*/ } #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orq %1,%0", "er", v); ATOMIC_ASM(clear, long, "andq %1,%0", "er", ~v); ATOMIC_ASM(add, long, "addq %1,%0", "er", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "er", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS /* Read the current value and store a new value in the destination. */ static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { __asm __volatile( " xchgq %1,%0 ; " "# atomic_swap_long" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_cmpset_acq_char atomic_cmpset_char #define atomic_cmpset_rel_char atomic_cmpset_char #define atomic_fcmpset_acq_char atomic_fcmpset_char #define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_cmpset_acq_short atomic_cmpset_short #define atomic_cmpset_rel_short atomic_cmpset_short #define atomic_fcmpset_acq_short atomic_fcmpset_short #define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) #define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subtract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_testandset_64 atomic_testandset_long #define atomic_testandclear_64 atomic_testandclear_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long +#define atomic_testandset_ptr atomic_testandset_long +#define atomic_testandclear_ptr atomic_testandclear_long #endif /* !WANT_FUNCTIONS */ #endif /* !SAN_NEEDS_INTERCEPTORS || SAN_RUNTIME */ #endif /* !_MACHINE_ATOMIC_H_ */ #endif /* __i386__ */ diff --git a/sys/arm/include/atomic.h b/sys/arm/include/atomic.h index 1fef40aea2e8..33116d0a6aee 100644 --- a/sys/arm/include/atomic.h +++ b/sys/arm/include/atomic.h @@ -1,1111 +1,1113 @@ /* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ /*- * SPDX-License-Identifier: BSD-4-Clause * * Copyright (C) 2003-2004 Olivier Houchard * Copyright (C) 1994-1997 Mark Brinicombe * Copyright (C) 1994 Brini * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of Brini may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #include #define isb() __asm __volatile("isb" : : : "memory") #define dsb() __asm __volatile("dsb" : : : "memory") #define dmb() __asm __volatile("dmb" : : : "memory") #define mb() dmb() #define wmb() dmb() #define rmb() dmb() #define ARM_HAVE_ATOMIC64 #define ATOMIC_ACQ_REL_LONG(NAME) \ static __inline void \ atomic_##NAME##_acq_long(__volatile u_long *p, u_long v) \ { \ atomic_##NAME##_long(p, v); \ dmb(); \ } \ \ static __inline void \ atomic_##NAME##_rel_long(__volatile u_long *p, u_long v) \ { \ dmb(); \ atomic_##NAME##_long(p, v); \ } #define ATOMIC_ACQ_REL(NAME, WIDTH) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ atomic_##NAME##_##WIDTH(p, v); \ dmb(); \ } \ \ static __inline void \ atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ dmb(); \ atomic_##NAME##_##WIDTH(p, v); \ } static __inline void atomic_add_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " add %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2) ,"+r" (p), "+r" (val) : : "cc", "memory"); } static __inline void atomic_add_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " adds %Q[tmp], %Q[val] \n" " adc %R[tmp], %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_add_long(volatile u_long *p, u_long val) { atomic_add_32((volatile uint32_t *)p, val); } ATOMIC_ACQ_REL(add, 32) ATOMIC_ACQ_REL(add, 64) ATOMIC_ACQ_REL_LONG(add) static __inline void atomic_clear_32(volatile uint32_t *address, uint32_t setmask) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " bic %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask) : : "cc", "memory"); } static __inline void atomic_clear_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " bic %Q[tmp], %Q[val] \n" " bic %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_clear_long(volatile u_long *address, u_long setmask) { atomic_clear_32((volatile uint32_t *)address, setmask); } ATOMIC_ACQ_REL(clear, 32) ATOMIC_ACQ_REL(clear, 64) ATOMIC_ACQ_REL_LONG(clear) #define ATOMIC_FCMPSET_CODE(RET, TYPE, SUF) \ { \ TYPE tmp; \ \ __asm __volatile( \ "1: ldrex" SUF " %[tmp], [%[ptr]] \n" \ " ldr" SUF " %[ret], [%[oldv]] \n" \ " teq %[tmp], %[ret] \n" \ " ittee ne \n" \ " str" SUF "ne %[tmp], [%[oldv]] \n" \ " movne %[ret], #0 \n" \ " strex" SUF "eq %[ret], %[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET), \ [tmp] "=&r" (tmp) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } #define ATOMIC_FCMPSET_CODE64(RET) \ { \ uint64_t cmp, tmp; \ \ __asm __volatile( \ "1: ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" \ " ldrd %Q[cmp], %R[cmp], [%[oldv]] \n" \ " teq %Q[tmp], %Q[cmp] \n" \ " it eq \n" \ " teqeq %R[tmp], %R[cmp] \n" \ " ittee ne \n" \ " movne %[ret], #0 \n" \ " strdne %[cmp], [%[oldv]] \n" \ " strexdeq %[ret], %Q[newv], %R[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET), \ [cmp] "=&r" (cmp), \ [tmp] "=&r" (tmp) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } static __inline int atomic_fcmpset_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint8_t, "b"); return (ret); } #define atomic_fcmpset_8 atomic_fcmpset_8 static __inline int atomic_fcmpset_acq_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint8_t, "b"); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, uint8_t, "b"); return (ret); } static __inline int atomic_fcmpset_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint16_t, "h"); return (ret); } #define atomic_fcmpset_16 atomic_fcmpset_16 static __inline int atomic_fcmpset_acq_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint16_t, "h"); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, uint16_t, "h"); return (ret); } static __inline int atomic_fcmpset_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint32_t, ""); return (ret); } static __inline int atomic_fcmpset_acq_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint32_t, ""); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, uint32_t, ""); return (ret); } static __inline int atomic_fcmpset_long(volatile u_long *_ptr, u_long *_old, u_long _new) { int ret; ATOMIC_FCMPSET_CODE(ret, u_long, ""); return (ret); } static __inline int atomic_fcmpset_acq_long(volatile u_long *_ptr, u_long *_old, u_long _new) { int ret; ATOMIC_FCMPSET_CODE(ret, u_long, ""); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_long(volatile u_long *_ptr, u_long *_old, u_long _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, u_long, ""); return (ret); } static __inline int atomic_fcmpset_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new) { int ret; ATOMIC_FCMPSET_CODE64(ret); return (ret); } static __inline int atomic_fcmpset_acq_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new) { int ret; ATOMIC_FCMPSET_CODE64(ret); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE64(ret); return (ret); } #define ATOMIC_CMPSET_CODE(RET, SUF) \ { \ __asm __volatile( \ "1: ldrex" SUF " %[ret], [%[ptr]] \n" \ " teq %[ret], %[oldv] \n" \ " itee ne \n" \ " movne %[ret], #0 \n" \ " strex" SUF "eq %[ret], %[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } #define ATOMIC_CMPSET_CODE64(RET) \ { \ uint64_t tmp; \ \ __asm __volatile( \ "1: ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" \ " teq %Q[tmp], %Q[oldv] \n" \ " it eq \n" \ " teqeq %R[tmp], %R[oldv] \n" \ " itee ne \n" \ " movne %[ret], #0 \n" \ " strexdeq %[ret], %Q[newv], %R[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET), \ [tmp] "=&r" (tmp) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } static __inline int atomic_cmpset_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "b"); return (ret); } #define atomic_cmpset_8 atomic_cmpset_8 static __inline int atomic_cmpset_acq_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "b"); dmb(); return (ret); } static __inline int atomic_cmpset_rel_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, "b"); return (ret); } static __inline int atomic_cmpset_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "h"); return (ret); } #define atomic_cmpset_16 atomic_cmpset_16 static __inline int atomic_cmpset_acq_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "h"); dmb(); return (ret); } static __inline int atomic_cmpset_rel_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, "h"); return (ret); } static __inline int atomic_cmpset_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_acq_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); dmb(); return (ret); } static __inline int atomic_cmpset_rel_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_long(volatile u_long *_ptr, u_long _old, u_long _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_acq_long(volatile u_long *_ptr, u_long _old, u_long _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); dmb(); return (ret); } static __inline int atomic_cmpset_rel_long(volatile u_long *_ptr, u_long _old, u_long _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new) { int ret; ATOMIC_CMPSET_CODE64(ret); return (ret); } static __inline int atomic_cmpset_acq_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new) { int ret; ATOMIC_CMPSET_CODE64(ret); dmb(); return (ret); } static __inline int atomic_cmpset_rel_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE64(ret); return (ret); } static __inline uint32_t atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp = 0, tmp2 = 0, ret = 0; __asm __volatile( "1: ldrex %0, [%3] \n" " add %1, %0, %4 \n" " strex %2, %1, [%3] \n" " cmp %2, #0 \n" " it ne \n" " bne 1b \n" : "+r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val) : : "cc", "memory"); return (ret); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) { uint64_t ret, tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[ret], %R[ret], [%[ptr]] \n" " adds %Q[tmp], %Q[ret], %Q[val] \n" " adc %R[tmp], %R[ret], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); return (ret); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long val) { return (atomic_fetchadd_32((volatile uint32_t *)p, val)); } static __inline uint32_t atomic_load_acq_32(volatile uint32_t *p) { uint32_t v; v = *p; dmb(); return (v); } static __inline uint64_t atomic_load_64(volatile uint64_t *p) { uint64_t ret; /* * The only way to atomically load 64 bits is with LDREXD which puts the * exclusive monitor into the exclusive state, so reset it to open state * with CLREX because we don't actually need to store anything. */ __asm __volatile( "ldrexd %Q[ret], %R[ret], [%[ptr]] \n" "clrex \n" : [ret] "=&r" (ret) : [ptr] "r" (p) : "cc", "memory"); return (ret); } static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { uint64_t ret; ret = atomic_load_64(p); dmb(); return (ret); } static __inline u_long atomic_load_acq_long(volatile u_long *p) { u_long v; v = *p; dmb(); return (v); } static __inline uint32_t atomic_readandclear_32(volatile uint32_t *p) { uint32_t ret, tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%3] \n" " mov %1, #0 \n" " strex %2, %1, [%3] \n" " cmp %2, #0 \n" " it ne \n" " bne 1b \n" : "=r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p) : : "cc", "memory"); return (ret); } static __inline uint64_t atomic_readandclear_64(volatile uint64_t *p) { uint64_t ret, tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[ret], %R[ret], [%[ptr]] \n" " mov %Q[tmp], #0 \n" " mov %R[tmp], #0 \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p) : "cc", "memory"); return (ret); } static __inline u_long atomic_readandclear_long(volatile u_long *p) { return (atomic_readandclear_32((volatile uint32_t *)p)); } static __inline void atomic_set_32(volatile uint32_t *address, uint32_t setmask) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " orr %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask) : : "cc", "memory"); } static __inline void atomic_set_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " orr %Q[tmp], %Q[val] \n" " orr %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_set_long(volatile u_long *address, u_long setmask) { atomic_set_32((volatile uint32_t *)address, setmask); } ATOMIC_ACQ_REL(set, 32) ATOMIC_ACQ_REL(set, 64) ATOMIC_ACQ_REL_LONG(set) static __inline void atomic_subtract_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " sub %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val) : : "cc", "memory"); } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " subs %Q[tmp], %Q[val] \n" " sbc %R[tmp], %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_subtract_long(volatile u_long *p, u_long val) { atomic_subtract_32((volatile uint32_t *)p, val); } ATOMIC_ACQ_REL(subtract, 32) ATOMIC_ACQ_REL(subtract, 64) ATOMIC_ACQ_REL_LONG(subtract) static __inline void atomic_store_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; /* * The only way to atomically store 64 bits is with STREXD, which will * succeed only if paired up with a preceeding LDREXD using the same * address, so we read and discard the existing value before storing. */ __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " strexd %[exf], %Q[val], %R[val], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [tmp] "=&r" (tmp), [exf] "=&r" (exflag) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_store_rel_32(volatile uint32_t *p, uint32_t v) { dmb(); *p = v; } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t val) { dmb(); atomic_store_64(p, val); } static __inline void atomic_store_rel_long(volatile u_long *p, u_long v) { dmb(); *p = v; } static __inline int atomic_testandclear_32(volatile uint32_t *ptr, u_int bit) { int newv, oldv, result; __asm __volatile( " mov ip, #1 \n" " lsl ip, ip, %[bit] \n" /* Done with %[bit] as input, reuse below as output. */ "1: \n" " ldrex %[oldv], [%[ptr]] \n" " bic %[newv], %[oldv], ip \n" " strex %[bit], %[newv], [%[ptr]] \n" " teq %[bit], #0 \n" " it ne \n" " bne 1b \n" " ands %[bit], %[oldv], ip \n" " it ne \n" " movne %[bit], #1 \n" : [bit] "=&r" (result), [oldv] "=&r" (oldv), [newv] "=&r" (newv) : [ptr] "r" (ptr), "[bit]" (bit & 0x1f) : "cc", "ip", "memory"); return (result); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { return (atomic_testandclear_32((volatile uint32_t *)p, v)); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { return (atomic_testandclear_32((volatile uint32_t *)p, v)); } static __inline int atomic_testandclear_64(volatile uint64_t *p, u_int v) { volatile uint32_t *p32; p32 = (volatile uint32_t *)p; /* * Assume little-endian, * atomic_testandclear_32() uses only last 5 bits of v */ if ((v & 0x20) != 0) p32++; return (atomic_testandclear_32(p32, v)); } static __inline int atomic_testandset_32(volatile uint32_t *ptr, u_int bit) { int newv, oldv, result; __asm __volatile( " mov ip, #1 \n" " lsl ip, ip, %[bit] \n" /* Done with %[bit] as input, reuse below as output. */ "1: \n" " ldrex %[oldv], [%[ptr]] \n" " orr %[newv], %[oldv], ip \n" " strex %[bit], %[newv], [%[ptr]] \n" " teq %[bit], #0 \n" " it ne \n" " bne 1b \n" " ands %[bit], %[oldv], ip \n" " it ne \n" " movne %[bit], #1 \n" : [bit] "=&r" (result), [oldv] "=&r" (oldv), [newv] "=&r" (newv) : [ptr] "r" (ptr), "[bit]" (bit & 0x1f) : "cc", "ip", "memory"); return (result); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { return (atomic_testandset_32((volatile uint32_t *)p, v)); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { return (atomic_testandset_32((volatile uint32_t *)p, v)); } static __inline int atomic_testandset_acq_long(volatile u_long *p, u_int v) { int ret; ret = atomic_testandset_32((volatile uint32_t *)p, v); dmb(); return (ret); } static __inline int atomic_testandset_64(volatile uint64_t *p, u_int v) { volatile uint32_t *p32; p32 = (volatile uint32_t *)p; /* * Assume little-endian, * atomic_testandset_32() uses only last 5 bits of v */ if ((v & 0x20) != 0) p32++; return (atomic_testandset_32(p32, v)); } static __inline uint32_t atomic_swap_32(volatile uint32_t *p, uint32_t v) { uint32_t ret, exflag; __asm __volatile( "1: ldrex %[ret], [%[ptr]] \n" " strex %[exf], %[val], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag) : [val] "r" (v), [ptr] "r" (p) : "cc", "memory"); return (ret); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { return (atomic_swap_32((volatile uint32_t *)p, v)); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t v) { uint64_t ret; uint32_t exflag; __asm __volatile( "1: ldrexd %Q[ret], %R[ret], [%[ptr]] \n" " strexd %[exf], %Q[val], %R[val], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag) : [val] "r" (v), [ptr] "r" (p) : "cc", "memory"); return (ret); } #undef ATOMIC_ACQ_REL #undef ATOMIC_ACQ_REL_LONG static __inline void atomic_thread_fence_acq(void) { dmb(); } static __inline void atomic_thread_fence_rel(void) { dmb(); } static __inline void atomic_thread_fence_acq_rel(void) { dmb(); } static __inline void atomic_thread_fence_seq_cst(void) { dmb(); } #define atomic_add_ptr atomic_add_32 #define atomic_add_acq_ptr atomic_add_acq_32 #define atomic_add_rel_ptr atomic_add_rel_32 #define atomic_subtract_ptr atomic_subtract_32 #define atomic_subtract_acq_ptr atomic_subtract_acq_32 #define atomic_subtract_rel_ptr atomic_subtract_rel_32 #define atomic_clear_ptr atomic_clear_32 #define atomic_clear_acq_ptr atomic_clear_acq_32 #define atomic_clear_rel_ptr atomic_clear_rel_32 #define atomic_set_ptr atomic_set_32 #define atomic_set_acq_ptr atomic_set_acq_32 #define atomic_set_rel_ptr atomic_set_rel_32 #define atomic_fcmpset_ptr atomic_fcmpset_32 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_32 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_32 #define atomic_cmpset_ptr atomic_cmpset_32 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_32 #define atomic_fetchadd_ptr atomic_fetchadd_32 #define atomic_readandclear_ptr atomic_readandclear_32 #define atomic_load_acq_ptr atomic_load_acq_32 #define atomic_store_rel_ptr atomic_store_rel_32 #define atomic_swap_ptr atomic_swap_32 +#define atomic_testandset_ptr atomic_testandset_32 +#define atomic_testandclear_ptr atomic_testandclear_32 #define atomic_add_int atomic_add_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_clear_int atomic_clear_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_set_int atomic_set_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_swap_int atomic_swap_32 /* * For: * - atomic_load_acq_8 * - atomic_load_acq_16 */ #include #endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/arm64/include/atomic.h b/sys/arm64/include/atomic.h index c7aaa76d9de9..f7018f2f9e0b 100644 --- a/sys/arm64/include/atomic.h +++ b/sys/arm64/include/atomic.h @@ -1,677 +1,679 @@ /*- * Copyright (c) 2013 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifdef __arm__ #include #else /* !__arm__ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #define isb() __asm __volatile("isb" : : : "memory") /* * Options for DMB and DSB: * oshld Outer Shareable, load * oshst Outer Shareable, store * osh Outer Shareable, all * nshld Non-shareable, load * nshst Non-shareable, store * nsh Non-shareable, all * ishld Inner Shareable, load * ishst Inner Shareable, store * ish Inner Shareable, all * ld Full system, load * st Full system, store * sy Full system, all */ #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") #define mb() dmb(sy) /* Full system memory barrier all */ #define wmb() dmb(st) /* Full system memory barrier store */ #define rmb() dmb(ld) /* Full system memory barrier load */ #ifdef _KERNEL extern _Bool lse_supported; #endif #if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) #include #else #include #if defined(__ARM_FEATURE_ATOMICS) #define _ATOMIC_LSE_SUPPORTED 1 #elif defined(_KERNEL) #ifdef LSE_ATOMICS #define _ATOMIC_LSE_SUPPORTED 1 #else #define _ATOMIC_LSE_SUPPORTED lse_supported #endif #else #define _ATOMIC_LSE_SUPPORTED 0 #endif #define _ATOMIC_OP_PROTO(t, op, bar, flav) \ static __inline void \ atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ _ATOMIC_OP_PROTO(t, op, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ \ pre; \ __asm __volatile( \ "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \ " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \ " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \ " cbnz %w1, 1b\n" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } \ \ _ATOMIC_OP_PROTO(t, op, bar, _lse) \ { \ uint##t##_t tmp; \ \ pre; \ __asm __volatile( \ ".arch_extension lse\n" \ "ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (tmp) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } \ \ _ATOMIC_OP_PROTO(t, op, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ atomic_##op##_##bar##t##_lse(p, val); \ else \ atomic_##op##_##bar##t##_llsc(p, val); \ } #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l) _ATOMIC_OP(add, add, add, ) _ATOMIC_OP(clear, bic, clr, ) _ATOMIC_OP(set, orr, set, ) _ATOMIC_OP(subtract, add, add, val = -val) #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \ static __inline int \ atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \ uint##t##_t cmpval, uint##t##_t newval) #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \ static __inline int \ atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \ uint##t##_t *cmpval, uint##t##_t newval) #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \ _ATOMIC_CMPSET_PROTO(t, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ \ __asm __volatile( \ "1: mov %w1, #1\n" \ " ld"#a"xr"#s" %"#w"0, [%2]\n" \ " cmp %"#w"0, %"#w"3\n" \ " b.ne 2f\n" \ " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ " cbnz %w1, 1b\n" \ "2:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (cmpval), "r" (newval) \ : "cc", "memory" \ ); \ \ return (!res); \ } \ \ _ATOMIC_CMPSET_PROTO(t, bar, _lse) \ { \ uint##t##_t oldval; \ int res; \ \ oldval = cmpval; \ __asm __volatile( \ ".arch_extension lse\n" \ "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ "cmp %"#w"1, %"#w"2\n" \ "cset %w0, eq\n" \ ".arch_extension nolse\n" \ : "=r" (res), "+&r" (cmpval) \ : "r" (oldval), "r" (p), "r" (newval) \ : "cc", "memory" \ ); \ \ return (res); \ } \ \ _ATOMIC_CMPSET_PROTO(t, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_cmpset_##bar##t##_lse(p, cmpval, \ newval)); \ else \ return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \ newval)); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ { \ uint##t##_t _cmpval, tmp; \ int res; \ \ _cmpval = *cmpval; \ __asm __volatile( \ " mov %w1, #1\n" \ " ld"#a"xr"#s" %"#w"0, [%2]\n" \ " cmp %"#w"0, %"#w"3\n" \ " b.ne 1f\n" \ " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ "1:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (_cmpval), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (!res); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \ { \ uint##t##_t _cmpval, tmp; \ int res; \ \ _cmpval = tmp = *cmpval; \ __asm __volatile( \ ".arch_extension lse\n" \ "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ "cmp %"#w"1, %"#w"2\n" \ "cset %w0, eq\n" \ ".arch_extension nolse\n" \ : "=r" (res), "+&r" (tmp) \ : "r" (_cmpval), "r" (p), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (res); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \ newval)); \ else \ return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \ newval)); \ } #define _ATOMIC_CMPSET(bar, a, l) \ _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \ _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \ _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \ _ATOMIC_CMPSET_IMPL(64, , , bar, a, l) #define atomic_cmpset_8 atomic_cmpset_8 #define atomic_fcmpset_8 atomic_fcmpset_8 #define atomic_cmpset_16 atomic_cmpset_16 #define atomic_fcmpset_16 atomic_fcmpset_16 _ATOMIC_CMPSET( , , ) _ATOMIC_CMPSET(acq_, a, ) _ATOMIC_CMPSET(rel_, ,l) #define _ATOMIC_FETCHADD_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_FETCHADD_IMPL(t, w) \ _ATOMIC_FETCHADD_PROTO(t, _llsc) \ { \ uint##t##_t ret, tmp; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"2, [%3]\n" \ " add %"#w"0, %"#w"2, %"#w"4\n" \ " stxr %w1, %"#w"0, [%3]\n" \ " cbnz %w1, 1b\n" \ : "=&r" (tmp), "=&r" (res), "=&r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_FETCHADD_PROTO(t, _lse) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ ".arch_extension lse\n" \ "ldadd %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_FETCHADD_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_fetchadd_##t##_lse(p, val)); \ else \ return (atomic_fetchadd_##t##_llsc(p, val)); \ } _ATOMIC_FETCHADD_IMPL(32, w) _ATOMIC_FETCHADD_IMPL(64, ) #define _ATOMIC_SWAP_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_READANDCLEAR_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_readandclear_##t##flav(volatile uint##t##_t *p) #define _ATOMIC_SWAP_IMPL(t, w, zreg) \ _ATOMIC_SWAP_PROTO(t, _llsc) \ { \ uint##t##_t ret; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"1, [%2]\n" \ " stxr %w0, %"#w"3, [%2]\n" \ " cbnz %w0, 1b\n" \ : "=&r" (res), "=&r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_SWAP_PROTO(t, _lse) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ ".arch_extension lse\n" \ "swp %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_SWAP_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_swap_##t##_lse(p, val)); \ else \ return (atomic_swap_##t##_llsc(p, val)); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ { \ uint##t##_t ret; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"1, [%2]\n" \ " stxr %w0, "#zreg", [%2]\n" \ " cbnz %w0, 1b\n" \ : "=&r" (res), "=&r" (ret) \ : "r" (p) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, _lse) \ { \ return (atomic_swap_##t##_lse(p, 0)); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_readandclear_##t##_lse(p)); \ else \ return (atomic_readandclear_##t##_llsc(p)); \ } _ATOMIC_SWAP_IMPL(32, w, wzr) _ATOMIC_SWAP_IMPL(64, , xzr) #define _ATOMIC_TEST_OP_PROTO(t, op, bar, flav) \ static __inline int \ atomic_testand##op##_##bar##t##flav(volatile uint##t##_t *p, u_int val) #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op, bar, a) \ _ATOMIC_TEST_OP_PROTO(t, op, bar, _llsc) \ { \ uint##t##_t mask, old, tmp; \ int res; \ \ mask = ((uint##t##_t)1) << (val & (t - 1)); \ __asm __volatile( \ "1: ld"#a"xr %"#w"2, [%3]\n" \ " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \ " stxr %w1, %"#w"0, [%3]\n" \ " cbnz %w1, 1b\n" \ : "=&r" (tmp), "=&r" (res), "=&r" (old) \ : "r" (p), "r" (mask) \ : "memory" \ ); \ \ return ((old & mask) != 0); \ } \ \ _ATOMIC_TEST_OP_PROTO(t, op, bar, _lse) \ { \ uint##t##_t mask, old; \ \ mask = ((uint##t##_t)1) << (val & (t - 1)); \ __asm __volatile( \ ".arch_extension lse\n" \ "ld"#lse_asm_op#a" %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (old) \ : "r" (p), "r" (mask) \ : "memory" \ ); \ \ return ((old & mask) != 0); \ } \ \ _ATOMIC_TEST_OP_PROTO(t, op, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_testand##op##_##bar##t##_lse(p, val)); \ else \ return (atomic_testand##op##_##bar##t##_llsc(p, val)); \ } #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \ _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, , ) \ _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, acq_, a) \ _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, , ) \ _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, acq_, a) _ATOMIC_TEST_OP(clear, bic, clr) _ATOMIC_TEST_OP(set, orr, set) #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \ static __inline uint##t##_t \ atomic_load_acq_##t(volatile uint##t##_t *p) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ "ldar"#s" %"#w"0, [%1]\n" \ : "=&r" (ret) \ : "r" (p) \ : "memory"); \ \ return (ret); \ } #define atomic_load_acq_8 atomic_load_acq_8 #define atomic_load_acq_16 atomic_load_acq_16 _ATOMIC_LOAD_ACQ_IMPL(8, w, b) _ATOMIC_LOAD_ACQ_IMPL(16, w, h) _ATOMIC_LOAD_ACQ_IMPL(32, w, ) _ATOMIC_LOAD_ACQ_IMPL(64, , ) #define _ATOMIC_STORE_REL_IMPL(t, w, s) \ static __inline void \ atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \ { \ __asm __volatile( \ "stlr"#s" %"#w"0, [%1]\n" \ : \ : "r" (val), "r" (p) \ : "memory"); \ } _ATOMIC_STORE_REL_IMPL(8, w, b) _ATOMIC_STORE_REL_IMPL(16, w, h) _ATOMIC_STORE_REL_IMPL(32, w, ) _ATOMIC_STORE_REL_IMPL(64, , ) #define atomic_add_char atomic_add_8 #define atomic_fcmpset_char atomic_fcmpset_8 #define atomic_clear_char atomic_clear_8 #define atomic_cmpset_char atomic_cmpset_8 #define atomic_fetchadd_char atomic_fetchadd_8 #define atomic_readandclear_char atomic_readandclear_8 #define atomic_set_char atomic_set_8 #define atomic_swap_char atomic_swap_8 #define atomic_subtract_char atomic_subtract_8 #define atomic_testandclear_char atomic_testandclear_8 #define atomic_testandset_char atomic_testandset_8 #define atomic_add_acq_char atomic_add_acq_8 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 #define atomic_clear_acq_char atomic_clear_acq_8 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 #define atomic_load_acq_char atomic_load_acq_8 #define atomic_set_acq_char atomic_set_acq_8 #define atomic_subtract_acq_char atomic_subtract_acq_8 #define atomic_testandset_acq_char atomic_testandset_acq_8 #define atomic_add_rel_char atomic_add_rel_8 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 #define atomic_clear_rel_char atomic_clear_rel_8 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 #define atomic_set_rel_char atomic_set_rel_8 #define atomic_subtract_rel_char atomic_subtract_rel_8 #define atomic_store_rel_char atomic_store_rel_8 #define atomic_add_short atomic_add_16 #define atomic_fcmpset_short atomic_fcmpset_16 #define atomic_clear_short atomic_clear_16 #define atomic_cmpset_short atomic_cmpset_16 #define atomic_fetchadd_short atomic_fetchadd_16 #define atomic_readandclear_short atomic_readandclear_16 #define atomic_set_short atomic_set_16 #define atomic_swap_short atomic_swap_16 #define atomic_subtract_short atomic_subtract_16 #define atomic_testandclear_short atomic_testandclear_16 #define atomic_testandset_short atomic_testandset_16 #define atomic_add_acq_short atomic_add_acq_16 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 #define atomic_clear_acq_short atomic_clear_acq_16 #define atomic_cmpset_acq_short atomic_cmpset_acq_16 #define atomic_load_acq_short atomic_load_acq_16 #define atomic_set_acq_short atomic_set_acq_16 #define atomic_subtract_acq_short atomic_subtract_acq_16 #define atomic_testandset_acq_short atomic_testandset_acq_16 #define atomic_add_rel_short atomic_add_rel_16 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 #define atomic_clear_rel_short atomic_clear_rel_16 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 #define atomic_set_rel_short atomic_set_rel_16 #define atomic_subtract_rel_short atomic_subtract_rel_16 #define atomic_store_rel_short atomic_store_rel_16 #define atomic_add_int atomic_add_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_swap_int atomic_swap_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_testandclear_int atomic_testandclear_32 #define atomic_testandset_int atomic_testandset_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_testandset_acq_int atomic_testandset_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_add_long atomic_add_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_swap_long atomic_swap_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_testandclear_long atomic_testandclear_64 #define atomic_testandset_long atomic_testandset_64 #define atomic_add_ptr atomic_add_64 #define atomic_fcmpset_ptr atomic_fcmpset_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_swap_ptr atomic_swap_64 #define atomic_subtract_ptr atomic_subtract_64 +#define atomic_testandclear_ptr atomic_testandclear_64 +#define atomic_testandset_ptr atomic_testandset_64 #define atomic_add_acq_long atomic_add_acq_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_testandset_acq_long atomic_testandset_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 #define atomic_clear_acq_ptr atomic_clear_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 static __inline void atomic_thread_fence_acq(void) { dmb(ld); } static __inline void atomic_thread_fence_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_acq_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_seq_cst(void) { dmb(sy); } #endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* _MACHINE_ATOMIC_H_ */ #endif /* !__arm__ */ diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index 144b8f1d114c..f48ad55b8029 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -1,881 +1,885 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #include #ifdef _KERNEL #include #include #endif #ifndef __OFFSETOF_MONITORBUF /* * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in i386/vm_machdep.c ensures that the value is correct. */ #define __OFFSETOF_MONITORBUF 0x80 static __inline void __mbk(void) { __asm __volatile("lock; addl $0,%%fs:%c0" : : "i" (__OFFSETOF_MONITORBUF) : "memory", "cc"); } static __inline void __mbu(void) { __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); } #endif /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ /* * Always use lock prefixes. The result is slightly less optimal for * UP systems, but it matters less now, and sometimes UP is emulated * over SMP. * * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions. * * cmpset: * if (*dst == expect) * *dst = src * * fcmpset: * if (*dst == *expect) * *dst = src * else * *expect = *dst * * Returns 0 on failure, non-zero on success. */ #define ATOMIC_CMPSET(TYPE, CONS) \ static __inline int \ atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ " sete %0 ; " \ "# atomic_cmpset_" #TYPE " " \ : "=q" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (expect) /* 2 */ \ : CONS (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } \ \ static __inline int \ atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ " sete %0 ; " \ "# atomic_fcmpset_" #TYPE " " \ : "=q" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (*expect) /* 2 */ \ : CONS (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } ATOMIC_CMPSET(char, "q"); ATOMIC_CMPSET(short, "r"); ATOMIC_CMPSET(int, "r"); /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " lock; xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btsl %2,%1 ; " " setc %0 ; " "# atomic_testandset_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btrl %2,%1 ; " " setc %0 ; " "# atomic_testandclear_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. In the kernel, we use a private per-cpu * cache line for "mem", to avoid introducing false data * dependencies. In user space, we use the word at the top of the * stack. * * For UP kernels, however, the memory of the single processor is * always consistent, so we only need to stop the compiler from * reordering accesses in a way that violates the semantics of acquire * and release. */ #if defined(_KERNEL) #define __storeload_barrier() __mbk() #else /* !_KERNEL */ #define __storeload_barrier() __mbu() #endif /* _KERNEL*/ #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #ifdef _KERNEL #ifdef WANT_FUNCTIONS int atomic_cmpset_64_i386(volatile uint64_t *, uint64_t, uint64_t); int atomic_cmpset_64_i586(volatile uint64_t *, uint64_t, uint64_t); uint64_t atomic_load_acq_64_i386(volatile uint64_t *); uint64_t atomic_load_acq_64_i586(volatile uint64_t *); void atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); void atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64_i386(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64_i586(volatile uint64_t *, uint64_t); #endif /* I486 does not support SMP or CMPXCHG8B. */ static __inline int atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) { volatile uint32_t *p; u_char res; p = (volatile uint32_t *)dst; __asm __volatile( " pushfl ; " " cli ; " " xorl %1,%%eax ; " " xorl %2,%%edx ; " " orl %%edx,%%eax ; " " jne 1f ; " " movl %4,%1 ; " " movl %5,%2 ; " "1: " " sete %3 ; " " popfl" : "+A" (expect), /* 0 */ "+m" (*p), /* 1 */ "+m" (*(p + 1)), /* 2 */ "=q" (res) /* 3 */ : "r" ((uint32_t)src), /* 4 */ "r" ((uint32_t)(src >> 32)) /* 5 */ : "memory", "cc"); return (res); } static __inline int atomic_fcmpset_64_i386(volatile uint64_t *dst, uint64_t *expect, uint64_t src) { if (atomic_cmpset_64_i386(dst, *expect, src)) { return (1); } else { *expect = *dst; return (0); } } static __inline uint64_t atomic_load_acq_64_i386(volatile uint64_t *p) { volatile uint32_t *q; uint64_t res; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %1,%%eax ; " " movl %2,%%edx ; " " popfl" : "=&A" (res) /* 0 */ : "m" (*q), /* 1 */ "m" (*(q + 1)) /* 2 */ : "memory"); return (res); } static __inline void atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) { volatile uint32_t *q; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %%eax,%0 ; " " movl %%edx,%1 ; " " popfl" : "=m" (*q), /* 0 */ "=m" (*(q + 1)) /* 1 */ : "A" (v) /* 2 */ : "memory"); } static __inline uint64_t atomic_swap_64_i386(volatile uint64_t *p, uint64_t v) { volatile uint32_t *q; uint64_t res; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %1,%%eax ; " " movl %2,%%edx ; " " movl %4,%2 ; " " movl %3,%1 ; " " popfl" : "=&A" (res), /* 0 */ "+m" (*q), /* 1 */ "+m" (*(q + 1)) /* 2 */ : "r" ((uint32_t)v), /* 3 */ "r" ((uint32_t)(v >> 32))); /* 4 */ return (res); } static __inline int atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) { u_char res; __asm __volatile( " lock; cmpxchg8b %1 ; " " sete %0" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+A" (expect) /* 2 */ : "b" ((uint32_t)src), /* 3 */ "c" ((uint32_t)(src >> 32)) /* 4 */ : "memory", "cc"); return (res); } static __inline int atomic_fcmpset_64_i586(volatile uint64_t *dst, uint64_t *expect, uint64_t src) { u_char res; __asm __volatile( " lock; cmpxchg8b %1 ; " " sete %0" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+A" (*expect) /* 2 */ : "b" ((uint32_t)src), /* 3 */ "c" ((uint32_t)(src >> 32)) /* 4 */ : "memory", "cc"); return (res); } static __inline uint64_t atomic_load_acq_64_i586(volatile uint64_t *p) { uint64_t res; __asm __volatile( " movl %%ebx,%%eax ; " " movl %%ecx,%%edx ; " " lock; cmpxchg8b %1" : "=&A" (res), /* 0 */ "+m" (*p) /* 1 */ : : "memory", "cc"); return (res); } static __inline void atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) { __asm __volatile( " movl %%eax,%%ebx ; " " movl %%edx,%%ecx ; " "1: " " lock; cmpxchg8b %0 ; " " jne 1b" : "+m" (*p), /* 0 */ "+A" (v) /* 1 */ : : "ebx", "ecx", "memory", "cc"); } static __inline uint64_t atomic_swap_64_i586(volatile uint64_t *p, uint64_t v) { __asm __volatile( " movl %%eax,%%ebx ; " " movl %%edx,%%ecx ; " "1: " " lock; cmpxchg8b %0 ; " " jne 1b" : "+m" (*p), /* 0 */ "+A" (v) /* 1 */ : : "ebx", "ecx", "memory", "cc"); return (v); } static __inline int atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_cmpset_64_i386(dst, expect, src)); else return (atomic_cmpset_64_i586(dst, expect, src)); } static __inline int atomic_fcmpset_64(volatile uint64_t *dst, uint64_t *expect, uint64_t src) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_fcmpset_64_i386(dst, expect, src)); else return (atomic_fcmpset_64_i586(dst, expect, src)); } static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_load_acq_64_i386(p)); else return (atomic_load_acq_64_i586(p)); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t v) { if ((cpu_feature & CPUID_CX8) == 0) atomic_store_rel_64_i386(p, v); else atomic_store_rel_64_i586(p, v); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t v) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_swap_64_i386(p, v)); else return (atomic_swap_64_i586(p, v)); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) { for (;;) { uint64_t t = *p; if (atomic_cmpset_64(p, t, t + v)) return (t); } } static __inline void atomic_add_64(volatile uint64_t *p, uint64_t v) { uint64_t t; for (;;) { t = *p; if (atomic_cmpset_64(p, t, t + v)) break; } } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t v) { uint64_t t; for (;;) { t = *p; if (atomic_cmpset_64(p, t, t - v)) break; } } #endif /* _KERNEL */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, long, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS static __inline int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) { return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect, (u_int)src)); } static __inline int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src) { return (atomic_fcmpset_int((volatile u_int *)dst, (u_int *)expect, (u_int)src)); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { return (atomic_testandset_int((volatile u_int *)p, v)); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { return (atomic_testandclear_int((volatile u_int *)p, v)); } /* Read the current value and store a new value in the destination. */ static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { return (atomic_swap_int((volatile u_int *)p, (u_int)v)); } #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_cmpset_acq_char atomic_cmpset_char #define atomic_cmpset_rel_char atomic_cmpset_char #define atomic_fcmpset_acq_char atomic_fcmpset_char #define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_cmpset_acq_short atomic_cmpset_short #define atomic_cmpset_rel_short atomic_cmpset_short #define atomic_fcmpset_acq_short atomic_fcmpset_short #define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) #define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int #ifdef _KERNEL /* Operations on 64-bit quad words. */ #define atomic_cmpset_acq_64 atomic_cmpset_64 #define atomic_cmpset_rel_64 atomic_cmpset_64 #define atomic_fcmpset_acq_64 atomic_fcmpset_64 #define atomic_fcmpset_rel_64 atomic_fcmpset_64 #define atomic_fetchadd_acq_64 atomic_fetchadd_64 #define atomic_fetchadd_rel_64 atomic_fetchadd_64 #define atomic_add_acq_64 atomic_add_64 #define atomic_add_rel_64 atomic_add_64 #define atomic_subtract_acq_64 atomic_subtract_64 #define atomic_subtract_rel_64 atomic_subtract_64 #define atomic_load_64 atomic_load_acq_64 #define atomic_store_64 atomic_store_rel_64 #endif /* Operations on pointers. */ #define atomic_set_ptr(p, v) \ atomic_set_int((volatile u_int *)(p), (u_int)(v)) #define atomic_set_acq_ptr(p, v) \ atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_set_rel_ptr(p, v) \ atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_ptr(p, v) \ atomic_clear_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_acq_ptr(p, v) \ atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_rel_ptr(p, v) \ atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_ptr(p, v) \ atomic_add_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_acq_ptr(p, v) \ atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_rel_ptr(p, v) \ atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_ptr(p, v) \ atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_acq_ptr(p, v) \ atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_rel_ptr(p, v) \ atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_load_acq_ptr(p) \ atomic_load_acq_int((volatile u_int *)(p)) #define atomic_store_rel_ptr(p, v) \ atomic_store_rel_int((volatile u_int *)(p), (v)) #define atomic_cmpset_ptr(dst, old, new) \ atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) #define atomic_cmpset_acq_ptr(dst, old, new) \ atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) #define atomic_fcmpset_ptr(dst, old, new) \ atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new)) #define atomic_fcmpset_acq_ptr(dst, old, new) \ atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \ (u_int)(new)) #define atomic_fcmpset_rel_ptr(dst, old, new) \ atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \ (u_int)(new)) #define atomic_swap_ptr(p, v) \ atomic_swap_int((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_ptr(p) \ atomic_readandclear_int((volatile u_int *)(p)) +#define atomic_testandclear_ptr(p, val) \ + atomic_testandclear_int((volatile u_int *)(p), (val)) +#define atomic_testandset_ptr(p, val) \ + atomic_testandset_int((volatile u_int *)(p), (val)) #endif /* !WANT_FUNCTIONS */ #if defined(_KERNEL) #define mb() __mbk() #define wmb() __mbk() #define rmb() __mbk() #else #define mb() __mbu() #define wmb() __mbu() #define rmb() __mbu() #endif #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/kern/subr_asan.c b/sys/kern/subr_asan.c index 75d9e75c531a..61466ffbf2ab 100644 --- a/sys/kern/subr_asan.c +++ b/sys/kern/subr_asan.c @@ -1,1237 +1,1239 @@ /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */ /* * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net * All rights reserved. * * This code is part of the KASAN subsystem of the NetBSD kernel. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #define SAN_RUNTIME #include #if 0 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $"); #endif #include #include #include #include #include #include #include #include #include /* ASAN constants. Part of the compiler ABI. */ #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1) #define KASAN_ALLOCA_SCALE_SIZE 32 /* ASAN ABI version. */ #if defined(__clang__) && (__clang_major__ - 0 >= 6) #define ASAN_ABI_VERSION 8 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) #define ASAN_ABI_VERSION 8 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) #define ASAN_ABI_VERSION 6 #else #error "Unsupported compiler version" #endif #define __RET_ADDR (unsigned long)__builtin_return_address(0) /* Global variable descriptor. Part of the compiler ABI. */ struct __asan_global_source_location { const char *filename; int line_no; int column_no; }; struct __asan_global { const void *beg; /* address of the global variable */ size_t size; /* size of the global variable */ size_t size_with_redzone; /* size with the redzone */ const void *name; /* name of the variable */ const void *module_name; /* name of the module where the var is declared */ unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ struct __asan_global_source_location *location; #if ASAN_ABI_VERSION >= 7 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ #endif }; FEATURE(kasan, "Kernel address sanitizer"); static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "KASAN options"); static int panic_on_violation = 1; SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN, &panic_on_violation, 0, "Panic if an invalid access is detected"); #define kasan_enabled (!kasan_disabled) static bool kasan_disabled __read_mostly = true; SYSCTL_BOOL(_debug_kasan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &kasan_disabled, 0, "KASAN is disabled"); /* -------------------------------------------------------------------------- */ void kasan_shadow_map(vm_offset_t addr, size_t size) { size_t sz, npages, i; vm_offset_t sva, eva; KASSERT(addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %#lx", __func__, addr)); sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE; sva = kasan_md_addr_to_shad(addr); eva = kasan_md_addr_to_shad(addr) + sz; sva = rounddown(sva, PAGE_SIZE); eva = roundup(eva, PAGE_SIZE); npages = (eva - sva) / PAGE_SIZE; KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS, ("%s: invalid address range %#lx-%#lx", __func__, sva, eva)); for (i = 0; i < npages; i++) pmap_san_enter(sva + ptoa(i)); } void kasan_init(void) { int disabled; disabled = 0; TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled); if (disabled) return; /* MD initialization. */ kasan_md_init(); /* Now officially enabled. */ kasan_disabled = false; } void kasan_init_early(vm_offset_t stack, size_t size) { kasan_md_init_early(stack, size); } static inline const char * kasan_code_name(uint8_t code) { switch (code) { case KASAN_GENERIC_REDZONE: return "GenericRedZone"; case KASAN_MALLOC_REDZONE: return "MallocRedZone"; case KASAN_KMEM_REDZONE: return "KmemRedZone"; case KASAN_UMA_FREED: return "UMAUseAfterFree"; case KASAN_KSTACK_FREED: return "KernelStack"; case KASAN_EXEC_ARGS_FREED: return "ExecKVA"; case 1 ... 7: return "RedZonePartial"; case KASAN_STACK_LEFT: return "StackLeft"; case KASAN_STACK_MID: return "StackMiddle"; case KASAN_STACK_RIGHT: return "StackRight"; case KASAN_USE_AFTER_RET: return "UseAfterRet"; case KASAN_USE_AFTER_SCOPE: return "UseAfterScope"; default: return "Unknown"; } } #define REPORT(f, ...) do { \ if (panic_on_violation) { \ kasan_disabled = true; \ panic(f, __VA_ARGS__); \ } else { \ struct stack st; \ \ stack_save(&st); \ printf(f "\n", __VA_ARGS__); \ stack_print_ddb(&st); \ } \ } while (0) static void kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, uint8_t code) { REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)", size, (write ? "write" : "read"), addr, kasan_code_name(code), code); } static __always_inline void kasan_shadow_1byte_markvalid(unsigned long addr) { int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); int8_t last = (addr & KASAN_SHADOW_MASK) + 1; *byte = last; } static __always_inline void kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) { size_t i; for (i = 0; i < size; i++) { kasan_shadow_1byte_markvalid((unsigned long)addr + i); } } static __always_inline void kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) { void *shad; if (__predict_false(size == 0)) return; if (__predict_false(kasan_md_unsupported((vm_offset_t)addr))) return; KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %p", __func__, addr)); KASSERT(size % KASAN_SHADOW_SCALE == 0, ("%s: invalid size %zu", __func__, size)); shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr); size = size >> KASAN_SHADOW_SCALE_SHIFT; __builtin_memset(shad, code, size); } /* * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, * and the rest as invalid. There are generally two use cases: * * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks * the redzone at the end of the buffer as invalid. If the entire is to be * marked invalid, origsize will be 0. * * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. */ void kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code) { size_t i, n, redz; int8_t *shad; if (__predict_false(!kasan_enabled)) return; if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS && (vm_offset_t)addr < DMAP_MAX_ADDRESS) return; KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS && (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS, ("%s: invalid address %p", __func__, addr)); KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %p", __func__, addr)); redz = redzsize - roundup(size, KASAN_SHADOW_SCALE); KASSERT(redz % KASAN_SHADOW_SCALE == 0, ("%s: invalid size %zu", __func__, redz)); shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr); /* Chunks of 8 bytes, valid. */ n = size / KASAN_SHADOW_SCALE; for (i = 0; i < n; i++) { *shad++ = 0; } /* Possibly one chunk, mid. */ if ((size & KASAN_SHADOW_MASK) != 0) { *shad++ = (size & KASAN_SHADOW_MASK); } /* Chunks of 8 bytes, invalid. */ n = redz / KASAN_SHADOW_SCALE; for (i = 0; i < n; i++) { *shad++ = code; } } void kasan_thread_alloc(struct thread *td) { if (td->td_kstack != 0) { kasan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), ptoa(td->td_kstack_pages), 0); } } /* -------------------------------------------------------------------------- */ #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) static __always_inline bool kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); int8_t last = (addr & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { return (kasan_shadow_1byte_isvalid(addr, code) && kasan_shadow_1byte_isvalid(addr+1, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { return (kasan_shadow_2byte_isvalid(addr, code) && kasan_shadow_2byte_isvalid(addr+2, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { return (kasan_shadow_4byte_isvalid(addr, code) && kasan_shadow_4byte_isvalid(addr+4, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) { size_t i; for (i = 0; i < size; i++) { if (!kasan_shadow_1byte_isvalid(addr+i, code)) return (false); } return (true); } static __always_inline void kasan_shadow_check(unsigned long addr, size_t size, bool write, unsigned long retaddr) { uint8_t code; bool valid; if (__predict_false(!kasan_enabled)) return; if (__predict_false(size == 0)) return; if (__predict_false(kasan_md_unsupported(addr))) return; if (KERNEL_PANICKED()) return; if (__builtin_constant_p(size)) { switch (size) { case 1: valid = kasan_shadow_1byte_isvalid(addr, &code); break; case 2: valid = kasan_shadow_2byte_isvalid(addr, &code); break; case 4: valid = kasan_shadow_4byte_isvalid(addr, &code); break; case 8: valid = kasan_shadow_8byte_isvalid(addr, &code); break; default: valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); break; } } else { valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); } if (__predict_false(!valid)) { kasan_report(addr, size, write, retaddr, code); } } /* -------------------------------------------------------------------------- */ void * kasan_memcpy(void *dst, const void *src, size_t len) { kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); return (__builtin_memcpy(dst, src, len)); } int kasan_memcmp(const void *b1, const void *b2, size_t len) { kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); return (__builtin_memcmp(b1, b2, len)); } void * kasan_memset(void *b, int c, size_t len) { kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); return (__builtin_memset(b, c, len)); } void * kasan_memmove(void *dst, const void *src, size_t len) { kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); return (__builtin_memmove(dst, src, len)); } size_t kasan_strlen(const char *str) { const char *s; s = str; while (1) { kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); if (*s == '\0') break; s++; } return (s - str); } char * kasan_strcpy(char *dst, const char *src) { char *save = dst; while (1) { kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); *dst = *src; if (*src == '\0') break; src++, dst++; } return save; } int kasan_strcmp(const char *s1, const char *s2) { while (1) { kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); if (*s1 != *s2) break; if (*s1 == '\0') return 0; s1++, s2++; } return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } int kasan_copyin(const void *uaddr, void *kaddr, size_t len) { kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); return (copyin(uaddr, kaddr, len)); } int kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); return (copyinstr(uaddr, kaddr, len, done)); } int kasan_copyout(const void *kaddr, void *uaddr, size_t len) { kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); return (copyout(kaddr, uaddr, len)); } /* -------------------------------------------------------------------------- */ int kasan_fubyte(volatile const void *base) { return (fubyte(base)); } int kasan_fuword16(volatile const void *base) { return (fuword16(base)); } int kasan_fueword(volatile const void *base, long *val) { kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR); return (fueword(base, val)); } int kasan_fueword32(volatile const void *base, int32_t *val) { kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR); return (fueword32(base, val)); } int kasan_fueword64(volatile const void *base, int64_t *val) { kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR); return (fueword64(base, val)); } int kasan_subyte(volatile void *base, int byte) { return (subyte(base, byte)); } int kasan_suword(volatile void *base, long word) { return (suword(base, word)); } int kasan_suword16(volatile void *base, int word) { return (suword16(base, word)); } int kasan_suword32(volatile void *base, int32_t word) { return (suword32(base, word)); } int kasan_suword64(volatile void *base, int64_t word) { return (suword64(base, word)); } int kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval) { kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true, __RET_ADDR); return (casueword32(base, oldval, oldvalp, newval)); } int kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval) { kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true, __RET_ADDR); return (casueword(base, oldval, oldvalp, newval)); } /* -------------------------------------------------------------------------- */ #include #include #define _ASAN_ATOMIC_FUNC_ADD(name, type) \ void kasan_atomic_add_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_ADD(name, type) \ _ASAN_ATOMIC_FUNC_ADD(name, type) \ _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _ASAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kasan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _ASAN_ATOMIC_FUNC_SET(name, type) \ void kasan_atomic_set_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_SET(name, type) \ _ASAN_ATOMIC_FUNC_SET(name, type) \ _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_SET(rel_##name, type) #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ void kasan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kasan_atomic_readandclear_##name(volatile type *ptr) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_testandclear_##name(ptr, v)); \ } #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, v)); \ } #define ASAN_ATOMIC_FUNC_SWAP(name, type) \ type kasan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_swap_##name(ptr, val)); \ } #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \ type nval) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, oval, nval)); \ } #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ type nval) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, oval, nval)); \ } #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kasan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \ type kasan_atomic_load_##name(volatile type *ptr) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } #define ASAN_ATOMIC_FUNC_LOAD(name, type) \ _ASAN_ATOMIC_FUNC_LOAD(name, type) \ _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type) #define _ASAN_ATOMIC_FUNC_STORE(name, type) \ void kasan_atomic_store_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_STORE(name, type) \ _ASAN_ATOMIC_FUNC_STORE(name, type) \ _ASAN_ATOMIC_FUNC_STORE(rel_##name, type) ASAN_ATOMIC_FUNC_ADD(8, uint8_t); ASAN_ATOMIC_FUNC_ADD(16, uint16_t); ASAN_ATOMIC_FUNC_ADD(32, uint32_t); ASAN_ATOMIC_FUNC_ADD(64, uint64_t); ASAN_ATOMIC_FUNC_ADD(int, u_int); ASAN_ATOMIC_FUNC_ADD(long, u_long); ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int); ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long); ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SET(8, uint8_t); ASAN_ATOMIC_FUNC_SET(16, uint16_t); ASAN_ATOMIC_FUNC_SET(32, uint32_t); ASAN_ATOMIC_FUNC_SET(64, uint64_t); ASAN_ATOMIC_FUNC_SET(int, u_int); ASAN_ATOMIC_FUNC_SET(long, u_long); ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_CLEAR(int, u_int); ASAN_ATOMIC_FUNC_CLEAR(long, u_long); ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); ASAN_ATOMIC_FUNC_FETCHADD(int, u_int); ASAN_ATOMIC_FUNC_FETCHADD(long, u_long); ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); +ASAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int); ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long); +ASAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SWAP(32, uint32_t); ASAN_ATOMIC_FUNC_SWAP(64, uint64_t); ASAN_ATOMIC_FUNC_SWAP(int, u_int); ASAN_ATOMIC_FUNC_SWAP(long, u_long); ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t); ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t); ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t); ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t); ASAN_ATOMIC_FUNC_CMPSET(int, u_int); ASAN_ATOMIC_FUNC_CMPSET(long, u_long); ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); ASAN_ATOMIC_FUNC_FCMPSET(int, u_int); ASAN_ATOMIC_FUNC_FCMPSET(long, u_long); ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); _ASAN_ATOMIC_FUNC_LOAD(bool, bool); ASAN_ATOMIC_FUNC_LOAD(8, uint8_t); ASAN_ATOMIC_FUNC_LOAD(16, uint16_t); ASAN_ATOMIC_FUNC_LOAD(32, uint32_t); ASAN_ATOMIC_FUNC_LOAD(64, uint64_t); ASAN_ATOMIC_FUNC_LOAD(char, u_char); ASAN_ATOMIC_FUNC_LOAD(short, u_short); ASAN_ATOMIC_FUNC_LOAD(int, u_int); ASAN_ATOMIC_FUNC_LOAD(long, u_long); ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); _ASAN_ATOMIC_FUNC_STORE(bool, bool); ASAN_ATOMIC_FUNC_STORE(8, uint8_t); ASAN_ATOMIC_FUNC_STORE(16, uint16_t); ASAN_ATOMIC_FUNC_STORE(32, uint32_t); ASAN_ATOMIC_FUNC_STORE(64, uint64_t); ASAN_ATOMIC_FUNC_STORE(char, u_char); ASAN_ATOMIC_FUNC_STORE(short, u_short); ASAN_ATOMIC_FUNC_STORE(int, u_int); ASAN_ATOMIC_FUNC_STORE(long, u_long); ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); ASAN_ATOMIC_FUNC_THREAD_FENCE(acq); ASAN_ATOMIC_FUNC_THREAD_FENCE(rel); ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); void kasan_atomic_interrupt_fence(void) { } /* -------------------------------------------------------------------------- */ #include #include #include int kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } void kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #define ASAN_BUS_READ_FUNC(func, width, type) \ type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ return (bus_space_read##func##_##width(tag, hnd, \ offset)); \ } \ #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \ void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ false, __RET_ADDR); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } ASAN_BUS_READ_FUNC(, 1, uint8_t) ASAN_BUS_READ_FUNC(_stream, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) ASAN_BUS_READ_FUNC(, 2, uint16_t) ASAN_BUS_READ_FUNC(_stream, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) ASAN_BUS_READ_FUNC(, 4, uint32_t) ASAN_BUS_READ_FUNC(_stream, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) ASAN_BUS_READ_FUNC(, 8, uint64_t) #if defined(__aarch64__) ASAN_BUS_READ_FUNC(_stream, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) #endif #define ASAN_BUS_WRITE_FUNC(func, width, type) \ void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value);\ } \ #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ true, __RET_ADDR); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } ASAN_BUS_WRITE_FUNC(, 1, uint8_t) ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) ASAN_BUS_WRITE_FUNC(, 2, uint16_t) ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) ASAN_BUS_WRITE_FUNC(, 4, uint32_t) ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) ASAN_BUS_WRITE_FUNC(, 8, uint64_t) #define ASAN_BUS_SET_FUNC(func, width, type) \ void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } ASAN_BUS_SET_FUNC(multi, 1, uint8_t) ASAN_BUS_SET_FUNC(region, 1, uint8_t) ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t) ASAN_BUS_SET_FUNC(multi, 2, uint16_t) ASAN_BUS_SET_FUNC(region, 2, uint16_t) ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t) ASAN_BUS_SET_FUNC(multi, 4, uint32_t) ASAN_BUS_SET_FUNC(region, 4, uint32_t) ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #define ASAN_BUS_PEEK_FUNC(width, type) \ int kasan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *valuep) \ { \ return (bus_space_peek_##width(tag, hnd, offset, \ valuep)); \ } ASAN_BUS_PEEK_FUNC(1, uint8_t) ASAN_BUS_PEEK_FUNC(2, uint16_t) ASAN_BUS_PEEK_FUNC(4, uint32_t) ASAN_BUS_PEEK_FUNC(8, uint64_t) #define ASAN_BUS_POKE_FUNC(width, type) \ int kasan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, \ value)); \ } ASAN_BUS_POKE_FUNC(1, uint8_t) ASAN_BUS_POKE_FUNC(2, uint16_t) ASAN_BUS_POKE_FUNC(4, uint32_t) ASAN_BUS_POKE_FUNC(8, uint64_t) /* -------------------------------------------------------------------------- */ void __asan_register_globals(struct __asan_global *, size_t); void __asan_unregister_globals(struct __asan_global *, size_t); void __asan_register_globals(struct __asan_global *globals, size_t n) { size_t i; for (i = 0; i < n; i++) { kasan_mark(globals[i].beg, globals[i].size, globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); } } void __asan_unregister_globals(struct __asan_global *globals, size_t n) { size_t i; for (i = 0; i < n; i++) { kasan_mark(globals[i].beg, globals[i].size_with_redzone, globals[i].size_with_redzone, 0); } } #define ASAN_LOAD_STORE(size) \ void __asan_load##size(unsigned long); \ void __asan_load##size(unsigned long addr) \ { \ kasan_shadow_check(addr, size, false, __RET_ADDR);\ } \ void __asan_load##size##_noabort(unsigned long); \ void __asan_load##size##_noabort(unsigned long addr) \ { \ kasan_shadow_check(addr, size, false, __RET_ADDR);\ } \ void __asan_store##size(unsigned long); \ void __asan_store##size(unsigned long addr) \ { \ kasan_shadow_check(addr, size, true, __RET_ADDR);\ } \ void __asan_store##size##_noabort(unsigned long); \ void __asan_store##size##_noabort(unsigned long addr) \ { \ kasan_shadow_check(addr, size, true, __RET_ADDR);\ } ASAN_LOAD_STORE(1); ASAN_LOAD_STORE(2); ASAN_LOAD_STORE(4); ASAN_LOAD_STORE(8); ASAN_LOAD_STORE(16); void __asan_loadN(unsigned long, size_t); void __asan_loadN_noabort(unsigned long, size_t); void __asan_storeN(unsigned long, size_t); void __asan_storeN_noabort(unsigned long, size_t); void __asan_handle_no_return(void); void __asan_loadN(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, false, __RET_ADDR); } void __asan_loadN_noabort(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, false, __RET_ADDR); } void __asan_storeN(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, true, __RET_ADDR); } void __asan_storeN_noabort(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, true, __RET_ADDR); } void __asan_handle_no_return(void) { /* nothing */ } #define ASAN_SET_SHADOW(byte) \ void __asan_set_shadow_##byte(void *, size_t); \ void __asan_set_shadow_##byte(void *addr, size_t size) \ { \ __builtin_memset((void *)addr, 0x##byte, size); \ } ASAN_SET_SHADOW(00); ASAN_SET_SHADOW(f1); ASAN_SET_SHADOW(f2); ASAN_SET_SHADOW(f3); ASAN_SET_SHADOW(f5); ASAN_SET_SHADOW(f8); void __asan_poison_stack_memory(const void *, size_t); void __asan_unpoison_stack_memory(const void *, size_t); void __asan_poison_stack_memory(const void *addr, size_t size) { size = roundup(size, KASAN_SHADOW_SCALE); kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); } void __asan_unpoison_stack_memory(const void *addr, size_t size) { kasan_shadow_Nbyte_markvalid(addr, size); } void __asan_alloca_poison(const void *, size_t); void __asan_allocas_unpoison(const void *, const void *); void __asan_alloca_poison(const void *addr, size_t size) { const void *l, *r; KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0, ("%s: invalid address %p", __func__, addr)); l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), KASAN_STACK_MID); kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); } void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) { size_t size; if (__predict_false(!stkbegin)) return; if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) return; size = (uintptr_t)stkend - (uintptr_t)stkbegin; kasan_shadow_Nbyte_fill(stkbegin, size, 0); } void __asan_poison_memory_region(const void *addr, size_t size); void __asan_unpoison_memory_region(const void *addr, size_t size); void __asan_poison_memory_region(const void *addr, size_t size) { } void __asan_unpoison_memory_region(const void *addr, size_t size) { } diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c index 580e632b05fd..5d0e6607403c 100644 --- a/sys/kern/subr_csan.c +++ b/sys/kern/subr_csan.c @@ -1,906 +1,904 @@ /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */ /* * Copyright (c) 2019 The NetBSD Foundation, Inc. * All rights reserved. * Copyright (c) 2019 Andrew Turner * * This code is derived from software contributed to The NetBSD Foundation * by Maxime Villard. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #define SAN_RUNTIME #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #ifdef KCSAN_PANIC #define REPORT panic #else #define REPORT printf #endif typedef struct { uintptr_t addr; uint32_t size; bool write:1; bool atomic:1; uintptr_t pc; } csan_cell_t; typedef struct { bool inited; uint32_t cnt; csan_cell_t cell; } csan_cpu_t; static csan_cpu_t kcsan_cpus[MAXCPU]; static bool kcsan_enabled __read_mostly; #define __RET_ADDR (uintptr_t)__builtin_return_address(0) #define KCSAN_NACCESSES 1024 #define KCSAN_DELAY 10 /* 10 microseconds */ /* -------------------------------------------------------------------------- */ /* The MD code. */ #include /* -------------------------------------------------------------------------- */ static void kcsan_enable(void *dummy __unused) { printf("Enabling KCSCAN, expect reduced performance.\n"); kcsan_enabled = true; } SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); void kcsan_cpu_init(u_int cpu) { kcsan_cpus[cpu].inited = true; } /* -------------------------------------------------------------------------- */ static inline void kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) { const char *newsym, *oldsym; #ifdef DDB c_db_sym_t sym; db_expr_t offset; sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &newsym, NULL); sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &oldsym, NULL); #else newsym = ""; oldsym = ""; #endif REPORT("CSan: Racy Access " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", newcpu, (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), (void *)new->addr, new->size, (void *)new->pc, newsym, oldcpu, (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), (void *)old->addr, old->size, (void *)old->pc, oldsym); kcsan_md_unwind(); } static inline bool kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) { if (new->write && !new->atomic) return false; if (old->write && !old->atomic) return false; return true; } static inline void kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) { csan_cell_t old, new; csan_cpu_t *cpu; uint64_t intr; size_t i; if (__predict_false(!kcsan_enabled)) return; if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr))) return; if (KERNEL_PANICKED()) return; new.addr = addr; new.size = size; new.write = write; new.atomic = atomic; new.pc = pc; CPU_FOREACH(i) { __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); if (old.addr + old.size <= new.addr) continue; if (new.addr + new.size <= old.addr) continue; if (__predict_true(!old.write && !new.write)) continue; if (__predict_true(kcsan_access_is_atomic(&new, &old))) continue; kcsan_report(&new, PCPU_GET(cpuid), &old, i); break; } if (__predict_false(!kcsan_md_is_avail())) return; kcsan_md_disable_intrs(&intr); cpu = &kcsan_cpus[PCPU_GET(cpuid)]; if (__predict_false(!cpu->inited)) goto out; cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; if (__predict_true(cpu->cnt != 0)) goto out; __builtin_memcpy(&cpu->cell, &new, sizeof(new)); kcsan_md_delay(KCSAN_DELAY); __builtin_memset(&cpu->cell, 0, sizeof(new)); out: kcsan_md_enable_intrs(&intr); } #define CSAN_READ(size) \ void __tsan_read##size(uintptr_t); \ void __tsan_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } \ void __tsan_unaligned_read##size(uintptr_t); \ void __tsan_unaligned_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } CSAN_READ(1) CSAN_READ(2) CSAN_READ(4) CSAN_READ(8) CSAN_READ(16) #define CSAN_WRITE(size) \ void __tsan_write##size(uintptr_t); \ void __tsan_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } \ void __tsan_unaligned_write##size(uintptr_t); \ void __tsan_unaligned_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } CSAN_WRITE(1) CSAN_WRITE(2) CSAN_WRITE(4) CSAN_WRITE(8) CSAN_WRITE(16) void __tsan_read_range(uintptr_t, size_t); void __tsan_write_range(uintptr_t, size_t); void __tsan_read_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, false, false, __RET_ADDR); } void __tsan_write_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, true, false, __RET_ADDR); } void __tsan_init(void); void __tsan_func_entry(void *); void __tsan_func_exit(void); void __tsan_init(void) { } void __tsan_func_entry(void *call_pc) { } void __tsan_func_exit(void) { } /* -------------------------------------------------------------------------- */ void * kcsan_memcpy(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memcpy(dst, src, len); } int kcsan_memcmp(const void *b1, const void *b2, size_t len) { kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); return __builtin_memcmp(b1, b2, len); } void * kcsan_memset(void *b, int c, size_t len) { kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); return __builtin_memset(b, c, len); } void * kcsan_memmove(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memmove(dst, src, len); } __strong_reference(kcsan_memcpy, __tsan_memcpy); __strong_reference(kcsan_memset, __tsan_memset); __strong_reference(kcsan_memmove, __tsan_memmove); char * kcsan_strcpy(char *dst, const char *src) { char *save = dst; while (1) { kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); *dst = *src; if (*src == '\0') break; src++, dst++; } return save; } int kcsan_strcmp(const char *s1, const char *s2) { while (1) { kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); if (*s1 != *s2) break; if (*s1 == '\0') return 0; s1++, s2++; } return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } size_t kcsan_strlen(const char *str) { const char *s; s = str; while (1) { kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); if (*s == '\0') break; s++; } return (s - str); } int kcsan_copyin(const void *uaddr, void *kaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyin(uaddr, kaddr, len); } int kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyinstr(uaddr, kaddr, len, done); } int kcsan_copyout(const void *kaddr, void *uaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); return copyout(kaddr, uaddr, len); } /* -------------------------------------------------------------------------- */ #include #include #define _CSAN_ATOMIC_FUNC_ADD(name, type) \ void kcsan_atomic_add_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ type kcsan_atomic_load_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } #define CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kcsan_atomic_readandclear_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } #define _CSAN_ATOMIC_FUNC_SET(name, type) \ void kcsan_atomic_set_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _CSAN_ATOMIC_FUNC_STORE(name, type) \ void kcsan_atomic_store_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) #define CSAN_ATOMIC_FUNC_SWAP(name, type) \ type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_swap_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_testandclear_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, val)); \ } _CSAN_ATOMIC_FUNC_LOAD(bool, bool) _CSAN_ATOMIC_FUNC_STORE(bool, bool) CSAN_ATOMIC_FUNC_ADD(8, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) CSAN_ATOMIC_FUNC_SET(8, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) _CSAN_ATOMIC_FUNC_STORE(8, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(16, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) CSAN_ATOMIC_FUNC_SET(16, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) _CSAN_ATOMIC_FUNC_STORE(16, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(32, uint32_t) CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_SET(32, uint32_t) CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) CSAN_ATOMIC_FUNC_STORE(32, uint32_t) CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) CSAN_ATOMIC_FUNC_ADD(64, uint64_t) CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_SET(64, uint64_t) CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) CSAN_ATOMIC_FUNC_STORE(64, uint64_t) CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) CSAN_ATOMIC_FUNC_ADD(char, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_LOAD(char, uint8_t) CSAN_ATOMIC_FUNC_SET(char, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t) _CSAN_ATOMIC_FUNC_STORE(char, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_SWAP(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(short, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_LOAD(short, uint16_t) CSAN_ATOMIC_FUNC_SET(short, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t) _CSAN_ATOMIC_FUNC_STORE(short, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_SWAP(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(int, u_int) CSAN_ATOMIC_FUNC_CLEAR(int, u_int) CSAN_ATOMIC_FUNC_CMPSET(int, u_int) CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) CSAN_ATOMIC_FUNC_LOAD(int, u_int) CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_SET(int, u_int) CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) CSAN_ATOMIC_FUNC_STORE(int, u_int) CSAN_ATOMIC_FUNC_SWAP(int, u_int) CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) CSAN_ATOMIC_FUNC_ADD(long, u_long) CSAN_ATOMIC_FUNC_CLEAR(long, u_long) CSAN_ATOMIC_FUNC_CMPSET(long, u_long) CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) CSAN_ATOMIC_FUNC_LOAD(long, u_long) CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_SET(long, u_long) CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) CSAN_ATOMIC_FUNC_STORE(long, u_long) CSAN_ATOMIC_FUNC_SWAP(long, u_long) CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) #if !defined(__amd64__) CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) #endif CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) -#if 0 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) -#endif #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kcsan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) void kcsan_atomic_interrupt_fence(void) { atomic_interrupt_fence(); } /* -------------------------------------------------------------------------- */ #include #include #include int kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } #if !defined(__amd64__) int kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *handlep) { return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, boundary, flags, addrp, handlep)); } #endif void kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #define CSAN_BUS_READ_FUNC(func, width, type) \ type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ return (bus_space_read##func##_##width(tag, hnd, \ offset)); \ } \ #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ false, false, __RET_ADDR); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_READ_FUNC(, 1, uint8_t) CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_READ_FUNC(, 2, uint16_t) CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_READ_FUNC(, 4, uint32_t) CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_READ_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_WRITE_FUNC(func, width, type) \ void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value); \ } \ #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ true, false, __RET_ADDR); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_WRITE_FUNC(, 1, uint8_t) CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_WRITE_FUNC(, 2, uint16_t) CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_WRITE_FUNC(, 4, uint32_t) CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_WRITE_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_SET_FUNC(func, width, type) \ void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } CSAN_BUS_SET_FUNC(multi, 1, uint8_t) CSAN_BUS_SET_FUNC(region, 1, uint8_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) #endif CSAN_BUS_SET_FUNC(multi, 2, uint16_t) CSAN_BUS_SET_FUNC(region, 2, uint16_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) #endif CSAN_BUS_SET_FUNC(multi, 4, uint32_t) CSAN_BUS_SET_FUNC(region, 4, uint32_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #endif #if !defined(__amd64__) CSAN_BUS_SET_FUNC(multi, 8, uint64_t) CSAN_BUS_SET_FUNC(region, 8, uint64_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) #endif #endif #define CSAN_BUS_PEEK_FUNC(width, type) \ int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ kcsan_access((uintptr_t)value, sizeof(type), true, false, \ __RET_ADDR); \ return (bus_space_peek_##width(tag, hnd, offset, value)); \ } CSAN_BUS_PEEK_FUNC(1, uint8_t) CSAN_BUS_PEEK_FUNC(2, uint16_t) CSAN_BUS_PEEK_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_PEEK_FUNC(8, uint64_t) #endif #define CSAN_BUS_POKE_FUNC(width, type) \ int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, value)); \ } CSAN_BUS_POKE_FUNC(1, uint8_t) CSAN_BUS_POKE_FUNC(2, uint16_t) CSAN_BUS_POKE_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_POKE_FUNC(8, uint64_t) #endif diff --git a/sys/kern/subr_msan.c b/sys/kern/subr_msan.c index ee60d6baf3fa..52ec5ce13bd1 100644 --- a/sys/kern/subr_msan.c +++ b/sys/kern/subr_msan.c @@ -1,1585 +1,1587 @@ /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ /* * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net * All rights reserved. * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Mark Johnston under sponsorship * from the FreeBSD Foundation. * * This code is part of the KMSAN subsystem of the NetBSD kernel. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #define SAN_RUNTIME #include #if 0 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include void kmsan_init_arg(size_t); void kmsan_init_ret(size_t); /* -------------------------------------------------------------------------- */ /* * Part of the compiler ABI. */ typedef struct { uint8_t *shad; msan_orig_t *orig; } msan_meta_t; #define MSAN_PARAM_SIZE 800 #define MSAN_RETVAL_SIZE 800 typedef struct { uint8_t param_shadow[MSAN_PARAM_SIZE]; uint8_t retval_shadow[MSAN_RETVAL_SIZE]; uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; uint8_t va_arg_origin[MSAN_PARAM_SIZE]; uint64_t va_arg_overflow_size; msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; msan_orig_t retval_origin; } msan_tls_t; /* -------------------------------------------------------------------------- */ #define MSAN_NCONTEXT 4 #define MSAN_ORIG_MASK (~0x3) typedef struct kmsan_td { size_t ctx; msan_tls_t tls[MSAN_NCONTEXT]; } msan_td_t; static msan_tls_t dummy_tls; /* * Use separate dummy regions for loads and stores: stores may mark the region * as uninitialized, and that can trigger false positives. */ static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); static msan_td_t msan_thread0; static bool kmsan_reporting = false; /* * Avoid clobbering any thread-local state before we panic. */ #define kmsan_panic(f, ...) do { \ kmsan_disabled = true; \ panic(f, __VA_ARGS__); \ } while (0) #define REPORT(f, ...) do { \ if (panic_on_violation) { \ kmsan_panic(f, __VA_ARGS__); \ } else { \ struct stack st; \ \ stack_save(&st); \ printf(f "\n", __VA_ARGS__); \ stack_print_ddb(&st); \ } \ } while (0) FEATURE(kmsan, "Kernel memory sanitizer"); static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "KMSAN options"); static bool panic_on_violation = 1; SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, &panic_on_violation, 0, "Panic if an invalid access is detected"); static bool kmsan_disabled __read_mostly = true; #define kmsan_enabled (!kmsan_disabled) SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &kmsan_disabled, 0, "KMSAN is disabled"); static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); /* -------------------------------------------------------------------------- */ static inline const char * kmsan_orig_name(int type) { switch (type) { case KMSAN_TYPE_STACK: return ("stack"); case KMSAN_TYPE_KMEM: return ("kmem"); case KMSAN_TYPE_MALLOC: return ("malloc"); case KMSAN_TYPE_UMA: return ("UMA"); default: return ("unknown"); } } static void kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off, const char *hook) { const char *typename; char *var, *fn; uintptr_t ptr; long foff; char buf[128]; int type; if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) return; kmsan_reporting = true; __compiler_membar(); if (*orig == 0) { REPORT("MSan: Uninitialized memory in %s, offset %zu", hook, off); goto out; } kmsan_md_orig_decode(*orig, &type, &ptr); typename = kmsan_orig_name(type); if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, sizeof(buf), &foff) == 0) { REPORT("MSan: Uninitialized %s memory in %s, " "offset %zu/%zu, addr %p, from %s+%#lx", typename, hook, off, size, addr, buf, foff); } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { /* * The format of the string is: "----var@function". Parse it to * display a nice warning. */ var = (char *)ptr + 4; strlcpy(buf, var, sizeof(buf)); var = buf; fn = strchr(buf, '@'); *fn++ = '\0'; REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " "variable '%s' from %s", typename, hook, off, var, fn); } else { REPORT("MSan: Uninitialized %s memory in %s, " "offset %zu/%zu, addr %p, PC %p", typename, hook, off, size, addr, (void *)ptr); } out: __compiler_membar(); kmsan_reporting = false; } static void kmsan_report_inline(msan_orig_t orig, unsigned long pc) { const char *typename; char *var, *fn; uintptr_t ptr; char buf[128]; long foff; int type; if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) return; kmsan_reporting = true; __compiler_membar(); if (orig == 0) { REPORT("MSan: uninitialized variable in %p", (void *)pc); goto out; } kmsan_md_orig_decode(orig, &type, &ptr); typename = kmsan_orig_name(type); if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, sizeof(buf), &foff) == 0) { REPORT("MSan: Uninitialized %s memory from %s+%#lx", typename, buf, foff); } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { /* * The format of the string is: "----var@function". Parse it to * display a nice warning. */ var = (char *)ptr + 4; strlcpy(buf, var, sizeof(buf)); var = buf; fn = strchr(buf, '@'); *fn++ = '\0'; REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); } else { REPORT("MSan: Uninitialized %s memory, origin %x", typename, orig); } out: __compiler_membar(); kmsan_reporting = false; } /* -------------------------------------------------------------------------- */ static inline msan_meta_t kmsan_meta_get(const void *addr, size_t size, const bool write) { msan_meta_t ret; if (__predict_false(!kmsan_enabled)) { ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; ret.orig = (msan_orig_t *)msan_dummy_orig; } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; ret.orig = (msan_orig_t *)msan_dummy_orig; } else { ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & MSAN_ORIG_MASK); } return (ret); } static inline void kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) { msan_orig_t *orig; size_t i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) return; orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); size += ((uintptr_t)orig & (sizeof(*orig) - 1)); orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); for (i = 0; i < size; i += 4) { orig[i / 4] = o; } } static inline void kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) { uint8_t *shad; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported(addr))) return; shad = (uint8_t *)kmsan_md_addr_to_shad(addr); __builtin_memset(shad, c, size); } static inline void kmsan_meta_copy(void *dst, const void *src, size_t size) { uint8_t *orig_src, *orig_dst; uint8_t *shad_src, *shad_dst; msan_orig_t *_src, *_dst; size_t i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) return; if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); return; } shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); __builtin_memmove(shad_dst, shad_src, size); orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); for (i = 0; i < size; i++) { _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); *_dst = *_src; orig_src++; orig_dst++; } } static inline void kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) { msan_orig_t *orig; uint8_t *shad; size_t i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported(addr))) return; shad = (uint8_t *)kmsan_md_addr_to_shad(addr); for (i = 0; i < size; i++) { if (__predict_true(shad[i] == 0)) continue; orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i); orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); kmsan_report_hook((const char *)addr + i, orig, size, i, hook); break; } } void kmsan_init_arg(size_t n) { msan_td_t *mtd; uint8_t *arg; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(curthread == NULL)) return; mtd = curthread->td_kmsan; arg = mtd->tls[mtd->ctx].param_shadow; __builtin_memset(arg, 0, n); } void kmsan_init_ret(size_t n) { msan_td_t *mtd; uint8_t *arg; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(curthread == NULL)) return; mtd = curthread->td_kmsan; arg = mtd->tls[mtd->ctx].retval_shadow; __builtin_memset(arg, 0, n); } static void kmsan_check_arg(size_t size, const char *hook) { msan_orig_t *orig; msan_td_t *mtd; uint8_t *arg; size_t ctx, i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(curthread == NULL)) return; mtd = curthread->td_kmsan; ctx = mtd->ctx; arg = mtd->tls[ctx].param_shadow; for (i = 0; i < size; i++) { if (__predict_true(arg[i] == 0)) continue; orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)]; kmsan_report_hook((const char *)arg + i, orig, size, i, hook); break; } } void kmsan_thread_alloc(struct thread *td) { msan_td_t *mtd; if (!kmsan_enabled) return; mtd = td->td_kmsan; if (mtd == NULL) { /* We might be recycling a thread. */ kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + sizeof(int)); mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); } __builtin_memset(mtd, 0, sizeof(*mtd)); mtd->ctx = 0; if (td->td_kstack != 0) kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), KMSAN_STATE_UNINIT); td->td_kmsan = mtd; } void kmsan_thread_free(struct thread *td) { msan_td_t *mtd; if (!kmsan_enabled) return; if (__predict_false(td == curthread)) kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); mtd = td->td_kmsan; kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); free(mtd, M_KMSAN); td->td_kmsan = NULL; } void kmsan_intr_enter(void); void kmsan_intr_leave(void); void kmsan_intr_enter(void) { msan_td_t *mtd; if (__predict_false(!kmsan_enabled)) return; mtd = curthread->td_kmsan; mtd->ctx++; if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); } void kmsan_intr_leave(void) { msan_td_t *mtd; if (__predict_false(!kmsan_enabled)) return; mtd = curthread->td_kmsan; if (__predict_false(mtd->ctx == 0)) kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); mtd->ctx--; } /* -------------------------------------------------------------------------- */ void kmsan_shadow_map(vm_offset_t addr, size_t size) { size_t npages, i; vm_offset_t va; MPASS(addr % PAGE_SIZE == 0); MPASS(size % PAGE_SIZE == 0); if (!kmsan_enabled) return; npages = atop(size); va = kmsan_md_addr_to_shad(addr); for (i = 0; i < npages; i++) { pmap_san_enter(va + ptoa(i)); } va = kmsan_md_addr_to_orig(addr); for (i = 0; i < npages; i++) { pmap_san_enter(va + ptoa(i)); } } void kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) { msan_orig_t orig; orig = kmsan_md_orig_encode(type, pc); kmsan_origin_fill(addr, orig, size); } void kmsan_mark(const void *addr, size_t size, uint8_t c) { kmsan_shadow_fill((uintptr_t)addr, c, size); } void kmsan_mark_bio(const struct bio *bp, uint8_t c) { kmsan_mark(bp->bio_data, bp->bio_length, c); } void kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) { do { if ((m->m_flags & M_EXTPG) == 0) kmsan_mark(m->m_data, m->m_len, c); m = m->m_next; } while (m != NULL); } void kmsan_check(const void *p, size_t sz, const char *descr) { kmsan_shadow_check((uintptr_t)p, sz, descr); } void kmsan_check_bio(const struct bio *bp, const char *descr) { kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); } void kmsan_check_mbuf(const struct mbuf *m, const char *descr) { do { kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); } while ((m = m->m_next) != NULL); } void kmsan_check_uio(const struct uio *uio, const char *descr) { for (int i = 0; i < uio->uio_iovcnt; i++) { kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len, descr); } } void kmsan_init(void) { int disabled; disabled = 0; TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); if (disabled) return; /* Initialize the TLS for curthread. */ msan_thread0.ctx = 0; thread0.td_kmsan = &msan_thread0; /* Now officially enabled. */ kmsan_disabled = false; } /* -------------------------------------------------------------------------- */ msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) { return (kmsan_meta_get(addr, size, false)); } msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) { return (kmsan_meta_get(addr, size, true)); } #define MSAN_META_FUNC(size) \ msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ { \ return (kmsan_meta_get(addr, size, false)); \ } \ msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ { \ return (kmsan_meta_get(addr, size, true)); \ } MSAN_META_FUNC(1) MSAN_META_FUNC(2) MSAN_META_FUNC(4) MSAN_META_FUNC(8) void __msan_instrument_asm_store(const void *, size_t); msan_orig_t __msan_chain_origin(msan_orig_t); void __msan_poison(const void *, size_t); void __msan_unpoison(const void *, size_t); void __msan_poison_alloca(const void *, uint64_t, const char *); void __msan_unpoison_alloca(const void *, uint64_t); void __msan_warning(msan_orig_t); msan_tls_t *__msan_get_context_state(void); void __msan_instrument_asm_store(const void *addr, size_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); } msan_orig_t __msan_chain_origin(msan_orig_t origin) { return (origin); } void __msan_poison(const void *addr, size_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); } void __msan_unpoison(const void *addr, size_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); } void __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) { msan_orig_t orig; orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); kmsan_origin_fill(addr, orig, size); kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); } void __msan_unpoison_alloca(const void *addr, uint64_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); } void __msan_warning(msan_orig_t origin) { if (__predict_false(!kmsan_enabled)) return; kmsan_report_inline(origin, KMSAN_RET_ADDR); } msan_tls_t * __msan_get_context_state(void) { msan_td_t *mtd; /* * When APs are started, they execute some C code before curthread is * set. We have to handle that here. */ if (__predict_false(!kmsan_enabled || curthread == NULL)) return (&dummy_tls); mtd = curthread->td_kmsan; return (&mtd->tls[mtd->ctx]); } /* -------------------------------------------------------------------------- */ /* * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle * initialized areas properly. */ void * kmsan_memcpy(void *dst, const void *src, size_t len) { /* No kmsan_check_arg, because inlined. */ kmsan_init_ret(sizeof(void *)); if (__predict_true(len != 0)) { kmsan_meta_copy(dst, src, len); } return (__builtin_memcpy(dst, src, len)); } int kmsan_memcmp(const void *b1, const void *b2, size_t len) { const uint8_t *_b1 = b1, *_b2 = b2; size_t i; kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), "memcmp():args"); kmsan_init_ret(sizeof(int)); for (i = 0; i < len; i++) { if (*_b1 != *_b2) { kmsan_shadow_check((uintptr_t)b1, i + 1, "memcmp():arg1"); kmsan_shadow_check((uintptr_t)b2, i + 1, "memcmp():arg2"); return (*_b1 - *_b2); } _b1++, _b2++; } return (0); } void * kmsan_memset(void *dst, int c, size_t len) { /* No kmsan_check_arg, because inlined. */ kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); kmsan_init_ret(sizeof(void *)); return (__builtin_memset(dst, c, len)); } void * kmsan_memmove(void *dst, const void *src, size_t len) { /* No kmsan_check_arg, because inlined. */ if (__predict_true(len != 0)) { kmsan_meta_copy(dst, src, len); } kmsan_init_ret(sizeof(void *)); return (__builtin_memmove(dst, src, len)); } __strong_reference(kmsan_memcpy, __msan_memcpy); __strong_reference(kmsan_memset, __msan_memset); __strong_reference(kmsan_memmove, __msan_memmove); char * kmsan_strcpy(char *dst, const char *src) { const char *_src = src; char *_dst = dst; size_t len = 0; kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); while (1) { len++; *dst = *src; if (*src == '\0') break; src++, dst++; } kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); kmsan_init_ret(sizeof(char *)); return (_dst); } int kmsan_strcmp(const char *s1, const char *s2) { const char *_s1 = s1, *_s2 = s2; size_t len = 0; kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); kmsan_init_ret(sizeof(int)); while (1) { len++; if (*s1 != *s2) break; if (*s1 == '\0') { kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); return (0); } s1++, s2++; } kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } size_t kmsan_strlen(const char *str) { const char *s; kmsan_check_arg(sizeof(str), "strlen():args"); s = str; while (1) { if (*s == '\0') break; s++; } kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); kmsan_init_ret(sizeof(size_t)); return (s - str); } int kmsan_copyin(const void *, void *, size_t); int kmsan_copyout(const void *, void *, size_t); int kmsan_copyinstr(const void *, void *, size_t, size_t *); int kmsan_copyin(const void *uaddr, void *kaddr, size_t len) { int ret; kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), "copyin():args"); ret = copyin(uaddr, kaddr, len); if (ret == 0) kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_copyout(const void *kaddr, void *uaddr, size_t len) { kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), "copyout():args"); kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); kmsan_init_ret(sizeof(int)); return (copyout(kaddr, uaddr, len)); } int kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { size_t _done; int ret; kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len) + sizeof(done), "copyinstr():args"); ret = copyinstr(uaddr, kaddr, len, &_done); if (ret == 0) kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); if (done != NULL) { *done = _done; kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); } kmsan_init_ret(sizeof(int)); return (ret); } /* -------------------------------------------------------------------------- */ int kmsan_fubyte(volatile const void *base) { int ret; kmsan_check_arg(sizeof(base), "fubyte(): args"); ret = fubyte(base); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fuword16(volatile const void *base) { int ret; kmsan_check_arg(sizeof(base), "fuword16(): args"); ret = fuword16(base); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fueword(volatile const void *base, long *val) { int ret; kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); ret = fueword(base, val); if (ret == 0) kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, sizeof(*val)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fueword32(volatile const void *base, int32_t *val) { int ret; kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); ret = fueword32(base, val); if (ret == 0) kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, sizeof(*val)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fueword64(volatile const void *base, int64_t *val) { int ret; kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); ret = fueword64(base, val); if (ret == 0) kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, sizeof(*val)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_subyte(volatile void *base, int byte) { int ret; kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); ret = subyte(base, byte); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword(volatile void *base, long word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); ret = suword(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword16(volatile void *base, int word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); ret = suword16(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword32(volatile void *base, int32_t word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); ret = suword32(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword64(volatile void *base, int64_t word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); ret = suword64(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval) { int ret; kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + sizeof(newval), "casueword32(): args"); ret = casueword32(base, oldval, oldvalp, newval); kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, sizeof(*oldvalp)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval) { int ret; kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + sizeof(newval), "casueword32(): args"); ret = casueword(base, oldval, oldvalp, newval); kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, sizeof(*oldvalp)); kmsan_init_ret(sizeof(int)); return (ret); } /* -------------------------------------------------------------------------- */ #include #include #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ void kmsan_atomic_add_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_add_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_add_" #name "():ptr"); \ atomic_add_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_ADD(name, type) \ _MSAN_ATOMIC_FUNC_ADD(name, type) \ _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_subtract_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_subtract_" #name "():ptr"); \ atomic_subtract_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _MSAN_ATOMIC_FUNC_SET(name, type) \ void kmsan_atomic_set_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_set_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_set_" #name "():ptr"); \ atomic_set_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_SET(name, type) \ _MSAN_ATOMIC_FUNC_SET(name, type) \ _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ _MSAN_ATOMIC_FUNC_SET(rel_##name, type) #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_clear_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_clear_" #name "():ptr"); \ atomic_clear_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_fetchadd_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_fetchadd_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kmsan_atomic_readandclear_##name(volatile type *ptr) \ { \ kmsan_check_arg(sizeof(ptr), \ "atomic_readandclear_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_readandclear_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_readandclear_##name(ptr)); \ } #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(v), \ "atomic_testandclear_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_testandclear_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_testandclear_##name(ptr, v)); \ } #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(v), \ "atomic_testandset_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_testandset_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_testandset_##name(ptr, v)); \ } #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_swap_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_swap_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_swap_##name(ptr, val)); \ } #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ type nval) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ sizeof(nval), "atomic_cmpset_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_cmpset_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_cmpset_##name(ptr, oval, nval)); \ } #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ type nval) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ sizeof(nval), "atomic_fcmpset_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_fcmpset_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_fcmpset_##name(ptr, oval, nval)); \ } #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kmsan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ type kmsan_atomic_load_##name(volatile type *ptr) \ { \ kmsan_check_arg(sizeof(ptr), \ "atomic_load_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_load_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_load_##name(ptr)); \ } #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ _MSAN_ATOMIC_FUNC_LOAD(name, type) \ _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ void kmsan_atomic_store_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_store_" #name "():args"); \ kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ sizeof(type)); \ atomic_store_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_STORE(name, type) \ _MSAN_ATOMIC_FUNC_STORE(name, type) \ _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) MSAN_ATOMIC_FUNC_ADD(8, uint8_t); MSAN_ATOMIC_FUNC_ADD(16, uint16_t); MSAN_ATOMIC_FUNC_ADD(32, uint32_t); MSAN_ATOMIC_FUNC_ADD(64, uint64_t); MSAN_ATOMIC_FUNC_ADD(int, u_int); MSAN_ATOMIC_FUNC_ADD(long, u_long); MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); MSAN_ATOMIC_FUNC_SET(8, uint8_t); MSAN_ATOMIC_FUNC_SET(16, uint16_t); MSAN_ATOMIC_FUNC_SET(32, uint32_t); MSAN_ATOMIC_FUNC_SET(64, uint64_t); MSAN_ATOMIC_FUNC_SET(int, u_int); MSAN_ATOMIC_FUNC_SET(long, u_long); MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); MSAN_ATOMIC_FUNC_CLEAR(int, u_int); MSAN_ATOMIC_FUNC_CLEAR(long, u_long); MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); +MSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t); MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); +MSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t); MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); MSAN_ATOMIC_FUNC_SWAP(int, u_int); MSAN_ATOMIC_FUNC_SWAP(long, u_long); MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); MSAN_ATOMIC_FUNC_CMPSET(int, u_int); MSAN_ATOMIC_FUNC_CMPSET(long, u_long); MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); _MSAN_ATOMIC_FUNC_LOAD(bool, bool); MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); MSAN_ATOMIC_FUNC_LOAD(char, u_char); MSAN_ATOMIC_FUNC_LOAD(short, u_short); MSAN_ATOMIC_FUNC_LOAD(int, u_int); MSAN_ATOMIC_FUNC_LOAD(long, u_long); MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); _MSAN_ATOMIC_FUNC_STORE(bool, bool); MSAN_ATOMIC_FUNC_STORE(8, uint8_t); MSAN_ATOMIC_FUNC_STORE(16, uint16_t); MSAN_ATOMIC_FUNC_STORE(32, uint32_t); MSAN_ATOMIC_FUNC_STORE(64, uint64_t); MSAN_ATOMIC_FUNC_STORE(char, u_char); MSAN_ATOMIC_FUNC_STORE(short, u_short); MSAN_ATOMIC_FUNC_STORE(int, u_int); MSAN_ATOMIC_FUNC_STORE(long, u_long); MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); void kmsan_atomic_interrupt_fence(void) { atomic_interrupt_fence(); } /* -------------------------------------------------------------------------- */ #include #include #include int kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } void kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #if defined(__amd64__) #define BUS_SPACE_IO(tag) ((tag) == X86_BUS_SPACE_IO) #else #define BUS_SPACE_IO(tag) (false) #endif #define MSAN_BUS_READ_FUNC(func, width, type) \ type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ type ret; \ if (!BUS_SPACE_IO(tag)) \ kmsan_shadow_fill((uintptr_t)(hnd + offset), \ KMSAN_STATE_INITED, (width)); \ ret = bus_space_read##func##_##width(tag, hnd, offset); \ kmsan_init_ret(sizeof(type)); \ return (ret); \ } \ #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ (width) * count); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } MSAN_BUS_READ_FUNC(, 1, uint8_t) MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) MSAN_BUS_READ_FUNC(, 2, uint16_t) MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) MSAN_BUS_READ_FUNC(, 4, uint32_t) MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) MSAN_BUS_READ_FUNC(, 8, uint64_t) #ifndef __amd64__ MSAN_BUS_READ_FUNC(_stream, 8, uint64_t) MSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) MSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) MSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) MSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) #endif #define MSAN_BUS_WRITE_FUNC(func, width, type) \ void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value);\ } \ #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ "bus_space_write()"); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } MSAN_BUS_WRITE_FUNC(, 1, uint8_t) MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) MSAN_BUS_WRITE_FUNC(, 2, uint16_t) MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) MSAN_BUS_WRITE_FUNC(, 4, uint32_t) MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) MSAN_BUS_WRITE_FUNC(, 8, uint64_t) #define MSAN_BUS_SET_FUNC(func, width, type) \ void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } MSAN_BUS_SET_FUNC(multi, 1, uint8_t) MSAN_BUS_SET_FUNC(region, 1, uint8_t) MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) MSAN_BUS_SET_FUNC(multi, 2, uint16_t) MSAN_BUS_SET_FUNC(region, 2, uint16_t) MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) MSAN_BUS_SET_FUNC(multi, 4, uint32_t) MSAN_BUS_SET_FUNC(region, 4, uint32_t) MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #define MSAN_BUS_PEEK_FUNC(width, type) \ int kmsan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ return (bus_space_peek_##width(tag, hnd, offset, value)); \ } MSAN_BUS_PEEK_FUNC(1, uint8_t) MSAN_BUS_PEEK_FUNC(2, uint16_t) MSAN_BUS_PEEK_FUNC(4, uint32_t) #define MSAN_BUS_POKE_FUNC(width, type) \ int kmsan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, value)); \ } MSAN_BUS_POKE_FUNC(1, uint8_t) MSAN_BUS_POKE_FUNC(2, uint16_t) MSAN_BUS_POKE_FUNC(4, uint32_t) /* -------------------------------------------------------------------------- */ void kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) { /* * Some drivers, e.g., nvme, use the same code path for loading device * read and write requests, and will thus specify both flags. In this * case we should not do any checking since it will generally lead to * false positives. */ if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == BUS_DMASYNC_PREWRITE) { switch (desc->md_type) { case MEMDESC_VADDR: kmsan_check(desc->u.md_vaddr, desc->md_len, "dmasync"); break; case MEMDESC_MBUF: kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); break; case 0: break; default: kmsan_panic("%s: unhandled memdesc type %d", __func__, desc->md_type); } } if ((op & BUS_DMASYNC_POSTREAD) != 0) { switch (desc->md_type) { case MEMDESC_VADDR: kmsan_mark(desc->u.md_vaddr, desc->md_len, KMSAN_STATE_INITED); break; case MEMDESC_MBUF: kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); break; case 0: break; default: kmsan_panic("%s: unhandled memdesc type %d", __func__, desc->md_type); } } } diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h index 47fd9ce30fd6..0c3a57698342 100644 --- a/sys/powerpc/include/atomic.h +++ b/sys/powerpc/include/atomic.h @@ -1,1142 +1,1152 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2008 Marcel Moolenaar * Copyright (c) 2001 Benno Rice * Copyright (c) 2001 David E. O'Brien * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #include #ifndef __powerpc64__ #include #endif /* * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction * with the atomic lXarx/stXcx. sequences below. They are not exposed outside * of this file. See also Appendix B.2 of Book II of the architecture manual. * * Note that not all Book-E processors accept the light-weight sync variant. * In particular, early models of E500 cores are known to wedge. Bank on all * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs * to use the heavier-weight sync. */ #ifdef __powerpc64__ #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("lwsync" : : : "memory") #define wmb() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #else #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("sync" : : : "memory") #define wmb() __asm __volatile("sync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #endif static __inline void powerpc_lwsync(void) { #ifdef __powerpc64__ __asm __volatile("lwsync" : : : "memory"); #else __asm __volatile("sync" : : : "memory"); #endif } /* * atomic_add(p, v) * { *p += v; } */ #define __atomic_add_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_int */ #ifdef __powerpc64__ #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #else #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #endif #define _ATOMIC_ADD(type) \ static __inline void \ atomic_add_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ } \ \ static __inline void \ atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_add_##type(p, v, t); \ } \ /* _ATOMIC_ADD */ _ATOMIC_ADD(int) _ATOMIC_ADD(long) #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #ifdef __powerpc64__ #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #else #define atomic_add_ptr atomic_add_int #define atomic_add_acq_ptr atomic_add_acq_int #define atomic_add_rel_ptr atomic_add_rel_int #endif #undef _ATOMIC_ADD #undef __atomic_add_long #undef __atomic_add_int /* * atomic_clear(p, v) * { *p &= ~v; } */ #define __atomic_clear_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_int */ #ifdef __powerpc64__ #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #else #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #endif #define _ATOMIC_CLEAR(type) \ static __inline void \ atomic_clear_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ } \ \ static __inline void \ atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_clear_##type(p, v, t); \ } \ /* _ATOMIC_CLEAR */ _ATOMIC_CLEAR(int) _ATOMIC_CLEAR(long) #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #ifdef __powerpc64__ #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #else #define atomic_clear_ptr atomic_clear_int #define atomic_clear_acq_ptr atomic_clear_acq_int #define atomic_clear_rel_ptr atomic_clear_rel_int #endif #undef _ATOMIC_CLEAR #undef __atomic_clear_long #undef __atomic_clear_int /* * atomic_cmpset(p, o, n) */ /* TODO -- see below */ /* * atomic_load_acq(p) */ /* TODO -- see below */ /* * atomic_readandclear(p) */ /* TODO -- see below */ /* * atomic_set(p, v) * { *p |= v; } */ #define __atomic_set_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_int */ #ifdef __powerpc64__ #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #else #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #endif #define _ATOMIC_SET(type) \ static __inline void \ atomic_set_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ } \ \ static __inline void \ atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_set_##type(p, v, t); \ } \ /* _ATOMIC_SET */ _ATOMIC_SET(int) _ATOMIC_SET(long) #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #ifdef __powerpc64__ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #else #define atomic_set_ptr atomic_set_int #define atomic_set_acq_ptr atomic_set_acq_int #define atomic_set_rel_ptr atomic_set_rel_int #endif #undef _ATOMIC_SET #undef __atomic_set_long #undef __atomic_set_int /* * atomic_subtract(p, v) * { *p -= v; } */ #define __atomic_subtract_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_int */ #ifdef __powerpc64__ #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #else #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #endif #define _ATOMIC_SUBTRACT(type) \ static __inline void \ atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ } \ \ static __inline void \ atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_subtract_##type(p, v, t); \ } \ /* _ATOMIC_SUBTRACT */ _ATOMIC_SUBTRACT(int) _ATOMIC_SUBTRACT(long) #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #ifdef __powerpc64__ #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #else #define atomic_subtract_ptr atomic_subtract_int #define atomic_subtract_acq_ptr atomic_subtract_acq_int #define atomic_subtract_rel_ptr atomic_subtract_rel_int #endif #undef _ATOMIC_SUBTRACT #undef __atomic_subtract_long #undef __atomic_subtract_int /* * atomic_store_rel(p, v) */ /* TODO -- see below */ /* * Old/original implementations that still need revisiting. */ static __inline u_int atomic_readandclear_int(volatile u_int *addr) { u_int result,temp; __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tlwarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stwcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); return (result); } #ifdef __powerpc64__ static __inline u_long atomic_readandclear_long(volatile u_long *addr) { u_long result,temp; __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tldarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stdcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); return (result); } #endif #define atomic_readandclear_32 atomic_readandclear_int #ifdef __powerpc64__ #define atomic_readandclear_64 atomic_readandclear_long #define atomic_readandclear_ptr atomic_readandclear_long #else static __inline u_long atomic_readandclear_long(volatile u_long *addr) { return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); } #define atomic_readandclear_ptr atomic_readandclear_int #endif /* * We assume that a = b will do atomic loads and stores. */ #define ATOMIC_STORE_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE v; \ \ v = *p; \ powerpc_lwsync(); \ return (v); \ } \ \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ powerpc_lwsync(); \ *p = v; \ } ATOMIC_STORE_LOAD(int) #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #ifdef __powerpc64__ ATOMIC_STORE_LOAD(long) #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #else static __inline u_long atomic_load_acq_long(volatile u_long *addr) { return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); } static __inline void atomic_store_rel_long(volatile u_long *addr, u_long val) { atomic_store_rel_int((volatile u_int *)addr, (u_int)val); } #define atomic_load_acq_ptr atomic_load_acq_int #define atomic_store_rel_ptr atomic_store_rel_int #endif #undef ATOMIC_STORE_LOAD /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ #ifdef ISA_206_ATOMICS static __inline int atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) { int ret; __asm __volatile ( "1:\tlbarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stbcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } static __inline int atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) { int ret; __asm __volatile ( "1:\tlharx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "sthcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } #else static __inline int atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, uint32_t mask) { int ret; uint32_t tmp; __asm __volatile ( "1:\tlwarx %2, 0, %3\n\t" /* load old value */ "and %0, %2, %7\n\t" "cmplw %4, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "andc %2, %2, %7\n\t" "or %2, %2, %5\n\t" "stwcx. %2, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p), "+&r" (tmp) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), "r" (mask) : "cr0", "memory"); return (ret); } #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) #endif static __inline int atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { int ret; __asm __volatile ( "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } static __inline int atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) { int ret; __asm __volatile ( #ifdef __powerpc64__ "1:\tldarx %0, 0, %2\n\t" /* load old value */ "cmpld %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stdcx. %4, 0, %2\n\t" /* attempt to store */ #else "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ #endif "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #else "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #endif "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } #define ATOMIC_CMPSET_ACQ_REL(type) \ static __inline int \ atomic_cmpset_acq_##type(volatile u_##type *p, \ u_##type cmpval, u_##type newval)\ {\ u_##type retval; \ retval = atomic_cmpset_##type(p, cmpval, newval);\ __ATOMIC_ACQ();\ return (retval);\ }\ static __inline int \ atomic_cmpset_rel_##type(volatile u_##type *p, \ u_##type cmpval, u_##type newval)\ {\ __ATOMIC_REL();\ return (atomic_cmpset_##type(p, cmpval, newval));\ }\ struct hack ATOMIC_CMPSET_ACQ_REL(int); ATOMIC_CMPSET_ACQ_REL(long); #ifdef ISA_206_ATOMICS #define atomic_cmpset_8 atomic_cmpset_char #endif #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #ifdef ISA_206_ATOMICS #define atomic_cmpset_16 atomic_cmpset_short #endif #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #ifdef __powerpc64__ #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #else #define atomic_cmpset_ptr atomic_cmpset_int #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int #endif /* * Atomically compare the value stored at *p with *cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed and sets *cmpval to the read value from *p, * nonzero otherwise. */ #ifdef ISA_206_ATOMICS static __inline int atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) { int ret; __asm __volatile ( "lbarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stbcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stbx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } static __inline int atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) { int ret; __asm __volatile ( "lharx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "sthcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "sthx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } #endif /* ISA_206_ATOMICS */ static __inline int atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) { int ret; __asm __volatile ( "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } static __inline int atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) { int ret; __asm __volatile ( #ifdef __powerpc64__ "ldarx %0, 0, %3\n\t" /* load old value */ "cmpld %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stdcx. %5, 0, %3\n\t" /* attempt to store */ #else "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ #endif "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stdx %0, 0, %7\n\t" #else "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" #endif "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } #define ATOMIC_FCMPSET_ACQ_REL(type) \ static __inline int \ atomic_fcmpset_acq_##type(volatile u_##type *p, \ u_##type *cmpval, u_##type newval)\ {\ u_##type retval; \ retval = atomic_fcmpset_##type(p, cmpval, newval);\ __ATOMIC_ACQ();\ return (retval);\ }\ static __inline int \ atomic_fcmpset_rel_##type(volatile u_##type *p, \ u_##type *cmpval, u_##type newval)\ {\ __ATOMIC_REL();\ return (atomic_fcmpset_##type(p, cmpval, newval));\ }\ struct hack ATOMIC_FCMPSET_ACQ_REL(int); ATOMIC_FCMPSET_ACQ_REL(long); #ifdef ISA_206_ATOMICS #define atomic_fcmpset_8 atomic_fcmpset_char #endif #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char #ifdef ISA_206_ATOMICS #define atomic_fcmpset_16 atomic_fcmpset_short #endif #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #ifdef __powerpc64__ #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #else #define atomic_fcmpset_ptr atomic_fcmpset_int #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int #endif static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { u_int value; do { value = *p; } while (!atomic_cmpset_int(p, value, value + v)); return (value); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { u_long value; do { value = *p; } while (!atomic_cmpset_long(p, value, value + v)); return (value); } static __inline u_int atomic_swap_32(volatile u_int *p, u_int v) { u_int prev; __asm __volatile( "1: lwarx %0,0,%2\n" " stwcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_int *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #ifdef __powerpc64__ static __inline u_long atomic_swap_64(volatile u_long *p, u_long v) { u_long prev; __asm __volatile( "1: ldarx %0,0,%2\n" " stdcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_long *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #endif #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_swap_int atomic_swap_32 #ifdef __powerpc64__ #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_swap_long atomic_swap_64 #define atomic_swap_ptr atomic_swap_64 #else #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) #endif static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_int m = (1u << (v & 0x1f)); u_int res; u_int tmp; __asm __volatile( "1: lwarx %0,0,%3\n" " and %1,%0,%4\n" " or %0,%0,%4\n" " stwcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_int m = (1u << (v & 0x1f)); u_int res; u_int tmp; __asm __volatile( "1: lwarx %0,0,%3\n" " and %1,%0,%4\n" " andc %0,%0,%4\n" " stwcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } #ifdef __powerpc64__ static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_long m = (1ul << (v & 0x3f)); u_long res; u_long tmp; __asm __volatile( "1: ldarx %0,0,%3\n" " and %1,%0,%4\n" " or %0,%0,%4\n" " stdcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_long m = (1ul << (v & 0x3f)); u_long res; u_long tmp; __asm __volatile( "1: ldarx %0,0,%3\n" " and %1,%0,%4\n" " andc %0,%0,%4\n" " stdcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } #else static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { return (atomic_testandset_int((volatile u_int *)p, v)); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { return (atomic_testandclear_int((volatile u_int *)p, v)); } #endif #define atomic_testandclear_32 atomic_testandclear_int #define atomic_testandset_32 atomic_testandset_int static __inline int atomic_testandset_acq_long(volatile u_long *p, u_int v) { u_int a = atomic_testandset_long(p, v); __ATOMIC_ACQ(); return (a); } +#ifdef __powerpc64__ +#define atomic_testandclear_ptr atomic_testandclear_long +#define atomic_testandset_ptr atomic_testandset_long +#else +#define atomic_testandclear_ptr(p,v) \ + atomic_testandclear_32((volatile u_int *)(p), v) +#define atomic_testandset_ptr(p,v) \ + atomic_testandset_32((volatile u_int *)(p), v) +#endif + static __inline void atomic_thread_fence_acq(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_acq_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_seq_cst(void) { __asm __volatile("sync" : : : "memory"); } #ifndef ISA_206_ATOMICS #include #define atomic_cmpset_char atomic_cmpset_8 #define atomic_cmpset_short atomic_cmpset_16 #define atomic_fcmpset_char atomic_fcmpset_8 #define atomic_fcmpset_short atomic_fcmpset_16 #endif /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ ATOMIC_CMPSET_ACQ_REL(char); ATOMIC_CMPSET_ACQ_REL(short); ATOMIC_FCMPSET_ACQ_REL(char); ATOMIC_FCMPSET_ACQ_REL(short); #undef __ATOMIC_REL #undef __ATOMIC_ACQ #endif /* ! _MACHINE_ATOMIC_H_ */ diff --git a/sys/riscv/include/atomic.h b/sys/riscv/include/atomic.h index 38261e95cf31..aaa7add6894b 100644 --- a/sys/riscv/include/atomic.h +++ b/sys/riscv/include/atomic.h @@ -1,657 +1,659 @@ /*- * Copyright (c) 2015-2024 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #include #define fence() __asm __volatile("fence" ::: "memory"); #define mb() fence() #define rmb() fence() #define wmb() fence() static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *, uint16_t); #define ATOMIC_ACQ_REL(NAME, WIDTH) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ atomic_##NAME##_##WIDTH(p, v); \ fence(); \ } \ \ static __inline void \ atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ fence(); \ atomic_##NAME##_##WIDTH(p, v); \ } #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \ static __inline int \ atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ { \ int retval; \ \ retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \ fence(); \ return (retval); \ } \ \ static __inline int \ atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ { \ fence(); \ return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \ } #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \ static __inline int \ atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ { \ int retval; \ \ retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \ fence(); \ return (retval); \ } \ \ static __inline int \ atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ { \ fence(); \ return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \ } ATOMIC_CMPSET_ACQ_REL(8); ATOMIC_FCMPSET_ACQ_REL(8); #define atomic_cmpset_char atomic_cmpset_8 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 #define atomic_fcmpset_char atomic_fcmpset_8 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 #define atomic_cmpset_short atomic_cmpset_16 #define atomic_fcmpset_short atomic_fcmpset_16 ATOMIC_CMPSET_ACQ_REL(16); ATOMIC_FCMPSET_ACQ_REL(16); #define atomic_load_acq_16 atomic_load_acq_16 static __inline uint16_t atomic_load_acq_16(volatile uint16_t *p) { uint16_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_16(volatile uint16_t *p, uint16_t val) { fence(); *p = val; } #define atomic_cmpset_acq_short atomic_cmpset_acq_16 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 #define atomic_load_acq_short atomic_load_acq_16 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 #define atomic_store_rel_short atomic_store_rel_16 static __inline void atomic_add_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoadd.w zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_subtract_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoadd.w zero, %1, %0" : "+A" (*p) : "r" (-val) : "memory"); } static __inline void atomic_set_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoor.w zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_clear_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoand.w zero, %1, %0" : "+A" (*p) : "r" (~val) : "memory"); } static __inline int atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.w %0, %2\n" "bne %0, %z3, 1f\n" "sc.w %1, %z4, %2\n" "bnez %1, 0b\n" "1:" : "=&r" (tmp), "=&r" (res), "+A" (*p) : "rJ" ((long)(int32_t)cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline int atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { uint32_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.w %0, %2\n" /* Load old value */ "bne %0, %z4, 1f\n" /* Compare */ "sc.w %1, %z5, %2\n" /* Try to store new value */ "j 2f\n" "1:" "sw %0, %3\n" /* Save old value */ "2:" : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline uint32_t atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) { uint32_t ret; __asm __volatile("amoadd.w %0, %2, %1" : "=&r" (ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint32_t atomic_readandclear_32(volatile uint32_t *p) { uint32_t ret; uint32_t val; val = 0; __asm __volatile("amoswap.w %0, %2, %1" : "=&r"(ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline int atomic_testandclear_32(volatile uint32_t *p, u_int val) { uint32_t mask, old; mask = 1u << (val & 31); __asm __volatile("amoand.w %0, %2, %1" : "=&r" (old), "+A" (*p) : "r" (~mask) : "memory"); return ((old & mask) != 0); } static __inline int atomic_testandset_32(volatile uint32_t *p, u_int val) { uint32_t mask, old; mask = 1u << (val & 31); __asm __volatile("amoor.w %0, %2, %1" : "=&r" (old), "+A" (*p) : "r" (mask) : "memory"); return ((old & mask) != 0); } #define atomic_add_int atomic_add_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_subtract_int atomic_subtract_32 ATOMIC_ACQ_REL(set, 32) ATOMIC_ACQ_REL(clear, 32) ATOMIC_ACQ_REL(add, 32) ATOMIC_ACQ_REL(subtract, 32) ATOMIC_CMPSET_ACQ_REL(32); ATOMIC_FCMPSET_ACQ_REL(32); static __inline uint32_t atomic_load_acq_32(volatile uint32_t *p) { uint32_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_32(volatile uint32_t *p, uint32_t val) { fence(); *p = val; } #define atomic_add_acq_int atomic_add_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 static __inline void atomic_add_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoadd.d zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoadd.d zero, %1, %0" : "+A" (*p) : "r" (-val) : "memory"); } static __inline void atomic_set_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoor.d zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_clear_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoand.d zero, %1, %0" : "+A" (*p) : "r" (~val) : "memory"); } static __inline int atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.d %0, %2\n" "bne %0, %z3, 1f\n" "sc.d %1, %z4, %2\n" "bnez %1, 0b\n" "1:" : "=&r" (tmp), "=&r" (res), "+A" (*p) : "rJ" (cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline int atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { uint64_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.d %0, %2\n" /* Load old value */ "bne %0, %z4, 1f\n" /* Compare */ "sc.d %1, %z5, %2\n" /* Try to store new value */ "j 2f\n" "1:" "sd %0, %3\n" /* Save old value */ "2:" : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) : "rJ" (*cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) { uint64_t ret; __asm __volatile("amoadd.d %0, %2, %1" : "=&r" (ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint64_t atomic_readandclear_64(volatile uint64_t *p) { uint64_t ret; uint64_t val; val = 0; __asm __volatile("amoswap.d %0, %2, %1" : "=&r"(ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline int atomic_testandclear_64(volatile uint64_t *p, u_int val) { uint64_t mask, old; mask = 1ul << (val & 63); __asm __volatile("amoand.d %0, %2, %1" : "=&r" (old), "+A" (*p) : "r" (~mask) : "memory"); return ((old & mask) != 0); } static __inline int atomic_testandset_64(volatile uint64_t *p, u_int val) { uint64_t mask, old; mask = 1ul << (val & 63); __asm __volatile("amoor.d %0, %2, %1" : "=&r" (old), "+A" (*p) : "r" (mask) : "memory"); return ((old & mask) != 0); } static __inline int atomic_testandset_acq_64(volatile uint64_t *p, u_int val) { uint64_t mask, old; mask = 1ul << (val & 63); __asm __volatile("amoor.d.aq %0, %2, %1" : "=&r" (old), "+A" (*p) : "r" (mask) : "memory"); return ((old & mask) != 0); } static __inline uint32_t atomic_swap_32(volatile uint32_t *p, uint32_t val) { uint32_t old; __asm __volatile("amoswap.w %0, %2, %1" : "=&r"(old), "+A" (*p) : "r" (val) : "memory"); return (old); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t val) { uint64_t old; __asm __volatile("amoswap.d %0, %2, %1" : "=&r"(old), "+A" (*p) : "r" (val) : "memory"); return (old); } #define atomic_swap_int atomic_swap_32 #define atomic_add_long atomic_add_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_swap_long atomic_swap_64 #define atomic_testandclear_long atomic_testandclear_64 #define atomic_testandset_long atomic_testandset_64 #define atomic_testandset_acq_long atomic_testandset_acq_64 #define atomic_add_ptr atomic_add_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fcmpset_ptr atomic_fcmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_subtract_ptr atomic_subtract_64 #define atomic_swap_ptr atomic_swap_64 +#define atomic_testandclear_ptr atomic_testandclear_64 +#define atomic_testandset_ptr atomic_testandset_64 ATOMIC_ACQ_REL(set, 64) ATOMIC_ACQ_REL(clear, 64) ATOMIC_ACQ_REL(add, 64) ATOMIC_ACQ_REL(subtract, 64) ATOMIC_CMPSET_ACQ_REL(64); ATOMIC_FCMPSET_ACQ_REL(64); static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { uint64_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t val) { fence(); *p = val; } #define atomic_add_acq_long atomic_add_acq_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_clear_acq_ptr atomic_clear_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 #undef ATOMIC_ACQ_REL static __inline void atomic_thread_fence_acq(void) { fence(); } static __inline void atomic_thread_fence_rel(void) { fence(); } static __inline void atomic_thread_fence_acq_rel(void) { fence(); } static __inline void atomic_thread_fence_seq_cst(void) { fence(); } #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 #include #endif /* _MACHINE_ATOMIC_H_ */