diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 399d4ce62daa..c1816ac4fb8b 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -1,639 +1,639 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif /* * To express interprocessor (as opposed to processor and device) memory * ordering constraints, use the atomic_*() functions with acquire and release * semantics rather than the *mb() functions. An architecture's memory * ordering (or memory consistency) model governs the order in which a * program's accesses to different locations may be performed by an * implementation of that architecture. In general, for memory regions * defined as writeback cacheable, the memory ordering implemented by amd64 * processors preserves the program ordering of a load followed by a load, a * load followed by a store, and a store followed by a store. Only a store * followed by a load to a different memory location may be reordered. * Therefore, except for special cases, like non-temporal memory accesses or * memory regions defined as write combining, the memory ordering effects * provided by the sfence instruction in the wmb() function and the lfence * instruction in the rmb() function are redundant. In contrast, the * atomic_*() functions with acquire and release semantics do not perform * redundant instructions for ordinary cases of interprocessor memory * ordering on any architecture. */ #define mb() __asm __volatile("mfence;" : : : "memory") #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") #ifdef _KERNEL /* * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ #define OFFSETOF_MONITORBUF 0x100 #endif #if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) #include #else #include /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ #if !defined(__GNUCLIKE_ASM) #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); int atomic_fcmpset_short(volatile u_short *dst, u_short *expect, u_short src); int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandset_long(volatile u_long *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); int atomic_testandclear_long(volatile u_long *p, u_int v); void atomic_thread_fence_acq(void); void atomic_thread_fence_acq_rel(void); void atomic_thread_fence_rel(void); void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) #else /* !__GNUCLIKE_ASM */ /* * Always use lock prefixes. The result is slighly less optimal for * UP systems, but it matters less now, and sometimes UP is emulated * over SMP. * * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions. * * cmpset: * if (*dst == expect) * *dst = src * * fcmpset: * if (*dst == *expect) * *dst = src * else * *expect = *dst * * Returns 0 on failure, non-zero on success. */ #define ATOMIC_CMPSET(TYPE) \ static __inline int \ atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ "# atomic_cmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } \ \ static __inline int \ atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ "# atomic_fcmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (*expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } ATOMIC_CMPSET(char); ATOMIC_CMPSET(short); ATOMIC_CMPSET(int); ATOMIC_CMPSET(long); /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " lock; xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } /* * Atomically add the value of v to the long integer pointed to by p and return * the previous value of *p. */ static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { __asm __volatile( " lock; xaddq %0,%1 ; " "# atomic_fetchadd_long" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btsl %2,%1 ; " "# atomic_testandset_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " lock; btsq %2,%1 ; " "# atomic_testandset_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btrl %2,%1 ; " "# atomic_testandclear_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " lock; btrq %2,%1 ; " "# atomic_testandclear_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need a Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. To avoid false data dependencies, we use a * special address for "mem". In the kernel, we use a private per-cpu * cache line. In user space, we use a word in the stack's red zone * (-8(%rsp)). */ static __inline void __storeload_barrier(void) { #if defined(_KERNEL) __asm __volatile("lock; addl $0,%%gs:%0" : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); #else /* !_KERNEL */ __asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc"); #endif /* _KERNEL*/ } #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ -atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ +atomic_load_acq_##TYPE(const volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #endif /* !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orq %1,%0", "er", v); ATOMIC_ASM(clear, long, "andq %1,%0", "er", ~v); ATOMIC_ASM(add, long, "addq %1,%0", "er", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "er", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { __asm __volatile( " xchgq %1,%0 ; " "# atomic_swap_long" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } #else /* !__GNUCLIKE_ASM */ u_int atomic_swap_int(volatile u_int *p, u_int v); u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_cmpset_acq_char atomic_cmpset_char #define atomic_cmpset_rel_char atomic_cmpset_char #define atomic_fcmpset_acq_char atomic_fcmpset_char #define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_cmpset_acq_short atomic_cmpset_short #define atomic_cmpset_rel_short atomic_cmpset_short #define atomic_fcmpset_acq_short atomic_fcmpset_short #define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) #define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subtract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_testandset_64 atomic_testandset_long #define atomic_testandclear_64 atomic_testandclear_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ #endif /* !SAN_NEEDS_INTERCEPTORS || SAN_RUNTIME */ #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/arm/include/atomic-v6.h b/sys/arm/include/atomic-v6.h index c7a4de4b7e7d..ae844296f09f 100644 --- a/sys/arm/include/atomic-v6.h +++ b/sys/arm/include/atomic-v6.h @@ -1,1048 +1,1048 @@ /* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ /*- * Copyright (C) 2003-2004 Olivier Houchard * Copyright (C) 1994-1997 Mark Brinicombe * Copyright (C) 1994 Brini * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of Brini may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_V6_H_ #define _MACHINE_ATOMIC_V6_H_ #ifndef _MACHINE_ATOMIC_H_ #error Do not include this file directly, use #endif #if __ARM_ARCH >= 7 #define isb() __asm __volatile("isb" : : : "memory") #define dsb() __asm __volatile("dsb" : : : "memory") #define dmb() __asm __volatile("dmb" : : : "memory") #else #define isb() __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") #define dsb() __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") #define dmb() __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory") #endif #define mb() dmb() #define wmb() dmb() #define rmb() dmb() #define ARM_HAVE_ATOMIC64 #define ATOMIC_ACQ_REL_LONG(NAME) \ static __inline void \ atomic_##NAME##_acq_long(__volatile u_long *p, u_long v) \ { \ atomic_##NAME##_long(p, v); \ dmb(); \ } \ \ static __inline void \ atomic_##NAME##_rel_long(__volatile u_long *p, u_long v) \ { \ dmb(); \ atomic_##NAME##_long(p, v); \ } #define ATOMIC_ACQ_REL(NAME, WIDTH) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ atomic_##NAME##_##WIDTH(p, v); \ dmb(); \ } \ \ static __inline void \ atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ dmb(); \ atomic_##NAME##_##WIDTH(p, v); \ } static __inline void atomic_add_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " add %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2) ,"+r" (p), "+r" (val) : : "cc", "memory"); } static __inline void atomic_add_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " adds %Q[tmp], %Q[val] \n" " adc %R[tmp], %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_add_long(volatile u_long *p, u_long val) { atomic_add_32((volatile uint32_t *)p, val); } ATOMIC_ACQ_REL(add, 32) ATOMIC_ACQ_REL(add, 64) ATOMIC_ACQ_REL_LONG(add) static __inline void atomic_clear_32(volatile uint32_t *address, uint32_t setmask) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " bic %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask) : : "cc", "memory"); } static __inline void atomic_clear_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " bic %Q[tmp], %Q[val] \n" " bic %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_clear_long(volatile u_long *address, u_long setmask) { atomic_clear_32((volatile uint32_t *)address, setmask); } ATOMIC_ACQ_REL(clear, 32) ATOMIC_ACQ_REL(clear, 64) ATOMIC_ACQ_REL_LONG(clear) #define ATOMIC_FCMPSET_CODE(RET, TYPE, SUF) \ { \ TYPE tmp; \ \ __asm __volatile( \ "1: ldrex" SUF " %[tmp], [%[ptr]] \n" \ " ldr" SUF " %[ret], [%[oldv]] \n" \ " teq %[tmp], %[ret] \n" \ " ittee ne \n" \ " str" SUF "ne %[tmp], [%[oldv]] \n" \ " movne %[ret], #0 \n" \ " strex" SUF "eq %[ret], %[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET), \ [tmp] "=&r" (tmp) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } #define ATOMIC_FCMPSET_CODE64(RET) \ { \ uint64_t cmp, tmp; \ \ __asm __volatile( \ "1: ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" \ " ldrd %Q[cmp], %R[cmp], [%[oldv]] \n" \ " teq %Q[tmp], %Q[cmp] \n" \ " it eq \n" \ " teqeq %R[tmp], %R[cmp] \n" \ " ittee ne \n" \ " movne %[ret], #0 \n" \ " strdne %[cmp], [%[oldv]] \n" \ " strexdeq %[ret], %Q[newv], %R[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET), \ [cmp] "=&r" (cmp), \ [tmp] "=&r" (tmp) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } static __inline int atomic_fcmpset_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint8_t, "b"); return (ret); } #define atomic_fcmpset_8 atomic_fcmpset_8 static __inline int atomic_fcmpset_acq_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint8_t, "b"); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, uint8_t, "b"); return (ret); } static __inline int atomic_fcmpset_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint16_t, "h"); return (ret); } #define atomic_fcmpset_16 atomic_fcmpset_16 static __inline int atomic_fcmpset_acq_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint16_t, "h"); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, uint16_t, "h"); return (ret); } static __inline int atomic_fcmpset_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint32_t, ""); return (ret); } static __inline int atomic_fcmpset_acq_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new) { int ret; ATOMIC_FCMPSET_CODE(ret, uint32_t, ""); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_32(volatile uint32_t *_ptr, uint32_t *_old, uint32_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, uint32_t, ""); return (ret); } static __inline int atomic_fcmpset_long(volatile u_long *_ptr, u_long *_old, u_long _new) { int ret; ATOMIC_FCMPSET_CODE(ret, u_long, ""); return (ret); } static __inline int atomic_fcmpset_acq_long(volatile u_long *_ptr, u_long *_old, u_long _new) { int ret; ATOMIC_FCMPSET_CODE(ret, u_long, ""); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_long(volatile u_long *_ptr, u_long *_old, u_long _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE(ret, u_long, ""); return (ret); } static __inline int atomic_fcmpset_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new) { int ret; ATOMIC_FCMPSET_CODE64(ret); return (ret); } static __inline int atomic_fcmpset_acq_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new) { int ret; ATOMIC_FCMPSET_CODE64(ret); dmb(); return (ret); } static __inline int atomic_fcmpset_rel_64(volatile uint64_t *_ptr, uint64_t *_old, uint64_t _new) { int ret; dmb(); ATOMIC_FCMPSET_CODE64(ret); return (ret); } #define ATOMIC_CMPSET_CODE(RET, SUF) \ { \ __asm __volatile( \ "1: ldrex" SUF " %[ret], [%[ptr]] \n" \ " teq %[ret], %[oldv] \n" \ " itee ne \n" \ " movne %[ret], #0 \n" \ " strex" SUF "eq %[ret], %[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } #define ATOMIC_CMPSET_CODE64(RET) \ { \ uint64_t tmp; \ \ __asm __volatile( \ "1: ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" \ " teq %Q[tmp], %Q[oldv] \n" \ " it eq \n" \ " teqeq %R[tmp], %R[oldv] \n" \ " itee ne \n" \ " movne %[ret], #0 \n" \ " strexdeq %[ret], %Q[newv], %R[newv], [%[ptr]] \n" \ " eorseq %[ret], #1 \n" \ " beq 1b \n" \ : [ret] "=&r" (RET), \ [tmp] "=&r" (tmp) \ : [ptr] "r" (_ptr), \ [oldv] "r" (_old), \ [newv] "r" (_new) \ : "cc", "memory"); \ } static __inline int atomic_cmpset_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "b"); return (ret); } #define atomic_cmpset_8 atomic_cmpset_8 static __inline int atomic_cmpset_acq_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "b"); dmb(); return (ret); } static __inline int atomic_cmpset_rel_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, "b"); return (ret); } static __inline int atomic_cmpset_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "h"); return (ret); } #define atomic_cmpset_16 atomic_cmpset_16 static __inline int atomic_cmpset_acq_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, "h"); dmb(); return (ret); } static __inline int atomic_cmpset_rel_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, "h"); return (ret); } static __inline int atomic_cmpset_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_acq_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); dmb(); return (ret); } static __inline int atomic_cmpset_rel_32(volatile uint32_t *_ptr, uint32_t _old, uint32_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_long(volatile u_long *_ptr, u_long _old, u_long _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_acq_long(volatile u_long *_ptr, u_long _old, u_long _new) { int ret; ATOMIC_CMPSET_CODE(ret, ""); dmb(); return (ret); } static __inline int atomic_cmpset_rel_long(volatile u_long *_ptr, u_long _old, u_long _new) { int ret; dmb(); ATOMIC_CMPSET_CODE(ret, ""); return (ret); } static __inline int atomic_cmpset_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new) { int ret; ATOMIC_CMPSET_CODE64(ret); return (ret); } static __inline int atomic_cmpset_acq_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new) { int ret; ATOMIC_CMPSET_CODE64(ret); dmb(); return (ret); } static __inline int atomic_cmpset_rel_64(volatile uint64_t *_ptr, uint64_t _old, uint64_t _new) { int ret; dmb(); ATOMIC_CMPSET_CODE64(ret); return (ret); } static __inline uint32_t atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp = 0, tmp2 = 0, ret = 0; __asm __volatile( "1: ldrex %0, [%3] \n" " add %1, %0, %4 \n" " strex %2, %1, [%3] \n" " cmp %2, #0 \n" " it ne \n" " bne 1b \n" : "+r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val) : : "cc", "memory"); return (ret); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) { uint64_t ret, tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[ret], %R[ret], [%[ptr]] \n" " adds %Q[tmp], %Q[ret], %Q[val] \n" " adc %R[tmp], %R[ret], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); return (ret); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long val) { return (atomic_fetchadd_32((volatile uint32_t *)p, val)); } static __inline uint32_t -atomic_load_acq_32(volatile uint32_t *p) +atomic_load_acq_32(const volatile uint32_t *p) { uint32_t v; v = *p; dmb(); return (v); } static __inline uint64_t -atomic_load_64(volatile uint64_t *p) +atomic_load_64(const volatile uint64_t *p) { uint64_t ret; /* * The only way to atomically load 64 bits is with LDREXD which puts the * exclusive monitor into the exclusive state, so reset it to open state * with CLREX because we don't actually need to store anything. */ __asm __volatile( "ldrexd %Q[ret], %R[ret], [%[ptr]] \n" "clrex \n" : [ret] "=&r" (ret) : [ptr] "r" (p) : "cc", "memory"); return (ret); } static __inline uint64_t -atomic_load_acq_64(volatile uint64_t *p) +atomic_load_acq_64(const volatile uint64_t *p) { uint64_t ret; ret = atomic_load_64(p); dmb(); return (ret); } static __inline u_long -atomic_load_acq_long(volatile u_long *p) +atomic_load_acq_long(const volatile u_long *p) { u_long v; v = *p; dmb(); return (v); } static __inline uint32_t atomic_readandclear_32(volatile uint32_t *p) { uint32_t ret, tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%3] \n" " mov %1, #0 \n" " strex %2, %1, [%3] \n" " cmp %2, #0 \n" " it ne \n" " bne 1b \n" : "=r" (ret), "=&r" (tmp), "+r" (tmp2), "+r" (p) : : "cc", "memory"); return (ret); } static __inline uint64_t atomic_readandclear_64(volatile uint64_t *p) { uint64_t ret, tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[ret], %R[ret], [%[ptr]] \n" " mov %Q[tmp], #0 \n" " mov %R[tmp], #0 \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p) : "cc", "memory"); return (ret); } static __inline u_long atomic_readandclear_long(volatile u_long *p) { return (atomic_readandclear_32((volatile uint32_t *)p)); } static __inline void atomic_set_32(volatile uint32_t *address, uint32_t setmask) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " orr %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2), "+r" (address), "+r" (setmask) : : "cc", "memory"); } static __inline void atomic_set_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " orr %Q[tmp], %Q[val] \n" " orr %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_set_long(volatile u_long *address, u_long setmask) { atomic_set_32((volatile uint32_t *)address, setmask); } ATOMIC_ACQ_REL(set, 32) ATOMIC_ACQ_REL(set, 64) ATOMIC_ACQ_REL_LONG(set) static __inline void atomic_subtract_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp = 0, tmp2 = 0; __asm __volatile( "1: ldrex %0, [%2] \n" " sub %0, %0, %3 \n" " strex %1, %0, [%2] \n" " cmp %1, #0 \n" " it ne \n" " bne 1b \n" : "=&r" (tmp), "+r" (tmp2), "+r" (p), "+r" (val) : : "cc", "memory"); } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " subs %Q[tmp], %Q[val] \n" " sbc %R[tmp], %R[tmp], %R[val] \n" " strexd %[exf], %Q[tmp], %R[tmp], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [exf] "=&r" (exflag), [tmp] "=&r" (tmp) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_subtract_long(volatile u_long *p, u_long val) { atomic_subtract_32((volatile uint32_t *)p, val); } ATOMIC_ACQ_REL(subtract, 32) ATOMIC_ACQ_REL(subtract, 64) ATOMIC_ACQ_REL_LONG(subtract) static __inline void atomic_store_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp; uint32_t exflag; /* * The only way to atomically store 64 bits is with STREXD, which will * succeed only if paired up with a preceeding LDREXD using the same * address, so we read and discard the existing value before storing. */ __asm __volatile( "1: \n" " ldrexd %Q[tmp], %R[tmp], [%[ptr]] \n" " strexd %[exf], %Q[val], %R[val], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [tmp] "=&r" (tmp), [exf] "=&r" (exflag) : [ptr] "r" (p), [val] "r" (val) : "cc", "memory"); } static __inline void atomic_store_rel_32(volatile uint32_t *p, uint32_t v) { dmb(); *p = v; } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t val) { dmb(); atomic_store_64(p, val); } static __inline void atomic_store_rel_long(volatile u_long *p, u_long v) { dmb(); *p = v; } static __inline int atomic_testandclear_32(volatile uint32_t *ptr, u_int bit) { int newv, oldv, result; __asm __volatile( " mov ip, #1 \n" " lsl ip, ip, %[bit] \n" /* Done with %[bit] as input, reuse below as output. */ "1: \n" " ldrex %[oldv], [%[ptr]] \n" " bic %[newv], %[oldv], ip \n" " strex %[bit], %[newv], [%[ptr]] \n" " teq %[bit], #0 \n" " it ne \n" " bne 1b \n" " ands %[bit], %[oldv], ip \n" " it ne \n" " movne %[bit], #1 \n" : [bit] "=&r" (result), [oldv] "=&r" (oldv), [newv] "=&r" (newv) : [ptr] "r" (ptr), "[bit]" (bit & 0x1f) : "cc", "ip", "memory"); return (result); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { return (atomic_testandclear_32((volatile uint32_t *)p, v)); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { return (atomic_testandclear_32((volatile uint32_t *)p, v)); } #define atomic_testandclear_long atomic_testandclear_long static __inline int atomic_testandclear_64(volatile uint64_t *p, u_int v) { volatile uint32_t *p32; p32 = (volatile uint32_t *)p; /* * Assume little-endian, * atomic_testandclear_32() uses only last 5 bits of v */ if ((v & 0x20) != 0) p32++; return (atomic_testandclear_32(p32, v)); } static __inline int atomic_testandset_32(volatile uint32_t *ptr, u_int bit) { int newv, oldv, result; __asm __volatile( " mov ip, #1 \n" " lsl ip, ip, %[bit] \n" /* Done with %[bit] as input, reuse below as output. */ "1: \n" " ldrex %[oldv], [%[ptr]] \n" " orr %[newv], %[oldv], ip \n" " strex %[bit], %[newv], [%[ptr]] \n" " teq %[bit], #0 \n" " it ne \n" " bne 1b \n" " ands %[bit], %[oldv], ip \n" " it ne \n" " movne %[bit], #1 \n" : [bit] "=&r" (result), [oldv] "=&r" (oldv), [newv] "=&r" (newv) : [ptr] "r" (ptr), "[bit]" (bit & 0x1f) : "cc", "ip", "memory"); return (result); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { return (atomic_testandset_32((volatile uint32_t *)p, v)); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { return (atomic_testandset_32((volatile uint32_t *)p, v)); } #define atomic_testandset_long atomic_testandset_long static __inline int atomic_testandset_64(volatile uint64_t *p, u_int v) { volatile uint32_t *p32; p32 = (volatile uint32_t *)p; /* * Assume little-endian, * atomic_testandset_32() uses only last 5 bits of v */ if ((v & 0x20) != 0) p32++; return (atomic_testandset_32(p32, v)); } static __inline uint32_t atomic_swap_32(volatile uint32_t *p, uint32_t v) { uint32_t ret, exflag; __asm __volatile( "1: ldrex %[ret], [%[ptr]] \n" " strex %[exf], %[val], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag) : [val] "r" (v), [ptr] "r" (p) : "cc", "memory"); return (ret); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t v) { uint64_t ret; uint32_t exflag; __asm __volatile( "1: ldrexd %Q[ret], %R[ret], [%[ptr]] \n" " strexd %[exf], %Q[val], %R[val], [%[ptr]] \n" " teq %[exf], #0 \n" " it ne \n" " bne 1b \n" : [ret] "=&r" (ret), [exf] "=&r" (exflag) : [val] "r" (v), [ptr] "r" (p) : "cc", "memory"); return (ret); } #undef ATOMIC_ACQ_REL #undef ATOMIC_ACQ_REL_LONG static __inline void atomic_thread_fence_acq(void) { dmb(); } static __inline void atomic_thread_fence_rel(void) { dmb(); } static __inline void atomic_thread_fence_acq_rel(void) { dmb(); } static __inline void atomic_thread_fence_seq_cst(void) { dmb(); } #endif /* _MACHINE_ATOMIC_V6_H_ */ diff --git a/sys/arm64/include/atomic.h b/sys/arm64/include/atomic.h index c2a8caceeb64..72fae5703fb7 100644 --- a/sys/arm64/include/atomic.h +++ b/sys/arm64/include/atomic.h @@ -1,667 +1,667 @@ /*- * Copyright (c) 2013 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #define isb() __asm __volatile("isb" : : : "memory") /* * Options for DMB and DSB: * oshld Outer Shareable, load * oshst Outer Shareable, store * osh Outer Shareable, all * nshld Non-shareable, load * nshst Non-shareable, store * nsh Non-shareable, all * ishld Inner Shareable, load * ishst Inner Shareable, store * ish Inner Shareable, all * ld Full system, load * st Full system, store * sy Full system, all */ #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") #define mb() dmb(sy) /* Full system memory barrier all */ #define wmb() dmb(st) /* Full system memory barrier store */ #define rmb() dmb(ld) /* Full system memory barrier load */ #if defined(KCSAN) && !defined(KCSAN_RUNTIME) #include #else #include #ifdef _KERNEL extern bool lse_supported; #ifdef LSE_ATOMICS #define _ATOMIC_LSE_SUPPORTED 1 #else #define _ATOMIC_LSE_SUPPORTED lse_supported #endif #else #define _ATOMIC_LSE_SUPPORTED 0 #endif #define _ATOMIC_OP_PROTO(t, op, bar, flav) \ static __inline void \ atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ _ATOMIC_OP_PROTO(t, op, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ \ pre; \ __asm __volatile( \ "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \ " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \ " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \ " cbnz %w1, 1b\n" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } \ \ _ATOMIC_OP_PROTO(t, op, bar, _lse) \ { \ uint##t##_t tmp; \ \ pre; \ __asm __volatile( \ ".arch_extension lse\n" \ "ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (tmp) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } \ \ _ATOMIC_OP_PROTO(t, op, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ atomic_##op##_##bar##t##_lse(p, val); \ else \ atomic_##op##_##bar##t##_llsc(p, val); \ } #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \ _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) \ _ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \ bar, a, l) #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l) _ATOMIC_OP(add, add, add, ) _ATOMIC_OP(clear, bic, clr, ) _ATOMIC_OP(set, orr, set, ) _ATOMIC_OP(subtract, add, add, val = -val) #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \ static __inline int \ atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \ uint##t##_t cmpval, uint##t##_t newval) #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \ static __inline int \ atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \ uint##t##_t *cmpval, uint##t##_t newval) #define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \ _ATOMIC_CMPSET_PROTO(t, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ \ __asm __volatile( \ "1: mov %w1, #1\n" \ " ld"#a"xr"#s" %"#w"0, [%2]\n" \ " cmp %"#w"0, %"#w"3\n" \ " b.ne 2f\n" \ " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ " cbnz %w1, 1b\n" \ "2:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (cmpval), "r" (newval) \ : "cc", "memory" \ ); \ \ return (!res); \ } \ \ _ATOMIC_CMPSET_PROTO(t, bar, _lse) \ { \ uint##t##_t oldval; \ int res; \ \ oldval = cmpval; \ __asm __volatile( \ ".arch_extension lse\n" \ "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ "cmp %"#w"1, %"#w"2\n" \ "cset %w0, eq\n" \ ".arch_extension nolse\n" \ : "=r" (res), "+&r" (cmpval) \ : "r" (oldval), "r" (p), "r" (newval) \ : "cc", "memory" \ ); \ \ return (res); \ } \ \ _ATOMIC_CMPSET_PROTO(t, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_cmpset_##bar##t##_lse(p, cmpval, \ newval)); \ else \ return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \ newval)); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ { \ uint##t##_t _cmpval, tmp; \ int res; \ \ _cmpval = *cmpval; \ __asm __volatile( \ " mov %w1, #1\n" \ " ld"#a"xr"#s" %"#w"0, [%2]\n" \ " cmp %"#w"0, %"#w"3\n" \ " b.ne 1f\n" \ " st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \ "1:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (_cmpval), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (!res); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \ { \ uint##t##_t _cmpval, tmp; \ int res; \ \ _cmpval = tmp = *cmpval; \ __asm __volatile( \ ".arch_extension lse\n" \ "cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \ "cmp %"#w"1, %"#w"2\n" \ "cset %w0, eq\n" \ ".arch_extension nolse\n" \ : "=r" (res), "+&r" (tmp) \ : "r" (_cmpval), "r" (p), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (res); \ } \ \ _ATOMIC_FCMPSET_PROTO(t, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \ newval)); \ else \ return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \ newval)); \ } #define _ATOMIC_CMPSET(bar, a, l) \ _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \ _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \ _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \ _ATOMIC_CMPSET_IMPL(64, , , bar, a, l) #define atomic_cmpset_8 atomic_cmpset_8 #define atomic_fcmpset_8 atomic_fcmpset_8 #define atomic_cmpset_16 atomic_cmpset_16 #define atomic_fcmpset_16 atomic_fcmpset_16 _ATOMIC_CMPSET( , , ) _ATOMIC_CMPSET(acq_, a, ) _ATOMIC_CMPSET(rel_, ,l) #define _ATOMIC_FETCHADD_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_FETCHADD_IMPL(t, w) \ _ATOMIC_FETCHADD_PROTO(t, _llsc) \ { \ uint##t##_t ret, tmp; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"2, [%3]\n" \ " add %"#w"0, %"#w"2, %"#w"4\n" \ " stxr %w1, %"#w"0, [%3]\n" \ " cbnz %w1, 1b\n" \ : "=&r" (tmp), "=&r" (res), "=&r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_FETCHADD_PROTO(t, _lse) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ ".arch_extension lse\n" \ "ldadd %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_FETCHADD_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_fetchadd_##t##_lse(p, val)); \ else \ return (atomic_fetchadd_##t##_llsc(p, val)); \ } _ATOMIC_FETCHADD_IMPL(32, w) _ATOMIC_FETCHADD_IMPL(64, ) #define _ATOMIC_SWAP_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val) #define _ATOMIC_READANDCLEAR_PROTO(t, flav) \ static __inline uint##t##_t \ atomic_readandclear_##t##flav(volatile uint##t##_t *p) #define _ATOMIC_SWAP_IMPL(t, w, zreg) \ _ATOMIC_SWAP_PROTO(t, _llsc) \ { \ uint##t##_t ret; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"1, [%2]\n" \ " stxr %w0, %"#w"3, [%2]\n" \ " cbnz %w0, 1b\n" \ : "=&r" (res), "=&r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_SWAP_PROTO(t, _lse) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ ".arch_extension lse\n" \ "swp %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (ret) \ : "r" (p), "r" (val) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_SWAP_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_swap_##t##_lse(p, val)); \ else \ return (atomic_swap_##t##_llsc(p, val)); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ { \ uint##t##_t ret; \ int res; \ \ __asm __volatile( \ "1: ldxr %"#w"1, [%2]\n" \ " stxr %w0, "#zreg", [%2]\n" \ " cbnz %w0, 1b\n" \ : "=&r" (res), "=&r" (ret) \ : "r" (p) \ : "memory" \ ); \ \ return (ret); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, _lse) \ { \ return (atomic_swap_##t##_lse(p, 0)); \ } \ \ _ATOMIC_READANDCLEAR_PROTO(t, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_readandclear_##t##_lse(p)); \ else \ return (atomic_readandclear_##t##_llsc(p)); \ } _ATOMIC_SWAP_IMPL(32, w, wzr) _ATOMIC_SWAP_IMPL(64, , xzr) #define _ATOMIC_TEST_OP_PROTO(t, op, bar, flav) \ static __inline int \ atomic_testand##op##_##bar##t##flav(volatile uint##t##_t *p, u_int val) #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op, bar, a) \ _ATOMIC_TEST_OP_PROTO(t, op, bar, _llsc) \ { \ uint##t##_t mask, old, tmp; \ int res; \ \ mask = ((uint##t##_t)1) << (val & (t - 1)); \ __asm __volatile( \ "1: ld"#a"xr %"#w"2, [%3]\n" \ " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \ " stxr %w1, %"#w"0, [%3]\n" \ " cbnz %w1, 1b\n" \ : "=&r" (tmp), "=&r" (res), "=&r" (old) \ : "r" (p), "r" (mask) \ : "memory" \ ); \ \ return ((old & mask) != 0); \ } \ \ _ATOMIC_TEST_OP_PROTO(t, op, bar, _lse) \ { \ uint##t##_t mask, old; \ \ mask = ((uint##t##_t)1) << (val & (t - 1)); \ __asm __volatile( \ ".arch_extension lse\n" \ "ld"#lse_asm_op#a" %"#w"2, %"#w"0, [%1]\n" \ ".arch_extension nolse\n" \ : "=r" (old) \ : "r" (p), "r" (mask) \ : "memory" \ ); \ \ return ((old & mask) != 0); \ } \ \ _ATOMIC_TEST_OP_PROTO(t, op, bar, ) \ { \ if (_ATOMIC_LSE_SUPPORTED) \ return (atomic_testand##op##_##bar##t##_lse(p, val)); \ else \ return (atomic_testand##op##_##bar##t##_llsc(p, val)); \ } #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \ _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, , ) \ _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, acq_, a) \ _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, , ) \ _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, acq_, a) _ATOMIC_TEST_OP(clear, bic, clr) _ATOMIC_TEST_OP(set, orr, set) #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \ static __inline uint##t##_t \ -atomic_load_acq_##t(volatile uint##t##_t *p) \ +atomic_load_acq_##t(const volatile uint##t##_t *p) \ { \ uint##t##_t ret; \ \ __asm __volatile( \ "ldar"#s" %"#w"0, [%1]\n" \ : "=&r" (ret) \ : "r" (p) \ : "memory"); \ \ return (ret); \ } #define atomic_load_acq_8 atomic_load_acq_8 #define atomic_load_acq_16 atomic_load_acq_16 _ATOMIC_LOAD_ACQ_IMPL(8, w, b) _ATOMIC_LOAD_ACQ_IMPL(16, w, h) _ATOMIC_LOAD_ACQ_IMPL(32, w, ) _ATOMIC_LOAD_ACQ_IMPL(64, , ) #define _ATOMIC_STORE_REL_IMPL(t, w, s) \ static __inline void \ atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \ { \ __asm __volatile( \ "stlr"#s" %"#w"0, [%1]\n" \ : \ : "r" (val), "r" (p) \ : "memory"); \ } _ATOMIC_STORE_REL_IMPL(8, w, b) _ATOMIC_STORE_REL_IMPL(16, w, h) _ATOMIC_STORE_REL_IMPL(32, w, ) _ATOMIC_STORE_REL_IMPL(64, , ) #define atomic_add_char atomic_add_8 #define atomic_fcmpset_char atomic_fcmpset_8 #define atomic_clear_char atomic_clear_8 #define atomic_cmpset_char atomic_cmpset_8 #define atomic_fetchadd_char atomic_fetchadd_8 #define atomic_readandclear_char atomic_readandclear_8 #define atomic_set_char atomic_set_8 #define atomic_swap_char atomic_swap_8 #define atomic_subtract_char atomic_subtract_8 #define atomic_testandclear_char atomic_testandclear_8 #define atomic_testandset_char atomic_testandset_8 #define atomic_add_acq_char atomic_add_acq_8 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 #define atomic_clear_acq_char atomic_clear_acq_8 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 #define atomic_load_acq_char atomic_load_acq_8 #define atomic_set_acq_char atomic_set_acq_8 #define atomic_subtract_acq_char atomic_subtract_acq_8 #define atomic_testandset_acq_char atomic_testandset_acq_8 #define atomic_add_rel_char atomic_add_rel_8 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 #define atomic_clear_rel_char atomic_clear_rel_8 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 #define atomic_set_rel_char atomic_set_rel_8 #define atomic_subtract_rel_char atomic_subtract_rel_8 #define atomic_store_rel_char atomic_store_rel_8 #define atomic_add_short atomic_add_16 #define atomic_fcmpset_short atomic_fcmpset_16 #define atomic_clear_short atomic_clear_16 #define atomic_cmpset_short atomic_cmpset_16 #define atomic_fetchadd_short atomic_fetchadd_16 #define atomic_readandclear_short atomic_readandclear_16 #define atomic_set_short atomic_set_16 #define atomic_swap_short atomic_swap_16 #define atomic_subtract_short atomic_subtract_16 #define atomic_testandclear_short atomic_testandclear_16 #define atomic_testandset_short atomic_testandset_16 #define atomic_add_acq_short atomic_add_acq_16 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 #define atomic_clear_acq_short atomic_clear_acq_16 #define atomic_cmpset_acq_short atomic_cmpset_acq_16 #define atomic_load_acq_short atomic_load_acq_16 #define atomic_set_acq_short atomic_set_acq_16 #define atomic_subtract_acq_short atomic_subtract_acq_16 #define atomic_testandset_acq_short atomic_testandset_acq_16 #define atomic_add_rel_short atomic_add_rel_16 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 #define atomic_clear_rel_short atomic_clear_rel_16 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 #define atomic_set_rel_short atomic_set_rel_16 #define atomic_subtract_rel_short atomic_subtract_rel_16 #define atomic_store_rel_short atomic_store_rel_16 #define atomic_add_int atomic_add_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_swap_int atomic_swap_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_testandclear_int atomic_testandclear_32 #define atomic_testandset_int atomic_testandset_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_testandset_acq_int atomic_testandset_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_add_long atomic_add_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_swap_long atomic_swap_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_testandclear_long atomic_testandclear_64 #define atomic_testandset_long atomic_testandset_64 #define atomic_add_ptr atomic_add_64 #define atomic_fcmpset_ptr atomic_fcmpset_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_swap_ptr atomic_swap_64 #define atomic_subtract_ptr atomic_subtract_64 #define atomic_add_acq_long atomic_add_acq_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_testandset_acq_long atomic_testandset_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 #define atomic_clear_acq_ptr atomic_clear_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 static __inline void atomic_thread_fence_acq(void) { dmb(ld); } static __inline void atomic_thread_fence_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_acq_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_seq_cst(void) { dmb(sy); } #endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index 96fcfcaed2c7..2ccd206a35ec 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -1,932 +1,936 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif #include #ifdef _KERNEL #include #include #endif #ifndef __OFFSETOF_MONITORBUF /* * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in i386/vm_machdep.c ensures that the value is correct. */ #define __OFFSETOF_MONITORBUF 0x80 static __inline void __mbk(void) { __asm __volatile("lock; addl $0,%%fs:%0" : "+m" (*(u_int *)__OFFSETOF_MONITORBUF) : : "memory", "cc"); } static __inline void __mbu(void) { __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); } #endif /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ #if !defined(__GNUCLIKE_ASM) #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); int atomic_fcmpset_short(volatile u_short *dst, u_short *expect, u_short src); int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); void atomic_thread_fence_acq(void); void atomic_thread_fence_acq_rel(void); void atomic_thread_fence_rel(void); void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_64(volatile uint64_t *, uint64_t, uint64_t); int atomic_fcmpset_64(volatile uint64_t *, uint64_t *, uint64_t); uint64_t atomic_load_acq_64(volatile uint64_t *); void atomic_store_rel_64(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64(volatile uint64_t *, uint64_t); uint64_t atomic_fetchadd_64(volatile uint64_t *, uint64_t); void atomic_add_64(volatile uint64_t *, uint64_t); void atomic_subtract_64(volatile uint64_t *, uint64_t); #else /* !__GNUCLIKE_ASM */ /* * Always use lock prefixes. The result is slightly less optimal for * UP systems, but it matters less now, and sometimes UP is emulated * over SMP. * * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions. * * cmpset: * if (*dst == expect) * *dst = src * * fcmpset: * if (*dst == *expect) * *dst = src * else * *expect = *dst * * Returns 0 on failure, non-zero on success. */ #define ATOMIC_CMPSET(TYPE, CONS) \ static __inline int \ atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ " sete %0 ; " \ "# atomic_cmpset_" #TYPE " " \ : "=q" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (expect) /* 2 */ \ : CONS (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } \ \ static __inline int \ atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ " sete %0 ; " \ "# atomic_fcmpset_" #TYPE " " \ : "=q" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (*expect) /* 2 */ \ : CONS (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } ATOMIC_CMPSET(char, "q"); ATOMIC_CMPSET(short, "r"); ATOMIC_CMPSET(int, "r"); /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " lock; xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btsl %2,%1 ; " " setc %0 ; " "# atomic_testandset_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btrl %2,%1 ; " " setc %0 ; " "# atomic_testandclear_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. In the kernel, we use a private per-cpu * cache line for "mem", to avoid introducing false data * dependencies. In user space, we use the word at the top of the * stack. * * For UP kernels, however, the memory of the single processor is * always consistent, so we only need to stop the compiler from * reordering accesses in a way that violates the semantics of acquire * and release. */ #if defined(_KERNEL) #define __storeload_barrier() __mbk() #else /* !_KERNEL */ #define __storeload_barrier() __mbu() #endif /* _KERNEL*/ #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ -atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ +atomic_load_acq_##TYPE(const volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #ifdef _KERNEL #ifdef WANT_FUNCTIONS int atomic_cmpset_64_i386(volatile uint64_t *, uint64_t, uint64_t); int atomic_cmpset_64_i586(volatile uint64_t *, uint64_t, uint64_t); -uint64_t atomic_load_acq_64_i386(volatile uint64_t *); -uint64_t atomic_load_acq_64_i586(volatile uint64_t *); +uint64_t atomic_load_acq_64_i386(const volatile uint64_t *); +uint64_t atomic_load_acq_64_i586(const volatile uint64_t *); void atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); void atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64_i386(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64_i586(volatile uint64_t *, uint64_t); #endif /* I486 does not support SMP or CMPXCHG8B. */ static __inline int atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) { volatile uint32_t *p; u_char res; p = (volatile uint32_t *)dst; __asm __volatile( " pushfl ; " " cli ; " " xorl %1,%%eax ; " " xorl %2,%%edx ; " " orl %%edx,%%eax ; " " jne 1f ; " " movl %4,%1 ; " " movl %5,%2 ; " "1: " " sete %3 ; " " popfl" : "+A" (expect), /* 0 */ "+m" (*p), /* 1 */ "+m" (*(p + 1)), /* 2 */ "=q" (res) /* 3 */ : "r" ((uint32_t)src), /* 4 */ "r" ((uint32_t)(src >> 32)) /* 5 */ : "memory", "cc"); return (res); } static __inline int atomic_fcmpset_64_i386(volatile uint64_t *dst, uint64_t *expect, uint64_t src) { if (atomic_cmpset_64_i386(dst, *expect, src)) { return (1); } else { *expect = *dst; return (0); } } static __inline uint64_t -atomic_load_acq_64_i386(volatile uint64_t *p) +atomic_load_acq_64_i386(const volatile uint64_t *p) { - volatile uint32_t *q; + const volatile uint32_t *q; uint64_t res; - q = (volatile uint32_t *)p; + q = (const volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %1,%%eax ; " " movl %2,%%edx ; " " popfl" : "=&A" (res) /* 0 */ : "m" (*q), /* 1 */ "m" (*(q + 1)) /* 2 */ : "memory"); return (res); } static __inline void atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) { volatile uint32_t *q; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %%eax,%0 ; " " movl %%edx,%1 ; " " popfl" : "=m" (*q), /* 0 */ "=m" (*(q + 1)) /* 1 */ : "A" (v) /* 2 */ : "memory"); } static __inline uint64_t atomic_swap_64_i386(volatile uint64_t *p, uint64_t v) { volatile uint32_t *q; uint64_t res; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %1,%%eax ; " " movl %2,%%edx ; " " movl %4,%2 ; " " movl %3,%1 ; " " popfl" : "=&A" (res), /* 0 */ "+m" (*q), /* 1 */ "+m" (*(q + 1)) /* 2 */ : "r" ((uint32_t)v), /* 3 */ "r" ((uint32_t)(v >> 32))); /* 4 */ return (res); } static __inline int atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) { u_char res; __asm __volatile( " lock; cmpxchg8b %1 ; " " sete %0" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+A" (expect) /* 2 */ : "b" ((uint32_t)src), /* 3 */ "c" ((uint32_t)(src >> 32)) /* 4 */ : "memory", "cc"); return (res); } static __inline int atomic_fcmpset_64_i586(volatile uint64_t *dst, uint64_t *expect, uint64_t src) { u_char res; __asm __volatile( " lock; cmpxchg8b %1 ; " " sete %0" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+A" (*expect) /* 2 */ : "b" ((uint32_t)src), /* 3 */ "c" ((uint32_t)(src >> 32)) /* 4 */ : "memory", "cc"); return (res); } +/* + * Architecturally always writes back some value to '*p' so will trigger + * a #GP(0) on read-only mappings. + */ static __inline uint64_t -atomic_load_acq_64_i586(volatile uint64_t *p) +atomic_load_acq_64_i586(const volatile uint64_t *p) { uint64_t res; __asm __volatile( " movl %%ebx,%%eax ; " " movl %%ecx,%%edx ; " " lock; cmpxchg8b %1" - : "=&A" (res), /* 0 */ - "+m" (*p) /* 1 */ - : : "memory", "cc"); + : "=&A" (res) /* 0 */ + : "m" (*p) /* 1 */ + : "memory", "cc"); return (res); } static __inline void atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) { __asm __volatile( " movl %%eax,%%ebx ; " " movl %%edx,%%ecx ; " "1: " " lock; cmpxchg8b %0 ; " " jne 1b" : "+m" (*p), /* 0 */ "+A" (v) /* 1 */ : : "ebx", "ecx", "memory", "cc"); } static __inline uint64_t atomic_swap_64_i586(volatile uint64_t *p, uint64_t v) { __asm __volatile( " movl %%eax,%%ebx ; " " movl %%edx,%%ecx ; " "1: " " lock; cmpxchg8b %0 ; " " jne 1b" : "+m" (*p), /* 0 */ "+A" (v) /* 1 */ : : "ebx", "ecx", "memory", "cc"); return (v); } static __inline int atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_cmpset_64_i386(dst, expect, src)); else return (atomic_cmpset_64_i586(dst, expect, src)); } static __inline int atomic_fcmpset_64(volatile uint64_t *dst, uint64_t *expect, uint64_t src) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_fcmpset_64_i386(dst, expect, src)); else return (atomic_fcmpset_64_i586(dst, expect, src)); } static __inline uint64_t -atomic_load_acq_64(volatile uint64_t *p) +atomic_load_acq_64(const volatile uint64_t *p) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_load_acq_64_i386(p)); else return (atomic_load_acq_64_i586(p)); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t v) { if ((cpu_feature & CPUID_CX8) == 0) atomic_store_rel_64_i386(p, v); else atomic_store_rel_64_i586(p, v); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t v) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_swap_64_i386(p, v)); else return (atomic_swap_64_i586(p, v)); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) { for (;;) { uint64_t t = *p; if (atomic_cmpset_64(p, t, t + v)) return (t); } } static __inline void atomic_add_64(volatile uint64_t *p, uint64_t v) { uint64_t t; for (;;) { t = *p; if (atomic_cmpset_64(p, t, t + v)) break; } } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t v) { uint64_t t; for (;;) { t = *p; if (atomic_cmpset_64(p, t, t - v)) break; } } #endif /* _KERNEL */ #endif /* !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, long, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS static __inline int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) { return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect, (u_int)src)); } static __inline int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src) { return (atomic_fcmpset_int((volatile u_int *)dst, (u_int *)expect, (u_int)src)); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { return (atomic_testandset_int((volatile u_int *)p, v)); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { return (atomic_testandclear_int((volatile u_int *)p, v)); } /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { return (atomic_swap_int((volatile u_int *)p, (u_int)v)); } #else /* !__GNUCLIKE_ASM */ u_int atomic_swap_int(volatile u_int *p, u_int v); u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_cmpset_acq_char atomic_cmpset_char #define atomic_cmpset_rel_char atomic_cmpset_char #define atomic_fcmpset_acq_char atomic_fcmpset_char #define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_cmpset_acq_short atomic_cmpset_short #define atomic_cmpset_rel_short atomic_cmpset_short #define atomic_fcmpset_acq_short atomic_fcmpset_short #define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) #define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int #ifdef _KERNEL /* Operations on 64-bit quad words. */ #define atomic_cmpset_acq_64 atomic_cmpset_64 #define atomic_cmpset_rel_64 atomic_cmpset_64 #define atomic_fcmpset_acq_64 atomic_fcmpset_64 #define atomic_fcmpset_rel_64 atomic_fcmpset_64 #define atomic_fetchadd_acq_64 atomic_fetchadd_64 #define atomic_fetchadd_rel_64 atomic_fetchadd_64 #define atomic_add_acq_64 atomic_add_64 #define atomic_add_rel_64 atomic_add_64 #define atomic_subtract_acq_64 atomic_subtract_64 #define atomic_subtract_rel_64 atomic_subtract_64 #define atomic_load_64 atomic_load_acq_64 #define atomic_store_64 atomic_store_rel_64 #endif /* Operations on pointers. */ #define atomic_set_ptr(p, v) \ atomic_set_int((volatile u_int *)(p), (u_int)(v)) #define atomic_set_acq_ptr(p, v) \ atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_set_rel_ptr(p, v) \ atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_ptr(p, v) \ atomic_clear_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_acq_ptr(p, v) \ atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_rel_ptr(p, v) \ atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_ptr(p, v) \ atomic_add_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_acq_ptr(p, v) \ atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_rel_ptr(p, v) \ atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_ptr(p, v) \ atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_acq_ptr(p, v) \ atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_rel_ptr(p, v) \ atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_load_acq_ptr(p) \ - atomic_load_acq_int((volatile u_int *)(p)) + atomic_load_acq_int((const volatile u_int *)(p)) #define atomic_store_rel_ptr(p, v) \ atomic_store_rel_int((volatile u_int *)(p), (v)) #define atomic_cmpset_ptr(dst, old, new) \ atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) #define atomic_cmpset_acq_ptr(dst, old, new) \ atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) #define atomic_fcmpset_ptr(dst, old, new) \ atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new)) #define atomic_fcmpset_acq_ptr(dst, old, new) \ atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \ (u_int)(new)) #define atomic_fcmpset_rel_ptr(dst, old, new) \ atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \ (u_int)(new)) #define atomic_swap_ptr(p, v) \ atomic_swap_int((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_ptr(p) \ atomic_readandclear_int((volatile u_int *)(p)) #endif /* !WANT_FUNCTIONS */ #if defined(_KERNEL) #define mb() __mbk() #define wmb() __mbk() #define rmb() __mbk() #else #define mb() __mbu() #define wmb() __mbu() #define rmb() __mbu() #endif #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h index e326f5aaabfb..ef42def6fa5e 100644 --- a/sys/powerpc/include/atomic.h +++ b/sys/powerpc/include/atomic.h @@ -1,1152 +1,1152 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2008 Marcel Moolenaar * Copyright (c) 2001 Benno Rice * Copyright (c) 2001 David E. O'Brien * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif #include #ifndef __powerpc64__ #include #endif /* * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction * with the atomic lXarx/stXcx. sequences below. They are not exposed outside * of this file. See also Appendix B.2 of Book II of the architecture manual. * * Note that not all Book-E processors accept the light-weight sync variant. * In particular, early models of E500 cores are known to wedge. Bank on all * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs * to use the heavier-weight sync. */ #ifdef __powerpc64__ #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("lwsync" : : : "memory") #define wmb() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #else #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("sync" : : : "memory") #define wmb() __asm __volatile("sync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #endif static __inline void powerpc_lwsync(void) { #ifdef __powerpc64__ __asm __volatile("lwsync" : : : "memory"); #else __asm __volatile("sync" : : : "memory"); #endif } /* * atomic_add(p, v) * { *p += v; } */ #define __atomic_add_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_int */ #ifdef __powerpc64__ #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #else #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #endif #define _ATOMIC_ADD(type) \ static __inline void \ atomic_add_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ } \ \ static __inline void \ atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_add_##type(p, v, t); \ } \ /* _ATOMIC_ADD */ _ATOMIC_ADD(int) _ATOMIC_ADD(long) #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #ifdef __powerpc64__ #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #else #define atomic_add_ptr atomic_add_int #define atomic_add_acq_ptr atomic_add_acq_int #define atomic_add_rel_ptr atomic_add_rel_int #endif #undef _ATOMIC_ADD #undef __atomic_add_long #undef __atomic_add_int /* * atomic_clear(p, v) * { *p &= ~v; } */ #define __atomic_clear_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_int */ #ifdef __powerpc64__ #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #else #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #endif #define _ATOMIC_CLEAR(type) \ static __inline void \ atomic_clear_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ } \ \ static __inline void \ atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_clear_##type(p, v, t); \ } \ /* _ATOMIC_CLEAR */ _ATOMIC_CLEAR(int) _ATOMIC_CLEAR(long) #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #ifdef __powerpc64__ #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #else #define atomic_clear_ptr atomic_clear_int #define atomic_clear_acq_ptr atomic_clear_acq_int #define atomic_clear_rel_ptr atomic_clear_rel_int #endif #undef _ATOMIC_CLEAR #undef __atomic_clear_long #undef __atomic_clear_int /* * atomic_cmpset(p, o, n) */ /* TODO -- see below */ /* * atomic_load_acq(p) */ /* TODO -- see below */ /* * atomic_readandclear(p) */ /* TODO -- see below */ /* * atomic_set(p, v) * { *p |= v; } */ #define __atomic_set_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_int */ #ifdef __powerpc64__ #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #else #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #endif #define _ATOMIC_SET(type) \ static __inline void \ atomic_set_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ } \ \ static __inline void \ atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_set_##type(p, v, t); \ } \ /* _ATOMIC_SET */ _ATOMIC_SET(int) _ATOMIC_SET(long) #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #ifdef __powerpc64__ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #else #define atomic_set_ptr atomic_set_int #define atomic_set_acq_ptr atomic_set_acq_int #define atomic_set_rel_ptr atomic_set_rel_int #endif #undef _ATOMIC_SET #undef __atomic_set_long #undef __atomic_set_int /* * atomic_subtract(p, v) * { *p -= v; } */ #define __atomic_subtract_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_int */ #ifdef __powerpc64__ #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #else #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #endif #define _ATOMIC_SUBTRACT(type) \ static __inline void \ atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ } \ \ static __inline void \ atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_subtract_##type(p, v, t); \ } \ /* _ATOMIC_SUBTRACT */ _ATOMIC_SUBTRACT(int) _ATOMIC_SUBTRACT(long) #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #ifdef __powerpc64__ #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #else #define atomic_subtract_ptr atomic_subtract_int #define atomic_subtract_acq_ptr atomic_subtract_acq_int #define atomic_subtract_rel_ptr atomic_subtract_rel_int #endif #undef _ATOMIC_SUBTRACT #undef __atomic_subtract_long #undef __atomic_subtract_int /* * atomic_store_rel(p, v) */ /* TODO -- see below */ /* * Old/original implementations that still need revisiting. */ static __inline u_int atomic_readandclear_int(volatile u_int *addr) { u_int result,temp; __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tlwarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stwcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); return (result); } #ifdef __powerpc64__ static __inline u_long atomic_readandclear_long(volatile u_long *addr) { u_long result,temp; __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tldarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stdcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); return (result); } #endif #define atomic_readandclear_32 atomic_readandclear_int #ifdef __powerpc64__ #define atomic_readandclear_64 atomic_readandclear_long #define atomic_readandclear_ptr atomic_readandclear_long #else static __inline u_long atomic_readandclear_long(volatile u_long *addr) { return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); } #define atomic_readandclear_ptr atomic_readandclear_int #endif /* * We assume that a = b will do atomic loads and stores. */ #define ATOMIC_STORE_LOAD(TYPE) \ static __inline u_##TYPE \ -atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ +atomic_load_acq_##TYPE(const volatile u_##TYPE *p) \ { \ u_##TYPE v; \ \ v = *p; \ powerpc_lwsync(); \ return (v); \ } \ \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ powerpc_lwsync(); \ *p = v; \ } ATOMIC_STORE_LOAD(int) #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #ifdef __powerpc64__ ATOMIC_STORE_LOAD(long) #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #else static __inline u_long -atomic_load_acq_long(volatile u_long *addr) +atomic_load_acq_long(const volatile u_long *addr) { - return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); + return ((u_long)atomic_load_acq_int((const volatile u_int *)addr)); } static __inline void atomic_store_rel_long(volatile u_long *addr, u_long val) { atomic_store_rel_int((volatile u_int *)addr, (u_int)val); } #define atomic_load_acq_ptr atomic_load_acq_int #define atomic_store_rel_ptr atomic_store_rel_int #endif #undef ATOMIC_STORE_LOAD /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ #ifdef ISA_206_ATOMICS static __inline int atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) { int ret; __asm __volatile ( "1:\tlbarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stbcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } static __inline int atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) { int ret; __asm __volatile ( "1:\tlharx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "sthcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } #else static __inline int atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, uint32_t mask) { int ret; uint32_t tmp; __asm __volatile ( "1:\tlwarx %2, 0, %3\n\t" /* load old value */ "and %0, %2, %7\n\t" "cmplw %4, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "andc %2, %2, %7\n\t" "or %2, %2, %5\n\t" "stwcx. %2, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p), "+&r" (tmp) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), "r" (mask) : "cr0", "memory"); return (ret); } #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) #endif static __inline int atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { int ret; __asm __volatile ( "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } static __inline int atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) { int ret; __asm __volatile ( #ifdef __powerpc64__ "1:\tldarx %0, 0, %2\n\t" /* load old value */ "cmpld %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stdcx. %4, 0, %2\n\t" /* attempt to store */ #else "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne- 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ #endif "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #else "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #endif "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } #define ATOMIC_CMPSET_ACQ_REL(type) \ static __inline int \ atomic_cmpset_acq_##type(volatile u_##type *p, \ u_##type cmpval, u_##type newval)\ {\ u_##type retval; \ retval = atomic_cmpset_##type(p, cmpval, newval);\ __ATOMIC_ACQ();\ return (retval);\ }\ static __inline int \ atomic_cmpset_rel_##type(volatile u_##type *p, \ u_##type cmpval, u_##type newval)\ {\ __ATOMIC_REL();\ return (atomic_cmpset_##type(p, cmpval, newval));\ }\ struct hack ATOMIC_CMPSET_ACQ_REL(int); ATOMIC_CMPSET_ACQ_REL(long); #ifdef ISA_206_ATOMICS #define atomic_cmpset_8 atomic_cmpset_char #endif #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #ifdef ISA_206_ATOMICS #define atomic_cmpset_16 atomic_cmpset_short #endif #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #ifdef __powerpc64__ #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #else #define atomic_cmpset_ptr atomic_cmpset_int #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int #endif /* * Atomically compare the value stored at *p with *cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed and sets *cmpval to the read value from *p, * nonzero otherwise. */ #ifdef ISA_206_ATOMICS static __inline int atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) { int ret; __asm __volatile ( "lbarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stbcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } static __inline int atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) { int ret; __asm __volatile ( "lharx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "sthcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } #endif /* ISA_206_ATOMICS */ static __inline int atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) { int ret; __asm __volatile ( "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } static __inline int atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) { int ret; __asm __volatile ( #ifdef __powerpc64__ "ldarx %0, 0, %3\n\t" /* load old value */ "cmpld %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stdcx. %5, 0, %3\n\t" /* attempt to store */ #else "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ #endif "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stdx %0, 0, %7\n\t" #else "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" #endif "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } #define ATOMIC_FCMPSET_ACQ_REL(type) \ static __inline int \ atomic_fcmpset_acq_##type(volatile u_##type *p, \ u_##type *cmpval, u_##type newval)\ {\ u_##type retval; \ retval = atomic_fcmpset_##type(p, cmpval, newval);\ __ATOMIC_ACQ();\ return (retval);\ }\ static __inline int \ atomic_fcmpset_rel_##type(volatile u_##type *p, \ u_##type *cmpval, u_##type newval)\ {\ __ATOMIC_REL();\ return (atomic_fcmpset_##type(p, cmpval, newval));\ }\ struct hack ATOMIC_FCMPSET_ACQ_REL(int); ATOMIC_FCMPSET_ACQ_REL(long); #ifdef ISA_206_ATOMICS #define atomic_fcmpset_8 atomic_fcmpset_char #endif #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char #ifdef ISA_206_ATOMICS #define atomic_fcmpset_16 atomic_fcmpset_short #endif #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #ifdef __powerpc64__ #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #else #define atomic_fcmpset_ptr atomic_fcmpset_int #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int #endif static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { u_int value; do { value = *p; } while (!atomic_cmpset_int(p, value, value + v)); return (value); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { u_long value; do { value = *p; } while (!atomic_cmpset_long(p, value, value + v)); return (value); } static __inline u_int atomic_swap_32(volatile u_int *p, u_int v) { u_int prev; __asm __volatile( "1: lwarx %0,0,%2\n" " stwcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_int *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #ifdef __powerpc64__ static __inline u_long atomic_swap_64(volatile u_long *p, u_long v) { u_long prev; __asm __volatile( "1: ldarx %0,0,%2\n" " stdcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_long *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #endif #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_swap_int atomic_swap_32 #ifdef __powerpc64__ #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_swap_long atomic_swap_64 #define atomic_swap_ptr atomic_swap_64 #else #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) #endif static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_int m = (1u << (v & 0x1f)); u_int res; u_int tmp; __asm __volatile( "1: lwarx %0,0,%3\n" " and %1,%0,%4\n" " or %0,%0,%4\n" " stwcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_int m = (1u << (v & 0x1f)); u_int res; u_int tmp; __asm __volatile( "1: lwarx %0,0,%3\n" " and %1,%0,%4\n" " andc %0,%0,%4\n" " stwcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } #ifdef __powerpc64__ static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_long m = (1ul << (v & 0x3f)); u_long res; u_long tmp; __asm __volatile( "1: ldarx %0,0,%3\n" " and %1,%0,%4\n" " or %0,%0,%4\n" " stdcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_long m = (1ul << (v & 0x3f)); u_long res; u_long tmp; __asm __volatile( "1: ldarx %0,0,%3\n" " and %1,%0,%4\n" " andc %0,%0,%4\n" " stdcx. %0,0,%3\n" " bne- 1b\n" : "=&r"(tmp), "=&r"(res), "+m"(*p) : "r"(p), "r"(m) : "cr0", "memory"); return (res != 0); } #else static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { return (atomic_testandset_int((volatile u_int *)p, v)); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { return (atomic_testandclear_int((volatile u_int *)p, v)); } #endif #define atomic_testandclear_32 atomic_testandclear_int #define atomic_testandset_32 atomic_testandset_int static __inline int atomic_testandset_acq_long(volatile u_long *p, u_int v) { u_int a = atomic_testandset_long(p, v); __ATOMIC_ACQ(); return (a); } #define atomic_testandclear_int atomic_testandclear_int #define atomic_testandset_int atomic_testandset_int #define atomic_testandclear_long atomic_testandclear_long #define atomic_testandset_long atomic_testandset_long #define atomic_testandset_acq_long atomic_testandset_acq_long static __inline void atomic_thread_fence_acq(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_acq_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_seq_cst(void) { __asm __volatile("sync" : : : "memory"); } #ifndef ISA_206_ATOMICS #include #define atomic_cmpset_char atomic_cmpset_8 #define atomic_cmpset_short atomic_cmpset_16 #define atomic_fcmpset_char atomic_fcmpset_8 #define atomic_fcmpset_short atomic_fcmpset_16 #endif /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ ATOMIC_CMPSET_ACQ_REL(char); ATOMIC_CMPSET_ACQ_REL(short); ATOMIC_FCMPSET_ACQ_REL(char); ATOMIC_FCMPSET_ACQ_REL(short); #undef __ATOMIC_REL #undef __ATOMIC_ACQ #endif /* ! _MACHINE_ATOMIC_H_ */ diff --git a/sys/riscv/include/atomic.h b/sys/riscv/include/atomic.h index 03c1327d74da..32ceb9a6341d 100644 --- a/sys/riscv/include/atomic.h +++ b/sys/riscv/include/atomic.h @@ -1,584 +1,584 @@ /*- * Copyright (c) 2015-2024 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #include #define fence() __asm __volatile("fence" ::: "memory"); #define mb() fence() #define rmb() fence() #define wmb() fence() static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *, uint16_t); #define ATOMIC_ACQ_REL(NAME, WIDTH) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ atomic_##NAME##_##WIDTH(p, v); \ fence(); \ } \ \ static __inline void \ atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ fence(); \ atomic_##NAME##_##WIDTH(p, v); \ } #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \ static __inline int \ atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ { \ int retval; \ \ retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \ fence(); \ return (retval); \ } \ \ static __inline int \ atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \ { \ fence(); \ return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \ } #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \ static __inline int \ atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ { \ int retval; \ \ retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \ fence(); \ return (retval); \ } \ \ static __inline int \ atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \ uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \ { \ fence(); \ return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \ } ATOMIC_CMPSET_ACQ_REL(8); ATOMIC_FCMPSET_ACQ_REL(8); #define atomic_cmpset_char atomic_cmpset_8 #define atomic_cmpset_acq_char atomic_cmpset_acq_8 #define atomic_cmpset_rel_char atomic_cmpset_rel_8 #define atomic_fcmpset_char atomic_fcmpset_8 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8 #define atomic_cmpset_short atomic_cmpset_16 #define atomic_fcmpset_short atomic_fcmpset_16 ATOMIC_CMPSET_ACQ_REL(16); ATOMIC_FCMPSET_ACQ_REL(16); #define atomic_load_acq_16 atomic_load_acq_16 static __inline uint16_t -atomic_load_acq_16(volatile uint16_t *p) +atomic_load_acq_16(const volatile uint16_t *p) { uint16_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_16(volatile uint16_t *p, uint16_t val) { fence(); *p = val; } #define atomic_cmpset_acq_short atomic_cmpset_acq_16 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16 #define atomic_load_acq_short atomic_load_acq_16 #define atomic_cmpset_rel_short atomic_cmpset_rel_16 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16 #define atomic_store_rel_short atomic_store_rel_16 static __inline void atomic_add_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoadd.w zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_subtract_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoadd.w zero, %1, %0" : "+A" (*p) : "r" (-val) : "memory"); } static __inline void atomic_set_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoor.w zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_clear_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoand.w zero, %1, %0" : "+A" (*p) : "r" (~val) : "memory"); } static __inline int atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.w %0, %2\n" "bne %0, %z3, 1f\n" "sc.w %1, %z4, %2\n" "bnez %1, 0b\n" "1:" : "=&r" (tmp), "=&r" (res), "+A" (*p) : "rJ" ((long)(int32_t)cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline int atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { uint32_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.w %0, %2\n" /* Load old value */ "bne %0, %z4, 1f\n" /* Compare */ "sc.w %1, %z5, %2\n" /* Try to store new value */ "j 2f\n" "1:" "sw %0, %3\n" /* Save old value */ "2:" : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline uint32_t atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) { uint32_t ret; __asm __volatile("amoadd.w %0, %2, %1" : "=&r" (ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint32_t atomic_readandclear_32(volatile uint32_t *p) { uint32_t ret; uint32_t val; val = 0; __asm __volatile("amoswap.w %0, %2, %1" : "=&r"(ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } #define atomic_add_int atomic_add_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_subtract_int atomic_subtract_32 ATOMIC_ACQ_REL(set, 32) ATOMIC_ACQ_REL(clear, 32) ATOMIC_ACQ_REL(add, 32) ATOMIC_ACQ_REL(subtract, 32) ATOMIC_CMPSET_ACQ_REL(32); ATOMIC_FCMPSET_ACQ_REL(32); static __inline uint32_t -atomic_load_acq_32(volatile uint32_t *p) +atomic_load_acq_32(const volatile uint32_t *p) { uint32_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_32(volatile uint32_t *p, uint32_t val) { fence(); *p = val; } #define atomic_add_acq_int atomic_add_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 static __inline void atomic_add_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoadd.d zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoadd.d zero, %1, %0" : "+A" (*p) : "r" (-val) : "memory"); } static __inline void atomic_set_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoor.d zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_clear_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoand.d zero, %1, %0" : "+A" (*p) : "r" (~val) : "memory"); } static __inline int atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.d %0, %2\n" "bne %0, %z3, 1f\n" "sc.d %1, %z4, %2\n" "bnez %1, 0b\n" "1:" : "=&r" (tmp), "=&r" (res), "+A" (*p) : "rJ" (cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline int atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { uint64_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.d %0, %2\n" /* Load old value */ "bne %0, %z4, 1f\n" /* Compare */ "sc.d %1, %z5, %2\n" /* Try to store new value */ "j 2f\n" "1:" "sd %0, %3\n" /* Save old value */ "2:" : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) : "rJ" (*cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) { uint64_t ret; __asm __volatile("amoadd.d %0, %2, %1" : "=&r" (ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint64_t atomic_readandclear_64(volatile uint64_t *p) { uint64_t ret; uint64_t val; val = 0; __asm __volatile("amoswap.d %0, %2, %1" : "=&r"(ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint32_t atomic_swap_32(volatile uint32_t *p, uint32_t val) { uint32_t old; __asm __volatile("amoswap.w %0, %2, %1" : "=&r"(old), "+A" (*p) : "r" (val) : "memory"); return (old); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t val) { uint64_t old; __asm __volatile("amoswap.d %0, %2, %1" : "=&r"(old), "+A" (*p) : "r" (val) : "memory"); return (old); } #define atomic_swap_int atomic_swap_32 #define atomic_add_long atomic_add_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_swap_long atomic_swap_64 #define atomic_add_ptr atomic_add_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fcmpset_ptr atomic_fcmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_subtract_ptr atomic_subtract_64 #define atomic_swap_ptr atomic_swap_64 ATOMIC_ACQ_REL(set, 64) ATOMIC_ACQ_REL(clear, 64) ATOMIC_ACQ_REL(add, 64) ATOMIC_ACQ_REL(subtract, 64) ATOMIC_CMPSET_ACQ_REL(64); ATOMIC_FCMPSET_ACQ_REL(64); static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { uint64_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t val) { fence(); *p = val; } #define atomic_add_acq_long atomic_add_acq_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_clear_acq_ptr atomic_clear_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 #undef ATOMIC_ACQ_REL static __inline void atomic_thread_fence_acq(void) { fence(); } static __inline void atomic_thread_fence_rel(void) { fence(); } static __inline void atomic_thread_fence_acq_rel(void) { fence(); } static __inline void atomic_thread_fence_seq_cst(void) { fence(); } #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 #include #endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/sys/_atomic64e.h b/sys/sys/_atomic64e.h index f7245dafb98a..82fe817f307b 100644 --- a/sys/sys/_atomic64e.h +++ b/sys/sys/_atomic64e.h @@ -1,77 +1,77 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Justin Hibbits * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_ATOMIC64E_H_ #define _SYS_ATOMIC64E_H_ #ifndef _MACHINE_ATOMIC_H_ #error "This should not be included directly. Include " #endif #ifdef _KERNEL #define HAS_EMULATED_ATOMIC64 /* Emulated versions of 64-bit atomic operations. */ void atomic_add_64(volatile u_int64_t *, u_int64_t); #define atomic_add_acq_64 atomic_add_64 #define atomic_add_rel_64 atomic_add_64 int atomic_cmpset_64(volatile u_int64_t *, u_int64_t, u_int64_t); #define atomic_cmpset_acq_64 atomic_cmpset_64 #define atomic_cmpset_rel_64 atomic_cmpset_64 void atomic_clear_64(volatile u_int64_t *, u_int64_t); #define atomic_clear_acq_64 atomic_clear_64 #define atomic_clear_rel_64 atomic_clear_64 int atomic_fcmpset_64(volatile u_int64_t *, u_int64_t *, u_int64_t); #define atomic_fcmpset_acq_64 atomic_fcmpset_64 #define atomic_fcmpset_rel_64 atomic_fcmpset_64 u_int64_t atomic_fetchadd_64(volatile u_int64_t *, u_int64_t); -u_int64_t atomic_load_64(volatile u_int64_t *); +u_int64_t atomic_load_64(const volatile u_int64_t *); #define atomic_load_acq_64 atomic_load_64 void atomic_readandclear_64(volatile u_int64_t *); void atomic_set_64(volatile u_int64_t *, u_int64_t); #define atomic_set_acq_64 atomic_set_64 #define atomic_set_rel_64 atomic_set_64 void atomic_subtract_64(volatile u_int64_t *, u_int64_t); #define atomic_subtract_acq_64 atomic_subtract_64 #define atomic_subtract_rel_64 atomic_subtract_64 void atomic_store_64(volatile u_int64_t *, u_int64_t); #define atomic_store_rel_64 atomic_store_64 u_int64_t atomic_swap_64(volatile u_int64_t *, u_int64_t); #endif /* _KERNEL */ #endif /* _SYS_ATOMIC64E_H_ */ diff --git a/sys/sys/_atomic_subword.h b/sys/sys/_atomic_subword.h index dc1b2f23e731..491b869b46c4 100644 --- a/sys/sys/_atomic_subword.h +++ b/sys/sys/_atomic_subword.h @@ -1,272 +1,272 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Kyle Evans * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS__ATOMIC_SUBWORD_H_ #define _SYS__ATOMIC_SUBWORD_H_ /* * This header is specifically for platforms that either do not have ways to or * simply do not do sub-word atomic operations. These are not ideal as they * require a little more effort to make sure our atomic operations are failing * because of the bits of the word we're trying to write rather than the rest * of the word. */ #ifndef _MACHINE_ATOMIC_H_ #error do not include this header, use machine/atomic.h #endif #include #ifndef _KERNEL #include #endif #ifndef NBBY #define NBBY 8 #endif #define _ATOMIC_WORD_ALIGNED(p) \ (uint32_t *)((__uintptr_t)(p) - ((__uintptr_t)(p) % 4)) #if _BYTE_ORDER == _BIG_ENDIAN #define _ATOMIC_BYTE_SHIFT(p) \ ((3 - ((__uintptr_t)(p) % 4)) * NBBY) #define _ATOMIC_HWORD_SHIFT(p) \ ((2 - ((__uintptr_t)(p) % 4)) * NBBY) #else #define _ATOMIC_BYTE_SHIFT(p) \ ((((__uintptr_t)(p) % 4)) * NBBY) #define _ATOMIC_HWORD_SHIFT(p) \ ((((__uintptr_t)(p) % 4)) * NBBY) #endif #ifndef _atomic_cmpset_masked_word /* * Pass these bad boys a couple words and a mask of the bits you care about, * they'll loop until we either succeed or fail because of those bits rather * than the ones we're not masking. old and val should already be preshifted to * the proper position. */ static __inline int _atomic_cmpset_masked_word(uint32_t *addr, uint32_t old, uint32_t val, uint32_t mask) { int ret; uint32_t wcomp; wcomp = old; /* * We'll attempt the cmpset on the entire word. Loop here in case the * operation fails due to the other half-word resident in that word, * rather than the half-word we're trying to operate on. Ideally we * only take one trip through here. We'll have to recalculate the old * value since it's the other part of the word changing. */ do { old = (*addr & ~mask) | wcomp; ret = atomic_fcmpset_32(addr, &old, (old & ~mask) | val); } while (ret == 0 && (old & mask) == wcomp); return (ret); } #endif #ifndef _atomic_fcmpset_masked_word static __inline int _atomic_fcmpset_masked_word(uint32_t *addr, uint32_t *old, uint32_t val, uint32_t mask) { /* * fcmpset_* is documented in atomic(9) to allow spurious failures where * *old == val on ll/sc architectures because the sc may fail due to * parallel writes or other reasons. We take advantage of that here * and only attempt once, because the caller should be compensating for * that possibility. */ *old = (*addr & ~mask) | *old; return (atomic_fcmpset_32(addr, old, (*old & ~mask) | val)); } #endif #ifndef atomic_cmpset_8 static __inline int atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, uint8_t val) { int shift; shift = _ATOMIC_BYTE_SHIFT(addr); return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr), old << shift, val << shift, 0xff << shift)); } #endif #ifndef atomic_fcmpset_8 static __inline int atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *old, uint8_t val) { int ret, shift; uint32_t wold; shift = _ATOMIC_BYTE_SHIFT(addr); wold = *old << shift; ret = _atomic_fcmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr), &wold, val << shift, 0xff << shift); if (ret == 0) *old = (wold >> shift) & 0xff; return (ret); } #endif #ifndef atomic_cmpset_16 static __inline int atomic_cmpset_16(__volatile uint16_t *addr, uint16_t old, uint16_t val) { int shift; shift = _ATOMIC_HWORD_SHIFT(addr); return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr), old << shift, val << shift, 0xffff << shift)); } #endif #ifndef atomic_fcmpset_16 static __inline int atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old, uint16_t val) { int ret, shift; uint32_t wold; shift = _ATOMIC_HWORD_SHIFT(addr); wold = *old << shift; ret = _atomic_fcmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr), &wold, val << shift, 0xffff << shift); if (ret == 0) *old = (wold >> shift) & 0xffff; return (ret); } #endif #ifndef atomic_load_acq_8 static __inline uint8_t -atomic_load_acq_8(volatile uint8_t *p) +atomic_load_acq_8(const volatile uint8_t *p) { int shift; uint8_t ret; shift = _ATOMIC_BYTE_SHIFT(p); ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & 0xff; return (ret); } #endif #ifndef atomic_load_acq_16 static __inline uint16_t -atomic_load_acq_16(volatile uint16_t *p) +atomic_load_acq_16(const volatile uint16_t *p) { int shift; uint16_t ret; shift = _ATOMIC_HWORD_SHIFT(p); ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & 0xffff; return (ret); } #endif #undef _ATOMIC_WORD_ALIGNED #undef _ATOMIC_BYTE_SHIFT #undef _ATOMIC_HWORD_SHIFT /* * Provide generic testandset_long implementation based on fcmpset long * primitive. It may not be ideal for any given arch, so machine/atomic.h * should define the macro atomic_testandset_long to override with an * MD-specific version. * * (Organizationally, this isn't really subword atomics. But atomic_common is * included too early in machine/atomic.h, so it isn't a good place for derived * primitives like this.) */ #ifndef atomic_testandset_acq_long static __inline int atomic_testandset_acq_long(volatile u_long *p, u_int v) { u_long bit, old; bool ret; bit = (1ul << (v % (sizeof(*p) * NBBY))); old = atomic_load_acq_long(p); ret = false; while (!ret && (old & bit) == 0) ret = atomic_fcmpset_acq_long(p, &old, old | bit); return (!ret); } #endif #ifndef atomic_testandset_long static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_long bit, old; bool ret; bit = (1ul << (v % (sizeof(*p) * NBBY))); old = atomic_load_long(p); ret = false; while (!ret && (old & bit) == 0) ret = atomic_fcmpset_long(p, &old, old | bit); return (!ret); } #endif #ifndef atomic_testandclear_long static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_long bit, old; bool ret; bit = (1ul << (v % (sizeof(*p) * NBBY))); old = atomic_load_long(p); ret = false; while (!ret && (old & bit) != 0) ret = atomic_fcmpset_long(p, &old, old & ~bit); return (ret); } #endif #endif /* _SYS__ATOMIC_SUBWORD_H_ */ diff --git a/sys/sys/atomic_common.h b/sys/sys/atomic_common.h index 83e0d5af583d..e03cd93c2d4a 100644 --- a/sys/sys/atomic_common.h +++ b/sys/sys/atomic_common.h @@ -1,140 +1,140 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017 The FreeBSD Foundation * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_ATOMIC_COMMON_H_ #define _SYS_ATOMIC_COMMON_H_ #ifndef _MACHINE_ATOMIC_H_ #error do not include this header, use machine/atomic.h #endif #include -#define __atomic_load_bool_relaxed(p) (*(volatile _Bool *)(p)) +#define __atomic_load_bool_relaxed(p) (*(const volatile _Bool *)(p)) #define __atomic_store_bool_relaxed(p, v) \ (*(volatile _Bool *)(p) = (_Bool)(v)) -#define __atomic_load_char_relaxed(p) (*(volatile u_char *)(p)) -#define __atomic_load_short_relaxed(p) (*(volatile u_short *)(p)) -#define __atomic_load_int_relaxed(p) (*(volatile u_int *)(p)) -#define __atomic_load_long_relaxed(p) (*(volatile u_long *)(p)) -#define __atomic_load_8_relaxed(p) (*(volatile uint8_t *)(p)) -#define __atomic_load_16_relaxed(p) (*(volatile uint16_t *)(p)) -#define __atomic_load_32_relaxed(p) (*(volatile uint32_t *)(p)) -#define __atomic_load_64_relaxed(p) (*(volatile uint64_t *)(p)) +#define __atomic_load_char_relaxed(p) (*(const volatile u_char *)(p)) +#define __atomic_load_short_relaxed(p) (*(const volatile u_short *)(p)) +#define __atomic_load_int_relaxed(p) (*(const volatile u_int *)(p)) +#define __atomic_load_long_relaxed(p) (*(const volatile u_long *)(p)) +#define __atomic_load_8_relaxed(p) (*(const volatile uint8_t *)(p)) +#define __atomic_load_16_relaxed(p) (*(const volatile uint16_t *)(p)) +#define __atomic_load_32_relaxed(p) (*(const volatile uint32_t *)(p)) +#define __atomic_load_64_relaxed(p) (*(const volatile uint64_t *)(p)) #define __atomic_store_char_relaxed(p, v) \ (*(volatile u_char *)(p) = (u_char)(v)) #define __atomic_store_short_relaxed(p, v) \ (*(volatile u_short *)(p) = (u_short)(v)) #define __atomic_store_int_relaxed(p, v) \ (*(volatile u_int *)(p) = (u_int)(v)) #define __atomic_store_long_relaxed(p, v) \ (*(volatile u_long *)(p) = (u_long)(v)) #define __atomic_store_8_relaxed(p, v) \ (*(volatile uint8_t *)(p) = (uint8_t)(v)) #define __atomic_store_16_relaxed(p, v) \ (*(volatile uint16_t *)(p) = (uint16_t)(v)) #define __atomic_store_32_relaxed(p, v) \ (*(volatile uint32_t *)(p) = (uint32_t)(v)) #define __atomic_store_64_relaxed(p, v) \ (*(volatile uint64_t *)(p) = (uint64_t)(v)) /* * When _Generic is available, try to provide some type checking. */ #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ __has_extension(c_generic_selections) #define atomic_load_bool(p) \ _Generic(*(p), _Bool: __atomic_load_bool_relaxed(p)) #define atomic_store_bool(p, v) \ _Generic(*(p), _Bool: __atomic_store_bool_relaxed(p, v)) #define __atomic_load_generic(p, t, ut, n) \ _Generic(*(p), \ t: __atomic_load_ ## n ## _relaxed(p), \ ut: __atomic_load_ ## n ## _relaxed(p)) #define __atomic_store_generic(p, v, t, ut, n) \ _Generic(*(p), \ t: __atomic_store_ ## n ## _relaxed(p, v), \ ut: __atomic_store_ ## n ## _relaxed(p, v)) #else #define atomic_load_bool(p) \ __atomic_load_bool_relaxed(p) #define atomic_store_bool(p, v) \ __atomic_store_bool_relaxed(p, v) #define __atomic_load_generic(p, t, ut, n) \ __atomic_load_ ## n ## _relaxed(p) #define __atomic_store_generic(p, v, t, ut, n) \ __atomic_store_ ## n ## _relaxed(p, v) #endif #define atomic_load_char(p) __atomic_load_generic(p, char, u_char, char) #define atomic_load_short(p) __atomic_load_generic(p, short, u_short, short) #define atomic_load_int(p) __atomic_load_generic(p, int, u_int, int) #define atomic_load_long(p) __atomic_load_generic(p, long, u_long, long) #define atomic_load_8(p) __atomic_load_generic(p, int8_t, uint8_t, 8) #define atomic_load_16(p) __atomic_load_generic(p, int16_t, uint16_t, 16) #define atomic_load_32(p) __atomic_load_generic(p, int32_t, uint32_t, 32) #ifdef __LP64__ #define atomic_load_64(p) __atomic_load_generic(p, int64_t, uint64_t, 64) #endif #define atomic_store_char(p, v) \ __atomic_store_generic(p, v, char, u_char, char) #define atomic_store_short(p, v) \ __atomic_store_generic(p, v, short, u_short, short) #define atomic_store_int(p, v) \ __atomic_store_generic(p, v, int, u_int, int) #define atomic_store_long(p, v) \ __atomic_store_generic(p, v, long, u_long, long) #define atomic_store_8(p, v) \ __atomic_store_generic(p, v, int8_t, uint8_t, 8) #define atomic_store_16(p, v) \ __atomic_store_generic(p, v, int16_t, uint16_t, 16) #define atomic_store_32(p, v) \ __atomic_store_generic(p, v, int32_t, uint32_t, 32) #ifdef __LP64__ #define atomic_store_64(p, v) \ __atomic_store_generic(p, v, int64_t, uint64_t, 64) #endif -#define atomic_load_ptr(p) (*(volatile __typeof(*p) *)(p)) +#define atomic_load_ptr(p) (*(const volatile __typeof(*p) *)(p)) #define atomic_store_ptr(p, v) (*(volatile __typeof(*p) *)(p) = (v)) /* * Currently all architectures provide acquire and release fences on their own, * but they don't provide consume. Kludge below allows relevant code to stop * openly resorting to the stronger acquire fence, to be sorted out. */ #define atomic_load_consume_ptr(p) \ ((__typeof(*p)) atomic_load_acq_ptr((uintptr_t *)p)) #define atomic_interrupt_fence() __compiler_membar() #endif /* !_SYS_ATOMIC_COMMON_H_ */