Index: stable/11/sys/amd64/include/atomic.h =================================================================== --- stable/11/sys/amd64/include/atomic.h (revision 327194) +++ stable/11/sys/amd64/include/atomic.h (revision 327195) @@ -1,676 +1,678 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif /* * To express interprocessor (as opposed to processor and device) memory * ordering constraints, use the atomic_*() functions with acquire and release * semantics rather than the *mb() functions. An architecture's memory * ordering (or memory consistency) model governs the order in which a * program's accesses to different locations may be performed by an * implementation of that architecture. In general, for memory regions * defined as writeback cacheable, the memory ordering implemented by amd64 * processors preserves the program ordering of a load followed by a load, a * load followed by a store, and a store followed by a store. Only a store * followed by a load to a different memory location may be reordered. * Therefore, except for special cases, like non-temporal memory accesses or * memory regions defined as write combining, the memory ordering effects * provided by the sfence instruction in the wmb() function and the lfence * instruction in the rmb() function are redundant. In contrast, the * atomic_*() functions with acquire and release semantics do not perform * redundant instructions for ordinary cases of interprocessor memory * ordering on any architecture. */ #define mb() __asm __volatile("mfence;" : : : "memory") #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") +#include + /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ /* * The above functions are expanded inline in the statically-linked * kernel. Lock prefixes are generated if an SMP kernel is being * built. * * Kernel modules call real functions which are built into the kernel. * This allows kernel modules to be portable between UP and SMP systems. */ #if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM) #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandset_long(volatile u_long *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); int atomic_testandclear_long(volatile u_long *p, u_int v); void atomic_thread_fence_acq(void); void atomic_thread_fence_acq_rel(void); void atomic_thread_fence_rel(void); void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) #else /* !KLD_MODULE && __GNUCLIKE_ASM */ /* * For userland, always use lock prefixes so that the binaries will run * on both SMP and !SMP systems. */ #if defined(SMP) || !defined(_KERNEL) #define MPLOCKED "lock ; " #else #define MPLOCKED #endif /* * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions * * if (*dst == expect) *dst = src (all 32 bit words) * * Returns 0 on failure, non-zero on success */ static __inline int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) { u_char res; __asm __volatile( " " MPLOCKED " " " cmpxchgl %3,%1 ; " " sete %0 ; " "# atomic_cmpset_int" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+a" (expect) /* 2 */ : "r" (src) /* 3 */ : "memory", "cc"); return (res); } static __inline int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) { u_char res; __asm __volatile( " " MPLOCKED " " " cmpxchgq %3,%1 ; " " sete %0 ; " "# atomic_cmpset_long" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+a" (expect) /* 2 */ : "r" (src) /* 3 */ : "memory", "cc"); return (res); } static __inline int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src) { u_char res; __asm __volatile( " " MPLOCKED " " " cmpxchgl %3,%1 ; " " sete %0 ; " "# atomic_fcmpset_int" : "=r" (res), /* 0 */ "+m" (*dst), /* 1 */ "+a" (*expect) /* 2 */ : "r" (src) /* 3 */ : "memory", "cc"); return (res); } static __inline int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src) { u_char res; __asm __volatile( " " MPLOCKED " " " cmpxchgq %3,%1 ; " " sete %0 ; " "# atomic_fcmpset_long" : "=r" (res), /* 0 */ "+m" (*dst), /* 1 */ "+a" (*expect) /* 2 */ : "r" (src) /* 3 */ : "memory", "cc"); return (res); } /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " " MPLOCKED " " " xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } /* * Atomically add the value of v to the long integer pointed to by p and return * the previous value of *p. */ static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { __asm __volatile( " " MPLOCKED " " " xaddq %0,%1 ; " "# atomic_fetchadd_long" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btsl %2,%1 ; " " setc %0 ; " "# atomic_testandset_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btsq %2,%1 ; " " setc %0 ; " "# atomic_testandset_long" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btrl %2,%1 ; " " setc %0 ; " "# atomic_testandclear_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btrq %2,%1 ; " " setc %0 ; " "# atomic_testandclear_long" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need a Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. To avoid false data dependencies, we use a * special address for "mem". In the kernel, we use a private per-cpu * cache line. In user space, we use a word in the stack's red zone * (-8(%rsp)). * * For UP kernels, however, the memory of the single processor is * always consistent, so we only need to stop the compiler from * reordering accesses in a way that violates the semantics of acquire * and release. */ #if defined(_KERNEL) /* * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ #define OFFSETOF_MONITORBUF 0x180 #if defined(SMP) static __inline void __storeload_barrier(void) { __asm __volatile("lock; addl $0,%%gs:%0" : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); } #else /* _KERNEL && UP */ static __inline void __storeload_barrier(void) { __compiler_membar(); } #endif /* SMP */ #else /* !_KERNEL */ static __inline void __storeload_barrier(void) { __asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc"); } #endif /* _KERNEL*/ #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orq %1,%0", "ir", v); ATOMIC_ASM(clear, long, "andq %1,%0", "ir", ~v); ATOMIC_ASM(add, long, "addq %1,%0", "ir", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "ir", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { __asm __volatile( " xchgq %1,%0 ; " "# atomic_swap_long" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } #else /* !__GNUCLIKE_ASM */ u_int atomic_swap_int(volatile u_int *p, u_int v); u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subtract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_testandset_64 atomic_testandset_long #define atomic_testandclear_64 atomic_testandclear_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ #endif /* !_MACHINE_ATOMIC_H_ */ Index: stable/11/sys/arm/include/atomic.h =================================================================== --- stable/11/sys/arm/include/atomic.h (revision 327194) +++ stable/11/sys/arm/include/atomic.h (revision 327195) @@ -1,122 +1,97 @@ /* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ /*- * Copyright (C) 2003-2004 Olivier Houchard * Copyright (C) 1994-1997 Mark Brinicombe * Copyright (C) 1994 Brini * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of Brini may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ +#include + #include #ifndef _KERNEL #include #endif #if __ARM_ARCH >= 6 #include #else /* < armv6 */ #include #endif /* Arch >= v6 */ -static __inline int -atomic_load_32(volatile uint32_t *v) -{ - - return (*v); -} - -static __inline void -atomic_store_32(volatile uint32_t *dst, uint32_t src) -{ - *dst = src; -} - -static __inline int -atomic_load_long(volatile u_long *v) -{ - - return (*v); -} - -static __inline void -atomic_store_long(volatile u_long *dst, u_long src) -{ - *dst = src; -} - #define atomic_clear_ptr atomic_clear_32 #define atomic_clear_acq_ptr atomic_clear_acq_32 #define atomic_clear_rel_ptr atomic_clear_rel_32 #define atomic_set_ptr atomic_set_32 #define atomic_set_acq_ptr atomic_set_acq_32 #define atomic_set_rel_ptr atomic_set_rel_32 #define atomic_fcmpset_ptr atomic_fcmpset_32 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_32 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_32 #define atomic_cmpset_ptr atomic_cmpset_32 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_32 #define atomic_load_acq_ptr atomic_load_acq_32 -#define atomic_store_ptr atomic_store_32 #define atomic_store_rel_ptr atomic_store_rel_32 #define atomic_swap_ptr atomic_swap_32 #define atomic_readandclear_ptr atomic_readandclear_32 #define atomic_add_int atomic_add_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_clear_int atomic_clear_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_set_int atomic_set_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_swap_int atomic_swap_32 #endif /* _MACHINE_ATOMIC_H_ */ Index: stable/11/sys/arm64/include/atomic.h =================================================================== --- stable/11/sys/arm64/include/atomic.h (revision 327194) +++ stable/11/sys/arm64/include/atomic.h (revision 327195) @@ -1,475 +1,477 @@ /*- * Copyright (c) 2013 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ +#include + #define isb() __asm __volatile("isb" : : : "memory") /* * Options for DMB and DSB: * oshld Outer Shareable, load * oshst Outer Shareable, store * osh Outer Shareable, all * nshld Non-shareable, load * nshst Non-shareable, store * nsh Non-shareable, all * ishld Inner Shareable, load * ishst Inner Shareable, store * ish Inner Shareable, all * ld Full system, load * st Full system, store * sy Full system, all */ #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") #define mb() dmb(sy) /* Full system memory barrier all */ #define wmb() dmb(st) /* Full system memory barrier store */ #define rmb() dmb(ld) /* Full system memory barrier load */ #define ATOMIC_OP(op, asm_op, bar, a, l) \ static __inline void \ atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val) \ { \ uint32_t tmp; \ int res; \ \ __asm __volatile( \ "1: ld"#a"xr %w0, [%2] \n" \ " "#asm_op" %w0, %w0, %w3 \n" \ " st"#l"xr %w1, %w0, [%2] \n" \ " cbnz %w1, 1b \n" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } \ \ static __inline void \ atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val) \ { \ uint64_t tmp; \ int res; \ \ __asm __volatile( \ "1: ld"#a"xr %0, [%2] \n" \ " "#asm_op" %0, %0, %3 \n" \ " st"#l"xr %w1, %0, [%2] \n" \ " cbnz %w1, 1b \n" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (val) \ : "memory" \ ); \ } #define ATOMIC(op, asm_op) \ ATOMIC_OP(op, asm_op, , , ) \ ATOMIC_OP(op, asm_op, acq_, a, ) \ ATOMIC_OP(op, asm_op, rel_, , l) \ ATOMIC(add, add) ATOMIC(clear, bic) ATOMIC(set, orr) ATOMIC(subtract, sub) #define ATOMIC_FCMPSET(bar, a, l) \ static __inline int \ atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval, \ uint32_t newval) \ { \ uint32_t tmp; \ uint32_t _cmpval = *cmpval; \ int res; \ \ __asm __volatile( \ "1: mov %w1, #1 \n" \ " ld"#a"xr %w0, [%2] \n" \ " cmp %w0, %w3 \n" \ " b.ne 2f \n" \ " st"#l"xr %w1, %w4, [%2] \n" \ "2:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (_cmpval), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (!res); \ } \ \ static __inline int \ atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval, \ uint64_t newval) \ { \ uint64_t tmp; \ uint64_t _cmpval = *cmpval; \ int res; \ \ __asm __volatile( \ "1: mov %w1, #1 \n" \ " ld"#a"xr %0, [%2] \n" \ " cmp %0, %3 \n" \ " b.ne 2f \n" \ " st"#l"xr %w1, %4, [%2] \n" \ "2:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (_cmpval), "r" (newval) \ : "cc", "memory" \ ); \ *cmpval = tmp; \ \ return (!res); \ } ATOMIC_FCMPSET( , , ) ATOMIC_FCMPSET(acq_, a, ) ATOMIC_FCMPSET(rel_, ,l) #undef ATOMIC_FCMPSET #define ATOMIC_CMPSET(bar, a, l) \ static __inline int \ atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ uint32_t newval) \ { \ uint32_t tmp; \ int res; \ \ __asm __volatile( \ "1: mov %w1, #1 \n" \ " ld"#a"xr %w0, [%2] \n" \ " cmp %w0, %w3 \n" \ " b.ne 2f \n" \ " st"#l"xr %w1, %w4, [%2] \n" \ " cbnz %w1, 1b \n" \ "2:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (cmpval), "r" (newval) \ : "cc", "memory" \ ); \ \ return (!res); \ } \ \ static __inline int \ atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval, \ uint64_t newval) \ { \ uint64_t tmp; \ int res; \ \ __asm __volatile( \ "1: mov %w1, #1 \n" \ " ld"#a"xr %0, [%2] \n" \ " cmp %0, %3 \n" \ " b.ne 2f \n" \ " st"#l"xr %w1, %4, [%2] \n" \ " cbnz %w1, 1b \n" \ "2:" \ : "=&r"(tmp), "=&r"(res) \ : "r" (p), "r" (cmpval), "r" (newval) \ : "cc", "memory" \ ); \ \ return (!res); \ } ATOMIC_CMPSET( , , ) ATOMIC_CMPSET(acq_, a, ) ATOMIC_CMPSET(rel_, ,l) static __inline uint32_t atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) { uint32_t tmp, ret; int res; __asm __volatile( "1: ldxr %w2, [%3] \n" " add %w0, %w2, %w4 \n" " stxr %w1, %w0, [%3] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "=&r"(ret) : "r" (p), "r" (val) : "memory" ); return (ret); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) { uint64_t tmp, ret; int res; __asm __volatile( "1: ldxr %2, [%3] \n" " add %0, %2, %4 \n" " stxr %w1, %0, [%3] \n" " cbnz %w1, 1b \n" : "=&r"(tmp), "=&r"(res), "=&r"(ret) : "r" (p), "r" (val) : "memory" ); return (ret); } static __inline uint32_t atomic_readandclear_32(volatile uint32_t *p) { uint32_t ret; int res; __asm __volatile( "1: ldxr %w1, [%2] \n" " stxr %w0, wzr, [%2] \n" " cbnz %w0, 1b \n" : "=&r"(res), "=&r"(ret) : "r" (p) : "memory" ); return (ret); } static __inline uint64_t atomic_readandclear_64(volatile uint64_t *p) { uint64_t ret; int res; __asm __volatile( "1: ldxr %1, [%2] \n" " stxr %w0, xzr, [%2] \n" " cbnz %w0, 1b \n" : "=&r"(res), "=&r"(ret) : "r" (p) : "memory" ); return (ret); } static __inline uint32_t atomic_swap_32(volatile uint32_t *p, uint32_t val) { uint32_t ret; int res; __asm __volatile( "1: ldxr %w0, [%2] \n" " stxr %w1, %w3, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(ret), "=&r"(res) : "r" (p), "r" (val) : "memory" ); return (ret); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t val) { uint64_t ret; int res; __asm __volatile( "1: ldxr %0, [%2] \n" " stxr %w1, %3, [%2] \n" " cbnz %w1, 1b \n" : "=&r"(ret), "=&r"(res) : "r" (p), "r" (val) : "memory" ); return (ret); } static __inline uint32_t atomic_load_acq_32(volatile uint32_t *p) { uint32_t ret; __asm __volatile( "ldar %w0, [%1] \n" : "=&r" (ret) : "r" (p) : "memory"); return (ret); } static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { uint64_t ret; __asm __volatile( "ldar %0, [%1] \n" : "=&r" (ret) : "r" (p) : "memory"); return (ret); } static __inline void atomic_store_rel_32(volatile uint32_t *p, uint32_t val) { __asm __volatile( "stlr %w0, [%1] \n" : : "r" (val), "r" (p) : "memory"); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t val) { __asm __volatile( "stlr %0, [%1] \n" : : "r" (val), "r" (p) : "memory"); } #define atomic_add_int atomic_add_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_swap_int atomic_swap_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_add_long atomic_add_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_swap_long atomic_swap_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_add_ptr atomic_add_64 #define atomic_fcmpset_ptr atomic_fcmpset_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_swap_ptr atomic_swap_64 #define atomic_subtract_ptr atomic_subtract_64 #define atomic_add_acq_long atomic_add_acq_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 #define atomic_clear_acq_ptr atomic_clear_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 static __inline void atomic_thread_fence_acq(void) { dmb(ld); } static __inline void atomic_thread_fence_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_acq_rel(void) { dmb(sy); } static __inline void atomic_thread_fence_seq_cst(void) { dmb(sy); } #endif /* _MACHINE_ATOMIC_H_ */ Index: stable/11/sys/i386/include/atomic.h =================================================================== --- stable/11/sys/i386/include/atomic.h (revision 327194) +++ stable/11/sys/i386/include/atomic.h (revision 327195) @@ -1,829 +1,831 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif +#include + #ifdef _KERNEL #include #include #endif #ifndef __OFFSETOF_MONITORBUF /* * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in i386/vm_machdep.c ensures that the value is correct. */ #define __OFFSETOF_MONITORBUF 0x180 static __inline void __mbk(void) { __asm __volatile("lock; addl $0,%%fs:%0" : "+m" (*(u_int *)__OFFSETOF_MONITORBUF) : : "memory", "cc"); } static __inline void __mbu(void) { __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); } #endif /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ /* * The above functions are expanded inline in the statically-linked * kernel. Lock prefixes are generated if an SMP kernel is being * built. * * Kernel modules call real functions which are built into the kernel. * This allows kernel modules to be portable between UP and SMP systems. */ #if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM) #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); void atomic_thread_fence_acq(void); void atomic_thread_fence_acq_rel(void); void atomic_thread_fence_rel(void); void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_64(volatile uint64_t *, uint64_t, uint64_t); uint64_t atomic_load_acq_64(volatile uint64_t *); void atomic_store_rel_64(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64(volatile uint64_t *, uint64_t); uint64_t atomic_fetchadd_64(volatile uint64_t *, uint64_t); #else /* !KLD_MODULE && __GNUCLIKE_ASM */ /* * For userland, always use lock prefixes so that the binaries will run * on both SMP and !SMP systems. */ #if defined(SMP) || !defined(_KERNEL) #define MPLOCKED "lock ; " #else #define MPLOCKED #endif /* * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions * * if (*dst == expect) *dst = src (all 32 bit words) * * Returns 0 on failure, non-zero on success */ static __inline int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) { u_char res; __asm __volatile( " " MPLOCKED " " " cmpxchgl %3,%1 ; " " sete %0 ; " "# atomic_cmpset_int" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+a" (expect) /* 2 */ : "r" (src) /* 3 */ : "memory", "cc"); return (res); } static __inline int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src) { u_char res; __asm __volatile( " " MPLOCKED " " " cmpxchgl %3,%1 ; " " sete %0 ; " "# atomic_cmpset_int" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+a" (*expect) /* 2 */ : "r" (src) /* 3 */ : "memory", "cc"); return (res); } /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " " MPLOCKED " " " xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btsl %2,%1 ; " " setc %0 ; " "# atomic_testandset_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btrl %2,%1 ; " " setc %0 ; " "# atomic_testandclear_int" : "=q" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. In the kernel, we use a private per-cpu * cache line for "mem", to avoid introducing false data * dependencies. In user space, we use the word at the top of the * stack. * * For UP kernels, however, the memory of the single processor is * always consistent, so we only need to stop the compiler from * reordering accesses in a way that violates the semantics of acquire * and release. */ #if defined(_KERNEL) #if defined(SMP) #define __storeload_barrier() __mbk() #else /* _KERNEL && UP */ #define __storeload_barrier() __compiler_membar() #endif /* SMP */ #else /* !_KERNEL */ #define __storeload_barrier() __mbu() #endif /* _KERNEL*/ #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #ifdef _KERNEL #ifdef WANT_FUNCTIONS int atomic_cmpset_64_i386(volatile uint64_t *, uint64_t, uint64_t); int atomic_cmpset_64_i586(volatile uint64_t *, uint64_t, uint64_t); uint64_t atomic_load_acq_64_i386(volatile uint64_t *); uint64_t atomic_load_acq_64_i586(volatile uint64_t *); void atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); void atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64_i386(volatile uint64_t *, uint64_t); uint64_t atomic_swap_64_i586(volatile uint64_t *, uint64_t); #endif /* I486 does not support SMP or CMPXCHG8B. */ static __inline int atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) { volatile uint32_t *p; u_char res; p = (volatile uint32_t *)dst; __asm __volatile( " pushfl ; " " cli ; " " xorl %1,%%eax ; " " xorl %2,%%edx ; " " orl %%edx,%%eax ; " " jne 1f ; " " movl %4,%1 ; " " movl %5,%2 ; " "1: " " sete %3 ; " " popfl" : "+A" (expect), /* 0 */ "+m" (*p), /* 1 */ "+m" (*(p + 1)), /* 2 */ "=q" (res) /* 3 */ : "r" ((uint32_t)src), /* 4 */ "r" ((uint32_t)(src >> 32)) /* 5 */ : "memory", "cc"); return (res); } static __inline uint64_t atomic_load_acq_64_i386(volatile uint64_t *p) { volatile uint32_t *q; uint64_t res; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %1,%%eax ; " " movl %2,%%edx ; " " popfl" : "=&A" (res) /* 0 */ : "m" (*q), /* 1 */ "m" (*(q + 1)) /* 2 */ : "memory"); return (res); } static __inline void atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) { volatile uint32_t *q; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %%eax,%0 ; " " movl %%edx,%1 ; " " popfl" : "=m" (*q), /* 0 */ "=m" (*(q + 1)) /* 1 */ : "A" (v) /* 2 */ : "memory"); } static __inline uint64_t atomic_swap_64_i386(volatile uint64_t *p, uint64_t v) { volatile uint32_t *q; uint64_t res; q = (volatile uint32_t *)p; __asm __volatile( " pushfl ; " " cli ; " " movl %1,%%eax ; " " movl %2,%%edx ; " " movl %4,%2 ; " " movl %3,%1 ; " " popfl" : "=&A" (res), /* 0 */ "+m" (*q), /* 1 */ "+m" (*(q + 1)) /* 2 */ : "r" ((uint32_t)v), /* 3 */ "r" ((uint32_t)(v >> 32))); /* 4 */ return (res); } static __inline int atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) { u_char res; __asm __volatile( " " MPLOCKED " " " cmpxchg8b %1 ; " " sete %0" : "=q" (res), /* 0 */ "+m" (*dst), /* 1 */ "+A" (expect) /* 2 */ : "b" ((uint32_t)src), /* 3 */ "c" ((uint32_t)(src >> 32)) /* 4 */ : "memory", "cc"); return (res); } static __inline uint64_t atomic_load_acq_64_i586(volatile uint64_t *p) { uint64_t res; __asm __volatile( " movl %%ebx,%%eax ; " " movl %%ecx,%%edx ; " " " MPLOCKED " " " cmpxchg8b %1" : "=&A" (res), /* 0 */ "+m" (*p) /* 1 */ : : "memory", "cc"); return (res); } static __inline void atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) { __asm __volatile( " movl %%eax,%%ebx ; " " movl %%edx,%%ecx ; " "1: " " " MPLOCKED " " " cmpxchg8b %0 ; " " jne 1b" : "+m" (*p), /* 0 */ "+A" (v) /* 1 */ : : "ebx", "ecx", "memory", "cc"); } static __inline uint64_t atomic_swap_64_i586(volatile uint64_t *p, uint64_t v) { __asm __volatile( " movl %%eax,%%ebx ; " " movl %%edx,%%ecx ; " "1: " " " MPLOCKED " " " cmpxchg8b %0 ; " " jne 1b" : "+m" (*p), /* 0 */ "+A" (v) /* 1 */ : : "ebx", "ecx", "memory", "cc"); return (v); } static __inline int atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_cmpset_64_i386(dst, expect, src)); else return (atomic_cmpset_64_i586(dst, expect, src)); } static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_load_acq_64_i386(p)); else return (atomic_load_acq_64_i586(p)); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t v) { if ((cpu_feature & CPUID_CX8) == 0) atomic_store_rel_64_i386(p, v); else atomic_store_rel_64_i586(p, v); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t v) { if ((cpu_feature & CPUID_CX8) == 0) return (atomic_swap_64_i386(p, v)); else return (atomic_swap_64_i586(p, v)); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) { for (;;) { uint64_t t = *p; if (atomic_cmpset_64(p, t, t + v)) return (t); } } #endif /* _KERNEL */ #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, long, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS static __inline int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) { return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect, (u_int)src)); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { return (atomic_testandset_int((volatile u_int *)p, v)); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { return (atomic_testandclear_int((volatile u_int *)p, v)); } /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { return (atomic_swap_int((volatile u_int *)p, (u_int)v)); } #else /* !__GNUCLIKE_ASM */ u_int atomic_swap_int(volatile u_int *p, u_int v); u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on pointers. */ #define atomic_set_ptr(p, v) \ atomic_set_int((volatile u_int *)(p), (u_int)(v)) #define atomic_set_acq_ptr(p, v) \ atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_set_rel_ptr(p, v) \ atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_ptr(p, v) \ atomic_clear_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_acq_ptr(p, v) \ atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_rel_ptr(p, v) \ atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_ptr(p, v) \ atomic_add_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_acq_ptr(p, v) \ atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_add_rel_ptr(p, v) \ atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_ptr(p, v) \ atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_acq_ptr(p, v) \ atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_rel_ptr(p, v) \ atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) #define atomic_load_acq_ptr(p) \ atomic_load_acq_int((volatile u_int *)(p)) #define atomic_store_rel_ptr(p, v) \ atomic_store_rel_int((volatile u_int *)(p), (v)) #define atomic_cmpset_ptr(dst, old, new) \ atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) #define atomic_cmpset_acq_ptr(dst, old, new) \ atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) #define atomic_fcmpset_ptr(dst, old, new) \ atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new)) #define atomic_fcmpset_acq_ptr(dst, old, new) \ atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \ (u_int)(new)) #define atomic_fcmpset_rel_ptr(dst, old, new) \ atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \ (u_int)(new)) #define atomic_swap_ptr(p, v) \ atomic_swap_int((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_ptr(p) \ atomic_readandclear_int((volatile u_int *)(p)) #endif /* !WANT_FUNCTIONS */ #if defined(_KERNEL) #define mb() __mbk() #define wmb() __mbk() #define rmb() __mbk() #else #define mb() __mbu() #define wmb() __mbu() #define rmb() __mbu() #endif #endif /* !_MACHINE_ATOMIC_H_ */ Index: stable/11/sys/mips/include/atomic.h =================================================================== --- stable/11/sys/mips/include/atomic.h (revision 327194) +++ stable/11/sys/mips/include/atomic.h (revision 327195) @@ -1,771 +1,756 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: src/sys/alpha/include/atomic.h,v 1.21.2.3 2005/10/06 18:12:05 jhb * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif +#include + /* * Note: All the 64-bit atomic operations are only atomic when running * in 64-bit mode. It is assumed that code compiled for n32 and n64 * fits into this definition and no further safeties are needed. * * It is also assumed that the add, subtract and other arithmetic is * done on numbers not pointers. The special rules for n32 pointers * do not have atomic operations defined for them, but generally shouldn't * need atomic operations. */ #ifndef __MIPS_PLATFORM_SYNC_NOPS #define __MIPS_PLATFORM_SYNC_NOPS "" #endif static __inline void mips_sync(void) { __asm __volatile (".set noreorder\n" "\tsync\n" __MIPS_PLATFORM_SYNC_NOPS ".set reorder\n" : : : "memory"); } #define mb() mips_sync() #define wmb() mips_sync() #define rmb() mips_sync() /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and SMP safe. */ void atomic_set_8(__volatile uint8_t *, uint8_t); void atomic_clear_8(__volatile uint8_t *, uint8_t); void atomic_add_8(__volatile uint8_t *, uint8_t); void atomic_subtract_8(__volatile uint8_t *, uint8_t); void atomic_set_16(__volatile uint16_t *, uint16_t); void atomic_clear_16(__volatile uint16_t *, uint16_t); void atomic_add_16(__volatile uint16_t *, uint16_t); void atomic_subtract_16(__volatile uint16_t *, uint16_t); static __inline void atomic_set_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "or %0, %2, %0\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_clear_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; v = ~v; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "and %0, %2, %0\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_add_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "addu %0, %2, %0\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_subtract_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "subu %0, %2\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline uint32_t atomic_readandclear_32(__volatile uint32_t *addr) { uint32_t result,temp; __asm __volatile ( "1:\tll %0,%3\n\t" /* load current value, asserting lock */ "li %1,0\n\t" /* value to store */ "sc %1,%2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr) : "memory"); return result; } static __inline uint32_t atomic_readandset_32(__volatile uint32_t *addr, uint32_t value) { uint32_t result,temp; __asm __volatile ( "1:\tll %0,%3\n\t" /* load current value, asserting lock */ "or %1,$0,%4\n\t" "sc %1,%2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr), "r" (value) : "memory"); return result; } #if defined(__mips_n64) || defined(__mips_n32) static __inline void atomic_set_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "or %0, %2, %0\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_clear_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; v = ~v; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "and %0, %2, %0\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_add_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "daddu %0, %2, %0\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_subtract_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "dsubu %0, %2\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline uint64_t atomic_readandclear_64(__volatile uint64_t *addr) { uint64_t result,temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* value to store */ "scd %1, %2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr) : "memory"); return result; } static __inline uint64_t atomic_readandset_64(__volatile uint64_t *addr, uint64_t value) { uint64_t result,temp; __asm __volatile ( "1:\n\t" "lld %0,%3\n\t" /* Load old value*/ "or %1,$0,%4\n\t" "scd %1,%2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr), "r" (value) : "memory"); return result; } #endif #define ATOMIC_ACQ_REL(NAME, WIDTH) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ atomic_##NAME##_##WIDTH(p, v); \ mips_sync(); \ } \ \ static __inline void \ atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ mips_sync(); \ atomic_##NAME##_##WIDTH(p, v); \ } /* Variants of simple arithmetic with memory barriers. */ ATOMIC_ACQ_REL(set, 8) ATOMIC_ACQ_REL(clear, 8) ATOMIC_ACQ_REL(add, 8) ATOMIC_ACQ_REL(subtract, 8) ATOMIC_ACQ_REL(set, 16) ATOMIC_ACQ_REL(clear, 16) ATOMIC_ACQ_REL(add, 16) ATOMIC_ACQ_REL(subtract, 16) ATOMIC_ACQ_REL(set, 32) ATOMIC_ACQ_REL(clear, 32) ATOMIC_ACQ_REL(add, 32) ATOMIC_ACQ_REL(subtract, 32) #if defined(__mips_n64) || defined(__mips_n32) ATOMIC_ACQ_REL(set, 64) ATOMIC_ACQ_REL(clear, 64) ATOMIC_ACQ_REL(add, 64) ATOMIC_ACQ_REL(subtract, 64) #endif #undef ATOMIC_ACQ_REL /* * We assume that a = b will do atomic loads and stores. */ #define ATOMIC_STORE_LOAD(WIDTH) \ static __inline uint##WIDTH##_t \ atomic_load_acq_##WIDTH(__volatile uint##WIDTH##_t *p) \ { \ uint##WIDTH##_t v; \ \ v = *p; \ mips_sync(); \ return (v); \ } \ \ static __inline void \ atomic_store_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ mips_sync(); \ *p = v; \ } ATOMIC_STORE_LOAD(32) ATOMIC_STORE_LOAD(64) -#if !defined(__mips_n64) && !defined(__mips_n32) -void atomic_store_64(__volatile uint64_t *, uint64_t *); -void atomic_load_64(__volatile uint64_t *, uint64_t *); -#else -static __inline void -atomic_store_64(__volatile uint64_t *p, uint64_t *v) -{ - *p = *v; -} - -static __inline void -atomic_load_64(__volatile uint64_t *p, uint64_t *v) -{ - *v = *p; -} -#endif - #undef ATOMIC_STORE_LOAD /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint32_t atomic_cmpset_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t ret; __asm __volatile ( "1:\tll %0, %4\n\t" /* load old value */ "bne %0, %2, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "li %0, 0\n\t" "3:\n" : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); return ret; } /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint32_t atomic_cmpset_acq_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { int retval; retval = atomic_cmpset_32(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint32_t atomic_cmpset_rel_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { mips_sync(); return (atomic_cmpset_32(p, cmpval, newval)); } static __inline uint32_t atomic_fcmpset_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { uint32_t ret; __asm __volatile ( "1:\n\t" "ll %0, %1\n\t" /* load old value */ "bne %0, %4, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "sw %0, %2\n\t" /* save old value */ "li %0, 0\n\t" "3:\n" : "=&r" (ret), "+m" (*p), "=m" (*cmpval) : "r" (newval), "r" (*cmpval) : "memory"); return ret; } static __inline uint32_t atomic_fcmpset_acq_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { int retval; retval = atomic_fcmpset_32(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint32_t atomic_fcmpset_rel_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { mips_sync(); return (atomic_fcmpset_32(p, cmpval, newval)); } /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline uint32_t atomic_fetchadd_32(__volatile uint32_t *p, uint32_t v) { uint32_t value, temp; __asm __volatile ( "1:\tll %0, %1\n\t" /* load old value */ "addu %2, %3, %0\n\t" /* calculate new value */ "sc %2, %1\n\t" /* attempt to store */ "beqz %2, 1b\n\t" /* spin if failed */ : "=&r" (value), "=m" (*p), "=&r" (temp) : "r" (v), "m" (*p)); return (value); } #if defined(__mips_n64) || defined(__mips_n32) /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint64_t atomic_cmpset_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t ret; __asm __volatile ( "1:\n\t" "lld %0, %4\n\t" /* load old value */ "bne %0, %2, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "li %0, 0\n\t" "3:\n" : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); return ret; } /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint64_t atomic_cmpset_acq_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { int retval; retval = atomic_cmpset_64(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint64_t atomic_cmpset_rel_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { mips_sync(); return (atomic_cmpset_64(p, cmpval, newval)); } static __inline uint32_t atomic_fcmpset_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { uint32_t ret; __asm __volatile ( "1:\n\t" "lld %0, %1\n\t" /* load old value */ "bne %0, %4, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "sd %0, %2\n\t" /* save old value */ "li %0, 0\n\t" "3:\n" : "=&r" (ret), "+m" (*p), "=m" (*cmpval) : "r" (newval), "r" (*cmpval) : "memory"); return ret; } static __inline uint64_t atomic_fcmpset_acq_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { int retval; retval = atomic_fcmpset_64(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint64_t atomic_fcmpset_rel_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { mips_sync(); return (atomic_fcmpset_64(p, cmpval, newval)); } /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline uint64_t atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v) { uint64_t value, temp; __asm __volatile ( "1:\n\t" "lld %0, %1\n\t" /* load old value */ "daddu %2, %3, %0\n\t" /* calculate new value */ "scd %2, %1\n\t" /* attempt to store */ "beqz %2, 1b\n\t" /* spin if failed */ : "=&r" (value), "=m" (*p), "=&r" (temp) : "r" (v), "m" (*p)); return (value); } #endif static __inline void atomic_thread_fence_acq(void) { mips_sync(); } static __inline void atomic_thread_fence_rel(void) { mips_sync(); } static __inline void atomic_thread_fence_acq_rel(void) { mips_sync(); } static __inline void atomic_thread_fence_seq_cst(void) { mips_sync(); } /* Operations on chars. */ #define atomic_set_char atomic_set_8 #define atomic_set_acq_char atomic_set_acq_8 #define atomic_set_rel_char atomic_set_rel_8 #define atomic_clear_char atomic_clear_8 #define atomic_clear_acq_char atomic_clear_acq_8 #define atomic_clear_rel_char atomic_clear_rel_8 #define atomic_add_char atomic_add_8 #define atomic_add_acq_char atomic_add_acq_8 #define atomic_add_rel_char atomic_add_rel_8 #define atomic_subtract_char atomic_subtract_8 #define atomic_subtract_acq_char atomic_subtract_acq_8 #define atomic_subtract_rel_char atomic_subtract_rel_8 /* Operations on shorts. */ #define atomic_set_short atomic_set_16 #define atomic_set_acq_short atomic_set_acq_16 #define atomic_set_rel_short atomic_set_rel_16 #define atomic_clear_short atomic_clear_16 #define atomic_clear_acq_short atomic_clear_acq_16 #define atomic_clear_rel_short atomic_clear_rel_16 #define atomic_add_short atomic_add_16 #define atomic_add_acq_short atomic_add_acq_16 #define atomic_add_rel_short atomic_add_rel_16 #define atomic_subtract_short atomic_subtract_16 #define atomic_subtract_acq_short atomic_subtract_acq_16 #define atomic_subtract_rel_short atomic_subtract_rel_16 /* Operations on ints. */ #define atomic_set_int atomic_set_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_clear_int atomic_clear_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_add_int atomic_add_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_readandset_int atomic_readandset_32 #define atomic_fetchadd_int atomic_fetchadd_32 /* * I think the following is right, even for n32. For n32 the pointers * are still 32-bits, so we need to operate on them as 32-bit quantities, * even though they are sign extended in operation. For longs, there's * no question because they are always 32-bits. */ #ifdef __mips_n64 /* Operations on longs. */ #define atomic_set_long atomic_set_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_clear_long atomic_clear_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_add_long atomic_add_64 #define atomic_add_acq_long atomic_add_acq_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #else /* !__mips_n64 */ /* Operations on longs. */ #define atomic_set_long(p, v) \ atomic_set_32((volatile u_int *)(p), (u_int)(v)) #define atomic_set_acq_long(p, v) \ atomic_set_acq_32((volatile u_int *)(p), (u_int)(v)) #define atomic_set_rel_long(p, v) \ atomic_set_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_long(p, v) \ atomic_clear_32((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_acq_long(p, v) \ atomic_clear_acq_32((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_rel_long(p, v) \ atomic_clear_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_add_long(p, v) \ atomic_add_32((volatile u_int *)(p), (u_int)(v)) #define atomic_add_acq_long(p, v) \ atomic_add_32((volatile u_int *)(p), (u_int)(v)) #define atomic_add_rel_long(p, v) \ atomic_add_32((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_long(p, v) \ atomic_subtract_32((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_acq_long(p, v) \ atomic_subtract_acq_32((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_rel_long(p, v) \ atomic_subtract_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_cmpset_long(p, cmpval, newval) \ atomic_cmpset_32((volatile u_int *)(p), (u_int)(cmpval), \ (u_int)(newval)) #define atomic_cmpset_acq_long(p, cmpval, newval) \ atomic_cmpset_acq_32((volatile u_int *)(p), (u_int)(cmpval), \ (u_int)(newval)) #define atomic_cmpset_rel_long(p, cmpval, newval) \ atomic_cmpset_rel_32((volatile u_int *)(p), (u_int)(cmpval), \ (u_int)(newval)) #define atomic_fcmpset_long(p, cmpval, newval) \ atomic_fcmpset_32((volatile u_int *)(p), (u_int *)(cmpval), \ (u_int)(newval)) #define atomic_fcmpset_acq_long(p, cmpval, newval) \ atomic_fcmpset_acq_32((volatile u_int *)(p), (u_int *)(cmpval), \ (u_int)(newval)) #define atomic_fcmpset_rel_long(p, cmpval, newval) \ atomic_fcmpset_rel_32((volatile u_int *)(p), (u_int *)(cmpval), \ (u_int)(newval)) #define atomic_load_acq_long(p) \ (u_long)atomic_load_acq_32((volatile u_int *)(p)) #define atomic_store_rel_long(p, v) \ atomic_store_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_fetchadd_long(p, v) \ atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_long(p) \ atomic_readandclear_32((volatile u_int *)(p)) #endif /* __mips_n64 */ /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* ! _MACHINE_ATOMIC_H_ */ Index: stable/11/sys/mips/mips/db_interface.c =================================================================== --- stable/11/sys/mips/mips/db_interface.c (revision 327194) +++ stable/11/sys/mips/mips/db_interface.c (revision 327195) @@ -1,348 +1,348 @@ /* $OpenBSD: db_machdep.c,v 1.2 1998/09/15 10:50:13 pefo Exp $ */ /*- * Copyright (c) 1998 Per Fogelstrom, Opsycon AB * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed under OpenBSD by * Per Fogelstrom, Opsycon AB, Sweden. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * JNPR: db_interface.c,v 1.6.2.1 2007/08/29 12:24:49 girish */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static db_varfcn_t db_frame; #define DB_OFFSET(x) (db_expr_t *)offsetof(struct trapframe, x) struct db_variable db_regs[] = { { "at", DB_OFFSET(ast), db_frame }, { "v0", DB_OFFSET(v0), db_frame }, { "v1", DB_OFFSET(v1), db_frame }, { "a0", DB_OFFSET(a0), db_frame }, { "a1", DB_OFFSET(a1), db_frame }, { "a2", DB_OFFSET(a2), db_frame }, { "a3", DB_OFFSET(a3), db_frame }, #if defined(__mips_n32) || defined(__mips_n64) { "a4", DB_OFFSET(a4), db_frame }, { "a5", DB_OFFSET(a5), db_frame }, { "a6", DB_OFFSET(a6), db_frame }, { "a7", DB_OFFSET(a7), db_frame }, { "t0", DB_OFFSET(t0), db_frame }, { "t1", DB_OFFSET(t1), db_frame }, { "t2", DB_OFFSET(t2), db_frame }, { "t3", DB_OFFSET(t3), db_frame }, #else { "t0", DB_OFFSET(t0), db_frame }, { "t1", DB_OFFSET(t1), db_frame }, { "t2", DB_OFFSET(t2), db_frame }, { "t3", DB_OFFSET(t3), db_frame }, { "t4", DB_OFFSET(t4), db_frame }, { "t5", DB_OFFSET(t5), db_frame }, { "t6", DB_OFFSET(t6), db_frame }, { "t7", DB_OFFSET(t7), db_frame }, #endif { "s0", DB_OFFSET(s0), db_frame }, { "s1", DB_OFFSET(s1), db_frame }, { "s2", DB_OFFSET(s2), db_frame }, { "s3", DB_OFFSET(s3), db_frame }, { "s4", DB_OFFSET(s4), db_frame }, { "s5", DB_OFFSET(s5), db_frame }, { "s6", DB_OFFSET(s6), db_frame }, { "s7", DB_OFFSET(s7), db_frame }, { "t8", DB_OFFSET(t8), db_frame }, { "t9", DB_OFFSET(t9), db_frame }, { "k0", DB_OFFSET(k0), db_frame }, { "k1", DB_OFFSET(k1), db_frame }, { "gp", DB_OFFSET(gp), db_frame }, { "sp", DB_OFFSET(sp), db_frame }, { "s8", DB_OFFSET(s8), db_frame }, { "ra", DB_OFFSET(ra), db_frame }, { "sr", DB_OFFSET(sr), db_frame }, { "lo", DB_OFFSET(mullo), db_frame }, { "hi", DB_OFFSET(mulhi), db_frame }, { "bad", DB_OFFSET(badvaddr), db_frame }, { "cs", DB_OFFSET(cause), db_frame }, { "pc", DB_OFFSET(pc), db_frame }, }; struct db_variable *db_eregs = db_regs + nitems(db_regs); int (*do_db_log_stack_trace_cmd)(char *); static int db_frame(struct db_variable *vp, db_expr_t *valuep, int op) { register_t *reg; if (kdb_frame == NULL) return (0); reg = (register_t *)((uintptr_t)kdb_frame + (size_t)(intptr_t)vp->valuep); if (op == DB_VAR_GET) *valuep = *reg; else *reg = *valuep; return (1); } int db_read_bytes(vm_offset_t addr, size_t size, char *data) { jmp_buf jb; void *prev_jb; int ret; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { /* * 'addr' could be a memory-mapped I/O address. Try to * do atomic load/store in unit of size requested. + * size == 8 is only atomic on 64bit or n32 kernel. */ if ((size == 2 || size == 4 || size == 8) && ((addr & (size -1)) == 0) && (((vm_offset_t)data & (size -1)) == 0)) { switch (size) { case 2: *(uint16_t *)data = *(uint16_t *)addr; break; case 4: *(uint32_t *)data = *(uint32_t *)addr; break; case 8: - atomic_load_64((volatile u_int64_t *)addr, - (u_int64_t *)data); - break; + *(uint64_t *)data = *(uint64_t *)addr; + break; } } else { char *src; src = (char *)addr; while (size-- > 0) *data++ = *src++; } } (void)kdb_jmpbuf(prev_jb); return (ret); } int db_write_bytes(vm_offset_t addr, size_t size, char *data) { int ret; jmp_buf jb; void *prev_jb; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { /* * 'addr' could be a memory-mapped I/O address. Try to * do atomic load/store in unit of size requested. + * size == 8 is only atomic on 64bit or n32 kernel. */ if ((size == 2 || size == 4 || size == 8) && ((addr & (size -1)) == 0) && (((vm_offset_t)data & (size -1)) == 0)) { switch (size) { case 2: *(uint16_t *)addr = *(uint16_t *)data; break; case 4: *(uint32_t *)addr = *(uint32_t *)data; break; case 8: - atomic_store_64((volatile u_int64_t *)addr, - (u_int64_t *)data); - break; + *(uint64_t *)addr = *(uint64_t *)data; + break; } } else { char *dst; size_t len = size; dst = (char *)addr; while (len-- > 0) *dst++ = *data++; } mips_icache_sync_range((db_addr_t) addr, size); mips_dcache_wbinv_range((db_addr_t) addr, size); } (void)kdb_jmpbuf(prev_jb); return (ret); } /* * To do a single step ddb needs to know the next address * that we will get to. It means that we need to find out * both the address for a branch taken and for not taken, NOT! :-) * MipsEmulateBranch will do the job to find out _exactly_ which * address we will end up at so the 'dual bp' method is not * requiered. */ db_addr_t next_instr_address(db_addr_t pc, boolean_t bd) { db_addr_t next; next = (db_addr_t)MipsEmulateBranch(kdb_frame, pc, 0, 0); return (next); } /* * Decode instruction and figure out type. */ int db_inst_type(int ins) { InstFmt inst; int ityp = 0; inst.word = ins; switch ((int)inst.JType.op) { case OP_SPECIAL: switch ((int)inst.RType.func) { case OP_JR: ityp = IT_BRANCH; break; case OP_JALR: case OP_SYSCALL: ityp = IT_CALL; break; } break; case OP_BCOND: switch ((int)inst.IType.rt) { case OP_BLTZ: case OP_BLTZL: case OP_BGEZ: case OP_BGEZL: ityp = IT_BRANCH; break; case OP_BLTZAL: case OP_BLTZALL: case OP_BGEZAL: case OP_BGEZALL: ityp = IT_CALL; break; } break; case OP_JAL: ityp = IT_CALL; break; case OP_J: case OP_BEQ: case OP_BEQL: case OP_BNE: case OP_BNEL: case OP_BLEZ: case OP_BLEZL: case OP_BGTZ: case OP_BGTZL: ityp = IT_BRANCH; break; case OP_COP1: switch (inst.RType.rs) { case OP_BCx: case OP_BCy: ityp = IT_BRANCH; break; } break; case OP_LB: case OP_LH: case OP_LW: case OP_LD: case OP_LBU: case OP_LHU: case OP_LWU: case OP_LWC1: ityp = IT_LOAD; break; case OP_SB: case OP_SH: case OP_SW: case OP_SD: case OP_SWC1: ityp = IT_STORE; break; } return (ityp); } /* * Return the next pc if the given branch is taken. * MachEmulateBranch() runs analysis for branch delay slot. */ db_addr_t branch_taken(int inst, db_addr_t pc) { db_addr_t ra; register_t fpucsr; /* TBD: when is fsr set */ fpucsr = (curthread) ? curthread->td_pcb->pcb_regs.fsr : 0; ra = (db_addr_t)MipsEmulateBranch(kdb_frame, pc, fpucsr, 0); return (ra); } Index: stable/11/sys/mips/mips/support.S =================================================================== --- stable/11/sys/mips/mips/support.S (revision 327194) +++ stable/11/sys/mips/mips/support.S (revision 327195) @@ -1,1099 +1,1032 @@ /* $OpenBSD: locore.S,v 1.18 1998/09/15 10:58:53 pefo Exp $ */ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Digital Equipment Corporation and Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Copyright (C) 1989 Digital Equipment Corporation. * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby granted, * provided that the above copyright notice appears in all copies. * Digital Equipment Corporation makes no representations about the * suitability of this software for any purpose. It is provided "as is" * without express or implied warranty. * * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/loMem.s, * v 1.1 89/07/11 17:55:04 nelson Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/machAsm.s, * v 9.2 90/01/29 18:00:39 shirriff Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/vm/ds3100.md/vmPmaxAsm.s, * v 1.1 89/07/10 14:27:41 nelson Exp SPRITE (DECWRL) * * from: @(#)locore.s 8.5 (Berkeley) 1/4/94 * JNPR: support.S,v 1.5.2.2 2007/08/29 10:03:49 girish * $FreeBSD$ */ /* * Copyright (c) 1997 Jonathan Stone (hereinafter referred to as the author) * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Jonathan R. Stone for * the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Contains assembly language support routines. */ #include "opt_ddb.h" #include #include #include #include #include #include #include "assym.s" .set noreorder # Noreorder is default style! /* * Primitives */ .text /* * See if access to addr with a len type instruction causes a machine check. * len is length of access (1=byte, 2=short, 4=int) * * badaddr(addr, len) * char *addr; * int len; */ LEAF(badaddr) PTR_LA v0, baderr GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) bne a1, 1, 2f PTR_S v0, U_PCB_ONFAULT(v1) b 5f lbu v0, (a0) 2: bne a1, 2, 4f nop b 5f lhu v0, (a0) 4: lw v0, (a0) 5: PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero # made it w/o errors baderr: j ra li v0, 1 # trap sends us here END(badaddr) /* * int copystr(void *kfaddr, void *kdaddr, size_t maxlen, size_t *lencopied) * Copy a NIL-terminated string, at most maxlen characters long. Return the * number of characters copied (including the NIL) in *lencopied. If the * string is too long, return ENAMETOOLONG; else return 0. */ LEAF(copystr) move t0, a2 beq a2, zero, 4f 1: lbu v0, 0(a0) PTR_SUBU a2, a2, 1 beq v0, zero, 2f sb v0, 0(a1) # each byte until NIL PTR_ADDU a0, a0, 1 bne a2, zero, 1b # less than maxlen PTR_ADDU a1, a1, 1 4: li v0, ENAMETOOLONG # run out of space 2: beq a3, zero, 3f # return num. of copied bytes PTR_SUBU a2, t0, a2 # if the 4th arg was non-NULL PTR_S a2, 0(a3) 3: j ra # v0 is 0 or ENAMETOOLONG nop END(copystr) /* * Copy a null terminated string from the user address space into * the kernel address space. * * copyinstr(fromaddr, toaddr, maxlength, &lencopied) * caddr_t fromaddr; * caddr_t toaddr; * u_int maxlength; * u_int *lencopied; */ NESTED(copyinstr, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a0, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(copystr) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S zero, U_PCB_ONFAULT(v1) j ra PTR_ADDU sp, sp, CALLFRAME_SIZ END(copyinstr) /* * Copy a null terminated string from the kernel address space into * the user address space. * * copyoutstr(fromaddr, toaddr, maxlength, &lencopied) * caddr_t fromaddr; * caddr_t toaddr; * u_int maxlength; * u_int *lencopied; */ NESTED(copyoutstr, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a1, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(copystr) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S zero, U_PCB_ONFAULT(v1) j ra PTR_ADDU sp, sp, CALLFRAME_SIZ END(copyoutstr) /* * Copy specified amount of data from user space into the kernel * copyin(from, to, len) * caddr_t *from; (user source address) * caddr_t *to; (kernel destination address) * unsigned len; */ NESTED(copyin, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a0, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(bcopy) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) # bcopy modified v1, so reload PTR_S zero, U_PCB_ONFAULT(v1) PTR_ADDU sp, sp, CALLFRAME_SIZ j ra move v0, zero END(copyin) /* * Copy specified amount of data from kernel to the user space * copyout(from, to, len) * caddr_t *from; (kernel source address) * caddr_t *to; (user destination address) * unsigned len; */ NESTED(copyout, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a1, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(bcopy) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) # bcopy modified v1, so reload PTR_S zero, U_PCB_ONFAULT(v1) PTR_ADDU sp, sp, CALLFRAME_SIZ j ra move v0, zero END(copyout) LEAF(copyerr) REG_L ra, CALLFRAME_RA(sp) PTR_ADDU sp, sp, CALLFRAME_SIZ j ra li v0, EFAULT # return error END(copyerr) /* * {fu,su},{ibyte,isword,iword}, fetch or store a byte, short or word to * user text space. * {fu,su},{byte,sword,word}, fetch or store a byte, short or word to * user data space. */ #ifdef __mips_n64 LEAF(fuword64) XLEAF(fuword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) ld v0, 0(a0) # fetch word j ra PTR_S zero, U_PCB_ONFAULT(v1) END(fuword64) #endif LEAF(fuword32) #ifndef __mips_n64 XLEAF(fuword) #endif PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lw v0, 0(a0) # fetch word j ra PTR_S zero, U_PCB_ONFAULT(v1) END(fuword32) LEAF(fusword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lhu v0, 0(a0) # fetch short j ra PTR_S zero, U_PCB_ONFAULT(v1) END(fusword) LEAF(fubyte) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lbu v0, 0(a0) # fetch byte j ra PTR_S zero, U_PCB_ONFAULT(v1) END(fubyte) LEAF(suword32) #ifndef __mips_n64 XLEAF(suword) #endif PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sw a1, 0(a0) # store word PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(suword32) #ifdef __mips_n64 LEAF(suword64) XLEAF(suword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sd a1, 0(a0) # store word PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(suword64) #endif /* * casuword(9) * u_long casuword(u_long *p, u_long oldval, u_long newval) */ /* * casuword32(9) * uint32_t casuword(uint32_t *p, uint32_t oldval, * uint32_t newval) */ LEAF(casuword32) #ifndef __mips_n64 XLEAF(casuword) #endif PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) 1: move t0, a2 ll v0, 0(a0) bne a1, v0, 2f nop sc t0, 0(a0) # store word beqz t0, 1b nop j 3f nop 2: li v0, -1 3: PTR_S zero, U_PCB_ONFAULT(v1) jr ra nop END(casuword32) #ifdef __mips_n64 LEAF(casuword64) XLEAF(casuword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) 1: move t0, a2 lld v0, 0(a0) bne a1, v0, 2f nop scd t0, 0(a0) # store double word beqz t0, 1b nop j 3f nop 2: li v0, -1 3: PTR_S zero, U_PCB_ONFAULT(v1) jr ra nop END(casuword64) #endif /* * Will have to flush the instruction cache if byte merging is done in hardware. */ LEAF(susword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sh a1, 0(a0) # store short PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(susword) LEAF(subyte) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sb a1, 0(a0) # store byte PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(subyte) LEAF(fswberr) j ra li v0, -1 END(fswberr) /* * fuswintr and suswintr are just like fusword and susword except that if * the page is not in memory or would cause a trap, then we return an error. * The important thing is to prevent sleep() and switch(). */ LEAF(fuswintr) PTR_LA v0, fswintrberr blt a0, zero, fswintrberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lhu v0, 0(a0) # fetch short j ra PTR_S zero, U_PCB_ONFAULT(v1) END(fuswintr) LEAF(suswintr) PTR_LA v0, fswintrberr blt a0, zero, fswintrberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sh a1, 0(a0) # store short PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(suswintr) LEAF(fswintrberr) j ra li v0, -1 END(fswintrberr) /* * memset(void *s1, int c, int len) * NetBSD: memset.S,v 1.3 2001/10/16 15:40:53 uch Exp */ LEAF(memset) .set noreorder blt a2, 12, memsetsmallclr # small amount to clear? move v0, a0 # save s1 for result sll t1, a1, 8 # compute c << 8 in t1 or t1, t1, a1 # compute c << 8 | c in 11 sll t2, t1, 16 # shift that left 16 or t1, t2, t1 # or together PTR_SUBU t0, zero, a0 # compute # bytes to word align address and t0, t0, 3 beq t0, zero, 1f # skip if word aligned PTR_SUBU a2, a2, t0 # subtract from remaining count SWHI t1, 0(a0) # store 1, 2, or 3 bytes to align PTR_ADDU a0, a0, t0 1: and v1, a2, 3 # compute number of whole words left PTR_SUBU t0, a2, v1 PTR_SUBU a2, a2, t0 PTR_ADDU t0, t0, a0 # compute ending address 2: PTR_ADDU a0, a0, 4 # clear words bne a0, t0, 2b # unrolling loop does not help sw t1, -4(a0) # since we are limited by memory speed memsetsmallclr: ble a2, zero, 2f PTR_ADDU t0, a2, a0 # compute ending address 1: PTR_ADDU a0, a0, 1 # clear bytes bne a0, t0, 1b sb a1, -1(a0) 2: j ra nop .set reorder END(memset) /* * bzero(s1, n) */ LEAF(bzero) XLEAF(blkclr) .set noreorder blt a1, 12, smallclr # small amount to clear? PTR_SUBU a3, zero, a0 # compute # bytes to word align address and a3, a3, 3 beq a3, zero, 1f # skip if word aligned PTR_SUBU a1, a1, a3 # subtract from remaining count SWHI zero, 0(a0) # clear 1, 2, or 3 bytes to align PTR_ADDU a0, a0, a3 1: and v0, a1, 3 # compute number of words left PTR_SUBU a3, a1, v0 move a1, v0 PTR_ADDU a3, a3, a0 # compute ending address 2: PTR_ADDU a0, a0, 4 # clear words bne a0, a3, 2b # unrolling loop does not help sw zero, -4(a0) # since we are limited by memory speed smallclr: ble a1, zero, 2f PTR_ADDU a3, a1, a0 # compute ending address 1: PTR_ADDU a0, a0, 1 # clear bytes bne a0, a3, 1b sb zero, -1(a0) 2: j ra nop END(bzero) /* * bcmp(s1, s2, n) */ LEAF(bcmp) .set noreorder blt a2, 16, smallcmp # is it worth any trouble? xor v0, a0, a1 # compare low two bits of addresses and v0, v0, 3 PTR_SUBU a3, zero, a1 # compute # bytes to word align address bne v0, zero, unalignedcmp # not possible to align addresses and a3, a3, 3 beq a3, zero, 1f PTR_SUBU a2, a2, a3 # subtract from remaining count move v0, v1 # init v0,v1 so unmodified bytes match LWHI v0, 0(a0) # read 1, 2, or 3 bytes LWHI v1, 0(a1) PTR_ADDU a1, a1, a3 bne v0, v1, nomatch PTR_ADDU a0, a0, a3 1: and a3, a2, ~3 # compute number of whole words left PTR_SUBU a2, a2, a3 # which has to be >= (16-3) & ~3 PTR_ADDU a3, a3, a0 # compute ending address 2: lw v0, 0(a0) # compare words lw v1, 0(a1) PTR_ADDU a0, a0, 4 bne v0, v1, nomatch PTR_ADDU a1, a1, 4 bne a0, a3, 2b nop b smallcmp # finish remainder nop unalignedcmp: beq a3, zero, 2f PTR_SUBU a2, a2, a3 # subtract from remaining count PTR_ADDU a3, a3, a0 # compute ending address 1: lbu v0, 0(a0) # compare bytes until a1 word aligned lbu v1, 0(a1) PTR_ADDU a0, a0, 1 bne v0, v1, nomatch PTR_ADDU a1, a1, 1 bne a0, a3, 1b nop 2: and a3, a2, ~3 # compute number of whole words left PTR_SUBU a2, a2, a3 # which has to be >= (16-3) & ~3 PTR_ADDU a3, a3, a0 # compute ending address 3: LWHI v0, 0(a0) # compare words a0 unaligned, a1 aligned LWLO v0, 3(a0) lw v1, 0(a1) PTR_ADDU a0, a0, 4 bne v0, v1, nomatch PTR_ADDU a1, a1, 4 bne a0, a3, 3b nop smallcmp: ble a2, zero, match PTR_ADDU a3, a2, a0 # compute ending address 1: lbu v0, 0(a0) lbu v1, 0(a1) PTR_ADDU a0, a0, 1 bne v0, v1, nomatch PTR_ADDU a1, a1, 1 bne a0, a3, 1b nop match: j ra move v0, zero nomatch: j ra li v0, 1 END(bcmp) /* * bit = ffs(value) */ LEAF(ffs) .set noreorder beq a0, zero, 2f move v0, zero 1: and v1, a0, 1 # bit set? addu v0, v0, 1 beq v1, zero, 1b # no, continue srl a0, a0, 1 2: j ra nop END(ffs) /** * void * atomic_set_16(u_int16_t *a, u_int16_t b) * { * *a |= b; * } */ LEAF(atomic_set_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 andi a1, a1, 0xffff 1: ll t0, 0(a0) or t0, t0, a1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_set_16) /** * void * atomic_clear_16(u_int16_t *a, u_int16_t b) * { * *a &= ~b; * } */ LEAF(atomic_clear_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 nor a1, zero, a1 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xffff # t1 has the original lower 16 bits and t1, t1, a1 # t1 has the new lower 16 bits srl t0, t0, 16 # preserve original top 16 bits sll t0, t0, 16 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_clear_16) /** * void * atomic_subtract_16(uint16_t *a, uint16_t b) * { * *a -= b; * } */ LEAF(atomic_subtract_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xffff # t1 has the original lower 16 bits subu t1, t1, a1 andi t1, t1, 0xffff # t1 has the new lower 16 bits srl t0, t0, 16 # preserve original top 16 bits sll t0, t0, 16 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_subtract_16) /** * void * atomic_add_16(uint16_t *a, uint16_t b) * { * *a += b; * } */ LEAF(atomic_add_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xffff # t1 has the original lower 16 bits addu t1, t1, a1 andi t1, t1, 0xffff # t1 has the new lower 16 bits srl t0, t0, 16 # preserve original top 16 bits sll t0, t0, 16 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_add_16) /** * void * atomic_add_8(uint8_t *a, uint8_t b) * { * *a += b; * } */ LEAF(atomic_add_8) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xff # t1 has the original lower 8 bits addu t1, t1, a1 andi t1, t1, 0xff # t1 has the new lower 8 bits srl t0, t0, 8 # preserve original top 24 bits sll t0, t0, 8 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_add_8) /** * void * atomic_subtract_8(uint8_t *a, uint8_t b) * { * *a += b; * } */ LEAF(atomic_subtract_8) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xff # t1 has the original lower 8 bits subu t1, t1, a1 andi t1, t1, 0xff # t1 has the new lower 8 bits srl t0, t0, 8 # preserve original top 24 bits sll t0, t0, 8 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_subtract_8) -/* - * atomic 64-bit register read/write assembly language support routines. - */ - .set noreorder # Noreorder is default style! - -#if !defined(__mips_n64) && !defined(__mips_n32) - /* - * I don't know if these routines have the right number of - * NOPs in it for all processors. XXX - * - * Maybe it would be better to just leave this undefined in that case. - * - * XXX These routines are not safe in the case of a TLB miss on a1 or - * a0 unless the trapframe is 64-bit, which it just isn't with O32. - * If we take any exception, not just an interrupt, the upper - * 32-bits will be clobbered. Use only N32 and N64 kernels if you - * want to use 64-bit registers while interrupts are enabled or - * with memory operations. Since this isn't even using load-linked - * and store-conditional, perhaps it should just use two registers - * instead, as is right and good with the O32 ABI. - */ -LEAF(atomic_store_64) - mfc0 t1, MIPS_COP_0_STATUS - and t2, t1, ~MIPS_SR_INT_IE - mtc0 t2, MIPS_COP_0_STATUS - nop - nop - nop - nop - ld t0, (a1) - nop - nop - sd t0, (a0) - nop - nop - mtc0 t1,MIPS_COP_0_STATUS - nop - nop - nop - nop - j ra - nop -END(atomic_store_64) - -LEAF(atomic_load_64) - mfc0 t1, MIPS_COP_0_STATUS - and t2, t1, ~MIPS_SR_INT_IE - mtc0 t2, MIPS_COP_0_STATUS - nop - nop - nop - nop - ld t0, (a0) - nop - nop - sd t0, (a1) - nop - nop - mtc0 t1,MIPS_COP_0_STATUS - nop - nop - nop - nop - j ra - nop -END(atomic_load_64) -#endif #if defined(DDB) || defined(DEBUG) LEAF(kdbpeek) PTR_LA v1, ddberr and v0, a0, 3 # unaligned ? GET_CPU_PCPU(t1) PTR_L t1, PC_CURPCB(t1) bne v0, zero, 1f PTR_S v1, U_PCB_ONFAULT(t1) lw v0, (a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) 1: LWHI v0, 0(a0) LWLO v0, 3(a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) END(kdbpeek) LEAF(kdbpeekd) PTR_LA v1, ddberr and v0, a0, 3 # unaligned ? GET_CPU_PCPU(t1) PTR_L t1, PC_CURPCB(t1) bne v0, zero, 1f PTR_S v1, U_PCB_ONFAULT(t1) ld v0, (a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) 1: REG_LHI v0, 0(a0) REG_LLO v0, 7(a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) END(kdbpeekd) ddberr: jr ra nop #if defined(DDB) LEAF(kdbpoke) PTR_LA v1, ddberr and v0, a0, 3 # unaligned ? GET_CPU_PCPU(t1) PTR_L t1, PC_CURPCB(t1) bne v0, zero, 1f PTR_S v1, U_PCB_ONFAULT(t1) sw a1, (a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) 1: SWHI a1, 0(a0) SWLO a1, 3(a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) END(kdbpoke) .data .globl esym esym: .word 0 #endif /* DDB */ #endif /* DDB || DEBUG */ .text LEAF(breakpoint) break MIPS_BREAK_SOVER_VAL jr ra nop END(breakpoint) LEAF(setjmp) mfc0 v0, MIPS_COP_0_STATUS # Later the "real" spl value! REG_S s0, (SZREG * PCB_REG_S0)(a0) REG_S s1, (SZREG * PCB_REG_S1)(a0) REG_S s2, (SZREG * PCB_REG_S2)(a0) REG_S s3, (SZREG * PCB_REG_S3)(a0) REG_S s4, (SZREG * PCB_REG_S4)(a0) REG_S s5, (SZREG * PCB_REG_S5)(a0) REG_S s6, (SZREG * PCB_REG_S6)(a0) REG_S s7, (SZREG * PCB_REG_S7)(a0) REG_S s8, (SZREG * PCB_REG_S8)(a0) REG_S sp, (SZREG * PCB_REG_SP)(a0) REG_S ra, (SZREG * PCB_REG_RA)(a0) REG_S v0, (SZREG * PCB_REG_SR)(a0) jr ra li v0, 0 # setjmp return END(setjmp) LEAF(longjmp) REG_L v0, (SZREG * PCB_REG_SR)(a0) REG_L ra, (SZREG * PCB_REG_RA)(a0) REG_L s0, (SZREG * PCB_REG_S0)(a0) REG_L s1, (SZREG * PCB_REG_S1)(a0) REG_L s2, (SZREG * PCB_REG_S2)(a0) REG_L s3, (SZREG * PCB_REG_S3)(a0) REG_L s4, (SZREG * PCB_REG_S4)(a0) REG_L s5, (SZREG * PCB_REG_S5)(a0) REG_L s6, (SZREG * PCB_REG_S6)(a0) REG_L s7, (SZREG * PCB_REG_S7)(a0) REG_L s8, (SZREG * PCB_REG_S8)(a0) REG_L sp, (SZREG * PCB_REG_SP)(a0) mtc0 v0, MIPS_COP_0_STATUS # Later the "real" spl value! ITLBNOPFIX jr ra li v0, 1 # longjmp return END(longjmp) LEAF(mips3_ld) .set push .set noreorder .set mips64 #if defined(__mips_o32) mfc0 t0, MIPS_COP_0_STATUS # turn off interrupts and t1, t0, ~(MIPS_SR_INT_IE) mtc0 t1, MIPS_COP_0_STATUS COP0_SYNC nop nop nop ld v0, 0(a0) #if _BYTE_ORDER == _BIG_ENDIAN dsll v1, v0, 32 dsra v1, v1, 32 # low word in v1 dsra v0, v0, 32 # high word in v0 #else dsra v1, v0, 32 # high word in v1 dsll v0, v0, 32 dsra v0, v0, 32 # low word in v0 #endif mtc0 t0, MIPS_COP_0_STATUS # restore intr status. COP0_SYNC nop #else /* !__mips_o32 */ ld v0, 0(a0) #endif /* !__mips_o32 */ jr ra nop .set pop END(mips3_ld) LEAF(mips3_sd) .set push .set mips64 .set noreorder #if defined(__mips_o32) mfc0 t0, MIPS_COP_0_STATUS # turn off interrupts and t1, t0, ~(MIPS_SR_INT_IE) mtc0 t1, MIPS_COP_0_STATUS COP0_SYNC nop nop nop # NOTE: a1 is padding! #if _BYTE_ORDER == _BIG_ENDIAN dsll a2, a2, 32 # high word in a2 dsll a3, a3, 32 # low word in a3 dsrl a3, a3, 32 #else dsll a2, a2, 32 # low word in a2 dsrl a2, a2, 32 dsll a3, a3, 32 # high word in a3 #endif or a1, a2, a3 sd a1, 0(a0) mtc0 t0, MIPS_COP_0_STATUS # restore intr status. COP0_SYNC nop #else /* !__mips_o32 */ sd a1, 0(a0) #endif /* !__mips_o32 */ jr ra nop .set pop END(mips3_sd) Index: stable/11/sys/powerpc/include/atomic.h =================================================================== --- stable/11/sys/powerpc/include/atomic.h (revision 327194) +++ stable/11/sys/powerpc/include/atomic.h (revision 327195) @@ -1,896 +1,898 @@ /*- * Copyright (c) 2008 Marcel Moolenaar * Copyright (c) 2001 Benno Rice * Copyright (c) 2001 David E. O'Brien * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif +#include + /* * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction * with the atomic lXarx/stXcx. sequences below. They are not exposed outside * of this file. See also Appendix B.2 of Book II of the architecture manual. * * Note that not all Book-E processors accept the light-weight sync variant. * In particular, early models of E500 cores are known to wedge. Bank on all * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs * to use the heavier-weight sync. */ #ifdef __powerpc64__ #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("lwsync" : : : "memory") #define wmb() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #else #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("sync" : : : "memory") #define wmb() __asm __volatile("sync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #endif static __inline void powerpc_lwsync(void) { #ifdef __powerpc64__ __asm __volatile("lwsync" : : : "memory"); #else __asm __volatile("sync" : : : "memory"); #endif } /* * atomic_add(p, v) * { *p += v; } */ #define __atomic_add_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_int */ #ifdef __powerpc64__ #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #else #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #endif #define _ATOMIC_ADD(type) \ static __inline void \ atomic_add_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ } \ \ static __inline void \ atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_add_##type(p, v, t); \ } \ /* _ATOMIC_ADD */ _ATOMIC_ADD(int) _ATOMIC_ADD(long) #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #ifdef __powerpc64__ #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #else #define atomic_add_ptr atomic_add_int #define atomic_add_acq_ptr atomic_add_acq_int #define atomic_add_rel_ptr atomic_add_rel_int #endif #undef _ATOMIC_ADD #undef __atomic_add_long #undef __atomic_add_int /* * atomic_clear(p, v) * { *p &= ~v; } */ #define __atomic_clear_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_int */ #ifdef __powerpc64__ #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #else #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #endif #define _ATOMIC_CLEAR(type) \ static __inline void \ atomic_clear_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ } \ \ static __inline void \ atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_clear_##type(p, v, t); \ } \ /* _ATOMIC_CLEAR */ _ATOMIC_CLEAR(int) _ATOMIC_CLEAR(long) #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #ifdef __powerpc64__ #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #else #define atomic_clear_ptr atomic_clear_int #define atomic_clear_acq_ptr atomic_clear_acq_int #define atomic_clear_rel_ptr atomic_clear_rel_int #endif #undef _ATOMIC_CLEAR #undef __atomic_clear_long #undef __atomic_clear_int /* * atomic_cmpset(p, o, n) */ /* TODO -- see below */ /* * atomic_load_acq(p) */ /* TODO -- see below */ /* * atomic_readandclear(p) */ /* TODO -- see below */ /* * atomic_set(p, v) * { *p |= v; } */ #define __atomic_set_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_int */ #ifdef __powerpc64__ #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #else #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #endif #define _ATOMIC_SET(type) \ static __inline void \ atomic_set_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ } \ \ static __inline void \ atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_set_##type(p, v, t); \ } \ /* _ATOMIC_SET */ _ATOMIC_SET(int) _ATOMIC_SET(long) #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #ifdef __powerpc64__ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #else #define atomic_set_ptr atomic_set_int #define atomic_set_acq_ptr atomic_set_acq_int #define atomic_set_rel_ptr atomic_set_rel_int #endif #undef _ATOMIC_SET #undef __atomic_set_long #undef __atomic_set_int /* * atomic_subtract(p, v) * { *p -= v; } */ #define __atomic_subtract_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_int */ #ifdef __powerpc64__ #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #else #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #endif #define _ATOMIC_SUBTRACT(type) \ static __inline void \ atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ } \ \ static __inline void \ atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_subtract_##type(p, v, t); \ } \ /* _ATOMIC_SUBTRACT */ _ATOMIC_SUBTRACT(int) _ATOMIC_SUBTRACT(long) #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #ifdef __powerpc64__ #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #else #define atomic_subtract_ptr atomic_subtract_int #define atomic_subtract_acq_ptr atomic_subtract_acq_int #define atomic_subtract_rel_ptr atomic_subtract_rel_int #endif #undef _ATOMIC_SUBTRACT #undef __atomic_subtract_long #undef __atomic_subtract_int /* * atomic_store_rel(p, v) */ /* TODO -- see below */ /* * Old/original implementations that still need revisiting. */ static __inline u_int atomic_readandclear_int(volatile u_int *addr) { u_int result,temp; #ifdef __GNUCLIKE_ASM __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tlwarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stwcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); #endif return (result); } #ifdef __powerpc64__ static __inline u_long atomic_readandclear_long(volatile u_long *addr) { u_long result,temp; #ifdef __GNUCLIKE_ASM __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tldarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stdcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); #endif return (result); } #endif #define atomic_readandclear_32 atomic_readandclear_int #ifdef __powerpc64__ #define atomic_readandclear_64 atomic_readandclear_long #define atomic_readandclear_ptr atomic_readandclear_long #else static __inline u_long atomic_readandclear_long(volatile u_long *addr) { return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); } #define atomic_readandclear_ptr atomic_readandclear_int #endif /* * We assume that a = b will do atomic loads and stores. */ #define ATOMIC_STORE_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE v; \ \ v = *p; \ mb(); \ return (v); \ } \ \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ powerpc_lwsync(); \ *p = v; \ } ATOMIC_STORE_LOAD(int) #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #ifdef __powerpc64__ ATOMIC_STORE_LOAD(long) #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #else static __inline u_long atomic_load_acq_long(volatile u_long *addr) { return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); } static __inline void atomic_store_rel_long(volatile u_long *addr, u_long val) { atomic_store_rel_int((volatile u_int *)addr, (u_int)val); } #define atomic_load_acq_ptr atomic_load_acq_int #define atomic_store_rel_ptr atomic_store_rel_int #endif #undef ATOMIC_STORE_LOAD /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline int atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { int ret; #ifdef __GNUCLIKE_ASM __asm __volatile ( "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); #endif return (ret); } static __inline int atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) { int ret; #ifdef __GNUCLIKE_ASM __asm __volatile ( #ifdef __powerpc64__ "1:\tldarx %0, 0, %2\n\t" /* load old value */ "cmpld %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stdcx. %4, 0, %2\n\t" /* attempt to store */ #else "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ #endif "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #else "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #endif "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); #endif return (ret); } static __inline int atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) { int retval; retval = atomic_cmpset_int(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) { __ATOMIC_REL(); return (atomic_cmpset_int(p, cmpval, newval)); } static __inline int atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) { u_long retval; retval = atomic_cmpset_long(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) { __ATOMIC_REL(); return (atomic_cmpset_long(p, cmpval, newval)); } #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #ifdef __powerpc64__ #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #else #define atomic_cmpset_ptr atomic_cmpset_int #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int #endif /* * Atomically compare the value stored at *p with *cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed and sets *cmpval to the read value from *p, * nonzero otherwise. */ static __inline int atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) { int ret; #ifdef __GNUCLIKE_ASM __asm __volatile ( "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); #endif return (ret); } static __inline int atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) { int ret; #ifdef __GNUCLIKE_ASM __asm __volatile ( #ifdef __powerpc64__ "ldarx %0, 0, %3\n\t" /* load old value */ "cmpld %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stdcx. %5, 0, %3\n\t" /* attempt to store */ #else "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ #endif "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stdx %0, 0, %7\n\t" #else "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" #endif "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); #endif return (ret); } static __inline int atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval) { int retval; retval = atomic_fcmpset_int(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval) { __ATOMIC_REL(); return (atomic_fcmpset_int(p, cmpval, newval)); } static __inline int atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval) { u_long retval; retval = atomic_fcmpset_long(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval) { __ATOMIC_REL(); return (atomic_fcmpset_long(p, cmpval, newval)); } #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #ifdef __powerpc64__ #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #else #define atomic_fcmpset_ptr atomic_fcmpset_int #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int #endif static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { u_int value; do { value = *p; } while (!atomic_cmpset_int(p, value, value + v)); return (value); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { u_long value; do { value = *p; } while (!atomic_cmpset_long(p, value, value + v)); return (value); } static __inline u_int atomic_swap_32(volatile u_int *p, u_int v) { u_int prev; __asm __volatile( "1: lwarx %0,0,%2\n" " stwcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_int *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #ifdef __powerpc64__ static __inline u_long atomic_swap_64(volatile u_long *p, u_long v) { u_long prev; __asm __volatile( "1: ldarx %0,0,%2\n" " stdcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_long *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #endif #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_swap_int atomic_swap_32 #ifdef __powerpc64__ #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_swap_long atomic_swap_64 #define atomic_swap_ptr atomic_swap_64 #endif #undef __ATOMIC_REL #undef __ATOMIC_ACQ static __inline void atomic_thread_fence_acq(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_acq_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_seq_cst(void) { __asm __volatile("sync" : : : "memory"); } #endif /* ! _MACHINE_ATOMIC_H_ */ Index: stable/11/sys/riscv/include/atomic.h =================================================================== --- stable/11/sys/riscv/include/atomic.h (revision 327194) +++ stable/11/sys/riscv/include/atomic.h (revision 327195) @@ -1,562 +1,564 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ +#include + #define fence() __asm __volatile("fence" ::: "memory"); #define mb() fence() #define rmb() fence() #define wmb() fence() #define ATOMIC_ACQ_REL(NAME, WIDTH) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ atomic_##NAME##_##WIDTH(p, v); \ fence(); \ } \ \ static __inline void \ atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ fence(); \ atomic_##NAME##_##WIDTH(p, v); \ } static __inline void atomic_add_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoadd.w zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_subtract_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoadd.w zero, %1, %0" : "+A" (*p) : "r" (-val) : "memory"); } static __inline void atomic_set_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoor.w zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_clear_32(volatile uint32_t *p, uint32_t val) { __asm __volatile("amoand.w zero, %1, %0" : "+A" (*p) : "r" (~val) : "memory"); } static __inline int atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.w %0, %2\n" "bne %0, %z3, 1f\n" "sc.w %1, %z4, %2\n" "bnez %1, 0b\n" "1:" : "=&r" (tmp), "=&r" (res), "+A" (*p) : "rJ" (cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline int atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { uint32_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.w %0, %2\n" /* Load old value */ "bne %0, %z4, 1f\n" /* Compare */ "sc.w %1, %z5, %2\n" /* Try to store new value */ "j 2f\n" "1:" "sw %0, %3\n" /* Save old value */ "2:" : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) : "rJ" (*cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline uint32_t atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) { uint32_t ret; __asm __volatile("amoadd.w %0, %2, %1" : "=&r" (ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint32_t atomic_readandclear_32(volatile uint32_t *p) { uint32_t ret; uint32_t val; val = 0; __asm __volatile("amoswap.w %0, %2, %1" : "=&r"(ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } #define atomic_add_int atomic_add_32 #define atomic_clear_int atomic_clear_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_fetchadd_int atomic_fetchadd_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_int atomic_set_32 #define atomic_subtract_int atomic_subtract_32 ATOMIC_ACQ_REL(set, 32) ATOMIC_ACQ_REL(clear, 32) ATOMIC_ACQ_REL(add, 32) ATOMIC_ACQ_REL(subtract, 32) static __inline int atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { int res; res = atomic_cmpset_32(p, cmpval, newval); fence(); return (res); } static __inline int atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { fence(); return (atomic_cmpset_32(p, cmpval, newval)); } static __inline int atomic_fcmpset_acq_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { int res; res = atomic_fcmpset_32(p, cmpval, newval); fence(); return (res); } static __inline int atomic_fcmpset_rel_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { fence(); return (atomic_fcmpset_32(p, cmpval, newval)); } static __inline uint32_t atomic_load_acq_32(volatile uint32_t *p) { uint32_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_32(volatile uint32_t *p, uint32_t val) { fence(); *p = val; } #define atomic_add_acq_int atomic_add_acq_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_clear_rel_int atomic_add_rel_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_store_rel_int atomic_store_rel_32 static __inline void atomic_add_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoadd.d zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_subtract_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoadd.d zero, %1, %0" : "+A" (*p) : "r" (-val) : "memory"); } static __inline void atomic_set_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoor.d zero, %1, %0" : "+A" (*p) : "r" (val) : "memory"); } static __inline void atomic_clear_64(volatile uint64_t *p, uint64_t val) { __asm __volatile("amoand.d zero, %1, %0" : "+A" (*p) : "r" (~val) : "memory"); } static __inline int atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.d %0, %2\n" "bne %0, %z3, 1f\n" "sc.d %1, %z4, %2\n" "bnez %1, 0b\n" "1:" : "=&r" (tmp), "=&r" (res), "+A" (*p) : "rJ" (cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline int atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { uint64_t tmp; int res; res = 0; __asm __volatile( "0:" "li %1, 1\n" /* Preset to fail */ "lr.d %0, %2\n" /* Load old value */ "bne %0, %z4, 1f\n" /* Compare */ "sc.d %1, %z5, %2\n" /* Try to store new value */ "j 2f\n" "1:" "sd %0, %3\n" /* Save old value */ "2:" : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) : "rJ" (*cmpval), "rJ" (newval) : "memory"); return (!res); } static __inline uint64_t atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) { uint64_t ret; __asm __volatile("amoadd.d %0, %2, %1" : "=&r" (ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint64_t atomic_readandclear_64(volatile uint64_t *p) { uint64_t ret; uint64_t val; val = 0; __asm __volatile("amoswap.d %0, %2, %1" : "=&r"(ret), "+A" (*p) : "r" (val) : "memory"); return (ret); } static __inline uint32_t atomic_swap_32(volatile uint32_t *p, uint32_t val) { uint32_t old; __asm __volatile("amoswap.w %0, %2, %1" : "=&r"(old), "+A" (*p) : "r" (val) : "memory"); return (old); } static __inline uint64_t atomic_swap_64(volatile uint64_t *p, uint64_t val) { uint64_t old; __asm __volatile("amoswap.d %0, %2, %1" : "=&r"(old), "+A" (*p) : "r" (val) : "memory"); return (old); } #define atomic_add_long atomic_add_64 #define atomic_clear_long atomic_clear_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #define atomic_set_long atomic_set_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_add_ptr atomic_add_64 #define atomic_clear_ptr atomic_clear_64 #define atomic_cmpset_ptr atomic_cmpset_64 #define atomic_fcmpset_ptr atomic_fcmpset_64 #define atomic_fetchadd_ptr atomic_fetchadd_64 #define atomic_readandclear_ptr atomic_readandclear_64 #define atomic_set_ptr atomic_set_64 #define atomic_subtract_ptr atomic_subtract_64 ATOMIC_ACQ_REL(set, 64) ATOMIC_ACQ_REL(clear, 64) ATOMIC_ACQ_REL(add, 64) ATOMIC_ACQ_REL(subtract, 64) static __inline int atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { int res; res = atomic_cmpset_64(p, cmpval, newval); fence(); return (res); } static __inline int atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { fence(); return (atomic_cmpset_64(p, cmpval, newval)); } static __inline int atomic_fcmpset_acq_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { int res; res = atomic_fcmpset_64(p, cmpval, newval); fence(); return (res); } static __inline int atomic_fcmpset_rel_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { fence(); return (atomic_fcmpset_64(p, cmpval, newval)); } static __inline uint64_t atomic_load_acq_64(volatile uint64_t *p) { uint64_t ret; ret = *p; fence(); return (ret); } static __inline void atomic_store_rel_64(volatile uint64_t *p, uint64_t val) { fence(); *p = val; } #define atomic_add_acq_long atomic_add_acq_64 #define atomic_clear_acq_long atomic_add_acq_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_add_acq_ptr atomic_add_acq_64 #define atomic_clear_acq_ptr atomic_add_acq_64 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 #define atomic_load_acq_ptr atomic_load_acq_64 #define atomic_set_acq_ptr atomic_set_acq_64 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 static __inline void atomic_thread_fence_acq(void) { fence(); } static __inline void atomic_thread_fence_rel(void) { fence(); } static __inline void atomic_thread_fence_acq_rel(void) { fence(); } static __inline void atomic_thread_fence_seq_cst(void) { fence(); } #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_add_rel_ptr atomic_add_rel_64 #define atomic_clear_rel_ptr atomic_clear_rel_64 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 #define atomic_set_rel_ptr atomic_set_rel_64 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 #define atomic_store_rel_ptr atomic_store_rel_64 #endif /* _MACHINE_ATOMIC_H_ */ Index: stable/11/sys/sparc64/include/atomic.h =================================================================== --- stable/11/sys/sparc64/include/atomic.h (revision 327194) +++ stable/11/sys/sparc64/include/atomic.h (revision 327195) @@ -1,371 +1,368 @@ /*- * Copyright (c) 1998 Doug Rabson. * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11 * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #include #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") #define wmb() mb() #define rmb() mb() +#include + /* Userland needs different ASI's. */ #ifdef _KERNEL #define __ASI_ATOMIC ASI_N #else #define __ASI_ATOMIC ASI_P #endif /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and multiple processors. See atomic(9) for details. * Note that efficient hardware support exists only for the 32 and 64 * bit variants; the 8 and 16 bit versions are not provided and should * not be used in MI code. * * This implementation takes advantage of the fact that the sparc64 * cas instruction is both a load and a store. The loop is often coded * as follows: * * do { * expect = *p; * new = expect + 1; * } while (cas(p, expect, new) != expect); * * which performs an unnnecessary load on each iteration that the cas * operation fails. Modified as follows: * * expect = *p; * for (;;) { * new = expect + 1; * result = cas(p, expect, new); * if (result == expect) * break; * expect = result; * } * * the return value of cas is used to avoid the extra reload. * * We only include a memory barrier in the rel variants as in total store * order which we use for running the kernel and all of the userland atomic * loads and stores behave as if the were followed by a membar with a mask * of #LoadLoad | #LoadStore | #StoreStore. In order to be also sufficient * for use of relaxed memory ordering, the atomic_cas() in the acq variants * additionally would have to be followed by a membar #LoadLoad | #LoadStore. * Due to the suggested assembly syntax of the membar operands containing a * # character, they cannot be used in macros. The cmask and mmask bits thus * are hard coded in machine/cpufunc.h and used here through macros. * Hopefully the bit numbers won't change in the future. */ #define itype(sz) uint ## sz ## _t #define atomic_cas_32(p, e, s) casa((p), (e), (s), __ASI_ATOMIC) #define atomic_cas_64(p, e, s) casxa((p), (e), (s), __ASI_ATOMIC) #define atomic_cas(p, e, s, sz) \ atomic_cas_ ## sz((p), (e), (s)) #define atomic_cas_acq(p, e, s, sz) ({ \ itype(sz) v; \ v = atomic_cas((p), (e), (s), sz); \ __compiler_membar(); \ v; \ }) #define atomic_cas_rel(p, e, s, sz) ({ \ itype(sz) v; \ membar(LoadStore | StoreStore); \ v = atomic_cas((p), (e), (s), sz); \ v; \ }) #define atomic_op(p, op, v, sz) ({ \ itype(sz) e, r, s; \ for (e = *(volatile itype(sz) *)(p);; e = r) { \ s = e op (v); \ r = atomic_cas_ ## sz((p), e, s); \ if (r == e) \ break; \ } \ e; \ }) #define atomic_op_acq(p, op, v, sz) ({ \ itype(sz) t; \ t = atomic_op((p), op, (v), sz); \ __compiler_membar(); \ t; \ }) #define atomic_op_rel(p, op, v, sz) ({ \ itype(sz) t; \ membar(LoadStore | StoreStore); \ t = atomic_op((p), op, (v), sz); \ t; \ }) #define atomic_ld_acq(p, sz) ({ \ itype(sz) v; \ v = atomic_cas((p), 0, 0, sz); \ __compiler_membar(); \ v; \ }) #define atomic_ld_clear(p, sz) ({ \ itype(sz) e, r; \ for (e = *(volatile itype(sz) *)(p);; e = r) { \ r = atomic_cas((p), e, 0, sz); \ if (r == e) \ break; \ } \ e; \ }) #define atomic_st(p, v, sz) do { \ itype(sz) e, r; \ for (e = *(volatile itype(sz) *)(p);; e = r) { \ r = atomic_cas((p), e, (v), sz); \ if (r == e) \ break; \ } \ } while (0) #define atomic_st_acq(p, v, sz) do { \ atomic_st((p), (v), sz); \ __compiler_membar(); \ } while (0) #define atomic_st_rel(p, v, sz) do { \ membar(LoadStore | StoreStore); \ atomic_st((p), (v), sz); \ } while (0) #define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ \ static __inline vtype \ atomic_add_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op((p), +, (v), sz)); \ } \ static __inline vtype \ atomic_add_acq_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_acq((p), +, (v), sz)); \ } \ static __inline vtype \ atomic_add_rel_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_rel((p), +, (v), sz)); \ } \ \ static __inline vtype \ atomic_clear_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op((p), &, ~(v), sz)); \ } \ static __inline vtype \ atomic_clear_acq_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_acq((p), &, ~(v), sz)); \ } \ static __inline vtype \ atomic_clear_rel_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_rel((p), &, ~(v), sz)); \ } \ \ static __inline int \ atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ { \ return (((vtype)atomic_cas((p), (e), (s), sz)) == (e)); \ } \ static __inline int \ atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ { \ return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e)); \ } \ static __inline int \ atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ { \ return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \ } \ \ static __inline int \ atomic_fcmpset_ ## name(volatile ptype p, vtype *ep, vtype s) \ { \ vtype t; \ \ t = (vtype)atomic_cas((p), (*ep), (s), sz); \ if (t == (*ep)) \ return (1); \ *ep = t; \ return (0); \ } \ static __inline int \ atomic_fcmpset_acq_ ## name(volatile ptype p, vtype *ep, vtype s) \ { \ vtype t; \ \ t = (vtype)atomic_cas_acq((p), (*ep), (s), sz); \ if (t == (*ep)) \ return (1); \ *ep = t; \ return (0); \ } \ static __inline int \ atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *ep, vtype s) \ { \ vtype t; \ \ t = (vtype)atomic_cas_rel((p), (*ep), (s), sz); \ if (t == (*ep)) \ return (1); \ *ep = t; \ return (0); \ } \ \ -static __inline vtype \ -atomic_load_ ## name(volatile ptype p) \ -{ \ - return ((vtype)atomic_cas((p), 0, 0, sz)); \ -} \ static __inline vtype \ atomic_load_acq_ ## name(volatile ptype p) \ { \ return ((vtype)atomic_cas_acq((p), 0, 0, sz)); \ } \ \ static __inline vtype \ atomic_readandclear_ ## name(volatile ptype p) \ { \ return ((vtype)atomic_ld_clear((p), sz)); \ } \ \ static __inline vtype \ atomic_set_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op((p), |, (v), sz)); \ } \ static __inline vtype \ atomic_set_acq_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_acq((p), |, (v), sz)); \ } \ static __inline vtype \ atomic_set_rel_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_rel((p), |, (v), sz)); \ } \ \ static __inline vtype \ atomic_subtract_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op((p), -, (v), sz)); \ } \ static __inline vtype \ atomic_subtract_acq_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_acq((p), -, (v), sz)); \ } \ static __inline vtype \ atomic_subtract_rel_ ## name(volatile ptype p, atype v) \ { \ return ((vtype)atomic_op_rel((p), -, (v), sz)); \ } \ \ static __inline void \ atomic_store_acq_ ## name(volatile ptype p, vtype v) \ { \ atomic_st_acq((p), (v), sz); \ } \ static __inline void \ atomic_store_rel_ ## name(volatile ptype p, vtype v) \ { \ atomic_st_rel((p), (v), sz); \ } static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { membar(LoadLoad | LoadStore | StoreStore | StoreLoad); } ATOMIC_GEN(int, u_int *, u_int, u_int, 32); ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32); ATOMIC_GEN(long, u_long *, u_long, u_long, 64); ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64); ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64); #define atomic_fetchadd_int atomic_add_int #define atomic_fetchadd_32 atomic_add_32 #define atomic_fetchadd_long atomic_add_long #define atomic_fetchadd_64 atomic_add_64 #undef ATOMIC_GEN #undef atomic_cas #undef atomic_cas_acq #undef atomic_cas_rel #undef atomic_op #undef atomic_op_acq #undef atomic_op_rel #undef atomic_ld_acq #undef atomic_ld_clear #undef atomic_st #undef atomic_st_acq #undef atomic_st_rel #endif /* !_MACHINE_ATOMIC_H_ */ Index: stable/11/sys/sys/atomic_common.h =================================================================== --- stable/11/sys/sys/atomic_common.h (nonexistent) +++ stable/11/sys/sys/atomic_common.h (revision 327195) @@ -0,0 +1,73 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2017 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _SYS_ATOMIC_COMMON_H_ +#define _SYS_ATOMIC_COMMON_H_ + +#ifndef _MACHINE_ATOMIC_H_ +#error do not include this header, use machine/atomic.h +#endif + +#define atomic_load_char(p) (*(volatile u_char *)(p)) +#define atomic_load_short(p) (*(volatile u_short *)(p)) +#define atomic_load_int(p) (*(volatile u_int *)(p)) +#define atomic_load_long(p) (*(volatile u_long *)(p)) +#define atomic_load_ptr(p) (*(volatile uintptr_t*)(p)) +#define atomic_load_8(p) (*(volatile uint8_t *)(p)) +#define atomic_load_16(p) (*(volatile uint16_t *)(p)) +#define atomic_load_32(p) (*(volatile uint32_t *)(p)) +#ifdef _LP64 +#define atomic_load_64(p) (*(volatile uint64_t *)(p)) +#endif + +#define atomic_store_char(p, v) \ + (*(volatile u_char *)(p) = (u_char)(v)) +#define atomic_store_short(p, v) \ + (*(volatile u_short *)(p) = (u_short)(v)) +#define atomic_store_int(p, v) \ + (*(volatile u_int *)(p) = (u_int)(v)) +#define atomic_store_long(p, v) \ + (*(volatile u_long *)(p) = (u_long)(v)) +#define atomic_store_ptr(p, v) \ + (*(uintptr_t *)(p) = (uintptr_t)(v)) +#define atomic_store_8(p, v) \ + (*(volatile uint8_t *)(p) = (uint8_t)(v)) +#define atomic_store_16(p, v) \ + (*(volatile uint16_t *)(p) = (uint16_t)(v)) +#define atomic_store_32(p, v) \ + (*(volatile uint32_t *)(p) = (uint32_t)(v)) +#ifdef _LP64 +#define atomic_store_64(p, v) \ + (*(volatile uint64_t *)(p) = (uint64_t)(v)) +#endif + +#endif Property changes on: stable/11/sys/sys/atomic_common.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: stable/11 =================================================================== --- stable/11 (revision 327194) +++ stable/11 (revision 327195) Property changes on: stable/11 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r326971,327047,327053,327074,327097