Index: sys/amd64/include/atomic.h =================================================================== --- sys/amd64/include/atomic.h +++ sys/amd64/include/atomic.h @@ -73,6 +73,85 @@ #else #include +#if defined(__GNUCLIKE_ASM) && defined(SMP) && !defined(WANT_FUNCTIONS) +#include +#define __USING_GCC_ATOMICS + +__ATOMIC_CMPSET_GCC(char); +__ATOMIC_FCMPSET_GCC(char, false); +__ATOMIC_SET_GCC(char); +__ATOMIC_CLEAR_GCC(char); +__ATOMIC_ADD_GCC(char); +__ATOMIC_SUBTRACT_GCC(char); + +__ATOMIC_CMPSET_GCC(short); +__ATOMIC_FCMPSET_GCC(short, false); +__ATOMIC_SET_GCC(short); +__ATOMIC_CLEAR_GCC(short); +__ATOMIC_ADD_GCC(short); +__ATOMIC_SUBTRACT_GCC(short); + +__ATOMIC_CMPSET_GCC(int); +__ATOMIC_FCMPSET_GCC(int, false); +__ATOMIC_FETCHADD_GCC(int); +__ATOMIC_SET_GCC(int); +__ATOMIC_CLEAR_GCC(int); +__ATOMIC_ADD_GCC(int); +__ATOMIC_SUBTRACT_GCC(int); +__ATOMIC_SWAP_GCC(int); + +__ATOMIC_CMPSET_GCC(long); +__ATOMIC_FCMPSET_GCC(long, false); +__ATOMIC_FETCHADD_GCC(long); +__ATOMIC_SET_GCC(long); +__ATOMIC_CLEAR_GCC(long); +__ATOMIC_ADD_GCC(long); +__ATOMIC_SUBTRACT_GCC(long); +__ATOMIC_SWAP_GCC(long); + +__ATOMIC_CMPSET_ALIAS_GCC(8, char); +__ATOMIC_FCMPSET_ALIAS_GCC(8, char); +__ATOMIC_SET_ALIAS_GCC(8, char); +__ATOMIC_CLEAR_ALIAS_GCC(8, char); +__ATOMIC_ADD_ALIAS_GCC(8, char); +__ATOMIC_SUBTRACT_ALIAS_GCC(8, char); + +__ATOMIC_CMPSET_ALIAS_GCC(16, short); +__ATOMIC_FCMPSET_ALIAS_GCC(16, short); +__ATOMIC_SET_ALIAS_GCC(16, short); +__ATOMIC_CLEAR_ALIAS_GCC(16, short); +__ATOMIC_ADD_ALIAS_GCC(16, short); +__ATOMIC_SUBTRACT_ALIAS_GCC(16, short); + +__ATOMIC_CMPSET_ALIAS_GCC(32, int); +__ATOMIC_FCMPSET_ALIAS_GCC(32, int); +__ATOMIC_FETCHADD_ALIAS_GCC(32, int); +__ATOMIC_SET_ALIAS_GCC(32, int); +__ATOMIC_CLEAR_ALIAS_GCC(32, int); +__ATOMIC_ADD_ALIAS_GCC(32, int); +__ATOMIC_SUBTRACT_ALIAS_GCC(32, int); +__ATOMIC_SWAP_ALIAS_GCC(32, int); + +__ATOMIC_CMPSET_ALIAS_GCC(64, long); +__ATOMIC_FCMPSET_ALIAS_GCC(64, long); +__ATOMIC_FETCHADD_ALIAS_GCC(64, long); +__ATOMIC_SET_ALIAS_GCC(64, long); +__ATOMIC_CLEAR_ALIAS_GCC(64, long); +__ATOMIC_ADD_ALIAS_GCC(64, long); +__ATOMIC_SUBTRACT_ALIAS_GCC(64, long); +__ATOMIC_SWAP_ALIAS_GCC(64, long); + +__ATOMIC_CMPSET_ALIAS_GCC(ptr, long); +__ATOMIC_FCMPSET_ALIAS_GCC(ptr, long); +__ATOMIC_FETCHADD_ALIAS_GCC(ptr, long); +__ATOMIC_SET_ALIAS_GCC(ptr, long); +__ATOMIC_CLEAR_ALIAS_GCC(ptr, long); +__ATOMIC_ADD_ALIAS_GCC(ptr, long); +__ATOMIC_SUBTRACT_ALIAS_GCC(ptr, long); +__ATOMIC_SWAP_ALIAS_GCC(ptr, long); + +#endif + /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. @@ -229,11 +308,14 @@ return (res); \ } +#ifndef __USING_GCC_ATOMICS ATOMIC_CMPSET(char); ATOMIC_CMPSET(short); ATOMIC_CMPSET(int); ATOMIC_CMPSET(long); +#endif +#ifndef __USING_GCC_ATOMICS /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. @@ -269,6 +351,7 @@ : : "cc"); return (v); } +#endif static __inline int atomic_testandset_int(volatile u_int *p, u_int v) @@ -436,6 +519,7 @@ #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ +#ifndef __USING_GCC_ATOMICS ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); @@ -455,6 +539,7 @@ ATOMIC_ASM(clear, long, "andq %1,%0", "er", ~v); ATOMIC_ASM(add, long, "addq %1,%0", "er", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "er", v); +#endif #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ @@ -474,6 +559,7 @@ /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM +#ifndef __USING_GCC_ATOMICS static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { @@ -497,6 +583,7 @@ "+m" (*p)); /* 1 */ return (v); } +#endif #else /* !__GNUCLIKE_ASM */ @@ -505,6 +592,7 @@ #endif /* __GNUCLIKE_ASM */ +#ifndef __USING_GCC_ATOMICS #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char @@ -556,11 +644,13 @@ #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long +#endif #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) /* Operations on 8-bit bytes. */ +#ifndef __USING_GCC_ATOMICS #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char @@ -573,16 +663,18 @@ #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char -#define atomic_load_acq_8 atomic_load_acq_char -#define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char +#endif +#define atomic_load_acq_8 atomic_load_acq_char +#define atomic_store_rel_8 atomic_store_rel_char /* Operations on 16-bit words. */ +#ifndef __USING_GCC_ATOMICS #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short @@ -595,16 +687,18 @@ #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short -#define atomic_load_acq_16 atomic_load_acq_short -#define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short +#endif +#define atomic_load_acq_16 atomic_load_acq_short +#define atomic_store_rel_16 atomic_store_rel_short /* Operations on 32-bit double words. */ +#ifndef __USING_GCC_ATOMICS #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int @@ -617,8 +711,6 @@ #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int -#define atomic_load_acq_32 atomic_load_acq_int -#define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int @@ -626,12 +718,16 @@ #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int -#define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#endif +#define atomic_load_acq_32 atomic_load_acq_int +#define atomic_store_rel_32 atomic_store_rel_int +#define atomic_readandclear_32 atomic_readandclear_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on 64-bit quad words. */ +#ifndef __USING_GCC_ATOMICS #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long @@ -644,8 +740,6 @@ #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subtract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long -#define atomic_load_acq_64 atomic_load_acq_long -#define atomic_store_rel_64 atomic_store_rel_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long @@ -653,12 +747,16 @@ #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long -#define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long +#endif +#define atomic_load_acq_64 atomic_load_acq_long +#define atomic_store_rel_64 atomic_store_rel_long +#define atomic_readandclear_64 atomic_readandclear_long #define atomic_testandset_64 atomic_testandset_long #define atomic_testandclear_64 atomic_testandclear_long /* Operations on pointers. */ +#ifndef __USING_GCC_ATOMICS #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long @@ -671,8 +769,6 @@ #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long -#define atomic_load_acq_ptr atomic_load_acq_long -#define atomic_store_rel_ptr atomic_store_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long @@ -680,8 +776,13 @@ #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long +#endif +#define atomic_load_acq_ptr atomic_load_acq_long +#define atomic_store_rel_ptr atomic_store_rel_long #define atomic_readandclear_ptr atomic_readandclear_long +#undef __USING_GCC_ATOMICS + #endif /* !WANT_FUNCTIONS */ #endif /* KCSAN && !KCSAN_RUNTIME */ Index: sys/sys/_gcc_atomic.h =================================================================== --- /dev/null +++ sys/sys/_gcc_atomic.h @@ -0,0 +1,406 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 Mateusz Guzik + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _SYS__GCC_ATOMIC_H_ +#define _SYS__GCC_ATOMIC_H_ + +/* + * This file provides a translation layer between FreeBSD API for atomics and + * the one provided by GCC. Primary motivation is that custom routines written + * in assembly tend to be no better than what the compiler provides and are + * prone to being worse -- they can have overzealous clobbers, bad constraints + * or just suffer from limitations of the compiler when faced with inline + * assembly. + */ + +#ifndef _MACHINE_ATOMIC_H_ +#error do not include this header, use machine/atomic.h +#endif + +#ifndef SMP +#error generated code is pessimal for UP kernels +#endif + +#define __atomic_cmpset_gcc(p, e, s, w, f) ({ \ + __typeof(e) __var = (e); \ + __atomic_compare_exchange_n(p, &__var, s, w, __ATOMIC_RELAXED, f); \ +}) + +#define __ATOMIC_CMPSET_GCC(TYPE) \ +static __inline int \ +atomic_cmpset_##TYPE(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ +{ \ + \ + return (__atomic_cmpset_gcc(p, e, s, false, __ATOMIC_RELAXED)); \ +} \ + \ +static __inline int \ +atomic_cmpset_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ +{ \ + \ + return (__atomic_cmpset_gcc(p, e, s, false, __ATOMIC_ACQUIRE)); \ +} \ + \ +static __inline int \ +atomic_cmpset_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ +{ \ + \ + return (__atomic_cmpset_gcc(p, e, s, false, __ATOMIC_RELEASE)); \ +} + +#define __ATOMIC_CMPSET_ALIAS_GCC(ALIAS, TYPE) \ +static __inline int \ +atomic_cmpset_##ALIAS(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ +{ \ + \ + return (atomic_cmpset_##TYPE(p, e, s)); \ +} \ + \ +static __inline int \ +atomic_cmpset_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ +{ \ + \ + return (atomic_cmpset_acq_##TYPE(p, e, s)); \ +} \ + \ +static __inline int \ +atomic_cmpset_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ +{ \ + \ + return (atomic_cmpset_rel_##TYPE(p, e, s)); \ +} + +#define __atomic_fcmpset_gcc(p, e, s, w, f) ({ \ + __atomic_compare_exchange_n(p, e, s, w, __ATOMIC_RELAXED, f); \ +}) + +#define __ATOMIC_FCMPSET_GCC(TYPE, WEAK) \ +static __inline int \ +atomic_fcmpset_##TYPE(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ +{ \ + \ + return (__atomic_fcmpset_gcc(p, e, s, WEAK, __ATOMIC_RELAXED)); \ +} \ + \ +static __inline int \ +atomic_fcmpset_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ +{ \ + \ + return (__atomic_fcmpset_gcc(p, e, s, WEAK, __ATOMIC_ACQUIRE)); \ +} \ + \ +static __inline int \ +atomic_fcmpset_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ +{ \ + \ + return (__atomic_fcmpset_gcc(p, e, s, WEAK, __ATOMIC_RELEASE)); \ +} + +#define __ATOMIC_FCMPSET_ALIAS_GCC(ALIAS, TYPE) \ +static __inline int \ +atomic_fcmpset_##ALIAS(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ +{ \ + \ + return (atomic_fcmpset_##TYPE(p, e, s)); \ +} \ + \ +static __inline int \ +atomic_fcmpset_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ +{ \ + \ + return (atomic_fcmpset_acq_##TYPE(p, e, s)); \ +} \ + \ +static __inline int \ +atomic_fcmpset_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ +{ \ + \ + return (atomic_fcmpset_rel_##TYPE(p, e, s)); \ +} + +#define __atomic_fetchadd_gcc(p, v, f) ({ \ + __atomic_fetch_add(p, v, f); \ +}) + +#define __ATOMIC_FETCHADD_GCC(TYPE) \ +static __inline u_##TYPE \ +atomic_fetchadd_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (__atomic_fetchadd_gcc(p, v, __ATOMIC_RELAXED)); \ +} + +#define __ATOMIC_FETCHADD_ALIAS_GCC(ALIAS, TYPE) \ +static __inline u_##TYPE \ +atomic_fetchadd_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (atomic_fetchadd_##TYPE(p, v)); \ +} + +#define __atomic_set_gcc(p, v, f) ({ \ + __atomic_or_fetch(p, v, f); \ +}) + +#define __ATOMIC_SET_GCC(TYPE) \ +static __inline void \ +atomic_set_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_set_gcc(p, v, __ATOMIC_RELAXED); \ +} \ + \ +static __inline void \ +atomic_set_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_set_gcc(p, v, __ATOMIC_ACQUIRE); \ +} \ + \ +static __inline void \ +atomic_set_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_set_gcc(p, v, __ATOMIC_RELEASE); \ +} + +#define __ATOMIC_SET_ALIAS_GCC(ALIAS, TYPE) \ +static __inline void \ +atomic_set_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_set_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_set_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_set_acq_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_set_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_set_rel_##TYPE(p, v); \ +} + +#define __atomic_clear_gcc(p, v, f) ({ \ + __atomic_and_fetch(p, ~v, f); \ +}) + +#define __ATOMIC_CLEAR_GCC(TYPE) \ +static __inline void \ +atomic_clear_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_clear_gcc(p, v, __ATOMIC_RELAXED); \ +} \ + \ +static __inline void \ +atomic_clear_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_clear_gcc(p, v, __ATOMIC_ACQUIRE); \ +} \ + \ +static __inline void \ +atomic_clear_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_clear_gcc(p, v, __ATOMIC_RELEASE); \ +} + +#define __ATOMIC_CLEAR_ALIAS_GCC(ALIAS, TYPE) \ +static __inline void \ +atomic_clear_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_clear_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_clear_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_clear_acq_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_clear_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_clear_rel_##TYPE(p, v); \ +} + +#define __atomic_add_gcc(p, v, f) ({ \ + __atomic_fetch_add(p, v, f); \ +}) + +#define __ATOMIC_ADD_GCC(TYPE) \ +static __inline void \ +atomic_add_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_add_gcc(p, v, __ATOMIC_RELAXED); \ +} \ + \ +static __inline void \ +atomic_add_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_add_gcc(p, v, __ATOMIC_ACQUIRE); \ +} \ + \ +static __inline void \ +atomic_add_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_add_gcc(p, v, __ATOMIC_RELEASE); \ +} + +#define __ATOMIC_ADD_ALIAS_GCC(ALIAS, TYPE) \ +static __inline void \ +atomic_add_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_add_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_add_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_add_acq_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_add_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_add_rel_##TYPE(p, v); \ +} + +#define __atomic_subtract_gcc(p, v, f) ({ \ + __atomic_sub_fetch(p, v, f); \ +}) + +#define __ATOMIC_SUBTRACT_GCC(TYPE) \ +static __inline void \ +atomic_subtract_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_subtract_gcc(p, v, __ATOMIC_RELAXED); \ +} \ + \ +static __inline void \ +atomic_subtract_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_subtract_gcc(p, v, __ATOMIC_ACQUIRE); \ +} \ + \ +static __inline void \ +atomic_subtract_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + __atomic_subtract_gcc(p, v, __ATOMIC_RELEASE); \ +} + +#define __ATOMIC_SUBTRACT_ALIAS_GCC(ALIAS, TYPE) \ +static __inline void \ +atomic_subtract_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_subtract_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_subtract_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_subtract_acq_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_subtract_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + atomic_subtract_rel_##TYPE(p, v); \ +} + +#define __atomic_swap_gcc(p, v, f) ({ \ + __atomic_exchange_n(p, v, f); \ +}) + +#define __ATOMIC_SWAP_GCC(TYPE) \ +static __inline u_##TYPE \ +atomic_swap_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (__atomic_swap_gcc(p, v, __ATOMIC_RELAXED)); \ +} \ + \ +static __inline u_##TYPE \ +atomic_swap_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (__atomic_swap_gcc(p, v, __ATOMIC_ACQUIRE)); \ +} \ + \ +static __inline u_##TYPE \ +atomic_swap_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (__atomic_swap_gcc(p, v, __ATOMIC_RELEASE)); \ +} + +#define __ATOMIC_SWAP_ALIAS_GCC(ALIAS, TYPE) \ +static __inline u_##TYPE \ +atomic_swap_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (atomic_swap_##TYPE(p, v)); \ +} \ + \ +static __inline u_##TYPE \ +atomic_swap_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (atomic_swap_acq_##TYPE(p, v)); \ +} \ + \ +static __inline u_##TYPE \ +atomic_swap_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + return (atomic_swap_rel_##TYPE(p, v)); \ +} + +#endif /* _SYS__GCC_ATOMIC_H_ */