Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/include/atomic.h
Show First 20 Lines • Show All 147 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
#if defined(SMP) || !defined(_KERNEL) || defined(KLD_MODULE) | #if defined(SMP) || !defined(_KERNEL) || defined(KLD_MODULE) | ||||
#define MPLOCKED "lock ; " | #define MPLOCKED "lock ; " | ||||
#else | #else | ||||
#define MPLOCKED | #define MPLOCKED | ||||
#endif | #endif | ||||
/* | /* | ||||
* Newer versions of clang and gcc, including the in-tree compiler, support | |||||
* outputting status flags directly from asm blocks. Provide support for | |||||
* older cross compilers, where the flag is extracted via setcc. | |||||
*/ | |||||
#ifdef __GCC_ASM_FLAG_OUTPUTS__ | |||||
#define CC_FLAG_OUT_INST(cc, loc) | |||||
#define CC_FLAG_OUT_CONS(cc) "=@cc" #cc | |||||
#else | |||||
#define CC_FLAG_OUT_INST(cc, loc) "set" #cc " " loc " ; " | |||||
#define CC_FLAG_OUT_CONS(cc) "=q" | |||||
#endif | |||||
/* | |||||
* The assembly is volatilized to avoid code chunk removal by the compiler. | * The assembly is volatilized to avoid code chunk removal by the compiler. | ||||
* GCC aggressively reorders operations and memory clobbering is necessary | * GCC aggressively reorders operations and memory clobbering is necessary | ||||
* in order to avoid that for memory barriers. | * in order to avoid that for memory barriers. | ||||
*/ | */ | ||||
#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ | #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ | ||||
static __inline void \ | static __inline void \ | ||||
atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ | atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ | ||||
{ \ | { \ | ||||
Show All 32 Lines | |||||
static __inline int \ | static __inline int \ | ||||
atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ | atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ | ||||
{ \ | { \ | ||||
u_char res; \ | u_char res; \ | ||||
\ | \ | ||||
__asm __volatile( \ | __asm __volatile( \ | ||||
" " MPLOCKED " " \ | " " MPLOCKED " " \ | ||||
" cmpxchg %3,%1 ; " \ | " cmpxchg %3,%1 ; " \ | ||||
CC_FLAG_OUT_INST(e, "%0") \ | |||||
"# atomic_cmpset_" #TYPE " " \ | "# atomic_cmpset_" #TYPE " " \ | ||||
: "=@cce" (res), /* 0 */ \ | : CC_FLAG_OUT_CONS(e) (res), /* 0 */ \ | ||||
"+m" (*dst), /* 1 */ \ | "+m" (*dst), /* 1 */ \ | ||||
"+a" (expect) /* 2 */ \ | "+a" (expect) /* 2 */ \ | ||||
: "r" (src) /* 3 */ \ | : "r" (src) /* 3 */ \ | ||||
: "memory", "cc"); \ | : "memory", "cc"); \ | ||||
return (res); \ | return (res); \ | ||||
} \ | } \ | ||||
\ | \ | ||||
static __inline int \ | static __inline int \ | ||||
atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ | atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ | ||||
{ \ | { \ | ||||
u_char res; \ | u_char res; \ | ||||
\ | \ | ||||
__asm __volatile( \ | __asm __volatile( \ | ||||
" " MPLOCKED " " \ | " " MPLOCKED " " \ | ||||
" cmpxchg %3,%1 ; " \ | " cmpxchg %3,%1 ; " \ | ||||
CC_FLAG_OUT_INST(e, "%0") \ | |||||
"# atomic_fcmpset_" #TYPE " " \ | "# atomic_fcmpset_" #TYPE " " \ | ||||
: "=@cce" (res), /* 0 */ \ | : CC_FLAG_OUT_CONS(e) (res), /* 0 */ \ | ||||
"+m" (*dst), /* 1 */ \ | "+m" (*dst), /* 1 */ \ | ||||
"+a" (*expect) /* 2 */ \ | "+a" (*expect) /* 2 */ \ | ||||
: "r" (src) /* 3 */ \ | : "r" (src) /* 3 */ \ | ||||
: "memory", "cc"); \ | : "memory", "cc"); \ | ||||
return (res); \ | return (res); \ | ||||
} | } | ||||
ATOMIC_CMPSET(char); | ATOMIC_CMPSET(char); | ||||
Show All 40 Lines | |||||
static __inline int | static __inline int | ||||
atomic_testandset_int(volatile u_int *p, u_int v) | atomic_testandset_int(volatile u_int *p, u_int v) | ||||
{ | { | ||||
u_char res; | u_char res; | ||||
__asm __volatile( | __asm __volatile( | ||||
" " MPLOCKED " " | " " MPLOCKED " " | ||||
" btsl %2,%1 ; " | " btsl %2,%1 ; " | ||||
CC_FLAG_OUT_INST(c, "%0") | |||||
"# atomic_testandset_int" | "# atomic_testandset_int" | ||||
: "=@ccc" (res), /* 0 */ | : CC_FLAG_OUT_CONS(c) (res), /* 0 */ | ||||
"+m" (*p) /* 1 */ | "+m" (*p) /* 1 */ | ||||
: "Ir" (v & 0x1f) /* 2 */ | : "Ir" (v & 0x1f) /* 2 */ | ||||
: "cc"); | : "cc"); | ||||
return (res); | return (res); | ||||
} | } | ||||
static __inline int | static __inline int | ||||
atomic_testandset_long(volatile u_long *p, u_int v) | atomic_testandset_long(volatile u_long *p, u_int v) | ||||
{ | { | ||||
u_char res; | u_char res; | ||||
__asm __volatile( | __asm __volatile( | ||||
" " MPLOCKED " " | " " MPLOCKED " " | ||||
" btsq %2,%1 ; " | " btsq %2,%1 ; " | ||||
CC_FLAG_OUT_INST(c, "%0") | |||||
"# atomic_testandset_long" | "# atomic_testandset_long" | ||||
: "=@ccc" (res), /* 0 */ | : CC_FLAG_OUT_CONS(c) (res), /* 0 */ | ||||
"+m" (*p) /* 1 */ | "+m" (*p) /* 1 */ | ||||
: "Jr" ((u_long)(v & 0x3f)) /* 2 */ | : "Jr" ((u_long)(v & 0x3f)) /* 2 */ | ||||
: "cc"); | : "cc"); | ||||
return (res); | return (res); | ||||
} | } | ||||
static __inline int | static __inline int | ||||
atomic_testandclear_int(volatile u_int *p, u_int v) | atomic_testandclear_int(volatile u_int *p, u_int v) | ||||
{ | { | ||||
u_char res; | u_char res; | ||||
__asm __volatile( | __asm __volatile( | ||||
" " MPLOCKED " " | " " MPLOCKED " " | ||||
" btrl %2,%1 ; " | " btrl %2,%1 ; " | ||||
CC_FLAG_OUT_INST(c, "%0") | |||||
"# atomic_testandclear_int" | "# atomic_testandclear_int" | ||||
: "=@ccc" (res), /* 0 */ | : CC_FLAG_OUT_CONS(c) (res), /* 0 */ | ||||
"+m" (*p) /* 1 */ | "+m" (*p) /* 1 */ | ||||
: "Ir" (v & 0x1f) /* 2 */ | : "Ir" (v & 0x1f) /* 2 */ | ||||
: "cc"); | : "cc"); | ||||
return (res); | return (res); | ||||
} | } | ||||
static __inline int | static __inline int | ||||
atomic_testandclear_long(volatile u_long *p, u_int v) | atomic_testandclear_long(volatile u_long *p, u_int v) | ||||
{ | { | ||||
u_char res; | u_char res; | ||||
__asm __volatile( | __asm __volatile( | ||||
" " MPLOCKED " " | " " MPLOCKED " " | ||||
" btrq %2,%1 ; " | " btrq %2,%1 ; " | ||||
CC_FLAG_OUT_INST(c, "%0") | |||||
"# atomic_testandclear_long" | "# atomic_testandclear_long" | ||||
: "=@ccc" (res), /* 0 */ | : CC_FLAG_OUT_CONS(c) (res), /* 0 */ | ||||
"+m" (*p) /* 1 */ | "+m" (*p) /* 1 */ | ||||
: "Jr" ((u_long)(v & 0x3f)) /* 2 */ | : "Jr" ((u_long)(v & 0x3f)) /* 2 */ | ||||
: "cc"); | : "cc"); | ||||
return (res); | return (res); | ||||
} | } | ||||
/* | /* | ||||
* We assume that a = b will do atomic loads and stores. Due to the | * We assume that a = b will do atomic loads and stores. Due to the | ||||
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | |||||
ATOMIC_LOADSTORE(short); | ATOMIC_LOADSTORE(short); | ||||
ATOMIC_LOADSTORE(int); | ATOMIC_LOADSTORE(int); | ||||
ATOMIC_LOADSTORE(long); | ATOMIC_LOADSTORE(long); | ||||
#undef ATOMIC_ASM | #undef ATOMIC_ASM | ||||
#undef ATOMIC_LOAD | #undef ATOMIC_LOAD | ||||
#undef ATOMIC_STORE | #undef ATOMIC_STORE | ||||
#undef ATOMIC_LOADSTORE | #undef ATOMIC_LOADSTORE | ||||
#undef CC_FLAG_OUT_INST | |||||
#undef CC_FLAG_OUT_CONS | |||||
#ifndef WANT_FUNCTIONS | #ifndef WANT_FUNCTIONS | ||||
/* Read the current value and store a new value in the destination. */ | /* Read the current value and store a new value in the destination. */ | ||||
#ifdef __GNUCLIKE_ASM | #ifdef __GNUCLIKE_ASM | ||||
static __inline u_int | static __inline u_int | ||||
atomic_swap_int(volatile u_int *p, u_int v) | atomic_swap_int(volatile u_int *p, u_int v) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 210 Lines • Show Last 20 Lines |