Page MenuHomeFreeBSD
Differential D23324 Diff 67729 head/sys/arm64/include/atomic.h

Changeset View

Standalone View

View Options

head/sys/arm64/include/atomic.h

Show First 20 Lines • Show All 57 Lines • ▼ Show 20 Lines
#else #else
#include <sys/atomic_common.h> #include <sys/atomic_common.h>
#define _ATOMIC_OP_PROTO(t, op, bar, flav) \ #define _ATOMIC_OP_PROTO(t, op, bar, flav) \
static __inline void \ static __inline void \
atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
#define _ATOMIC_OP_IMPL(t, w, s, op, asm_op, bar, a, l) \ #define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
_ATOMIC_OP_PROTO(t, op, bar, _llsc) \ _ATOMIC_OP_PROTO(t, op, bar, _llsc) \
{ \ { \
uint##t##_t tmp; \ uint##t##_t tmp; \
int res; \ int res; \
\ \
pre; \
__asm __volatile( \ __asm __volatile( \
"1: ld"#a"xr"#s" %"#w"0, [%2]\n" \ "1: ld"#a"xr"#s" %"#w"0, [%2]\n" \
" "#asm_op" %"#w"0, %"#w"0, %"#w"3\n" \ " "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \
" st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \ " st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \
" cbnz %w1, 1b\n" \ " cbnz %w1, 1b\n" \
: "=&r"(tmp), "=&r"(res) \ : "=&r"(tmp), "=&r"(res) \
: "r" (p), "r" (val) \ : "r" (p), "r" (val) \
: "memory" \ : "memory" \
); \ ); \
} \ } \
\ \
_ATOMIC_OP_PROTO(t, op, bar, _lse) \
{ \
uint##t##_t tmp; \
\
pre; \
__asm __volatile( \
".arch_extension lse\n" \
"ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \
".arch_extension nolse\n" \
: "=r" (tmp) \
: "r" (p), "r" (val) \
: "memory" \
); \
} \
\
_ATOMIC_OP_PROTO(t, op, bar, ) \ _ATOMIC_OP_PROTO(t, op, bar, ) \
{ \ { \
atomic_##op##_##bar##t##_llsc(p, val); \ atomic_##op##_##bar##t##_llsc(p, val); \
} }
#define __ATOMIC_OP(op, asm_op, bar, a, l) \ #define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
_ATOMIC_OP_IMPL(8, w, b, op, asm_op, bar, a, l) \ _ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \
_ATOMIC_OP_IMPL(16, w, h, op, asm_op, bar, a, l) \ bar, a, l) \
_ATOMIC_OP_IMPL(32, w, , op, asm_op, bar, a, l) \ _ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \
_ATOMIC_OP_IMPL(64, , , op, asm_op, bar, a, l) bar, a, l) \
_ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \
bar, a, l) \
_ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \
bar, a, l)
#define _ATOMIC_OP(op, asm_op) \ #define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \
__ATOMIC_OP(op, asm_op, , , ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \
__ATOMIC_OP(op, asm_op, acq_, a, ) \ __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \
__ATOMIC_OP(op, asm_op, rel_, , l) __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l)
_ATOMIC_OP(add, add) _ATOMIC_OP(add, add, add, )
_ATOMIC_OP(clear, bic) _ATOMIC_OP(clear, bic, clr, )
_ATOMIC_OP(set, orr) _ATOMIC_OP(set, orr, set, )
_ATOMIC_OP(subtract, sub) _ATOMIC_OP(subtract, add, add, val = -val)
#define _ATOMIC_CMPSET_PROTO(t, bar, flav) \ #define _ATOMIC_CMPSET_PROTO(t, bar, flav) \
static __inline int \ static __inline int \
atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \ atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \
uint##t##_t cmpval, uint##t##_t newval) uint##t##_t cmpval, uint##t##_t newval)
#define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \ #define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \
static __inline int \ static __inline int \
Show All 17 Lines __asm __volatile( \
: "=&r"(tmp), "=&r"(res) \ : "=&r"(tmp), "=&r"(res) \
: "r" (p), "r" (cmpval), "r" (newval) \ : "r" (p), "r" (cmpval), "r" (newval) \
: "cc", "memory" \ : "cc", "memory" \
); \ ); \
\ \
return (!res); \ return (!res); \
} \ } \
\ \
_ATOMIC_CMPSET_PROTO(t, bar, _lse) \
{ \
uint##t##_t oldval; \
int res; \
\
oldval = cmpval; \
__asm __volatile( \
".arch_extension lse\n" \
"cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
"cmp %"#w"1, %"#w"2\n" \
"cset %w0, eq\n" \
".arch_extension nolse\n" \
: "=r" (res), "+&r" (cmpval) \
: "r" (oldval), "r" (p), "r" (newval) \
: "cc", "memory" \
); \
\
return (res); \
} \
\
_ATOMIC_CMPSET_PROTO(t, bar, ) \ _ATOMIC_CMPSET_PROTO(t, bar, ) \
{ \ { \
return (atomic_cmpset_##bar##t##_llsc(p, cmpval, newval)); \ return (atomic_cmpset_##bar##t##_llsc(p, cmpval, newval)); \
} \ } \
\ \
_ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ _ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \
{ \ { \
uint##t##_t _cmpval, tmp; \ uint##t##_t _cmpval, tmp; \
Show All 11 Lines __asm __volatile( \
: "r" (p), "r" (_cmpval), "r" (newval) \ : "r" (p), "r" (_cmpval), "r" (newval) \
: "cc", "memory" \ : "cc", "memory" \
); \ ); \
*cmpval = tmp; \ *cmpval = tmp; \
\ \
return (!res); \ return (!res); \
} \ } \
\ \
_ATOMIC_FCMPSET_PROTO(t, bar, _lse) \
{ \
uint##t##_t _cmpval, tmp; \
int res; \
\
_cmpval = tmp = *cmpval; \
__asm __volatile( \
".arch_extension lse\n" \
"cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
"cmp %"#w"1, %"#w"2\n" \
"cset %w0, eq\n" \
".arch_extension nolse\n" \
: "=r" (res), "+&r" (tmp) \
: "r" (_cmpval), "r" (p), "r" (newval) \
: "cc", "memory" \
); \
*cmpval = tmp; \
\
return (res); \
} \
\
_ATOMIC_FCMPSET_PROTO(t, bar, ) \ _ATOMIC_FCMPSET_PROTO(t, bar, ) \
{ \ { \
return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, newval)); \ return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, newval)); \
} }
#define _ATOMIC_CMPSET(bar, a, l) \ #define _ATOMIC_CMPSET(bar, a, l) \
_ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \ _ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \
_ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \ _ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \
_ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \ _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \
_ATOMIC_CMPSET_IMPL(64, , , bar, a, l) _ATOMIC_CMPSET_IMPL(64, , , bar, a, l)
_ATOMIC_CMPSET( , , ) _ATOMIC_CMPSET( , , )
_ATOMIC_CMPSET(acq_, a, ) _ATOMIC_CMPSET(acq_, a, )
_ATOMIC_CMPSET(rel_, ,l) _ATOMIC_CMPSET(rel_, ,l)
#define _ATOMIC_FETCHADD_PROTO(t, flav) \ #define _ATOMIC_FETCHADD_PROTO(t, flav) \
static __inline uint##t##_t \ static __inline uint##t##_t \
atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val) atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
#define _ATOMIC_FETCHADD_IMPL(t, w) \ #define _ATOMIC_FETCHADD_IMPL(t, w) \
_ATOMIC_FETCHADD_PROTO(t, _llsc) \ _ATOMIC_FETCHADD_PROTO(t, _llsc) \
{ \ { \
uint##t##_t tmp, ret; \ uint##t##_t ret, tmp; \
int res; \ int res; \
\ \
__asm __volatile( \ __asm __volatile( \
"1: ldxr %"#w"2, [%3]\n" \ "1: ldxr %"#w"2, [%3]\n" \
" add %"#w"0, %"#w"2, %"#w"4\n" \ " add %"#w"0, %"#w"2, %"#w"4\n" \
" stxr %w1, %"#w"0, [%3]\n" \ " stxr %w1, %"#w"0, [%3]\n" \
" cbnz %w1, 1b\n" \ " cbnz %w1, 1b\n" \
: "=&r" (tmp), "=&r" (res), "=&r" (ret) \ : "=&r" (tmp), "=&r" (res), "=&r" (ret) \
: "r" (p), "r" (val) \ : "r" (p), "r" (val) \
: "memory" \ : "memory" \
); \ ); \
\ \
return (ret); \ return (ret); \
} \ } \
\ \
_ATOMIC_FETCHADD_PROTO(t, _lse) \
{ \
uint##t##_t ret; \
\
__asm __volatile( \
".arch_extension lse\n" \
"ldadd %"#w"2, %"#w"0, [%1]\n" \
".arch_extension nolse\n" \
: "=r" (ret) \
: "r" (p), "r" (val) \
: "memory" \
); \
\
return (ret); \
} \
\
_ATOMIC_FETCHADD_PROTO(t, ) \ _ATOMIC_FETCHADD_PROTO(t, ) \
{ \ { \
return (atomic_fetchadd_##t##_llsc(p, val)); \ return (atomic_fetchadd_##t##_llsc(p, val)); \
} }
_ATOMIC_FETCHADD_IMPL(32, w) _ATOMIC_FETCHADD_IMPL(32, w)
_ATOMIC_FETCHADD_IMPL(64, ) _ATOMIC_FETCHADD_IMPL(64, )
Show All 18 Lines __asm __volatile( \
: "=&r" (res), "=&r" (ret) \ : "=&r" (res), "=&r" (ret) \
: "r" (p), "r" (val) \ : "r" (p), "r" (val) \
: "memory" \ : "memory" \
); \ ); \
\ \
return (ret); \ return (ret); \
} \ } \
\ \
_ATOMIC_SWAP_PROTO(t, _lse) \
{ \
uint##t##_t ret; \
\
__asm __volatile( \
".arch_extension lse\n" \
"swp %"#w"2, %"#w"0, [%1]\n" \
".arch_extension nolse\n" \
: "=r" (ret) \
: "r" (p), "r" (val) \
: "memory" \
); \
\
return (ret); \
} \
\
_ATOMIC_SWAP_PROTO(t, ) \ _ATOMIC_SWAP_PROTO(t, ) \
{ \ { \
return (atomic_swap_##t##_llsc(p, val)); \ return (atomic_swap_##t##_llsc(p, val)); \
} \ } \
\ \
_ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ _ATOMIC_READANDCLEAR_PROTO(t, _llsc) \
{ \ { \
uint##t##_t ret; \ uint##t##_t ret; \
int res; \ int res; \
\ \
__asm __volatile( \ __asm __volatile( \
"1: ldxr %"#w"1, [%2]\n" \ "1: ldxr %"#w"1, [%2]\n" \
" stxr %w0, "#zreg", [%2]\n" \ " stxr %w0, "#zreg", [%2]\n" \
" cbnz %w0, 1b\n" \ " cbnz %w0, 1b\n" \
: "=&r" (res), "=&r" (ret) \ : "=&r" (res), "=&r" (ret) \
: "r" (p) \ : "r" (p) \
: "memory" \ : "memory" \
); \ ); \
\ \
return (ret); \ return (ret); \
} \ } \
\ \
_ATOMIC_READANDCLEAR_PROTO(t, _lse) \
{ \
return (atomic_swap_##t##_lse(p, 0)); \
} \
\
_ATOMIC_READANDCLEAR_PROTO(t, ) \ _ATOMIC_READANDCLEAR_PROTO(t, ) \
{ \ { \
return (atomic_readandclear_##t##_llsc(p)); \ return (atomic_readandclear_##t##_llsc(p)); \
} }
_ATOMIC_SWAP_IMPL(32, w, wzr) _ATOMIC_SWAP_IMPL(32, w, wzr)
_ATOMIC_SWAP_IMPL(64, , xzr) _ATOMIC_SWAP_IMPL(64, , xzr)
#define _ATOMIC_TEST_OP_PROTO(t, op, flav) \ #define _ATOMIC_TEST_OP_PROTO(t, op, flav) \
static __inline int \ static __inline int \
atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val) atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val)
#define _ATOMIC_TEST_OP_IMPL(t, w, op, asm_op) \ #define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op) \
_ATOMIC_TEST_OP_PROTO(t, op, _llsc) \ _ATOMIC_TEST_OP_PROTO(t, op, _llsc) \
{ \ { \
uint##t##_t mask, old, tmp; \ uint##t##_t mask, old, tmp; \
int res; \ int res; \
\ \
mask = 1u << (val & 0x1f); \ mask = 1u << (val & 0x1f); \
__asm __volatile( \ __asm __volatile( \
"1: ldxr %"#w"2, [%3]\n" \ "1: ldxr %"#w"2, [%3]\n" \
" "#asm_op" %"#w"0, %"#w"2, %"#w"4\n" \ " "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \
" stxr %w1, %"#w"0, [%3]\n" \ " stxr %w1, %"#w"0, [%3]\n" \
" cbnz %w1, 1b\n" \ " cbnz %w1, 1b\n" \
: "=&r" (tmp), "=&r" (res), "=&r" (old) \ : "=&r" (tmp), "=&r" (res), "=&r" (old) \
: "r" (p), "r" (mask) \ : "r" (p), "r" (mask) \
: "memory" \ : "memory" \
); \ ); \
\ \
return ((old & mask) != 0); \ return ((old & mask) != 0); \
} \ } \
\ \
_ATOMIC_TEST_OP_PROTO(t, op, _lse) \
{ \
uint##t##_t mask, old; \
\
mask = 1u << (val & 0x1f); \
__asm __volatile( \
".arch_extension lse\n" \
"ld"#lse_asm_op" %"#w"2, %"#w"0, [%1]\n" \
".arch_extension nolse\n" \
: "=r" (old) \
: "r" (p), "r" (mask) \
: "memory" \
); \
\
return ((old & mask) != 0); \
} \
\
_ATOMIC_TEST_OP_PROTO(t, op, ) \ _ATOMIC_TEST_OP_PROTO(t, op, ) \
{ \ { \
return (atomic_testand##op##_##t##_llsc(p, val)); \ return (atomic_testand##op##_##t##_llsc(p, val)); \
} }
#define _ATOMIC_TEST_OP(op, asm_op) \ #define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \
_ATOMIC_TEST_OP_IMPL(32, w, op, asm_op) \ _ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op) \
_ATOMIC_TEST_OP_IMPL(64, , op, asm_op) _ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op)
_ATOMIC_TEST_OP(clear, bic) _ATOMIC_TEST_OP(clear, bic, clr)
_ATOMIC_TEST_OP(set, orr) _ATOMIC_TEST_OP(set, orr, set)
#define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \ #define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \
static __inline uint##t##_t \ static __inline uint##t##_t \
atomic_load_acq_##t(volatile uint##t##_t *p) \ atomic_load_acq_##t(volatile uint##t##_t *p) \
{ \ { \
uint##t##_t ret; \ uint##t##_t ret; \
\ \
__asm __volatile( \ __asm __volatile( \
▲ Show 20 Lines • Show All 141 Lines • Show Last 20 Lines