Differential D23325 Diff 67730 head/sys/arm64/include/atomic.h

Changeset View

Standalone View

head/sys/arm64/include/atomic.h

Show First 20 Lines • Show All 53 Lines • ▼ Show 20 Lines
#define rmb() dmb(ld) /* Full system memory barrier load */		#define rmb() dmb(ld) /* Full system memory barrier load */

#if defined(KCSAN) && !defined(KCSAN_RUNTIME)		#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
#include <sys/_cscan_atomic.h>		#include <sys/_cscan_atomic.h>
#else		#else

#include <sys/atomic_common.h>		#include <sys/atomic_common.h>

		#ifdef _KERNEL
		extern bool lse_supported;

		#ifdef LSE_ATOMICS
		#define _ATOMIC_LSE_SUPPORTED 1
		#else
		#define _ATOMIC_LSE_SUPPORTED lse_supported
		#endif
		#else
		#define _ATOMIC_LSE_SUPPORTED 0
		#endif

#define _ATOMIC_OP_PROTO(t, op, bar, flav) \		#define _ATOMIC_OP_PROTO(t, op, bar, flav) \
static __inline void \		static __inline void \
atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)		atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)

#define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \		#define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
_ATOMIC_OP_PROTO(t, op, bar, _llsc) \		_ATOMIC_OP_PROTO(t, op, bar, _llsc) \
{ \		{ \
uint##t##_t tmp; \		uint##t##_t tmp; \
Show All 23 Lines	__asm __volatile( \
: "=r" (tmp) \		: "=r" (tmp) \
: "r" (p), "r" (val) \		: "r" (p), "r" (val) \
: "memory" \		: "memory" \
); \		); \
} \		} \
\		\
_ATOMIC_OP_PROTO(t, op, bar, ) \		_ATOMIC_OP_PROTO(t, op, bar, ) \
{ \		{ \
		if (_ATOMIC_LSE_SUPPORTED) \
		atomic_##op##_##bar##t##_lse(p, val); \
		else \
atomic_##op##_##bar##t##_llsc(p, val); \		atomic_##op##_##bar##t##_llsc(p, val); \
}		}

#define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \		#define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
_ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \		_ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \
bar, a, l) \		bar, a, l) \
_ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \		_ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \
bar, a, l) \		bar, a, l) \
_ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \		_ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines	__asm __volatile( \
: "cc", "memory" \		: "cc", "memory" \
); \		); \
\		\
return (res); \		return (res); \
} \		} \
\		\
_ATOMIC_CMPSET_PROTO(t, bar, ) \		_ATOMIC_CMPSET_PROTO(t, bar, ) \
{ \		{ \
return (atomic_cmpset_##bar##t##_llsc(p, cmpval, newval)); \		if (_ATOMIC_LSE_SUPPORTED) \
		return (atomic_cmpset_##bar##t##_lse(p, cmpval, \
		newval)); \
		else \
		return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \
		newval)); \
} \		} \
\		\
_ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \		_ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \
{ \		{ \
uint##t##_t _cmpval, tmp; \		uint##t##_t _cmpval, tmp; \
int res; \		int res; \
\		\
_cmpval = *cmpval; \		_cmpval = *cmpval; \
Show All 31 Lines	_ATOMIC_FCMPSET_PROTO(t, bar, _lse) \
); \		); \
*cmpval = tmp; \		*cmpval = tmp; \
\		\
return (res); \		return (res); \
} \		} \
\		\
_ATOMIC_FCMPSET_PROTO(t, bar, ) \		_ATOMIC_FCMPSET_PROTO(t, bar, ) \
{ \		{ \
return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, newval)); \		if (_ATOMIC_LSE_SUPPORTED) \
		return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \
		newval)); \
		else \
		return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \
		newval)); \
}		}

#define _ATOMIC_CMPSET(bar, a, l) \		#define _ATOMIC_CMPSET(bar, a, l) \
_ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \		_ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \
_ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \		_ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \
_ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \		_ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \
_ATOMIC_CMPSET_IMPL(64, , , bar, a, l)		_ATOMIC_CMPSET_IMPL(64, , , bar, a, l)

Show All 37 Lines	__asm __volatile( \
: "memory" \		: "memory" \
); \		); \
\		\
return (ret); \		return (ret); \
} \		} \
\		\
_ATOMIC_FETCHADD_PROTO(t, ) \		_ATOMIC_FETCHADD_PROTO(t, ) \
{ \		{ \
		if (_ATOMIC_LSE_SUPPORTED) \
		return (atomic_fetchadd_##t##_lse(p, val)); \
		else \
return (atomic_fetchadd_##t##_llsc(p, val)); \		return (atomic_fetchadd_##t##_llsc(p, val)); \
}		}

_ATOMIC_FETCHADD_IMPL(32, w)		_ATOMIC_FETCHADD_IMPL(32, w)
_ATOMIC_FETCHADD_IMPL(64, )		_ATOMIC_FETCHADD_IMPL(64, )

#define _ATOMIC_SWAP_PROTO(t, flav) \		#define _ATOMIC_SWAP_PROTO(t, flav) \
static __inline uint##t##_t \		static __inline uint##t##_t \
atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val)		atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
Show All 33 Lines	__asm __volatile( \
: "memory" \		: "memory" \
); \		); \
\		\
return (ret); \		return (ret); \
} \		} \
\		\
_ATOMIC_SWAP_PROTO(t, ) \		_ATOMIC_SWAP_PROTO(t, ) \
{ \		{ \
		if (_ATOMIC_LSE_SUPPORTED) \
		return (atomic_swap_##t##_lse(p, val)); \
		else \
return (atomic_swap_##t##_llsc(p, val)); \		return (atomic_swap_##t##_llsc(p, val)); \
} \		} \
\		\
_ATOMIC_READANDCLEAR_PROTO(t, _llsc) \		_ATOMIC_READANDCLEAR_PROTO(t, _llsc) \
{ \		{ \
uint##t##_t ret; \		uint##t##_t ret; \
int res; \		int res; \
\		\
__asm __volatile( \		__asm __volatile( \
Show All 10 Lines	return (ret); \
\		\
_ATOMIC_READANDCLEAR_PROTO(t, _lse) \		_ATOMIC_READANDCLEAR_PROTO(t, _lse) \
{ \		{ \
return (atomic_swap_##t##_lse(p, 0)); \		return (atomic_swap_##t##_lse(p, 0)); \
} \		} \
\		\
_ATOMIC_READANDCLEAR_PROTO(t, ) \		_ATOMIC_READANDCLEAR_PROTO(t, ) \
{ \		{ \
		if (_ATOMIC_LSE_SUPPORTED) \
		return (atomic_readandclear_##t##_lse(p)); \
		else \
return (atomic_readandclear_##t##_llsc(p)); \		return (atomic_readandclear_##t##_llsc(p)); \
}		}

_ATOMIC_SWAP_IMPL(32, w, wzr)		_ATOMIC_SWAP_IMPL(32, w, wzr)
_ATOMIC_SWAP_IMPL(64, , xzr)		_ATOMIC_SWAP_IMPL(64, , xzr)

#define _ATOMIC_TEST_OP_PROTO(t, op, flav) \		#define _ATOMIC_TEST_OP_PROTO(t, op, flav) \
static __inline int \		static __inline int \
atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val)		atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val)
Show All 32 Lines	__asm __volatile( \
: "memory" \		: "memory" \
); \		); \
\		\
return ((old & mask) != 0); \		return ((old & mask) != 0); \
} \		} \
\		\
_ATOMIC_TEST_OP_PROTO(t, op, ) \		_ATOMIC_TEST_OP_PROTO(t, op, ) \
{ \		{ \
		if (_ATOMIC_LSE_SUPPORTED) \
		return (atomic_testand##op##_##t##_lse(p, val)); \
		else \
return (atomic_testand##op##_##t##_llsc(p, val)); \		return (atomic_testand##op##_##t##_llsc(p, val)); \
}		}

#define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \		#define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \
_ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op) \		_ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op) \
_ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op)		_ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op)

_ATOMIC_TEST_OP(clear, bic, clr)		_ATOMIC_TEST_OP(clear, bic, clr)
_ATOMIC_TEST_OP(set, orr, set)		_ATOMIC_TEST_OP(set, orr, set)
▲ Show 20 Lines • Show All 149 Lines • Show Last 20 Lines