Differential D4318 Diff 10632 head/sys/arm64/include/atomic.h

Changeset View

Standalone View

head/sys/arm64/include/atomic.h

	Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Lines
	*/			*/
	#define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory")			#define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory")
	#define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory")			#define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory")

	#define mb() dmb(sy) /* Full system memory barrier all */			#define mb() dmb(sy) /* Full system memory barrier all */
	#define wmb() dmb(st) /* Full system memory barrier store */			#define wmb() dmb(st) /* Full system memory barrier store */
	#define rmb() dmb(ld) /* Full system memory barrier load */			#define rmb() dmb(ld) /* Full system memory barrier load */

	static __inline void			#define ATOMIC_OP(op, asm_op, bar, a, l) \
	atomic_add_32(volatile uint32_t *p, uint32_t val)			static __inline void \
	{			atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val) \
	uint32_t tmp;			{ \
	int res;			uint32_t tmp; \
				int res; \
	__asm __volatile(			\
	"1: ldxr %w0, [%2] \n"			__asm __volatile( \
	" add %w0, %w0, %w3 \n"			"1: ld"#a"xr %w0, [%2] \n" \
	" stxr %w1, %w0, [%2] \n"			" "#asm_op" %w0, %w0, %w3 \n" \
	" cbnz %w1, 1b \n"			" st"#l"xr %w1, %w0, [%2] \n" \
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc"			" cbnz %w1, 1b \n" \
	);			: "=&r"(tmp), "=&r"(res) \
				: "r" (p), "r" (val) \
				: "memory" \
				); \
				} \
				\
				static __inline void \
				atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val) \
				{ \
				uint64_t tmp; \
				int res; \
				\
				__asm __volatile( \
				"1: ld"#a"xr %0, [%2] \n" \
				" "#asm_op" %0, %0, %3 \n" \
				" st"#l"xr %w1, %0, [%2] \n" \
				" cbnz %w1, 1b \n" \
				: "=&r"(tmp), "=&r"(res) \
				: "r" (p), "r" (val) \
				: "memory" \
				); \
	}			}

	static __inline void			#define ATOMIC(op, asm_op) \
	atomic_clear_32(volatile uint32_t *p, uint32_t val)			ATOMIC_OP(op, asm_op, , , ) \
	{			ATOMIC_OP(op, asm_op, acq_, a, ) \
	uint32_t tmp;			ATOMIC_OP(op, asm_op, rel_, , l) \
	int res;

	__asm __volatile(			ATOMIC(add, add)
	"1: ldxr %w0, [%2] \n"			ATOMIC(clear, bic)
	" bic %w0, %w0, %w3 \n"			ATOMIC(set, orr)
	" stxr %w1, %w0, [%2] \n"			ATOMIC(subtract, sub)
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc"			#define ATOMIC_CMPSET(bar, a, l) \
	);			static __inline int \
				atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \
				uint32_t newval) \
				{ \
				uint32_t tmp; \
				int res; \
				\
				__asm __volatile( \
				"1: mov %w1, #1 \n" \
				" ld"#a"xr %w0, [%2] \n" \
				" cmp %w0, %w3 \n" \
				" b.ne 2f \n" \
				" st"#l"xr %w1, %w4, [%2] \n" \
				" cbnz %w1, 1b \n" \
				"2:" \
				: "=&r"(tmp), "=&r"(res) \
				: "r" (p), "r" (cmpval), "r" (newval) \
				: "cc", "memory" \
				); \
				\
				return (!res); \
				} \
				\
				static __inline int \
				atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval, \
				uint64_t newval) \
				{ \
				uint64_t tmp; \
				int res; \
				\
				__asm __volatile( \
				"1: mov %w1, #1 \n" \
				" ld"#a"xr %0, [%2] \n" \
				" cmp %0, %3 \n" \
				" b.ne 2f \n" \
				" st"#l"xr %w1, %4, [%2] \n" \
				" cbnz %w1, 1b \n" \
				"2:" \
				: "=&r"(tmp), "=&r"(res) \
				: "r" (p), "r" (cmpval), "r" (newval) \
				: "cc", "memory" \
				); \
				\
				return (!res); \
	}			}

	static __inline int			ATOMIC_CMPSET( , , )
	atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)			ATOMIC_CMPSET(acq_, a, )
				ATOMIC_CMPSET(rel_, ,l)

				static __inline uint32_t
				atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
	{			{
	uint32_t tmp;			uint32_t tmp, ret;
	int res;			int res;

	__asm __volatile(			__asm __volatile(
	"1: mov %w1, #1 \n"			"1: ldxr %w2, [%3] \n"
	" ldxr %w0, [%2] \n"			" add %w0, %w2, %w4 \n"
	" cmp %w0, %w3 \n"			" stxr %w1, %w0, [%3] \n"
	" b.ne 2f \n"
	" stxr %w1, %w4, [%2] \n"
	" cbnz %w1, 1b \n"			" cbnz %w1, 1b \n"
	"2:"			: "=&r"(tmp), "=&r"(res), "=&r"(ret)
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval)			: "r" (p), "r" (val)
	: : "cc"			: "memory"
	);			);

	return (!res);			return (ret);
	}			}

	static __inline uint32_t			static __inline uint64_t
	atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)			atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
	{			{
	uint32_t tmp, ret;			uint64_t tmp, ret;
	int res;			int res;

	__asm __volatile(			__asm __volatile(
	"1: ldxr %w4, [%2] \n"			"1: ldxr %2, [%3] \n"
	" add %w0, %w4, %w3 \n"			" add %0, %2, %4 \n"
	" stxr %w1, %w0, [%2] \n"			" stxr %w1, %0, [%3] \n"
	" cbnz %w1, 1b \n"			" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc"			: "=&r"(tmp), "=&r"(res), "=&r"(ret)
				: "r" (p), "r" (val)
				: "memory"
	);			);

	return (ret);			return (ret);
	}			}

	static __inline uint32_t			static __inline uint32_t
	atomic_readandclear_32(volatile uint32_t *p)			atomic_readandclear_32(volatile uint32_t *p)
	{			{
	uint32_t tmp, ret;			uint32_t ret;
	int res;			int res;

	__asm __volatile(			__asm __volatile(
	" mov %w0, #0 \n"			"1: ldxr %w1, [%2] \n"
	"1: ldxr %w3, [%2] \n"			" stxr %w0, wzr, [%2] \n"
	" stxr %w1, %w0, [%2] \n"			" cbnz %w0, 1b \n"
	" cbnz %w1, 1b \n"			: "=&r"(res), "=&r"(ret)
	: "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc"			: "r" (p)
				: "memory"
	);			);

	return (ret);			return (ret);
	}			}

	static __inline void			static __inline uint64_t
	atomic_set_32(volatile uint32_t *p, uint32_t val)			atomic_readandclear_64(volatile uint64_t *p)
	{			{
	uint32_t tmp;			uint64_t ret;
	int res;			int res;

	__asm __volatile(			__asm __volatile(
	"1: ldxr %w0, [%2] \n"			"1: ldxr %1, [%2] \n"
	" orr %w0, %w0, %w3 \n"			" stxr %w0, xzr, [%2] \n"
	" stxr %w1, %w0, [%2] \n"			" cbnz %w0, 1b \n"
	" cbnz %w1, 1b \n"			: "=&r"(res), "=&r"(ret)
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc"			: "r" (p)
				: "memory"
	);			);

				return (ret);
	}			}

	static __inline uint32_t			static __inline uint32_t
	atomic_swap_32(volatile uint32_t *p, uint32_t val)			atomic_swap_32(volatile uint32_t *p, uint32_t val)
	{			{
	uint32_t tmp;			uint32_t ret;
	int res;			int res;

	__asm __volatile(			__asm __volatile(
	"1: ldxr %w0, [%2] \n"			"1: ldxr %w0, [%2] \n"
	" stxr %w1, %w3, [%2] \n"			" stxr %w1, %w3, [%2] \n"
	" cbnz %w1, 1b \n"			" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"			: "=&r"(ret), "=&r"(res)
				: "r" (p), "r" (val)
				: "memory"
	);			);

	return (tmp);			return (ret);
	}			}

	static __inline void			static __inline uint64_t
	atomic_subtract_32(volatile uint32_t *p, uint32_t val)			atomic_swap_64(volatile uint64_t *p, uint64_t val)
	{			{
	uint32_t tmp;			uint64_t ret;
	int res;			int res;

	__asm __volatile(			__asm __volatile(
	"1: ldxr %w0, [%2] \n"			"1: ldxr %0, [%2] \n"
	" sub %w0, %w0, %w3 \n"			" stxr %w1, %3, [%2] \n"
	" stxr %w1, %w0, [%2] \n"
	" cbnz %w1, 1b \n"			" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc"			: "=&r"(ret), "=&r"(res)
				: "r" (p), "r" (val)
				: "memory"
	);			);
	}

	#define atomic_add_int atomic_add_32			return (ret);
	#define atomic_clear_int atomic_clear_32
	#define atomic_cmpset_int atomic_cmpset_32
	#define atomic_fetchadd_int atomic_fetchadd_32
	#define atomic_readandclear_int atomic_readandclear_32
	#define atomic_set_int atomic_set_32
	#define atomic_swap_int atomic_swap_32
	#define atomic_subtract_int atomic_subtract_32

	static __inline void
	atomic_add_acq_32(volatile uint32_t *p, uint32_t val)
	{
	uint32_t tmp;
	int res;

	__asm __volatile(
	"1: ldaxr %w0, [%2] \n"
	" add %w0, %w0, %w3 \n"
	" stxr %w1, %w0, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}			}

	static __inline void			static __inline uint32_t
	atomic_clear_acq_32(volatile uint32_t *p, uint32_t val)			atomic_load_acq_32(volatile uint32_t *p)
	{			{
	uint32_t tmp;			uint32_t ret;
	int res;

	__asm __volatile(			__asm __volatile(
	"1: ldaxr %w0, [%2] \n"			"ldar %w0, [%1] \n"
	" bic %w0, %w0, %w3 \n"			: "=&r" (ret)
	" stxr %w1, %w0, [%2] \n"			: "r" (p)
	" cbnz %w1, 1b \n"			: "memory");
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline int			return (ret);
	atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
	{
	uint32_t tmp;
	int res;

	__asm __volatile(
	"1: mov %w1, #1 \n"
	" ldaxr %w0, [%2] \n"
	" cmp %w0, %w3 \n"
	" b.ne 2f \n"
	" stxr %w1, %w4, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval)
	: : "cc", "memory"
	);

	return (!res);
	}			}

	static __inline uint32_t			static __inline uint64_t
	atomic_load_acq_32(volatile uint32_t *p)			atomic_load_acq_64(volatile uint64_t *p)
	{			{
	uint32_t ret;			uint64_t ret;

	__asm __volatile(			__asm __volatile(
	"ldar %w0, [%1] \n"			"ldar %0, [%1] \n"
	: "=&r" (ret) : "r" (p) : "memory");			: "=&r" (ret)
				: "r" (p)
				: "memory");

	return (ret);			return (ret);
	}			}

	static __inline void			static __inline void
	atomic_set_acq_32(volatile uint32_t *p, uint32_t val)			atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
	{			{
	uint32_t tmp;
	int res;

	__asm __volatile(			__asm __volatile(
	"1: ldaxr %w0, [%2] \n"			"stlr %w0, [%1] \n"
	" orr %w0, %w0, %w3 \n"			:
	" stxr %w1, %w0, [%2] \n"			: "r" (val), "r" (p)
	" cbnz %w1, 1b \n"			: "memory");
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}			}

	static __inline void			static __inline void
	atomic_subtract_acq_32(volatile uint32_t *p, uint32_t val)			atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
	{			{
	uint32_t tmp;
	int res;

	__asm __volatile(			__asm __volatile(
	"1: ldaxr %w0, [%2] \n"			"stlr %0, [%1] \n"
	" sub %w0, %w0, %w3 \n"			:
	" stxr %w1, %w0, [%2] \n"			: "r" (val), "r" (p)
	" cbnz %w1, 1b \n"			: "memory");
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}			}


				#define atomic_add_int atomic_add_32
				#define atomic_clear_int atomic_clear_32
				#define atomic_cmpset_int atomic_cmpset_32
				#define atomic_fetchadd_int atomic_fetchadd_32
				#define atomic_readandclear_int atomic_readandclear_32
				#define atomic_set_int atomic_set_32
				#define atomic_swap_int atomic_swap_32
				#define atomic_subtract_int atomic_subtract_32

	#define atomic_add_acq_int atomic_add_acq_32			#define atomic_add_acq_int atomic_add_acq_32
	#define atomic_clear_acq_int atomic_clear_acq_32			#define atomic_clear_acq_int atomic_clear_acq_32
	#define atomic_cmpset_acq_int atomic_cmpset_acq_32			#define atomic_cmpset_acq_int atomic_cmpset_acq_32
	#define atomic_load_acq_int atomic_load_acq_32			#define atomic_load_acq_int atomic_load_acq_32
	#define atomic_set_acq_int atomic_set_acq_32			#define atomic_set_acq_int atomic_set_acq_32
	#define atomic_subtract_acq_int atomic_subtract_acq_32			#define atomic_subtract_acq_int atomic_subtract_acq_32

	/* The atomic functions currently are both acq and rel, we should fix this. */

	static __inline void
	atomic_add_rel_32(volatile uint32_t *p, uint32_t val)
	{
	uint32_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %w0, [%2] \n"
	" add %w0, %w0, %w3 \n"
	" stlxr %w1, %w0, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline void
	atomic_clear_rel_32(volatile uint32_t *p, uint32_t val)
	{
	uint32_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %w0, [%2] \n"
	" bic %w0, %w0, %w3 \n"
	" stlxr %w1, %w0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline int
	atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
	{
	uint32_t tmp;
	int res;

	__asm __volatile(
	"1: mov %w1, #1 \n"
	" ldxr %w0, [%2] \n"
	" cmp %w0, %w3 \n"
	" b.ne 2f \n"
	" stlxr %w1, %w4, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval)
	: : "cc", "memory"
	);

	return (!res);
	}

	static __inline void
	atomic_set_rel_32(volatile uint32_t *p, uint32_t val)
	{
	uint32_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %w0, [%2] \n"
	" orr %w0, %w0, %w3 \n"
	" stlxr %w1, %w0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline void
	atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
	{

	__asm __volatile(
	"stlr %w0, [%1] \n"
	: : "r" (val), "r" (p) : "memory");
	}

	static __inline void
	atomic_subtract_rel_32(volatile uint32_t *p, uint32_t val)
	{
	uint32_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %w0, [%2] \n"
	" sub %w0, %w0, %w3 \n"
	" stlxr %w1, %w0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	#define atomic_add_rel_int atomic_add_rel_32			#define atomic_add_rel_int atomic_add_rel_32
	#define atomic_clear_rel_int atomic_add_rel_32			#define atomic_clear_rel_int atomic_add_rel_32
	#define atomic_cmpset_rel_int atomic_cmpset_rel_32			#define atomic_cmpset_rel_int atomic_cmpset_rel_32
	#define atomic_set_rel_int atomic_set_rel_32			#define atomic_set_rel_int atomic_set_rel_32
	#define atomic_subtract_rel_int atomic_subtract_rel_32			#define atomic_subtract_rel_int atomic_subtract_rel_32
	#define atomic_store_rel_int atomic_store_rel_32			#define atomic_store_rel_int atomic_store_rel_32


	static __inline void
	atomic_add_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" add %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (val) : : "cc"
	);
	}

	static __inline void
	atomic_clear_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" bic %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc"
	);
	}

	static __inline int
	atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: mov %w1, #1 \n"
	" ldxr %0, [%2] \n"
	" cmp %0, %3 \n"
	" b.ne 2f \n"
	" stxr %w1, %4, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r" (tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval)
	: : "cc", "memory"
	);

	return (!res);
	}

	static __inline uint64_t
	atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp, ret;
	int res;

	__asm __volatile(
	"1: ldxr %4, [%2] \n"
	" add %0, %4, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc"
	);

	return (ret);
	}

	static __inline uint64_t
	atomic_readandclear_64(volatile uint64_t *p)
	{
	uint64_t tmp, ret;
	int res;

	__asm __volatile(
	" mov %0, #0 \n"
	"1: ldxr %3, [%2] \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc"
	);

	return (ret);
	}

	static __inline void
	atomic_set_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" orr %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc"
	);
	}

	static __inline void
	atomic_subtract_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" sub %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc"
	);
	}

	static __inline uint64_t
	atomic_swap_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t old;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" stxr %w1, %3, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(old), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);

	return (old);
	}

	#define atomic_add_long atomic_add_64			#define atomic_add_long atomic_add_64
	#define atomic_clear_long atomic_clear_64			#define atomic_clear_long atomic_clear_64
	#define atomic_cmpset_long atomic_cmpset_64			#define atomic_cmpset_long atomic_cmpset_64
	#define atomic_fetchadd_long atomic_fetchadd_64			#define atomic_fetchadd_long atomic_fetchadd_64
	#define atomic_readandclear_long atomic_readandclear_64			#define atomic_readandclear_long atomic_readandclear_64
	#define atomic_set_long atomic_set_64			#define atomic_set_long atomic_set_64
	#define atomic_swap_long atomic_swap_64			#define atomic_swap_long atomic_swap_64
	#define atomic_subtract_long atomic_subtract_64			#define atomic_subtract_long atomic_subtract_64

	#define atomic_add_ptr atomic_add_64			#define atomic_add_ptr atomic_add_64
	#define atomic_clear_ptr atomic_clear_64			#define atomic_clear_ptr atomic_clear_64
	#define atomic_cmpset_ptr atomic_cmpset_64			#define atomic_cmpset_ptr atomic_cmpset_64
	#define atomic_fetchadd_ptr atomic_fetchadd_64			#define atomic_fetchadd_ptr atomic_fetchadd_64
	#define atomic_readandclear_ptr atomic_readandclear_64			#define atomic_readandclear_ptr atomic_readandclear_64
	#define atomic_set_ptr atomic_set_64			#define atomic_set_ptr atomic_set_64
	#define atomic_swap_ptr atomic_swap_64			#define atomic_swap_ptr atomic_swap_64
	#define atomic_subtract_ptr atomic_subtract_64			#define atomic_subtract_ptr atomic_subtract_64

	static __inline void
	atomic_add_acq_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldaxr %0, [%2] \n"
	" add %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline void
	atomic_clear_acq_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldaxr %0, [%2] \n"
	" bic %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline int
	atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: mov %w1, #1 \n"
	" ldaxr %0, [%2] \n"
	" cmp %0, %3 \n"
	" b.ne 2f \n"
	" stxr %w1, %4, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (cmpval), "+r" (newval)
	: : "cc", "memory"
	);

	return (!res);
	}

	static __inline uint64_t
	atomic_load_acq_64(volatile uint64_t *p)
	{
	uint64_t ret;

	__asm __volatile(
	"ldar %0, [%1] \n"
	: "=&r" (ret) : "r" (p) : "memory");

	return (ret);
	}

	static __inline void
	atomic_set_acq_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldaxr %0, [%2] \n"
	" orr %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline void
	atomic_subtract_acq_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldaxr %0, [%2] \n"
	" sub %0, %0, %3 \n"
	" stxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	#define atomic_add_acq_long atomic_add_acq_64			#define atomic_add_acq_long atomic_add_acq_64
	#define atomic_clear_acq_long atomic_add_acq_64			#define atomic_clear_acq_long atomic_add_acq_64
	#define atomic_cmpset_acq_long atomic_cmpset_acq_64			#define atomic_cmpset_acq_long atomic_cmpset_acq_64
	#define atomic_load_acq_long atomic_load_acq_64			#define atomic_load_acq_long atomic_load_acq_64
	#define atomic_set_acq_long atomic_set_acq_64			#define atomic_set_acq_long atomic_set_acq_64
	#define atomic_subtract_acq_long atomic_subtract_acq_64			#define atomic_subtract_acq_long atomic_subtract_acq_64

	#define atomic_add_acq_ptr atomic_add_acq_64			#define atomic_add_acq_ptr atomic_add_acq_64
	#define atomic_clear_acq_ptr atomic_add_acq_64			#define atomic_clear_acq_ptr atomic_add_acq_64
	#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64			#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
	#define atomic_load_acq_ptr atomic_load_acq_64			#define atomic_load_acq_ptr atomic_load_acq_64
	#define atomic_set_acq_ptr atomic_set_acq_64			#define atomic_set_acq_ptr atomic_set_acq_64
	#define atomic_subtract_acq_ptr atomic_subtract_acq_64			#define atomic_subtract_acq_ptr atomic_subtract_acq_64

	/*			#define atomic_add_rel_long atomic_add_rel_64
	* TODO: The atomic functions currently are both acq and rel, we should fix			#define atomic_clear_rel_long atomic_clear_rel_64
	* this.			#define atomic_cmpset_rel_long atomic_cmpset_rel_64
	*/			#define atomic_set_rel_long atomic_set_rel_64
	static __inline void			#define atomic_subtract_rel_long atomic_subtract_rel_64
	atomic_add_rel_64(volatile uint64_t *p, uint64_t val)			#define atomic_store_rel_long atomic_store_rel_64
	{
	uint64_t tmp;
	int res;

	__asm __volatile(			#define atomic_add_rel_ptr atomic_add_rel_64
	"1: ldxr %0, [%2] \n"			#define atomic_clear_rel_ptr atomic_clear_rel_64
	" add %0, %0, %3 \n"			#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
	" stlxr %w1, %0, [%2] \n"			#define atomic_set_rel_ptr atomic_set_rel_64
	" cbnz %w1, 1b \n"			#define atomic_subtract_rel_ptr atomic_subtract_rel_64
	"2:"			#define atomic_store_rel_ptr atomic_store_rel_64
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline void			static __inline void
	atomic_clear_rel_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" bic %0, %0, %3 \n"
	" stlxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline int
	atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: mov %w1, #1 \n"
	" ldxr %0, [%2] \n"
	" cmp %0, %3 \n"
	" b.ne 2f \n"
	" stlxr %w1, %4, [%2] \n"
	" cbnz %w1, 1b \n"
	"2:"
	: "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (cmpval), "+r" (newval)
	: : "cc", "memory"
	);

	return (!res);
	}

	static __inline void
	atomic_set_rel_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" orr %0, %0, %3 \n"
	" stlxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline void
	atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
	{

	__asm __volatile(
	"stlr %0, [%1] \n"
	: : "r" (val), "r" (p) : "memory");
	}

	static __inline void
	atomic_subtract_rel_64(volatile uint64_t *p, uint64_t val)
	{
	uint64_t tmp;
	int res;

	__asm __volatile(
	"1: ldxr %0, [%2] \n"
	" sub %0, %0, %3 \n"
	" stlxr %w1, %0, [%2] \n"
	" cbnz %w1, 1b \n"
	: "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory"
	);
	}

	static __inline void
	atomic_thread_fence_acq(void)			atomic_thread_fence_acq(void)
	{			{

	dmb(ld);			dmb(ld);
	}			}

	static __inline void			static __inline void
	atomic_thread_fence_rel(void)			atomic_thread_fence_rel(void)
	Show All 10 Lines
	}			}

	static __inline void			static __inline void
	atomic_thread_fence_seq_cst(void)			atomic_thread_fence_seq_cst(void)
	{			{

	dmb(sy);			dmb(sy);
	}			}

	#define atomic_add_rel_long atomic_add_rel_64
	#define atomic_clear_rel_long atomic_clear_rel_64
	#define atomic_cmpset_rel_long atomic_cmpset_rel_64
	#define atomic_set_rel_long atomic_set_rel_64
	#define atomic_subtract_rel_long atomic_subtract_rel_64
	#define atomic_store_rel_long atomic_store_rel_64

	#define atomic_add_rel_ptr atomic_add_rel_64
	#define atomic_clear_rel_ptr atomic_clear_rel_64
	#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
	#define atomic_set_rel_ptr atomic_set_rel_64
	#define atomic_subtract_rel_ptr atomic_subtract_rel_64
	#define atomic_store_rel_ptr atomic_store_rel_64

	#endif /* _MACHINE_ATOMIC_H_ */			#endif /* _MACHINE_ATOMIC_H_ */