Differential D21682 Diff 63018 head/sys/powerpc/include/atomic.h

Changeset View

Standalone View

head/sys/powerpc/include/atomic.h

Show First 20 Lines • Show All 554 Lines • ▼ Show 20 Lines
#endif		#endif
#undef ATOMIC_STORE_LOAD		#undef ATOMIC_STORE_LOAD

/*		/*
* Atomically compare the value stored at *p with cmpval and if the		* Atomically compare the value stored at *p with cmpval and if the
* two values are equal, update the value of *p with newval. Returns		* two values are equal, update the value of *p with newval. Returns
* zero if the compare failed, nonzero otherwise.		* zero if the compare failed, nonzero otherwise.
*/		*/
		#ifdef ISA_206_ATOMICS
static __inline int		static __inline int
		atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval)
		{
		int ret;

		__asm __volatile (
		"1:\tlbarx %0, 0, %2\n\t" /* load old value */
		"cmplw %3, %0\n\t" /* compare */
		"bne- 2f\n\t" /* exit if not equal */
		"stbcx. %4, 0, %2\n\t" /* attempt to store */
		"bne- 1b\n\t" /* spin if failed */
		"li %0, 1\n\t" /* success - retval = 1 */
		"b 3f\n\t" /* we've succeeded */
		"2:\n\t"
		"stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
		"li %0, 0\n\t" /* failure - retval = 0 */
		"3:\n\t"
		: "=&r" (ret), "=m" (*p)
		: "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
		: "cr0", "memory");

		return (ret);
		}

		static __inline int
		atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval)
		{
		int ret;

		__asm __volatile (
		"1:\tlharx %0, 0, %2\n\t" /* load old value */
		"cmplw %3, %0\n\t" /* compare */
		"bne- 2f\n\t" /* exit if not equal */
		"sthcx. %4, 0, %2\n\t" /* attempt to store */
		"bne- 1b\n\t" /* spin if failed */
		"li %0, 1\n\t" /* success - retval = 1 */
		"b 3f\n\t" /* we've succeeded */
		"2:\n\t"
		"sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
		"li %0, 0\n\t" /* failure - retval = 0 */
		"3:\n\t"
		: "=&r" (ret), "=m" (*p)
		: "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
		: "cr0", "memory");

		return (ret);
		}
		#endif

		static __inline int
atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)		atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
{		{
int ret;		int ret;

__asm __volatile (		__asm __volatile (
"1:\tlwarx %0, 0, %2\n\t" /* load old value */		"1:\tlwarx %0, 0, %2\n\t" /* load old value */
"cmplw %3, %0\n\t" /* compare */		"cmplw %3, %0\n\t" /* compare */
"bne- 2f\n\t" /* exit if not equal */		"bne- 2f\n\t" /* exit if not equal */
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines	__asm __volatile (
"3:\n\t"		"3:\n\t"
: "=&r" (ret), "=m" (*p)		: "=&r" (ret), "=m" (*p)
: "r" (p), "r" (cmpval), "r" (newval), "m" (*p)		: "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
: "cr0", "memory");		: "cr0", "memory");

return (ret);		return (ret);
}		}

static __inline int		#define ATOMIC_CMPSET_ACQ_REL(type) \
atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval)		static __inline int \
{		atomic_cmpset_acq_##type(volatile u_##type *p, \
int retval;		u_##type cmpval, u_##type newval)\
		{\
		u_##type retval; \
		retval = atomic_cmpset_##type(p, cmpval, newval);\
		__ATOMIC_ACQ();\
		return (retval);\
		}\
		static __inline int \
		atomic_cmpset_rel_##type(volatile u_##type *p, \
		u_##type cmpval, u_##type newval)\
		{\
		__ATOMIC_ACQ();\
		return (atomic_cmpset_##type(p, cmpval, newval));\
		}\
		struct hack

retval = atomic_cmpset_int(p, cmpval, newval);		ATOMIC_CMPSET_ACQ_REL(int);
__ATOMIC_ACQ();		ATOMIC_CMPSET_ACQ_REL(long);
return (retval);
}

static __inline int
atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval)
{
__ATOMIC_REL();
return (atomic_cmpset_int(p, cmpval, newval));
}

static __inline int		#define atomic_cmpset_8 atomic_cmpset_char
atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)		#define atomic_cmpset_acq_8 atomic_cmpset_acq_char
{		#define atomic_cmpset_rel_8 atomic_cmpset_rel_char
u_long retval;

retval = atomic_cmpset_long(p, cmpval, newval);		#define atomic_cmpset_16 atomic_cmpset_short
__ATOMIC_ACQ();		#define atomic_cmpset_acq_16 atomic_cmpset_acq_short
return (retval);		#define atomic_cmpset_rel_16 atomic_cmpset_rel_short
}

static __inline int
atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
{
__ATOMIC_REL();
return (atomic_cmpset_long(p, cmpval, newval));
}

#define atomic_cmpset_32 atomic_cmpset_int		#define atomic_cmpset_32 atomic_cmpset_int
#define atomic_cmpset_acq_32 atomic_cmpset_acq_int		#define atomic_cmpset_acq_32 atomic_cmpset_acq_int
#define atomic_cmpset_rel_32 atomic_cmpset_rel_int		#define atomic_cmpset_rel_32 atomic_cmpset_rel_int

#ifdef __powerpc64__		#ifdef __powerpc64__
#define atomic_cmpset_64 atomic_cmpset_long		#define atomic_cmpset_64 atomic_cmpset_long
#define atomic_cmpset_acq_64 atomic_cmpset_acq_long		#define atomic_cmpset_acq_64 atomic_cmpset_acq_long
#define atomic_cmpset_rel_64 atomic_cmpset_rel_long		#define atomic_cmpset_rel_64 atomic_cmpset_rel_long

#define atomic_cmpset_ptr atomic_cmpset_long		#define atomic_cmpset_ptr atomic_cmpset_long
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long		#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long		#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
#else		#else
#define atomic_cmpset_ptr atomic_cmpset_int		#define atomic_cmpset_ptr atomic_cmpset_int
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int		#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int		#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int
#endif		#endif

/*		/*
* Atomically compare the value stored at p with cmpval and if the		* Atomically compare the value stored at p with cmpval and if the
* two values are equal, update the value of *p with newval. Returns		* two values are equal, update the value of *p with newval. Returns
* zero if the compare failed and sets cmpval to the read value from p,		* zero if the compare failed and sets cmpval to the read value from p,
* nonzero otherwise.		* nonzero otherwise.
*/		*/
		#ifdef ISA_206_ATOMICS
static __inline int		static __inline int
		atomic_fcmpset_char(volatile u_char p, u_char cmpval, u_char newval)
		{
		int ret;

		__asm __volatile (
		"lbarx %0, 0, %3\n\t" /* load old value */
		"cmplw %4, %0\n\t" /* compare */
		"bne- 1f\n\t" /* exit if not equal */
		"stbcx. %5, 0, %3\n\t" /* attempt to store */
		"bne- 1f\n\t" /* exit if failed */
		"li %0, 1\n\t" /* success - retval = 1 */
		"b 2f\n\t" /* we've succeeded */
		"1:\n\t"
		"stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
		"stwx %0, 0, %7\n\t"
		"li %0, 0\n\t" /* failure - retval = 0 */
		"2:\n\t"
		: "=&r" (ret), "=m" (p), "=m" (cmpval)
		: "r" (p), "r" (cmpval), "r" (newval), "m" (p), "r"(cmpval)
		: "cr0", "memory");

		return (ret);
		}

		static __inline int
		atomic_fcmpset_short(volatile u_short p, u_short cmpval, u_short newval)
		{
		int ret;

		__asm __volatile (
		"lharx %0, 0, %3\n\t" /* load old value */
		"cmplw %4, %0\n\t" /* compare */
		"bne- 1f\n\t" /* exit if not equal */
		"sthcx. %5, 0, %3\n\t" /* attempt to store */
		"bne- 1f\n\t" /* exit if failed */
		"li %0, 1\n\t" /* success - retval = 1 */
		"b 2f\n\t" /* we've succeeded */
		"1:\n\t"
		"sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
		"stwx %0, 0, %7\n\t"
		"li %0, 0\n\t" /* failure - retval = 0 */
		"2:\n\t"
		: "=&r" (ret), "=m" (p), "=m" (cmpval)
		: "r" (p), "r" (cmpval), "r" (newval), "m" (p), "r"(cmpval)
		: "cr0", "memory");

		return (ret);
		}
		#endif /* ISA_206_ATOMICS */

		static __inline int
atomic_fcmpset_int(volatile u_int p, u_int cmpval, u_int newval)		atomic_fcmpset_int(volatile u_int p, u_int cmpval, u_int newval)
{		{
int ret;		int ret;

__asm __volatile (		__asm __volatile (
"lwarx %0, 0, %3\n\t" /* load old value */		"lwarx %0, 0, %3\n\t" /* load old value */
"cmplw %4, %0\n\t" /* compare */		"cmplw %4, %0\n\t" /* compare */
"bne- 1f\n\t" /* exit if not equal */		"bne- 1f\n\t" /* exit if not equal */
"stwcx. %5, 0, %3\n\t" /* attempt to store */		"stwcx. %5, 0, %3\n\t" /* attempt to store */
"bne- 1f\n\t" /* exit if failed */		"bne- 1f\n\t" /* exit if failed */
"li %0, 1\n\t" /* success - retval = 1 */		"li %0, 1\n\t" /* success - retval = 1 */
"b 2f\n\t" /* we've succeeded */		"b 2f\n\t" /* we've succeeded */
"1:\n\t"		"1:\n\t"
"stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */		"stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
"stwx %0, 0, %7\n\t"		"stwx %0, 0, %7\n\t"
"li %0, 0\n\t" /* failure - retval = 0 */		"li %0, 0\n\t" /* failure - retval = 0 */
"2:\n\t"		"2:\n\t"
: "=&r" (ret), "=m" (p), "=m" (cmpval)		: "=&r" (ret), "=m" (p), "=m" (cmpval)
: "r" (p), "r" (cmpval), "r" (newval), "m" (p), "r"(cmpval)		: "r" (p), "r" (cmpval), "r" (newval), "m" (p), "r"(cmpval)
: "cr0", "memory");		: "cr0", "memory");

return (ret);		return (ret);
}		}
static __inline int		static __inline int
atomic_fcmpset_long(volatile u_long p, u_long cmpval, u_long newval)		atomic_fcmpset_long(volatile u_long p, u_long cmpval, u_long newval)
{		{
int ret;		int ret;

__asm __volatile (		__asm __volatile (
#ifdef __powerpc64__		#ifdef __powerpc64__
"ldarx %0, 0, %3\n\t" /* load old value */		"ldarx %0, 0, %3\n\t" /* load old value */
"cmpld %4, %0\n\t" /* compare */		"cmpld %4, %0\n\t" /* compare */
"bne- 1f\n\t" /* exit if not equal */		"bne- 1f\n\t" /* exit if not equal */
"stdcx. %5, 0, %3\n\t" /* attempt to store */		"stdcx. %5, 0, %3\n\t" /* attempt to store */
#else		#else
"lwarx %0, 0, %3\n\t" /* load old value */		"lwarx %0, 0, %3\n\t" /* load old value */
"cmplw %4, %0\n\t" /* compare */		"cmplw %4, %0\n\t" /* compare */
"bne- 1f\n\t" /* exit if not equal */		"bne- 1f\n\t" /* exit if not equal */
"stwcx. %5, 0, %3\n\t" /* attempt to store */		"stwcx. %5, 0, %3\n\t" /* attempt to store */
#endif		#endif
"bne- 1f\n\t" /* exit if failed */		"bne- 1f\n\t" /* exit if failed */
"li %0, 1\n\t" /* success - retval = 1 */		"li %0, 1\n\t" /* success - retval = 1 */
"b 2f\n\t" /* we've succeeded */		"b 2f\n\t" /* we've succeeded */
"1:\n\t"		"1:\n\t"
#ifdef __powerpc64__		#ifdef __powerpc64__
"stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */		"stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
"stdx %0, 0, %7\n\t"		"stdx %0, 0, %7\n\t"
#else		#else
"stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */		"stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
"stwx %0, 0, %7\n\t"		"stwx %0, 0, %7\n\t"
#endif		#endif
"li %0, 0\n\t" /* failure - retval = 0 */		"li %0, 0\n\t" /* failure - retval = 0 */
"2:\n\t"		"2:\n\t"
: "=&r" (ret), "=m" (p), "=m" (cmpval)		: "=&r" (ret), "=m" (p), "=m" (cmpval)
: "r" (p), "r" (cmpval), "r" (newval), "m" (p), "r"(cmpval)		: "r" (p), "r" (cmpval), "r" (newval), "m" (p), "r"(cmpval)
: "cr0", "memory");		: "cr0", "memory");

return (ret);		return (ret);
}		}

static __inline int		#define ATOMIC_FCMPSET_ACQ_REL(type) \
atomic_fcmpset_acq_int(volatile u_int p, u_int cmpval, u_int newval)		static __inline int \
{		atomic_fcmpset_acq_##type(volatile u_##type *p, \
int retval;		u_##type *cmpval, u_##type newval)\
		{\
		u_##type retval; \
		retval = atomic_fcmpset_##type(p, cmpval, newval);\
		__ATOMIC_ACQ();\
		return (retval);\
		}\
		static __inline int \
		atomic_fcmpset_rel_##type(volatile u_##type *p, \
		u_##type *cmpval, u_##type newval)\
		{\
		__ATOMIC_REL();\
		return (atomic_fcmpset_##type(p, cmpval, newval));\
		}\
		struct hack

retval = atomic_fcmpset_int(p, cmpval, newval);		ATOMIC_FCMPSET_ACQ_REL(int);
__ATOMIC_ACQ();		ATOMIC_FCMPSET_ACQ_REL(long);
return (retval);
}

static __inline int		#define atomic_fcmpset_8 atomic_fcmpset_char
atomic_fcmpset_rel_int(volatile u_int p, u_int cmpval, u_int newval)		#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
{		#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
__ATOMIC_REL();
return (atomic_fcmpset_int(p, cmpval, newval));
}

static __inline int		#define atomic_fcmpset_16 atomic_fcmpset_short
atomic_fcmpset_acq_long(volatile u_long p, u_long cmpval, u_long newval)		#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
{		#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
u_long retval;

retval = atomic_fcmpset_long(p, cmpval, newval);
__ATOMIC_ACQ();
return (retval);
}

static __inline int
atomic_fcmpset_rel_long(volatile u_long p, u_long cmpval, u_long newval)
{
__ATOMIC_REL();
return (atomic_fcmpset_long(p, cmpval, newval));
}

#define atomic_fcmpset_32 atomic_fcmpset_int		#define atomic_fcmpset_32 atomic_fcmpset_int
#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int		#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int		#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int

#ifdef __powerpc64__		#ifdef __powerpc64__
#define atomic_fcmpset_64 atomic_fcmpset_long		#define atomic_fcmpset_64 atomic_fcmpset_long
#define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long		#define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long
#define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long		#define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines
#define atomic_fetchadd_64 atomic_fetchadd_long		#define atomic_fetchadd_64 atomic_fetchadd_long
#define atomic_swap_long atomic_swap_64		#define atomic_swap_long atomic_swap_64
#define atomic_swap_ptr atomic_swap_64		#define atomic_swap_ptr atomic_swap_64
#else		#else
#define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v)		#define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v)
#define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v)		#define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v)
#endif		#endif

#undef __ATOMIC_REL
#undef __ATOMIC_ACQ

static __inline void		static __inline void
atomic_thread_fence_acq(void)		atomic_thread_fence_acq(void)
{		{

powerpc_lwsync();		powerpc_lwsync();
}		}

static __inline void		static __inline void
Show All 11 Lines
}		}

static __inline void		static __inline void
atomic_thread_fence_seq_cst(void)		atomic_thread_fence_seq_cst(void)
{		{

__asm __volatile("sync" : : : "memory");		__asm __volatile("sync" : : : "memory");
}		}

		#ifndef ISA_206_ATOMICS
		#include <sys/_atomic_subword.h>
		#endif

		/* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
		ATOMIC_CMPSET_ACQ_REL(char);
		ATOMIC_CMPSET_ACQ_REL(short);

		ATOMIC_FCMPSET_ACQ_REL(char);
		ATOMIC_FCMPSET_ACQ_REL(short);

		#undef __ATOMIC_REL
		#undef __ATOMIC_ACQ

#endif /* ! _MACHINE_ATOMIC_H_ */		#endif /* ! _MACHINE_ATOMIC_H_ */