Index: head/sys/alpha/include/atomic.h =================================================================== --- head/sys/alpha/include/atomic.h (revision 67350) +++ head/sys/alpha/include/atomic.h (revision 67351) @@ -1,320 +1,489 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ +#include + /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and SMP safe. */ void atomic_set_8(volatile u_int8_t *, u_int8_t); void atomic_clear_8(volatile u_int8_t *, u_int8_t); void atomic_add_8(volatile u_int8_t *, u_int8_t); void atomic_subtract_8(volatile u_int8_t *, u_int8_t); void atomic_set_16(volatile u_int16_t *, u_int16_t); void atomic_clear_16(volatile u_int16_t *, u_int16_t); void atomic_add_16(volatile u_int16_t *, u_int16_t); void atomic_subtract_16(volatile u_int16_t *, u_int16_t); static __inline void atomic_set_32(volatile u_int32_t *p, u_int32_t v) { u_int32_t temp; __asm __volatile ( "1:\tldl_l %0, %2\n\t" /* load old value */ "bis %0, %3, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline void atomic_clear_32(volatile u_int32_t *p, u_int32_t v) { u_int32_t temp; __asm __volatile ( "1:\tldl_l %0, %2\n\t" /* load old value */ "bic %0, %3, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline void atomic_add_32(volatile u_int32_t *p, u_int32_t v) { u_int32_t temp; __asm __volatile ( "1:\tldl_l %0, %2\n\t" /* load old value */ "addl %0, %3, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline void atomic_subtract_32(volatile u_int32_t *p, u_int32_t v) { u_int32_t temp; __asm __volatile ( "1:\tldl_l %0, %2\n\t" /* load old value */ "subl %0, %3, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline u_int32_t atomic_readandclear_32(volatile u_int32_t *addr) { u_int32_t result,temp; __asm __volatile ( "wmb\n" /* ensure pending writes have drained */ "1:\tldl_l %0,%3\n\t" /* load current value, asserting lock */ "ldiq %1,0\n\t" /* value to store */ "stl_c %1,%2\n\t" /* attempt to store */ "beq %1,2f\n\t" /* if the store failed, spin */ "br 3f\n" /* it worked, exit */ "2:\tbr 1b\n" /* *addr not updated, loop */ "3:\tmb\n" /* it worked */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m"(*addr) : "memory"); return result; } static __inline void atomic_set_64(volatile u_int64_t *p, u_int64_t v) { u_int64_t temp; __asm __volatile ( "1:\tldq_l %0, %2\n\t" /* load old value */ "bis %0, %3, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline void atomic_clear_64(volatile u_int64_t *p, u_int64_t v) { u_int64_t temp; __asm __volatile ( "1:\tldq_l %0, %2\n\t" /* load old value */ "bic %0, %3, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline void atomic_add_64(volatile u_int64_t *p, u_int64_t v) { u_int64_t temp; __asm __volatile ( "1:\tldq_l %0, %2\n\t" /* load old value */ "addq %0, %3, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline void atomic_subtract_64(volatile u_int64_t *p, u_int64_t v) { u_int64_t temp; __asm __volatile ( "1:\tldq_l %0, %2\n\t" /* load old value */ "subq %0, %3, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 2f\n\t" /* spin if failed */ "mb\n\t" /* drain to memory */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "2:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (temp), "=m" (*p) : "m" (*p), "r" (v) : "memory"); } static __inline u_int64_t atomic_readandclear_64(volatile u_int64_t *addr) { u_int64_t result,temp; __asm __volatile ( "wmb\n" /* ensure pending writes have drained */ "1:\tldq_l %0,%3\n\t" /* load current value, asserting lock */ "ldiq %1,0\n\t" /* value to store */ "stq_c %1,%2\n\t" /* attempt to store */ "beq %1,2f\n\t" /* if the store failed, spin */ "br 3f\n" /* it worked, exit */ "2:\tbr 1b\n" /* *addr not updated, loop */ "3:\tmb\n" /* it worked */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m"(*addr) : "memory"); return result; } #define atomic_set_char atomic_set_8 #define atomic_clear_char atomic_clear_8 #define atomic_add_char atomic_add_8 #define atomic_subtract_char atomic_subtract_8 #define atomic_set_short atomic_set_16 #define atomic_clear_short atomic_clear_16 #define atomic_add_short atomic_add_16 #define atomic_subtract_short atomic_subtract_16 #define atomic_set_int atomic_set_32 #define atomic_clear_int atomic_clear_32 #define atomic_add_int atomic_add_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_set_long atomic_set_64 #define atomic_clear_long atomic_clear_64 #define atomic_add_long atomic_add_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_readandclear_long atomic_readandclear_64 +#define ATOMIC_ACQ_REL(NAME, WIDTH, TYPE) \ +static __inline void \ +atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ +{ \ + alpha_mb(); \ + atomic_##NAME##_##WIDTH(p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ +{ \ + atomic_##NAME##_##WIDTH(p, v); \ + alpha_wmb(); \ +} \ + \ +static __inline void \ +atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ +{ \ + alpha_mb(); \ + atomic_##NAME##_##WIDTH(p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ +{ \ + atomic_##NAME##_##WIDTH(p, v); \ + alpha_wmb(); \ +} + +ATOMIC_ACQ_REL(set, 8, char) +ATOMIC_ACQ_REL(clear, 8, char) +ATOMIC_ACQ_REL(add, 8, char) +ATOMIC_ACQ_REL(subtract, 8, char) +ATOMIC_ACQ_REL(set, 16, short) +ATOMIC_ACQ_REL(clear, 16, short) +ATOMIC_ACQ_REL(add, 16, short) +ATOMIC_ACQ_REL(subtract, 16, short) +ATOMIC_ACQ_REL(set, 32, int) +ATOMIC_ACQ_REL(clear, 32, int) +ATOMIC_ACQ_REL(add, 32, int) +ATOMIC_ACQ_REL(subtract, 32, int) +ATOMIC_ACQ_REL(set, 64, long) +ATOMIC_ACQ_REL(clear, 64, long) +ATOMIC_ACQ_REL(add, 64, long) +ATOMIC_ACQ_REL(subtract, 64, long) + +#undef ATOMIC_ACQ_REL + /* + * We assume that a = b will do atomic loads and stores. + */ +#define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ +static __inline u_##TYPE \ +atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ +{ \ + alpha_mb(); \ + return (*p); \ +} \ + \ +static __inline void \ +atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + *p = v; \ + alpha_wmb(); \ +} \ +static __inline u_##TYPE \ +atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ +{ \ + alpha_mb(); \ + return (*p); \ +} \ + \ +static __inline void \ +atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + *p = v; \ + alpha_wmb(); \ +} + +ATOMIC_STORE_LOAD(char, 8) +ATOMIC_STORE_LOAD(short, 16) +ATOMIC_STORE_LOAD(int, 32) +ATOMIC_STORE_LOAD(long, 64) + +#undef ATOMIC_STORE_LOAD + +/* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline u_int32_t atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) { u_int32_t ret; __asm __volatile ( "1:\tldl_l %0, %4\n\t" /* load old value */ "cmpeq %0, %2, %0\n\t" /* compare */ "beq %0, 2f\n\t" /* exit if not equal */ "mov %3, %0\n\t" /* value to store */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 3f\n\t" /* if it failed, spin */ "2:\n" /* done */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "3:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); return ret; } /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline u_int64_t atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) { u_int64_t ret; __asm __volatile ( "1:\tldq_l %0, %4\n\t" /* load old value */ "cmpeq %0, %2, %0\n\t" /* compare */ "beq %0, 2f\n\t" /* exit if not equal */ "mov %3, %0\n\t" /* value to store */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 3f\n\t" /* if it failed, spin */ "2:\n" /* done */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "3:\tbr 1b\n" /* try again */ ".previous\n" : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); return ret; } #define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_long atomic_cmpset_64 static __inline int atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) { return ( atomic_cmpset_long((volatile u_long *)dst, (u_long)exp, (u_long)src)); } + +static __inline u_int32_t +atomic_cmpset_acq_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) +{ + alpha_mb(); + return (atomic_cmpset_32(p, cmpval, newval)); +} + +static __inline u_int32_t +atomic_cmpset_rel_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) +{ + int retval; + + retval = atomic_cmpset_32(p, cmpval, newval); + alpha_wmb(); + return (retval); +} + +static __inline u_int64_t +atomic_cmpset_acq_64(volatile u_int64_t *p, u_int64_t cmpval, u_int64_t newval) +{ + alpha_mb(); + return (atomic_cmpset_64(p, cmpval, newval)); +} + +static __inline u_int64_t +atomic_cmpset_rel_64(volatile u_int64_t *p, u_int64_t cmpval, u_int64_t newval) +{ + int retval; + + retval = atomic_cmpset_64(p, cmpval, newval); + alpha_wmb(); + return (retval); +} + +#define atomic_cmpset_acq_int atomic_cmpset_acq_32 +#define atomic_cmpset_rel_int atomic_cmpset_rel_32 +#define atomic_cmpset_acq_long atomic_cmpset_acq_64 +#define atomic_cmpset_rel_long atomic_cmpset_rel_64 +#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long +#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long + +static __inline void * +atomic_load_acq_ptr(volatile void *p) +{ + return (void *)atomic_load_acq_long((volatile u_long *)p); +} + +static __inline void +atomic_store_rel_ptr(volatile void *p, void *v) +{ + atomic_store_rel_long((volatile u_long *)p, (u_long)v); +} + +#define ATOMIC_PTR(NAME) \ +static __inline void \ +atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_long((volatile u_long *)p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ +{ \ + alpha_mb(); \ + atomic_##NAME##_acq_long((volatile u_long *)p, v);\ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_rel_long((volatile u_long *)p, v);\ + alpha_wmb(); \ +} + +ATOMIC_PTR(set) +ATOMIC_PTR(clear) +ATOMIC_PTR(add) +ATOMIC_PTR(subtract) + +#undef ATOMIC_PTR #endif /* ! _MACHINE_ATOMIC_H_ */ Index: head/sys/amd64/include/atomic.h =================================================================== --- head/sys/amd64/include/atomic.h (revision 67350) +++ head/sys/amd64/include/atomic.h (revision 67351) @@ -1,255 +1,361 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char*)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char*)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char*)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char*)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short*)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short*)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short*)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short*)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int*)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int*)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int*)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int*)(P) -= (V)) * atomic_readandclear_int(P) (return *(u_int*)P; *(u_int*)P = 0;) * * atomic_set_long(P, V) (*(u_long*)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long*)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long*)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long*)(P) -= (V)) * atomic_readandclear_long(P) (return *(u_long*)P; *(u_long*)P = 0;) */ /* * The above functions are expanded inline in the statically-linked * kernel. Lock prefixes are generated if an SMP kernel is being * built. * * Kernel modules call real functions which are built into the kernel. * This allows kernel modules to be portable between UP and SMP systems. */ #if defined(KLD_MODULE) #define ATOMIC_ASM(NAME, TYPE, OP, V) \ - void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); +void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src); #else /* !KLD_MODULE */ #if defined(SMP) #if defined(LOCORE) #define MPLOCKED lock ; #else #define MPLOCKED "lock ; " #endif #else #define MPLOCKED #endif /* * The assembly is volatilized to demark potential before-and-after side * effects if an interrupt or SMP collision were to occur. */ #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 9) /* egcs 1.1.2+ version */ #define ATOMIC_ASM(NAME, TYPE, OP, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "=m" (*p) \ : "0" (*p), "ir" (V)); \ } /* * Atomic compare and set, used by the mutex functions * * if (*dst == exp) *dst = src (all 32 bit words) * * Returns 0 on failure, non-zero on success */ #if defined(I386_CPU) static __inline int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) { int res = exp; __asm __volatile( " pushfl ; " " cli ; " " cmpl %1,%3 ; " " jne 1f ; " " movl %2,%3 ; " "1: " " sete %%al; " " movzbl %%al,%0 ; " " popfl ; " "# atomic_cmpset_int" : "=a" (res) /* 0 (result) */ : "0" (exp), /* 1 */ "r" (src), /* 2 */ "m" (*(dst)) /* 3 */ : "memory"); return (res); } #else /* defined(I386_CPU) */ static __inline int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) { int res = exp; __asm __volatile ( " " MPLOCKED " " " cmpxchgl %2,%3 ; " " setz %%al ; " " movzbl %%al,%0 ; " "1: " "# atomic_cmpset_int" : "=a" (res) /* 0 (result) */ : "0" (exp), /* 1 */ "r" (src), /* 2 */ "m" (*(dst)) /* 3 */ : "memory"); return (res); } #endif /* defined(I386_CPU) */ +#define atomic_cmpset_acq_int atomic_cmpset_int +#define atomic_cmpset_rel_int atomic_cmpset_int + #else /* gcc <= 2.8 version */ #define ATOMIC_ASM(NAME, TYPE, OP, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "=m" (*p) \ : "ir" (V)); \ -} +} \ + \ + #endif #endif /* KLD_MODULE */ #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 9) /* egcs 1.1.2+ version */ ATOMIC_ASM(set, char, "orb %b2,%0", v) ATOMIC_ASM(clear, char, "andb %b2,%0", ~v) ATOMIC_ASM(add, char, "addb %b2,%0", v) ATOMIC_ASM(subtract, char, "subb %b2,%0", v) ATOMIC_ASM(set, short, "orw %w2,%0", v) ATOMIC_ASM(clear, short, "andw %w2,%0", ~v) ATOMIC_ASM(add, short, "addw %w2,%0", v) ATOMIC_ASM(subtract, short, "subw %w2,%0", v) ATOMIC_ASM(set, int, "orl %2,%0", v) ATOMIC_ASM(clear, int, "andl %2,%0", ~v) ATOMIC_ASM(add, int, "addl %2,%0", v) ATOMIC_ASM(subtract, int, "subl %2,%0", v) ATOMIC_ASM(set, long, "orl %2,%0", v) ATOMIC_ASM(clear, long, "andl %2,%0", ~v) ATOMIC_ASM(add, long, "addl %2,%0", v) ATOMIC_ASM(subtract, long, "subl %2,%0", v) #else /* gcc <= 2.8 version */ ATOMIC_ASM(set, char, "orb %1,%0", v) ATOMIC_ASM(clear, char, "andb %1,%0", ~v) ATOMIC_ASM(add, char, "addb %1,%0", v) ATOMIC_ASM(subtract, char, "subb %1,%0", v) ATOMIC_ASM(set, short, "orw %1,%0", v) ATOMIC_ASM(clear, short, "andw %1,%0", ~v) ATOMIC_ASM(add, short, "addw %1,%0", v) ATOMIC_ASM(subtract, short, "subw %1,%0", v) ATOMIC_ASM(set, int, "orl %1,%0", v) ATOMIC_ASM(clear, int, "andl %1,%0", ~v) ATOMIC_ASM(add, int, "addl %1,%0", v) ATOMIC_ASM(subtract, int, "subl %1,%0", v) ATOMIC_ASM(set, long, "orl %1,%0", v) ATOMIC_ASM(clear, long, "andl %1,%0", ~v) ATOMIC_ASM(add, long, "addl %1,%0", v) ATOMIC_ASM(subtract, long, "subl %1,%0", v) #endif +#undef ATOMIC_ASM + #ifndef WANT_FUNCTIONS +#define ATOMIC_ACQ_REL(NAME, TYPE) \ +static __inline void \ +atomic_##NAME##_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + __asm __volatile("lock; addl $0,0(%esp)" : : : "memory");\ + atomic_##NAME##_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + atomic_##NAME##_##TYPE(p, v); \ +} + +ATOMIC_ACQ_REL(set, char) +ATOMIC_ACQ_REL(clear, char) +ATOMIC_ACQ_REL(add, char) +ATOMIC_ACQ_REL(subtract, char) +ATOMIC_ACQ_REL(set, short) +ATOMIC_ACQ_REL(clear, short) +ATOMIC_ACQ_REL(add, short) +ATOMIC_ACQ_REL(subtract, short) +ATOMIC_ACQ_REL(set, int) +ATOMIC_ACQ_REL(clear, int) +ATOMIC_ACQ_REL(add, int) +ATOMIC_ACQ_REL(subtract, int) +ATOMIC_ACQ_REL(set, long) +ATOMIC_ACQ_REL(clear, long) +ATOMIC_ACQ_REL(add, long) +ATOMIC_ACQ_REL(subtract, long) + +#undef ATOMIC_ACQ_REL + +/* + * We assume that a = b will do atomic loads and stores. + */ +#define ATOMIC_STORE_LOAD(TYPE) \ +static __inline u_##TYPE \ +atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ +{ \ + __asm __volatile("lock; addl $0,0(%esp)" : : : "memory");\ + return (*p); \ +} \ + \ +static __inline void \ +atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + *p = v; \ + __asm __volatile("" : : : "memory"); \ +} + +ATOMIC_STORE_LOAD(char) +ATOMIC_STORE_LOAD(short) +ATOMIC_STORE_LOAD(int) +ATOMIC_STORE_LOAD(long) + +#undef ATOMIC_STORE_LOAD + static __inline int atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) { return ( atomic_cmpset_int((volatile u_int *)dst, (u_int)exp, (u_int)src)); } + +#define atomic_cmpset_acq_ptr atomic_cmpset_ptr +#define atomic_cmpset_rel_ptr atomic_cmpset_ptr + +static __inline void * +atomic_load_acq_ptr(volatile void *p) +{ + return (void *)atomic_load_acq_int((volatile u_int *)p); +} + +static __inline void +atomic_store_rel_ptr(volatile void *p, void *v) +{ + atomic_store_rel_int((volatile u_int *)p, (u_int)v); +} + +#define ATOMIC_PTR(NAME) \ +static __inline void \ +atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_int((volatile u_int *)p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_acq_int((volatile u_int *)p, v);\ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_rel_int((volatile u_int *)p, v);\ +} + +ATOMIC_PTR(set) +ATOMIC_PTR(clear) +ATOMIC_PTR(add) +ATOMIC_PTR(subtract) + +#undef ATOMIC_PTR static __inline u_int atomic_readandclear_int(volatile u_int *addr) { u_int result; __asm __volatile ( " xorl %0,%0 ; " " xchgl %1,%0 ; " "# atomic_readandclear_int" : "=&r" (result) /* 0 (result) */ : "m" (*addr)); /* 1 (addr) */ return (result); } static __inline u_long atomic_readandclear_long(volatile u_long *addr) { u_long result; __asm __volatile ( " xorl %0,%0 ; " " xchgl %1,%0 ; " "# atomic_readandclear_int" : "=&r" (result) /* 0 (result) */ : "m" (*addr)); /* 1 (addr) */ return (result); } #endif #endif /* ! _MACHINE_ATOMIC_H_ */ Index: head/sys/i386/include/atomic.h =================================================================== --- head/sys/i386/include/atomic.h (revision 67350) +++ head/sys/i386/include/atomic.h (revision 67351) @@ -1,255 +1,361 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char*)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char*)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char*)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char*)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short*)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short*)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short*)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short*)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int*)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int*)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int*)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int*)(P) -= (V)) * atomic_readandclear_int(P) (return *(u_int*)P; *(u_int*)P = 0;) * * atomic_set_long(P, V) (*(u_long*)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long*)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long*)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long*)(P) -= (V)) * atomic_readandclear_long(P) (return *(u_long*)P; *(u_long*)P = 0;) */ /* * The above functions are expanded inline in the statically-linked * kernel. Lock prefixes are generated if an SMP kernel is being * built. * * Kernel modules call real functions which are built into the kernel. * This allows kernel modules to be portable between UP and SMP systems. */ #if defined(KLD_MODULE) #define ATOMIC_ASM(NAME, TYPE, OP, V) \ - void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); +void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src); #else /* !KLD_MODULE */ #if defined(SMP) #if defined(LOCORE) #define MPLOCKED lock ; #else #define MPLOCKED "lock ; " #endif #else #define MPLOCKED #endif /* * The assembly is volatilized to demark potential before-and-after side * effects if an interrupt or SMP collision were to occur. */ #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 9) /* egcs 1.1.2+ version */ #define ATOMIC_ASM(NAME, TYPE, OP, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "=m" (*p) \ : "0" (*p), "ir" (V)); \ } /* * Atomic compare and set, used by the mutex functions * * if (*dst == exp) *dst = src (all 32 bit words) * * Returns 0 on failure, non-zero on success */ #if defined(I386_CPU) static __inline int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) { int res = exp; __asm __volatile( " pushfl ; " " cli ; " " cmpl %1,%3 ; " " jne 1f ; " " movl %2,%3 ; " "1: " " sete %%al; " " movzbl %%al,%0 ; " " popfl ; " "# atomic_cmpset_int" : "=a" (res) /* 0 (result) */ : "0" (exp), /* 1 */ "r" (src), /* 2 */ "m" (*(dst)) /* 3 */ : "memory"); return (res); } #else /* defined(I386_CPU) */ static __inline int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) { int res = exp; __asm __volatile ( " " MPLOCKED " " " cmpxchgl %2,%3 ; " " setz %%al ; " " movzbl %%al,%0 ; " "1: " "# atomic_cmpset_int" : "=a" (res) /* 0 (result) */ : "0" (exp), /* 1 */ "r" (src), /* 2 */ "m" (*(dst)) /* 3 */ : "memory"); return (res); } #endif /* defined(I386_CPU) */ +#define atomic_cmpset_acq_int atomic_cmpset_int +#define atomic_cmpset_rel_int atomic_cmpset_int + #else /* gcc <= 2.8 version */ #define ATOMIC_ASM(NAME, TYPE, OP, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "=m" (*p) \ : "ir" (V)); \ -} +} \ + \ + #endif #endif /* KLD_MODULE */ #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 9) /* egcs 1.1.2+ version */ ATOMIC_ASM(set, char, "orb %b2,%0", v) ATOMIC_ASM(clear, char, "andb %b2,%0", ~v) ATOMIC_ASM(add, char, "addb %b2,%0", v) ATOMIC_ASM(subtract, char, "subb %b2,%0", v) ATOMIC_ASM(set, short, "orw %w2,%0", v) ATOMIC_ASM(clear, short, "andw %w2,%0", ~v) ATOMIC_ASM(add, short, "addw %w2,%0", v) ATOMIC_ASM(subtract, short, "subw %w2,%0", v) ATOMIC_ASM(set, int, "orl %2,%0", v) ATOMIC_ASM(clear, int, "andl %2,%0", ~v) ATOMIC_ASM(add, int, "addl %2,%0", v) ATOMIC_ASM(subtract, int, "subl %2,%0", v) ATOMIC_ASM(set, long, "orl %2,%0", v) ATOMIC_ASM(clear, long, "andl %2,%0", ~v) ATOMIC_ASM(add, long, "addl %2,%0", v) ATOMIC_ASM(subtract, long, "subl %2,%0", v) #else /* gcc <= 2.8 version */ ATOMIC_ASM(set, char, "orb %1,%0", v) ATOMIC_ASM(clear, char, "andb %1,%0", ~v) ATOMIC_ASM(add, char, "addb %1,%0", v) ATOMIC_ASM(subtract, char, "subb %1,%0", v) ATOMIC_ASM(set, short, "orw %1,%0", v) ATOMIC_ASM(clear, short, "andw %1,%0", ~v) ATOMIC_ASM(add, short, "addw %1,%0", v) ATOMIC_ASM(subtract, short, "subw %1,%0", v) ATOMIC_ASM(set, int, "orl %1,%0", v) ATOMIC_ASM(clear, int, "andl %1,%0", ~v) ATOMIC_ASM(add, int, "addl %1,%0", v) ATOMIC_ASM(subtract, int, "subl %1,%0", v) ATOMIC_ASM(set, long, "orl %1,%0", v) ATOMIC_ASM(clear, long, "andl %1,%0", ~v) ATOMIC_ASM(add, long, "addl %1,%0", v) ATOMIC_ASM(subtract, long, "subl %1,%0", v) #endif +#undef ATOMIC_ASM + #ifndef WANT_FUNCTIONS +#define ATOMIC_ACQ_REL(NAME, TYPE) \ +static __inline void \ +atomic_##NAME##_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + __asm __volatile("lock; addl $0,0(%esp)" : : : "memory");\ + atomic_##NAME##_##TYPE(p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + atomic_##NAME##_##TYPE(p, v); \ +} + +ATOMIC_ACQ_REL(set, char) +ATOMIC_ACQ_REL(clear, char) +ATOMIC_ACQ_REL(add, char) +ATOMIC_ACQ_REL(subtract, char) +ATOMIC_ACQ_REL(set, short) +ATOMIC_ACQ_REL(clear, short) +ATOMIC_ACQ_REL(add, short) +ATOMIC_ACQ_REL(subtract, short) +ATOMIC_ACQ_REL(set, int) +ATOMIC_ACQ_REL(clear, int) +ATOMIC_ACQ_REL(add, int) +ATOMIC_ACQ_REL(subtract, int) +ATOMIC_ACQ_REL(set, long) +ATOMIC_ACQ_REL(clear, long) +ATOMIC_ACQ_REL(add, long) +ATOMIC_ACQ_REL(subtract, long) + +#undef ATOMIC_ACQ_REL + +/* + * We assume that a = b will do atomic loads and stores. + */ +#define ATOMIC_STORE_LOAD(TYPE) \ +static __inline u_##TYPE \ +atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ +{ \ + __asm __volatile("lock; addl $0,0(%esp)" : : : "memory");\ + return (*p); \ +} \ + \ +static __inline void \ +atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + *p = v; \ + __asm __volatile("" : : : "memory"); \ +} + +ATOMIC_STORE_LOAD(char) +ATOMIC_STORE_LOAD(short) +ATOMIC_STORE_LOAD(int) +ATOMIC_STORE_LOAD(long) + +#undef ATOMIC_STORE_LOAD + static __inline int atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) { return ( atomic_cmpset_int((volatile u_int *)dst, (u_int)exp, (u_int)src)); } + +#define atomic_cmpset_acq_ptr atomic_cmpset_ptr +#define atomic_cmpset_rel_ptr atomic_cmpset_ptr + +static __inline void * +atomic_load_acq_ptr(volatile void *p) +{ + return (void *)atomic_load_acq_int((volatile u_int *)p); +} + +static __inline void +atomic_store_rel_ptr(volatile void *p, void *v) +{ + atomic_store_rel_int((volatile u_int *)p, (u_int)v); +} + +#define ATOMIC_PTR(NAME) \ +static __inline void \ +atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_int((volatile u_int *)p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_acq_int((volatile u_int *)p, v);\ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_rel_int((volatile u_int *)p, v);\ +} + +ATOMIC_PTR(set) +ATOMIC_PTR(clear) +ATOMIC_PTR(add) +ATOMIC_PTR(subtract) + +#undef ATOMIC_PTR static __inline u_int atomic_readandclear_int(volatile u_int *addr) { u_int result; __asm __volatile ( " xorl %0,%0 ; " " xchgl %1,%0 ; " "# atomic_readandclear_int" : "=&r" (result) /* 0 (result) */ : "m" (*addr)); /* 1 (addr) */ return (result); } static __inline u_long atomic_readandclear_long(volatile u_long *addr) { u_long result; __asm __volatile ( " xorl %0,%0 ; " " xchgl %1,%0 ; " "# atomic_readandclear_int" : "=&r" (result) /* 0 (result) */ : "m" (*addr)); /* 1 (addr) */ return (result); } #endif #endif /* ! _MACHINE_ATOMIC_H_ */ Index: head/sys/ia64/include/atomic.h =================================================================== --- head/sys/ia64/include/atomic.h (revision 67350) +++ head/sys/ia64/include/atomic.h (revision 67351) @@ -1,213 +1,363 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and SMP safe. */ /* * Everything is built out of cmpxchg. */ #define IA64_CMPXCHG(sz, sem, type, p, cmpval, newval) \ ({ \ type _cmpval = cmpval; \ type _newval = newval; \ volatile type *_p = (volatile type *) p; \ type _ret; \ \ __asm __volatile ( \ "mov ar.ccv=%2;;\n\t" \ "cmpxchg" #sz "." #sem " %0=%4,%3,ar.ccv\n\t" \ : "=r" (_ret), "=m" (*_p) \ : "r" (_cmpval), "r" (_newval), "m" (*_p) \ : "memory"); \ _ret; \ }) /* * Some common forms of cmpxch. */ static __inline u_int32_t ia64_cmpxchg_acq_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) { return IA64_CMPXCHG(4, acq, u_int32_t, p, cmpval, newval); } static __inline u_int32_t ia64_cmpxchg_rel_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) { return IA64_CMPXCHG(4, rel, u_int32_t, p, cmpval, newval); } static __inline u_int64_t ia64_cmpxchg_acq_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) { return IA64_CMPXCHG(8, acq, u_int64_t, p, cmpval, newval); } static __inline u_int64_t ia64_cmpxchg_rel_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) { return IA64_CMPXCHG(8, rel, u_int64_t, p, cmpval, newval); } -/* - * Store with release semantics is used to release locks. - */ -static __inline void -ia64_st_rel_32(volatile u_int32_t* p, u_int32_t v) -{ - __asm __volatile ("st4.rel %0=%1" - : "=m" (*p) - : "r" (v) - : "memory"); +#define ATOMIC_STORE_LOAD(type, width, size) \ +static __inline u_int##width##_t \ +ia64_ld_acq_##width(volatile u_int##width##_t* p) \ +{ \ + u_int##width_t v; \ + \ + __asm __volatile ("ld" size ".acq %0=%1" \ + : "r" (v) \ + : "=m" (*p) \ + : "memory"); \ + return (v); \ +} \ + \ +static __inline u_int##width##_t \ +atomic_load_acq_##width(volatile u_int##width##_t* p) \ +{ \ + u_int##width_t v; \ + \ + __asm __volatile ("ld" size ".acq %0=%1" \ + : "r" (v) \ + : "=m" (*p) \ + : "memory"); \ + return (v); \ +} \ + \ +static __inline u_int##width##_t \ +atomic_load_acq_##type(volatile u_int##width##_t* p) \ +{ \ + u_int##width_t v; \ + \ + __asm __volatile ("ld" size ".acq %0=%1" \ + : "r" (v) \ + : "=m" (*p) \ + : "memory"); \ + return (v); \ +} \ + \ +static __inline void \ +ia64_st_rel_##width(volatile u_int##width##_t* p, u_int##width##_t v)\ +{ \ + __asm __volatile ("st" size ".rel %0=%1" \ + : "=m" (*p) \ + : "r" (v) \ + : "memory"); \ +} \ + \ +static __inline void \ +atomic_store_rel_##width(volatile u_int##width##_t* p, u_int##width##_t v)\ +{ \ + __asm __volatile ("st" size ".rel %0=%1" \ + : "=m" (*p) \ + : "r" (v) \ + : "memory"); \ +} \ + \ +static __inline void \ +atomic_store_rel_##type(volatile u_int##width##_t* p, u_int##width##_t v)\ +{ \ + __asm __volatile ("st" size ".rel %0=%1" \ + : "=m" (*p) \ + : "r" (v) \ + : "memory"); \ } -static __inline void -ia64_st_rel_64(volatile u_int64_t* p, u_int64_t v) -{ - __asm __volatile ("st8.rel %0=%1" - : "=m" (*p) - : "r" (v) - : "memory"); -} +ATOMIC_STORE_LOAD(char, 8, "1") +ATOMIC_STORE_LOAD(short, 16, "2") +ATOMIC_STORE_LOAD(int, 32, "4") +ATOMIC_STORE_LOAD(long, 64, "8") -#define IA64_ATOMIC(sz, type, name, op) \ +#undef ATOMIC_STORE_LOAD + +#define IA64_ATOMIC(sz, type, name, width, op) \ \ static __inline void \ -atomic_##name(volatile type *p, type v) \ +atomic_##name##_acq_##width(volatile type *p, type v) \ { \ type old; \ do { \ old = *p; \ } while (IA64_CMPXCHG(sz, acq, type, p, old, old op v) != old); \ +} \ + \ +static __inline void \ +atomic_##name##_rel_##width(volatile type *p, type v) \ +{ \ + type old; \ + do { \ + old = *p; \ + } while (IA64_CMPXCHG(sz, rel, type, p, old, old op v) != old); \ } -IA64_ATOMIC(1, u_int8_t, set_8, |) -IA64_ATOMIC(2, u_int16_t, set_16, |) -IA64_ATOMIC(4, u_int32_t, set_32, |) -IA64_ATOMIC(8, u_int64_t, set_64, |) +IA64_ATOMIC(1, u_int8_t, set, 8, |) +IA64_ATOMIC(2, u_int16_t, set, 16, |) +IA64_ATOMIC(4, u_int32_t, set, 32, |) +IA64_ATOMIC(8, u_int64_t, set, 64, |) -IA64_ATOMIC(1, u_int8_t, clear_8, &~) -IA64_ATOMIC(2, u_int16_t, clear_16, &~) -IA64_ATOMIC(4, u_int32_t, clear_32, &~) -IA64_ATOMIC(8, u_int64_t, clear_64, &~) +IA64_ATOMIC(1, u_int8_t, clear, 8, &~) +IA64_ATOMIC(2, u_int16_t, clear, 16, &~) +IA64_ATOMIC(4, u_int32_t, clear, 32, &~) +IA64_ATOMIC(8, u_int64_t, clear, 64, &~) -IA64_ATOMIC(1, u_int8_t, add_8, +) -IA64_ATOMIC(2, u_int16_t, add_16, +) -IA64_ATOMIC(4, u_int32_t, add_32, +) -IA64_ATOMIC(8, u_int64_t, add_64, +) +IA64_ATOMIC(1, u_int8_t, add, 8, +) +IA64_ATOMIC(2, u_int16_t, add, 16, +) +IA64_ATOMIC(4, u_int32_t, add, 32, +) +IA64_ATOMIC(8, u_int64_t, add, 64, +) -IA64_ATOMIC(1, u_int8_t, subtract_8, -) -IA64_ATOMIC(2, u_int16_t, subtract_16, -) -IA64_ATOMIC(4, u_int32_t, subtract_32, -) -IA64_ATOMIC(8, u_int64_t, subtract_64, -) +IA64_ATOMIC(1, u_int8_t, subtract, 8, -) +IA64_ATOMIC(2, u_int16_t, subtract, 16, -) +IA64_ATOMIC(4, u_int32_t, subtract, 32, -) +IA64_ATOMIC(8, u_int64_t, subtract, 64, -) #undef IA64_ATOMIC #undef IA64_CMPXCHG -#define atomic_set_char atomic_set_8 -#define atomic_clear_char atomic_clear_8 -#define atomic_add_char atomic_add_8 -#define atomic_subtract_char atomic_subtract_8 +#define atomic_set_8 atomic_set_acq_8 +#define atomic_clear_8 atomic_clear_acq_8 +#define atomic_add_8 atomic_add_acq_8 +#define atomic_subtract_8 atomic_subtract_acq_8 -#define atomic_set_short atomic_set_16 -#define atomic_clear_short atomic_clear_16 -#define atomic_add_short atomic_add_16 -#define atomic_subtract_short atomic_subtract_16 +#define atomic_set_16 atomic_set_acq_16 +#define atomic_clear_16 atomic_clear_acq_16 +#define atomic_add_16 atomic_add_acq_16 +#define atomic_subtract_16 atomic_subtract_acq_16 -#define atomic_set_int atomic_set_32 -#define atomic_clear_int atomic_clear_32 -#define atomic_add_int atomic_add_32 -#define atomic_subtract_int atomic_subtract_32 +#define atomic_set_32 atomic_set_acq_32 +#define atomic_clear_32 atomic_clear_acq_32 +#define atomic_add_32 atomic_add_acq_32 +#define atomic_subtract_32 atomic_subtract_acq_32 -#define atomic_set_long atomic_set_64 -#define atomic_clear_long atomic_clear_64 -#define atomic_add_long atomic_add_64 -#define atomic_subtract_long atomic_subtract_64 +#define atomic_set_64 atomic_set_acq_64 +#define atomic_clear_64 atomic_clear_acq_64 +#define atomic_add_64 atomic_add_acq_64 +#define atomic_subtract_64 atomic_subtract_acq_64 +#define atomic_set_char atomic_set_8 +#define atomic_clear_char atomic_clear_8 +#define atomic_add_char atomic_add_8 +#define atomic_subtract_char atomic_subtract_8 +#define atomic_set_acq_char atomic_set_acq_8 +#define atomic_clear_acq_char atomic_clear_acq_8 +#define atomic_add_acq_char atomic_add_acq_8 +#define atomic_subtract_acq_char atomic_subtract_acq_8 +#define atomic_set_rel_char atomic_set_rel_8 +#define atomic_clear_rel_char atomic_clear_rel_8 +#define atomic_add_rel_char atomic_add_rel_8 +#define atomic_subtract_rel_char atomic_subtract_rel_8 + +#define atomic_set_short atomic_set_16 +#define atomic_clear_short atomic_clear_16 +#define atomic_add_short atomic_add_16 +#define atomic_subtract_short atomic_subtract_16 +#define atomic_set_acq_short atomic_set_acq_16 +#define atomic_clear_acq_short atomic_clear_acq_16 +#define atomic_add_acq_short atomic_add_acq_16 +#define atomic_subtract_acq_short atomic_subtract_acq_16 +#define atomic_set_rel_short atomic_set_rel_16 +#define atomic_clear_rel_short atomic_clear_rel_16 +#define atomic_add_rel_short atomic_add_rel_16 +#define atomic_subtract_rel_short atomic_subtract_rel_16 + +#define atomic_set_int atomic_set_32 +#define atomic_clear_int atomic_clear_32 +#define atomic_add_int atomic_add_32 +#define atomic_subtract_int atomic_subtract_32 +#define atomic_set_acq_int atomic_set_acq_32 +#define atomic_clear_acq_int atomic_clear_acq_32 +#define atomic_add_acq_int atomic_add_acq_32 +#define atomic_subtract_acq_int atomic_subtract_acq_32 +#define atomic_set_rel_int atomic_set_rel_32 +#define atomic_clear_rel_int atomic_clear_rel_32 +#define atomic_add_rel_int atomic_add_rel_32 +#define atomic_subtract_rel_int atomic_subtract_rel_32 + +#define atomic_set_long atomic_set_64 +#define atomic_clear_long atomic_clear_64 +#define atomic_add_long atomic_add_64 +#define atomic_subtract_long atomic_subtract_64 +#define atomic_set_acq_long atomic_set_acq_64 +#define atomic_clear_acq_long atomic_clear_acq_64 +#define atomic_add_acq_long atomic_add_acq_64 +#define atomic_subtract_acq_long atomic_subtract_acq_64 +#define atomic_set_rel_long atomic_set_rel_64 +#define atomic_clear_rel_long atomic_clear_rel_64 +#define atomic_add_rel_long atomic_add_rel_64 +#define atomic_subtract_rel_long atomic_subtract_rel_64 + /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline int -atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) +atomic_cmpset_acq_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) { return ia64_cmpxchg_acq_32(p, cmpval, newval) == cmpval; } +static __inline int +atomic_cmpset_rel_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) +{ + return ia64_cmpxchg_rel_32(p, cmpval, newval) == cmpval; +} + /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline int -atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) +atomic_cmpset_acq_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) { return ia64_cmpxchg_acq_64(p, cmpval, newval) == cmpval; } +static __inline int +atomic_cmpset_rel_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) +{ + return ia64_cmpxchg_rel_64(p, cmpval, newval) == cmpval; +} + +#define atomic_cmpset_32 atomic_cmpset_acq_32 +#define atomic_cmpset_64 atomic_cmpset_acq_64 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_long atomic_cmpset_64 +#define atomic_cmpset_acq_int atomic_cmpset_acq_32 +#define atomic_cmpset_rel_int atomic_cmpset_rel_32 +#define atomic_cmpset_acq_long atomic_cmpset_acq_64 +#define atomic_cmpset_rel_long atomic_cmpset_rel_64 static __inline int -atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) +atomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src) { - return atomic_cmpset_long((volatile u_long *)dst, - (u_long)exp, (u_long)src); + return atomic_cmpset_acq_long((volatile u_long *)dst, + (u_long)exp, (u_long)src); +} + +static __inline int +atomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src) +{ + return atomic_cmpset_rel_long((volatile u_long *)dst, + (u_long)exp, (u_long)src); +} + +#define atomic_cmpset_ptr atomic_cmpset_acq_ptr + +static __inline void * +atomic_load_acq_ptr(volatile void *p) +{ + return (void *)atomic_load_acq_long((volatile u_long *)p); +} + +static __inline void +atomic_store_rel_ptr(volatile void *p, void *v) +{ + atomic_store_rel_long((volatile u_long *)p, (u_long)v); } static __inline u_int32_t atomic_readandclear_32(volatile u_int32_t* p) { u_int32_t val; do { val = *p; } while (!atomic_cmpset_32(p, val, 0)); return val; } static __inline u_int64_t atomic_readandclear_64(volatile u_int64_t* p) { u_int64_t val; do { val = *p; } while (!atomic_cmpset_64(p, val, 0)); return val; } #define atomic_readandclear_int atomic_readandclear_32 #define atomic_readandclear_long atomic_readandclear_64 #endif /* ! _MACHINE_ATOMIC_H_ */