Index: head/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c =================================================================== --- head/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c (revision 353339) +++ head/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c (revision 353340) @@ -1,161 +1,128 @@ /*- * Copyright (c) 2007 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include +#if !defined(__LP64__) && !defined(__mips_n32) && \ + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) + #ifdef _KERNEL #include struct mtx atomic_mtx; MTX_SYSINIT(atomic, &atomic_mtx, "atomic", MTX_DEF); #else #include #define mtx_lock(lock) pthread_mutex_lock(lock) #define mtx_unlock(lock) pthread_mutex_unlock(lock) static pthread_mutex_t atomic_mtx; static __attribute__((constructor)) void atomic_init(void) { pthread_mutex_init(&atomic_mtx, NULL); } #endif -#if !defined(__LP64__) && !defined(__mips_n32) && \ - !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) void atomic_add_64(volatile uint64_t *target, int64_t delta) { mtx_lock(&atomic_mtx); *target += delta; mtx_unlock(&atomic_mtx); } void atomic_dec_64(volatile uint64_t *target) { mtx_lock(&atomic_mtx); *target -= 1; mtx_unlock(&atomic_mtx); } uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value) { uint64_t ret; mtx_lock(&atomic_mtx); ret = *a; *a = value; mtx_unlock(&atomic_mtx); return (ret); } uint64_t atomic_load_64(volatile uint64_t *a) { uint64_t ret; mtx_lock(&atomic_mtx); ret = *a; mtx_unlock(&atomic_mtx); return (ret); } -#endif uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta) { uint64_t newval; mtx_lock(&atomic_mtx); newval = (*target += delta); mtx_unlock(&atomic_mtx); return (newval); } -#if defined(__powerpc__) || defined(__arm__) || defined(__mips__) -void -atomic_or_8(volatile uint8_t *target, uint8_t value) -{ - mtx_lock(&atomic_mtx); - *target |= value; - mtx_unlock(&atomic_mtx); -} -#endif - -uint8_t -atomic_or_8_nv(volatile uint8_t *target, uint8_t value) -{ - uint8_t newval; - - mtx_lock(&atomic_mtx); - newval = (*target |= value); - mtx_unlock(&atomic_mtx); - return (newval); -} - uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) { uint64_t oldval; mtx_lock(&atomic_mtx); oldval = *target; if (oldval == cmp) *target = newval; mtx_unlock(&atomic_mtx); return (oldval); } - -uint32_t -atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) -{ - uint32_t oldval; - - mtx_lock(&atomic_mtx); - oldval = *target; - if (oldval == cmp) - *target = newval; - mtx_unlock(&atomic_mtx); - return (oldval); -} +#endif void membar_producer(void) { /* nothing */ } Index: head/sys/cddl/compat/opensolaris/sys/atomic.h =================================================================== --- head/sys/cddl/compat/opensolaris/sys/atomic.h (revision 353339) +++ head/sys/cddl/compat/opensolaris/sys/atomic.h (revision 353340) @@ -1,150 +1,155 @@ /*- * Copyright (c) 2007 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _OPENSOLARIS_SYS_ATOMIC_H_ #define _OPENSOLARIS_SYS_ATOMIC_H_ #include #include -#define casptr(_a, _b, _c) \ - atomic_cmpset_ptr((volatile uintptr_t *)(_a), (uintptr_t)(_b), (uintptr_t) (_c)) -#define cas32 atomic_cmpset_32 - #if defined(__i386__) && (defined(_KERNEL) || defined(KLD_MODULE)) #define I386_HAVE_ATOMIC64 #endif #if !defined(__LP64__) && !defined(__mips_n32) && \ !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) extern void atomic_add_64(volatile uint64_t *target, int64_t delta); extern void atomic_dec_64(volatile uint64_t *target); extern uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value); extern uint64_t atomic_load_64(volatile uint64_t *a); -#endif -#ifndef __sparc64__ -extern uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp, - uint32_t newval); +extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta); extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval); #endif -extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta); -extern uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value); + extern void membar_producer(void); -#if defined(__sparc64__) || defined(__powerpc__) || defined(__arm__) || \ - defined(__mips__) || defined(__aarch64__) || defined(__riscv) -extern void atomic_or_8(volatile uint8_t *target, uint8_t value); -#else -static __inline void -atomic_or_8(volatile uint8_t *target, uint8_t value) -{ - atomic_set_8(target, value); -} -#endif - static __inline uint32_t atomic_add_32_nv(volatile uint32_t *target, int32_t delta) { return (atomic_fetchadd_32(target, delta) + delta); } static __inline u_int atomic_add_int_nv(volatile u_int *target, int delta) { return (atomic_add_32_nv(target, delta)); } static __inline void +atomic_inc_32(volatile uint32_t *target) +{ + atomic_add_32(target, 1); +} + +static __inline uint32_t +atomic_inc_32_nv(volatile uint32_t *target) +{ + return (atomic_add_32_nv(target, 1)); +} + +static __inline void atomic_dec_32(volatile uint32_t *target) { atomic_subtract_32(target, 1); } static __inline uint32_t atomic_dec_32_nv(volatile uint32_t *target) { - return (atomic_fetchadd_32(target, -1) - 1); + return (atomic_add_32_nv(target, -1)); } +#ifndef __sparc64__ +static inline uint32_t +atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) +{ + (void)atomic_fcmpset_32(target, &cmp, newval); + return (cmp); +} +#endif + #if defined(__LP64__) || defined(__mips_n32) || \ defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) static __inline void atomic_dec_64(volatile uint64_t *target) { atomic_subtract_64(target, 1); } -#endif -static __inline void -atomic_inc_32(volatile uint32_t *target) +static inline uint64_t +atomic_add_64_nv(volatile uint64_t *target, int64_t delta) { - atomic_add_32(target, 1); + return (atomic_fetchadd_64(target, delta) + delta); } -static __inline uint32_t -atomic_inc_32_nv(volatile uint32_t *target) +#ifndef __sparc64__ +static inline uint64_t +atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) { - return (atomic_add_32_nv(target, 1)); + (void)atomic_fcmpset_64(target, &cmp, newval); + return (cmp); } +#endif +#endif static __inline void atomic_inc_64(volatile uint64_t *target) { atomic_add_64(target, 1); } static __inline uint64_t atomic_inc_64_nv(volatile uint64_t *target) { return (atomic_add_64_nv(target, 1)); } static __inline uint64_t atomic_dec_64_nv(volatile uint64_t *target) { return (atomic_add_64_nv(target, -1)); } #if !defined(COMPAT_32BIT) && defined(__LP64__) static __inline void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return ((void *)atomic_cas_64((volatile uint64_t *)target, (uint64_t)cmp, (uint64_t)newval)); } #else static __inline void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return ((void *)atomic_cas_32((volatile uint32_t *)target, (uint32_t)cmp, (uint32_t)newval)); } #endif /* !defined(COMPAT_32BIT) && defined(__LP64__) */ #endif /* !_OPENSOLARIS_SYS_ATOMIC_H_ */ Index: head/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S =================================================================== --- head/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S (revision 353339) +++ head/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S (revision 353340) @@ -1,87 +1,35 @@ /*- * Copyright (C) 2015 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include -/* - * uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta) - */ -ENTRY(atomic_add_64_nv) -1: ldxr x2, [x0] /* Load *target */ - add x2, x2, x1 /* x2 = x2 + delta */ - stxr w3, x2, [x0] /* Store *target */ - cbnz w3, 1b /* Check if the store succeeded */ - mov x0, x2 /* Return the new value */ - ret -END(atomic_add_64_nv) - -/* - * uint32_t - * atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) - */ -ENTRY(atomic_cas_32) -1: ldxr w3, [x0] /* Load *target */ - cmp w3, w1 /* Does *targe == cmp? */ - b.ne 2f /* If not exit */ - stxr w4, w2, [x0] /* Store newval to *target */ - cbnz w4, 1b /* Check if the store succeeded */ -2: mov w0, w3 /* Return the old value */ - ret -END(atomic_cas_32) - -/* - * uint64_t - * atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) - */ -ENTRY(atomic_cas_64) -1: ldxr x3, [x0] /* Load *target */ - cmp x3, x1 /* Does *targe == cmp? */ - b.ne 2f /* If not exit */ - stxr w4, x2, [x0] /* Store newval to *target */ - cbnz w4, 1b /* Check if the store succeeded */ -2: mov x0, x3 /* Return the old value */ - ret -END(atomic_cas_64) - -/* - * uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value) - */ -ENTRY(atomic_or_8_nv) -1: ldxrb w2, [x0] /* Load *target */ - orr w2, w2, w1 /* x2 = x2 | delta */ - stxrb w3, w2, [x0] /* Store *target */ - cbnz w3, 1b /* Check if the store succeeded */ - mov w0, w2 /* Return the new value */ - ret -END(atomic_or_8_nv) - ENTRY(membar_producer) dmb ish ret END(membar_producer) Index: head/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S =================================================================== --- head/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S (revision 353339) +++ head/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S (revision 353340) @@ -1,68 +1,34 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. */ .file "atomic.s" #define _ASM #include - ENTRY(atomic_add_64_nv) - mov %rsi, %rax // %rax = delta addend - lock - xaddq %rsi, (%rdi) // %rsi = old value, (%rdi) = sum - addq %rsi, %rax // new value = original value + delta - ret - SET_SIZE(atomic_add_64_nv) - - ENTRY(atomic_or_8_nv) - movb (%rdi), %al // %al = old value -1: - movb %sil, %cl - orb %al, %cl // %cl = new value - lock - cmpxchgb %cl, (%rdi) // try to stick it in - jne 1b - movzbl %cl, %eax // return new value - ret - SET_SIZE(atomic_or_8_nv) - - ENTRY(atomic_cas_32) - movl %esi, %eax - lock - cmpxchgl %edx, (%rdi) - ret - SET_SIZE(atomic_cas_32) - - ENTRY(atomic_cas_64) - movq %rsi, %rax - lock - cmpxchgq %rdx, (%rdi) - ret - SET_SIZE(atomic_cas_64) - ENTRY(membar_producer) sfence ret SET_SIZE(membar_producer) Index: head/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S =================================================================== --- head/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S (revision 353339) +++ head/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S (revision 353340) @@ -1,161 +1,139 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ .file "atomic.s" #define _ASM #include /* * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever * separated, it is important to edit the libc i386 platform * specific mapfile and remove the NODYNSORT attribute * from atomic_dec_64_nv. */ ENTRY(atomic_dec_64) ALTENTRY(atomic_dec_64_nv) pushl %edi pushl %ebx movl 12(%esp), %edi // %edi = target address movl (%edi), %eax movl 4(%edi), %edx // %edx:%eax = old value 1: xorl %ebx, %ebx xorl %ecx, %ecx not %ecx not %ebx // %ecx:%ebx = -1 addl %eax, %ebx adcl %edx, %ecx // add in the carry from inc lock cmpxchg8b (%edi) // try to stick it in jne 1b movl %ebx, %eax movl %ecx, %edx // return new value popl %ebx popl %edi ret SET_SIZE(atomic_dec_64_nv) SET_SIZE(atomic_dec_64) /* * NOTE: If atomic_add_64 and atomic_add_64_nv are ever * separated, it is important to edit the libc i386 platform * specific mapfile and remove the NODYNSORT attribute * from atomic_add_64_nv. */ ENTRY(atomic_add_64) ALTENTRY(atomic_add_64_nv) pushl %edi pushl %ebx movl 12(%esp), %edi // %edi = target address movl (%edi), %eax movl 4(%edi), %edx // %edx:%eax = old value 1: movl 16(%esp), %ebx movl 20(%esp), %ecx // %ecx:%ebx = delta addl %eax, %ebx adcl %edx, %ecx // %ecx:%ebx = new value lock cmpxchg8b (%edi) // try to stick it in jne 1b movl %ebx, %eax movl %ecx, %edx // return new value popl %ebx popl %edi ret SET_SIZE(atomic_add_64_nv) SET_SIZE(atomic_add_64) - ENTRY(atomic_or_8_nv) - movl 4(%esp), %edx // %edx = target address - movb (%edx), %al // %al = old value -1: - movl 8(%esp), %ecx // %ecx = delta - orb %al, %cl // %cl = new value - lock - cmpxchgb %cl, (%edx) // try to stick it in - jne 1b - movzbl %cl, %eax // return new value - ret - SET_SIZE(atomic_or_8_nv) - - ENTRY(atomic_cas_32) - movl 4(%esp), %edx - movl 8(%esp), %eax - movl 12(%esp), %ecx - lock - cmpxchgl %ecx, (%edx) - ret - SET_SIZE(atomic_cas_32) - ENTRY(atomic_cas_64) pushl %ebx pushl %esi movl 12(%esp), %esi movl 16(%esp), %eax movl 20(%esp), %edx movl 24(%esp), %ebx movl 28(%esp), %ecx lock cmpxchg8b (%esi) popl %esi popl %ebx ret SET_SIZE(atomic_cas_64) ENTRY(atomic_swap_64) pushl %esi pushl %ebx movl 12(%esp), %esi movl 16(%esp), %ebx movl 20(%esp), %ecx movl (%esi), %eax movl 4(%esi), %edx // %edx:%eax = old value 1: lock cmpxchg8b (%esi) jne 1b popl %ebx popl %esi ret SET_SIZE(atomic_swap_64) ENTRY(atomic_load_64) pushl %esi movl 8(%esp), %esi movl %ebx, %eax // make old and new values equal, so that movl %ecx, %edx // destination is never changed lock cmpxchg8b (%esi) popl %esi ret SET_SIZE(atomic_load_64) ENTRY(membar_producer) lock xorl $0, (%esp) ret SET_SIZE(membar_producer) Index: head/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S =================================================================== --- head/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S (revision 353339) +++ head/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S (revision 353340) @@ -1,88 +1,33 @@ /*- * Copyright (C) 2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #include -ENTRY(atomic_add_64_nv) - 1: ldarx %r5,0,%r3 - add %r5,%r4,%r5 - stdcx. %r5,0,%r3 - bne- 1b - - mr %r3,%r5 - blr - -ENTRY(atomic_cas_32) - 1: lwarx %r6,0,%r3 - cmplw %r6,%r4 - bne 2f - stwcx. %r5,0,%r3 - bne- 1b - b 3f - - 2: stwcx. %r6,0,%r3 /* clear reservation */ - - 3: mr %r3,%r6 - blr - -ENTRY(atomic_cas_64) - 1: ldarx %r6,0,%r3 - cmpld %r6,%r4 - bne 2f - stdcx. %r5,0,%r3 - bne- 1b - b 3f - - 2: stdcx. %r6,0,%r3 /* clear reservation */ - - 3: mr %r3,%r6 - blr - -ENTRY(atomic_or_8_nv) - li %r6,3 - andc. %r6,%r3,%r6 /* r6 = r3 & ~4 */ - addi %r7,%r6,3 - sub %r7,%r7,%r3 /* offset in r7 */ - sldi %r7,%r7,3 /* bits to shift in r7 */ - - rlwinm %r4,%r4,0,24,31 /* mask and rotate the argument */ - slw %r4,%r4,%r7 - - 1: lwarx %r5,0,%r6 - or %r5,%r4,%r5 - stwcx. %r5,0,%r6 - bne- 1b - - srw %r3,%r5,%r7 - rlwinm %r3,%r3,0,24,31 /* mask return value */ - - blr - ENTRY(membar_producer) eieio blr Index: head/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S =================================================================== --- head/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S (revision 353339) +++ head/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S (revision 353340) @@ -1,113 +1,52 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ .ident "%Z%%M% %I% %E% SMI" .file "%M%" #define _ASM #include #include /* Userland needs different ASIs. */ #ifdef _KERNEL #define __ASI_ATOMIC ASI_N #else #define __ASI_ATOMIC ASI_P #endif /* - * NOTE: If atomic_add_64 and atomic_add_64_nv are ever - * separated, you need to also edit the libc sparcv9 platform - * specific mapfile and remove the NODYNSORT attribute - * from atomic_add_64_nv. - */ - ENTRY(atomic_add_64) - ALTENTRY(atomic_add_64_nv) - ALTENTRY(atomic_add_ptr) - ALTENTRY(atomic_add_ptr_nv) - ALTENTRY(atomic_add_long) - ALTENTRY(atomic_add_long_nv) -add_64: - ldx [%o0], %o2 -1: - add %o2, %o1, %o3 - casxa [%o0] __ASI_ATOMIC, %o2, %o3 - cmp %o2, %o3 - bne,a,pn %xcc, 1b - mov %o3, %o2 - retl - add %o2, %o1, %o0 ! return new value - SET_SIZE(atomic_add_long_nv) - SET_SIZE(atomic_add_long) - SET_SIZE(atomic_add_ptr_nv) - SET_SIZE(atomic_add_ptr) - SET_SIZE(atomic_add_64_nv) - SET_SIZE(atomic_add_64) - - /* - * NOTE: If atomic_or_8 and atomic_or_8_nv are ever - * separated, you need to also edit the libc sparcv9 platform - * specific mapfile and remove the NODYNSORT attribute - * from atomic_or_8_nv. - */ - ENTRY(atomic_or_8) - ALTENTRY(atomic_or_8_nv) - ALTENTRY(atomic_or_uchar) - and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right - xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left - sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left - set 0xff, %o3 ! %o3 = mask - sll %o3, %g1, %o3 ! %o3 = shifted to bit offset - sll %o1, %g1, %o1 ! %o1 = shifted to bit offset - and %o1, %o3, %o1 ! %o1 = single byte value - andn %o0, 0x3, %o0 ! %o0 = word address - ld [%o0], %o2 ! read old value -1: - or %o2, %o1, %o5 ! or in the new value - casa [%o0] __ASI_ATOMIC, %o2, %o5 - cmp %o2, %o5 - bne,a,pn %icc, 1b - mov %o5, %o2 ! %o2 = old value - or %o2, %o1, %o5 - and %o5, %o3, %o5 - retl - srl %o5, %g1, %o0 ! %o0 = new value - SET_SIZE(atomic_or_uchar) - SET_SIZE(atomic_or_8_nv) - SET_SIZE(atomic_or_8) - - /* * Spitfires and Blackbirds have a problem with membars in the * delay slot (SF_ERRATA_51). For safety's sake, we assume * that the whole world needs the workaround. */ ENTRY(membar_producer) membar #StoreStore retl nop SET_SIZE(membar_producer)