Index: stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c =================================================================== --- stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c (revision 353880) +++ stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c (revision 353881) @@ -1,138 +1,161 @@ /*- * Copyright (c) 2007 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #ifdef _KERNEL #include struct mtx atomic_mtx; MTX_SYSINIT(atomic, &atomic_mtx, "atomic", MTX_DEF); #else #include #define mtx_lock(lock) pthread_mutex_lock(lock) #define mtx_unlock(lock) pthread_mutex_unlock(lock) static pthread_mutex_t atomic_mtx; static __attribute__((constructor)) void atomic_init(void) { pthread_mutex_init(&atomic_mtx, NULL); } #endif #if !defined(__LP64__) && !defined(__mips_n32) && \ !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) void atomic_add_64(volatile uint64_t *target, int64_t delta) { mtx_lock(&atomic_mtx); *target += delta; mtx_unlock(&atomic_mtx); } void atomic_dec_64(volatile uint64_t *target) { mtx_lock(&atomic_mtx); *target -= 1; mtx_unlock(&atomic_mtx); } + +uint64_t +atomic_swap_64(volatile uint64_t *a, uint64_t value) +{ + uint64_t ret; + + mtx_lock(&atomic_mtx); + ret = *a; + *a = value; + mtx_unlock(&atomic_mtx); + return (ret); +} + +uint64_t +atomic_load_64(volatile uint64_t *a) +{ + uint64_t ret; + + mtx_lock(&atomic_mtx); + ret = *a; + mtx_unlock(&atomic_mtx); + return (ret); +} #endif uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta) { uint64_t newval; mtx_lock(&atomic_mtx); newval = (*target += delta); mtx_unlock(&atomic_mtx); return (newval); } #if defined(__powerpc__) || defined(__arm__) || defined(__mips__) void atomic_or_8(volatile uint8_t *target, uint8_t value) { mtx_lock(&atomic_mtx); *target |= value; mtx_unlock(&atomic_mtx); } #endif uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value) { uint8_t newval; mtx_lock(&atomic_mtx); newval = (*target |= value); mtx_unlock(&atomic_mtx); return (newval); } uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) { uint64_t oldval; mtx_lock(&atomic_mtx); oldval = *target; if (oldval == cmp) *target = newval; mtx_unlock(&atomic_mtx); return (oldval); } uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) { uint32_t oldval; mtx_lock(&atomic_mtx); oldval = *target; if (oldval == cmp) *target = newval; mtx_unlock(&atomic_mtx); return (oldval); } void membar_producer(void) { /* nothing */ } Index: stable/12/sys/cddl/compat/opensolaris/sys/atomic.h =================================================================== --- stable/12/sys/cddl/compat/opensolaris/sys/atomic.h (revision 353880) +++ stable/12/sys/cddl/compat/opensolaris/sys/atomic.h (revision 353881) @@ -1,148 +1,150 @@ /*- * Copyright (c) 2007 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _OPENSOLARIS_SYS_ATOMIC_H_ #define _OPENSOLARIS_SYS_ATOMIC_H_ #include #include #define casptr(_a, _b, _c) \ atomic_cmpset_ptr((volatile uintptr_t *)(_a), (uintptr_t)(_b), (uintptr_t) (_c)) #define cas32 atomic_cmpset_32 #if defined(__i386__) && (defined(_KERNEL) || defined(KLD_MODULE)) #define I386_HAVE_ATOMIC64 #endif #if !defined(__LP64__) && !defined(__mips_n32) && \ !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) extern void atomic_add_64(volatile uint64_t *target, int64_t delta); extern void atomic_dec_64(volatile uint64_t *target); +extern uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value); +extern uint64_t atomic_load_64(volatile uint64_t *a); #endif #ifndef __sparc64__ extern uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval); extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval); #endif extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta); extern uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value); extern void membar_producer(void); #if defined(__sparc64__) || defined(__powerpc__) || defined(__arm__) || \ defined(__mips__) || defined(__aarch64__) || defined(__riscv) extern void atomic_or_8(volatile uint8_t *target, uint8_t value); #else static __inline void atomic_or_8(volatile uint8_t *target, uint8_t value) { atomic_set_8(target, value); } #endif static __inline uint32_t atomic_add_32_nv(volatile uint32_t *target, int32_t delta) { return (atomic_fetchadd_32(target, delta) + delta); } static __inline u_int atomic_add_int_nv(volatile u_int *target, int delta) { return (atomic_add_32_nv(target, delta)); } static __inline void atomic_dec_32(volatile uint32_t *target) { atomic_subtract_32(target, 1); } static __inline uint32_t atomic_dec_32_nv(volatile uint32_t *target) { return (atomic_fetchadd_32(target, -1) - 1); } #if defined(__LP64__) || defined(__mips_n32) || \ defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) static __inline void atomic_dec_64(volatile uint64_t *target) { atomic_subtract_64(target, 1); } #endif static __inline void atomic_inc_32(volatile uint32_t *target) { atomic_add_32(target, 1); } static __inline uint32_t atomic_inc_32_nv(volatile uint32_t *target) { return (atomic_add_32_nv(target, 1)); } static __inline void atomic_inc_64(volatile uint64_t *target) { atomic_add_64(target, 1); } static __inline uint64_t atomic_inc_64_nv(volatile uint64_t *target) { return (atomic_add_64_nv(target, 1)); } static __inline uint64_t atomic_dec_64_nv(volatile uint64_t *target) { return (atomic_add_64_nv(target, -1)); } #if !defined(COMPAT_32BIT) && defined(__LP64__) static __inline void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return ((void *)atomic_cas_64((volatile uint64_t *)target, (uint64_t)cmp, (uint64_t)newval)); } #else static __inline void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return ((void *)atomic_cas_32((volatile uint32_t *)target, (uint32_t)cmp, (uint32_t)newval)); } #endif /* !defined(COMPAT_32BIT) && defined(__LP64__) */ #endif /* !_OPENSOLARIS_SYS_ATOMIC_H_ */ Index: stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S =================================================================== --- stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S (revision 353880) +++ stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S (revision 353881) @@ -1,133 +1,161 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ .file "atomic.s" #define _ASM #include /* * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever * separated, it is important to edit the libc i386 platform * specific mapfile and remove the NODYNSORT attribute * from atomic_dec_64_nv. */ ENTRY(atomic_dec_64) ALTENTRY(atomic_dec_64_nv) pushl %edi pushl %ebx movl 12(%esp), %edi // %edi = target address movl (%edi), %eax movl 4(%edi), %edx // %edx:%eax = old value 1: xorl %ebx, %ebx xorl %ecx, %ecx not %ecx not %ebx // %ecx:%ebx = -1 addl %eax, %ebx adcl %edx, %ecx // add in the carry from inc lock cmpxchg8b (%edi) // try to stick it in jne 1b movl %ebx, %eax movl %ecx, %edx // return new value popl %ebx popl %edi ret SET_SIZE(atomic_dec_64_nv) SET_SIZE(atomic_dec_64) /* * NOTE: If atomic_add_64 and atomic_add_64_nv are ever * separated, it is important to edit the libc i386 platform * specific mapfile and remove the NODYNSORT attribute * from atomic_add_64_nv. */ ENTRY(atomic_add_64) ALTENTRY(atomic_add_64_nv) pushl %edi pushl %ebx movl 12(%esp), %edi // %edi = target address movl (%edi), %eax movl 4(%edi), %edx // %edx:%eax = old value 1: movl 16(%esp), %ebx movl 20(%esp), %ecx // %ecx:%ebx = delta addl %eax, %ebx adcl %edx, %ecx // %ecx:%ebx = new value lock cmpxchg8b (%edi) // try to stick it in jne 1b movl %ebx, %eax movl %ecx, %edx // return new value popl %ebx popl %edi ret SET_SIZE(atomic_add_64_nv) SET_SIZE(atomic_add_64) ENTRY(atomic_or_8_nv) movl 4(%esp), %edx // %edx = target address movb (%edx), %al // %al = old value 1: movl 8(%esp), %ecx // %ecx = delta orb %al, %cl // %cl = new value lock cmpxchgb %cl, (%edx) // try to stick it in jne 1b movzbl %cl, %eax // return new value ret SET_SIZE(atomic_or_8_nv) ENTRY(atomic_cas_32) movl 4(%esp), %edx movl 8(%esp), %eax movl 12(%esp), %ecx lock cmpxchgl %ecx, (%edx) ret SET_SIZE(atomic_cas_32) ENTRY(atomic_cas_64) pushl %ebx pushl %esi movl 12(%esp), %esi movl 16(%esp), %eax movl 20(%esp), %edx movl 24(%esp), %ebx movl 28(%esp), %ecx lock cmpxchg8b (%esi) popl %esi popl %ebx ret SET_SIZE(atomic_cas_64) + ENTRY(atomic_swap_64) + pushl %esi + pushl %ebx + movl 12(%esp), %esi + movl 16(%esp), %ebx + movl 20(%esp), %ecx + movl (%esi), %eax + movl 4(%esi), %edx // %edx:%eax = old value +1: + lock + cmpxchg8b (%esi) + jne 1b + popl %ebx + popl %esi + ret + SET_SIZE(atomic_swap_64) + + ENTRY(atomic_load_64) + pushl %esi + movl 8(%esp), %esi + movl %ebx, %eax // make old and new values equal, so that + movl %ecx, %edx // destination is never changed + lock + cmpxchg8b (%esi) + popl %esi + ret + SET_SIZE(atomic_load_64) + ENTRY(membar_producer) lock xorl $0, (%esp) ret SET_SIZE(membar_producer) Index: stable/12 =================================================================== --- stable/12 (revision 353880) +++ stable/12 (revision 353881) Property changes on: stable/12 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r353167,353270