diff --git a/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c b/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c index 89cd1db46742..99a79b306580 100644 --- a/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c +++ b/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c @@ -1,122 +1,123 @@ /*- * Copyright (c) 2007 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #if !defined(__LP64__) && !defined(__mips_n32) && \ - !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) && \ + !defined(HAS_EMULATED_ATOMIC64) #ifdef _KERNEL #include struct mtx atomic_mtx; MTX_SYSINIT(atomic, &atomic_mtx, "atomic", MTX_DEF); #else #include #define mtx_lock(lock) pthread_mutex_lock(lock) #define mtx_unlock(lock) pthread_mutex_unlock(lock) static pthread_mutex_t atomic_mtx; static __attribute__((constructor)) void atomic_init(void) { pthread_mutex_init(&atomic_mtx, NULL); } #endif void atomic_add_64(volatile uint64_t *target, int64_t delta) { mtx_lock(&atomic_mtx); *target += delta; mtx_unlock(&atomic_mtx); } void atomic_dec_64(volatile uint64_t *target) { mtx_lock(&atomic_mtx); *target -= 1; mtx_unlock(&atomic_mtx); } uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value) { uint64_t ret; mtx_lock(&atomic_mtx); ret = *a; *a = value; mtx_unlock(&atomic_mtx); return (ret); } uint64_t atomic_load_64(volatile uint64_t *a) { uint64_t ret; mtx_lock(&atomic_mtx); ret = *a; mtx_unlock(&atomic_mtx); return (ret); } uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta) { uint64_t newval; mtx_lock(&atomic_mtx); newval = (*target += delta); mtx_unlock(&atomic_mtx); return (newval); } uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) { uint64_t oldval; mtx_lock(&atomic_mtx); oldval = *target; if (oldval == cmp) *target = newval; mtx_unlock(&atomic_mtx); return (oldval); } #endif diff --git a/sys/cddl/compat/opensolaris/sys/atomic.h b/sys/cddl/compat/opensolaris/sys/atomic.h index 9e30f3da6669..b3a2f393afa4 100644 --- a/sys/cddl/compat/opensolaris/sys/atomic.h +++ b/sys/cddl/compat/opensolaris/sys/atomic.h @@ -1,178 +1,180 @@ /*- * Copyright (c) 2007 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _OPENSOLARIS_SYS_ATOMIC_H_ #define _OPENSOLARIS_SYS_ATOMIC_H_ #include #include #if defined(__i386__) && (defined(_KERNEL) || defined(KLD_MODULE)) #define I386_HAVE_ATOMIC64 #endif #if defined(__i386__) || defined(__amd64__) || defined(__arm__) /* No spurious failures from fcmpset. */ #define STRONG_FCMPSET #endif #if !defined(__LP64__) && !defined(__mips_n32) && \ - !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) && \ + !defined(HAS_EMULATED_ATOMIC64) extern void atomic_add_64(volatile uint64_t *target, int64_t delta); extern void atomic_dec_64(volatile uint64_t *target); extern uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value); extern uint64_t atomic_load_64(volatile uint64_t *a); extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta); extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval); #endif #define membar_producer atomic_thread_fence_rel static __inline uint32_t atomic_add_32_nv(volatile uint32_t *target, int32_t delta) { return (atomic_fetchadd_32(target, delta) + delta); } static __inline u_int atomic_add_int_nv(volatile u_int *target, int delta) { return (atomic_add_32_nv(target, delta)); } static __inline void atomic_inc_32(volatile uint32_t *target) { atomic_add_32(target, 1); } static __inline uint32_t atomic_inc_32_nv(volatile uint32_t *target) { return (atomic_add_32_nv(target, 1)); } static __inline void atomic_dec_32(volatile uint32_t *target) { atomic_subtract_32(target, 1); } static __inline uint32_t atomic_dec_32_nv(volatile uint32_t *target) { return (atomic_add_32_nv(target, -1)); } #ifndef __sparc64__ static inline uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) { #ifdef STRONG_FCMPSET (void)atomic_fcmpset_32(target, &cmp, newval); #else uint32_t expected = cmp; do { if (atomic_fcmpset_32(target, &cmp, newval)) break; } while (cmp == expected); #endif return (cmp); } #endif #if defined(__LP64__) || defined(__mips_n32) || \ - defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) + defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) || \ + defined(HAS_EMULATED_ATOMIC64) static __inline void atomic_dec_64(volatile uint64_t *target) { atomic_subtract_64(target, 1); } static inline uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta) { return (atomic_fetchadd_64(target, delta) + delta); } #ifndef __sparc64__ static inline uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) { #ifdef STRONG_FCMPSET (void)atomic_fcmpset_64(target, &cmp, newval); #else uint64_t expected = cmp; do { if (atomic_fcmpset_64(target, &cmp, newval)) break; } while (cmp == expected); #endif return (cmp); } #endif #endif static __inline void atomic_inc_64(volatile uint64_t *target) { atomic_add_64(target, 1); } static __inline uint64_t atomic_inc_64_nv(volatile uint64_t *target) { return (atomic_add_64_nv(target, 1)); } static __inline uint64_t atomic_dec_64_nv(volatile uint64_t *target) { return (atomic_add_64_nv(target, -1)); } #if !defined(COMPAT_32BIT) && defined(__LP64__) static __inline void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return ((void *)atomic_cas_64((volatile uint64_t *)target, (uint64_t)cmp, (uint64_t)newval)); } #else static __inline void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return ((void *)atomic_cas_32((volatile uint32_t *)target, (uint32_t)cmp, (uint32_t)newval)); } #endif /* !defined(COMPAT_32BIT) && defined(__LP64__) */ #endif /* !_OPENSOLARIS_SYS_ATOMIC_H_ */ diff --git a/sys/conf/files.mips b/sys/conf/files.mips index 1977fb9dcce2..4781c9adb4bb 100644 --- a/sys/conf/files.mips +++ b/sys/conf/files.mips @@ -1,117 +1,118 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # Arch dependent files mips/mips/autoconf.c standard mips/mips/bus_space_generic.c standard mips/mips/busdma_machdep.c standard mips/mips/cache.c standard mips/mips/cache_mipsNN.c standard mips/mips/cpu.c standard mips/mips/db_disasm.c optional ddb mips/mips/db_interface.c optional ddb mips/mips/db_trace.c optional ddb mips/mips/dump_machdep.c standard mips/mips/elf_machdep.c standard mips/mips/exception.S standard mips/mips/fp.S standard mips/mips/freebsd32_machdep.c optional compat_freebsd32 mips/mips/gdb_machdep.c standard mips/mips/in_cksum.c optional inet mips/mips/libkern_machdep.c standard mips/mips/locore.S standard no-obj mips/mips/machdep.c standard mips/mips/mem.c optional mem mips/mips/minidump_machdep.c standard mips/mips/mp_machdep.c optional smp mips/mips/mpboot.S optional smp mips/mips/nexus.c standard mips/mips/ofw_machdep.c optional fdt mips/mips/pm_machdep.c standard mips/mips/pmap.c standard mips/mips/ptrace_machdep.c standard mips/mips/sc_machdep.c standard mips/mips/stack_machdep.c optional ddb | stack mips/mips/stdatomic.c standard \ compile-with "${NORMAL_C:N-Wmissing-prototypes}" mips/mips/support.S standard mips/mips/bcopy.S standard mips/mips/swtch.S standard mips/mips/sys_machdep.c standard mips/mips/tlb.c standard mips/mips/trap.c standard mips/mips/uio_machdep.c standard mips/mips/uma_machdep.c standard mips/mips/vm_machdep.c standard # misc opt-in bits kern/kern_clocksource.c standard kern/link_elf_obj.c standard +kern/subr_atomic64.c optional mips | mipsel | mipshf | mipselhf kern/subr_busdma_bufalloc.c standard kern/subr_dummy_vdso_tc.c standard kern/subr_sfbuf.c optional mips | mipsel | mipsn32 kern/subr_sfbuf.c optional mipshf | mipselhf # gcc/clang runtime libkern/ffsl.c standard libkern/ffsll.c standard libkern/fls.c standard libkern/flsl.c standard libkern/flsll.c standard libkern/cmpdi2.c optional mips | mipshf | mipsel | mipselhf libkern/ucmpdi2.c optional mips | mipshf | mipsel | mipselhf libkern/ashldi3.c standard libkern/ashrdi3.c standard libkern/memcmp.c standard # cfe support dev/cfe/cfe_api.c optional cfe dev/cfe/cfe_console.c optional cfe_console dev/cfe/cfe_env.c optional cfe_env # syscons support dev/fb/fb.c optional sc dev/syscons/scgfbrndr.c optional sc dev/syscons/scterm-teken.c optional sc dev/syscons/scvtb.c optional sc mips/mips/sc_machdep.c optional sc # FDT support dev/uart/uart_cpu_fdt.c optional uart fdt # crypto support -- use generic crypto/blowfish/bf_enc.c optional crypto | ipsec | \ ipsec_support crypto/des/des_enc.c optional crypto | ipsec | \ ipsec_support | netsmb # AP common nvram interface MIPS specific, but maybe should be more generic dev/nvram2env/nvram2env_mips.c optional nvram2env dev/nvram2env/nvram2env.c optional nvram2env # hwpmc support dev/hwpmc/hwpmc_mips.c optional hwpmc dev/hwpmc/hwpmc_mips24k.c optional hwpmc_mips24k dev/hwpmc/hwpmc_mips74k.c optional hwpmc_mips74k # ofw support dev/ofw/ofwpci.c optional fdt pci # INTRNG support code kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_intr.c optional intrng # INTRNG compatible MIPS32 interrupt controller mips/mips/mips_pic.c optional intrng # DTrace cddl/compat/opensolaris/kern/opensolaris_atomic.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/dev/dtrace/mips/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/mips/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/mips/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" # Zstd contrib/zstd/lib/freebsd/zstd_kfreebsd.c optional zstdio compile-with ${ZSTD_C} diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index eb55f955ebaa..90f683f7a890 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -1,316 +1,317 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # # font.h optional sc \ compile-with "uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'u_char dflt_font_16[16*256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'u_char dflt_font_14[14*256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" # # There is only an asm version on ppc64. cddl/compat/opensolaris/kern/opensolaris_atomic.c optional zfs powerpc | dtrace powerpc | zfs powerpcspe | dtrace powerpcspe compile-with "${ZFS_C}" cddl/dev/dtrace/powerpc/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/powerpc/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/powerpc/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/blowfish/bf_enc.c optional crypto | ipsec | ipsec_support crypto/des/des_enc.c optional crypto | ipsec | ipsec_support | netsmb dev/bm/if_bm.c optional bm powermac dev/adb/adb_bus.c optional adb dev/adb/adb_kbd.c optional adb dev/adb/adb_mouse.c optional adb dev/adb/adb_hb_if.m optional adb dev/adb/adb_if.m optional adb dev/adb/adb_buttons.c optional adb dev/agp/agp_apple.c optional agp powermac dev/fb/fb.c optional sc dev/hwpmc/hwpmc_e500.c optional hwpmc dev/hwpmc/hwpmc_mpc7xxx.c optional hwpmc dev/hwpmc/hwpmc_powerpc.c optional hwpmc dev/hwpmc/hwpmc_ppc970.c optional hwpmc dev/iicbus/ad7417.c optional ad7417 powermac dev/iicbus/adm1030.c optional powermac windtunnel | adm1030 powermac dev/iicbus/adt746x.c optional adt746x powermac dev/iicbus/ds1631.c optional ds1631 powermac dev/iicbus/ds1775.c optional ds1775 powermac dev/iicbus/max6690.c optional max6690 powermac dev/iicbus/ofw_iicbus.c optional iicbus aim dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_opal.c optional powernv ipmi dev/ixl/if_ixl.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_main.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iflib.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_qmgr.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iov.c optional ixl pci pci_iov powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_i2c.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/if_iavf.c optional iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/iavf_vc.c optional iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_txrx.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_osdep.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_lan_hmc.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_hmc.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_common.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_nvm.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_adminq.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_dcb.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/nand/nfc_fsl.c optional nand mpc85xx dev/nand/nfc_rb.c optional nand mpc85xx # Most ofw stuff below is brought in by conf/files for options FDT, but # we always want it, even on non-FDT platforms. dev/fdt/simplebus.c standard dev/ofw/openfirm.c standard dev/ofw/openfirmio.c standard dev/ofw/ofw_bus_if.m standard dev/ofw/ofw_cpu.c standard dev/ofw/ofw_if.m standard dev/ofw/ofw_bus_subr.c standard dev/ofw/ofw_console.c optional aim dev/ofw/ofw_disk.c optional ofwd aim dev/ofw/ofwbus.c standard dev/ofw/ofwpci.c optional pci dev/ofw/ofw_standard.c optional aim powerpc dev/ofw/ofw_subr.c standard dev/powermac_nvram/powermac_nvram.c optional powermac_nvram powermac dev/quicc/quicc_bfe_fdt.c optional quicc mpc85xx dev/random/darn.c optional powerpc64 random dev/scc/scc_bfe_macio.c optional scc powermac dev/sdhci/fsl_sdhci.c optional mpc85xx sdhci dev/sec/sec.c optional sec mpc85xx dev/sound/macio/aoa.c optional snd_davbus | snd_ai2s powermac dev/sound/macio/davbus.c optional snd_davbus powermac dev/sound/macio/i2s.c optional snd_ai2s powermac dev/sound/macio/onyx.c optional snd_ai2s iicbus powermac dev/sound/macio/snapper.c optional snd_ai2s iicbus powermac dev/sound/macio/tumbler.c optional snd_ai2s iicbus powermac dev/syscons/scgfbrndr.c optional sc dev/syscons/scterm-teken.c optional sc dev/syscons/scvtb.c optional sc dev/tsec/if_tsec.c optional tsec dev/tsec/if_tsec_fdt.c optional tsec dev/uart/uart_cpu_powerpc.c optional uart dev/usb/controller/ehci_fsl.c optional ehci mpc85xx dev/vt/hw/ofwfb/ofwfb.c optional vt aim kern/kern_clocksource.c standard +kern/subr_atomic64.c optional powerpc | powerpcspe kern/subr_dummy_vdso_tc.c standard kern/syscalls.c optional ktr kern/subr_sfbuf.c standard libkern/ashldi3.c optional powerpc | powerpcspe libkern/ashrdi3.c optional powerpc | powerpcspe libkern/bcmp.c standard libkern/bcopy.c standard libkern/cmpdi2.c optional powerpc | powerpcspe libkern/divdi3.c optional powerpc | powerpcspe libkern/ffs.c standard libkern/ffsl.c standard libkern/ffsll.c standard libkern/fls.c standard libkern/flsl.c standard libkern/flsll.c standard libkern/lshrdi3.c optional powerpc | powerpcspe libkern/memcmp.c standard libkern/memset.c standard libkern/moddi3.c optional powerpc | powerpcspe libkern/qdivrem.c optional powerpc | powerpcspe libkern/ucmpdi2.c optional powerpc | powerpcspe libkern/udivdi3.c optional powerpc | powerpcspe libkern/umoddi3.c optional powerpc | powerpcspe powerpc/aim/isa3_hashtb.c optional aim powerpc64 powerpc/aim/locore.S optional aim no-obj powerpc/aim/aim_machdep.c optional aim powerpc/aim/mmu_oea.c optional aim powerpc powerpc/aim/mmu_oea64.c optional aim powerpc/aim/moea64_if.m optional aim powerpc/aim/moea64_native.c optional aim powerpc/aim/mp_cpudep.c optional aim powerpc/aim/slb.c optional aim powerpc64 powerpc/booke/locore.S optional booke no-obj powerpc/booke/booke_machdep.c optional booke powerpc/booke/machdep_e500.c optional booke_e500 powerpc/booke/mp_cpudep.c optional booke smp powerpc/booke/platform_bare.c optional booke powerpc/booke/pmap.c optional booke powerpc/booke/spe.c optional powerpcspe powerpc/cpufreq/dfs.c optional cpufreq powerpc/cpufreq/mpc85xx_jog.c optional cpufreq mpc85xx powerpc/cpufreq/pcr.c optional cpufreq aim powerpc/cpufreq/pmcr.c optional cpufreq aim powerpc64 powerpc/cpufreq/pmufreq.c optional cpufreq aim pmu powerpc/fpu/fpu_add.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_compare.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_div.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_emu.c optional fpu_emu powerpc/fpu/fpu_explode.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_implode.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_mul.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_sqrt.c optional fpu_emu powerpc/fpu/fpu_subr.c optional fpu_emu | powerpcspe powerpc/mambo/mambocall.S optional mambo powerpc/mambo/mambo.c optional mambo powerpc/mambo/mambo_console.c optional mambo powerpc/mambo/mambo_disk.c optional mambo powerpc/mikrotik/platform_rb.c optional mikrotik powerpc/mikrotik/rb_led.c optional mikrotik powerpc/mpc85xx/atpic.c optional mpc85xx isa powerpc/mpc85xx/ds1553_bus_fdt.c optional ds1553 powerpc/mpc85xx/ds1553_core.c optional ds1553 powerpc/mpc85xx/fsl_diu.c optional mpc85xx diu powerpc/mpc85xx/fsl_espi.c optional mpc85xx spibus powerpc/mpc85xx/fsl_sata.c optional mpc85xx ata powerpc/mpc85xx/i2c.c optional iicbus powerpc/mpc85xx/isa.c optional mpc85xx isa powerpc/mpc85xx/lbc.c optional mpc85xx powerpc/mpc85xx/mpc85xx.c optional mpc85xx powerpc/mpc85xx/mpc85xx_cache.c optional mpc85xx powerpc/mpc85xx/mpc85xx_gpio.c optional mpc85xx gpio powerpc/mpc85xx/platform_mpc85xx.c optional mpc85xx powerpc/mpc85xx/pci_mpc85xx.c optional pci mpc85xx powerpc/mpc85xx/pci_mpc85xx_pcib.c optional pci mpc85xx powerpc/mpc85xx/qoriq_gpio.c optional mpc85xx gpio powerpc/ofw/ofw_machdep.c standard powerpc/ofw/ofw_pcibus.c optional pci powerpc/ofw/ofw_pcib_pci.c optional pci powerpc/ofw/ofw_real.c optional aim powerpc/ofw/ofw_syscons.c optional sc aim powerpc/ofw/ofwcall32.S optional aim powerpc powerpc/ofw/ofwcall64.S optional aim powerpc64 powerpc/ofw/openpic_ofw.c standard powerpc/ofw/rtas.c optional aim powerpc/ofw/ofw_initrd.c optional md_root_mem powerpc64 powerpc/powermac/ata_kauai.c optional powermac ata | powermac atamacio powerpc/powermac/ata_macio.c optional powermac ata | powermac atamacio powerpc/powermac/ata_dbdma.c optional powermac ata | powermac atamacio powerpc/powermac/atibl.c optional powermac atibl powerpc/powermac/cuda.c optional powermac cuda powerpc/powermac/cpcht.c optional powermac pci powerpc/powermac/dbdma.c optional powermac pci powerpc/powermac/fcu.c optional powermac fcu powerpc/powermac/grackle.c optional powermac pci powerpc/powermac/hrowpic.c optional powermac pci powerpc/powermac/kiic.c optional powermac kiic powerpc/powermac/macgpio.c optional powermac pci powerpc/powermac/macio.c optional powermac pci powerpc/powermac/nvbl.c optional powermac nvbl powerpc/powermac/platform_powermac.c optional powermac powerpc/powermac/powermac_thermal.c optional powermac powerpc/powermac/pswitch.c optional powermac pswitch powerpc/powermac/pmu.c optional powermac pmu powerpc/powermac/smu.c optional powermac smu powerpc/powermac/smusat.c optional powermac smu powerpc/powermac/uninorth.c optional powermac powerpc/powermac/uninorthpci.c optional powermac pci powerpc/powermac/vcoregpio.c optional powermac powerpc/powernv/opal.c optional powernv powerpc/powernv/opal_console.c optional powernv powerpc/powernv/opal_dev.c optional powernv powerpc/powernv/opal_hmi.c optional powernv powerpc/powernv/opal_i2c.c optional iicbus fdt powernv powerpc/powernv/opal_i2cm.c optional iicbus fdt powernv powerpc/powernv/opal_pci.c optional powernv pci powerpc/powernv/opal_sensor.c optional powernv powerpc/powernv/opalcall.S optional powernv powerpc/powernv/platform_powernv.c optional powernv powerpc/powernv/powernv_centaur.c optional powernv powerpc/powernv/powernv_xscom.c optional powernv powerpc/powerpc/altivec.c optional powerpc | powerpc64 powerpc/powerpc/autoconf.c standard powerpc/powerpc/bus_machdep.c standard powerpc/powerpc/busdma_machdep.c standard powerpc/powerpc/clock.c standard powerpc/powerpc/copyinout.c standard powerpc/powerpc/copystr.c standard powerpc/powerpc/cpu.c standard powerpc/powerpc/cpu_subr64.S optional powerpc64 powerpc/powerpc/db_disasm.c optional ddb powerpc/powerpc/db_hwwatch.c optional ddb powerpc/powerpc/db_interface.c optional ddb powerpc/powerpc/db_trace.c optional ddb powerpc/powerpc/dump_machdep.c standard powerpc/powerpc/elf32_machdep.c optional powerpc | powerpcspe | compat_freebsd32 powerpc/powerpc/elf64_machdep.c optional powerpc64 powerpc/powerpc/exec_machdep.c standard powerpc/powerpc/fpu.c standard powerpc/powerpc/gdb_machdep.c optional gdb powerpc/powerpc/in_cksum.c optional inet | inet6 powerpc/powerpc/interrupt.c standard powerpc/powerpc/intr_machdep.c standard powerpc/powerpc/iommu_if.m standard powerpc/powerpc/machdep.c standard powerpc/powerpc/mem.c optional mem powerpc/powerpc/mmu_if.m standard powerpc/powerpc/mp_machdep.c optional smp powerpc/powerpc/nexus.c standard powerpc/powerpc/openpic.c standard powerpc/powerpc/pic_if.m standard powerpc/powerpc/pmap_dispatch.c standard powerpc/powerpc/platform.c standard powerpc/powerpc/platform_if.m standard powerpc/powerpc/ptrace_machdep.c standard powerpc/powerpc/sc_machdep.c optional sc powerpc/powerpc/setjmp.S standard powerpc/powerpc/sigcode32.S optional powerpc | powerpcspe | compat_freebsd32 powerpc/powerpc/sigcode64.S optional powerpc64 powerpc/powerpc/swtch32.S optional powerpc | powerpcspe powerpc/powerpc/swtch64.S optional powerpc64 powerpc/powerpc/stack_machdep.c optional ddb | stack powerpc/powerpc/syncicache.c standard powerpc/powerpc/sys_machdep.c standard powerpc/powerpc/trap.c standard powerpc/powerpc/uio_machdep.c standard powerpc/powerpc/uma_machdep.c standard powerpc/powerpc/vm_machdep.c standard powerpc/ps3/ehci_ps3.c optional ps3 ehci powerpc/ps3/ohci_ps3.c optional ps3 ohci powerpc/ps3/if_glc.c optional ps3 glc powerpc/ps3/mmu_ps3.c optional ps3 powerpc/ps3/platform_ps3.c optional ps3 powerpc/ps3/ps3bus.c optional ps3 powerpc/ps3/ps3cdrom.c optional ps3 scbus powerpc/ps3/ps3disk.c optional ps3 powerpc/ps3/ps3pic.c optional ps3 powerpc/ps3/ps3_syscons.c optional ps3 vt powerpc/ps3/ps3-hvcall.S optional ps3 powerpc/pseries/phyp-hvcall.S optional pseries powerpc64 powerpc/pseries/mmu_phyp.c optional pseries powerpc64 powerpc/pseries/phyp_console.c optional pseries powerpc64 uart powerpc/pseries/phyp_llan.c optional llan powerpc/pseries/phyp_vscsi.c optional pseries powerpc64 scbus powerpc/pseries/platform_chrp.c optional pseries powerpc/pseries/plpar_iommu.c optional pseries powerpc64 powerpc/pseries/plpar_pcibus.c optional pseries powerpc64 pci powerpc/pseries/rtas_dev.c optional pseries powerpc/pseries/rtas_pci.c optional pseries pci powerpc/pseries/vdevice.c optional pseries powerpc64 powerpc/pseries/xics.c optional pseries powerpc64 powerpc/psim/iobus.c optional psim powerpc/psim/ata_iobus.c optional ata psim powerpc/psim/openpic_iobus.c optional psim powerpc/psim/uart_iobus.c optional uart psim diff --git a/sys/kern/subr_atomic64.c b/sys/kern/subr_atomic64.c new file mode 100644 index 000000000000..266b4ee20a10 --- /dev/null +++ b/sys/kern/subr_atomic64.c @@ -0,0 +1,140 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 Justin Hibbits + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +enum { + ATOMIC64_ADD, + ATOMIC64_CLEAR, + ATOMIC64_CMPSET, + ATOMIC64_FCMPSET, + ATOMIC64_FETCHADD, + ATOMIC64_LOAD, + ATOMIC64_SET, + ATOMIC64_SUBTRACT, + ATOMIC64_STORE, + ATOMIC64_SWAP +}; + +#ifdef _KERNEL +#define A64_POOL_SIZE MAXCPU +/* Estimated size of a cacheline */ +#define CACHE_ALIGN CACHE_LINE_SIZE + +#define GET_MUTEX(p) \ + (&a64_mtx_pool[(pmap_kextract((vm_offset_t)p) / CACHE_ALIGN) % (A64_POOL_SIZE)]) + +#define LOCK_A64() \ + struct mtx *_amtx = GET_MUTEX(p); \ + if (smp_started) mtx_lock(_amtx) + +#define UNLOCK_A64() if (smp_started) mtx_unlock(_amtx) + +#define ATOMIC64_EMU_UN(op, rt, block, ret) \ + rt \ + atomic_##op##_64(volatile u_int64_t *p) { \ + u_int64_t tmp __unused; \ + LOCK_A64(); \ + block; \ + UNLOCK_A64(); \ + ret; } struct hack + +#define ATOMIC64_EMU_BIN(op, rt, block, ret) \ + rt \ + atomic_##op##_64(volatile u_int64_t *p, u_int64_t v) { \ + u_int64_t tmp __unused; \ + LOCK_A64(); \ + block; \ + UNLOCK_A64(); \ + ret; } struct hack + +static struct mtx a64_mtx_pool[A64_POOL_SIZE]; + +ATOMIC64_EMU_BIN(add, void, (*p = *p + v), return); +ATOMIC64_EMU_BIN(clear, void, *p &= ~v, return); +ATOMIC64_EMU_BIN(fetchadd, u_int64_t, (*p = *p + v, v = *p - v), return (v)); +ATOMIC64_EMU_UN(load, u_int64_t, (tmp = *p), return (tmp)); +ATOMIC64_EMU_BIN(set, void, *p |= v, return); +ATOMIC64_EMU_BIN(subtract, void, (*p = *p - v), return); +ATOMIC64_EMU_BIN(store, void, *p = v, return); +ATOMIC64_EMU_BIN(swap, u_int64_t, tmp = *p; *p = v; v = tmp, return(v)); + +int atomic_cmpset_64(volatile u_int64_t *p, u_int64_t old, u_int64_t new) +{ + u_int64_t tmp; + + LOCK_A64(); + tmp = *p; + if (tmp == old) + *p = new; + UNLOCK_A64(); + + return (tmp == old); +} + +int atomic_fcmpset_64(volatile u_int64_t *p, u_int64_t *old, u_int64_t new) +{ + u_int64_t tmp, tmp_old; + + LOCK_A64(); + tmp = *p; + tmp_old = *old; + if (tmp == tmp_old) + *p = new; + else + *old = tmp; + UNLOCK_A64(); + + return (tmp == tmp_old); +} + +static void +atomic64_mtxinit(void *x __unused) +{ + int i; + + for (i = 0; i < A64_POOL_SIZE; i++) + mtx_init(&a64_mtx_pool[i], "atomic64 mutex", NULL, MTX_DEF); +} + +SYSINIT(atomic64_mtxinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, atomic64_mtxinit, NULL); + +#endif /* _KERNEL */ diff --git a/sys/mips/include/atomic.h b/sys/mips/include/atomic.h index 709af5cdcf59..0ede70d82202 100644 --- a/sys/mips/include/atomic.h +++ b/sys/mips/include/atomic.h @@ -1,829 +1,833 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: src/sys/alpha/include/atomic.h,v 1.21.2.3 2005/10/06 18:12:05 jhb * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif #include +#if !defined(__mips_n64) && !defined(__mips_n32) +#include +#endif + /* * Note: All the 64-bit atomic operations are only atomic when running * in 64-bit mode. It is assumed that code compiled for n32 and n64 * fits into this definition and no further safeties are needed. * * It is also assumed that the add, subtract and other arithmetic is * done on numbers not pointers. The special rules for n32 pointers * do not have atomic operations defined for them, but generally shouldn't * need atomic operations. */ #ifndef __MIPS_PLATFORM_SYNC_NOPS #define __MIPS_PLATFORM_SYNC_NOPS "" #endif static __inline void mips_sync(void) { __asm __volatile (".set noreorder\n" "\tsync\n" __MIPS_PLATFORM_SYNC_NOPS ".set reorder\n" : : : "memory"); } #define mb() mips_sync() #define wmb() mips_sync() #define rmb() mips_sync() /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and SMP safe. */ void atomic_set_8(__volatile uint8_t *, uint8_t); void atomic_clear_8(__volatile uint8_t *, uint8_t); void atomic_add_8(__volatile uint8_t *, uint8_t); void atomic_subtract_8(__volatile uint8_t *, uint8_t); void atomic_set_16(__volatile uint16_t *, uint16_t); void atomic_clear_16(__volatile uint16_t *, uint16_t); void atomic_add_16(__volatile uint16_t *, uint16_t); void atomic_subtract_16(__volatile uint16_t *, uint16_t); static __inline void atomic_set_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "or %0, %2, %0\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_clear_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; v = ~v; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "and %0, %2, %0\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_add_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "addu %0, %2, %0\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_subtract_32(__volatile uint32_t *p, uint32_t v) { uint32_t temp; __asm __volatile ( "1:\tll %0, %3\n\t" /* load old value */ "subu %0, %2\n\t" /* calculate new value */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline uint32_t atomic_readandclear_32(__volatile uint32_t *addr) { uint32_t result,temp; __asm __volatile ( "1:\tll %0,%3\n\t" /* load current value, asserting lock */ "li %1,0\n\t" /* value to store */ "sc %1,%2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr) : "memory"); return result; } static __inline uint32_t atomic_readandset_32(__volatile uint32_t *addr, uint32_t value) { uint32_t result,temp; __asm __volatile ( "1:\tll %0,%3\n\t" /* load current value, asserting lock */ "or %1,$0,%4\n\t" "sc %1,%2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr), "r" (value) : "memory"); return result; } #if defined(__mips_n64) || defined(__mips_n32) static __inline void atomic_set_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "or %0, %2, %0\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_clear_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; v = ~v; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "and %0, %2, %0\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_add_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "daddu %0, %2, %0\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline void atomic_subtract_64(__volatile uint64_t *p, uint64_t v) { uint64_t temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "dsubu %0, %2\n\t" /* calculate new value */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* spin if failed */ : "=&r" (temp), "=m" (*p) : "r" (v), "m" (*p) : "memory"); } static __inline uint64_t atomic_readandclear_64(__volatile uint64_t *addr) { uint64_t result,temp; __asm __volatile ( "1:\n\t" "lld %0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* value to store */ "scd %1, %2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr) : "memory"); return result; } static __inline uint64_t atomic_readandset_64(__volatile uint64_t *addr, uint64_t value) { uint64_t result,temp; __asm __volatile ( "1:\n\t" "lld %0,%3\n\t" /* Load old value*/ "or %1,$0,%4\n\t" "scd %1,%2\n\t" /* attempt to store */ "beqz %1, 1b\n\t" /* if the store failed, spin */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "m" (*addr), "r" (value) : "memory"); return result; } #endif #define ATOMIC_ACQ_REL(NAME, WIDTH) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ atomic_##NAME##_##WIDTH(p, v); \ mips_sync(); \ } \ \ static __inline void \ atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ mips_sync(); \ atomic_##NAME##_##WIDTH(p, v); \ } /* Variants of simple arithmetic with memory barriers. */ ATOMIC_ACQ_REL(set, 8) ATOMIC_ACQ_REL(clear, 8) ATOMIC_ACQ_REL(add, 8) ATOMIC_ACQ_REL(subtract, 8) ATOMIC_ACQ_REL(set, 16) ATOMIC_ACQ_REL(clear, 16) ATOMIC_ACQ_REL(add, 16) ATOMIC_ACQ_REL(subtract, 16) ATOMIC_ACQ_REL(set, 32) ATOMIC_ACQ_REL(clear, 32) ATOMIC_ACQ_REL(add, 32) ATOMIC_ACQ_REL(subtract, 32) #if defined(__mips_n64) || defined(__mips_n32) ATOMIC_ACQ_REL(set, 64) ATOMIC_ACQ_REL(clear, 64) ATOMIC_ACQ_REL(add, 64) ATOMIC_ACQ_REL(subtract, 64) #endif #undef ATOMIC_ACQ_REL /* * We assume that a = b will do atomic loads and stores. */ #define ATOMIC_STORE_LOAD(WIDTH) \ static __inline uint##WIDTH##_t \ atomic_load_acq_##WIDTH(__volatile uint##WIDTH##_t *p) \ { \ uint##WIDTH##_t v; \ \ v = *p; \ mips_sync(); \ return (v); \ } \ \ static __inline void \ atomic_store_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ { \ mips_sync(); \ *p = v; \ } ATOMIC_STORE_LOAD(32) ATOMIC_STORE_LOAD(64) #undef ATOMIC_STORE_LOAD #ifdef __mips_n32 #define atomic_load_64 atomic_load_acq_64 #endif /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint32_t atomic_cmpset_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { uint32_t ret; __asm __volatile ( "1:\tll %0, %4\n\t" /* load old value */ "bne %0, %2, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "li %0, 0\n\t" "3:\n" : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); return ret; } /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint32_t atomic_cmpset_acq_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { int retval; retval = atomic_cmpset_32(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint32_t atomic_cmpset_rel_32(__volatile uint32_t *p, uint32_t cmpval, uint32_t newval) { mips_sync(); return (atomic_cmpset_32(p, cmpval, newval)); } static __inline uint32_t atomic_fcmpset_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { uint32_t ret; __asm __volatile ( "1:\n\t" "ll %0, %1\n\t" /* load old value */ "bne %0, %4, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "sc %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "sw %0, %2\n\t" /* save old value */ "li %0, 0\n\t" "3:\n" : "=&r" (ret), "+m" (*p), "=m" (*cmpval) : "r" (newval), "r" (*cmpval) : "memory"); return ret; } static __inline uint32_t atomic_fcmpset_acq_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { int retval; retval = atomic_fcmpset_32(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint32_t atomic_fcmpset_rel_32(__volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) { mips_sync(); return (atomic_fcmpset_32(p, cmpval, newval)); } /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline uint32_t atomic_fetchadd_32(__volatile uint32_t *p, uint32_t v) { uint32_t value, temp; __asm __volatile ( "1:\tll %0, %1\n\t" /* load old value */ "addu %2, %3, %0\n\t" /* calculate new value */ "sc %2, %1\n\t" /* attempt to store */ "beqz %2, 1b\n\t" /* spin if failed */ : "=&r" (value), "=m" (*p), "=&r" (temp) : "r" (v), "m" (*p)); return (value); } #if defined(__mips_n64) || defined(__mips_n32) /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint64_t atomic_cmpset_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { uint64_t ret; __asm __volatile ( "1:\n\t" "lld %0, %4\n\t" /* load old value */ "bne %0, %2, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "li %0, 0\n\t" "3:\n" : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); return ret; } /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline uint64_t atomic_cmpset_acq_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { int retval; retval = atomic_cmpset_64(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint64_t atomic_cmpset_rel_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval) { mips_sync(); return (atomic_cmpset_64(p, cmpval, newval)); } static __inline uint32_t atomic_fcmpset_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { uint32_t ret; __asm __volatile ( "1:\n\t" "lld %0, %1\n\t" /* load old value */ "bne %0, %4, 2f\n\t" /* compare */ "move %0, %3\n\t" /* value to store */ "scd %0, %1\n\t" /* attempt to store */ "beqz %0, 1b\n\t" /* if it failed, spin */ "j 3f\n\t" "2:\n\t" "sd %0, %2\n\t" /* save old value */ "li %0, 0\n\t" "3:\n" : "=&r" (ret), "+m" (*p), "=m" (*cmpval) : "r" (newval), "r" (*cmpval) : "memory"); return ret; } static __inline uint64_t atomic_fcmpset_acq_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { int retval; retval = atomic_fcmpset_64(p, cmpval, newval); mips_sync(); return (retval); } static __inline uint64_t atomic_fcmpset_rel_64(__volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) { mips_sync(); return (atomic_fcmpset_64(p, cmpval, newval)); } /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline uint64_t atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v) { uint64_t value, temp; __asm __volatile ( "1:\n\t" "lld %0, %1\n\t" /* load old value */ "daddu %2, %3, %0\n\t" /* calculate new value */ "scd %2, %1\n\t" /* attempt to store */ "beqz %2, 1b\n\t" /* spin if failed */ : "=&r" (value), "=m" (*p), "=&r" (temp) : "r" (v), "m" (*p)); return (value); } #endif static __inline void atomic_thread_fence_acq(void) { mips_sync(); } static __inline void atomic_thread_fence_rel(void) { mips_sync(); } static __inline void atomic_thread_fence_acq_rel(void) { mips_sync(); } static __inline void atomic_thread_fence_seq_cst(void) { mips_sync(); } /* Operations on chars. */ #define atomic_set_char atomic_set_8 #define atomic_set_acq_char atomic_set_acq_8 #define atomic_set_rel_char atomic_set_rel_8 #define atomic_clear_char atomic_clear_8 #define atomic_clear_acq_char atomic_clear_acq_8 #define atomic_clear_rel_char atomic_clear_rel_8 #define atomic_add_char atomic_add_8 #define atomic_add_acq_char atomic_add_acq_8 #define atomic_add_rel_char atomic_add_rel_8 #define atomic_subtract_char atomic_subtract_8 #define atomic_subtract_acq_char atomic_subtract_acq_8 #define atomic_subtract_rel_char atomic_subtract_rel_8 /* Operations on shorts. */ #define atomic_set_short atomic_set_16 #define atomic_set_acq_short atomic_set_acq_16 #define atomic_set_rel_short atomic_set_rel_16 #define atomic_clear_short atomic_clear_16 #define atomic_clear_acq_short atomic_clear_acq_16 #define atomic_clear_rel_short atomic_clear_rel_16 #define atomic_add_short atomic_add_16 #define atomic_add_acq_short atomic_add_acq_16 #define atomic_add_rel_short atomic_add_rel_16 #define atomic_subtract_short atomic_subtract_16 #define atomic_subtract_acq_short atomic_subtract_acq_16 #define atomic_subtract_rel_short atomic_subtract_rel_16 /* Operations on ints. */ #define atomic_set_int atomic_set_32 #define atomic_set_acq_int atomic_set_acq_32 #define atomic_set_rel_int atomic_set_rel_32 #define atomic_clear_int atomic_clear_32 #define atomic_clear_acq_int atomic_clear_acq_32 #define atomic_clear_rel_int atomic_clear_rel_32 #define atomic_add_int atomic_add_32 #define atomic_add_acq_int atomic_add_acq_32 #define atomic_add_rel_int atomic_add_rel_32 #define atomic_subtract_int atomic_subtract_32 #define atomic_subtract_acq_int atomic_subtract_acq_32 #define atomic_subtract_rel_int atomic_subtract_rel_32 #define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 #define atomic_fcmpset_int atomic_fcmpset_32 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 #define atomic_load_acq_int atomic_load_acq_32 #define atomic_store_rel_int atomic_store_rel_32 #define atomic_readandclear_int atomic_readandclear_32 #define atomic_readandset_int atomic_readandset_32 #define atomic_fetchadd_int atomic_fetchadd_32 /* * I think the following is right, even for n32. For n32 the pointers * are still 32-bits, so we need to operate on them as 32-bit quantities, * even though they are sign extended in operation. For longs, there's * no question because they are always 32-bits. */ #ifdef __mips_n64 /* Operations on longs. */ #define atomic_set_long atomic_set_64 #define atomic_set_acq_long atomic_set_acq_64 #define atomic_set_rel_long atomic_set_rel_64 #define atomic_clear_long atomic_clear_64 #define atomic_clear_acq_long atomic_clear_acq_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_add_long atomic_add_64 #define atomic_add_acq_long atomic_add_acq_64 #define atomic_add_rel_long atomic_add_rel_64 #define atomic_subtract_long atomic_subtract_64 #define atomic_subtract_acq_long atomic_subtract_acq_64 #define atomic_subtract_rel_long atomic_subtract_rel_64 #define atomic_cmpset_long atomic_cmpset_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 #define atomic_fcmpset_long atomic_fcmpset_64 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 #define atomic_load_acq_long atomic_load_acq_64 #define atomic_store_rel_long atomic_store_rel_64 #define atomic_fetchadd_long atomic_fetchadd_64 #define atomic_readandclear_long atomic_readandclear_64 #else /* !__mips_n64 */ /* Operations on longs. */ #define atomic_set_long(p, v) \ atomic_set_32((volatile u_int *)(p), (u_int)(v)) #define atomic_set_acq_long(p, v) \ atomic_set_acq_32((volatile u_int *)(p), (u_int)(v)) #define atomic_set_rel_long(p, v) \ atomic_set_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_long(p, v) \ atomic_clear_32((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_acq_long(p, v) \ atomic_clear_acq_32((volatile u_int *)(p), (u_int)(v)) #define atomic_clear_rel_long(p, v) \ atomic_clear_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_add_long(p, v) \ atomic_add_32((volatile u_int *)(p), (u_int)(v)) #define atomic_add_acq_long(p, v) \ atomic_add_32((volatile u_int *)(p), (u_int)(v)) #define atomic_add_rel_long(p, v) \ atomic_add_32((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_long(p, v) \ atomic_subtract_32((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_acq_long(p, v) \ atomic_subtract_acq_32((volatile u_int *)(p), (u_int)(v)) #define atomic_subtract_rel_long(p, v) \ atomic_subtract_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_cmpset_long(p, cmpval, newval) \ atomic_cmpset_32((volatile u_int *)(p), (u_int)(cmpval), \ (u_int)(newval)) #define atomic_cmpset_acq_long(p, cmpval, newval) \ atomic_cmpset_acq_32((volatile u_int *)(p), (u_int)(cmpval), \ (u_int)(newval)) #define atomic_cmpset_rel_long(p, cmpval, newval) \ atomic_cmpset_rel_32((volatile u_int *)(p), (u_int)(cmpval), \ (u_int)(newval)) #define atomic_fcmpset_long(p, cmpval, newval) \ atomic_fcmpset_32((volatile u_int *)(p), (u_int *)(cmpval), \ (u_int)(newval)) #define atomic_fcmpset_acq_long(p, cmpval, newval) \ atomic_fcmpset_acq_32((volatile u_int *)(p), (u_int *)(cmpval), \ (u_int)(newval)) #define atomic_fcmpset_rel_long(p, cmpval, newval) \ atomic_fcmpset_rel_32((volatile u_int *)(p), (u_int *)(cmpval), \ (u_int)(newval)) #define atomic_load_acq_long(p) \ (u_long)atomic_load_acq_32((volatile u_int *)(p)) #define atomic_store_rel_long(p, v) \ atomic_store_rel_32((volatile u_int *)(p), (u_int)(v)) #define atomic_fetchadd_long(p, v) \ atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_long(p) \ atomic_readandclear_32((volatile u_int *)(p)) #endif /* __mips_n64 */ /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_readandclear_ptr atomic_readandclear_long static __inline unsigned int atomic_swap_int(volatile unsigned int *ptr, const unsigned int value) { unsigned int retval; retval = *ptr; while (!atomic_fcmpset_int(ptr, &retval, value)) ; return (retval); } static __inline uint32_t atomic_swap_32(volatile uint32_t *ptr, const uint32_t value) { uint32_t retval; retval = *ptr; while (!atomic_fcmpset_32(ptr, &retval, value)) ; return (retval); } #if defined(__mips_n64) || defined(__mips_n32) static __inline uint64_t atomic_swap_64(volatile uint64_t *ptr, const uint64_t value) { uint64_t retval; retval = *ptr; while (!atomic_fcmpset_64(ptr, &retval, value)) ; return (retval); } #endif #ifdef __mips_n64 static __inline unsigned long atomic_swap_long(volatile unsigned long *ptr, const unsigned long value) { unsigned long retval; retval = *ptr; while (!atomic_fcmpset_64((volatile uint64_t *)ptr, (uint64_t *)&retval, value)) ; return (retval); } #else static __inline unsigned long atomic_swap_long(volatile unsigned long *ptr, const unsigned long value) { unsigned long retval; retval = *ptr; while (!atomic_fcmpset_32((volatile uint32_t *)ptr, (uint32_t *)&retval, value)) ; return (retval); } #endif #define atomic_swap_ptr(ptr, value) atomic_swap_long((unsigned long *)(ptr), value) #endif /* ! _MACHINE_ATOMIC_H_ */ diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h index afffdd7bb4ed..e49a8473a747 100644 --- a/sys/powerpc/include/atomic.h +++ b/sys/powerpc/include/atomic.h @@ -1,891 +1,895 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Marcel Moolenaar * Copyright (c) 2001 Benno Rice * Copyright (c) 2001 David E. O'Brien * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif #include +#ifndef __powerpc64__ +#include +#endif + /* * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction * with the atomic lXarx/stXcx. sequences below. They are not exposed outside * of this file. See also Appendix B.2 of Book II of the architecture manual. * * Note that not all Book-E processors accept the light-weight sync variant. * In particular, early models of E500 cores are known to wedge. Bank on all * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs * to use the heavier-weight sync. */ #ifdef __powerpc64__ #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("lwsync" : : : "memory") #define wmb() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #else #define mb() __asm __volatile("sync" : : : "memory") #define rmb() __asm __volatile("sync" : : : "memory") #define wmb() __asm __volatile("sync" : : : "memory") #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") #endif static __inline void powerpc_lwsync(void) { #ifdef __powerpc64__ __asm __volatile("lwsync" : : : "memory"); #else __asm __volatile("sync" : : : "memory"); #endif } /* * atomic_add(p, v) * { *p += v; } */ #define __atomic_add_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_int */ #ifdef __powerpc64__ #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #else #define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_add_long */ #endif #define _ATOMIC_ADD(type) \ static __inline void \ atomic_add_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ } \ \ static __inline void \ atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_add_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_add_##type(p, v, t); \ } \ /* _ATOMIC_ADD */ _ATOMIC_ADD(int) _ATOMIC_ADD(long) #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #ifdef __powerpc64__ #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #else #define atomic_add_ptr atomic_add_int #define atomic_add_acq_ptr atomic_add_acq_int #define atomic_add_rel_ptr atomic_add_rel_int #endif #undef _ATOMIC_ADD #undef __atomic_add_long #undef __atomic_add_int /* * atomic_clear(p, v) * { *p &= ~v; } */ #define __atomic_clear_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_int */ #ifdef __powerpc64__ #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #else #define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_clear_long */ #endif #define _ATOMIC_CLEAR(type) \ static __inline void \ atomic_clear_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ } \ \ static __inline void \ atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_clear_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_clear_##type(p, v, t); \ } \ /* _ATOMIC_CLEAR */ _ATOMIC_CLEAR(int) _ATOMIC_CLEAR(long) #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #ifdef __powerpc64__ #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #else #define atomic_clear_ptr atomic_clear_int #define atomic_clear_acq_ptr atomic_clear_acq_int #define atomic_clear_rel_ptr atomic_clear_rel_int #endif #undef _ATOMIC_CLEAR #undef __atomic_clear_long #undef __atomic_clear_int /* * atomic_cmpset(p, o, n) */ /* TODO -- see below */ /* * atomic_load_acq(p) */ /* TODO -- see below */ /* * atomic_readandclear(p) */ /* TODO -- see below */ /* * atomic_set(p, v) * { *p |= v; } */ #define __atomic_set_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_int */ #ifdef __powerpc64__ #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #else #define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_set_long */ #endif #define _ATOMIC_SET(type) \ static __inline void \ atomic_set_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ } \ \ static __inline void \ atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_set_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_set_##type(p, v, t); \ } \ /* _ATOMIC_SET */ _ATOMIC_SET(int) _ATOMIC_SET(long) #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #ifdef __powerpc64__ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #else #define atomic_set_ptr atomic_set_int #define atomic_set_acq_ptr atomic_set_acq_int #define atomic_set_rel_ptr atomic_set_rel_int #endif #undef _ATOMIC_SET #undef __atomic_set_long #undef __atomic_set_int /* * atomic_subtract(p, v) * { *p -= v; } */ #define __atomic_subtract_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_int */ #ifdef __powerpc64__ #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stdcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #else #define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cr0", "memory") \ /* __atomic_subtract_long */ #endif #define _ATOMIC_SUBTRACT(type) \ static __inline void \ atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ } \ \ static __inline void \ atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __atomic_subtract_##type(p, v, t); \ __ATOMIC_ACQ(); \ } \ \ static __inline void \ atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ u_##type t; \ __ATOMIC_REL(); \ __atomic_subtract_##type(p, v, t); \ } \ /* _ATOMIC_SUBTRACT */ _ATOMIC_SUBTRACT(int) _ATOMIC_SUBTRACT(long) #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #ifdef __powerpc64__ #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #else #define atomic_subtract_ptr atomic_subtract_int #define atomic_subtract_acq_ptr atomic_subtract_acq_int #define atomic_subtract_rel_ptr atomic_subtract_rel_int #endif #undef _ATOMIC_SUBTRACT #undef __atomic_subtract_long #undef __atomic_subtract_int /* * atomic_store_rel(p, v) */ /* TODO -- see below */ /* * Old/original implementations that still need revisiting. */ static __inline u_int atomic_readandclear_int(volatile u_int *addr) { u_int result,temp; __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tlwarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stwcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); return (result); } #ifdef __powerpc64__ static __inline u_long atomic_readandclear_long(volatile u_long *addr) { u_long result,temp; __asm __volatile ( "\tsync\n" /* drain writes */ "1:\tldarx %0, 0, %3\n\t" /* load old value */ "li %1, 0\n\t" /* load new value */ "stdcx. %1, 0, %3\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ : "=&r"(result), "=&r"(temp), "=m" (*addr) : "r" (addr), "m" (*addr) : "cr0", "memory"); return (result); } #endif #define atomic_readandclear_32 atomic_readandclear_int #ifdef __powerpc64__ #define atomic_readandclear_64 atomic_readandclear_long #define atomic_readandclear_ptr atomic_readandclear_long #else static __inline u_long atomic_readandclear_long(volatile u_long *addr) { return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); } #define atomic_readandclear_ptr atomic_readandclear_int #endif /* * We assume that a = b will do atomic loads and stores. */ #define ATOMIC_STORE_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE v; \ \ v = *p; \ mb(); \ return (v); \ } \ \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ powerpc_lwsync(); \ *p = v; \ } ATOMIC_STORE_LOAD(int) #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #ifdef __powerpc64__ ATOMIC_STORE_LOAD(long) #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #else static __inline u_long atomic_load_acq_long(volatile u_long *addr) { return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); } static __inline void atomic_store_rel_long(volatile u_long *addr, u_long val) { atomic_store_rel_int((volatile u_int *)addr, (u_int)val); } #define atomic_load_acq_ptr atomic_load_acq_int #define atomic_store_rel_ptr atomic_store_rel_int #endif #undef ATOMIC_STORE_LOAD /* * Atomically compare the value stored at *p with cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ static __inline int atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { int ret; __asm __volatile ( "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } static __inline int atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) { int ret; __asm __volatile ( #ifdef __powerpc64__ "1:\tldarx %0, 0, %2\n\t" /* load old value */ "cmpld %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stdcx. %4, 0, %2\n\t" /* attempt to store */ #else "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ #endif "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #else "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ #endif "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cr0", "memory"); return (ret); } static __inline int atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) { int retval; retval = atomic_cmpset_int(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) { __ATOMIC_REL(); return (atomic_cmpset_int(p, cmpval, newval)); } static __inline int atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) { u_long retval; retval = atomic_cmpset_long(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) { __ATOMIC_REL(); return (atomic_cmpset_long(p, cmpval, newval)); } #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #ifdef __powerpc64__ #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #else #define atomic_cmpset_ptr atomic_cmpset_int #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int #endif /* * Atomically compare the value stored at *p with *cmpval and if the * two values are equal, update the value of *p with newval. Returns * zero if the compare failed and sets *cmpval to the read value from *p, * nonzero otherwise. */ static __inline int atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) { int ret; __asm __volatile ( "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } static __inline int atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) { int ret; __asm __volatile ( #ifdef __powerpc64__ "ldarx %0, 0, %3\n\t" /* load old value */ "cmpld %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stdcx. %5, 0, %3\n\t" /* attempt to store */ #else "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ #endif "bne- 1f\n\t" /* exit if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 2f\n\t" /* we've succeeded */ "1:\n\t" #ifdef __powerpc64__ "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stdx %0, 0, %7\n\t" #else "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ "stwx %0, 0, %7\n\t" #endif "li %0, 0\n\t" /* failure - retval = 0 */ "2:\n\t" : "=&r" (ret), "=m" (*p), "=m" (*cmpval) : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) : "cr0", "memory"); return (ret); } static __inline int atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval) { int retval; retval = atomic_fcmpset_int(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval) { __ATOMIC_REL(); return (atomic_fcmpset_int(p, cmpval, newval)); } static __inline int atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval) { u_long retval; retval = atomic_fcmpset_long(p, cmpval, newval); __ATOMIC_ACQ(); return (retval); } static __inline int atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval) { __ATOMIC_REL(); return (atomic_fcmpset_long(p, cmpval, newval)); } #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #ifdef __powerpc64__ #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #else #define atomic_fcmpset_ptr atomic_fcmpset_int #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int #endif static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { u_int value; do { value = *p; } while (!atomic_cmpset_int(p, value, value + v)); return (value); } static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { u_long value; do { value = *p; } while (!atomic_cmpset_long(p, value, value + v)); return (value); } static __inline u_int atomic_swap_32(volatile u_int *p, u_int v) { u_int prev; __asm __volatile( "1: lwarx %0,0,%2\n" " stwcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_int *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #ifdef __powerpc64__ static __inline u_long atomic_swap_64(volatile u_long *p, u_long v) { u_long prev; __asm __volatile( "1: ldarx %0,0,%2\n" " stdcx. %3,0,%2\n" " bne- 1b\n" : "=&r" (prev), "+m" (*(volatile u_long *)p) : "r" (p), "r" (v) : "cr0", "memory"); return (prev); } #endif #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_swap_int atomic_swap_32 #ifdef __powerpc64__ #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_swap_long atomic_swap_64 #define atomic_swap_ptr atomic_swap_64 #else #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) #endif #undef __ATOMIC_REL #undef __ATOMIC_ACQ static __inline void atomic_thread_fence_acq(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_acq_rel(void) { powerpc_lwsync(); } static __inline void atomic_thread_fence_seq_cst(void) { __asm __volatile("sync" : : : "memory"); } #endif /* ! _MACHINE_ATOMIC_H_ */ diff --git a/sys/sys/_atomic64e.h b/sys/sys/_atomic64e.h new file mode 100644 index 000000000000..289e5117ee0b --- /dev/null +++ b/sys/sys/_atomic64e.h @@ -0,0 +1,80 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 Justin Hibbits + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef _SYS_ATOMIC64E_H_ +#define _SYS_ATOMIC64E_H_ + +#ifndef _MACHINE_ATOMIC_H_ +#error "This should not be included directly. Include " +#endif + +#ifdef _KERNEL +#define HAS_EMULATED_ATOMIC64 + +/* Emulated versions of 64-bit atomic operations. */ + +void atomic_add_64(volatile u_int64_t *, u_int64_t); +#define atomic_add_acq_64 atomic_add_64 +#define atomic_add_rel_64 atomic_add_64 + +int atomic_cmpset_64(volatile u_int64_t *, u_int64_t, u_int64_t); +#define atomic_cmpset_acq_64 atomic_cmpset_64 +#define atomic_cmpset_rel_64 atomic_cmpset_64 + +void atomic_clear_64(volatile u_int64_t *, u_int64_t); +#define atomic_clear_acq_64 atomic_clear_64 +#define atomic_clear_rel_64 atomic_clear_64 + +int atomic_fcmpset_64(volatile u_int64_t *, u_int64_t *, u_int64_t); +#define atomic_fcmpset_acq_64 atomic_fcmpset_64 +#define atomic_fcmpset_rel_64 atomic_fcmpset_64 + +u_int64_t atomic_fetchadd_64(volatile u_int64_t *, u_int64_t); + +u_int64_t atomic_load_64(volatile u_int64_t *); +#define atomic_load_acq_64 atomic_load_64 + +void atomic_readandclear_64(volatile u_int64_t *); + +void atomic_set_64(volatile u_int64_t *, u_int64_t); +#define atomic_set_acq_64 atomic_set_64 +#define atomic_set_rel_64 atomic_set_64 + +void atomic_subtract_64(volatile u_int64_t *, u_int64_t); +#define atomic_subtract_acq_64 atomic_subtract_64 +#define atomic_subtract_rel_64 atomic_subtract_64 + +void atomic_store_64(volatile u_int64_t *, u_int64_t); +#define atomic_store_rel_64 atomic_store_64 + +u_int64_t atomic_swap_64(volatile u_int64_t *, u_int64_t); + +#endif /* _KERNEL */ +#endif /* _SYS_ATOMIC64E_H_ */