Changeset View
Changeset View
Standalone View
Standalone View
sys/sys/_gcc_atomic.h
- This file was added.
/*- | |||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD | |||||
* | |||||
* Copyright (c) 2020 Mateusz Guzik | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND | |||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | |||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |||||
* SUCH DAMAGE. | |||||
* | |||||
* $FreeBSD$ | |||||
*/ | |||||
#ifndef _SYS__GCC_ATOMIC_H_ | |||||
#define _SYS__GCC_ATOMIC_H_ | |||||
/* | |||||
* This file provides a translation layer between FreeBSD API for atomics and | |||||
* the one provided by GCC. Primary motivation is that custom routines written | |||||
* in assembly tend to be no better than what the compiler provides and are | |||||
* prone to being worse -- they can have overzealous clobbers, bad constraints | |||||
* or just suffer from limitations of the compiler when faced with inline | |||||
* assembly. | |||||
*/ | |||||
#ifndef _MACHINE_ATOMIC_H_ | |||||
#error do not include this header, use machine/atomic.h | |||||
#endif | |||||
#ifndef SMP | |||||
#error generated code is pessimal for UP kernels | |||||
#endif | |||||
#define __atomic_cmpset_gcc(p, e, s, w, f) ({ \ | |||||
__typeof(e) __var = (e); \ | |||||
__atomic_compare_exchange_n(p, &__var, s, w, __ATOMIC_RELAXED, f); \ | |||||
arichardson: Why is success_memorder always relaxed? | |||||
mjgAuthorUnsubmitted Done Inline Actionsops, that's breakage introduced in subsequent update. of course it was meant to be for success, will fix later mjg: ops, that's breakage introduced in subsequent update. of course it was meant to be for success… | |||||
mjgAuthorUnsubmitted Done Inline Actionssigh, I meant the actual barrier was meant for success and relaxed for failure. mjg: sigh, I meant the actual barrier was meant for success and relaxed for failure. | |||||
}) | |||||
#define __ATOMIC_CMPSET_GCC(TYPE) \ | |||||
static __inline int \ | |||||
atomic_cmpset_##TYPE(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_cmpset_gcc(p, e, s, false, __ATOMIC_RELAXED)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_cmpset_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_cmpset_gcc(p, e, s, false, __ATOMIC_ACQUIRE)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_cmpset_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_cmpset_gcc(p, e, s, false, __ATOMIC_RELEASE)); \ | |||||
} | |||||
#define __ATOMIC_CMPSET_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline int \ | |||||
atomic_cmpset_##ALIAS(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_cmpset_##TYPE(p, e, s)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_cmpset_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_cmpset_acq_##TYPE(p, e, s)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_cmpset_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_cmpset_rel_##TYPE(p, e, s)); \ | |||||
} | |||||
#define __atomic_fcmpset_gcc(p, e, s, w, f) ({ \ | |||||
__atomic_compare_exchange_n(p, e, s, w, __ATOMIC_RELAXED, f); \ | |||||
}) | |||||
#define __ATOMIC_FCMPSET_GCC(TYPE, WEAK) \ | |||||
static __inline int \ | |||||
atomic_fcmpset_##TYPE(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_fcmpset_gcc(p, e, s, WEAK, __ATOMIC_RELAXED)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_fcmpset_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_fcmpset_gcc(p, e, s, WEAK, __ATOMIC_ACQUIRE)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_fcmpset_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_fcmpset_gcc(p, e, s, WEAK, __ATOMIC_RELEASE)); \ | |||||
} | |||||
#define __ATOMIC_FCMPSET_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline int \ | |||||
atomic_fcmpset_##ALIAS(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_fcmpset_##TYPE(p, e, s)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_fcmpset_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_fcmpset_acq_##TYPE(p, e, s)); \ | |||||
} \ | |||||
\ | |||||
static __inline int \ | |||||
atomic_fcmpset_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE *e, u_##TYPE s) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_fcmpset_rel_##TYPE(p, e, s)); \ | |||||
} | |||||
#define __atomic_fetchadd_gcc(p, v, f) ({ \ | |||||
__atomic_fetch_add(p, v, f); \ | |||||
}) | |||||
#define __ATOMIC_FETCHADD_GCC(TYPE) \ | |||||
static __inline u_##TYPE \ | |||||
atomic_fetchadd_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_fetchadd_gcc(p, v, __ATOMIC_RELAXED)); \ | |||||
} | |||||
#define __ATOMIC_FETCHADD_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline u_##TYPE \ | |||||
atomic_fetchadd_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_fetchadd_##TYPE(p, v)); \ | |||||
} | |||||
#define __atomic_set_gcc(p, v, f) ({ \ | |||||
__atomic_or_fetch(p, v, f); \ | |||||
}) | |||||
#define __ATOMIC_SET_GCC(TYPE) \ | |||||
static __inline void \ | |||||
atomic_set_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_set_gcc(p, v, __ATOMIC_RELAXED); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_set_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_set_gcc(p, v, __ATOMIC_ACQUIRE); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_set_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_set_gcc(p, v, __ATOMIC_RELEASE); \ | |||||
} | |||||
#define __ATOMIC_SET_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline void \ | |||||
atomic_set_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_set_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_set_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_set_acq_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_set_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_set_rel_##TYPE(p, v); \ | |||||
} | |||||
#define __atomic_clear_gcc(p, v, f) ({ \ | |||||
__atomic_and_fetch(p, ~v, f); \ | |||||
}) | |||||
#define __ATOMIC_CLEAR_GCC(TYPE) \ | |||||
static __inline void \ | |||||
atomic_clear_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_clear_gcc(p, v, __ATOMIC_RELAXED); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_clear_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_clear_gcc(p, v, __ATOMIC_ACQUIRE); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_clear_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_clear_gcc(p, v, __ATOMIC_RELEASE); \ | |||||
} | |||||
#define __ATOMIC_CLEAR_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline void \ | |||||
atomic_clear_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_clear_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_clear_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_clear_acq_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_clear_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_clear_rel_##TYPE(p, v); \ | |||||
} | |||||
#define __atomic_add_gcc(p, v, f) ({ \ | |||||
__atomic_fetch_add(p, v, f); \ | |||||
}) | |||||
#define __ATOMIC_ADD_GCC(TYPE) \ | |||||
static __inline void \ | |||||
atomic_add_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_add_gcc(p, v, __ATOMIC_RELAXED); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_add_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_add_gcc(p, v, __ATOMIC_ACQUIRE); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_add_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_add_gcc(p, v, __ATOMIC_RELEASE); \ | |||||
} | |||||
#define __ATOMIC_ADD_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline void \ | |||||
atomic_add_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_add_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_add_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_add_acq_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_add_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_add_rel_##TYPE(p, v); \ | |||||
} | |||||
#define __atomic_subtract_gcc(p, v, f) ({ \ | |||||
__atomic_sub_fetch(p, v, f); \ | |||||
}) | |||||
#define __ATOMIC_SUBTRACT_GCC(TYPE) \ | |||||
static __inline void \ | |||||
atomic_subtract_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_subtract_gcc(p, v, __ATOMIC_RELAXED); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_subtract_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_subtract_gcc(p, v, __ATOMIC_ACQUIRE); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_subtract_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
__atomic_subtract_gcc(p, v, __ATOMIC_RELEASE); \ | |||||
} | |||||
#define __ATOMIC_SUBTRACT_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline void \ | |||||
atomic_subtract_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_subtract_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_subtract_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_subtract_acq_##TYPE(p, v); \ | |||||
} \ | |||||
\ | |||||
static __inline void \ | |||||
atomic_subtract_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
atomic_subtract_rel_##TYPE(p, v); \ | |||||
} | |||||
#define __atomic_swap_gcc(p, v, f) ({ \ | |||||
__atomic_exchange_n(p, v, f); \ | |||||
}) | |||||
#define __ATOMIC_SWAP_GCC(TYPE) \ | |||||
static __inline u_##TYPE \ | |||||
atomic_swap_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_swap_gcc(p, v, __ATOMIC_RELAXED)); \ | |||||
} \ | |||||
\ | |||||
static __inline u_##TYPE \ | |||||
atomic_swap_acq_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_swap_gcc(p, v, __ATOMIC_ACQUIRE)); \ | |||||
} \ | |||||
\ | |||||
static __inline u_##TYPE \ | |||||
atomic_swap_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (__atomic_swap_gcc(p, v, __ATOMIC_RELEASE)); \ | |||||
} | |||||
#define __ATOMIC_SWAP_ALIAS_GCC(ALIAS, TYPE) \ | |||||
static __inline u_##TYPE \ | |||||
atomic_swap_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_swap_##TYPE(p, v)); \ | |||||
} \ | |||||
\ | |||||
static __inline u_##TYPE \ | |||||
atomic_swap_acq_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_swap_acq_##TYPE(p, v)); \ | |||||
} \ | |||||
\ | |||||
static __inline u_##TYPE \ | |||||
atomic_swap_rel_##ALIAS(volatile u_##TYPE *p, u_##TYPE v) \ | |||||
{ \ | |||||
\ | |||||
return (atomic_swap_rel_##TYPE(p, v)); \ | |||||
} | |||||
#endif /* _SYS__GCC_ATOMIC_H_ */ |
Why is success_memorder always relaxed?