Index: vendor/llvm-openmp/dist-release_80/runtime/src/kmp_atomic.h =================================================================== --- vendor/llvm-openmp/dist-release_80/runtime/src/kmp_atomic.h (revision 349788) +++ vendor/llvm-openmp/dist-release_80/runtime/src/kmp_atomic.h (revision 349789) @@ -1,1776 +1,1776 @@ /* * kmp_atomic.h - ATOMIC header file */ //===----------------------------------------------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.txt for details. // //===----------------------------------------------------------------------===// #ifndef KMP_ATOMIC_H #define KMP_ATOMIC_H #include "kmp_lock.h" #include "kmp_os.h" #if OMPT_SUPPORT #include "ompt-specific.h" #endif // C++ build port. // Intel compiler does not support _Complex datatype on win. // Intel compiler supports _Complex datatype on lin and mac. // On the other side, there is a problem of stack alignment on lin_32 and mac_32 // if the rhs is cmplx80 or cmplx128 typedef'ed datatype. // The decision is: to use compiler supported _Complex type on lin and mac, // to use typedef'ed types on win. // Condition for WIN64 was modified in anticipation of 10.1 build compiler. #if defined(__cplusplus) && (KMP_OS_WINDOWS) // create shortcuts for c99 complex types // Visual Studio cannot have function parameters that have the // align __declspec attribute, so we must remove it. (Compiler Error C2719) #if KMP_COMPILER_MSVC #undef KMP_DO_ALIGN #define KMP_DO_ALIGN(alignment) /* Nothing */ #endif #if (_MSC_VER < 1600) && defined(_DEBUG) // Workaround for the problem of _DebugHeapTag unresolved external. // This problem prevented to use our static debug library for C tests // compiled with /MDd option (the library itself built with /MTd), #undef _DEBUG #define _DEBUG_TEMPORARILY_UNSET_ #endif #include template std::complex __kmp_lhs_div_rhs(const std::complex &lhs, const std::complex &rhs) { type_lhs a = lhs.real(); type_lhs b = lhs.imag(); type_rhs c = rhs.real(); type_rhs d = rhs.imag(); type_rhs den = c * c + d * d; type_rhs r = (a * c + b * d); type_rhs i = (b * c - a * d); std::complex ret(r / den, i / den); return ret; } // complex8 struct __kmp_cmplx64_t : std::complex { __kmp_cmplx64_t() : std::complex() {} __kmp_cmplx64_t(const std::complex &cd) : std::complex(cd) {} void operator/=(const __kmp_cmplx64_t &rhs) { std::complex lhs = *this; *this = __kmp_lhs_div_rhs(lhs, rhs); } __kmp_cmplx64_t operator/(const __kmp_cmplx64_t &rhs) { std::complex lhs = *this; return __kmp_lhs_div_rhs(lhs, rhs); } }; typedef struct __kmp_cmplx64_t kmp_cmplx64; // complex4 struct __kmp_cmplx32_t : std::complex { __kmp_cmplx32_t() : std::complex() {} __kmp_cmplx32_t(const std::complex &cf) : std::complex(cf) {} __kmp_cmplx32_t operator+(const __kmp_cmplx32_t &b) { std::complex lhs = *this; std::complex rhs = b; return (lhs + rhs); } __kmp_cmplx32_t operator-(const __kmp_cmplx32_t &b) { std::complex lhs = *this; std::complex rhs = b; return (lhs - rhs); } __kmp_cmplx32_t operator*(const __kmp_cmplx32_t &b) { std::complex lhs = *this; std::complex rhs = b; return (lhs * rhs); } __kmp_cmplx32_t operator+(const kmp_cmplx64 &b) { kmp_cmplx64 t = kmp_cmplx64(*this) + b; std::complex d(t); std::complex f(d); __kmp_cmplx32_t r(f); return r; } __kmp_cmplx32_t operator-(const kmp_cmplx64 &b) { kmp_cmplx64 t = kmp_cmplx64(*this) - b; std::complex d(t); std::complex f(d); __kmp_cmplx32_t r(f); return r; } __kmp_cmplx32_t operator*(const kmp_cmplx64 &b) { kmp_cmplx64 t = kmp_cmplx64(*this) * b; std::complex d(t); std::complex f(d); __kmp_cmplx32_t r(f); return r; } void operator/=(const __kmp_cmplx32_t &rhs) { std::complex lhs = *this; *this = __kmp_lhs_div_rhs(lhs, rhs); } __kmp_cmplx32_t operator/(const __kmp_cmplx32_t &rhs) { std::complex lhs = *this; return __kmp_lhs_div_rhs(lhs, rhs); } void operator/=(const kmp_cmplx64 &rhs) { std::complex lhs = *this; *this = __kmp_lhs_div_rhs(lhs, rhs); } __kmp_cmplx32_t operator/(const kmp_cmplx64 &rhs) { std::complex lhs = *this; return __kmp_lhs_div_rhs(lhs, rhs); } }; typedef struct __kmp_cmplx32_t kmp_cmplx32; // complex10 struct KMP_DO_ALIGN(16) __kmp_cmplx80_t : std::complex { __kmp_cmplx80_t() : std::complex() {} __kmp_cmplx80_t(const std::complex &cld) : std::complex(cld) {} void operator/=(const __kmp_cmplx80_t &rhs) { std::complex lhs = *this; *this = __kmp_lhs_div_rhs(lhs, rhs); } __kmp_cmplx80_t operator/(const __kmp_cmplx80_t &rhs) { std::complex lhs = *this; return __kmp_lhs_div_rhs(lhs, rhs); } }; typedef KMP_DO_ALIGN(16) struct __kmp_cmplx80_t kmp_cmplx80; // complex16 #if KMP_HAVE_QUAD struct __kmp_cmplx128_t : std::complex<_Quad> { __kmp_cmplx128_t() : std::complex<_Quad>() {} __kmp_cmplx128_t(const std::complex<_Quad> &cq) : std::complex<_Quad>(cq) {} void operator/=(const __kmp_cmplx128_t &rhs) { std::complex<_Quad> lhs = *this; *this = __kmp_lhs_div_rhs(lhs, rhs); } __kmp_cmplx128_t operator/(const __kmp_cmplx128_t &rhs) { std::complex<_Quad> lhs = *this; return __kmp_lhs_div_rhs(lhs, rhs); } }; typedef struct __kmp_cmplx128_t kmp_cmplx128; #endif /* KMP_HAVE_QUAD */ #ifdef _DEBUG_TEMPORARILY_UNSET_ #undef _DEBUG_TEMPORARILY_UNSET_ // Set it back now #define _DEBUG 1 #endif #else // create shortcuts for c99 complex types typedef float _Complex kmp_cmplx32; typedef double _Complex kmp_cmplx64; typedef long double _Complex kmp_cmplx80; #if KMP_HAVE_QUAD typedef _Quad _Complex kmp_cmplx128; #endif #endif // Compiler 12.0 changed alignment of 16 and 32-byte arguments (like _Quad // and kmp_cmplx128) on IA-32 architecture. The following aligned structures // are implemented to support the old alignment in 10.1, 11.0, 11.1 and // introduce the new alignment in 12.0. See CQ88405. #if KMP_ARCH_X86 && KMP_HAVE_QUAD // 4-byte aligned structures for backward compatibility. #pragma pack(push, 4) struct KMP_DO_ALIGN(4) Quad_a4_t { _Quad q; Quad_a4_t() : q() {} Quad_a4_t(const _Quad &cq) : q(cq) {} Quad_a4_t operator+(const Quad_a4_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a4_t)(lhs + rhs); } Quad_a4_t operator-(const Quad_a4_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a4_t)(lhs - rhs); } Quad_a4_t operator*(const Quad_a4_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a4_t)(lhs * rhs); } Quad_a4_t operator/(const Quad_a4_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a4_t)(lhs / rhs); } }; struct KMP_DO_ALIGN(4) kmp_cmplx128_a4_t { kmp_cmplx128 q; kmp_cmplx128_a4_t() : q() {} kmp_cmplx128_a4_t(const kmp_cmplx128 &c128) : q(c128) {} kmp_cmplx128_a4_t operator+(const kmp_cmplx128_a4_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a4_t)(lhs + rhs); } kmp_cmplx128_a4_t operator-(const kmp_cmplx128_a4_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a4_t)(lhs - rhs); } kmp_cmplx128_a4_t operator*(const kmp_cmplx128_a4_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a4_t)(lhs * rhs); } kmp_cmplx128_a4_t operator/(const kmp_cmplx128_a4_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a4_t)(lhs / rhs); } }; #pragma pack(pop) // New 16-byte aligned structures for 12.0 compiler. struct KMP_DO_ALIGN(16) Quad_a16_t { _Quad q; Quad_a16_t() : q() {} Quad_a16_t(const _Quad &cq) : q(cq) {} Quad_a16_t operator+(const Quad_a16_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a16_t)(lhs + rhs); } Quad_a16_t operator-(const Quad_a16_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a16_t)(lhs - rhs); } Quad_a16_t operator*(const Quad_a16_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a16_t)(lhs * rhs); } Quad_a16_t operator/(const Quad_a16_t &b) { _Quad lhs = (*this).q; _Quad rhs = b.q; return (Quad_a16_t)(lhs / rhs); } }; struct KMP_DO_ALIGN(16) kmp_cmplx128_a16_t { kmp_cmplx128 q; kmp_cmplx128_a16_t() : q() {} kmp_cmplx128_a16_t(const kmp_cmplx128 &c128) : q(c128) {} kmp_cmplx128_a16_t operator+(const kmp_cmplx128_a16_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a16_t)(lhs + rhs); } kmp_cmplx128_a16_t operator-(const kmp_cmplx128_a16_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a16_t)(lhs - rhs); } kmp_cmplx128_a16_t operator*(const kmp_cmplx128_a16_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a16_t)(lhs * rhs); } kmp_cmplx128_a16_t operator/(const kmp_cmplx128_a16_t &b) { kmp_cmplx128 lhs = (*this).q; kmp_cmplx128 rhs = b.q; return (kmp_cmplx128_a16_t)(lhs / rhs); } }; #endif #if (KMP_ARCH_X86) #define QUAD_LEGACY Quad_a4_t #define CPLX128_LEG kmp_cmplx128_a4_t #else #define QUAD_LEGACY _Quad #define CPLX128_LEG kmp_cmplx128 #endif #ifdef __cplusplus extern "C" { #endif extern int __kmp_atomic_mode; // Atomic locks can easily become contended, so we use queuing locks for them. typedef kmp_queuing_lock_t kmp_atomic_lock_t; static inline void __kmp_acquire_atomic_lock(kmp_atomic_lock_t *lck, kmp_int32 gtid) { #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( - ompt_mutex_atomic, 0, kmp_mutex_impl_queuing, (ompt_wait_id_t)lck, + ompt_mutex_atomic, 0, kmp_mutex_impl_queuing, (ompt_wait_id_t)(uintptr_t)lck, OMPT_GET_RETURN_ADDRESS(0)); } #endif __kmp_acquire_queuing_lock(lck, gtid); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_atomic, (ompt_wait_id_t)lck, OMPT_GET_RETURN_ADDRESS(0)); + ompt_mutex_atomic, (ompt_wait_id_t)(uintptr_t)lck, OMPT_GET_RETURN_ADDRESS(0)); } #endif } static inline int __kmp_test_atomic_lock(kmp_atomic_lock_t *lck, kmp_int32 gtid) { return __kmp_test_queuing_lock(lck, gtid); } static inline void __kmp_release_atomic_lock(kmp_atomic_lock_t *lck, kmp_int32 gtid) { __kmp_release_queuing_lock(lck, gtid); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_mutex_released) { ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_atomic, (ompt_wait_id_t)lck, OMPT_GET_RETURN_ADDRESS(0)); + ompt_mutex_atomic, (ompt_wait_id_t)(uintptr_t)lck, OMPT_GET_RETURN_ADDRESS(0)); } #endif } static inline void __kmp_init_atomic_lock(kmp_atomic_lock_t *lck) { __kmp_init_queuing_lock(lck); } static inline void __kmp_destroy_atomic_lock(kmp_atomic_lock_t *lck) { __kmp_destroy_queuing_lock(lck); } // Global Locks extern kmp_atomic_lock_t __kmp_atomic_lock; /* Control access to all user coded atomics in Gnu compat mode */ extern kmp_atomic_lock_t __kmp_atomic_lock_1i; /* Control access to all user coded atomics for 1-byte fixed data types */ extern kmp_atomic_lock_t __kmp_atomic_lock_2i; /* Control access to all user coded atomics for 2-byte fixed data types */ extern kmp_atomic_lock_t __kmp_atomic_lock_4i; /* Control access to all user coded atomics for 4-byte fixed data types */ extern kmp_atomic_lock_t __kmp_atomic_lock_4r; /* Control access to all user coded atomics for kmp_real32 data type */ extern kmp_atomic_lock_t __kmp_atomic_lock_8i; /* Control access to all user coded atomics for 8-byte fixed data types */ extern kmp_atomic_lock_t __kmp_atomic_lock_8r; /* Control access to all user coded atomics for kmp_real64 data type */ extern kmp_atomic_lock_t __kmp_atomic_lock_8c; /* Control access to all user coded atomics for complex byte data type */ extern kmp_atomic_lock_t __kmp_atomic_lock_10r; /* Control access to all user coded atomics for long double data type */ extern kmp_atomic_lock_t __kmp_atomic_lock_16r; /* Control access to all user coded atomics for _Quad data type */ extern kmp_atomic_lock_t __kmp_atomic_lock_16c; /* Control access to all user coded atomics for double complex data type*/ extern kmp_atomic_lock_t __kmp_atomic_lock_20c; /* Control access to all user coded atomics for long double complex type*/ extern kmp_atomic_lock_t __kmp_atomic_lock_32c; /* Control access to all user coded atomics for _Quad complex data type */ // Below routines for atomic UPDATE are listed // 1-byte void __kmpc_atomic_fixed1_add(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_andb(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_div(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1u_div(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs); void __kmpc_atomic_fixed1_mul(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_orb(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_shl(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_shr(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1u_shr(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs); void __kmpc_atomic_fixed1_sub(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_xor(ident_t *id_ref, int gtid, char *lhs, char rhs); // 2-byte void __kmpc_atomic_fixed2_add(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_andb(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_div(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2u_div(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs); void __kmpc_atomic_fixed2_mul(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_orb(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_shl(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_shr(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2u_shr(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs); void __kmpc_atomic_fixed2_sub(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_xor(ident_t *id_ref, int gtid, short *lhs, short rhs); // 4-byte add / sub fixed void __kmpc_atomic_fixed4_add(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_sub(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); // 4-byte add / sub float void __kmpc_atomic_float4_add(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs); void __kmpc_atomic_float4_sub(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs); // 8-byte add / sub fixed void __kmpc_atomic_fixed8_add(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_sub(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); // 8-byte add / sub float void __kmpc_atomic_float8_add(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs); void __kmpc_atomic_float8_sub(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs); // 4-byte fixed void __kmpc_atomic_fixed4_andb(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_div(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4u_div(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs); void __kmpc_atomic_fixed4_mul(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_orb(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_shl(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_shr(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4u_shr(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs); void __kmpc_atomic_fixed4_xor(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); // 8-byte fixed void __kmpc_atomic_fixed8_andb(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_div(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8u_div(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs); void __kmpc_atomic_fixed8_mul(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_orb(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_shl(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_shr(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8u_shr(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs); void __kmpc_atomic_fixed8_xor(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); // 4-byte float void __kmpc_atomic_float4_div(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs); void __kmpc_atomic_float4_mul(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs); // 8-byte float void __kmpc_atomic_float8_div(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs); void __kmpc_atomic_float8_mul(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs); // 1-, 2-, 4-, 8-byte logical (&&, ||) void __kmpc_atomic_fixed1_andl(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_orl(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed2_andl(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_orl(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed4_andl(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_orl(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed8_andl(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_orl(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); // MIN / MAX void __kmpc_atomic_fixed1_max(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_min(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed2_max(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_min(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed4_max(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_min(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed8_max(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_min(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_float4_max(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs); void __kmpc_atomic_float4_min(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs); void __kmpc_atomic_float8_max(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs); void __kmpc_atomic_float8_min(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs); #if KMP_HAVE_QUAD void __kmpc_atomic_float16_max(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); void __kmpc_atomic_float16_min(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); #if (KMP_ARCH_X86) // Routines with 16-byte arguments aligned to 16-byte boundary; IA-32 // architecture only void __kmpc_atomic_float16_max_a16(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); void __kmpc_atomic_float16_min_a16(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); #endif #endif // .NEQV. (same as xor) void __kmpc_atomic_fixed1_neqv(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed2_neqv(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed4_neqv(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed8_neqv(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); // .EQV. (same as ~xor) void __kmpc_atomic_fixed1_eqv(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed2_eqv(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed4_eqv(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed8_eqv(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); // long double type void __kmpc_atomic_float10_add(ident_t *id_ref, int gtid, long double *lhs, long double rhs); void __kmpc_atomic_float10_sub(ident_t *id_ref, int gtid, long double *lhs, long double rhs); void __kmpc_atomic_float10_mul(ident_t *id_ref, int gtid, long double *lhs, long double rhs); void __kmpc_atomic_float10_div(ident_t *id_ref, int gtid, long double *lhs, long double rhs); // _Quad type #if KMP_HAVE_QUAD void __kmpc_atomic_float16_add(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); void __kmpc_atomic_float16_sub(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); void __kmpc_atomic_float16_mul(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); void __kmpc_atomic_float16_div(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); #if (KMP_ARCH_X86) // Routines with 16-byte arguments aligned to 16-byte boundary void __kmpc_atomic_float16_add_a16(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); void __kmpc_atomic_float16_sub_a16(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); void __kmpc_atomic_float16_mul_a16(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); void __kmpc_atomic_float16_div_a16(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); #endif #endif // routines for complex types void __kmpc_atomic_cmplx4_add(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs); void __kmpc_atomic_cmplx4_sub(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs); void __kmpc_atomic_cmplx4_mul(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs); void __kmpc_atomic_cmplx4_div(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs); void __kmpc_atomic_cmplx8_add(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx8_sub(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx8_mul(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx8_div(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx10_add(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); void __kmpc_atomic_cmplx10_sub(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); void __kmpc_atomic_cmplx10_mul(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); void __kmpc_atomic_cmplx10_div(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); #if KMP_HAVE_QUAD void __kmpc_atomic_cmplx16_add(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); void __kmpc_atomic_cmplx16_sub(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); void __kmpc_atomic_cmplx16_mul(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); void __kmpc_atomic_cmplx16_div(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); #if (KMP_ARCH_X86) // Routines with 16-byte arguments aligned to 16-byte boundary void __kmpc_atomic_cmplx16_add_a16(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); void __kmpc_atomic_cmplx16_sub_a16(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); void __kmpc_atomic_cmplx16_mul_a16(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); void __kmpc_atomic_cmplx16_div_a16(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); #endif #endif #if OMP_40_ENABLED // OpenMP 4.0: x = expr binop x for non-commutative operations. // Supported only on IA-32 architecture and Intel(R) 64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 void __kmpc_atomic_fixed1_sub_rev(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_div_rev(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1u_div_rev(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs); void __kmpc_atomic_fixed1_shl_rev(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1_shr_rev(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed1u_shr_rev(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs); void __kmpc_atomic_fixed2_sub_rev(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_div_rev(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2u_div_rev(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs); void __kmpc_atomic_fixed2_shl_rev(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2_shr_rev(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed2u_shr_rev(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs); void __kmpc_atomic_fixed4_sub_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_div_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4u_div_rev(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs); void __kmpc_atomic_fixed4_shl_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4_shr_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed4u_shr_rev(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs); void __kmpc_atomic_fixed8_sub_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_div_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8u_div_rev(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs); void __kmpc_atomic_fixed8_shl_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8_shr_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_fixed8u_shr_rev(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs); void __kmpc_atomic_float4_sub_rev(ident_t *id_ref, int gtid, float *lhs, float rhs); void __kmpc_atomic_float4_div_rev(ident_t *id_ref, int gtid, float *lhs, float rhs); void __kmpc_atomic_float8_sub_rev(ident_t *id_ref, int gtid, double *lhs, double rhs); void __kmpc_atomic_float8_div_rev(ident_t *id_ref, int gtid, double *lhs, double rhs); void __kmpc_atomic_float10_sub_rev(ident_t *id_ref, int gtid, long double *lhs, long double rhs); void __kmpc_atomic_float10_div_rev(ident_t *id_ref, int gtid, long double *lhs, long double rhs); #if KMP_HAVE_QUAD void __kmpc_atomic_float16_sub_rev(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); void __kmpc_atomic_float16_div_rev(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); #endif void __kmpc_atomic_cmplx4_sub_rev(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs); void __kmpc_atomic_cmplx4_div_rev(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs); void __kmpc_atomic_cmplx8_sub_rev(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx8_div_rev(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx10_sub_rev(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); void __kmpc_atomic_cmplx10_div_rev(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); #if KMP_HAVE_QUAD void __kmpc_atomic_cmplx16_sub_rev(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); void __kmpc_atomic_cmplx16_div_rev(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); #if (KMP_ARCH_X86) // Routines with 16-byte arguments aligned to 16-byte boundary void __kmpc_atomic_float16_sub_a16_rev(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); void __kmpc_atomic_float16_div_a16_rev(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); void __kmpc_atomic_cmplx16_sub_a16_rev(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); void __kmpc_atomic_cmplx16_div_a16_rev(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); #endif #endif // KMP_HAVE_QUAD #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 #endif // OMP_40_ENABLED // routines for mixed types // RHS=float8 void __kmpc_atomic_fixed1_mul_float8(ident_t *id_ref, int gtid, char *lhs, kmp_real64 rhs); void __kmpc_atomic_fixed1_div_float8(ident_t *id_ref, int gtid, char *lhs, kmp_real64 rhs); void __kmpc_atomic_fixed2_mul_float8(ident_t *id_ref, int gtid, short *lhs, kmp_real64 rhs); void __kmpc_atomic_fixed2_div_float8(ident_t *id_ref, int gtid, short *lhs, kmp_real64 rhs); void __kmpc_atomic_fixed4_mul_float8(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_real64 rhs); void __kmpc_atomic_fixed4_div_float8(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_real64 rhs); void __kmpc_atomic_fixed8_mul_float8(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_real64 rhs); void __kmpc_atomic_fixed8_div_float8(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_real64 rhs); void __kmpc_atomic_float4_add_float8(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real64 rhs); void __kmpc_atomic_float4_sub_float8(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real64 rhs); void __kmpc_atomic_float4_mul_float8(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real64 rhs); void __kmpc_atomic_float4_div_float8(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real64 rhs); // RHS=float16 (deprecated, to be removed when we are sure the compiler does not // use them) #if KMP_HAVE_QUAD void __kmpc_atomic_fixed1_add_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs); void __kmpc_atomic_fixed1u_add_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs); void __kmpc_atomic_fixed1_sub_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs); void __kmpc_atomic_fixed1u_sub_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs); void __kmpc_atomic_fixed1_mul_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs); void __kmpc_atomic_fixed1u_mul_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs); void __kmpc_atomic_fixed1_div_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs); void __kmpc_atomic_fixed1u_div_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs); void __kmpc_atomic_fixed2_add_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs); void __kmpc_atomic_fixed2u_add_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs); void __kmpc_atomic_fixed2_sub_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs); void __kmpc_atomic_fixed2u_sub_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs); void __kmpc_atomic_fixed2_mul_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs); void __kmpc_atomic_fixed2u_mul_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs); void __kmpc_atomic_fixed2_div_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs); void __kmpc_atomic_fixed2u_div_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs); void __kmpc_atomic_fixed4_add_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4u_add_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4_sub_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4u_sub_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4_mul_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4u_mul_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4_div_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4u_div_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs); void __kmpc_atomic_fixed8_add_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8u_add_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8_sub_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8u_sub_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8_mul_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8u_mul_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8_div_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8u_div_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs); void __kmpc_atomic_float4_add_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs); void __kmpc_atomic_float4_sub_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs); void __kmpc_atomic_float4_mul_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs); void __kmpc_atomic_float4_div_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs); void __kmpc_atomic_float8_add_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs); void __kmpc_atomic_float8_sub_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs); void __kmpc_atomic_float8_mul_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs); void __kmpc_atomic_float8_div_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs); void __kmpc_atomic_float10_add_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs); void __kmpc_atomic_float10_sub_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs); void __kmpc_atomic_float10_mul_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs); void __kmpc_atomic_float10_div_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs); // Reverse operations void __kmpc_atomic_fixed1_sub_rev_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs); void __kmpc_atomic_fixed1u_sub_rev_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs); void __kmpc_atomic_fixed1_div_rev_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs); void __kmpc_atomic_fixed1u_div_rev_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs); void __kmpc_atomic_fixed2_sub_rev_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs); void __kmpc_atomic_fixed2u_sub_rev_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs); void __kmpc_atomic_fixed2_div_rev_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs); void __kmpc_atomic_fixed2u_div_rev_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs); void __kmpc_atomic_fixed4_sub_rev_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4u_sub_rev_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4_div_rev_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs); void __kmpc_atomic_fixed4u_div_rev_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs); void __kmpc_atomic_fixed8_sub_rev_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8u_sub_rev_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8_div_rev_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs); void __kmpc_atomic_fixed8u_div_rev_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs); void __kmpc_atomic_float4_sub_rev_fp(ident_t *id_ref, int gtid, float *lhs, _Quad rhs); void __kmpc_atomic_float4_div_rev_fp(ident_t *id_ref, int gtid, float *lhs, _Quad rhs); void __kmpc_atomic_float8_sub_rev_fp(ident_t *id_ref, int gtid, double *lhs, _Quad rhs); void __kmpc_atomic_float8_div_rev_fp(ident_t *id_ref, int gtid, double *lhs, _Quad rhs); void __kmpc_atomic_float10_sub_rev_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs); void __kmpc_atomic_float10_div_rev_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs); #endif // KMP_HAVE_QUAD // RHS=cmplx8 void __kmpc_atomic_cmplx4_add_cmplx8(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx4_sub_cmplx8(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx4_mul_cmplx8(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx4_div_cmplx8(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx64 rhs); // generic atomic routines void __kmpc_atomic_1(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); void __kmpc_atomic_2(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); void __kmpc_atomic_4(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); void __kmpc_atomic_8(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); void __kmpc_atomic_10(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); void __kmpc_atomic_16(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); void __kmpc_atomic_20(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); void __kmpc_atomic_32(ident_t *id_ref, int gtid, void *lhs, void *rhs, void (*f)(void *, void *, void *)); // READ, WRITE, CAPTURE are supported only on IA-32 architecture and Intel(R) 64 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 // Below routines for atomic READ are listed char __kmpc_atomic_fixed1_rd(ident_t *id_ref, int gtid, char *loc); short __kmpc_atomic_fixed2_rd(ident_t *id_ref, int gtid, short *loc); kmp_int32 __kmpc_atomic_fixed4_rd(ident_t *id_ref, int gtid, kmp_int32 *loc); kmp_int64 __kmpc_atomic_fixed8_rd(ident_t *id_ref, int gtid, kmp_int64 *loc); kmp_real32 __kmpc_atomic_float4_rd(ident_t *id_ref, int gtid, kmp_real32 *loc); kmp_real64 __kmpc_atomic_float8_rd(ident_t *id_ref, int gtid, kmp_real64 *loc); long double __kmpc_atomic_float10_rd(ident_t *id_ref, int gtid, long double *loc); #if KMP_HAVE_QUAD QUAD_LEGACY __kmpc_atomic_float16_rd(ident_t *id_ref, int gtid, QUAD_LEGACY *loc); #endif // Fix for CQ220361: cmplx4 READ will return void on Windows* OS; read value // will be returned through an additional parameter #if (KMP_OS_WINDOWS) void __kmpc_atomic_cmplx4_rd(kmp_cmplx32 *out, ident_t *id_ref, int gtid, kmp_cmplx32 *loc); #else kmp_cmplx32 __kmpc_atomic_cmplx4_rd(ident_t *id_ref, int gtid, kmp_cmplx32 *loc); #endif kmp_cmplx64 __kmpc_atomic_cmplx8_rd(ident_t *id_ref, int gtid, kmp_cmplx64 *loc); kmp_cmplx80 __kmpc_atomic_cmplx10_rd(ident_t *id_ref, int gtid, kmp_cmplx80 *loc); #if KMP_HAVE_QUAD CPLX128_LEG __kmpc_atomic_cmplx16_rd(ident_t *id_ref, int gtid, CPLX128_LEG *loc); #if (KMP_ARCH_X86) // Routines with 16-byte arguments aligned to 16-byte boundary Quad_a16_t __kmpc_atomic_float16_a16_rd(ident_t *id_ref, int gtid, Quad_a16_t *loc); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_a16_rd(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *loc); #endif #endif // Below routines for atomic WRITE are listed void __kmpc_atomic_fixed1_wr(ident_t *id_ref, int gtid, char *lhs, char rhs); void __kmpc_atomic_fixed2_wr(ident_t *id_ref, int gtid, short *lhs, short rhs); void __kmpc_atomic_fixed4_wr(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); void __kmpc_atomic_fixed8_wr(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); void __kmpc_atomic_float4_wr(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs); void __kmpc_atomic_float8_wr(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs); void __kmpc_atomic_float10_wr(ident_t *id_ref, int gtid, long double *lhs, long double rhs); #if KMP_HAVE_QUAD void __kmpc_atomic_float16_wr(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); #endif void __kmpc_atomic_cmplx4_wr(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs); void __kmpc_atomic_cmplx8_wr(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); void __kmpc_atomic_cmplx10_wr(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); #if KMP_HAVE_QUAD void __kmpc_atomic_cmplx16_wr(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); #if (KMP_ARCH_X86) // Routines with 16-byte arguments aligned to 16-byte boundary void __kmpc_atomic_float16_a16_wr(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); void __kmpc_atomic_cmplx16_a16_wr(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); #endif #endif // Below routines for atomic CAPTURE are listed // 1-byte char __kmpc_atomic_fixed1_add_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_andb_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_div_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); unsigned char __kmpc_atomic_fixed1u_div_cpt(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs, int flag); char __kmpc_atomic_fixed1_mul_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_orb_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_shl_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_shr_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); unsigned char __kmpc_atomic_fixed1u_shr_cpt(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs, int flag); char __kmpc_atomic_fixed1_sub_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_xor_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); // 2-byte short __kmpc_atomic_fixed2_add_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_andb_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_div_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); unsigned short __kmpc_atomic_fixed2u_div_cpt(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs, int flag); short __kmpc_atomic_fixed2_mul_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_orb_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_shl_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_shr_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); unsigned short __kmpc_atomic_fixed2u_shr_cpt(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs, int flag); short __kmpc_atomic_fixed2_sub_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_xor_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); // 4-byte add / sub fixed kmp_int32 __kmpc_atomic_fixed4_add_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_sub_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); // 4-byte add / sub float kmp_real32 __kmpc_atomic_float4_add_cpt(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs, int flag); kmp_real32 __kmpc_atomic_float4_sub_cpt(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs, int flag); // 8-byte add / sub fixed kmp_int64 __kmpc_atomic_fixed8_add_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_sub_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); // 8-byte add / sub float kmp_real64 __kmpc_atomic_float8_add_cpt(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs, int flag); kmp_real64 __kmpc_atomic_float8_sub_cpt(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs, int flag); // 4-byte fixed kmp_int32 __kmpc_atomic_fixed4_andb_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_div_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_div_cpt(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_mul_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_orb_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_shl_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_shr_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_shr_cpt(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_xor_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); // 8-byte fixed kmp_int64 __kmpc_atomic_fixed8_andb_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_div_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_div_cpt(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_mul_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_orb_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_shl_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_shr_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_shr_cpt(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_xor_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); // 4-byte float kmp_real32 __kmpc_atomic_float4_div_cpt(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs, int flag); kmp_real32 __kmpc_atomic_float4_mul_cpt(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs, int flag); // 8-byte float kmp_real64 __kmpc_atomic_float8_div_cpt(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs, int flag); kmp_real64 __kmpc_atomic_float8_mul_cpt(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs, int flag); // 1-, 2-, 4-, 8-byte logical (&&, ||) char __kmpc_atomic_fixed1_andl_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_orl_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); short __kmpc_atomic_fixed2_andl_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_orl_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_andl_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_orl_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_andl_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_orl_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); // MIN / MAX char __kmpc_atomic_fixed1_max_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_min_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); short __kmpc_atomic_fixed2_max_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_min_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_max_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_min_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_max_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_min_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_real32 __kmpc_atomic_float4_max_cpt(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs, int flag); kmp_real32 __kmpc_atomic_float4_min_cpt(ident_t *id_ref, int gtid, kmp_real32 *lhs, kmp_real32 rhs, int flag); kmp_real64 __kmpc_atomic_float8_max_cpt(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs, int flag); kmp_real64 __kmpc_atomic_float8_min_cpt(ident_t *id_ref, int gtid, kmp_real64 *lhs, kmp_real64 rhs, int flag); #if KMP_HAVE_QUAD QUAD_LEGACY __kmpc_atomic_float16_max_cpt(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); QUAD_LEGACY __kmpc_atomic_float16_min_cpt(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); #endif // .NEQV. (same as xor) char __kmpc_atomic_fixed1_neqv_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); short __kmpc_atomic_fixed2_neqv_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_neqv_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_neqv_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); // .EQV. (same as ~xor) char __kmpc_atomic_fixed1_eqv_cpt(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); short __kmpc_atomic_fixed2_eqv_cpt(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_eqv_cpt(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_eqv_cpt(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); // long double type long double __kmpc_atomic_float10_add_cpt(ident_t *id_ref, int gtid, long double *lhs, long double rhs, int flag); long double __kmpc_atomic_float10_sub_cpt(ident_t *id_ref, int gtid, long double *lhs, long double rhs, int flag); long double __kmpc_atomic_float10_mul_cpt(ident_t *id_ref, int gtid, long double *lhs, long double rhs, int flag); long double __kmpc_atomic_float10_div_cpt(ident_t *id_ref, int gtid, long double *lhs, long double rhs, int flag); #if KMP_HAVE_QUAD // _Quad type QUAD_LEGACY __kmpc_atomic_float16_add_cpt(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); QUAD_LEGACY __kmpc_atomic_float16_sub_cpt(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); QUAD_LEGACY __kmpc_atomic_float16_mul_cpt(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); QUAD_LEGACY __kmpc_atomic_float16_div_cpt(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); #endif // routines for complex types // Workaround for cmplx4 routines - return void; captured value is returned via // the argument void __kmpc_atomic_cmplx4_add_cpt(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs, kmp_cmplx32 *out, int flag); void __kmpc_atomic_cmplx4_sub_cpt(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs, kmp_cmplx32 *out, int flag); void __kmpc_atomic_cmplx4_mul_cpt(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs, kmp_cmplx32 *out, int flag); void __kmpc_atomic_cmplx4_div_cpt(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs, kmp_cmplx32 *out, int flag); kmp_cmplx64 __kmpc_atomic_cmplx8_add_cpt(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs, int flag); kmp_cmplx64 __kmpc_atomic_cmplx8_sub_cpt(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs, int flag); kmp_cmplx64 __kmpc_atomic_cmplx8_mul_cpt(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs, int flag); kmp_cmplx64 __kmpc_atomic_cmplx8_div_cpt(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs, int flag); kmp_cmplx80 __kmpc_atomic_cmplx10_add_cpt(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs, int flag); kmp_cmplx80 __kmpc_atomic_cmplx10_sub_cpt(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs, int flag); kmp_cmplx80 __kmpc_atomic_cmplx10_mul_cpt(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs, int flag); kmp_cmplx80 __kmpc_atomic_cmplx10_div_cpt(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs, int flag); #if KMP_HAVE_QUAD CPLX128_LEG __kmpc_atomic_cmplx16_add_cpt(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs, int flag); CPLX128_LEG __kmpc_atomic_cmplx16_sub_cpt(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs, int flag); CPLX128_LEG __kmpc_atomic_cmplx16_mul_cpt(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs, int flag); CPLX128_LEG __kmpc_atomic_cmplx16_div_cpt(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs, int flag); #if (KMP_ARCH_X86) // Routines with 16-byte arguments aligned to 16-byte boundary Quad_a16_t __kmpc_atomic_float16_add_a16_cpt(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); Quad_a16_t __kmpc_atomic_float16_sub_a16_cpt(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); Quad_a16_t __kmpc_atomic_float16_mul_a16_cpt(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); Quad_a16_t __kmpc_atomic_float16_div_a16_cpt(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); Quad_a16_t __kmpc_atomic_float16_max_a16_cpt(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); Quad_a16_t __kmpc_atomic_float16_min_a16_cpt(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_add_a16_cpt(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs, int flag); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_sub_a16_cpt(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs, int flag); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_mul_a16_cpt(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs, int flag); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_div_a16_cpt(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs, int flag); #endif #endif void __kmpc_atomic_start(void); void __kmpc_atomic_end(void); #if OMP_40_ENABLED // OpenMP 4.0: v = x = expr binop x; { v = x; x = expr binop x; } { x = expr // binop x; v = x; } for non-commutative operations. char __kmpc_atomic_fixed1_sub_cpt_rev(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_div_cpt_rev(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); unsigned char __kmpc_atomic_fixed1u_div_cpt_rev(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs, int flag); char __kmpc_atomic_fixed1_shl_cpt_rev(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); char __kmpc_atomic_fixed1_shr_cpt_rev(ident_t *id_ref, int gtid, char *lhs, char rhs, int flag); unsigned char __kmpc_atomic_fixed1u_shr_cpt_rev(ident_t *id_ref, int gtid, unsigned char *lhs, unsigned char rhs, int flag); short __kmpc_atomic_fixed2_sub_cpt_rev(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_div_cpt_rev(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); unsigned short __kmpc_atomic_fixed2u_div_cpt_rev(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs, int flag); short __kmpc_atomic_fixed2_shl_cpt_rev(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); short __kmpc_atomic_fixed2_shr_cpt_rev(ident_t *id_ref, int gtid, short *lhs, short rhs, int flag); unsigned short __kmpc_atomic_fixed2u_shr_cpt_rev(ident_t *id_ref, int gtid, unsigned short *lhs, unsigned short rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_sub_cpt_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_div_cpt_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_div_cpt_rev(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_shl_cpt_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_shr_cpt_rev(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_shr_cpt_rev(ident_t *id_ref, int gtid, kmp_uint32 *lhs, kmp_uint32 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_sub_cpt_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_div_cpt_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_div_cpt_rev(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_shl_cpt_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_shr_cpt_rev(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_shr_cpt_rev(ident_t *id_ref, int gtid, kmp_uint64 *lhs, kmp_uint64 rhs, int flag); float __kmpc_atomic_float4_sub_cpt_rev(ident_t *id_ref, int gtid, float *lhs, float rhs, int flag); float __kmpc_atomic_float4_div_cpt_rev(ident_t *id_ref, int gtid, float *lhs, float rhs, int flag); double __kmpc_atomic_float8_sub_cpt_rev(ident_t *id_ref, int gtid, double *lhs, double rhs, int flag); double __kmpc_atomic_float8_div_cpt_rev(ident_t *id_ref, int gtid, double *lhs, double rhs, int flag); long double __kmpc_atomic_float10_sub_cpt_rev(ident_t *id_ref, int gtid, long double *lhs, long double rhs, int flag); long double __kmpc_atomic_float10_div_cpt_rev(ident_t *id_ref, int gtid, long double *lhs, long double rhs, int flag); #if KMP_HAVE_QUAD QUAD_LEGACY __kmpc_atomic_float16_sub_cpt_rev(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); QUAD_LEGACY __kmpc_atomic_float16_div_cpt_rev(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs, int flag); #endif // Workaround for cmplx4 routines - return void; captured value is returned via // the argument void __kmpc_atomic_cmplx4_sub_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs, kmp_cmplx32 *out, int flag); void __kmpc_atomic_cmplx4_div_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs, kmp_cmplx32 *out, int flag); kmp_cmplx64 __kmpc_atomic_cmplx8_sub_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs, int flag); kmp_cmplx64 __kmpc_atomic_cmplx8_div_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs, int flag); kmp_cmplx80 __kmpc_atomic_cmplx10_sub_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs, int flag); kmp_cmplx80 __kmpc_atomic_cmplx10_div_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs, int flag); #if KMP_HAVE_QUAD CPLX128_LEG __kmpc_atomic_cmplx16_sub_cpt_rev(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs, int flag); CPLX128_LEG __kmpc_atomic_cmplx16_div_cpt_rev(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs, int flag); #if (KMP_ARCH_X86) Quad_a16_t __kmpc_atomic_float16_sub_a16_cpt_rev(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); Quad_a16_t __kmpc_atomic_float16_div_a16_cpt_rev(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs, int flag); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_sub_a16_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs, int flag); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_div_a16_cpt_rev(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs, int flag); #endif #endif // OpenMP 4.0 Capture-write (swap): {v = x; x = expr;} char __kmpc_atomic_fixed1_swp(ident_t *id_ref, int gtid, char *lhs, char rhs); short __kmpc_atomic_fixed2_swp(ident_t *id_ref, int gtid, short *lhs, short rhs); kmp_int32 __kmpc_atomic_fixed4_swp(ident_t *id_ref, int gtid, kmp_int32 *lhs, kmp_int32 rhs); kmp_int64 __kmpc_atomic_fixed8_swp(ident_t *id_ref, int gtid, kmp_int64 *lhs, kmp_int64 rhs); float __kmpc_atomic_float4_swp(ident_t *id_ref, int gtid, float *lhs, float rhs); double __kmpc_atomic_float8_swp(ident_t *id_ref, int gtid, double *lhs, double rhs); long double __kmpc_atomic_float10_swp(ident_t *id_ref, int gtid, long double *lhs, long double rhs); #if KMP_HAVE_QUAD QUAD_LEGACY __kmpc_atomic_float16_swp(ident_t *id_ref, int gtid, QUAD_LEGACY *lhs, QUAD_LEGACY rhs); #endif // !!! TODO: check if we need a workaround here void __kmpc_atomic_cmplx4_swp(ident_t *id_ref, int gtid, kmp_cmplx32 *lhs, kmp_cmplx32 rhs, kmp_cmplx32 *out); // kmp_cmplx32 __kmpc_atomic_cmplx4_swp( ident_t *id_ref, int gtid, // kmp_cmplx32 * lhs, kmp_cmplx32 rhs ); kmp_cmplx64 __kmpc_atomic_cmplx8_swp(ident_t *id_ref, int gtid, kmp_cmplx64 *lhs, kmp_cmplx64 rhs); kmp_cmplx80 __kmpc_atomic_cmplx10_swp(ident_t *id_ref, int gtid, kmp_cmplx80 *lhs, kmp_cmplx80 rhs); #if KMP_HAVE_QUAD CPLX128_LEG __kmpc_atomic_cmplx16_swp(ident_t *id_ref, int gtid, CPLX128_LEG *lhs, CPLX128_LEG rhs); #if (KMP_ARCH_X86) Quad_a16_t __kmpc_atomic_float16_a16_swp(ident_t *id_ref, int gtid, Quad_a16_t *lhs, Quad_a16_t rhs); kmp_cmplx128_a16_t __kmpc_atomic_cmplx16_a16_swp(ident_t *id_ref, int gtid, kmp_cmplx128_a16_t *lhs, kmp_cmplx128_a16_t rhs); #endif #endif // Capture routines for mixed types (RHS=float16) #if KMP_HAVE_QUAD char __kmpc_atomic_fixed1_add_cpt_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs, int flag); char __kmpc_atomic_fixed1_sub_cpt_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs, int flag); char __kmpc_atomic_fixed1_mul_cpt_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs, int flag); char __kmpc_atomic_fixed1_div_cpt_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs, int flag); unsigned char __kmpc_atomic_fixed1u_add_cpt_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs, int flag); unsigned char __kmpc_atomic_fixed1u_sub_cpt_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs, int flag); unsigned char __kmpc_atomic_fixed1u_mul_cpt_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs, int flag); unsigned char __kmpc_atomic_fixed1u_div_cpt_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs, int flag); short __kmpc_atomic_fixed2_add_cpt_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs, int flag); short __kmpc_atomic_fixed2_sub_cpt_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs, int flag); short __kmpc_atomic_fixed2_mul_cpt_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs, int flag); short __kmpc_atomic_fixed2_div_cpt_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs, int flag); unsigned short __kmpc_atomic_fixed2u_add_cpt_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs, int flag); unsigned short __kmpc_atomic_fixed2u_sub_cpt_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs, int flag); unsigned short __kmpc_atomic_fixed2u_mul_cpt_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs, int flag); unsigned short __kmpc_atomic_fixed2u_div_cpt_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_add_cpt_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_sub_cpt_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_mul_cpt_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_div_cpt_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_add_cpt_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_sub_cpt_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_mul_cpt_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_div_cpt_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_add_cpt_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_sub_cpt_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_mul_cpt_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_div_cpt_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_add_cpt_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_sub_cpt_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_mul_cpt_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_div_cpt_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs, int flag); float __kmpc_atomic_float4_add_cpt_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs, int flag); float __kmpc_atomic_float4_sub_cpt_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs, int flag); float __kmpc_atomic_float4_mul_cpt_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs, int flag); float __kmpc_atomic_float4_div_cpt_fp(ident_t *id_ref, int gtid, kmp_real32 *lhs, _Quad rhs, int flag); double __kmpc_atomic_float8_add_cpt_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs, int flag); double __kmpc_atomic_float8_sub_cpt_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs, int flag); double __kmpc_atomic_float8_mul_cpt_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs, int flag); double __kmpc_atomic_float8_div_cpt_fp(ident_t *id_ref, int gtid, kmp_real64 *lhs, _Quad rhs, int flag); long double __kmpc_atomic_float10_add_cpt_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs, int flag); long double __kmpc_atomic_float10_sub_cpt_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs, int flag); long double __kmpc_atomic_float10_mul_cpt_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs, int flag); long double __kmpc_atomic_float10_div_cpt_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs, int flag); char __kmpc_atomic_fixed1_sub_cpt_rev_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs, int flag); unsigned char __kmpc_atomic_fixed1u_sub_cpt_rev_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs, int flag); char __kmpc_atomic_fixed1_div_cpt_rev_fp(ident_t *id_ref, int gtid, char *lhs, _Quad rhs, int flag); unsigned char __kmpc_atomic_fixed1u_div_cpt_rev_fp(ident_t *id_ref, int gtid, unsigned char *lhs, _Quad rhs, int flag); short __kmpc_atomic_fixed2_sub_cpt_rev_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs, int flag); unsigned short __kmpc_atomic_fixed2u_sub_cpt_rev_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs, int flag); short __kmpc_atomic_fixed2_div_cpt_rev_fp(ident_t *id_ref, int gtid, short *lhs, _Quad rhs, int flag); unsigned short __kmpc_atomic_fixed2u_div_cpt_rev_fp(ident_t *id_ref, int gtid, unsigned short *lhs, _Quad rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_sub_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_sub_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs, int flag); kmp_int32 __kmpc_atomic_fixed4_div_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_int32 *lhs, _Quad rhs, int flag); kmp_uint32 __kmpc_atomic_fixed4u_div_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_uint32 *lhs, _Quad rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_sub_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_sub_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs, int flag); kmp_int64 __kmpc_atomic_fixed8_div_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_int64 *lhs, _Quad rhs, int flag); kmp_uint64 __kmpc_atomic_fixed8u_div_cpt_rev_fp(ident_t *id_ref, int gtid, kmp_uint64 *lhs, _Quad rhs, int flag); float __kmpc_atomic_float4_sub_cpt_rev_fp(ident_t *id_ref, int gtid, float *lhs, _Quad rhs, int flag); float __kmpc_atomic_float4_div_cpt_rev_fp(ident_t *id_ref, int gtid, float *lhs, _Quad rhs, int flag); double __kmpc_atomic_float8_sub_cpt_rev_fp(ident_t *id_ref, int gtid, double *lhs, _Quad rhs, int flag); double __kmpc_atomic_float8_div_cpt_rev_fp(ident_t *id_ref, int gtid, double *lhs, _Quad rhs, int flag); long double __kmpc_atomic_float10_sub_cpt_rev_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs, int flag); long double __kmpc_atomic_float10_div_cpt_rev_fp(ident_t *id_ref, int gtid, long double *lhs, _Quad rhs, int flag); #endif // KMP_HAVE_QUAD // End of OpenMP 4.0 capture #endif // OMP_40_ENABLED #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 /* ------------------------------------------------------------------------ */ #ifdef __cplusplus } // extern "C" #endif #endif /* KMP_ATOMIC_H */ // end of file Index: vendor/llvm-openmp/dist-release_80/runtime/src/kmp_csupport.cpp =================================================================== --- vendor/llvm-openmp/dist-release_80/runtime/src/kmp_csupport.cpp (revision 349788) +++ vendor/llvm-openmp/dist-release_80/runtime/src/kmp_csupport.cpp (revision 349789) @@ -1,4164 +1,4173 @@ /* * kmp_csupport.cpp -- kfront linkage support for OpenMP. */ //===----------------------------------------------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.txt for details. // //===----------------------------------------------------------------------===// #define __KMP_IMP #include "omp.h" /* extern "C" declarations of user-visible routines */ #include "kmp.h" #include "kmp_error.h" #include "kmp_i18n.h" #include "kmp_itt.h" #include "kmp_lock.h" #include "kmp_stats.h" #if OMPT_SUPPORT #include "ompt-specific.h" #endif #define MAX_MESSAGE 512 // flags will be used in future, e.g. to implement openmp_strict library // restrictions /*! * @ingroup STARTUP_SHUTDOWN * @param loc in source location information * @param flags in for future use (currently ignored) * * Initialize the runtime library. This call is optional; if it is not made then * it will be implicitly called by attempts to use other library functions. */ void __kmpc_begin(ident_t *loc, kmp_int32 flags) { // By default __kmpc_begin() is no-op. char *env; if ((env = getenv("KMP_INITIAL_THREAD_BIND")) != NULL && __kmp_str_match_true(env)) { __kmp_middle_initialize(); KC_TRACE(10, ("__kmpc_begin: middle initialization called\n")); } else if (__kmp_ignore_mppbeg() == FALSE) { // By default __kmp_ignore_mppbeg() returns TRUE. __kmp_internal_begin(); KC_TRACE(10, ("__kmpc_begin: called\n")); } } /*! * @ingroup STARTUP_SHUTDOWN * @param loc source location information * * Shutdown the runtime library. This is also optional, and even if called will * not do anything unless the `KMP_IGNORE_MPPEND` environment variable is set to * zero. */ void __kmpc_end(ident_t *loc) { // By default, __kmp_ignore_mppend() returns TRUE which makes __kmpc_end() // call no-op. However, this can be overridden with KMP_IGNORE_MPPEND // environment variable. If KMP_IGNORE_MPPEND is 0, __kmp_ignore_mppend() // returns FALSE and __kmpc_end() will unregister this root (it can cause // library shut down). if (__kmp_ignore_mppend() == FALSE) { KC_TRACE(10, ("__kmpc_end: called\n")); KA_TRACE(30, ("__kmpc_end\n")); __kmp_internal_end_thread(-1); } #if KMP_OS_WINDOWS && OMPT_SUPPORT // Normal exit process on Windows does not allow worker threads of the final // parallel region to finish reporting their events, so shutting down the // library here fixes the issue at least for the cases where __kmpc_end() is // placed properly. if (ompt_enabled.enabled) __kmp_internal_end_library(__kmp_gtid_get_specific()); #endif } /*! @ingroup THREAD_STATES @param loc Source location information. @return The global thread index of the active thread. This function can be called in any context. If the runtime has ony been entered at the outermost level from a single (necessarily non-OpenMP*) thread, then the thread number is that which would be returned by omp_get_thread_num() in the outermost active parallel construct. (Or zero if there is no active parallel construct, since the master thread is necessarily thread zero). If multiple non-OpenMP threads all enter an OpenMP construct then this will be a unique thread identifier among all the threads created by the OpenMP runtime (but the value cannote be defined in terms of OpenMP thread ids returned by omp_get_thread_num()). */ kmp_int32 __kmpc_global_thread_num(ident_t *loc) { kmp_int32 gtid = __kmp_entry_gtid(); KC_TRACE(10, ("__kmpc_global_thread_num: T#%d\n", gtid)); return gtid; } /*! @ingroup THREAD_STATES @param loc Source location information. @return The number of threads under control of the OpenMP* runtime This function can be called in any context. It returns the total number of threads under the control of the OpenMP runtime. That is not a number that can be determined by any OpenMP standard calls, since the library may be called from more than one non-OpenMP thread, and this reflects the total over all such calls. Similarly the runtime maintains underlying threads even when they are not active (since the cost of creating and destroying OS threads is high), this call counts all such threads even if they are not waiting for work. */ kmp_int32 __kmpc_global_num_threads(ident_t *loc) { KC_TRACE(10, ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth)); return TCR_4(__kmp_all_nth); } /*! @ingroup THREAD_STATES @param loc Source location information. @return The thread number of the calling thread in the innermost active parallel construct. */ kmp_int32 __kmpc_bound_thread_num(ident_t *loc) { KC_TRACE(10, ("__kmpc_bound_thread_num: called\n")); return __kmp_tid_from_gtid(__kmp_entry_gtid()); } /*! @ingroup THREAD_STATES @param loc Source location information. @return The number of threads in the innermost active parallel construct. */ kmp_int32 __kmpc_bound_num_threads(ident_t *loc) { KC_TRACE(10, ("__kmpc_bound_num_threads: called\n")); return __kmp_entry_thread()->th.th_team->t.t_nproc; } /*! * @ingroup DEPRECATED * @param loc location description * * This function need not be called. It always returns TRUE. */ kmp_int32 __kmpc_ok_to_fork(ident_t *loc) { #ifndef KMP_DEBUG return TRUE; #else const char *semi2; const char *semi3; int line_no; if (__kmp_par_range == 0) { return TRUE; } semi2 = loc->psource; if (semi2 == NULL) { return TRUE; } semi2 = strchr(semi2, ';'); if (semi2 == NULL) { return TRUE; } semi2 = strchr(semi2 + 1, ';'); if (semi2 == NULL) { return TRUE; } if (__kmp_par_range_filename[0]) { const char *name = semi2 - 1; while ((name > loc->psource) && (*name != '/') && (*name != ';')) { name--; } if ((*name == '/') || (*name == ';')) { name++; } if (strncmp(__kmp_par_range_filename, name, semi2 - name)) { return __kmp_par_range < 0; } } semi3 = strchr(semi2 + 1, ';'); if (__kmp_par_range_routine[0]) { if ((semi3 != NULL) && (semi3 > semi2) && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) { return __kmp_par_range < 0; } } if (KMP_SSCANF(semi3 + 1, "%d", &line_no) == 1) { if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) { return __kmp_par_range > 0; } return __kmp_par_range < 0; } return TRUE; #endif /* KMP_DEBUG */ } /*! @ingroup THREAD_STATES @param loc Source location information. @return 1 if this thread is executing inside an active parallel region, zero if not. */ kmp_int32 __kmpc_in_parallel(ident_t *loc) { return __kmp_entry_thread()->th.th_root->r.r_active; } /*! @ingroup PARALLEL @param loc source location information @param global_tid global thread number @param num_threads number of threads requested for this parallel construct Set the number of threads to be used by the next fork spawned by this thread. This call is only required if the parallel construct has a `num_threads` clause. */ void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads) { KA_TRACE(20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n", global_tid, num_threads)); __kmp_push_num_threads(loc, global_tid, num_threads); } void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid) { KA_TRACE(20, ("__kmpc_pop_num_threads: enter\n")); /* the num_threads are automatically popped */ } #if OMP_40_ENABLED void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind) { KA_TRACE(20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid, proc_bind)); __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind); } #endif /* OMP_40_ENABLED */ /*! @ingroup PARALLEL @param loc source location information @param argc total number of arguments in the ellipsis @param microtask pointer to callback routine consisting of outlined parallel construct @param ... pointers to shared variables that aren't global Do the actual fork and call the microtask in the relevant number of threads. */ void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...) { int gtid = __kmp_entry_gtid(); #if (KMP_STATS_ENABLED) // If we were in a serial region, then stop the serial timer, record // the event, and start parallel region timer stats_state_e previous_state = KMP_GET_THREAD_STATE(); if (previous_state == stats_state_e::SERIAL_REGION) { KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead); } else { KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead); } int inParallel = __kmpc_in_parallel(loc); if (inParallel) { KMP_COUNT_BLOCK(OMP_NESTED_PARALLEL); } else { KMP_COUNT_BLOCK(OMP_PARALLEL); } #endif // maybe to save thr_state is enough here { va_list ap; va_start(ap, microtask); #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { kmp_info_t *master_th = __kmp_threads[gtid]; kmp_team_t *parent_team = master_th->th.th_team; ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info; if (lwt) ompt_frame = &(lwt->ompt_task_info.frame); else { int tid = __kmp_tid_from_gtid(gtid); ompt_frame = &( parent_team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame); } ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(gtid); } #endif #if INCLUDE_SSC_MARKS SSC_MARK_FORKING(); #endif __kmp_fork_call(loc, gtid, fork_context_intel, argc, VOLATILE_CAST(microtask_t) microtask, // "wrapped" task VOLATILE_CAST(launch_t) __kmp_invoke_task_func, /* TODO: revert workaround for Intel(R) 64 tracker #96 */ #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX &ap #else ap #endif ); #if INCLUDE_SSC_MARKS SSC_MARK_JOINING(); #endif __kmp_join_call(loc, gtid #if OMPT_SUPPORT , fork_context_intel #endif ); va_end(ap); } #if KMP_STATS_ENABLED if (previous_state == stats_state_e::SERIAL_REGION) { KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial); } else { KMP_POP_PARTITIONED_TIMER(); } #endif // KMP_STATS_ENABLED } #if OMP_40_ENABLED /*! @ingroup PARALLEL @param loc source location information @param global_tid global thread number @param num_teams number of teams requested for the teams construct @param num_threads number of threads per team requested for the teams construct Set the number of teams to be used by the teams construct. This call is only required if the teams construct has a `num_teams` clause or a `thread_limit` clause (or both). */ void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads) { KA_TRACE(20, ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n", global_tid, num_teams, num_threads)); __kmp_push_num_teams(loc, global_tid, num_teams, num_threads); } /*! @ingroup PARALLEL @param loc source location information @param argc total number of arguments in the ellipsis @param microtask pointer to callback routine consisting of outlined teams construct @param ... pointers to shared variables that aren't global Do the actual fork and call the microtask in the relevant number of threads. */ void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...) { int gtid = __kmp_entry_gtid(); kmp_info_t *this_thr = __kmp_threads[gtid]; va_list ap; va_start(ap, microtask); KMP_COUNT_BLOCK(OMP_TEAMS); // remember teams entry point and nesting level this_thr->th.th_teams_microtask = microtask; this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level; // AC: can be >0 on host #if OMPT_SUPPORT kmp_team_t *parent_team = this_thr->th.th_team; int tid = __kmp_tid_from_gtid(gtid); if (ompt_enabled.enabled) { parent_team->t.t_implicit_task_taskdata[tid] .ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); } OMPT_STORE_RETURN_ADDRESS(gtid); #endif // check if __kmpc_push_num_teams called, set default number of teams // otherwise if (this_thr->th.th_teams_size.nteams == 0) { __kmp_push_num_teams(loc, gtid, 0, 0); } KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1); KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1); KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1); __kmp_fork_call(loc, gtid, fork_context_intel, argc, VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task VOLATILE_CAST(launch_t) __kmp_invoke_teams_master, #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX &ap #else ap #endif ); __kmp_join_call(loc, gtid #if OMPT_SUPPORT , fork_context_intel #endif ); this_thr->th.th_teams_microtask = NULL; this_thr->th.th_teams_level = 0; *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L; va_end(ap); } #endif /* OMP_40_ENABLED */ // I don't think this function should ever have been exported. // The __kmpc_ prefix was misapplied. I'm fairly certain that no generated // openmp code ever called it, but it's been exported from the RTL for so // long that I'm afraid to remove the definition. int __kmpc_invoke_task_func(int gtid) { return __kmp_invoke_task_func(gtid); } /*! @ingroup PARALLEL @param loc source location information @param global_tid global thread number Enter a serialized parallel construct. This interface is used to handle a conditional parallel region, like this, @code #pragma omp parallel if (condition) @endcode when the condition is false. */ void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid) { // The implementation is now in kmp_runtime.cpp so that it can share static // functions with kmp_fork_call since the tasks to be done are similar in // each case. #if OMPT_SUPPORT OMPT_STORE_RETURN_ADDRESS(global_tid); #endif __kmp_serialized_parallel(loc, global_tid); } /*! @ingroup PARALLEL @param loc source location information @param global_tid global thread number Leave a serialized parallel construct. */ void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) { kmp_internal_control_t *top; kmp_info_t *this_thr; kmp_team_t *serial_team; KC_TRACE(10, ("__kmpc_end_serialized_parallel: called by T#%d\n", global_tid)); /* skip all this code for autopar serialized loops since it results in unacceptable overhead */ if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR)) return; // Not autopar code if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); this_thr = __kmp_threads[global_tid]; serial_team = this_thr->th.th_serial_team; #if OMP_45_ENABLED kmp_task_team_t *task_team = this_thr->th.th_task_team; // we need to wait for the proxy tasks before finishing the thread if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL)); #endif KMP_MB(); KMP_DEBUG_ASSERT(serial_team); KMP_ASSERT(serial_team->t.t_serialized); KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team); KMP_DEBUG_ASSERT(serial_team->t.t_threads); KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); #if OMPT_SUPPORT if (ompt_enabled.enabled && this_thr->th.ompt_thread_info.state != ompt_state_overhead) { OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = ompt_data_none; if (ompt_enabled.ompt_callback_implicit_task) { ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1, OMPT_CUR_TASK_INFO(this_thr)->thread_num, ompt_task_implicit); } // reset clear the task id only after unlinking the task ompt_data_t *parent_task_data; __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL); if (ompt_enabled.ompt_callback_parallel_end) { ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( &(serial_team->t.ompt_team_info.parallel_data), parent_task_data, ompt_parallel_invoker_program, OMPT_LOAD_RETURN_ADDRESS(global_tid)); } __ompt_lw_taskteam_unlink(this_thr); this_thr->th.ompt_thread_info.state = ompt_state_overhead; } #endif /* If necessary, pop the internal control stack values and replace the team * values */ top = serial_team->t.t_control_stack_top; if (top && top->serial_nesting_level == serial_team->t.t_serialized) { copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top); serial_team->t.t_control_stack_top = top->next; __kmp_free(top); } // if( serial_team -> t.t_serialized > 1 ) serial_team->t.t_level--; /* pop dispatch buffers stack */ KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer); { dispatch_private_info_t *disp_buffer = serial_team->t.t_dispatch->th_disp_buffer; serial_team->t.t_dispatch->th_disp_buffer = serial_team->t.t_dispatch->th_disp_buffer->next; __kmp_free(disp_buffer); } #if OMP_50_ENABLED this_thr->th.th_def_allocator = serial_team->t.t_def_allocator; // restore #endif --serial_team->t.t_serialized; if (serial_team->t.t_serialized == 0) { /* return to the parallel section */ #if KMP_ARCH_X86 || KMP_ARCH_X86_64 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) { __kmp_clear_x87_fpu_status_word(); __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word); __kmp_load_mxcsr(&serial_team->t.t_mxcsr); } #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ this_thr->th.th_team = serial_team->t.t_parent; this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid; /* restore values cached in the thread */ this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc; /* JPH */ this_thr->th.th_team_master = serial_team->t.t_parent->t.t_threads[0]; /* JPH */ this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized; /* TODO the below shouldn't need to be adjusted for serialized teams */ this_thr->th.th_dispatch = &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid]; __kmp_pop_current_task_from_thread(this_thr); KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0); this_thr->th.th_current_task->td_flags.executing = 1; if (__kmp_tasking_mode != tskm_immediate_exec) { // Copy the task team from the new child / old parent team to the thread. this_thr->th.th_task_team = this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]; KA_TRACE(20, ("__kmpc_end_serialized_parallel: T#%d restoring task_team %p / " "team %p\n", global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); } } else { if (__kmp_tasking_mode != tskm_immediate_exec) { KA_TRACE(20, ("__kmpc_end_serialized_parallel: T#%d decreasing nesting " "depth of serial team %p to %d\n", global_tid, serial_team, serial_team->t.t_serialized)); } } if (__kmp_env_consistency_check) __kmp_pop_parallel(global_tid, NULL); #if OMPT_SUPPORT if (ompt_enabled.enabled) this_thr->th.ompt_thread_info.state = ((this_thr->th.th_team_serialized) ? ompt_state_work_serial : ompt_state_work_parallel); #endif } /*! @ingroup SYNCHRONIZATION @param loc source location information. Execute flush. This is implemented as a full memory fence. (Though depending on the memory ordering convention obeyed by the compiler even that may not be necessary). */ void __kmpc_flush(ident_t *loc) { KC_TRACE(10, ("__kmpc_flush: called\n")); /* need explicit __mf() here since use volatile instead in library */ KMP_MB(); /* Flush all pending memory write invalidates. */ #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) #if KMP_MIC // fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used. // We shouldn't need it, though, since the ABI rules require that // * If the compiler generates NGO stores it also generates the fence // * If users hand-code NGO stores they should insert the fence // therefore no incomplete unordered stores should be visible. #else // C74404 // This is to address non-temporal store instructions (sfence needed). // The clflush instruction is addressed either (mfence needed). // Probably the non-temporal load monvtdqa instruction should also be // addressed. // mfence is a SSE2 instruction. Do not execute it if CPU is not SSE2. if (!__kmp_cpuinfo.initialized) { __kmp_query_cpuid(&__kmp_cpuinfo); } if (!__kmp_cpuinfo.sse2) { // CPU cannot execute SSE2 instructions. } else { #if KMP_COMPILER_ICC _mm_mfence(); #elif KMP_COMPILER_MSVC MemoryBarrier(); #else __sync_synchronize(); #endif // KMP_COMPILER_ICC } #endif // KMP_MIC #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64) // Nothing to see here move along #elif KMP_ARCH_PPC64 // Nothing needed here (we have a real MB above). #if KMP_OS_CNK // The flushing thread needs to yield here; this prevents a // busy-waiting thread from saturating the pipeline. flush is // often used in loops like this: // while (!flag) { // #pragma omp flush(flag) // } // and adding the yield here is good for at least a 10x speedup // when running >2 threads per core (on the NAS LU benchmark). __kmp_yield(TRUE); #endif #else #error Unknown or unsupported architecture #endif #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_flush) { ompt_callbacks.ompt_callback(ompt_callback_flush)( __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0)); } #endif } /* -------------------------------------------------------------------------- */ /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid thread id. Execute a barrier. */ void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) { KMP_COUNT_BLOCK(OMP_BARRIER); KC_TRACE(10, ("__kmpc_barrier: called T#%d\n", global_tid)); if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); if (__kmp_env_consistency_check) { if (loc == 0) { KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user? } __kmp_check_barrier(global_tid, ct_barrier, loc); } #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif __kmp_threads[global_tid]->th.th_ident = loc; // TODO: explicit barrier_wait_id: // this function is called when 'barrier' directive is present or // implicit barrier at the end of a worksharing construct. // 1) better to add a per-thread barrier counter to a thread data structure // 2) set to 0 when a new team is created // 4) no sync is required __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif } /* The BARRIER for a MASTER section is always explicit */ /*! @ingroup WORK_SHARING @param loc source location information. @param global_tid global thread number . @return 1 if this thread should execute the master block, 0 otherwise. */ kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid) { int status = 0; KC_TRACE(10, ("__kmpc_master: called T#%d\n", global_tid)); if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); if (KMP_MASTER_GTID(global_tid)) { KMP_COUNT_BLOCK(OMP_MASTER); KMP_PUSH_PARTITIONED_TIMER(OMP_master); status = 1; } #if OMPT_SUPPORT && OMPT_OPTIONAL if (status) { if (ompt_enabled.ompt_callback_master) { kmp_info_t *this_thr = __kmp_threads[global_tid]; kmp_team_t *team = this_thr->th.th_team; int tid = __kmp_tid_from_gtid(global_tid); ompt_callbacks.ompt_callback(ompt_callback_master)( ompt_scope_begin, &(team->t.ompt_team_info.parallel_data), &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), OMPT_GET_RETURN_ADDRESS(0)); } } #endif if (__kmp_env_consistency_check) { #if KMP_USE_DYNAMIC_LOCK if (status) __kmp_push_sync(global_tid, ct_master, loc, NULL, 0); else __kmp_check_sync(global_tid, ct_master, loc, NULL, 0); #else if (status) __kmp_push_sync(global_tid, ct_master, loc, NULL); else __kmp_check_sync(global_tid, ct_master, loc, NULL); #endif } return status; } /*! @ingroup WORK_SHARING @param loc source location information. @param global_tid global thread number . Mark the end of a master region. This should only be called by the thread that executes the master region. */ void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid) { KC_TRACE(10, ("__kmpc_end_master: called T#%d\n", global_tid)); KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid)); KMP_POP_PARTITIONED_TIMER(); #if OMPT_SUPPORT && OMPT_OPTIONAL kmp_info_t *this_thr = __kmp_threads[global_tid]; kmp_team_t *team = this_thr->th.th_team; if (ompt_enabled.ompt_callback_master) { int tid = __kmp_tid_from_gtid(global_tid); ompt_callbacks.ompt_callback(ompt_callback_master)( ompt_scope_end, &(team->t.ompt_team_info.parallel_data), &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), OMPT_GET_RETURN_ADDRESS(0)); } #endif if (__kmp_env_consistency_check) { if (global_tid < 0) KMP_WARNING(ThreadIdentInvalid); if (KMP_MASTER_GTID(global_tid)) __kmp_pop_sync(global_tid, ct_master, loc); } } /*! @ingroup WORK_SHARING @param loc source location information. @param gtid global thread number. Start execution of an ordered construct. */ void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) { int cid = 0; kmp_info_t *th; KMP_DEBUG_ASSERT(__kmp_init_serial); KC_TRACE(10, ("__kmpc_ordered: called T#%d\n", gtid)); if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); #if USE_ITT_BUILD __kmp_itt_ordered_prep(gtid); // TODO: ordered_wait_id #endif /* USE_ITT_BUILD */ th = __kmp_threads[gtid]; #if OMPT_SUPPORT && OMPT_OPTIONAL kmp_team_t *team; ompt_wait_id_t lck; void *codeptr_ra; if (ompt_enabled.enabled) { OMPT_STORE_RETURN_ADDRESS(gtid); team = __kmp_team_from_gtid(gtid); - lck = (ompt_wait_id_t)&team->t.t_ordered.dt.t_value; + lck = (ompt_wait_id_t)(uintptr_t)&team->t.t_ordered.dt.t_value; /* OMPT state update */ th->th.ompt_thread_info.wait_id = lck; th->th.ompt_thread_info.state = ompt_state_wait_ordered; /* OMPT event callback */ codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid); if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( - ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin, - (ompt_wait_id_t)lck, codeptr_ra); + ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin, lck, + codeptr_ra); } } #endif if (th->th.th_dispatch->th_deo_fcn != 0) (*th->th.th_dispatch->th_deo_fcn)(>id, &cid, loc); else __kmp_parallel_deo(>id, &cid, loc); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { /* OMPT state update */ th->th.ompt_thread_info.state = ompt_state_work_parallel; th->th.ompt_thread_info.wait_id = 0; /* OMPT event callback */ if (ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_ordered, (ompt_wait_id_t)lck, codeptr_ra); + ompt_mutex_ordered, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra); } } #endif #if USE_ITT_BUILD __kmp_itt_ordered_start(gtid); #endif /* USE_ITT_BUILD */ } /*! @ingroup WORK_SHARING @param loc source location information. @param gtid global thread number. End execution of an ordered construct. */ void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid) { int cid = 0; kmp_info_t *th; KC_TRACE(10, ("__kmpc_end_ordered: called T#%d\n", gtid)); #if USE_ITT_BUILD __kmp_itt_ordered_end(gtid); // TODO: ordered_wait_id #endif /* USE_ITT_BUILD */ th = __kmp_threads[gtid]; if (th->th.th_dispatch->th_dxo_fcn != 0) (*th->th.th_dispatch->th_dxo_fcn)(>id, &cid, loc); else __kmp_parallel_dxo(>id, &cid, loc); #if OMPT_SUPPORT && OMPT_OPTIONAL OMPT_STORE_RETURN_ADDRESS(gtid); if (ompt_enabled.ompt_callback_mutex_released) { ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( ompt_mutex_ordered, - (ompt_wait_id_t)&__kmp_team_from_gtid(gtid)->t.t_ordered.dt.t_value, + (ompt_wait_id_t)(uintptr_t)&__kmp_team_from_gtid(gtid) + ->t.t_ordered.dt.t_value, OMPT_LOAD_RETURN_ADDRESS(gtid)); } #endif } #if KMP_USE_DYNAMIC_LOCK static __forceinline void __kmp_init_indirect_csptr(kmp_critical_name *crit, ident_t const *loc, kmp_int32 gtid, kmp_indirect_locktag_t tag) { // Pointer to the allocated indirect lock is written to crit, while indexing // is ignored. void *idx; kmp_indirect_lock_t **lck; lck = (kmp_indirect_lock_t **)crit; kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag); KMP_I_LOCK_FUNC(ilk, init)(ilk->lock); KMP_SET_I_LOCK_LOCATION(ilk, loc); KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section); KA_TRACE(20, ("__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag)); #if USE_ITT_BUILD __kmp_itt_critical_creating(ilk->lock, loc); #endif int status = KMP_COMPARE_AND_STORE_PTR(lck, nullptr, ilk); if (status == 0) { #if USE_ITT_BUILD __kmp_itt_critical_destroyed(ilk->lock); #endif // We don't really need to destroy the unclaimed lock here since it will be // cleaned up at program exit. // KMP_D_LOCK_FUNC(&idx, destroy)((kmp_dyna_lock_t *)&idx); } KMP_DEBUG_ASSERT(*lck != NULL); } // Fast-path acquire tas lock #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \ { \ kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ kmp_uint32 spins; \ KMP_FSYNC_PREPARE(l); \ KMP_INIT_YIELD(spins); \ if (TCR_4(__kmp_nth) > \ (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ KMP_YIELD(TRUE); \ } else { \ KMP_YIELD_SPIN(spins); \ } \ kmp_backoff_t backoff = __kmp_spin_backoff_params; \ while ( \ KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ __kmp_spin_backoff(&backoff); \ if (TCR_4(__kmp_nth) > \ (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ KMP_YIELD(TRUE); \ } else { \ KMP_YIELD_SPIN(spins); \ } \ } \ } \ KMP_FSYNC_ACQUIRED(l); \ } // Fast-path test tas lock #define KMP_TEST_TAS_LOCK(lock, gtid, rc) \ { \ kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \ __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \ } // Fast-path release tas lock #define KMP_RELEASE_TAS_LOCK(lock, gtid) \ { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); } #if KMP_USE_FUTEX #include #include #ifndef FUTEX_WAIT #define FUTEX_WAIT 0 #endif #ifndef FUTEX_WAKE #define FUTEX_WAKE 1 #endif // Fast-path acquire futex lock #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \ { \ kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ kmp_int32 gtid_code = (gtid + 1) << 1; \ KMP_MB(); \ KMP_FSYNC_PREPARE(ftx); \ kmp_int32 poll_val; \ while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \ &(ftx->lk.poll), KMP_LOCK_FREE(futex), \ KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ if (!cond) { \ if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \ poll_val | \ KMP_LOCK_BUSY(1, futex))) { \ continue; \ } \ poll_val |= KMP_LOCK_BUSY(1, futex); \ } \ kmp_int32 rc; \ if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \ NULL, NULL, 0)) != 0) { \ continue; \ } \ gtid_code |= 1; \ } \ KMP_FSYNC_ACQUIRED(ftx); \ } // Fast-path test futex lock #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \ { \ kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \ KMP_FSYNC_ACQUIRED(ftx); \ rc = TRUE; \ } else { \ rc = FALSE; \ } \ } // Fast-path release futex lock #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \ { \ kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ KMP_MB(); \ KMP_FSYNC_RELEASING(ftx); \ kmp_int32 poll_val = \ KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ if (KMP_LOCK_STRIP(poll_val) & 1) { \ syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \ KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ } \ KMP_MB(); \ KMP_YIELD(TCR_4(__kmp_nth) > \ (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ } #endif // KMP_USE_FUTEX #else // KMP_USE_DYNAMIC_LOCK static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit, ident_t const *loc, kmp_int32 gtid) { kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit; // Because of the double-check, the following load doesn't need to be volatile kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp); if (lck == NULL) { void *idx; // Allocate & initialize the lock. // Remember alloc'ed locks in table in order to free them in __kmp_cleanup() lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section); __kmp_init_user_lock_with_checks(lck); __kmp_set_user_lock_location(lck, loc); #if USE_ITT_BUILD __kmp_itt_critical_creating(lck); // __kmp_itt_critical_creating() should be called *before* the first usage // of underlying lock. It is the only place where we can guarantee it. There // are chances the lock will destroyed with no usage, but it is not a // problem, because this is not real event seen by user but rather setting // name for object (lock). See more details in kmp_itt.h. #endif /* USE_ITT_BUILD */ // Use a cmpxchg instruction to slam the start of the critical section with // the lock pointer. If another thread beat us to it, deallocate the lock, // and use the lock that the other thread allocated. int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck); if (status == 0) { // Deallocate the lock and reload the value. #if USE_ITT_BUILD __kmp_itt_critical_destroyed(lck); // Let ITT know the lock is destroyed and the same memory location may be reused // for another purpose. #endif /* USE_ITT_BUILD */ __kmp_destroy_user_lock_with_checks(lck); __kmp_user_lock_free(&idx, gtid, lck); lck = (kmp_user_lock_p)TCR_PTR(*lck_pp); KMP_DEBUG_ASSERT(lck != NULL); } } return lck; } #endif // KMP_USE_DYNAMIC_LOCK /*! @ingroup WORK_SHARING @param loc source location information. @param global_tid global thread number . @param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or some other suitably unique value. Enter code protected by a `critical` construct. This function blocks until the executing thread can enter the critical section. */ void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit) { #if KMP_USE_DYNAMIC_LOCK #if OMPT_SUPPORT && OMPT_OPTIONAL OMPT_STORE_RETURN_ADDRESS(global_tid); #endif // OMPT_SUPPORT __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none); #else KMP_COUNT_BLOCK(OMP_CRITICAL); #if OMPT_SUPPORT && OMPT_OPTIONAL ompt_state_t prev_state = ompt_state_undefined; ompt_thread_info_t ti; #endif kmp_user_lock_p lck; KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid)); // TODO: add THR_OVHD_STATE KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait); KMP_CHECK_USER_LOCK_INIT(); if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) { lck = (kmp_user_lock_p)crit; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) { lck = (kmp_user_lock_p)crit; } #endif else { // ticket, queuing or drdpa lck = __kmp_get_critical_section_ptr(crit, loc, global_tid); } if (__kmp_env_consistency_check) __kmp_push_sync(global_tid, ct_critical, loc, lck); // since the critical directive binds to all threads, not just the current // team we have to check this even if we are in a serialized team. // also, even if we are the uber thread, we still have to conduct the lock, // as we have to contend with sibling threads. #if USE_ITT_BUILD __kmp_itt_critical_acquiring(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL OMPT_STORE_RETURN_ADDRESS(gtid); void *codeptr_ra = NULL; if (ompt_enabled.enabled) { ti = __kmp_threads[global_tid]->th.ompt_thread_info; /* OMPT state update */ prev_state = ti.state; - ti.wait_id = (ompt_wait_id_t)lck; + ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck; ti.state = ompt_state_wait_critical; /* OMPT event callback */ codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid); if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(), - (ompt_wait_id_t)crit, codeptr_ra); + (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra); } } #endif // Value of 'crit' should be good for using as a critical_id of the critical // section directive. __kmp_acquire_user_lock_with_checks(lck, global_tid); #if USE_ITT_BUILD __kmp_itt_critical_acquired(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { /* OMPT state update */ ti.state = prev_state; ti.wait_id = 0; /* OMPT event callback */ if (ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr_ra); + ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra); } } #endif KMP_POP_PARTITIONED_TIMER(); KMP_PUSH_PARTITIONED_TIMER(OMP_critical); KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid)); #endif // KMP_USE_DYNAMIC_LOCK } #if KMP_USE_DYNAMIC_LOCK // Converts the given hint to an internal lock implementation static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) { #if KMP_USE_TSX #define KMP_TSX_LOCK(seq) lockseq_##seq #else #define KMP_TSX_LOCK(seq) __kmp_user_lock_seq #endif #if KMP_ARCH_X86 || KMP_ARCH_X86_64 #define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm) #else #define KMP_CPUINFO_RTM 0 #endif // Hints that do not require further logic if (hint & kmp_lock_hint_hle) return KMP_TSX_LOCK(hle); if (hint & kmp_lock_hint_rtm) return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm) : __kmp_user_lock_seq; if (hint & kmp_lock_hint_adaptive) return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq; // Rule out conflicting hints first by returning the default lock if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended)) return __kmp_user_lock_seq; if ((hint & omp_lock_hint_speculative) && (hint & omp_lock_hint_nonspeculative)) return __kmp_user_lock_seq; // Do not even consider speculation when it appears to be contended if (hint & omp_lock_hint_contended) return lockseq_queuing; // Uncontended lock without speculation if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative)) return lockseq_tas; // HLE lock for speculation if (hint & omp_lock_hint_speculative) return KMP_TSX_LOCK(hle); return __kmp_user_lock_seq; } #if OMPT_SUPPORT && OMPT_OPTIONAL #if KMP_USE_DYNAMIC_LOCK static kmp_mutex_impl_t __ompt_get_mutex_impl_type(void *user_lock, kmp_indirect_lock_t *ilock = 0) { if (user_lock) { switch (KMP_EXTRACT_D_TAG(user_lock)) { case 0: break; #if KMP_USE_FUTEX case locktag_futex: return kmp_mutex_impl_queuing; #endif case locktag_tas: return kmp_mutex_impl_spin; #if KMP_USE_TSX case locktag_hle: return kmp_mutex_impl_speculative; #endif default: return kmp_mutex_impl_none; } ilock = KMP_LOOKUP_I_LOCK(user_lock); } KMP_ASSERT(ilock); switch (ilock->type) { #if KMP_USE_TSX case locktag_adaptive: case locktag_rtm: return kmp_mutex_impl_speculative; #endif case locktag_nested_tas: return kmp_mutex_impl_spin; #if KMP_USE_FUTEX case locktag_nested_futex: #endif case locktag_ticket: case locktag_queuing: case locktag_drdpa: case locktag_nested_ticket: case locktag_nested_queuing: case locktag_nested_drdpa: return kmp_mutex_impl_queuing; default: return kmp_mutex_impl_none; } } #else // For locks without dynamic binding static kmp_mutex_impl_t __ompt_get_mutex_impl_type() { switch (__kmp_user_lock_kind) { case lk_tas: return kmp_mutex_impl_spin; #if KMP_USE_FUTEX case lk_futex: #endif case lk_ticket: case lk_queuing: case lk_drdpa: return kmp_mutex_impl_queuing; #if KMP_USE_TSX case lk_hle: case lk_rtm: case lk_adaptive: return kmp_mutex_impl_speculative; #endif default: return kmp_mutex_impl_none; } } #endif // KMP_USE_DYNAMIC_LOCK #endif // OMPT_SUPPORT && OMPT_OPTIONAL /*! @ingroup WORK_SHARING @param loc source location information. @param global_tid global thread number. @param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or some other suitably unique value. @param hint the lock hint. Enter code protected by a `critical` construct with a hint. The hint value is used to suggest a lock implementation. This function blocks until the executing thread can enter the critical section unless the hint suggests use of speculative execution and the hardware supports it. */ void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit, uint32_t hint) { KMP_COUNT_BLOCK(OMP_CRITICAL); kmp_user_lock_p lck; #if OMPT_SUPPORT && OMPT_OPTIONAL ompt_state_t prev_state = ompt_state_undefined; ompt_thread_info_t ti; // This is the case, if called from __kmpc_critical: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); #endif KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid)); kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit; // Check if it is initialized. KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait); if (*lk == 0) { kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint); if (KMP_IS_D_LOCK(lckseq)) { KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(lckseq)); } else { __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq)); } } // Branch for accessing the actual lock object and set operation. This // branching is inevitable since this lock initialization does not follow the // normal dispatch path (lock table is not used). if (KMP_EXTRACT_D_TAG(lk) != 0) { lck = (kmp_user_lock_p)lk; if (__kmp_env_consistency_check) { __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint)); } #if USE_ITT_BUILD __kmp_itt_critical_acquiring(lck); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ti = __kmp_threads[global_tid]->th.ompt_thread_info; /* OMPT state update */ prev_state = ti.state; - ti.wait_id = (ompt_wait_id_t)lck; + ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck; ti.state = ompt_state_wait_critical; /* OMPT event callback */ if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_critical, (unsigned int)hint, - __ompt_get_mutex_impl_type(crit), (ompt_wait_id_t)crit, codeptr); + __ompt_get_mutex_impl_type(crit), (ompt_wait_id_t)(uintptr_t)lck, + codeptr); } } #endif #if KMP_USE_INLINED_TAS if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) { KMP_ACQUIRE_TAS_LOCK(lck, global_tid); } else #elif KMP_USE_INLINED_FUTEX if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) { KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid); } else #endif { KMP_D_LOCK_FUNC(lk, set)(lk, global_tid); } } else { kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk); lck = ilk->lock; if (__kmp_env_consistency_check) { __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint)); } #if USE_ITT_BUILD __kmp_itt_critical_acquiring(lck); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ti = __kmp_threads[global_tid]->th.ompt_thread_info; /* OMPT state update */ prev_state = ti.state; - ti.wait_id = (ompt_wait_id_t)lck; + ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck; ti.state = ompt_state_wait_critical; /* OMPT event callback */ if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_critical, (unsigned int)hint, - __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)crit, codeptr); + __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)(uintptr_t)lck, + codeptr); } } #endif KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid); } KMP_POP_PARTITIONED_TIMER(); #if USE_ITT_BUILD __kmp_itt_critical_acquired(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { /* OMPT state update */ ti.state = prev_state; ti.wait_id = 0; /* OMPT event callback */ if (ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr); + ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } #endif KMP_PUSH_PARTITIONED_TIMER(OMP_critical); KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid)); } // __kmpc_critical_with_hint #endif // KMP_USE_DYNAMIC_LOCK /*! @ingroup WORK_SHARING @param loc source location information. @param global_tid global thread number . @param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or some other suitably unique value. Leave a critical section, releasing any lock that was held during its execution. */ void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit) { kmp_user_lock_p lck; KC_TRACE(10, ("__kmpc_end_critical: called T#%d\n", global_tid)); #if KMP_USE_DYNAMIC_LOCK if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) { lck = (kmp_user_lock_p)crit; KMP_ASSERT(lck != NULL); if (__kmp_env_consistency_check) { __kmp_pop_sync(global_tid, ct_critical, loc); } #if USE_ITT_BUILD __kmp_itt_critical_releasing(lck); #endif #if KMP_USE_INLINED_TAS if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) { KMP_RELEASE_TAS_LOCK(lck, global_tid); } else #elif KMP_USE_INLINED_FUTEX if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) { KMP_RELEASE_FUTEX_LOCK(lck, global_tid); } else #endif { KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid); } } else { kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit)); KMP_ASSERT(ilk != NULL); lck = ilk->lock; if (__kmp_env_consistency_check) { __kmp_pop_sync(global_tid, ct_critical, loc); } #if USE_ITT_BUILD __kmp_itt_critical_releasing(lck); #endif KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid); } #else // KMP_USE_DYNAMIC_LOCK if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) { lck = (kmp_user_lock_p)crit; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) { lck = (kmp_user_lock_p)crit; } #endif else { // ticket, queuing or drdpa lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit)); } KMP_ASSERT(lck != NULL); if (__kmp_env_consistency_check) __kmp_pop_sync(global_tid, ct_critical, loc); #if USE_ITT_BUILD __kmp_itt_critical_releasing(lck); #endif /* USE_ITT_BUILD */ // Value of 'crit' should be good for using as a critical_id of the critical // section directive. __kmp_release_user_lock_with_checks(lck, global_tid); #endif // KMP_USE_DYNAMIC_LOCK #if OMPT_SUPPORT && OMPT_OPTIONAL /* OMPT release event triggers after lock is released; place here to trigger * for all #if branches */ OMPT_STORE_RETURN_ADDRESS(global_tid); if (ompt_enabled.ompt_callback_mutex_released) { ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_critical, (ompt_wait_id_t)crit, OMPT_LOAD_RETURN_ADDRESS(0)); + ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, + OMPT_LOAD_RETURN_ADDRESS(0)); } #endif KMP_POP_PARTITIONED_TIMER(); KA_TRACE(15, ("__kmpc_end_critical: done T#%d\n", global_tid)); } /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid thread id. @return one if the thread should execute the master block, zero otherwise Start execution of a combined barrier and master. The barrier is executed inside this function. */ kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid) { int status; KC_TRACE(10, ("__kmpc_barrier_master: called T#%d\n", global_tid)); if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); if (__kmp_env_consistency_check) __kmp_check_barrier(global_tid, ct_barrier, loc); #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif #if USE_ITT_NOTIFY __kmp_threads[global_tid]->th.th_ident = loc; #endif status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif return (status != 0) ? 0 : 1; } /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid thread id. Complete the execution of a combined barrier and master. This function should only be called at the completion of the master code. Other threads will still be waiting at the barrier and this call releases them. */ void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid) { KC_TRACE(10, ("__kmpc_end_barrier_master: called T#%d\n", global_tid)); __kmp_end_split_barrier(bs_plain_barrier, global_tid); } /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid thread id. @return one if the thread should execute the master block, zero otherwise Start execution of a combined barrier and master(nowait) construct. The barrier is executed inside this function. There is no equivalent "end" function, since the */ kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid) { kmp_int32 ret; KC_TRACE(10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid)); if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); if (__kmp_env_consistency_check) { if (loc == 0) { KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user? } __kmp_check_barrier(global_tid, ct_barrier, loc); } #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif #if USE_ITT_NOTIFY __kmp_threads[global_tid]->th.th_ident = loc; #endif __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif ret = __kmpc_master(loc, global_tid); if (__kmp_env_consistency_check) { /* there's no __kmpc_end_master called; so the (stats) */ /* actions of __kmpc_end_master are done here */ if (global_tid < 0) { KMP_WARNING(ThreadIdentInvalid); } if (ret) { /* only one thread should do the pop since only */ /* one did the push (see __kmpc_master()) */ __kmp_pop_sync(global_tid, ct_master, loc); } } return (ret); } /* The BARRIER for a SINGLE process section is always explicit */ /*! @ingroup WORK_SHARING @param loc source location information @param global_tid global thread number @return One if this thread should execute the single construct, zero otherwise. Test whether to execute a single construct. There are no implicit barriers in the two "single" calls, rather the compiler should introduce an explicit barrier if it is required. */ kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid) { kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE); if (rc) { // We are going to execute the single statement, so we should count it. KMP_COUNT_BLOCK(OMP_SINGLE); KMP_PUSH_PARTITIONED_TIMER(OMP_single); } #if OMPT_SUPPORT && OMPT_OPTIONAL kmp_info_t *this_thr = __kmp_threads[global_tid]; kmp_team_t *team = this_thr->th.th_team; int tid = __kmp_tid_from_gtid(global_tid); if (ompt_enabled.enabled) { if (rc) { if (ompt_enabled.ompt_callback_work) { ompt_callbacks.ompt_callback(ompt_callback_work)( ompt_work_single_executor, ompt_scope_begin, &(team->t.ompt_team_info.parallel_data), &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1, OMPT_GET_RETURN_ADDRESS(0)); } } else { if (ompt_enabled.ompt_callback_work) { ompt_callbacks.ompt_callback(ompt_callback_work)( ompt_work_single_other, ompt_scope_begin, &(team->t.ompt_team_info.parallel_data), &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1, OMPT_GET_RETURN_ADDRESS(0)); ompt_callbacks.ompt_callback(ompt_callback_work)( ompt_work_single_other, ompt_scope_end, &(team->t.ompt_team_info.parallel_data), &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1, OMPT_GET_RETURN_ADDRESS(0)); } } } #endif return rc; } /*! @ingroup WORK_SHARING @param loc source location information @param global_tid global thread number Mark the end of a single construct. This function should only be called by the thread that executed the block of code protected by the `single` construct. */ void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid) { __kmp_exit_single(global_tid); KMP_POP_PARTITIONED_TIMER(); #if OMPT_SUPPORT && OMPT_OPTIONAL kmp_info_t *this_thr = __kmp_threads[global_tid]; kmp_team_t *team = this_thr->th.th_team; int tid = __kmp_tid_from_gtid(global_tid); if (ompt_enabled.ompt_callback_work) { ompt_callbacks.ompt_callback(ompt_callback_work)( ompt_work_single_executor, ompt_scope_end, &(team->t.ompt_team_info.parallel_data), &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1, OMPT_GET_RETURN_ADDRESS(0)); } #endif } /*! @ingroup WORK_SHARING @param loc Source location @param global_tid Global thread id Mark the end of a statically scheduled loop. */ void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid) { KMP_POP_PARTITIONED_TIMER(); KE_TRACE(10, ("__kmpc_for_static_fini called T#%d\n", global_tid)); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_work) { ompt_work_t ompt_work_type = ompt_work_loop; ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); ompt_task_info_t *task_info = __ompt_get_task_info_object(0); // Determine workshare type if (loc != NULL) { if ((loc->flags & KMP_IDENT_WORK_LOOP) != 0) { ompt_work_type = ompt_work_loop; } else if ((loc->flags & KMP_IDENT_WORK_SECTIONS) != 0) { ompt_work_type = ompt_work_sections; } else if ((loc->flags & KMP_IDENT_WORK_DISTRIBUTE) != 0) { ompt_work_type = ompt_work_distribute; } else { // use default set above. // a warning about this case is provided in __kmpc_for_static_init } KMP_DEBUG_ASSERT(ompt_work_type); } ompt_callbacks.ompt_callback(ompt_callback_work)( ompt_work_type, ompt_scope_end, &(team_info->parallel_data), &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0)); } #endif if (__kmp_env_consistency_check) __kmp_pop_workshare(global_tid, ct_pdo, loc); } // User routines which take C-style arguments (call by value) // different from the Fortran equivalent routines void ompc_set_num_threads(int arg) { // !!!!! TODO: check the per-task binding __kmp_set_num_threads(arg, __kmp_entry_gtid()); } void ompc_set_dynamic(int flag) { kmp_info_t *thread; /* For the thread-private implementation of the internal controls */ thread = __kmp_entry_thread(); __kmp_save_internal_controls(thread); set__dynamic(thread, flag ? TRUE : FALSE); } void ompc_set_nested(int flag) { kmp_info_t *thread; /* For the thread-private internal controls implementation */ thread = __kmp_entry_thread(); __kmp_save_internal_controls(thread); set__nested(thread, flag ? TRUE : FALSE); } void ompc_set_max_active_levels(int max_active_levels) { /* TO DO */ /* we want per-task implementation of this internal control */ /* For the per-thread internal controls implementation */ __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels); } void ompc_set_schedule(omp_sched_t kind, int modifier) { // !!!!! TODO: check the per-task binding __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier); } int ompc_get_ancestor_thread_num(int level) { return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level); } int ompc_get_team_size(int level) { return __kmp_get_team_size(__kmp_entry_gtid(), level); } #if OMP_50_ENABLED /* OpenMP 5.0 Affinity Format API */ void ompc_set_affinity_format(char const *format) { if (!__kmp_init_serial) { __kmp_serial_initialize(); } __kmp_strncpy_truncate(__kmp_affinity_format, KMP_AFFINITY_FORMAT_SIZE, format, KMP_STRLEN(format) + 1); } size_t ompc_get_affinity_format(char *buffer, size_t size) { size_t format_size; if (!__kmp_init_serial) { __kmp_serial_initialize(); } format_size = KMP_STRLEN(__kmp_affinity_format); if (buffer && size) { __kmp_strncpy_truncate(buffer, size, __kmp_affinity_format, format_size + 1); } return format_size; } void ompc_display_affinity(char const *format) { int gtid; if (!TCR_4(__kmp_init_middle)) { __kmp_middle_initialize(); } gtid = __kmp_get_gtid(); __kmp_aux_display_affinity(gtid, format); } size_t ompc_capture_affinity(char *buffer, size_t buf_size, char const *format) { int gtid; size_t num_required; kmp_str_buf_t capture_buf; if (!TCR_4(__kmp_init_middle)) { __kmp_middle_initialize(); } gtid = __kmp_get_gtid(); __kmp_str_buf_init(&capture_buf); num_required = __kmp_aux_capture_affinity(gtid, format, &capture_buf); if (buffer && buf_size) { __kmp_strncpy_truncate(buffer, buf_size, capture_buf.str, capture_buf.used + 1); } __kmp_str_buf_free(&capture_buf); return num_required; } #endif /* OMP_50_ENABLED */ void kmpc_set_stacksize(int arg) { // __kmp_aux_set_stacksize initializes the library if needed __kmp_aux_set_stacksize(arg); } void kmpc_set_stacksize_s(size_t arg) { // __kmp_aux_set_stacksize initializes the library if needed __kmp_aux_set_stacksize(arg); } void kmpc_set_blocktime(int arg) { int gtid, tid; kmp_info_t *thread; gtid = __kmp_entry_gtid(); tid = __kmp_tid_from_gtid(gtid); thread = __kmp_thread_from_gtid(gtid); __kmp_aux_set_blocktime(arg, thread, tid); } void kmpc_set_library(int arg) { // __kmp_user_set_library initializes the library if needed __kmp_user_set_library((enum library_type)arg); } void kmpc_set_defaults(char const *str) { // __kmp_aux_set_defaults initializes the library if needed __kmp_aux_set_defaults(str, KMP_STRLEN(str)); } void kmpc_set_disp_num_buffers(int arg) { // ignore after initialization because some teams have already // allocated dispatch buffers if (__kmp_init_serial == 0 && arg > 0) __kmp_dispatch_num_buffers = arg; } int kmpc_set_affinity_mask_proc(int proc, void **mask) { #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED return -1; #else if (!TCR_4(__kmp_init_middle)) { __kmp_middle_initialize(); } return __kmp_aux_set_affinity_mask_proc(proc, mask); #endif } int kmpc_unset_affinity_mask_proc(int proc, void **mask) { #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED return -1; #else if (!TCR_4(__kmp_init_middle)) { __kmp_middle_initialize(); } return __kmp_aux_unset_affinity_mask_proc(proc, mask); #endif } int kmpc_get_affinity_mask_proc(int proc, void **mask) { #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED return -1; #else if (!TCR_4(__kmp_init_middle)) { __kmp_middle_initialize(); } return __kmp_aux_get_affinity_mask_proc(proc, mask); #endif } /* -------------------------------------------------------------------------- */ /*! @ingroup THREADPRIVATE @param loc source location information @param gtid global thread number @param cpy_size size of the cpy_data buffer @param cpy_data pointer to data to be copied @param cpy_func helper function to call for copying data @param didit flag variable: 1=single thread; 0=not single thread __kmpc_copyprivate implements the interface for the private data broadcast needed for the copyprivate clause associated with a single region in an OpenMP* program (both C and Fortran). All threads participating in the parallel region call this routine. One of the threads (called the single thread) should have the didit variable set to 1 and all other threads should have that variable set to 0. All threads pass a pointer to a data buffer (cpy_data) that they have built. The OpenMP specification forbids the use of nowait on the single region when a copyprivate clause is present. However, @ref __kmpc_copyprivate implements a barrier internally to avoid race conditions, so the code generation for the single region should avoid generating a barrier after the call to @ref __kmpc_copyprivate. The gtid parameter is the global thread id for the current thread. The loc parameter is a pointer to source location information. Internal implementation: The single thread will first copy its descriptor address (cpy_data) to a team-private location, then the other threads will each call the function pointed to by the parameter cpy_func, which carries out the copy by copying the data using the cpy_data buffer. The cpy_func routine used for the copy and the contents of the data area defined by cpy_data and cpy_size may be built in any fashion that will allow the copy to be done. For instance, the cpy_data buffer can hold the actual data to be copied or it may hold a list of pointers to the data. The cpy_func routine must interpret the cpy_data buffer appropriately. The interface to cpy_func is as follows: @code void cpy_func( void *destination, void *source ) @endcode where void *destination is the cpy_data pointer for the thread being copied to and void *source is the cpy_data pointer for the thread being copied from. */ void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void (*cpy_func)(void *, void *), kmp_int32 didit) { void **data_ptr; KC_TRACE(10, ("__kmpc_copyprivate: called T#%d\n", gtid)); KMP_MB(); data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data; if (__kmp_env_consistency_check) { if (loc == 0) { KMP_WARNING(ConstructIdentInvalid); } } // ToDo: Optimize the following two barriers into some kind of split barrier if (didit) *data_ptr = cpy_data; #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(gtid); } #endif /* This barrier is not a barrier region boundary */ #if USE_ITT_NOTIFY __kmp_threads[gtid]->th.th_ident = loc; #endif __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); if (!didit) (*cpy_func)(cpy_data, *data_ptr); // Consider next barrier a user-visible barrier for barrier region boundaries // Nesting checks are already handled by the single construct checks #if OMPT_SUPPORT if (ompt_enabled.enabled) { OMPT_STORE_RETURN_ADDRESS(gtid); } #endif #if USE_ITT_NOTIFY __kmp_threads[gtid]->th.th_ident = loc; // TODO: check if it is needed (e.g. // tasks can overwrite the location) #endif __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif } /* -------------------------------------------------------------------------- */ #define INIT_LOCK __kmp_init_user_lock_with_checks #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks #define ACQUIRE_NESTED_LOCK_TIMED \ __kmp_acquire_nested_user_lock_with_checks_timed #define RELEASE_LOCK __kmp_release_user_lock_with_checks #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks #define TEST_LOCK __kmp_test_user_lock_with_checks #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks // TODO: Make check abort messages use location info & pass it into // with_checks routines #if KMP_USE_DYNAMIC_LOCK // internal lock initializer static __forceinline void __kmp_init_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq) { if (KMP_IS_D_LOCK(seq)) { KMP_INIT_D_LOCK(lock, seq); #if USE_ITT_BUILD __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL); #endif } else { KMP_INIT_I_LOCK(lock, seq); #if USE_ITT_BUILD kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock); __kmp_itt_lock_creating(ilk->lock, loc); #endif } } // internal nest lock initializer static __forceinline void __kmp_init_nest_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq) { #if KMP_USE_TSX // Don't have nested lock implementation for speculative locks if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive) seq = __kmp_user_lock_seq; #endif switch (seq) { case lockseq_tas: seq = lockseq_nested_tas; break; #if KMP_USE_FUTEX case lockseq_futex: seq = lockseq_nested_futex; break; #endif case lockseq_ticket: seq = lockseq_nested_ticket; break; case lockseq_queuing: seq = lockseq_nested_queuing; break; case lockseq_drdpa: seq = lockseq_nested_drdpa; break; default: seq = lockseq_nested_queuing; } KMP_INIT_I_LOCK(lock, seq); #if USE_ITT_BUILD kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock); __kmp_itt_lock_creating(ilk->lock, loc); #endif } /* initialize the lock with a hint */ void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint) { KMP_DEBUG_ASSERT(__kmp_init_serial); if (__kmp_env_consistency_check && user_lock == NULL) { KMP_FATAL(LockIsUninitialized, "omp_init_lock_with_hint"); } __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint)); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_init) { ompt_callbacks.ompt_callback(ompt_callback_lock_init)( ompt_mutex_lock, (omp_lock_hint_t)hint, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif } /* initialize the lock with a hint */ void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint) { KMP_DEBUG_ASSERT(__kmp_init_serial); if (__kmp_env_consistency_check && user_lock == NULL) { KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock_with_hint"); } __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint)); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_init) { ompt_callbacks.ompt_callback(ompt_callback_lock_init)( ompt_mutex_nest_lock, (omp_lock_hint_t)hint, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif } #endif // KMP_USE_DYNAMIC_LOCK /* initialize the lock */ void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK KMP_DEBUG_ASSERT(__kmp_init_serial); if (__kmp_env_consistency_check && user_lock == NULL) { KMP_FATAL(LockIsUninitialized, "omp_init_lock"); } __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_init) { ompt_callbacks.ompt_callback(ompt_callback_lock_init)( ompt_mutex_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #else // KMP_USE_DYNAMIC_LOCK static char const *const func = "omp_init_lock"; kmp_user_lock_p lck; KMP_DEBUG_ASSERT(__kmp_init_serial); if (__kmp_env_consistency_check) { if (user_lock == NULL) { KMP_FATAL(LockIsUninitialized, func); } } KMP_CHECK_USER_LOCK_INIT(); if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_user_lock_allocate(user_lock, gtid, 0); } INIT_LOCK(lck); __kmp_set_user_lock_location(lck, loc); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_init) { ompt_callbacks.ompt_callback(ompt_callback_lock_init)( ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(), - (ompt_wait_id_t)user_lock, codeptr); + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #if USE_ITT_BUILD __kmp_itt_lock_creating(lck); #endif /* USE_ITT_BUILD */ #endif // KMP_USE_DYNAMIC_LOCK } // __kmpc_init_lock /* initialize the lock */ void __kmpc_init_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK KMP_DEBUG_ASSERT(__kmp_init_serial); if (__kmp_env_consistency_check && user_lock == NULL) { KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock"); } __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_init) { ompt_callbacks.ompt_callback(ompt_callback_lock_init)( ompt_mutex_nest_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #else // KMP_USE_DYNAMIC_LOCK static char const *const func = "omp_init_nest_lock"; kmp_user_lock_p lck; KMP_DEBUG_ASSERT(__kmp_init_serial); if (__kmp_env_consistency_check) { if (user_lock == NULL) { KMP_FATAL(LockIsUninitialized, func); } } KMP_CHECK_USER_LOCK_INIT(); if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_user_lock_allocate(user_lock, gtid, 0); } INIT_NESTED_LOCK(lck); __kmp_set_user_lock_location(lck, loc); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_init) { ompt_callbacks.ompt_callback(ompt_callback_lock_init)( ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(), - (ompt_wait_id_t)user_lock, codeptr); + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #if USE_ITT_BUILD __kmp_itt_lock_creating(lck); #endif /* USE_ITT_BUILD */ #endif // KMP_USE_DYNAMIC_LOCK } // __kmpc_init_nest_lock void __kmpc_destroy_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK #if USE_ITT_BUILD kmp_user_lock_p lck; if (KMP_EXTRACT_D_TAG(user_lock) == 0) { lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock; } else { lck = (kmp_user_lock_p)user_lock; } __kmp_itt_lock_destroyed(lck); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_destroy) { kmp_user_lock_p lck; if (KMP_EXTRACT_D_TAG(user_lock) == 0) { lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock; } else { lck = (kmp_user_lock_p)user_lock; } ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)( - ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock); #else kmp_user_lock_p lck; if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_destroy_lock"); } #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_destroy) { ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)( - ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #if USE_ITT_BUILD __kmp_itt_lock_destroyed(lck); #endif /* USE_ITT_BUILD */ DESTROY_LOCK(lck); if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) { ; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) { ; } #endif else { __kmp_user_lock_free(user_lock, gtid, lck); } #endif // KMP_USE_DYNAMIC_LOCK } // __kmpc_destroy_lock /* destroy the lock */ void __kmpc_destroy_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK #if USE_ITT_BUILD kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock); __kmp_itt_lock_destroyed(ilk->lock); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_destroy) { ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)( - ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock); #else // KMP_USE_DYNAMIC_LOCK kmp_user_lock_p lck; if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_destroy_nest_lock"); } #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_lock_destroy) { ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)( - ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #if USE_ITT_BUILD __kmp_itt_lock_destroyed(lck); #endif /* USE_ITT_BUILD */ DESTROY_NESTED_LOCK(lck); if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { ; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { ; } #endif else { __kmp_user_lock_free(user_lock, gtid, lck); } #endif // KMP_USE_DYNAMIC_LOCK } // __kmpc_destroy_nest_lock void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { KMP_COUNT_BLOCK(OMP_set_lock); #if KMP_USE_DYNAMIC_LOCK int tag = KMP_EXTRACT_D_TAG(user_lock); #if USE_ITT_BUILD __kmp_itt_lock_acquiring( (kmp_user_lock_p) user_lock); // itt function will get to the right lock object. #endif #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #if KMP_USE_INLINED_TAS if (tag == locktag_tas && !__kmp_env_consistency_check) { KMP_ACQUIRE_TAS_LOCK(user_lock, gtid); } else #elif KMP_USE_INLINED_FUTEX if (tag == locktag_futex && !__kmp_env_consistency_check) { KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid); } else #endif { __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid); } #if USE_ITT_BUILD __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #else // KMP_USE_DYNAMIC_LOCK kmp_user_lock_p lck; if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_set_lock"); } #if USE_ITT_BUILD __kmp_itt_lock_acquiring(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(), - (ompt_wait_id_t)lck, codeptr); + (ompt_wait_id_t)(uintptr_t)lck, codeptr); } #endif ACQUIRE_LOCK(lck, gtid); #if USE_ITT_BUILD __kmp_itt_lock_acquired(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } #endif #endif // KMP_USE_DYNAMIC_LOCK } void __kmpc_set_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK #if USE_ITT_BUILD __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.enabled) { if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_nest_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } } #endif int acquire_status = KMP_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid); (void) acquire_status; #if USE_ITT_BUILD __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) { if (ompt_enabled.ompt_callback_mutex_acquired) { // lock_first ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, + codeptr); } } else { if (ompt_enabled.ompt_callback_nest_lock) { // lock_next ompt_callbacks.ompt_callback(ompt_callback_nest_lock)( - ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr); + ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } } } #endif #else // KMP_USE_DYNAMIC_LOCK int acquire_status; kmp_user_lock_p lck; if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_set_nest_lock"); } #if USE_ITT_BUILD __kmp_itt_lock_acquiring(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.enabled) { if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_nest_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr); + __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)lck, + codeptr); } } #endif ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status); #if USE_ITT_BUILD __kmp_itt_lock_acquired(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) { if (ompt_enabled.ompt_callback_mutex_acquired) { // lock_first ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } else { if (ompt_enabled.ompt_callback_nest_lock) { // lock_next ompt_callbacks.ompt_callback(ompt_callback_nest_lock)( - ompt_scope_begin, (ompt_wait_id_t)lck, codeptr); + ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } } #endif #endif // KMP_USE_DYNAMIC_LOCK } void __kmpc_unset_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK int tag = KMP_EXTRACT_D_TAG(user_lock); #if USE_ITT_BUILD __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock); #endif #if KMP_USE_INLINED_TAS if (tag == locktag_tas && !__kmp_env_consistency_check) { KMP_RELEASE_TAS_LOCK(user_lock, gtid); } else #elif KMP_USE_INLINED_FUTEX if (tag == locktag_futex && !__kmp_env_consistency_check) { KMP_RELEASE_FUTEX_LOCK(user_lock, gtid); } else #endif { __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid); } #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_released) { ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #else // KMP_USE_DYNAMIC_LOCK kmp_user_lock_p lck; /* Can't use serial interval since not block structured */ /* release the lock */ if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) { #if KMP_OS_LINUX && \ (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) // "fast" path implemented to fix customer performance issue #if USE_ITT_BUILD __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock); #endif /* USE_ITT_BUILD */ TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0); KMP_MB(); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_released) { ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } #endif return; #else lck = (kmp_user_lock_p)user_lock; #endif } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_unset_lock"); } #if USE_ITT_BUILD __kmp_itt_lock_releasing(lck); #endif /* USE_ITT_BUILD */ RELEASE_LOCK(lck, gtid); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_released) { ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } #endif #endif // KMP_USE_DYNAMIC_LOCK } /* release the lock */ void __kmpc_unset_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK #if USE_ITT_BUILD __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock); #endif int release_status = KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid); (void) release_status; #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.enabled) { if (release_status == KMP_LOCK_RELEASED) { if (ompt_enabled.ompt_callback_mutex_released) { // release_lock_last ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, + codeptr); } } else if (ompt_enabled.ompt_callback_nest_lock) { // release_lock_prev ompt_callbacks.ompt_callback(ompt_callback_nest_lock)( - ompt_scope_end, (ompt_wait_id_t)user_lock, codeptr); + ompt_scope_end, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } } #endif #else // KMP_USE_DYNAMIC_LOCK kmp_user_lock_p lck; /* Can't use serial interval since not block structured */ if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { #if KMP_OS_LINUX && \ (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) // "fast" path implemented to fix customer performance issue kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock; #if USE_ITT_BUILD __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL int release_status = KMP_LOCK_STILL_HELD; #endif if (--(tl->lk.depth_locked) == 0) { TCW_4(tl->lk.poll, 0); #if OMPT_SUPPORT && OMPT_OPTIONAL release_status = KMP_LOCK_RELEASED; #endif } KMP_MB(); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.enabled) { if (release_status == KMP_LOCK_RELEASED) { if (ompt_enabled.ompt_callback_mutex_released) { // release_lock_last ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } else if (ompt_enabled.ompt_callback_nest_lock) { // release_lock_previous ompt_callbacks.ompt_callback(ompt_callback_nest_lock)( - ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } #endif return; #else lck = (kmp_user_lock_p)user_lock; #endif } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_unset_nest_lock"); } #if USE_ITT_BUILD __kmp_itt_lock_releasing(lck); #endif /* USE_ITT_BUILD */ int release_status; release_status = RELEASE_NESTED_LOCK(lck, gtid); #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.enabled) { if (release_status == KMP_LOCK_RELEASED) { if (ompt_enabled.ompt_callback_mutex_released) { // release_lock_last ompt_callbacks.ompt_callback(ompt_callback_mutex_released)( - ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } else if (ompt_enabled.ompt_callback_nest_lock) { // release_lock_previous ompt_callbacks.ompt_callback(ompt_callback_nest_lock)( - ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } #endif #endif // KMP_USE_DYNAMIC_LOCK } /* try to acquire the lock */ int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { KMP_COUNT_BLOCK(OMP_test_lock); #if KMP_USE_DYNAMIC_LOCK int rc; int tag = KMP_EXTRACT_D_TAG(user_lock); #if USE_ITT_BUILD __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif #if KMP_USE_INLINED_TAS if (tag == locktag_tas && !__kmp_env_consistency_check) { KMP_TEST_TAS_LOCK(user_lock, gtid, rc); } else #elif KMP_USE_INLINED_FUTEX if (tag == locktag_futex && !__kmp_env_consistency_check) { KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc); } else #endif { rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid); } if (rc) { #if USE_ITT_BUILD __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif return FTN_TRUE; } else { #if USE_ITT_BUILD __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock); #endif return FTN_FALSE; } #else // KMP_USE_DYNAMIC_LOCK kmp_user_lock_p lck; int rc; if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_test_lock"); } #if USE_ITT_BUILD __kmp_itt_lock_acquiring(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(), - (ompt_wait_id_t)lck, codeptr); + (ompt_wait_id_t)(uintptr_t)lck, codeptr); } #endif rc = TEST_LOCK(lck, gtid); #if USE_ITT_BUILD if (rc) { __kmp_itt_lock_acquired(lck); } else { __kmp_itt_lock_cancelled(lck); } #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL if (rc && ompt_enabled.ompt_callback_mutex_acquired) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } #endif return (rc ? FTN_TRUE : FTN_FALSE); /* Can't use serial interval since not block structured */ #endif // KMP_USE_DYNAMIC_LOCK } /* try to acquire the lock */ int __kmpc_test_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) { #if KMP_USE_DYNAMIC_LOCK int rc; #if USE_ITT_BUILD __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); #endif #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_nest_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock, - codeptr); + __ompt_get_mutex_impl_type(user_lock), + (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } #endif rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid); #if USE_ITT_BUILD if (rc) { __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); } else { __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock); } #endif #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled && rc) { if (rc == 1) { if (ompt_enabled.ompt_callback_mutex_acquired) { // lock_first ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, + codeptr); } } else { if (ompt_enabled.ompt_callback_nest_lock) { // lock_next ompt_callbacks.ompt_callback(ompt_callback_nest_lock)( - ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr); + ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr); } } } #endif return rc; #else // KMP_USE_DYNAMIC_LOCK kmp_user_lock_p lck; int rc; if ((__kmp_user_lock_kind == lk_tas) && (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #if KMP_USE_FUTEX else if ((__kmp_user_lock_kind == lk_futex) && (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <= OMP_NEST_LOCK_T_SIZE)) { lck = (kmp_user_lock_p)user_lock; } #endif else { lck = __kmp_lookup_user_lock(user_lock, "omp_test_nest_lock"); } #if USE_ITT_BUILD __kmp_itt_lock_acquiring(lck); #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL // This is the case, if called from omp_init_lock_with_hint: void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); if (!codeptr) codeptr = OMPT_GET_RETURN_ADDRESS(0); if (ompt_enabled.enabled) && ompt_enabled.ompt_callback_mutex_acquire) { ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)( ompt_mutex_nest_lock, omp_lock_hint_none, - __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr); + __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)lck, + codeptr); } #endif rc = TEST_NESTED_LOCK(lck, gtid); #if USE_ITT_BUILD if (rc) { __kmp_itt_lock_acquired(lck); } else { __kmp_itt_lock_cancelled(lck); } #endif /* USE_ITT_BUILD */ #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled && rc) { if (rc == 1) { if (ompt_enabled.ompt_callback_mutex_acquired) { // lock_first ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)( - ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } else { if (ompt_enabled.ompt_callback_nest_lock) { // lock_next ompt_callbacks.ompt_callback(ompt_callback_nest_lock)( - ompt_mutex_scope_begin, (ompt_wait_id_t)lck, codeptr); + ompt_mutex_scope_begin, (ompt_wait_id_t)(uintptr_t)lck, codeptr); } } } #endif return rc; /* Can't use serial interval since not block structured */ #endif // KMP_USE_DYNAMIC_LOCK } // Interface to fast scalable reduce methods routines // keep the selected method in a thread local structure for cross-function // usage: will be used in __kmpc_end_reduce* functions; // another solution: to re-determine the method one more time in // __kmpc_end_reduce* functions (new prototype required then) // AT: which solution is better? #define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \ ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod)) #define __KMP_GET_REDUCTION_METHOD(gtid) \ (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) // description of the packed_reduction_method variable: look at the macros in // kmp.h // used in a critical section reduce block static __forceinline void __kmp_enter_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit) { // this lock was visible to a customer and to the threading profile tool as a // serial overhead span (although it's used for an internal purpose only) // why was it visible in previous implementation? // should we keep it visible in new reduce block? kmp_user_lock_p lck; #if KMP_USE_DYNAMIC_LOCK kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit; // Check if it is initialized. if (*lk == 0) { if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) { KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(__kmp_user_lock_seq)); } else { __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(__kmp_user_lock_seq)); } } // Branch for accessing the actual lock object and set operation. This // branching is inevitable since this lock initialization does not follow the // normal dispatch path (lock table is not used). if (KMP_EXTRACT_D_TAG(lk) != 0) { lck = (kmp_user_lock_p)lk; KMP_DEBUG_ASSERT(lck != NULL); if (__kmp_env_consistency_check) { __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq); } KMP_D_LOCK_FUNC(lk, set)(lk, global_tid); } else { kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk); lck = ilk->lock; KMP_DEBUG_ASSERT(lck != NULL); if (__kmp_env_consistency_check) { __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq); } KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid); } #else // KMP_USE_DYNAMIC_LOCK // We know that the fast reduction code is only emitted by Intel compilers // with 32 byte critical sections. If there isn't enough space, then we // have to use a pointer. if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) { lck = (kmp_user_lock_p)crit; } else { lck = __kmp_get_critical_section_ptr(crit, loc, global_tid); } KMP_DEBUG_ASSERT(lck != NULL); if (__kmp_env_consistency_check) __kmp_push_sync(global_tid, ct_critical, loc, lck); __kmp_acquire_user_lock_with_checks(lck, global_tid); #endif // KMP_USE_DYNAMIC_LOCK } // used in a critical section reduce block static __forceinline void __kmp_end_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit) { kmp_user_lock_p lck; #if KMP_USE_DYNAMIC_LOCK if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) { lck = (kmp_user_lock_p)crit; if (__kmp_env_consistency_check) __kmp_pop_sync(global_tid, ct_critical, loc); KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid); } else { kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit)); if (__kmp_env_consistency_check) __kmp_pop_sync(global_tid, ct_critical, loc); KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid); } #else // KMP_USE_DYNAMIC_LOCK // We know that the fast reduction code is only emitted by Intel compilers // with 32 byte critical sections. If there isn't enough space, then we have // to use a pointer. if (__kmp_base_user_lock_size > 32) { lck = *((kmp_user_lock_p *)crit); KMP_ASSERT(lck != NULL); } else { lck = (kmp_user_lock_p)crit; } if (__kmp_env_consistency_check) __kmp_pop_sync(global_tid, ct_critical, loc); __kmp_release_user_lock_with_checks(lck, global_tid); #endif // KMP_USE_DYNAMIC_LOCK } // __kmp_end_critical_section_reduce_block #if OMP_40_ENABLED static __forceinline int __kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p, int *task_state) { kmp_team_t *team; // Check if we are inside the teams construct? if (th->th.th_teams_microtask) { *team_p = team = th->th.th_team; if (team->t.t_level == th->th.th_teams_level) { // This is reduction at teams construct. KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid); // AC: check that tid == 0 // Let's swap teams temporarily for the reduction. th->th.th_info.ds.ds_tid = team->t.t_master_tid; th->th.th_team = team->t.t_parent; th->th.th_team_nproc = th->th.th_team->t.t_nproc; th->th.th_task_team = th->th.th_team->t.t_task_team[0]; *task_state = th->th.th_task_state; th->th.th_task_state = 0; return 1; } } return 0; } static __forceinline void __kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team, int task_state) { // Restore thread structure swapped in __kmp_swap_teams_for_teams_reduction. th->th.th_info.ds.ds_tid = 0; th->th.th_team = team; th->th.th_team_nproc = team->t.t_nproc; th->th.th_task_team = team->t.t_task_team[task_state]; th->th.th_task_state = task_state; } #endif /* 2.a.i. Reduce Block without a terminating barrier */ /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid global thread number @param num_vars number of items (variables) to be reduced @param reduce_size size of data in bytes to be reduced @param reduce_data pointer to data to be reduced @param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data @param lck pointer to the unique lock data structure @result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed The nowait version is used for a reduce clause with the nowait argument. */ kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck) { KMP_COUNT_BLOCK(REDUCE_nowait); int retval = 0; PACKED_REDUCTION_METHOD_T packed_reduction_method; #if OMP_40_ENABLED kmp_info_t *th; kmp_team_t *team; int teams_swapped = 0, task_state; #endif KA_TRACE(10, ("__kmpc_reduce_nowait() enter: called T#%d\n", global_tid)); // why do we need this initialization here at all? // Reduction clause can not be used as a stand-alone directive. // do not call __kmp_serial_initialize(), it will be called by // __kmp_parallel_initialize() if needed // possible detection of false-positive race by the threadchecker ??? if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); // check correctness of reduce block nesting #if KMP_USE_DYNAMIC_LOCK if (__kmp_env_consistency_check) __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0); #else if (__kmp_env_consistency_check) __kmp_push_sync(global_tid, ct_reduce, loc, NULL); #endif #if OMP_40_ENABLED th = __kmp_thread_from_gtid(global_tid); teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state); #endif // OMP_40_ENABLED // packed_reduction_method value will be reused by __kmp_end_reduce* function, // the value should be kept in a variable // the variable should be either a construct-specific or thread-specific // property, not a team specific property // (a thread can reach the next reduce block on the next construct, reduce // method may differ on the next construct) // an ident_t "loc" parameter could be used as a construct-specific property // (what if loc == 0?) // (if both construct-specific and team-specific variables were shared, // then unness extra syncs should be needed) // a thread-specific variable is better regarding two issues above (next // construct and extra syncs) // a thread-specific "th_local.reduction_method" variable is used currently // each thread executes 'determine' and 'set' lines (no need to execute by one // thread, to avoid unness extra syncs) packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck); __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method); if (packed_reduction_method == critical_reduce_block) { __kmp_enter_critical_section_reduce_block(loc, global_tid, lck); retval = 1; } else if (packed_reduction_method == empty_reduce_block) { // usage: if team size == 1, no synchronization is required ( Intel // platforms only ) retval = 1; } else if (packed_reduction_method == atomic_reduce_block) { retval = 2; // all threads should do this pop here (because __kmpc_end_reduce_nowait() // won't be called by the code gen) // (it's not quite good, because the checking block has been closed by // this 'pop', // but atomic operation has not been executed yet, will be executed // slightly later, literally on next instruction) if (__kmp_env_consistency_check) __kmp_pop_sync(global_tid, ct_reduce, loc); } else if (TEST_REDUCTION_METHOD(packed_reduction_method, tree_reduce_block)) { // AT: performance issue: a real barrier here // AT: (if master goes slow, other threads are blocked here waiting for the // master to come and release them) // AT: (it's not what a customer might expect specifying NOWAIT clause) // AT: (specifying NOWAIT won't result in improvement of performance, it'll // be confusing to a customer) // AT: another implementation of *barrier_gather*nowait() (or some other design) // might go faster and be more in line with sense of NOWAIT // AT: TO DO: do epcc test and compare times // this barrier should be invisible to a customer and to the threading profile // tool (it's neither a terminating barrier nor customer's code, it's // used for an internal purpose) #if OMPT_SUPPORT // JP: can this barrier potentially leed to task scheduling? // JP: as long as there is a barrier in the implementation, OMPT should and // will provide the barrier events // so we set-up the necessary frame/return addresses. ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif #if USE_ITT_NOTIFY __kmp_threads[global_tid]->th.th_ident = loc; #endif retval = __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method), global_tid, FALSE, reduce_size, reduce_data, reduce_func); retval = (retval != 0) ? (0) : (1); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif // all other workers except master should do this pop here // ( none of other workers will get to __kmpc_end_reduce_nowait() ) if (__kmp_env_consistency_check) { if (retval == 0) { __kmp_pop_sync(global_tid, ct_reduce, loc); } } } else { // should never reach this block KMP_ASSERT(0); // "unexpected method" } #if OMP_40_ENABLED if (teams_swapped) { __kmp_restore_swapped_teams(th, team, task_state); } #endif KA_TRACE( 10, ("__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval)); return retval; } /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid global thread id. @param lck pointer to the unique lock data structure Finish the execution of a reduce nowait. */ void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck) { PACKED_REDUCTION_METHOD_T packed_reduction_method; KA_TRACE(10, ("__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid)); packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid); if (packed_reduction_method == critical_reduce_block) { __kmp_end_critical_section_reduce_block(loc, global_tid, lck); } else if (packed_reduction_method == empty_reduce_block) { // usage: if team size == 1, no synchronization is required ( on Intel // platforms only ) } else if (packed_reduction_method == atomic_reduce_block) { // neither master nor other workers should get here // (code gen does not generate this call in case 2: atomic reduce block) // actually it's better to remove this elseif at all; // after removal this value will checked by the 'else' and will assert } else if (TEST_REDUCTION_METHOD(packed_reduction_method, tree_reduce_block)) { // only master gets here } else { // should never reach this block KMP_ASSERT(0); // "unexpected method" } if (__kmp_env_consistency_check) __kmp_pop_sync(global_tid, ct_reduce, loc); KA_TRACE(10, ("__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method)); return; } /* 2.a.ii. Reduce Block with a terminating barrier */ /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid global thread number @param num_vars number of items (variables) to be reduced @param reduce_size size of data in bytes to be reduced @param reduce_data pointer to data to be reduced @param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data @param lck pointer to the unique lock data structure @result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed A blocking reduce that includes an implicit barrier. */ kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck) { KMP_COUNT_BLOCK(REDUCE_wait); int retval = 0; PACKED_REDUCTION_METHOD_T packed_reduction_method; #if OMP_40_ENABLED kmp_info_t *th; kmp_team_t *team; int teams_swapped = 0, task_state; #endif KA_TRACE(10, ("__kmpc_reduce() enter: called T#%d\n", global_tid)); // why do we need this initialization here at all? // Reduction clause can not be a stand-alone directive. // do not call __kmp_serial_initialize(), it will be called by // __kmp_parallel_initialize() if needed // possible detection of false-positive race by the threadchecker ??? if (!TCR_4(__kmp_init_parallel)) __kmp_parallel_initialize(); // check correctness of reduce block nesting #if KMP_USE_DYNAMIC_LOCK if (__kmp_env_consistency_check) __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0); #else if (__kmp_env_consistency_check) __kmp_push_sync(global_tid, ct_reduce, loc, NULL); #endif #if OMP_40_ENABLED th = __kmp_thread_from_gtid(global_tid); teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state); #endif // OMP_40_ENABLED packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck); __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method); if (packed_reduction_method == critical_reduce_block) { __kmp_enter_critical_section_reduce_block(loc, global_tid, lck); retval = 1; } else if (packed_reduction_method == empty_reduce_block) { // usage: if team size == 1, no synchronization is required ( Intel // platforms only ) retval = 1; } else if (packed_reduction_method == atomic_reduce_block) { retval = 2; } else if (TEST_REDUCTION_METHOD(packed_reduction_method, tree_reduce_block)) { // case tree_reduce_block: // this barrier should be visible to a customer and to the threading profile // tool (it's a terminating barrier on constructs if NOWAIT not specified) #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif #if USE_ITT_NOTIFY __kmp_threads[global_tid]->th.th_ident = loc; // needed for correct notification of frames #endif retval = __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method), global_tid, TRUE, reduce_size, reduce_data, reduce_func); retval = (retval != 0) ? (0) : (1); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif // all other workers except master should do this pop here // ( none of other workers except master will enter __kmpc_end_reduce() ) if (__kmp_env_consistency_check) { if (retval == 0) { // 0: all other workers; 1: master __kmp_pop_sync(global_tid, ct_reduce, loc); } } } else { // should never reach this block KMP_ASSERT(0); // "unexpected method" } #if OMP_40_ENABLED if (teams_swapped) { __kmp_restore_swapped_teams(th, team, task_state); } #endif KA_TRACE(10, ("__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval)); return retval; } /*! @ingroup SYNCHRONIZATION @param loc source location information @param global_tid global thread id. @param lck pointer to the unique lock data structure Finish the execution of a blocking reduce. The lck pointer must be the same as that used in the corresponding start function. */ void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck) { PACKED_REDUCTION_METHOD_T packed_reduction_method; #if OMP_40_ENABLED kmp_info_t *th; kmp_team_t *team; int teams_swapped = 0, task_state; #endif KA_TRACE(10, ("__kmpc_end_reduce() enter: called T#%d\n", global_tid)); #if OMP_40_ENABLED th = __kmp_thread_from_gtid(global_tid); teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state); #endif // OMP_40_ENABLED packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid); // this barrier should be visible to a customer and to the threading profile // tool (it's a terminating barrier on constructs if NOWAIT not specified) if (packed_reduction_method == critical_reduce_block) { __kmp_end_critical_section_reduce_block(loc, global_tid, lck); // TODO: implicit barrier: should be exposed #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif #if USE_ITT_NOTIFY __kmp_threads[global_tid]->th.th_ident = loc; #endif __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif } else if (packed_reduction_method == empty_reduce_block) { // usage: if team size==1, no synchronization is required (Intel platforms only) // TODO: implicit barrier: should be exposed #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif #if USE_ITT_NOTIFY __kmp_threads[global_tid]->th.th_ident = loc; #endif __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif } else if (packed_reduction_method == atomic_reduce_block) { #if OMPT_SUPPORT ompt_frame_t *ompt_frame; if (ompt_enabled.enabled) { __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); if (ompt_frame->enter_frame.ptr == NULL) ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); OMPT_STORE_RETURN_ADDRESS(global_tid); } #endif // TODO: implicit barrier: should be exposed #if USE_ITT_NOTIFY __kmp_threads[global_tid]->th.th_ident = loc; #endif __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL); #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.enabled) { ompt_frame->enter_frame = ompt_data_none; } #endif } else if (TEST_REDUCTION_METHOD(packed_reduction_method, tree_reduce_block)) { // only master executes here (master releases all other workers) __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method), global_tid); } else { // should never reach this block KMP_ASSERT(0); // "unexpected method" } #if OMP_40_ENABLED if (teams_swapped) { __kmp_restore_swapped_teams(th, team, task_state); } #endif if (__kmp_env_consistency_check) __kmp_pop_sync(global_tid, ct_reduce, loc); KA_TRACE(10, ("__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method)); return; } #undef __KMP_GET_REDUCTION_METHOD #undef __KMP_SET_REDUCTION_METHOD /* end of interface to fast scalable reduce routines */ kmp_uint64 __kmpc_get_taskid() { kmp_int32 gtid; kmp_info_t *thread; gtid = __kmp_get_gtid(); if (gtid < 0) { return 0; } thread = __kmp_thread_from_gtid(gtid); return thread->th.th_current_task->td_task_id; } // __kmpc_get_taskid kmp_uint64 __kmpc_get_parent_taskid() { kmp_int32 gtid; kmp_info_t *thread; kmp_taskdata_t *parent_task; gtid = __kmp_get_gtid(); if (gtid < 0) { return 0; } thread = __kmp_thread_from_gtid(gtid); parent_task = thread->th.th_current_task->td_parent; return (parent_task == NULL ? 0 : parent_task->td_task_id); } // __kmpc_get_parent_taskid #if OMP_45_ENABLED /*! @ingroup WORK_SHARING @param loc source location information. @param gtid global thread number. @param num_dims number of associated doacross loops. @param dims info on loops bounds. Initialize doacross loop information. Expect compiler send us inclusive bounds, e.g. for(i=2;i<9;i+=2) lo=2, up=8, st=2. */ void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, const struct kmp_dim *dims) { int j, idx; kmp_int64 last, trace_count; kmp_info_t *th = __kmp_threads[gtid]; kmp_team_t *team = th->th.th_team; kmp_uint32 *flags; kmp_disp_t *pr_buf = th->th.th_dispatch; dispatch_shared_info_t *sh_buf; KA_TRACE( 20, ("__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n", gtid, num_dims, !team->t.t_serialized)); KMP_DEBUG_ASSERT(dims != NULL); KMP_DEBUG_ASSERT(num_dims > 0); if (team->t.t_serialized) { KA_TRACE(20, ("__kmpc_doacross_init() exit: serialized team\n")); return; // no dependencies if team is serialized } KMP_DEBUG_ASSERT(team->t.t_nproc > 1); idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for // the next loop sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers]; // Save bounds info into allocated private buffer KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL); pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc( th, sizeof(kmp_int64) * (4 * num_dims + 1)); KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL); pr_buf->th_doacross_info[0] = (kmp_int64)num_dims; // first element is number of dimensions // Save also address of num_done in order to access it later without knowing // the buffer index pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done; pr_buf->th_doacross_info[2] = dims[0].lo; pr_buf->th_doacross_info[3] = dims[0].up; pr_buf->th_doacross_info[4] = dims[0].st; last = 5; for (j = 1; j < num_dims; ++j) { kmp_int64 range_length; // To keep ranges of all dimensions but the first dims[0] if (dims[j].st == 1) { // most common case // AC: should we care of ranges bigger than LLONG_MAX? (not for now) range_length = dims[j].up - dims[j].lo + 1; } else { if (dims[j].st > 0) { KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo); range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1; } else { // negative increment KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up); range_length = (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1; } } pr_buf->th_doacross_info[last++] = range_length; pr_buf->th_doacross_info[last++] = dims[j].lo; pr_buf->th_doacross_info[last++] = dims[j].up; pr_buf->th_doacross_info[last++] = dims[j].st; } // Compute total trip count. // Start with range of dims[0] which we don't need to keep in the buffer. if (dims[0].st == 1) { // most common case trace_count = dims[0].up - dims[0].lo + 1; } else if (dims[0].st > 0) { KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo); trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1; } else { // negative increment KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up); trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1; } for (j = 1; j < num_dims; ++j) { trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges } KMP_DEBUG_ASSERT(trace_count > 0); // Check if shared buffer is not occupied by other loop (idx - // __kmp_dispatch_num_buffers) if (idx != sh_buf->doacross_buf_idx) { // Shared buffer is occupied, wait for it to be free __kmp_wait_yield_4((volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx, __kmp_eq_4, NULL); } #if KMP_32_BIT_ARCH // Check if we are the first thread. After the CAS the first thread gets 0, // others get 1 if initialization is in progress, allocated pointer otherwise. // Treat pointer as volatile integer (value 0 or 1) until memory is allocated. flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32( (volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1); #else flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64( (volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL); #endif if (flags == NULL) { // we are the first thread, allocate the array of flags size_t size = trace_count / 8 + 8; // in bytes, use single bit per iteration flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1); KMP_MB(); sh_buf->doacross_flags = flags; } else if (flags == (kmp_uint32 *)1) { #if KMP_32_BIT_ARCH // initialization is still in progress, need to wait while (*(volatile kmp_int32 *)&sh_buf->doacross_flags == 1) #else while (*(volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL) #endif KMP_YIELD(TRUE); KMP_MB(); } else { KMP_MB(); } KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1); // check ptr value pr_buf->th_doacross_flags = sh_buf->doacross_flags; // save private copy in order to not // touch shared buffer on each iteration KA_TRACE(20, ("__kmpc_doacross_init() exit: T#%d\n", gtid)); } void __kmpc_doacross_wait(ident_t *loc, int gtid, const kmp_int64 *vec) { kmp_int32 shft, num_dims, i; kmp_uint32 flag; kmp_int64 iter_number; // iteration number of "collapsed" loop nest kmp_info_t *th = __kmp_threads[gtid]; kmp_team_t *team = th->th.th_team; kmp_disp_t *pr_buf; kmp_int64 lo, up, st; KA_TRACE(20, ("__kmpc_doacross_wait() enter: called T#%d\n", gtid)); if (team->t.t_serialized) { KA_TRACE(20, ("__kmpc_doacross_wait() exit: serialized team\n")); return; // no dependencies if team is serialized } // calculate sequential iteration number and check out-of-bounds condition pr_buf = th->th.th_dispatch; KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL); num_dims = pr_buf->th_doacross_info[0]; lo = pr_buf->th_doacross_info[2]; up = pr_buf->th_doacross_info[3]; st = pr_buf->th_doacross_info[4]; if (st == 1) { // most common case if (vec[0] < lo || vec[0] > up) { KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of " "bounds [%lld,%lld]\n", gtid, vec[0], lo, up)); return; } iter_number = vec[0] - lo; } else if (st > 0) { if (vec[0] < lo || vec[0] > up) { KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of " "bounds [%lld,%lld]\n", gtid, vec[0], lo, up)); return; } iter_number = (kmp_uint64)(vec[0] - lo) / st; } else { // negative increment if (vec[0] > lo || vec[0] < up) { KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of " "bounds [%lld,%lld]\n", gtid, vec[0], lo, up)); return; } iter_number = (kmp_uint64)(lo - vec[0]) / (-st); } for (i = 1; i < num_dims; ++i) { kmp_int64 iter, ln; kmp_int32 j = i * 4; ln = pr_buf->th_doacross_info[j + 1]; lo = pr_buf->th_doacross_info[j + 2]; up = pr_buf->th_doacross_info[j + 3]; st = pr_buf->th_doacross_info[j + 4]; if (st == 1) { if (vec[i] < lo || vec[i] > up) { KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of " "bounds [%lld,%lld]\n", gtid, vec[i], lo, up)); return; } iter = vec[i] - lo; } else if (st > 0) { if (vec[i] < lo || vec[i] > up) { KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of " "bounds [%lld,%lld]\n", gtid, vec[i], lo, up)); return; } iter = (kmp_uint64)(vec[i] - lo) / st; } else { // st < 0 if (vec[i] > lo || vec[i] < up) { KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of " "bounds [%lld,%lld]\n", gtid, vec[i], lo, up)); return; } iter = (kmp_uint64)(lo - vec[i]) / (-st); } iter_number = iter + ln * iter_number; } shft = iter_number % 32; // use 32-bit granularity iter_number >>= 5; // divided by 32 flag = 1 << shft; while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) { KMP_YIELD(TRUE); } KMP_MB(); KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n", gtid, (iter_number << 5) + shft)); } void __kmpc_doacross_post(ident_t *loc, int gtid, const kmp_int64 *vec) { kmp_int32 shft, num_dims, i; kmp_uint32 flag; kmp_int64 iter_number; // iteration number of "collapsed" loop nest kmp_info_t *th = __kmp_threads[gtid]; kmp_team_t *team = th->th.th_team; kmp_disp_t *pr_buf; kmp_int64 lo, st; KA_TRACE(20, ("__kmpc_doacross_post() enter: called T#%d\n", gtid)); if (team->t.t_serialized) { KA_TRACE(20, ("__kmpc_doacross_post() exit: serialized team\n")); return; // no dependencies if team is serialized } // calculate sequential iteration number (same as in "wait" but no // out-of-bounds checks) pr_buf = th->th.th_dispatch; KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL); num_dims = pr_buf->th_doacross_info[0]; lo = pr_buf->th_doacross_info[2]; st = pr_buf->th_doacross_info[4]; if (st == 1) { // most common case iter_number = vec[0] - lo; } else if (st > 0) { iter_number = (kmp_uint64)(vec[0] - lo) / st; } else { // negative increment iter_number = (kmp_uint64)(lo - vec[0]) / (-st); } for (i = 1; i < num_dims; ++i) { kmp_int64 iter, ln; kmp_int32 j = i * 4; ln = pr_buf->th_doacross_info[j + 1]; lo = pr_buf->th_doacross_info[j + 2]; st = pr_buf->th_doacross_info[j + 4]; if (st == 1) { iter = vec[i] - lo; } else if (st > 0) { iter = (kmp_uint64)(vec[i] - lo) / st; } else { // st < 0 iter = (kmp_uint64)(lo - vec[i]) / (-st); } iter_number = iter + ln * iter_number; } shft = iter_number % 32; // use 32-bit granularity iter_number >>= 5; // divided by 32 flag = 1 << shft; KMP_MB(); if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag); KA_TRACE(20, ("__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid, (iter_number << 5) + shft)); } void __kmpc_doacross_fini(ident_t *loc, int gtid) { kmp_int32 num_done; kmp_info_t *th = __kmp_threads[gtid]; kmp_team_t *team = th->th.th_team; kmp_disp_t *pr_buf = th->th.th_dispatch; KA_TRACE(20, ("__kmpc_doacross_fini() enter: called T#%d\n", gtid)); if (team->t.t_serialized) { KA_TRACE(20, ("__kmpc_doacross_fini() exit: serialized team %p\n", team)); return; // nothing to do } num_done = KMP_TEST_THEN_INC32((kmp_int32 *)pr_buf->th_doacross_info[1]) + 1; if (num_done == th->th.th_team_nproc) { // we are the last thread, need to free shared resources int idx = pr_buf->th_doacross_buf_idx - 1; dispatch_shared_info_t *sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers]; KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] == (kmp_int64)&sh_buf->doacross_num_done); KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done); KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx); __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags)); sh_buf->doacross_flags = NULL; sh_buf->doacross_num_done = 0; sh_buf->doacross_buf_idx += __kmp_dispatch_num_buffers; // free buffer for future re-use } // free private resources (need to keep buffer index forever) pr_buf->th_doacross_flags = NULL; __kmp_thread_free(th, (void *)pr_buf->th_doacross_info); pr_buf->th_doacross_info = NULL; KA_TRACE(20, ("__kmpc_doacross_fini() exit: T#%d\n", gtid)); } #endif #if OMP_50_ENABLED int __kmpc_get_target_offload(void) { if (!__kmp_init_serial) { __kmp_serial_initialize(); } return __kmp_target_offload; } #endif // OMP_50_ENABLED // end of file // Index: vendor/llvm-openmp/dist-release_80/runtime/src/ompt-specific.cpp =================================================================== --- vendor/llvm-openmp/dist-release_80/runtime/src/ompt-specific.cpp (revision 349788) +++ vendor/llvm-openmp/dist-release_80/runtime/src/ompt-specific.cpp (revision 349789) @@ -1,451 +1,451 @@ /* * ompt-specific.cpp -- OMPT internal functions */ //===----------------------------------------------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.txt for details. // //===----------------------------------------------------------------------===// //****************************************************************************** // include files //****************************************************************************** #include "kmp.h" #include "ompt-specific.h" #if KMP_OS_UNIX #include #endif #if KMP_OS_WINDOWS #define THREAD_LOCAL __declspec(thread) #else #define THREAD_LOCAL __thread #endif #define OMPT_WEAK_ATTRIBUTE KMP_WEAK_ATTRIBUTE //****************************************************************************** // macros //****************************************************************************** #define LWT_FROM_TEAM(team) (team)->t.ompt_serialized_team_info #define OMPT_THREAD_ID_BITS 16 //****************************************************************************** // private operations //****************************************************************************** //---------------------------------------------------------- // traverse the team and task hierarchy // note: __ompt_get_teaminfo and __ompt_get_task_info_object // traverse the hierarchy similarly and need to be // kept consistent //---------------------------------------------------------- ompt_team_info_t *__ompt_get_teaminfo(int depth, int *size) { kmp_info_t *thr = ompt_get_thread(); if (thr) { kmp_team *team = thr->th.th_team; if (team == NULL) return NULL; ompt_lw_taskteam_t *next_lwt = LWT_FROM_TEAM(team), *lwt = NULL; while (depth > 0) { // next lightweight team (if any) if (lwt) lwt = lwt->parent; // next heavyweight team (if any) after // lightweight teams are exhausted if (!lwt && team) { if (next_lwt) { lwt = next_lwt; next_lwt = NULL; } else { team = team->t.t_parent; if (team) { next_lwt = LWT_FROM_TEAM(team); } } } depth--; } if (lwt) { // lightweight teams have one task if (size) *size = 1; // return team info for lightweight team return &lwt->ompt_team_info; } else if (team) { // extract size from heavyweight team if (size) *size = team->t.t_nproc; // return team info for heavyweight team return &team->t.ompt_team_info; } } return NULL; } ompt_task_info_t *__ompt_get_task_info_object(int depth) { ompt_task_info_t *info = NULL; kmp_info_t *thr = ompt_get_thread(); if (thr) { kmp_taskdata_t *taskdata = thr->th.th_current_task; ompt_lw_taskteam_t *lwt = NULL, *next_lwt = LWT_FROM_TEAM(taskdata->td_team); while (depth > 0) { // next lightweight team (if any) if (lwt) lwt = lwt->parent; // next heavyweight team (if any) after // lightweight teams are exhausted if (!lwt && taskdata) { if (next_lwt) { lwt = next_lwt; next_lwt = NULL; } else { taskdata = taskdata->td_parent; if (taskdata) { next_lwt = LWT_FROM_TEAM(taskdata->td_team); } } } depth--; } if (lwt) { info = &lwt->ompt_task_info; } else if (taskdata) { info = &taskdata->ompt_task_info; } } return info; } ompt_task_info_t *__ompt_get_scheduling_taskinfo(int depth) { ompt_task_info_t *info = NULL; kmp_info_t *thr = ompt_get_thread(); if (thr) { kmp_taskdata_t *taskdata = thr->th.th_current_task; ompt_lw_taskteam_t *lwt = NULL, *next_lwt = LWT_FROM_TEAM(taskdata->td_team); while (depth > 0) { // next lightweight team (if any) if (lwt) lwt = lwt->parent; // next heavyweight team (if any) after // lightweight teams are exhausted if (!lwt && taskdata) { // first try scheduling parent (for explicit task scheduling) if (taskdata->ompt_task_info.scheduling_parent) { taskdata = taskdata->ompt_task_info.scheduling_parent; } else if (next_lwt) { lwt = next_lwt; next_lwt = NULL; } else { // then go for implicit tasks taskdata = taskdata->td_parent; if (taskdata) { next_lwt = LWT_FROM_TEAM(taskdata->td_team); } } } depth--; } if (lwt) { info = &lwt->ompt_task_info; } else if (taskdata) { info = &taskdata->ompt_task_info; } } return info; } //****************************************************************************** // interface operations //****************************************************************************** //---------------------------------------------------------- // thread support //---------------------------------------------------------- ompt_data_t *__ompt_get_thread_data_internal() { if (__kmp_get_gtid() >= 0) { kmp_info_t *thread = ompt_get_thread(); if (thread == NULL) return NULL; return &(thread->th.ompt_thread_info.thread_data); } return NULL; } //---------------------------------------------------------- // state support //---------------------------------------------------------- void __ompt_thread_assign_wait_id(void *variable) { kmp_info_t *ti = ompt_get_thread(); - ti->th.ompt_thread_info.wait_id = (ompt_wait_id_t)variable; + ti->th.ompt_thread_info.wait_id = (ompt_wait_id_t)(uintptr_t)variable; } int __ompt_get_state_internal(ompt_wait_id_t *omp_wait_id) { kmp_info_t *ti = ompt_get_thread(); if (ti) { if (omp_wait_id) *omp_wait_id = ti->th.ompt_thread_info.wait_id; return ti->th.ompt_thread_info.state; } return ompt_state_undefined; } //---------------------------------------------------------- // parallel region support //---------------------------------------------------------- int __ompt_get_parallel_info_internal(int ancestor_level, ompt_data_t **parallel_data, int *team_size) { if (__kmp_get_gtid() >= 0) { ompt_team_info_t *info; if (team_size) { info = __ompt_get_teaminfo(ancestor_level, team_size); } else { info = __ompt_get_teaminfo(ancestor_level, NULL); } if (parallel_data) { *parallel_data = info ? &(info->parallel_data) : NULL; } return info ? 2 : 0; } else { return 0; } } //---------------------------------------------------------- // lightweight task team support //---------------------------------------------------------- void __ompt_lw_taskteam_init(ompt_lw_taskteam_t *lwt, kmp_info_t *thr, int gtid, ompt_data_t *ompt_pid, void *codeptr) { // initialize parallel_data with input, return address to parallel_data on // exit lwt->ompt_team_info.parallel_data = *ompt_pid; lwt->ompt_team_info.master_return_address = codeptr; lwt->ompt_task_info.task_data.value = 0; lwt->ompt_task_info.frame.enter_frame = ompt_data_none; lwt->ompt_task_info.frame.exit_frame = ompt_data_none; lwt->ompt_task_info.scheduling_parent = NULL; lwt->ompt_task_info.deps = NULL; lwt->ompt_task_info.ndeps = 0; lwt->heap = 0; lwt->parent = 0; } void __ompt_lw_taskteam_link(ompt_lw_taskteam_t *lwt, kmp_info_t *thr, int on_heap) { ompt_lw_taskteam_t *link_lwt = lwt; if (thr->th.th_team->t.t_serialized > 1) { // we already have a team, so link the new team and swap values if (on_heap) { // the lw_taskteam cannot stay on stack, allocate it on heap link_lwt = (ompt_lw_taskteam_t *)__kmp_allocate(sizeof(ompt_lw_taskteam_t)); } link_lwt->heap = on_heap; // would be swap in the (on_stack) case. ompt_team_info_t tmp_team = lwt->ompt_team_info; link_lwt->ompt_team_info = *OMPT_CUR_TEAM_INFO(thr); *OMPT_CUR_TEAM_INFO(thr) = tmp_team; ompt_task_info_t tmp_task = lwt->ompt_task_info; link_lwt->ompt_task_info = *OMPT_CUR_TASK_INFO(thr); *OMPT_CUR_TASK_INFO(thr) = tmp_task; // link the taskteam into the list of taskteams: ompt_lw_taskteam_t *my_parent = thr->th.th_team->t.ompt_serialized_team_info; link_lwt->parent = my_parent; thr->th.th_team->t.ompt_serialized_team_info = link_lwt; } else { // this is the first serialized team, so we just store the values in the // team and drop the taskteam-object *OMPT_CUR_TEAM_INFO(thr) = lwt->ompt_team_info; *OMPT_CUR_TASK_INFO(thr) = lwt->ompt_task_info; } } void __ompt_lw_taskteam_unlink(kmp_info_t *thr) { ompt_lw_taskteam_t *lwtask = thr->th.th_team->t.ompt_serialized_team_info; if (lwtask) { thr->th.th_team->t.ompt_serialized_team_info = lwtask->parent; ompt_team_info_t tmp_team = lwtask->ompt_team_info; lwtask->ompt_team_info = *OMPT_CUR_TEAM_INFO(thr); *OMPT_CUR_TEAM_INFO(thr) = tmp_team; ompt_task_info_t tmp_task = lwtask->ompt_task_info; lwtask->ompt_task_info = *OMPT_CUR_TASK_INFO(thr); *OMPT_CUR_TASK_INFO(thr) = tmp_task; if (lwtask->heap) { __kmp_free(lwtask); lwtask = NULL; } } // return lwtask; } //---------------------------------------------------------- // task support //---------------------------------------------------------- int __ompt_get_task_info_internal(int ancestor_level, int *type, ompt_data_t **task_data, ompt_frame_t **task_frame, ompt_data_t **parallel_data, int *thread_num) { if (__kmp_get_gtid() < 0) return 0; if (ancestor_level < 0) return 0; // copied from __ompt_get_scheduling_taskinfo ompt_task_info_t *info = NULL; ompt_team_info_t *team_info = NULL; kmp_info_t *thr = ompt_get_thread(); int level = ancestor_level; if (thr) { kmp_taskdata_t *taskdata = thr->th.th_current_task; if (taskdata == NULL) return 0; kmp_team *team = thr->th.th_team, *prev_team = NULL; if (team == NULL) return 0; ompt_lw_taskteam_t *lwt = NULL, *next_lwt = LWT_FROM_TEAM(taskdata->td_team), *prev_lwt = NULL; while (ancestor_level > 0) { // needed for thread_num prev_team = team; prev_lwt = lwt; // next lightweight team (if any) if (lwt) lwt = lwt->parent; // next heavyweight team (if any) after // lightweight teams are exhausted if (!lwt && taskdata) { // first try scheduling parent (for explicit task scheduling) if (taskdata->ompt_task_info.scheduling_parent) { taskdata = taskdata->ompt_task_info.scheduling_parent; } else if (next_lwt) { lwt = next_lwt; next_lwt = NULL; } else { // then go for implicit tasks taskdata = taskdata->td_parent; if (team == NULL) return 0; team = team->t.t_parent; if (taskdata) { next_lwt = LWT_FROM_TEAM(taskdata->td_team); } } } ancestor_level--; } if (lwt) { info = &lwt->ompt_task_info; team_info = &lwt->ompt_team_info; if (type) { *type = ompt_task_implicit; } } else if (taskdata) { info = &taskdata->ompt_task_info; team_info = &team->t.ompt_team_info; if (type) { if (taskdata->td_parent) { *type = (taskdata->td_flags.tasktype ? ompt_task_explicit : ompt_task_implicit) | TASK_TYPE_DETAILS_FORMAT(taskdata); } else { *type = ompt_task_initial; } } } if (task_data) { *task_data = info ? &info->task_data : NULL; } if (task_frame) { // OpenMP spec asks for the scheduling task to be returned. *task_frame = info ? &info->frame : NULL; } if (parallel_data) { *parallel_data = team_info ? &(team_info->parallel_data) : NULL; } if (thread_num) { if (level == 0) *thread_num = __kmp_get_tid(); else if (prev_lwt) *thread_num = 0; else *thread_num = prev_team->t.t_master_tid; // *thread_num = team->t.t_master_tid; } return info ? 2 : 0; } return 0; } //---------------------------------------------------------- // team support //---------------------------------------------------------- void __ompt_team_assign_id(kmp_team_t *team, ompt_data_t ompt_pid) { team->t.ompt_team_info.parallel_data = ompt_pid; } //---------------------------------------------------------- // misc //---------------------------------------------------------- static uint64_t __ompt_get_unique_id_internal() { static uint64_t thread = 1; static THREAD_LOCAL uint64_t ID = 0; if (ID == 0) { uint64_t new_thread = KMP_TEST_THEN_INC64((kmp_int64 *)&thread); ID = new_thread << (sizeof(uint64_t) * 8 - OMPT_THREAD_ID_BITS); } return ++ID; }