From 1306432ae180deb962380bd051012cdb9b8585aa Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Thu, 1 Sep 2016 03:23:28 -0500 Subject: [PATCH] Switch atomic_helpers.h from C11 atomics to GCC __atomic builtins. C11 atomics are not statically guaranteed to be lock free and so may not be atomic with respect to atomic operations originating outside the standard library, such as platform atomics. C11 macros to statically discover always lock free operations (ATOMIC_*_LOCK_FREE) do not cover uint64_t in GCC and std::atomic is not a type alias of any covered type. All use of __atomic by atomic_helpers.h is statically checked to be always lock free. GCC builtin fencing does not appear to be strong enough for WC memory. Added an option (enabled) to enforce consistency for WC memory on x64. __sync builtin's were not used as they were declared legacy by GCC. Added a strongly conservative option (ALWAYS_CONSERVATIVE) to enable use of full memory fences in place of partial fences and compiler driven processor specific optimization. Change-Id: Id7aaaca626144070f58759f6a348cbee4612bbc0 [ROCm/ROCR-Runtime commit: 1bc15bbf79c4b8b53a7afe585070378094b30ed4] --- .../hsa-runtime/core/util/atomic_helpers.h | 276 ++++++++++++------ 1 file changed, 194 insertions(+), 82 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/atomic_helpers.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/atomic_helpers.h index d75bc205e8..69a2a58ab1 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/atomic_helpers.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/atomic_helpers.h @@ -40,45 +40,119 @@ // //////////////////////////////////////////////////////////////////////////////// -// Helpers to use non-atomic types with C++11 atomic operations. - +/* + Helpers to use native types with C++11 atomic operations. + Fixes GCC builtin functionality for x86 with respect to WC and non-temporal + stores. +*/ #ifndef HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_ #define HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_ #include #include "utils.h" -/// @brief: Special assert used here to check each atomic variable for lock free -/// implementation. -/// ANY locked atomics are very likely incompatable with out-of-library -/// concurrent access (HW access for instance) -#define lockless_check(exp) assert(exp) +//ALWAYS_CONSERVATIVE will very likely overfence your code. +//For use as a debugging aid only. +#define ALWAYS_CONSERVATIVE 0 + +#if !ALWAYS_CONSERVATIVE +#ifdef __x86_64 +#define X64_ORDER_WC 1 +#endif +#if X64_ORDER_WC +#include +#endif +#endif namespace atomic { -/// @brief: Checks if type T is compatible with its atomic representation. -/// @param: ptr(Input), a pointer to type T for check. -/// @return: void. + +static constexpr int c11ToBuiltInFlags(std::memory_order order) +{ +#if ALWAYS_CONSERVATIVE + return __ATOMIC_RELAXED; +#elif X64_ORDER_WC + return __ATOMIC_RELAXED; +#else + return (order == std::memory_order_relaxed) ? __ATOMIC_RELAXED : + (order == std::memory_order_acquire) ? __ATOMIC_ACQUIRE : + (order == std::memory_order_release) ? __ATOMIC_RELEASE : + (order == std::memory_order_seq_cst) ? __ATOMIC_SEQ_CST : + (order == std::memory_order_consume) ? __ATOMIC_CONSUME : + (order == std::memory_order_acq_rel) ? __ATOMIC_ACQ_REL : + __ATOMIC_SEQ_CST; +#endif +} + +static __forceinline void PreFence(std::memory_order order) { +#if ALWAYS_CONSERVATIVE + switch (order) { + case std::memory_order_release: + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + __atomic_thread_fence(__ATOMIC_SEQ_CST); + default:; + } +#elif X64_ORDER_WC + switch (order) { + case std::memory_order_release: + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + _mm_sfence(); + default:; + } +#endif +} + +static __forceinline void PostFence(std::memory_order order) { +#if ALWAYS_CONSERVATIVE + switch (order) { + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + case std::memory_order_acquire: + __atomic_thread_fence(__ATOMIC_SEQ_CST); + default:; + } +#elif X64_ORDER_WC + switch (order) { + case std::memory_order_seq_cst: + return _mm_mfence(); + case std::memory_order_acq_rel: + case std::memory_order_acquire: + return _mm_lfence(); + default:; + } +#endif +} + +static __forceinline void Fence(std::memory_order order=std::memory_order_seq_cst) { +#if ALWAYS_CONSERVATIVE + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif X64_ORDER_WC + switch (order) { + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + return _mm_mfence(); + case std::memory_order_acquire: + return _mm_lfence(); + case std::memory_order_release: + return _mm_sfence(); + default:; + } +#else + std::atomic_thread_fence(order); +#endif +} + template static __forceinline void BasicCheck(const T* ptr) { - static_assert(sizeof(T) == sizeof(std::atomic), - "Type is size incompatible with its atomic representation!"); - lockless_check( - reinterpret_cast*>(ptr)->is_lock_free() && - "Atomic operation is not lock free! Use may conflict with peripheral HW " - "atomics!"); + constexpr bool value = __atomic_always_lock_free(sizeof(T), 0); + static_assert(value, "Atomic type may not be compatible with peripheral atomics."); }; -/// @brief: function overloading, for more info, see previous one. -/// @param: ptr(Input), a pointer to a volatile type. -/// @return: void. template static __forceinline void BasicCheck(const volatile T* ptr) { - static_assert(sizeof(T) == sizeof(std::atomic), - "Type is size incompatible with its atomic representation!"); - lockless_check( - reinterpret_cast*>(ptr)->is_lock_free() && - "Atomic operation is not lock free! Use may conflict with peripheral HW " - "atomics!"); + constexpr bool value = __atomic_always_lock_free(sizeof(T), 0); + static_assert(value, "Atomic type may not be compatible with peripheral atomics."); }; /// @brief: Load value of type T atomically with specified memory order. @@ -89,8 +163,11 @@ template static __forceinline T Load(const T* ptr, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - const std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->load(order); + T ret; + PreFence(order); + __atomic_load(ptr, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: function overloading, for more info, see previous one. @@ -102,9 +179,11 @@ static __forceinline T Load(const volatile T* ptr, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile const std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->load(order); + T ret; + PreFence(order); + __atomic_load(ptr, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Store value of type T with specified memory order. @@ -116,8 +195,9 @@ template static __forceinline void Store( T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - aptr->store(val, order); + PreFence(order); + __atomic_store(ptr, &val, c11ToBuiltInFlags(order)); + PostFence(order); } /// @brief: Function overloading, for more info, see previous one. @@ -130,9 +210,9 @@ static __forceinline void Store( volatile T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - aptr->store(val, order); + PreFence(order); + __atomic_store(ptr, &val, c11ToBuiltInFlags(order)); + PostFence(order); } /// @brief: Compare and swap value atomically with specified memory order. @@ -146,8 +226,9 @@ static __forceinline T Cas(T* ptr, T val, T expected, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - aptr->compare_exchange_strong(expected, val, order); + PreFence(order); + __atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED); + PostFence(order); return expected; } @@ -162,9 +243,9 @@ static __forceinline T Cas(volatile T* ptr, T val, T expected, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - aptr->compare_exchange_strong(expected, val, order); + PreFence(order); + __atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED); + PostFence(order); return expected; } @@ -178,8 +259,11 @@ static __forceinline T Exchange(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->exchange(val, order); + T ret; + PreFence(order); + __atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Function overloading, for more info, see previous one. @@ -192,9 +276,11 @@ static __forceinline T Exchange(volatile T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->exchange(val, order); + T ret; + PreFence(order); + __atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Add value to variable atomically with specified memory order. @@ -206,8 +292,10 @@ template static __forceinline T Add(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->fetch_add(val, order); + PreFence(order); + T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Subtract value from the variable atomically with specified memory @@ -220,8 +308,10 @@ template static __forceinline T Sub(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->fetch_sub(val, order); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Bit And operation on variable atomically with specified memory @@ -234,8 +324,10 @@ template static __forceinline T And(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->fetch_and(val, order); + PreFence(order); + T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Bit Or operation on variable atomically with specified memory order. @@ -247,8 +339,10 @@ template static __forceinline T Or(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->fetch_or(val, order); + PreFence(order); + T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Bit Xor operation on variable atomically with specified memory @@ -261,8 +355,10 @@ template static __forceinline T Xor(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->fetch_xor(val, order); + PreFence(order); + T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Increase the value of variable atomically with specified memory @@ -274,8 +370,10 @@ template static __forceinline T Increment(T* ptr, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->fetch_add(1, order); + PreFence(order); + T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Decrease the value of the variable atomically with specified memory @@ -287,8 +385,10 @@ template static __forceinline T Decrement(T* ptr, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - std::atomic* aptr = reinterpret_cast*>(ptr); - return aptr->fetch_sub(1, order); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Add value to variable atomically with specified memory order. @@ -301,9 +401,10 @@ static __forceinline T Add(volatile T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->fetch_add(val, order); + PreFence(order); + T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Subtract value from the variable atomically with specified memory @@ -317,9 +418,10 @@ static __forceinline T Sub(volatile T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->fetch_sub(val, order); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Bit And operation on variable atomically with specified memory @@ -333,9 +435,10 @@ static __forceinline T And(volatile T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->fetch_and(val, order); + PreFence(order); + T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Bit Or operation on variable atomically with specified memory order. @@ -347,9 +450,10 @@ template static __forceinline T Or(volatile T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->fetch_or(val, order); + PreFence(order); + T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Bit Xor operation on variable atomically with specified memory @@ -363,9 +467,10 @@ static __forceinline T Xor(volatile T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->fetch_xor(val, order); + PreFence(order); + T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Increase the value of variable atomically with specified memory @@ -378,9 +483,10 @@ static __forceinline T Increment(volatile T* ptr, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->fetch_add(1, order); + PreFence(order); + T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } /// @brief: Decrease the value of the variable atomically with specified memory @@ -393,13 +499,19 @@ static __forceinline T Decrement(volatile T* ptr, std::memory_order order = std::memory_order_relaxed) { BasicCheck(ptr); - volatile std::atomic* aptr = - reinterpret_cast*>(ptr); - return aptr->fetch_sub(1, order); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; } } -// Remove special assert to avoid name polution -#undef lockless_check +#ifdef X64_ORDER_WC +#undef X64_ORDER_WC +#endif + +#ifdef ALWAYS_CONSERVATIVE +#undef ALWAYS_CONSERVATIVE +#endif #endif // HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_