From 8f6983d69b10c8b63149486b18acc5ae82eed9e4 Mon Sep 17 00:00:00 2001 From: foreman Date: Tue, 30 Sep 2014 17:40:37 -0400 Subject: [PATCH] P4 to Git Change 1082964 by lmoriche@lmoriche_opencl_dev on 2014/09/30 17:09:28 ECR #304775 - Replace amd::Atomic with std::atomic (cont'd) Pre-checkin: http://ocltc.amd.com:8111/viewModification.html?modId=40674&personal=true&buildTypeId=&tab=vcsModificationBuilds&show_all_builds=true Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/thread/atomic.hpp#6 edit ... //depot/stg/opencl/drivers/opencl/runtime/thread/monitor.cpp#6 edit ... //depot/stg/opencl/drivers/opencl/runtime/thread/monitor.hpp#7 edit ... //depot/stg/opencl/drivers/opencl/runtime/utils/concurrent.hpp#7 edit [ROCm/clr commit: d780c32a1b3e9c2e11d24c8959f1615e6d249a0b] --- projects/clr/rocclr/runtime/thread/atomic.hpp | 135 +----------------- .../clr/rocclr/runtime/thread/monitor.cpp | 100 +++++++------ .../clr/rocclr/runtime/thread/monitor.hpp | 45 +++--- .../clr/rocclr/runtime/utils/concurrent.hpp | 2 +- 4 files changed, 78 insertions(+), 204 deletions(-) diff --git a/projects/clr/rocclr/runtime/thread/atomic.hpp b/projects/clr/rocclr/runtime/thread/atomic.hpp index db13a9f061..8477ece1e0 100644 --- a/projects/clr/rocclr/runtime/thread/atomic.hpp +++ b/projects/clr/rocclr/runtime/thread/atomic.hpp @@ -23,6 +23,7 @@ # include #endif // !_WIN32 +#include #include namespace amd { @@ -30,59 +31,6 @@ namespace amd { /*! \addtogroup Threads * @{ * - * \defgroup MemOrder Memory ordering - * @{ - */ - -/*! \brief Memory order access operations. - */ -class MemoryOrder : AllStatic -{ -public: - /*! \brief Execute a memory fence. - * - * Perform a serializing operation on loads and stores which guarantees - * that all memory operations dispatched prior to the fence will be - * globally visible before any other memory operation following the fence. - */ - static void fence() { -# if defined(ATI_ARCH_X86) - _mm_mfence(); -# else // !ATI_ARCH_X86 - __sync_synchronize(); -# endif // !ATI_ARCH_X86 - } - - /*! \brief Execute a loads fence. - * - * Perform a serializing operation on loads which guarantees that all - * load from memory operations dispatched prior to the lfence will be - * globally visible before any other load following the lfence. - */ - static void lfence() { -# if defined(ATI_ARCH_X86) - _mm_lfence(); -# else // !ATI_ARCH_X86 - fence(); -# endif // !ATI_ARCH_X86 - } - - /*! \brief Execute a stores fence. - * - * Perform a serializing operation on stores which guarantees that all - * store to memory operations dispatched prior to the sfence will be - * globally visible before any other store following the sfence. - */ - static void sfence() { -# if defined(ATI_ARCH_X86) - _mm_sfence(); -# else // !ATI_ARCH_X86 - fence(); -# endif // !ATI_ARCH_X86 - } -}; - -/*! @} * \addtogroup Atomic Atomic Operations * @{ */ @@ -564,7 +512,7 @@ public: */ void storeRelease(T value) { - MemoryOrder::fence(); + std::atomic_thread_fence(std::memory_order_release); value_ = value; } @@ -577,7 +525,7 @@ public: T loadAcquire() const { T value = value_; - MemoryOrder::fence(); + std::atomic_thread_fence(std::memory_order_acquire); return value; } }; @@ -591,83 +539,6 @@ make_atomic(T& t) } -template -class AtomicMarkableReference -{ -private: - static const intptr_t kMarkBitMask = 0x1; - -private: - Atomic reference_; - -private: - static intptr_t markMask(bool mark) - { - return mark ? kMarkBitMask : intptr_t(0); - } - -public: - AtomicMarkableReference() - : reference_(NULL) - { } - - AtomicMarkableReference(T* ptr, bool mark = false) - : reference_((T*)((intptr_t) ptr | markMask(mark))) - { } - - bool compareAndSet( - T* expectedPtr, T* newPtr, - bool expectedMark, bool newMark) - { - return reference_.compareAndSet( - (T*)((intptr_t) expectedPtr | markMask(expectedMark)), - (T*)((intptr_t) newPtr | markMask(newMark))); - } - - std::pair swap(T* newPtr, bool newMark) - { - T* prev = reference_.swap( - (T*)((intptr_t) newPtr | markMask(newMark))); - return std::make_pair( - (T*) ((intptr_t) prev & ~kMarkBitMask), - ((intptr_t) prev & kMarkBitMask) != 0); - } - - bool tryMark(T* expectedPtr, bool newMark) - { - T* current = reference_; - if (((intptr_t) current & ~kMarkBitMask) != (intptr_t) expectedPtr) { - return false; - } - bool currentMark = ((intptr_t) current & kMarkBitMask) != 0; - return currentMark == newMark || reference_.compareAndSet(current, - (T*)((intptr_t) expectedPtr | markMask(newMark))); - } - - bool isMarked() const - { - return ((intptr_t)(T*) reference_ & kMarkBitMask) != 0; - } - - std::pair get() const - { - T* current = reference_; - return std::make_pair( - (T*) ((intptr_t) current & ~kMarkBitMask), - ((intptr_t) current & kMarkBitMask) != 0); - } - - T* getReference() const - { - return (T*) ((intptr_t)(T*) reference_ & ~kMarkBitMask); - } - - void set(T* ptr, bool mark) - { - reference_ = (T*)((intptr_t) ptr | markMask(mark)); - } -}; - /*! @} * @} */ diff --git a/projects/clr/rocclr/runtime/thread/monitor.cpp b/projects/clr/rocclr/runtime/thread/monitor.cpp index 6c9ffb11c4..9a3d7de433 100644 --- a/projects/clr/rocclr/runtime/thread/monitor.cpp +++ b/projects/clr/rocclr/runtime/thread/monitor.cpp @@ -15,7 +15,7 @@ namespace amd { Monitor::Monitor(const char* name, bool recursive) : - contendersList_(NULL), onDeck_(NULL), waitersList_(NULL), + contendersList_(0), onDeck_(0), waitersList_(NULL), owner_(NULL), recursive_(recursive) { const size_t maxNameLen = sizeof(name_); @@ -68,19 +68,17 @@ Monitor::finishLock() /* The lock is contended. Push the thread's semaphore onto * the contention list. */ - Semaphore& sem = thread->lockSemaphore(); - sem.reset(); + Semaphore& semaphore = thread->lockSemaphore(); + semaphore.reset(); LinkedNode newHead; - newHead.setItem(&sem); - - while (true) { - LinkedNode* head; bool isLocked; + newHead.setItem(&semaphore); + intptr_t head = contendersList_.load(std::memory_order_acquire); + for (;;) { // The assumption is that lockWord is locked. Make sure we do not // continue unless the lock bit is set. - std::tie(head, isLocked) = contendersList_.get(); - if (!isLocked) { + if ((head & kLockBit) == 0) { if (tryLock()) { return; } @@ -88,8 +86,10 @@ Monitor::finishLock() } // Set the new contention list head if lockWord is unchanged. - newHead.setNext(head); - if (contendersList_.compareAndSet(head, &newHead, kLocked, kLocked)) { + newHead.setNext(reinterpret_cast(head & ~kLockBit)); + if (contendersList_.compare_exchange_weak(head, + reinterpret_cast(&newHead) | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) { break; } @@ -99,7 +99,7 @@ Monitor::finishLock() int32_t spinCount = 0; // Go to sleep until we become the on-deck thread. - while (onDeck_.getReference() != &sem) { + while ((onDeck_ & ~kLockBit) != reinterpret_cast(&semaphore)) { // First, be SMT friendly if (spinCount < kMaxReadSpinIter) { Os::spinPause(); @@ -110,7 +110,7 @@ Monitor::finishLock() } // now go to sleep else { - sem.wait(); + semaphore.wait(); } spinCount++; } @@ -120,8 +120,9 @@ Monitor::finishLock() // From now-on, we are the on-deck thread. It will stay that way until // we successfuly acquire the lock. // - while (true) { - assert(onDeck_.getReference() == &sem && "just checking"); + for (;;) { + assert((onDeck_ & ~kLockBit) == reinterpret_cast(&semaphore) + && "just checking"); if (tryLock()) { break; } @@ -138,13 +139,13 @@ Monitor::finishLock() } // now go to sleep else { - sem.wait(); + semaphore.wait(); } spinCount++; } assert(newHead.next() == NULL && "Should not be linked"); - onDeck_ = NULL; + onDeck_ = 0; } void @@ -154,53 +155,58 @@ Monitor::finishUnlock() // list waiting to acquire the lock. We need to select a successor and // place it on-deck. - while (true) { + for (;;) { // Grab the onDeck_ microlock to protect the next loop (make sure only // one semaphore is removed from the contention list). // - if (!onDeck_.compareAndSet(NULL, NULL, kUnlocked, kLocked)) { + intptr_t ptr = 0; + if (!onDeck_.compare_exchange_strong(ptr, ptr | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) { return; // Somebody else has the microlock, let him select onDeck_ } - LinkedNode* head; bool isLocked; - while (true) { - std::tie(head, isLocked) = contendersList_.get(); - - if (head == NULL) { + intptr_t head = contendersList_.load(std::memory_order_acquire); + for (;;) { + if (head == 0) { break; // There's nothing else to do. } - if (isLocked) { + if ((head & kLockBit) != 0) { // Somebody could have acquired then released the lock // and failed to grab the onDeck_ microlock. - head = NULL; + head = 0; break; } - if (contendersList_.compareAndSet( - head, head->next(), kUnlocked, kUnlocked)) { + if (contendersList_.compare_exchange_weak( + head, reinterpret_cast( + reinterpret_cast(head)->next()), + std::memory_order_acq_rel, std::memory_order_acquire)) { #ifdef ASSERT - head->setNext(NULL); + reinterpret_cast(head)->setNext(NULL); #endif // ASSERT break; } } - Semaphore* sem = (head != NULL) ? head->item() : NULL; - onDeck_ = sem; - MemoryOrder::fence(); + Semaphore* semaphore = (head != 0) + ? reinterpret_cast(head)->item() + : NULL; + + onDeck_.store(reinterpret_cast(semaphore), + std::memory_order_release); // // Release the onDeck_ microlock (end of critical region); - if (sem != NULL) { - sem->post(); + if (semaphore != NULL) { + semaphore->post(); return; } - // We do not have an on-deck thread (sem == NULL). Return if + // We do not have an on-deck thread (semaphore == NULL). Return if // the contention list is empty or if the lock got acquired again. - std::tie(head, isLocked) = contendersList_.get(); - if (isLocked || head == NULL) { + head = contendersList_; + if (head == 0 || (head & kLockBit) != 0) { return; } } @@ -230,7 +236,7 @@ Monitor::wait() // Go to sleep until we become the on-deck thread. int32_t spinCount = 0; - while (onDeck_.getReference() != &suspend) { + while ((onDeck_ & ~kLockBit) != reinterpret_cast(&suspend)) { // First, be SMT friendly if (spinCount < kMaxReadSpinIter) { Os::spinPause(); @@ -247,8 +253,9 @@ Monitor::wait() } spinCount = 0; - while (true) { - assert(onDeck_.getReference() == &suspend && "just checking"); + for (;;) { + assert((onDeck_ & ~kLockBit) == reinterpret_cast(&suspend) + && "just checking"); if (trySpinLock()) { break; @@ -274,8 +281,7 @@ Monitor::wait() // Restore the lock count (for recursive mutexes) lockCount_ = lockCount; - onDeck_ = NULL; - MemoryOrder::fence(); + onDeck_.store(0, std::memory_order_release); } void @@ -290,11 +296,13 @@ Monitor::notify() // Dequeue a waiter from the wait list and add it to the contention list. waitersList_ = waiter->next(); - while (true) { - LinkedNode* node = contendersList_.getReference(); - waiter->setNext(node); - if (contendersList_.compareAndSet(node, waiter, kLocked, kLocked)) { + intptr_t node = contendersList_.load(std::memory_order_acquire); + for (;;) { + waiter->setNext(reinterpret_cast(node & ~kLockBit)); + if (contendersList_.compare_exchange_weak(node, + reinterpret_cast(waiter) | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) { break; } } diff --git a/projects/clr/rocclr/runtime/thread/monitor.hpp b/projects/clr/rocclr/runtime/thread/monitor.hpp index 4729ea0876..fac75b72d8 100644 --- a/projects/clr/rocclr/runtime/thread/monitor.hpp +++ b/projects/clr/rocclr/runtime/thread/monitor.hpp @@ -62,8 +62,7 @@ class Monitor : public HeapObject typedef details::SimplyLinkedNode LinkedNode; private: - static const bool kUnlocked = false; - static const bool kLocked = true; + static const intptr_t kLockBit = 0x1; static const int kMaxSpinIter = 55; //!< Total number of spin iterations. static const int kMaxReadSpinIter = 50; //!< Read iterations before yielding @@ -71,12 +70,12 @@ private: /*! Linked list of semaphores the contending threads are waiting on * and main lock. */ - AtomicMarkableReference contendersList_; + std::atomic_intptr_t contendersList_; //! The Mutex's name char name_[64]; //! Semaphore of the next thread to contend for the lock. - AtomicMarkableReference onDeck_; + std::atomic_intptr_t onDeck_; //! Linked list of the suspended threads resume semaphores. LinkedNode* volatile waitersList_; @@ -101,7 +100,7 @@ protected: * * \note The user is responsible for the memory ordering. */ - bool isLocked() const { return contendersList_.isMarked(); } + bool isLocked() const { return (contendersList_ & kLockBit) != 0; } //! Return this monitor's owner thread (NULL if unlocked). Thread* owner() const { return owner_; } @@ -179,10 +178,9 @@ Monitor::tryLock() Thread* thread = Thread::current(); assert(thread != NULL && "cannot lock() from (null)"); - LinkedNode* ptr; bool isLocked; - std::tie(ptr, isLocked) = contendersList_.get(); + intptr_t ptr = contendersList_.load(std::memory_order_acquire); - if (unlikely(isLocked)) { + if (unlikely((ptr & kLockBit) != 0)) { if (recursive_ && thread == owner_) { // Recursive lock: increment the lock count and return. ++lockCount_; @@ -191,8 +189,8 @@ Monitor::tryLock() return false; // Already locked! } - if (unlikely(!contendersList_.compareAndSet( - ptr, ptr, kUnlocked, kLocked))) { + if (unlikely(!contendersList_.compare_exchange_weak(ptr, ptr | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire))) { return false; // We failed the CAS from unlocked to locked. } @@ -227,23 +225,21 @@ Monitor::unlock() setOwner(NULL); - while (true) { - LinkedNode* ptr = contendersList_.getReference(); - // Clear the lock bit. - if (contendersList_.compareAndSet(ptr, ptr, kLocked, kUnlocked)) { - break; // We succeeded the CAS from locked to unlocked. - } - } + // Clear the lock bit. + intptr_t ptr = contendersList_.load(std::memory_order_acquire); + while (!contendersList_.compare_exchange_weak(ptr, ptr & ~kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) + ; // + // We succeeded the CAS from locked to unlocked. // This is the end of the critical region. // Check if we have an on-deck thread that needs signaling. - Semaphore* onDeck; bool isMarked; - std::tie(onDeck, isMarked) = onDeck_.get(); - if (onDeck != NULL) { - if (!isMarked) { + intptr_t onDeck = onDeck_; + if (onDeck != 0) { + if ((onDeck & kLockBit) == 0) { // Only signal if it is unmarked. - onDeck->post(); + reinterpret_cast(onDeck)->post(); } return; // We are done. } @@ -253,9 +249,8 @@ Monitor::unlock() // so return if the list is empty or if the lock got acquired again (it's // somebody else's problem now!) - LinkedNode* head; bool isLocked; - std::tie(head, isLocked) = contendersList_.get(); - if (isLocked || head == NULL) { + intptr_t head = contendersList_; + if (head == 0 || (head & kLockBit) != 0) { return; } diff --git a/projects/clr/rocclr/runtime/utils/concurrent.hpp b/projects/clr/rocclr/runtime/utils/concurrent.hpp index a52c8701ef..679291b1b8 100644 --- a/projects/clr/rocclr/runtime/utils/concurrent.hpp +++ b/projects/clr/rocclr/runtime/utils/concurrent.hpp @@ -126,7 +126,7 @@ ConcurrentLinkedQueue::ConcurrentLinkedQueue() // Make sure the instance is fully initialized before it becomes // globally visible. - MemoryOrder::sfence(); + std::atomic_thread_fence(std::memory_order_release); } template