diff --git a/rocclr/platform/command.hpp b/rocclr/platform/command.hpp index 5655f06602..fa8c315b57 100644 --- a/rocclr/platform/command.hpp +++ b/rocclr/platform/command.hpp @@ -183,7 +183,7 @@ class Event : public RuntimeObject { //! Signal all threads waiting on this event. void signal() { - ScopedLock lock(lock_); + ScopedLock lock(lock_);// Unnecessary lock_.notifyAll(); } diff --git a/rocclr/thread/monitor.cpp b/rocclr/thread/monitor.cpp index d7d4e23002..159ea1f19a 100644 --- a/rocclr/thread/monitor.cpp +++ b/rocclr/thread/monitor.cpp @@ -29,11 +29,14 @@ #include namespace amd { +MonitorBase::~MonitorBase() {} + +namespace legacy_monitor { Monitor::Monitor(const char* name, bool recursive) : contendersList_(0), onDeck_(0), waitersList_(NULL), owner_(NULL), recursive_(recursive) { if (name == NULL) { - const char* unknownName = "@unknown@"; + const char unknownName[] = "@unknown@"; assert(sizeof(unknownName) < sizeof(name_) && "just checking"); ::strncpy(name_, unknownName, sizeof(name_) - 1); } else { @@ -316,4 +319,89 @@ void Monitor::notifyAll() { } } +bool Monitor::tryLock() { + Thread* thread = Thread::current(); + assert(thread != NULL && "cannot lock() from (null)"); + + intptr_t ptr = contendersList_.load(std::memory_order_acquire); + + if (unlikely((ptr & kLockBit) != 0)) { + if (recursive_ && thread == owner_) { + // Recursive lock: increment the lock count and return. + ++lockCount_; + return true; + } + return false; // Already locked! + } + + if (unlikely(!contendersList_.compare_exchange_weak( + ptr, ptr | kLockBit, std::memory_order_acq_rel, std::memory_order_acquire))) { + return false; // We failed the CAS from unlocked to locked. + } + + setOwner(thread); // cannot move above the CAS. + lockCount_ = 1; + + return true; +} + +void Monitor::lock() { + if (unlikely(!tryLock())) { + // The lock is contented. + finishLock(); + } + + // This is the beginning of the critical region. From now-on, everything + // executes single-threaded! + // +} + +void Monitor::unlock() { + assert(isLocked() && owner_ == Thread::current() && "invariant"); + + if (recursive_ && --lockCount_ > 0) { + // was a recursive lock case, simply return. + return; + } + + setOwner(NULL); + + // Clear the lock bit. + intptr_t ptr = contendersList_.load(std::memory_order_acquire); + while (!contendersList_.compare_exchange_weak(ptr, ptr & ~kLockBit, std::memory_order_acq_rel, + std::memory_order_acquire)) + ; + + // A StoreLoad barrier is required to make sure future loads do not happen before the + // contendersList_ store is published. + std::atomic_thread_fence(std::memory_order_seq_cst); + + // + // We succeeded the CAS from locked to unlocked. + // This is the end of the critical region. + + // Check if we have an on-deck thread that needs signaling. + intptr_t onDeck = onDeck_; + if (onDeck != 0) { + if ((onDeck & kLockBit) == 0) { + // Only signal if it is unmarked. + reinterpret_cast(onDeck)->post(); + } + return; // We are done. + } + + // We do not have an on-deck thread yet, we might have to walk the list in + // order to select the next onDeck_. Only one thread needs to fill onDeck_, + // so return if the list is empty or if the lock got acquired again (it's + // somebody else's problem now!) + + intptr_t head = contendersList_; + if (head == 0 || (head & kLockBit) != 0) { + return; + } + + // Finish the unlock operation: find a thread to wake up. + finishUnlock(); +} +} // namespace legacy_monitor } // namespace amd diff --git a/rocclr/thread/monitor.hpp b/rocclr/thread/monitor.hpp index 9b5f73d4f4..f778849689 100644 --- a/rocclr/thread/monitor.hpp +++ b/rocclr/thread/monitor.hpp @@ -22,9 +22,11 @@ #define MONITOR_HPP_ #include "top.hpp" +#include "utils/flags.hpp" #include "thread/semaphore.hpp" #include "thread/thread.hpp" - +#include +#include #include #include #include @@ -69,7 +71,20 @@ template struct SimplyLinkedNode : publ } // namespace details -class Monitor : public HeapObject { +class MonitorBase { +public: + virtual ~MonitorBase() = 0; + virtual bool tryLock() = 0; + virtual void lock() = 0; + virtual void unlock() = 0; + virtual void wait() = 0; + virtual void notify() = 0; + virtual void notifyAll() = 0; + virtual const char* name() const = 0; +}; + +namespace legacy_monitor { +class Monitor final: public HeapObject, public MonitorBase { typedef details::SimplyLinkedNode LinkedNode; private: @@ -124,13 +139,13 @@ class Monitor : public HeapObject { ~Monitor() {} //! Try to acquire the lock, return true if successful. - inline bool tryLock(); + bool tryLock(); //! Acquire the lock or suspend the calling thread. - inline void lock(); + void lock(); //! Release the lock and wake a single waiting thread if any. - inline void unlock(); + void unlock(); /*! \brief Give up the lock and go to sleep. * @@ -155,10 +170,119 @@ class Monitor : public HeapObject { const char* name() const { return name_; } }; -class ScopedLock : StackObject { - private: - Monitor* lock_; +} // namespace legacy_monitor + +namespace mutex_monitor { +class Monitor final: public HeapObject, public MonitorBase { + public: + explicit Monitor(const char* name = nullptr, bool recursive = false) + : recursive_(recursive) { + if (name == NULL) { + const char unknownName[] = "@unknown@"; + assert(sizeof(unknownName) < sizeof(name_) && "just checking"); + ::strncpy(name_, unknownName, sizeof(name_) - 1); + } else { + ::strncpy(name_, name, sizeof(name_) - 1); + } + name_[sizeof(name_) - 1] = '\0'; + + if (recursive) + new (&rec_mutex_) std::recursive_mutex(); + else + new (&mutex_) std::mutex(); + } + + ~Monitor() { + // Caller must make sure the mutext is unlocked. + if (recursive_) + rec_mutex_.~recursive_mutex(); + else + mutex_.~mutex(); + } + + //! Try to acquire the lock, return true if successful, false if failed. + bool tryLock() { + return recursive_ ? rec_mutex_.try_lock() : mutex_.try_lock(); + } + + //! Acquire the lock or suspend the calling thread. + void lock() { + recursive_ ? rec_mutex_.lock() : mutex_.lock(); + } + + //! Release the lock and wake a single waiting thread if any. + void unlock() { + recursive_ ? rec_mutex_.unlock() : mutex_.unlock(); + } + + /*! \brief Give up the lock and go to sleep. + * + * Calling wait() causes the current thread to go to sleep until + * another thread calls notify()/notifyAll(). + * + * \note The monitor must be owned before calling wait(). + */ + void wait() { + assert(recursive_ == false && "wait() doesn't support recursive mode"); + // the mutex must be locked by caller + std::unique_lock lk(mutex_, std::adopt_lock); + cv_.wait(lk); + // the mutex is locked again + lk.release(); // Release the ownership so that the caller should unlock the mutex + } + + /*! \brief Wake up a single thread waiting on this monitor. + * + * \note The monitor may or may not be owned before calling notify(). + */ + void notify() { cv_.notify_one(); } + + /*! \brief Wake up all threads that are waiting on this monitor. + * + * \note The monitor may or may not be owned before calling notifyAll(). + */ + void notifyAll() { cv_.notify_all(); } + + //! Return this lock's name. + const char* name() const { return name_; } + + private: + union { + std::mutex mutex_; + std::recursive_mutex rec_mutex_; + }; + std::condition_variable cv_; //!< The condition variable for sync on the mutex + char name_[64]; //!< The mutex's name + const bool recursive_; //!< True if this is a recursive mutex, false otherwise. +}; +} // namespace mutex_monitor + +// Monitor API wrapper to user +class Monitor { +public: + explicit Monitor(const char* name = nullptr, bool recursive = false) { + if (DEBUG_CLR_USE_STDMUTEX_IN_AMD_MONITOR) { + monitor_ = new mutex_monitor::Monitor(name, recursive); + } + else { + monitor_ = new legacy_monitor::Monitor(name, recursive); + } + } + inline ~Monitor() { delete monitor_; }; + inline bool tryLock() { return monitor_->tryLock(); } + inline void lock() { monitor_->lock(); } + inline void unlock() { monitor_->unlock(); } + inline void wait() { monitor_->wait(); } + inline void notify() { monitor_->notify(); } + inline void notifyAll() { monitor_->notifyAll(); } + inline const char* name() { return monitor_->name(); } + +private: + MonitorBase* monitor_; +}; + +class ScopedLock : StackObject { public: ScopedLock(Monitor& lock) : lock_(&lock) { lock_->lock(); } @@ -169,97 +293,11 @@ class ScopedLock : StackObject { ~ScopedLock() { if (lock_) lock_->unlock(); } + + private: + Monitor* lock_; }; -/*! @} - * @} - */ - -inline bool Monitor::tryLock() { - Thread* thread = Thread::current(); - assert(thread != NULL && "cannot lock() from (null)"); - - intptr_t ptr = contendersList_.load(std::memory_order_acquire); - - if (unlikely((ptr & kLockBit) != 0)) { - if (recursive_ && thread == owner_) { - // Recursive lock: increment the lock count and return. - ++lockCount_; - return true; - } - return false; // Already locked! - } - - if (unlikely(!contendersList_.compare_exchange_weak( - ptr, ptr | kLockBit, std::memory_order_acq_rel, std::memory_order_acquire))) { - return false; // We failed the CAS from unlocked to locked. - } - - setOwner(thread); // cannot move above the CAS. - lockCount_ = 1; - - return true; -} - -inline void Monitor::lock() { - if (unlikely(!tryLock())) { - // The lock is contented. - finishLock(); - } - - // This is the beginning of the critical region. From now-on, everything - // executes single-threaded! - // -} - -inline void Monitor::unlock() { - assert(isLocked() && owner_ == Thread::current() && "invariant"); - - if (recursive_ && --lockCount_ > 0) { - // was a recursive lock case, simply return. - return; - } - - setOwner(NULL); - - // Clear the lock bit. - intptr_t ptr = contendersList_.load(std::memory_order_acquire); - while (!contendersList_.compare_exchange_weak(ptr, ptr & ~kLockBit, std::memory_order_acq_rel, - std::memory_order_acquire)) - ; - - // A StoreLoad barrier is required to make sure future loads do not happen before the - // contendersList_ store is published. - std::atomic_thread_fence(std::memory_order_seq_cst); - - // - // We succeeded the CAS from locked to unlocked. - // This is the end of the critical region. - - // Check if we have an on-deck thread that needs signaling. - intptr_t onDeck = onDeck_; - if (onDeck != 0) { - if ((onDeck & kLockBit) == 0) { - // Only signal if it is unmarked. - reinterpret_cast(onDeck)->post(); - } - return; // We are done. - } - - // We do not have an on-deck thread yet, we might have to walk the list in - // order to select the next onDeck_. Only one thread needs to fill onDeck_, - // so return if the list is empty or if the lock got acquired again (it's - // somebody else's problem now!) - - intptr_t head = contendersList_; - if (head == 0 || (head & kLockBit) != 0) { - return; - } - - // Finish the unlock operation: find a thread to wake up. - finishUnlock(); -} - } // namespace amd #endif /*MONITOR_HPP_*/ diff --git a/rocclr/utils/flags.hpp b/rocclr/utils/flags.hpp index c48eeb1523..5df612a07e 100644 --- a/rocclr/utils/flags.hpp +++ b/rocclr/utils/flags.hpp @@ -252,7 +252,9 @@ release(bool, DEBUG_HIP_GRAPH_DOT_PRINT, false, \ release(bool, HIP_ALWAYS_USE_NEW_COMGR_UNBUNDLING_ACTION, false, \ "Force to always use new comgr unbundling action") \ release(bool, DEBUG_HIP_KERNARG_COPY_OPT, true, \ - "Enable/Disable multiple kern arg copies") \ + "Enable/Disable multiple kern arg copies") \ +release(bool, DEBUG_CLR_USE_STDMUTEX_IN_AMD_MONITOR, false, \ + "Use std::mutext in amd::monotor") \ namespace amd {