From 8d3fee5095262ce4b1878031201370ce6ace92f2 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Tue, 19 Dec 2023 15:18:38 +0000 Subject: [PATCH] Use HybridMutex for signal mutexes Implement HybridMutex to improve latencies compared to KernelMutex when there is contention between several threads calling hsa_signal_create and hsa_amd_signal_async_handler. Change-Id: If53377033e749b0050727964c9303f09b02527cc --- .../hsa-runtime/core/inc/interrupt_signal.h | 2 +- runtime/hsa-runtime/core/inc/runtime.h | 2 +- runtime/hsa-runtime/core/inc/signal.h | 2 +- .../core/runtime/interrupt_signal.cpp | 4 +- runtime/hsa-runtime/core/runtime/runtime.cpp | 4 +- runtime/hsa-runtime/core/runtime/signal.cpp | 4 +- .../hsa-runtime/core/util/lnx/os_linux.cpp | 25 +++++++++ runtime/hsa-runtime/core/util/locks.h | 55 +++++++++++++++++++ runtime/hsa-runtime/core/util/os.h | 22 ++++++++ runtime/hsa-runtime/core/util/win/os_win.cpp | 24 ++++++++ 10 files changed, 135 insertions(+), 9 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/interrupt_signal.h b/runtime/hsa-runtime/core/inc/interrupt_signal.h index fea227ba9a..1652c1b751 100644 --- a/runtime/hsa-runtime/core/inc/interrupt_signal.h +++ b/runtime/hsa-runtime/core/inc/interrupt_signal.h @@ -83,7 +83,7 @@ class InterruptSignal : private LocalSignal, public Signal { } private: - KernelMutex lock_; + HybridMutex lock_; std::vector events_; bool allEventsAllocated; }; diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index d2d9d48e16..d566e93665 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -501,7 +501,7 @@ class Runtime { hsa_signal_t wake; os::Thread async_events_thread_; - KernelMutex lock; + HybridMutex lock; bool exit; }; diff --git a/runtime/hsa-runtime/core/inc/signal.h b/runtime/hsa-runtime/core/inc/signal.h index f3e42fe7e7..39e5321867 100644 --- a/runtime/hsa-runtime/core/inc/signal.h +++ b/runtime/hsa-runtime/core/inc/signal.h @@ -172,7 +172,7 @@ class SharedSignalPool_t : private BaseShared { private: static const size_t minblock_ = 4096 / sizeof(SharedSignal); - KernelMutex lock_; + HybridMutex lock_; std::vector free_list_; std::vector> block_list_; size_t block_size_; diff --git a/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp b/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp index 4fd6a1c0d2..428d7ab99d 100644 --- a/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp +++ b/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp @@ -54,7 +54,7 @@ namespace rocr { namespace core { HsaEvent* InterruptSignal::EventPool::alloc() { - ScopedAcquire lock(&lock_); + ScopedAcquire lock(&lock_); if (events_.empty()) { if (!allEventsAllocated) { HsaEvent* evt = InterruptSignal::CreateEvent(HSA_EVENTTYPE_SIGNAL, false); @@ -70,7 +70,7 @@ HsaEvent* InterruptSignal::EventPool::alloc() { void InterruptSignal::EventPool::free(HsaEvent* evt) { if (evt == nullptr) return; - ScopedAcquire lock(&lock_); + ScopedAcquire lock(&lock_); events_.push_back(unique_event_ptr(evt)); } diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 344e7bf9b4..9b1417ce02 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -754,7 +754,7 @@ hsa_status_t Runtime::SetAsyncSignalHandler(hsa_signal_t signal, // Indicate that this signal is in use. if (signal.handle != 0) hsa_signal_handle(signal)->Retain(); - ScopedAcquire scope_lock(&async_events_control_.lock); + ScopedAcquire scope_lock(&async_events_control_.lock); // Lazy initializer if (async_events_control_.async_events_thread_ == NULL) { @@ -1233,7 +1233,7 @@ void Runtime::AsyncEventsLoop(void*) { typedef std::pair func_arg_t; std::vector functions; { - ScopedAcquire scope_lock(&async_events_control_.lock); + ScopedAcquire scope_lock(&async_events_control_.lock); for (size_t i = 0; i < new_async_events_.Size(); i++) { if (new_async_events_.signal_[i].handle == 0) { functions.push_back( diff --git a/runtime/hsa-runtime/core/runtime/signal.cpp b/runtime/hsa-runtime/core/runtime/signal.cpp index 06fd9eba90..017d24689d 100644 --- a/runtime/hsa-runtime/core/runtime/signal.cpp +++ b/runtime/hsa-runtime/core/runtime/signal.cpp @@ -70,7 +70,7 @@ void SharedSignalPool_t::clear() { } SharedSignal* SharedSignalPool_t::alloc() { - ScopedAcquire lock(&lock_); + ScopedAcquire lock(&lock_); if (free_list_.empty()) { SharedSignal* block = reinterpret_cast( allocate_(block_size_ * sizeof(SharedSignal), __alignof(SharedSignal), 0)); @@ -103,7 +103,7 @@ void SharedSignalPool_t::free(SharedSignal* ptr) { if (ptr == nullptr) return; ptr->~SharedSignal(); - ScopedAcquire lock(&lock_); + ScopedAcquire lock(&lock_); ifdebug { bool valid = false; diff --git a/runtime/hsa-runtime/core/util/lnx/os_linux.cpp b/runtime/hsa-runtime/core/util/lnx/os_linux.cpp index 54a9c99ffc..7850490e22 100644 --- a/runtime/hsa-runtime/core/util/lnx/os_linux.cpp +++ b/runtime/hsa-runtime/core/util/lnx/os_linux.cpp @@ -59,6 +59,7 @@ #include #include #include +#include #include "core/inc/runtime.h" #if defined(__i386__) || defined(__x86_64__) #include @@ -201,6 +202,7 @@ class os_thread { }; static_assert(sizeof(LibHandle) == sizeof(void*), "OS abstraction size mismatch"); +static_assert(sizeof(Semaphore) == sizeof(sem_t*), "OS abstraction size mismatch"); static_assert(sizeof(Mutex) == sizeof(pthread_mutex_t*), "OS abstraction size mismatch"); static_assert(sizeof(SharedMutex) == sizeof(pthread_rwlock_t*), "OS abstraction size mismatch"); static_assert(sizeof(Thread) == sizeof(os_thread*), "OS abstraction size mismatch"); @@ -345,6 +347,29 @@ std::string GetLibraryName(LibHandle lib) { return map->l_name; } +Semaphore CreateSemaphore() { + sem_t *sem = new sem_t; + sem_init(sem, 0, 0); + return *(Semaphore*)&sem; +} + +bool WaitSemaphore(Semaphore sem) { + while(sem_wait(*(sem_t**)&sem)) + if (errno != EINTR) return false; + + return true; +} + +void PostSemaphore(Semaphore sem) { + if (sem_post(*(sem_t**)&sem)) + assert(false && "Failed to post semaphore"); +} + +void DestroySemaphore(Semaphore sem) { + sem_destroy(*(sem_t**)&sem); + delete *(sem_t**)&sem; +} + Mutex CreateMutex() { pthread_mutex_t* mutex = new pthread_mutex_t; pthread_mutex_init(mutex, NULL); diff --git a/runtime/hsa-runtime/core/util/locks.h b/runtime/hsa-runtime/core/util/locks.h index fd4d4569ba..6c0de49a07 100644 --- a/runtime/hsa-runtime/core/util/locks.h +++ b/runtime/hsa-runtime/core/util/locks.h @@ -50,6 +50,57 @@ namespace rocr { +class HybridMutex { + public: + HybridMutex():lock_(0) { + sem_ = os::CreateSemaphore(); + } + + ~HybridMutex() { + os::DestroySemaphore(sem_); + } + + bool Try() { + int old = 0; + return lock_.compare_exchange_strong(old, 1); + } + + bool Acquire() { + int cnt = maxSpinIterPause + maxSpinIterYield; + + int old = 0; + while (!lock_.compare_exchange_strong(old, 1)) { + cnt--; + if (cnt > maxSpinIterPause) { + _mm_pause(); + } else if (cnt-- > maxSpinIterYield) { + os::YieldThread(); + } else { + os::WaitSemaphore(sem_); + cnt = maxSpinIterPause + maxSpinIterYield; + } + old = 0; + } + return true; + } + + void Release() { + int old = 1; + if (lock_.compare_exchange_strong(old, 0)) + os::PostSemaphore(sem_); + } + + private: + std::atomic lock_; + os::Semaphore sem_; + const uint32_t maxSpinIterPause = 55; + const uint32_t maxSpinIterYield = 55; + + /// @brief: Disable copiable and assignable ability. + DISALLOW_COPY_AND_ASSIGN(HybridMutex); +}; + + /// @brief: a class represents a kernel mutex. /// Uses the kernel's scheduler to keep the waiting thread from being scheduled /// until the lock is released (Best for long waits, though anything using @@ -160,6 +211,10 @@ template class isMutex { public: enum { value = false }; }; +template <> class isMutex { + public: + enum { value = true }; +}; template <> class isMutex { public: enum { value = true }; diff --git a/runtime/hsa-runtime/core/util/os.h b/runtime/hsa-runtime/core/util/os.h index 8f7dcdffd4..50a7b0fbd4 100644 --- a/runtime/hsa-runtime/core/util/os.h +++ b/runtime/hsa-runtime/core/util/os.h @@ -52,6 +52,7 @@ namespace rocr { namespace os { typedef void* LibHandle; +typedef void* Semaphore; typedef void* Mutex; typedef void* SharedMutex; typedef void* Thread; @@ -96,6 +97,27 @@ std::vector GetLoadedToolsLib(); /// @return: Path name of library std::string GetLibraryName(LibHandle lib); +/// @brief: Creates a Semaphore, will return NULL if failed. +/// @param: void. +/// @return: Semaphore. +Semaphore CreateSemaphore(); + +/// @brief: Waits for the semaphore. This is a blocking wait. +/// If the Semaphore is signalled, this function will return. +/// @param: sem(Input), handle to the semaphore. +/// @return: void. +bool WaitSemaphore(Semaphore sem); + +/// @brief: Post/Signal/Wake-up the semaphore +/// @param: sem(Input), handle to the semaphore. +/// @return: void. +void PostSemaphore(Semaphore sem); + +/// @brief: Destroys the semaphore. +/// @param: sem(Input), handle to the semaphore. +/// @return: void. +void DestroySemaphore(Semaphore sem); + /// @brief: Creates a mutex, will return NULL if failed. /// @param: void. /// @return: Mutex. diff --git a/runtime/hsa-runtime/core/util/win/os_win.cpp b/runtime/hsa-runtime/core/util/win/os_win.cpp index da5c8b0d1a..81c90cd266 100644 --- a/runtime/hsa-runtime/core/util/win/os_win.cpp +++ b/runtime/hsa-runtime/core/util/win/os_win.cpp @@ -64,6 +64,8 @@ static_assert(sizeof(LibHandle) == sizeof(HMODULE), "OS abstraction size mismatch"); static_assert(sizeof(LibHandle) == sizeof(::HANDLE), "OS abstraction size mismatch"); +static_assert(sizeof(Semaphore) == sizeof(::HANDLE), + "OS abstraction size mismatch"); static_assert(sizeof(Mutex) == sizeof(::HANDLE), "OS abstraction size mismatch"); static_assert(sizeof(Thread) == sizeof(::HANDLE), @@ -91,6 +93,28 @@ std::string GetLibraryName(LibHandle lib) { static_assert(false, "Not implemented."); } +Semaphore CreateSemaphore() { + sem = static_cast(CreateSemaphore(NULL, 0, LONG_MAX, NULL)); + assert(sem != NULL && "CreateSemaphore failed"); + + return *(Semaphore*)&sem; +} + +bool WaitSemaphore(Semaphore sem) { + return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0; +} + +void PostSemaphore(Semaphore sem) { + ReleaseSemaphore(static_cast(*sem), 1, NULL); +} + +void DestroySemaphore(Semaphore sem) { + if (!CloseHandle(static_cast(*sem))) { + assert("CloseHandle() failed"); + } + *sem = NULL; +} + Mutex CreateMutex() { return CreateEvent(NULL, false, true, NULL); } bool TryAcquireMutex(Mutex lock) {