Use HybridMutex for signal mutexes

Implement HybridMutex to improve latencies compared to KernelMutex when
there is contention between several threads calling hsa_signal_create
and hsa_amd_signal_async_handler.

Change-Id: If53377033e749b0050727964c9303f09b02527cc

runtime/hsa-runtime/core/inc/interrupt_signal.h

@@ -83,7 +83,7 @@ class InterruptSignal : private LocalSignal, public Signal {
     }
 
    private:
-    KernelMutex lock_;
+    HybridMutex lock_;
     std::vector<unique_event_ptr> events_;
     bool allEventsAllocated;
   };

runtime/hsa-runtime/core/inc/runtime.h

@@ -501,7 +501,7 @@ class Runtime {
 
     hsa_signal_t wake;
     os::Thread async_events_thread_;
-    KernelMutex lock;
+    HybridMutex lock;
     bool exit;
   };
 

runtime/hsa-runtime/core/inc/signal.h

@@ -172,7 +172,7 @@ class SharedSignalPool_t : private BaseShared {
 
  private:
   static const size_t minblock_ = 4096 / sizeof(SharedSignal);
-  KernelMutex lock_;
+  HybridMutex lock_;
   std::vector<SharedSignal*> free_list_;
   std::vector<std::pair<void*, size_t>> block_list_;
   size_t block_size_;

runtime/hsa-runtime/core/runtime/interrupt_signal.cpp

@@ -54,7 +54,7 @@ namespace rocr {
 namespace core {
 
 HsaEvent* InterruptSignal::EventPool::alloc() {
-  ScopedAcquire<KernelMutex> lock(&lock_);
+  ScopedAcquire<HybridMutex> lock(&lock_);
   if (events_.empty()) {
     if (!allEventsAllocated) {
       HsaEvent* evt = InterruptSignal::CreateEvent(HSA_EVENTTYPE_SIGNAL, false);
@@ -70,7 +70,7 @@ HsaEvent* InterruptSignal::EventPool::alloc() {
 
 void InterruptSignal::EventPool::free(HsaEvent* evt) {
   if (evt == nullptr) return;
-  ScopedAcquire<KernelMutex> lock(&lock_);
+  ScopedAcquire<HybridMutex> lock(&lock_);
   events_.push_back(unique_event_ptr(evt));
 }
 

runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -754,7 +754,7 @@ hsa_status_t Runtime::SetAsyncSignalHandler(hsa_signal_t signal,
   // Indicate that this signal is in use.
   if (signal.handle != 0) hsa_signal_handle(signal)->Retain();
 
-  ScopedAcquire<KernelMutex> scope_lock(&async_events_control_.lock);
+  ScopedAcquire<HybridMutex> scope_lock(&async_events_control_.lock);
 
   // Lazy initializer
   if (async_events_control_.async_events_thread_ == NULL) {
@@ -1233,7 +1233,7 @@ void Runtime::AsyncEventsLoop(void*) {
     typedef std::pair<void (*)(void*), void*> func_arg_t;
     std::vector<func_arg_t> functions;
     {
-      ScopedAcquire<KernelMutex> scope_lock(&async_events_control_.lock);
+      ScopedAcquire<HybridMutex> scope_lock(&async_events_control_.lock);
       for (size_t i = 0; i < new_async_events_.Size(); i++) {
         if (new_async_events_.signal_[i].handle == 0) {
           functions.push_back(

runtime/hsa-runtime/core/runtime/signal.cpp

@@ -70,7 +70,7 @@ void SharedSignalPool_t::clear() {
 }
 
 SharedSignal* SharedSignalPool_t::alloc() {
-  ScopedAcquire<KernelMutex> lock(&lock_);
+  ScopedAcquire<HybridMutex> lock(&lock_);
   if (free_list_.empty()) {
     SharedSignal* block = reinterpret_cast<SharedSignal*>(
         allocate_(block_size_ * sizeof(SharedSignal), __alignof(SharedSignal), 0));
@@ -103,7 +103,7 @@ void SharedSignalPool_t::free(SharedSignal* ptr) {
   if (ptr == nullptr) return;
 
   ptr->~SharedSignal();
-  ScopedAcquire<KernelMutex> lock(&lock_);
+  ScopedAcquire<HybridMutex> lock(&lock_);
 
   ifdebug {
     bool valid = false;

runtime/hsa-runtime/core/util/lnx/os_linux.cpp

@@ -59,6 +59,7 @@
 #include <memory>
 #include <string>
 #include <utility>
+#include <semaphore.h>
 #include "core/inc/runtime.h"
 #if defined(__i386__) || defined(__x86_64__)
 #include <cpuid.h>
@@ -201,6 +202,7 @@ class os_thread {
 };
 
 static_assert(sizeof(LibHandle) == sizeof(void*), "OS abstraction size mismatch");
+static_assert(sizeof(Semaphore) == sizeof(sem_t*), "OS abstraction size mismatch");
 static_assert(sizeof(Mutex) == sizeof(pthread_mutex_t*), "OS abstraction size mismatch");
 static_assert(sizeof(SharedMutex) == sizeof(pthread_rwlock_t*), "OS abstraction size mismatch");
 static_assert(sizeof(Thread) == sizeof(os_thread*), "OS abstraction size mismatch");
@@ -345,6 +347,29 @@ std::string GetLibraryName(LibHandle lib) {
   return map->l_name;
 }
 
+Semaphore CreateSemaphore() {
+  sem_t *sem = new sem_t;
+  sem_init(sem, 0, 0);
+  return *(Semaphore*)&sem;
+}
+
+bool WaitSemaphore(Semaphore sem) {
+  while(sem_wait(*(sem_t**)&sem))
+    if (errno != EINTR) return false;
+
+  return true;
+}
+
+void PostSemaphore(Semaphore sem) {
+  if (sem_post(*(sem_t**)&sem))
+    assert(false && "Failed to post semaphore");
+}
+
+void DestroySemaphore(Semaphore sem) {
+  sem_destroy(*(sem_t**)&sem);
+  delete *(sem_t**)&sem;
+}
+
 Mutex CreateMutex() {
   pthread_mutex_t* mutex = new pthread_mutex_t;
   pthread_mutex_init(mutex, NULL);

runtime/hsa-runtime/core/util/locks.h

@@ -50,6 +50,57 @@
 
 namespace rocr {
 
+class HybridMutex {
+ public:
+  HybridMutex():lock_(0) { 
+    sem_ = os::CreateSemaphore(); 
+  }
+
+  ~HybridMutex() { 
+    os::DestroySemaphore(sem_); 
+  }
+
+  bool Try() {
+    int old = 0;
+    return lock_.compare_exchange_strong(old, 1);
+  }
+
+  bool Acquire() {
+    int cnt = maxSpinIterPause + maxSpinIterYield;
+
+    int old = 0;
+    while (!lock_.compare_exchange_strong(old, 1)) {
+      cnt--;
+      if (cnt > maxSpinIterPause) {
+        _mm_pause();
+      } else if (cnt-- > maxSpinIterYield) {
+        os::YieldThread();
+      } else {
+        os::WaitSemaphore(sem_);
+        cnt = maxSpinIterPause + maxSpinIterYield;
+      }
+      old = 0;
+    }
+    return true;
+  }
+
+  void Release() {
+    int old = 1;
+    if (lock_.compare_exchange_strong(old, 0))
+      os::PostSemaphore(sem_);
+  }
+
+ private:
+  std::atomic<int> lock_;
+  os::Semaphore sem_;
+  const uint32_t maxSpinIterPause = 55;
+  const uint32_t maxSpinIterYield = 55;
+
+  /// @brief: Disable copiable and assignable ability.
+  DISALLOW_COPY_AND_ASSIGN(HybridMutex);
+};
+
+
 /// @brief: a class represents a kernel mutex.
 /// Uses the kernel's scheduler to keep the waiting thread from being scheduled
 /// until the lock is released (Best for long waits, though anything using
@@ -160,6 +211,10 @@ template <class T> class isMutex {
  public:
   enum { value = false };
 };
+template <> class isMutex<HybridMutex> {
+ public:
+  enum { value = true };
+};
 template <> class isMutex<KernelMutex> {
  public:
   enum { value = true };

runtime/hsa-runtime/core/util/os.h

@@ -52,6 +52,7 @@
 namespace rocr {
 namespace os {
 typedef void* LibHandle;
+typedef void* Semaphore;
 typedef void* Mutex;
 typedef void* SharedMutex;
 typedef void* Thread;
@@ -96,6 +97,27 @@ std::vector<LibHandle> GetLoadedToolsLib();
 /// @return: Path name of library
 std::string GetLibraryName(LibHandle lib);
 
+/// @brief: Creates a Semaphore, will return NULL if failed.
+/// @param: void.
+/// @return: Semaphore.
+Semaphore CreateSemaphore();
+
+/// @brief: Waits for the semaphore. This is a blocking wait.
+/// If the Semaphore is signalled, this function will return.
+/// @param: sem(Input), handle to the semaphore.
+/// @return: void.
+bool WaitSemaphore(Semaphore sem);
+
+/// @brief: Post/Signal/Wake-up the semaphore
+/// @param: sem(Input), handle to the semaphore.
+/// @return: void.
+void PostSemaphore(Semaphore sem);
+
+/// @brief: Destroys the semaphore.
+/// @param: sem(Input), handle to the semaphore.
+/// @return: void.
+void DestroySemaphore(Semaphore sem);
+
 /// @brief: Creates a mutex, will return NULL if failed.
 /// @param: void.
 /// @return: Mutex.

runtime/hsa-runtime/core/util/win/os_win.cpp

@@ -64,6 +64,8 @@ static_assert(sizeof(LibHandle) == sizeof(HMODULE),
               "OS abstraction size mismatch");
 static_assert(sizeof(LibHandle) == sizeof(::HANDLE),
               "OS abstraction size mismatch");
+static_assert(sizeof(Semaphore) == sizeof(::HANDLE),
+              "OS abstraction size mismatch");
 static_assert(sizeof(Mutex) == sizeof(::HANDLE),
               "OS abstraction size mismatch");
 static_assert(sizeof(Thread) == sizeof(::HANDLE),
@@ -91,6 +93,28 @@ std::string GetLibraryName(LibHandle lib) {
   static_assert(false, "Not implemented.");
 }
 
+Semaphore CreateSemaphore() {
+  sem = static_cast<void*>(CreateSemaphore(NULL, 0, LONG_MAX, NULL));
+  assert(sem != NULL && "CreateSemaphore failed");
+
+  return *(Semaphore*)&sem;
+}
+
+bool WaitSemaphore(Semaphore sem) {
+  return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0;
+}
+
+void PostSemaphore(Semaphore sem) {
+  ReleaseSemaphore(static_cast<HANDLE>(*sem), 1, NULL);
+}
+
+void DestroySemaphore(Semaphore sem) {
+  if (!CloseHandle(static_cast<HANDLE>(*sem))) {
+    assert("CloseHandle() failed");
+  }
+  *sem = NULL;
+}
+
 Mutex CreateMutex() { return CreateEvent(NULL, false, true, NULL); }
 
 bool TryAcquireMutex(Mutex lock) {