Rework memory locks to allow device parallelism in alloc/free.

Prior solution used a single global lock to protect the memory tracking structures.
This change protects the memory tracking structure with a shared mutex (rw lock) in
shared (r) mode for memory allocations and frees so that long duration processes,
calling to kfd, can be done in parallel.  Operations which must modify the memory map
take the mutex in exclusive mode (w) and must not call to the thunk while holding
the mutex.

The fragment allocator now requires separate protection and is protected with a
mutex at the device level.  Protecting at the device level, rather than pool,
allows retention of the current recursive design and allows calling Trim from
withing Allocate.  This could be made finer (pool level locks) but would
require backing out of Allocate entirely to call Trim.  Trim and any retried
Allocation must be done in isolation (per device) or we may report OOM when
memory is actually available in some pool's fragment cache.  So some device
level serialization is required in at least some paths.

Change-Id: I7c1e94d6965ffcc602b12fefdd3a6e97b84b5e00

runtime/hsa-runtime/core/util/lnx/os_linux.cpp

@@ -168,6 +168,7 @@ class os_thread {
 
 static_assert(sizeof(LibHandle) == sizeof(void*), "OS abstraction size mismatch");
 static_assert(sizeof(Mutex) == sizeof(pthread_mutex_t*), "OS abstraction size mismatch");
+static_assert(sizeof(SharedMutex) == sizeof(pthread_rwlock_t*), "OS abstraction size mismatch");
 static_assert(sizeof(Thread) == sizeof(os_thread*), "OS abstraction size mismatch");
 
 LibHandle LoadLib(std::string filename) {
@@ -457,6 +458,63 @@ uint64_t AccurateClockFrequency() {
   if (invPeriod == 0.0) invPeriod = 1.0 / double(time.tv_nsec);
   return 1000000000ull / uint64_t(time.tv_nsec);
 }
+
+SharedMutex CreateSharedMutex() {
+  pthread_rwlockattr_t attrib;
+  int err = pthread_rwlockattr_init(&attrib);
+  if (err != 0) {
+    assert(false && "rw lock attribute init failed.");
+    return nullptr;
+  }
+  err = pthread_rwlockattr_setkind_np(&attrib, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+  assert(err == 0 && "Set rw lock attribute failure.");
+
+  pthread_rwlock_t* lock = new pthread_rwlock_t;
+  err = pthread_rwlock_init(lock, &attrib);
+  if (err != 0) {
+    assert(false && "rw lock init failed.");
+    return nullptr;
+  }
+
+  pthread_rwlockattr_destroy(&attrib);
+  return lock;
+}
+
+bool TryAcquireSharedMutex(SharedMutex lock) {
+  int err = pthread_rwlock_trywrlock(*(pthread_rwlock_t**)&lock);
+  return err == 0;
+}
+
+bool AcquireSharedMutex(SharedMutex lock) {
+  int err = pthread_rwlock_wrlock(*(pthread_rwlock_t**)&lock);
+  return err == 0;
+}
+
+void ReleaseSharedMutex(SharedMutex lock) {
+  int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock);
+  assert(err == 0 && "SharedMutex unlock failed.");
+}
+
+bool TrySharedAcquireSharedMutex(SharedMutex lock) {
+  int err = pthread_rwlock_tryrdlock(*(pthread_rwlock_t**)&lock);
+  return err == 0;
+}
+
+bool SharedAcquireSharedMutex(SharedMutex lock) {
+  int err = pthread_rwlock_rdlock(*(pthread_rwlock_t**)&lock);
+  return err == 0;
+}
+
+void SharedReleaseSharedMutex(SharedMutex lock) {
+  int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock);
+  assert(err == 0 && "SharedMutex unlock failed.");
+}
+
+void DestroySharedMutex(SharedMutex lock) {
+  pthread_rwlock_destroy(*(pthread_rwlock_t**)&lock);
+  delete *(pthread_rwlock_t**)&lock;
+}
+
 }   //  namespace os
 }   //  namespace rocr
 

runtime/hsa-runtime/core/util/locks.h

@@ -50,34 +50,6 @@
 
 namespace rocr {
 
-/// @brief: A class behaves as a lock in a scope. When trying to enter into the
-/// critical section, creat a object of this class. After the control path goes
-/// out of the scope, it will release the lock automatically.
-template <class LockType>
-class ScopedAcquire {
- public:
-  /// @brief: When constructing, acquire the lock.
-  /// @param: lock(Input), pointer to an existing lock.
-  explicit ScopedAcquire(LockType* lock) : lock_(lock), doRelease(true) { lock_->Acquire(); }
-
-  /// @brief: when destructing, release the lock.
-  ~ScopedAcquire() {
-    if (doRelease) lock_->Release();
-  }
-
-  /// @brief: Release the lock early.  Avoid using when possible.
-  void Release() {
-    lock_->Release();
-    doRelease = false;
-  }
-
- private:
-  LockType* lock_;
-  bool doRelease;
-  /// @brief: Disable copiable and assignable ability.
-  DISALLOW_COPY_AND_ASSIGN(ScopedAcquire);
-};
-
 /// @brief: a class represents a kernel mutex.
 /// Uses the kernel's scheduler to keep the waiting thread from being scheduled
 /// until the lock is released (Best for long waits, though anything using
@@ -144,6 +116,120 @@ class KernelEvent {
   DISALLOW_COPY_AND_ASSIGN(KernelEvent);
 };
 
+/// @brief: represents a yielding shared mutex.
+/// aka read/write mutex
+class KernelSharedMutex {
+ public:
+  /// @brief: Interfaces ScopedAcquire to shared operations.
+  class Shared {
+   public:
+    explicit Shared(KernelSharedMutex* lock) : lock_(lock) {}
+    bool Try() { return lock_->TryShared(); }
+    bool Acquire() { return lock_->AcquireShared(); }
+    void Release() { lock_->ReleaseShared(); }
+
+   private:
+    KernelSharedMutex* lock_;
+  };
+
+  KernelSharedMutex() { lock_ = os::CreateSharedMutex(); }
+  ~KernelSharedMutex() { os::DestroySharedMutex(lock_); }
+
+  // Exclusive mode operations
+  bool Try() { return os::TryAcquireSharedMutex(lock_); }
+  bool Acquire() { return os::AcquireSharedMutex(lock_); }
+  void Release() { os::ReleaseSharedMutex(lock_); }
+
+  // Shared mode operations
+  bool TryShared() { return os::TrySharedAcquireSharedMutex(lock_); }
+  bool AcquireShared() { return os::SharedAcquireSharedMutex(lock_); }
+  void ReleaseShared() { os::SharedReleaseSharedMutex(lock_); }
+
+  // Return shared operations interface
+  Shared shared() { return Shared(this); }
+
+ private:
+  os::SharedMutex lock_;
+
+  /// @brief: Disable copiable and assignable ability.
+  DISALLOW_COPY_AND_ASSIGN(KernelSharedMutex);
+};
+
+/// @brief: Type trait to identify mutex types
+template <class T> class isMutex {
+ public:
+  enum { value = false };
+};
+template <> class isMutex<KernelMutex> {
+ public:
+  enum { value = true };
+};
+template <> class isMutex<SpinMutex> {
+ public:
+  enum { value = true };
+};
+template <> class isMutex<KernelSharedMutex> {
+ public:
+  enum { value = true };
+};
+
+/// @brief: A class behaves as a lock in a scope. When trying to enter into the
+/// critical section, creat a object of this class. After the control path goes
+/// out of the scope, it will release the lock automatically.
+template <class LockType> class ScopedAcquire {
+ public:
+  /// @brief: When constructing, acquire the lock.
+  /// @param: lock(Input), pointer to an existing lock.
+  explicit ScopedAcquire(LockType* lock) : lock_(lock), doRelease(true) {
+    static_assert(isMutex<LockType>::value, "ScopedAcquire requires a mutex type.");
+    lock_.Acquire();
+  }
+  explicit ScopedAcquire(LockType lock) : lock_(lock), doRelease(true) {
+    static_assert(!isMutex<LockType>::value, "Mutex types are not copyable.");
+    lock_.Acquire();
+  }
+
+  /// @brief: when destructing, release the lock.
+  ~ScopedAcquire() {
+    if (doRelease) lock_.Release();
+  }
+
+  /// @brief: Release the lock early.  Avoid using when possible.
+  void Release() {
+    lock_.Release();
+    doRelease = false;
+  }
+
+ private:
+  /// @brief: Adapts between pointers to mutex types and mutex pointer types.
+  template <class T, bool B> class container {
+   public:
+    container(T* lock) : lock_(lock) {}
+    __forceinline bool Acquire() { return lock_->Acquire(); }
+    __forceinline void Release() { return lock_->Release(); }
+
+   private:
+    T* lock_;
+  };
+
+  /// @brief: Specialization for mutex pointer types.
+  template <class T> class container<T, false> {
+   public:
+    container(T lock) : lock_(lock) {}
+    __forceinline bool Acquire() { return lock_.Acquire(); }
+    __forceinline void Release() { return lock_.Release(); }
+
+   private:
+    T lock_;
+  };
+
+  container<LockType, isMutex<LockType>::value> lock_;
+  bool doRelease;
+
+  /// @brief: Disable copiable and assignable ability.
+  DISALLOW_COPY_AND_ASSIGN(ScopedAcquire);
+};
+
 }  // namespace rocr
 
 #endif  // HSA_RUNTIME_CORE_SUTIL_LOCKS_H_

runtime/hsa-runtime/core/util/os.h

@@ -52,6 +52,7 @@ namespace rocr {
 namespace os {
 typedef void* LibHandle;
 typedef void* Mutex;
+typedef void* SharedMutex;
 typedef void* Thread;
 typedef void* EventHandle;
 
@@ -110,6 +111,48 @@ void ReleaseMutex(Mutex lock);
 /// @return: void.
 void DestroyMutex(Mutex lock);
 
+/// @brief: Creates a shared mutex, will return NULL if failed.
+/// @param: void.
+/// @return: SharedMutex.
+SharedMutex CreateSharedMutex();
+
+/// @brief: Tries to acquire the mutex in exclusive mode once, if successed, return true.
+/// @param: lock(Input), handle to the shared mutex.
+/// @return: bool.
+bool TryAcquireSharedMutex(SharedMutex lock);
+
+/// @brief: Aquires the mutex in exclusive mode, if the mutex is locked, it will wait until it is
+/// released. If the mutex is acquired successfully, it will return true.
+/// @param: lock(Input), handle to the mutex.
+/// @return: bool.
+bool AcquireSharedMutex(SharedMutex lock);
+
+/// @brief: Releases the mutex from exclusive mode.
+/// @param: lock(Input), handle to the mutex.
+/// @return: void.
+void ReleaseSharedMutex(SharedMutex lock);
+
+/// @brief: Tries to acquire the mutex in shared mode once, if successed, return true.
+/// @param: lock(Input), handle to the mutex.
+/// @return: bool.
+bool TrySharedAcquireSharedMutex(SharedMutex lock);
+
+/// @brief: Aquires the mutex in shared mode, if the mutex in exclusive mode, it will wait until it
+/// is released. If the mutex is acquired successfully, it will return true.
+/// @param: lock(Input), handle to the mutex.
+/// @return: bool.
+bool SharedAcquireSharedMutex(SharedMutex lock);
+
+/// @brief: Releases the mutex from shared mode.
+/// @param: lock(Input), handle to the mutex.
+/// @return: void.
+void SharedReleaseSharedMutex(SharedMutex lock);
+
+/// @brief: Destroys the mutex.
+/// @param: lock(Input), handle to the mutex.
+/// @return: void.
+void DestroySharedMutex(SharedMutex lock);
+
 /// @brief: Puts current thread to sleep.
 /// @param: delayInMs(Input), time in millisecond for sleeping.
 /// @return: void.

runtime/hsa-runtime/core/util/win/os_win.cpp

@@ -224,6 +224,52 @@ uint64_t AccurateClockFrequency() {
   QueryPerformanceFrequency((LARGE_INTEGER*)&ret);
   return ret;
 }
+
+SharedMutex CreateSharedMutex() {
+  assert(false && "Not implemented.");
+  abort();
+  return nullptr;
+}
+
+bool TryAcquireSharedMutex(SharedMutex lock) {
+  assert(false && "Not implemented.");
+  abort();
+  return false;
+}
+
+bool AcquireSharedMutex(SharedMutex lock) {
+  assert(false && "Not implemented.");
+  abort();
+  return false;
+}
+
+void ReleaseSharedMutex(SharedMutex lock) {
+  assert(false && "Not implemented.");
+  abort();
+}
+
+bool TrySharedAcquireSharedMutex(SharedMutex lock) {
+  assert(false && "Not implemented.");
+  abort();
+  return false;
+}
+
+bool SharedAcquireSharedMutex(SharedMutex lock) {
+  assert(false && "Not implemented.");
+  abort();
+  return false;
+}
+
+void SharedReleaseSharedMutex(SharedMutex lock) {
+  assert(false && "Not implemented.");
+  abort();
+}
+
+void DestroySharedMutex(SharedMutex lock) {
+  assert(false && "Not implemented.");
+  abort();
+}
+
 }   //  namespace os
 }   //  namespace rocr