From 9ec37b51035768597b1558c41c49d2b6f1294f11 Mon Sep 17 00:00:00 2001
From: Sean Keely <Sean.Keely@amd.com>
Date: Mon, 22 Oct 2018 19:15:22 -0500
Subject: [PATCH] Ensure runtime cleanup when hsa_init ref count reaches 0.

Delete the runtime object when the last hsa_shut_down occurs.

Change-Id: I2005d52d06702eaef166714fd5e471cc277924db
---
 runtime/hsa-runtime/core/inc/runtime.h       |  9 +++----
 runtime/hsa-runtime/core/runtime/runtime.cpp | 25 +++++++++++---------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h
index 8ee4d10615..651ced228e 100644
--- a/runtime/hsa-runtime/core/inc/runtime.h
+++ b/runtime/hsa-runtime/core/inc/runtime.h
@@ -104,6 +104,9 @@ class Runtime {
   /// @brief Open connection to kernel driver and increment reference count.
   static hsa_status_t Acquire();
 
+  /// @brief Decrement reference count and close connection to kernel driver.
+  static hsa_status_t Release();
+
   /// @brief Checks if connection to kernel driver is opened.
   /// @retval True if the connection to kernel driver is opened.
   static bool IsOpen();
@@ -114,9 +117,6 @@ class Runtime {
   /// @brief Singleton object of the runtime.
   static Runtime* runtime_singleton_;
 
-  /// @brief Decrement reference count and close connection to kernel driver.
-  hsa_status_t Release();
-
   /// @brief Insert agent into agent list ::agents_.
   /// @param [in] agent Pointer to the agent object.
   void RegisterAgent(Agent* agent);
@@ -420,9 +420,6 @@ class Runtime {
   /// @retval Index in ::link_matrix_.
   uint32_t GetIndexLinkInfo(uint32_t node_id_from, uint32_t node_id_to);
 
-  // Mutex object to protect multithreaded access to ::Acquire and ::Release.
-  KernelMutex kernel_lock_;
-
   // Mutex object to protect multithreaded access to ::allocation_map_,
   // KFD map/unmap, register/unregister, and access to hsaKmtQueryPointerInfo
   // registered & mapped arrays.
diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp
index c95ebfd425..9e2175c93b 100644
--- a/runtime/hsa-runtime/core/runtime/runtime.cpp
+++ b/runtime/hsa-runtime/core/runtime/runtime.cpp
@@ -93,16 +93,12 @@ hsa_status_t Runtime::Acquire() {
   // Check to see if HSA has been cleaned up (process exit)
   if (!loaded) return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
 
-  // Handle initialization races
   ScopedAcquire<KernelMutex> boot(&bootstrap_lock_);
 
   if (runtime_singleton_ == NULL) {
     runtime_singleton_ = new Runtime();
   }
 
-  // Serialize with release
-  ScopedAcquire<KernelMutex> lock(&runtime_singleton_->kernel_lock_);
-
   if (runtime_singleton_->ref_count_ == INT32_MAX) {
     return HSA_STATUS_ERROR_REFCOUNT_OVERFLOW;
   }
@@ -123,17 +119,24 @@ hsa_status_t Runtime::Acquire() {
 }
 
 hsa_status_t Runtime::Release() {
-  ScopedAcquire<KernelMutex> lock(&kernel_lock_);
-  if (ref_count_ == 0) {
-    return HSA_STATUS_ERROR_NOT_INITIALIZED;
-  }
+  // Check to see if HSA has been cleaned up (process exit)
+  if (!loaded) return HSA_STATUS_SUCCESS;
 
-  if (ref_count_ == 1) {
+  ScopedAcquire<KernelMutex> boot(&bootstrap_lock_);
+
+  if (runtime_singleton_ == nullptr) return HSA_STATUS_ERROR_NOT_INITIALIZED;
+
+  if (runtime_singleton_->ref_count_ == 1) {
     // Release all registered memory, then unload backends
-    Unload();
+    runtime_singleton_->Unload();
   }
 
-  ref_count_--;
+  runtime_singleton_->ref_count_--;
+
+  if (runtime_singleton_->ref_count_ == 0) {
+    delete runtime_singleton_;
+    runtime_singleton_ = nullptr;
+  }
 
   return HSA_STATUS_SUCCESS;
 }