rocr: Release agent resources before pools

Adding a general stage for agents to release their resources on
shutdown. This avoids a circular dependency during shutdown because
we have to delete allocated resources before deleting memory pools, but
we also have to delete memory pools before destroying agents.


[ROCm/ROCR-Runtime commit: 947391deac]

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -247,35 +247,6 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna
 }
 
 GpuAgent::~GpuAgent() {
-  if (this->Enabled()) {
-    for (auto& blit : blits_) {
-      if (!blit.empty()) {
-        hsa_status_t status = blit->Destroy(*this);
-        assert(status == HSA_STATUS_SUCCESS);
-      }
-    }
-
-    if (ape1_base_ != 0) {
-      _aligned_free(reinterpret_cast<void*>(ape1_base_));
-    }
-
-    scratch_cache_.trim(true);
-    scratch_cache_.free_reserve();
-
-    if (scratch_pool_.base() != NULL) {
-      hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size());
-    }
-
-    for (int i = 0; i < QueueCount; i++)
-      queues_[i].reset();
-
-    system_deallocator()(doorbell_queue_map_);
-
-    if (trap_code_buf_ != NULL) {
-      ReleaseShader(trap_code_buf_, trap_code_buf_size_);
-    }
-  }
-
   std::for_each(regions_.begin(), regions_.end(), DeleteObject());
   regions_.clear();
 }
@@ -947,6 +918,37 @@ void GpuAgent::PreloadBlits() {
   }
 }
 
+void GpuAgent::ReleaseResources() {
+  if (this->Enabled()) {
+    this->Disable();
+    for (auto& blit : blits_) {
+      if (!blit.empty()) {
+        hsa_status_t status = blit->Destroy(*this);
+        assert(status == HSA_STATUS_SUCCESS);
+      }
+    }
+
+    if (ape1_base_ != 0) {
+      _aligned_free(reinterpret_cast<void*>(ape1_base_));
+    }
+
+    scratch_cache_.trim(true);
+    scratch_cache_.free_reserve();
+
+    if (scratch_pool_.base() != NULL) {
+      hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size());
+    }
+
+    for (int i = 0; i < QueueCount; i++)
+      queues_[i].reset();
+
+    system_deallocator()(doorbell_queue_map_);
+
+    if (trap_code_buf_ != NULL)
+      system_deallocator()(trap_code_buf_);
+  }
+}
+
 hsa_status_t GpuAgent::PostToolsInit() {
   // Defer memory allocation until agents have been discovered.
   InitAllocators();

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -2098,6 +2098,11 @@ void Runtime::Unload() {
   amd::hsa::loader::Loader::Destroy(loader_);
   loader_ = nullptr;
 
+  for(auto nodeAgent: agents_by_node_) {
+    for (auto agent: nodeAgent.second)
+      agent->ReleaseResources();
+  }
+
   asyncSignals_.control.Shutdown();
   asyncExceptions_.control.Shutdown();