SWDEV-352197 - Destroy virtual device in thread destructor

Windows kills threads on exit without any notification. However,
runtime can still destroy VirtualGPU object from the host thread with
HostQueue destruction.
This change also forces RGP trace transfer on the last capture without
any delays.

Change-Id: I768e87e99e1d23a021e63c12f36e450817743759


[ROCm/clr commit: ad33a021cb]

projects/clr/rocclr/device/pal/palgpuopen.cpp

@@ -278,6 +278,21 @@ void RgpCaptureMgr::PostDispatch(VirtualGPU* gpu) {
           // continue until we find the right queue...
         } else if (Pal::Result::Success == res) {
           trace_.sqtt_disp_count_ = 0;
+          // Stop the trace and save the result. Currently runtime can't delay upload in HIP,
+          // because default stream doesn't have explicit destruction and
+          // OS kills all threads on exit without any notification. That includes PAL RGP threads.
+          {
+            if (trace_.status_ == TraceStatus::WaitingForSqtt) {
+              auto result = EndRGPTrace(gpu);
+            }
+            // Check if runtime is waiting for the final trace results
+            if (trace_.status_ == TraceStatus::WaitingForResults) {
+              // If results are ready, then finish the trace
+              if (CheckForTraceResults() == Pal::Result::Success) {
+                FinishRGPTrace(gpu, false);
+              }
+            }
+          }
         } else {
           FinishRGPTrace(gpu, true);
         }

projects/clr/rocclr/device/pal/palvirtual.cpp

@@ -1072,15 +1072,15 @@ bool VirtualGPU::allocHsaQueueMem() {
 }
 
 VirtualGPU::~VirtualGPU() {
+  // Not safe to remove a queue. So lock the device
+  amd::ScopedLock k(dev().lockAsyncOps());
+  amd::ScopedLock lock(dev().vgpusAccess());
+
   // Destroy RGP trace
   if (rgpCaptureEna()) {
     dev().rgpCaptureMgr()->FinishRGPTrace(this, true);
   }
 
-  // Not safe to remove a queue. So lock the device
-  amd::ScopedLock k(dev().lockAsyncOps());
-  amd::ScopedLock lock(dev().vgpusAccess());
-
   while (!freeCbQueue_.empty()) {
     auto cb = freeCbQueue_.front();
     delete cb;

projects/clr/rocclr/platform/commandqueue.cpp

@@ -67,7 +67,6 @@ bool HostQueue::terminate() {
       marker->release();
     }
     thread_.acceptingCommands_ = false;
-    thread_.Release();
   } else {
     if (Os::isThreadAlive(thread_)) {
       Command* marker = nullptr;

projects/clr/rocclr/platform/commandqueue.hpp

@@ -162,15 +162,21 @@ class HostQueue : public CommandQueue {
     Thread()
         : amd::Thread("Command Queue Thread", CQ_THREAD_STACK_SIZE, !AMD_DIRECT_DISPATCH),
           acceptingCommands_(false),
-          virtualDevice_(NULL) {}
+          virtualDevice_(nullptr) {}
+
+    virtual ~Thread() {
+      if (virtualDevice_ != nullptr) {
+        delete virtualDevice_;
+        virtualDevice_ = nullptr;
+      }
+    }
 
     //! The command queue thread entry point.
     void run(void* data) {
       HostQueue* queue = static_cast<HostQueue*>(data);
       virtualDevice_ = queue->device().createVirtualDevice(queue);
-      if (virtualDevice_ != NULL) {
+      if (virtualDevice_ != nullptr) {
         queue->loop(virtualDevice_);
-        Release();
       } else {
         acceptingCommands_ = false;
         queue->flush();
@@ -184,8 +190,6 @@ class HostQueue : public CommandQueue {
       }
     }
 
-    void Release() const { delete virtualDevice_; }
-
     //! Get virtual device for the current thread
     device::VirtualDevice* vdev() const { return virtualDevice_; }