From 6ab92931466b8110e4ee1243d67535179f2d4d79 Mon Sep 17 00:00:00 2001
From: German <German.Andryeyev@amd.com>
Date: Mon, 17 Oct 2022 13:47:13 -0400
Subject: [PATCH] SWDEV-360183 - Finish the trace if queue was destroyed - Make
 sure SQQT trace is captured for RGP server if the queue is destroyed before
 normal capture is done. - Remove prepare queue from the logic. It's not
 really used for any HW capture and can cause RGP server abort if destroyed
 before capture is even started(delayed capture)

Change-Id: I6eb19963190a5769c6477a5496c1b831a6d59b89


[ROCm/clr commit: c1c5127875007324537ba6d9d4910996287ee61b]
---
 projects/clr/rocclr/device/pal/palgpuopen.cpp | 42 +++++++++++--------
 projects/clr/rocclr/device/pal/palgpuopen.hpp |  1 -
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/projects/clr/rocclr/device/pal/palgpuopen.cpp b/projects/clr/rocclr/device/pal/palgpuopen.cpp
index f473bc2dbd..7c6357e91c 100644
--- a/projects/clr/rocclr/device/pal/palgpuopen.cpp
+++ b/projects/clr/rocclr/device/pal/palgpuopen.cpp
@@ -540,10 +540,8 @@ Pal::Result RgpCaptureMgr::PrepareRGPTrace(VirtualGPU* gpu) {
   }
 
   if (result == Pal::Result::Success) {
-    // Remember which queue started the trace
-    trace_.prepare_queue_ = gpu;
-    trace_.begin_queue_ = nullptr;
 
+    trace_.begin_queue_ = nullptr;
     trace_.status_ = TraceStatus::Preparing;
   } else {
     // We failed to prepare for the trace so abort it.
@@ -571,13 +569,6 @@ Pal::Result RgpCaptureMgr::BeginRGPTrace(VirtualGPU* gpu) {
   // resources against this new one if the device is changing.
   Pal::Result result = Pal::Result::Success;
 
-  if (result == Pal::Result::Success) {
-    // Only allow trace to start if the queue family at prep-time matches the queue
-    // family at begin time because the command buffer engine type must match
-    if (trace_.prepare_queue_ != gpu) {
-      result = Pal::Result::ErrorIncompatibleQueue;
-    }
-  }
 
   // Start a GPA tracing sample with SQTT enabled
   if (result == Pal::Result::Success) {
@@ -711,14 +702,32 @@ Pal::Result RgpCaptureMgr::EndRGPTrace(VirtualGPU* gpu) {
 // This function resets and possibly cancels a currently active (between begin/end) RGP trace.
 // It frees any dependent resources.
 void RgpCaptureMgr::FinishRGPTrace(VirtualGPU* gpu, bool aborted) {
-  if (trace_.prepare_queue_ == nullptr) {
+  // Make sure current queue matches the capture queue
+  if ((trace_.begin_queue_ == nullptr) || (trace_.begin_queue_ != gpu)) {
     return;
   }
 
-  // Finish the trace if the queue was destroyed before
-  // OCL reached the number of captured dispatches
-  if ((trace_.sqtt_disp_count_ != 0) && (gpu != nullptr)) {
-    EndRGPHardwareTrace(gpu);
+  // Finish the trace if the queue was destroyed before OCL reached
+  // the number of captured dispatches
+  if (trace_.sqtt_disp_count_ != 0) {
+    if (EndRGPHardwareTrace(gpu) != Pal::Result::Success) {
+      aborted = true;
+    }
+  }
+  // If the trace was aborted, then make sure the current results are sent to RGP server
+  if (aborted) {
+    if (trace_.status_ == TraceStatus::WaitingForSqtt) {
+      auto result = EndRGPTrace(gpu);
+      // The logic always checks for the trace status below and error can be ignored, since
+      // runtime aborts the trace
+    }
+    // Check if runtime is waiting for the final trace results
+    if (trace_.status_ == TraceStatus::WaitingForResults) {
+      // If results are ready, then finish the trace
+      if (CheckForTraceResults() == Pal::Result::Success) {
+        rgp_server_->EndTrace();
+      }
+    }
   }
 
   // Inform RGP protocol that we're done with the trace, either by aborting it or finishing normally
@@ -727,17 +736,14 @@ void RgpCaptureMgr::FinishRGPTrace(VirtualGPU* gpu, bool aborted) {
   } else {
     rgp_server_->EndTrace();
   }
-
   if (trace_.gpa_session_ != nullptr) {
     trace_.gpa_session_->Reset();
   }
-
   // Reset tracing state to idle
   trace_.prepared_disp_count_ = 0;
   trace_.sqtt_disp_count_ = 0;
   trace_.gpa_sample_id_ = 0;
   trace_.status_ = TraceStatus::Idle;
-  trace_.prepare_queue_ = nullptr;
   trace_.begin_queue_ = nullptr;
 }
 
diff --git a/projects/clr/rocclr/device/pal/palgpuopen.hpp b/projects/clr/rocclr/device/pal/palgpuopen.hpp
index 6396c44082..ddae799b14 100644
--- a/projects/clr/rocclr/device/pal/palgpuopen.hpp
+++ b/projects/clr/rocclr/device/pal/palgpuopen.hpp
@@ -361,7 +361,6 @@ class RgpCaptureMgr {
     GpuEvent end_sqtt_event_;    // Event that is signaled when a trace-end cmdbuf retires
     GpuEvent end_event_;         // Event that is signaled when a trace-end cmdbuf retires
 
-    VirtualGPU* prepare_queue_;  // The queue that triggered the full start of a trace
     VirtualGPU* begin_queue_;    // The queue that triggered starting SQTT
 
     GpuUtil::GpaSession* gpa_session_;  // GPA session helper object for building RGP data