From 5d96e499bece761219d383d718277cabc8fa8f7f Mon Sep 17 00:00:00 2001
From: Tony <Tony.Tye@amd.com>
Date: Tue, 2 Jun 2020 21:59:20 -0400
Subject: [PATCH] Initialize HSA Queue object before creating KFD queue

Initialize all the fields in the HSA queue object to known values
before calling the thunk to create the KFD queue. This ensures that
when the debugger detects that a KFD queue is created it can access
the values it requires. The values it requires include the apperture
addresses, queue scratch memory base, and the HSA queue kind.

Change-Id: Ic985755b0402c6794d5987e60aff50d223f09eb9


[ROCm/ROCR-Runtime commit: a74660c69a0e3962787d1f13d0263b07ed0dbde5]
---
 .../core/runtime/amd_aql_queue.cpp            | 42 +++++++++++--------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
index 6ee90cbb29..4106f3ee32 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
@@ -151,31 +151,20 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr
     queue_rsrc.Queue_write_ptr_aql = (uint64_t*)&amd_queue_.max_legacy_doorbell_dispatch_id_plus_1;
   }
 
-  HSAKMT_STATUS kmt_status;
-  kmt_status = hsaKmtCreateQueue(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, ring_buf_,
-                                 ring_buf_alloc_bytes_, NULL, &queue_rsrc);
-  if (kmt_status != HSAKMT_STATUS_SUCCESS)
-    throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES,
-                             "Queue create failed at hsaKmtCreateQueue\n");
-  queue_id_ = queue_rsrc.QueueId;
-  MAKE_NAMED_SCOPE_GUARD(QueueGuard, [&]() { hsaKmtDestroyQueue(queue_id_); });
-
-  // Populate doorbell signal structure.
-  memset(&signal_, 0, sizeof(signal_));
-  signal_.kind = (doorbell_type_ == 2) ? AMD_SIGNAL_KIND_DOORBELL : AMD_SIGNAL_KIND_LEGACY_DOORBELL;
-  signal_.legacy_hardware_doorbell_ptr =
-      (volatile uint32_t*)queue_rsrc.Queue_DoorBell;
-  signal_.queue_ptr = &amd_queue_;
-
   // Populate amd_queue_ structure.
   amd_queue_.hsa_queue.type = HSA_QUEUE_TYPE_MULTI;
   amd_queue_.hsa_queue.features = HSA_QUEUE_FEATURE_KERNEL_DISPATCH;
   amd_queue_.hsa_queue.base_address = ring_buf_;
   amd_queue_.hsa_queue.doorbell_signal = Signal::Convert(this);
   amd_queue_.hsa_queue.size = queue_size_pkts;
-  amd_queue_.hsa_queue.id = queue_id_;
+  amd_queue_.hsa_queue.id = INVALID_QUEUEID;
   amd_queue_.read_dispatch_id_field_base_byte_offset = uint32_t(
       uintptr_t(&amd_queue_.read_dispatch_id) - uintptr_t(&amd_queue_));
+  // Initialize the doorbell signal structure.
+  memset(&signal_, 0, sizeof(signal_));
+  signal_.kind = (doorbell_type_ == 2) ? AMD_SIGNAL_KIND_DOORBELL : AMD_SIGNAL_KIND_LEGACY_DOORBELL;
+  signal_.legacy_hardware_doorbell_ptr = nullptr;
+  signal_.queue_ptr = &amd_queue_;
 
   const auto& props = agent->properties();
   amd_queue_.max_cu_id = (props.NumFComputeCores / props.NumSIMDPerCU) - 1;
@@ -221,6 +210,25 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr
     assert(amd_queue_.private_segment_aperture_base_hi != 0 && "No private region found.");
   }
 
+  // Ensure the amd_queue_ is fully initialized before creating the KFD queue.
+  // This ensures that the debugger can access the fields once it detects there
+  // is a KFD queue. The debugger may access the aperture addresses, queue
+  // scratch base, and queue type.
+
+  HSAKMT_STATUS kmt_status;
+  kmt_status = hsaKmtCreateQueue(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, ring_buf_,
+                                 ring_buf_alloc_bytes_, NULL, &queue_rsrc);
+  if (kmt_status != HSAKMT_STATUS_SUCCESS)
+    throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES,
+                             "Queue create failed at hsaKmtCreateQueue\n");
+  // Complete populating the doorbell signal structure.
+  signal_.legacy_hardware_doorbell_ptr =
+      (volatile uint32_t*)queue_rsrc.Queue_DoorBell;
+  // Complete populating the amd_queue_ structure.
+  amd_queue_.hsa_queue.id = queue_id_ = queue_rsrc.QueueId;
+
+  MAKE_NAMED_SCOPE_GUARD(QueueGuard, [&]() { hsaKmtDestroyQueue(queue_id_); });
+
   MAKE_NAMED_SCOPE_GUARD(EventGuard, [&]() {
     ScopedAcquire<KernelMutex> _lock(&queue_lock_);
     queue_count_--;