From 2fa724e0bd8a6390ab3943132601b7f5b8bf3193 Mon Sep 17 00:00:00 2001
From: Alex Xie <AlexBin.Xie@amd.com>
Date: Wed, 29 Nov 2023 16:26:40 -0500
Subject: [PATCH] SWDEV-433820 - Optimize queue initialization (ROCM)

Change-Id: I0619a9f66ae3d64a0f3fc36384f46adb302e725d


[ROCm/clr commit: 5e1de3cc26fd966ca7d05417b9278d8fabcfa1a1]
---
 .../clr/rocclr/device/rocm/rocvirtual.cpp     | 55 ++++++++-----------
 1 file changed, 24 insertions(+), 31 deletions(-)

diff --git a/projects/clr/rocclr/device/rocm/rocvirtual.cpp b/projects/clr/rocclr/device/rocm/rocvirtual.cpp
index 0046f130b3..74bc741db1 100644
--- a/projects/clr/rocclr/device/rocm/rocvirtual.cpp
+++ b/projects/clr/rocclr/device/rocm/rocvirtual.cpp
@@ -2857,50 +2857,43 @@ bool VirtualGPU::createVirtualQueue(uint deviceQueueSize)
   }
 
   uint64_t vqVA = reinterpret_cast<uint64_t>(vqMem->getDeviceMemory());
-  uint64_t pattern = 0;
-  amd::Coord3D origin(0, 0, 0);
-  amd::Coord3D region(virtualQueue_->getSize(), 1, 1);
 
-  if (!dev().xferMgr().fillBuffer(*vqMem, &pattern, sizeof(pattern), region, origin, region)) {
-    return false;
-  }
+  // Use shadow to prepare the data structure in host.
+  auto shadow = std::make_unique<uint8_t[]>(allocSize);
 
-  AmdVQueueHeader header = {};
+  std::memset(&shadow[0], 0, allocSize);
+
+  AmdVQueueHeader* header = reinterpret_cast<AmdVQueueHeader*>(&shadow[0]);
   // Initialize the virtual queue header
-  header.aql_slot_num = numSlots;
-  header.event_slot_num = dev().settings().numDeviceEvents_;
-  header.event_slot_mask = vqVA + eventMaskOffs;
-  header.event_slots = vqVA + eventsOffs;
-  header.aql_slot_mask = vqVA + slotMaskOffs;
-  header.wait_size = dev().settings().numWaitEvents_;
-  header.arg_size = dev().info().maxParameterSize_ + 64;
-  header.mask_groups = maskGroups_;
-
-  amd::Coord3D origin_header(0);
-  amd::Coord3D region_header(sizeof(AmdVQueueHeader));
-
-  if (!dev().xferMgr().writeBuffer(&header, *vqMem, origin_header, region_header)) {
-    return false;
-  }
+  header->aql_slot_num = numSlots;
+  header->event_slot_num = dev().settings().numDeviceEvents_;
+  header->event_slot_mask = vqVA + eventMaskOffs;
+  header->event_slots = vqVA + eventsOffs;
+  header->aql_slot_mask = vqVA + slotMaskOffs;
+  header->wait_size = dev().settings().numWaitEvents_;
+  header->arg_size = dev().info().maxParameterSize_ + 64;
+  header->mask_groups = maskGroups_;
 
   // Go over all slots and perform initialization
-  AmdAqlWrap slot = {};
   size_t offset = sizeof(AmdVQueueHeader);
   for (uint i = 0; i < numSlots; ++i) {
+    AmdAqlWrap * slot = reinterpret_cast<AmdAqlWrap*>(&shadow[0] + offset);
     uint64_t argStart = vqVA + argOffs + i * singleArgSize;
-    amd::Coord3D origin_slot(offset);
-    amd::Coord3D region_slot(sizeof(AmdAqlWrap));
 
-    slot.aql.kernarg_address = reinterpret_cast<void*>(argStart);
-    slot.wait_list = argStart + dev().info().maxParameterSize_ + 64;
-
-    if (!dev().xferMgr().writeBuffer(&slot, *vqMem, origin_slot, region_slot)) {
-      return false;
-    }
+    slot->aql.kernarg_address = reinterpret_cast<void*>(argStart);
+    slot->wait_list = argStart + dev().info().maxParameterSize_ + 64;
 
     offset += sizeof(AmdAqlWrap);
   }
 
+  amd::Coord3D origin (0, 0, 0);
+  amd::Coord3D region (allocSize, 1, 1);
+
+  // copy the data structure from host to GPU
+  if (!dev().xferMgr().writeBuffer(&shadow[0], *vqMem, origin, region)) {
+    return false;
+  }
+
   deviceQueueSize_ = deviceQueueSize;
   schedulerThreads_ = numSlots / (DeviceQueueMaskSize * maskGroups_);