From 2ec5a47c88267cea1cd99b5cf443c23270ddafc8 Mon Sep 17 00:00:00 2001
From: Vladislav Sytchenko <Vladislav.Sytchenko@amd.com>
Date: Tue, 13 Oct 2020 14:31:40 -0400
Subject: [PATCH] [PAL] Allow for embedding debug info into IBs

Change-Id: I4473b9c5aa36370d9af37f22a78f4414eaa21e01
---
 rocclr/device/pal/palvirtual.cpp | 21 +++++++++++++++------
 rocclr/device/pal/palvirtual.hpp | 24 +++++++++++++++++++++++-
 rocclr/utils/flags.hpp           |  4 +++-
 3 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/rocclr/device/pal/palvirtual.cpp b/rocclr/device/pal/palvirtual.cpp
index aed53a9809..4a85bb10fe 100644
--- a/rocclr/device/pal/palvirtual.cpp
+++ b/rocclr/device/pal/palvirtual.cpp
@@ -2428,7 +2428,19 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
   }
   size_t ldsSize;
 
-  ClPrint(amd::LOG_INFO, amd::LOG_KERN, "!\tShaderName : %s\n", hsaKernel.name().c_str());
+  ClPrint(amd::LOG_INFO, amd::LOG_KERN, "!\tkernel : %s\n", hsaKernel.name().c_str());
+
+  if (PAL_EMBED_KERNEL_MD) {
+    char buf[256];
+    sprintf(buf,
+            "kernel: %s\n"
+            "private mem size: %x\n"
+            "group mem size: %x\n",
+            hsaKernel.name().c_str(),
+            hsaKernel.spillSegSize(),
+            hsaKernel.ldsSize());
+    iCmd()->CmdCommentString(buf);
+  }
 
   // Check memory dependency and SVM objects
   if (!processMemObjectsHSA(kernel, parameters, nativeMem, ldsSize)) {
@@ -3414,11 +3426,7 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p
 
           addVmMemory(gpuMem);
           const void* globalAddress = *reinterpret_cast<const void* const*>(params + desc.offset_);
-          ClPrint(amd::LOG_INFO, amd::LOG_KERN, "!\targ%d: %s %s = ptr:%p obj:[%p-%p] threadId : %zx\n", index,
-                  desc.typeName_.c_str(), desc.name_.c_str(), globalAddress,
-                  reinterpret_cast<void*>(gpuMem->vmAddress()),
-                  reinterpret_cast<void*>(gpuMem->vmAddress() + gpuMem->size()),
-                  std::this_thread::get_id());
+          logVmMemory(desc.name_, gpuMem);
 
           //! Check if compiler expects read/write.
           //! Note: SVM with subbuffers has an issue with tracking.
@@ -3542,6 +3550,7 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p
        memoryDependency().validate(*this, scratch->memObj_, IsReadOnly);
     }
     addVmMemory(scratch->memObj_);
+    logVmMemory("scratch", scratch->memObj_);
   }
 
   // Synchronize dispatches unconditionally in case memory tracking is disabled
diff --git a/rocclr/device/pal/palvirtual.hpp b/rocclr/device/pal/palvirtual.hpp
index 696749d7c2..e85778a9f6 100644
--- a/rocclr/device/pal/palvirtual.hpp
+++ b/rocclr/device/pal/palvirtual.hpp
@@ -393,7 +393,12 @@ class VirtualGPU : public device::VirtualDevice {
                                const amd::Event* waitingEvent  //!< Waiting event
   );
 
-  //! Adds a memory handle into the GSL memory array for Virtual Heap
+  //! Embeds memory handle info into the CB associated with this VGPU
+  inline void logVmMemory(const std::string name, //!< Brief description of the memory object
+                          const Memory* memory //!< GPU memory object
+  );
+
+  //! Adds a memory handle into the PAL memory array for Virtual Heap
   inline void addVmMemory(const Memory* memory  //!< GPU memory object
   );
 
@@ -671,6 +676,23 @@ class VirtualGPU : public device::VirtualDevice {
   std::vector<Image*> wrtBackImageBuffer_;  //!< Array of images for write back
 };
 
+inline void VirtualGPU::logVmMemory(const std::string name, const Memory* memory) {
+  if (PAL_EMBED_KERNEL_MD || (AMD_LOG_LEVEL >= amd::LOG_INFO)) {
+    char buf[256];
+    sprintf(buf,
+            "%s = ptr:[%p-%p] obj:[%p-%p]",
+            name.c_str(),
+            reinterpret_cast<void*>(memory->vmAddress()),
+            reinterpret_cast<void*>(memory->vmAddress() + memory->size()),
+            reinterpret_cast<void*>(memory->iMem()->Desc().gpuVirtAddr),
+            reinterpret_cast<void*>(memory->iMem()->Desc().gpuVirtAddr + memory->iMem()->Desc().size));
+    if (PAL_EMBED_KERNEL_MD) {
+      iCmd()->CmdCommentString(buf);
+    }
+    LogPrintfInfo("%s threadId : %zx\n", buf, std::this_thread::get_id());
+  }
+}
+
 inline void VirtualGPU::addVmMemory(const Memory* memory) {
   queues_[MainEngine]->addCmdMemRef(memory->memRef());
   memory->setBusy(*this, queues_[MainEngine]->cmdBufId());
diff --git a/rocclr/utils/flags.hpp b/rocclr/utils/flags.hpp
index d8c48d31f4..da9fae9f30 100644
--- a/rocclr/utils/flags.hpp
+++ b/rocclr/utils/flags.hpp
@@ -245,7 +245,9 @@ release(bool, ROC_ENABLE_LARGE_BAR, true,                                     \
 release(bool, HIP_FORCE_QUEUE_PROFILING, false,                               \
         "Force command queue profiling by default")                           \
 release(uint, PAL_FORCE_ASIC_REVISION, 0,                                     \
-        "Force a specific asic revision for all devices")
+        "Force a specific asic revision for all devices")                     \
+release(bool, PAL_EMBED_KERNEL_MD, false,                                     \
+        "Enables writing kernel metadata into command buffers.")
 
 namespace amd {