Correct reported info in ROC profiler

OCL can't distinguish different copy types, but ROC profiler
expects SDMA transfer visibility. Add extra code to detect
a transfer with the host memory and substitute OCL command

Change-Id: I5290acd0e10bc082e00c1d4ae1474a075de7f165

rocclr/device/rocm/rocblit.cpp

@@ -1630,6 +1630,17 @@ bool KernelBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory
   address parameters = captureArguments(kernels_[blitType]);
   result = gpu().submitKernelInternal(ndrange, *kernels_[blitType], parameters, nullptr);
   releaseArguments(parameters);
+
+  if (amd::IS_HIP) {
+    // Update the command type for ROC profiler
+    if (srcMemory.isHostMemDirectAccess()) {
+      gpu().SetCopyCommandType(CL_COMMAND_WRITE_BUFFER_RECT);
+    }
+    if (dstMemory.isHostMemDirectAccess()) {
+      gpu().SetCopyCommandType(CL_COMMAND_READ_BUFFER_RECT);
+    }
+  }
+
   synchronize();
 
   return result;
@@ -1857,6 +1868,7 @@ bool KernelBlitManager::writeBufferRect(const void* srcHost, device::Memory& dst
   return result;
 }
 
+// ================================================================================================
 bool KernelBlitManager::fillBuffer(device::Memory& memory, const void* pattern, size_t patternSize,
                                    const amd::Coord3D& origin, const amd::Coord3D& size,
                                    bool entire) const {
@@ -1919,6 +1931,7 @@ bool KernelBlitManager::fillBuffer(device::Memory& memory, const void* pattern,
   return result;
 }
 
+// ================================================================================================
 bool KernelBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& dstMemory,
                                    const amd::Coord3D& srcOrigin, const amd::Coord3D& dstOrigin,
                                    const amd::Coord3D& sizeIn, bool entire) const {
@@ -1975,12 +1988,10 @@ bool KernelBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& ds
     setArgument(kernels_[blitType], 1, sizeof(cl_mem), &mem);
     // Program source origin
     uint64_t srcOffset = srcOrigin[0] / CopyBuffAlignment[i];
-    ;
     setArgument(kernels_[blitType], 2, sizeof(srcOffset), &srcOffset);
 
     // Program destinaiton origin
     uint64_t dstOffset = dstOrigin[0] / CopyBuffAlignment[i];
-    ;
     setArgument(kernels_[blitType], 3, sizeof(dstOffset), &dstOffset);
 
     uint64_t copySize = size[0];
@@ -2001,7 +2012,15 @@ bool KernelBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& ds
     result = gpu().submitKernelInternal(ndrange, *kernels_[blitType], parameters, nullptr);
     releaseArguments(parameters);
   } else {
-    //printf("rocm!\n");
+    if (amd::IS_HIP) {
+      // Update the command type for ROC profiler
+      if (srcMemory.isHostMemDirectAccess()) {
+        gpu().SetCopyCommandType(CL_COMMAND_WRITE_BUFFER);
+      }
+      if (dstMemory.isHostMemDirectAccess()) {
+        gpu().SetCopyCommandType(CL_COMMAND_READ_BUFFER);
+      }
+    }
     result = DmaBlitManager::copyBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, sizeIn, entire);
   }
 
@@ -2010,6 +2029,7 @@ bool KernelBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& ds
   return result;
 }
 
+// ================================================================================================
 bool KernelBlitManager::fillImage(device::Memory& memory, const void* pattern,
                                   const amd::Coord3D& origin, const amd::Coord3D& size,
                                   bool entire) const {

rocclr/device/rocm/rocvirtual.cpp

@@ -694,7 +694,8 @@ VirtualGPU::VirtualGPU(Device& device, bool profiling, bool cooperative,
       schedulerQueue_(nullptr),
       schedulerSignal_({0}),
       cuMask_(cuMask),
-      priority_(priority)
+      priority_(priority),
+      copy_command_type_(0)
 {
   index_ = device.numOfVgpus_++;
   gpu_device_ = device.getBackendDevice();
@@ -1360,6 +1361,7 @@ bool VirtualGPU::copyMemory(cl_command_type type, amd::Memory& srcMem, amd::Memo
   return true;
 }
 
+// ================================================================================================
 void VirtualGPU::submitCopyMemory(amd::CopyMemoryCommand& cmd) {
   // Make sure VirtualGPU has an exclusive access to the resources
   amd::ScopedLock lock(execution());
@@ -1374,9 +1376,15 @@ void VirtualGPU::submitCopyMemory(amd::CopyMemoryCommand& cmd) {
     cmd.setStatus(CL_INVALID_OPERATION);
   }
 
+  // Runtime may change the command type to report a more accurate info in ROC profiler
+  if (copy_command_type_ != 0) {
+    cmd.OverrrideCommandType(copy_command_type_);
+    copy_command_type_ = 0;
+  }
   profilingEnd(cmd);
 }
 
+// ================================================================================================
 void VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& cmd) {
   // Make sure VirtualGPU has an exclusive access to the resources
   amd::ScopedLock lock(execution());

rocclr/device/rocm/rocvirtual.hpp

@@ -295,6 +295,7 @@ class VirtualGPU : public device::VirtualDevice {
 
   void hasPendingDispatch() { hasPendingDispatch_ = true; }
   void addSystemScope() { addSystemScope_ = true; }
+  void SetCopyCommandType(cl_command_type type) { copy_command_type_ = type; }
 
   // } roc OpenCL integration
  private:
@@ -403,6 +404,10 @@ class VirtualGPU : public device::VirtualDevice {
   //!< bit-vector representing the CU mask. Each active bit represents using one CU
   const std::vector<uint32_t> cuMask_;
   amd::CommandQueue::Priority priority_; //!< The priority for the hsa queue
+
+  cl_command_type copy_command_type_;   //!< Type of the copy command, used for ROC profiler
+                                        //!< OCL doesn't distinguish diffrent copy types,
+                                        //!< but ROC profiler expects D2H or H2D detection
 };
 
 template <typename T>