SWDEV-257937 - ROC_BARRIER_SYNC fix for missing SDMA flush

Change-Id: I93e8902bfcb16bac8ea594e16ea397b1ceafbd79


[ROCm/clr commit: f134b90199]

projects/clr/rocclr/device/rocm/rocblit.cpp

@@ -466,6 +466,7 @@ bool DmaBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory& d
               hsa_amd_memory_async_copy((reinterpret_cast<address>(dst) + dstOffset), dstAgent,
                                         (reinterpret_cast<const_address>(src) + srcOffset), srcAgent,
                                         size[0], 0, nullptr, completion_signal_);
+          gpu().setLastCommandSDMA(true) ;
           if (status != HSA_STATUS_SUCCESS) {
             LogPrintfError("DMA buffer failed with code %d", status);
             return false;
@@ -489,7 +490,11 @@ bool DmaBlitManager::copyImageToBuffer(device::Memory& srcMemory, device::Memory
                                        const amd::Coord3D& size, bool entire, size_t rowPitch,
                                        size_t slicePitch) const {
   // HSA copy functionality with a possible async operaiton, hence make sure GPU is done
-  gpu().releaseGpuMemoryFence();
+  if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) {
+    gpu().releaseGpuMemoryFence(true);
+  } else {
+    gpu().releaseGpuMemoryFence();
+  }
 
   bool result = false;
 
@@ -536,7 +541,11 @@ bool DmaBlitManager::copyBufferToImage(device::Memory& srcMemory, device::Memory
                                        const amd::Coord3D& size, bool entire, size_t rowPitch,
                                        size_t slicePitch) const {
   // HSA copy functionality with a possible async operaiton, hence make sure GPU is done
-  gpu().releaseGpuMemoryFence();
+  if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) {
+    gpu().releaseGpuMemoryFence(true);
+  } else {
+    gpu().releaseGpuMemoryFence();
+  }
 
   bool result = false;
 
@@ -601,6 +610,10 @@ bool DmaBlitManager::hsaCopy(const Memory& srcMemory, const Memory& dstMemory,
   address src = reinterpret_cast<address>(srcMemory.getDeviceMemory());
   address dst = reinterpret_cast<address>(dstMemory.getDeviceMemory());
 
+  if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) {
+    gpu().releaseGpuMemoryFence(true);
+  }
+
   src += srcOrigin[0];
   dst += dstOrigin[0];
 
@@ -641,7 +654,7 @@ bool DmaBlitManager::hsaCopy(const Memory& srcMemory, const Memory& dstMemory,
   // Use SDMA to transfer the data
   status = hsa_amd_memory_async_copy(dst, dstAgent, src, srcAgent, size[0], 0, nullptr,
                                      completion_signal_);
-
+  gpu().setLastCommandSDMA(true);
   if (status == HSA_STATUS_SUCCESS) {
     hsa_signal_value_t val;
 
@@ -675,6 +688,9 @@ bool DmaBlitManager::hsaCopyStaged(const_address hostSrc, address hostDst, size_
   size_t offset = 0;
 
   address hsaBuffer = staging;
+  if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) {
+    gpu().releaseGpuMemoryFence(true);
+  }
 
   // Allocate requested size of memory
   while (totalSize > 0) {
@@ -692,6 +708,7 @@ bool DmaBlitManager::hsaCopyStaged(const_address hostSrc, address hostDst, size_
       memcpy(hsaBuffer, hostSrc + offset, size);
       status = hsa_amd_memory_async_copy(hostDst + offset, dev().getBackendDevice(), hsaBuffer,
                                          srcAgent, size, 0, nullptr, completion_signal_);
+      gpu().setLastCommandSDMA(true);
       if (status == HSA_STATUS_SUCCESS) {
         if (!WaitForSignal(completion_signal_)) {
           LogError("Async copy failed");
@@ -716,6 +733,7 @@ bool DmaBlitManager::hsaCopyStaged(const_address hostSrc, address hostDst, size_
     status =
         hsa_amd_memory_async_copy(hsaBuffer, dstAgent, hostSrc + offset,
                                   dev().getBackendDevice(), size, 0, nullptr, completion_signal_);
+    gpu().setLastCommandSDMA(true);
     if (status == HSA_STATUS_SUCCESS) {
       if (!WaitForSignal(completion_signal_)) {
         LogError("Async copy failed");
@@ -1065,7 +1083,11 @@ bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory,
   releaseArguments(parameters);
   if (releaseView) {
     // todo SRD programming could be changed to avoid a stall
-    gpu().releaseGpuMemoryFence();
+    if(!dev().settings().barrier_sync_) {
+      gpu().releaseGpuMemoryFence(true);
+   } else {
+     gpu().releaseGpuMemoryFence();
+   }
     dstView->owner()->release();
   }
 
@@ -1263,7 +1285,11 @@ bool KernelBlitManager::copyImageToBufferKernel(device::Memory& srcMemory,
   releaseArguments(parameters);
   if (releaseView) {
     // todo SRD programming could be changed to avoid a stall
-    gpu().releaseGpuMemoryFence();
+    if(!dev().settings().barrier_sync_) {
+      gpu().releaseGpuMemoryFence(true);
+    } else {
+      gpu().releaseGpuMemoryFence();
+    }
     srcView->owner()->release();
   }
 

projects/clr/rocclr/device/rocm/rocvirtual.cpp

@@ -715,7 +715,9 @@ void VirtualGPU::ResetQueueStates() {
 bool VirtualGPU::releaseGpuMemoryFence(bool force_barrier) {
   // Return if there is no pending dispatch
   if (!hasPendingDispatch_) {
-    return false;
+    if (dev().settings().barrier_sync_ || !force_barrier) {
+      return false;
+    }
   }
   hsa_signal_t wait_signal = barrier_signal_;
 

projects/clr/rocclr/device/rocm/rocvirtual.hpp

@@ -290,6 +290,8 @@ class VirtualGPU : public device::VirtualDevice {
   amd::Memory* findPinnedMem(void* addr, size_t size);
 
   void enableSyncBlit() const;
+  bool isLastCommandSDMA() const { return isLastCommandSDMA_; }
+  void setLastCommandSDMA(bool s) { isLastCommandSDMA_ = s; }
 
   void hasPendingDispatch() { hasPendingDispatch_ = true; }
   void addSystemScope() { addSystemScope_ = true; }
@@ -365,6 +367,8 @@ class VirtualGPU : public device::VirtualDevice {
       uint32_t profiling_          : 1; //!< Profiling is enabled
       uint32_t cooperative_        : 1; //!< Cooperative launch is enabled
       uint32_t addSystemScope_     : 1; //!< Insert a system scope to the next aql
+      uint32_t isLastCommandSDMA_  : 1; //!< Keep track if the last command was SDMA and 
+                                               //!< not send Barrier packets if barrier_sync is 0
     };
     uint32_t  state_;
   };