SWDEV-257937 - Updated fix for ROC_BARRIER_SYNC=0

Change-Id: I7e28e541b654db57fb0890d7dbb7519cfb2d93db

rocclr/device/device.hpp

@@ -1203,6 +1203,9 @@ class VirtualDevice : public amd::HeapObject {
   //! Returns true if device has active wait setting
   bool ActiveWait() const;
 
+  bool isLastCommandSDMA() const { return isLastCommandSDMA_; }
+  void setLastCommandSDMA(bool s) { isLastCommandSDMA_ = s; }
+
  private:
   //! Disable default copy constructor
   VirtualDevice& operator=(const VirtualDevice&);
@@ -1216,6 +1219,8 @@ class VirtualDevice : public amd::HeapObject {
  protected:
   device::BlitManager* blitMgr_;  //!< Blit manager
 
+  //!< Keep track if the last command was SDMA and not send Barrier packets if barrier_sync is 0
+  std::atomic_bool isLastCommandSDMA_;
   amd::Monitor execution_;  //!< Lock to serialise access to all device objects
   uint index_;              //!< The virtual device unique index
 };

rocclr/device/rocm/rocblit.cpp

@@ -395,7 +395,8 @@ bool DmaBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory& d
     gpu().releaseGpuMemoryFence();
     return HostBlitManager::copyBufferRect(srcMemory, dstMemory, srcRect, dstRect, size, entire);
   } else {
-    gpu().releaseGpuMemoryFence(kIgnoreBarrier, kSkipCpuWait);
+    bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA();
+    gpu().releaseGpuMemoryFence(force_barrier, kSkipCpuWait);
 
     void* src = gpuMem(srcMemory).getDeviceMemory();
     void* dst = gpuMem(dstMemory).getDeviceMemory();
@@ -1117,7 +1118,8 @@ bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory,
   releaseArguments(parameters);
   if (releaseView) {
     // todo SRD programming could be changed to avoid a stall
-    gpu().releaseGpuMemoryFence();
+    bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA();
+    gpu().releaseGpuMemoryFence(force_barrier);
     dstView->owner()->release();
   }
 
@@ -1315,7 +1317,8 @@ bool KernelBlitManager::copyImageToBufferKernel(device::Memory& srcMemory,
   releaseArguments(parameters);
   if (releaseView) {
     // todo SRD programming could be changed to avoid a stall
-    gpu().releaseGpuMemoryFence();
+    bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA();
+    gpu().releaseGpuMemoryFence(force_barrier);
     srcView->owner()->release();
   }
 
@@ -1446,7 +1449,8 @@ bool KernelBlitManager::copyImage(device::Memory& srcMemory, device::Memory& dst
   releaseArguments(parameters);
   if (releaseView) {
     // todo SRD programming could be changed to avoid a stall
-    gpu().releaseGpuMemoryFence();
+    bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA();
+    gpu().releaseGpuMemoryFence(force_barrier);
     srcView->owner()->release();
     dstView->owner()->release();
   }
@@ -2269,7 +2273,8 @@ bool KernelBlitManager::fillImage(device::Memory& memory, const void* pattern,
   releaseArguments(parameters);
   if (releaseView) {
     // todo SRD programming could be changed to avoid a stall
-    gpu().releaseGpuMemoryFence();
+    bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA();
+    gpu().releaseGpuMemoryFence(force_barrier);
     memView->owner()->release();
   }
 

rocclr/device/rocm/rocdevice.cpp

@@ -1584,12 +1584,6 @@ bool Device::populateOCLDeviceConstants() {
 device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) {
   amd::ScopedLock lock(vgpusAccess());
 
-  // If barrier is disabled, then profiling should be enabled to make sure HSA signal is
-  // attached for every dispatch
-  if (!settings().barrier_sync_) {
-    queue->properties().set(CL_QUEUE_PROFILING_ENABLE);
-  }
-
   bool profiling = (queue != nullptr) && queue->properties().test(CL_QUEUE_PROFILING_ENABLE);
   bool cooperative = false;
 
@@ -1599,7 +1593,11 @@ device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) {
     cooperative = amd::IS_HIP && settings().enableCoopGroups_;
     profiling = amd::IS_HIP;
   }
-
+  // If barrier is disabled, then profiling should be enabled to make sure HSA signal is
+  // attached for every dispatch
+  else if (!settings().barrier_sync_) {
+    queue->properties().set(CL_QUEUE_PROFILING_ENABLE);
+  }
   // Initialization of heap and other resources occur during the command
   // queue creation time.
   const std::vector<uint32_t> defaultCuMask = {};

rocclr/device/rocm/rocvirtual.cpp

@@ -2336,7 +2336,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
   device::Kernel* devKernel = const_cast<device::Kernel*>(kernel.getDeviceKernel(dev()));
   Kernel& gpuKernel = static_cast<Kernel&>(*devKernel);
   size_t ldsUsage = gpuKernel.WorkgroupGroupSegmentByteSize();
-
+  setLastCommandSDMA(false);
   // Check memory dependency and SVM objects
   bool coopGroups = (vcmd != nullptr) ? vcmd->cooperativeGroups() : false;
   if (!processMemObjects(kernel, parameters, ldsUsage, coopGroups)) {
@@ -2617,7 +2617,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
 void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) {
   if (vcmd.cooperativeGroups() || vcmd.cooperativeMultiDeviceGroups()) {
     // Wait for the execution on the current queue, since the coop groups will use the device queue
-    releaseGpuMemoryFence(kIgnoreBarrier, kSkipCpuWait);
+    bool force_barrier = !dev().settings().barrier_sync_ && !isLastCommandSDMA();
+    releaseGpuMemoryFence(force_barrier, kSkipCpuWait);
 
     // Get device queue for exclusive GPU access
     VirtualGPU* queue = dev().xferQueue();
@@ -2726,6 +2727,7 @@ void VirtualGPU::submitReleaseExtObjects(amd::ReleaseExtObjectsCommand& vcmd) {
 void VirtualGPU::flush(amd::Command* list, bool wait) {
   // Direct dispatch relies on HSA signal callback
   bool skip_cpu_wait = AMD_DIRECT_DISPATCH;
+  bool force_barrier = !dev().settings().barrier_sync_ && !isLastCommandSDMA();
 
   if (skip_cpu_wait) {
     // Search for the last command in the batch to track GPU state
@@ -2742,7 +2744,7 @@ void VirtualGPU::flush(amd::Command* list, bool wait) {
     // CPU wait is also forced if pinned buffers were used
     skip_cpu_wait &= hasPendingDispatch_ && (pinnedMems_.size() == 0);
 
-    releaseGpuMemoryFence(kIgnoreBarrier, skip_cpu_wait);
+    releaseGpuMemoryFence(force_barrier, skip_cpu_wait);
     profilingEnd(*current);
   } else {
     // If barrier is requested, then wait for everything, otherwise
@@ -2754,7 +2756,7 @@ void VirtualGPU::flush(amd::Command* list, bool wait) {
 
   // If CPU waited for GPU, then the queue is idle
   if (!skip_cpu_wait) {
-      updateCommandsState(list);
+    updateCommandsState(list);
 
     // Add extra clean up for resources if releaseGpuMemoryFence() was skipped
     if (!dev().settings().barrier_sync_) {

rocclr/device/rocm/rocvirtual.hpp

@@ -36,6 +36,7 @@ class Device;
 class Memory;
 class Timestamp;
 
+
 struct ProfilingSignal : public amd::HeapObject {
   hsa_signal_t  signal_;  //!< HSA signal to track profiling information
   Timestamp*    ts_;      //!< Timestamp object associated with the signal
@@ -312,8 +313,6 @@ class VirtualGPU : public device::VirtualDevice {
   amd::Memory* findPinnedMem(void* addr, size_t size);
 
   void enableSyncBlit() const;
-  bool isLastCommandSDMA() const { return isLastCommandSDMA_; }
-  void setLastCommandSDMA(bool s) { isLastCommandSDMA_ = s; }
 
   void hasPendingDispatch() { hasPendingDispatch_ = true; }
   void addSystemScope() { addSystemScope_ = true; }
@@ -390,8 +389,6 @@ class VirtualGPU : public device::VirtualDevice {
       uint32_t profiling_          : 1; //!< Profiling is enabled
       uint32_t cooperative_        : 1; //!< Cooperative launch is enabled
       uint32_t addSystemScope_     : 1; //!< Insert a system scope to the next aql
-      uint32_t isLastCommandSDMA_  : 1; //!< Keep track if the last command was SDMA and
-                                        //!< not send Barrier packets if barrier_sync is 0
       uint32_t tracking_created_   : 1; //!< Enabled if tracking object was properly initialized
     };
     uint32_t  state_;