From a2e0b0495ce66c5f97a640e607db76f44a236b0e Mon Sep 17 00:00:00 2001 From: Payam Date: Mon, 1 Feb 2021 16:13:42 -0500 Subject: [PATCH] SWDEV-257937 - Updated fix for ROC_BARRIER_SYNC=0 Change-Id: I7e28e541b654db57fb0890d7dbb7519cfb2d93db --- rocclr/device/device.hpp | 5 +++++ rocclr/device/rocm/rocblit.cpp | 15 ++++++++++----- rocclr/device/rocm/rocdevice.cpp | 12 +++++------- rocclr/device/rocm/rocvirtual.cpp | 10 ++++++---- rocclr/device/rocm/rocvirtual.hpp | 5 +---- 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/rocclr/device/device.hpp b/rocclr/device/device.hpp index 5c6f12e79b..3389fd3c00 100644 --- a/rocclr/device/device.hpp +++ b/rocclr/device/device.hpp @@ -1203,6 +1203,9 @@ class VirtualDevice : public amd::HeapObject { //! Returns true if device has active wait setting bool ActiveWait() const; + bool isLastCommandSDMA() const { return isLastCommandSDMA_; } + void setLastCommandSDMA(bool s) { isLastCommandSDMA_ = s; } + private: //! Disable default copy constructor VirtualDevice& operator=(const VirtualDevice&); @@ -1216,6 +1219,8 @@ class VirtualDevice : public amd::HeapObject { protected: device::BlitManager* blitMgr_; //!< Blit manager + //!< Keep track if the last command was SDMA and not send Barrier packets if barrier_sync is 0 + std::atomic_bool isLastCommandSDMA_; amd::Monitor execution_; //!< Lock to serialise access to all device objects uint index_; //!< The virtual device unique index }; diff --git a/rocclr/device/rocm/rocblit.cpp b/rocclr/device/rocm/rocblit.cpp index 807ba4f28f..c010500c0c 100644 --- a/rocclr/device/rocm/rocblit.cpp +++ b/rocclr/device/rocm/rocblit.cpp @@ -395,7 +395,8 @@ bool DmaBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory& d gpu().releaseGpuMemoryFence(); return HostBlitManager::copyBufferRect(srcMemory, dstMemory, srcRect, dstRect, size, entire); } else { - gpu().releaseGpuMemoryFence(kIgnoreBarrier, kSkipCpuWait); + bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA(); + gpu().releaseGpuMemoryFence(force_barrier, kSkipCpuWait); void* src = gpuMem(srcMemory).getDeviceMemory(); void* dst = gpuMem(dstMemory).getDeviceMemory(); @@ -1117,7 +1118,8 @@ bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory, releaseArguments(parameters); if (releaseView) { // todo SRD programming could be changed to avoid a stall - gpu().releaseGpuMemoryFence(); + bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA(); + gpu().releaseGpuMemoryFence(force_barrier); dstView->owner()->release(); } @@ -1315,7 +1317,8 @@ bool KernelBlitManager::copyImageToBufferKernel(device::Memory& srcMemory, releaseArguments(parameters); if (releaseView) { // todo SRD programming could be changed to avoid a stall - gpu().releaseGpuMemoryFence(); + bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA(); + gpu().releaseGpuMemoryFence(force_barrier); srcView->owner()->release(); } @@ -1446,7 +1449,8 @@ bool KernelBlitManager::copyImage(device::Memory& srcMemory, device::Memory& dst releaseArguments(parameters); if (releaseView) { // todo SRD programming could be changed to avoid a stall - gpu().releaseGpuMemoryFence(); + bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA(); + gpu().releaseGpuMemoryFence(force_barrier); srcView->owner()->release(); dstView->owner()->release(); } @@ -2269,7 +2273,8 @@ bool KernelBlitManager::fillImage(device::Memory& memory, const void* pattern, releaseArguments(parameters); if (releaseView) { // todo SRD programming could be changed to avoid a stall - gpu().releaseGpuMemoryFence(); + bool force_barrier = !dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA(); + gpu().releaseGpuMemoryFence(force_barrier); memView->owner()->release(); } diff --git a/rocclr/device/rocm/rocdevice.cpp b/rocclr/device/rocm/rocdevice.cpp index a344dd8825..4b3302f76c 100644 --- a/rocclr/device/rocm/rocdevice.cpp +++ b/rocclr/device/rocm/rocdevice.cpp @@ -1584,12 +1584,6 @@ bool Device::populateOCLDeviceConstants() { device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) { amd::ScopedLock lock(vgpusAccess()); - // If barrier is disabled, then profiling should be enabled to make sure HSA signal is - // attached for every dispatch - if (!settings().barrier_sync_) { - queue->properties().set(CL_QUEUE_PROFILING_ENABLE); - } - bool profiling = (queue != nullptr) && queue->properties().test(CL_QUEUE_PROFILING_ENABLE); bool cooperative = false; @@ -1599,7 +1593,11 @@ device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) { cooperative = amd::IS_HIP && settings().enableCoopGroups_; profiling = amd::IS_HIP; } - + // If barrier is disabled, then profiling should be enabled to make sure HSA signal is + // attached for every dispatch + else if (!settings().barrier_sync_) { + queue->properties().set(CL_QUEUE_PROFILING_ENABLE); + } // Initialization of heap and other resources occur during the command // queue creation time. const std::vector defaultCuMask = {}; diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index 28b75c9027..3efda825ac 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -2336,7 +2336,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const device::Kernel* devKernel = const_cast(kernel.getDeviceKernel(dev())); Kernel& gpuKernel = static_cast(*devKernel); size_t ldsUsage = gpuKernel.WorkgroupGroupSegmentByteSize(); - + setLastCommandSDMA(false); // Check memory dependency and SVM objects bool coopGroups = (vcmd != nullptr) ? vcmd->cooperativeGroups() : false; if (!processMemObjects(kernel, parameters, ldsUsage, coopGroups)) { @@ -2617,7 +2617,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) { if (vcmd.cooperativeGroups() || vcmd.cooperativeMultiDeviceGroups()) { // Wait for the execution on the current queue, since the coop groups will use the device queue - releaseGpuMemoryFence(kIgnoreBarrier, kSkipCpuWait); + bool force_barrier = !dev().settings().barrier_sync_ && !isLastCommandSDMA(); + releaseGpuMemoryFence(force_barrier, kSkipCpuWait); // Get device queue for exclusive GPU access VirtualGPU* queue = dev().xferQueue(); @@ -2726,6 +2727,7 @@ void VirtualGPU::submitReleaseExtObjects(amd::ReleaseExtObjectsCommand& vcmd) { void VirtualGPU::flush(amd::Command* list, bool wait) { // Direct dispatch relies on HSA signal callback bool skip_cpu_wait = AMD_DIRECT_DISPATCH; + bool force_barrier = !dev().settings().barrier_sync_ && !isLastCommandSDMA(); if (skip_cpu_wait) { // Search for the last command in the batch to track GPU state @@ -2742,7 +2744,7 @@ void VirtualGPU::flush(amd::Command* list, bool wait) { // CPU wait is also forced if pinned buffers were used skip_cpu_wait &= hasPendingDispatch_ && (pinnedMems_.size() == 0); - releaseGpuMemoryFence(kIgnoreBarrier, skip_cpu_wait); + releaseGpuMemoryFence(force_barrier, skip_cpu_wait); profilingEnd(*current); } else { // If barrier is requested, then wait for everything, otherwise @@ -2754,7 +2756,7 @@ void VirtualGPU::flush(amd::Command* list, bool wait) { // If CPU waited for GPU, then the queue is idle if (!skip_cpu_wait) { - updateCommandsState(list); + updateCommandsState(list); // Add extra clean up for resources if releaseGpuMemoryFence() was skipped if (!dev().settings().barrier_sync_) { diff --git a/rocclr/device/rocm/rocvirtual.hpp b/rocclr/device/rocm/rocvirtual.hpp index cc4c96c635..626a684efd 100644 --- a/rocclr/device/rocm/rocvirtual.hpp +++ b/rocclr/device/rocm/rocvirtual.hpp @@ -36,6 +36,7 @@ class Device; class Memory; class Timestamp; + struct ProfilingSignal : public amd::HeapObject { hsa_signal_t signal_; //!< HSA signal to track profiling information Timestamp* ts_; //!< Timestamp object associated with the signal @@ -312,8 +313,6 @@ class VirtualGPU : public device::VirtualDevice { amd::Memory* findPinnedMem(void* addr, size_t size); void enableSyncBlit() const; - bool isLastCommandSDMA() const { return isLastCommandSDMA_; } - void setLastCommandSDMA(bool s) { isLastCommandSDMA_ = s; } void hasPendingDispatch() { hasPendingDispatch_ = true; } void addSystemScope() { addSystemScope_ = true; } @@ -390,8 +389,6 @@ class VirtualGPU : public device::VirtualDevice { uint32_t profiling_ : 1; //!< Profiling is enabled uint32_t cooperative_ : 1; //!< Cooperative launch is enabled uint32_t addSystemScope_ : 1; //!< Insert a system scope to the next aql - uint32_t isLastCommandSDMA_ : 1; //!< Keep track if the last command was SDMA and - //!< not send Barrier packets if barrier_sync is 0 uint32_t tracking_created_ : 1; //!< Enabled if tracking object was properly initialized }; uint32_t state_;