From 53d3c09599fee053870f238b098aec5d6aeb3aea Mon Sep 17 00:00:00 2001 From: Payam Date: Mon, 14 Dec 2020 16:44:32 -0500 Subject: [PATCH] SWDEV-257937 - ROC_BARRIER_SYNC fix for missing SDMA flush Change-Id: I93e8902bfcb16bac8ea594e16ea397b1ceafbd79 [ROCm/clr commit: f134b901993d4421051dc56a3408adf81ddb101e] --- projects/clr/rocclr/device/rocm/rocblit.cpp | 36 ++++++++++++++++--- .../clr/rocclr/device/rocm/rocvirtual.cpp | 4 ++- .../clr/rocclr/device/rocm/rocvirtual.hpp | 4 +++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/projects/clr/rocclr/device/rocm/rocblit.cpp b/projects/clr/rocclr/device/rocm/rocblit.cpp index ad065c63d9..f992e841ea 100644 --- a/projects/clr/rocclr/device/rocm/rocblit.cpp +++ b/projects/clr/rocclr/device/rocm/rocblit.cpp @@ -466,6 +466,7 @@ bool DmaBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory& d hsa_amd_memory_async_copy((reinterpret_cast
(dst) + dstOffset), dstAgent, (reinterpret_cast(src) + srcOffset), srcAgent, size[0], 0, nullptr, completion_signal_); + gpu().setLastCommandSDMA(true) ; if (status != HSA_STATUS_SUCCESS) { LogPrintfError("DMA buffer failed with code %d", status); return false; @@ -489,7 +490,11 @@ bool DmaBlitManager::copyImageToBuffer(device::Memory& srcMemory, device::Memory const amd::Coord3D& size, bool entire, size_t rowPitch, size_t slicePitch) const { // HSA copy functionality with a possible async operaiton, hence make sure GPU is done - gpu().releaseGpuMemoryFence(); + if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) { + gpu().releaseGpuMemoryFence(true); + } else { + gpu().releaseGpuMemoryFence(); + } bool result = false; @@ -536,7 +541,11 @@ bool DmaBlitManager::copyBufferToImage(device::Memory& srcMemory, device::Memory const amd::Coord3D& size, bool entire, size_t rowPitch, size_t slicePitch) const { // HSA copy functionality with a possible async operaiton, hence make sure GPU is done - gpu().releaseGpuMemoryFence(); + if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) { + gpu().releaseGpuMemoryFence(true); + } else { + gpu().releaseGpuMemoryFence(); + } bool result = false; @@ -601,6 +610,10 @@ bool DmaBlitManager::hsaCopy(const Memory& srcMemory, const Memory& dstMemory, address src = reinterpret_cast
(srcMemory.getDeviceMemory()); address dst = reinterpret_cast
(dstMemory.getDeviceMemory()); + if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) { + gpu().releaseGpuMemoryFence(true); + } + src += srcOrigin[0]; dst += dstOrigin[0]; @@ -641,7 +654,7 @@ bool DmaBlitManager::hsaCopy(const Memory& srcMemory, const Memory& dstMemory, // Use SDMA to transfer the data status = hsa_amd_memory_async_copy(dst, dstAgent, src, srcAgent, size[0], 0, nullptr, completion_signal_); - + gpu().setLastCommandSDMA(true); if (status == HSA_STATUS_SUCCESS) { hsa_signal_value_t val; @@ -675,6 +688,9 @@ bool DmaBlitManager::hsaCopyStaged(const_address hostSrc, address hostDst, size_ size_t offset = 0; address hsaBuffer = staging; + if (!dev().settings().barrier_sync_ && !gpu().isLastCommandSDMA()) { + gpu().releaseGpuMemoryFence(true); + } // Allocate requested size of memory while (totalSize > 0) { @@ -692,6 +708,7 @@ bool DmaBlitManager::hsaCopyStaged(const_address hostSrc, address hostDst, size_ memcpy(hsaBuffer, hostSrc + offset, size); status = hsa_amd_memory_async_copy(hostDst + offset, dev().getBackendDevice(), hsaBuffer, srcAgent, size, 0, nullptr, completion_signal_); + gpu().setLastCommandSDMA(true); if (status == HSA_STATUS_SUCCESS) { if (!WaitForSignal(completion_signal_)) { LogError("Async copy failed"); @@ -716,6 +733,7 @@ bool DmaBlitManager::hsaCopyStaged(const_address hostSrc, address hostDst, size_ status = hsa_amd_memory_async_copy(hsaBuffer, dstAgent, hostSrc + offset, dev().getBackendDevice(), size, 0, nullptr, completion_signal_); + gpu().setLastCommandSDMA(true); if (status == HSA_STATUS_SUCCESS) { if (!WaitForSignal(completion_signal_)) { LogError("Async copy failed"); @@ -1065,7 +1083,11 @@ bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory, releaseArguments(parameters); if (releaseView) { // todo SRD programming could be changed to avoid a stall - gpu().releaseGpuMemoryFence(); + if(!dev().settings().barrier_sync_) { + gpu().releaseGpuMemoryFence(true); + } else { + gpu().releaseGpuMemoryFence(); + } dstView->owner()->release(); } @@ -1263,7 +1285,11 @@ bool KernelBlitManager::copyImageToBufferKernel(device::Memory& srcMemory, releaseArguments(parameters); if (releaseView) { // todo SRD programming could be changed to avoid a stall - gpu().releaseGpuMemoryFence(); + if(!dev().settings().barrier_sync_) { + gpu().releaseGpuMemoryFence(true); + } else { + gpu().releaseGpuMemoryFence(); + } srcView->owner()->release(); } diff --git a/projects/clr/rocclr/device/rocm/rocvirtual.cpp b/projects/clr/rocclr/device/rocm/rocvirtual.cpp index 4ae1126449..16edb11fdf 100644 --- a/projects/clr/rocclr/device/rocm/rocvirtual.cpp +++ b/projects/clr/rocclr/device/rocm/rocvirtual.cpp @@ -715,7 +715,9 @@ void VirtualGPU::ResetQueueStates() { bool VirtualGPU::releaseGpuMemoryFence(bool force_barrier) { // Return if there is no pending dispatch if (!hasPendingDispatch_) { - return false; + if (dev().settings().barrier_sync_ || !force_barrier) { + return false; + } } hsa_signal_t wait_signal = barrier_signal_; diff --git a/projects/clr/rocclr/device/rocm/rocvirtual.hpp b/projects/clr/rocclr/device/rocm/rocvirtual.hpp index 3bd075b700..7f8b93f49d 100644 --- a/projects/clr/rocclr/device/rocm/rocvirtual.hpp +++ b/projects/clr/rocclr/device/rocm/rocvirtual.hpp @@ -290,6 +290,8 @@ class VirtualGPU : public device::VirtualDevice { amd::Memory* findPinnedMem(void* addr, size_t size); void enableSyncBlit() const; + bool isLastCommandSDMA() const { return isLastCommandSDMA_; } + void setLastCommandSDMA(bool s) { isLastCommandSDMA_ = s; } void hasPendingDispatch() { hasPendingDispatch_ = true; } void addSystemScope() { addSystemScope_ = true; } @@ -365,6 +367,8 @@ class VirtualGPU : public device::VirtualDevice { uint32_t profiling_ : 1; //!< Profiling is enabled uint32_t cooperative_ : 1; //!< Cooperative launch is enabled uint32_t addSystemScope_ : 1; //!< Insert a system scope to the next aql + uint32_t isLastCommandSDMA_ : 1; //!< Keep track if the last command was SDMA and + //!< not send Barrier packets if barrier_sync is 0 }; uint32_t state_; };