From 2ce6bbebc479cbbdbed1c7e0b04a7882628ef9c7 Mon Sep 17 00:00:00 2001 From: German Andryeyev Date: Tue, 19 May 2020 17:25:14 -0400 Subject: [PATCH] Fix async mem clear Optimization for the fence release removed a sync for mem fill. Add simple const buffer management forr the filled pattern to avoid pattern overwriting with the async fills. Change-Id: I63773ac09ceec31d5396d24570e4647ff096326b --- rocclr/device/rocm/rocblit.cpp | 7 +++++-- rocclr/device/rocm/rocblit.hpp | 34 +++++++++++++++++++++++-------- rocclr/device/rocm/rocvirtual.cpp | 4 ++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/rocclr/device/rocm/rocblit.cpp b/rocclr/device/rocm/rocblit.cpp index 56ed6c3c17..570ab35d46 100644 --- a/rocclr/device/rocm/rocblit.cpp +++ b/rocclr/device/rocm/rocblit.cpp @@ -758,6 +758,7 @@ KernelBlitManager::KernelBlitManager(VirtualGPU& gpu, Setup setup) : DmaBlitManager(gpu, setup), program_(nullptr), constantBuffer_(nullptr), + constantBufferOffset_(0), xferBufferSize_(0), lockXferOps_("Transfer Ops Lock", true) { for (uint i = 0; i < BlitTotal; ++i) { @@ -1900,11 +1901,13 @@ bool KernelBlitManager::fillBuffer(device::Memory& memory, const void* pattern, if (gpuCB == nullptr) { return false; } - void* constBuf = constantBuffer_->getHostMem(); + // Find offset in the current constant buffer to allow multipel fills + uint32_t constBufOffset = ConstantBufferOffset(); + auto constBuf = reinterpret_cast
(constantBuffer_->getHostMem()) + constBufOffset; memcpy(constBuf, pattern, patternSize); mem = as_cl(gpuCB->owner()); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), &mem); + setArgument(kernels_[fillType], 2, sizeof(cl_mem), &mem, constBufOffset); uint64_t offset = origin[0]; if (dwordAligned) { patternSize /= sizeof(uint32_t); diff --git a/rocclr/device/rocm/rocblit.hpp b/rocclr/device/rocm/rocblit.hpp index 4ef96da434..ea4ac996a9 100644 --- a/rocclr/device/rocm/rocblit.hpp +++ b/rocclr/device/rocm/rocblit.hpp @@ -440,7 +440,22 @@ class KernelBlitManager : public DmaBlitManager { address captureArguments(const amd::Kernel* kernel) const; void releaseArguments(address args) const; - inline void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const; + inline void setArgument(amd::Kernel* kernel, size_t index, + size_t size, const void* value, uint32_t offset = 0) const; + + uint32_t ConstantBufferOffset() const { + // Make sure it can fit at least 128 bytes for OCL memory fill of double16 + constexpr uint32_t kManagedSize = 0x80; + // Adjust the ofset to the new location + constantBufferOffset_ += kManagedSize; + // Check if the allocation exceeds the limit + if ((constantBufferOffset_ + kManagedSize) > constantBuffer_->getSize()) { + // Stall GPU and reset the ofset + gpu().releaseGpuMemoryFence(); + constantBufferOffset_ = 0; + } + return constantBufferOffset_; + } //! Disable copy constructor KernelBlitManager(const KernelBlitManager&); @@ -448,11 +463,12 @@ class KernelBlitManager : public DmaBlitManager { //! Disable operator= KernelBlitManager& operator=(const KernelBlitManager&); - amd::Program* program_; //!< GPU program obejct - amd::Kernel* kernels_[BlitTotal]; //!< GPU kernels for blit - amd::Memory* constantBuffer_; //!< An internal CB for blits - size_t xferBufferSize_; //!< Transfer buffer size - mutable amd::Monitor lockXferOps_; //!< Lock transfer operation + amd::Program* program_; //!< GPU program obejct + amd::Kernel* kernels_[BlitTotal]; //!< GPU kernels for blit + amd::Memory* constantBuffer_; //!< An internal CB for blits + mutable uint32_t constantBufferOffset_; //!< Current offset in the constant buffer + size_t xferBufferSize_; //!< Transfer buffer size + mutable amd::Monitor lockXferOps_; //!< Lock transfer operation }; static const char* BlitName[KernelBlitManager::BlitTotal] = { @@ -462,7 +478,8 @@ static const char* BlitName[KernelBlitManager::BlitTotal] = { "fillImage", "scheduler", "gwsInit" }; -inline void KernelBlitManager::setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) const { +inline void KernelBlitManager::setArgument(amd::Kernel* kernel, size_t index, + size_t size, const void* value, uint32_t offset) const { const amd::KernelParameterDescriptor& desc = kernel->signature().at(index); void* param = kernel->parameters().values() + desc.offset_; @@ -483,7 +500,8 @@ inline void KernelBlitManager::setArgument(amd::Kernel* kernel, size_t index, si // convert cl_mem to amd::Memory*, return false if invalid. reinterpret_cast(kernel->parameters().values() + kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = mem; - LP64_SWITCH(uint32_value, uint64_value) = static_cast(mem->getDeviceMemory(dev())->virtualAddress()); + LP64_SWITCH(uint32_value, uint64_value) = static_cast( + mem->getDeviceMemory(dev())->virtualAddress()) + offset; } } else if (desc.type_ == T_SAMPLER) { assert(false && "No sampler support in blit manager! Use internal samplers!"); diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index b1a5843eab..e936468fd8 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -1318,6 +1318,8 @@ void VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& cmd) { cmd.setStatus(CL_INVALID_OPERATION); } } else { + // Stall GPU for CPU access to memory + releaseGpuMemoryFence(); // direct memcpy for FGS enabled system amd::SvmBuffer::memFill(cmd.dst(), cmd.src(), cmd.srcSize(), 1); } @@ -1785,6 +1787,8 @@ void VirtualGPU::submitSvmFillMemory(amd::SvmFillMemoryCommand& cmd) { // Mark this as the most-recently written cache of the destination dstMemory->signalWrite(&dev()); } else { + // Stall GPU for CPU access to memory + releaseGpuMemoryFence(); // for FGS capable device, fill CPU memory directly amd::SvmBuffer::memFill(cmd.dst(), cmd.pattern(), cmd.patternSize(), cmd.times()); }