diff --git a/rocclr/device/blit.hpp b/rocclr/device/blit.hpp index 165dc2359d..21b7b37c76 100644 --- a/rocclr/device/blit.hpp +++ b/rocclr/device/blit.hpp @@ -53,6 +53,7 @@ class BlitManager : public amd::HeapObject { uint disableFillImage_ : 1; uint disableCopyBufferToImageOpt_ : 1; uint disableHwlCopyBuffer_ : 1; + uint disableHostCopyBuffer_ : 1; }; uint32_t value_; Setup() : value_(0) {} diff --git a/rocclr/device/rocm/rocblit.cpp b/rocclr/device/rocm/rocblit.cpp index 4be93dd4ad..1cdd232dac 100644 --- a/rocclr/device/rocm/rocblit.cpp +++ b/rocclr/device/rocm/rocblit.cpp @@ -69,7 +69,8 @@ bool DmaBlitManager::readBuffer(device::Memory& srcMemory, void* dstHost, // Use host copy if memory has direct access if (setup_.disableReadBuffer_ || - (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached())) { + (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached() && + !setup_.disableHostCopyBuffer_)) { // Stall GPU before CPU access gpu().Barriers().WaitCurrent(); return HostBlitManager::readBuffer(srcMemory, dstHost, origin, size, entire, copyMetadata); @@ -162,10 +163,12 @@ bool DmaBlitManager::readBufferRect(device::Memory& srcMemory, void* dstHost, // Use host copy if memory has direct access if (setup_.disableReadBufferRect_ || - (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached())) { + (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached() && + !setup_.disableHostCopyBuffer_)) { // Stall GPU before CPU access gpu().Barriers().WaitCurrent(); - return HostBlitManager::readBufferRect(srcMemory, dstHost, bufRect, hostRect, size, entire, copyMetadata); + return HostBlitManager::readBufferRect(srcMemory, dstHost, bufRect, hostRect, size, + entire, copyMetadata); } else { Memory& xferBuf = dev().xferRead().acquire(); address staging = xferBuf.getDeviceMemory(); @@ -233,8 +236,8 @@ bool DmaBlitManager::writeBuffer(const void* srcHost, device::Memory& dstMemory, const amd::Coord3D& origin, const amd::Coord3D& size, bool entire, amd::CopyMetadata copyMetadata) const { // Use host copy if memory has direct access - if (setup_.disableWriteBuffer_ || dstMemory.isHostMemDirectAccess() || - gpuMem(dstMemory).IsPersistentDirectMap()) { + if ((setup_.disableWriteBuffer_ || dstMemory.isHostMemDirectAccess() || + gpuMem(dstMemory).IsPersistentDirectMap()) && !setup_.disableHostCopyBuffer_) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); return HostBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire, copyMetadata); @@ -330,8 +333,8 @@ bool DmaBlitManager::writeBufferRect(const void* srcHost, device::Memory& dstMem gpu().releaseGpuMemoryFence(); // Use host copy if memory has direct access - if (setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess() || - gpuMem(dstMemory).IsPersistentDirectMap()) { + if ((setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess() || + gpuMem(dstMemory).IsPersistentDirectMap()) && !setup_.disableHostCopyBuffer_) { return HostBlitManager::writeBufferRect(srcHost, dstMemory, hostRect, bufRect, size, entire, copyMetadata); } else { @@ -387,8 +390,9 @@ bool DmaBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& dstMe const amd::Coord3D& size, bool entire, amd::CopyMetadata copyMetadata) const { if (setup_.disableCopyBuffer_ || - (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached() && - (dev().agent_profile() != HSA_PROFILE_FULL) && dstMemory.isHostMemDirectAccess())) { + (!setup_.disableHostCopyBuffer_ && srcMemory.isHostMemDirectAccess() && + !srcMemory.isCpuUncached() && (dev().agent_profile() != HSA_PROFILE_FULL) && + dstMemory.isHostMemDirectAccess())) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); return HostBlitManager::copyBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, size, false, @@ -406,8 +410,8 @@ bool DmaBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory& d const amd::Coord3D& size, bool entire, amd::CopyMetadata copyMetadata) const { if (setup_.disableCopyBufferRect_ || - (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached() && - dstMemory.isHostMemDirectAccess())) { + (!setup_.disableHostCopyBuffer_ && srcMemory.isHostMemDirectAccess() && + !srcMemory.isCpuUncached() && dstMemory.isHostMemDirectAccess())) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); return HostBlitManager::copyBufferRect(srcMemory, dstMemory, srcRect, dstRect, size, entire, @@ -1796,7 +1800,7 @@ bool KernelBlitManager::readBuffer(device::Memory& srcMemory, void* dstHost, // Use host copy if memory has direct access if (setup_.disableReadBuffer_ || (srcMemory.isHostMemDirectAccess() && - !srcMemory.isCpuUncached())) { + !srcMemory.isCpuUncached() && !setup_.disableHostCopyBuffer_)) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); result = HostBlitManager::readBuffer(srcMemory, dstHost, origin, size, entire, copyMetadata); @@ -1848,7 +1852,8 @@ bool KernelBlitManager::readBufferRect(device::Memory& srcMemory, void* dstHost, // Use host copy if memory has direct access if (setup_.disableReadBufferRect_ || - (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached())) { + (srcMemory.isHostMemDirectAccess() && !srcMemory.isCpuUncached() && + !setup_.disableHostCopyBuffer_)) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); result = HostBlitManager::readBufferRect(srcMemory, dstHost, bufRect, hostRect, size, entire, @@ -1899,7 +1904,7 @@ bool KernelBlitManager::writeBuffer(const void* srcHost, device::Memory& dstMemo // Use host copy if memory has direct access if (setup_.disableWriteBuffer_ || dstMemory.isHostMemDirectAccess() || - gpuMem(dstMemory).IsPersistentDirectMap()) { + gpuMem(dstMemory).IsPersistentDirectMap() && !setup_.disableHostCopyBuffer_) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); result = HostBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire, copyMetadata); @@ -1950,8 +1955,8 @@ bool KernelBlitManager::writeBufferRect(const void* srcHost, device::Memory& dst bool result = false; // Use host copy if memory has direct access - if (setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess() || - gpuMem(dstMemory).IsPersistentDirectMap()) { + if ((setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess() || + gpuMem(dstMemory).IsPersistentDirectMap()) && !setup_.disableHostCopyBuffer_) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); result = HostBlitManager::writeBufferRect(srcHost, dstMemory, hostRect, bufRect, size, entire, @@ -2031,7 +2036,8 @@ bool KernelBlitManager::fillBuffer1D(device::Memory& memory, const void* pattern bool result = false; // Use host fill if memory has direct access - if (setup_.disableFillBuffer_ || (!forceBlit && memory.isHostMemDirectAccess())) { + if (setup_.disableFillBuffer_ || (!forceBlit && memory.isHostMemDirectAccess() && + !setup_.disableHostCopyBuffer_)) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); result = HostBlitManager::fillBuffer(memory, pattern, patternSize, size, origin, size, entire); @@ -2130,7 +2136,8 @@ bool KernelBlitManager::fillBuffer2D(device::Memory& memory, const void* pattern bool result = false; // Use host fill if memory has direct access - if (setup_.disableFillBuffer_ || (!forceBlit && memory.isHostMemDirectAccess())) { + if (setup_.disableFillBuffer_ || (!forceBlit && memory.isHostMemDirectAccess() && + !setup_.disableHostCopyBuffer_)) { // Stall GPU before CPU access gpu().releaseGpuMemoryFence(); result = HostBlitManager::fillBuffer(memory, pattern, patternSize, size, origin, size, entire); diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index dbac1d3df3..97c38d2437 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -1358,6 +1358,8 @@ bool VirtualGPU::create() { } device::BlitManager::Setup blitSetup; + // Disable HostBlit copy path for HIP + blitSetup.disableHostCopyBuffer_ = amd::IS_HIP; blitMgr_ = new KernelBlitManager(*this, blitSetup); if ((nullptr == blitMgr_) || !blitMgr_->create(roc_device_)) { LogError("Could not create BlitManager!");