From 008133cf41583d3454f2c696992b5ea3e1b8c171 Mon Sep 17 00:00:00 2001 From: German Andryeyev Date: Tue, 7 Dec 2021 11:56:17 -0500 Subject: [PATCH] SWDEV-305016 - Improve MGPU scaling in Tensorflow Add a threshold for ROCR/SDMA P2P transfers. ROCR copy path requires extra barriers in compute for synchronization. That costs extra performance with tiny transfers. Reduce active wait time to 10us. Tensorflow uses extra thread per GPU with constant hipEventQuery() calls. Longer active waits in ROCr affect CPU performance. Change-Id: I9020358438615fa2d4617f862f00a562f0a588e7 --- rocclr/device/rocm/rocblit.cpp | 3 ++- rocclr/device/rocm/rocvirtual.cpp | 2 +- rocclr/utils/flags.hpp | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/rocclr/device/rocm/rocblit.cpp b/rocclr/device/rocm/rocblit.cpp index d759e6ec6a..6bc9e8cb38 100644 --- a/rocclr/device/rocm/rocblit.cpp +++ b/rocclr/device/rocm/rocblit.cpp @@ -2056,7 +2056,8 @@ bool KernelBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& ds const amd::Coord3D& sizeIn, bool entire) const { amd::ScopedLock k(lockXferOps_); bool result = false; - bool p2p = (&gpuMem(srcMemory).dev() != &gpuMem(dstMemory).dev()); + bool p2p = (&gpuMem(srcMemory).dev() != &gpuMem(dstMemory).dev()) && + (sizeIn[0] > ROC_P2P_SDMA_SIZE * Ki); bool asan = false; #if defined(__clang__) #if __has_feature(address_sanitizer) diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index 20fe2972db..f853a700b9 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -1249,7 +1249,7 @@ void* VirtualGPU::allocKernArg(size_t size, size_t alignment) { // ================================================================================================ address VirtualGPU::allocKernelArguments(size_t size, size_t alignment) { - if (ROCR_SKIP_KERNEL_ARG_COPY) { + if (ROC_SKIP_KERNEL_ARG_COPY) { // Make sure VirtualGPU has an exclusive access to the resources amd::ScopedLock lock(execution()); return reinterpret_cast
(allocKernArg(size, alignment)); diff --git a/rocclr/utils/flags.hpp b/rocclr/utils/flags.hpp index 5f0891af98..8f0228cc95 100644 --- a/rocclr/utils/flags.hpp +++ b/rocclr/utils/flags.hpp @@ -235,7 +235,7 @@ release(uint, HIP_HIDDEN_FREE_MEM, 0, \ "0 = Disable") \ release(size_t, GPU_FORCE_BLIT_COPY_SIZE, 0, \ "Size in KB of the threshold below which to force blit instead for sdma") \ -release(uint, ROC_ACTIVE_WAIT_TIMEOUT, 50, \ +release(uint, ROC_ACTIVE_WAIT_TIMEOUT, 10, \ "Forces active wait of GPU interrup for the timeout(us)") \ release(bool, ROC_ENABLE_LARGE_BAR, true, \ "Enable Large Bar if supported by the device") \ @@ -264,9 +264,11 @@ release(bool, AMD_CPU_AFFINITY, false, \ "Reset CPU affinity of any runtime threads") \ release(bool, ROC_USE_FGS_KERNARG, true, \ "Use fine grain kernel args segment for supported asics") \ +release(uint, ROC_P2P_SDMA_SIZE, 1024, \ + "The minimum size in MB for P2P transfer with SDMA") \ release(uint, ROC_AQL_QUEUE_SIZE, 4096, \ "AQL queue size in AQL packets") \ -release(bool, ROCR_SKIP_KERNEL_ARG_COPY, false, \ +release(bool, ROC_SKIP_KERNEL_ARG_COPY, false, \ "If true, then runtime can skip kernel arg copy") \ release(bool, GPU_STREAMOPS_CP_WAIT, false, \ "Force the stream wait memory operation to wait on CP.")