SWDEV-305016 - Improve MGPU scaling in Tensorflow

Add a threshold for ROCR/SDMA P2P transfers. ROCR copy path
requires extra barriers in compute for synchronization. That costs
extra performance with tiny transfers.
Reduce active wait time to 10us. Tensorflow uses extra thread
per GPU with constant hipEventQuery() calls. Longer active waits
in ROCr affect CPU performance.

Change-Id: I9020358438615fa2d4617f862f00a562f0a588e7


[ROCm/clr commit: 008133cf41]

projects/clr/rocclr/device/rocm/rocblit.cpp

@@ -2056,7 +2056,8 @@ bool KernelBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& ds
                                    const amd::Coord3D& sizeIn, bool entire) const {
   amd::ScopedLock k(lockXferOps_);
   bool result = false;
-  bool p2p = (&gpuMem(srcMemory).dev() != &gpuMem(dstMemory).dev());
+  bool p2p = (&gpuMem(srcMemory).dev() != &gpuMem(dstMemory).dev()) &&
+             (sizeIn[0] > ROC_P2P_SDMA_SIZE * Ki);
   bool asan = false;
 #if defined(__clang__)
 #if __has_feature(address_sanitizer)

projects/clr/rocclr/device/rocm/rocvirtual.cpp

@@ -1249,7 +1249,7 @@ void* VirtualGPU::allocKernArg(size_t size, size_t alignment) {
 
 // ================================================================================================
 address VirtualGPU::allocKernelArguments(size_t size, size_t alignment) {
-  if (ROCR_SKIP_KERNEL_ARG_COPY) {
+  if (ROC_SKIP_KERNEL_ARG_COPY) {
     // Make sure VirtualGPU has an exclusive access to the resources
     amd::ScopedLock lock(execution());
     return reinterpret_cast<address>(allocKernArg(size, alignment));

projects/clr/rocclr/utils/flags.hpp

@@ -235,7 +235,7 @@ release(uint, HIP_HIDDEN_FREE_MEM, 0,                                         \
         "0 = Disable")                                                        \
 release(size_t, GPU_FORCE_BLIT_COPY_SIZE, 0,                                  \
         "Size in KB of the threshold below which to force blit instead for sdma") \
-release(uint, ROC_ACTIVE_WAIT_TIMEOUT, 50,                                    \
+release(uint, ROC_ACTIVE_WAIT_TIMEOUT, 10,                                    \
         "Forces active wait of GPU interrup for the timeout(us)")             \
 release(bool, ROC_ENABLE_LARGE_BAR, true,                                     \
         "Enable Large Bar if supported by the device")                        \
@@ -264,9 +264,11 @@ release(bool, AMD_CPU_AFFINITY, false,                                        \
         "Reset CPU affinity of any runtime threads")                          \
 release(bool, ROC_USE_FGS_KERNARG, true,                                      \
         "Use fine grain kernel args segment for supported asics")             \
+release(uint, ROC_P2P_SDMA_SIZE, 1024,                                        \
+        "The minimum size in MB for P2P transfer with SDMA")                  \
 release(uint, ROC_AQL_QUEUE_SIZE, 4096,                                       \
         "AQL queue size in AQL packets")                                      \
-release(bool, ROCR_SKIP_KERNEL_ARG_COPY, false,                               \
+release(bool, ROC_SKIP_KERNEL_ARG_COPY, false,                                \
         "If true, then runtime can skip kernel arg copy")                     \
 release(bool, GPU_STREAMOPS_CP_WAIT, false,                                   \
         "Force the stream wait memory operation to wait on CP.")