AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-305016 - Improve MGPU scaling in Tensorflow

Browse Source 
Add a threshold for ROCR/SDMA P2P transfers. ROCR copy path
requires extra barriers in compute for synchronization. That costs
extra performance with tiny transfers.
Reduce active wait time to 10us. Tensorflow uses extra thread
per GPU with constant hipEventQuery() calls. Longer active waits
in ROCr affect CPU performance.

Change-Id: I9020358438615fa2d4617f862f00a562f0a588e7
This commit is contained in:
German Andryeyev
2021-12-07 11:56:17 -05:00
parent 587cda0c91
commit 008133cf41
3 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rocclr/utils/flags.hpp
+4 -2
View File
@@ -235,7 +235,7 @@ release(uint, HIP_HIDDEN_FREE_MEM, 0, \
"0 = Disable") \
release(size_t, GPU_FORCE_BLIT_COPY_SIZE, 0, \
"Size in KB of the threshold below which to force blit instead for sdma") \
release(uint, ROC_ACTIVE_WAIT_TIMEOUT, 50, \
release(uint, ROC_ACTIVE_WAIT_TIMEOUT, 10, \
"Forces active wait of GPU interrup for the timeout(us)") \
release(bool, ROC_ENABLE_LARGE_BAR, true, \
"Enable Large Bar if supported by the device") \
@@ -264,9 +264,11 @@ release(bool, AMD_CPU_AFFINITY, false, \
"Reset CPU affinity of any runtime threads") \
release(bool, ROC_USE_FGS_KERNARG, true, \
"Use fine grain kernel args segment for supported asics") \
release(uint, ROC_P2P_SDMA_SIZE, 1024, \
"The minimum size in MB for P2P transfer with SDMA") \
release(uint, ROC_AQL_QUEUE_SIZE, 4096, \
"AQL queue size in AQL packets") \
release(bool, ROCR_SKIP_KERNEL_ARG_COPY, false, \
release(bool, ROC_SKIP_KERNEL_ARG_COPY, false, \
"If true, then runtime can skip kernel arg copy") \
release(bool, GPU_STREAMOPS_CP_WAIT, false, \
"Force the stream wait memory operation to wait on CP.")