From f9a6578b6b08f4629aff64fea6fc231bac1f0d94 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 28 Feb 2024 14:18:27 -0500 Subject: [PATCH] Disable SDMA ganging on non-APU multi-partition modes Work around SDMA hang in non-SPX modes for non-APU devices by disabling ganging. Root cause of hang not found. non-APU xGMI modes have only 1 link between socket devices anyways so there's likely no real system level gain in ganging intra-socket. Change-Id: Ia4eda2f85cbf25151d3dbcf50cc45b8b775c60e2 [ROCm/ROCR-Runtime commit: ed462035fabc5dfe8ddd1c5a2d8709a768122515] --- .../hsa-runtime/core/runtime/amd_topology.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp index 89c5cd2600..e595bffac7 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -367,6 +367,17 @@ void BuildTopology() { } const_cast(core::Runtime::runtime_singleton_->flag()).parse_masks(maxGpu, maxCu); + // Temporary work-around, disable SDMA ganging on non-APUs in non-SPX modes + // Check xGMI APU status + bool isXgmiApu = false; + auto& firstCpu = core::Runtime::runtime_singleton_->cpu_agents()[0]; + for (auto& peer_gpu : core::Runtime::runtime_singleton_->gpu_agents()) { + auto linfo = core::Runtime::runtime_singleton_->GetLinkInfo(firstCpu->node_id(), + peer_gpu->node_id()); + isXgmiApu = linfo.info.link_type == HSA_AMD_LINK_INFO_TYPE_XGMI; + if (isXgmiApu) break; + } + // Register destination agents that can SDMA gang copy for source agents for (auto& src_gpu : core::Runtime::runtime_singleton_->gpu_agents()) { uint32_t src_id = src_gpu->node_id(); @@ -383,7 +394,7 @@ void BuildTopology() { // Weight of 41 - Inter-socket GPU link in multi-partition mode if (linfo.info.link_type == HSA_AMD_LINK_INFO_TYPE_XGMI) { if (linfo.info.numa_distance == 13 || linfo.info.numa_distance == 41) - gang_factor = 2; + gang_factor = isXgmiApu ? 2 : 1; else if (linfo.info.numa_distance == 15 && linfo.info.min_bandwidth) gang_factor = linfo.info.max_bandwidth/linfo.info.min_bandwidth; else gang_factor = 1;