From 8d34f4e12d21ba066384847151c588d23d1f92f9 Mon Sep 17 00:00:00 2001 From: Shane Xiao Date: Fri, 11 Apr 2025 18:20:44 +0800 Subject: [PATCH] rocr: Add rec sdma engines with limited XGMI SDMA engine This patch will adds recommended sdma supports with limited XGMI SDMA engine. It will use one PCIe SDMA to do gpu <-> gpu copies which will help improve all to all copy performance. Signed-off-by: Shane Xiao [ROCm/ROCR-Runtime commit: 6a63170b38ba35e8264251735827c3dfd0298873] --- .../hsa-runtime/core/inc/amd_gpu_agent.h | 6 ++++ .../core/runtime/amd_gpu_agent.cpp | 5 +-- .../hsa-runtime/core/runtime/amd_topology.cpp | 35 +++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index 051879c4ba..8efc271c90 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -171,6 +171,8 @@ class GpuAgentInt : public core::Agent { virtual void RegisterRecSdmaEngIdMaskPeer(core::Agent &gang_peer, uint32_t rec_sdma_eng_id_mask) = 0; + virtual void SetRecSdmaEngOverride(bool flag) = 0; + // @brief Query if agent represent Kaveri GPU. // // @retval true if agent is Kaveri GPU. @@ -376,6 +378,9 @@ class GpuAgent : public GpuAgentInt { return properties_; } + // @brief set rec_sdma_eng_override_ + __forceinline void SetRecSdmaEngOverride(bool flag) override { rec_sdma_eng_override_ = flag; } + // @brief Returns number of data caches. __forceinline size_t num_cache() const { return cache_props_.size(); } @@ -798,6 +803,7 @@ class GpuAgent : public GpuAgentInt { std::map rec_sdma_eng_id_peers_info_; bool uses_rec_sdma_eng_id_mask_; + bool rec_sdma_eng_override_; // structure for host trap sampling pcs_data_t pcs_hosttrap_data_; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index d1a8c52707..9fd6e035fa 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -117,7 +117,8 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna trap_handler_tma_region_(NULL), pcs_hosttrap_data_(), pcs_stochastic_data_(), - xgmi_cpu_gpu_(false) { + xgmi_cpu_gpu_(false), + rec_sdma_eng_override_(false) { const bool is_apu_node = (properties_.NumCPUCores > 0); profile_ = (is_apu_node) ? HSA_PROFILE_FULL : HSA_PROFILE_BASE; @@ -1160,7 +1161,7 @@ hsa_status_t GpuAgent::DmaCopyOnEngine(void* dst, core::Agent& dst_agent, bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10); // Ensure engine selection is within proper range based on transfer type - if ((is_xgmi && engine_offset <= properties_.NumSdmaEngines) || + if ((is_xgmi && !rec_sdma_eng_override_ && engine_offset <= properties_.NumSdmaEngines) || (!is_xgmi && engine_offset > (properties_.NumSdmaEngines + properties_.NumSdmaXgmiEngines)) || (!is_h2d_blit && !is_same_gpu && limit_h2d_blit && diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp index c609e9e639..d30e11b0ae 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -381,6 +381,32 @@ bool BuildTopology() { } const_cast(rt->flag()).parse_masks(maxGpu, maxCu); + // Front load the rec_sdma_eng_id_mask to check whether needs to override old mask + bool rec_sdma_engine_override = false; + for (auto& src_gpu : rt->gpu_agents()) { + uint32_t src_id = src_gpu->node_id(); + + // skip the pre-loop if NumSdmaXgmiEngines != 6 + if (((AMD::GpuAgent*)src_gpu)->properties().NumSdmaXgmiEngines != 6) + break; + + for (auto& dst_gpu : rt->gpu_agents()) { + uint32_t dst_id = dst_gpu->node_id(); + if (src_id != dst_id) { + auto linfo = rt->GetLinkInfo(src_id, dst_id); + if (IsPowerOfTwo(linfo.rec_sdma_eng_id_mask)) { + rec_sdma_engine_override = true; + ((AMD::GpuAgent*)src_gpu)->SetRecSdmaEngOverride(rec_sdma_engine_override); + break; + } + } + } + + // skip the pre-loop if RecSdmaEngOverride is true + if (rec_sdma_engine_override) + break; + } + // Register destination agents that can SDMA gang copy for source agents for (auto& src_gpu : rt->gpu_agents()) { uint32_t src_id = src_gpu->node_id(); @@ -406,6 +432,15 @@ bool BuildTopology() { else gang_factor = 1; rec_sdma_eng_id_mask = linfo.rec_sdma_eng_id_mask; + + // Override the old mask if rec sdma eng verride is true + // Using one pcie sdma for device to device copy with limited XGMI SDMA engine. + // This will help improve all to all copy with limited XGMI SDMA engine. + if (rec_sdma_engine_override) { + uint32_t sdma_engine_mask = (1 << ((AMD::GpuAgent*)src_gpu)->properties().NumSdmaEngines - 1); + rec_sdma_eng_id_mask = !IsPowerOfTwo(rec_sdma_eng_id_mask) ? + sdma_engine_mask : rec_sdma_eng_id_mask; + } } }