From a397373cea3f2981b8069bb774c0e8eccc5cc382 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Thu, 18 May 2023 00:10:20 +0000 Subject: [PATCH] Add HSA_ENABLE_PEER_SDMA env variable Add support for HSA_ENABLE_PEER_SDMA env variable that can be used to disable use of SDMA engines for device-to-device transfers. Note that setting HSA_ENABLE_SDMA=0 will disable all SDMA transfers and override HSA_ENABLE_PEER_SDMA values. Change-Id: I737b3c2b2efcf3ff237f98bc748f49b8252ed24a --- .../core/runtime/amd_gpu_agent.cpp | 54 ++++++++++++------- runtime/hsa-runtime/core/util/flag.h | 6 +++ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 9ccbf6b893..a606159892 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -831,29 +831,41 @@ hsa_status_t GpuAgent::DmaCopyOnEngine(void* dst, core::Agent& dst_agent, // check if dst and src are the same gpu or over xGMI. bool is_same_gpu = (src_agent.public_handle().handle == dst_agent.public_handle().handle) && (dst_agent.public_handle().handle == public_handle_.handle); - bool is_xgmi = !is_same_gpu && - src_agent.device_type() == core::Agent::kAmdGpuDevice && - dst_agent.device_type() == core::Agent::kAmdGpuDevice && - dst_agent.HiveId() && src_agent.HiveId() == dst_agent.HiveId() && + + bool is_p2p = !is_same_gpu && src_agent.device_type() == core::Agent::kAmdGpuDevice && + dst_agent.device_type() == core::Agent::kAmdGpuDevice; + + if (is_p2p && + core::Runtime::runtime_singleton_->flag().enable_peer_sdma() == Flag::SDMA_DISABLE) { + // Note that VDI/HIP will call DmaCopy instead of DmaCopyOnEngine for P2P copies, but + // we still want to handle force Blit Kernels in this function in case other libraries + // decide to use DmaCopyOnEngine for P2P copies + + engine_offset = BlitDevToDev; + } else { + bool is_xgmi = is_p2p && dst_agent.HiveId() && src_agent.HiveId() == dst_agent.HiveId() && properties_.NumSdmaXgmiEngines; - // Due to a RAS issue, GFX90a can only support H2D copies on SDMA0 - bool is_h2d_blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice && - dst_agent.device_type() == core::Agent::kAmdGpuDevice); - bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10); - // Ensure engine selection is within proper range based on transfer type - if ((is_xgmi && engine_offset <= properties_.NumSdmaEngines) || - (!is_xgmi && engine_offset > (properties_.NumSdmaEngines + - properties_.NumSdmaXgmiEngines)) || - (!is_h2d_blit && !is_same_gpu && limit_h2d_blit && - engine_offset == BlitHostToDev)) { - return HSA_STATUS_ERROR_INVALID_ARGUMENT; + // Due to a RAS issue, GFX90a can only support H2D copies on SDMA0 + bool is_h2d_blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice && + dst_agent.device_type() == core::Agent::kAmdGpuDevice); + bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10); + + // Ensure engine selection is within proper range based on transfer type + if ((is_xgmi && engine_offset <= properties_.NumSdmaEngines) || + (!is_xgmi && engine_offset > (properties_.NumSdmaEngines + + properties_.NumSdmaXgmiEngines)) || + (!is_h2d_blit && !is_same_gpu && limit_h2d_blit && + engine_offset == BlitHostToDev)) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + engine_offset = is_same_gpu ?(force_copy_on_sdma ? BlitDevToHost : + BlitDevToDev) : engine_offset; } - lazy_ptr& blit = is_same_gpu ? - (force_copy_on_sdma ? blits_[BlitDevToHost] : - blits_[BlitDevToDev]) : blits_[engine_offset]; + lazy_ptr& blit = blits_[engine_offset]; if (profiling_enabled()) { // Track the agent so we could translate the resulting timestamp to system @@ -1826,6 +1838,12 @@ lazy_ptr& GpuAgent::GetBlitObject(const core::Agent& dst_agent, return blits_[BlitDevToDev]; } + if (core::Runtime::runtime_singleton_->flag().enable_peer_sdma() == Flag::SDMA_DISABLE + && src_agent.device_type() == core::Agent::kAmdGpuDevice + && dst_agent.device_type() == core::Agent::kAmdGpuDevice) { + return blits_[BlitDevToDev]; + } + // Acquire Hive Id of Src and Dst devices - ignore hive id for CPU devices. // CPU-GPU connections should always use the host (aka pcie) facing SDMA engines, even if the // connection is XGMI. diff --git a/runtime/hsa-runtime/core/util/flag.h b/runtime/hsa-runtime/core/util/flag.h index 2f7f726902..507b299014 100644 --- a/runtime/hsa-runtime/core/util/flag.h +++ b/runtime/hsa-runtime/core/util/flag.h @@ -87,6 +87,9 @@ class Flag { var = os::GetEnvVar("HSA_ENABLE_SDMA"); enable_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + var = os::GetEnvVar("HSA_ENABLE_PEER_SDMA"); + enable_peer_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + visible_gpus_ = os::GetEnvVar("ROCR_VISIBLE_DEVICES"); filter_visible_gpus_ = os::IsEnvVarSet("ROCR_VISIBLE_DEVICES"); @@ -225,6 +228,8 @@ class Flag { SDMA_OVERRIDE enable_sdma() const { return enable_sdma_; } + SDMA_OVERRIDE enable_peer_sdma() const { return enable_peer_sdma_; } + std::string visible_gpus() const { return visible_gpus_; } bool filter_visible_gpus() const { return filter_visible_gpus_; } @@ -302,6 +307,7 @@ class Flag { bool enable_mwaitx_; SDMA_OVERRIDE enable_sdma_; + SDMA_OVERRIDE enable_peer_sdma_; bool filter_visible_gpus_; std::string visible_gpus_;