diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h index 672a03c95e..4eea7c746d 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h @@ -65,7 +65,8 @@ class BlitSdmaBase : public core::Blit { static const size_t kMaxSingleCopySize; static const size_t kMaxSingleFillSize; virtual bool isSDMA() const override { return true; } - virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi) = 0; + virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi, + size_t linear_copy_size_override) = 0; virtual hsa_status_t SubmitCopyRectCommand(const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, @@ -90,7 +91,8 @@ class BlitSdma : public BlitSdmaBase { /// @param agent Pointer to the agent that will execute the PM4 commands. /// /// @return hsa_status_t - virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi) override; + virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi, + size_t linear_copy_size_override) override; /// @brief Marks the queue object as invalid and uncouples its link with /// the underlying compute device's control block. Use of queue object diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/sdma_registers.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/sdma_registers.h index 254744f6a8..d94eed43fd 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/sdma_registers.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/sdma_registers.h @@ -80,7 +80,11 @@ typedef struct SDMA_PKT_COPY_LINEAR_TAG { struct { unsigned int count : 22; unsigned int reserved_0 : 10; - }; + } count; + struct { + unsigned int count : 30; + unsigned int reserved_0 : 2; + } count_ext; unsigned int DW_1_DATA; } COUNT_UNION; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index 830cb98929..ff33788fe2 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -131,7 +131,7 @@ BlitSdma::~BlitSdma() template hsa_status_t BlitSdma::Initialize( - const core::Agent& agent, bool use_xgmi) { + const core::Agent& agent, bool use_xgmi, size_t linear_copy_size_override) { if (queue_start_addr_ != NULL) { // Already initialized. return HSA_STATUS_SUCCESS; @@ -201,6 +201,8 @@ hsa_status_t BlitSdma: signals_[0].reset(new core::InterruptSignal(0)); signals_[1].reset(new core::InterruptSignal(0)); + max_single_linear_copy_size_ = linear_copy_size_override; + cleanupOnException.Dismiss(); return HSA_STATUS_SUCCESS; } @@ -492,7 +494,9 @@ hsa_status_t BlitSdma::SubmitLinearCopyCommand(void* dst, const void* src, size_t size) { // Break the copy into multiple copy operation incase the copy size exceeds // the SDMA linear copy limit. - const uint32_t num_copy_command = (size + kMaxSingleCopySize - 1) / kMaxSingleCopySize; + const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ : + kMaxSingleCopySize; + const uint32_t num_copy_command = (size + max_copy_size - 1) / max_copy_size; std::vector buff(num_copy_command); BuildCopyCommand(reinterpret_cast(&buff[0]), num_copy_command, dst, src, size); @@ -508,7 +512,9 @@ hsa_status_t BlitSdma& gang_signals) { // Break the copy into multiple copy operations when the copy size exceeds // the SDMA linear copy limit. - const uint32_t num_copy_command = (size + kMaxSingleCopySize - 1) / kMaxSingleCopySize; + const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ : + kMaxSingleCopySize; + const uint32_t num_copy_command = (size + max_copy_size - 1) / max_copy_size; // Assemble copy packets. std::vector buff(num_copy_command); @@ -774,9 +780,11 @@ template ::BuildCopyCommand( char* cmd_addr, uint32_t num_copy_command, void* dst, const void* src, size_t size) { size_t cur_size = 0; + const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ : + kMaxSingleCopySize; for (uint32_t i = 0; i < num_copy_command; ++i) { const uint32_t copy_size = - static_cast(std::min((size - cur_size), kMaxSingleCopySize)); + static_cast(std::min((size - cur_size), max_copy_size)); void* cur_dst = static_cast(dst) + cur_size; const void* cur_src = static_cast(src) + cur_size; @@ -789,7 +797,10 @@ void BlitSdma::BuildCo packet_addr->HEADER_UNION.op = SDMA_OP_COPY; packet_addr->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR; - packet_addr->COUNT_UNION.count = copy_size + SizeToCountOffset; + if (max_copy_size == (1 << 30) -1) + packet_addr->COUNT_UNION.count_ext.count = copy_size + SizeToCountOffset; + else + packet_addr->COUNT_UNION.count.count = copy_size + SizeToCountOffset; packet_addr->SRC_ADDR_LO_UNION.src_addr_31_0 = ptrlow32(cur_src); packet_addr->SRC_ADDR_HI_UNION.src_addr_63_32 = ptrhigh32(cur_src); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index f5d276f0be..ec1875c0a6 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -661,6 +661,8 @@ core::Queue* GpuAgent::CreateInterceptibleQueue(void (*callback)(hsa_status_t st core::Blit* GpuAgent::CreateBlitSdma(bool use_xgmi) { AMD::BlitSdmaBase* sdma; + size_t copy_size_override = 0; + const size_t copy_size_overrides[2] = {0x3fffff, 0x3fffffff}; switch (isa_->GetMajorVersion()) { case 7: @@ -669,19 +671,29 @@ core::Blit* GpuAgent::CreateBlitSdma(bool use_xgmi) { break; case 9: sdma = new BlitSdmaV4(); + copy_size_override = (isa_->GetMinorVersion() == 0 && isa_->GetStepping() == 10) || + isa_->GetMinorVersion() > 0 ? copy_size_overrides[1] : + copy_size_overrides[0]; break; case 10: sdma = new BlitSdmaV5(); + copy_size_override = isa_->GetMinorVersion() < 3 ? copy_size_overrides[0] : + copy_size_overrides[1]; break; case 11: sdma = new BlitSdmaV5(); + copy_size_override = copy_size_overrides[1]; break; default: assert(false && "Unexpected device major version."); return nullptr; } - if (sdma->Initialize(*this, use_xgmi) != HSA_STATUS_SUCCESS) { + Flag::SDMA_OVERRIDE copy_size_override_setting = + core::Runtime::runtime_singleton_->flag().enable_sdma_copy_size_override(); + if (copy_size_override_setting == Flag::SDMA_DISABLE) copy_size_override = 0; + + if (sdma->Initialize(*this, use_xgmi, copy_size_override) != HSA_STATUS_SUCCESS) { sdma->Destroy(*this); delete sdma; sdma = nullptr; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h index a75e1ccb85..bd18f42559 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h @@ -94,6 +94,10 @@ class Flag { enable_sdma_gang_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + var = os::GetEnvVar("HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE"); + enable_sdma_copy_size_override_ = (var == "0") ? SDMA_DISABLE : + ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + visible_gpus_ = os::GetEnvVar("ROCR_VISIBLE_DEVICES"); filter_visible_gpus_ = os::IsEnvVarSet("ROCR_VISIBLE_DEVICES"); @@ -236,6 +240,8 @@ class Flag { SDMA_OVERRIDE enable_sdma_gang() const { return enable_sdma_gang_; } + SDMA_OVERRIDE enable_sdma_copy_size_override() const { return enable_sdma_copy_size_override_; } + std::string visible_gpus() const { return visible_gpus_; } bool filter_visible_gpus() const { return filter_visible_gpus_; } @@ -315,6 +321,7 @@ class Flag { SDMA_OVERRIDE enable_sdma_; SDMA_OVERRIDE enable_peer_sdma_; SDMA_OVERRIDE enable_sdma_gang_; + SDMA_OVERRIDE enable_sdma_copy_size_override_; bool filter_visible_gpus_; std::string visible_gpus_;