diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h index fb0eb1abf8..463a83c34b 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h @@ -69,7 +69,7 @@ class BlitSdmaBase : public core::Blit { template class BlitSdma : public BlitSdmaBase { public: - explicit BlitSdma(); + explicit BlitSdma(bool copy_direction); virtual ~BlitSdma() override; @@ -125,7 +125,7 @@ class BlitSdma : public BlitSdmaBase { virtual hsa_status_t EnableProfiling(bool enable) override; - protected: + private: /// @brief Acquires the address into queue buffer where a new command /// packet of specified size could be written. The address that is /// returned is guaranteed to be unique even in a multi-threaded access @@ -170,6 +170,9 @@ class BlitSdma : public BlitSdmaBase { void BuildFenceCommand(char* fence_command_addr, uint32_t* fence, uint32_t fence_value); + /// @brief Build Hdp Flush command + void BuildHdpFlushCommand(char* cmd_addr); + uint32_t* ObtainFenceObject(); void WaitFence(uint32_t* fence, uint32_t fence_value); @@ -204,19 +207,25 @@ class BlitSdma : public BlitSdmaBase { RingIndexTy cached_reserve_index_; RingIndexTy cached_commit_index_; - uint32_t linear_copy_command_size_; + static const uint32_t linear_copy_command_size_; - uint32_t fill_command_size_; + static const uint32_t fill_command_size_; - uint32_t fence_command_size_; + static const uint32_t fence_command_size_; - uint32_t poll_command_size_; + static const uint32_t poll_command_size_; - uint32_t atomic_command_size_; + static const uint32_t flush_command_size_; - uint32_t timestamp_command_size_; + static const uint32_t atomic_command_size_; - uint32_t trap_command_size_; + static const uint32_t timestamp_command_size_; + + static const uint32_t trap_command_size_; + + // Flag to indicate if sDMA queue is used for H2D copy operations + // true if used for H2D operations, false otherwise + const bool sdma_h2d_; // Max copy size of a single linear copy command packet. size_t max_single_linear_copy_size_; @@ -232,19 +241,20 @@ class BlitSdma : public BlitSdmaBase { /// True if platform atomic is supported. bool platform_atomic_support_; + + /// True if sDMA supports HDP flush + bool hdp_flush_support_; }; -class BlitSdmaV2V3 - // Ring indices are 32-bit. - // HW ring indices are not monotonic (wrap at end of ring). - // Count fields of SDMA commands are 0-based. - : public BlitSdma {}; +// Ring indices are 32-bit. +// HW ring indices are not monotonic (wrap at end of ring). +// Count fields of SDMA commands are 0-based. +typedef BlitSdma BlitSdmaV2V3; -class BlitSdmaV4 - // Ring indices are 64-bit. - // HW ring indices are monotonic (do not wrap at end of ring). - // Count fields of SDMA commands are 1-based. - : public BlitSdma {}; +// Ring indices are 64-bit. +// HW ring indices are monotonic (do not wrap at end of ring). +// Count fields of SDMA commands are 1-based. +typedef BlitSdma BlitSdmaV4; } // namespace amd diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index fd5c8f8d56..3e36be0f52 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -192,6 +192,8 @@ class GpuAgent : public GpuAgentInt { uint16_t GetMicrocodeVersion() const; + uint16_t GetSdmaMicrocodeVersion() const; + // @brief Assembles SP3 shader source into ISA or AQL code object. // // @param [in] src_sp3 SP3 shader source text representation. @@ -325,7 +327,7 @@ class GpuAgent : public GpuAgentInt { // @brief Create SDMA blit object. // // @retval NULL if SDMA blit creation and initialization failed. - core::Blit* CreateBlitSdma(); + core::Blit* CreateBlitSdma(bool h2d); // @brief Create Kernel blit object using provided compute queue. // diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index 2ea06a1a15..bab08c6598 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -360,6 +360,20 @@ typedef struct SDMA_PKT_TRAP_TAG { } INT_CONTEXT_UNION; } SDMA_PKT_TRAP; +// Initialize Hdp flush packet for use on sDMA of devices +// from Gfx9 or new family +static const SDMA_PKT_POLL_REGMEM hdp_flush_cmd_ { + { SDMA_OP_POLL_REGMEM }, + { 0x00 }, + { 0x80000000 }, + { 0x00 }, + { 0x00 }, + { 0x00 }, +}; + +// Version of sDMA microcode supporting Hdp flush +static const uint16_t sdma_version_ = 0x01A5; + inline uint32_t ptrlow32(const void* p) { return static_cast(reinterpret_cast(p)); } @@ -377,8 +391,33 @@ const size_t BlitSdmaBase::kCopyPacketSize = sizeof(SDMA_PKT_COPY_LINEAR); const size_t BlitSdmaBase::kMaxSingleCopySize = 0x3fffe0; // From HW documentation const size_t BlitSdmaBase::kMaxSingleFillSize = 0x3fffe0; +// Initialize size of various sDMA commands use by this module template -BlitSdma::BlitSdma() +const uint32_t BlitSdma::linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR); + +template +const uint32_t BlitSdma::fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL); + +template +const uint32_t BlitSdma::fence_command_size_ = sizeof(SDMA_PKT_FENCE); + +template +const uint32_t BlitSdma::poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); + +template +const uint32_t BlitSdma::flush_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); + +template +const uint32_t BlitSdma::atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC); + +template +const uint32_t BlitSdma::timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP); + +template +const uint32_t BlitSdma::trap_command_size_ = sizeof(SDMA_PKT_TRAP); + +template +BlitSdma::BlitSdma(bool copy_direction) : agent_(NULL), queue_start_addr_(NULL), fence_base_addr_(NULL), @@ -386,7 +425,9 @@ BlitSdma::BlitSdma() fence_pool_counter_(0), cached_reserve_index_(0), cached_commit_index_(0), - platform_atomic_support_(true) { + sdma_h2d_(copy_direction), + platform_atomic_support_(true), + hdp_flush_support_(false) { std::memset(&queue_resource_, 0, sizeof(queue_resource_)); } @@ -407,14 +448,6 @@ hsa_status_t BlitSdma::Initial return HSA_STATUS_ERROR; } - linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR); - fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL); - fence_command_size_ = sizeof(SDMA_PKT_FENCE); - poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); - atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC); - timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP); - trap_command_size_ = sizeof(SDMA_PKT_TRAP); - const amd::GpuAgentInt& amd_gpu_agent = static_cast(agent); @@ -428,6 +461,11 @@ hsa_status_t BlitSdma::Initial platform_atomic_support_ = false; } + // Determine if sDMA microcode supports HDP flush command + if (agent_->GetSdmaMicrocodeVersion() >= sdma_version_) { + hdp_flush_support_ = true; + } + // Allocate queue buffer. queue_start_addr_ = (char*)core::Runtime::runtime_singleton_->system_allocator()( kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable); @@ -508,8 +546,16 @@ hsa_status_t BlitSdma::SubmitL const uint32_t total_copy_command_size = num_copy_command * linear_copy_command_size_; + // Add space for acquire or release Hdp flush command + uint32_t flush_cmd_size = 0; + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if (HwIndexMonotonic) { + flush_cmd_size = flush_command_size_; + } + } + const uint32_t total_command_size = - total_copy_command_size + fence_command_size_; + total_copy_command_size + fence_command_size_ + flush_cmd_size; const uint32_t kFenceValue = 2015; uint32_t* fence_addr = ObtainFenceObject(); @@ -522,10 +568,25 @@ hsa_status_t BlitSdma::SubmitL return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } - BuildCopyCommand(command_addr, num_copy_command, dst, src, size); + // Determine if a Hdp flush cmd is required at the top of cmd stream + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_ == false)) { + BuildHdpFlushCommand(command_addr); + command_addr += flush_command_size_; + } + } + BuildCopyCommand(command_addr, num_copy_command, dst, src, size); command_addr += total_copy_command_size; + // Determine if a Hdp flush cmd is required at the end of cmd stream + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_)) { + BuildHdpFlushCommand(command_addr); + command_addr += flush_command_size_; + } + } + BuildFenceCommand(command_addr, fence_addr, kFenceValue); ReleaseWriteAddress(curr_index, total_command_size); @@ -593,9 +654,17 @@ hsa_status_t BlitSdma::SubmitL ? (fence_command_size_ + trap_command_size_) : 0; + // Add space for acquire or release Hdp flush command + uint32_t flush_cmd_size = 0; + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if (HwIndexMonotonic) { + flush_cmd_size = flush_command_size_; + } + } + const uint32_t total_command_size = total_poll_command_size + total_copy_command_size + sync_command_size + - total_timestamp_command_size + interrupt_command_size; + total_timestamp_command_size + interrupt_command_size + flush_cmd_size; RingIndexTy curr_index; char* command_addr = AcquireWriteAddress(total_command_size, curr_index); @@ -621,11 +690,26 @@ hsa_status_t BlitSdma::SubmitL command_addr += timestamp_command_size_; } + // Determine if a Hdp flush cmd is required at the top of cmd stream + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_ == false)) { + BuildHdpFlushCommand(command_addr); + command_addr += flush_command_size_; + } + } + // Do the transfer after all polls are satisfied. BuildCopyCommand(command_addr, num_copy_command, dst, src, size); - command_addr += total_copy_command_size; + // Determine if a Hdp flush cmd is required at the end of cmd stream + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_)) { + BuildHdpFlushCommand(command_addr); + command_addr += flush_command_size_; + } + } + if (profiling_enabled) { assert(IsMultipleOf(end_ts_addr, 32)); BuildGetGlobalTimestampCommand(command_addr, @@ -685,8 +769,16 @@ hsa_status_t BlitSdma::SubmitL const uint32_t total_fill_command_size = num_fill_command * fill_command_size_; + // Add space for acquire or release Hdp flush command + uint32_t flush_cmd_size = 0; + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if (HwIndexMonotonic) { + flush_cmd_size = flush_command_size_; + } + } + const uint32_t total_command_size = - total_fill_command_size + fence_command_size_; + total_fill_command_size + fence_command_size_ + flush_cmd_size; RingIndexTy curr_index; char* command_addr = AcquireWriteAddress(total_command_size, curr_index); @@ -724,6 +816,14 @@ hsa_status_t BlitSdma::SubmitL assert(cur_size == size); + // Determine if a Hdp flush cmd is required at the end of cmd stream + if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { + if ((HwIndexMonotonic) && (hdp_flush_support_)) { + BuildHdpFlushCommand(command_addr); + command_addr += flush_command_size_; + } + } + const uint32_t kFenceValue = 2015; uint32_t* fence_addr = ObtainFenceObject(); *fence_addr = 0; @@ -1018,6 +1118,14 @@ void BlitSdma::BuildTrapComman packet_addr->HEADER_UNION.op = SDMA_OP_TRAP; } +template +void BlitSdma::BuildHdpFlushCommand( + char* cmd_addr) { + assert(cmd_addr != NULL); + SDMA_PKT_POLL_REGMEM* addr = reinterpret_cast(cmd_addr); + memcpy(addr, &hdp_flush_cmd_, flush_command_size_); +} + template class BlitSdma; template class BlitSdma; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 413f4e49c1..83a69520e3 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -515,13 +515,13 @@ core::Queue* GpuAgent::CreateInterceptibleQueue() { return queue; } -core::Blit* GpuAgent::CreateBlitSdma() { +core::Blit* GpuAgent::CreateBlitSdma(bool h2d) { core::Blit* sdma; if (isa_->GetMajorVersion() <= 8) { - sdma = new BlitSdmaV2V3; + sdma = new BlitSdmaV2V3(h2d); } else { - sdma = new BlitSdmaV4; + sdma = new BlitSdmaV4(h2d); } if (sdma->Initialize(*this) != HSA_STATUS_SUCCESS) { @@ -561,10 +561,10 @@ void GpuAgent::InitDma() { // Blits, try create SDMA blit first. // Disable SDMA on specific ISA targets until they are fully qualified. - auto blit_lambda = [this](lazy_ptr& queue) { + auto blit_lambda = [this](bool h2d, lazy_ptr& queue) { if ((isa_->GetMajorVersion() != 8) && core::Runtime::runtime_singleton_->flag().enable_sdma() && (HSA_PROFILE_BASE == profile_)) { - auto ret = CreateBlitSdma(); + auto ret = CreateBlitSdma(h2d); if (ret != nullptr) return ret; } auto ret = CreateBlitKernel((*queue).get()); @@ -573,8 +573,8 @@ void GpuAgent::InitDma() { return ret; }; - blits_[BlitHostToDev].reset([blit_lambda, this]() { return blit_lambda(queues_[QueueBlitOnly]); }); - blits_[BlitDevToHost].reset([blit_lambda, this]() { return blit_lambda(queues_[QueueUtility]); }); + blits_[BlitHostToDev].reset([blit_lambda, this]() { return blit_lambda(true, queues_[QueueBlitOnly]); }); + blits_[BlitDevToHost].reset([blit_lambda, this]() { return blit_lambda(false, queues_[QueueUtility]); }); blits_[BlitDevToDev].reset([this]() { auto ret = CreateBlitKernel((*queues_[QueueUtility]).get()); if (ret == nullptr) @@ -614,7 +614,8 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent, : (src_agent.device_type() == core::Agent::kAmdGpuDevice && dst_agent.device_type() == core::Agent::kAmdCpuDevice) ? blits_[BlitDevToHost] - : blits_[BlitDevToDev]; + : (src_agent.node_id() == dst_agent.node_id()) + ? blits_[BlitDevToDev] : blits_[BlitDevToHost]; if (profiling_enabled()) { // Track the agent so we could translate the resulting timestamp to system @@ -1063,6 +1064,10 @@ uint16_t GpuAgent::GetMicrocodeVersion() const { return (properties_.EngineId.ui32.uCode); } +uint16_t GpuAgent::GetSdmaMicrocodeVersion() const { + return (properties_.uCodeEngineVersions.uCodeSDMA); +} + void GpuAgent::SyncClocks() { HSAKMT_STATUS err = hsaKmtGetClockCounters(node_id(), &t1_); assert(err == HSAKMT_STATUS_SUCCESS && "hsaGetClockCounters error"); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h index 1de3b15abc..e9049f36b0 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h @@ -91,18 +91,23 @@ class Flag { var = os::GetEnvVar("HSA_DISABLE_FRAGMENT_ALLOCATOR"); disable_fragment_alloc_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_ENABLE_SDMA_HDP_FLUSH"); + enable_sdma_hdp_flush_ = (var == "0") ? false : true; } bool check_flat_scratch() const { return check_flat_scratch_; } bool enable_vm_fault_message() const { return enable_vm_fault_message_; } - + bool enable_queue_fault_message() const { return enable_queue_fault_message_; } bool enable_interrupt() const { return enable_interrupt_; } bool enable_sdma() const { return enable_sdma_; } + bool enable_sdma_hdp_flush() const { return enable_sdma_hdp_flush_; } + bool running_valgrind() const { return running_valgrind_; } bool sdma_wait_idle() const { return sdma_wait_idle_; } @@ -122,6 +127,7 @@ class Flag { bool enable_vm_fault_message_; bool enable_interrupt_; bool enable_sdma_; + bool enable_sdma_hdp_flush_; bool running_valgrind_; bool sdma_wait_idle_; bool enable_queue_fault_message_;