diff --git a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index 1d417b738e..2fec272d58 100644 --- a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -969,6 +969,21 @@ hsa_status_t HSA_API num_dep_signals, dep_signals, completion_signal); } +// Mirrors Amd Extension Apis +hsa_status_t HSA_API + hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent, const void* src, + hsa_agent_t src_agent, size_t size, + uint32_t num_dep_signals, + const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal, + hsa_amd_sdma_engine_id_t engine_id, + bool force_copy_on_sdma) { + return amdExtTable->hsa_amd_memory_async_copy_on_engine_fn( + dst, dst_agent, src, src_agent, size, + num_dep_signals, dep_signals, completion_signal, + engine_id, force_copy_on_sdma); +} + // Mirrors Amd Extension Apis hsa_status_t HSA_API hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent, diff --git a/runtime/hsa-runtime/core/inc/agent.h b/runtime/hsa-runtime/core/inc/agent.h index 5861a6cbcd..d0b675aded 100644 --- a/runtime/hsa-runtime/core/inc/agent.h +++ b/runtime/hsa-runtime/core/inc/agent.h @@ -171,6 +171,27 @@ class Agent : public Checked<0xF6BC25EB17E6F917> { return HSA_STATUS_ERROR; } + // @brief Submit DMA copy command to move data from src to dst on engine_id. + // This call does not wait until the copy is finished + // + // @details All semantics and params are identical to DmaCopy except for engine_id. + // + // @param [in] engine_offset Target engine + // @param [in] force_copy_on_sdma By default, blit kernel copies are used if + // dst_agent == src_agent. Setting this true forces the copy over SDMA1. + // + // + // @retval HSA_STATUS_SUCCESS The memory copy is finished and successful. + virtual hsa_status_t DmaCopyOnEngine(void* dst, core::Agent& dst_agent, + const void* src, core::Agent& src_agent, + size_t size, + std::vector& dep_signals, + core::Signal& out_signal, + int engine_offset, + bool force_copy_on_sdma) { + return HSA_STATUS_ERROR; + } + // @brief Return DMA availability status for copy direction. // // @param [in] dst_agent Destination agent. diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index 4b1e9e181e..938c87ef7f 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -231,6 +231,13 @@ class GpuAgent : public GpuAgentInt { std::vector& dep_signals, core::Signal& out_signal) override; + // @brief Override from core::Agent. + hsa_status_t DmaCopyOnEngine(void* dst, core::Agent& dst_agent, const void* src, + core::Agent& src_agent, size_t size, + std::vector& dep_signals, + core::Signal& out_signal, int engine_offset, + bool force_copy_on_sdma) override; + // @brief Override from core::Agent. hsa_status_t DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent, uint32_t *engine_ids_mask) override; diff --git a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index fc0020e40a..75b5972cdb 100644 --- a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -145,6 +145,16 @@ hsa_status_t const hsa_signal_t* dep_signals, hsa_signal_t completion_signal); +// Mirrors Amd Extension Apis +hsa_status_t + hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent, const void* src, + hsa_agent_t src_agent, size_t size, + uint32_t num_dep_signals, + const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal, + hsa_amd_sdma_engine_id_t engine_id, + bool force_copy_on_sdma); + // Mirrors Amd Extension Apis hsa_status_t hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent, diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index 93fd365dda..680fd833a3 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -229,6 +229,24 @@ class Runtime { core::Agent* src_agent, size_t size, std::vector& dep_signals, core::Signal& completion_signal); + /// @brief Non-blocking memory copy from src to dst on engine_id. + /// + /// @details All semantics and params are dentical to CopyMemory + /// with the exception of engine_id. + /// + /// @param [in] engine_id Target engine to copy on. + /// + /// @param [in] force_copy_on_sdma By default, a blit kernel copy is used + /// when dst_agent == src_agent. Setting this to true will force the copy + /// over SDMA1. + /// + /// @retval ::HSA_STATUS_SUCCESS if copy command has been submitted + /// successfully to the agent DMA queue. + hsa_status_t CopyMemoryOnEngine(void* dst, core::Agent* dst_agent, const void* src, + core::Agent* src_agent, size_t size, + std::vector& dep_signals, core::Signal& completion_signal, + hsa_amd_sdma_engine_id_t engine_id, bool force_copy_on_sdma); + /// @brief Return SDMA availability status for copy direction /// /// @param [in] dst_agent Destination agent. diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 4157fea49f..c4acfa6d02 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -775,6 +775,59 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent, return stat; } +hsa_status_t GpuAgent::DmaCopyOnEngine(void* dst, core::Agent& dst_agent, + const void* src, core::Agent& src_agent, + size_t size, + std::vector& dep_signals, + core::Signal& out_signal, + int engine_offset, + bool force_copy_on_sdma) { + // At this point it is guaranteed that one of + // the two devices is a GPU, potentially both + assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) || + (dst_agent.device_type() == core::Agent::kAmdGpuDevice)) && + ("Both devices are CPU agents which is not expected")); + + if (engine_offset >= properties_.NumSdmaEngines + properties_.NumSdmaXgmiEngines) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + // check if dst and src are the same gpu or over xGMI. + bool is_same_gpu = (src_agent.public_handle().handle == dst_agent.public_handle().handle) && + (dst_agent.public_handle().handle == public_handle_.handle); + bool is_xgmi = !is_same_gpu && + src_agent.device_type() == core::Agent::kAmdGpuDevice && + dst_agent.device_type() == core::Agent::kAmdGpuDevice && + dst_agent.HiveId() && src_agent.HiveId() == dst_agent.HiveId() && + properties_.NumSdmaXgmiEngines; + + // Due to a RAS issue, GFX90a can only support H2D copies on SDMA0 + bool is_h2d_blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice && + dst_agent.device_type() == core::Agent::kAmdGpuDevice); + bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10); + + // Ensure engine selection is within proper range based on transfer type + if ((is_xgmi && engine_offset < properties_.NumSdmaEngines) || + (!is_xgmi && engine_offset >= properties_.NumSdmaEngines) || + (!is_h2d_blit && !is_same_gpu && limit_h2d_blit && !engine_offset)) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + lazy_ptr& blit = is_same_gpu ? + (force_copy_on_sdma ? blits_[BlitDevToHost] : + blits_[BlitDevToDev]) : blits_[engine_offset]; + + if (profiling_enabled()) { + // Track the agent so we could translate the resulting timestamp to system + // domain correctly. + out_signal.async_copy_agent(core::Agent::Convert(this->public_handle())); + } + + hsa_status_t stat = blit->SubmitLinearCopyCommand(dst, src, size, dep_signals, out_signal); + + return stat; +} + hsa_status_t GpuAgent::DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent, uint32_t *engine_ids_mask) { assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) || diff --git a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index d7dca6fc16..5f81eec690 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -365,6 +365,7 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_memory_pool_allocate_fn = AMD::hsa_amd_memory_pool_allocate; amd_ext_api.hsa_amd_memory_pool_free_fn = AMD::hsa_amd_memory_pool_free; amd_ext_api.hsa_amd_memory_async_copy_fn = AMD::hsa_amd_memory_async_copy; + amd_ext_api.hsa_amd_memory_async_copy_on_engine_fn = AMD::hsa_amd_memory_async_copy_on_engine; amd_ext_api.hsa_amd_memory_copy_engine_status_fn = AMD::hsa_amd_memory_copy_engine_status; amd_ext_api.hsa_amd_agent_memory_pool_get_info_fn = AMD::hsa_amd_agent_memory_pool_get_info; amd_ext_api.hsa_amd_agents_allow_access_fn = AMD::hsa_amd_agents_allow_access; diff --git a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index f91edef7f4..dbd5827945 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -276,6 +276,50 @@ hsa_status_t hsa_amd_memory_async_copy(void* dst, hsa_agent_t dst_agent_handle, CATCH; } +hsa_status_t hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent_handle, + const void* src, hsa_agent_t src_agent_handle, size_t size, + uint32_t num_dep_signals, const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal, hsa_amd_sdma_engine_id_t engine_id, + bool force_copy_on_sdma) { + TRY; + IS_BAD_PTR(dst); + IS_BAD_PTR(src); + + if ((num_dep_signals == 0 && dep_signals != nullptr) || + (num_dep_signals > 0 && dep_signals == nullptr)) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle); + IS_VALID(dst_agent); + + core::Agent* src_agent = core::Agent::Convert(src_agent_handle); + IS_VALID(src_agent); + + std::vector dep_signal_list(num_dep_signals); + if (num_dep_signals > 0) { + for (size_t i = 0; i < num_dep_signals; ++i) { + core::Signal* dep_signal_obj = core::Signal::Convert(dep_signals[i]); + IS_VALID(dep_signal_obj); + dep_signal_list[i] = dep_signal_obj; + } + } + + core::Signal* out_signal_obj = core::Signal::Convert(completion_signal); + IS_VALID(out_signal_obj); + + bool rev_copy_dir = core::Runtime::runtime_singleton_->flag().rev_copy_dir(); + if (size > 0) { + return core::Runtime::runtime_singleton_->CopyMemoryOnEngine( + dst, (rev_copy_dir ? src_agent : dst_agent), + src, (rev_copy_dir ? dst_agent : src_agent), + size, dep_signal_list, *out_signal_obj, engine_id, force_copy_on_sdma); + } + + return HSA_STATUS_SUCCESS; + CATCH; +} + hsa_status_t hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent_handle, hsa_agent_t src_agent_handle, uint32_t *engine_ids_mask) { core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle); diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 3a89c35efd..283c6cc0a7 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -501,6 +501,24 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent* dst_agent, const void* completion_signal); } +hsa_status_t Runtime::CopyMemoryOnEngine(void* dst, core::Agent* dst_agent, const void* src, + core::Agent* src_agent, size_t size, + std::vector& dep_signals, + core::Signal& completion_signal, + hsa_amd_sdma_engine_id_t engine_id, bool force_copy_on_sdma) { + const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice); + core::Agent* copy_agent = (src_gpu) ? src_agent : dst_agent; + + // engine_id is single bitset unique. + int engine_offset = ffs(engine_id) - 1; + if (!engine_id || !!((engine_id >> (engine_offset + 1)))) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + return copy_agent->DmaCopyOnEngine(dst, *dst_agent, src, *src_agent, size, dep_signals, + completion_signal, engine_offset, force_copy_on_sdma); +} + hsa_status_t Runtime::CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent, uint32_t *engine_ids_mask) { const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice); diff --git a/runtime/hsa-runtime/hsacore.so.def b/runtime/hsa-runtime/hsacore.so.def index 8691ce25bc..07f4e93749 100644 --- a/runtime/hsa-runtime/hsacore.so.def +++ b/runtime/hsa-runtime/hsacore.so.def @@ -182,6 +182,7 @@ global: hsa_amd_queue_cu_get_mask; hsa_amd_memory_fill; hsa_amd_memory_async_copy; + hsa_amd_memory_async_copy_on_engine; hsa_amd_memory_copy_engine_status; hsa_amd_memory_async_copy_rect; hsa_amd_memory_lock; diff --git a/runtime/hsa-runtime/inc/hsa_api_trace.h b/runtime/hsa-runtime/inc/hsa_api_trace.h index 4a57e22a94..3954b51a31 100644 --- a/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -155,6 +155,7 @@ struct AmdExtTable { decltype(hsa_amd_memory_pool_allocate)* hsa_amd_memory_pool_allocate_fn; decltype(hsa_amd_memory_pool_free)* hsa_amd_memory_pool_free_fn; decltype(hsa_amd_memory_async_copy)* hsa_amd_memory_async_copy_fn; + decltype(hsa_amd_memory_async_copy_on_engine)* hsa_amd_memory_async_copy_on_engine_fn; decltype(hsa_amd_memory_copy_engine_status)* hsa_amd_memory_copy_engine_status_fn; decltype(hsa_amd_agent_memory_pool_get_info)* hsa_amd_agent_memory_pool_get_info_fn; decltype(hsa_amd_agents_allow_access)* hsa_amd_agents_allow_access_fn; diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index 39332b36f0..a094001c2b 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -1273,6 +1273,41 @@ hsa_status_t HSA_API uint32_t num_dep_signals, const hsa_signal_t* dep_signals, hsa_signal_t completion_signal); + +/** + * @brief Asynchronously copy a block of memory from the location pointed to by + * @p src on the @p src_agent to the memory block pointed to by @p dst on the @p + * dst_agent on engine_id. + * + * WARNING: Concurrent use of this call with hsa_amd_memory_async_copy can result + * in resource conflicts as HSA runtime will auto assign engines with the latter + * call. Approach using both calls concurrently with caution. + * + * All param definitions are identical to hsa_amd_memory_async_copy with the + * exception of engine_id and force_copy_on_sdma. + * + * @param[in] - engine_id Target engine defined by hsa_amd_sdma_engine_id_t. + * Client should use hsa_amd_memory_copy_engine_status first to get the ID + * availability. + * + * @param[in] - force_copy_on_sdma By default, blit kernel copies are used when + * dst_agent == src_agent. Setting this to true will force the copy over SDMA1. + * + * All return definitions are identical to hsa_amd_memory_async_copy with the + * following ammendments: + * + * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT The source or destination + * pointers are NULL, or the completion signal is 0 or engine_id is improperly + * bounded. + */ +hsa_status_t HSA_API + hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent, const void* src, + hsa_agent_t src_agent, size_t size, + uint32_t num_dep_signals, + const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal, + hsa_amd_sdma_engine_id_t engine_id, + bool force_copy_on_sdma); /** * @brief Reports the availability of SDMA copy engines. *