From 8f27f495c65785cd86e3b8325de480e144350041 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 1 Feb 2023 12:52:00 -0500 Subject: [PATCH] Make SDMA engine availability status queryable. Report the availability of SDMA engines for memory copies. Change-Id: Ie31b02d6b65355122bb8c98bc73700a59bee166e --- .../core/common/hsa_table_interface.cpp | 7 ++++ runtime/hsa-runtime/core/inc/agent.h | 13 +++++++ runtime/hsa-runtime/core/inc/amd_gpu_agent.h | 4 ++ .../hsa-runtime/core/inc/hsa_ext_amd_impl.h | 5 +++ runtime/hsa-runtime/core/inc/runtime.h | 11 ++++++ .../core/runtime/amd_gpu_agent.cpp | 37 +++++++++++++++++++ .../core/runtime/hsa_api_trace.cpp | 1 + .../hsa-runtime/core/runtime/hsa_ext_amd.cpp | 11 ++++++ runtime/hsa-runtime/core/runtime/runtime.cpp | 14 ++++++- runtime/hsa-runtime/hsacore.so.def | 1 + runtime/hsa-runtime/inc/hsa_api_trace.h | 1 + runtime/hsa-runtime/inc/hsa_ext_amd.h | 34 +++++++++++++++++ 12 files changed, 138 insertions(+), 1 deletion(-) diff --git a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index e438314d32..1d417b738e 100644 --- a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -969,6 +969,13 @@ hsa_status_t HSA_API num_dep_signals, dep_signals, completion_signal); } +// Mirrors Amd Extension Apis +hsa_status_t HSA_API + hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent, + uint32_t *engine_ids_mask) { + return amdExtTable->hsa_amd_memory_copy_engine_status_fn(dst_agent, src_agent, engine_ids_mask); +} + // Mirrors Amd Extension Apis hsa_status_t HSA_API hsa_amd_memory_async_copy_rect( const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, diff --git a/runtime/hsa-runtime/core/inc/agent.h b/runtime/hsa-runtime/core/inc/agent.h index d4dac6f2e3..5861a6cbcd 100644 --- a/runtime/hsa-runtime/core/inc/agent.h +++ b/runtime/hsa-runtime/core/inc/agent.h @@ -171,6 +171,19 @@ class Agent : public Checked<0xF6BC25EB17E6F917> { return HSA_STATUS_ERROR; } + // @brief Return DMA availability status for copy direction. + // + // @param [in] dst_agent Destination agent. + // @param [in] src_agent Source agent. + // @param [out] engine_ids_mask Mask of engine ids. + // + // @retval HSA_STATUS_SUCCESS DMA engines are available + // @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES DMA engines are not available + virtual hsa_status_t DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent, + uint32_t *engine_ids_mask) { + return HSA_STATUS_ERROR; + } + // @brief Submit DMA command to set the content of a pointer and wait // until it is finished. // diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index 1ccb5a0bb3..4b1e9e181e 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -231,6 +231,10 @@ class GpuAgent : public GpuAgentInt { std::vector& dep_signals, core::Signal& out_signal) override; + // @brief Override from core::Agent. + hsa_status_t DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent, + uint32_t *engine_ids_mask) override; + // @brief Override from core::Agent. hsa_status_t DmaCopyRect(const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, const hsa_dim3_t* src_offset, diff --git a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index a0ab357e01..fc0020e40a 100644 --- a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -145,6 +145,11 @@ hsa_status_t const hsa_signal_t* dep_signals, hsa_signal_t completion_signal); +// Mirrors Amd Extension Apis +hsa_status_t + hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent, + uint32_t *engine_ids_mask); + // Mirrors Amd Extension Apis hsa_status_t hsa_amd_memory_async_copy_rect( const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index f612f2c534..93fd365dda 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -229,6 +229,17 @@ class Runtime { core::Agent* src_agent, size_t size, std::vector& dep_signals, core::Signal& completion_signal); + /// @brief Return SDMA availability status for copy direction + /// + /// @param [in] dst_agent Destination agent. + /// @param [in] src_agent Source agent. + /// @param [out] engine_ids_mask Mask of engine_ids. + /// + /// @retval HSA_STATUS_SUCCESS DMA engines are available + /// @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES DMA engines are not available + hsa_status_t CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent, + uint32_t *engine_ids_mask); + /// @brief Fill the first @p count of uint32_t in ptr with value. /// /// @param [in] ptr Memory address to be filled. diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index a240b74680..4157fea49f 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -775,6 +775,43 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent, return stat; } +hsa_status_t GpuAgent::DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent, + uint32_t *engine_ids_mask) { + assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) || + (dst_agent.device_type() == core::Agent::kAmdGpuDevice)) && + ("Both devices are CPU agents which is not expected")); + + *engine_ids_mask = 0; + if (src_agent.device_type() == core::Agent::kAmdGpuDevice && + dst_agent.device_type() == core::Agent::kAmdGpuDevice && + dst_agent.HiveId() && src_agent.HiveId() == dst_agent.HiveId() && + properties_.NumSdmaXgmiEngines) { + // Find a free xGMI SDMA engine + for (int i = 0; i < properties_.NumSdmaXgmiEngines; i++) { + if (!!!blits_[DefaultBlitCount + i]->PendingBytes()) { + *engine_ids_mask |= (HSA_AMD_SDMA_ENGINE_2 << i); + } + } + } else { + bool is_h2d_blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice && + dst_agent.device_type() == core::Agent::kAmdGpuDevice); + // Due to a RAS issue, GFX90a can only support H2D copies on SDMA0 + bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10); + + if (!!!blits_[BlitHostToDev]->PendingBytes()) { + if (is_h2d_blit || !limit_h2d_blit) { + *engine_ids_mask |= HSA_AMD_SDMA_ENGINE_0; + } + } + + if (!!!blits_[BlitDevToHost]->PendingBytes()) { + *engine_ids_mask |= HSA_AMD_SDMA_ENGINE_1; + } + } + + return !!(*engine_ids_mask) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR_OUT_OF_RESOURCES; +} + hsa_status_t GpuAgent::DmaCopyRect(const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, const hsa_dim3_t* src_offset, const hsa_dim3_t* range, hsa_amd_copy_direction_t dir, diff --git a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index 7f71adcdf5..d7dca6fc16 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -365,6 +365,7 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_memory_pool_allocate_fn = AMD::hsa_amd_memory_pool_allocate; amd_ext_api.hsa_amd_memory_pool_free_fn = AMD::hsa_amd_memory_pool_free; amd_ext_api.hsa_amd_memory_async_copy_fn = AMD::hsa_amd_memory_async_copy; + amd_ext_api.hsa_amd_memory_copy_engine_status_fn = AMD::hsa_amd_memory_copy_engine_status; amd_ext_api.hsa_amd_agent_memory_pool_get_info_fn = AMD::hsa_amd_agent_memory_pool_get_info; amd_ext_api.hsa_amd_agents_allow_access_fn = AMD::hsa_amd_agents_allow_access; amd_ext_api.hsa_amd_memory_pool_can_migrate_fn = AMD::hsa_amd_memory_pool_can_migrate; diff --git a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index 162041c388..f91edef7f4 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -276,6 +276,17 @@ hsa_status_t hsa_amd_memory_async_copy(void* dst, hsa_agent_t dst_agent_handle, CATCH; } +hsa_status_t hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent_handle, hsa_agent_t src_agent_handle, + uint32_t *engine_ids_mask) { + core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle); + IS_VALID(dst_agent); + + core::Agent* src_agent = core::Agent::Convert(src_agent_handle); + IS_VALID(src_agent); + + return core::Runtime::runtime_singleton_->CopyMemoryStatus(dst_agent, src_agent, engine_ids_mask); +} + hsa_status_t hsa_amd_memory_async_copy_rect( const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, const hsa_dim3_t* src_offset, const hsa_dim3_t* range, hsa_agent_t copy_agent, diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 61cdb04786..3a89c35efd 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -459,7 +459,7 @@ hsa_status_t Runtime::CopyMemory(void* dst, const void* src, size_t size) { /* GPU-GPU - functional support, not a performance path. - + This goes through system memory because we have to support copying between non-peer GPUs and we can't use P2P pointers even if the GPUs are peers. Because hsa_amd_agents_allow_access requires the caller to specify all allowed agents we can't assume that a peer mapped pointer @@ -501,6 +501,18 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent* dst_agent, const void* completion_signal); } +hsa_status_t Runtime::CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent, + uint32_t *engine_ids_mask) { + const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice); + core::Agent* copy_agent = (src_gpu) ? src_agent : dst_agent; + + if (dst_agent == src_agent) { + return HSA_STATUS_ERROR_INVALID_AGENT; + } + + return copy_agent->DmaCopyStatus(*dst_agent, *src_agent, engine_ids_mask); +} + hsa_status_t Runtime::FillMemory(void* ptr, uint32_t value, size_t count) { // Choose blit agent from pointer info hsa_amd_pointer_info_t info; diff --git a/runtime/hsa-runtime/hsacore.so.def b/runtime/hsa-runtime/hsacore.so.def index 9a57be2936..8691ce25bc 100644 --- a/runtime/hsa-runtime/hsacore.so.def +++ b/runtime/hsa-runtime/hsacore.so.def @@ -182,6 +182,7 @@ global: hsa_amd_queue_cu_get_mask; hsa_amd_memory_fill; hsa_amd_memory_async_copy; + hsa_amd_memory_copy_engine_status; hsa_amd_memory_async_copy_rect; hsa_amd_memory_lock; hsa_amd_memory_lock_to_pool; diff --git a/runtime/hsa-runtime/inc/hsa_api_trace.h b/runtime/hsa-runtime/inc/hsa_api_trace.h index f02322215f..4a57e22a94 100644 --- a/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -155,6 +155,7 @@ struct AmdExtTable { decltype(hsa_amd_memory_pool_allocate)* hsa_amd_memory_pool_allocate_fn; decltype(hsa_amd_memory_pool_free)* hsa_amd_memory_pool_free_fn; decltype(hsa_amd_memory_async_copy)* hsa_amd_memory_async_copy_fn; + decltype(hsa_amd_memory_copy_engine_status)* hsa_amd_memory_copy_engine_status_fn; decltype(hsa_amd_agent_memory_pool_get_info)* hsa_amd_agent_memory_pool_get_info_fn; decltype(hsa_amd_agents_allow_access)* hsa_amd_agents_allow_access_fn; decltype(hsa_amd_memory_pool_can_migrate)* hsa_amd_memory_pool_can_migrate_fn; diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index 3f933c2109..39332b36f0 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -381,6 +381,20 @@ typedef enum hsa_amd_agent_info_s { HSA_AMD_AGENT_INFO_IOMMU_SUPPORT = 0xA110 } hsa_amd_agent_info_t; +/** + * @brief SDMA engine IDs unique by single set bit position. + */ +typedef enum hsa_amd_sdma_engine_id { + HSA_AMD_SDMA_ENGINE_0 = 0x1, + HSA_AMD_SDMA_ENGINE_1 = 0x2, + HSA_AMD_SDMA_ENGINE_2 = 0x4, + HSA_AMD_SDMA_ENGINE_3 = 0x8, + HSA_AMD_SDMA_ENGINE_4 = 0x10, + HSA_AMD_SDMA_ENGINE_5 = 0x20, + HSA_AMD_SDMA_ENGINE_6 = 0x40, + HSA_AMD_SDMA_ENGINE_7 = 0x80 +} hsa_amd_sdma_engine_id_t; + typedef struct hsa_amd_hdp_flush_s { uint32_t* HDP_MEM_FLUSH_CNTL; uint32_t* HDP_REG_FLUSH_CNTL; @@ -1259,6 +1273,26 @@ hsa_status_t HSA_API uint32_t num_dep_signals, const hsa_signal_t* dep_signals, hsa_signal_t completion_signal); +/** + * @brief Reports the availability of SDMA copy engines. + * + * @param[in] dst_agent Destination agent of copy status direction. + * + * @param[in] src_agent Source agent of copy status direction. + * + * @param[out] engine_ids_mask returns available SDMA engine IDs that can be masked + * with hsa_amd_sdma_engine_id_t. + * + * @retval ::HSA_STATUS_SUCCESS Agent has available SDMA engines. + * + * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Agent does not have available SDMA engines. + * + * @retval ::HSA_STATUS_ERROR_INVALID_AGENT dst_agent and src_agent are the same as + * dst_agent == src_agent is generally used for shader copies. + */ +hsa_status_t HSA_API + hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent, + uint32_t *engine_ids_mask); /* [Provisional API]