AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make SDMA engine availability status queryable.

Browse Source 
Report the availability of SDMA engines for memory copies.

Change-Id: Ie31b02d6b65355122bb8c98bc73700a59bee166e
This commit is contained in:
Jonathan Kim
2023-02-01 12:52:00 -05:00
parent 4f283d9bb3
commit 8f27f495c6
12 changed files with 138 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/hsa-runtime/core/common/hsa_table_interface.cpp
+7
View File
@@ -969,6 +969,13 @@ hsa_status_t HSA_API
num_dep_signals, dep_signals, completion_signal);
}
// Mirrors Amd Extension Apis
hsa_status_t HSA_API
hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent,
uint32_t *engine_ids_mask) {
return amdExtTable->hsa_amd_memory_copy_engine_status_fn(dst_agent, src_agent, engine_ids_mask);
}
// Mirrors Amd Extension Apis
hsa_status_t HSA_API hsa_amd_memory_async_copy_rect(
const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src,
runtime/hsa-runtime/core/inc/agent.h
+13
View File
@@ -171,6 +171,19 @@ class Agent : public Checked<0xF6BC25EB17E6F917> {
return HSA_STATUS_ERROR;
}
// @brief Return DMA availability status for copy direction.
//
// @param [in] dst_agent Destination agent.
// @param [in] src_agent Source agent.
// @param [out] engine_ids_mask Mask of engine ids.
//
// @retval HSA_STATUS_SUCCESS DMA engines are available
// @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES DMA engines are not available
virtual hsa_status_t DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent,
uint32_t *engine_ids_mask) {
return HSA_STATUS_ERROR;
}
// @brief Submit DMA command to set the content of a pointer and wait
// until it is finished.
//
runtime/hsa-runtime/core/inc/amd_gpu_agent.h
+4
View File
@@ -231,6 +231,10 @@ class GpuAgent : public GpuAgentInt {
std::vector<core::Signal*>& dep_signals,
core::Signal& out_signal) override;
// @brief Override from core::Agent.
hsa_status_t DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent,
uint32_t *engine_ids_mask) override;
// @brief Override from core::Agent.
hsa_status_t DmaCopyRect(const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset,
const hsa_pitched_ptr_t* src, const hsa_dim3_t* src_offset,
runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h
+5
View File
@@ -145,6 +145,11 @@ hsa_status_t
const hsa_signal_t* dep_signals,
hsa_signal_t completion_signal);
// Mirrors Amd Extension Apis
hsa_status_t
hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent,
uint32_t *engine_ids_mask);
// Mirrors Amd Extension Apis
hsa_status_t hsa_amd_memory_async_copy_rect(
const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src,
runtime/hsa-runtime/core/inc/runtime.h
+11
View File
@@ -229,6 +229,17 @@ class Runtime {
core::Agent* src_agent, size_t size,
std::vector<core::Signal*>& dep_signals, core::Signal& completion_signal);
/// @brief Return SDMA availability status for copy direction
///
/// @param [in] dst_agent Destination agent.
/// @param [in] src_agent Source agent.
/// @param [out] engine_ids_mask Mask of engine_ids.
///
/// @retval HSA_STATUS_SUCCESS DMA engines are available
/// @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES DMA engines are not available
hsa_status_t CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent,
uint32_t *engine_ids_mask);
/// @brief Fill the first @p count of uint32_t in ptr with value.
///
/// @param [in] ptr Memory address to be filled.
runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+37
View File
@@ -775,6 +775,43 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
return stat;
}
hsa_status_t GpuAgent::DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent,
uint32_t *engine_ids_mask) {
assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) ||
(dst_agent.device_type() == core::Agent::kAmdGpuDevice)) &&
("Both devices are CPU agents which is not expected"));
*engine_ids_mask = 0;
if (src_agent.device_type() == core::Agent::kAmdGpuDevice &&
dst_agent.device_type() == core::Agent::kAmdGpuDevice &&
dst_agent.HiveId() && src_agent.HiveId() == dst_agent.HiveId() &&
properties_.NumSdmaXgmiEngines) {
// Find a free xGMI SDMA engine
for (int i = 0; i < properties_.NumSdmaXgmiEngines; i++) {
if (!!!blits_[DefaultBlitCount + i]->PendingBytes()) {
*engine_ids_mask |= (HSA_AMD_SDMA_ENGINE_2 << i);
}
}
} else {
bool is_h2d_blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice &&
dst_agent.device_type() == core::Agent::kAmdGpuDevice);
// Due to a RAS issue, GFX90a can only support H2D copies on SDMA0
bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10);
if (!!!blits_[BlitHostToDev]->PendingBytes()) {
if (is_h2d_blit || !limit_h2d_blit) {
*engine_ids_mask |= HSA_AMD_SDMA_ENGINE_0;
}
}
if (!!!blits_[BlitDevToHost]->PendingBytes()) {
*engine_ids_mask |= HSA_AMD_SDMA_ENGINE_1;
}
}
return !!(*engine_ids_mask) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR_OUT_OF_RESOURCES;
}
hsa_status_t GpuAgent::DmaCopyRect(const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset,
const hsa_pitched_ptr_t* src, const hsa_dim3_t* src_offset,
const hsa_dim3_t* range, hsa_amd_copy_direction_t dir,
runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp
+1
View File
@@ -365,6 +365,7 @@ void HsaApiTable::UpdateAmdExts() {
amd_ext_api.hsa_amd_memory_pool_allocate_fn = AMD::hsa_amd_memory_pool_allocate;
amd_ext_api.hsa_amd_memory_pool_free_fn = AMD::hsa_amd_memory_pool_free;
amd_ext_api.hsa_amd_memory_async_copy_fn = AMD::hsa_amd_memory_async_copy;
amd_ext_api.hsa_amd_memory_copy_engine_status_fn = AMD::hsa_amd_memory_copy_engine_status;
amd_ext_api.hsa_amd_agent_memory_pool_get_info_fn = AMD::hsa_amd_agent_memory_pool_get_info;
amd_ext_api.hsa_amd_agents_allow_access_fn = AMD::hsa_amd_agents_allow_access;
amd_ext_api.hsa_amd_memory_pool_can_migrate_fn = AMD::hsa_amd_memory_pool_can_migrate;
runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp
+11
View File
@@ -276,6 +276,17 @@ hsa_status_t hsa_amd_memory_async_copy(void* dst, hsa_agent_t dst_agent_handle,
CATCH;
}
hsa_status_t hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent_handle, hsa_agent_t src_agent_handle,
uint32_t *engine_ids_mask) {
core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle);
IS_VALID(dst_agent);
core::Agent* src_agent = core::Agent::Convert(src_agent_handle);
IS_VALID(src_agent);
return core::Runtime::runtime_singleton_->CopyMemoryStatus(dst_agent, src_agent, engine_ids_mask);
}
hsa_status_t hsa_amd_memory_async_copy_rect(
const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src,
const hsa_dim3_t* src_offset, const hsa_dim3_t* range, hsa_agent_t copy_agent,
runtime/hsa-runtime/core/runtime/runtime.cpp
+13 -1
View File
@@ -459,7 +459,7 @@ hsa_status_t Runtime::CopyMemory(void* dst, const void* src, size_t size) {
/*
GPU-GPU - functional support, not a performance path.
This goes through system memory because we have to support copying between non-peer GPUs
and we can't use P2P pointers even if the GPUs are peers. Because hsa_amd_agents_allow_access
requires the caller to specify all allowed agents we can't assume that a peer mapped pointer
@@ -501,6 +501,18 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent* dst_agent, const void*
completion_signal);
}
hsa_status_t Runtime::CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent,
uint32_t *engine_ids_mask) {
const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice);
core::Agent* copy_agent = (src_gpu) ? src_agent : dst_agent;
if (dst_agent == src_agent) {
return HSA_STATUS_ERROR_INVALID_AGENT;
}
return copy_agent->DmaCopyStatus(*dst_agent, *src_agent, engine_ids_mask);
}
hsa_status_t Runtime::FillMemory(void* ptr, uint32_t value, size_t count) {
// Choose blit agent from pointer info
hsa_amd_pointer_info_t info;
runtime/hsa-runtime/hsacore.so.def
+1
View File
@@ -182,6 +182,7 @@ global:
hsa_amd_queue_cu_get_mask;
hsa_amd_memory_fill;
hsa_amd_memory_async_copy;
hsa_amd_memory_copy_engine_status;
hsa_amd_memory_async_copy_rect;
hsa_amd_memory_lock;
hsa_amd_memory_lock_to_pool;
runtime/hsa-runtime/inc/hsa_api_trace.h
+1
View File
@@ -155,6 +155,7 @@ struct AmdExtTable {
decltype(hsa_amd_memory_pool_allocate)* hsa_amd_memory_pool_allocate_fn;
decltype(hsa_amd_memory_pool_free)* hsa_amd_memory_pool_free_fn;
decltype(hsa_amd_memory_async_copy)* hsa_amd_memory_async_copy_fn;
decltype(hsa_amd_memory_copy_engine_status)* hsa_amd_memory_copy_engine_status_fn;
decltype(hsa_amd_agent_memory_pool_get_info)* hsa_amd_agent_memory_pool_get_info_fn;
decltype(hsa_amd_agents_allow_access)* hsa_amd_agents_allow_access_fn;
decltype(hsa_amd_memory_pool_can_migrate)* hsa_amd_memory_pool_can_migrate_fn;
runtime/hsa-runtime/inc/hsa_ext_amd.h
+34
View File
@@ -381,6 +381,20 @@ typedef enum hsa_amd_agent_info_s {
HSA_AMD_AGENT_INFO_IOMMU_SUPPORT = 0xA110
} hsa_amd_agent_info_t;
/**
* @brief SDMA engine IDs unique by single set bit position.
*/
typedef enum hsa_amd_sdma_engine_id {
HSA_AMD_SDMA_ENGINE_0 = 0x1,
HSA_AMD_SDMA_ENGINE_1 = 0x2,
HSA_AMD_SDMA_ENGINE_2 = 0x4,
HSA_AMD_SDMA_ENGINE_3 = 0x8,
HSA_AMD_SDMA_ENGINE_4 = 0x10,
HSA_AMD_SDMA_ENGINE_5 = 0x20,
HSA_AMD_SDMA_ENGINE_6 = 0x40,
HSA_AMD_SDMA_ENGINE_7 = 0x80
} hsa_amd_sdma_engine_id_t;
typedef struct hsa_amd_hdp_flush_s {
uint32_t* HDP_MEM_FLUSH_CNTL;
uint32_t* HDP_REG_FLUSH_CNTL;
@@ -1259,6 +1273,26 @@ hsa_status_t HSA_API
uint32_t num_dep_signals,
const hsa_signal_t* dep_signals,
hsa_signal_t completion_signal);
/**
* @brief Reports the availability of SDMA copy engines.
*
* @param[in] dst_agent Destination agent of copy status direction.
*
* @param[in] src_agent Source agent of copy status direction.
*
* @param[out] engine_ids_mask returns available SDMA engine IDs that can be masked
* with hsa_amd_sdma_engine_id_t.
*
* @retval ::HSA_STATUS_SUCCESS Agent has available SDMA engines.
*
* @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Agent does not have available SDMA engines.
*
* @retval ::HSA_STATUS_ERROR_INVALID_AGENT dst_agent and src_agent are the same as
* dst_agent == src_agent is generally used for shader copies.
*/
hsa_status_t HSA_API
hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent,
uint32_t *engine_ids_mask);
/*
[Provisional API]