diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index fb8eb0ecec..cbf06be28c 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -72,6 +72,11 @@ class GpuAgentInt : public core::Agent { GpuAgentInt(uint32_t node_id) : core::Agent(node_id, core::Agent::DeviceType::kAmdGpuDevice) {} + // @brief Initialize DMA queue. + // + // @retval HSA_STATUS_SUCCESS DMA queue initialization is successful. + virtual void InitDma() = 0; + // @brief Invoke the user provided callback for each region accessible by // this agent. // @@ -158,10 +163,13 @@ class GpuAgent : public GpuAgentInt { // @brief GPU agent destructor. ~GpuAgent(); - // @brief Initialize DMA queue. + // @brief Override from core::Agent. + void InitDma() override; + + // @brief Initialize blit kernel object based on AQL queue. // - // @retval HSA_STATUS_SUCCESS DMA queue initialization is successful. - hsa_status_t InitDma(); + // @retval HSA_STATUS_SUCCESS blit kernel object initialization is successful. + hsa_status_t InitBlitKernel(); uint16_t GetMicrocodeVersion() const; @@ -333,10 +341,13 @@ class GpuAgent : public GpuAgentInt { // @brief Blit object to handle memory copy from system to device memory. core::Blit* blit_h2d_; - // @brief Blit object to handle memory copy from device to system, device to - // device, and memory fill. + // @brief Blit object to handle memory copy from device to system memory. core::Blit* blit_d2h_; + // @brief Blit object to handle memory copy from device to device memory, and + // memory fill. + core::Blit* blit_d2d_; + // @brief Mutex to protect the update to coherency type. KernelMutex coherency_lock_; @@ -346,6 +357,9 @@ class GpuAgent : public GpuAgentInt { // @brief Mutex to protect access to ::t1_. KernelMutex t1_lock_; + // @brief Mutex to protect access to blit objects. + KernelMutex blit_lock_; + // @brief GPU tick on initialization. HsaClockCounters t0_; @@ -391,6 +405,9 @@ class GpuAgent : public GpuAgentInt { // @brief Alternative aperture size. Only on KV. size_t ape1_size_; + // @brief True if blit objects are initialized. + bool blit_initialized_; + DISALLOW_COPY_AND_ASSIGN(GpuAgent); }; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 49361b1f40..5aad565789 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -71,13 +71,15 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) current_coherency_type_(HSA_AMD_COHERENCY_TYPE_COHERENT), blit_h2d_(NULL), blit_d2h_(NULL), + blit_d2d_(NULL), is_kv_device_(false), trap_code_buf_(NULL), trap_code_buf_size_(0), memory_bus_width_(0), memory_max_frequency_(0), ape1_base_(0), - ape1_size_(0) { + ape1_size_(0), + blit_initialized_(false) { const bool is_apu_node = (properties_.NumCPUCores > 0); profile_ = (is_apu_node) ? HSA_PROFILE_FULL : HSA_PROFILE_BASE; @@ -126,6 +128,14 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) } GpuAgent::~GpuAgent() { + if (blit_d2d_ != NULL) { + hsa_status_t status = blit_d2d_->Destroy(*this); + assert(status == HSA_STATUS_SUCCESS); + + delete blit_d2d_; + blit_d2d_ = NULL; + } + if (blit_h2d_ != NULL) { hsa_status_t status = blit_h2d_->Destroy(*this); assert(status == HSA_STATUS_SUCCESS); @@ -445,41 +455,60 @@ core::Blit* GpuAgent::CreateBlitKernel() { return kernl; } -hsa_status_t GpuAgent::InitDma() { - // Try create SDMA blit first. - if (core::Runtime::runtime_singleton_->flag().enable_sdma() && - isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0 && - isa_->GetStepping() == 3) { - blit_h2d_ = CreateBlitSdma(); - blit_d2h_ = CreateBlitSdma(); +void GpuAgent::InitDma() { + // This provides the ability to lazy init the blit objects on places that + // could give indication of DMA usage in the future. E.g.: + // 1. Call to allow access API. + // 2. Call to memory lock API. + if (!atomic::Load(&blit_initialized_, std::memory_order_acquire)) { + ScopedAcquire lock(&blit_lock_); + if (!atomic::Load(&blit_initialized_, std::memory_order_relaxed)) { + // Try create SDMA blit first. + if (core::Runtime::runtime_singleton_->flag().enable_sdma() && + isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0 && + isa_->GetStepping() == 3) { + blit_h2d_ = CreateBlitSdma(); + blit_d2h_ = CreateBlitSdma(); - if (blit_h2d_ != NULL && blit_d2h_ != NULL) { - return HSA_STATUS_SUCCESS; + if (blit_h2d_ != NULL && blit_d2h_ != NULL) { + atomic::Store(&blit_initialized_, true, std::memory_order_release); + return; + } + } + + // Fall back to blit kernel if SDMA is unavailable. + assert(blit_h2d_ == NULL || blit_d2h_ == NULL); + + if (blit_h2d_ == NULL) { + blit_h2d_ = CreateBlitKernel(); + } + + if (blit_d2h_ == NULL) { + blit_d2h_ = CreateBlitKernel(); + } + + atomic::Store(&blit_initialized_, true, std::memory_order_release); } } +} - // Fall back to blit kernel if SDMA is unavailable. - assert(blit_h2d_ == NULL || blit_d2h_ == NULL); - - if (blit_h2d_ == NULL) { - blit_h2d_ = CreateBlitKernel(); +hsa_status_t GpuAgent::InitBlitKernel() { + // Unlike InitDma, this function is not designed for lazy initialization. + // So checking the state without double checked locking is fine. + if (blit_d2d_ == NULL) { + blit_d2d_ = CreateBlitKernel(); } - if (blit_d2h_ == NULL) { - blit_d2h_ = CreateBlitKernel(); - } - - return (blit_h2d_ != NULL && blit_d2h_ != NULL) - ? HSA_STATUS_SUCCESS - : HSA_STATUS_ERROR_OUT_OF_RESOURCES; + return (blit_d2d_ != NULL) ? HSA_STATUS_SUCCESS + : HSA_STATUS_ERROR_OUT_OF_RESOURCES; } hsa_status_t GpuAgent::DmaCopy(void* dst, const void* src, size_t size) { - if (blit_d2h_ == NULL) { + if (blit_d2d_ == NULL) { return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } - return blit_d2h_->SubmitLinearCopyCommand(dst, src, size); + return blit_d2d_->SubmitLinearCopyCommand(dst, src, size); } hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent, @@ -487,10 +516,14 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent, size_t size, std::vector& dep_signals, core::Signal& out_signal) { - core::Blit* blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice && - dst_agent.device_type() == core::Agent::kAmdGpuDevice) - ? blit_h2d_ - : blit_d2h_; + core::Blit* blit = + (src_agent.device_type() == core::Agent::kAmdCpuDevice && + dst_agent.device_type() == core::Agent::kAmdGpuDevice) + ? blit_h2d_ + : (src_agent.device_type() == core::Agent::kAmdGpuDevice && + dst_agent.device_type() == core::Agent::kAmdCpuDevice) + ? blit_d2h_ + : blit_d2d_; if (blit == NULL) { return HSA_STATUS_ERROR_OUT_OF_RESOURCES; @@ -507,11 +540,11 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent, } hsa_status_t GpuAgent::DmaFill(void* ptr, uint32_t value, size_t count) { - if (blit_d2h_ == NULL) { + if (blit_d2d_ == NULL) { return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } - return blit_d2h_->SubmitLinearFillCommand(ptr, value, count); + return blit_d2d_->SubmitLinearFillCommand(ptr, value, count); } hsa_status_t GpuAgent::GetInfo(hsa_agent_info_t attribute, void* value) const { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp index f981b423de..678f79e76b 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp @@ -43,6 +43,7 @@ #include "core/inc/amd_memory_region.h" #include +#include #include "core/inc/runtime.h" #include "core/inc/amd_cpu_agent.h" @@ -431,15 +432,17 @@ hsa_status_t MemoryRegion::AllowAccess(uint32_t num_agents, bool cpu_in_list = false; + std::set whitelist_gpus; std::vector whitelist_nodes; for (uint32_t i = 0; i < num_agents; ++i) { - const core::Agent* agent = core::Agent::Convert(agents[i]); + core::Agent* agent = core::Agent::Convert(agents[i]); if (agent == NULL || !agent->IsValid()) { return HSA_STATUS_ERROR_INVALID_AGENT; } if (agent->device_type() == core::Agent::kAmdGpuDevice) { whitelist_nodes.push_back(agent->node_id()); + whitelist_gpus.insert(reinterpret_cast(agent)); } else { cpu_in_list = true; } @@ -458,17 +461,24 @@ hsa_status_t MemoryRegion::AllowAccess(uint32_t num_agents, std::find(whitelist_nodes.begin(), whitelist_nodes.end(), owner()->node_id()) == whitelist_nodes.end()) { whitelist_nodes.push_back(owner()->node_id()); + whitelist_gpus.insert(reinterpret_cast(owner())); } HsaMemMapFlags map_flag = map_flag_; map_flag.ui32.HostAccess |= (cpu_in_list) ? 1 : 0; uint64_t alternate_va = 0; - return (amd::MemoryRegion::MakeKfdMemoryResident( - whitelist_nodes.size(), &whitelist_nodes[0], - const_cast(ptr), size, &alternate_va, map_flag)) - ? HSA_STATUS_SUCCESS - : HSA_STATUS_ERROR_OUT_OF_RESOURCES; + if (!amd::MemoryRegion::MakeKfdMemoryResident( + whitelist_nodes.size(), &whitelist_nodes[0], const_cast(ptr), + size, &alternate_va, map_flag)) { + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + for (GpuAgentInt* gpu : whitelist_gpus) { + gpu->InitDma(); + } + + return HSA_STATUS_SUCCESS; } hsa_status_t MemoryRegion::CanMigrate(const MemoryRegion& dst, @@ -496,10 +506,15 @@ hsa_status_t MemoryRegion::Lock(uint32_t num_agents, const hsa_agent_t* agents, return HSA_STATUS_SUCCESS; } + std::set whitelist_gpus; std::vector whitelist_nodes; if (num_agents == 0 || agents == NULL) { // Map to all GPU agents. whitelist_nodes = core::Runtime::runtime_singleton_->gpu_ids(); + + whitelist_gpus.insert( + core::Runtime::runtime_singleton_->gpu_agents().begin(), + core::Runtime::runtime_singleton_->gpu_agents().end()); } else { for (int i = 0; i < num_agents; ++i) { core::Agent* agent = core::Agent::Convert(agents[i]); @@ -509,6 +524,7 @@ hsa_status_t MemoryRegion::Lock(uint32_t num_agents, const hsa_agent_t* agents, if (agent->device_type() == core::Agent::kAmdGpuDevice) { whitelist_nodes.push_back(agent->node_id()); + whitelist_gpus.insert(reinterpret_cast(agent)); } } } @@ -528,6 +544,11 @@ hsa_status_t MemoryRegion::Lock(uint32_t num_agents, const hsa_agent_t* agents, host_ptr, size, &alternate_va, map_flag_)) { assert(alternate_va != 0); *agent_ptr = reinterpret_cast(alternate_va); + + for (core::Agent* gpu : whitelist_gpus) { + reinterpret_cast(gpu)->InitDma(); + } + return HSA_STATUS_SUCCESS; } amd::MemoryRegion::DeregisterMemory(host_ptr); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp index e6a348330d..e008261ee4 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -78,7 +78,7 @@ GpuAgent* DiscoverGpu(HSAuint32 node_id, HsaNodeProperties& node_prop) { GpuAgent* gpu = new GpuAgent(node_id, node_prop); core::Runtime::runtime_singleton_->RegisterAgent(gpu); - if (HSA_STATUS_SUCCESS != gpu->InitDma()) { + if (HSA_STATUS_SUCCESS != gpu->InitBlitKernel()) { assert(false && "Fail init blit"); delete gpu; gpu = NULL;