Use lazy initialization to create Blit objects

Change-Id: I388865030dc2538c5c881c055e38af52a57f6d87


[ROCm/ROCR-Runtime commit: bc589048a9]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h

@@ -72,6 +72,11 @@ class GpuAgentInt : public core::Agent {
   GpuAgentInt(uint32_t node_id)
       : core::Agent(node_id, core::Agent::DeviceType::kAmdGpuDevice) {}
 
+  // @brief Initialize DMA queue.
+  //
+  // @retval HSA_STATUS_SUCCESS DMA queue initialization is successful.
+  virtual void InitDma() = 0;
+
   // @brief Invoke the user provided callback for each region accessible by
   // this agent.
   //
@@ -158,10 +163,13 @@ class GpuAgent : public GpuAgentInt {
   // @brief GPU agent destructor.
   ~GpuAgent();
 
-  // @brief Initialize DMA queue.
+  // @brief Override from core::Agent.
+  void InitDma() override;
+
+  // @brief Initialize blit kernel object based on AQL queue.
   //
-  // @retval HSA_STATUS_SUCCESS DMA queue initialization is successful.
-  hsa_status_t InitDma();
+  // @retval HSA_STATUS_SUCCESS blit kernel object initialization is successful.
+  hsa_status_t InitBlitKernel();
 
   uint16_t GetMicrocodeVersion() const;
 
@@ -333,10 +341,13 @@ class GpuAgent : public GpuAgentInt {
   // @brief Blit object to handle memory copy from system to device memory.
   core::Blit* blit_h2d_;
 
-  // @brief Blit object to handle memory copy from device to system, device to
-  // device, and memory fill.
+  // @brief Blit object to handle memory copy from device to system memory.
   core::Blit* blit_d2h_;
 
+  // @brief Blit object to handle memory copy from device to device memory, and
+  // memory fill.
+  core::Blit* blit_d2d_;
+
   // @brief Mutex to protect the update to coherency type.
   KernelMutex coherency_lock_;
 
@@ -346,6 +357,9 @@ class GpuAgent : public GpuAgentInt {
   // @brief Mutex to protect access to ::t1_.
   KernelMutex t1_lock_;
 
+  // @brief Mutex to protect access to blit objects.
+  KernelMutex blit_lock_;
+
   // @brief GPU tick on initialization.
   HsaClockCounters t0_;
 
@@ -391,6 +405,9 @@ class GpuAgent : public GpuAgentInt {
   // @brief Alternative aperture size. Only on KV.
   size_t ape1_size_;
 
+  // @brief True if blit objects are initialized.
+  bool blit_initialized_;
+
   DISALLOW_COPY_AND_ASSIGN(GpuAgent);
 };
 

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -71,13 +71,15 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props)
       current_coherency_type_(HSA_AMD_COHERENCY_TYPE_COHERENT),
       blit_h2d_(NULL),
       blit_d2h_(NULL),
+      blit_d2d_(NULL),
       is_kv_device_(false),
       trap_code_buf_(NULL),
       trap_code_buf_size_(0),
       memory_bus_width_(0),
       memory_max_frequency_(0),
       ape1_base_(0),
-      ape1_size_(0) {
+      ape1_size_(0),
+      blit_initialized_(false) {
   const bool is_apu_node = (properties_.NumCPUCores > 0);
   profile_ = (is_apu_node) ? HSA_PROFILE_FULL : HSA_PROFILE_BASE;
 
@@ -126,6 +128,14 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props)
 }
 
 GpuAgent::~GpuAgent() {
+  if (blit_d2d_ != NULL) {
+    hsa_status_t status = blit_d2d_->Destroy(*this);
+    assert(status == HSA_STATUS_SUCCESS);
+
+    delete blit_d2d_;
+    blit_d2d_ = NULL;
+  }
+
   if (blit_h2d_ != NULL) {
     hsa_status_t status = blit_h2d_->Destroy(*this);
     assert(status == HSA_STATUS_SUCCESS);
@@ -445,41 +455,60 @@ core::Blit* GpuAgent::CreateBlitKernel() {
   return kernl;
 }
 
-hsa_status_t GpuAgent::InitDma() {
-  // Try create SDMA blit first.
-  if (core::Runtime::runtime_singleton_->flag().enable_sdma() &&
-      isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0 &&
-      isa_->GetStepping() == 3) {
-    blit_h2d_ = CreateBlitSdma();
-    blit_d2h_ = CreateBlitSdma();
+void GpuAgent::InitDma() {
+  // This provides the ability to lazy init the blit objects on places that
+  // could give indication of DMA usage in the future. E.g.:
+  // 1. Call to allow access API.
+  // 2. Call to memory lock API.
+  if (!atomic::Load(&blit_initialized_, std::memory_order_acquire)) {
+    ScopedAcquire<KernelMutex> lock(&blit_lock_);
+    if (!atomic::Load(&blit_initialized_, std::memory_order_relaxed)) {
+      // Try create SDMA blit first.
+      if (core::Runtime::runtime_singleton_->flag().enable_sdma() &&
+          isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0 &&
+          isa_->GetStepping() == 3) {
+        blit_h2d_ = CreateBlitSdma();
+        blit_d2h_ = CreateBlitSdma();
 
-    if (blit_h2d_ != NULL && blit_d2h_ != NULL) {
-      return HSA_STATUS_SUCCESS;
+        if (blit_h2d_ != NULL && blit_d2h_ != NULL) {
+          atomic::Store(&blit_initialized_, true, std::memory_order_release);
+          return;
+        }
+      }
+
+      // Fall back to blit kernel if SDMA is unavailable.
+      assert(blit_h2d_ == NULL || blit_d2h_ == NULL);
+
+      if (blit_h2d_ == NULL) {
+        blit_h2d_ = CreateBlitKernel();
+      }
+
+      if (blit_d2h_ == NULL) {
+        blit_d2h_ = CreateBlitKernel();
+      }
+
+      atomic::Store(&blit_initialized_, true, std::memory_order_release);
     }
   }
+}
 
-  // Fall back to blit kernel if SDMA is unavailable.
-  assert(blit_h2d_ == NULL || blit_d2h_ == NULL);
-
-  if (blit_h2d_ == NULL) {
-    blit_h2d_ = CreateBlitKernel();
+hsa_status_t GpuAgent::InitBlitKernel() {
+  // Unlike InitDma, this function is not designed for lazy initialization.
+  // So checking the state without  double checked locking is fine.
+  if (blit_d2d_ == NULL) {
+    blit_d2d_ = CreateBlitKernel();
   }
 
-  if (blit_d2h_ == NULL) {
-    blit_d2h_ = CreateBlitKernel();
-  }
-
-  return (blit_h2d_ != NULL && blit_d2h_ != NULL)
-             ? HSA_STATUS_SUCCESS
-             : HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  return (blit_d2d_ != NULL) ? HSA_STATUS_SUCCESS
+                             : HSA_STATUS_ERROR_OUT_OF_RESOURCES;
 }
 
 hsa_status_t GpuAgent::DmaCopy(void* dst, const void* src, size_t size) {
-  if (blit_d2h_ == NULL) {
+  if (blit_d2d_ == NULL) {
     return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
   }
 
-  return blit_d2h_->SubmitLinearCopyCommand(dst, src, size);
+  return blit_d2d_->SubmitLinearCopyCommand(dst, src, size);
 }
 
 hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
@@ -487,10 +516,14 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
                                size_t size,
                                std::vector<core::Signal*>& dep_signals,
                                core::Signal& out_signal) {
-  core::Blit* blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice &&
-                      dst_agent.device_type() == core::Agent::kAmdGpuDevice)
-                         ? blit_h2d_
-                         : blit_d2h_;
+  core::Blit* blit =
+      (src_agent.device_type() == core::Agent::kAmdCpuDevice &&
+       dst_agent.device_type() == core::Agent::kAmdGpuDevice)
+          ? blit_h2d_
+          : (src_agent.device_type() == core::Agent::kAmdGpuDevice &&
+             dst_agent.device_type() == core::Agent::kAmdCpuDevice)
+                ? blit_d2h_
+                : blit_d2d_;
 
   if (blit == NULL) {
     return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
@@ -507,11 +540,11 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
 }
 
 hsa_status_t GpuAgent::DmaFill(void* ptr, uint32_t value, size_t count) {
-  if (blit_d2h_ == NULL) {
+  if (blit_d2d_ == NULL) {
     return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
   }
 
-  return blit_d2h_->SubmitLinearFillCommand(ptr, value, count);
+  return blit_d2d_->SubmitLinearFillCommand(ptr, value, count);
 }
 
 hsa_status_t GpuAgent::GetInfo(hsa_agent_info_t attribute, void* value) const {

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp

@@ -43,6 +43,7 @@
 #include "core/inc/amd_memory_region.h"
 
 #include <algorithm>
+#include <set>
 
 #include "core/inc/runtime.h"
 #include "core/inc/amd_cpu_agent.h"
@@ -431,15 +432,17 @@ hsa_status_t MemoryRegion::AllowAccess(uint32_t num_agents,
 
   bool cpu_in_list = false;
 
+  std::set<GpuAgentInt*> whitelist_gpus;
   std::vector<uint32_t> whitelist_nodes;
   for (uint32_t i = 0; i < num_agents; ++i) {
-    const core::Agent* agent = core::Agent::Convert(agents[i]);
+    core::Agent* agent = core::Agent::Convert(agents[i]);
     if (agent == NULL || !agent->IsValid()) {
       return HSA_STATUS_ERROR_INVALID_AGENT;
     }
 
     if (agent->device_type() == core::Agent::kAmdGpuDevice) {
       whitelist_nodes.push_back(agent->node_id());
+      whitelist_gpus.insert(reinterpret_cast<GpuAgentInt*>(agent));
     } else {
       cpu_in_list = true;
     }
@@ -458,17 +461,24 @@ hsa_status_t MemoryRegion::AllowAccess(uint32_t num_agents,
       std::find(whitelist_nodes.begin(), whitelist_nodes.end(),
                 owner()->node_id()) == whitelist_nodes.end()) {
     whitelist_nodes.push_back(owner()->node_id());
+    whitelist_gpus.insert(reinterpret_cast<GpuAgentInt*>(owner()));
   }
 
   HsaMemMapFlags map_flag = map_flag_;
   map_flag.ui32.HostAccess |= (cpu_in_list) ? 1 : 0;
 
   uint64_t alternate_va = 0;
-  return (amd::MemoryRegion::MakeKfdMemoryResident(
-             whitelist_nodes.size(), &whitelist_nodes[0],
-             const_cast<void*>(ptr), size, &alternate_va, map_flag))
-             ? HSA_STATUS_SUCCESS
-             : HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  if (!amd::MemoryRegion::MakeKfdMemoryResident(
+          whitelist_nodes.size(), &whitelist_nodes[0], const_cast<void*>(ptr),
+          size, &alternate_va, map_flag)) {
+    return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  }
+
+  for (GpuAgentInt* gpu : whitelist_gpus) {
+    gpu->InitDma();
+  }
+
+  return HSA_STATUS_SUCCESS;
 }
 
 hsa_status_t MemoryRegion::CanMigrate(const MemoryRegion& dst,
@@ -496,10 +506,15 @@ hsa_status_t MemoryRegion::Lock(uint32_t num_agents, const hsa_agent_t* agents,
     return HSA_STATUS_SUCCESS;
   }
 
+  std::set<core::Agent*> whitelist_gpus;
   std::vector<HSAuint32> whitelist_nodes;
   if (num_agents == 0 || agents == NULL) {
     // Map to all GPU agents.
     whitelist_nodes = core::Runtime::runtime_singleton_->gpu_ids();
+
+    whitelist_gpus.insert(
+        core::Runtime::runtime_singleton_->gpu_agents().begin(),
+        core::Runtime::runtime_singleton_->gpu_agents().end());
   } else {
     for (int i = 0; i < num_agents; ++i) {
       core::Agent* agent = core::Agent::Convert(agents[i]);
@@ -509,6 +524,7 @@ hsa_status_t MemoryRegion::Lock(uint32_t num_agents, const hsa_agent_t* agents,
 
       if (agent->device_type() == core::Agent::kAmdGpuDevice) {
         whitelist_nodes.push_back(agent->node_id());
+        whitelist_gpus.insert(reinterpret_cast<GpuAgentInt*>(agent));
       }
     }
   }
@@ -528,6 +544,11 @@ hsa_status_t MemoryRegion::Lock(uint32_t num_agents, const hsa_agent_t* agents,
                               host_ptr, size, &alternate_va, map_flag_)) {
       assert(alternate_va != 0);
       *agent_ptr = reinterpret_cast<void*>(alternate_va);
+
+      for (core::Agent* gpu : whitelist_gpus) {
+        reinterpret_cast<GpuAgentInt*>(gpu)->InitDma();
+      }
+
       return HSA_STATUS_SUCCESS;
     }
     amd::MemoryRegion::DeregisterMemory(host_ptr);

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp

@@ -78,7 +78,7 @@ GpuAgent* DiscoverGpu(HSAuint32 node_id, HsaNodeProperties& node_prop) {
   GpuAgent* gpu = new GpuAgent(node_id, node_prop);
   core::Runtime::runtime_singleton_->RegisterAgent(gpu);
 
-  if (HSA_STATUS_SUCCESS != gpu->InitDma()) {
+  if (HSA_STATUS_SUCCESS != gpu->InitBlitKernel()) {
     assert(false && "Fail init blit");
     delete gpu;
     gpu = NULL;