Add interface to DMA copy directly to a target engine.

Change-Id: Ic87cfeabb11c1a465f98f3f444d39955f5300525

runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -775,6 +775,59 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
   return stat;
 }
 
+hsa_status_t GpuAgent::DmaCopyOnEngine(void* dst, core::Agent& dst_agent,
+                               const void* src, core::Agent& src_agent,
+                               size_t size,
+                               std::vector<core::Signal*>& dep_signals,
+                               core::Signal& out_signal,
+                               int engine_offset,
+                               bool force_copy_on_sdma) {
+  // At this point it is guaranteed that one of
+  // the two devices is a GPU, potentially both
+  assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) ||
+          (dst_agent.device_type() == core::Agent::kAmdGpuDevice)) &&
+         ("Both devices are CPU agents which is not expected"));
+
+  if (engine_offset >= properties_.NumSdmaEngines + properties_.NumSdmaXgmiEngines) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  // check if dst and src are the same gpu or over xGMI.
+  bool is_same_gpu = (src_agent.public_handle().handle == dst_agent.public_handle().handle) &&
+      (dst_agent.public_handle().handle == public_handle_.handle);
+  bool is_xgmi = !is_same_gpu &&
+                   src_agent.device_type() == core::Agent::kAmdGpuDevice &&
+                     dst_agent.device_type() == core::Agent::kAmdGpuDevice &&
+                       dst_agent.HiveId() && src_agent.HiveId() == dst_agent.HiveId() &&
+                         properties_.NumSdmaXgmiEngines;
+
+  // Due to a RAS issue, GFX90a can only support H2D copies on SDMA0
+  bool is_h2d_blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice &&
+    dst_agent.device_type() == core::Agent::kAmdGpuDevice);
+  bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10);
+
+  // Ensure engine selection is within proper range based on transfer type
+  if ((is_xgmi && engine_offset < properties_.NumSdmaEngines) ||
+       (!is_xgmi && engine_offset >= properties_.NumSdmaEngines) ||
+         (!is_h2d_blit && !is_same_gpu && limit_h2d_blit && !engine_offset)) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  lazy_ptr<core::Blit>& blit = is_same_gpu ?
+                                 (force_copy_on_sdma ? blits_[BlitDevToHost] :
+                                   blits_[BlitDevToDev]) : blits_[engine_offset];
+
+  if (profiling_enabled()) {
+    // Track the agent so we could translate the resulting timestamp to system
+    // domain correctly.
+    out_signal.async_copy_agent(core::Agent::Convert(this->public_handle()));
+  }
+
+  hsa_status_t stat = blit->SubmitLinearCopyCommand(dst, src, size, dep_signals, out_signal);
+
+  return stat;
+}
+
 hsa_status_t GpuAgent::DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent,
                                      uint32_t *engine_ids_mask) {
   assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) ||

runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp

@@ -365,6 +365,7 @@ void HsaApiTable::UpdateAmdExts() {
   amd_ext_api.hsa_amd_memory_pool_allocate_fn = AMD::hsa_amd_memory_pool_allocate;
   amd_ext_api.hsa_amd_memory_pool_free_fn = AMD::hsa_amd_memory_pool_free;
   amd_ext_api.hsa_amd_memory_async_copy_fn = AMD::hsa_amd_memory_async_copy;
+  amd_ext_api.hsa_amd_memory_async_copy_on_engine_fn = AMD::hsa_amd_memory_async_copy_on_engine;
   amd_ext_api.hsa_amd_memory_copy_engine_status_fn = AMD::hsa_amd_memory_copy_engine_status;
   amd_ext_api.hsa_amd_agent_memory_pool_get_info_fn = AMD::hsa_amd_agent_memory_pool_get_info;
   amd_ext_api.hsa_amd_agents_allow_access_fn = AMD::hsa_amd_agents_allow_access;

runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp

@@ -276,6 +276,50 @@ hsa_status_t hsa_amd_memory_async_copy(void* dst, hsa_agent_t dst_agent_handle,
   CATCH;
 }
 
+hsa_status_t hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent_handle,
+                                       const void* src, hsa_agent_t src_agent_handle, size_t size,
+                                       uint32_t num_dep_signals, const hsa_signal_t* dep_signals,
+                                       hsa_signal_t completion_signal, hsa_amd_sdma_engine_id_t engine_id,
+                                       bool force_copy_on_sdma) {
+  TRY;
+  IS_BAD_PTR(dst);
+  IS_BAD_PTR(src);
+
+  if ((num_dep_signals == 0 && dep_signals != nullptr) ||
+      (num_dep_signals > 0 && dep_signals == nullptr)) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle);
+  IS_VALID(dst_agent);
+
+  core::Agent* src_agent = core::Agent::Convert(src_agent_handle);
+  IS_VALID(src_agent);
+
+  std::vector<core::Signal*> dep_signal_list(num_dep_signals);
+  if (num_dep_signals > 0) {
+    for (size_t i = 0; i < num_dep_signals; ++i) {
+      core::Signal* dep_signal_obj = core::Signal::Convert(dep_signals[i]);
+      IS_VALID(dep_signal_obj);
+      dep_signal_list[i] = dep_signal_obj;
+    }
+  }
+
+  core::Signal* out_signal_obj = core::Signal::Convert(completion_signal);
+  IS_VALID(out_signal_obj);
+
+  bool rev_copy_dir = core::Runtime::runtime_singleton_->flag().rev_copy_dir();
+  if (size > 0) {
+    return core::Runtime::runtime_singleton_->CopyMemoryOnEngine(
+        dst, (rev_copy_dir ? src_agent : dst_agent),
+        src, (rev_copy_dir ? dst_agent : src_agent),
+        size, dep_signal_list, *out_signal_obj, engine_id, force_copy_on_sdma);
+  }
+
+  return HSA_STATUS_SUCCESS;
+  CATCH;
+}
+
 hsa_status_t hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent_handle, hsa_agent_t src_agent_handle,
                                                uint32_t *engine_ids_mask) {
   core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle);

runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -501,6 +501,24 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent* dst_agent, const void*
                              completion_signal);
 }
 
+hsa_status_t Runtime::CopyMemoryOnEngine(void* dst, core::Agent* dst_agent, const void* src,
+                                 core::Agent* src_agent, size_t size,
+                                 std::vector<core::Signal*>& dep_signals,
+                                 core::Signal& completion_signal,
+                                 hsa_amd_sdma_engine_id_t engine_id, bool force_copy_on_sdma) {
+  const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice);
+  core::Agent* copy_agent = (src_gpu) ? src_agent : dst_agent;
+
+  // engine_id is single bitset unique.
+  int engine_offset = ffs(engine_id) - 1;
+  if (!engine_id || !!((engine_id >> (engine_offset + 1)))) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  return copy_agent->DmaCopyOnEngine(dst, *dst_agent, src, *src_agent, size, dep_signals,
+                             completion_signal, engine_offset, force_copy_on_sdma);
+}
+
 hsa_status_t Runtime::CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent,
                                        uint32_t *engine_ids_mask) {
   const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice);