Add interface to DMA copy directly to a target engine.

Change-Id: Ic87cfeabb11c1a465f98f3f444d39955f5300525

runtime/hsa-runtime/core/common/hsa_table_interface.cpp

@@ -969,6 +969,21 @@ hsa_status_t HSA_API
                                      num_dep_signals, dep_signals, completion_signal);
 }
 
+// Mirrors Amd Extension Apis
+hsa_status_t HSA_API
+    hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent, const void* src,
+                              hsa_agent_t src_agent, size_t size,
+                              uint32_t num_dep_signals,
+                              const hsa_signal_t* dep_signals,
+                              hsa_signal_t completion_signal,
+                              hsa_amd_sdma_engine_id_t engine_id,
+                              bool force_copy_on_sdma) {
+  return amdExtTable->hsa_amd_memory_async_copy_on_engine_fn(
+                                     dst, dst_agent, src, src_agent, size,
+                                     num_dep_signals, dep_signals, completion_signal,
+                                     engine_id, force_copy_on_sdma);
+}
+
 // Mirrors Amd Extension Apis
 hsa_status_t HSA_API
     hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent,

runtime/hsa-runtime/core/inc/agent.h

@@ -171,6 +171,27 @@ class Agent : public Checked<0xF6BC25EB17E6F917> {
     return HSA_STATUS_ERROR;
   }
 
+  // @brief Submit DMA copy command to move data from src to dst on engine_id.
+  // This call does not wait until the copy is finished
+  //
+  // @details All semantics and params are identical to DmaCopy except for engine_id.
+  //
+  // @param [in] engine_offset Target engine
+  // @param [in] force_copy_on_sdma By default, blit kernel copies are used if
+  // dst_agent == src_agent.  Setting this true forces the copy over SDMA1.
+  //
+  //
+  // @retval HSA_STATUS_SUCCESS The memory copy is finished and successful.
+  virtual hsa_status_t DmaCopyOnEngine(void* dst, core::Agent& dst_agent,
+                               const void* src, core::Agent& src_agent,
+                               size_t size,
+                               std::vector<core::Signal*>& dep_signals,
+                               core::Signal& out_signal,
+                               int engine_offset,
+                               bool force_copy_on_sdma) {
+    return HSA_STATUS_ERROR;
+  }
+
   // @brief Return DMA availability status for copy direction.
   //
   // @param [in] dst_agent Destination agent.

runtime/hsa-runtime/core/inc/amd_gpu_agent.h

@@ -231,6 +231,13 @@ class GpuAgent : public GpuAgentInt {
                        std::vector<core::Signal*>& dep_signals,
                        core::Signal& out_signal) override;
 
+  // @brief Override from core::Agent.
+  hsa_status_t DmaCopyOnEngine(void* dst, core::Agent& dst_agent, const void* src,
+                       core::Agent& src_agent, size_t size,
+                       std::vector<core::Signal*>& dep_signals,
+                       core::Signal& out_signal, int engine_offset,
+                       bool force_copy_on_sdma) override;
+
   // @brief Override from core::Agent.
   hsa_status_t DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent,
                              uint32_t *engine_ids_mask) override;

runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h

@@ -145,6 +145,16 @@ hsa_status_t
                               const hsa_signal_t* dep_signals,
                               hsa_signal_t completion_signal);
 
+// Mirrors Amd Extension Apis
+hsa_status_t
+    hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent, const void* src,
+                              hsa_agent_t src_agent, size_t size,
+                              uint32_t num_dep_signals,
+                              const hsa_signal_t* dep_signals,
+                              hsa_signal_t completion_signal,
+                              hsa_amd_sdma_engine_id_t engine_id,
+                              bool force_copy_on_sdma);
+
 // Mirrors Amd Extension Apis
 hsa_status_t
     hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent, hsa_agent_t src_agent,

runtime/hsa-runtime/core/inc/runtime.h

@@ -229,6 +229,24 @@ class Runtime {
                           core::Agent* src_agent, size_t size,
                           std::vector<core::Signal*>& dep_signals, core::Signal& completion_signal);
 
+  /// @brief Non-blocking memory copy from src to dst on engine_id.
+  ///
+  /// @details All semantics and params are dentical to CopyMemory
+  ///  with the exception of engine_id.
+  ///
+  /// @param [in] engine_id Target engine to copy on.
+  ///
+  /// @param [in] force_copy_on_sdma By default, a blit kernel copy is used
+  /// when dst_agent == src_agent.  Setting this to true will force the copy
+  /// over SDMA1.
+  ///
+  /// @retval ::HSA_STATUS_SUCCESS if copy command has been submitted
+  /// successfully to the agent DMA queue.
+  hsa_status_t CopyMemoryOnEngine(void* dst, core::Agent* dst_agent, const void* src,
+                          core::Agent* src_agent, size_t size,
+                          std::vector<core::Signal*>& dep_signals, core::Signal& completion_signal,
+                          hsa_amd_sdma_engine_id_t  engine_id, bool force_copy_on_sdma);
+
   /// @brief Return SDMA availability status for copy direction
   ///
   /// @param [in] dst_agent Destination agent.

runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -775,6 +775,59 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
   return stat;
 }
 
+hsa_status_t GpuAgent::DmaCopyOnEngine(void* dst, core::Agent& dst_agent,
+                               const void* src, core::Agent& src_agent,
+                               size_t size,
+                               std::vector<core::Signal*>& dep_signals,
+                               core::Signal& out_signal,
+                               int engine_offset,
+                               bool force_copy_on_sdma) {
+  // At this point it is guaranteed that one of
+  // the two devices is a GPU, potentially both
+  assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) ||
+          (dst_agent.device_type() == core::Agent::kAmdGpuDevice)) &&
+         ("Both devices are CPU agents which is not expected"));
+
+  if (engine_offset >= properties_.NumSdmaEngines + properties_.NumSdmaXgmiEngines) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  // check if dst and src are the same gpu or over xGMI.
+  bool is_same_gpu = (src_agent.public_handle().handle == dst_agent.public_handle().handle) &&
+      (dst_agent.public_handle().handle == public_handle_.handle);
+  bool is_xgmi = !is_same_gpu &&
+                   src_agent.device_type() == core::Agent::kAmdGpuDevice &&
+                     dst_agent.device_type() == core::Agent::kAmdGpuDevice &&
+                       dst_agent.HiveId() && src_agent.HiveId() == dst_agent.HiveId() &&
+                         properties_.NumSdmaXgmiEngines;
+
+  // Due to a RAS issue, GFX90a can only support H2D copies on SDMA0
+  bool is_h2d_blit = (src_agent.device_type() == core::Agent::kAmdCpuDevice &&
+    dst_agent.device_type() == core::Agent::kAmdGpuDevice);
+  bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10);
+
+  // Ensure engine selection is within proper range based on transfer type
+  if ((is_xgmi && engine_offset < properties_.NumSdmaEngines) ||
+       (!is_xgmi && engine_offset >= properties_.NumSdmaEngines) ||
+         (!is_h2d_blit && !is_same_gpu && limit_h2d_blit && !engine_offset)) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  lazy_ptr<core::Blit>& blit = is_same_gpu ?
+                                 (force_copy_on_sdma ? blits_[BlitDevToHost] :
+                                   blits_[BlitDevToDev]) : blits_[engine_offset];
+
+  if (profiling_enabled()) {
+    // Track the agent so we could translate the resulting timestamp to system
+    // domain correctly.
+    out_signal.async_copy_agent(core::Agent::Convert(this->public_handle()));
+  }
+
+  hsa_status_t stat = blit->SubmitLinearCopyCommand(dst, src, size, dep_signals, out_signal);
+
+  return stat;
+}
+
 hsa_status_t GpuAgent::DmaCopyStatus(core::Agent& dst_agent, core::Agent& src_agent,
                                      uint32_t *engine_ids_mask) {
   assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) ||

runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp

@@ -365,6 +365,7 @@ void HsaApiTable::UpdateAmdExts() {
   amd_ext_api.hsa_amd_memory_pool_allocate_fn = AMD::hsa_amd_memory_pool_allocate;
   amd_ext_api.hsa_amd_memory_pool_free_fn = AMD::hsa_amd_memory_pool_free;
   amd_ext_api.hsa_amd_memory_async_copy_fn = AMD::hsa_amd_memory_async_copy;
+  amd_ext_api.hsa_amd_memory_async_copy_on_engine_fn = AMD::hsa_amd_memory_async_copy_on_engine;
   amd_ext_api.hsa_amd_memory_copy_engine_status_fn = AMD::hsa_amd_memory_copy_engine_status;
   amd_ext_api.hsa_amd_agent_memory_pool_get_info_fn = AMD::hsa_amd_agent_memory_pool_get_info;
   amd_ext_api.hsa_amd_agents_allow_access_fn = AMD::hsa_amd_agents_allow_access;

runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp

@@ -276,6 +276,50 @@ hsa_status_t hsa_amd_memory_async_copy(void* dst, hsa_agent_t dst_agent_handle,
   CATCH;
 }
 
+hsa_status_t hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent_handle,
+                                       const void* src, hsa_agent_t src_agent_handle, size_t size,
+                                       uint32_t num_dep_signals, const hsa_signal_t* dep_signals,
+                                       hsa_signal_t completion_signal, hsa_amd_sdma_engine_id_t engine_id,
+                                       bool force_copy_on_sdma) {
+  TRY;
+  IS_BAD_PTR(dst);
+  IS_BAD_PTR(src);
+
+  if ((num_dep_signals == 0 && dep_signals != nullptr) ||
+      (num_dep_signals > 0 && dep_signals == nullptr)) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle);
+  IS_VALID(dst_agent);
+
+  core::Agent* src_agent = core::Agent::Convert(src_agent_handle);
+  IS_VALID(src_agent);
+
+  std::vector<core::Signal*> dep_signal_list(num_dep_signals);
+  if (num_dep_signals > 0) {
+    for (size_t i = 0; i < num_dep_signals; ++i) {
+      core::Signal* dep_signal_obj = core::Signal::Convert(dep_signals[i]);
+      IS_VALID(dep_signal_obj);
+      dep_signal_list[i] = dep_signal_obj;
+    }
+  }
+
+  core::Signal* out_signal_obj = core::Signal::Convert(completion_signal);
+  IS_VALID(out_signal_obj);
+
+  bool rev_copy_dir = core::Runtime::runtime_singleton_->flag().rev_copy_dir();
+  if (size > 0) {
+    return core::Runtime::runtime_singleton_->CopyMemoryOnEngine(
+        dst, (rev_copy_dir ? src_agent : dst_agent),
+        src, (rev_copy_dir ? dst_agent : src_agent),
+        size, dep_signal_list, *out_signal_obj, engine_id, force_copy_on_sdma);
+  }
+
+  return HSA_STATUS_SUCCESS;
+  CATCH;
+}
+
 hsa_status_t hsa_amd_memory_copy_engine_status(hsa_agent_t dst_agent_handle, hsa_agent_t src_agent_handle,
                                                uint32_t *engine_ids_mask) {
   core::Agent* dst_agent = core::Agent::Convert(dst_agent_handle);

runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -501,6 +501,24 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent* dst_agent, const void*
                              completion_signal);
 }
 
+hsa_status_t Runtime::CopyMemoryOnEngine(void* dst, core::Agent* dst_agent, const void* src,
+                                 core::Agent* src_agent, size_t size,
+                                 std::vector<core::Signal*>& dep_signals,
+                                 core::Signal& completion_signal,
+                                 hsa_amd_sdma_engine_id_t engine_id, bool force_copy_on_sdma) {
+  const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice);
+  core::Agent* copy_agent = (src_gpu) ? src_agent : dst_agent;
+
+  // engine_id is single bitset unique.
+  int engine_offset = ffs(engine_id) - 1;
+  if (!engine_id || !!((engine_id >> (engine_offset + 1)))) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  return copy_agent->DmaCopyOnEngine(dst, *dst_agent, src, *src_agent, size, dep_signals,
+                             completion_signal, engine_offset, force_copy_on_sdma);
+}
+
 hsa_status_t Runtime::CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent,
                                        uint32_t *engine_ids_mask) {
   const bool src_gpu = (src_agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice);

runtime/hsa-runtime/hsacore.so.def

@@ -182,6 +182,7 @@ global:
 	hsa_amd_queue_cu_get_mask;
 	hsa_amd_memory_fill;
 	hsa_amd_memory_async_copy;
+	hsa_amd_memory_async_copy_on_engine;
 	hsa_amd_memory_copy_engine_status;
 	hsa_amd_memory_async_copy_rect;
 	hsa_amd_memory_lock;

runtime/hsa-runtime/inc/hsa_api_trace.h

@@ -155,6 +155,7 @@ struct AmdExtTable {
   decltype(hsa_amd_memory_pool_allocate)* hsa_amd_memory_pool_allocate_fn;
   decltype(hsa_amd_memory_pool_free)* hsa_amd_memory_pool_free_fn;
   decltype(hsa_amd_memory_async_copy)* hsa_amd_memory_async_copy_fn;
+  decltype(hsa_amd_memory_async_copy_on_engine)* hsa_amd_memory_async_copy_on_engine_fn;
   decltype(hsa_amd_memory_copy_engine_status)* hsa_amd_memory_copy_engine_status_fn;
   decltype(hsa_amd_agent_memory_pool_get_info)* hsa_amd_agent_memory_pool_get_info_fn;
   decltype(hsa_amd_agents_allow_access)* hsa_amd_agents_allow_access_fn;

runtime/hsa-runtime/inc/hsa_ext_amd.h

@@ -1273,6 +1273,41 @@ hsa_status_t HSA_API
                               uint32_t num_dep_signals,
                               const hsa_signal_t* dep_signals,
                               hsa_signal_t completion_signal);
+
+/**
+ * @brief Asynchronously copy a block of memory from the location pointed to by
+ * @p src on the @p src_agent to the memory block pointed to by @p dst on the @p
+ * dst_agent on engine_id.
+ *
+ * WARNING: Concurrent use of this call with hsa_amd_memory_async_copy can result
+ * in resource conflicts as HSA runtime will auto assign engines with the latter
+ * call.  Approach using both calls concurrently with caution.
+ *
+ * All param definitions are identical to hsa_amd_memory_async_copy with the
+ * exception of engine_id and force_copy_on_sdma.
+ *
+ * @param[in] - engine_id Target engine defined by hsa_amd_sdma_engine_id_t.
+ * Client should use hsa_amd_memory_copy_engine_status first to get the ID
+ * availability.
+ *
+ * @param[in] - force_copy_on_sdma By default, blit kernel copies are used when
+ * dst_agent == src_agent.  Setting this to true will force the copy over SDMA1.
+ *
+ * All return definitions are identical to hsa_amd_memory_async_copy with the
+ * following ammendments:
+ *
+ * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT The source or destination
+ * pointers are NULL, or the completion signal is 0 or engine_id is improperly
+ * bounded.
+ */
+hsa_status_t HSA_API
+    hsa_amd_memory_async_copy_on_engine(void* dst, hsa_agent_t dst_agent, const void* src,
+                              hsa_agent_t src_agent, size_t size,
+                              uint32_t num_dep_signals,
+                              const hsa_signal_t* dep_signals,
+                              hsa_signal_t completion_signal,
+                              hsa_amd_sdma_engine_id_t engine_id,
+                              bool force_copy_on_sdma);
 /**
  * @brief Reports the availability of SDMA copy engines.
  *