rocr: Add rec sdma engines with limited XGMI SDMA engine

This patch will adds recommended sdma supports with
limited XGMI SDMA engine. It will use one PCIe SDMA
to do gpu <-> gpu copies which will help improve all
to all copy performance.

Signed-off-by: Shane Xiao <shane.xiao@amd.com>


[ROCm/ROCR-Runtime commit: 6a63170b38]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h

@@ -171,6 +171,8 @@ class GpuAgentInt : public core::Agent {
    virtual void RegisterRecSdmaEngIdMaskPeer(core::Agent &gang_peer,
                                              uint32_t rec_sdma_eng_id_mask) = 0;
 
+   virtual void SetRecSdmaEngOverride(bool flag) = 0;
+
    // @brief Query if agent represent Kaveri GPU.
    //
    // @retval true if agent is Kaveri GPU.
@@ -376,6 +378,9 @@ class GpuAgent : public GpuAgentInt {
     return properties_;
   }
 
+  // @brief set rec_sdma_eng_override_
+  __forceinline void SetRecSdmaEngOverride(bool flag) override { rec_sdma_eng_override_ = flag; }
+
   // @brief Returns number of data caches.
   __forceinline size_t num_cache() const { return cache_props_.size(); }
 
@@ -798,6 +803,7 @@ class GpuAgent : public GpuAgentInt {
   std::map<uint64_t, uint32_t> rec_sdma_eng_id_peers_info_;
 
   bool uses_rec_sdma_eng_id_mask_;
+  bool rec_sdma_eng_override_;
 
   // structure for host trap sampling
   pcs_data_t pcs_hosttrap_data_;

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -117,7 +117,8 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna
       trap_handler_tma_region_(NULL),
       pcs_hosttrap_data_(),
       pcs_stochastic_data_(),
-      xgmi_cpu_gpu_(false) {
+      xgmi_cpu_gpu_(false),
+      rec_sdma_eng_override_(false) {
   const bool is_apu_node = (properties_.NumCPUCores > 0);
   profile_ = (is_apu_node) ? HSA_PROFILE_FULL : HSA_PROFILE_BASE;
 
@@ -1160,7 +1161,7 @@ hsa_status_t GpuAgent::DmaCopyOnEngine(void* dst, core::Agent& dst_agent,
     bool limit_h2d_blit = isa_->GetVersion() == core::Isa::Version(9, 0, 10);
 
     // Ensure engine selection is within proper range based on transfer type
-    if ((is_xgmi && engine_offset <= properties_.NumSdmaEngines) ||
+    if ((is_xgmi && !rec_sdma_eng_override_ && engine_offset <= properties_.NumSdmaEngines) ||
         (!is_xgmi && engine_offset > (properties_.NumSdmaEngines +
                                       properties_.NumSdmaXgmiEngines)) ||
           (!is_h2d_blit && !is_same_gpu && limit_h2d_blit &&

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp

@@ -381,6 +381,32 @@ bool BuildTopology() {
   }
   const_cast<Flag&>(rt->flag()).parse_masks(maxGpu, maxCu);
 
+  // Front load the rec_sdma_eng_id_mask to check whether needs to override old mask
+  bool rec_sdma_engine_override = false;
+  for (auto& src_gpu : rt->gpu_agents()) {
+    uint32_t src_id = src_gpu->node_id();
+
+    // skip the pre-loop if NumSdmaXgmiEngines != 6
+    if (((AMD::GpuAgent*)src_gpu)->properties().NumSdmaXgmiEngines != 6)
+      break;
+
+    for (auto& dst_gpu : rt->gpu_agents()) {
+      uint32_t dst_id = dst_gpu->node_id();
+      if (src_id != dst_id) {
+        auto linfo = rt->GetLinkInfo(src_id, dst_id);
+        if (IsPowerOfTwo(linfo.rec_sdma_eng_id_mask)) {
+          rec_sdma_engine_override = true;
+          ((AMD::GpuAgent*)src_gpu)->SetRecSdmaEngOverride(rec_sdma_engine_override);
+          break;
+        }
+      }
+    }
+
+    // skip the pre-loop if RecSdmaEngOverride is true
+    if (rec_sdma_engine_override)
+      break;
+  }
+
   // Register destination agents that can SDMA gang copy for source agents
   for (auto& src_gpu : rt->gpu_agents()) {
     uint32_t src_id = src_gpu->node_id();
@@ -406,6 +432,15 @@ bool BuildTopology() {
           else gang_factor = 1;
 
           rec_sdma_eng_id_mask = linfo.rec_sdma_eng_id_mask;
+
+          // Override the old mask if rec sdma eng verride is true
+          // Using one pcie sdma for device to device copy with limited XGMI SDMA engine.
+          // This will help improve all to all copy with limited XGMI SDMA engine.
+          if (rec_sdma_engine_override) {
+            uint32_t sdma_engine_mask = (1 << ((AMD::GpuAgent*)src_gpu)->properties().NumSdmaEngines - 1);
+            rec_sdma_eng_id_mask = !IsPowerOfTwo(rec_sdma_eng_id_mask) ?
+              sdma_engine_mask : rec_sdma_eng_id_mask;
+          }
         }
       }