P4 to Git Change 2029039 by kjayapra@0_HIPWS_LNX1_ROCM on 2019/11/12 12:52:46

SWDEV-210844 - Implementing hipExtGetLinkTypeAndHopCount

Affected files ...

... //depot/stg/opencl/drivers/opencl/api/hip/hip_device_runtime.cpp#22 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#344 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#47 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#141 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#44 edit

rocclr/runtime/device/device.hpp

@@ -1411,6 +1411,12 @@ class Device : public RuntimeObject {
   //! Returns index of current device
   uint32_t index() const { return index_; }
 
+  virtual bool findLinkTypeAndHopCount(amd::Device* other_device, uint32_t* link_type,
+                                       uint32_t* hop_count) {
+    ShouldNotReachHere();
+    return false;
+  }
+
  protected:
   //! Enable the specified extension
   char* getExtensionString();

rocclr/runtime/device/pal/paldevice.hpp

@@ -545,6 +545,13 @@ class Device : public NullDevice {
   std::map<Pal::IQueue*, QueueRecycleInfo*>& QueuePool() { return queue_pool_; }
   const std::map<Pal::IQueue*, QueueRecycleInfo*>& QueuePool() const { return queue_pool_; }
 
+  virtual bool findLinkTypeAndHopCount(amd::Device* other_device, uint32_t* link_type,
+                                       uint32_t* hop_count) {
+    /* Not Supported in PAL yet */
+    ShouldNotReachHere();
+    return false;
+  }
+
  private:
   static void PAL_STDCALL PalDeveloperCallback(void* pPrivateData, const Pal::uint32 deviceIndex,
                                                Pal::Developer::CallbackType type, void* pCbData);

rocclr/runtime/device/rocm/rocdevice.cpp

@@ -875,7 +875,11 @@ hsa_status_t Device::iterateGpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, vo
 
         if ((global_flag & HSA_REGION_GLOBAL_FLAG_FINE_GRAINED) != 0) {
           dev->gpu_fine_grained_segment_ = pool;
-        } else {
+        } else if ((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) != 0) {
+          dev->gpuvm_segment_ = pool;
+        }
+
+        if (dev->gpuvm_segment_.handle == 0) {
           dev->gpuvm_segment_ = pool;
         }
       }
@@ -1914,5 +1918,28 @@ void Device::releaseQueue(hsa_queue_t* queue) {
   queuePool_.erase(qIter);
 }
 
+bool Device::findLinkTypeAndHopCount(amd::Device* other_device,
+                                     uint32_t* link_type, uint32_t* hop_count) {
+  hsa_amd_memory_pool_link_info_t link_info;
+  hsa_amd_memory_pool_t pool = (static_cast<roc::Device*>(other_device))->gpuvm_segment_;
+
+  if (pool.handle != 0) {
+    if (HSA_STATUS_SUCCESS
+        != hsa_amd_agent_memory_pool_get_info(this->getBackendDevice(), pool,
+                                              HSA_AMD_AGENT_MEMORY_POOL_INFO_LINK_INFO,
+                                              &link_info)) {
+      return false;
+    }
+
+    *link_type = link_info.link_type;
+    if (link_info.numa_distance < 30) {
+      *hop_count = 1;
+    } else {
+      *hop_count = 2;
+    }
+  }
+  return true;
+}
+
 } // namespace roc
 #endif  // WITHOUT_HSA_BACKEND

rocclr/runtime/device/rocm/rocdevice.hpp

@@ -420,6 +420,9 @@ class Device : public NullDevice {
   //! Return multi GPU grid launch sync buffer
   address MGSync() const { return mg_sync_; }
 
+  virtual bool findLinkTypeAndHopCount(amd::Device* other_device, uint32_t* link_type,
+                                       uint32_t* hop_count);
+
  private:
   static hsa_ven_amd_loader_1_00_pfn_t amd_loader_ext_table;