From 7970890e4b2dea5d30552784eb3fec5dec75b3bb Mon Sep 17 00:00:00 2001
From: kjayapra-amd <karthik.jayaprakash@amd.com>
Date: Thu, 13 Aug 2020 16:09:51 -0400
Subject: [PATCH] SWDEV-240800 - P2P device attributes support

Change-Id: Ia32b87ffea17e0d98b69a07f2633ba14e7637b8a


[ROCm/hip commit: 1f8543560f7c7ac7745232498329fef2fd11a663]
---
 projects/hip/rocclr/hip_device_runtime.cpp | 26 -------
 projects/hip/rocclr/hip_peer.cpp           | 86 ++++++++++++++++++----
 2 files changed, 73 insertions(+), 39 deletions(-)

diff --git a/projects/hip/rocclr/hip_device_runtime.cpp b/projects/hip/rocclr/hip_device_runtime.cpp
index 470b088f02..350363b1b6 100755
--- a/projects/hip/rocclr/hip_device_runtime.cpp
+++ b/projects/hip/rocclr/hip_device_runtime.cpp
@@ -551,29 +551,3 @@ hipError_t hipSetValidDevices ( int* device_arr, int  len ) {
 
   HIP_RETURN(hipErrorNotSupported);
 }
-
-hipError_t hipExtGetLinkTypeAndHopCount(int device1, int device2, uint32_t* linktype, uint32_t* hopcount) {
-  HIP_INIT_API(hipExtGetLinkTypeAndHopCount, device1, device2, linktype, hopcount);
-
-  amd::Device* amd_dev_obj1 = nullptr;
-  amd::Device* amd_dev_obj2 = nullptr;
-  const int numDevices = static_cast<int>(g_devices.size());
-
-  if ((device1 < 0) || (device1 >= numDevices) || (device2 < 0) || (device2 >= numDevices)) {
-    HIP_RETURN(hipErrorInvalidDevice);
-  }
-
-  if ((linktype == nullptr) || (hopcount == nullptr)) {
-    HIP_RETURN(hipErrorInvalidValue);
-  }
-
-  amd_dev_obj1 = g_devices[device1]->devices()[0];
-  amd_dev_obj2 = g_devices[device2]->devices()[0];
-
-  if (!amd_dev_obj1->findLinkTypeAndHopCount(amd_dev_obj2, linktype, hopcount)) {
-    HIP_RETURN(hipErrorInvalidHandle);
-  }
-
-  HIP_RETURN(hipSuccess);
-}
-
diff --git a/projects/hip/rocclr/hip_peer.cpp b/projects/hip/rocclr/hip_peer.cpp
index 24207b52c6..ded6843957 100755
--- a/projects/hip/rocclr/hip_peer.cpp
+++ b/projects/hip/rocclr/hip_peer.cpp
@@ -72,12 +72,48 @@ hipError_t canAccessPeer(int* canAccessPeer, int deviceId, int peerDeviceId){
   return hipSuccess;
 }
 
+hipError_t findLinkInfo(int device1, int device2,
+                        std::vector<amd::Device::LinkAttrType>* link_attrs) {
+
+  amd::Device* amd_dev_obj1 = nullptr;
+  amd::Device* amd_dev_obj2 = nullptr;
+  const int numDevices = static_cast<int>(g_devices.size());
+
+  if ((device1 < 0) || (device1 >= numDevices) || (device2 < 0) || (device2 >= numDevices)) {
+    return hipErrorInvalidDevice;
+  }
+
+  amd_dev_obj1 = g_devices[device1]->devices()[0];
+  amd_dev_obj2 = g_devices[device2]->devices()[0];
+
+  if (!amd_dev_obj1->findLinkInfo(*amd_dev_obj2, link_attrs)) {
+    return hipErrorInvalidHandle;
+  }
+
+  return hipSuccess;
+}
+
+hipError_t hipExtGetLinkTypeAndHopCount(int device1, int device2,
+                                        uint32_t* linktype, uint32_t* hopcount) {
+  HIP_INIT_API(hipExtGetLinkTypeAndHopCount, device1, device2, linktype, hopcount);
+
+  // Fill out the list of LinkAttributes
+  std::vector<amd::Device::LinkAttrType> link_attrs;
+  link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0));
+  link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkHopCount, 0));
+
+  HIP_RETURN_ONFAIL(findLinkInfo(device1, device2, &link_attrs));
+
+  *linktype = static_cast<uint32_t>(link_attrs[0].second);
+  *hopcount = static_cast<uint32_t>(link_attrs[1].second);
+
+  HIP_RETURN(hipSuccess);
+}
+
 hipError_t hipDeviceGetP2PAttribute(int* value, hipDeviceP2PAttr attr,
                                     int srcDevice, int dstDevice) {
   HIP_INIT_API(hipDeviceGetP2PAttribute, value, attr, srcDevice, dstDevice);
 
-  hipError_t hip_error = hipSuccess;
-
   if (value == nullptr) {
     HIP_RETURN(hipErrorInvalidValue);
   }
@@ -87,26 +123,50 @@ hipError_t hipDeviceGetP2PAttribute(int* value, hipDeviceP2PAttr attr,
     HIP_RETURN(hipErrorInvalidDevice);
   }
 
+  std::vector<amd::Device::LinkAttrType> link_attrs;
+
   switch (attr) {
-    case hipDevP2PAttrPerformanceRank :
-      assert(0 && "Unimplemented");
+    case hipDevP2PAttrPerformanceRank : {
+      link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0));
       break;
-    case hipDevP2PAttrAccessSupported :
-      hip_error = canAccessPeer(value, srcDevice, dstDevice);
+    }
+    case hipDevP2PAttrAccessSupported : {
+      HIP_RETURN_ONFAIL(canAccessPeer(value, srcDevice, dstDevice));
       break;
-    case hipDevP2PAttrNativeAtomicSupported :
-      assert(0 && "Unimplemented");
+    }
+    case hipDevP2PAttrNativeAtomicSupported : {
+      link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0));
       break;
-    case hipDevP2PAttrHipArrayAccessSupported :
-      assert(0 && "Unimplemented");
+    }
+    case hipDevP2PAttrHipArrayAccessSupported : {
+      hipDeviceProp_t srcDeviceProp;
+      hipDeviceProp_t dstDeviceProp;
+      HIP_RETURN_ONFAIL(hipGetDeviceProperties(&srcDeviceProp, srcDevice));
+      HIP_RETURN_ONFAIL(hipGetDeviceProperties(&dstDeviceProp, dstDevice));
+
+      // Linear layout access is supported if P2P is enabled
+      // Opaque Images are supported only on homogeneous systems
+      // Might have more conditions to check, in future.
+      if (srcDeviceProp.gcnArch == dstDeviceProp.gcnArch) {
+        HIP_RETURN_ONFAIL(canAccessPeer(value, srcDevice, dstDevice));
+      } else {
+        value = 0;
+      }
       break;
-    default :
+    }
+    default : {
       DevLogPrintfError("Invalid attribute attr: %d ", attr);
-      hip_error = hipErrorInvalidValue;
+      HIP_RETURN(hipErrorInvalidValue);
       break;
+    }
   }
 
-  HIP_RETURN(hip_error);
+  if ((attr != hipDevP2PAttrAccessSupported) && (attr != hipDevP2PAttrHipArrayAccessSupported)) {
+    HIP_RETURN_ONFAIL(findLinkInfo(srcDevice, dstDevice, &link_attrs));
+    *value = static_cast<int>(link_attrs[0].second);
+  }
+
+  HIP_RETURN(hipSuccess);
 }
 
 hipError_t hipDeviceCanAccessPeer(int* canAccess, int deviceId, int peerDeviceId) {