kfdtest: check peer accessible with new function

check GPU peer accessible with p2p_links in system

Signed-off-by: Gang Ba <gaba@amd.com>
Change-Id: I026f16564303b687811d6648f0b7f84be6819979

tests/kfdtest/src/KFDMemoryTest.cpp

@@ -1992,7 +1992,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) {
     const HsaNodeProperties *pNodeProperties;
     HSAuint32 *mmioBase = NULL;
     unsigned int *nullPtr = NULL;
-    std::vector<HSAuint32> nodes;
+    std::vector<int> nodes;
     int numPeers;
 
     const std::vector<int> gpuNodes = m_NodeInfo.GetNodesWithGPU();
@@ -2006,8 +2006,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) {
         nodes.push_back(g_TestNodeId);
         nodes.push_back(g_TestDstNodeId);
 
-        if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) &&
-            !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) {
+        if (!m_NodeInfo.IsPeerAccessibleByNode(g_TestDstNodeId, g_TestNodeId)) {
             LOG() << "Skipping test: first GPU specified is not peer-accessible." << std::endl;
             return;
         }
@@ -2018,7 +2017,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) {
         }
     } else {
         HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode();
-        m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, false);
+        m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU);
         if (nodes.size() < 2) {
             LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl;
             LOG() << "               or two GPUs are XGMI connected." << std::endl;

tests/kfdtest/src/KFDPerformanceTest.cpp

@@ -145,7 +145,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
     }
 
     const std::vector<int> gpuNodes = m_NodeInfo.GetNodesWithGPU();
-    std::vector<HSAuint32> nodes;
+    std::vector<int> nodes;
     const bool isSpecified = g_TestDstNodeId != -1 && g_TestNodeId != -1;
     int numPeers = 0;
 
@@ -153,14 +153,13 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
         if (g_TestNodeId != g_TestDstNodeId) {
             nodes.push_back(g_TestNodeId);
             nodes.push_back(g_TestDstNodeId);
-            if ((m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) &&
-                 m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId)) ||
-                m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId))
+            if ((m_NodeInfo.IsPeerAccessibleByNode(g_TestNodeId, g_TestDstNodeId) &&
+                 m_NodeInfo.IsPeerAccessibleByNode(g_TestDstNodeId, g_TestNodeId)))
                 numPeers = 2;
         }
     } else {
-        HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode();
-        numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true);
+        nodes = m_NodeInfo.GetNodesWithGPU();
+        numPeers = nodes.size();
     }
 
     if (numPeers < 2) {
@@ -168,7 +167,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
         return;
     }
 
-    std::vector<HSAuint32> sysNodes(nodes); // include sysMem node 0...
+    std::vector<int> sysNodes(nodes); // include sysMem node 0...
     sysNodes.insert(sysNodes.begin(),0);
 
     const int total_tests = 7;
@@ -230,6 +229,9 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
                 if (n1 == n2)
                     continue;
 
+                if (!m_NodeInfo.IsPeerAccessibleByNode(n2, n1))
+                    continue;
+
                 snprintf(str, sizeof(str), "[%d -> %d] ", n1, n2);
                 msg << str << std::endl;
                 testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg);
@@ -251,6 +253,10 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
             for (unsigned j = i + 1; j < nodes.size(); j++) {
                 HSAuint32 n2 = nodes[j];
 
+                if (!m_NodeInfo.IsPeerAccessibleByNode(n2, n1) ||
+                    !m_NodeInfo.IsPeerAccessibleByNode(n1, n2))
+                    continue;
+
                 snprintf(str, sizeof(str), "[%d <-> %d] ", n1, n2);
                 msg << str << std::endl;
                 testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg);
@@ -265,8 +271,8 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
         LOG() << test_suits_string[s] << std::endl;
         msg << test_suits_string[s] << std::endl;
         /* Just use GPU nodes to do copy.*/
-        std::vector<HSAuint32> &src = test_suits[s][0] != NONE ? nodes : sysNodes;
-        std::vector<HSAuint32> &dst = test_suits[s][1] != NONE ? nodes : sysNodes;
+        std::vector<int> &src = test_suits[s][0] != NONE ? nodes : sysNodes;
+        std::vector<int> &dst = test_suits[s][1] != NONE ? nodes : sysNodes;
 
         for (unsigned i = 0; i < src.size(); i++) {
             HSAuint32 n1 = src[i];
@@ -275,9 +281,18 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
             int n = 0;
             char str[64];
 
-            for (unsigned j = 0; j < dst.size(); j++)
-                if (dst[j] != n1)
+            for (unsigned j = 0; j < dst.size(); j++) {
+                if (dst[j] != n1) {
+                    if (test_suits[s][0] != NONE &&
+                        !m_NodeInfo.IsPeerAccessibleByNode(dst[j], n1))
+                            continue;
+                    if (test_suits[s][1] != NONE &&
+                        !m_NodeInfo.IsPeerAccessibleByNode(n1, dst[j]))
+                            continue;
                     n2[n++] = dst[j];
+                }
+            }
+
             /* At least 2 dst GPUs.*/
             if (n < 2)
                 continue;
@@ -293,7 +308,6 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
                                         (float)speed2 / 1024 << " GB/s" << std::endl;
         }
     }
-
 exit:
     /* New line.*/
     LOG() << std::endl << msg.str() << std::endl;
@@ -309,17 +323,17 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) {
     }
 
     const std::vector<int> gpuNodes = m_NodeInfo.GetNodesWithGPU();
-    std::vector<HSAuint32> nodes;
+    std::vector<int> nodes;
 
-    HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode();
-    int numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true);
+    nodes = m_NodeInfo.GetNodesWithGPU();
+    int numPeers = nodes.size();
 
     if (numPeers < 2) {
         LOG() << "Skipping test: Need at least two large bar GPU or XGMI connected." << std::endl;
         return;
     }
 
-    std::vector<HSAuint32> sysNodes(nodes); // include sysMem node 0...
+    std::vector<int> sysNodes(nodes); // include sysMem node 0...
     sysNodes.insert(sysNodes.begin(),0);
 
     /* size should be small.*/
@@ -352,6 +366,9 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) {
                 HSAuint32 n2 = sysNodes[j];
                 std::stringstream msg;
 
+                if (n1 != n2 && !m_NodeInfo.IsPeerAccessibleByNode(n2, n1))
+                    continue;
+
                 msg << test_suits_string[s] << "[" << n1 << " -> " << n2 << "]";
                 for (auto &size : sizeArray) {
                     testNodeToNodes(n1, &n2, 1, test_suits[s], NONE, size, 0, 0, 0, 1, &time);

tests/kfdtest/src/KFDQMTest.cpp

@@ -1625,7 +1625,7 @@ TEST_F(KFDQMTest, P2PTest) {
         LOG() << "Skipping test: At least two GPUs are required." << std::endl;
         return;
     }
-    std::vector<HSAuint32> nodes;
+    std::vector<int> nodes;
 
     /* This test simulates RT team's P2P part in IPCtest:
      *
@@ -1647,8 +1647,7 @@ TEST_F(KFDQMTest, P2PTest) {
         nodes.push_back(g_TestNodeId);
         nodes.push_back(g_TestDstNodeId);
 
-        if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId) &&
-            !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) {
+        if (!m_NodeInfo.IsPeerAccessibleByNode(g_TestNodeId, g_TestDstNodeId)) {
             LOG() << "Skipping test: Dst GPU specified is not peer-accessible." << std::endl;
             return;
         }
@@ -1657,8 +1656,7 @@ TEST_F(KFDQMTest, P2PTest) {
             return;
         }
     } else {
-        HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode();
-        m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true);
+        nodes = m_NodeInfo.GetNodesWithGPU();
         if (nodes.size() < 2) {
             LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl;
             LOG() << "               or two GPUs are XGMI connected." << std::endl;
@@ -1685,7 +1683,7 @@ TEST_F(KFDQMTest, P2PTest) {
     EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags,
                                      reinterpret_cast<void **>(&sysBuf)));
     EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL,
-                                             mapFlags, nodes.size(), &nodes[0]));
+                                             mapFlags, nodes.size(), (HSAuint32 *)&nodes[0]));
 #define MAGIC_NUM 0xdeadbeaf
 
     /* First GPU fills mem with MAGIC_NUM */
@@ -1707,6 +1705,11 @@ TEST_F(KFDQMTest, P2PTest) {
         } else {
             n = 2;
             next = nodes[i];
+
+            /* check if cur access next node */
+            if (!m_NodeInfo.IsPeerAccessibleByNode(next, cur))
+                continue;
+
             ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, reinterpret_cast<void**>(&dst)));
             ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dst, size, NULL));
         }

tests/kfdtest/src/KFDTestUtil.cpp

@@ -616,6 +616,23 @@ const bool HsaNodeInfo::IsGPUNodeLargeBar(int node) const {
     return false;
 }
 
+const bool HsaNodeInfo::IsPeerAccessibleByNode(int peer, int node) const {
+    const HsaNodeProperties *pNodeProperties;
+
+    pNodeProperties = GetNodeProperties(node);
+    if (pNodeProperties) {
+        HsaIoLinkProperties p2pLinksProperties[pNodeProperties->NumIOLinks];
+        EXPECT_SUCCESS(hsaKmtGetNodeIoLinkProperties(node,
+					pNodeProperties->NumIOLinks, p2pLinksProperties));
+
+        for (unsigned link = 0; link < pNodeProperties->NumIOLinks; link++)
+            if (p2pLinksProperties[link].NodeTo == peer)
+                return true;
+    }
+
+    return false;
+}
+
 const int HsaNodeInfo::FindLargeBarGPUNode() const {
     const std::vector<int> gpuNodes = GetNodesWithGPU();
 
@@ -637,27 +654,16 @@ const bool HsaNodeInfo::AreGPUNodesXGMI(int node0, int node1) const {
     return false;
 }
 
-int HsaNodeInfo::FindAccessiblePeers(std::vector<HSAuint32> *peers, HSAuint32 dstNode,
-        bool bidirectional) const {
-    peers->push_back(dstNode);
-    if (IsGPUNodeLargeBar(dstNode)) {
-        for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) {
-            if (m_NodesWithGPU.at(i) == dstNode)
-                continue;
+int HsaNodeInfo::FindAccessiblePeers(std::vector<int> *peers,
+		                             HSAuint32 node) const {
+    peers->push_back(node);
 
-            if (!bidirectional || IsGPUNodeLargeBar(m_NodesWithGPU.at(i)) ||
-                AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i)))
-                peers->push_back(m_NodesWithGPU.at(i));
-        }
-    } else {
-        for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) {
-            if (m_NodesWithGPU.at(i) == dstNode)
-                continue;
+    for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) {
+        if (m_NodesWithGPU.at(i) == node)
+            continue;
 
-            if (AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i)))
-                peers->push_back(m_NodesWithGPU.at(i));
-        }
+        if (IsPeerAccessibleByNode(m_NodesWithGPU.at(i), node))
+            peers->push_back(m_NodesWithGPU.at(i));
     }
-
     return peers->size();
 }

tests/kfdtest/src/KFDTestUtil.hpp

@@ -190,12 +190,13 @@ class HsaNodeInfo {
 
     void PrintNodeInfo() const;
     const bool IsGPUNodeLargeBar(int node) const;
+    const bool IsPeerAccessibleByNode(int peer, int node) const;
     // @brief Find the first available Large-BAR GPU node
     // @return: Node ID if successful or -1
     const int FindLargeBarGPUNode() const;
     const bool AreGPUNodesXGMI(int node0, int node1) const;
-    int FindAccessiblePeers(std::vector<HSAuint32> *peers, HSAuint32 dstNode,
-            bool bidirectional) const;
+    int FindAccessiblePeers(std::vector<int> *peers,
+                                        HSAuint32 node) const;
 };
 
 #endif  // __KFD__TEST__UTIL__H__