diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index f5e102be55..ef58813329 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1992,7 +1992,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { const HsaNodeProperties *pNodeProperties; HSAuint32 *mmioBase = NULL; unsigned int *nullPtr = NULL; - std::vector nodes; + std::vector nodes; int numPeers; const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); @@ -2006,8 +2006,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && - !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + if (!m_NodeInfo.IsPeerAccessibleByNode(g_TestDstNodeId, g_TestNodeId)) { LOG() << "Skipping test: first GPU specified is not peer-accessible." << std::endl; return; } @@ -2018,7 +2017,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { } } else { HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, false); + m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU); if (nodes.size() < 2) { LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; LOG() << " or two GPUs are XGMI connected." << std::endl; diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index e80e4e71d6..32f342bd4d 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -145,7 +145,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { } const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); - std::vector nodes; + std::vector nodes; const bool isSpecified = g_TestDstNodeId != -1 && g_TestNodeId != -1; int numPeers = 0; @@ -153,14 +153,13 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { if (g_TestNodeId != g_TestDstNodeId) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if ((m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && - m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId)) || - m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) + if ((m_NodeInfo.IsPeerAccessibleByNode(g_TestNodeId, g_TestDstNodeId) && + m_NodeInfo.IsPeerAccessibleByNode(g_TestDstNodeId, g_TestNodeId))) numPeers = 2; } } else { - HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + nodes = m_NodeInfo.GetNodesWithGPU(); + numPeers = nodes.size(); } if (numPeers < 2) { @@ -168,7 +167,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { return; } - std::vector sysNodes(nodes); // include sysMem node 0... + std::vector sysNodes(nodes); // include sysMem node 0... sysNodes.insert(sysNodes.begin(),0); const int total_tests = 7; @@ -230,6 +229,9 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { if (n1 == n2) continue; + if (!m_NodeInfo.IsPeerAccessibleByNode(n2, n1)) + continue; + snprintf(str, sizeof(str), "[%d -> %d] ", n1, n2); msg << str << std::endl; testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); @@ -251,6 +253,10 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { for (unsigned j = i + 1; j < nodes.size(); j++) { HSAuint32 n2 = nodes[j]; + if (!m_NodeInfo.IsPeerAccessibleByNode(n2, n1) || + !m_NodeInfo.IsPeerAccessibleByNode(n1, n2)) + continue; + snprintf(str, sizeof(str), "[%d <-> %d] ", n1, n2); msg << str << std::endl; testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); @@ -265,8 +271,8 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { LOG() << test_suits_string[s] << std::endl; msg << test_suits_string[s] << std::endl; /* Just use GPU nodes to do copy.*/ - std::vector &src = test_suits[s][0] != NONE ? nodes : sysNodes; - std::vector &dst = test_suits[s][1] != NONE ? nodes : sysNodes; + std::vector &src = test_suits[s][0] != NONE ? nodes : sysNodes; + std::vector &dst = test_suits[s][1] != NONE ? nodes : sysNodes; for (unsigned i = 0; i < src.size(); i++) { HSAuint32 n1 = src[i]; @@ -275,9 +281,18 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { int n = 0; char str[64]; - for (unsigned j = 0; j < dst.size(); j++) - if (dst[j] != n1) + for (unsigned j = 0; j < dst.size(); j++) { + if (dst[j] != n1) { + if (test_suits[s][0] != NONE && + !m_NodeInfo.IsPeerAccessibleByNode(dst[j], n1)) + continue; + if (test_suits[s][1] != NONE && + !m_NodeInfo.IsPeerAccessibleByNode(n1, dst[j])) + continue; n2[n++] = dst[j]; + } + } + /* At least 2 dst GPUs.*/ if (n < 2) continue; @@ -293,7 +308,6 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { (float)speed2 / 1024 << " GB/s" << std::endl; } } - exit: /* New line.*/ LOG() << std::endl << msg.str() << std::endl; @@ -309,17 +323,17 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) { } const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); - std::vector nodes; + std::vector nodes; - HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - int numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + nodes = m_NodeInfo.GetNodesWithGPU(); + int numPeers = nodes.size(); if (numPeers < 2) { LOG() << "Skipping test: Need at least two large bar GPU or XGMI connected." << std::endl; return; } - std::vector sysNodes(nodes); // include sysMem node 0... + std::vector sysNodes(nodes); // include sysMem node 0... sysNodes.insert(sysNodes.begin(),0); /* size should be small.*/ @@ -352,6 +366,9 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) { HSAuint32 n2 = sysNodes[j]; std::stringstream msg; + if (n1 != n2 && !m_NodeInfo.IsPeerAccessibleByNode(n2, n1)) + continue; + msg << test_suits_string[s] << "[" << n1 << " -> " << n2 << "]"; for (auto &size : sizeArray) { testNodeToNodes(n1, &n2, 1, test_suits[s], NONE, size, 0, 0, 0, 1, &time); diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index eab1b54885..ae561fccb2 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1625,7 +1625,7 @@ TEST_F(KFDQMTest, P2PTest) { LOG() << "Skipping test: At least two GPUs are required." << std::endl; return; } - std::vector nodes; + std::vector nodes; /* This test simulates RT team's P2P part in IPCtest: * @@ -1647,8 +1647,7 @@ TEST_F(KFDQMTest, P2PTest) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId) && - !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + if (!m_NodeInfo.IsPeerAccessibleByNode(g_TestNodeId, g_TestDstNodeId)) { LOG() << "Skipping test: Dst GPU specified is not peer-accessible." << std::endl; return; } @@ -1657,8 +1656,7 @@ TEST_F(KFDQMTest, P2PTest) { return; } } else { - HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + nodes = m_NodeInfo.GetNodesWithGPU(); if (nodes.size() < 2) { LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; LOG() << " or two GPUs are XGMI connected." << std::endl; @@ -1685,7 +1683,7 @@ TEST_F(KFDQMTest, P2PTest) { EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, reinterpret_cast(&sysBuf))); EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL, - mapFlags, nodes.size(), &nodes[0])); + mapFlags, nodes.size(), (HSAuint32 *)&nodes[0])); #define MAGIC_NUM 0xdeadbeaf /* First GPU fills mem with MAGIC_NUM */ @@ -1707,6 +1705,11 @@ TEST_F(KFDQMTest, P2PTest) { } else { n = 2; next = nodes[i]; + + /* check if cur access next node */ + if (!m_NodeInfo.IsPeerAccessibleByNode(next, cur)) + continue; + ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, reinterpret_cast(&dst))); ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dst, size, NULL)); } diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index b55cd80247..1e1b85abda 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -616,6 +616,23 @@ const bool HsaNodeInfo::IsGPUNodeLargeBar(int node) const { return false; } +const bool HsaNodeInfo::IsPeerAccessibleByNode(int peer, int node) const { + const HsaNodeProperties *pNodeProperties; + + pNodeProperties = GetNodeProperties(node); + if (pNodeProperties) { + HsaIoLinkProperties p2pLinksProperties[pNodeProperties->NumIOLinks]; + EXPECT_SUCCESS(hsaKmtGetNodeIoLinkProperties(node, + pNodeProperties->NumIOLinks, p2pLinksProperties)); + + for (unsigned link = 0; link < pNodeProperties->NumIOLinks; link++) + if (p2pLinksProperties[link].NodeTo == peer) + return true; + } + + return false; +} + const int HsaNodeInfo::FindLargeBarGPUNode() const { const std::vector gpuNodes = GetNodesWithGPU(); @@ -637,27 +654,16 @@ const bool HsaNodeInfo::AreGPUNodesXGMI(int node0, int node1) const { return false; } -int HsaNodeInfo::FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, - bool bidirectional) const { - peers->push_back(dstNode); - if (IsGPUNodeLargeBar(dstNode)) { - for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { - if (m_NodesWithGPU.at(i) == dstNode) - continue; +int HsaNodeInfo::FindAccessiblePeers(std::vector *peers, + HSAuint32 node) const { + peers->push_back(node); - if (!bidirectional || IsGPUNodeLargeBar(m_NodesWithGPU.at(i)) || - AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) - peers->push_back(m_NodesWithGPU.at(i)); - } - } else { - for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { - if (m_NodesWithGPU.at(i) == dstNode) - continue; + for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { + if (m_NodesWithGPU.at(i) == node) + continue; - if (AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) - peers->push_back(m_NodesWithGPU.at(i)); - } + if (IsPeerAccessibleByNode(m_NodesWithGPU.at(i), node)) + peers->push_back(m_NodesWithGPU.at(i)); } - return peers->size(); } diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 28847f370e..e55ca95062 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -190,12 +190,13 @@ class HsaNodeInfo { void PrintNodeInfo() const; const bool IsGPUNodeLargeBar(int node) const; + const bool IsPeerAccessibleByNode(int peer, int node) const; // @brief Find the first available Large-BAR GPU node // @return: Node ID if successful or -1 const int FindLargeBarGPUNode() const; const bool AreGPUNodesXGMI(int node0, int node1) const; - int FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, - bool bidirectional) const; + int FindAccessiblePeers(std::vector *peers, + HSAuint32 node) const; }; #endif // __KFD__TEST__UTIL__H__