From 0174377351d7124de2d57366ddabd9fb11ffe550 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Fri, 4 Oct 2019 16:49:38 -0400 Subject: [PATCH] kfdtest: add xgmi path for p2p tests When large bar is not available, we can use xgmi to do p2p tests. Change-Id: Ib7b59fb8a4d41f605739a0428973f6b2f1a3450f Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDMemoryTest.cpp | 20 ++++++------- tests/kfdtest/src/KFDPerformanceTest.cpp | 32 +++++++++++++-------- tests/kfdtest/src/KFDQMTest.cpp | 13 +++++---- tests/kfdtest/src/KFDTestUtil.cpp | 36 ++++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 3 ++ 5 files changed, 75 insertions(+), 29 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 10426392ed..33ef74862c 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1967,6 +1967,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { HSAuint32 *mmioBase = NULL; unsigned int *nullPtr = NULL; std::vector nodes; + int numPeers; const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); if (gpuNodes.size() < 2) { @@ -1978,26 +1979,23 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[0])) { - LOG() << "Skipping test: first GPU specified is not a large bar GPU." << std::endl; + + if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && + !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + LOG() << "Skipping test: first GPU specified is not peer-accessible." << std::endl; return; } + if (nodes[0] == nodes[1]) { LOG() << "Skipping test: Different GPUs must be specified (2 GPUs required)." << std::endl; return; } } else { HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPU)) { - LOG() << "Skipping test: Default GPUs must be large bar." << std::endl; - return; - } - nodes.push_back(defaultGPU); - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (gpuNodes.at(i) != defaultGPU) - nodes.push_back(gpuNodes.at(i)); + m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, false); if (nodes.size() < 2) { - LOG() << "Skipping test: At least 2 GPUs required." << std::endl; + LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; + LOG() << " or two GPUs are XGMI connected." << std::endl; return; } } diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index e4f5af2b51..50cff1544d 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -147,15 +147,24 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); std::vector nodes; const bool isSpecified = g_TestDstNodeId != -1 && g_TestNodeId != -1; + int numPeers = 0; - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && - /* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */ - (!isSpecified || gpuNodes.at(i) == g_TestDstNodeId || gpuNodes.at(i) == g_TestNodeId)) - nodes.push_back(gpuNodes.at(i)); + if (isSpecified) { + if (g_TestNodeId != g_TestDstNodeId) { + nodes.push_back(g_TestNodeId); + nodes.push_back(g_TestDstNodeId); + if ((m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && + m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId)) || + m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) + numPeers = 2; + } + } else { + HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); + numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + } - if (nodes.size() < 2) { - LOG() << "Skipping test: Need at least two large bar GPU." << std::endl; + if (numPeers < 2) { + LOG() << "Skipping test: Need at least two large bar GPU or XGMI connected." << std::endl; return; } @@ -302,12 +311,11 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) { const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); std::vector nodes; - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i))) - nodes.push_back(gpuNodes.at(i)); + HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); + int numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); - if (nodes.size() < 2) { - LOG() << "Skipping test: Need at least two large bar GPU." << std::endl; + if (numPeers < 2) { + LOG() << "Skipping test: Need at least two large bar GPU or XGMI connected." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 20dbf598e0..c5abb9ff5e 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -23,6 +23,7 @@ #include #include +#include #include "KFDQMTest.hpp" #include "PM4Queue.hpp" #include "PM4Packet.hpp" @@ -1459,8 +1460,10 @@ TEST_F(KFDQMTest, P2PTest) { if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[1])) { - LOG() << "Skipping test: Dst GPU is not a large bar GPU." << std::endl; + + if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId) && + !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + LOG() << "Skipping test: Dst GPU specified is not peer-accessible." << std::endl; return; } if (nodes[0] == nodes[1]) { @@ -1469,12 +1472,10 @@ TEST_F(KFDQMTest, P2PTest) { } } else { HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - nodes.push_back(defaultGPU); - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && gpuNodes.at(i) != defaultGPU) - nodes.push_back(gpuNodes.at(i)); + m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); if (nodes.size() < 2) { LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; + LOG() << " or two GPUs are XGMI connected." << std::endl; return; } } diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 41d13f9e9a..991bea2e11 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -589,3 +589,39 @@ const int HsaNodeInfo::FindLargeBarGPUNode() const { return -1; } + +const bool HsaNodeInfo::AreGPUNodesXGMI(int node0, int node1) const { + const HsaNodeProperties *pNodeProperties0 = GetNodeProperties(node0); + const HsaNodeProperties *pNodeProperties1 = GetNodeProperties(node1); + + if ((pNodeProperties0->HiveID != 0) && (pNodeProperties1->HiveID != 0) && + (pNodeProperties0->HiveID == pNodeProperties1->HiveID)) + return true; + + return false; +} + +int HsaNodeInfo::FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, + bool bidirectional) const { + peers->push_back(dstNode); + if (IsGPUNodeLargeBar(dstNode)) { + for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { + if (m_NodesWithGPU.at(i) == dstNode) + continue; + + if (!bidirectional || IsGPUNodeLargeBar(m_NodesWithGPU.at(i)) || + AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) + peers->push_back(m_NodesWithGPU.at(i)); + } + } else { + for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { + if (m_NodesWithGPU.at(i) == dstNode) + continue; + + if (AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) + peers->push_back(m_NodesWithGPU.at(i)); + } + } + + return peers->size(); +} diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 1e40e8b4e4..4cbaa395e7 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -188,6 +188,9 @@ class HsaNodeInfo { // @brief Find the first available Large-BAR GPU node // @return: Node ID if successful or -1 const int FindLargeBarGPUNode() const; + const bool AreGPUNodesXGMI(int node0, int node1) const; + int FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, + bool bidirectional) const; }; #endif // __KFD__TEST__UTIL__H__