From 5b7d3a16c533a4e5174ae1e09d932a0cdc207497 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Wed, 19 Sep 2018 15:12:04 +0800 Subject: [PATCH] kfdtest: add P2POverheadTest This is to measure the laterncy + overhead of sdma packet consumption on p2p. It is Similar with QueueLatency test. What's more, the queue's overhead with different workload show more details. test result on two gfx900. [ RUN ] KFDPerformanceTest.P2POverheadTest [ ] Test (avg. ns) | Size 4 8 16 64 256 1024 [ ] ----------------------------------------------------------------------- [ ] [push] [1 -> 0] 333 148 185 111 148 148 [ ] [push] [1 -> 1] 370 222 333 74 148 111 [ ] [push] [1 -> 2] 333 148 148 148 148 148 [ ] [push] [2 -> 0] 111 333 259 148 148 148 [ ] [push] [2 -> 1] 222 148 185 148 148 148 [ ] [push] [2 -> 2] 222 111 370 111 74 148 [ ] [pull] [1 -> 0] 370 296 296 148 185 148 [ ] [pull] [1 -> 1] 185 333 222 148 222 148 [ ] [pull] [1 -> 2] 222 444 259 148 185 111 [ ] [pull] [2 -> 0] 148 148 148 148 148 148 [ ] [pull] [2 -> 1] 148 148 148 148 148 148 [ ] [pull] [2 -> 2] 185 148 148 74 222 296 [ ] [push|pull][1 -> 0] 1259 1222 1259 1074 1037 962 [ ] [push|pull][1 -> 1] 1037 1037 1037 740 740 1000 [ ] [push|pull][1 -> 2] 1259 1259 1296 1037 1000 1074 [ ] [push|pull][2 -> 0] 1037 1037 1037 1074 1037 1148 [ ] [push|pull][2 -> 1] 1037 1037 1037 1037 925 1074 [ ] [push|pull][2 -> 2] 666 666 740 740 703 925 [ OK ] KFDPerformanceTest.P2POverheadTest (459 ms) Change-Id: I422263cb52f7ce184f6f1ff4466d04c239fbe9c9 Signed-off-by: xinhui pan [ROCm/ROCR-Runtime commit: 918a45a43053bc21957ed71dd3f01950b7ce399f] --- .../tests/kfdtest/scripts/kfdtest.exclude | 3 +- .../tests/kfdtest/src/KFDPerformanceTest.cpp | 107 +++++++++++++++--- 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/projects/rocr-runtime/tests/kfdtest/scripts/kfdtest.exclude b/projects/rocr-runtime/tests/kfdtest/scripts/kfdtest.exclude index 6607cacdb8..fcdaeb48fe 100644 --- a/projects/rocr-runtime/tests/kfdtest/scripts/kfdtest.exclude +++ b/projects/rocr-runtime/tests/kfdtest/scripts/kfdtest.exclude @@ -44,7 +44,8 @@ SDMA_BLACKLIST=\ "KFDMemoryTest.MMBench:"\ "KFDQMTest.*Sdma*:"\ "KFDQMTest.P2PTest:"\ -"KFDPerformanceTest.P2PBandWidthTest" +"KFDPerformanceTest.P2PBandWidthTest:"\ +"KFDPerformanceTest.P2POverheadTest" # Anything involving CP queue creation is failing on Kaveri. Separate them here for convenience (KFD-336) KV_QUEUE_BLACKLIST=\ diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDPerformanceTest.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDPerformanceTest.cpp index 8b86d98555..4d49f392cc 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -66,7 +66,8 @@ enum P2PDirection { */ static void testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirection n1Direction, - P2PDirection n2Direction, HSAuint64 size, HSAuint64 &speed, HSAuint64 &speed2, std::stringstream &msg) { + P2PDirection n2Direction, HSAuint64 size, HSAuint64 *speed, HSAuint64 *speed2, std::stringstream *msg, + bool isTestOverhead = false, HSAuint64 *time = 0) { ASSERT_GT(16, unsigned(n - 1)); HSAuint32 n2[n]; void *n1Mem, *n2Mem[n]; @@ -76,15 +77,17 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio memFlags.ui32.NonPaged = 1; SDMACopyParams array[n * 4]; int array_count = 0; + HSAuint64 alloc_size = ALIGN_UP(size, PAGE_SIZE); + std::vector copyArray; int i; - ASSERT_SUCCESS(hsaKmtAllocMemory(n1, size, memFlags, &n1Mem)); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n1Mem, size, NULL)); + ASSERT_SUCCESS(hsaKmtAllocMemory(n1, alloc_size, memFlags, &n1Mem)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n1Mem, alloc_size, NULL)); for (i = 0; i < n; i++) { n2[i] = n2Array[i]; - ASSERT_SUCCESS(hsaKmtAllocMemory(n2[i], size, memFlags, &n2Mem[i])); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n2Mem[i], size, NULL)); + ASSERT_SUCCESS(hsaKmtAllocMemory(n2[i], alloc_size, memFlags, &n2Mem[i])); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n2Mem[i], alloc_size, NULL)); } for (i = 0; i < n; i++) { @@ -96,10 +99,10 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio do { if (n1Direction == IN || n1Direction == IN_OUT) /* n2Mem -> n1Mem*/ - array[array_count++] = {n1, n2Mem[i], n1Mem, size}; + array[array_count++] = {n1, n2Mem[i], n1Mem, size, n1/*group id, just a hint*/}; if (n1Direction == OUT || n1Direction == IN_OUT) /* n1Mem -> n2Mem*/ - array[array_count++] = {n1, n1Mem, n2Mem[i], size}; + array[array_count++] = {n1, n1Mem, n2Mem[i], size, n1}; /* Issue two copies to make full use of sdma.*/ } while (n1Direction < IN_OUT && n == 1 && array_count % 2); /* Do nothing if no IN or OUT specified.*/ @@ -107,21 +110,30 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio do { if (n2Direction == IN || n2Direction == IN_OUT) /* n1Mem -> n2Mem*/ - array[array_count++] = {n2[i], n1Mem, n2Mem[i], size}; + array[array_count++] = {n2[i], n1Mem, n2Mem[i], size, n2[i]}; if (n2Direction == OUT || n2Direction == IN_OUT) /* n2Mem -> n1Mem*/ - array[array_count++] = {n2[i], n2Mem[i], n1Mem, size}; + array[array_count++] = {n2[i], n2Mem[i], n1Mem, size, n2[i]}; } while (n2Direction < IN_OUT && array_count % 2); } - sdma_multicopy(array, array_count, &speed, &speed2, &msg); + /* We measure a bunch of packets.*/ + if (isTestOverhead) { + for (i = 0; i < 1000; i++) + for (int j = 0; j < array_count; j++) + copyArray.push_back(array[j]); + sdma_multicopy(copyArray, 1, HEAD_TAIL); + *time = CounterToNanoSec(copyArray[0].timeConsumption / (1000 * array_count)); + } else + /* It did not respect the group id we set above.*/ + sdma_multicopy(array, array_count, speed, speed2, msg); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(n1Mem)); - EXPECT_SUCCESS(hsaKmtFreeMemory(n1Mem, size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(n1Mem, alloc_size)); for (i = 0; i < n; i++) { EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(n2Mem[i])); - EXPECT_SUCCESS(hsaKmtFreeMemory(n2Mem[i], size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(n2Mem[i], alloc_size)); } } @@ -196,7 +208,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { snprintf(str, sizeof(str), "[%d -> %d] ", n1, n2); msg << str << std::endl; - testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); LOG() << std::dec << str << (float)speed / 1024 << " - " << (float)speed2 / 1024 << " GB/s" << std::endl; @@ -217,7 +229,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { snprintf(str, sizeof(str), "[%d <-> %d] ", n1, n2); msg << str << std::endl; - testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); LOG() << std::dec << str << (float)speed / 1024 << " - " << (float)speed2 / 1024 << " GB/s" << std::endl; @@ -251,7 +263,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { else snprintf(str, sizeof(str), "[%d -> [%d...%d]] ", n1, dst.front(), dst.back()); msg << str << std::endl; - testNodeToNodes(n1, n2, n, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + testNodeToNodes(n1, n2, n, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); LOG() << std::dec << str << (float)speed / 1024 << " - " << (float)speed2 / 1024 << " GB/s" << std::endl; @@ -263,3 +275,68 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { TEST_END } + +TEST_F(KFDPerformanceTest, P2POverheadTest) { + TEST_START(TESTPROFILE_RUNALL); + if (!is_dgpu()) { + LOG() << "Skipping test: Can't have 2 APUs on the same system." << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + std::vector nodes; + + for (unsigned i = 0; i < gpuNodes.size(); i++) + if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i))) + nodes.push_back(gpuNodes.at(i)); + + if (nodes.size() < 2) { + LOG() << "Skipping test: Need at least two large bar GPU." << std::endl; + return; + } + + std::vector sysNodes(nodes); // include sysMem node 0... + sysNodes.insert(sysNodes.begin(),0); + + /* size should be small.*/ + const HSAuint32 sizeArray[] = {4, 8, 16, 64, 256, 1024}; + const int total_tests = 3; + const char *test_suits_string[total_tests] = { + "[push] ", + "[pull] ", + "[push|pull]", + }; + const P2PDirection test_suits[total_tests] = {OUT, IN, IN_OUT}; + std::stringstream msg; + int s; //test index; + + msg << "Test (avg. ns) | Size"; + for (auto &size : sizeArray) + msg << "\t" << size; + LOG() << msg.str() << std::endl; + LOG() << "-----------------------------------------------------------------------" << std::endl; + + for (s = 0; s < total_tests; s++) { + + for (unsigned i = 0; i < nodes.size(); i++) { + /* Src node is a GPU.*/ + HSAuint32 n1 = nodes[i]; + HSAuint64 time; + + /* Pick up dst node which can be sysMem.*/ + for (unsigned j = 0; j < sysNodes.size(); j++) { + HSAuint32 n2 = sysNodes[j]; + std::stringstream msg; + + msg << test_suits_string[s] << "[" << n1 << " -> " << n2 << "]"; + for (auto &size : sizeArray) { + testNodeToNodes(n1, &n2, 1, test_suits[s], NONE, size, 0, 0, 0, 1, &time); + msg << "\t" << time; + } + LOG() << msg.str() << std::endl; + } + } + } + + TEST_END +}