diff --git a/projects/rocr-runtime/tests/kfdtest/include/sdma_pkt_struct.h b/projects/rocr-runtime/tests/kfdtest/include/sdma_pkt_struct.h index aa13006256..bc37b5189f 100644 --- a/projects/rocr-runtime/tests/kfdtest/include/sdma_pkt_struct.h +++ b/projects/rocr-runtime/tests/kfdtest/include/sdma_pkt_struct.h @@ -102,23 +102,26 @@ typedef struct SDMA_PKT_COPY_LINEAR_TAG unsigned int DW_4_DATA; } SRC_ADDR_HI_UNION; - union + struct { - struct + union { - unsigned int dst_addr_31_0:32; - }; - unsigned int DW_5_DATA; - } DST_ADDR_LO_UNION; + struct + { + unsigned int dst_addr_31_0:32; + }; + unsigned int DW_5_DATA; + } DST_ADDR_LO_UNION; - union - { - struct + union { - unsigned int dst_addr_63_32:32; - }; - unsigned int DW_6_DATA; - } DST_ADDR_HI_UNION; + struct + { + unsigned int dst_addr_63_32:32; + }; + unsigned int DW_6_DATA; + } DST_ADDR_HI_UNION; + } DST_ADDR[0]; } SDMA_PKT_COPY_LINEAR, *PSDMA_PKT_COPY_LINEAR; /* diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDQMTest.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDQMTest.cpp index d88e33c7c1..25e4c8df73 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDQMTest.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDQMTest.cpp @@ -1352,6 +1352,31 @@ TEST_F(KFDQMTest, mGPUShareBO) { TEST_END } + +static void sdma_copy(HSAint32 node, void *src, void *const dst[], int n, unsigned int size) { + ROUTINE_START; + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(node)); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(dst, src, n, size)); + sdmaQueue.Wait4PacketConsumption(); + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + ROUTINE_END; +} + +static void sdma_fill(HSAint32 node, void *dst, unsigned int data, unsigned int size) { + ROUTINE_START; + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(node)); + sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(dst, data, size)); + sdmaQueue.Wait4PacketConsumption(); + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + ROUTINE_END; +} + TEST_F(KFDQMTest, P2PTest) { TEST_START(TESTPROFILE_RUNALL); if (!is_dgpu()) { @@ -1364,92 +1389,106 @@ TEST_F(KFDQMTest, P2PTest) { LOG() << "Skipping test: Need at least two GPUs" << std::endl; return; } - HSAint32 gpuNode1 = m_NodeInfo.HsaDefaultGPUNode(); - HSAint32 gpuNode2 = 0; + std::vector nodes; /* This test simulates RT team's P2P part in IPCtest: * - * +--------------------------------------------+ - * | gpu1 gpu2 | - * |gpu1 mem ----> gpu2 mem ----> system buffer | - * +--------------------------------------------+ + * +------------------------------------------------+ + * | gpu1 gpu2 gpuX | + * |gpu1 mem ----> gpu2 mem ----> gpuX mem | + * | \ \ \ | + * | \ \ \ | + * | system buffer system buffer system buffer| + * +------------------------------------------------+ * - * Copy data from GPU-1 memory to GPU-2 memory using GPU-1, then GPU-2 - * memory to system buffer using GPU-2. Verify the system buffer - * (initialized with 0) has the same content as gpu1 memory (0x5). + * Copy data from current GPU memory to next GPU memory and system memory + * Using current GPU, aka p2p push. + * Verify the system buffer has the expected content after each push. */ /* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */ if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { - gpuNode1 = g_TestNodeId; - gpuNode2 = g_TestDstNodeId; - } - - /* GPU-2 must have public memory(large bar) to do GPU-to-GPU copy. If - * not specified in the command line, find one. - */ - gpuNode2 = m_NodeInfo.FindLargeBarGPUNode(); - if (gpuNode2 < 0) { - LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; - return; - } - if (gpuNode1 == gpuNode2) { - for (unsigned i = 0; i < gpuNodes.size(); i++) { - if (gpuNodes.at(i) != gpuNode2) { - gpuNode1 = gpuNodes.at(i); - break; - } + nodes.push_back(g_TestNodeId); + nodes.push_back(g_TestDstNodeId); + if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[1])) { + LOG() << "Skipping test: Dst GPU is not a large bar GPU" << std::endl; + return; + } + if (nodes[0] == nodes[1]) { + LOG() << "Skipping test: Need different GPUs specified" << std::endl; + return; + } + } else { + HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); + nodes.push_back(defaultGPU); + for (unsigned i = 0; i < gpuNodes.size(); i++) + if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && gpuNodes.at(i) != defaultGPU) + nodes.push_back(gpuNodes.at(i)); + if (nodes.size() < 2) { + LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + return; } } HSAuint32 *sysBuf; - HSAuint32 size = 0xc00000; // bigger than 4MB to test non-contiguous memory + HSAuint32 size = 16ULL<<20; // bigger than 16MB to test non-contiguous memory HsaMemFlags memFlags = {0}; HsaMemMapFlags mapFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 1; + memFlags.ui32.NonPaged = 1; + unsigned int end = size / sizeof(HSAuint32) - 1; - /* 1. Allocate a system buffer and allow the access to GPU-2 */ + /* 1. Allocate a system buffer and allow the access to GPUs */ EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, (void **)&sysBuf)); EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL, - mapFlags, 1, (HSAuint32*)&gpuNode2)); + mapFlags, nodes.size(), &nodes[0])); +#define MAGIC_NUM 0xdeadbeaf - /* 2.- Allocate local memory on GPU-1 - * - Allocate local memory on GPU-2 and allow access to both GPUs - */ - HsaMemoryBuffer gpu1Mem(size, gpuNode1, false, true /*isLocal*/); - HsaMemoryBuffer gpu2Mem(size, gpuNode2, false, true); + /* First GPU fills mem with MAGIC_NUM*/ + void *src, *dst; + HSAuint32 cur = nodes[0], next; + ASSERT_SUCCESS(hsaKmtAllocMemory(cur, size, memFlags, (void**)&src)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(src, size, NULL)); + sdma_fill(cur, src, MAGIC_NUM, size); - SDMAQueue sdmaQueue1, sdmaQueue2; - ASSERT_SUCCESS(sdmaQueue1.Create(gpuNode1)); - ASSERT_SUCCESS(sdmaQueue2.Create(gpuNode2)); + for (unsigned i = 1; i <= nodes.size(); i++) { + int n; + memset(sysBuf, 0, size); - /* initialize sysBuf as 0 and fill up gpu1 mem with 0x5 */ - memset(sysBuf, 0, size); - sdmaQueue1.PlaceAndSubmitPacket(SDMAFillDataPacket(gpu1Mem.As(), - 0x5, size)); - sdmaQueue1.Wait4PacketConsumption(); + /* Last GPU just copy mem to sysBuf*/ + if (i == nodes.size()) { + n = 1; + next = 0;/*system memory node*/ + dst = 0; + } else { + n = 2; + next = nodes[i]; + ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, (void**)&dst)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dst, size, NULL)); + } - /* 3. Copy data from gpuNode1 to gpuNode2 */ - sdmaQueue1.PlaceAndSubmitPacket(SDMACopyDataPacket(gpu2Mem.As(), - gpu1Mem.As(), size)); - sdmaQueue1.Wait4PacketConsumption(); + LOG() << "Test " << cur << " -> " << next << std::endl; + /* copy to sysBuf and next GPU*/ + void *dst_array[] = {sysBuf, dst}; + sdma_copy(cur, src, dst_array, n, size); - /* 4. Copy data from gpuNode2 to system buffer */ - sdmaQueue2.PlaceAndSubmitPacket(SDMACopyDataPacket(sysBuf, - gpu2Mem.As(), size)); - sdmaQueue2.Wait4PacketConsumption(); + /* verify the data*/ + ASSERT_EQ(sysBuf[0], MAGIC_NUM); + ASSERT_EQ(sysBuf[end], MAGIC_NUM); - /* 5. Verify the data */ - ASSERT_EQ(sysBuf[0], 0x5); - unsigned int end = size / sizeof(HSAuint32) - 1; - ASSERT_EQ(sysBuf[end], 0x5); + LOG() << "PASS " << cur << " -> " << next << std::endl; + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(src)); + EXPECT_SUCCESS(hsaKmtFreeMemory(src, size)); + + cur = next; + src = dst; + } EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(sysBuf)); EXPECT_SUCCESS(hsaKmtFreeMemory(sysBuf, size)); - EXPECT_SUCCESS(sdmaQueue1.Destroy()); - EXPECT_SUCCESS(sdmaQueue2.Destroy()); TEST_END } diff --git a/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp b/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp index e7d1020546..45c7a6cd3e 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp @@ -72,14 +72,24 @@ void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int ndw, memcpy(&packetData->DATA0_UNION.DW_4_DATA, data, ndw*sizeof(unsigned int)); } -#define TWO_MEG (1 << 21) +#define BITS (21) +#define TWO_MEG (1 << BITS) SDMACopyDataPacket::~SDMACopyDataPacket(void) { free(packetData); } -SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsize) { - int32_t size = 0; - packetSize = ((surfsize + TWO_MEG - 1) >> 21) * sizeof(SDMA_PKT_COPY_LINEAR); +SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, unsigned int surfsize) { + int32_t size = 0, i; + void **dst = (void**)malloc(sizeof(void*) * n); + const int singlePacketSize = sizeof(SDMA_PKT_COPY_LINEAR) + + sizeof(SDMA_PKT_COPY_LINEAR::DST_ADDR[0]) * n; + + if (n > 2) + WARN() << "SDMACopyDataPacket does not support more than 2 dst addresses!" << std::endl; + + memcpy(dst, dsts, sizeof(void*) * n); + + packetSize = ((surfsize + TWO_MEG - 1) >> BITS) * singlePacketSize; SDMA_PKT_COPY_LINEAR *pSDMA = (SDMA_PKT_COPY_LINEAR *)malloc(packetSize); packetData = pSDMA; @@ -91,22 +101,31 @@ SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsi else size = surfsize; - memset(pSDMA, 0, sizeof(SDMA_PKT_COPY_LINEAR)); + memset(pSDMA, 0, singlePacketSize); pSDMA->HEADER_UNION.op = SDMA_OP_COPY; pSDMA->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR; + pSDMA->HEADER_UNION.broadcast = n > 1 ? 1 : 0; pSDMA->COUNT_UNION.count = SDMA_COUNT(size); SplitU64(reinterpret_cast(src), pSDMA->SRC_ADDR_LO_UNION.DW_3_DATA, // src_addr_31_0 pSDMA->SRC_ADDR_HI_UNION.DW_4_DATA); // src_addr_63_32 - SplitU64(reinterpret_cast(dst), - pSDMA->DST_ADDR_LO_UNION.DW_5_DATA, // dst_addr_31_0 - pSDMA->DST_ADDR_HI_UNION.DW_6_DATA); // dst_addr_63_32 - pSDMA++; + for (i = 0; i < n; i++) + SplitU64(reinterpret_cast(dst[i]), + pSDMA->DST_ADDR[i].DST_ADDR_LO_UNION.DW_5_DATA, // dst_addr_31_0 + pSDMA->DST_ADDR[i].DST_ADDR_HI_UNION.DW_6_DATA); // dst_addr_63_32 + + pSDMA = (SDMA_PKT_COPY_LINEAR *)((char *)pSDMA + singlePacketSize); + for (i = 0; i < n; i++) + dst[i] = (char *)dst[i] + size; src = (char *)src + size; - dst = (char *)dst + size; surfsize -= size; } + free(dst); +} + +SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsize) { + new (this)SDMACopyDataPacket(&dst, src, 1, surfsize); } SDMAFillDataPacket::~SDMAFillDataPacket() { @@ -118,7 +137,7 @@ SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned in SDMA_PKT_CONSTANT_FILL *pSDMA; /* SDMA support maximum 0x3fffe0 byte in one copy. Use 2M copy_size */ - m_PacketSize = ((size + TWO_MEG - 1) >> 21) * sizeof(SDMA_PKT_CONSTANT_FILL); + m_PacketSize = ((size + TWO_MEG - 1) >> BITS) * sizeof(SDMA_PKT_CONSTANT_FILL); pSDMA = (SDMA_PKT_CONSTANT_FILL *)calloc(1, m_PacketSize); m_PacketData = pSDMA; diff --git a/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.hpp b/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.hpp index 89ad4cac07..f7ef53f733 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.hpp +++ b/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.hpp @@ -64,6 +64,7 @@ class SDMACopyDataPacket : public SDMAPacket { public: // this contructor will also init the packet, no need for adittional calls SDMACopyDataPacket(void *dest, void *src, unsigned int size); + SDMACopyDataPacket(void *const dst[], void *src, int n, unsigned int surfsize); virtual ~SDMACopyDataPacket(void);