kfdtest: Add RDMATest ContiguousVRAMAllocation

Update amdp2ptest.h to sync with the same file from rdma test driver
folder.

Add ContiguousVRAMAllocation to verify rdma get pages will get
contiguous VRAM pages, skipped RDMA getpages if amdp2ptest.ko is not
loaded.

Change rdma buffer mmap with MAP_SHARED flag, because MAP_PRIVATE goes
to COW path, which requires mmap the entire vma and cannot support
multiple sg nents.

Change-Id: I5fbb1902251f1454616d4404a4b048a88996d4f7
Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Signed-off-by: Chris Freehill <cfreehil@amd.com>


[ROCm/ROCR-Runtime commit: e076a4ee82]

projects/rocr-runtime/libhsakmt/tests/kfdtest/src/RDMATest.cpp

@@ -112,3 +112,92 @@ TEST_F(RDMATest, GPUDirect) {
 
     TEST_END
 }
+
+TEST_F(RDMATest, ContiguousVRAMAllocation) {
+    TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX);
+    TEST_START(TESTPROFILE_RUNALL);
+
+    HSAuint64 AlternateVAGPU;
+
+    PM4Queue queue;
+    unsigned long BufferSize = 4UL << 30;
+
+    int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode();
+    ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node";
+
+    HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode);
+    HsaMemoryBuffer srcSysBuffer(PAGE_SIZE, defaultGPUNode, false);
+    void *LocalBuffer;
+    HsaMemFlags memFlags = {0};
+    int ret;
+
+    memFlags.ui32.NonPaged = 1;
+    memFlags.ui32.Contiguous = 1;
+    ret = hsaKmtAllocMemory(defaultGPUNode, BufferSize, memFlags, &LocalBuffer);
+    if (ret == HSAKMT_STATUS_NOT_SUPPORTED) {
+        LOG() << "KFD does not support contiguous memory, skipping the test" << std::endl;
+        return;
+    }
+
+    ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(srcSysBuffer.As<void*>(),
+                                        srcSysBuffer.Size(),
+                                        &AlternateVAGPU));
+    ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer, BufferSize, &AlternateVAGPU));
+
+    /* Fill up srcSysBuffer */
+    srcSysBuffer.Fill(0xfe);
+
+    /* Put 'copy dword' command to ISA buffer */
+    ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As<char*>()));
+
+    ASSERT_SUCCESS(queue.Create(defaultGPUNode));
+    Dispatch dispatch(isaBuffer);
+
+    /* Submit the command to GPU so GPU will copy from system memory
+     * (srcSysBuffer) to local memory(LocalBuffer)
+     */
+    dispatch.SetArgs(srcSysBuffer.As<void*>(), LocalBuffer);
+    dispatch.Submit(queue);
+    dispatch.Sync(g_TestTimeOut);  // GPU executed the command
+
+    EXPECT_SUCCESS(queue.Destroy());
+
+    LocalMemoryAccess Rdma;
+    void *gpuAddr;
+
+    Rdma.Open();
+    if (Rdma.fd < 0) {
+        LOG() << "amdp2ptest.ko driver not loaded, skipping RDMA getpages" << std::endl;
+        goto exit;
+    }
+
+    /* GetPages asks the test driver to convert GPU virtual memory to DMA/
+     * Physical memory and save it in the list. rdma_mmap maps the memory to
+     * user space memory.
+     */
+    ret = Rdma.GetPages((uint64_t)LocalBuffer, BufferSize);
+    ASSERT_EQ(ret, 0) << "Failed to get pages";
+
+    gpuAddr = Rdma.MMap((uint64_t)LocalBuffer, BufferSize);
+    ASSERT_GE((int64_t)gpuAddr, 0) << "Failed to map RDMA address.";
+
+    printf("contiguous VRAM address %p size 0x%lx bytes\n", LocalBuffer, BufferSize);
+    printf("Pause to dump page table to check if allocation is contiguous\n");
+    printf("Press Enter key to continue\n");
+    getchar();
+
+    /* Read the memory to confirm that application can read the local memory
+     * correctly from the mapped address.
+     */
+    EXPECT_EQ(memcmp(gpuAddr, srcSysBuffer.As<void*>(), 4), 0);
+
+    Rdma.UnMap(gpuAddr, PAGE_SIZE);
+    Rdma.Close();
+
+exit:
+    EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(srcSysBuffer.As<void*>()));
+    EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer));
+    EXPECT_SUCCESS(hsaKmtFreeMemory(LocalBuffer, BufferSize));
+
+    TEST_END
+}