rdmatest: Add rdma contiguous memory allocation test

Update CMakeLists.txt to use Thunk pkgconfig.

Add rdma contiguous memory allocation test, to verify if KFD rdma get
pages to pin buffer on contiguous VRAM pages.

Change-Id: I7cc617fc083ce1998c214c327c130f033ce41d6f
Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Signed-off-by: Chris Freehill <cfreehil@amd.com>


[ROCm/ROCR-Runtime commit: 3f00c88910]

projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt

@@ -2,11 +2,38 @@ cmake_minimum_required (VERSION 2.6)
 
 project (rdma_test)
 
-link_directories($ENV{ROOT_OF_ROOTS}/out/lib)
+find_package(PkgConfig)
+pkg_check_modules(DRM REQUIRED libdrm)
+pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu)
+include_directories(${DRM_AMDGPU_INCLUDE_DIRS})
+
+if( DEFINED ENV{LIBHSAKMT_PATH} )
+    set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} )
+    message ( "LIBHSAKMT_PATH environment variable is set" )
+else()
+    if ( ${ROCM_INSTALL_PATH} )
+       set ( ENV{PKG_CONFIG_PATH} ${ROCM_INSTALL_PATH}/lib/pkgconfig )
+    else()
+       set ( ENV{PKG_CONFIG_PATH} /opt/rocm/lib/pkgconfig )
+    endif()
+
+    pkg_check_modules(HSAKMT libhsakmt)
+
+    if( NOT HSAKMT_FOUND )
+       set ( LIBHSAKMT_PATH $ENV{OUT_DIR} )
+    endif()
+endif()
+
+if( DEFINED LIBHSAKMT_PATH )
+    set ( HSAKMT_LIBRARY_DIRS ${LIBHSAKMT_PATH}/lib )
+    set ( HSAKMT_LIBRARIES hsakmt )
+endif()
+
+
+link_directories(${HSAKMT_LIBRARY_DIRS})
 
 include_directories($ENV{LIBHSAKMT_ROOT}/include)
 include_directories(../drv)
 
 add_executable(rdma_test rdma_test.cpp)
-target_link_libraries(rdma_test libhsakmt.so)
-
+target_link_libraries(rdma_test libhsakmt.a dl pthread numa drm drm_amdgpu)

projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp

@@ -29,8 +29,7 @@
 #include <errno.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
-
-#include "hsakmt.h"
+#include "hsakmt/hsakmt.h"
 #include "amdp2ptest.h"
 
 int rdma_fd = -1;
@@ -166,6 +165,166 @@ void run_rdma_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty)
     hsaKmtFreeMemory(MemoryAddress, SizeInBytes);
 }
 
+int getSysMemorySize(unsigned long *memSize)
+{
+    FILE *meminfo = fopen("/proc/meminfo", "r");
+
+    if(meminfo == NULL)
+        return -1;
+
+    char buff[256];
+    while (fgets(buff, sizeof(buff), meminfo))
+    {
+        long ramKB;
+        if (sscanf(buff, "MemTotal: %ld kB", &ramKB) == 1)
+        {
+            *memSize = ramKB * 1024;
+            break;
+        }
+    }
+
+    fclose(meminfo);
+    printf("Total system memory size 0x%lx\n", *memSize);
+    return 0;
+}
+
+/*
+ * RDMA contiguous memory allocation test
+ *
+ * Test steps:
+ * 1. fragment the entire VRAM, alloc all VRAM using multiple buffers, then free 1 buffer from every
+      other buffers
+ * 2. alloc memFlags.ui32.Contiguous=1 buffer for contiguous VRAM allocation
+ * 3. Call AMD2P2PTEST_IOCTL_GET_PAGES to get contiguous VRAM buffer pages
+ * 4. Test fails if any above step failed
+ */
+void run_rdma_contiguous_mem_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty)
+{
+    unsigned int *nullPtr = NULL;
+    unsigned long bufSize = 512ULL << 20;
+    unsigned long nBuf;
+
+    HSAuint64 vramSize;
+    unsigned long sysMemSize;
+    HsaMemFlags memFlags = {0};
+    HSAKMT_STATUS status;
+
+    if (getSysMemorySize(&sysMemSize) < 0) {
+        fprintf(stderr, "Failed to get system memory size\n");
+        exit(EXIT_FAILURE);
+    }
+    status = hsaKmtAvailableMemory(Node, &vramSize);
+    if (status != HSAKMT_STATUS_SUCCESS) {
+        fprintf(stderr, "Failed %d to get VRAM size\n", status);
+        exit(EXIT_FAILURE);
+    }
+    if (sysMemSize < (16UL << 30) || vramSize < (4UL << 30)) {
+        fprintf(stderr, "No enough system memory or VRAM\n");
+        exit(0);
+    }
+    nBuf = vramSize / bufSize;
+
+    void **pBuf = (void **)malloc(sizeof(*pBuf) * nBuf);
+    memFlags.ui32.NonPaged = 1;
+
+    for (int i = 0; i < nBuf; i++) {
+        status = hsaKmtAllocMemory(Node, bufSize, memFlags, &pBuf[i]);
+        if (status != HSAKMT_STATUS_SUCCESS) {
+            fprintf(stderr, "Failed %d to alloc buf %d\n", status, i);
+            exit(EXIT_FAILURE);
+        }
+
+        status = hsaKmtMapMemoryToGPU(pBuf[i], bufSize, NULL);
+        if (status != HSAKMT_STATUS_SUCCESS) {
+            fprintf(stderr, "Failed %d to map buf %d\n", status, i);
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    //printf("Freeing every other BO to fragment VRAM\n");
+    for (int i = 0; i < nBuf; i+=2) {
+        status = hsaKmtUnmapMemoryToGPU(pBuf[i]);
+        if (status != HSAKMT_STATUS_SUCCESS) {
+            fprintf(stderr, "Failed %d to unmap buf %d from GPU\n", status, i);
+            exit(EXIT_FAILURE);
+        }
+        status = hsaKmtFreeMemory(pBuf[i], bufSize);
+        if (status != HSAKMT_STATUS_SUCCESS) {
+            fprintf(stderr, "Failed %d to free buf %d\n", status, i);
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    printf("Node %d Size 0x%lx (%ld MB)\n", Node, MemoryProperty->SizeInBytes,
+                                        MemoryProperty->SizeInBytes / (1024 * 1024));
+
+    void *cpu_ptr;
+    int ret = 0;
+    void *MemoryAddress = 0;
+    HSAuint64 SizeInBytes = 1UL <<  30;
+
+    memFlags.ui32.Contiguous = 1;
+
+    status = hsaKmtAllocMemory(Node, SizeInBytes, memFlags, &MemoryAddress);
+    if (status != HSAKMT_STATUS_SUCCESS)
+    {
+        fprintf(stderr, "Failure to allocate memory 0x%lx. Status %d\n", SizeInBytes, status);
+        exit(EXIT_FAILURE);
+    }
+
+    status = hsaKmtMapMemoryToGPU(MemoryAddress, SizeInBytes, NULL);
+    if (status != HSAKMT_STATUS_SUCCESS)
+    {
+        fprintf(stderr, "Failure to map memory. Status %d\n", status);
+        exit(EXIT_FAILURE);
+    }
+
+    printf("VRAM allocated. Address %p size 0x%lx bytes\n", MemoryAddress, SizeInBytes);
+    //printf("Press Enter key to continue\n");
+    //getchar();
+
+    struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM get_page_size = {0};
+    get_page_size.addr   = (uint64_t) MemoryAddress;
+    get_page_size.length = SizeInBytes;
+
+    ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGE_SIZE, &get_page_size);
+    if (ret != 0)
+    {
+        fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGE_SIZE error (errno=%d/%s)\n",
+                ret, strerror(ret));
+        exit(EXIT_FAILURE);
+    }
+
+    printf("GPU Page size: 0x%ld\n", get_page_size.page_size);
+
+    struct AMDRDMA_IOCTL_GET_PAGES_PARAM get_cpu_ptr = {0};
+    get_cpu_ptr.addr    = (uint64_t) MemoryAddress;
+    get_cpu_ptr.length  = SizeInBytes;
+
+    ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGES, &get_cpu_ptr);
+    if (ret != 0)
+    {
+        fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGES error (errno=%d/%s)\n",
+                         ret, strerror(ret));
+		//printf("IOCTL_GET_PAGES failed, Press Enter key to continue\n");
+		//getchar();
+        exit(EXIT_FAILURE);
+    }
+
+    printf("IOCTL_GET_PAGES return contiguous VRAM address %p size 0x%lx bytes\n", MemoryAddress, SizeInBytes);
+    printf("Pause to dump page table to check if allocation is contiguous\n");
+    printf("Press Enter key to continue\n");
+    getchar();
+
+    ret = rdma_map((uint64_t)MemoryAddress, 4096, &cpu_ptr);
+    if (ret < 0)
+    {
+        exit(EXIT_FAILURE);
+    }
+
+    hsaKmtFreeMemory(MemoryAddress, SizeInBytes);
+}
+
 int main(void)
 {
     HsaVersionInfo      VersionInfo;
@@ -234,6 +393,7 @@ int main(void)
                 // We found local memory available for RDMA operation.
                 // Run some tests on it.
                 run_rdma_tests(iNode, &MemoryProperties[iMemBank]);
+                run_rdma_contiguous_mem_tests(iNode, &MemoryProperties[iMemBank]);
             }
         }
     }