From 20566a9b38702eacb1d242608d7238cc182d4a45 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 6 Apr 2024 22:35:08 -0400 Subject: [PATCH] rdmatest: Add rdma contiguous memory allocation test Update CMakeLists.txt to use Thunk pkgconfig. Add rdma contiguous memory allocation test, to verify if KFD rdma get pages to pin buffer on contiguous VRAM pages. Change-Id: I7cc617fc083ce1998c214c327c130f033ce41d6f Signed-off-by: Philip Yang Signed-off-by: Chris Freehill [ROCm/ROCR-Runtime commit: 3f00c889105719ad2c6a35b96998165ad670a023] --- .../tests/rdma/simple/app/CMakeLists.txt | 33 +++- .../tests/rdma/simple/app/rdma_test.cpp | 164 +++++++++++++++++- 2 files changed, 192 insertions(+), 5 deletions(-) diff --git a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt index 762420d461..5285975514 100644 --- a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt +++ b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt @@ -2,11 +2,38 @@ cmake_minimum_required (VERSION 2.6) project (rdma_test) -link_directories($ENV{ROOT_OF_ROOTS}/out/lib) +find_package(PkgConfig) +pkg_check_modules(DRM REQUIRED libdrm) +pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) +include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) + +if( DEFINED ENV{LIBHSAKMT_PATH} ) + set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} ) + message ( "LIBHSAKMT_PATH environment variable is set" ) +else() + if ( ${ROCM_INSTALL_PATH} ) + set ( ENV{PKG_CONFIG_PATH} ${ROCM_INSTALL_PATH}/lib/pkgconfig ) + else() + set ( ENV{PKG_CONFIG_PATH} /opt/rocm/lib/pkgconfig ) + endif() + + pkg_check_modules(HSAKMT libhsakmt) + + if( NOT HSAKMT_FOUND ) + set ( LIBHSAKMT_PATH $ENV{OUT_DIR} ) + endif() +endif() + +if( DEFINED LIBHSAKMT_PATH ) + set ( HSAKMT_LIBRARY_DIRS ${LIBHSAKMT_PATH}/lib ) + set ( HSAKMT_LIBRARIES hsakmt ) +endif() + + +link_directories(${HSAKMT_LIBRARY_DIRS}) include_directories($ENV{LIBHSAKMT_ROOT}/include) include_directories(../drv) add_executable(rdma_test rdma_test.cpp) -target_link_libraries(rdma_test libhsakmt.so) - +target_link_libraries(rdma_test libhsakmt.a dl pthread numa drm drm_amdgpu) diff --git a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp index 211299fd10..73a3d641ad 100644 --- a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp +++ b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp @@ -29,8 +29,7 @@ #include #include #include - -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #include "amdp2ptest.h" int rdma_fd = -1; @@ -166,6 +165,166 @@ void run_rdma_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty) hsaKmtFreeMemory(MemoryAddress, SizeInBytes); } +int getSysMemorySize(unsigned long *memSize) +{ + FILE *meminfo = fopen("/proc/meminfo", "r"); + + if(meminfo == NULL) + return -1; + + char buff[256]; + while (fgets(buff, sizeof(buff), meminfo)) + { + long ramKB; + if (sscanf(buff, "MemTotal: %ld kB", &ramKB) == 1) + { + *memSize = ramKB * 1024; + break; + } + } + + fclose(meminfo); + printf("Total system memory size 0x%lx\n", *memSize); + return 0; +} + +/* + * RDMA contiguous memory allocation test + * + * Test steps: + * 1. fragment the entire VRAM, alloc all VRAM using multiple buffers, then free 1 buffer from every + other buffers + * 2. alloc memFlags.ui32.Contiguous=1 buffer for contiguous VRAM allocation + * 3. Call AMD2P2PTEST_IOCTL_GET_PAGES to get contiguous VRAM buffer pages + * 4. Test fails if any above step failed + */ +void run_rdma_contiguous_mem_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty) +{ + unsigned int *nullPtr = NULL; + unsigned long bufSize = 512ULL << 20; + unsigned long nBuf; + + HSAuint64 vramSize; + unsigned long sysMemSize; + HsaMemFlags memFlags = {0}; + HSAKMT_STATUS status; + + if (getSysMemorySize(&sysMemSize) < 0) { + fprintf(stderr, "Failed to get system memory size\n"); + exit(EXIT_FAILURE); + } + status = hsaKmtAvailableMemory(Node, &vramSize); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to get VRAM size\n", status); + exit(EXIT_FAILURE); + } + if (sysMemSize < (16UL << 30) || vramSize < (4UL << 30)) { + fprintf(stderr, "No enough system memory or VRAM\n"); + exit(0); + } + nBuf = vramSize / bufSize; + + void **pBuf = (void **)malloc(sizeof(*pBuf) * nBuf); + memFlags.ui32.NonPaged = 1; + + for (int i = 0; i < nBuf; i++) { + status = hsaKmtAllocMemory(Node, bufSize, memFlags, &pBuf[i]); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to alloc buf %d\n", status, i); + exit(EXIT_FAILURE); + } + + status = hsaKmtMapMemoryToGPU(pBuf[i], bufSize, NULL); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to map buf %d\n", status, i); + exit(EXIT_FAILURE); + } + } + + //printf("Freeing every other BO to fragment VRAM\n"); + for (int i = 0; i < nBuf; i+=2) { + status = hsaKmtUnmapMemoryToGPU(pBuf[i]); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to unmap buf %d from GPU\n", status, i); + exit(EXIT_FAILURE); + } + status = hsaKmtFreeMemory(pBuf[i], bufSize); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to free buf %d\n", status, i); + exit(EXIT_FAILURE); + } + } + + printf("Node %d Size 0x%lx (%ld MB)\n", Node, MemoryProperty->SizeInBytes, + MemoryProperty->SizeInBytes / (1024 * 1024)); + + void *cpu_ptr; + int ret = 0; + void *MemoryAddress = 0; + HSAuint64 SizeInBytes = 1UL << 30; + + memFlags.ui32.Contiguous = 1; + + status = hsaKmtAllocMemory(Node, SizeInBytes, memFlags, &MemoryAddress); + if (status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "Failure to allocate memory 0x%lx. Status %d\n", SizeInBytes, status); + exit(EXIT_FAILURE); + } + + status = hsaKmtMapMemoryToGPU(MemoryAddress, SizeInBytes, NULL); + if (status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "Failure to map memory. Status %d\n", status); + exit(EXIT_FAILURE); + } + + printf("VRAM allocated. Address %p size 0x%lx bytes\n", MemoryAddress, SizeInBytes); + //printf("Press Enter key to continue\n"); + //getchar(); + + struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM get_page_size = {0}; + get_page_size.addr = (uint64_t) MemoryAddress; + get_page_size.length = SizeInBytes; + + ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGE_SIZE, &get_page_size); + if (ret != 0) + { + fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGE_SIZE error (errno=%d/%s)\n", + ret, strerror(ret)); + exit(EXIT_FAILURE); + } + + printf("GPU Page size: 0x%ld\n", get_page_size.page_size); + + struct AMDRDMA_IOCTL_GET_PAGES_PARAM get_cpu_ptr = {0}; + get_cpu_ptr.addr = (uint64_t) MemoryAddress; + get_cpu_ptr.length = SizeInBytes; + + ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGES, &get_cpu_ptr); + if (ret != 0) + { + fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGES error (errno=%d/%s)\n", + ret, strerror(ret)); + //printf("IOCTL_GET_PAGES failed, Press Enter key to continue\n"); + //getchar(); + exit(EXIT_FAILURE); + } + + printf("IOCTL_GET_PAGES return contiguous VRAM address %p size 0x%lx bytes\n", MemoryAddress, SizeInBytes); + printf("Pause to dump page table to check if allocation is contiguous\n"); + printf("Press Enter key to continue\n"); + getchar(); + + ret = rdma_map((uint64_t)MemoryAddress, 4096, &cpu_ptr); + if (ret < 0) + { + exit(EXIT_FAILURE); + } + + hsaKmtFreeMemory(MemoryAddress, SizeInBytes); +} + int main(void) { HsaVersionInfo VersionInfo; @@ -234,6 +393,7 @@ int main(void) // We found local memory available for RDMA operation. // Run some tests on it. run_rdma_tests(iNode, &MemoryProperties[iMemBank]); + run_rdma_contiguous_mem_tests(iNode, &MemoryProperties[iMemBank]); } } }