diff --git a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt index 762420d461..5285975514 100644 --- a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt +++ b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/CMakeLists.txt @@ -2,11 +2,38 @@ cmake_minimum_required (VERSION 2.6) project (rdma_test) -link_directories($ENV{ROOT_OF_ROOTS}/out/lib) +find_package(PkgConfig) +pkg_check_modules(DRM REQUIRED libdrm) +pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) +include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) + +if( DEFINED ENV{LIBHSAKMT_PATH} ) + set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} ) + message ( "LIBHSAKMT_PATH environment variable is set" ) +else() + if ( ${ROCM_INSTALL_PATH} ) + set ( ENV{PKG_CONFIG_PATH} ${ROCM_INSTALL_PATH}/lib/pkgconfig ) + else() + set ( ENV{PKG_CONFIG_PATH} /opt/rocm/lib/pkgconfig ) + endif() + + pkg_check_modules(HSAKMT libhsakmt) + + if( NOT HSAKMT_FOUND ) + set ( LIBHSAKMT_PATH $ENV{OUT_DIR} ) + endif() +endif() + +if( DEFINED LIBHSAKMT_PATH ) + set ( HSAKMT_LIBRARY_DIRS ${LIBHSAKMT_PATH}/lib ) + set ( HSAKMT_LIBRARIES hsakmt ) +endif() + + +link_directories(${HSAKMT_LIBRARY_DIRS}) include_directories($ENV{LIBHSAKMT_ROOT}/include) include_directories(../drv) add_executable(rdma_test rdma_test.cpp) -target_link_libraries(rdma_test libhsakmt.so) - +target_link_libraries(rdma_test libhsakmt.a dl pthread numa drm drm_amdgpu) diff --git a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp index 211299fd10..73a3d641ad 100644 --- a/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp +++ b/projects/rocr-runtime/libhsakmt/tests/rdma/simple/app/rdma_test.cpp @@ -29,8 +29,7 @@ #include #include #include - -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #include "amdp2ptest.h" int rdma_fd = -1; @@ -166,6 +165,166 @@ void run_rdma_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty) hsaKmtFreeMemory(MemoryAddress, SizeInBytes); } +int getSysMemorySize(unsigned long *memSize) +{ + FILE *meminfo = fopen("/proc/meminfo", "r"); + + if(meminfo == NULL) + return -1; + + char buff[256]; + while (fgets(buff, sizeof(buff), meminfo)) + { + long ramKB; + if (sscanf(buff, "MemTotal: %ld kB", &ramKB) == 1) + { + *memSize = ramKB * 1024; + break; + } + } + + fclose(meminfo); + printf("Total system memory size 0x%lx\n", *memSize); + return 0; +} + +/* + * RDMA contiguous memory allocation test + * + * Test steps: + * 1. fragment the entire VRAM, alloc all VRAM using multiple buffers, then free 1 buffer from every + other buffers + * 2. alloc memFlags.ui32.Contiguous=1 buffer for contiguous VRAM allocation + * 3. Call AMD2P2PTEST_IOCTL_GET_PAGES to get contiguous VRAM buffer pages + * 4. Test fails if any above step failed + */ +void run_rdma_contiguous_mem_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty) +{ + unsigned int *nullPtr = NULL; + unsigned long bufSize = 512ULL << 20; + unsigned long nBuf; + + HSAuint64 vramSize; + unsigned long sysMemSize; + HsaMemFlags memFlags = {0}; + HSAKMT_STATUS status; + + if (getSysMemorySize(&sysMemSize) < 0) { + fprintf(stderr, "Failed to get system memory size\n"); + exit(EXIT_FAILURE); + } + status = hsaKmtAvailableMemory(Node, &vramSize); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to get VRAM size\n", status); + exit(EXIT_FAILURE); + } + if (sysMemSize < (16UL << 30) || vramSize < (4UL << 30)) { + fprintf(stderr, "No enough system memory or VRAM\n"); + exit(0); + } + nBuf = vramSize / bufSize; + + void **pBuf = (void **)malloc(sizeof(*pBuf) * nBuf); + memFlags.ui32.NonPaged = 1; + + for (int i = 0; i < nBuf; i++) { + status = hsaKmtAllocMemory(Node, bufSize, memFlags, &pBuf[i]); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to alloc buf %d\n", status, i); + exit(EXIT_FAILURE); + } + + status = hsaKmtMapMemoryToGPU(pBuf[i], bufSize, NULL); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to map buf %d\n", status, i); + exit(EXIT_FAILURE); + } + } + + //printf("Freeing every other BO to fragment VRAM\n"); + for (int i = 0; i < nBuf; i+=2) { + status = hsaKmtUnmapMemoryToGPU(pBuf[i]); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to unmap buf %d from GPU\n", status, i); + exit(EXIT_FAILURE); + } + status = hsaKmtFreeMemory(pBuf[i], bufSize); + if (status != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, "Failed %d to free buf %d\n", status, i); + exit(EXIT_FAILURE); + } + } + + printf("Node %d Size 0x%lx (%ld MB)\n", Node, MemoryProperty->SizeInBytes, + MemoryProperty->SizeInBytes / (1024 * 1024)); + + void *cpu_ptr; + int ret = 0; + void *MemoryAddress = 0; + HSAuint64 SizeInBytes = 1UL << 30; + + memFlags.ui32.Contiguous = 1; + + status = hsaKmtAllocMemory(Node, SizeInBytes, memFlags, &MemoryAddress); + if (status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "Failure to allocate memory 0x%lx. Status %d\n", SizeInBytes, status); + exit(EXIT_FAILURE); + } + + status = hsaKmtMapMemoryToGPU(MemoryAddress, SizeInBytes, NULL); + if (status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "Failure to map memory. Status %d\n", status); + exit(EXIT_FAILURE); + } + + printf("VRAM allocated. Address %p size 0x%lx bytes\n", MemoryAddress, SizeInBytes); + //printf("Press Enter key to continue\n"); + //getchar(); + + struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM get_page_size = {0}; + get_page_size.addr = (uint64_t) MemoryAddress; + get_page_size.length = SizeInBytes; + + ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGE_SIZE, &get_page_size); + if (ret != 0) + { + fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGE_SIZE error (errno=%d/%s)\n", + ret, strerror(ret)); + exit(EXIT_FAILURE); + } + + printf("GPU Page size: 0x%ld\n", get_page_size.page_size); + + struct AMDRDMA_IOCTL_GET_PAGES_PARAM get_cpu_ptr = {0}; + get_cpu_ptr.addr = (uint64_t) MemoryAddress; + get_cpu_ptr.length = SizeInBytes; + + ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGES, &get_cpu_ptr); + if (ret != 0) + { + fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGES error (errno=%d/%s)\n", + ret, strerror(ret)); + //printf("IOCTL_GET_PAGES failed, Press Enter key to continue\n"); + //getchar(); + exit(EXIT_FAILURE); + } + + printf("IOCTL_GET_PAGES return contiguous VRAM address %p size 0x%lx bytes\n", MemoryAddress, SizeInBytes); + printf("Pause to dump page table to check if allocation is contiguous\n"); + printf("Press Enter key to continue\n"); + getchar(); + + ret = rdma_map((uint64_t)MemoryAddress, 4096, &cpu_ptr); + if (ret < 0) + { + exit(EXIT_FAILURE); + } + + hsaKmtFreeMemory(MemoryAddress, SizeInBytes); +} + int main(void) { HsaVersionInfo VersionInfo; @@ -234,6 +393,7 @@ int main(void) // We found local memory available for RDMA operation. // Run some tests on it. run_rdma_tests(iNode, &MemoryProperties[iMemBank]); + run_rdma_contiguous_mem_tests(iNode, &MemoryProperties[iMemBank]); } } }