From 680c8ca5a9a355b67bb72e4276aca27f6805c462 Mon Sep 17 00:00:00 2001 From: David Francis Date: Tue, 17 Jan 2023 10:42:03 -0500 Subject: [PATCH] kfdtest: Make queue evict tests use constant number of wavefronts. Previously, KFDEvictTest.QueueTest and KFDSVMEvictTest.QueueTest would create a variable number of wavefronts, one for each 64MB of memory under test. This ran into limits on the buffers used by the wavefronts, and may at some point have exceeded the wavefront limit. Restrict the number of wavefronts to 512, and adjust the shader to accomodate a variable buffer size Signed-off-by: David Francis Change-Id: I2ec292e2900e2efa62a08313bca3d2f4bdabca8b --- tests/kfdtest/src/KFDEvictTest.cpp | 11 +++++++--- tests/kfdtest/src/KFDSVMEvictTest.cpp | 31 +++++++++++++++++++++++---- tests/kfdtest/src/KFDSVMEvictTest.hpp | 1 + tests/kfdtest/src/ShaderStore.cpp | 11 ++++++++-- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 1effa639b0..417123af37 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -33,6 +33,7 @@ #define N_PROCESSES (2) /* Number of processes running in parallel, must be at least 2 */ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +#define MAX_WAVEFRONTS (512) #define SDMA_NOP 0x0 @@ -382,7 +383,7 @@ TEST_F(KFDEvictTest, QueueTest) { HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + unsigned int count = MAX_WAVEFRONTS; const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); @@ -406,9 +407,10 @@ TEST_F(KFDEvictTest, QueueTest) { // Use 7/8 of VRAM between all processes HSAuint64 testSize = vramSize * 7 / 8; - HSAuint32 count = testSize / (vramBufSize * N_PROCESSES); + HSAuint32 vramBufSize = testSize / (count * N_PROCESSES); + vramBufSize = (vramBufSize / (1024 * 1024)) * (1024 * 1024); - if (count == 0) { + if (vramBufSize == 0) { LOG() << "Skipping test: Not enough system memory available." << std::endl; return; } @@ -455,6 +457,9 @@ TEST_F(KFDEvictTest, QueueTest) { for (i = 0; i < wavefront_num; i++) *(localBufAddr + i) = pBuffers[i]; + for (i = 0; i < wavefront_num; i++) + *(result + i) = vramBufSize; + dispatch0.SetArgs(localBufAddr, result); dispatch0.SetDim(wavefront_num, 1, 1); /* Submit the packet and start shader */ diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index cd61de3c3e..b97e6c7089 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -34,6 +34,7 @@ #define N_PROCESSES (4) /* number of processes running in parallel, at least 2 */ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +#define MAX_WAVEFRONTS (512) void KFDSVMEvictTest::SetUp() { ROUTINE_START @@ -64,7 +65,7 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; /* use one third of total system memory for eviction buffer to test - * limit max allocate size to duoble of vramSize + * limit max allocate size to double of vramSize * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) */ size = sysMemSize / 3 + vramSize; @@ -81,6 +82,25 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf return count; } +HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count) { + HSAuint64 sysMemSize = GetSysMemSize(); + HSAuint64 size, sizeInPages; + HSAuint64 vramBufSizeInPages; + + LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; + + /* use one third of total system memory for eviction buffer to test + * limit max allocate size to duoble of vramSize + * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) + */ + size = sysMemSize / 3 + vramSize; + size = size > vramSize << 1 ? vramSize << 1 : size; + sizeInPages = size >> PAGE_SHIFT; + vramBufSizeInPages = sizeInPages / (count * N_PROCESSES); + + return vramBufSizeInPages << 20; +} + void KFDSVMEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, std::vector &pBuffers) { HSAuint64 totalMB; @@ -266,7 +286,7 @@ TEST_F(KFDSVMEvictTest, QueueTest) { HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + unsigned int count = MAX_WAVEFRONTS; const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); @@ -286,8 +306,8 @@ TEST_F(KFDSVMEvictTest, QueueTest) { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; } - HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); - if (count == 0) { + HSAuint64 vramBufSize = GetBufferSize(vramSize, count); + if (vramBufSize == 0) { LOG() << "Not enough system memory, skipping the test" << std::endl; return; } @@ -315,6 +335,9 @@ TEST_F(KFDSVMEvictTest, QueueTest) { for (i = 0; i < wavefront_num; i++) *(localBufAddr + i) = pBuffers[i]; + for (i = 0; i < wavefront_num; i++) + *(result + i) = vramBufSize; + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadMemoryIsa, isaBuffer.As())); PM4Queue pm4Queue; diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/tests/kfdtest/src/KFDSVMEvictTest.hpp index 3f26287cc0..7e772105ae 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.hpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.hpp @@ -60,6 +60,7 @@ class KFDSVMEvictTest : public KFDLocalMemoryTest { void ForkChildProcesses(int nprocesses); void WaitChildProcesses(); HSAint32 GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize); + HSAint64 GetBufferSize(HSAuint64 vramSize, HSAuint32 count); protected: // members std::string m_psName; diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index a5fbbc2d64..d9b32c7e37 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -523,6 +523,7 @@ const char *PersistentIterateIsa = SHADER_MACROS R"( * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 * v[6:7] - local buf address used for read test + * v11 - size of local buffer in MB */ const char *ReadMemoryIsa = SHADER_MACROS R"( // Compute address of corresponding output buffer @@ -538,6 +539,13 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v0 * 8 V_ADD_CO_CI_U32 v3, v3, 0 // v[2:3] = s[0:1] + v0 * 8 + //Load local buffer size from output buffer + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + flat_load_dword v11, v[4:5] nt sc1 sc0 + .else + flat_load_dword v11, v[4:5] slc + .endif + // Load 64bit local buffer address stored at v[2:3] to v[6:7] .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) flat_load_dwordx2 v[6:7], v[2:3] nt sc1 sc0 @@ -553,11 +561,10 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( s_cmp_eq_i32 s16, s8 s_cbranch_scc1 L_QUIT // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] + // Loop read local buffer starting at v[6:7] // every 4k page only read once v_mov_b32 v9, 0 v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size v_mov_b32 v12, v6 v_mov_b32 v13, v7 L_LOOP_READ: