AI/rocm-systems
2
0
Garpu 0
Kode Masalah Tarik Permintaan Actions Packages Projects Rilis Wiki Kegiatan

KFDTest: fix failure when run KFDTest on multi-GPU small bar system

Telusuri Sumber 
On small bar multi-gpu system, hsaKmtMemoryMapToGPU will fail due to latest
kernel P2P sanity check. Swith to use hsaKmtMemoryMapToGPUNodes to fix
the failure

Change-Id: Id8b6329d1243df0e908cc9a171b5c7f9156f4a8b
Signed-off-by: shaoyunl <shaoyun.liu@amd.com>
This commit is contained in:
shaoyunl
2018-11-19 11:38:38 -05:00
orang tua 29b45b8c0a
melakukan d8009b4fd3
5 mengubah file dengan 58 tambahan dan 25 penghapusan
Download Patch File Download Diff File Expand all files Collapse all files
tests/kfdtest/src/KFDEvictTest.cpp
+3 -1
Melihat File
@@ -44,6 +44,7 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui
<< totalMB << ")MB VRAM in KFD" << std::endl;
}
HsaMemMapFlags mapFlags = {0};
HSAKMT_STATUS ret;
HSAuint32 retry = 0;
@@ -56,7 +57,8 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui
ret = hsaKmtAllocMemory(defaultGPUNode, vramBufSize, m_Flags, &m_pBuf);
if (ret == HSAKMT_STATUS_SUCCESS) {
if (is_dgpu()) {
if (hsaKmtMapMemoryToGPU(m_pBuf, vramBufSize, NULL) == HSAKMT_STATUS_ERROR) {
if (hsaKmtMapMemoryToGPUNodes(m_pBuf, vramBufSize, NULL,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)) == HSAKMT_STATUS_ERROR) {
EXPECT_SUCCESS(hsaKmtFreeMemory(m_pBuf, vramBufSize));
break;
}
tests/kfdtest/src/KFDIPCTest.cpp
+6 -2
Melihat File
@@ -75,13 +75,15 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) {
HSAuint64 size = PAGE_SIZE, sharedSize;
HsaMemoryBuffer tempSysBuffer(size, defaultGPUNode, false);
HSAuint32 *sharedLocalBuffer = NULL;
HsaMemMapFlags mapFlags = {0};
/* Read from Pipe the shared Handle. Import shared Local Memory */
ASSERT_GE(read(pipefd[0], reinterpret_cast<void*>(&sharedHandleLM), sizeof(sharedHandleLM)), 0);
ASSERT_SUCCESS(hsaKmtRegisterSharedHandle(&sharedHandleLM,
reinterpret_cast<void**>(&sharedLocalBuffer), &sharedSize));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(sharedLocalBuffer, sharedSize, NULL));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sharedLocalBuffer, sharedSize, NULL,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
/* Check for pattern in the shared Local Memory */
ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode));
@@ -114,9 +116,11 @@ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pip
HsaMemoryBuffer tempSysBuffer(PAGE_SIZE, defaultGPUNode, false);
SDMAQueue sdmaQueue;
HsaSharedMemoryHandle sharedHandleLM;
HsaMemMapFlags mapFlags = {0};
/* Fill a Local Buffer with a pattern */
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(toShareLocalBuffer.As<void*>(), toShareLocalBuffer.Size(), &AlternateVAGPU));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(toShareLocalBuffer.As<void*>(), toShareLocalBuffer.Size(), &AlternateVAGPU,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
tempSysBuffer.Fill(0xAAAAAAAA);
/* Copy pattern in Local Memory before sharing it */
tests/kfdtest/src/KFDLocalMemoryTest.cpp
+13 -6
Melihat File
@@ -57,6 +57,7 @@ TEST_F(KFDLocalMemoryTest, BasicTest) {
PM4Queue queue;
HSAuint64 AlternateVAGPU;
unsigned int BufferSize = PAGE_SIZE;
HsaMemMapFlags mapFlags = {0};
int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode();
ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node";
@@ -76,8 +77,10 @@ TEST_F(KFDLocalMemoryTest, BasicTest) {
m_pIsaGen->GetCopyDwordIsa(isaBuffer);
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(srcLocalBuffer.As<void*>(), srcLocalBuffer.Size(), &AlternateVAGPU));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dstLocalBuffer.As<void*>(), dstLocalBuffer.Size(), &AlternateVAGPU));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(srcLocalBuffer.As<void*>(), srcLocalBuffer.Size(), &AlternateVAGPU,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(dstLocalBuffer.As<void*>(), dstLocalBuffer.Size(), &AlternateVAGPU,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
queue.SetSkipWaitConsump(0);
@@ -112,6 +115,7 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) {
PM4Queue queue;
HSAuint64 AlternateVAGPU;
unsigned int BufferSize = PAGE_SIZE;
HsaMemMapFlags mapFlags = {0};
int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode();
ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node";
@@ -134,7 +138,8 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) {
queue.SetSkipWaitConsump(0);
if (!is_dgpu())
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As<void*>(), LocalBuffer.Size(), &AlternateVAGPU));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(LocalBuffer.As<void*>(), LocalBuffer.Size(), &AlternateVAGPU,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
Dispatch dispatch(isaBuffer);
@@ -143,7 +148,8 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) {
dispatch.Sync(g_TestTimeOut);
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As<void*>()));
EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As<void*>(), LocalBuffer.Size(), &AlternateVAGPU));
EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(LocalBuffer.As<void*>(), LocalBuffer.Size(), &AlternateVAGPU,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
dispatch.SetArgs(LocalBuffer.As<void*>(), SysBufferB.As<void*>());
dispatch.Submit(queue);
@@ -270,6 +276,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) {
/* Allocate and test memory using the strategy explained at the top */
HSAKMT_STATUS status;
HsaMemFlags memFlags = {0};
HsaMemMapFlags mapFlags = {0};
memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB;
memFlags.ui32.HostAccess = 0;
memFlags.ui32.NonPaged = 1;
@@ -318,8 +325,8 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) {
+ size - sizeof(unsigned));
sysBuffer.As<unsigned *>()[0] = ++value;
status = hsaKmtMapMemoryToGPU(pages[order].pointers[p],
size, NULL);
status = hsaKmtMapMemoryToGPUNodes(pages[order].pointers[p], size, NULL,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode));
if (status != HSAKMT_STATUS_SUCCESS) {
ASSERT_SUCCESS(hsaKmtFreeMemory(pages[order].pointers[p],
size));
tests/kfdtest/src/KFDMemoryTest.cpp
+29 -15
Melihat File
@@ -896,6 +896,16 @@ TEST_F(KFDMemoryTest, MMBench) {
HSAuint64 vramSizeMB = GetVramSize(defaultGPUNode) >> 20;
const std::vector<int> gpuNodes = m_NodeInfo.GetNodesWithGPU();
bool is_all_large_bar = true;
for (unsigned i = 0; i < gpuNodes.size(); i++) {
if (!m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i))) {
is_all_large_bar = false;
break;
}
}
LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl;
if (vramSizeMB == 0)
@@ -986,22 +996,24 @@ TEST_F(KFDMemoryTest, MMBench) {
IDLE_SDMA();
/* Map to all GPUs */
start = GetSystemTickCountInMicroSec();
for (i = 0; i < nBufs; i++) {
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(bufs[i], bufSize, &altVa));
INTERLEAVE_SDMA();
}
mapAllTime = GetSystemTickCountInMicroSec() - start;
IDLE_SDMA();
if (is_all_large_bar) {
start = GetSystemTickCountInMicroSec();
for (i = 0; i < nBufs; i++) {
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(bufs[i], bufSize, &altVa));
INTERLEAVE_SDMA();
}
mapAllTime = GetSystemTickCountInMicroSec() - start;
IDLE_SDMA();
/* Unmap from all GPUs */
start = GetSystemTickCountInMicroSec();
for (i = 0; i < nBufs; i++) {
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i]));
INTERLEAVE_SDMA();
/* Unmap from all GPUs */
start = GetSystemTickCountInMicroSec();
for (i = 0; i < nBufs; i++) {
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i]));
INTERLEAVE_SDMA();
}
unmapAllTime = GetSystemTickCountInMicroSec() - start;
IDLE_SDMA();
}
unmapAllTime = GetSystemTickCountInMicroSec() - start;
IDLE_SDMA();
/* Free */
start = GetSystemTickCountInMicroSec();
@@ -1292,6 +1304,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) {
int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode();
ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node";
HsaMemMapFlags mapFlags = {0};
HsaMemFlags memFlags = {0};
memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB;
/* Allocate host not accessible vram */
@@ -1307,7 +1320,8 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) {
const HSAuint64 VRAM_OFFSET = (4 << 20) - sizeof(HSAuint64);
ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, size, memFlags, &mem));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(mem, size, NULL));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(mem, size, NULL,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
/* Set the word before 4M boundary to 0xdeadbeefdeadbeef
* and the word after 4M boundary to 0xcafebabecafebabe
*/
tests/kfdtest/src/KFDTestUtil.cpp
+7 -1
Melihat File
@@ -148,6 +148,9 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b
m_Node(node) {
m_Flags.Value = 0;
HsaMemMapFlags mapFlags = {0};
bool map_specific_gpu = (node && !isScratch);
if (isScratch) {
m_Flags.ui32.Scratch = 1;
m_Flags.ui32.HostAccess = 1;
@@ -170,7 +173,10 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b
EXPECT_SUCCESS(hsaKmtAllocMemory(m_Node, m_Size, m_Flags, &m_pBuf));
if (is_dgpu()) {
EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(m_pBuf, m_Size, NULL));
if (map_specific_gpu)
EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(m_pBuf, m_Size, NULL, mapFlags, 1, &m_Node));
else
EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(m_pBuf, m_Size, NULL));
m_MappedNodes = 1 << m_Node;
}