From 949f8fc7aaafbf37d3976a9b4c76c88a0cde6800 Mon Sep 17 00:00:00 2001
From: Philip Yang <Philip.Yang@amd.com>
Date: Mon, 4 Apr 2022 20:16:59 -0400
Subject: [PATCH] kfdtest: Set XNACK mode according HSA_XNACK env setting

According env setting HSA_XNACK=1 or 0, set XNACK mode ON or OFF to run
KFDSVMRangeTest and KFDSVMEvictTest. If HSA_XNACK is not defined, use
system boot-time XNACK mode setting.

Restore to the original XNACK mode when test finished.

Change-Id: Ia896a1b0a90854646c8a79acca38a7d46098efde
Signed-off-by: Philip Yang <Philip.Yang@amd.com>


[ROCm/ROCR-Runtime commit: 23ec6e880e25bf1246d04839f828d39fcb3b0766]
---
 .../kfdtest/src/KFDBaseComponentTest.hpp      | 34 +++++++++++++++++++
 .../tests/kfdtest/src/KFDSVMEvictTest.cpp     | 20 +++++++++++
 .../tests/kfdtest/src/KFDSVMEvictTest.hpp     |  4 +++
 .../tests/kfdtest/src/KFDSVMRangeTest.cpp     | 10 +++---
 4 files changed, 64 insertions(+), 4 deletions(-)

diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp b/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp
index 187d9fbd73..e27baf1cc1 100644
--- a/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp
+++ b/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp
@@ -73,6 +73,7 @@ class KFDBaseComponentTest : public testing::Test {
     unsigned int m_numSdmaQueuesPerEngine;
     HsaMemFlags m_MemoryFlags;
     HsaNodeInfo m_NodeInfo;
+    HSAint32 m_xnack;
 
     // @brief Executed before every test that uses KFDBaseComponentTest class and sets all common settings for the tests.
     virtual void SetUp();
@@ -86,6 +87,39 @@ class KFDBaseComponentTest : public testing::Test {
             LOG() << "SVM API not supported" << std::endl;
         return supported;
     }
+
+    void SVMSetXNACKMode() {
+        if (!SVMAPISupported())
+            return;
+
+        m_xnack = -1;
+
+        char *hsa_xnack = getenv("HSA_XNACK");
+        if (!hsa_xnack)
+            return;
+
+        HSAKMT_STATUS ret = hsaKmtGetXNACKMode(&m_xnack);
+        if (ret != HSAKMT_STATUS_SUCCESS) {
+            LOG() << "Failed " << ret << " to get XNACK mode" << std::endl;
+            return;
+        }
+
+        // XNACK OFF if defined HSA_XNACK=0
+        HSAint32 xnack_on = strncmp(hsa_xnack, "0", 1);
+        ret = hsaKmtSetXNACKMode(xnack_on);
+        if (ret != HSAKMT_STATUS_SUCCESS)
+            LOG() << "Failed " << ret << " to set XNACK mode " << xnack_on << std::endl;
+    }
+
+    void SVMRestoreXNACKMode() {
+        if (!SVMAPISupported())
+             return;
+
+        if (m_xnack == -1)
+            return;
+
+        hsaKmtSetXNACKMode(m_xnack);
+    }
 };
 
 extern KFDBaseComponentTest* g_baseTest;
diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.cpp
index f326d32093..d41aedac74 100644
--- a/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.cpp
+++ b/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.cpp
@@ -35,6 +35,26 @@
 #define ALLOCATE_BUF_SIZE_MB    (64)
 #define ALLOCATE_RETRY_TIMES    (3)
 
+void KFDSVMEvictTest::SetUp() {
+    ROUTINE_START
+
+    KFDLocalMemoryTest::SetUp();
+
+    SVMSetXNACKMode();
+
+    ROUTINE_END
+}
+
+void KFDSVMEvictTest::TearDown() {
+    ROUTINE_START
+
+    SVMRestoreXNACKMode();
+
+    KFDLocalMemoryTest::TearDown();
+
+    ROUTINE_END
+}
+
 HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) {
     HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT;
     HSAuint64 sysMemSize = GetSysMemSize();
diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.hpp b/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.hpp
index 7749ac2dee..2b8a1de957 100644
--- a/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.hpp
+++ b/projects/rocr-runtime/tests/kfdtest/src/KFDSVMEvictTest.hpp
@@ -49,6 +49,10 @@ class KFDSVMEvictTest :  public KFDLocalMemoryTest {
         } catch (...) {}
     }
 
+ protected:
+    virtual void SetUp();
+    virtual void TearDown();
+
  protected:
     std::string CreateShader();
     void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize,
diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDSVMRangeTest.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDSVMRangeTest.cpp
index 655ad4ed66..6aad683f91 100644
--- a/projects/rocr-runtime/tests/kfdtest/src/KFDSVMRangeTest.cpp
+++ b/projects/rocr-runtime/tests/kfdtest/src/KFDSVMRangeTest.cpp
@@ -36,6 +36,8 @@ void KFDSVMRangeTest::SetUp() {
 
     m_pIsaGen = IsaGenerator::Create(m_FamilyId);
 
+    SVMSetXNACKMode();
+
     ROUTINE_END
 }
 
@@ -46,6 +48,8 @@ void KFDSVMRangeTest::TearDown() {
         delete m_pIsaGen;
     m_pIsaGen = NULL;
 
+    SVMRestoreXNACKMode();
+
     KFDBaseComponentTest::TearDown();
 
     ROUTINE_END
@@ -1287,10 +1291,8 @@ TEST_F(KFDSVMRangeTest, ReadOnlyRangeTest) {
      */
     int pid = fork();
     if (pid == 0) {
-        if (hsaKmtOpenKFD() != HSAKMT_STATUS_SUCCESS) {
-            WARN() << "KFD open failed in child process" << std::endl;
-            exit(1);
-        }
+        TearDown();
+        SetUp();
     } else {
         int childStatus;