From e1fdd6fb5f3dec98eed821e29051e2397dddae28 Mon Sep 17 00:00:00 2001
From: Xiaogang Chen <xiaogang.chen@amd.com>
Date: Thu, 12 Dec 2024 11:53:03 -0600
Subject: [PATCH] kfdtest: Separate LLVM initialization and instantiation

Assembler function from LLVM is not multi-thread safe and is ASIC dependent.
To extend kfdtest to test on multiple GPU separate LLVM initialization and
assembler instantiation. Each test uses its own assembler.

Signed-off-by: Xiaogang Chen <Xiaogang.Chen@amd.com>
Change-Id: I94c996e807b81d8b19361b450be985b12042477c
---
 libhsakmt/tests/kfdtest/src/Assemble.cpp    | 26 ++++++++++++---------
 libhsakmt/tests/kfdtest/src/Assemble.hpp    |  6 ++++-
 libhsakmt/tests/kfdtest/src/KFDTestMain.cpp | 13 ++++++++++-
 3 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/libhsakmt/tests/kfdtest/src/Assemble.cpp b/libhsakmt/tests/kfdtest/src/Assemble.cpp
index 9b799813a0..eda7fa841e 100644
--- a/libhsakmt/tests/kfdtest/src/Assemble.cpp
+++ b/libhsakmt/tests/kfdtest/src/Assemble.cpp
@@ -79,16 +79,29 @@
 
 using namespace llvm;
 
+/* Assembler implementation is not multi-thread safe and is
+ * asic type dependent. Instantiate it per thread/gpu use case,
+ * delete each assembler after assembling
+ */
+
+void Init_LLVM() {
+    LLVMInitializeAMDGPUTargetInfo();
+    LLVMInitializeAMDGPUTargetMC();
+    LLVMInitializeAMDGPUAsmParser();
+}
+
+void Shutdown_LLVM() {
+    llvm_shutdown();
+}
+
 Assembler::Assembler(const uint32_t Gfxv) {
     SetTargetAsic(Gfxv);
     TextData = nullptr;
     TextSize = 0;
-    LLVMInit();
 }
 
 Assembler::~Assembler() {
     FlushText();
-    llvm_shutdown();
 }
 
 const char* Assembler::GetInstrStream() {
@@ -123,15 +136,6 @@ void Assembler::SetTargetAsic(const uint32_t Gfxv) {
     snprintf(MCPU, ASM_MCPU_LEN, "gfx%d%d%x", Major, Minor, Step);
 }
 
-/**
- * Initialize LLVM targets and assembly printers/parsers
- */
-void Assembler::LLVMInit() {
-    LLVMInitializeAMDGPUTargetInfo();
-    LLVMInitializeAMDGPUTargetMC();
-    LLVMInitializeAMDGPUAsmParser();
-}
-
 /**
  * Flush/reset TextData and TextSize to initial state
  */
diff --git a/libhsakmt/tests/kfdtest/src/Assemble.hpp b/libhsakmt/tests/kfdtest/src/Assemble.hpp
index a519129ed8..fc1ad05473 100644
--- a/libhsakmt/tests/kfdtest/src/Assemble.hpp
+++ b/libhsakmt/tests/kfdtest/src/Assemble.hpp
@@ -47,6 +47,11 @@
 
 #define ASM_MCPU_LEN 16
 
+/* initialize LLVM targets and assembly printers/parsers */
+void Init_LLVM();
+/* shutdown LLVM */
+void Shutdown_LLVM();
+
 class Assembler {
   private:
       const char* ArchName = "amdgcn";
@@ -62,7 +67,6 @@ class Assembler {
 
       void SetTargetAsic(const uint32_t Gfxv);
 
-      void LLVMInit();
       void FlushText();
       void PrintELFHex(const std::string Data);
       int ExtractELFText(const char* RawData);
diff --git a/libhsakmt/tests/kfdtest/src/KFDTestMain.cpp b/libhsakmt/tests/kfdtest/src/KFDTestMain.cpp
index 8d2cb40fff..1a806b08f4 100644
--- a/libhsakmt/tests/kfdtest/src/KFDTestMain.cpp
+++ b/libhsakmt/tests/kfdtest/src/KFDTestMain.cpp
@@ -26,6 +26,7 @@
 #include "KFDTestUtil.hpp"
 #include "GoogleTestExtension.hpp"
 #include "OSWrapper.hpp"
+#include "Assemble.hpp"
 
 #define KFD_TEST_DEFAULT_TIMEOUT 60000
 
@@ -72,6 +73,7 @@ GTEST_API_ int main(int argc, char **argv) {
     bool success = GetCommandLineArguments(argc, argv, args);
 
     if (success) {
+        int r;
         if ((GetHwCapabilityHWS() || args.HwsEnabled == HWCAP__FORCE_ENABLED) &&
                 (args.HwsEnabled != HWCAP__FORCE_DISABLED))
             g_TestENVCaps |= ENVCAPS_HWSCHEDULING;
@@ -104,6 +106,15 @@ GTEST_API_ int main(int argc, char **argv) {
             LOG() << "Sleep time in seconds as specified by user: " << std::dec << g_SleepTime << std::endl;
         }
 
-        return RUN_ALL_TESTS();
+        /* init LLVM one time*/
+        Init_LLVM();
+
+        r = RUN_ALL_TESTS();
+
+        /* shutdown LLVM after tests finish */
+        Shutdown_LLVM();
+
+        LOG() << "kfdtest finished with return code: " << r << std::endl;
+        return r;
     }
 }