From 1bf9b39cf87b17fcf264d490fb597fe24e45f4d6 Mon Sep 17 00:00:00 2001
From: Saleel Kudchadker <Saleel.Kudchadker@amd.com>
Date: Wed, 1 Sep 2021 10:42:34 -0700
Subject: [PATCH] SWDEV-301667 - Kern arg placement

Add a env var ROC_USE_FGS_KERNARG to toggle kernel arg placement
By default its in Fine Grain Kernel arg segment for supported asics.

Change-Id: I3d57ed69a1a4db2b392b0438ead499f3ddca4716


[ROCm/clr commit: e29b9c00ee68be9836f8857345e1d7a597a094d0]
---
 projects/clr/rocclr/device/rocm/rocdevice.cpp   | 7 ++++---
 projects/clr/rocclr/device/rocm/rocsettings.cpp | 8 ++++++--
 projects/clr/rocclr/device/rocm/rocsettings.hpp | 7 ++++---
 projects/clr/rocclr/utils/flags.hpp             | 4 +++-
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/projects/clr/rocclr/device/rocm/rocdevice.cpp b/projects/clr/rocclr/device/rocm/rocdevice.cpp
index f31ec77c02..77e3430cc0 100644
--- a/projects/clr/rocclr/device/rocm/rocdevice.cpp
+++ b/projects/clr/rocclr/device/rocm/rocdevice.cpp
@@ -120,7 +120,7 @@ bool NullDevice::create(const amd::Isa &isa) {
   roc::Settings* hsaSettings = new roc::Settings();
   settings_ = hsaSettings;
   if (!hsaSettings ||
-      !hsaSettings->create(false, isa.versionMajor(), isa.versionMinor(),
+      !hsaSettings->create(false, isa.versionMajor(), isa.versionMinor(), isa.versionStepping(),
                            isa.xnack() == amd::Isa::Feature::Enabled)) {
     LogPrintfError("Error creating settings for offline HSA device %s", isa.targetId());
     return false;
@@ -631,7 +631,8 @@ bool Device::create() {
   settings_ = hsaSettings;
   if (!hsaSettings ||
       !hsaSettings->create((agent_profile_ == HSA_PROFILE_FULL), isa->versionMajor(),
-                           isa->versionMinor(), isa->xnack() == amd::Isa::Feature::Enabled,
+                           isa->versionMinor(), isa->versionStepping(),
+                           isa->xnack() == amd::Isa::Feature::Enabled,
                            coop_groups)) {
     LogPrintfError("Unable to create settings for HSA device %s (PCI ID %x)", agent_name,
                    pciDeviceId_);
@@ -1847,7 +1848,7 @@ void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) co
   hsa_amd_memory_pool_t segment{0};
   switch (mem_seg) {
     case kKernArg : {
-      if (::strcmp(isa().processorName().c_str(), "gfx90a") == 0) {
+      if (settings().fgs_kernel_arg_) {
         segment = system_kernarg_segment_;
         break;
       }
diff --git a/projects/clr/rocclr/device/rocm/rocsettings.cpp b/projects/clr/rocclr/device/rocm/rocsettings.cpp
index 17c940e3f3..eaa2f2ea63 100644
--- a/projects/clr/rocclr/device/rocm/rocsettings.cpp
+++ b/projects/clr/rocclr/device/rocm/rocsettings.cpp
@@ -100,8 +100,8 @@ Settings::Settings() {
 }
 
 // ================================================================================================
-bool Settings::create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor, bool enableXNACK,
-                      bool coop_groups) {
+bool Settings::create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor,
+                      uint32_t gfxStepping, bool enableXNACK, bool coop_groups) {
   customHostAllocator_ = false;
 
   if (fullProfile) {
@@ -172,6 +172,10 @@ bool Settings::create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor
 
   lcWavefrontSize64_ = !enableWave32Mode_;
 
+  // Enable fgs kernel arg segment only for a specific HW
+  fgs_kernel_arg_ = gfxipMajor == 9 && gfxStepping == 10;
+  fgs_kernel_arg_ = (!flagIsDefault(ROC_USE_FGS_KERNARG)) ?
+                          ROC_USE_FGS_KERNARG : fgs_kernel_arg_;
   // Override current device settings
   override();
 
diff --git a/projects/clr/rocclr/device/rocm/rocsettings.hpp b/projects/clr/rocclr/device/rocm/rocsettings.hpp
index a7107ce861..f12e5cd774 100644
--- a/projects/clr/rocclr/device/rocm/rocsettings.hpp
+++ b/projects/clr/rocclr/device/rocm/rocsettings.hpp
@@ -52,7 +52,8 @@ class Settings : public device::Settings {
       uint cpu_wait_for_signal_ : 1;    //!< Wait for HSA signal on CPU
       uint system_scope_signal_ : 1;    //!< HSA signal is visibile to the entire system
       uint skip_copy_sync_ : 1;         //!< Ignore explicit HSA signal waits for copy functionality
-      uint reserved_ : 21;
+      uint fgs_kernel_arg_ : 1;         //!< Use fine grain kernel arg segment
+      uint reserved_ : 20;
     };
     uint value_;
   };
@@ -89,8 +90,8 @@ class Settings : public device::Settings {
   Settings();
 
   //! Creates settings
-  bool create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor, bool enableXNACK,
-              bool coop_groups = false);
+  bool create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor, uint32_t gfxStepping,
+              bool enableXNACK, bool coop_groups = false);
 
  private:
   //! Disable copy constructor
diff --git a/projects/clr/rocclr/utils/flags.hpp b/projects/clr/rocclr/utils/flags.hpp
index 4197d1f9e2..db18ab3fd2 100644
--- a/projects/clr/rocclr/utils/flags.hpp
+++ b/projects/clr/rocclr/utils/flags.hpp
@@ -261,7 +261,9 @@ release(cstring, AMD_LOG_LEVEL_FILE, "",                                      \
 release(size_t, PAL_PREPINNED_MEMORY_SIZE, 64,                                \
         "Size in KBytes of prepinned memory")                                 \
 release(bool, AMD_CPU_AFFINITY, false,                                        \
-        "Reset CPU affinity of any runtime threads")
+        "Reset CPU affinity of any runtime threads")                          \
+release(bool, ROC_USE_FGS_KERNARG, true,                                      \
+        "Use fine grain kernel args segment for supported asics")
 
 namespace amd {