From 0ddfa04517a1899c768581f98e325d8c9ecf869c Mon Sep 17 00:00:00 2001
From: Saleel Kudchadker <Saleel.Kudchadker@amd.com>
Date: Thu, 13 Feb 2020 11:42:04 -0800
Subject: [PATCH] Implement HIP_HIDDEN_FREE_MEM env var

Set value to 256Mb to reflect what HIP/HCC reserves
Change-Id: Icaadf79f60d3916965ac168da237d15b975b1fe4


[ROCm/clr commit: 0730b39adb7322598c99348811285a68b0cb0d00]
---
 projects/clr/rocclr/device/device.cpp         | 13 +++++++++----
 projects/clr/rocclr/device/pal/paldevice.cpp  |  3 +++
 projects/clr/rocclr/device/rocm/rocdevice.cpp |  3 ++-
 projects/clr/rocclr/utils/flags.hpp           |  6 +++++-
 4 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/projects/clr/rocclr/device/device.cpp b/projects/clr/rocclr/device/device.cpp
index 6b0930f0f2..d88adfe5fe 100644
--- a/projects/clr/rocclr/device/device.cpp
+++ b/projects/clr/rocclr/device/device.cpp
@@ -469,11 +469,16 @@ Settings::Settings() : value_(0) {
       ? 4
       : 0;
 
-  if (amd::IS_HIP) {
-    GPU_SINGLE_ALLOC_PERCENT = 100;
-  }
-
   fenceScopeAgent_ = AMD_OPT_FLUSH;
+  if (amd::IS_HIP) {
+    if (flagIsDefault(GPU_SINGLE_ALLOC_PERCENT)) {
+      GPU_SINGLE_ALLOC_PERCENT = 100;
+    }
+
+    if (flagIsDefault(HIP_HIDDEN_FREE_MEM)) {
+      HIP_HIDDEN_FREE_MEM = 256;
+    }
+  }
 }
 
 void Memory::saveMapInfo(const void* mapAddress, const amd::Coord3D origin,
diff --git a/projects/clr/rocclr/device/pal/paldevice.cpp b/projects/clr/rocclr/device/pal/paldevice.cpp
index 51094ba944..f3e705b8cd 100644
--- a/projects/clr/rocclr/device/pal/paldevice.cpp
+++ b/projects/clr/rocclr/device/pal/paldevice.cpp
@@ -1866,6 +1866,9 @@ bool Device::globalFreeMemory(size_t* freeMemory) const {
   }
   freeMemory[LargestFreeBlock] = static_cast<size_t>(invisible) / Ki;
 
+  freeMemory[TotalFreeMemory] -= (freeMemory[TotalFreeMemory] > HIP_HIDDEN_FREE_MEM * Ki) ?
+                                  HIP_HIDDEN_FREE_MEM * Ki : 0;
+
   if (settings().apuSystem_) {
     Pal::gpusize sysMem = allocedMem[Pal::GpuHeapGartCacheable] + allocedMem[Pal::GpuHeapGartUswc] -
         resourceCache().cacheSize() + resourceCache().lclCacheSize();
diff --git a/projects/clr/rocclr/device/rocm/rocdevice.cpp b/projects/clr/rocclr/device/rocm/rocdevice.cpp
index ebc7f803ce..0812b4e38a 100755
--- a/projects/clr/rocclr/device/rocm/rocdevice.cpp
+++ b/projects/clr/rocclr/device/rocm/rocdevice.cpp
@@ -1475,7 +1475,8 @@ bool Device::globalFreeMemory(size_t* freeMemory) const {
   const uint LargestFreeBlock = 1;
 
   freeMemory[TotalFreeMemory] = freeMem_ / Ki;
-
+  freeMemory[TotalFreeMemory] -= (freeMemory[TotalFreeMemory] > HIP_HIDDEN_FREE_MEM * Ki) ?
+                                  HIP_HIDDEN_FREE_MEM * Ki : 0;
   // since there is no memory heap on ROCm, the biggest free block is
   // equal to total free local memory
   freeMemory[LargestFreeBlock] = freeMemory[TotalFreeMemory];
diff --git a/projects/clr/rocclr/utils/flags.hpp b/projects/clr/rocclr/utils/flags.hpp
index 24731f738c..a37c0a27c9 100644
--- a/projects/clr/rocclr/utils/flags.hpp
+++ b/projects/clr/rocclr/utils/flags.hpp
@@ -228,7 +228,11 @@ release(uint, HIP_HOST_COHERENT, 0,                                           \
         "0x0 = memory is not coherent between host and GPU")                  \
 release(uint, AMD_OPT_FLUSH, 0,                                               \
         "Kernel flush option , 0x0 = Use system-scope fence operations."      \
-        "0x1 = Use device-scope fence operations when possible.")
+        "0x1 = Use device-scope fence operations when possible.")             \
+release(uint, HIP_HIDDEN_FREE_MEM, 0,                                         \
+        "Reserve free mem reporting in Mb"                                    \
+        "0 = Disable")
+
 namespace amd {
 
 extern bool IS_HIP;