From 04e5354d887fbb01a33028b34eedf748a5d2bdbc Mon Sep 17 00:00:00 2001
From: Satyanvesh Dittakavi <Satyanvesh.Dittakavi@amd.com>
Date: Tue, 20 Jul 2021 15:14:32 +0000
Subject: [PATCH] SWDEV-292021 - Fix Device Reset

- Device Reset should not purge the allocations that were not by the user
- Addresses QMCPack Test abort due to the removal of all the mem objects during reset

Change-Id: I7b7a123e72bcc985d7e51d17c2382bc618d3e041


[ROCm/clr commit: 924695fb5e04cb2ffb7f2c200a5ebe92438391f1]
---
 projects/clr/rocclr/device/device.cpp          | 8 +++++---
 projects/clr/rocclr/device/device.hpp          | 2 +-
 projects/clr/rocclr/device/pal/palprogram.cpp  | 1 +
 projects/clr/rocclr/device/rocm/rocprogram.cpp | 5 +++--
 projects/clr/rocclr/platform/memory.hpp        | 3 ++-
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/projects/clr/rocclr/device/device.cpp b/projects/clr/rocclr/device/device.cpp
index 2b6b819957..40e124e517 100644
--- a/projects/clr/rocclr/device/device.cpp
+++ b/projects/clr/rocclr/device/device.cpp
@@ -333,9 +333,11 @@ void MemObjMap::Purge(amd::Device* dev) {
   assert(dev != nullptr);
 
   amd::ScopedLock lock(AllocatedLock_);
-  for (auto it = MemObjMap_.cbegin() ; it != MemObjMap_.cend() ;) {
-    const std::vector<Device*>& devices = it->second->getContext().devices();
-    if (devices.size() == 1 && devices[0] == dev) {
+  for (auto it = MemObjMap_.cbegin(); it != MemObjMap_.cend(); ) {
+    amd::Memory* memObj = it->second;
+    unsigned int flags = memObj->getMemFlags();
+    const std::vector<Device*>& devices = memObj->getContext().devices();
+    if (devices.size() == 1 && devices[0] == dev && !(flags & ROCCLR_MEM_INTERNAL_MEMORY)) {
       it = MemObjMap_.erase(it);
     } else {
       ++it;
diff --git a/projects/clr/rocclr/device/device.hpp b/projects/clr/rocclr/device/device.hpp
index 115176f5ef..845b65df2e 100644
--- a/projects/clr/rocclr/device/device.hpp
+++ b/projects/clr/rocclr/device/device.hpp
@@ -1246,7 +1246,7 @@ class MemObjMap : public AllStatic {
   static amd::Memory* FindMemObj(
       const void* k);  //!< find the mem object based on the input pointer
   static void UpdateAccess(amd::Device *peerDev);
-  static void Purge(amd::Device*dev); //!< Purge all the memories on the given device
+  static void Purge(amd::Device* dev); //!< Purge all user allocated memories on the given device
  private:
   static std::map<uintptr_t, amd::Memory*>
       MemObjMap_;                      //!< the mem object<->hostptr information container
diff --git a/projects/clr/rocclr/device/pal/palprogram.cpp b/projects/clr/rocclr/device/pal/palprogram.cpp
index 3cc2134661..f6e319b9b0 100644
--- a/projects/clr/rocclr/device/pal/palprogram.cpp
+++ b/projects/clr/rocclr/device/pal/palprogram.cpp
@@ -508,6 +508,7 @@ bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_p
   }
 
   /* Create a View from the global pal::Memory */
+  flags = ROCCLR_MEM_INTERNAL_MEMORY;
   parent = codeSegGpu_->owner();
   *amd_mem_obj = new (parent->getContext()) amd::Buffer(*parent, flags, offset, *bytes);
 
diff --git a/projects/clr/rocclr/device/rocm/rocprogram.cpp b/projects/clr/rocclr/device/rocm/rocprogram.cpp
index 1959f33a97..32aed3d55d 100644
--- a/projects/clr/rocclr/device/rocm/rocprogram.cpp
+++ b/projects/clr/rocclr/device/rocm/rocprogram.cpp
@@ -190,8 +190,9 @@ bool Program::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_pptr,
     }
 
     roc_device = &(rocDevice());
-    *amd_mem_obj = new(roc_device->context()) amd::Buffer(roc_device->context(), 0, *bytes,
-                                                          *device_pptr);
+    *amd_mem_obj = new(roc_device->context()) amd::Buffer(roc_device->context(),
+                                                          ROCCLR_MEM_INTERNAL_MEMORY,
+                                                          *bytes, *device_pptr);
 
     if (*amd_mem_obj == nullptr) {
       buildLog_ += "[OCL] Failed to create a mem object!";
diff --git a/projects/clr/rocclr/platform/memory.hpp b/projects/clr/rocclr/platform/memory.hpp
index 1b842dee3c..0906921e67 100644
--- a/projects/clr/rocclr/platform/memory.hpp
+++ b/projects/clr/rocclr/platform/memory.hpp
@@ -38,7 +38,8 @@
 #include <memory>
 #include <limits>
 #define CL_MEM_FOLLOW_USER_NUMA_POLICY              (1u << 31)
-#define ROCCLR_MEM_HSA_SIGNAL_MEMORY (1u << 30)
+#define ROCCLR_MEM_HSA_SIGNAL_MEMORY                (1u << 30)
+#define ROCCLR_MEM_INTERNAL_MEMORY                  (1u << 29)
 
 namespace device {
 class Memory;