SWDEV-292021 - Fix Device Reset

- Device Reset should not purge the allocations that were not by the user
- Addresses QMCPack Test abort due to the removal of all the mem objects during reset

Change-Id: I7b7a123e72bcc985d7e51d17c2382bc618d3e041


[ROCm/clr commit: 924695fb5e]

projects/clr/rocclr/device/device.cpp

@@ -333,9 +333,11 @@ void MemObjMap::Purge(amd::Device* dev) {
   assert(dev != nullptr);
 
   amd::ScopedLock lock(AllocatedLock_);
-  for (auto it = MemObjMap_.cbegin() ; it != MemObjMap_.cend() ;) {
-    const std::vector<Device*>& devices = it->second->getContext().devices();
-    if (devices.size() == 1 && devices[0] == dev) {
+  for (auto it = MemObjMap_.cbegin(); it != MemObjMap_.cend(); ) {
+    amd::Memory* memObj = it->second;
+    unsigned int flags = memObj->getMemFlags();
+    const std::vector<Device*>& devices = memObj->getContext().devices();
+    if (devices.size() == 1 && devices[0] == dev && !(flags & ROCCLR_MEM_INTERNAL_MEMORY)) {
       it = MemObjMap_.erase(it);
     } else {
       ++it;

projects/clr/rocclr/device/device.hpp

@@ -1246,7 +1246,7 @@ class MemObjMap : public AllStatic {
   static amd::Memory* FindMemObj(
       const void* k);  //!< find the mem object based on the input pointer
   static void UpdateAccess(amd::Device *peerDev);
-  static void Purge(amd::Device*dev); //!< Purge all the memories on the given device
+  static void Purge(amd::Device* dev); //!< Purge all user allocated memories on the given device
  private:
   static std::map<uintptr_t, amd::Memory*>
       MemObjMap_;                      //!< the mem object<->hostptr information container

projects/clr/rocclr/device/pal/palprogram.cpp

@@ -508,6 +508,7 @@ bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_p
   }
 
   /* Create a View from the global pal::Memory */
+  flags = ROCCLR_MEM_INTERNAL_MEMORY;
   parent = codeSegGpu_->owner();
   *amd_mem_obj = new (parent->getContext()) amd::Buffer(*parent, flags, offset, *bytes);
 

projects/clr/rocclr/device/rocm/rocprogram.cpp

@@ -190,8 +190,9 @@ bool Program::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_pptr,
     }
 
     roc_device = &(rocDevice());
-    *amd_mem_obj = new(roc_device->context()) amd::Buffer(roc_device->context(), 0, *bytes,
-                                                          *device_pptr);
+    *amd_mem_obj = new(roc_device->context()) amd::Buffer(roc_device->context(),
+                                                          ROCCLR_MEM_INTERNAL_MEMORY,
+                                                          *bytes, *device_pptr);
 
     if (*amd_mem_obj == nullptr) {
       buildLog_ += "[OCL] Failed to create a mem object!";

projects/clr/rocclr/platform/memory.hpp

@@ -38,7 +38,8 @@
 #include <memory>
 #include <limits>
 #define CL_MEM_FOLLOW_USER_NUMA_POLICY              (1u << 31)
-#define ROCCLR_MEM_HSA_SIGNAL_MEMORY (1u << 30)
+#define ROCCLR_MEM_HSA_SIGNAL_MEMORY                (1u << 30)
+#define ROCCLR_MEM_INTERNAL_MEMORY                  (1u << 29)
 
 namespace device {
 class Memory;