From 7d3aaa7a3922474ba86c67dbb02732d1348fa46c Mon Sep 17 00:00:00 2001
From: German Andryeyev <German.Andryeyev@amd.com>
Date: Tue, 12 Jan 2021 00:20:20 -0500
Subject: [PATCH] Explicit page table update on all devices

HMM with xnack enabled should automatically update page tables,
but currently it doesn't perform that. For now, runtime will
force page table update on all devices unconditionally.

Change-Id: Idfa6e1c145e6c114856214dce042b8a8349e5c58
---
 rocclr/device/rocm/rocdevice.cpp | 15 +++++++++++----
 rocclr/device/rocm/rocmemory.cpp |  8 --------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/rocclr/device/rocm/rocdevice.cpp b/rocclr/device/rocm/rocdevice.cpp
index a19987e3c2..c06b7d879e 100644
--- a/rocclr/device/rocm/rocdevice.cpp
+++ b/rocclr/device/rocm/rocdevice.cpp
@@ -2130,7 +2130,17 @@ bool Device::SetSvmAttributesInt(const void* dev_ptr, size_t count,
       if (use_cpu) {
         attr.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE, getCpuAgent().handle});
       } else {
-        attr.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE, getBackendDevice().handle});
+        if (first_alloc) {
+          // Provide access to all possible devices.
+          //! @note: HMM should support automatic page table update with xnack enabled,
+          //! but currently it doesn't and runtime explicitly enables access from all devices
+          for (const auto dev : devices()) {
+            attr.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE,
+                static_cast<Device*>(dev)->getBackendDevice().handle});
+          }
+        } else {
+          attr.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE, getBackendDevice().handle});
+        }
       }
       break;
     case amd::MemoryAdvice::UnsetAccessedBy:
@@ -2257,9 +2267,6 @@ bool Device::SvmAllocInit(void* memory, size_t size) const {
   if (settings().hmmFlags_ & Settings::Hmm::EnableSystemMemory) {
     advice = amd::MemoryAdvice::UnsetPreferredLocation;
     SetSvmAttributesInt(memory, size, advice);
-  } else {
-    advice = amd::MemoryAdvice::SetPreferredLocation;
-    SetSvmAttributesInt(memory, size, advice);
   }
 
   if ((settings().hmmFlags_ & Settings::Hmm::EnableMallocPrefetch) == 0) {
diff --git a/rocclr/device/rocm/rocmemory.cpp b/rocclr/device/rocm/rocmemory.cpp
index 9c1055d140..d8f4a0a6ef 100644
--- a/rocclr/device/rocm/rocmemory.cpp
+++ b/rocclr/device/rocm/rocmemory.cpp
@@ -745,14 +745,6 @@ bool Buffer::create() {
     } else {
       deviceMemory_ = owner()->getSvmPtr();
       kind_ = MEMORY_KIND_PTRGIVEN;
-#if AMD_HMM_SUPPORT
-      if (memFlags & CL_MEM_ALLOC_HOST_PTR) {
-        // Currently HMM requires cirtain initial calls to mark sysmem allocation as
-        // GPU accessible or prefetch memory into the current device
-        // @note: Skip any allocaiton here, since sysmem was allocated on another device.
-        dev().SvmAllocInit(deviceMemory_, size());
-      }
-#endif // AMD_HMM_SUPPORT
     }
 
     if ((deviceMemory_ != nullptr) && (dev().settings().apuSystem_ || !isFineGrain)) {