Using new KFD HSA extended coherent memory flag

Using new ExtendedCoherent KFD HSA memory flag to achieve system
scope coherence on atomic instructions. Non-compliant systems may
have the need to perform explicit HDP flushes to achieve system
scope coherence using this flag.

Change-Id: Ic6b47c0e97285086fa1f52bbfa4597b81cadafeb


[ROCm/ROCR-Runtime commit: 4eb6ed7799]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_memory_region.h

@@ -173,8 +173,6 @@ class MemoryRegion : public core::MemoryRegion {
     return static_cast<uint32_t>(mem_props_.MemoryClockMax);
   }
 
-  __forceinline bool extended_scope_fine_grain() const { return extended_scope_fine_grain_; }
-
   __forceinline size_t GetPageSize() const { return kPageSize_; }
 
  private:
@@ -184,9 +182,6 @@ class MemoryRegion : public core::MemoryRegion {
 
   HsaMemMapFlags map_flag_;
 
-  // Enables creating an extended scope fine grained memory pool region
-  const bool extended_scope_fine_grain_;
-
   size_t max_single_alloc_size_;
 
   // Used to collect total system memory

projects/rocr-runtime/runtime/hsa-runtime/core/inc/memory_region.h

@@ -57,8 +57,13 @@ class Agent;
 
 class MemoryRegion : public Checked<0x9C961F19EE175BB3> {
  public:
-  MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, core::Agent* owner)
-      : fine_grain_(fine_grain), kernarg_(kernarg), full_profile_(full_profile), owner_(owner) {
+  MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, bool extended_scope_fine_grain,
+               core::Agent* owner)
+      : fine_grain_(fine_grain),
+        kernarg_(kernarg),
+        full_profile_(full_profile),
+        extended_scope_fine_grain_(extended_scope_fine_grain),
+        owner_(owner) {
     assert(owner_ != NULL);
   }
 
@@ -117,6 +122,8 @@ class MemoryRegion : public Checked<0x9C961F19EE175BB3> {
 
   __forceinline bool fine_grain() const { return fine_grain_; }
 
+  __forceinline bool extended_scope_fine_grain() const { return extended_scope_fine_grain_; }
+
   __forceinline bool kernarg() const { return kernarg_; }
 
   __forceinline bool full_profile() const { return full_profile_; }
@@ -127,7 +134,7 @@ class MemoryRegion : public Checked<0x9C961F19EE175BB3> {
   const bool fine_grain_;
   const bool kernarg_;
   const bool full_profile_;
-
+  const bool extended_scope_fine_grain_;
   core::Agent* owner_;
 };
 }  // namespace core

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp

@@ -103,9 +103,8 @@ void MemoryRegion::MakeKfdMemoryUnresident(const void* ptr) {
 MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile,
                            bool extended_scope_fine_grain, core::Agent* owner,
                            const HsaMemoryProperties& mem_props)
-    : core::MemoryRegion(fine_grain, kernarg, full_profile, owner),
+    : core::MemoryRegion(fine_grain, kernarg, full_profile, extended_scope_fine_grain, owner),
       mem_props_(mem_props),
-      extended_scope_fine_grain_(extended_scope_fine_grain),
       max_single_alloc_size_(0),
       virtual_size_(0),
       fragment_allocator_(BlockAllocator(*this)) {
@@ -122,6 +121,12 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile,
   // coarse or fine grain or extended scope fine grain.
   mem_flag_.ui32.CoarseGrain = (fine_grain || extended_scope_fine_grain) ? 0 : 1;
 
+  // Extended scope fine-grained memory: Device scope atomics are promoted
+  // to system scope atomics. Non-compliant systems may require the
+  // application to perform device-specific actions, like HDP flushes,
+  // to achieve system-scope coherence
+  mem_flag_.ui32.ExtendedCoherent = (extended_scope_fine_grain) ? 1 : 0;
+
   if (IsLocalMemory()) {
     mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB;
     mem_flag_.ui32.NoSubstitute = 1;
@@ -131,19 +136,6 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile,
 
     virtual_size_ = kGpuVmSize;
 
-    // If memory region is extended scope fine grained
-    // mark the page table entries for this memory region
-    // as MTYPE_UC. Full read and write ordering are guaranteed
-    // to this address.
-    if (extended_scope_fine_grain) {
-      AMD::GpuAgent* agent_ =
-          const_cast<AMD::GpuAgent*>(reinterpret_cast<const AMD::GpuAgent*>(owner));
-      if (agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 0) ||
-          agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 1) ||
-          agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 2))
-        mem_flag_.ui32.Uncached = 1;
-    }
-
   } else if (IsSystem()) {
     mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB;
     mem_flag_.ui32.NoSubstitute = 0;