Defined a new extended scope memory region

Added HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_EXT_SCOPE_FINE_GRAINED flag to enable extended scope memory region
where the device-scope atomics act as system-scope atomics

Change-Id: I79fc3207cb630dfc68bed2f8aabd75f35fe80b12


[ROCm/ROCR-Runtime commit: 77bf357647]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_memory_region.h

@@ -95,8 +95,8 @@ class MemoryRegion : public core::MemoryRegion {
   /// @brief Unpin memory.
   static void MakeKfdMemoryUnresident(const void* ptr);
 
-  MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, core::Agent* owner,
-               const HsaMemoryProperties& mem_props);
+  MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, bool extended_scope_fine_grain,
+               core::Agent* owner, const HsaMemoryProperties& mem_props);
 
   ~MemoryRegion();
 
@@ -173,6 +173,8 @@ class MemoryRegion : public core::MemoryRegion {
     return static_cast<uint32_t>(mem_props_.MemoryClockMax);
   }
 
+  __forceinline bool extended_scope_fine_grain() const { return extended_scope_fine_grain_; }
+
  private:
   const HsaMemoryProperties mem_props_;
 
@@ -182,6 +184,9 @@ class MemoryRegion : public core::MemoryRegion {
 
   size_t max_single_alloc_size_;
 
+  // Enables creating an extended scope fine grained memory pool region
+  const bool extended_scope_fine_grain_;
+
   // Used to collect total system memory
   static size_t max_sysmem_alloc_size_;
 

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp

@@ -85,15 +85,15 @@ void CpuAgent::InitRegionList() {
     if (system_prop != mem_props.end()) system_props = *system_prop;
 
     MemoryRegion* system_region_fine =
-        new MemoryRegion(true, false, is_apu_node, this, system_props);
+        new MemoryRegion(true, false, is_apu_node, false, this, system_props);
     regions_.push_back(system_region_fine);
     MemoryRegion* system_region_kernarg =
-        new MemoryRegion(true, true, is_apu_node, this, system_props);
+        new MemoryRegion(true, true, is_apu_node, false, this, system_props);
     regions_.push_back(system_region_kernarg);
 
     if (!is_apu_node) {
       MemoryRegion* system_region_coarse =
-          new MemoryRegion(false, false, is_apu_node, this, system_props);
+          new MemoryRegion(false, false, is_apu_node, false, this, system_props);
       regions_.push_back(system_region_coarse);
     }
   }

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -447,15 +447,19 @@ void GpuAgent::InitRegionList() {
           memory_max_frequency_ = mem_props[mem_idx].MemoryClockMax;
         case HSA_HEAPTYPE_GPU_LDS:
         case HSA_HEAPTYPE_GPU_SCRATCH: {
-          MemoryRegion* region = new MemoryRegion(false, false, false, this, mem_props[mem_idx]);
+          MemoryRegion* region =
+              new MemoryRegion(false, false, false, false, this, mem_props[mem_idx]);
 
           regions_.push_back(region);
 
           if (region->IsLocalMemory()) {
+            regions_.push_back(
+                new MemoryRegion(false, false, false, true, this, mem_props[mem_idx]));
             // Expose VRAM as uncached/fine grain over PCIe (if enabled) or XGMI.
             if ((properties_.HiveID != 0) ||
                 (core::Runtime::runtime_singleton_->flag().fine_grain_pcie())) {
-              regions_.push_back(new MemoryRegion(true, false, false, this, mem_props[mem_idx]));
+              regions_.push_back(
+                  new MemoryRegion(true, false, false, false, this, mem_props[mem_idx]));
             }
           }
           break;

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp

@@ -100,20 +100,28 @@ void MemoryRegion::MakeKfdMemoryUnresident(const void* ptr) {
   hsaKmtUnmapMemoryToGPU(const_cast<void*>(ptr));
 }
 
-MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, core::Agent* owner,
+MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile,
+                           bool extended_scope_fine_grain, core::Agent* owner,
                            const HsaMemoryProperties& mem_props)
     : core::MemoryRegion(fine_grain, kernarg, full_profile, owner),
       mem_props_(mem_props),
+      extended_scope_fine_grain_(extended_scope_fine_grain),
       max_single_alloc_size_(0),
       virtual_size_(0),
       fragment_allocator_(BlockAllocator(*this)) {
   virtual_size_ = GetPhysicalSize();
 
+  // extended_scope_fine_grain and fine_grain memory regions are mutually exclusive
+  assert(!(fine_grain && extended_scope_fine_grain));
+
   mem_flag_.Value = 0;
   map_flag_.Value = 0;
-
   static const HSAuint64 kGpuVmSize = (1ULL << 40);
 
+  // Bind the memory region based on whether it is
+  // coarse or fine grain or extended scope fine grain.
+  mem_flag_.ui32.CoarseGrain = (fine_grain || extended_scope_fine_grain) ? 0 : 1;
+
   if (IsLocalMemory()) {
     mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB;
     mem_flag_.ui32.NoSubstitute = 1;
@@ -122,6 +130,20 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, cor
     mem_flag_.ui32.NonPaged = 1;
 
     virtual_size_ = kGpuVmSize;
+
+    // If memory region is extended scope fine grained
+    // mark the page table entries for this memory region
+    // as MTYPE_UC. Full read and write ordering are guaranteed
+    // to this address.
+    if (extended_scope_fine_grain) {
+      AMD::GpuAgent* agent_ =
+          const_cast<AMD::GpuAgent*>(reinterpret_cast<const AMD::GpuAgent*>(owner));
+      if (agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 0) ||
+          agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 1) ||
+          agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 2))
+        mem_flag_.ui32.Uncached = 1;
+    }
+
   } else if (IsSystem()) {
     mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB;
     mem_flag_.ui32.NoSubstitute = 0;
@@ -134,8 +156,6 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, cor
         (full_profile) ? os::GetUserModeVirtualMemorySize() : kGpuVmSize;
   }
 
-  // Bind if memory region is coarse or fine grain
-  mem_flag_.ui32.CoarseGrain = (fine_grain) ? 0 : 1;
 
   // Adjust allocatable size per page align
   max_single_alloc_size_ = AlignDown(static_cast<size_t>(GetPhysicalSize()), kPageSize_);
@@ -317,8 +337,12 @@ hsa_status_t MemoryRegion::GetInfo(hsa_region_info_t attribute,
         case HSA_HEAPTYPE_SYSTEM:
         case HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC:
         case HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE: {
-          uint32_t ret = fine_grain() ? HSA_REGION_GLOBAL_FLAG_FINE_GRAINED
-                                      : HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED;
+          uint32_t ret = 0;
+
+          ret = fine_grain()                ? HSA_REGION_GLOBAL_FLAG_FINE_GRAINED
+              : extended_scope_fine_grain() ? HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED
+                                            : HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED;
+
           if (kernarg()) ret |= HSA_REGION_GLOBAL_FLAG_KERNARG;
           *((uint32_t*)value) = ret;
           break;
@@ -480,7 +504,7 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo(
 
     // Return disallowed by default if memory is coarse
     // grained without regard to link type
-    if  (fine_grain() == false) {
+    if (extended_scope_fine_grain() == false && fine_grain() == false) {
       return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT;
     }