Defined a new extended scope memory region

Added HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_EXT_SCOPE_FINE_GRAINED flag to enable extended scope memory region where the device-scope atomics act as system-scope atomics Change-Id: I79fc3207cb630dfc68bed2f8aabd75f35fe80b12 [ROCm/ROCR-Runtime commit: 77bf357647]
2023-06-09 12:59:08 -04:00
@@ -85,7 +85,7 @@ if (ROCM_CCACHE_BUILD)
 endif() # if (ROCM_CCACHE_BUILD)

 ## Get version strings
-get_version ( "1.10.0" )
+get_version ( "1.11.0" )
 if ( ${ROCM_PATCH_VERSION} )
  set ( VERSION_PATCH ${ROCM_PATCH_VERSION})
 endif()
@@ -95,8 +95,8 @@ class MemoryRegion : public core::MemoryRegion {
  /// @brief Unpin memory.
  static void MakeKfdMemoryUnresident(const void* ptr);

-  MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, core::Agent* owner,
-               const HsaMemoryProperties& mem_props);
+  MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, bool extended_scope_fine_grain,
+               core::Agent* owner, const HsaMemoryProperties& mem_props);

  ~MemoryRegion();

@@ -173,6 +173,8 @@ class MemoryRegion : public core::MemoryRegion {
    return static_cast<uint32_t>(mem_props_.MemoryClockMax);
  }

+  __forceinline bool extended_scope_fine_grain() const { return extended_scope_fine_grain_; }
+
 private:
  const HsaMemoryProperties mem_props_;

@@ -182,6 +184,9 @@ class MemoryRegion : public core::MemoryRegion {

  size_t max_single_alloc_size_;

+  // Enables creating an extended scope fine grained memory pool region
+  const bool extended_scope_fine_grain_;
+
  // Used to collect total system memory
  static size_t max_sysmem_alloc_size_;

@@ -85,15 +85,15 @@ void CpuAgent::InitRegionList() {
    if (system_prop != mem_props.end()) system_props = *system_prop;

    MemoryRegion* system_region_fine =
-        new MemoryRegion(true, false, is_apu_node, this, system_props);
+        new MemoryRegion(true, false, is_apu_node, false, this, system_props);
    regions_.push_back(system_region_fine);
    MemoryRegion* system_region_kernarg =
-        new MemoryRegion(true, true, is_apu_node, this, system_props);
+        new MemoryRegion(true, true, is_apu_node, false, this, system_props);
    regions_.push_back(system_region_kernarg);

    if (!is_apu_node) {
      MemoryRegion* system_region_coarse =
-          new MemoryRegion(false, false, is_apu_node, this, system_props);
+          new MemoryRegion(false, false, is_apu_node, false, this, system_props);
      regions_.push_back(system_region_coarse);
    }
  }
@@ -447,15 +447,19 @@ void GpuAgent::InitRegionList() {
          memory_max_frequency_ = mem_props[mem_idx].MemoryClockMax;
        case HSA_HEAPTYPE_GPU_LDS:
        case HSA_HEAPTYPE_GPU_SCRATCH: {
-          MemoryRegion* region = new MemoryRegion(false, false, false, this, mem_props[mem_idx]);
+          MemoryRegion* region =
+              new MemoryRegion(false, false, false, false, this, mem_props[mem_idx]);

          regions_.push_back(region);

          if (region->IsLocalMemory()) {
+            regions_.push_back(
+                new MemoryRegion(false, false, false, true, this, mem_props[mem_idx]));
            // Expose VRAM as uncached/fine grain over PCIe (if enabled) or XGMI.
            if ((properties_.HiveID != 0) ||
                (core::Runtime::runtime_singleton_->flag().fine_grain_pcie())) {
-              regions_.push_back(new MemoryRegion(true, false, false, this, mem_props[mem_idx]));
+              regions_.push_back(
+                  new MemoryRegion(true, false, false, false, this, mem_props[mem_idx]));
            }
          }
          break;
@@ -100,20 +100,28 @@ void MemoryRegion::MakeKfdMemoryUnresident(const void* ptr) {
  hsaKmtUnmapMemoryToGPU(const_cast<void*>(ptr));
 }

-MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, core::Agent* owner,
+MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile,
+                           bool extended_scope_fine_grain, core::Agent* owner,
                           const HsaMemoryProperties& mem_props)
    : core::MemoryRegion(fine_grain, kernarg, full_profile, owner),
      mem_props_(mem_props),
+      extended_scope_fine_grain_(extended_scope_fine_grain),
      max_single_alloc_size_(0),
      virtual_size_(0),
      fragment_allocator_(BlockAllocator(*this)) {
  virtual_size_ = GetPhysicalSize();

+  // extended_scope_fine_grain and fine_grain memory regions are mutually exclusive
+  assert(!(fine_grain && extended_scope_fine_grain));
+
  mem_flag_.Value = 0;
  map_flag_.Value = 0;
-
  static const HSAuint64 kGpuVmSize = (1ULL << 40);

+  // Bind the memory region based on whether it is
+  // coarse or fine grain or extended scope fine grain.
+  mem_flag_.ui32.CoarseGrain = (fine_grain || extended_scope_fine_grain) ? 0 : 1;
+
  if (IsLocalMemory()) {
    mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB;
    mem_flag_.ui32.NoSubstitute = 1;
@@ -122,6 +130,20 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, cor
    mem_flag_.ui32.NonPaged = 1;

    virtual_size_ = kGpuVmSize;
+
+    // If memory region is extended scope fine grained
+    // mark the page table entries for this memory region
+    // as MTYPE_UC. Full read and write ordering are guaranteed
+    // to this address.
+    if (extended_scope_fine_grain) {
+      AMD::GpuAgent* agent_ =
+          const_cast<AMD::GpuAgent*>(reinterpret_cast<const AMD::GpuAgent*>(owner));
+      if (agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 0) ||
+          agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 1) ||
+          agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 2))
+        mem_flag_.ui32.Uncached = 1;
+    }
+
  } else if (IsSystem()) {
    mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB;
    mem_flag_.ui32.NoSubstitute = 0;
@@ -134,8 +156,6 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, cor
        (full_profile) ? os::GetUserModeVirtualMemorySize() : kGpuVmSize;
  }

-  // Bind if memory region is coarse or fine grain
-  mem_flag_.ui32.CoarseGrain = (fine_grain) ? 0 : 1;

  // Adjust allocatable size per page align
  max_single_alloc_size_ = AlignDown(static_cast<size_t>(GetPhysicalSize()), kPageSize_);
@@ -317,8 +337,12 @@ hsa_status_t MemoryRegion::GetInfo(hsa_region_info_t attribute,
        case HSA_HEAPTYPE_SYSTEM:
        case HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC:
        case HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE: {
-          uint32_t ret = fine_grain() ? HSA_REGION_GLOBAL_FLAG_FINE_GRAINED
-                                      : HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED;
+          uint32_t ret = 0;
+
+          ret = fine_grain()                ? HSA_REGION_GLOBAL_FLAG_FINE_GRAINED
+              : extended_scope_fine_grain() ? HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED
+                                            : HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED;
+
          if (kernarg()) ret |= HSA_REGION_GLOBAL_FLAG_KERNARG;
          *((uint32_t*)value) = ret;
          break;
@@ -480,7 +504,7 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo(

    // Return disallowed by default if memory is coarse
    // grained without regard to link type
-    if  (fine_grain() == false) {
+    if (extended_scope_fine_grain() == false && fine_grain() == false) {
      return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT;
    }

@@ -3218,7 +3218,16 @@ typedef enum {
   * region, the application must explicitely invoke ::hsa_memory_assign_agent
   * in order to transfer ownership to that agent for a particular buffer.
   */
-  HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED = 4
+  HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED = 4,
+
+  /**
+   * Updates to memory in this region have extended scope, where the device-scope atomics
+   * to this memory type act as system-scope with respect to all variables located in
+   * memory regions of this type.
+   * Note: On non-compliant systems, the application may still be responsible for performing
+   * device-specific actions necessary to achieve system-scope coherence.
+   */
+  HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED = 8
 } hsa_region_global_flag_t;

 /**
@@ -52,9 +52,10 @@
 * - 1.0 - initial version
 * - 1.1 - dmabuf export
 * - 1.2 - hsa_amd_memory_async_copy_on_engine
+ * - 1.3 - HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED pool
 */
 #define HSA_AMD_INTERFACE_VERSION_MAJOR 1
-#define HSA_AMD_INTERFACE_VERSION_MINOR 2
+#define HSA_AMD_INTERFACE_VERSION_MINOR 3

 #ifdef __cplusplus
 extern "C" {
@@ -1014,7 +1015,14 @@ typedef enum hsa_amd_memory_pool_global_flag_s {
  /**
   * Writes to memory in this pool can be performed by a single agent at a time.
   */
-  HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED = 4
+  HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED = 4,
+
+  /** Updates to memory in this memory pool have extended scope, acting as
+   * system-scope atomics for variables in memory regions of this type.
+   * Note: On non-compliant systems, device-specific actions may be required
+   * for system-scope coherence. */
+  HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED = 8,
+
 } hsa_amd_memory_pool_global_flag_t;

 typedef enum hsa_amd_memory_pool_location_s {