From 76cc9034ff4befc41e8de31c77ee0ed3536727b7 Mon Sep 17 00:00:00 2001 From: Shweta Khatri Date: Fri, 9 Jun 2023 12:59:08 -0400 Subject: [PATCH] Defined a new extended scope memory region Added HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_EXT_SCOPE_FINE_GRAINED flag to enable extended scope memory region where the device-scope atomics act as system-scope atomics Change-Id: I79fc3207cb630dfc68bed2f8aabd75f35fe80b12 [ROCm/ROCR-Runtime commit: 77bf357647e8e72d2a803e690d364477515eadd4] --- .../runtime/hsa-runtime/CMakeLists.txt | 2 +- .../hsa-runtime/core/inc/amd_memory_region.h | 9 ++++- .../core/runtime/amd_cpu_agent.cpp | 6 +-- .../core/runtime/amd_gpu_agent.cpp | 8 +++- .../core/runtime/amd_memory_region.cpp | 38 +++++++++++++++---- .../runtime/hsa-runtime/inc/hsa.h | 11 +++++- .../runtime/hsa-runtime/inc/hsa_ext_amd.h | 12 +++++- 7 files changed, 68 insertions(+), 18 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt b/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt index 3a826da3cd..e65f7abdc5 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt +++ b/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt @@ -85,7 +85,7 @@ if (ROCM_CCACHE_BUILD) endif() # if (ROCM_CCACHE_BUILD) ## Get version strings -get_version ( "1.10.0" ) +get_version ( "1.11.0" ) if ( ${ROCM_PATCH_VERSION} ) set ( VERSION_PATCH ${ROCM_PATCH_VERSION}) endif() diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_memory_region.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_memory_region.h index e12f0d760d..cb5d17e2b7 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_memory_region.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_memory_region.h @@ -95,8 +95,8 @@ class MemoryRegion : public core::MemoryRegion { /// @brief Unpin memory. static void MakeKfdMemoryUnresident(const void* ptr); - MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, core::Agent* owner, - const HsaMemoryProperties& mem_props); + MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, bool extended_scope_fine_grain, + core::Agent* owner, const HsaMemoryProperties& mem_props); ~MemoryRegion(); @@ -173,6 +173,8 @@ class MemoryRegion : public core::MemoryRegion { return static_cast(mem_props_.MemoryClockMax); } + __forceinline bool extended_scope_fine_grain() const { return extended_scope_fine_grain_; } + private: const HsaMemoryProperties mem_props_; @@ -182,6 +184,9 @@ class MemoryRegion : public core::MemoryRegion { size_t max_single_alloc_size_; + // Enables creating an extended scope fine grained memory pool region + const bool extended_scope_fine_grain_; + // Used to collect total system memory static size_t max_sysmem_alloc_size_; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp index c890098ca4..0b40031cc8 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp @@ -85,15 +85,15 @@ void CpuAgent::InitRegionList() { if (system_prop != mem_props.end()) system_props = *system_prop; MemoryRegion* system_region_fine = - new MemoryRegion(true, false, is_apu_node, this, system_props); + new MemoryRegion(true, false, is_apu_node, false, this, system_props); regions_.push_back(system_region_fine); MemoryRegion* system_region_kernarg = - new MemoryRegion(true, true, is_apu_node, this, system_props); + new MemoryRegion(true, true, is_apu_node, false, this, system_props); regions_.push_back(system_region_kernarg); if (!is_apu_node) { MemoryRegion* system_region_coarse = - new MemoryRegion(false, false, is_apu_node, this, system_props); + new MemoryRegion(false, false, is_apu_node, false, this, system_props); regions_.push_back(system_region_coarse); } } diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 9c64863145..589be527a5 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -447,15 +447,19 @@ void GpuAgent::InitRegionList() { memory_max_frequency_ = mem_props[mem_idx].MemoryClockMax; case HSA_HEAPTYPE_GPU_LDS: case HSA_HEAPTYPE_GPU_SCRATCH: { - MemoryRegion* region = new MemoryRegion(false, false, false, this, mem_props[mem_idx]); + MemoryRegion* region = + new MemoryRegion(false, false, false, false, this, mem_props[mem_idx]); regions_.push_back(region); if (region->IsLocalMemory()) { + regions_.push_back( + new MemoryRegion(false, false, false, true, this, mem_props[mem_idx])); // Expose VRAM as uncached/fine grain over PCIe (if enabled) or XGMI. if ((properties_.HiveID != 0) || (core::Runtime::runtime_singleton_->flag().fine_grain_pcie())) { - regions_.push_back(new MemoryRegion(true, false, false, this, mem_props[mem_idx])); + regions_.push_back( + new MemoryRegion(true, false, false, false, this, mem_props[mem_idx])); } } break; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp index a8e4c75c4e..bd87b9c5bd 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp @@ -100,20 +100,28 @@ void MemoryRegion::MakeKfdMemoryUnresident(const void* ptr) { hsaKmtUnmapMemoryToGPU(const_cast(ptr)); } -MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, core::Agent* owner, +MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, + bool extended_scope_fine_grain, core::Agent* owner, const HsaMemoryProperties& mem_props) : core::MemoryRegion(fine_grain, kernarg, full_profile, owner), mem_props_(mem_props), + extended_scope_fine_grain_(extended_scope_fine_grain), max_single_alloc_size_(0), virtual_size_(0), fragment_allocator_(BlockAllocator(*this)) { virtual_size_ = GetPhysicalSize(); + // extended_scope_fine_grain and fine_grain memory regions are mutually exclusive + assert(!(fine_grain && extended_scope_fine_grain)); + mem_flag_.Value = 0; map_flag_.Value = 0; - static const HSAuint64 kGpuVmSize = (1ULL << 40); + // Bind the memory region based on whether it is + // coarse or fine grain or extended scope fine grain. + mem_flag_.ui32.CoarseGrain = (fine_grain || extended_scope_fine_grain) ? 0 : 1; + if (IsLocalMemory()) { mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB; mem_flag_.ui32.NoSubstitute = 1; @@ -122,6 +130,20 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, cor mem_flag_.ui32.NonPaged = 1; virtual_size_ = kGpuVmSize; + + // If memory region is extended scope fine grained + // mark the page table entries for this memory region + // as MTYPE_UC. Full read and write ordering are guaranteed + // to this address. + if (extended_scope_fine_grain) { + AMD::GpuAgent* agent_ = + const_cast(reinterpret_cast(owner)); + if (agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 0) || + agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 1) || + agent_->isa()->GetVersion() == core::Isa::Version(9, 4, 2)) + mem_flag_.ui32.Uncached = 1; + } + } else if (IsSystem()) { mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB; mem_flag_.ui32.NoSubstitute = 0; @@ -134,8 +156,6 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, cor (full_profile) ? os::GetUserModeVirtualMemorySize() : kGpuVmSize; } - // Bind if memory region is coarse or fine grain - mem_flag_.ui32.CoarseGrain = (fine_grain) ? 0 : 1; // Adjust allocatable size per page align max_single_alloc_size_ = AlignDown(static_cast(GetPhysicalSize()), kPageSize_); @@ -317,8 +337,12 @@ hsa_status_t MemoryRegion::GetInfo(hsa_region_info_t attribute, case HSA_HEAPTYPE_SYSTEM: case HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC: case HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE: { - uint32_t ret = fine_grain() ? HSA_REGION_GLOBAL_FLAG_FINE_GRAINED - : HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED; + uint32_t ret = 0; + + ret = fine_grain() ? HSA_REGION_GLOBAL_FLAG_FINE_GRAINED + : extended_scope_fine_grain() ? HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED + : HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED; + if (kernarg()) ret |= HSA_REGION_GLOBAL_FLAG_KERNARG; *((uint32_t*)value) = ret; break; @@ -480,7 +504,7 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo( // Return disallowed by default if memory is coarse // grained without regard to link type - if (fine_grain() == false) { + if (extended_scope_fine_grain() == false && fine_grain() == false) { return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT; } diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h index 3c0db5d52b..a70fd0f061 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h @@ -3218,7 +3218,16 @@ typedef enum { * region, the application must explicitely invoke ::hsa_memory_assign_agent * in order to transfer ownership to that agent for a particular buffer. */ - HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED = 4 + HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED = 4, + + /** + * Updates to memory in this region have extended scope, where the device-scope atomics + * to this memory type act as system-scope with respect to all variables located in + * memory regions of this type. + * Note: On non-compliant systems, the application may still be responsible for performing + * device-specific actions necessary to achieve system-scope coherence. + */ + HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED = 8 } hsa_region_global_flag_t; /** diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h index ead1abd634..1e810f411a 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -52,9 +52,10 @@ * - 1.0 - initial version * - 1.1 - dmabuf export * - 1.2 - hsa_amd_memory_async_copy_on_engine + * - 1.3 - HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED pool */ #define HSA_AMD_INTERFACE_VERSION_MAJOR 1 -#define HSA_AMD_INTERFACE_VERSION_MINOR 2 +#define HSA_AMD_INTERFACE_VERSION_MINOR 3 #ifdef __cplusplus extern "C" { @@ -1014,7 +1015,14 @@ typedef enum hsa_amd_memory_pool_global_flag_s { /** * Writes to memory in this pool can be performed by a single agent at a time. */ - HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED = 4 + HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED = 4, + + /** Updates to memory in this memory pool have extended scope, acting as + * system-scope atomics for variables in memory regions of this type. + * Note: On non-compliant systems, device-specific actions may be required + * for system-scope coherence. */ + HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED = 8, + } hsa_amd_memory_pool_global_flag_t; typedef enum hsa_amd_memory_pool_location_s {