From 364715cbc6ff3107a6548cb75697c1b3f0b979f6 Mon Sep 17 00:00:00 2001 From: skhatri Date: Wed, 8 Jun 2022 18:30:24 -0400 Subject: [PATCH] Enabled allocation of pseudo fine grain memory where memory ordering is per point to point connection Atomic memory operations on these memory buffers are not guaranteed to be visible at system scope Change-Id: I4cccde114632071a000384502a83bc191e77e85b --- runtime/hsa-runtime/CMakeLists.txt | 2 +- runtime/hsa-runtime/core/inc/memory_region.h | 1 + .../core/runtime/amd_memory_region.cpp | 12 ++++------- .../hsa-runtime/core/runtime/hsa_ext_amd.cpp | 9 ++++++--- runtime/hsa-runtime/inc/hsa_ext_amd.h | 20 ++++++++++++++++++- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/runtime/hsa-runtime/CMakeLists.txt b/runtime/hsa-runtime/CMakeLists.txt index 20d6d43450..a0f08ae9cd 100644 --- a/runtime/hsa-runtime/CMakeLists.txt +++ b/runtime/hsa-runtime/CMakeLists.txt @@ -87,7 +87,7 @@ if (ROCM_CCACHE_BUILD) endif() # if (ROCM_CCACHE_BUILD) ## Get version strings -get_version ( "1.6.0" ) +get_version ( "1.7.0" ) if ( ${ROCM_PATCH_VERSION} ) set ( VERSION_PATCH ${ROCM_PATCH_VERSION}) endif() diff --git a/runtime/hsa-runtime/core/inc/memory_region.h b/runtime/hsa-runtime/core/inc/memory_region.h index 4f362fad2d..30fbc07ac1 100644 --- a/runtime/hsa-runtime/core/inc/memory_region.h +++ b/runtime/hsa-runtime/core/inc/memory_region.h @@ -89,6 +89,7 @@ class MemoryRegion : public Checked<0x9C961F19EE175BB3> { AllocateDoubleMap = (1 << 2), // Map twice VA allocation to backing store AllocateDirect = (1 << 3), // Bypass fragment cache. AllocateIPC = (1 << 4), // System memory that can be IPC-shared + AllocatePCIeRW = (1 << 5), // Enforce pseudo fine grain/RW memory }; typedef uint32_t AllocateFlags; diff --git a/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp b/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp index 7cbb543957..8327dec6cb 100644 --- a/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp @@ -188,6 +188,9 @@ hsa_status_t MemoryRegion::AllocateImpl(size_t& size, AllocateFlags alloc_flags, if (IsSystem() && (alloc_flags & AllocateIPC)) kmt_alloc_flags.ui32.NonPaged = 1; + // Allocate pseudo fine grain memory + kmt_alloc_flags.ui32.CoarseGrain = (alloc_flags & AllocatePCIeRW ? 0 : kmt_alloc_flags.ui32.CoarseGrain); + // Only allow using the suballocator for ordinary VRAM. if (IsLocalMemory()) { bool subAllocEnabled = !core::Runtime::runtime_singleton_->flag().disable_fragment_alloc(); @@ -451,8 +454,7 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo( // without regard to type of requesting device (CPU / GPU) // Return disallowed by default if framebuffer is fine grained // and requesting device is connected via xGMI link - // Return never allowed if framebuffer is fine grained and - // requesting device is connected via PCIe link + if (IsLocalMemory()) { // Return disallowed by default if memory is coarse @@ -461,12 +463,6 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo( return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT; } - // Determine if pool is pseudo fine-grained due to env flag - // Return disallowed by default - if (core::Runtime::runtime_singleton_->flag().fine_grain_pcie()) { - return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT; - } - // Return disallowed by default if memory is fine // grained and link type is xGMI. if (agent.HiveId() == owner()->HiveId()) { diff --git a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index 8ac975894d..2d39933577 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -686,7 +686,7 @@ hsa_status_t hsa_amd_memory_pool_allocate(hsa_amd_memory_pool_t memory_pool, siz TRY; IS_OPEN(); - if (size == 0 || ptr == NULL || flags != 0) { + if (size == 0 || ptr == NULL || (flags > HSA_AMD_MEMORY_POOL_PCIE_FLAG)) { return HSA_STATUS_ERROR_INVALID_ARGUMENT; } @@ -697,8 +697,11 @@ hsa_status_t hsa_amd_memory_pool_allocate(hsa_amd_memory_pool_t memory_pool, siz return (hsa_status_t)HSA_STATUS_ERROR_INVALID_MEMORY_POOL; } - return core::Runtime::runtime_singleton_->AllocateMemory( - mem_region, size, core::MemoryRegion::AllocateRestrict, ptr); + MemoryRegion::AllocateFlags alloc_flag = core::MemoryRegion::AllocateRestrict; + + if (flags == HSA_AMD_MEMORY_POOL_PCIE_FLAG) alloc_flag |= core::MemoryRegion::AllocatePCIeRW; + + return core::Runtime::runtime_singleton_->AllocateMemory(mem_region, size, alloc_flag, ptr); CATCH; } diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index 290c9ac845..9c9ef3d773 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -1000,6 +1000,24 @@ typedef enum { HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE = 16, } hsa_amd_memory_pool_info_t; +/** + * @brief Memory pool flag used to specify allocation directives + * + */ +typedef enum hsa_amd_memory_pool_flag_s { + /** + * Allocates memory that conforms to standard HSA memory consistency model + */ + HSA_AMD_MEMORY_POOL_STANDARD_FLAG = 0, + /** + * Allocates fine grain memory type where memory ordering is per point to point + * connection. Atomic memory operations on these memory buffers are not + * guaranteed to be visible at system scope. + */ + HSA_AMD_MEMORY_POOL_PCIE_FLAG = 1, + +} hsa_amd_memory_pool_flag_t; + /** * @brief Get the current value of an attribute of a memory pool. * @@ -1068,7 +1086,7 @@ hsa_status_t HSA_API hsa_amd_agent_iterate_memory_pools( * ::HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE in @p memory_pool. * * @param[in] flags A bit-field that is used to specify allocation - * directives. Reserved parameter, must be 0. + * directives. * * @param[out] ptr Pointer to the location where to store the base virtual * address of