diff --git a/runtime/hsa-runtime/CMakeLists.txt b/runtime/hsa-runtime/CMakeLists.txt index 20d6d43450..a0f08ae9cd 100644 --- a/runtime/hsa-runtime/CMakeLists.txt +++ b/runtime/hsa-runtime/CMakeLists.txt @@ -87,7 +87,7 @@ if (ROCM_CCACHE_BUILD) endif() # if (ROCM_CCACHE_BUILD) ## Get version strings -get_version ( "1.6.0" ) +get_version ( "1.7.0" ) if ( ${ROCM_PATCH_VERSION} ) set ( VERSION_PATCH ${ROCM_PATCH_VERSION}) endif() diff --git a/runtime/hsa-runtime/core/inc/memory_region.h b/runtime/hsa-runtime/core/inc/memory_region.h index 4f362fad2d..30fbc07ac1 100644 --- a/runtime/hsa-runtime/core/inc/memory_region.h +++ b/runtime/hsa-runtime/core/inc/memory_region.h @@ -89,6 +89,7 @@ class MemoryRegion : public Checked<0x9C961F19EE175BB3> { AllocateDoubleMap = (1 << 2), // Map twice VA allocation to backing store AllocateDirect = (1 << 3), // Bypass fragment cache. AllocateIPC = (1 << 4), // System memory that can be IPC-shared + AllocatePCIeRW = (1 << 5), // Enforce pseudo fine grain/RW memory }; typedef uint32_t AllocateFlags; diff --git a/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp b/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp index 7cbb543957..8327dec6cb 100644 --- a/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp @@ -188,6 +188,9 @@ hsa_status_t MemoryRegion::AllocateImpl(size_t& size, AllocateFlags alloc_flags, if (IsSystem() && (alloc_flags & AllocateIPC)) kmt_alloc_flags.ui32.NonPaged = 1; + // Allocate pseudo fine grain memory + kmt_alloc_flags.ui32.CoarseGrain = (alloc_flags & AllocatePCIeRW ? 0 : kmt_alloc_flags.ui32.CoarseGrain); + // Only allow using the suballocator for ordinary VRAM. if (IsLocalMemory()) { bool subAllocEnabled = !core::Runtime::runtime_singleton_->flag().disable_fragment_alloc(); @@ -451,8 +454,7 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo( // without regard to type of requesting device (CPU / GPU) // Return disallowed by default if framebuffer is fine grained // and requesting device is connected via xGMI link - // Return never allowed if framebuffer is fine grained and - // requesting device is connected via PCIe link + if (IsLocalMemory()) { // Return disallowed by default if memory is coarse @@ -461,12 +463,6 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo( return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT; } - // Determine if pool is pseudo fine-grained due to env flag - // Return disallowed by default - if (core::Runtime::runtime_singleton_->flag().fine_grain_pcie()) { - return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT; - } - // Return disallowed by default if memory is fine // grained and link type is xGMI. if (agent.HiveId() == owner()->HiveId()) { diff --git a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index 8ac975894d..2d39933577 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -686,7 +686,7 @@ hsa_status_t hsa_amd_memory_pool_allocate(hsa_amd_memory_pool_t memory_pool, siz TRY; IS_OPEN(); - if (size == 0 || ptr == NULL || flags != 0) { + if (size == 0 || ptr == NULL || (flags > HSA_AMD_MEMORY_POOL_PCIE_FLAG)) { return HSA_STATUS_ERROR_INVALID_ARGUMENT; } @@ -697,8 +697,11 @@ hsa_status_t hsa_amd_memory_pool_allocate(hsa_amd_memory_pool_t memory_pool, siz return (hsa_status_t)HSA_STATUS_ERROR_INVALID_MEMORY_POOL; } - return core::Runtime::runtime_singleton_->AllocateMemory( - mem_region, size, core::MemoryRegion::AllocateRestrict, ptr); + MemoryRegion::AllocateFlags alloc_flag = core::MemoryRegion::AllocateRestrict; + + if (flags == HSA_AMD_MEMORY_POOL_PCIE_FLAG) alloc_flag |= core::MemoryRegion::AllocatePCIeRW; + + return core::Runtime::runtime_singleton_->AllocateMemory(mem_region, size, alloc_flag, ptr); CATCH; } diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index 290c9ac845..9c9ef3d773 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -1000,6 +1000,24 @@ typedef enum { HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE = 16, } hsa_amd_memory_pool_info_t; +/** + * @brief Memory pool flag used to specify allocation directives + * + */ +typedef enum hsa_amd_memory_pool_flag_s { + /** + * Allocates memory that conforms to standard HSA memory consistency model + */ + HSA_AMD_MEMORY_POOL_STANDARD_FLAG = 0, + /** + * Allocates fine grain memory type where memory ordering is per point to point + * connection. Atomic memory operations on these memory buffers are not + * guaranteed to be visible at system scope. + */ + HSA_AMD_MEMORY_POOL_PCIE_FLAG = 1, + +} hsa_amd_memory_pool_flag_t; + /** * @brief Get the current value of an attribute of a memory pool. * @@ -1068,7 +1086,7 @@ hsa_status_t HSA_API hsa_amd_agent_iterate_memory_pools( * ::HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE in @p memory_pool. * * @param[in] flags A bit-field that is used to specify allocation - * directives. Reserved parameter, must be 0. + * directives. * * @param[out] ptr Pointer to the location where to store the base virtual * address of