AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update memory allocation guide in using pool apis

Browse Source 
This is to allow allocations in system memory that exceed sizes
reported by a CPU device

Change-Id: I3d10d192aafcefbe4107f69b7c5e30bf7f836619


[ROCm/ROCR-Runtime commit: 3201f68f72]
This commit is contained in:
Ramesh Errabolu
2019-06-05 11:49:44 -05:00
parent 2b9e13a56c
commit 61b9d4e8b2
7 changed files with 98 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_memory_region.h
+3
View File
@@ -175,6 +175,9 @@ class MemoryRegion : public core::MemoryRegion {
size_t max_single_alloc_size_;
// Used to collect total system memory
static size_t max_sysmem_alloc_size_;
HSAuint64 virtual_size_;
mutable KernelMutex access_lock_;
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp
+38 -5
View File
@@ -52,6 +52,10 @@
#include "core/inc/exceptions.h"
namespace amd {
// Tracks aggregate size of system memory available on platform
size_t MemoryRegion::max_sysmem_alloc_size_ = 0;
void* MemoryRegion::AllocateKfdMemory(const HsaMemFlags& flag,
HSAuint32 node_id, size_t size) {
void* ret = NULL;
@@ -119,7 +123,7 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool full_profile, core::Agent* owne
virtual_size_ = kGpuVmSize;
} else if (IsSystem()) {
mem_flag_.ui32.PageSize = HSA_PAGE_SIZE_4KB;
mem_flag_.ui32.NoSubstitute = 1;
mem_flag_.ui32.NoSubstitute = 0;
mem_flag_.ui32.HostAccess = 1;
mem_flag_.ui32.CachePolicy = HSA_CACHING_CACHED;
@@ -127,9 +131,20 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool full_profile, core::Agent* owne
(full_profile) ? os::GetUserModeVirtualMemorySize() : kGpuVmSize;
}
// Bind if memory region is coarse or fine grain
mem_flag_.ui32.CoarseGrain = (fine_grain) ? 0 : 1;
// Adjust allocatable size per page align
max_single_alloc_size_ = AlignDown(static_cast<size_t>(GetPhysicalSize()), kPageSize_);
mem_flag_.ui32.CoarseGrain = (fine_grain) ? 0 : 1;
// Keep track of total system memory available
// @note: System memory is surfaced as both coarse
// and fine grain memory regions. To track total system
// memory only fine grain is considered as it avoids
// double counting
if (IsSystem() && (fine_grain)) {
max_sysmem_alloc_size_ += max_single_alloc_size_;
}
assert(GetVirtualSize() != 0);
assert(GetPhysicalSize() <= GetVirtualSize());
@@ -147,7 +162,10 @@ hsa_status_t MemoryRegion::Allocate(size_t& size, AllocateFlags alloc_flags, voi
return HSA_STATUS_ERROR_INVALID_ALLOCATION;
}
if (size > max_single_alloc_size_) {
// Alocation requests for system memory considers aggregate
// memory available on all CPU devices
if (size > ((IsSystem() ?
max_sysmem_alloc_size_ : max_single_alloc_size_))) {
return HSA_STATUS_ERROR_INVALID_ALLOCATION;
}
@@ -285,9 +303,11 @@ hsa_status_t MemoryRegion::GetInfo(hsa_region_info_t attribute,
break;
case HSA_REGION_INFO_ALLOC_MAX_SIZE:
switch (mem_props_.HeapType) {
case HSA_HEAPTYPE_SYSTEM:
*((size_t*)value) = max_sysmem_alloc_size_;
break;
case HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE:
case HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC:
case HSA_HEAPTYPE_SYSTEM:
case HSA_HEAPTYPE_GPU_SCRATCH:
*((size_t*)value) = max_single_alloc_size_;
break;
@@ -365,10 +385,23 @@ hsa_status_t MemoryRegion::GetPoolInfo(hsa_amd_memory_pool_info_t attribute,
case HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE:
case HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALIGNMENT:
return GetInfo(static_cast<hsa_region_info_t>(attribute), value);
break;
case HSA_AMD_MEMORY_POOL_INFO_ACCESSIBLE_BY_ALL:
*((bool*)value) = IsSystem() ? true : false;
break;
case HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE:
switch (mem_props_.HeapType) {
case HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE:
case HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC:
case HSA_HEAPTYPE_GPU_SCRATCH:
return GetInfo(HSA_REGION_INFO_ALLOC_MAX_SIZE, value);
case HSA_HEAPTYPE_SYSTEM:
// Aggregate size available for allocation
*((size_t*)value) = max_sysmem_alloc_size_;
break;
default:
*((size_t*)value) = 0;
}
break;
default:
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
}
projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h
+27 -4
View File
@@ -648,8 +648,26 @@ typedef enum {
} hsa_amd_segment_t;
/**
* @brief A memory pool represents physical storage on an agent.
*/
* @brief A memory pool encapsulates physical storage on an agent
* along with a memory access model.
*
* @details A memory pool encapsulates a physical partition of an agent's
* memory system along with a memory access model. Division of a single
* memory system into separate pools allows querying each partition's access
* path properties (see ::hsa_amd_agent_memory_pool_get_info). Allocations
* from a pool are preferentially bound to that pool's physical partition.
* Binding to the pool's preferential physical partition may not be
* possible or persistent depending on the system's memory policy
* and/or state which is beyond the scope of HSA APIs.
*
* For example, a multi-node NUMA memory system may be represented by multiple
* pool's with each pool providing size and access path information for the
* partition it represents. Allocations from a pool are preferentially bound
* to the pool's partition (which in this example is a NUMA node) while
* following its memory access model. The actual placement may vary or migrate
* due to the system's NUMA policy and state, which is beyond the scope of
* HSA APIs.
*/
typedef struct hsa_amd_memory_pool_s {
/**
* Opaque handle.
@@ -729,6 +747,11 @@ typedef enum {
* attribute is bool.
*/
HSA_AMD_MEMORY_POOL_INFO_ACCESSIBLE_BY_ALL = 15,
/**
* Maximum aggregate allocation size in bytes. The type of this attribute
* is size_t.
*/
HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE = 16,
} hsa_amd_memory_pool_info_t;
/**
@@ -817,8 +840,8 @@ hsa_status_t HSA_API hsa_amd_agent_iterate_memory_pools(
* @retval ::HSA_STATUS_ERROR_INVALID_MEMORY_POOL The memory pool is invalid.
*
* @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION The host is not allowed to
* allocate memory in @p memory_pool, or @p size is greater than the value of
* HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE in @p memory_pool.
* allocate memory in @p memory_pool, or @p size is greater than
* the value of HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE in @p memory_pool.
*
* @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT @p ptr is NULL, or @p size is 0,
* or flags is not 0.