AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-497841 - Add virtual memory heap

Browse Source 
Add initial implementation of virtual memory heap with
dynamic virtual memory mapping support for memory pools.
DEBUG_HIP_MEM_POOL_VMHEAP controls the new method.

Change-Id: I8dc5be2e0f34ab472f1800f43bb6243639a5e500
This commit is contained in:
German Andryeyev
2025-01-24 19:26:39 -05:00
parent 63cf3057ba
commit 296dce5570
11 changed files with 670 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hipamd/src/hip_device.cpp
+1 -1
View File
@@ -315,7 +315,7 @@ bool Device::existsActiveStreamForDevice() {
// ================================================================================================
Device::~Device() {
if (default_mem_pool_ != nullptr) {
if ((IS_LINUX || !DEBUG_HIP_MEM_POOL_VMHEAP) && (default_mem_pool_ != nullptr)) {
default_mem_pool_->release();
}
hipamd/src/hip_mempool_impl.cpp
+27 -14
View File
@@ -1,4 +1,4 @@
/* Copyright (c) 2022-2023 Advanced Micro Devices, Inc.
/* Copyright (c) 2022-2025 Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -90,16 +90,19 @@ bool Heap::RemoveMemory(amd::Memory* memory, MemoryTimestamp* ts) {
}
// ================================================================================================
Heap::SortedMap::iterator Heap::EraseAllocaton(Heap::SortedMap::iterator& it) {
Heap::SortedMap::iterator Heap::EraseAllocation(Heap::SortedMap::iterator& it) {
auto memory = it->first.second;
const device::Memory* dev_mem = memory->getDeviceMemory(*device_->devices()[0]);
void* dev_mem_vaddr = reinterpret_cast<void*>(dev_mem->virtualAddress());
total_size_ -= it->first.first;
if (dev_mem_vaddr == nullptr) {
dev_mem_vaddr = memory->getSvmPtr();
}
if (dev_mem_vaddr != nullptr) {
amd::SvmBuffer::free(memory->getContext(), dev_mem_vaddr);
if (use_vm_heap_) {
vm_heap_.Free(memory);
} else {
amd::SvmBuffer::free(memory->getContext(), memory->getSvmPtr());
amd::SvmBuffer::free(memory->getContext(), dev_mem_vaddr);
}
// Clear HIP event
it->second.SetEvent(nullptr);
@@ -119,11 +122,15 @@ bool Heap::ReleaseAllMemory(size_t min_bytes_to_hold, bool safe_release) {
it->second.Wait();
}
if (it->second.IsSafeRelease()) {
it = EraseAllocaton(it);
it = EraseAllocation(it);
} else {
++it;
}
}
// Handle managed pool with trim
if (vm_heap_.FreeMappedSize() > min_bytes_to_hold) {
vm_heap_.TrimPhysMemory(min_bytes_to_hold);
}
return true;
}
@@ -131,11 +138,12 @@ bool Heap::ReleaseAllMemory(size_t min_bytes_to_hold, bool safe_release) {
bool Heap::ReleaseAllMemory() {
for (auto it = allocations_.begin(); it != allocations_.end();) {
// Make sure the heap holds the minimum number of bytes
if (total_size_ <= release_threshold_) {
// @note: Managed memory controls the threshold on its own
if (!use_vm_heap_ && (total_size_ <= release_threshold_)) {
return true;
}
if (it->second.IsSafeRelease()) {
it = EraseAllocaton(it);
it = EraseAllocation(it);
} else {
++it;
}
@@ -187,13 +195,17 @@ void* MemoryPool::AllocateMemory(size_t size, Stream* stream, void* dptr) {
}
cl_svm_mem_flags flags = (state_.interprocess_) ? ROCCLR_MEM_INTERPROCESS : 0;
flags |= (state_.phys_mem_) ? ROCCLR_MEM_PHYMEM : 0;
dev_ptr = amd::SvmBuffer::malloc(*context, flags, size, dev_info.memBaseAddrAlign_, nullptr);
if (state_.use_vm_heap_) {
dev_ptr = Alloc(size);
} else {
dev_ptr = amd::SvmBuffer::malloc(*context, flags, size, dev_info.memBaseAddrAlign_, nullptr);
}
if (dev_ptr == nullptr) {
size_t free = 0, total =0;
hipError_t err = hipMemGetInfo(&free, &total);
if (err == hipSuccess) {
LogPrintfError("Allocation failed : Device memory : required :%zu | free :%zu | total :%zu",
size, free, total);
LogPrintfError("Allocation failed : Device memory : required :\
%zu | free :%zu | total :%zu", size, free, total);
}
return nullptr;
}
@@ -236,7 +248,7 @@ bool MemoryPool::FreeMemory(amd::Memory* memory, Stream* stream, Event* event) {
{
amd::ScopedLock lock(lock_pool_ops_);
if (memory->getUserData().phys_mem_obj != nullptr) {
if (!state_.use_vm_heap_ && memory->getUserData().phys_mem_obj != nullptr) {
memory = memory->getUserData().phys_mem_obj;
}
@@ -408,8 +420,9 @@ hipError_t MemoryPool::GetAttribute(hipMemPoolAttr attr, void* value) {
*reinterpret_cast<uint64_t*>(value) = free_heap_.GetReleaseThreshold();
break;
case hipMemPoolAttrReservedMemCurrent:
// All allocate memory by the pool in OS
*reinterpret_cast<uint64_t*>(value) = busy_heap_.GetTotalSize() + free_heap_.GetTotalSize();
// All allocated memory by the pool in OS
*reinterpret_cast<uint64_t*>(value) = (state_.use_vm_heap_) ? MappedSize() :
(busy_heap_.GetTotalSize() + free_heap_.GetTotalSize());
break;
case hipMemPoolAttrReservedMemHigh:
// High watermark of all allocated memory in OS, since the last reset
hipamd/src/hip_mempool_impl.hpp
+38 -12
View File
@@ -1,4 +1,4 @@
/* Copyright (c) 2022 Advanced Micro Devices, Inc.
/* Copyright (c) 2022-2025 Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -23,6 +23,7 @@
#include <hip/hip_runtime.h>
#include "hip_event.hpp"
#include "hip_internal.hpp"
#include "platform/vmheap.hpp"
#include <unordered_map>
#include <unordered_set>
@@ -101,8 +102,12 @@ class Heap : public amd::EmbeddedObject {
public:
typedef std::map<std::pair<size_t, amd::Memory*>, MemoryTimestamp> SortedMap;
Heap(hip::Device* device):
total_size_(0), max_total_size_(0), release_threshold_(0), device_(device) {}
Heap(hip::Device* device, amd::VmHeap& vm_heap)
: total_size_(0)
, max_total_size_(0)
, release_threshold_(0)
, device_(device)
, vm_heap_(vm_heap) {}
~Heap() {}
/// Adds allocation into the heap on a specific stream
@@ -134,7 +139,10 @@ public:
bool IsEmpty() const { return (allocations_.size() == 0) ? true : false; }
/// Set the memory release threshold
void SetReleaseThreshold(uint64_t value) { release_threshold_ = value; }
void SetReleaseThreshold(uint64_t value) {
release_threshold_ = value;
vm_heap_.SetUnmapThreshold(value);
}
/// Set the memory release threshold
uint64_t GetReleaseThreshold() const { return release_threshold_; }
@@ -149,7 +157,7 @@ public:
void SetMaxTotalSize(uint64_t value) { max_total_size_ = value; }
/// Erases single allocation form the heap's map
SortedMap::iterator EraseAllocaton(SortedMap::iterator& it);
SortedMap::iterator EraseAllocation(SortedMap::iterator& it);
/// Add a safe stream for quick looks-ups in all allocations
void AddSafeStream(Stream* event_stream, Stream* wait_stream) {
@@ -162,6 +170,13 @@ public:
bool IsActiveMemory(amd::Memory* memory) const {
return (allocations_.find({memory->getSize(), memory}) != allocations_.end());
}
/// Enabled VM heap for memory, instead of direct allocations
void EnableVmHeap() { use_vm_heap_ = true; }
/// Returns true if heap uses virtual memory
bool UseVmHeap() const { return use_vm_heap_; }
const auto& Allocations() { return allocations_; }
private:
@@ -174,14 +189,16 @@ private:
uint64_t max_total_size_; //!< Maximum heap allocation size
uint64_t release_threshold_; //!< Threshold size in bytes for memory release from heap, default 0
hip::Device* device_; //!< Hip device the allocations will reside
hip::Device* device_; //!< Hip device the allocations will reside
amd::VmHeap& vm_heap_; //!< Managed heap for memory allocaitons
bool use_vm_heap_ = false; //!< Use virtual heap or direct allocations
};
/// Allocates memory in the pool on the specified stream and places the allocation into busy_heap_
/// @note: the logic also will look in free_heap for possible reuse.
/// hipMemPoolReuseAllowOpportunistic option will validate if HIP event,
/// associated with memory is done, then reuse can be performed.
class MemoryPool : public amd::ReferenceCountedObject {
class MemoryPool : public amd::ReferenceCountedObject, amd::VmHeap {
public:
struct SharedAccess {
int device_id_; //!< Device ID for access with a specified shared resource
@@ -197,9 +214,10 @@ class MemoryPool : public amd::ReferenceCountedObject {
};
MemoryPool(hip::Device* device, const hipMemPoolProps* props = nullptr, bool phys_mem = false)
: busy_heap_(device),
free_heap_(device),
lock_pool_ops_(true), /* Pool operations */
: VmHeap(device->asContext()->devices()[0], *device->NullStream()),
busy_heap_(device, *this),
free_heap_(device, *this),
lock_pool_ops_(true),
device_(device),
shared_(nullptr),
max_total_size_(0) {
@@ -220,9 +238,15 @@ class MemoryPool : public amd::ReferenceCountedObject {
.reserved = {}};
}
state_.interprocess_ = properties_.handleTypes != hipMemHandleTypeNone;
// Check if VM heap can be enabled
if (DEBUG_HIP_MEM_POOL_VMHEAP && !state_.phys_mem_ && !state_.interprocess_) {
state_.use_vm_heap_ = true;
busy_heap_.EnableVmHeap();
free_heap_.EnableVmHeap();
}
}
virtual ~MemoryPool() {
virtual ~MemoryPool() override {
if (!busy_heap_.IsEmpty()) {
LogError("Shouldn't destroy pool with busy allocations!");
}
@@ -320,6 +344,7 @@ class MemoryPool : public amd::ReferenceCountedObject {
uint32_t interprocess_ : 1; //!< Memory pool can be used in interprocess communications
uint32_t graph_in_use_ : 1; //!< Memory pool was used in a graph execution
uint32_t phys_mem_ : 1; //!< Mempool is used for graphs and will have physical allocations
uint32_t use_vm_heap_ : 1; //!< Use VM heap or direct allocations
};
uint32_t value_;
} state_;
@@ -327,7 +352,8 @@ class MemoryPool : public amd::ReferenceCountedObject {
hipMemPoolProps properties_; //!< Properties of the memory pool
amd::Monitor lock_pool_ops_; //!< Access to the pool must be lock protected
std::map<hip::Device*, hipMemAccessFlags> access_map_; //!< Map of access to the pool from devices
hip::Device* device_; //!< Hip device the heap will reside
hip::Device* device_; //!< Hip device the heap will reside
SharedMemPool* shared_; //!< Pointer to shared memory for IPC
uint64_t max_total_size_; //!< Max of total reserved memory in the pool since last reset
};