SWDEV-497841 - Add virtual memory heap

Add initial implementation of virtual memory heap with
dynamic virtual memory mapping support for memory pools.
DEBUG_HIP_MEM_POOL_VMHEAP controls the new method.

Change-Id: I8dc5be2e0f34ab472f1800f43bb6243639a5e500

hipamd/src/hip_mempool_impl.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2022-2023 Advanced Micro Devices, Inc.
+/* Copyright (c) 2022-2025 Advanced Micro Devices, Inc.
 
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
@@ -90,16 +90,19 @@ bool Heap::RemoveMemory(amd::Memory* memory, MemoryTimestamp* ts) {
 }
 
 // ================================================================================================
-Heap::SortedMap::iterator Heap::EraseAllocaton(Heap::SortedMap::iterator& it) {
+Heap::SortedMap::iterator Heap::EraseAllocation(Heap::SortedMap::iterator& it) {
   auto memory = it->first.second;
   const device::Memory* dev_mem = memory->getDeviceMemory(*device_->devices()[0]);
   void* dev_mem_vaddr = reinterpret_cast<void*>(dev_mem->virtualAddress());
   total_size_ -= it->first.first;
+  if (dev_mem_vaddr == nullptr) {
+    dev_mem_vaddr = memory->getSvmPtr();
+  }
 
-  if (dev_mem_vaddr != nullptr) {
-    amd::SvmBuffer::free(memory->getContext(), dev_mem_vaddr);
+  if (use_vm_heap_) {
+    vm_heap_.Free(memory);
   } else {
-    amd::SvmBuffer::free(memory->getContext(), memory->getSvmPtr());
+    amd::SvmBuffer::free(memory->getContext(), dev_mem_vaddr);
   }
   // Clear HIP event
   it->second.SetEvent(nullptr);
@@ -119,11 +122,15 @@ bool Heap::ReleaseAllMemory(size_t min_bytes_to_hold, bool safe_release) {
       it->second.Wait();
     }
     if (it->second.IsSafeRelease()) {
-      it = EraseAllocaton(it);
+      it = EraseAllocation(it);
     } else {
       ++it;
     }
   }
+  // Handle managed pool with trim
+  if (vm_heap_.FreeMappedSize() > min_bytes_to_hold) {
+    vm_heap_.TrimPhysMemory(min_bytes_to_hold);
+  }
   return true;
 }
 
@@ -131,11 +138,12 @@ bool Heap::ReleaseAllMemory(size_t min_bytes_to_hold, bool safe_release) {
 bool Heap::ReleaseAllMemory() {
   for (auto it = allocations_.begin(); it != allocations_.end();) {
     // Make sure the heap holds the minimum number of bytes
-    if (total_size_ <= release_threshold_) {
+    // @note: Managed memory controls the threshold on its own
+    if (!use_vm_heap_ && (total_size_ <= release_threshold_)) {
       return true;
     }
     if (it->second.IsSafeRelease()) {
-      it = EraseAllocaton(it);
+      it = EraseAllocation(it);
     } else {
       ++it;
     }
@@ -187,13 +195,17 @@ void* MemoryPool::AllocateMemory(size_t size, Stream* stream, void* dptr) {
     }
     cl_svm_mem_flags flags = (state_.interprocess_) ? ROCCLR_MEM_INTERPROCESS : 0;
     flags |= (state_.phys_mem_) ? ROCCLR_MEM_PHYMEM : 0;
-    dev_ptr = amd::SvmBuffer::malloc(*context, flags, size, dev_info.memBaseAddrAlign_, nullptr);
+    if (state_.use_vm_heap_) {
+      dev_ptr = Alloc(size);
+    } else {
+      dev_ptr = amd::SvmBuffer::malloc(*context, flags, size, dev_info.memBaseAddrAlign_, nullptr);
+    }
     if (dev_ptr == nullptr) {
       size_t free = 0, total =0;
       hipError_t err = hipMemGetInfo(&free, &total);
       if (err == hipSuccess) {
-        LogPrintfError("Allocation failed : Device memory : required :%zu | free :%zu | total :%zu",
-          size, free, total);
+        LogPrintfError("Allocation failed : Device memory : required :\
+          %zu | free :%zu | total :%zu", size, free, total);
       }
       return nullptr;
     }
@@ -236,7 +248,7 @@ bool MemoryPool::FreeMemory(amd::Memory* memory, Stream* stream, Event* event) {
   {
     amd::ScopedLock lock(lock_pool_ops_);
 
-    if (memory->getUserData().phys_mem_obj != nullptr) {
+    if (!state_.use_vm_heap_ && memory->getUserData().phys_mem_obj != nullptr) {
       memory = memory->getUserData().phys_mem_obj;
     }
 
@@ -408,8 +420,9 @@ hipError_t MemoryPool::GetAttribute(hipMemPoolAttr attr, void* value) {
       *reinterpret_cast<uint64_t*>(value) = free_heap_.GetReleaseThreshold();
       break;
     case hipMemPoolAttrReservedMemCurrent:
-      // All allocate memory by the pool in OS
-      *reinterpret_cast<uint64_t*>(value) = busy_heap_.GetTotalSize() + free_heap_.GetTotalSize();
+      // All allocated memory by the pool in OS
+      *reinterpret_cast<uint64_t*>(value) = (state_.use_vm_heap_) ? MappedSize() :
+        (busy_heap_.GetTotalSize() + free_heap_.GetTotalSize());
       break;
     case hipMemPoolAttrReservedMemHigh:
       // High watermark of all allocated memory in OS, since the last reset