From e4a84c4a9c41e0bf15a1a2af24fc135fef9b2688 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Sun, 18 Dec 2022 23:54:59 +0000 Subject: [PATCH] Support memory handles Add support for creating and releasing memory handles. Memory handles are memory allocations on device memory without a virtual address. This is part of patch series for Virtual Memory API. Change-Id: I5dfb162eb1661621cce171b2870a3c93b24d840e --- .../core/common/hsa_table_interface.cpp | 10 ++++ .../hsa-runtime/core/inc/amd_memory_region.h | 2 + .../hsa-runtime/core/inc/hsa_ext_amd_impl.h | 9 +++ runtime/hsa-runtime/core/inc/memory_region.h | 2 + runtime/hsa-runtime/core/inc/runtime.h | 38 +++++++++++++ .../core/runtime/hsa_api_trace.cpp | 2 + .../hsa-runtime/core/runtime/hsa_ext_amd.cpp | 34 +++++++++++ runtime/hsa-runtime/core/runtime/runtime.cpp | 46 +++++++++++++++ runtime/hsa-runtime/hsacore.so.def | 2 + runtime/hsa-runtime/inc/hsa_api_trace.h | 2 + runtime/hsa-runtime/inc/hsa_ext_amd.h | 56 +++++++++++++++++++ 11 files changed, 203 insertions(+) diff --git a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index 7ba78da1c9..be7191662a 100644 --- a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -1240,6 +1240,16 @@ hsa_status_t HSA_API hsa_amd_vmem_address_free(void* ptr, size_t size) { return amdExtTable->hsa_amd_vmem_address_free_fn(ptr, size); } +hsa_status_t HSA_API hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t pool, size_t size, + hsa_amd_memory_type_t type, uint64_t flags, + hsa_amd_vmem_alloc_handle_t* memory_handle) { + return amdExtTable->hsa_amd_vmem_handle_create_fn(pool, size, type, flags, memory_handle); +} + +hsa_status_t HSA_API hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle) { + return amdExtTable->hsa_amd_vmem_handle_release_fn(memory_handle); +} + // Tools only table interfaces. namespace rocr { diff --git a/runtime/hsa-runtime/core/inc/amd_memory_region.h b/runtime/hsa-runtime/core/inc/amd_memory_region.h index cb5d17e2b7..c3da1c770d 100644 --- a/runtime/hsa-runtime/core/inc/amd_memory_region.h +++ b/runtime/hsa-runtime/core/inc/amd_memory_region.h @@ -175,6 +175,8 @@ class MemoryRegion : public core::MemoryRegion { __forceinline bool extended_scope_fine_grain() const { return extended_scope_fine_grain_; } + __forceinline size_t GetPageSize() const { return kPageSize_; } + private: const HsaMemoryProperties mem_props_; diff --git a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index 8e3a792dca..8a04232e4e 100644 --- a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -304,6 +304,15 @@ hsa_status_t hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t addr // Mirrors Amd Extension Apis hsa_status_t hsa_amd_vmem_address_free(void* ptr, size_t size); + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t pool, size_t size, + hsa_amd_memory_type_t type, uint64_t flags, + hsa_amd_vmem_alloc_handle_t* memory_handle); + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle); + } // namespace amd } // namespace rocr diff --git a/runtime/hsa-runtime/core/inc/memory_region.h b/runtime/hsa-runtime/core/inc/memory_region.h index feba80695b..d9f953db89 100644 --- a/runtime/hsa-runtime/core/inc/memory_region.h +++ b/runtime/hsa-runtime/core/inc/memory_region.h @@ -92,6 +92,8 @@ class MemoryRegion : public Checked<0x9C961F19EE175BB3> { AllocateNonPaged = (1 << 4), // Non-paged system memory (AllocateIPC alias) AllocatePCIeRW = (1 << 5), // Enforce pseudo fine grain/RW memory AllocateAsan = (1 << 6), // ASAN - First page of allocation remapped to system memory + AllocatePinned = (1 << 7), // Currently treating Pinned memory as NoSubstitute + AllocateMemoryOnly = (1 << 8), // Memory only handle from thunk, no virtual address }; typedef uint32_t AllocateFlags; diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index 065d539f7d..c7f891f8e6 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -357,6 +357,13 @@ class Runtime { hsa_status_t VMemoryAddressReserve(void** ptr, size_t size, uint64_t address, uint64_t flags); hsa_status_t VMemoryAddressFree(void* ptr, size_t size); + + hsa_status_t VMemoryHandleCreate(const MemoryRegion* region, size_t size, + MemoryRegion::AllocateFlags alloc_flags, + uint64_t flags, hsa_amd_vmem_alloc_handle_t* memoryHandle); + + hsa_status_t VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryHandle); + const std::vector& cpu_agents() { return cpu_agents_; } const std::vector& gpu_agents() { return gpu_agents_; } @@ -677,6 +684,8 @@ class Runtime { bool virtual_mem_api_supported_; + typedef void* ThunkHandle; + struct AddressHandle { AddressHandle() : size(0), use_count(0) {} AddressHandle(size_t size) : size(size), use_count(0) {} @@ -686,6 +695,35 @@ class Runtime { }; std::map reserved_address_map_; // Indexed by VA + struct MemoryHandle { + MemoryHandle() : region(NULL), size(0), ref_count(0), thunk_handle(NULL), alloc_flag(0) {} + MemoryHandle(const MemoryRegion* region, size_t size, uint64_t flags_unused, + ThunkHandle thunk_handle, MemoryRegion::AllocateFlags alloc_flag) + : region(region), + size(size), + ref_count(1), + use_count(0), + thunk_handle(thunk_handle), + alloc_flag(alloc_flag) {} + + static __forceinline hsa_amd_vmem_alloc_handle_t Convert(void* handle) { + hsa_amd_vmem_alloc_handle_t ret_handle = { + static_cast(reinterpret_cast(handle))}; + return ret_handle; + } + + __forceinline core::Agent* agentOwner() const { return region->owner(); } + + const MemoryRegion* region; + size_t size; + int ref_count; + int use_count; + ThunkHandle thunk_handle; // handle returned by hsaKmtAllocMemory(NoAddress = 1) + MemoryRegion::AllocateFlags alloc_flag; + }; + + std::map memory_handle_map_; + // Frees runtime memory when the runtime library is unloaded if safe to do so. // Failure to release the runtime indicates an incorrect application but is // common (example: calls library routines at process exit). diff --git a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index fd2f9541eb..a4b2f1b6cf 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -404,6 +404,8 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_portable_close_dmabuf_fn = AMD::hsa_amd_portable_close_dmabuf; amd_ext_api.hsa_amd_vmem_address_reserve_fn = AMD::hsa_amd_vmem_address_reserve; amd_ext_api.hsa_amd_vmem_address_free_fn = AMD::hsa_amd_vmem_address_free; + amd_ext_api.hsa_amd_vmem_handle_create_fn = AMD::hsa_amd_vmem_handle_create; + amd_ext_api.hsa_amd_vmem_handle_release_fn = AMD::hsa_amd_vmem_handle_release; } void LoadInitialHsaApiTable() { diff --git a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index 5752d35695..6173e5fcf0 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -1220,5 +1220,39 @@ hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size) { return core::Runtime::runtime_singleton_->VMemoryAddressFree(va, size); CATCH; } + +hsa_status_t hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t memory_pool, size_t size, + hsa_amd_memory_type_t type, uint64_t flags, + hsa_amd_vmem_alloc_handle_t* memory_handle) { + TRY; + IS_OPEN(); + IS_ZERO(size); + IS_TRUE(core::Runtime::runtime_singleton_->VirtualMemApiSupported()); + + if (type != MEMORY_TYPE_NONE && type != MEMORY_TYPE_PINNED) + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + hsa_region_t region = {memory_pool.handle}; + const core::MemoryRegion* mem_region = core::MemoryRegion::Convert(region); + + if (mem_region == NULL || !mem_region->IsValid()) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + MemoryRegion::AllocateFlags alloc_flag = core::MemoryRegion::AllocateMemoryOnly; + if (type == MEMORY_TYPE_PINNED) alloc_flag |= core::MemoryRegion::AllocatePinned; + + return core::Runtime::runtime_singleton_->VMemoryHandleCreate(mem_region, size, alloc_flag, flags, + memory_handle); + CATCH; +} + +hsa_status_t hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle) { + TRY; + IS_OPEN(); + return core::Runtime::runtime_singleton_->VMemoryHandleRelease(memory_handle); + CATCH; +} + } // namespace amd } // namespace rocr diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 0572b69c22..4c4fbea4d5 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -2446,5 +2446,51 @@ hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) { return HSA_STATUS_SUCCESS; } +hsa_status_t Runtime::VMemoryHandleCreate(const MemoryRegion* region, size_t size, + MemoryRegion::AllocateFlags alloc_flags, + uint64_t flags_unused, + hsa_amd_vmem_alloc_handle_t* memoryOnlyHandle) { + const AMD::MemoryRegion* memRegion = static_cast(region); + if (!memRegion->IsLocalMemory()) return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + if (!IsMultipleOf(size, memRegion->GetPageSize())) + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + ScopedAcquire lock(&memory_lock_); + void* thunk_handle; + hsa_status_t status = region->Allocate(size, alloc_flags, &thunk_handle); + if (status == HSA_STATUS_SUCCESS) { + memory_handle_map_[thunk_handle] = + MemoryHandle(region, size, flags_unused, thunk_handle, alloc_flags); + *memoryOnlyHandle = MemoryHandle::Convert(thunk_handle); + } + return status; +} + +hsa_status_t Runtime::VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryOnlyHandle) { + ScopedAcquire lock(&memory_lock_); + auto memoryHandleIt = memory_handle_map_.find(reinterpret_cast(memoryOnlyHandle.handle)); + + if (memoryHandleIt == memory_handle_map_.end()) { + debug_warning(false && "Can't find memory handle"); + return HSA_STATUS_ERROR_INVALID_ALLOCATION; + } + + if (!memoryHandleIt->second.ref_count) return HSA_STATUS_ERROR_INVALID_ALLOCATION; + + if (--(memoryHandleIt->second.ref_count) == 0) { + // From documentation, the handle can be released while there are still outstanding mappings. If + // there are outstanding mappings, then we just decrement the ref count and exit. We will free + // this handle when the last MappedHandle is deleted + // and use_count == 0 and ref_count == 0. + + if (memoryHandleIt->second.use_count > 0) return HSA_STATUS_SUCCESS; + + memoryHandleIt->second.region->Free(memoryHandleIt->first, memoryHandleIt->second.size); + memory_handle_map_.erase(memoryHandleIt); + } + return HSA_STATUS_SUCCESS; +} + } // namespace core } // namespace rocr diff --git a/runtime/hsa-runtime/hsacore.so.def b/runtime/hsa-runtime/hsacore.so.def index 7496551a98..3688a31ef7 100644 --- a/runtime/hsa-runtime/hsacore.so.def +++ b/runtime/hsa-runtime/hsacore.so.def @@ -235,6 +235,8 @@ global: hsa_amd_portable_close_dmabuf; hsa_amd_vmem_address_reserve; hsa_amd_vmem_address_free; + hsa_amd_vmem_handle_create; + hsa_amd_vmem_handle_release; local: *; diff --git a/runtime/hsa-runtime/inc/hsa_api_trace.h b/runtime/hsa-runtime/inc/hsa_api_trace.h index 32cfa3c0fc..404a99cbea 100644 --- a/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -196,6 +196,8 @@ struct AmdExtTable { decltype(hsa_amd_portable_close_dmabuf)* hsa_amd_portable_close_dmabuf_fn; decltype(hsa_amd_vmem_address_reserve)* hsa_amd_vmem_address_reserve_fn; decltype(hsa_amd_vmem_address_free)* hsa_amd_vmem_address_free_fn; + decltype(hsa_amd_vmem_handle_create)* hsa_amd_vmem_handle_create_fn; + decltype(hsa_amd_vmem_handle_release)* hsa_amd_vmem_handle_release_fn; }; // Table to export HSA Core Runtime Apis diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index 15f128eab1..e5828e7ba9 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -2744,6 +2744,62 @@ hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t addre */ hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size); +/** + * @brief Struct containing an opaque handle to a memory allocation handle + */ +typedef struct hsa_amd_vmem_alloc_handle_s { + /** + * Opaque handle. Two handles reference the same object of the enclosing type + * if and only if they are equal. + */ + uint64_t handle; +} hsa_amd_vmem_alloc_handle_t; + +typedef enum { + MEMORY_TYPE_NONE, + MEMORY_TYPE_PINNED, +} hsa_amd_memory_type_t; + +/* + * @brief Create a virtual memory handle + * + * Create a virtual memory handle within this pool + * @p size must be a aligned to allocation granule size for this memory pool, see + * HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE + * To minimize internal memory fragmentation, align the size to the recommended allocation granule + * size, see HSA_AMD_REGION_INFO_RUNTIME_ALLOC_RECOMMENDED_GRANULE + * + * @param[in] pool memory to use + * @param[in] size of the memory allocation + * @param[in] type of memory + * @param[in] flags - currently unsupported + * @param[out] memory_handle - handle for the allocation + * + * @retval ::HSA_STATUS_SUCCESS memory allocated successfully + * + * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been initialized. + * + * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT Invalid arguments + * + * @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION This memory pool does not support allocations + * + * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Insufficient resources to allocate this memory + */ +hsa_status_t hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t pool, size_t size, + hsa_amd_memory_type_t type, uint64_t flags, + hsa_amd_vmem_alloc_handle_t* memory_handle); + +/* + * @brief Release a virtual memory handle + * + * @param[in] memory handle that was previously allocated + * + * @retval ::HSA_STATUS_SUCCESS Address range allocated successfully + * + * @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION Invalid memory handle + */ +hsa_status_t hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle); + #ifdef __cplusplus } // end extern "C" block #endif