From b219d0224d4ec496eccc8478a9a4dabcc2dc523c Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Sun, 18 Dec 2022 23:54:43 +0000 Subject: [PATCH] Support Virtual Address reservations Add support for reserving virtual address ranges. Virtual address ranges are addresses without any memory backing. These address ranges need to be mapped to memory handles later. This is part of patch series for Virtual Memory API. Change-Id: I5d066e7421d6896f933f524312afc230a13d594e [ROCm/ROCR-Runtime commit: 1085311f1ab8d8629a155619ec672145a53b81f7] --- .../core/common/hsa_table_interface.cpp | 9 ++++ .../hsa-runtime/core/inc/hsa_ext_amd_impl.h | 6 +++ .../runtime/hsa-runtime/core/inc/runtime.h | 12 ++++++ .../core/runtime/hsa_api_trace.cpp | 2 + .../hsa-runtime/core/runtime/hsa_ext_amd.cpp | 23 ++++++++++ .../hsa-runtime/core/runtime/runtime.cpp | 40 ++++++++++++++++++ .../runtime/hsa-runtime/hsacore.so.def | 2 + .../runtime/hsa-runtime/inc/hsa_api_trace.h | 2 + .../runtime/hsa-runtime/inc/hsa_ext_amd.h | 42 +++++++++++++++++++ 9 files changed, 138 insertions(+) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index 593ccb389e..7ba78da1c9 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -1231,6 +1231,15 @@ hsa_status_t HSA_API hsa_amd_portable_close_dmabuf(int dmabuf) { return amdExtTable->hsa_amd_portable_close_dmabuf_fn(dmabuf); } +hsa_status_t HSA_API hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t address, + uint64_t flags) { + return amdExtTable->hsa_amd_vmem_address_reserve_fn(ptr, size, address, flags); +} + +hsa_status_t HSA_API hsa_amd_vmem_address_free(void* ptr, size_t size) { + return amdExtTable->hsa_amd_vmem_address_free_fn(ptr, size); +} + // Tools only table interfaces. namespace rocr { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index bbe5b91b5c..8e3a792dca 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -298,6 +298,12 @@ hsa_status_t HSA_API hsa_amd_portable_export_dmabuf(const void* ptr, size_t size // Mirrors Amd Extension Apis hsa_status_t HSA_API hsa_amd_portable_close_dmabuf(int dmabuf); +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t address, + uint64_t flags); + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_vmem_address_free(void* ptr, size_t size); } // namespace amd } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h index 0bafa7b90f..065d539f7d 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h @@ -354,6 +354,9 @@ class Runtime { hsa_status_t DmaBufClose(int dmabuf); + hsa_status_t VMemoryAddressReserve(void** ptr, size_t size, uint64_t address, uint64_t flags); + + hsa_status_t VMemoryAddressFree(void* ptr, size_t size); const std::vector& cpu_agents() { return cpu_agents_; } const std::vector& gpu_agents() { return gpu_agents_; } @@ -674,6 +677,15 @@ class Runtime { bool virtual_mem_api_supported_; + struct AddressHandle { + AddressHandle() : size(0), use_count(0) {} + AddressHandle(size_t size) : size(size), use_count(0) {} + + size_t size; + int use_count; + }; + std::map reserved_address_map_; // Indexed by VA + // Frees runtime memory when the runtime library is unloaded if safe to do so. // Failure to release the runtime indicates an incorrect application but is // common (example: calls library routines at process exit). diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index fcaebf5b05..fd2f9541eb 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -402,6 +402,8 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_spm_set_dest_buffer_fn = AMD::hsa_amd_spm_set_dest_buffer; amd_ext_api.hsa_amd_portable_export_dmabuf_fn = AMD::hsa_amd_portable_export_dmabuf; amd_ext_api.hsa_amd_portable_close_dmabuf_fn = AMD::hsa_amd_portable_close_dmabuf; + amd_ext_api.hsa_amd_vmem_address_reserve_fn = AMD::hsa_amd_vmem_address_reserve; + amd_ext_api.hsa_amd_vmem_address_free_fn = AMD::hsa_amd_vmem_address_free; } void LoadInitialHsaApiTable() { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index 09c5976e42..5752d35695 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -95,6 +95,11 @@ struct ValidityError { enum { value = ValidityError::value }; }; +#define IS_TRUE(var) \ + do { \ + if ((var) != true) return HSA_STATUS_ERROR_INVALID_ARGUMENT; \ + } while (false) + #define IS_BAD_PTR(ptr) \ do { \ if ((ptr) == NULL) return HSA_STATUS_ERROR_INVALID_ARGUMENT; \ @@ -1197,5 +1202,23 @@ hsa_status_t hsa_amd_portable_close_dmabuf(int dmabuf) { CATCH; } +hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t address, + uint64_t flags) { + TRY; + IS_OPEN(); + IS_ZERO(size); + IS_TRUE(core::Runtime::runtime_singleton_->VirtualMemApiSupported()); + return core::Runtime::runtime_singleton_->VMemoryAddressReserve(va, size, address, flags); + CATCH; +} + +hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size) { + TRY; + IS_OPEN(); + IS_BAD_PTR(va); + IS_ZERO(size); + return core::Runtime::runtime_singleton_->VMemoryAddressFree(va, size); + CATCH; +} } // namespace amd } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp index 83239b7bbd..0572b69c22 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -2406,5 +2406,45 @@ hsa_status_t Runtime::DmaBufClose(int dmabuf) { #endif } +hsa_status_t Runtime::VMemoryAddressReserve(void** va, size_t size, uint64_t address, + uint64_t flags) { + void* addr = (void*)address; + HsaMemFlags memFlags = {0}; + + + ScopedAcquire lock(&memory_lock_); + + /* Try to reserving the VA requested by user */ + if (hsaKmtAllocMemory(0, size, memFlags, &addr) != HSAKMT_STATUS_SUCCESS) { + memFlags.ui32.FixedAddress = 0; + /* Could not reserved VA requested, allocate alternate VA */ + if (hsaKmtAllocMemory(0, size, memFlags, &addr) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + reserved_address_map_[addr] = AddressHandle(size); + *va = addr; + return HSA_STATUS_SUCCESS; +} + +hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) { + ScopedAcquire lock(&memory_lock_); + std::map::iterator it = reserved_address_map_.find(va); + + if (it == reserved_address_map_.end()) { + debug_warning(false && "Can't find address in reserved address"); + return HSA_STATUS_ERROR_INVALID_ALLOCATION; + } + + if (size != it->second.size) return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + if (it->second.use_count > 0) return HSA_STATUS_ERROR_RESOURCE_FREE; + + if (hsaKmtFreeMemory(va, size) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + reserved_address_map_.erase(it); + return HSA_STATUS_SUCCESS; +} + } // namespace core } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def b/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def index bd74f30661..7496551a98 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def +++ b/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def @@ -233,6 +233,8 @@ global: hsa_amd_spm_set_dest_buffer; hsa_amd_portable_export_dmabuf; hsa_amd_portable_close_dmabuf; + hsa_amd_vmem_address_reserve; + hsa_amd_vmem_address_free; local: *; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h index 3c613dc393..32cfa3c0fc 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -194,6 +194,8 @@ struct AmdExtTable { decltype(hsa_amd_queue_cu_get_mask)* hsa_amd_queue_cu_get_mask_fn; decltype(hsa_amd_portable_export_dmabuf)* hsa_amd_portable_export_dmabuf_fn; decltype(hsa_amd_portable_close_dmabuf)* hsa_amd_portable_close_dmabuf_fn; + decltype(hsa_amd_vmem_address_reserve)* hsa_amd_vmem_address_reserve_fn; + decltype(hsa_amd_vmem_address_free)* hsa_amd_vmem_address_free_fn; }; // Table to export HSA Core Runtime Apis diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h index 22e64860fb..15f128eab1 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -2702,6 +2702,48 @@ hsa_status_t hsa_amd_portable_export_dmabuf(const void* ptr, size_t size, int* d */ hsa_status_t hsa_amd_portable_close_dmabuf(int dmabuf); +/* + * @brief Allocate a reserved address range + * + * Reserve a virtual address range. The size must be a multiple of the system page size. + * If it is not possible to allocate the address specified by @p address, then @p va will be + * a different address range. + * Address range should be released by calling hsa_amd_vmem_address_free. + * + * @param[out] va virtual address allocated + * @param[in] size of address range requested + * @param[in] address requested + * @param[in] flags currently unsupported + * + * @retval ::HSA_STATUS_SUCCESS Address range allocated successfully + * + * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been + * initialized. + * + * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Insufficient resources to allocate an address + * range of this size. + */ +hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t address, + uint64_t flags); + +/* + * @brief Free a reserved address range + * + * Free a previously allocated address range. The size must match the size of a previously + * allocated address range. + * + * @param[out] va virtual address to be freed + * @param[in] size of address range + * + * @retval ::HSA_STATUS_SUCCESS Address range released successfully + * + * @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION Invalid va specified + * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT Invalid size specified + * @retval ::HSA_STATUS_ERROR_RESOURCE_FREE Address range is still in use + * @retval ::HSA_STATUS_ERROR Internal unexpected error + */ +hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size); + #ifdef __cplusplus } // end extern "C" block #endif