diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index 593ccb389e..7ba78da1c9 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -1231,6 +1231,15 @@ hsa_status_t HSA_API hsa_amd_portable_close_dmabuf(int dmabuf) { return amdExtTable->hsa_amd_portable_close_dmabuf_fn(dmabuf); } +hsa_status_t HSA_API hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t address, + uint64_t flags) { + return amdExtTable->hsa_amd_vmem_address_reserve_fn(ptr, size, address, flags); +} + +hsa_status_t HSA_API hsa_amd_vmem_address_free(void* ptr, size_t size) { + return amdExtTable->hsa_amd_vmem_address_free_fn(ptr, size); +} + // Tools only table interfaces. namespace rocr { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index bbe5b91b5c..8e3a792dca 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -298,6 +298,12 @@ hsa_status_t HSA_API hsa_amd_portable_export_dmabuf(const void* ptr, size_t size // Mirrors Amd Extension Apis hsa_status_t HSA_API hsa_amd_portable_close_dmabuf(int dmabuf); +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t address, + uint64_t flags); + +// Mirrors Amd Extension Apis +hsa_status_t hsa_amd_vmem_address_free(void* ptr, size_t size); } // namespace amd } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h index 0bafa7b90f..065d539f7d 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h @@ -354,6 +354,9 @@ class Runtime { hsa_status_t DmaBufClose(int dmabuf); + hsa_status_t VMemoryAddressReserve(void** ptr, size_t size, uint64_t address, uint64_t flags); + + hsa_status_t VMemoryAddressFree(void* ptr, size_t size); const std::vector& cpu_agents() { return cpu_agents_; } const std::vector& gpu_agents() { return gpu_agents_; } @@ -674,6 +677,15 @@ class Runtime { bool virtual_mem_api_supported_; + struct AddressHandle { + AddressHandle() : size(0), use_count(0) {} + AddressHandle(size_t size) : size(size), use_count(0) {} + + size_t size; + int use_count; + }; + std::map reserved_address_map_; // Indexed by VA + // Frees runtime memory when the runtime library is unloaded if safe to do so. // Failure to release the runtime indicates an incorrect application but is // common (example: calls library routines at process exit). diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index fcaebf5b05..fd2f9541eb 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -402,6 +402,8 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_spm_set_dest_buffer_fn = AMD::hsa_amd_spm_set_dest_buffer; amd_ext_api.hsa_amd_portable_export_dmabuf_fn = AMD::hsa_amd_portable_export_dmabuf; amd_ext_api.hsa_amd_portable_close_dmabuf_fn = AMD::hsa_amd_portable_close_dmabuf; + amd_ext_api.hsa_amd_vmem_address_reserve_fn = AMD::hsa_amd_vmem_address_reserve; + amd_ext_api.hsa_amd_vmem_address_free_fn = AMD::hsa_amd_vmem_address_free; } void LoadInitialHsaApiTable() { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index 09c5976e42..5752d35695 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -95,6 +95,11 @@ struct ValidityError { enum { value = ValidityError::value }; }; +#define IS_TRUE(var) \ + do { \ + if ((var) != true) return HSA_STATUS_ERROR_INVALID_ARGUMENT; \ + } while (false) + #define IS_BAD_PTR(ptr) \ do { \ if ((ptr) == NULL) return HSA_STATUS_ERROR_INVALID_ARGUMENT; \ @@ -1197,5 +1202,23 @@ hsa_status_t hsa_amd_portable_close_dmabuf(int dmabuf) { CATCH; } +hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t address, + uint64_t flags) { + TRY; + IS_OPEN(); + IS_ZERO(size); + IS_TRUE(core::Runtime::runtime_singleton_->VirtualMemApiSupported()); + return core::Runtime::runtime_singleton_->VMemoryAddressReserve(va, size, address, flags); + CATCH; +} + +hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size) { + TRY; + IS_OPEN(); + IS_BAD_PTR(va); + IS_ZERO(size); + return core::Runtime::runtime_singleton_->VMemoryAddressFree(va, size); + CATCH; +} } // namespace amd } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp index 83239b7bbd..0572b69c22 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -2406,5 +2406,45 @@ hsa_status_t Runtime::DmaBufClose(int dmabuf) { #endif } +hsa_status_t Runtime::VMemoryAddressReserve(void** va, size_t size, uint64_t address, + uint64_t flags) { + void* addr = (void*)address; + HsaMemFlags memFlags = {0}; + + + ScopedAcquire lock(&memory_lock_); + + /* Try to reserving the VA requested by user */ + if (hsaKmtAllocMemory(0, size, memFlags, &addr) != HSAKMT_STATUS_SUCCESS) { + memFlags.ui32.FixedAddress = 0; + /* Could not reserved VA requested, allocate alternate VA */ + if (hsaKmtAllocMemory(0, size, memFlags, &addr) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + reserved_address_map_[addr] = AddressHandle(size); + *va = addr; + return HSA_STATUS_SUCCESS; +} + +hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) { + ScopedAcquire lock(&memory_lock_); + std::map::iterator it = reserved_address_map_.find(va); + + if (it == reserved_address_map_.end()) { + debug_warning(false && "Can't find address in reserved address"); + return HSA_STATUS_ERROR_INVALID_ALLOCATION; + } + + if (size != it->second.size) return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + if (it->second.use_count > 0) return HSA_STATUS_ERROR_RESOURCE_FREE; + + if (hsaKmtFreeMemory(va, size) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + reserved_address_map_.erase(it); + return HSA_STATUS_SUCCESS; +} + } // namespace core } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def b/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def index bd74f30661..7496551a98 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def +++ b/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def @@ -233,6 +233,8 @@ global: hsa_amd_spm_set_dest_buffer; hsa_amd_portable_export_dmabuf; hsa_amd_portable_close_dmabuf; + hsa_amd_vmem_address_reserve; + hsa_amd_vmem_address_free; local: *; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h index 3c613dc393..32cfa3c0fc 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -194,6 +194,8 @@ struct AmdExtTable { decltype(hsa_amd_queue_cu_get_mask)* hsa_amd_queue_cu_get_mask_fn; decltype(hsa_amd_portable_export_dmabuf)* hsa_amd_portable_export_dmabuf_fn; decltype(hsa_amd_portable_close_dmabuf)* hsa_amd_portable_close_dmabuf_fn; + decltype(hsa_amd_vmem_address_reserve)* hsa_amd_vmem_address_reserve_fn; + decltype(hsa_amd_vmem_address_free)* hsa_amd_vmem_address_free_fn; }; // Table to export HSA Core Runtime Apis diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h index 22e64860fb..15f128eab1 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -2702,6 +2702,48 @@ hsa_status_t hsa_amd_portable_export_dmabuf(const void* ptr, size_t size, int* d */ hsa_status_t hsa_amd_portable_close_dmabuf(int dmabuf); +/* + * @brief Allocate a reserved address range + * + * Reserve a virtual address range. The size must be a multiple of the system page size. + * If it is not possible to allocate the address specified by @p address, then @p va will be + * a different address range. + * Address range should be released by calling hsa_amd_vmem_address_free. + * + * @param[out] va virtual address allocated + * @param[in] size of address range requested + * @param[in] address requested + * @param[in] flags currently unsupported + * + * @retval ::HSA_STATUS_SUCCESS Address range allocated successfully + * + * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been + * initialized. + * + * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Insufficient resources to allocate an address + * range of this size. + */ +hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t address, + uint64_t flags); + +/* + * @brief Free a reserved address range + * + * Free a previously allocated address range. The size must match the size of a previously + * allocated address range. + * + * @param[out] va virtual address to be freed + * @param[in] size of address range + * + * @retval ::HSA_STATUS_SUCCESS Address range released successfully + * + * @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION Invalid va specified + * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT Invalid size specified + * @retval ::HSA_STATUS_ERROR_RESOURCE_FREE Address range is still in use + * @retval ::HSA_STATUS_ERROR Internal unexpected error + */ +hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size); + #ifdef __cplusplus } // end extern "C" block #endif