Support Virtual Address reservations

Add support for reserving virtual address ranges. Virtual address
ranges are addresses without any memory backing. These address ranges
need to be mapped to memory handles later.

This is part of patch series for Virtual Memory API.

Change-Id: I5d066e7421d6896f933f524312afc230a13d594e


[ROCm/ROCR-Runtime commit: 1085311f1a]

projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp

@@ -1231,6 +1231,15 @@ hsa_status_t HSA_API hsa_amd_portable_close_dmabuf(int dmabuf) {
   return amdExtTable->hsa_amd_portable_close_dmabuf_fn(dmabuf);
 }
 
+hsa_status_t HSA_API hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t address,
+                                                  uint64_t flags) {
+  return amdExtTable->hsa_amd_vmem_address_reserve_fn(ptr, size, address, flags);
+}
+
+hsa_status_t HSA_API hsa_amd_vmem_address_free(void* ptr, size_t size) {
+  return amdExtTable->hsa_amd_vmem_address_free_fn(ptr, size);
+}
+
 // Tools only table interfaces.
 namespace rocr {
 

projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h

@@ -298,6 +298,12 @@ hsa_status_t HSA_API hsa_amd_portable_export_dmabuf(const void* ptr, size_t size
 // Mirrors Amd Extension Apis
 hsa_status_t HSA_API hsa_amd_portable_close_dmabuf(int dmabuf);
 
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t address,
+                                          uint64_t flags);
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_vmem_address_free(void* ptr, size_t size);
 }  // namespace amd
 }  // namespace rocr
 

projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h

@@ -354,6 +354,9 @@ class Runtime {
 
   hsa_status_t DmaBufClose(int dmabuf);
 
+  hsa_status_t VMemoryAddressReserve(void** ptr, size_t size, uint64_t address, uint64_t flags);
+
+  hsa_status_t VMemoryAddressFree(void* ptr, size_t size);
   const std::vector<Agent*>& cpu_agents() { return cpu_agents_; }
 
   const std::vector<Agent*>& gpu_agents() { return gpu_agents_; }
@@ -674,6 +677,15 @@ class Runtime {
 
   bool virtual_mem_api_supported_;
 
+  struct AddressHandle {
+    AddressHandle() : size(0), use_count(0) {}
+    AddressHandle(size_t size) : size(size), use_count(0) {}
+
+    size_t size;
+    int use_count;
+  };
+  std::map<const void*, AddressHandle> reserved_address_map_;  // Indexed by VA
+
   // Frees runtime memory when the runtime library is unloaded if safe to do so.
   // Failure to release the runtime indicates an incorrect application but is
   // common (example: calls library routines at process exit).

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp

@@ -402,6 +402,8 @@ void HsaApiTable::UpdateAmdExts() {
   amd_ext_api.hsa_amd_spm_set_dest_buffer_fn = AMD::hsa_amd_spm_set_dest_buffer;
   amd_ext_api.hsa_amd_portable_export_dmabuf_fn = AMD::hsa_amd_portable_export_dmabuf;
   amd_ext_api.hsa_amd_portable_close_dmabuf_fn = AMD::hsa_amd_portable_close_dmabuf;
+  amd_ext_api.hsa_amd_vmem_address_reserve_fn = AMD::hsa_amd_vmem_address_reserve;
+  amd_ext_api.hsa_amd_vmem_address_free_fn = AMD::hsa_amd_vmem_address_free;
 }
 
 void LoadInitialHsaApiTable() {

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp

@@ -95,6 +95,11 @@ struct ValidityError<const T*> {
   enum { value = ValidityError<T*>::value };
 };
 
+#define IS_TRUE(var)                                                                               \
+  do {                                                                                             \
+    if ((var) != true) return HSA_STATUS_ERROR_INVALID_ARGUMENT;                                   \
+  } while (false)
+
 #define IS_BAD_PTR(ptr)                                          \
   do {                                                           \
     if ((ptr) == NULL) return HSA_STATUS_ERROR_INVALID_ARGUMENT; \
@@ -1197,5 +1202,23 @@ hsa_status_t hsa_amd_portable_close_dmabuf(int dmabuf) {
   CATCH;
 }
 
+hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t address,
+                                          uint64_t flags) {
+  TRY;
+  IS_OPEN();
+  IS_ZERO(size);
+  IS_TRUE(core::Runtime::runtime_singleton_->VirtualMemApiSupported());
+  return core::Runtime::runtime_singleton_->VMemoryAddressReserve(va, size, address, flags);
+  CATCH;
+}
+
+hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size) {
+  TRY;
+  IS_OPEN();
+  IS_BAD_PTR(va);
+  IS_ZERO(size);
+  return core::Runtime::runtime_singleton_->VMemoryAddressFree(va, size);
+  CATCH;
+}
 }   //  namespace amd
 }   //  namespace rocr

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -2406,5 +2406,45 @@ hsa_status_t Runtime::DmaBufClose(int dmabuf) {
 #endif
 }
 
+hsa_status_t Runtime::VMemoryAddressReserve(void** va, size_t size, uint64_t address,
+                                            uint64_t flags) {
+  void* addr = (void*)address;
+  HsaMemFlags memFlags = {0};
+
+
+  ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
+
+  /* Try to reserving the VA requested by user */
+  if (hsaKmtAllocMemory(0, size, memFlags, &addr) != HSAKMT_STATUS_SUCCESS) {
+    memFlags.ui32.FixedAddress = 0;
+    /* Could not reserved VA requested, allocate alternate VA */
+    if (hsaKmtAllocMemory(0, size, memFlags, &addr) != HSAKMT_STATUS_SUCCESS)
+      return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  }
+
+  reserved_address_map_[addr] = AddressHandle(size);
+  *va = addr;
+  return HSA_STATUS_SUCCESS;
+}
+
+hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) {
+  ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
+  std::map<const void*, AddressHandle>::iterator it = reserved_address_map_.find(va);
+
+  if (it == reserved_address_map_.end()) {
+    debug_warning(false && "Can't find address in reserved address");
+    return HSA_STATUS_ERROR_INVALID_ALLOCATION;
+  }
+
+  if (size != it->second.size) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+
+  if (it->second.use_count > 0) return HSA_STATUS_ERROR_RESOURCE_FREE;
+
+  if (hsaKmtFreeMemory(va, size) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+
+  reserved_address_map_.erase(it);
+  return HSA_STATUS_SUCCESS;
+}
+
 }  // namespace core
 }  // namespace rocr

projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def

@@ -233,6 +233,8 @@ global:
 	hsa_amd_spm_set_dest_buffer;
 	hsa_amd_portable_export_dmabuf;
 	hsa_amd_portable_close_dmabuf;
+	hsa_amd_vmem_address_reserve;
+	hsa_amd_vmem_address_free;
 
 local:
     *;

projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h

@@ -194,6 +194,8 @@ struct AmdExtTable {
   decltype(hsa_amd_queue_cu_get_mask)* hsa_amd_queue_cu_get_mask_fn;
   decltype(hsa_amd_portable_export_dmabuf)* hsa_amd_portable_export_dmabuf_fn;
   decltype(hsa_amd_portable_close_dmabuf)* hsa_amd_portable_close_dmabuf_fn;
+  decltype(hsa_amd_vmem_address_reserve)* hsa_amd_vmem_address_reserve_fn;
+  decltype(hsa_amd_vmem_address_free)* hsa_amd_vmem_address_free_fn;
 };
 
 // Table to export HSA Core Runtime Apis

projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h

@@ -2702,6 +2702,48 @@ hsa_status_t hsa_amd_portable_export_dmabuf(const void* ptr, size_t size, int* d
  */
 hsa_status_t hsa_amd_portable_close_dmabuf(int dmabuf);
 
+/*
+ * @brief Allocate a reserved address range
+ *
+ * Reserve a virtual address range. The size must be a multiple of the system page size.
+ * If it is not possible to allocate the address specified by @p address, then @p va will be
+ * a different address range.
+ * Address range should be released by calling hsa_amd_vmem_address_free.
+ *
+ * @param[out] va virtual address allocated
+ * @param[in] size of address range requested
+ * @param[in] address requested
+ * @param[in] flags currently unsupported
+ *
+ * @retval ::HSA_STATUS_SUCCESS Address range allocated successfully
+ *
+ * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been
+ * initialized.
+ *
+ * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Insufficient resources to allocate an address
+ * range of this size.
+ */
+hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t address,
+                                          uint64_t flags);
+
+/*
+ * @brief Free a reserved address range
+ *
+ * Free a previously allocated address range. The size must match the size of a previously
+ * allocated address range.
+ *
+ * @param[out] va virtual address to be freed
+ * @param[in] size of address range
+ *
+ * @retval ::HSA_STATUS_SUCCESS Address range released successfully
+ *
+ * @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION Invalid va specified
+ * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT Invalid size specified
+ * @retval ::HSA_STATUS_ERROR_RESOURCE_FREE Address range is still in use
+ * @retval ::HSA_STATUS_ERROR Internal unexpected error
+ */
+hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size);
+
 #ifdef __cplusplus
 }  // end extern "C" block
 #endif