Support memory handles

Add support for creating and releasing memory handles. Memory
handles are memory allocations on device memory without a virtual
address.

This is part of patch series for Virtual Memory API.

Change-Id: I5dfb162eb1661621cce171b2870a3c93b24d840e

runtime/hsa-runtime/core/common/hsa_table_interface.cpp

@@ -1240,6 +1240,16 @@ hsa_status_t HSA_API hsa_amd_vmem_address_free(void* ptr, size_t size) {
   return amdExtTable->hsa_amd_vmem_address_free_fn(ptr, size);
 }
 
+hsa_status_t HSA_API hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t pool, size_t size,
+                                                hsa_amd_memory_type_t type, uint64_t flags,
+                                                hsa_amd_vmem_alloc_handle_t* memory_handle) {
+  return amdExtTable->hsa_amd_vmem_handle_create_fn(pool, size, type, flags, memory_handle);
+}
+
+hsa_status_t HSA_API hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle) {
+  return amdExtTable->hsa_amd_vmem_handle_release_fn(memory_handle);
+}
+
 // Tools only table interfaces.
 namespace rocr {
 

runtime/hsa-runtime/core/inc/amd_memory_region.h

@@ -175,6 +175,8 @@ class MemoryRegion : public core::MemoryRegion {
 
   __forceinline bool extended_scope_fine_grain() const { return extended_scope_fine_grain_; }
 
+  __forceinline size_t GetPageSize() const { return kPageSize_; }
+
  private:
   const HsaMemoryProperties mem_props_;
 

runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h

@@ -304,6 +304,15 @@ hsa_status_t hsa_amd_vmem_address_reserve(void** ptr, size_t size, uint64_t addr
 
 // Mirrors Amd Extension Apis
 hsa_status_t hsa_amd_vmem_address_free(void* ptr, size_t size);
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t pool, size_t size,
+                                        hsa_amd_memory_type_t type, uint64_t flags,
+                                        hsa_amd_vmem_alloc_handle_t* memory_handle);
+
+// Mirrors Amd Extension Apis
+hsa_status_t hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle);
+
 }  // namespace amd
 }  // namespace rocr
 

runtime/hsa-runtime/core/inc/memory_region.h

@@ -92,6 +92,8 @@ class MemoryRegion : public Checked<0x9C961F19EE175BB3> {
     AllocateNonPaged = (1 << 4),    // Non-paged system memory (AllocateIPC alias)
     AllocatePCIeRW = (1 << 5),      // Enforce pseudo fine grain/RW memory
     AllocateAsan = (1 << 6),        // ASAN - First page of allocation remapped to system memory
+    AllocatePinned = (1 << 7),      // Currently treating Pinned memory as NoSubstitute
+    AllocateMemoryOnly = (1 << 8),  // Memory only handle from thunk, no virtual address
   };
 
   typedef uint32_t AllocateFlags;

runtime/hsa-runtime/core/inc/runtime.h

@@ -357,6 +357,13 @@ class Runtime {
   hsa_status_t VMemoryAddressReserve(void** ptr, size_t size, uint64_t address, uint64_t flags);
 
   hsa_status_t VMemoryAddressFree(void* ptr, size_t size);
+
+  hsa_status_t VMemoryHandleCreate(const MemoryRegion* region, size_t size,
+                                   MemoryRegion::AllocateFlags alloc_flags,
+                                   uint64_t flags, hsa_amd_vmem_alloc_handle_t* memoryHandle);
+
+  hsa_status_t VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryHandle);
+
   const std::vector<Agent*>& cpu_agents() { return cpu_agents_; }
 
   const std::vector<Agent*>& gpu_agents() { return gpu_agents_; }
@@ -677,6 +684,8 @@ class Runtime {
 
   bool virtual_mem_api_supported_;
 
+  typedef void* ThunkHandle;
+
   struct AddressHandle {
     AddressHandle() : size(0), use_count(0) {}
     AddressHandle(size_t size) : size(size), use_count(0) {}
@@ -686,6 +695,35 @@ class Runtime {
   };
   std::map<const void*, AddressHandle> reserved_address_map_;  // Indexed by VA
 
+  struct MemoryHandle {
+    MemoryHandle() : region(NULL), size(0), ref_count(0), thunk_handle(NULL), alloc_flag(0) {}
+    MemoryHandle(const MemoryRegion* region, size_t size, uint64_t flags_unused,
+                 ThunkHandle thunk_handle, MemoryRegion::AllocateFlags alloc_flag)
+        : region(region),
+          size(size),
+          ref_count(1),
+          use_count(0),
+          thunk_handle(thunk_handle),
+          alloc_flag(alloc_flag) {}
+
+    static __forceinline hsa_amd_vmem_alloc_handle_t Convert(void* handle) {
+      hsa_amd_vmem_alloc_handle_t ret_handle = {
+          static_cast<uint64_t>(reinterpret_cast<uintptr_t>(handle))};
+      return ret_handle;
+    }
+
+    __forceinline core::Agent* agentOwner() const { return region->owner(); }
+
+    const MemoryRegion* region;
+    size_t size;
+    int ref_count;
+    int use_count;
+    ThunkHandle thunk_handle;  // handle returned by hsaKmtAllocMemory(NoAddress = 1)
+    MemoryRegion::AllocateFlags alloc_flag;
+  };
+
+  std::map<ThunkHandle, MemoryHandle> memory_handle_map_;
+
   // Frees runtime memory when the runtime library is unloaded if safe to do so.
   // Failure to release the runtime indicates an incorrect application but is
   // common (example: calls library routines at process exit).

runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp

@@ -404,6 +404,8 @@ void HsaApiTable::UpdateAmdExts() {
   amd_ext_api.hsa_amd_portable_close_dmabuf_fn = AMD::hsa_amd_portable_close_dmabuf;
   amd_ext_api.hsa_amd_vmem_address_reserve_fn = AMD::hsa_amd_vmem_address_reserve;
   amd_ext_api.hsa_amd_vmem_address_free_fn = AMD::hsa_amd_vmem_address_free;
+  amd_ext_api.hsa_amd_vmem_handle_create_fn = AMD::hsa_amd_vmem_handle_create;
+  amd_ext_api.hsa_amd_vmem_handle_release_fn = AMD::hsa_amd_vmem_handle_release;
 }
 
 void LoadInitialHsaApiTable() {

runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp

@@ -1220,5 +1220,39 @@ hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size) {
   return core::Runtime::runtime_singleton_->VMemoryAddressFree(va, size);
   CATCH;
 }
+
+hsa_status_t hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t memory_pool, size_t size,
+                                        hsa_amd_memory_type_t type, uint64_t flags,
+                                        hsa_amd_vmem_alloc_handle_t* memory_handle) {
+  TRY;
+  IS_OPEN();
+  IS_ZERO(size);
+  IS_TRUE(core::Runtime::runtime_singleton_->VirtualMemApiSupported());
+
+  if (type != MEMORY_TYPE_NONE && type != MEMORY_TYPE_PINNED)
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+
+  hsa_region_t region = {memory_pool.handle};
+  const core::MemoryRegion* mem_region = core::MemoryRegion::Convert(region);
+
+  if (mem_region == NULL || !mem_region->IsValid()) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  MemoryRegion::AllocateFlags alloc_flag = core::MemoryRegion::AllocateMemoryOnly;
+  if (type == MEMORY_TYPE_PINNED) alloc_flag |= core::MemoryRegion::AllocatePinned;
+
+  return core::Runtime::runtime_singleton_->VMemoryHandleCreate(mem_region, size, alloc_flag, flags,
+                                                                memory_handle);
+  CATCH;
+}
+
+hsa_status_t hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle) {
+  TRY;
+  IS_OPEN();
+  return core::Runtime::runtime_singleton_->VMemoryHandleRelease(memory_handle);
+  CATCH;
+}
+
 }   //  namespace amd
 }   //  namespace rocr

runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -2446,5 +2446,51 @@ hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) {
   return HSA_STATUS_SUCCESS;
 }
 
+hsa_status_t Runtime::VMemoryHandleCreate(const MemoryRegion* region, size_t size,
+                                          MemoryRegion::AllocateFlags alloc_flags,
+                                          uint64_t flags_unused,
+                                          hsa_amd_vmem_alloc_handle_t* memoryOnlyHandle) {
+  const AMD::MemoryRegion* memRegion = static_cast<const AMD::MemoryRegion*>(region);
+  if (!memRegion->IsLocalMemory()) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+
+  if (!IsMultipleOf(size, memRegion->GetPageSize()))
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+
+  ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
+  void* thunk_handle;
+  hsa_status_t status = region->Allocate(size, alloc_flags, &thunk_handle);
+  if (status == HSA_STATUS_SUCCESS) {
+    memory_handle_map_[thunk_handle] =
+        MemoryHandle(region, size, flags_unused, thunk_handle, alloc_flags);
+    *memoryOnlyHandle = MemoryHandle::Convert(thunk_handle);
+  }
+  return status;
+}
+
+hsa_status_t Runtime::VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryOnlyHandle) {
+  ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
+  auto memoryHandleIt = memory_handle_map_.find(reinterpret_cast<void*>(memoryOnlyHandle.handle));
+
+  if (memoryHandleIt == memory_handle_map_.end()) {
+    debug_warning(false && "Can't find memory handle");
+    return HSA_STATUS_ERROR_INVALID_ALLOCATION;
+  }
+
+  if (!memoryHandleIt->second.ref_count) return HSA_STATUS_ERROR_INVALID_ALLOCATION;
+
+  if (--(memoryHandleIt->second.ref_count) == 0) {
+    // From documentation, the handle can be released while there are still outstanding mappings. If
+    // there are outstanding mappings, then we just decrement the ref count and exit. We will free
+    // this handle when the last MappedHandle is deleted
+    // and use_count == 0 and ref_count == 0.
+
+    if (memoryHandleIt->second.use_count > 0) return HSA_STATUS_SUCCESS;
+
+    memoryHandleIt->second.region->Free(memoryHandleIt->first, memoryHandleIt->second.size);
+    memory_handle_map_.erase(memoryHandleIt);
+  }
+  return HSA_STATUS_SUCCESS;
+}
+
 }  // namespace core
 }  // namespace rocr

runtime/hsa-runtime/hsacore.so.def

@@ -235,6 +235,8 @@ global:
 	hsa_amd_portable_close_dmabuf;
 	hsa_amd_vmem_address_reserve;
 	hsa_amd_vmem_address_free;
+	hsa_amd_vmem_handle_create;
+	hsa_amd_vmem_handle_release;
 
 local:
     *;

runtime/hsa-runtime/inc/hsa_api_trace.h

@@ -196,6 +196,8 @@ struct AmdExtTable {
   decltype(hsa_amd_portable_close_dmabuf)* hsa_amd_portable_close_dmabuf_fn;
   decltype(hsa_amd_vmem_address_reserve)* hsa_amd_vmem_address_reserve_fn;
   decltype(hsa_amd_vmem_address_free)* hsa_amd_vmem_address_free_fn;
+  decltype(hsa_amd_vmem_handle_create)* hsa_amd_vmem_handle_create_fn;
+  decltype(hsa_amd_vmem_handle_release)* hsa_amd_vmem_handle_release_fn;
 };
 
 // Table to export HSA Core Runtime Apis

runtime/hsa-runtime/inc/hsa_ext_amd.h

@@ -2744,6 +2744,62 @@ hsa_status_t hsa_amd_vmem_address_reserve(void** va, size_t size, uint64_t addre
  */
 hsa_status_t hsa_amd_vmem_address_free(void* va, size_t size);
 
+/**
+ * @brief Struct containing an opaque handle to a memory allocation handle
+ */
+typedef struct hsa_amd_vmem_alloc_handle_s {
+  /**
+   * Opaque handle. Two handles reference the same object of the enclosing type
+   * if and only if they are equal.
+   */
+  uint64_t handle;
+} hsa_amd_vmem_alloc_handle_t;
+
+typedef enum {
+  MEMORY_TYPE_NONE,
+  MEMORY_TYPE_PINNED,
+} hsa_amd_memory_type_t;
+
+/*
+ * @brief Create a virtual memory handle
+ *
+ * Create a virtual memory handle within this pool
+ * @p size must be a aligned to allocation granule size for this memory pool, see
+ * HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE
+ * To minimize internal memory fragmentation, align the size to the recommended allocation granule
+ * size, see HSA_AMD_REGION_INFO_RUNTIME_ALLOC_RECOMMENDED_GRANULE
+ *
+ * @param[in] pool memory to use
+ * @param[in] size of the memory allocation
+ * @param[in] type of memory
+ * @param[in] flags - currently unsupported
+ * @param[out] memory_handle - handle for the allocation
+ *
+ * @retval ::HSA_STATUS_SUCCESS memory allocated successfully
+ *
+ * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been initialized.
+ *
+ * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT Invalid arguments
+ *
+ * @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION This memory pool does not support allocations
+ *
+ * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Insufficient resources to allocate this memory
+ */
+hsa_status_t hsa_amd_vmem_handle_create(hsa_amd_memory_pool_t pool, size_t size,
+                                        hsa_amd_memory_type_t type, uint64_t flags,
+                                        hsa_amd_vmem_alloc_handle_t* memory_handle);
+
+/*
+ * @brief Release a virtual memory handle
+ *
+ * @param[in] memory handle that was previously allocated
+ *
+ * @retval ::HSA_STATUS_SUCCESS Address range allocated successfully
+ *
+ * @retval ::HSA_STATUS_ERROR_INVALID_ALLOCATION Invalid memory handle
+ */
+hsa_status_t hsa_amd_vmem_handle_release(hsa_amd_vmem_alloc_handle_t memory_handle);
+
 #ifdef __cplusplus
 }  // end extern "C" block
 #endif