SWDEV-311270 - Add IPC support for memory pools

Initial implementation for hipMemPoolExportToShareableHandle,
hipMemPoolImportFromShareableHandle,
hipMemPoolExportPointer and hipMemPoolImportPointer

Change-Id: I0ebdc48e9163b394ded560adca6c38bbc5aee7d1


[ROCm/clr commit: 1a0c3e4dc4]

projects/clr/hipamd/src/hip_mempool.cpp

@@ -236,7 +236,7 @@ hipError_t hipMemPoolCreate(hipMemPool_t* mem_pool, const hipMemPoolProps* pool_
     HIP_RETURN(hipErrorInvalidValue);
   }
   auto device = g_devices[pool_props->location.id];
-  auto pool = new hip::MemoryPool(device);
+  auto pool = new hip::MemoryPool(device, pool_props->handleTypes != hipMemHandleTypeNone);
   if (pool == nullptr) {
     HIP_RETURN(hipErrorInvalidValue);
   }
@@ -298,7 +298,15 @@ hipError_t hipMemPoolExportToShareableHandle(
   if (mem_pool == nullptr || shared_handle == nullptr || flags == -1) {
     HIP_RETURN(hipErrorInvalidValue);
   }
-  HIP_RETURN(hipErrorNotSupported);
+
+  auto mpool = reinterpret_cast<hip::MemoryPool*>(mem_pool);
+  auto handle = mpool->Export();
+  if (!handle) {
+    HIP_RETURN(hipErrorInvalidValue);
+  }
+  *reinterpret_cast<amd::Os::FileDesc*>(shared_handle) = handle;
+
+  HIP_RETURN(hipSuccess);
 }
 
 // ================================================================================================
@@ -311,7 +319,26 @@ hipError_t hipMemPoolImportFromShareableHandle(
   if (mem_pool == nullptr || shared_handle == nullptr || flags == -1) {
     HIP_RETURN(hipErrorInvalidValue);
   }
-  HIP_RETURN(hipErrorNotSupported);
+
+  auto device = g_devices[0];
+  auto pool = new hip::MemoryPool(device);
+  if (pool == nullptr) {
+    HIP_RETURN(hipErrorOutOfMemory);
+  }
+  // Note: The interface casts the integer value of file handle under Linux into void*,
+  // but compiler may not allow to cast it back. Hence, make a cast with a union...
+  union {
+    amd::Os::FileDesc desc;
+    void* ptr;
+  } handle;
+  handle.ptr = shared_handle;
+  if (!pool->Import(handle.desc)) {
+    pool->release();
+    HIP_RETURN(hipErrorOutOfMemory);
+  }
+  *mem_pool = reinterpret_cast<hipMemPool_t>(pool);
+
+  HIP_RETURN(hipSuccess);
 }
 
 // ================================================================================================
@@ -320,7 +347,22 @@ hipError_t hipMemPoolExportPointer(hipMemPoolPtrExportData* export_data, void* p
   if (export_data == nullptr || ptr == nullptr) {
     HIP_RETURN(hipErrorInvalidValue);
   }
-  HIP_RETURN(hipErrorNotSupported);
+
+  size_t offset = 0;
+  auto memory = getMemoryObject(ptr, offset);
+  if (memory != nullptr) {
+    auto id = memory->getUserData().deviceId;
+    // Note: export_data must point to 64 bytes of shared memory
+    auto shared = reinterpret_cast<hip::SharedMemPointer*>(export_data);
+
+    if (!g_devices[id]->devices()[0]->IpcCreate(ptr,
+      &shared->size_, &shared->handle_[0], &shared->offset_)) {
+      HIP_RETURN(hipErrorOutOfMemory);
+    }
+  } else {
+    HIP_RETURN(hipErrorOutOfMemory);
+  }
+  HIP_RETURN(hipSuccess);
 }
 
 // ================================================================================================
@@ -332,6 +374,15 @@ hipError_t hipMemPoolImportPointer(
   if (mem_pool == nullptr || export_data == nullptr || ptr == nullptr) {
     HIP_RETURN(hipErrorInvalidValue);
   }
-
-  HIP_RETURN(hipErrorNotSupported);
+  auto mpool = reinterpret_cast<hip::MemoryPool*>(mem_pool);
+  auto shared = reinterpret_cast<hip::SharedMemPointer*>(export_data);
+  if (!mpool->Device()->devices()[0]->IpcAttach(
+      &shared->handle_[0], shared->size_, shared->offset_, 0, ptr)) {
+    HIP_RETURN(hipErrorOutOfMemory);
+  }
+  size_t offset = 0;
+  auto memory = getMemoryObject(*ptr, offset);
+  mpool->AddBusyMemory(memory);
+  mpool->retain();
+  HIP_RETURN(hipSuccess);
 }

projects/clr/hipamd/src/hip_mempool_impl.cpp

@@ -168,8 +168,8 @@ void* MemoryPool::AllocateMemory(size_t size, hip::Stream* stream, void* dptr) {
     if (dev_info.maxMemAllocSize_ < size) {
       return nullptr;
     }
-
-    dev_ptr = amd::SvmBuffer::malloc(*context, 0, size, dev_info.memBaseAddrAlign_, nullptr);
+    cl_svm_mem_flags flags = (state_.interprocess_) ? ROCCLR_MEM_INTERPROCESS : 0;
+    dev_ptr = amd::SvmBuffer::malloc(*context, flags, size, dev_info.memBaseAddrAlign_, nullptr);
     if (dev_ptr == nullptr) {
       size_t free = 0, total =0;
       hipError_t err = hipMemGetInfo(&free, &total);
@@ -425,10 +425,56 @@ void MemoryPool::GetAccess(hip::Device* device, hipMemAccessFlags* flags) {
   }
 }
 
+// ================================================================================================
 void MemoryPool::FreeAllMemory(hip::Stream* stream) {
   while (!busy_heap_.Allocations().empty()) {
     FreeMemory(busy_heap_.Allocations().begin()->first, stream);
   }
 }
 
+// ================================================================================================
+amd::Os::FileDesc MemoryPool::Export() {
+  amd::ScopedLock lock(lock_pool_ops_);
+  if (shared_ != nullptr) {
+    return shared_->handle_;
+  }
+
+  constexpr uint32_t kFileNameSize = 20;
+  char file_name[kFileNameSize];
+  // Generate a unique name from the mempool pointer
+  // Note: Windows can accept an unnamed allocation
+  snprintf(file_name, kFileNameSize, "%p", this);
+  amd::Os::FileDesc handle{};
+  shared_ = reinterpret_cast<SharedMemPool*>(amd::Os::CreateIpcMemory(
+      file_name, sizeof(SharedMemPool), &handle));
+  if (shared_ != nullptr) {
+    shared_->handle_ = handle;
+    shared_->state_ = state_.value_;
+    shared_->access_size_ = 0;
+    memset(shared_->access_, 0, sizeof(SharedAccess) * kMaxMgpuAccess);
+    assert((access_map_.size() <= kMaxMgpuAccess) && "Can't support more GPU(s) in shared access" );
+    for (auto it : access_map_) {
+      shared_->access_[shared_->access_size_] = SharedAccess{it.first->deviceId(), it.second};
+      shared_->access_size_++;
+    }
+  }
+  return handle;
+}
+
+// ================================================================================================
+bool MemoryPool::Import(amd::Os::FileDesc handle) {
+  amd::ScopedLock lock(lock_pool_ops_);
+  bool result = false;
+  auto shared = reinterpret_cast<SharedMemPool*>(
+      amd::Os::OpenIpcMemory(nullptr, handle, sizeof(SharedMemPool)));
+
+  if (shared != nullptr) {
+    state_.value_ = shared->state_;
+    for (uint32_t i = 0; i < shared->access_size_; ++i) {
+      access_map_[g_devices[shared->access_[i].device_id_]] = shared->access_[i].flags_;
+    }
+    result = true;
+  }
+  return result;
+}
 }

projects/clr/hipamd/src/hip_mempool_impl.hpp

@@ -31,6 +31,12 @@ namespace hip {
 class Device;
 class Stream;
 
+struct SharedMemPointer {
+  size_t offset_;
+  size_t size_;
+  char handle_[IHIP_IPC_MEM_HANDLE_SIZE];
+};
+
 struct MemoryTimestamp {
   MemoryTimestamp(hip::Stream* stream, hip::Event* event = nullptr): event_(event) {
     if (stream != nullptr) {
@@ -160,16 +166,34 @@ private:
 /// hipMemPoolReuseAllowOpportunistic option will validate if HIP event,
 /// associated with memory is done, then reuse can be performed.
 class MemoryPool : public amd::ReferenceCountedObject {
-public:
-  MemoryPool(hip::Device* device):
-    busy_heap_(device),
-    free_heap_(device),
-    lock_pool_ops_("Pool operations", true), device_(device) {
-      device_->AddMemoryPool(this);
-      state_.event_dependencies_ = 1;
-      state_.opportunistic_ = 1;
-      state_.internal_dependencies_ = 1;
-    }
+ public:
+  struct SharedAccess {
+    int device_id_;             //!< Device ID for access with a specified shared resource
+    hipMemAccessFlags flags_;   //!< Flags which define access type
+  };
+
+  static constexpr uint32_t kMaxMgpuAccess = 32;
+  struct SharedMemPool {
+    amd::Os::FileDesc handle_;            //!< File descriptor for shared memory
+    uint32_t state_;                      //!< Memory pool state
+    uint32_t access_size_;                //!< The number of entries in access array
+    SharedAccess access_[kMaxMgpuAccess]; //!< The list of devices for access
+  };
+
+  MemoryPool(hip::Device* device, bool interprocess = false)
+      : busy_heap_(device),
+        free_heap_(device),
+        lock_pool_ops_("Pool operations", true),
+        device_(device),
+        shared_(nullptr) {
+    device_->AddMemoryPool(this);
+    state_.value_ = 0;
+    state_.event_dependencies_ = 1;
+    state_.opportunistic_ = 1;
+    state_.internal_dependencies_ = 1;
+    state_.interprocess_ = interprocess;
+  }
+
   virtual ~MemoryPool() {
     if (!busy_heap_.IsEmpty()) {
       LogError("Shouldn't destroy pool with busy allocations!");
@@ -177,6 +201,10 @@ public:
     ReleaseAllMemory();
     // Remove memory pool from the list of all pool on the current device
     device_->RemoveMemoryPool(this);
+    if (shared_ != nullptr) {
+      // Note: The app supposes to close the handle... Double close in Windows will cause a crash
+      amd::Os::CloseIpcMemory(0, shared_, sizeof(SharedMemPool));
+    }
   }
 
   /// The same stream can reuse memory without HIP event validation
@@ -186,9 +214,7 @@ public:
   bool FreeMemory(amd::Memory* memory, hip::Stream* stream);
 
   /// Check if memory is active and belongs to the busy heap
-  bool IsBusyMemory(amd::Memory* memory) const {
-    return busy_heap_.IsActiveMemory(memory);
-  }
+  bool IsBusyMemory(amd::Memory* memory) const { return busy_heap_.IsActiveMemory(memory); }
 
   /// Releases all allocations from free_heap_. It can be called on Stream or Device synchronization
   /// @note The caller must make sure it's safe to release memory
@@ -200,6 +226,10 @@ public:
   /// Releases all allocations in MemoryPool
   void ReleaseAllMemory();
 
+  /// Place the allocated memory into the busy heap
+  void AddBusyMemory(amd::Memory* memory) {
+    busy_heap_.AddMemory(memory, nullptr);
+  }
   /// Trims the pool until it has only min_bytes_to_hold
   void TrimTo(size_t min_bytes_to_hold);
 
@@ -221,6 +251,12 @@ public:
   /// Frees all busy memory
   void FreeAllMemory(hip::Stream* stream = nullptr);
 
+  /// Exports memory pool into an OS specific handle
+  amd::Os::FileDesc Export();
+
+  /// Imports memory pool from an OS specific handle
+  bool Import(amd::Os::FileDesc handle);
+
   /// Accessors for the pool state
   bool EventDependencies() const { return (state_.event_dependencies_) ? true : false; }
   bool Opportunistic() const { return (state_.opportunistic_) ? true : false; }
@@ -233,15 +269,22 @@ private:
 
   Heap busy_heap_;    //!< Heap of busy allocations
   Heap free_heap_;    //!< Heap of freed allocations
-  struct {
-    uint32_t event_dependencies_ : 1;     //!< Event dependencies tracking is enabled
-    uint32_t opportunistic_ : 1;          //!< HIP event check is enabled
-    uint32_t internal_dependencies_ : 1;  //!< Runtime adds internal events to handle memory dependencies
+  union {
+    struct {
+      uint32_t event_dependencies_ : 1;     //!< Event dependencies tracking is enabled
+      uint32_t opportunistic_ : 1;          //!< HIP event check is enabled
+      uint32_t internal_dependencies_ : 1;  //!< Runtime adds internal events to handle memory
+                                            //!< dependencies
+      uint32_t interprocess_ : 1;  //!< Memory pool can be used in interprocess communications
+    };
+    uint32_t value_;
   } state_;
 
   amd::Monitor  lock_pool_ops_;  //!< Access to the pool must be lock protected
   std::map<hip::Device*, hipMemAccessFlags> access_map_;  //!< Map of access to the pool from devices
   hip::Device*  device_;    //!< Hip device the heap will reside
+  SharedMemPool* shared_;   //!< Pointer to shared memory for IPC
 };
 
+
 } // Mamespace hip