SWDEV-413997 - VMM IPC implementation for Linux.

Change-Id: Icfeb83ca51e96be35abb67a94d6e3e1a1ca5a934

hipamd/src/hip_vm.cpp

@@ -86,8 +86,8 @@ hipError_t hipMemCreate(hipMemGenericAllocationHandle_t* handle, size_t size,
     HIP_RETURN(hipErrorInvalidValue);
   }
 
-  // Currently only support non-IPC allocations
-  if (prop->requestedHandleType != hipMemHandleTypeNone) {
+  if (prop->requestedHandleType != hipMemHandleTypeNone
+      && prop->requestedHandleType != hipMemHandleTypePosixFileDescriptor) {
     HIP_RETURN(hipErrorNotSupported);
   }
 
@@ -139,11 +139,29 @@ hipError_t hipMemExportToShareableHandle(void* shareableHandle,
                                          unsigned long long flags) {
   HIP_INIT_API(hipMemExportToShareableHandle, shareableHandle, handle, handleType, flags);
 
-  if (flags != 0 || handle == nullptr || shareableHandle == nullptr) {
+  if (flags != 0 || handle == nullptr) {
     HIP_RETURN(hipErrorInvalidValue);
   }
 
-  HIP_RETURN(hipErrorNotSupported);
+  hip::GenericAllocation* ga = reinterpret_cast<hip::GenericAllocation*>(handle);
+  if (ga == nullptr) {
+    LogError("Generic Allocation is nullptr");
+    HIP_RETURN(hipErrorNotInitialized);
+  }
+
+  if (ga->GetProperties().requestedHandleType != handleType) {
+    LogPrintfError("HandleType mismatch memoryHandleType: %d, requestedHandleType: %d",
+                    ga->GetProperties().requestedHandleType, handleType);
+    HIP_RETURN(hipErrorInvalidValue);
+  }
+
+  if (!ga->asAmdMemory().getContext().devices()[0]->ExportShareableVMMHandle(
+        ga->asAmdMemory().getUserData().hsa_handle, flags, shareableHandle)) {
+    LogPrintfError("Exporting Handle failed with flags: %d", flags);
+    HIP_RETURN(hipErrorInvalidValue);
+  }
+
+  HIP_RETURN(hipSuccess);
 }
 
 hipError_t hipMemGetAccess(unsigned long long* flags, const hipMemLocation* location, void* ptr) {
@@ -205,7 +223,33 @@ hipError_t hipMemImportFromShareableHandle(hipMemGenericAllocationHandle_t* hand
     HIP_RETURN(hipErrorInvalidValue);
   }
 
-  HIP_RETURN(hipErrorNotSupported);
+  amd::Device* device = hip::getCurrentDevice()->devices()[0];
+  amd::Memory* phys_mem_obj = new (device->context()) amd::Buffer(device->context(),
+                                ROCCLR_MEM_PHYMEM | ROCCLR_MEM_INTERPROCESS, 0, osHandle);
+
+  if (phys_mem_obj == nullptr) {
+    LogError("failed to new a va range curr_mem_obj object!");
+    HIP_RETURN(hipErrorInvalidValue);
+  }
+
+  if (!phys_mem_obj->create(nullptr, false)) {
+    LogError("failed to create a va range mem object");
+    phys_mem_obj->release();
+    HIP_RETURN(hipErrorInvalidValue);
+  }
+
+  hipMemAllocationProp prop {};
+  prop.type = hipMemAllocationTypePinned;
+  prop.location.type = hipMemLocationTypeDevice;
+  prop.location.id = hip::getCurrentDevice()->deviceId();
+
+  phys_mem_obj->getUserData().deviceId = hip::getCurrentDevice()->deviceId();
+  phys_mem_obj->getUserData().data = new hip::GenericAllocation(*phys_mem_obj, 0, prop);
+  *handle = reinterpret_cast<hipMemGenericAllocationHandle_t>(phys_mem_obj->getUserData().data);
+
+  amd::MemObjMap::RemoveMemObj(phys_mem_obj->getSvmPtr());
+
+  HIP_RETURN(hipSuccess);
 }
 
 hipError_t hipMemMap(void* ptr, size_t size, size_t offset, hipMemGenericAllocationHandle_t handle,

rocclr/device/device.hpp

@@ -1859,6 +1859,29 @@ class Device : public RuntimeObject {
    */
   virtual void virtualFree(void* addr) = 0;
 
+  /**
+   * Export Shareable VMM Handle to FD
+   *
+   * @param hsa_handle hsa_handle which has the phys_mem info.
+   * @param flags any flags to be passed
+   * @param shareableHandle exported handle, points to fdesc.
+   */
+  virtual bool ExportShareableVMMHandle(uint64_t hsa_handle, int flags, void* shareableHandle) {
+    ShouldNotCallThis();
+    return false;
+  }
+
+  /**
+   * Import FD from Shareable VMM Handle
+   *
+   * @param osHandle os handle/fdesc
+   * @param hsa_handle_ptr hsa_handle which has the phys_mem info.
+   */
+  virtual bool ImportShareableVMMHandle(void* osHandle, uint64_t* hsa_handle_ptr) const {
+    ShouldNotCallThis();
+    return false;
+  }
+
   /**
    * @return True if the device successfully applied the SVM attributes in HMM for device memory
    */

rocclr/device/rocm/rocdevice.cpp

@@ -2418,10 +2418,7 @@ void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_
     }
 
     // add the information to context so that we can use it later.
-    if (mem->getMemFlags() & ROCCLR_MEM_PHYMEM) {
-      mem->setSvmPtr(reinterpret_cast<void*>(mem->getUserData().hsa_handle));
-      amd::MemObjMap::AddMemObj(mem->getSvmPtr(), mem);
-    } else if (mem->getSvmPtr() != nullptr) {
+    if (mem->getSvmPtr() != nullptr) {
       amd::MemObjMap::AddMemObj(mem->getSvmPtr(), mem);
     }
     svmPtr = mem->getSvmPtr();
@@ -2510,6 +2507,50 @@ bool Device::GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) {
   return true;
 }
 
+// ================================================================================================
+bool Device::ExportShareableVMMHandle(uint64_t hsa_handle, int flags, void* shareableHandle) {
+  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
+  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle {};
+
+  if (hsa_handle == 0) {
+    LogError("HSA Handle is not valid");
+    return false;
+  }
+
+  int dmabuf_fd = 0;
+  hsa_vmem_handle.handle = hsa_handle;
+  if ((hsa_status = hsa_amd_vmem_export_shareable_handle(&dmabuf_fd,
+                      hsa_vmem_handle, flags)) != HSA_STATUS_SUCCESS) {
+    LogPrintfError("Failed hsa_vmem_export_shareable_handle with status: %d \n", hsa_status);
+    return false;
+  }
+
+  *(reinterpret_cast<int*>(shareableHandle)) = dmabuf_fd;
+
+  return true;
+}
+
+// ================================================================================================
+bool Device::ImportShareableVMMHandle(void* osHandle, uint64_t* hsa_handle_ptr) const {
+  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
+  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle {};
+
+  if (hsa_handle_ptr == nullptr) {
+    LogError("HSA Handle ptr is null");
+    return false;
+  }
+
+  int dmabuf_fd = *(reinterpret_cast<int*>(osHandle));
+  if ((hsa_status = hsa_amd_vmem_import_shareable_handle(dmabuf_fd, &hsa_vmem_handle))
+                      != HSA_STATUS_SUCCESS) {
+    LogPrintfError("Failed hsa_amd_vmem_import_shareable_handle with status: %d \n", hsa_status);
+    return false;
+  }
+
+  *hsa_handle_ptr = hsa_vmem_handle.handle;
+  return true;
+}
+
 // ================================================================================================
 bool Device::SetSvmAttributesInt(const void* dev_ptr, size_t count,
                               amd::MemoryAdvice advice, bool first_alloc, bool use_cpu) const {

rocclr/device/rocm/rocdevice.hpp

@@ -473,6 +473,10 @@ class Device : public NullDevice {
   virtual bool SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags);
   virtual bool GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr);
 
+  virtual bool ExportShareableVMMHandle(uint64_t hsa_handle, int flags, void* shareableHandle);
+
+  virtual bool ImportShareableVMMHandle(void* osHandle, uint64_t* hsa_handle_ptr) const;
+
   virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput,
                             cl_set_device_clock_mode_output_amd* pSetClockModeOutput);
 

rocclr/device/rocm/rocmemory.cpp

@@ -769,12 +769,28 @@ bool Buffer::create(bool alloc_local) {
   cl_mem_flags memFlags = owner()->getMemFlags();
 
   if (memFlags & ROCCLR_MEM_PHYMEM) {
-    // If this is physical memory request, then get an handle and store it in user data
-    owner()->getUserData().hsa_handle = dev().deviceVmemAlloc(owner()->getSize(), 0);
+    if (memFlags & ROCCLR_MEM_INTERPROCESS) {
+      int dmabuf_fd = *(reinterpret_cast<int*>(owner()->getSvmPtr()));
+      // if interprocess flag is set, then the memory is importable.
+      if (!dev().ImportShareableVMMHandle(owner()->getSvmPtr(),
+            &owner()->getUserData().hsa_handle)) {
+        LogPrintfError("Importing Shareable Memory failed with os_handle: 0x%x \n",
+                        owner()->getSvmPtr());
+        return false;
+      }
+    } else {
+      // If this is physical memory request, then get an handle and store it in user data
+      owner()->getUserData().hsa_handle = dev().deviceVmemAlloc(owner()->getSize(), 0);
+    }
+
     if (owner()->getUserData().hsa_handle == 0) {
       LogError("HSA Opaque Handle returned was null");
       return false;
     }
+
+    owner()->setSvmPtr(reinterpret_cast<void*>(owner()->getUserData().hsa_handle));
+    amd::MemObjMap::AddMemObj(owner()->getSvmPtr(), owner());
+
     return true;
   }