From 56ebf5157af01fb9a25dd7a7a2ed9e01bf34c103 Mon Sep 17 00:00:00 2001 From: kjayapra-amd Date: Wed, 20 Mar 2024 06:09:38 -0400 Subject: [PATCH] SWDEV-413997 - VMM IPC implementation for Linux. Change-Id: Icfeb83ca51e96be35abb67a94d6e3e1a1ca5a934 --- hipamd/src/hip_vm.cpp | 54 +++++++++++++++++++++++++++++--- rocclr/device/device.hpp | 23 ++++++++++++++ rocclr/device/rocm/rocdevice.cpp | 49 ++++++++++++++++++++++++++--- rocclr/device/rocm/rocdevice.hpp | 4 +++ rocclr/device/rocm/rocmemory.cpp | 20 ++++++++++-- 5 files changed, 139 insertions(+), 11 deletions(-) diff --git a/hipamd/src/hip_vm.cpp b/hipamd/src/hip_vm.cpp index 7fcd92369e..79bd1230aa 100644 --- a/hipamd/src/hip_vm.cpp +++ b/hipamd/src/hip_vm.cpp @@ -86,8 +86,8 @@ hipError_t hipMemCreate(hipMemGenericAllocationHandle_t* handle, size_t size, HIP_RETURN(hipErrorInvalidValue); } - // Currently only support non-IPC allocations - if (prop->requestedHandleType != hipMemHandleTypeNone) { + if (prop->requestedHandleType != hipMemHandleTypeNone + && prop->requestedHandleType != hipMemHandleTypePosixFileDescriptor) { HIP_RETURN(hipErrorNotSupported); } @@ -139,11 +139,29 @@ hipError_t hipMemExportToShareableHandle(void* shareableHandle, unsigned long long flags) { HIP_INIT_API(hipMemExportToShareableHandle, shareableHandle, handle, handleType, flags); - if (flags != 0 || handle == nullptr || shareableHandle == nullptr) { + if (flags != 0 || handle == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - HIP_RETURN(hipErrorNotSupported); + hip::GenericAllocation* ga = reinterpret_cast(handle); + if (ga == nullptr) { + LogError("Generic Allocation is nullptr"); + HIP_RETURN(hipErrorNotInitialized); + } + + if (ga->GetProperties().requestedHandleType != handleType) { + LogPrintfError("HandleType mismatch memoryHandleType: %d, requestedHandleType: %d", + ga->GetProperties().requestedHandleType, handleType); + HIP_RETURN(hipErrorInvalidValue); + } + + if (!ga->asAmdMemory().getContext().devices()[0]->ExportShareableVMMHandle( + ga->asAmdMemory().getUserData().hsa_handle, flags, shareableHandle)) { + LogPrintfError("Exporting Handle failed with flags: %d", flags); + HIP_RETURN(hipErrorInvalidValue); + } + + HIP_RETURN(hipSuccess); } hipError_t hipMemGetAccess(unsigned long long* flags, const hipMemLocation* location, void* ptr) { @@ -205,7 +223,33 @@ hipError_t hipMemImportFromShareableHandle(hipMemGenericAllocationHandle_t* hand HIP_RETURN(hipErrorInvalidValue); } - HIP_RETURN(hipErrorNotSupported); + amd::Device* device = hip::getCurrentDevice()->devices()[0]; + amd::Memory* phys_mem_obj = new (device->context()) amd::Buffer(device->context(), + ROCCLR_MEM_PHYMEM | ROCCLR_MEM_INTERPROCESS, 0, osHandle); + + if (phys_mem_obj == nullptr) { + LogError("failed to new a va range curr_mem_obj object!"); + HIP_RETURN(hipErrorInvalidValue); + } + + if (!phys_mem_obj->create(nullptr, false)) { + LogError("failed to create a va range mem object"); + phys_mem_obj->release(); + HIP_RETURN(hipErrorInvalidValue); + } + + hipMemAllocationProp prop {}; + prop.type = hipMemAllocationTypePinned; + prop.location.type = hipMemLocationTypeDevice; + prop.location.id = hip::getCurrentDevice()->deviceId(); + + phys_mem_obj->getUserData().deviceId = hip::getCurrentDevice()->deviceId(); + phys_mem_obj->getUserData().data = new hip::GenericAllocation(*phys_mem_obj, 0, prop); + *handle = reinterpret_cast(phys_mem_obj->getUserData().data); + + amd::MemObjMap::RemoveMemObj(phys_mem_obj->getSvmPtr()); + + HIP_RETURN(hipSuccess); } hipError_t hipMemMap(void* ptr, size_t size, size_t offset, hipMemGenericAllocationHandle_t handle, diff --git a/rocclr/device/device.hpp b/rocclr/device/device.hpp index f5e814cb67..b4b1afd1af 100644 --- a/rocclr/device/device.hpp +++ b/rocclr/device/device.hpp @@ -1859,6 +1859,29 @@ class Device : public RuntimeObject { */ virtual void virtualFree(void* addr) = 0; + /** + * Export Shareable VMM Handle to FD + * + * @param hsa_handle hsa_handle which has the phys_mem info. + * @param flags any flags to be passed + * @param shareableHandle exported handle, points to fdesc. + */ + virtual bool ExportShareableVMMHandle(uint64_t hsa_handle, int flags, void* shareableHandle) { + ShouldNotCallThis(); + return false; + } + + /** + * Import FD from Shareable VMM Handle + * + * @param osHandle os handle/fdesc + * @param hsa_handle_ptr hsa_handle which has the phys_mem info. + */ + virtual bool ImportShareableVMMHandle(void* osHandle, uint64_t* hsa_handle_ptr) const { + ShouldNotCallThis(); + return false; + } + /** * @return True if the device successfully applied the SVM attributes in HMM for device memory */ diff --git a/rocclr/device/rocm/rocdevice.cpp b/rocclr/device/rocm/rocdevice.cpp index 9fd3600cb7..ab0d00015d 100644 --- a/rocclr/device/rocm/rocdevice.cpp +++ b/rocclr/device/rocm/rocdevice.cpp @@ -2418,10 +2418,7 @@ void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_ } // add the information to context so that we can use it later. - if (mem->getMemFlags() & ROCCLR_MEM_PHYMEM) { - mem->setSvmPtr(reinterpret_cast(mem->getUserData().hsa_handle)); - amd::MemObjMap::AddMemObj(mem->getSvmPtr(), mem); - } else if (mem->getSvmPtr() != nullptr) { + if (mem->getSvmPtr() != nullptr) { amd::MemObjMap::AddMemObj(mem->getSvmPtr(), mem); } svmPtr = mem->getSvmPtr(); @@ -2510,6 +2507,50 @@ bool Device::GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) { return true; } +// ================================================================================================ +bool Device::ExportShareableVMMHandle(uint64_t hsa_handle, int flags, void* shareableHandle) { + hsa_status_t hsa_status = HSA_STATUS_SUCCESS; + hsa_amd_vmem_alloc_handle_t hsa_vmem_handle {}; + + if (hsa_handle == 0) { + LogError("HSA Handle is not valid"); + return false; + } + + int dmabuf_fd = 0; + hsa_vmem_handle.handle = hsa_handle; + if ((hsa_status = hsa_amd_vmem_export_shareable_handle(&dmabuf_fd, + hsa_vmem_handle, flags)) != HSA_STATUS_SUCCESS) { + LogPrintfError("Failed hsa_vmem_export_shareable_handle with status: %d \n", hsa_status); + return false; + } + + *(reinterpret_cast(shareableHandle)) = dmabuf_fd; + + return true; +} + +// ================================================================================================ +bool Device::ImportShareableVMMHandle(void* osHandle, uint64_t* hsa_handle_ptr) const { + hsa_status_t hsa_status = HSA_STATUS_SUCCESS; + hsa_amd_vmem_alloc_handle_t hsa_vmem_handle {}; + + if (hsa_handle_ptr == nullptr) { + LogError("HSA Handle ptr is null"); + return false; + } + + int dmabuf_fd = *(reinterpret_cast(osHandle)); + if ((hsa_status = hsa_amd_vmem_import_shareable_handle(dmabuf_fd, &hsa_vmem_handle)) + != HSA_STATUS_SUCCESS) { + LogPrintfError("Failed hsa_amd_vmem_import_shareable_handle with status: %d \n", hsa_status); + return false; + } + + *hsa_handle_ptr = hsa_vmem_handle.handle; + return true; +} + // ================================================================================================ bool Device::SetSvmAttributesInt(const void* dev_ptr, size_t count, amd::MemoryAdvice advice, bool first_alloc, bool use_cpu) const { diff --git a/rocclr/device/rocm/rocdevice.hpp b/rocclr/device/rocm/rocdevice.hpp index c2240d8987..1cd47fb0ab 100644 --- a/rocclr/device/rocm/rocdevice.hpp +++ b/rocclr/device/rocm/rocdevice.hpp @@ -473,6 +473,10 @@ class Device : public NullDevice { virtual bool SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags); virtual bool GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr); + virtual bool ExportShareableVMMHandle(uint64_t hsa_handle, int flags, void* shareableHandle); + + virtual bool ImportShareableVMMHandle(void* osHandle, uint64_t* hsa_handle_ptr) const; + virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput); diff --git a/rocclr/device/rocm/rocmemory.cpp b/rocclr/device/rocm/rocmemory.cpp index cd94068904..75460957e4 100644 --- a/rocclr/device/rocm/rocmemory.cpp +++ b/rocclr/device/rocm/rocmemory.cpp @@ -769,12 +769,28 @@ bool Buffer::create(bool alloc_local) { cl_mem_flags memFlags = owner()->getMemFlags(); if (memFlags & ROCCLR_MEM_PHYMEM) { - // If this is physical memory request, then get an handle and store it in user data - owner()->getUserData().hsa_handle = dev().deviceVmemAlloc(owner()->getSize(), 0); + if (memFlags & ROCCLR_MEM_INTERPROCESS) { + int dmabuf_fd = *(reinterpret_cast(owner()->getSvmPtr())); + // if interprocess flag is set, then the memory is importable. + if (!dev().ImportShareableVMMHandle(owner()->getSvmPtr(), + &owner()->getUserData().hsa_handle)) { + LogPrintfError("Importing Shareable Memory failed with os_handle: 0x%x \n", + owner()->getSvmPtr()); + return false; + } + } else { + // If this is physical memory request, then get an handle and store it in user data + owner()->getUserData().hsa_handle = dev().deviceVmemAlloc(owner()->getSize(), 0); + } + if (owner()->getUserData().hsa_handle == 0) { LogError("HSA Opaque Handle returned was null"); return false; } + + owner()->setSvmPtr(reinterpret_cast(owner()->getUserData().hsa_handle)); + amd::MemObjMap::AddMemObj(owner()->getSvmPtr(), owner()); + return true; }