From 26bfa0b8f6b0efbdf275e89eb164c8c88d58f5be Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Mon, 27 Jan 2025 16:26:21 +0000 Subject: [PATCH] rocr/aie: Add dma-buf import support for AIEAgents via the Driver interface Change-Id: I70f8d8772dda7c06944d75042cb3034ddd89aff4 --- .../core/driver/kfd/amd_kfd_driver.cpp | 94 +++++++- .../core/driver/xdna/amd_xdna_driver.cpp | 63 +++++- runtime/hsa-runtime/core/inc/amd_kfd_driver.h | 11 +- .../hsa-runtime/core/inc/amd_xdna_driver.h | 15 +- runtime/hsa-runtime/core/inc/driver.h | 57 ++++- runtime/hsa-runtime/core/inc/runtime.h | 39 +--- runtime/hsa-runtime/core/runtime/runtime.cpp | 214 ++++++++---------- runtime/hsa-runtime/core/util/memory.h | 75 ++++++ 8 files changed, 414 insertions(+), 154 deletions(-) create mode 100644 runtime/hsa-runtime/core/util/memory.h diff --git a/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp b/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp index e1c41a2bf3..248a79174d 100644 --- a/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp +++ b/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp @@ -3,7 +3,7 @@ // The University of Illinois/NCSA // Open Source License (NCSA) // -// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // @@ -45,11 +45,13 @@ #include #include +#include #include #include #include "hsakmt/hsakmt.h" +#include "core/inc/amd_gpu_agent.h" #include "core/inc/amd_memory_region.h" #include "core/inc/runtime.h" @@ -58,6 +60,29 @@ extern r_debug _amdgpu_r_debug; namespace rocr { namespace AMD { +static_assert( + (sizeof(core::ShareableHandle::handle) >= sizeof(amdgpu_bo_handle)) && + (alignof(core::ShareableHandle::handle) >= alignof(amdgpu_bo_handle)), + "ShareableHandle cannot store a amdgpu_bo_handle"); + +namespace { + +__forceinline uint64_t drm_perm(hsa_access_permission_t perm) { + switch (perm) { + case HSA_ACCESS_PERMISSION_RO: + return AMDGPU_VM_PAGE_READABLE; + case HSA_ACCESS_PERMISSION_WO: + return AMDGPU_VM_PAGE_WRITEABLE; + case HSA_ACCESS_PERMISSION_RW: + return AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE; + case HSA_ACCESS_PERMISSION_NONE: + default: + return 0; + } +} + +} // namespace + KfdDriver::KfdDriver(std::string devnode_name) : core::Driver(core::DriverType::KFD, devnode_name) {} @@ -322,6 +347,73 @@ hsa_status_t KfdDriver::DestroyQueue(core::Queue &queue) const { return HSA_STATUS_SUCCESS; } +hsa_status_t KfdDriver::ExportDMABuf(void *mem, size_t size, int *dmabuf_fd, + size_t *offset) { + int dmabuf_fd_res = -1; + size_t offset_res = 0; + if (hsaKmtExportDMABufHandle(mem, size, &dmabuf_fd_res, &offset_res) != + HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + + *dmabuf_fd = dmabuf_fd_res; + *offset = offset_res; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdDriver::ImportDMABuf(int dmabuf_fd, core::Agent &agent, + core::ShareableHandle &handle) { + auto &gpu_agent = static_cast(agent); + amdgpu_bo_import_result res; + auto ret = amdgpu_bo_import( + gpu_agent.libDrmDev(), amdgpu_bo_handle_type_dma_buf_fd, dmabuf_fd, &res); + if (ret) + return HSA_STATUS_ERROR; + + handle.handle = reinterpret_cast(res.buf_handle); + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdDriver::Map(core::ShareableHandle handle, void *mem, + size_t offset, size_t size, + hsa_access_permission_t perms) { + const auto ldrm_bo = reinterpret_cast(handle.handle); + if (!ldrm_bo) + return HSA_STATUS_ERROR; + + if (amdgpu_bo_va_op(ldrm_bo, offset, size, reinterpret_cast(mem), + drm_perm(perms), AMDGPU_VA_OP_MAP) != 0) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdDriver::Unmap(core::ShareableHandle handle, void *mem, + size_t offset, size_t size) { + const auto ldrm_bo = reinterpret_cast(handle.handle); + if (!ldrm_bo) + return HSA_STATUS_ERROR; + + if (amdgpu_bo_va_op(ldrm_bo, offset, size, reinterpret_cast(mem), 0, + AMDGPU_VA_OP_UNMAP) != 0) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdDriver::ReleaseShareableHandle(core::ShareableHandle &handle) { + const auto ldrm_bo = reinterpret_cast(handle.handle); + if (!ldrm_bo) + return HSA_STATUS_ERROR; + + const auto ret = amdgpu_bo_free(ldrm_bo); + if (ret) + return HSA_STATUS_ERROR; + + handle = {}; + return HSA_STATUS_SUCCESS; +} + void *KfdDriver::AllocateKfdMemory(const HsaMemFlags &flags, uint32_t node_id, size_t size) { void *mem = nullptr; diff --git a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp index d9b8c6b3dd..b92b66d09b 100644 --- a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp +++ b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp @@ -3,7 +3,7 @@ // The University of Illinois/NCSA // Open Source License (NCSA) // -// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // @@ -52,12 +52,17 @@ #include "core/inc/amd_aie_aql_queue.h" #include "core/inc/amd_memory_region.h" #include "core/inc/runtime.h" +#include "core/util/memory.h" #include "core/util/utils.h" #include "uapi/amdxdna_accel.h" namespace rocr { namespace AMD { +static_assert((sizeof(core::ShareableHandle::handle) >= sizeof(uint32_t)) && + (alignof(core::ShareableHandle::handle) >= alignof(uint32_t)), + "ShareableHandle cannot store a XDNA handle"); + XdnaDriver::XdnaDriver(std::string devnode_name) : core::Driver(core::DriverType::XDNA, devnode_name) {} @@ -292,6 +297,62 @@ hsa_status_t XdnaDriver::DestroyQueue(core::Queue &queue) const { return HSA_STATUS_SUCCESS; } +hsa_status_t XdnaDriver::ExportDMABuf(void *mem, size_t size, int *dmabuf_fd, + size_t *offset) { + // Not implemented yet. + return HSA_STATUS_ERROR; +} + +hsa_status_t XdnaDriver::ImportDMABuf(int dmabuf_fd, core::Agent &agent, + core::ShareableHandle &handle) { + drm_prime_handle import_params = {}; + import_params.handle = AMDXDNA_INVALID_BO_HANDLE; + import_params.fd = dmabuf_fd; + if (ioctl(fd_, DRM_IOCTL_PRIME_FD_TO_HANDLE, &import_params) < 0) + return HSA_STATUS_ERROR; + + handle.handle = import_params.handle; + return HSA_STATUS_SUCCESS; +} + +hsa_status_t XdnaDriver::Map(core::ShareableHandle handle, void *mem, + size_t offset, size_t size, + hsa_access_permission_t perms) { + // Get fd associated with the handle. + drm_prime_handle params = {}; + params.handle = handle.handle; + params.fd = -1; + if (ioctl(fd_, DRM_IOCTL_PRIME_HANDLE_TO_FD, ¶ms) < 0) + return HSA_STATUS_ERROR; + + // Change permissions. + void *mapped_ptr = mmap(mem, size, PermissionsToMmapFlags(perms), + MAP_FIXED | MAP_SHARED, params.fd, offset); + if (mapped_ptr == MAP_FAILED) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t XdnaDriver::Unmap(core::ShareableHandle handle, void *mem, + size_t offset, size_t size) { + if (munmap(mem, size) != 0) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t XdnaDriver::ReleaseShareableHandle(core::ShareableHandle &handle) { + drm_gem_close close_params = {}; + close_params.handle = handle.handle; + if (ioctl(fd_, DRM_IOCTL_GEM_CLOSE, &close_params) < 0) + return HSA_STATUS_ERROR; + + handle = {}; + + return HSA_STATUS_SUCCESS; +} + hsa_status_t XdnaDriver::QueryDriverVersion() { amdxdna_drm_query_aie_version aie_version{0, 0}; amdxdna_drm_get_info args{DRM_AMDXDNA_QUERY_AIE_VERSION, sizeof(aie_version), diff --git a/runtime/hsa-runtime/core/inc/amd_kfd_driver.h b/runtime/hsa-runtime/core/inc/amd_kfd_driver.h index cd634a92b9..35d2b23a17 100644 --- a/runtime/hsa-runtime/core/inc/amd_kfd_driver.h +++ b/runtime/hsa-runtime/core/inc/amd_kfd_driver.h @@ -3,7 +3,7 @@ // The University of Illinois/NCSA // Open Source License (NCSA) // -// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // @@ -87,6 +87,15 @@ public: hsa_status_t FreeMemory(void *mem, size_t size) override; hsa_status_t CreateQueue(core::Queue &queue) const override; hsa_status_t DestroyQueue(core::Queue &queue) const override; + hsa_status_t ExportDMABuf(void *mem, size_t size, int *dmabuf_fd, + size_t *offset) override; + hsa_status_t ImportDMABuf(int dmabuf_fd, core::Agent &agent, + core::ShareableHandle &handle) override; + hsa_status_t Map(core::ShareableHandle handle, void *mem, size_t offset, + size_t size, hsa_access_permission_t perms) override; + hsa_status_t Unmap(core::ShareableHandle handle, void *mem, size_t offset, + size_t size) override; + hsa_status_t ReleaseShareableHandle(core::ShareableHandle &handle) override; private: /// @brief Allocate agent accessible memory (system / local memory). diff --git a/runtime/hsa-runtime/core/inc/amd_xdna_driver.h b/runtime/hsa-runtime/core/inc/amd_xdna_driver.h index 34af182d09..a170f72000 100644 --- a/runtime/hsa-runtime/core/inc/amd_xdna_driver.h +++ b/runtime/hsa-runtime/core/inc/amd_xdna_driver.h @@ -3,7 +3,7 @@ // The University of Illinois/NCSA // Open Source License (NCSA) // -// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // @@ -158,12 +158,17 @@ public: void **mem, size_t size, uint32_t node_id) override; hsa_status_t FreeMemory(void *mem, size_t size) override; - - /// @brief Creates a context on the AIE device for this queue. - /// @param queue Queue whose on-device context is being created. - /// @return hsa_status_t hsa_status_t CreateQueue(core::Queue &queue) const override; hsa_status_t DestroyQueue(core::Queue &queue) const override; + hsa_status_t ExportDMABuf(void *mem, size_t size, int *dmabuf_fd, + size_t *offset) override; + hsa_status_t ImportDMABuf(int dmabuf_fd, core::Agent &agent, + core::ShareableHandle &handle) override; + hsa_status_t Map(core::ShareableHandle handle, void *mem, size_t offset, + size_t size, hsa_access_permission_t perms) override; + hsa_status_t Unmap(core::ShareableHandle handle, void *mem, size_t offset, + size_t size) override; + hsa_status_t ReleaseShareableHandle(core::ShareableHandle &handle) override; // @brief Submits num_pkts packets in a command chain to the XDNA driver hsa_status_t SubmitCmdChain(hsa_amd_aie_ert_packet_t* first_pkt, uint32_t num_pkts, diff --git a/runtime/hsa-runtime/core/inc/driver.h b/runtime/hsa-runtime/core/inc/driver.h index c998a410b8..8a56060f7e 100644 --- a/runtime/hsa-runtime/core/inc/driver.h +++ b/runtime/hsa-runtime/core/inc/driver.h @@ -3,7 +3,7 @@ // The University of Illinois/NCSA // Open Source License (NCSA) // -// Copyright (c) 2023, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // @@ -59,14 +59,20 @@ enum class DriverQuery { GET_DRIVER_VERSION }; enum class DriverType { XDNA = 0, KFD, NUM_DRIVER_TYPES }; +/// @brief Handle for exported / imported memory. +struct ShareableHandle { + uint64_t handle{}; + + bool IsValid() const { return handle != 0; } +}; + /// @brief Kernel driver interface. /// /// @details A class used to provide an interface between the core runtime /// and agent kernel drivers. It also maintains state associated with active /// kernel drivers. class Driver { - public: - Driver() = delete; +public: Driver(DriverType kernel_driver_type, std::string devnode_name); virtual ~Driver() = default; @@ -126,7 +132,7 @@ class Driver { /// @brief Allocate agent-accessible memory (system or agent-local memory). /// - /// @param[out] pointer to newly allocated memory. + /// @param[out] mem pointer to newly allocated memory. /// /// @retval HSA_STATUS_SUCCESS if memory was successfully allocated or /// hsa_status_t error code if the memory allocation failed. @@ -141,6 +147,49 @@ class Driver { virtual hsa_status_t DestroyQueue(Queue &queue) const = 0; + /// @brief Imports memory using dma-buf. + /// + /// @param[in] mem virtual address + /// @param[in] size memory size in bytes + /// @param[out] dmabuf_fd dma-buf file descriptor + /// @param[out] offset memory offset in bytes + virtual hsa_status_t ExportDMABuf(void *mem, size_t size, int *dmabuf_fd, + size_t *offset) = 0; + + /// @brief Imports a memory chunk via dma-buf. + /// + /// @param[in] dmabuf_fd dma-buf file descriptor + /// @param[in] agent agent to import the memory for + /// @param[out] handle handle to the imported memory + virtual hsa_status_t ImportDMABuf(int dmabuf_fd, core::Agent &agent, + core::ShareableHandle &handle) = 0; + + /// @brief Maps the memory associated with the handle. + /// + /// @param[in] handle handle to the memory object + /// @param[in] mem virtual address associated with the handle + /// @param[in] offset memory offset in bytes + /// @param[in] size memory size in bytes + /// @param[perms] perms new permissions + virtual hsa_status_t Map(core::ShareableHandle handle, void *mem, + size_t offset, size_t size, + hsa_access_permission_t perms) = 0; + + /// @brief Unmaps the memory associated with the handle. + /// + /// @param[in] handle handle to the memory object + /// @param[in] mem virtual address associated with the handle + /// @param[in] offset memory offset in bytes + /// @param[in] size memory size in bytes + virtual hsa_status_t Unmap(core::ShareableHandle handle, void *mem, + size_t offset, size_t size) = 0; + + /// @brief Releases the object associated with the handle. + /// + /// @param[in] handle handle of the object to release + virtual hsa_status_t + ReleaseShareableHandle(core::ShareableHandle &handle) = 0; + /// Unique identifier for supported kernel-mode drivers. const DriverType kernel_driver_type_; diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index 0057d7c81c..1f88106a48 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -3,7 +3,7 @@ // The University of Illinois/NCSA // Open Source License (NCSA) // -// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2014-2025, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // @@ -776,7 +776,8 @@ class Runtime { private: void CheckVirtualMemApiSupport(); - int GetAmdgpuDeviceArgs(Agent* agent, amdgpu_bo_handle bo, int* drm_fd, uint64_t* cpu_addr); + int GetAmdgpuDeviceArgs(Agent *agent, ShareableHandle handle, int *drm_fd, + uint64_t *cpu_addr); bool virtual_mem_api_supported_; bool xnack_enabled_; @@ -822,8 +823,6 @@ class Runtime { struct MappedHandle; struct MappedHandleAllowedAgent { - MappedHandleAllowedAgent() - : va(NULL), size(0), permissions(HSA_ACCESS_PERMISSION_NONE), mappedHandle(NULL), ldrm_bo(0) {} MappedHandleAllowedAgent(MappedHandle* _mappedHandle, Agent* targetAgent, void* va, size_t size, hsa_access_permission_t perms); ~MappedHandleAllowedAgent(); @@ -836,42 +835,26 @@ class Runtime { Agent* targetAgent; hsa_access_permission_t permissions; MappedHandle* mappedHandle; - amdgpu_bo_handle ldrm_bo; + ShareableHandle shareable_handle; }; struct MappedHandle { - MappedHandle() - : mem_handle(NULL), - address_handle(NULL), - offset(0), - mmap_offset(0), - size(0), - drm_fd(-1), - drm_cpu_addr(NULL), - ldrm_bo(0) {} - - MappedHandle(MemoryHandle* mem_handle, AddressHandle* address_handle, uint64_t offset, - size_t size, int drm_fd, void* drm_cpu_addr, hsa_access_permission_t perm, - amdgpu_bo_handle bo) - : mem_handle(mem_handle), - address_handle(address_handle), - offset(offset), - mmap_offset(0), - size(size), - drm_fd(drm_fd), - drm_cpu_addr(drm_cpu_addr), - ldrm_bo(bo) {} + MappedHandle(MemoryHandle *mem_handle, AddressHandle *address_handle, + uint64_t offset, size_t size, int drm_fd, void *drm_cpu_addr, + hsa_access_permission_t perm, ShareableHandle shareable_handle) + : mem_handle(mem_handle), address_handle(address_handle), + offset(offset), size(size), drm_fd(drm_fd), + drm_cpu_addr(drm_cpu_addr), shareable_handle(shareable_handle) {} __forceinline core::Agent* agentOwner() const { return mem_handle->region->owner(); } MemoryHandle* mem_handle; AddressHandle* address_handle; uint64_t offset; - uint64_t mmap_offset; size_t size; int drm_fd; void* drm_cpu_addr; // CPU Buffer address - amdgpu_bo_handle ldrm_bo; + ShareableHandle shareable_handle; std::map allowed_agents; }; std::map mapped_handle_map_; // Indexed by VA diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 77bea8d974..4222be273a 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -3,7 +3,7 @@ // The University of Illinois/NCSA // Open Source License (NCSA) // -// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2014-2025, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // @@ -66,20 +66,21 @@ #endif #include "core/common/shared.h" -#include "core/inc/hsa_ext_interface.h" +#include "core/inc/amd_core_dump.hpp" #include "core/inc/amd_cpu_agent.h" #include "core/inc/amd_gpu_agent.h" #include "core/inc/amd_memory_region.h" #include "core/inc/amd_topology.h" -#include "core/inc/signal.h" -#include "core/inc/interrupt_signal.h" -#include "core/inc/hsa_ext_amd_impl.h" -#include "core/inc/hsa_api_trace_int.h" -#include "core/util/os.h" #include "core/inc/exceptions.h" -#include "inc/hsa_ven_amd_aqlprofile.h" -#include "core/inc/amd_core_dump.hpp" #include "core/inc/host_queue.h" +#include "core/inc/hsa_api_trace_int.h" +#include "core/inc/hsa_ext_amd_impl.h" +#include "core/inc/hsa_ext_interface.h" +#include "core/inc/interrupt_signal.h" +#include "core/inc/signal.h" +#include "core/util/memory.h" +#include "core/util/os.h" +#include "inc/hsa_ven_amd_aqlprofile.h" #ifndef HSA_VERSION_MAJOR #define HSA_VERSION_MAJOR 1 @@ -1977,8 +1978,8 @@ void Runtime::PrintMemoryMapNear(void* ptr) { info.size = sizeof(info); for (int i = 0; i < 3; i++) { if (it == runtime_singleton_->allocation_map_.end()) break; - hsa_status_t err = runtime_singleton_->PtrInfo(const_cast(it->first), &info, - malloc, &count, &canAccess, &block); + hsa_status_t err = runtime_singleton_->PtrInfo(const_cast(it->first), &info, malloc, + &count, &canAccess, &block); if (err == HSA_STATUS_SUCCESS) { fprintf(stderr, "PtrInfo:\n\tAddress: %p-%p/%p-%p\n\tSize: 0x%lx\n\tType: %u\n\tOwner: %p\n", info.agentBaseAddress, (char*)info.agentBaseAddress + info.sizeInBytes, @@ -2221,13 +2222,15 @@ int fn_amdgpu_device_get_fd_nosupport(HsaAMDGPUDeviceHandle device_handle) { return -1; } -int Runtime::GetAmdgpuDeviceArgs(Agent* agent, amdgpu_bo_handle bo, int* drm_fd, - uint64_t* cpu_addr) { +int Runtime::GetAmdgpuDeviceArgs(Agent *agent, ShareableHandle handle, + int *drm_fd, uint64_t *cpu_addr) { int renderFd = fn_amdgpu_device_get_fd(static_cast(agent)->libDrmDev()); if (renderFd < 0) return HSA_STATUS_ERROR; uint32_t gem_handle = 0; - if (amdgpu_bo_export(bo, amdgpu_bo_handle_type_kms, &gem_handle)) return HSA_STATUS_ERROR; + if (amdgpu_bo_export(reinterpret_cast(handle.handle), + amdgpu_bo_handle_type_kms, &gem_handle)) + return HSA_STATUS_ERROR; union drm_amdgpu_gem_mmap args; memset(&args, 0, sizeof(args)); @@ -3207,47 +3210,12 @@ hsa_status_t Runtime::VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryOnl return HSA_STATUS_SUCCESS; } -__forceinline uint64_t drm_perm(hsa_access_permission_t perm) { - switch (perm) { - case HSA_ACCESS_PERMISSION_RO: - return AMDGPU_VM_PAGE_READABLE; - case HSA_ACCESS_PERMISSION_WO: - return AMDGPU_VM_PAGE_WRITEABLE; - case HSA_ACCESS_PERMISSION_RW: - return AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE; - case HSA_ACCESS_PERMISSION_NONE: - return 0; - default: - break; - } - - return 0; -} - -__forceinline int mmap_perm(hsa_access_permission_t perms) { - switch (perms) { - case HSA_ACCESS_PERMISSION_RO: - return PROT_READ; - case HSA_ACCESS_PERMISSION_WO: - return PROT_WRITE; - case HSA_ACCESS_PERMISSION_RW: - return PROT_READ | PROT_WRITE; - case HSA_ACCESS_PERMISSION_NONE: - return PROT_NONE; - default: - break; - } - - return 0; -} - hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset, hsa_amd_vmem_alloc_handle_t memoryOnlyHandle, uint64_t flags) { int drm_fd, dmabuf_fd = 0; uint64_t offset = 0, ret; uint64_t drm_cpu_addr = 0; - amdgpu_bo_handle ldrm_bo = 0; bool reservedAddressFound = false; ScopedAcquire lock(&memory_lock_); @@ -3280,25 +3248,38 @@ hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset, return HSA_STATUS_ERROR_INVALID_ARGUMENT; } - ret = hsaKmtExportDMABufHandle(memoryHandleIt->first, size, &dmabuf_fd, &offset); - if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + auto *agent = memoryHandleIt->second.agentOwner(); + + // For now, this is only supported for KFD due to the call to + // GetAmdgpuDeviceArgs + if (agent->device_type() != core::Agent::DeviceType::kAmdGpuDevice) + return HSA_STATUS_ERROR_INVALID_AGENT; + + // Create handle by exporting and importing the memory from the owning agent + auto &agent_driver = agent->driver(); + hsa_status_t status = agent_driver.ExportDMABuf(memoryHandleIt->first, size, + &dmabuf_fd, &offset); + if (status != HSA_STATUS_SUCCESS) + return status; assert(offset == 0); - AMD::GpuAgent* agent = static_cast(memoryHandleIt->second.agentOwner()); - amdgpu_bo_import_result res; - ret = amdgpu_bo_import(agent->libDrmDev(), amdgpu_bo_handle_type_dma_buf_fd, dmabuf_fd, &res); - if (ret) return HSA_STATUS_ERROR; + ShareableHandle shareable_handle; + status = agent_driver.ImportDMABuf(dmabuf_fd, *agent, shareable_handle); + if (status != HSA_STATUS_SUCCESS) + return status; close(dmabuf_fd); - ldrm_bo = res.buf_handle; - ret = GetAmdgpuDeviceArgs(agent, ldrm_bo, &drm_fd, &drm_cpu_addr); + // Get address that memory is mapped to + ret = GetAmdgpuDeviceArgs(agent, shareable_handle, &drm_fd, &drm_cpu_addr); if (ret) return HSA_STATUS_ERROR; - mapped_handle_map_.emplace(std::piecewise_construct, - std::forward_as_tuple(va), - std::forward_as_tuple(&memoryHandleIt->second, &reservedAddressIt->second, offset, size, drm_fd, - reinterpret_cast(drm_cpu_addr), HSA_ACCESS_PERMISSION_NONE, ldrm_bo)); + mapped_handle_map_.emplace( + std::piecewise_construct, std::forward_as_tuple(va), + std::forward_as_tuple(&memoryHandleIt->second, &reservedAddressIt->second, + offset, size, drm_fd, + reinterpret_cast(drm_cpu_addr), + HSA_ACCESS_PERMISSION_NONE, shareable_handle)); reservedAddressIt->second.use_count++; memoryHandleIt->second.use_count++; @@ -3307,7 +3288,6 @@ hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset, } hsa_status_t Runtime::VMemoryHandleUnmap(void* va, size_t size) { - int ret; ScopedAcquire lock(&memory_lock_); auto mappedHandleIt = mapped_handle_map_.find(va); @@ -3315,21 +3295,23 @@ hsa_status_t Runtime::VMemoryHandleUnmap(void* va, size_t size) { if (mappedHandleIt->second.size != size) return HSA_STATUS_ERROR_INVALID_ARGUMENT; + // Remove access from all agents that were allowed access for (auto agentPermsIt = mappedHandleIt->second.allowed_agents.begin(); agentPermsIt != mappedHandleIt->second.allowed_agents.end();) { assert(va == agentPermsIt->second.va); - if (agentPermsIt->second.ldrm_bo) - ret = amdgpu_bo_va_op(agentPermsIt->second.ldrm_bo, mappedHandleIt->second.offset, size, - reinterpret_cast(va), 0, AMDGPU_VA_OP_UNMAP); - else - ret = munmap(va, size); - if (ret) return HSA_STATUS_ERROR; + + hsa_status_t status = agentPermsIt->second.RemoveAccess(); + if (status != HSA_STATUS_SUCCESS) + return status; + agentPermsIt = mappedHandleIt->second.allowed_agents.erase(agentPermsIt); } - if (mappedHandleIt->second.ldrm_bo) { - ret = amdgpu_bo_free(mappedHandleIt->second.ldrm_bo); - if (ret) return HSA_STATUS_ERROR; + if (mappedHandleIt->second.shareable_handle.IsValid()) { + hsa_status_t status = mappedHandleIt->second.agentOwner()->driver().ReleaseShareableHandle( + mappedHandleIt->second.shareable_handle); + if (status != HSA_STATUS_SUCCESS) + return status; } assert(mappedHandleIt->second.address_handle->use_count >= 1); @@ -3350,70 +3332,71 @@ hsa_status_t Runtime::VMemoryHandleUnmap(void* va, size_t size) { return HSA_STATUS_SUCCESS; } -Runtime::MappedHandleAllowedAgent::MappedHandleAllowedAgent(MappedHandle* _mappedHandle, Agent* targetAgent, void* va, size_t size, - hsa_access_permission_t perms) - : va(va), - size(size), - targetAgent(targetAgent), - permissions(perms), - mappedHandle(_mappedHandle), - ldrm_bo(NULL) { +Runtime::MappedHandleAllowedAgent::MappedHandleAllowedAgent( + MappedHandle *mappedHandle, Agent *targetAgent, void *va, size_t size, + hsa_access_permission_t perms) + : va(va), size(size), targetAgent(targetAgent), permissions(perms), + mappedHandle(mappedHandle) { + // CPU agents have access as the memory is already mapped to the host. if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) return; - AMD::GpuAgent* gpuAgent = static_cast(targetAgent); int dmabuf_fd = 0; uint64_t offset = 0; MemoryHandle *memHandle = mappedHandle->mem_handle; - int ret = hsaKmtExportDMABufHandle(memHandle->thunk_handle, mappedHandle->size, &dmabuf_fd, &offset); - assert(ret == HSAKMT_STATUS_SUCCESS); - - if (ret != HSAKMT_STATUS_SUCCESS) return; + // Export memory from owner agent. + hsa_status_t status = memHandle->agentOwner()->driver().ExportDMABuf( + memHandle->thunk_handle, mappedHandle->size, &dmabuf_fd, &offset); + assert(status == HSA_STATUS_SUCCESS); + if (status != HSA_STATUS_SUCCESS) + return; assert(offset == 0); - amdgpu_bo_import_result res; - ret = amdgpu_bo_import(gpuAgent->libDrmDev(), amdgpu_bo_handle_type_dma_buf_fd, dmabuf_fd, &res); - assert(ret == 0); - if (ret) return; + // Import to target agent. + status = targetAgent->driver().ImportDMABuf(dmabuf_fd, *targetAgent, + shareable_handle); + assert(status == HSA_STATUS_SUCCESS); + if (status != HSA_STATUS_SUCCESS) + return; close(dmabuf_fd); - ldrm_bo = res.buf_handle; } Runtime::MappedHandleAllowedAgent::~MappedHandleAllowedAgent() { if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) return; - amdgpu_bo_free(ldrm_bo); + hsa_status_t status = + targetAgent->driver().ReleaseShareableHandle(shareable_handle); + assert(status == HSA_STATUS_SUCCESS); } hsa_status_t Runtime::MappedHandleAllowedAgent::EnableAccess(hsa_access_permission_t perms) { if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) { - void* ret_cpu_addr = - mmap(va, size, mmap_perm(perms), MAP_SHARED | MAP_FIXED, mappedHandle->drm_fd, + void* mapped_ptr = + mmap(va, size, PermissionsToMmapFlags(perms), MAP_SHARED | MAP_FIXED, mappedHandle->drm_fd, reinterpret_cast(mappedHandle->drm_cpu_addr)); - assert(ret_cpu_addr == va); - } else { // GPU Memory - int ret; - if (!ldrm_bo) return HSA_STATUS_ERROR; - ret = amdgpu_bo_va_op(ldrm_bo, mappedHandle->offset, mappedHandle->size, - reinterpret_cast(va), drm_perm(perms), AMDGPU_VA_OP_MAP); - if (ret) return HSA_STATUS_ERROR; + if (mapped_ptr != va) + return HSA_STATUS_ERROR; + } else { + hsa_status_t status = targetAgent->driver().Map( + shareable_handle, va, mappedHandle->offset, size, perms); + if (status != HSA_STATUS_SUCCESS) + return status; } permissions = perms; return HSA_STATUS_SUCCESS; } hsa_status_t Runtime::MappedHandleAllowedAgent::RemoveAccess() { - int ret; - - if (!ldrm_bo) // Mapped to host - ret = munmap(va, mappedHandle->size); - else // Mapped to device - ret = amdgpu_bo_va_op(ldrm_bo, mappedHandle->offset, mappedHandle->size, - reinterpret_cast(va), 0, AMDGPU_VA_OP_UNMAP); - - return (ret) ? HSA_STATUS_ERROR : HSA_STATUS_SUCCESS; + if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) { + if (munmap(va, size) != 0) + return HSA_STATUS_ERROR; + return HSA_STATUS_SUCCESS; + } else { + return targetAgent->driver().Unmap( + shareable_handle, va, mappedHandle->offset, mappedHandle->size); + } } // Note: VMemorySetAccessPerHandle should be called with &memory_lock_ held @@ -3430,13 +3413,16 @@ Runtime::VMemorySetAccessPerHandle(void *va, MappedHandle &mappedHandle, auto agentPermsIt = mappedHandle.allowed_agents.find(targetAgent); if (agentPermsIt == mappedHandle.allowed_agents.end()) { /* Agent not previously allowed, we need a new entry */ - mappedHandle.allowed_agents.emplace( - std::piecewise_construct, std::forward_as_tuple(targetAgent), - std::forward_as_tuple(&mappedHandle, targetAgent, va, size, perm)); + agentPermsIt = + mappedHandle.allowed_agents + .emplace(std::piecewise_construct, + std::forward_as_tuple(targetAgent), + std::forward_as_tuple(&mappedHandle, targetAgent, va, + size, perm)) + .first; - if (mappedHandle.allowed_agents[targetAgent].EnableAccess(perm) != - HSA_STATUS_SUCCESS) { - mappedHandle.allowed_agents.erase(targetAgent); + if (agentPermsIt->second.EnableAccess(perm) != HSA_STATUS_SUCCESS) { + mappedHandle.allowed_agents.erase(agentPermsIt); return HSA_STATUS_ERROR; } } else { diff --git a/runtime/hsa-runtime/core/util/memory.h b/runtime/hsa-runtime/core/util/memory.h new file mode 100644 index 0000000000..13d5c19d84 --- /dev/null +++ b/runtime/hsa-runtime/core/util/memory.h @@ -0,0 +1,75 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +// Memory related utility functions. + +#ifndef HSA_RUNTIME_CORE_UTIL_MEMORY_H_ +#define HSA_RUNTIME_CORE_UTIL_MEMORY_H_ + +#ifdef __linux__ +#include "inc/hsa.h" +#include +#endif + +namespace rocr { + +#ifdef __linux__ +/// @brief Converts @ref hsa_access_permission_t to mmap memory protection +/// flags. +__forceinline int PermissionsToMmapFlags(hsa_access_permission_t perms) { + switch (perms) { + case HSA_ACCESS_PERMISSION_RO: + return PROT_READ; + case HSA_ACCESS_PERMISSION_WO: + return PROT_WRITE; + case HSA_ACCESS_PERMISSION_RW: + return PROT_READ | PROT_WRITE; + case HSA_ACCESS_PERMISSION_NONE: + default: + return PROT_NONE; + } +} +#endif + +} // namespace rocr + +#endif // HSA_RUNTIME_CORE_UTIL_MEMORY_H_