AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rocr/aie: Add dma-buf import support for AIEAgents via the Driver interface

Browse Source 
Change-Id: I70f8d8772dda7c06944d75042cb3034ddd89aff4
This commit is contained in:
Yiannis Papadopoulos
2025-01-27 16:26:21 +00:00
committed by David Yat Sin
parent 76052ba028
commit 26bfa0b8f6
8 changed files with 414 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp
+93 -1
View File
@@ -3,7 +3,7 @@
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
@@ -45,11 +45,13 @@
#include <memory>
#include <string>
#include <amdgpu_drm.h>
#include <link.h>
#include <sys/ioctl.h>
#include "hsakmt/hsakmt.h"
#include "core/inc/amd_gpu_agent.h"
#include "core/inc/amd_memory_region.h"
#include "core/inc/runtime.h"
@@ -58,6 +60,29 @@ extern r_debug _amdgpu_r_debug;
namespace rocr {
namespace AMD {
static_assert(
(sizeof(core::ShareableHandle::handle) >= sizeof(amdgpu_bo_handle)) &&
(alignof(core::ShareableHandle::handle) >= alignof(amdgpu_bo_handle)),
"ShareableHandle cannot store a amdgpu_bo_handle");
namespace {
__forceinline uint64_t drm_perm(hsa_access_permission_t perm) {
switch (perm) {
case HSA_ACCESS_PERMISSION_RO:
return AMDGPU_VM_PAGE_READABLE;
case HSA_ACCESS_PERMISSION_WO:
return AMDGPU_VM_PAGE_WRITEABLE;
case HSA_ACCESS_PERMISSION_RW:
return AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE;
case HSA_ACCESS_PERMISSION_NONE:
default:
return 0;
}
}
} // namespace
KfdDriver::KfdDriver(std::string devnode_name)
: core::Driver(core::DriverType::KFD, devnode_name) {}
@@ -322,6 +347,73 @@ hsa_status_t KfdDriver::DestroyQueue(core::Queue &queue) const {
return HSA_STATUS_SUCCESS;
}
hsa_status_t KfdDriver::ExportDMABuf(void *mem, size_t size, int *dmabuf_fd,
size_t *offset) {
int dmabuf_fd_res = -1;
size_t offset_res = 0;
if (hsaKmtExportDMABufHandle(mem, size, &dmabuf_fd_res, &offset_res) !=
HSAKMT_STATUS_SUCCESS)
return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
*dmabuf_fd = dmabuf_fd_res;
*offset = offset_res;
return HSA_STATUS_SUCCESS;
}
hsa_status_t KfdDriver::ImportDMABuf(int dmabuf_fd, core::Agent &agent,
core::ShareableHandle &handle) {
auto &gpu_agent = static_cast<GpuAgent &>(agent);
amdgpu_bo_import_result res;
auto ret = amdgpu_bo_import(
gpu_agent.libDrmDev(), amdgpu_bo_handle_type_dma_buf_fd, dmabuf_fd, &res);
if (ret)
return HSA_STATUS_ERROR;
handle.handle = reinterpret_cast<uint64_t>(res.buf_handle);
return HSA_STATUS_SUCCESS;
}
hsa_status_t KfdDriver::Map(core::ShareableHandle handle, void *mem,
size_t offset, size_t size,
hsa_access_permission_t perms) {
const auto ldrm_bo = reinterpret_cast<amdgpu_bo_handle>(handle.handle);
if (!ldrm_bo)
return HSA_STATUS_ERROR;
if (amdgpu_bo_va_op(ldrm_bo, offset, size, reinterpret_cast<uint64_t>(mem),
drm_perm(perms), AMDGPU_VA_OP_MAP) != 0)
return HSA_STATUS_ERROR;
return HSA_STATUS_SUCCESS;
}
hsa_status_t KfdDriver::Unmap(core::ShareableHandle handle, void *mem,
size_t offset, size_t size) {
const auto ldrm_bo = reinterpret_cast<amdgpu_bo_handle>(handle.handle);
if (!ldrm_bo)
return HSA_STATUS_ERROR;
if (amdgpu_bo_va_op(ldrm_bo, offset, size, reinterpret_cast<uint64_t>(mem), 0,
AMDGPU_VA_OP_UNMAP) != 0)
return HSA_STATUS_ERROR;
return HSA_STATUS_SUCCESS;
}
hsa_status_t KfdDriver::ReleaseShareableHandle(core::ShareableHandle &handle) {
const auto ldrm_bo = reinterpret_cast<amdgpu_bo_handle>(handle.handle);
if (!ldrm_bo)
return HSA_STATUS_ERROR;
const auto ret = amdgpu_bo_free(ldrm_bo);
if (ret)
return HSA_STATUS_ERROR;
handle = {};
return HSA_STATUS_SUCCESS;
}
void *KfdDriver::AllocateKfdMemory(const HsaMemFlags &flags, uint32_t node_id,
size_t size) {
void *mem = nullptr;
runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp
+62 -1
View File
@@ -3,7 +3,7 @@
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
@@ -52,12 +52,17 @@
#include "core/inc/amd_aie_aql_queue.h"
#include "core/inc/amd_memory_region.h"
#include "core/inc/runtime.h"
#include "core/util/memory.h"
#include "core/util/utils.h"
#include "uapi/amdxdna_accel.h"
namespace rocr {
namespace AMD {
static_assert((sizeof(core::ShareableHandle::handle) >= sizeof(uint32_t)) &&
(alignof(core::ShareableHandle::handle) >= alignof(uint32_t)),
"ShareableHandle cannot store a XDNA handle");
XdnaDriver::XdnaDriver(std::string devnode_name)
: core::Driver(core::DriverType::XDNA, devnode_name) {}
@@ -292,6 +297,62 @@ hsa_status_t XdnaDriver::DestroyQueue(core::Queue &queue) const {
return HSA_STATUS_SUCCESS;
}
hsa_status_t XdnaDriver::ExportDMABuf(void *mem, size_t size, int *dmabuf_fd,
size_t *offset) {
// Not implemented yet.
return HSA_STATUS_ERROR;
}
hsa_status_t XdnaDriver::ImportDMABuf(int dmabuf_fd, core::Agent &agent,
core::ShareableHandle &handle) {
drm_prime_handle import_params = {};
import_params.handle = AMDXDNA_INVALID_BO_HANDLE;
import_params.fd = dmabuf_fd;
if (ioctl(fd_, DRM_IOCTL_PRIME_FD_TO_HANDLE, &import_params) < 0)
return HSA_STATUS_ERROR;
handle.handle = import_params.handle;
return HSA_STATUS_SUCCESS;
}
hsa_status_t XdnaDriver::Map(core::ShareableHandle handle, void *mem,
size_t offset, size_t size,
hsa_access_permission_t perms) {
// Get fd associated with the handle.
drm_prime_handle params = {};
params.handle = handle.handle;
params.fd = -1;
if (ioctl(fd_, DRM_IOCTL_PRIME_HANDLE_TO_FD, &params) < 0)
return HSA_STATUS_ERROR;
// Change permissions.
void *mapped_ptr = mmap(mem, size, PermissionsToMmapFlags(perms),
MAP_FIXED | MAP_SHARED, params.fd, offset);
if (mapped_ptr == MAP_FAILED)
return HSA_STATUS_ERROR;
return HSA_STATUS_SUCCESS;
}
hsa_status_t XdnaDriver::Unmap(core::ShareableHandle handle, void *mem,
size_t offset, size_t size) {
if (munmap(mem, size) != 0)
return HSA_STATUS_ERROR;
return HSA_STATUS_SUCCESS;
}
hsa_status_t XdnaDriver::ReleaseShareableHandle(core::ShareableHandle &handle) {
drm_gem_close close_params = {};
close_params.handle = handle.handle;
if (ioctl(fd_, DRM_IOCTL_GEM_CLOSE, &close_params) < 0)
return HSA_STATUS_ERROR;
handle = {};
return HSA_STATUS_SUCCESS;
}
hsa_status_t XdnaDriver::QueryDriverVersion() {
amdxdna_drm_query_aie_version aie_version{0, 0};
amdxdna_drm_get_info args{DRM_AMDXDNA_QUERY_AIE_VERSION, sizeof(aie_version),
runtime/hsa-runtime/core/inc/amd_kfd_driver.h
+10 -1
View File
@@ -3,7 +3,7 @@
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
@@ -87,6 +87,15 @@ public:
hsa_status_t FreeMemory(void *mem, size_t size) override;
hsa_status_t CreateQueue(core::Queue &queue) const override;
hsa_status_t DestroyQueue(core::Queue &queue) const override;
hsa_status_t ExportDMABuf(void *mem, size_t size, int *dmabuf_fd,
size_t *offset) override;
hsa_status_t ImportDMABuf(int dmabuf_fd, core::Agent &agent,
core::ShareableHandle &handle) override;
hsa_status_t Map(core::ShareableHandle handle, void *mem, size_t offset,
size_t size, hsa_access_permission_t perms) override;
hsa_status_t Unmap(core::ShareableHandle handle, void *mem, size_t offset,
size_t size) override;
hsa_status_t ReleaseShareableHandle(core::ShareableHandle &handle) override;
private:
/// @brief Allocate agent accessible memory (system / local memory).
runtime/hsa-runtime/core/inc/amd_xdna_driver.h
+10 -5
View File
@@ -3,7 +3,7 @@
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
@@ -158,12 +158,17 @@ public:
void **mem, size_t size,
uint32_t node_id) override;
hsa_status_t FreeMemory(void *mem, size_t size) override;
/// @brief Creates a context on the AIE device for this queue.
/// @param queue Queue whose on-device context is being created.
/// @return hsa_status_t
hsa_status_t CreateQueue(core::Queue &queue) const override;
hsa_status_t DestroyQueue(core::Queue &queue) const override;
hsa_status_t ExportDMABuf(void *mem, size_t size, int *dmabuf_fd,
size_t *offset) override;
hsa_status_t ImportDMABuf(int dmabuf_fd, core::Agent &agent,
core::ShareableHandle &handle) override;
hsa_status_t Map(core::ShareableHandle handle, void *mem, size_t offset,
size_t size, hsa_access_permission_t perms) override;
hsa_status_t Unmap(core::ShareableHandle handle, void *mem, size_t offset,
size_t size) override;
hsa_status_t ReleaseShareableHandle(core::ShareableHandle &handle) override;
// @brief Submits num_pkts packets in a command chain to the XDNA driver
hsa_status_t SubmitCmdChain(hsa_amd_aie_ert_packet_t* first_pkt, uint32_t num_pkts,
runtime/hsa-runtime/core/inc/driver.h
+53 -4
View File
@@ -3,7 +3,7 @@
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2023, Advanced Micro Devices, Inc. All rights reserved.
// Copyright (c) 2023-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
@@ -59,14 +59,20 @@ enum class DriverQuery { GET_DRIVER_VERSION };
enum class DriverType { XDNA = 0, KFD, NUM_DRIVER_TYPES };
/// @brief Handle for exported / imported memory.
struct ShareableHandle {
uint64_t handle{};
bool IsValid() const { return handle != 0; }
};
/// @brief Kernel driver interface.
///
/// @details A class used to provide an interface between the core runtime
/// and agent kernel drivers. It also maintains state associated with active
/// kernel drivers.
class Driver {
public:
Driver() = delete;
public:
Driver(DriverType kernel_driver_type, std::string devnode_name);
virtual ~Driver() = default;
@@ -126,7 +132,7 @@ class Driver {
/// @brief Allocate agent-accessible memory (system or agent-local memory).
///
/// @param[out] pointer to newly allocated memory.
/// @param[out] mem pointer to newly allocated memory.
///
/// @retval HSA_STATUS_SUCCESS if memory was successfully allocated or
/// hsa_status_t error code if the memory allocation failed.
@@ -141,6 +147,49 @@ class Driver {
virtual hsa_status_t DestroyQueue(Queue &queue) const = 0;
/// @brief Imports memory using dma-buf.
///
/// @param[in] mem virtual address
/// @param[in] size memory size in bytes
/// @param[out] dmabuf_fd dma-buf file descriptor
/// @param[out] offset memory offset in bytes
virtual hsa_status_t ExportDMABuf(void *mem, size_t size, int *dmabuf_fd,
size_t *offset) = 0;
/// @brief Imports a memory chunk via dma-buf.
///
/// @param[in] dmabuf_fd dma-buf file descriptor
/// @param[in] agent agent to import the memory for
/// @param[out] handle handle to the imported memory
virtual hsa_status_t ImportDMABuf(int dmabuf_fd, core::Agent &agent,
core::ShareableHandle &handle) = 0;
/// @brief Maps the memory associated with the handle.
///
/// @param[in] handle handle to the memory object
/// @param[in] mem virtual address associated with the handle
/// @param[in] offset memory offset in bytes
/// @param[in] size memory size in bytes
/// @param[perms] perms new permissions
virtual hsa_status_t Map(core::ShareableHandle handle, void *mem,
size_t offset, size_t size,
hsa_access_permission_t perms) = 0;
/// @brief Unmaps the memory associated with the handle.
///
/// @param[in] handle handle to the memory object
/// @param[in] mem virtual address associated with the handle
/// @param[in] offset memory offset in bytes
/// @param[in] size memory size in bytes
virtual hsa_status_t Unmap(core::ShareableHandle handle, void *mem,
size_t offset, size_t size) = 0;
/// @brief Releases the object associated with the handle.
///
/// @param[in] handle handle of the object to release
virtual hsa_status_t
ReleaseShareableHandle(core::ShareableHandle &handle) = 0;
/// Unique identifier for supported kernel-mode drivers.
const DriverType kernel_driver_type_;
runtime/hsa-runtime/core/inc/runtime.h
+11 -28
View File
@@ -3,7 +3,7 @@
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
// Copyright (c) 2014-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
@@ -776,7 +776,8 @@ class Runtime {
private:
void CheckVirtualMemApiSupport();
int GetAmdgpuDeviceArgs(Agent* agent, amdgpu_bo_handle bo, int* drm_fd, uint64_t* cpu_addr);
int GetAmdgpuDeviceArgs(Agent *agent, ShareableHandle handle, int *drm_fd,
uint64_t *cpu_addr);
bool virtual_mem_api_supported_;
bool xnack_enabled_;
@@ -822,8 +823,6 @@ class Runtime {
struct MappedHandle;
struct MappedHandleAllowedAgent {
MappedHandleAllowedAgent()
: va(NULL), size(0), permissions(HSA_ACCESS_PERMISSION_NONE), mappedHandle(NULL), ldrm_bo(0) {}
MappedHandleAllowedAgent(MappedHandle* _mappedHandle, Agent* targetAgent, void* va, size_t size,
hsa_access_permission_t perms);
~MappedHandleAllowedAgent();
@@ -836,42 +835,26 @@ class Runtime {
Agent* targetAgent;
hsa_access_permission_t permissions;
MappedHandle* mappedHandle;
amdgpu_bo_handle ldrm_bo;
ShareableHandle shareable_handle;
};
struct MappedHandle {
MappedHandle()
: mem_handle(NULL),
address_handle(NULL),
offset(0),
mmap_offset(0),
size(0),
drm_fd(-1),
drm_cpu_addr(NULL),
ldrm_bo(0) {}
MappedHandle(MemoryHandle* mem_handle, AddressHandle* address_handle, uint64_t offset,
size_t size, int drm_fd, void* drm_cpu_addr, hsa_access_permission_t perm,
amdgpu_bo_handle bo)
: mem_handle(mem_handle),
address_handle(address_handle),
offset(offset),
mmap_offset(0),
size(size),
drm_fd(drm_fd),
drm_cpu_addr(drm_cpu_addr),
ldrm_bo(bo) {}
MappedHandle(MemoryHandle *mem_handle, AddressHandle *address_handle,
uint64_t offset, size_t size, int drm_fd, void *drm_cpu_addr,
hsa_access_permission_t perm, ShareableHandle shareable_handle)
: mem_handle(mem_handle), address_handle(address_handle),
offset(offset), size(size), drm_fd(drm_fd),
drm_cpu_addr(drm_cpu_addr), shareable_handle(shareable_handle) {}
__forceinline core::Agent* agentOwner() const { return mem_handle->region->owner(); }
MemoryHandle* mem_handle;
AddressHandle* address_handle;
uint64_t offset;
uint64_t mmap_offset;
size_t size;
int drm_fd;
void* drm_cpu_addr; // CPU Buffer address
amdgpu_bo_handle ldrm_bo;
ShareableHandle shareable_handle;
std::map<Agent*, MappedHandleAllowedAgent> allowed_agents;
};
std::map<const void*, MappedHandle> mapped_handle_map_; // Indexed by VA
runtime/hsa-runtime/core/runtime/runtime.cpp
+100 -114
View File
@@ -3,7 +3,7 @@
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
// Copyright (c) 2014-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
@@ -66,20 +66,21 @@
#endif
#include "core/common/shared.h"
#include "core/inc/hsa_ext_interface.h"
#include "core/inc/amd_core_dump.hpp"
#include "core/inc/amd_cpu_agent.h"
#include "core/inc/amd_gpu_agent.h"
#include "core/inc/amd_memory_region.h"
#include "core/inc/amd_topology.h"
#include "core/inc/signal.h"
#include "core/inc/interrupt_signal.h"
#include "core/inc/hsa_ext_amd_impl.h"
#include "core/inc/hsa_api_trace_int.h"
#include "core/util/os.h"
#include "core/inc/exceptions.h"
#include "inc/hsa_ven_amd_aqlprofile.h"
#include "core/inc/amd_core_dump.hpp"
#include "core/inc/host_queue.h"
#include "core/inc/hsa_api_trace_int.h"
#include "core/inc/hsa_ext_amd_impl.h"
#include "core/inc/hsa_ext_interface.h"
#include "core/inc/interrupt_signal.h"
#include "core/inc/signal.h"
#include "core/util/memory.h"
#include "core/util/os.h"
#include "inc/hsa_ven_amd_aqlprofile.h"
#ifndef HSA_VERSION_MAJOR
#define HSA_VERSION_MAJOR 1
@@ -1977,8 +1978,8 @@ void Runtime::PrintMemoryMapNear(void* ptr) {
info.size = sizeof(info);
for (int i = 0; i < 3; i++) {
if (it == runtime_singleton_->allocation_map_.end()) break;
hsa_status_t err = runtime_singleton_->PtrInfo(const_cast<void*>(it->first), &info,
malloc, &count, &canAccess, &block);
hsa_status_t err = runtime_singleton_->PtrInfo(const_cast<void*>(it->first), &info, malloc,
&count, &canAccess, &block);
if (err == HSA_STATUS_SUCCESS) {
fprintf(stderr, "PtrInfo:\n\tAddress: %p-%p/%p-%p\n\tSize: 0x%lx\n\tType: %u\n\tOwner: %p\n",
info.agentBaseAddress, (char*)info.agentBaseAddress + info.sizeInBytes,
@@ -2221,13 +2222,15 @@ int fn_amdgpu_device_get_fd_nosupport(HsaAMDGPUDeviceHandle device_handle) {
return -1;
}
int Runtime::GetAmdgpuDeviceArgs(Agent* agent, amdgpu_bo_handle bo, int* drm_fd,
uint64_t* cpu_addr) {
int Runtime::GetAmdgpuDeviceArgs(Agent *agent, ShareableHandle handle,
int *drm_fd, uint64_t *cpu_addr) {
int renderFd = fn_amdgpu_device_get_fd(static_cast<AMD::GpuAgent*>(agent)->libDrmDev());
if (renderFd < 0) return HSA_STATUS_ERROR;
uint32_t gem_handle = 0;
if (amdgpu_bo_export(bo, amdgpu_bo_handle_type_kms, &gem_handle)) return HSA_STATUS_ERROR;
if (amdgpu_bo_export(reinterpret_cast<amdgpu_bo_handle>(handle.handle),
amdgpu_bo_handle_type_kms, &gem_handle))
return HSA_STATUS_ERROR;
union drm_amdgpu_gem_mmap args;
memset(&args, 0, sizeof(args));
@@ -3207,47 +3210,12 @@ hsa_status_t Runtime::VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryOnl
return HSA_STATUS_SUCCESS;
}
__forceinline uint64_t drm_perm(hsa_access_permission_t perm) {
switch (perm) {
case HSA_ACCESS_PERMISSION_RO:
return AMDGPU_VM_PAGE_READABLE;
case HSA_ACCESS_PERMISSION_WO:
return AMDGPU_VM_PAGE_WRITEABLE;
case HSA_ACCESS_PERMISSION_RW:
return AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE;
case HSA_ACCESS_PERMISSION_NONE:
return 0;
default:
break;
}
return 0;
}
__forceinline int mmap_perm(hsa_access_permission_t perms) {
switch (perms) {
case HSA_ACCESS_PERMISSION_RO:
return PROT_READ;
case HSA_ACCESS_PERMISSION_WO:
return PROT_WRITE;
case HSA_ACCESS_PERMISSION_RW:
return PROT_READ | PROT_WRITE;
case HSA_ACCESS_PERMISSION_NONE:
return PROT_NONE;
default:
break;
}
return 0;
}
hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset,
hsa_amd_vmem_alloc_handle_t memoryOnlyHandle,
uint64_t flags) {
int drm_fd, dmabuf_fd = 0;
uint64_t offset = 0, ret;
uint64_t drm_cpu_addr = 0;
amdgpu_bo_handle ldrm_bo = 0;
bool reservedAddressFound = false;
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
@@ -3280,25 +3248,38 @@ hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset,
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
}
ret = hsaKmtExportDMABufHandle(memoryHandleIt->first, size, &dmabuf_fd, &offset);
if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
auto *agent = memoryHandleIt->second.agentOwner();
// For now, this is only supported for KFD due to the call to
// GetAmdgpuDeviceArgs
if (agent->device_type() != core::Agent::DeviceType::kAmdGpuDevice)
return HSA_STATUS_ERROR_INVALID_AGENT;
// Create handle by exporting and importing the memory from the owning agent
auto &agent_driver = agent->driver();
hsa_status_t status = agent_driver.ExportDMABuf(memoryHandleIt->first, size,
&dmabuf_fd, &offset);
if (status != HSA_STATUS_SUCCESS)
return status;
assert(offset == 0);
AMD::GpuAgent* agent = static_cast<AMD::GpuAgent*>(memoryHandleIt->second.agentOwner());
amdgpu_bo_import_result res;
ret = amdgpu_bo_import(agent->libDrmDev(), amdgpu_bo_handle_type_dma_buf_fd, dmabuf_fd, &res);
if (ret) return HSA_STATUS_ERROR;
ShareableHandle shareable_handle;
status = agent_driver.ImportDMABuf(dmabuf_fd, *agent, shareable_handle);
if (status != HSA_STATUS_SUCCESS)
return status;
close(dmabuf_fd);
ldrm_bo = res.buf_handle;
ret = GetAmdgpuDeviceArgs(agent, ldrm_bo, &drm_fd, &drm_cpu_addr);
// Get address that memory is mapped to
ret = GetAmdgpuDeviceArgs(agent, shareable_handle, &drm_fd, &drm_cpu_addr);
if (ret) return HSA_STATUS_ERROR;
mapped_handle_map_.emplace(std::piecewise_construct,
std::forward_as_tuple(va),
std::forward_as_tuple(&memoryHandleIt->second, &reservedAddressIt->second, offset, size, drm_fd,
reinterpret_cast<void*>(drm_cpu_addr), HSA_ACCESS_PERMISSION_NONE, ldrm_bo));
mapped_handle_map_.emplace(
std::piecewise_construct, std::forward_as_tuple(va),
std::forward_as_tuple(&memoryHandleIt->second, &reservedAddressIt->second,
offset, size, drm_fd,
reinterpret_cast<void *>(drm_cpu_addr),
HSA_ACCESS_PERMISSION_NONE, shareable_handle));
reservedAddressIt->second.use_count++;
memoryHandleIt->second.use_count++;
@@ -3307,7 +3288,6 @@ hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset,
}
hsa_status_t Runtime::VMemoryHandleUnmap(void* va, size_t size) {
int ret;
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
auto mappedHandleIt = mapped_handle_map_.find(va);
@@ -3315,21 +3295,23 @@ hsa_status_t Runtime::VMemoryHandleUnmap(void* va, size_t size) {
if (mappedHandleIt->second.size != size) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
// Remove access from all agents that were allowed access
for (auto agentPermsIt = mappedHandleIt->second.allowed_agents.begin();
agentPermsIt != mappedHandleIt->second.allowed_agents.end();) {
assert(va == agentPermsIt->second.va);
if (agentPermsIt->second.ldrm_bo)
ret = amdgpu_bo_va_op(agentPermsIt->second.ldrm_bo, mappedHandleIt->second.offset, size,
reinterpret_cast<uint64_t>(va), 0, AMDGPU_VA_OP_UNMAP);
else
ret = munmap(va, size);
if (ret) return HSA_STATUS_ERROR;
hsa_status_t status = agentPermsIt->second.RemoveAccess();
if (status != HSA_STATUS_SUCCESS)
return status;
agentPermsIt = mappedHandleIt->second.allowed_agents.erase(agentPermsIt);
}
if (mappedHandleIt->second.ldrm_bo) {
ret = amdgpu_bo_free(mappedHandleIt->second.ldrm_bo);
if (ret) return HSA_STATUS_ERROR;
if (mappedHandleIt->second.shareable_handle.IsValid()) {
hsa_status_t status = mappedHandleIt->second.agentOwner()->driver().ReleaseShareableHandle(
mappedHandleIt->second.shareable_handle);
if (status != HSA_STATUS_SUCCESS)
return status;
}
assert(mappedHandleIt->second.address_handle->use_count >= 1);
@@ -3350,70 +3332,71 @@ hsa_status_t Runtime::VMemoryHandleUnmap(void* va, size_t size) {
return HSA_STATUS_SUCCESS;
}
Runtime::MappedHandleAllowedAgent::MappedHandleAllowedAgent(MappedHandle* _mappedHandle, Agent* targetAgent, void* va, size_t size,
hsa_access_permission_t perms)
: va(va),
size(size),
targetAgent(targetAgent),
permissions(perms),
mappedHandle(_mappedHandle),
ldrm_bo(NULL) {
Runtime::MappedHandleAllowedAgent::MappedHandleAllowedAgent(
MappedHandle *mappedHandle, Agent *targetAgent, void *va, size_t size,
hsa_access_permission_t perms)
: va(va), size(size), targetAgent(targetAgent), permissions(perms),
mappedHandle(mappedHandle) {
// CPU agents have access as the memory is already mapped to the host.
if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) return;
AMD::GpuAgent* gpuAgent = static_cast<AMD::GpuAgent*>(targetAgent);
int dmabuf_fd = 0;
uint64_t offset = 0;
MemoryHandle *memHandle = mappedHandle->mem_handle;
int ret = hsaKmtExportDMABufHandle(memHandle->thunk_handle, mappedHandle->size, &dmabuf_fd, &offset);
assert(ret == HSAKMT_STATUS_SUCCESS);
if (ret != HSAKMT_STATUS_SUCCESS) return;
// Export memory from owner agent.
hsa_status_t status = memHandle->agentOwner()->driver().ExportDMABuf(
memHandle->thunk_handle, mappedHandle->size, &dmabuf_fd, &offset);
assert(status == HSA_STATUS_SUCCESS);
if (status != HSA_STATUS_SUCCESS)
return;
assert(offset == 0);
amdgpu_bo_import_result res;
ret = amdgpu_bo_import(gpuAgent->libDrmDev(), amdgpu_bo_handle_type_dma_buf_fd, dmabuf_fd, &res);
assert(ret == 0);
if (ret) return;
// Import to target agent.
status = targetAgent->driver().ImportDMABuf(dmabuf_fd, *targetAgent,
shareable_handle);
assert(status == HSA_STATUS_SUCCESS);
if (status != HSA_STATUS_SUCCESS)
return;
close(dmabuf_fd);
ldrm_bo = res.buf_handle;
}
Runtime::MappedHandleAllowedAgent::~MappedHandleAllowedAgent() {
if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) return;
amdgpu_bo_free(ldrm_bo);
hsa_status_t status =
targetAgent->driver().ReleaseShareableHandle(shareable_handle);
assert(status == HSA_STATUS_SUCCESS);
}
hsa_status_t Runtime::MappedHandleAllowedAgent::EnableAccess(hsa_access_permission_t perms) {
if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) {
void* ret_cpu_addr =
mmap(va, size, mmap_perm(perms), MAP_SHARED | MAP_FIXED, mappedHandle->drm_fd,
void* mapped_ptr =
mmap(va, size, PermissionsToMmapFlags(perms), MAP_SHARED | MAP_FIXED, mappedHandle->drm_fd,
reinterpret_cast<uint64_t>(mappedHandle->drm_cpu_addr));
assert(ret_cpu_addr == va);
} else { // GPU Memory
int ret;
if (!ldrm_bo) return HSA_STATUS_ERROR;
ret = amdgpu_bo_va_op(ldrm_bo, mappedHandle->offset, mappedHandle->size,
reinterpret_cast<uint64_t>(va), drm_perm(perms), AMDGPU_VA_OP_MAP);
if (ret) return HSA_STATUS_ERROR;
if (mapped_ptr != va)
return HSA_STATUS_ERROR;
} else {
hsa_status_t status = targetAgent->driver().Map(
shareable_handle, va, mappedHandle->offset, size, perms);
if (status != HSA_STATUS_SUCCESS)
return status;
}
permissions = perms;
return HSA_STATUS_SUCCESS;
}
hsa_status_t Runtime::MappedHandleAllowedAgent::RemoveAccess() {
int ret;
if (!ldrm_bo) // Mapped to host
ret = munmap(va, mappedHandle->size);
else // Mapped to device
ret = amdgpu_bo_va_op(ldrm_bo, mappedHandle->offset, mappedHandle->size,
reinterpret_cast<uint64_t>(va), 0, AMDGPU_VA_OP_UNMAP);
return (ret) ? HSA_STATUS_ERROR : HSA_STATUS_SUCCESS;
if (targetAgent->device_type() == core::Agent::DeviceType::kAmdCpuDevice) {
if (munmap(va, size) != 0)
return HSA_STATUS_ERROR;
return HSA_STATUS_SUCCESS;
} else {
return targetAgent->driver().Unmap(
shareable_handle, va, mappedHandle->offset, mappedHandle->size);
}
}
// Note: VMemorySetAccessPerHandle should be called with &memory_lock_ held
@@ -3430,13 +3413,16 @@ Runtime::VMemorySetAccessPerHandle(void *va, MappedHandle &mappedHandle,
auto agentPermsIt = mappedHandle.allowed_agents.find(targetAgent);
if (agentPermsIt == mappedHandle.allowed_agents.end()) {
/* Agent not previously allowed, we need a new entry */
mappedHandle.allowed_agents.emplace(
std::piecewise_construct, std::forward_as_tuple(targetAgent),
std::forward_as_tuple(&mappedHandle, targetAgent, va, size, perm));
agentPermsIt =
mappedHandle.allowed_agents
.emplace(std::piecewise_construct,
std::forward_as_tuple(targetAgent),
std::forward_as_tuple(&mappedHandle, targetAgent, va,
size, perm))
.first;
if (mappedHandle.allowed_agents[targetAgent].EnableAccess(perm) !=
HSA_STATUS_SUCCESS) {
mappedHandle.allowed_agents.erase(targetAgent);
if (agentPermsIt->second.EnableAccess(perm) != HSA_STATUS_SUCCESS) {
mappedHandle.allowed_agents.erase(agentPermsIt);
return HSA_STATUS_ERROR;
}
} else {
runtime/hsa-runtime/core/util/memory.h
+75
View File
@@ -0,0 +1,75 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
// Memory related utility functions.
#ifndef HSA_RUNTIME_CORE_UTIL_MEMORY_H_
#define HSA_RUNTIME_CORE_UTIL_MEMORY_H_
#ifdef __linux__
#include "inc/hsa.h"
#include <sys/mman.h>
#endif
namespace rocr {
#ifdef __linux__
/// @brief Converts @ref hsa_access_permission_t to mmap memory protection
/// flags.
__forceinline int PermissionsToMmapFlags(hsa_access_permission_t perms) {
switch (perms) {
case HSA_ACCESS_PERMISSION_RO:
return PROT_READ;
case HSA_ACCESS_PERMISSION_WO:
return PROT_WRITE;
case HSA_ACCESS_PERMISSION_RW:
return PROT_READ | PROT_WRITE;
case HSA_ACCESS_PERMISSION_NONE:
default:
return PROT_NONE;
}
}
#endif
} // namespace rocr
#endif // HSA_RUNTIME_CORE_UTIL_MEMORY_H_