libhsakmt/virtio: add virtio support for libhsakmt
This patch adds VirtIO support to the libhsakmt library, enabling communication
with AMD GPUs via VirtIO.
Details
- CMakeLists.txt: Added a new CMakeLists.txt file for the VirtIO component
of libhsakmt.
- hsakmt_virtio.c/h: Implemented the core VirtIO functionality, including
VirtIO GPU device initialization, command execution, and memory management.
- virtio_gpu.c/h: Contains the implementation of the VirtIO GPU device,
including ioctl handling, shared memory management, and command execution.
- hsakmt_virtio_events.c: Implements event handling for VirtIO, such as event
creation, destruction, setting, resetting, and querying event states.
- hsakmt_virtio_memory.c: Manages memory operations for VirtIO, including memory
allocation, freeing, mapping, and unmapping.
- hsakmt_virtio_queues.c: Implements queue management for VirtIO, including
queue creation, destruction, and updating.
- hsakmt_virtio_topology.c: Handles system and node properties for VirtIO.
- hsakmt_virtio_vm.c: Manages VM-related operations for VirtIO, such as
reserving and dereserving VA space.
- include/linux/virtgpu_drm.h: Contains DRM definitions for VirtIO GPU.
Key Features
- VirtIO GPU Initialization: The library can now initialize a VirtIO GPU device
and communicate with it.
- Command Execution: Supports executing commands on the VirtIO GPU device.
- Memory Management: Provides functions for allocating, freeing, mapping, and
unmapping memory for VirtIO operations.
- Event Handling: Implements a comprehensive event system for VirtIO.
- Queue Management: Allows for creating, destroying, and updating queues
on the VirtIO GPU device.
- System and Node Properties: Retrieves and manages system and node
properties for VirtIO.
Signed-off-by: Honglei Huang <Honglei1.Huang@amd.com>
[ROCm/ROCR-Runtime commit: 48d3719dba]
This commit is contained in:
Honglei Huang
committed by
Huang, Honglei1
Huang, Honglei1
parent
2bd0f85f80
commit
5e68bd163a
@@ -108,12 +108,20 @@ if (HSA_DEP_ROCPROFILER_REGISTER)
|
||||
string(APPEND CPACK_RPM_BINARY_PACKAGE_REQUIRES " rocprofiler-register")
|
||||
endif()
|
||||
|
||||
if (NOT DEFINED BUILD_THUNK_VIRTIO)
|
||||
set(BUILD_THUNK_VIRTIO OFF)
|
||||
endif()
|
||||
|
||||
add_rocm_subdir(libhsakmt "${THUNK_DEFINITIONS}")
|
||||
set_target_properties(hsakmt PROPERTIES
|
||||
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/archive"
|
||||
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/lib"
|
||||
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/runtime")
|
||||
|
||||
if (BUILD_THUNK_VIRTIO)
|
||||
add_rocm_subdir(libhsakmt/src/virtio "${THUNK_VIRTIO_DEFINITIONS}")
|
||||
endif()
|
||||
|
||||
if (BUILD_ROCR)
|
||||
add_rocm_subdir(runtime/hsa-runtime "${ROCR_DEFINITIONS}")
|
||||
set_target_properties(hsa-runtime64 PROPERTIES
|
||||
@@ -123,6 +131,9 @@ if (BUILD_ROCR)
|
||||
|
||||
if (BUILD_SHARED_LIBS)
|
||||
add_dependencies(hsa-runtime64 hsakmt)
|
||||
if (BUILD_THUNK_VIRTIO)
|
||||
add_dependencies(hsa-runtime64 hsakmt_virtio)
|
||||
endif()
|
||||
else()
|
||||
add_dependencies(hsa-runtime64 hsakmt-staticdrm)
|
||||
endif()
|
||||
|
||||
@@ -211,7 +211,7 @@ install ( TARGETS ${HSAKMT_TARGET}
|
||||
|
||||
# Install public headers
|
||||
install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/${HSAKMT_TARGET} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
||||
COMPONENT dev PATTERN "linux" EXCLUDE )
|
||||
COMPONENT dev PATTERN "linux" EXCLUDE PATTERN "*virtio*" EXCLUDE)
|
||||
|
||||
# Record our usage data for clients find_package calls.
|
||||
install ( EXPORT ${HSAKMT_TARGET}Targets
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
* Copyright © 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef HSAKMT_VIRTIO_H
|
||||
#define HSAKMT_VIRTIO_H
|
||||
|
||||
#include "hsakmt/linux/kfd_ioctl.h"
|
||||
#include "hsakmt/hsakmt.h"
|
||||
#include <libdrm/amdgpu.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtOpenKFD(void);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCloseKFD(void);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtAllocMemory(HSAuint32 PreferredNode, HSAuint64 SizeInBytes,
|
||||
HsaMemFlags MemFlags, void** MemoryAddress);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtFreeMemory(void* MemoryAddress, HSAuint64 SizeInBytes);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPUNodes(void* MemoryAddress, HSAuint64 MemorySizeInBytes,
|
||||
HSAuint64* AlternateVAGPU,
|
||||
HsaMemMapFlags MemMapFlags,
|
||||
HSAuint64 NumberOfNodes, HSAuint32* NodeArray);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtUnmapMemoryToGPU(void* MemoryAddress);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtAvailableMemory(HSAuint32 Node, HSAuint64* AvailableBytes);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPU(void* MemoryAddress, HSAuint64 MemorySizeInBytes,
|
||||
HSAuint64* AlternateVAGPU);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterMemoryWithFlags(void* MemoryAddress,
|
||||
HSAuint64 MemorySizeInBytes,
|
||||
HsaMemFlags MemFlags);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtDeregisterMemory(void* MemoryAddress);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetVersion(HsaVersionInfo* v);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtAcquireSystemProperties(HsaSystemProperties* SystemProperties);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtReleaseSystemProperties(void);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeProperties(HSAuint32 NodeId,
|
||||
HsaNodeProperties* NodeProperties);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetXNACKMode(HSAint32* enable);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeEnable(void* rDebug, bool setupTtmp);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeDisable(void);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAuint32 NumBanks,
|
||||
HsaMemoryProperties* MemoryProperties);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeCacheProperties(HSAuint32 NodeId, HSAuint32 ProcessorId,
|
||||
HSAuint32 NumCaches,
|
||||
HsaCacheProperties* CacheProperties);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks,
|
||||
HsaIoLinkProperties* IoLinkProperties);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetClockCounters(HSAuint32 NodeId, HsaClockCounters* Counters);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetAMDGPUDeviceHandle(HSAuint32 NodeId,
|
||||
HsaAMDGPUDeviceHandle* DeviceHandle);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryPointerInfo(const void* Pointer, HsaPointerInfo* PointerInfo);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig* config);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateEvent(HsaEventDescriptor* EventDesc, _Bool ManualReset,
|
||||
_Bool IsSignaled, HsaEvent** Event);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyEvent(HsaEvent* Event);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtSetEvent(HsaEvent* Event);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtResetEvent(HsaEvent* Event);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryEventState(HsaEvent* Event);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents(HsaEvent* Events[], HSAuint32 NumEvents,
|
||||
bool WaitOnAll, HSAuint32 Milliseconds);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent(HsaEvent* Event, HSAuint32 Milliseconds);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent_Ext(HsaEvent* Event, HSAuint32 Milliseconds,
|
||||
uint64_t* event_age);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents_Ext(HsaEvent* Events[], HSAuint32 NumEvents,
|
||||
bool WaitOnAll, HSAuint32 Milliseconds,
|
||||
uint64_t* event_age);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtSetTrapHandler(HSAuint32 NodeId, void* TrapHandlerBaseAddress,
|
||||
HSAuint64 TrapHandlerSizeInBytes,
|
||||
void* TrapBufferBaseAddress,
|
||||
HSAuint64 TrapBufferSizeInBytes);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueueExt(HSAuint32 NodeId, HSA_QUEUE_TYPE Type,
|
||||
HSAuint32 QueuePercentage,
|
||||
HSA_QUEUE_PRIORITY Priority, HSAuint32 SdmaEngineId,
|
||||
void* QueueAddress, HSAuint64 QueueSizeInBytes,
|
||||
HsaEvent* Event, HsaQueueResource* QueueResource);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueue(HSAuint32 NodeId, HSA_QUEUE_TYPE Type,
|
||||
HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority,
|
||||
void* QueueAddress, HSAuint64 QueueSizeInBytes,
|
||||
HsaEvent* Event, HsaQueueResource* QueueResource);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyQueue(HSA_QUEUEID QueueId);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterGraphicsHandleToNodes(
|
||||
HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo* GraphicsResourceInfo,
|
||||
HSAuint64 NumberOfNodes, HSAuint32* NodeArray);
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetRuntimeCapabilities(HSAuint32* caps_mask);
|
||||
|
||||
int vamdgpu_query_gpu_info(amdgpu_device_handle dev, void* out);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HSAKMT_VIRTIO_H */
|
||||
@@ -0,0 +1,88 @@
|
||||
|
||||
# Copyright 2025 Advanced Micro Devices, Inc.
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
# OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
cmake_minimum_required ( VERSION 3.7 )
|
||||
|
||||
set (CMAKE_VERBOSE_MAKEFILE ON)
|
||||
|
||||
set ( HSAKMT_VIRTIO "hsakmt_virtio" )
|
||||
set ( HSAKMT_VIRTIO_TARGET "${HSAKMT_VIRTIO}" )
|
||||
|
||||
project ( ${HSAKMT_VIRTIO_TARGET} VERSION 1.0)
|
||||
|
||||
## Compiler flags
|
||||
set ( HSAKMT_VIRTIO_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden )
|
||||
if ( CMAKE_COMPILER_IS_GNUCC )
|
||||
set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -Wlogical-op )
|
||||
endif ()
|
||||
|
||||
set ( HSAKMT_VIRTIO_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/libhsakmt_virtio.ver" )
|
||||
|
||||
set ( HSAKMT_VIRTIO_LINK_FLAGS "-Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_VIRTIO_LINKER_SCRIPT} -Wl,-z,nodelete")
|
||||
|
||||
if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release )
|
||||
set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -O2 )
|
||||
else ()
|
||||
set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -g )
|
||||
endif ()
|
||||
|
||||
set ( HSAKMT_VIRTIO_SRC "virtio_gpu.c"
|
||||
"hsakmt_virtio_vm.c"
|
||||
"hsakmt_virtio_device.c"
|
||||
"hsakmt_virtio_memory.c"
|
||||
"hsakmt_virtio_amdgpu.c"
|
||||
"hsakmt_virtio_events.c"
|
||||
"hsakmt_virtio_queues.c"
|
||||
"hsakmt_virtio_topology.c"
|
||||
"hsakmt_virtio_openclose.c"
|
||||
"../rbtree.c" )
|
||||
|
||||
add_library ( ${HSAKMT_VIRTIO_TARGET} STATIC ${HSAKMT_VIRTIO_SRC} )
|
||||
|
||||
target_sources ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${HSAKMT_VIRTIO_SRC} )
|
||||
|
||||
target_compile_options ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${HSAKMT_VIRTIO_C_FLAGS} )
|
||||
|
||||
target_include_directories ( ${HSAKMT_VIRTIO_TARGET}
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
||||
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
||||
PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/virtio
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../include
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/include/linux )
|
||||
|
||||
set_property(TARGET ${HSAKMT_VIRTIO_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_VIRTIO_LINK_FLAGS})
|
||||
|
||||
find_package ( PkgConfig )
|
||||
|
||||
## If environment variable DRM_DIR is set, the script
|
||||
## will pick up the corresponding libraries from that path.
|
||||
list ( PREPEND CMAKE_PREFIX_PATH "${DRM_DIR}" )
|
||||
|
||||
pkg_check_modules ( DRM REQUIRED IMPORTED_TARGET libdrm )
|
||||
pkg_check_modules ( DRM_AMDGPU REQUIRED IMPORTED_TARGET libdrm_amdgpu )
|
||||
target_include_directories ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${DRM_AMDGPU_INCLUDE_DIRS} )
|
||||
target_include_directories ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${DRM_INCLUDE_DIRS} )
|
||||
|
||||
target_link_libraries ( ${HSAKMT_VIRTIO_TARGET}
|
||||
PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt c ${CMAKE_DL_LIBS} )
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hsakmt/hsakmt_virtio.h"
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
int vamdgpu_query_gpu_info(amdgpu_device_handle handle, void* out) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GPU_INFO,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
int ret = vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
if (!ret) memcpy(out, &rsp->gpu_info, sizeof(struct amdgpu_gpu_info));
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS vhsaKmtGetAMDGPUDeviceHandle(HSAuint32 NodeId, HsaAMDGPUDeviceHandle* DeviceHandle) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* on the rights to use, copy, modify, merge, publish, distribute, sub
|
||||
* license, and/or sell copies of the Software, and to permit persons to whom
|
||||
* the Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
||||
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
int vhsakmt_execbuf_cpu(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, const char* from) {
|
||||
return virtio_gpu_exec_cmd(dev->vgdev, req, true);
|
||||
}
|
||||
|
||||
void* vhsakmt_alloc_rsp(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, uint32_t sz) {
|
||||
return virtio_gpu_alloc_rsp(dev->vgdev, req, sz);
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* on the rights to use, copy, modify, merge, publish, distribute, sub
|
||||
* license, and/or sell copies of the Software, and to permit persons to whom
|
||||
* the Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
||||
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef HSAKMT_VIRTIO_DEVICE_H
|
||||
#define HSAKMT_VIRTIO_DEVICE_H
|
||||
|
||||
#include "hsakmt_virtio_proto.h"
|
||||
#include "rbtree.h"
|
||||
#include "virtio_gpu.h"
|
||||
#include <stdatomic.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define vhsakmt_atomic_inc_return(ptr) (atomic_fetch_add((ptr), 1) + 1)
|
||||
#define vhsakmt_atomic_dec_return(ptr) (atomic_fetch_sub((ptr), 1) - 1)
|
||||
|
||||
#define VHSA_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr))
|
||||
#define VHSA_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v))
|
||||
|
||||
extern int vhsakmt_debug_level;
|
||||
#define vhsakmt_print(level, fmt, ...) \
|
||||
do { \
|
||||
if (level <= vhsakmt_debug_level) fprintf(stderr, fmt, ##__VA_ARGS__); \
|
||||
} while (0)
|
||||
#define VHSAKMT_DEBUG_LEVEL_DEFAULT -1
|
||||
#define VHSAKMT_DEBUG_LEVEL_ERR 3
|
||||
#define VHSAKMT_DEBUG_LEVEL_WARNING 4
|
||||
#define VHSAKMT_DEBUG_LEVEL_INFO 6
|
||||
#define VHSAKMT_DEBUG_LEVEL_DEBUG 7
|
||||
#define vhsa_err(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_ERR, fmt, ##__VA_ARGS__)
|
||||
#define vhsa_warn(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_WARNING, fmt, ##__VA_ARGS__)
|
||||
#define vhsa_info(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__)
|
||||
#define vhsa_debug(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__)
|
||||
|
||||
struct vhsakmt_device;
|
||||
struct vhsakmt_bo;
|
||||
|
||||
typedef struct vhsakmt_device* vhsakmt_device_handle;
|
||||
typedef struct vhsakmt_bo* vhsakmt_bo_handle;
|
||||
typedef rbtree_node_t* bo_entry;
|
||||
|
||||
extern pthread_mutex_t dev_mutex;
|
||||
extern vhsakmt_device_handle dev_list;
|
||||
|
||||
#define VHSA_BO_KFD_MEM 1 << 0 /* allocated from KFD (hsaKmtAllocMemory) */
|
||||
#define VHSA_BO_USERPTR 1 << 1
|
||||
#define VHSA_BO_QUEUE_BUFFER 1 << 2 /* allocated from KFD, but used for queue CMD submit */
|
||||
#define VHSA_BO_QUEUE_DOORBELL 1 << 3 /* doorbell memory */
|
||||
#define VHSA_BO_QUEUE_RW_PTR 1 << 4 /* queue read write ptr, from host map to guest*/
|
||||
/* allocated from KFD, but used for AQL queue read write ptr */
|
||||
#define VHSA_BO_QUEUE_AQL_RW_PTR 1 << 5
|
||||
#define VHSA_BO_CLGL 1 << 6 /* CLGL memory, imported from mesa GL */
|
||||
/* allocated from KFD, but is scratch memory, do not need map and unmap in ioctrl */
|
||||
#define VHSA_BO_SCRATCH 1 << 7
|
||||
#define VHSA_BO_QUEUE 1 << 8
|
||||
#define VHSA_BO_EVENT 1 << 9
|
||||
#define VHSA_BO_SCRATCH_MAP 1 << 10
|
||||
|
||||
#define VHSA_SDMA_NONE UINT32_MAX
|
||||
|
||||
#define CHECK_VIRTIO_KFD_OPEN() \
|
||||
do { \
|
||||
if (dev_list == NULL) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; \
|
||||
} while (0)
|
||||
|
||||
struct vhsakmt_node {
|
||||
HsaNodeProperties node_props;
|
||||
void* doorbell_base;
|
||||
uint64_t scratch_start;
|
||||
uint64_t scratch_size;
|
||||
};
|
||||
|
||||
struct vhsakmt_device {
|
||||
struct virtio_gpu_device* vgdev;
|
||||
int refcount;
|
||||
pthread_mutex_t bo_handles_mutex;
|
||||
rbtree_t bo_rbt;
|
||||
|
||||
struct vhsakmt_bo* shmem_bo;
|
||||
|
||||
uint32_t reqbuf_max;
|
||||
uint32_t next_blob_id;
|
||||
|
||||
uint64_t vm_start;
|
||||
uint64_t vm_size;
|
||||
|
||||
pthread_mutex_t vhsakmt_mutex;
|
||||
struct vhsakmt_node* vhsakmt_nodes;
|
||||
HsaSystemProperties* sys_props;
|
||||
};
|
||||
|
||||
struct vhsakmt_bo {
|
||||
rbtree_node_t rbtn;
|
||||
struct vhsakmt_device* dev;
|
||||
|
||||
int refcount;
|
||||
unsigned size;
|
||||
void* cpu_addr;
|
||||
void* host_addr;
|
||||
HsaMemFlags flags;
|
||||
uint32_t bo_type;
|
||||
uint32_t blob_id;
|
||||
pthread_mutex_t map_mutex;
|
||||
|
||||
union {
|
||||
struct {
|
||||
uint32_t handle;
|
||||
uint32_t res_id;
|
||||
uint64_t offset;
|
||||
uint64_t alloc_size;
|
||||
int map_count;
|
||||
} real;
|
||||
};
|
||||
|
||||
vHsaEvent* event;
|
||||
uint64_t queue_id;
|
||||
vhsakmt_bo_handle rw_bo;
|
||||
void* gl_meta_data;
|
||||
};
|
||||
|
||||
/*hsakmt_virtio_memory.c*/
|
||||
vhsakmt_bo_handle vhsakmt_entry_to_bo_handle(bo_entry e);
|
||||
bo_entry vhsakmt_bo_handle_to_entry(vhsakmt_bo_handle bo);
|
||||
|
||||
void vhsakmt_insert_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo, void* addr, uint64_t size);
|
||||
void vhsakmt_remove_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo);
|
||||
vhsakmt_bo_handle vhsakmt_find_bo_by_addr(vhsakmt_device_handle dev, void* addr);
|
||||
void* vhsakmt_gpu_va(vhsakmt_device_handle dev, void* va);
|
||||
|
||||
int vhsakmt_bo_cpu_unmap(vhsakmt_bo_handle bo);
|
||||
int vhsakmt_bo_cpu_map(vhsakmt_bo_handle bo_handle, void** cpu, void* fixed_cpu);
|
||||
int vhsakmt_create_mappable_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id,
|
||||
uint32_t bo_type, void* va_handle,
|
||||
vhsakmt_bo_handle* bo_handle);
|
||||
int vhsakmt_bo_free(vhsakmt_device_handle dev, vhsakmt_bo_handle bo);
|
||||
int vhsakmt_init_host_blob(vhsakmt_device_handle dev, size_t size, uint32_t blob_type,
|
||||
uint32_t blob_flag, uint32_t blob_id, uint32_t bo_type, void* va_handle,
|
||||
vhsakmt_bo_handle* bo_handle);
|
||||
|
||||
/*hsakmt_virtio_openclose.c*/
|
||||
vhsakmt_device_handle vhsakmt_dev(void);
|
||||
|
||||
/*hsakmt_virtio_vm.c*/
|
||||
void* vhsakmt_vm_start(void);
|
||||
int vhsakmt_reserve_va(uint64_t start, uint64_t size);
|
||||
void vhsakmt_dereserve_va(uint64_t start, uint64_t size);
|
||||
void vhsakmt_set_scratch_area(vhsakmt_device_handle dev, uint32_t node, uint64_t start,
|
||||
uint64_t size);
|
||||
void vhsakmt_set_vm_area(vhsakmt_device_handle dev, uint64_t start, uint64_t size);
|
||||
int vhsakmt_set_node_doorbell(vhsakmt_device_handle dev, uint32_t node, void* doorbell);
|
||||
void* vhsakmt_node_doorbell(vhsakmt_device_handle dev, uint32_t node);
|
||||
bool vhsakmt_is_scratch_mem(vhsakmt_device_handle dev, void* addr);
|
||||
bool vhsakmt_is_userptr(vhsakmt_device_handle dev, void* addr);
|
||||
|
||||
/*hsakmt_virtio_device.c*/
|
||||
int vhsakmt_execbuf_cpu(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, const char* from);
|
||||
void* vhsakmt_alloc_rsp(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, uint32_t sz);
|
||||
|
||||
/*hsakmt_virtio_event.c*/
|
||||
void* vhsakmt_event_host_handle(HsaEvent* h);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,189 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hsakmt/hsakmt_virtio.h"
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
int vhsakmt_debug_level;
|
||||
|
||||
void* vhsakmt_event_host_handle(HsaEvent* h) { return (void*)((vHsaEvent*)h)->event_handle; }
|
||||
|
||||
static inline int32_t vhsakmt_event_res_id(HsaEvent* h) { return ((vHsaEvent*)h)->res_id; }
|
||||
|
||||
static inline vhsakmt_bo_handle vhsakmt_event_bo_handle(HsaEvent* h) {
|
||||
return (vhsakmt_bo_handle)((vHsaEvent*)h)->bo_handle;
|
||||
}
|
||||
|
||||
static int vhsakmt_create_event_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id,
|
||||
vHsaEvent* vevent_handle, vhsakmt_bo_handle* bo_handle) {
|
||||
int r;
|
||||
|
||||
r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, 0, blob_id, VHSA_BO_EVENT,
|
||||
(void*)vevent_handle->event_handle, bo_handle);
|
||||
if (r) return r;
|
||||
|
||||
(*bo_handle)->event = vevent_handle;
|
||||
vevent_handle->bo_handle = (uint64_t)(*bo_handle);
|
||||
vevent_handle->res_id = (*bo_handle)->real.res_id;
|
||||
vhsakmt_insert_bo(dev, *bo_handle, vevent_handle, size);
|
||||
return r;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateEvent(HsaEventDescriptor* EventDesc, _Bool ManualReset,
|
||||
_Bool IsSignaled, HsaEvent** Event) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_event_rsp* rsp;
|
||||
vhsakmt_bo_handle event_bo;
|
||||
vHsaEvent* e;
|
||||
int r;
|
||||
struct vhsakmt_ccmd_event_req req = {
|
||||
.hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)),
|
||||
.type = VHSAKMT_CCMD_EVENT_CREATE,
|
||||
.create_args.EventDesc = *EventDesc,
|
||||
.create_args.ManualReset = ManualReset,
|
||||
.create_args.IsSignaled = IsSignaled,
|
||||
.blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id),
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (rsp->ret) return rsp->ret;
|
||||
|
||||
e = calloc(1, sizeof(vHsaEvent));
|
||||
if (!e) return -ENOMEM;
|
||||
|
||||
memcpy(e, &rsp->vevent, sizeof(vHsaEvent));
|
||||
|
||||
r = vhsakmt_create_event_blob_bo(dev, sizeof(vHsaEvent), req.blob_id, e, &event_bo);
|
||||
if (r) {
|
||||
free(e);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
*Event = (HsaEvent*)e;
|
||||
|
||||
vhsa_debug(
|
||||
"%s: event addr: %p, hw123: %lx, %lx, %x, type: %d, id: %x, host handle: 0x%lx, res id: %d\n",
|
||||
__FUNCTION__, e, e->event.EventData.HWData1, e->event.EventData.HWData2,
|
||||
e->event.EventData.HWData3, e->event.EventData.EventType, e->event.EventId, e->event_handle,
|
||||
event_bo->real.res_id);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyEvent(HsaEvent* Event) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_bo* bo;
|
||||
|
||||
if (Event == NULL) return HSAKMT_STATUS_SUCCESS;
|
||||
|
||||
bo = vhsakmt_event_bo_handle(Event);
|
||||
if (!bo) return HSAKMT_STATUS_SUCCESS;
|
||||
|
||||
return vhsakmt_bo_free(dev, bo);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtSetEvent(HsaEvent* Event) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_event_rsp* rsp;
|
||||
struct vhsakmt_ccmd_event_req req = {
|
||||
.hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)),
|
||||
.type = VHSAKMT_CCMD_EVENT_SET,
|
||||
.event_hanele = vhsakmt_event_host_handle(Event),
|
||||
.res_id = vhsakmt_event_res_id(Event),
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtResetEvent(HsaEvent* Event) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_event_rsp* rsp;
|
||||
struct vhsakmt_ccmd_event_req req = {
|
||||
.hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)),
|
||||
.type = VHSAKMT_CCMD_EVENT_RESET,
|
||||
.event_hanele = vhsakmt_event_host_handle(Event),
|
||||
.res_id = vhsakmt_event_res_id(Event),
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryEventState(HsaEvent* Event) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_event_rsp* rsp;
|
||||
struct vhsakmt_ccmd_event_req req = {
|
||||
.hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)),
|
||||
.type = VHSAKMT_CCMD_EVENT_QUERY_STATE,
|
||||
.event_hanele = vhsakmt_event_host_handle(Event),
|
||||
.res_id = vhsakmt_event_res_id(Event),
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents(HsaEvent* Events[], HSAuint32 NumEvents,
|
||||
bool WaitOnAll, HSAuint32 Milliseconds) {
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent(HsaEvent* Event, HSAuint32 Milliseconds) {
|
||||
return vhsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent_Ext(HsaEvent* Event, HSAuint32 Milliseconds,
|
||||
uint64_t* event_age) {
|
||||
return vhsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents_Ext(HsaEvent* Events[], HSAuint32 NumEvents,
|
||||
bool WaitOnAll, HSAuint32 Milliseconds,
|
||||
uint64_t* event_age) {
|
||||
return vhsaKmtWaitOnMultipleEvents(Events, NumEvents, WaitOnAll, Milliseconds);
|
||||
}
|
||||
@@ -0,0 +1,852 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hsakmt/hsakmt_virtio.h"
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
#define VHSA_GL_METADATA_MAX_SIZE (0x50)
|
||||
|
||||
vhsakmt_bo_handle vhsakmt_entry_to_bo_handle(bo_entry e) { return (vhsakmt_bo_handle)e; }
|
||||
bo_entry vhsakmt_bo_handle_to_entry(vhsakmt_bo_handle bo) { return &bo->rbtn; }
|
||||
static inline bool vhsakmt_is_mem_bo(vhsakmt_bo_handle bo) { return (!bo->queue_id && !bo->event); }
|
||||
|
||||
static bool vhsakmt_mappable(HsaMemFlags flags) { return (!flags.ui32.Scratch); }
|
||||
|
||||
static bool vhsakmt_bo_mappable(vhsakmt_bo_handle bo) { return vhsakmt_mappable(bo->flags); }
|
||||
|
||||
void vhsakmt_insert_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo, void* addr, uint64_t size) {
|
||||
bo->rbtn.key.addr = (unsigned long)addr;
|
||||
bo->rbtn.key.size = (unsigned long)size;
|
||||
|
||||
pthread_mutex_lock(&dev->bo_handles_mutex);
|
||||
hsakmt_rbtree_insert(&dev->bo_rbt, &bo->rbtn);
|
||||
pthread_mutex_unlock(&dev->bo_handles_mutex);
|
||||
}
|
||||
|
||||
static void vhsakmt_remove_entry(vhsakmt_device_handle dev, bo_entry entry) {
|
||||
if (!entry) return;
|
||||
|
||||
pthread_mutex_lock(&dev->bo_handles_mutex);
|
||||
hsakmt_rbtree_delete(&dev->bo_rbt, entry);
|
||||
pthread_mutex_unlock(&dev->bo_handles_mutex);
|
||||
}
|
||||
|
||||
void vhsakmt_remove_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) {
|
||||
bo_entry entry = vhsakmt_bo_handle_to_entry(bo);
|
||||
if (entry->key.addr == 0 && entry->key.size == 0) return;
|
||||
|
||||
vhsakmt_remove_entry(dev, entry);
|
||||
}
|
||||
|
||||
static bo_entry vhsakmt_rbt_search(vhsakmt_device_handle dev, void* addr) {
|
||||
vhsakmt_bo_handle bo;
|
||||
|
||||
rbtree_key_t key = rbtree_key((uint64_t)addr, 0);
|
||||
pthread_mutex_lock(&dev->bo_handles_mutex);
|
||||
bo_entry n = rbtree_lookup_nearest(&dev->bo_rbt, &key, LKP_ADDR, RIGHT);
|
||||
pthread_mutex_unlock(&dev->bo_handles_mutex);
|
||||
if (n) {
|
||||
bo = vhsakmt_entry_to_bo_handle(n);
|
||||
if (bo->cpu_addr != addr) return NULL;
|
||||
return n;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static bo_entry vhsakmt_find_entry_by_addr(vhsakmt_device_handle dev, void* addr) {
|
||||
return vhsakmt_rbt_search(dev, addr);
|
||||
}
|
||||
|
||||
vhsakmt_bo_handle vhsakmt_find_bo_by_addr(vhsakmt_device_handle dev, void* addr) {
|
||||
bo_entry entry = vhsakmt_find_entry_by_addr(dev, addr);
|
||||
|
||||
if (entry) {
|
||||
vhsakmt_bo_handle bo = vhsakmt_entry_to_bo_handle(entry);
|
||||
if (!vhsakmt_is_mem_bo(bo)) return NULL;
|
||||
|
||||
return bo;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void* vhsakmt_gpu_va(vhsakmt_device_handle dev, void* va) {
|
||||
if (!vhsakmt_is_userptr(dev, va)) return va;
|
||||
|
||||
bo_entry entry = vhsakmt_find_entry_by_addr(dev, va);
|
||||
|
||||
if (!entry) return NULL;
|
||||
|
||||
return vhsakmt_entry_to_bo_handle(entry)->host_addr;
|
||||
}
|
||||
|
||||
int vhsakmt_bo_cpu_map(vhsakmt_bo_handle bo, void** cpu, void* fixed_cpu) {
|
||||
int r;
|
||||
|
||||
if (!vhsakmt_bo_mappable(bo)) return 0;
|
||||
|
||||
pthread_mutex_lock(&bo->map_mutex);
|
||||
|
||||
if (!bo->cpu_addr) {
|
||||
r = virtio_gpu_map_handle(bo->dev->vgdev, bo->real.handle, bo->size, cpu, fixed_cpu);
|
||||
if (r) {
|
||||
pthread_mutex_unlock(&bo->map_mutex);
|
||||
return r;
|
||||
}
|
||||
bo->cpu_addr = *cpu;
|
||||
atomic_fetch_add(&bo->real.map_count, 1);
|
||||
}
|
||||
pthread_mutex_unlock(&bo->map_mutex);
|
||||
|
||||
return *cpu == MAP_FAILED;
|
||||
}
|
||||
|
||||
int vhsakmt_bo_cpu_unmap(vhsakmt_bo_handle bo) {
|
||||
int r = 0;
|
||||
|
||||
if (!vhsakmt_bo_mappable(bo)) return 0;
|
||||
|
||||
pthread_mutex_lock(&bo->map_mutex);
|
||||
|
||||
if (!bo->cpu_addr || bo->real.map_count == 0) {
|
||||
pthread_mutex_unlock(&bo->map_mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (vhsakmt_atomic_dec_return(&bo->real.map_count) <= 0) {
|
||||
if (bo->bo_type & VHSA_BO_KFD_MEM) {
|
||||
virtio_gpu_unmap(bo->cpu_addr, bo->size);
|
||||
vhsakmt_reserve_va(VHSA_VPTR_TO_UINT64(bo->cpu_addr), bo->size);
|
||||
bo->cpu_addr = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&bo->map_mutex);
|
||||
return r;
|
||||
}
|
||||
|
||||
static int vhsakmt_destroy_handle(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) {
|
||||
int r = virtio_gpu_destroy_handle(dev->vgdev, bo->real.handle);
|
||||
free(bo);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
int vhsakmt_init_host_blob(vhsakmt_device_handle dev, size_t size, uint32_t blob_type,
|
||||
uint32_t blob_flag, uint32_t blob_id, uint32_t bo_type, void* va_handle,
|
||||
vhsakmt_bo_handle* bo_handle) {
|
||||
int r;
|
||||
vhsakmt_bo_handle bo;
|
||||
struct drm_virtgpu_resource_create_blob args = {
|
||||
.blob_mem = blob_type,
|
||||
.size = size,
|
||||
.blob_id = blob_id,
|
||||
.blob_flags = blob_flag,
|
||||
};
|
||||
|
||||
r = virtio_gpu_create_blob(dev->vgdev, &args);
|
||||
if (r) return -EINVAL;
|
||||
|
||||
bo = calloc(1, sizeof(struct vhsakmt_bo));
|
||||
if (!bo) {
|
||||
virtio_gpu_destroy_handle(dev->vgdev, args.bo_handle);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
bo->dev = dev;
|
||||
bo->size = size;
|
||||
bo->real.alloc_size = size;
|
||||
bo->bo_type = bo_type;
|
||||
bo->host_addr = va_handle;
|
||||
pthread_mutex_init(&bo->map_mutex, NULL);
|
||||
atomic_store(&bo->real.map_count, 0);
|
||||
atomic_store(&bo->refcount, 1);
|
||||
bo->real.handle = args.bo_handle;
|
||||
|
||||
virtio_gpu_res_id(dev->vgdev, bo->real.handle, &bo->real.res_id);
|
||||
|
||||
*bo_handle = bo;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vhsakmt_init_userptr_blob(vhsakmt_device_handle dev, void* addr, size_t size,
|
||||
vhsakmt_bo_handle* bo_handle, uint64_t* offset) {
|
||||
int r;
|
||||
struct drm_virtgpu_resource_create_blob args = {
|
||||
.blob_mem = VIRTGPU_BLOB_MEM_HOST3D_GUEST,
|
||||
.blob_flags = VIRTGPU_BLOB_FLAG_USE_USERPTR,
|
||||
.size = size,
|
||||
.blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id),
|
||||
.blob_userptr = (uint64_t)addr,
|
||||
};
|
||||
|
||||
r = virtio_gpu_create_blob(dev->vgdev, &args);
|
||||
if (r < 0) return r;
|
||||
|
||||
vhsakmt_bo_handle userptr = calloc(1, sizeof(struct vhsakmt_bo));
|
||||
if (!userptr) {
|
||||
virtio_gpu_destroy_handle(dev->vgdev, args.bo_handle);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
userptr->dev = dev;
|
||||
userptr->size = size;
|
||||
userptr->real.alloc_size = size;
|
||||
userptr->bo_type = VHSA_BO_USERPTR;
|
||||
userptr->cpu_addr = addr;
|
||||
pthread_mutex_init(&userptr->map_mutex, NULL);
|
||||
atomic_store(&userptr->real.map_count, 0);
|
||||
atomic_store(&userptr->refcount, 1);
|
||||
userptr->real.handle = args.bo_handle;
|
||||
|
||||
virtio_gpu_res_id(dev->vgdev, userptr->real.handle, &userptr->real.res_id);
|
||||
|
||||
*bo_handle = userptr;
|
||||
*offset = args.offset;
|
||||
return r;
|
||||
}
|
||||
|
||||
int vhsakmt_create_mappable_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id,
|
||||
uint32_t bo_type, void* va_handle,
|
||||
vhsakmt_bo_handle* bo_handle) {
|
||||
int r;
|
||||
|
||||
r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, VIRTGPU_BLOB_FLAG_USE_MAPPABLE,
|
||||
blob_id, bo_type, va_handle, bo_handle);
|
||||
if (r) return r;
|
||||
|
||||
r = vhsakmt_bo_cpu_map(*bo_handle, &((*bo_handle)->cpu_addr), va_handle);
|
||||
if (r) {
|
||||
free(*bo_handle);
|
||||
*bo_handle = NULL;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (va_handle && (va_handle != (*bo_handle)->cpu_addr))
|
||||
vhsa_warn("%s: target map: %p != real map: %p\n", __FUNCTION__, va_handle,
|
||||
(*bo_handle)->cpu_addr);
|
||||
|
||||
vhsakmt_insert_bo(dev, *bo_handle, (*bo_handle)->cpu_addr, (*bo_handle)->size);
|
||||
return r;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtAllocMemory(HSAuint32 PreferredNode, HSAuint64 SizeInBytes,
|
||||
HsaMemFlags MemFlags, void** MemoryAddress) {
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp;
|
||||
vhsakmt_bo_handle bo;
|
||||
int r;
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_ALLOC,
|
||||
.blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id),
|
||||
.alloc_args =
|
||||
{
|
||||
.PreferredNode = PreferredNode,
|
||||
.SizeInBytes = SizeInBytes,
|
||||
.MemFlags = MemFlags,
|
||||
},
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (rsp->ret) return rsp->ret;
|
||||
|
||||
if (!rsp->memory_handle) return -ENOMEM;
|
||||
|
||||
r = vhsakmt_init_host_blob(dev, SizeInBytes, VIRTGPU_BLOB_MEM_HOST3D,
|
||||
vhsakmt_mappable(MemFlags) ? VIRTGPU_BLOB_FLAG_USE_MAPPABLE : 0,
|
||||
req.blob_id, VHSA_BO_KFD_MEM, (void*)rsp->memory_handle, &bo);
|
||||
if (r) return r;
|
||||
|
||||
if (!vhsakmt_mappable(MemFlags)) {
|
||||
bo->cpu_addr = bo->host_addr;
|
||||
if (MemFlags.ui32.Scratch) {
|
||||
vhsakmt_set_scratch_area(dev, PreferredNode, (uint64_t)bo->cpu_addr, SizeInBytes);
|
||||
bo->bo_type |= VHSA_BO_SCRATCH;
|
||||
}
|
||||
} else {
|
||||
r = vhsakmt_bo_cpu_map(bo, &bo->cpu_addr, bo->host_addr);
|
||||
if (r) {
|
||||
free(bo);
|
||||
return -ENOMEM;
|
||||
}
|
||||
}
|
||||
|
||||
if (!MemFlags.ui32.Scratch) vhsakmt_insert_bo(dev, bo, bo->cpu_addr, bo->size);
|
||||
|
||||
*MemoryAddress = bo->cpu_addr;
|
||||
|
||||
vhsa_debug("alloc mem addr: %p, host addr: %p, size: %lx, res-id: %d, handble: %d\n",
|
||||
*MemoryAddress, bo->host_addr, SizeInBytes, bo->real.res_id, bo->real.handle);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
int vhsakmt_bo_free(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) {
|
||||
bo_entry entry;
|
||||
int r;
|
||||
|
||||
if (vhsakmt_atomic_dec_return(&bo->refcount) > 0) return 0;
|
||||
|
||||
entry = vhsakmt_bo_handle_to_entry(bo);
|
||||
if (entry->key.addr == 0 && entry->key.size == 0) return -EINVAL;
|
||||
|
||||
/* do not free BOs of queue, let them be freed with queue */
|
||||
if (bo->bo_type & VHSA_BO_QUEUE_DOORBELL) {
|
||||
vhsa_err("%s: Try to free VHSA_BO_QUEUE_DOORBELL memory: %p\n", __FUNCTION__, bo->cpu_addr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
vhsakmt_remove_bo(dev, bo);
|
||||
|
||||
if (bo->cpu_addr) vhsakmt_bo_cpu_unmap(bo);
|
||||
|
||||
if (bo->event) free(bo->event);
|
||||
|
||||
if (bo->gl_meta_data) free(bo->gl_meta_data);
|
||||
|
||||
pthread_mutex_destroy(&bo->map_mutex);
|
||||
|
||||
r = vhsakmt_destroy_handle(dev, bo);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
/* Only remove bo in rbtree */
|
||||
static void vhsakmt_remove_userptr_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) {
|
||||
vhsakmt_remove_bo(dev, bo);
|
||||
free(bo);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtFreeMemory(void* MemoryAddress, HSAuint64 SizeInBytes) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress);
|
||||
if (!bo) return HSAKMT_STATUS_SUCCESS;
|
||||
|
||||
vhsa_debug("%s: addr: %p, size: %lx, res_id: %d\n", __FUNCTION__, MemoryAddress, SizeInBytes,
|
||||
bo->real.res_id);
|
||||
|
||||
return vhsakmt_bo_free(dev, bo);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPUNodes(void* MemoryAddress, HSAuint64 MemorySizeInBytes,
|
||||
HSAuint64* AlternateVAGPU,
|
||||
HsaMemMapFlags MemMapFlags,
|
||||
HSAuint64 NumberOfNodes, HSAuint32* NodeArray) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
size_t req_len =
|
||||
VHSA_ALIGN_UP(sizeof(struct vhsakmt_ccmd_memory_req) + NumberOfNodes * sizeof(*NodeArray), 8);
|
||||
struct vhsakmt_ccmd_memory_req* req;
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp;
|
||||
vhsakmt_bo_handle bo;
|
||||
|
||||
req = (void*)calloc(1, req_len);
|
||||
if (!req) return -ENOMEM;
|
||||
req->hdr = VHSAKMT_CCMD(MEMORY, req_len);
|
||||
req->type = VHSAKMT_CCMD_MEMORY_MAP_TO_GPU_NODES;
|
||||
req->map_to_GPU_nodes_args.MemorySizeInBytes = MemorySizeInBytes;
|
||||
req->map_to_GPU_nodes_args.MemMapFlags = MemMapFlags;
|
||||
req->map_to_GPU_nodes_args.NumberOfNodes = NumberOfNodes;
|
||||
|
||||
bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress);
|
||||
if (bo) {
|
||||
req->map_to_GPU_nodes_args.MemoryAddress = (uint64_t)bo->host_addr;
|
||||
if (bo->bo_type & VHSA_BO_USERPTR) vhsakmt_remove_userptr_bo(dev, bo);
|
||||
} else
|
||||
req->map_to_GPU_nodes_args.MemoryAddress = (uint64_t)MemoryAddress;
|
||||
|
||||
memcpy(req->payload, NodeArray, NumberOfNodes * sizeof(*NodeArray));
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req->hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) {
|
||||
free(req);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req->hdr, __FUNCTION__);
|
||||
|
||||
*AlternateVAGPU = rsp->alternate_vagpu;
|
||||
|
||||
vhsa_debug("%s: gva: %p, hva: 0x%lx, size: %lx, AlternateVAGPU: %lx, ret: %d\n", __FUNCTION__,
|
||||
MemoryAddress, req->map_to_GPU_nodes_args.MemoryAddress, MemorySizeInBytes,
|
||||
*AlternateVAGPU, rsp->ret);
|
||||
|
||||
free(req);
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtUnmapMemoryToGPU(void* MemoryAddress) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress);
|
||||
if (!bo) return HSAKMT_STATUS_SUCCESS;
|
||||
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp;
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_UNMAP_TO_GPU,
|
||||
.MemoryAddress = (uint64_t)bo->host_addr,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
vhsa_debug("%s: gva: %p, hva: 0x%lx\n", __FUNCTION__, MemoryAddress, req.MemoryAddress);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtAvailableMemory(HSAuint32 Node, HSAuint64* AvailableBytes) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp;
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_AVAIL_MEM,
|
||||
.Node = Node,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
*AvailableBytes = rsp->available_bytes;
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
static int vhsakmt_create_scratch_map_memory(vhsakmt_device_handle dev, void* MemoryAddress,
|
||||
HSAuint64 MemorySizeInBytes,
|
||||
HSAuint64* AlternateVAGPU) {
|
||||
vhsakmt_bo_handle out;
|
||||
int r;
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU,
|
||||
.blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id),
|
||||
.map_to_GPU_args =
|
||||
{
|
||||
.MemoryAddress = (uint64_t)MemoryAddress,
|
||||
.MemorySizeInBytes = MemorySizeInBytes,
|
||||
.need_create_bo = true,
|
||||
},
|
||||
};
|
||||
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp =
|
||||
vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
if (rsp->ret) return rsp->ret;
|
||||
|
||||
r = vhsakmt_init_host_blob(dev, MemorySizeInBytes, VIRTGPU_BLOB_MEM_HOST3D, 0, req.blob_id,
|
||||
VHSA_BO_SCRATCH_MAP, NULL, &out);
|
||||
if (r) return r;
|
||||
|
||||
// TODO: insert scratch bo into rbtree, or insert it in dev nodes.
|
||||
|
||||
out->cpu_addr = MemoryAddress;
|
||||
out->host_addr = (void*)rsp->memory_handle;
|
||||
*AlternateVAGPU = rsp->alternate_vagpu;
|
||||
|
||||
vhsa_debug(
|
||||
"%s: create scratch memory, gva: %p, memory_handle: 0x%p, alternate_vagpu: %p, size: %lx\n",
|
||||
__FUNCTION__, MemoryAddress, (void*)rsp->memory_handle, (void*)rsp->alternate_vagpu,
|
||||
MemorySizeInBytes);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPU(void* MemoryAddress, HSAuint64 MemorySizeInBytes,
|
||||
HSAuint64* AlternateVAGPU) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp;
|
||||
vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress);
|
||||
if (!bo && vhsakmt_is_scratch_mem(dev, MemoryAddress))
|
||||
return vhsakmt_create_scratch_map_memory(dev, MemoryAddress, MemorySizeInBytes, AlternateVAGPU);
|
||||
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU,
|
||||
.map_to_GPU_args =
|
||||
{
|
||||
.MemoryAddress = bo ? (uint64_t)bo->host_addr : (uint64_t)MemoryAddress,
|
||||
.MemorySizeInBytes = MemorySizeInBytes,
|
||||
},
|
||||
};
|
||||
|
||||
if (bo && (bo->bo_type & VHSA_BO_USERPTR)) vhsakmt_remove_userptr_bo(dev, bo);
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
vhsa_debug("%s: gva: %p, hva: 0x%lx, size: %lx\n", __FUNCTION__, MemoryAddress, req.MemoryAddress,
|
||||
MemorySizeInBytes);
|
||||
|
||||
*AlternateVAGPU = rsp->alternate_vagpu;
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
static int vhsakmt_map_userptr(vhsakmt_device_handle dev, void* addr, size_t size, uint32_t res_id,
|
||||
uint64_t* userptr_handle) {
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_MAP_USERPTR,
|
||||
.res_id = res_id,
|
||||
};
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp =
|
||||
vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
rsp->map_userptr_rsp.userptr_handle = 0;
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
*userptr_handle = rsp->map_userptr_rsp.userptr_handle;
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
static void* vhsakmt_map_to_gpu(void* addr, size_t size) {
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
size_t offset = (uint64_t)addr % getpagesize();
|
||||
size_t map_size = (VHSA_ALIGN_UP(size + offset, getpagesize()) / getpagesize()) * getpagesize();
|
||||
uint64_t userptr_offset, userptr_handle = 0;
|
||||
vhsakmt_bo_handle userptr;
|
||||
int r;
|
||||
|
||||
vhsa_debug("%s: addr: %p, size: 0x%lx, size + offset: 0x%lx, map_size: 0x%lx\n", __FUNCTION__,
|
||||
addr, size, size + offset, map_size);
|
||||
|
||||
r = vhsakmt_init_userptr_blob(dev, addr, size, &userptr, &userptr_offset);
|
||||
if (r < 0) {
|
||||
vhsa_debug("%s: userptr create failed at address: %p, ret = %d\n", __FUNCTION__, addr, r);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
vhsakmt_map_userptr(dev, addr, size, userptr->real.res_id, &userptr_handle);
|
||||
if (!userptr_handle) {
|
||||
vhsa_debug("%s: map userptr failed at address: %p, ret = %d\n", __FUNCTION__, addr, r);
|
||||
vhsakmt_destroy_handle(dev, userptr);
|
||||
vhsakmt_remove_userptr_bo(dev, userptr);
|
||||
return NULL;
|
||||
}
|
||||
userptr->host_addr = VHSA_UINT64_TO_VPTR(VHSA_VPTR_TO_UINT64(userptr_handle) + offset);
|
||||
|
||||
if (r > 0) {
|
||||
vhsa_debug("%s: userptr: %p already registered, offset: %lx\n", __FUNCTION__, addr,
|
||||
userptr_offset);
|
||||
userptr->host_addr =
|
||||
VHSA_UINT64_TO_VPTR(VHSA_VPTR_TO_UINT64(userptr->host_addr) + userptr_offset);
|
||||
}
|
||||
vhsakmt_insert_bo(dev, userptr, userptr->cpu_addr, userptr->size);
|
||||
|
||||
vhsa_debug("%s: real gva: %p, gva: %p, hva: %p, size: %lx, offset: %" PRIu64
|
||||
", map_size: 0x%lx\n",
|
||||
__FUNCTION__, addr, userptr->cpu_addr, userptr->host_addr, size, offset, map_size);
|
||||
return userptr->host_addr;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterMemoryWithFlags(void* MemoryAddress,
|
||||
HSAuint64 MemorySizeInBytes,
|
||||
HsaMemFlags MemFlags) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp;
|
||||
void* addr;
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_REG_MEM_WITH_FLAG,
|
||||
.reg_mem_with_flag =
|
||||
{
|
||||
.MemorySizeInBytes = MemorySizeInBytes,
|
||||
.MemFlags = MemFlags,
|
||||
},
|
||||
};
|
||||
|
||||
/* no need to register memory from lihsakmt / not a userptr */
|
||||
if (!vhsakmt_is_userptr(dev, MemoryAddress)) return HSAKMT_STATUS_SUCCESS;
|
||||
|
||||
addr = vhsakmt_map_to_gpu(MemoryAddress, MemorySizeInBytes);
|
||||
if (!addr) {
|
||||
vhsa_debug("%s: register memory failed, gva: %p, size: %lx\n", __FUNCTION__, MemoryAddress,
|
||||
MemorySizeInBytes);
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
req.reg_mem_with_flag.MemoryAddress = (uint64_t)addr;
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
static int vhsakmt_remove_clgl_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) {
|
||||
struct vhsakmt_ccmd_memory_rsp* rsp;
|
||||
struct vhsakmt_ccmd_memory_req req = {
|
||||
.hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)),
|
||||
.type = VHSAKMT_CCMD_MEMORY_DEREG_MEM,
|
||||
.res_id = bo->real.res_id,
|
||||
.MemoryAddress = (uint64_t)bo->cpu_addr,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (rsp->ret) vhsa_err("%s: deregister failed clgl memory gva: %p\n", __FUNCTION__, bo->cpu_addr);
|
||||
|
||||
vhsakmt_bo_free(dev, bo);
|
||||
|
||||
vhsa_debug("%s: deregister clgl memory gva: %p, ret: %d\n", __FUNCTION__, bo->cpu_addr, rsp->ret);
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtDeregisterMemory(void* MemoryAddress) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress);
|
||||
if (!bo) return HSAKMT_STATUS_SUCCESS;
|
||||
|
||||
vhsa_debug("%s: remove userptr %p size: 0x%lx, res id: %d\n", __FUNCTION__, MemoryAddress,
|
||||
(size_t)bo->size, bo->real.res_id);
|
||||
|
||||
if (bo->bo_type & VHSA_BO_CLGL)
|
||||
return vhsakmt_remove_clgl_bo(dev, bo);
|
||||
else {
|
||||
vhsakmt_remove_bo(dev, bo);
|
||||
free(bo);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryPointerInfo(const void* Pointer, HsaPointerInfo* PointerInfo) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
void* gpu_va = vhsakmt_gpu_va(dev, VHSA_UINT64_TO_VPTR(Pointer));
|
||||
if (!gpu_va) return -HSAKMT_STATUS_ERROR;
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_POINTER_INFO,
|
||||
.pointer = VHSA_VPTR_TO_UINT64(gpu_va),
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr,
|
||||
sizeof(struct vhsakmt_ccmd_query_info_rsp) +
|
||||
QUERY_PTR_INFO_MAX_MAPPED_NODES * sizeof(uint32_t));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
memcpy(PointerInfo, &rsp->ptr_info, sizeof(HsaPointerInfo));
|
||||
|
||||
if (PointerInfo->NMappedNodes && PointerInfo->MappedNodes) {
|
||||
if (PointerInfo->NMappedNodes > QUERY_PTR_INFO_MAX_MAPPED_NODES) {
|
||||
PointerInfo->NMappedNodes = QUERY_PTR_INFO_MAX_MAPPED_NODES;
|
||||
vhsa_debug(
|
||||
"%s: query pointer: %p info mapped nodes greater than QUERY_PTR_INFO_MAX_MAPPED_NODES\n",
|
||||
__FUNCTION__, Pointer);
|
||||
}
|
||||
|
||||
PointerInfo->MappedNodes = calloc(PointerInfo->NMappedNodes, sizeof(uint32_t));
|
||||
if (!PointerInfo->MappedNodes) {
|
||||
PointerInfo->NMappedNodes = 0;
|
||||
return -HSAKMT_STATUS_NO_MEMORY;
|
||||
}
|
||||
memcpy(VHSA_UINT64_TO_VPTR(PointerInfo->MappedNodes), rsp->payload,
|
||||
PointerInfo->NMappedNodes * sizeof(uint32_t));
|
||||
}
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig* config) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
uint8_t* config_cpy_addr = NULL;
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
unsigned req_len = sizeof(struct vhsakmt_ccmd_query_info_req);
|
||||
unsigned rsp_len = sizeof(struct vhsakmt_ccmd_query_info_rsp) +
|
||||
config->NumTileConfigs * sizeof(HSAuint32) + config->NumMacroTileConfigs * sizeof(HSAuint32);
|
||||
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, req_len),
|
||||
.type = VHSAKMT_CCMD_QUERY_TILE_CONFIG,
|
||||
.tile_config_args.NodeId = NodeId,
|
||||
.tile_config_args.config = *config,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, rsp_len);
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
memcpy(config, &rsp->tile_config_rsp, sizeof(HsaGpuTileConfig));
|
||||
config_cpy_addr = ((uint8_t*)rsp->payload);
|
||||
memcpy(config->TileConfig, config_cpy_addr, config->NumTileConfigs * sizeof(HSAuint32));
|
||||
config_cpy_addr += config->NumTileConfigs * sizeof(HSAuint32);
|
||||
memcpy(config->MacroTileConfig, config_cpy_addr, config->NumMacroTileConfigs * sizeof(HSAuint32));
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
static int vhsakmt_create_clgl_bo(vhsakmt_device_handle dev, void* addr, size_t size,
|
||||
uint32_t res_id, uint32_t bo_handle, void* meta_data) {
|
||||
vhsakmt_bo_handle out = calloc(1, sizeof(struct vhsakmt_bo));
|
||||
if (!out) return -ENOMEM;
|
||||
|
||||
out->dev = dev;
|
||||
out->size = size;
|
||||
atomic_store(&out->real.map_count, 0);
|
||||
atomic_store(&out->refcount, 1);
|
||||
|
||||
#ifdef CLGL_EXPORT_RESID
|
||||
out->real.res_id = GraphicsResourceHandle;
|
||||
#else
|
||||
out->real.res_id = res_id;
|
||||
#endif
|
||||
|
||||
/* GL bo handle from GL context*/
|
||||
out->real.handle = bo_handle;
|
||||
out->bo_type |= VHSA_BO_CLGL;
|
||||
if (meta_data) out->gl_meta_data = meta_data;
|
||||
|
||||
out->host_addr = addr;
|
||||
|
||||
vhsakmt_insert_bo(dev, out, addr, out->size);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vhsakmt_gfxhandle_to_resid(vhsakmt_device_handle dev, uint32_t gfx_handle,
|
||||
uint32_t* res_id, uint32_t* bo_handle) {
|
||||
int r = drmPrimeFDToHandle(dev->vgdev->fd, gfx_handle, bo_handle);
|
||||
if (r) {
|
||||
vhsa_err("%s: drmPrimeFDToHandle failed for handle: %u\n", __FUNCTION__, gfx_handle);
|
||||
return r;
|
||||
}
|
||||
|
||||
virtio_gpu_res_id(dev->vgdev, *bo_handle, res_id);
|
||||
|
||||
vhsa_debug("%s: register praphics handle: handle: %d, bo_handle: %d, res_id: %d\n", __FUNCTION__,
|
||||
gfx_handle, *bo_handle, *res_id);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterGraphicsHandleToNodes(
|
||||
HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo* GraphicsResourceInfo,
|
||||
HSAuint64 NumberOfNodes, HSAuint32* NodeArray) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
uint32_t bo_handle, res_id;
|
||||
uint64_t meta_data_size = VHSA_GL_METADATA_MAX_SIZE;
|
||||
unsigned req_len = sizeof(struct vhsakmt_ccmd_gl_inter_req) + NumberOfNodes * sizeof(NodeArray);
|
||||
struct vhsakmt_ccmd_gl_inter_req* req;
|
||||
struct vhsakmt_ccmd_gl_inter_rsp* rsp;
|
||||
int r;
|
||||
|
||||
req = calloc(1, req_len);
|
||||
if (!req) return -ENOMEM;
|
||||
|
||||
req->hdr = VHSAKMT_CCMD(GL_INTER, req_len);
|
||||
req->type = VHSAKMT_CCMD_GL_REG_GHD_TO_NODES;
|
||||
req->reg_ghd_to_nodes.NumberOfNodes = NumberOfNodes;
|
||||
req->reg_ghd_to_nodes.res_handle = GraphicsResourceHandle;
|
||||
|
||||
#ifdef CLGL_EXPORT_RESID
|
||||
req->reg_ghd_to_nodes.GraphicsResourceHandle = GraphicsResourceHandle;
|
||||
#else
|
||||
r = vhsakmt_gfxhandle_to_resid(dev, GraphicsResourceHandle, &res_id, &bo_handle);
|
||||
if (r) return r;
|
||||
|
||||
req->reg_ghd_to_nodes.GraphicsResourceHandle = bo_handle;
|
||||
req->reg_ghd_to_nodes.res_handle = res_id;
|
||||
#endif
|
||||
|
||||
memcpy(req->payload, NodeArray, NumberOfNodes * sizeof(NodeArray));
|
||||
|
||||
rsp =
|
||||
vhsakmt_alloc_rsp(dev, &req->hdr, sizeof(struct vhsakmt_ccmd_gl_inter_rsp) + meta_data_size);
|
||||
if (!rsp) {
|
||||
r = -ENOMEM;
|
||||
goto free_out;
|
||||
}
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req->hdr, __FUNCTION__);
|
||||
if (rsp->ret) return rsp->ret;
|
||||
|
||||
memcpy(GraphicsResourceInfo, &rsp->info, sizeof(HsaGraphicsResourceInfo));
|
||||
if (rsp->info.MetadataSizeInBytes) {
|
||||
GraphicsResourceInfo->Metadata = calloc(1, GraphicsResourceInfo->MetadataSizeInBytes);
|
||||
if (!GraphicsResourceInfo->Metadata) {
|
||||
r = -ENOMEM;
|
||||
goto free_out;
|
||||
}
|
||||
|
||||
memcpy(VHSA_UINT64_TO_VPTR(GraphicsResourceInfo->Metadata), rsp->payload,
|
||||
GraphicsResourceInfo->MetadataSizeInBytes);
|
||||
} else
|
||||
GraphicsResourceInfo->Metadata = NULL;
|
||||
|
||||
vhsa_debug("%s: register graphics handle: handle: %ld hva: %p, size: %lx\n", __FUNCTION__,
|
||||
GraphicsResourceHandle, GraphicsResourceInfo->MemoryAddress,
|
||||
GraphicsResourceInfo->SizeInBytes);
|
||||
|
||||
r = vhsakmt_create_clgl_bo(dev, GraphicsResourceInfo->MemoryAddress,
|
||||
GraphicsResourceInfo->SizeInBytes, res_id, bo_handle,
|
||||
VHSA_UINT64_TO_VPTR(GraphicsResourceInfo->Metadata));
|
||||
if (r) goto free_out;
|
||||
|
||||
r = rsp->ret;
|
||||
|
||||
free_out:
|
||||
/* close exported FD after register or close it when deregistre. Close after register here. */
|
||||
close(GraphicsResourceHandle);
|
||||
free(req);
|
||||
return r;
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hsakmt/hsakmt_virtio.h"
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
pthread_mutex_t dev_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
vhsakmt_device_handle dev_list = NULL;
|
||||
|
||||
vhsakmt_device_handle vhsakmt_dev(void) { return dev_list; }
|
||||
|
||||
static HSAKMT_STATUS vhsakmt_openKFD_cmd(vhsakmt_device_handle dev) {
|
||||
void* vm_start = vhsakmt_vm_start();
|
||||
if (!vm_start) return -HSAKMT_STATUS_NO_MEMORY;
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_OPEN_KFD,
|
||||
.open_kfd_args =
|
||||
{
|
||||
.cur_vm_start = VHSA_VPTR_TO_UINT64(vm_start),
|
||||
},
|
||||
};
|
||||
|
||||
if (!req.open_kfd_args.cur_vm_start) {
|
||||
vhsa_err("%s: failed to get current heap start address\n", __FUNCTION__);
|
||||
return -HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -HSAKMT_STATUS_NO_MEMORY;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (!rsp->open_kfd_rsp.vm_start || !rsp->open_kfd_rsp.vm_size) {
|
||||
vhsa_err("%s: failed to get KFD VM area\n", __FUNCTION__);
|
||||
return -HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
vhsakmt_set_vm_area(dev, rsp->open_kfd_rsp.vm_start, rsp->open_kfd_rsp.vm_size);
|
||||
if (vhsakmt_reserve_va(dev->vm_start, dev->vm_size)) {
|
||||
vhsa_err("%s: failed to reserve VM area: [%lx-%lx]-0x%lx\n", __FUNCTION__, dev->vm_start,
|
||||
dev->vm_start + dev->vm_size, dev->vm_size);
|
||||
return -HSAKMT_STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
vhsa_debug("%s: kfd vm range: [%lx-%lx]-0x%lx\n", __FUNCTION__, dev->vm_start,
|
||||
dev->vm_start + dev->vm_size, dev->vm_size);
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
static vhsakmt_device_handle vhsakmt_device_init(void) {
|
||||
int fd;
|
||||
vhsakmt_device_handle dev = NULL;
|
||||
|
||||
if (vhsakmt_dev()) return vhsakmt_dev();
|
||||
|
||||
pthread_mutex_lock(&dev_mutex);
|
||||
|
||||
fd = virtio_gpu_kfd_open();
|
||||
if (fd < 0) goto open_failed;
|
||||
|
||||
dev = calloc(1, sizeof(struct vhsakmt_device));
|
||||
if (!dev) goto open_failed;
|
||||
|
||||
dev->vgdev = virtio_gpu_init(fd, 0);
|
||||
if (!dev->vgdev) goto malloc_failed;
|
||||
|
||||
rbtree_init(&dev->bo_rbt);
|
||||
atomic_store(&dev->next_blob_id, 1);
|
||||
atomic_store(&dev->refcount, 1);
|
||||
pthread_mutex_init(&dev->bo_handles_mutex, NULL);
|
||||
pthread_mutex_init(&dev->vhsakmt_mutex, NULL);
|
||||
dev_list = dev;
|
||||
|
||||
pthread_mutex_unlock(&dev_mutex);
|
||||
return dev;
|
||||
|
||||
malloc_failed:
|
||||
free(dev);
|
||||
dev = NULL;
|
||||
open_failed:
|
||||
pthread_mutex_unlock(&dev_mutex);
|
||||
return dev;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtOpenKFD(void) {
|
||||
vhsakmt_device_handle dev;
|
||||
char* d = getenv("VHSAKMT_DEBUG_LEVEL");
|
||||
if (d) vhsakmt_debug_level = atoi(d);
|
||||
|
||||
dev = vhsakmt_device_init();
|
||||
if (!dev) return HSAKMT_STATUS_ERROR;
|
||||
|
||||
return vhsakmt_openKFD_cmd(vhsakmt_dev());
|
||||
}
|
||||
|
||||
static void vhsakmt_device_destroy(struct vhsakmt_device* dev) {
|
||||
pthread_mutex_destroy(&dev->bo_handles_mutex);
|
||||
vhsakmt_dereserve_va(dev->vm_start, dev->vm_size);
|
||||
|
||||
if (dev->sys_props) free(dev->sys_props);
|
||||
if (dev->vhsakmt_nodes) free(dev->vhsakmt_nodes);
|
||||
|
||||
virtio_gpu_close(dev->vgdev);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCloseKFD(void) {
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
pthread_mutex_lock(&dev_mutex);
|
||||
if (vhsakmt_atomic_dec_return(&dev->refcount) <= 0) vhsakmt_device_destroy(dev);
|
||||
pthread_mutex_unlock(&dev_mutex);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VHSAKMT_VIRTIO_PROTO_H
|
||||
#define VHSAKMT_VIRTIO_PROTO_H
|
||||
|
||||
#include "hsakmt/linux/kfd_ioctl.h"
|
||||
#include "hsakmt/hsakmt.h"
|
||||
|
||||
#include <drm/amdgpu_drm.h>
|
||||
#include <libdrm/amdgpu.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "virtio_gpu.h"
|
||||
|
||||
#ifdef __GNUC__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic error "-Wpadded"
|
||||
#endif
|
||||
|
||||
/* defined in other header file in virglrenderer */
|
||||
#define VHSAKMT_DEFINE_CAST(parent, child) \
|
||||
static inline struct child* to_##child(struct parent* x) { return (struct child*)x; }
|
||||
|
||||
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
|
||||
#define VHSAKMT_STATIC_ASSERT_SIZE(t) \
|
||||
static_assert(sizeof(struct t) % 8 == 0, "sizeof(struct " #t ") not multiple of 8"); \
|
||||
static_assert(_Alignof(struct t) <= 8, "alignof(struct " #t ") too large");
|
||||
#else
|
||||
#define VHSAKMT_STATIC_ASSERT_SIZE(t)
|
||||
#endif
|
||||
|
||||
enum vhsakmt_ccmd {
|
||||
VHSAKMT_CCMD_NOP = 1, /* No payload, can be used to sync with host */
|
||||
VHSAKMT_CCMD_QUERY_INFO,
|
||||
VHSAKMT_CCMD_EVENT,
|
||||
VHSAKMT_CCMD_MEMORY,
|
||||
VHSAKMT_CCMD_QUEUE,
|
||||
VHSAKMT_CCMD_GL_INTER,
|
||||
};
|
||||
|
||||
typedef struct _vHsaEvent {
|
||||
HsaEvent event;
|
||||
uint64_t event_handle;
|
||||
uint64_t bo_handle;
|
||||
uint32_t res_id;
|
||||
uint32_t pad;
|
||||
} vHsaEvent;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_vHsaEvent)
|
||||
|
||||
struct vhsakmt_event_shmem {
|
||||
uint32_t trigered_events_num;
|
||||
uint32_t pad;
|
||||
HsaEvent trigered_events[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_event_shmem)
|
||||
|
||||
#define VHSAKMT_CCMD(_cmd, _len) \
|
||||
((struct vhsakmt_ccmd_req){ \
|
||||
.cmd = VHSAKMT_CCMD_##_cmd, \
|
||||
.len = (_len), \
|
||||
})
|
||||
|
||||
struct vhsakmt_ccmd_nop_req {
|
||||
struct vhsakmt_ccmd_req hdr;
|
||||
};
|
||||
|
||||
/*
|
||||
* VHSAKMT_CCMD_QUERY
|
||||
*/
|
||||
enum vhsakmt_ccmd_query_type {
|
||||
VHSAKMT_CCMD_QUERY_GPU_INFO = 0,
|
||||
VHSAKMT_CCMD_QUERY_OPEN_KFD,
|
||||
VHSAKMT_CCMD_QUERY_GET_VER,
|
||||
VHSAKMT_CCMD_QUERY_REL_SYS_PROP,
|
||||
VHSAKMT_CCMD_QUERY_GET_SYS_PROP,
|
||||
VHSAKMT_CCMD_QUERY_GET_NODE_PROP,
|
||||
VHSAKMT_CCMD_QUERY_GET_XNACK_MODE,
|
||||
VHSAKMT_CCMD_QUERY_RUN_TIME_ENABLE,
|
||||
VHSAKMT_CCMD_QUERY_RUN_TIME_DISABLE,
|
||||
VHSAKMT_CCMD_QUERY_GET_NOD_MEM_PROP,
|
||||
VHSAKMT_CCMD_QUERY_GET_NOD_CACHE_PROP,
|
||||
VHSAKMT_CCMD_QUERY_GET_NOD_IO_LINK_PROP,
|
||||
VHSAKMT_CCMD_QUERY_GET_CLOCK_COUNTERS,
|
||||
VHSAKMT_CCMD_QUERY_POINTER_INFO,
|
||||
VHSAKMT_CCMD_QUERY_TILE_CONFIG,
|
||||
VHSAKMT_CCMD_QUERY_NANO_TIME,
|
||||
VHSAKMT_CCMD_QUERY_GET_RUNTIME_CAPS,
|
||||
};
|
||||
|
||||
#define QUERY_PTR_INFO_MAX_MAPPED_NODES 3
|
||||
|
||||
typedef struct _query_req_run_time_enable_args {
|
||||
/* void* rDebug, bypassed by payload */
|
||||
uint8_t pad[3];
|
||||
uint8_t setupTtmp;
|
||||
uint32_t __pad;
|
||||
} query_req_run_time_enable_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_req_run_time_enable_args)
|
||||
|
||||
typedef struct _query_req_node_mem_prop_args {
|
||||
uint32_t NodeId;
|
||||
uint32_t NumBanks;
|
||||
} query_req_node_mem_prop_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_mem_prop_args)
|
||||
|
||||
typedef struct _query_req_node_cache_prop_args {
|
||||
uint32_t NodeId;
|
||||
uint32_t ProcessorId;
|
||||
uint32_t NumCaches;
|
||||
uint32_t pad;
|
||||
} query_req_node_cache_prop_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_cache_prop_args)
|
||||
|
||||
typedef struct _query_req_node_io_link_args {
|
||||
uint32_t NodeId;
|
||||
uint32_t NumIoLinks;
|
||||
} query_req_node_io_link_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_io_link_args)
|
||||
|
||||
typedef struct _query_tile_config {
|
||||
HsaGpuTileConfig config;
|
||||
uint32_t NodeId;
|
||||
uint32_t pad;
|
||||
} query_tile_config;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_tile_config)
|
||||
|
||||
typedef struct _query_open_kfd_args {
|
||||
uint64_t cur_vm_start;
|
||||
} query_open_kfd_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_open_kfd_args)
|
||||
|
||||
typedef struct _query_open_kfd_rsp {
|
||||
uint64_t vm_start;
|
||||
uint64_t vm_size;
|
||||
} query_open_kfd_rsp;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_open_kfd_rsp)
|
||||
|
||||
typedef struct _query_nano_time_rsp {
|
||||
uint64_t nano_time;
|
||||
} query_nano_time_rsp;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_query_nano_time_rsp)
|
||||
|
||||
struct vhsakmt_ccmd_query_info_req {
|
||||
struct vhsakmt_ccmd_req hdr;
|
||||
struct drm_amdgpu_info info;
|
||||
uint32_t type;
|
||||
uint32_t pad;
|
||||
union {
|
||||
uint64_t pointer;
|
||||
uint32_t NodeID; /* some query API just need node ID */
|
||||
query_req_run_time_enable_args run_time_enable_args;
|
||||
query_req_node_mem_prop_args node_mem_prop_args;
|
||||
query_req_node_cache_prop_args node_cache_prop_args;
|
||||
query_req_node_io_link_args node_io_link_args;
|
||||
query_tile_config tile_config_args;
|
||||
query_open_kfd_args open_kfd_args;
|
||||
};
|
||||
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_query_info_req)
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_query_info_req)
|
||||
#define VHSAKMT_CCMD_QUERY_MAX_TILE_CONFIG 128
|
||||
#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_MEM_PROP 128
|
||||
#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_CACHE_PROP 128
|
||||
#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_IO_LINK_PROP 128
|
||||
|
||||
struct vhsakmt_ccmd_query_info_rsp {
|
||||
struct vhsakmt_ccmd_rsp hdr;
|
||||
int32_t ret;
|
||||
union {
|
||||
query_open_kfd_rsp open_kfd_rsp;
|
||||
query_nano_time_rsp nano_time_rsp;
|
||||
HsaGpuTileConfig tile_config_rsp;
|
||||
HsaPointerInfo ptr_info;
|
||||
struct amdgpu_gpu_info gpu_info;
|
||||
HsaVersionInfo kfd_version;
|
||||
HsaSystemProperties sys_props;
|
||||
HsaNodeProperties node_props;
|
||||
int32_t xnack_mode;
|
||||
HsaClockCounters clock_counters;
|
||||
uint32_t caps;
|
||||
uint64_t pad[9];
|
||||
};
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_query_info_rsp)
|
||||
|
||||
/*
|
||||
* VHSAKMT_CCMD_EVENT
|
||||
*/
|
||||
enum vhsakmt_ccmd_event_type {
|
||||
VHSAKMT_CCMD_EVENT_CREATE,
|
||||
VHSAKMT_CCMD_EVENT_DESTROY,
|
||||
VHSAKMT_CCMD_EVENT_SET,
|
||||
VHSAKMT_CCMD_EVENT_RESET,
|
||||
VHSAKMT_CCMD_EVENT_QUERY_STATE,
|
||||
VHSAKMT_CCMD_EVENT_WAIT_ON_MULTI_EVENTS,
|
||||
|
||||
VHSAKMT_CCMD_EVENT_SET_TRAP,
|
||||
|
||||
};
|
||||
typedef struct _event_req_create_args {
|
||||
HsaEventDescriptor EventDesc;
|
||||
uint8_t ManualReset;
|
||||
uint8_t IsSignaled;
|
||||
uint8_t pad[6];
|
||||
} event_req_create_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_event_req_create_args)
|
||||
|
||||
typedef struct _event_req_wait_args {
|
||||
HsaEvent Event;
|
||||
uint32_t Milliseconds;
|
||||
uint32_t pad;
|
||||
} event_req_wait_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_args)
|
||||
|
||||
typedef struct _event_req_wait_ext_args {
|
||||
HsaEvent Event;
|
||||
uint64_t event_age;
|
||||
uint32_t Milliseconds;
|
||||
uint32_t pad;
|
||||
} event_req_wait_ext_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_ext_args)
|
||||
|
||||
typedef struct _event_req_wait_on_multi_args {
|
||||
/*HsaEvent* Events[], in playloud*/
|
||||
uint32_t NumEvents;
|
||||
uint32_t Milliseconds;
|
||||
uint8_t WaitOnAll;
|
||||
uint8_t pad[7];
|
||||
} event_req_wait_on_multi_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_on_multi_args)
|
||||
|
||||
typedef struct _event_req_wait_on_multi_ext_args {
|
||||
/*HsaEvent* Events[], in playloud*/
|
||||
uint32_t NumEvents;
|
||||
uint32_t Milliseconds;
|
||||
uint64_t event_age;
|
||||
uint8_t WaitOnAll;
|
||||
uint8_t pad[7];
|
||||
} event_req_wait_on_multi_ext_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_on_multi_ext_args)
|
||||
|
||||
typedef struct _event_set_trap_handler_args {
|
||||
uint64_t TrapHandlerBaseAddress;
|
||||
uint64_t TrapHandlerSizeInBytes;
|
||||
uint64_t TrapBufferBaseAddress;
|
||||
uint64_t TrapBufferSizeInBytes;
|
||||
uint32_t NodeId;
|
||||
uint32_t pad;
|
||||
} event_set_trap_handler_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_event_set_trap_handler_args)
|
||||
|
||||
struct vhsakmt_ccmd_event_req {
|
||||
struct vhsakmt_ccmd_req hdr;
|
||||
union {
|
||||
HsaEvent Event; /* For set, reset, query. */
|
||||
HsaEvent* event_hanele;
|
||||
event_req_wait_args wait_args;
|
||||
event_req_create_args create_args;
|
||||
event_req_wait_ext_args wait_ext_args;
|
||||
event_req_wait_on_multi_args wait_on_multi_args;
|
||||
event_req_wait_on_multi_ext_args wait_on_multi_ext_args;
|
||||
event_set_trap_handler_args set_trap_handler_args;
|
||||
};
|
||||
uint32_t type;
|
||||
uint32_t sync_shmem_res_id;
|
||||
uint64_t blob_id;
|
||||
uint32_t res_id;
|
||||
uint32_t pad;
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_event_req)
|
||||
VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_event_req)
|
||||
|
||||
struct vhsakmt_ccmd_event_rsp {
|
||||
struct vhsakmt_ccmd_rsp hdr;
|
||||
int32_t ret;
|
||||
vHsaEvent vevent;
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_event_rsp)
|
||||
|
||||
/*
|
||||
* VHSAKMT_CCMD_MEMORY
|
||||
*/
|
||||
enum vhsakmt_ccmd_memory_type {
|
||||
VHSAKMT_CCMD_MEMORY_ALLOC,
|
||||
VHSAKMT_CCMD_MEMORY_MAP_TO_GPU_NODES,
|
||||
VHSAKMT_CCMD_MEMORY_FREE,
|
||||
VHSAKMT_CCMD_MEMORY_UNMAP_TO_GPU,
|
||||
VHSAKMT_CCMD_MEMORY_AVAIL_MEM,
|
||||
VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU,
|
||||
VHSAKMT_CCMD_MEMORY_REG_MEM_WITH_FLAG,
|
||||
VHSAKMT_CCMD_MEMORY_DEREG_MEM,
|
||||
VHSAKMT_CCMD_MEMORY_MAP_USERPTR,
|
||||
};
|
||||
|
||||
typedef struct _memory_req_alloc_args {
|
||||
uint32_t PreferredNode;
|
||||
HsaMemFlags MemFlags;
|
||||
uint64_t SizeInBytes;
|
||||
uint64_t MemoryAddress;
|
||||
} memory_req_alloc_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_alloc_args)
|
||||
|
||||
typedef struct _memory_req_free_args {
|
||||
uint64_t MemoryAddress;
|
||||
uint64_t SizeInBytes;
|
||||
} memory_req_free_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_free_args)
|
||||
|
||||
typedef struct _memory_req_map_to_GPU_nodes_args {
|
||||
uint64_t MemoryAddress;
|
||||
uint64_t MemorySizeInBytes;
|
||||
uint64_t AlternateVAGPU;
|
||||
HsaMemMapFlags MemMapFlags;
|
||||
uint32_t pad;
|
||||
uint64_t NumberOfNodes;
|
||||
uint32_t* NodeArray;
|
||||
} memory_req_map_to_GPU_nodes_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_map_to_GPU_nodes_args)
|
||||
|
||||
typedef struct _memory_map_mem_to_gpu_args {
|
||||
uint64_t MemoryAddress;
|
||||
uint64_t MemorySizeInBytes;
|
||||
uint8_t need_create_bo;
|
||||
uint8_t pad[7];
|
||||
} memory_map_mem_to_gpu_args;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_memory_map_mem_to_gpu_args)
|
||||
|
||||
typedef struct _memory_reg_mem_with_flag {
|
||||
uint64_t MemoryAddress;
|
||||
uint64_t MemorySizeInBytes;
|
||||
HsaMemFlags MemFlags;
|
||||
uint32_t pad;
|
||||
} memory_reg_mem_with_flag;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_memory_reg_mem_with_flag)
|
||||
|
||||
struct vhsakmt_ccmd_memory_req {
|
||||
struct vhsakmt_ccmd_req hdr;
|
||||
union {
|
||||
uint64_t MemoryAddress;
|
||||
uint32_t Node;
|
||||
memory_req_alloc_args alloc_args;
|
||||
memory_req_map_to_GPU_nodes_args map_to_GPU_nodes_args;
|
||||
memory_req_free_args free_args;
|
||||
memory_map_mem_to_gpu_args map_to_GPU_args;
|
||||
memory_reg_mem_with_flag reg_mem_with_flag;
|
||||
};
|
||||
uint64_t blob_id;
|
||||
uint32_t type;
|
||||
uint32_t res_id;
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_memory_req)
|
||||
VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_memory_req)
|
||||
|
||||
typedef struct _vhsakmt_ccmd_memory_map_userptr_rsp {
|
||||
uint64_t userptr_handle;
|
||||
uint32_t npfns;
|
||||
uint32_t pad;
|
||||
} vhsakmt_ccmd_memory_map_userptr_rsp;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_vhsakmt_ccmd_memory_map_userptr_rsp)
|
||||
|
||||
struct vhsakmt_ccmd_memory_rsp {
|
||||
struct vhsakmt_ccmd_rsp hdr;
|
||||
int32_t ret;
|
||||
union {
|
||||
vhsakmt_ccmd_memory_map_userptr_rsp map_userptr_rsp;
|
||||
uint64_t memory_handle;
|
||||
uint64_t alternate_vagpu;
|
||||
uint64_t available_bytes;
|
||||
};
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_memory_rsp)
|
||||
|
||||
/*
|
||||
* VHSAKMT_CCMD_QUEUE
|
||||
*/
|
||||
enum vhsakmt_ccmd_queue_type {
|
||||
VHSAKMT_CCMD_QUEUE_CREATE,
|
||||
VHSAKMT_CCMD_QUEUE_DESTROY,
|
||||
};
|
||||
|
||||
typedef struct _vHsaQueueResource {
|
||||
HsaQueueResource r;
|
||||
uint64_t host_doorbell;
|
||||
uint64_t host_doorbell_offset;
|
||||
uint64_t host_write_offset;
|
||||
uint64_t host_read_offset;
|
||||
uint64_t host_rw_handle;
|
||||
uint64_t queue_handle;
|
||||
} vHsaQueueResource;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_vHsaQueueResource)
|
||||
|
||||
typedef struct _queue_req_create {
|
||||
uint32_t NodeId;
|
||||
HSA_QUEUE_TYPE Type;
|
||||
uint32_t QueuePercentage;
|
||||
uint32_t pad;
|
||||
HSA_QUEUE_PRIORITY Priority;
|
||||
uint32_t pad1;
|
||||
uint32_t SdmaEngineId;
|
||||
uint64_t QueueAddress;
|
||||
uint64_t QueueSizeInBytes;
|
||||
HsaEvent* Event;
|
||||
HsaQueueResource* QueueResource;
|
||||
uint64_t* Queue_write_ptr_aql;
|
||||
uint64_t* Queue_read_ptr_aql;
|
||||
} queue_req_create;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_queue_req_create)
|
||||
|
||||
struct vhsakmt_ccmd_queue_req {
|
||||
struct vhsakmt_ccmd_req hdr;
|
||||
union {
|
||||
HSA_QUEUEID QueueId;
|
||||
queue_req_create create_queue_args;
|
||||
};
|
||||
uint64_t blob_id; /* For queue create, queue resource */
|
||||
uint64_t rw_ptr_blob_id; /* For queue create, r/w ptr memory mapping */
|
||||
uint64_t doorbell_blob_id; /* For queue create, doorbell ptr memory mapping */
|
||||
uint32_t res_id;
|
||||
uint32_t type;
|
||||
uint32_t queue_mem_res_id;
|
||||
uint32_t pad;
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_queue_req)
|
||||
VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_queue_req)
|
||||
|
||||
struct vhsakmt_ccmd_queue_rsp {
|
||||
struct vhsakmt_ccmd_rsp hdr;
|
||||
int32_t ret;
|
||||
vHsaQueueResource vqueue_res;
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_queue_rsp)
|
||||
|
||||
/*
|
||||
* VHSAKMT_CCMD_GL_INTER
|
||||
*/
|
||||
enum vhsakmt_ccmd_gl_inter_type {
|
||||
VHSAKMT_CCMD_GL_REG_GHD_TO_NODES,
|
||||
};
|
||||
|
||||
typedef struct _gl_inter_req_reg_ghd_to_nodes {
|
||||
uint64_t GraphicsResourceHandle;
|
||||
uint64_t NumberOfNodes; // NodeArray in payload
|
||||
uint32_t res_handle;
|
||||
uint32_t pad;
|
||||
} gl_inter_req_reg_ghd_to_nodes;
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(_gl_inter_req_reg_ghd_to_nodes)
|
||||
|
||||
struct vhsakmt_ccmd_gl_inter_req {
|
||||
struct vhsakmt_ccmd_req hdr;
|
||||
union {
|
||||
gl_inter_req_reg_ghd_to_nodes reg_ghd_to_nodes;
|
||||
};
|
||||
uint32_t type;
|
||||
uint32_t pad;
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_gl_inter_req)
|
||||
VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_gl_inter_req)
|
||||
|
||||
struct vhsakmt_ccmd_gl_inter_rsp {
|
||||
struct vhsakmt_ccmd_rsp hdr;
|
||||
int32_t ret;
|
||||
union {
|
||||
HsaGraphicsResourceInfo info;
|
||||
};
|
||||
uint8_t payload[];
|
||||
};
|
||||
VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_gl_inter_rsp)
|
||||
|
||||
#ifdef __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,253 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hsakmt/hsakmt_virtio.h"
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
static inline uint64_t vhsakmt_doorbell_page_size(void) { return 0x2000; }
|
||||
static inline uint64_t vhsakmt_queue_page_size(void) { return getpagesize(); }
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtSetTrapHandler(HSAuint32 NodeId, void* TrapHandlerBaseAddress,
|
||||
HSAuint64 TrapHandlerSizeInBytes,
|
||||
void* TrapBufferBaseAddress,
|
||||
HSAuint64 TrapBufferSizeInBytes) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_event_rsp* rsp;
|
||||
struct vhsakmt_ccmd_event_req req = {
|
||||
.hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)),
|
||||
.type = VHSAKMT_CCMD_EVENT_SET_TRAP,
|
||||
.set_trap_handler_args =
|
||||
{
|
||||
.NodeId = NodeId,
|
||||
.TrapHandlerBaseAddress = (uint64_t)TrapHandlerBaseAddress,
|
||||
.TrapHandlerSizeInBytes = TrapHandlerSizeInBytes,
|
||||
.TrapBufferBaseAddress = (uint64_t)TrapBufferBaseAddress,
|
||||
.TrapBufferSizeInBytes = TrapBufferSizeInBytes,
|
||||
},
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
static int vhsakmt_find_aql_rw_bo(vhsakmt_device_handle dev, uint64_t aql_ptr,
|
||||
uint32_t* aql_bo_res_id) {
|
||||
uint64_t aql_base_ptr = VHSA_ALIGN_DOWN(aql_ptr, getpagesize());
|
||||
|
||||
vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, (void*)aql_base_ptr);
|
||||
if (!bo) return -EINVAL;
|
||||
|
||||
bo->bo_type |= VHSA_BO_QUEUE_AQL_RW_PTR;
|
||||
*aql_bo_res_id = bo->real.res_id;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vhsakmt_create_doorbell_blob_bo(vhsakmt_device_handle dev, uint32_t node, size_t size,
|
||||
uint32_t blob_id, uint64_t host_handle,
|
||||
vhsakmt_bo_handle* bo_handle) {
|
||||
int r;
|
||||
|
||||
r = vhsakmt_create_mappable_blob_bo(dev, size, blob_id, VHSA_BO_QUEUE_DOORBELL,
|
||||
(void*)host_handle, bo_handle);
|
||||
if (r) return r;
|
||||
|
||||
r = vhsakmt_set_node_doorbell(dev, node, (*bo_handle)->cpu_addr);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static int vhsakmt_create_queue_rw_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id,
|
||||
uint64_t host_handle, vhsakmt_bo_handle* bo_handle) {
|
||||
int r;
|
||||
|
||||
r = vhsakmt_create_mappable_blob_bo(dev, size, blob_id, VHSA_BO_QUEUE_RW_PTR, NULL, bo_handle);
|
||||
if (r) return r;
|
||||
|
||||
(*bo_handle)->host_addr = (void*)host_handle;
|
||||
return r;
|
||||
}
|
||||
|
||||
static int vhsakmt_create_queue_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id,
|
||||
uint64_t queue_id, vhsakmt_bo_handle rw_bo_handle,
|
||||
vhsakmt_bo_handle* bo_handle) {
|
||||
int r;
|
||||
|
||||
r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, 0, blob_id, VHSA_BO_QUEUE, NULL,
|
||||
bo_handle);
|
||||
if (r) return r;
|
||||
|
||||
vhsakmt_insert_bo(dev, *bo_handle, *bo_handle, (*bo_handle)->size);
|
||||
|
||||
(*bo_handle)->queue_id = queue_id;
|
||||
(*bo_handle)->rw_bo = rw_bo_handle;
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueueExt(HSAuint32 NodeId, HSA_QUEUE_TYPE Type,
|
||||
HSAuint32 QueuePercentage,
|
||||
HSA_QUEUE_PRIORITY Priority, HSAuint32 SdmaEngineId,
|
||||
void* QueueAddress, HSAuint64 QueueSizeInBytes,
|
||||
HsaEvent* Event, HsaQueueResource* QueueResource) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
vhsakmt_bo_handle rw_bo_handle = NULL, doorbell_bo, queue_bo, queue_mem_bo;
|
||||
struct vhsakmt_ccmd_queue_rsp* rsp;
|
||||
struct vhsakmt_ccmd_queue_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUEUE, sizeof(struct vhsakmt_ccmd_queue_req)),
|
||||
.type = VHSAKMT_CCMD_QUEUE_CREATE,
|
||||
.create_queue_args =
|
||||
{
|
||||
.NodeId = NodeId,
|
||||
.Type = Type,
|
||||
.QueuePercentage = QueuePercentage,
|
||||
.Priority = Priority,
|
||||
.SdmaEngineId = SdmaEngineId,
|
||||
.QueueAddress = (uint64_t)QueueAddress,
|
||||
.QueueSizeInBytes = QueueSizeInBytes,
|
||||
.Event = Event ? vhsakmt_event_host_handle(Event) : 0,
|
||||
.Queue_write_ptr_aql = QueueResource->Queue_write_ptr_aql,
|
||||
.Queue_read_ptr_aql = QueueResource->Queue_read_ptr_aql,
|
||||
},
|
||||
.blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), /* For queue resource */
|
||||
.doorbell_blob_id = vhsakmt_node_doorbell(dev, NodeId)
|
||||
? 0
|
||||
: vhsakmt_atomic_inc_return(&dev->next_blob_id), /* For queue doorbell memory map */
|
||||
};
|
||||
int r;
|
||||
|
||||
/* Queue ptr memory is allocated by hsakmtallocmemory in host then mapped into guest, but their
|
||||
* address are not aligned. */
|
||||
if (Type == HSA_QUEUE_COMPUTE_AQL) {
|
||||
r = vhsakmt_find_aql_rw_bo(dev, QueueResource->QueueWptrValue, &req.res_id);
|
||||
if (r) {
|
||||
vhsa_debug("%s: can not find the AQL queue R/W BO: %p\n", __FUNCTION__,
|
||||
QueueResource->Queue_write_ptr_aql);
|
||||
return HSAKMT_STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
vhsa_debug("%s: create AQL queue, read ptr: %p, write ptr: %p, res id: %d\n", __FUNCTION__,
|
||||
QueueResource->Queue_read_ptr_aql, QueueResource->Queue_write_ptr_aql, req.res_id);
|
||||
} else
|
||||
/* For queue not CP AQL, it use r/w ptr by itself. */
|
||||
req.rw_ptr_blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id);
|
||||
|
||||
queue_mem_bo = vhsakmt_find_bo_by_addr(dev, QueueAddress);
|
||||
if (!queue_mem_bo) {
|
||||
vhsa_err("%s: can not find the queue memory BO: %p\n", __FUNCTION__, QueueAddress);
|
||||
return HSAKMT_STATUS_NO_MEMORY;
|
||||
}
|
||||
queue_mem_bo->bo_type |= VHSA_BO_QUEUE_AQL_RW_PTR;
|
||||
req.queue_mem_res_id = queue_mem_bo->real.res_id;
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_queue_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (rsp->ret) {
|
||||
vhsa_err("%s: queue create failed, ret: %d", __FUNCTION__, rsp->ret);
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
/* Map doorbell */
|
||||
if (req.doorbell_blob_id) {
|
||||
r = vhsakmt_create_doorbell_blob_bo(
|
||||
dev, NodeId, vhsakmt_doorbell_page_size(), req.doorbell_blob_id,
|
||||
rsp->vqueue_res.host_doorbell - rsp->vqueue_res.host_doorbell_offset, &doorbell_bo);
|
||||
if (r) {
|
||||
vhsa_err("%s: doorbell create failed, doorbell: %lx\n", __FUNCTION__,
|
||||
rsp->vqueue_res.host_doorbell);
|
||||
return r;
|
||||
}
|
||||
vhsa_debug("%s: create doorbell: %p, size: 0x%x\n", __FUNCTION__, doorbell_bo->cpu_addr,
|
||||
doorbell_bo->size);
|
||||
}
|
||||
|
||||
QueueResource->Queue_DoorBell_aql = (void*)rsp->vqueue_res.host_doorbell;
|
||||
vhsa_debug("%s: queue create, Doorbell: %p\n", __FUNCTION__, QueueResource->Queue_DoorBell_aql);
|
||||
|
||||
/* Map R/W pointer.
|
||||
* For a queue is not a COMPUTE AQL, the R/W PTR not using the input address,
|
||||
* uses the queue memory allocated by hsakmtallocmemory, a page align address.
|
||||
*/
|
||||
if (Type != HSA_QUEUE_COMPUTE_AQL) {
|
||||
r = vhsakmt_create_queue_rw_blob_bo(dev, vhsakmt_queue_page_size(), req.rw_ptr_blob_id,
|
||||
rsp->vqueue_res.host_rw_handle, &rw_bo_handle);
|
||||
if (r) {
|
||||
vhsa_debug("%s: queue rw ptr create failed, host addr: %p\n", __FUNCTION__,
|
||||
(void*)rsp->vqueue_res.host_rw_handle);
|
||||
return r;
|
||||
}
|
||||
|
||||
QueueResource->Queue_write_ptr_aql = VHSA_UINT64_TO_VPTR(
|
||||
VHSA_VPTR_TO_UINT64(rw_bo_handle->cpu_addr) + rsp->vqueue_res.host_write_offset);
|
||||
QueueResource->Queue_read_ptr_aql = VHSA_UINT64_TO_VPTR(
|
||||
VHSA_VPTR_TO_UINT64(rw_bo_handle->cpu_addr) + rsp->vqueue_res.host_read_offset);
|
||||
|
||||
vhsa_debug("%s: queue create: write ptr gva: %p, read ptr gva: %p, base hva: %lx\n",
|
||||
__FUNCTION__, QueueResource->Queue_write_ptr_aql, QueueResource->Queue_read_ptr_aql,
|
||||
rsp->vqueue_res.host_rw_handle);
|
||||
}
|
||||
|
||||
r = vhsakmt_create_queue_blob_bo(dev, QueueSizeInBytes, req.blob_id, rsp->vqueue_res.r.QueueId,
|
||||
rw_bo_handle, &queue_bo);
|
||||
if (r) {
|
||||
vhsa_err("%s: queue create failed, queue ID: 0x%lx\n", __FUNCTION__, rsp->vqueue_res.r.QueueId);
|
||||
return r;
|
||||
}
|
||||
QueueResource->QueueId = (uint64_t)queue_bo;
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueue(HSAuint32 NodeId, HSA_QUEUE_TYPE Type,
|
||||
HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority,
|
||||
void* QueueAddress, HSAuint64 QueueSizeInBytes,
|
||||
HsaEvent* Event, HsaQueueResource* QueueResource) {
|
||||
return vhsaKmtCreateQueueExt(NodeId, Type, QueuePercentage, Priority, VHSA_SDMA_NONE,
|
||||
QueueAddress, QueueSizeInBytes, Event, QueueResource);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyQueue(HSA_QUEUEID QueueId) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
int r;
|
||||
|
||||
/* queue ID: vhsakmt_bo_handle -> real queue ID*/
|
||||
vhsakmt_bo_handle bo = (vhsakmt_bo_handle)QueueId;
|
||||
vhsakmt_bo_handle rw_bo = bo->rw_bo;
|
||||
|
||||
r = vhsakmt_bo_free(dev, bo);
|
||||
if (rw_bo) vhsakmt_bo_free(dev, rw_bo);
|
||||
|
||||
vhsa_debug("%s: queue res id: %d, queue ID: %" PRIu64 ", ret = %d\n", __FUNCTION__,
|
||||
bo->real.res_id, bo->queue_id, r);
|
||||
|
||||
return r;
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hsakmt/hsakmt_virtio.h"
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
static int vhsakmt_set_sys_props(vhsakmt_device_handle dev, HsaSystemProperties* sys_props) {
|
||||
int r = 0;
|
||||
|
||||
pthread_mutex_lock(&dev->vhsakmt_mutex);
|
||||
if (dev->sys_props) {
|
||||
r = 0;
|
||||
goto out;
|
||||
}
|
||||
|
||||
dev->sys_props = calloc(1, sizeof(HsaSystemProperties));
|
||||
if (!dev->sys_props) {
|
||||
r = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
memcpy(dev->sys_props, sys_props, sizeof(HsaSystemProperties));
|
||||
|
||||
out:
|
||||
pthread_mutex_unlock(&dev->vhsakmt_mutex);
|
||||
return r;
|
||||
}
|
||||
|
||||
static int vhsakmt_set_node_props(vhsakmt_device_handle dev, uint32_t node,
|
||||
HsaNodeProperties* node_props) {
|
||||
int r = 0;
|
||||
if (!dev->sys_props) return -EINVAL;
|
||||
if (node >= dev->sys_props->NumNodes) return -EINVAL;
|
||||
|
||||
pthread_mutex_lock(&dev->vhsakmt_mutex);
|
||||
|
||||
if (!dev->vhsakmt_nodes) {
|
||||
dev->vhsakmt_nodes = calloc(dev->sys_props->NumNodes, sizeof(struct vhsakmt_node));
|
||||
if (!dev->vhsakmt_nodes) {
|
||||
r = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(&dev->vhsakmt_nodes[node].node_props, node_props, sizeof(HsaNodeProperties));
|
||||
|
||||
out:
|
||||
pthread_mutex_unlock(&dev->vhsakmt_mutex);
|
||||
return r;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetVersion(HsaVersionInfo* v) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_VER,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
memcpy(v, &rsp->kfd_version, sizeof(HsaVersionInfo));
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtAcquireSystemProperties(HsaSystemProperties* SystemProperties) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
int r;
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_SYS_PROP,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
memcpy(SystemProperties, &rsp->sys_props, sizeof(HsaSystemProperties));
|
||||
|
||||
r = vhsakmt_set_sys_props(dev, SystemProperties);
|
||||
if (r) return r;
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtReleaseSystemProperties(void) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_REL_SYS_PROP,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
if (dev->sys_props) {
|
||||
free(dev->sys_props);
|
||||
dev->sys_props = NULL;
|
||||
}
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeProperties(HSAuint32 NodeId,
|
||||
HsaNodeProperties* NodeProperties) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
int r;
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.NodeID = NodeId,
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_NODE_PROP,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
memcpy(NodeProperties, &rsp->node_props, sizeof(HsaNodeProperties));
|
||||
|
||||
r = vhsakmt_set_node_props(dev, NodeId, NodeProperties);
|
||||
if (r) return r;
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetXNACKMode(HSAint32* enable) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_XNACK_MODE,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
memcpy(enable, &rsp->xnack_mode, sizeof(HSAint32));
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeEnable(void* rDebug, bool setupTtmp) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.run_time_enable_args.setupTtmp = setupTtmp,
|
||||
.type = VHSAKMT_CCMD_QUERY_RUN_TIME_ENABLE,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeDisable(void) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_RUN_TIME_DISABLE,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAuint32 NumBanks,
|
||||
HsaMemoryProperties* MemoryProperties) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_NOD_MEM_PROP,
|
||||
.node_mem_prop_args.NodeId = NodeId,
|
||||
.node_mem_prop_args.NumBanks = NumBanks,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(
|
||||
dev, &req.hdr,
|
||||
sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumBanks * sizeof(HsaMemoryProperties));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
memcpy(MemoryProperties, rsp->payload, NumBanks * sizeof(HsaMemoryProperties));
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeCacheProperties(HSAuint32 NodeId, HSAuint32 ProcessorId,
|
||||
HSAuint32 NumCaches,
|
||||
HsaCacheProperties* CacheProperties) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_NOD_CACHE_PROP,
|
||||
.node_cache_prop_args.NodeId = NodeId,
|
||||
.node_cache_prop_args.ProcessorId = ProcessorId,
|
||||
.node_cache_prop_args.NumCaches = NumCaches,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(
|
||||
dev, &req.hdr,
|
||||
sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumCaches * sizeof(HsaCacheProperties));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
memcpy(CacheProperties, rsp->payload, NumCaches * sizeof(HsaCacheProperties));
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks,
|
||||
HsaIoLinkProperties* IoLinkProperties) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_NOD_IO_LINK_PROP,
|
||||
.node_io_link_args.NodeId = NodeId,
|
||||
.node_io_link_args.NumIoLinks = NumIoLinks,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(
|
||||
dev, &req.hdr,
|
||||
sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumIoLinks * sizeof(HsaIoLinkProperties));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
memcpy(IoLinkProperties, rsp->payload, NumIoLinks * sizeof(HsaIoLinkProperties));
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetClockCounters(HSAuint32 NodeId, HsaClockCounters* Counters) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_CLOCK_COUNTERS,
|
||||
.NodeID = NodeId,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
memcpy(Counters, &rsp->clock_counters, sizeof(HsaClockCounters));
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI vhsaKmtGetRuntimeCapabilities(HSAuint32* caps_mask) {
|
||||
CHECK_VIRTIO_KFD_OPEN();
|
||||
|
||||
vhsakmt_device_handle dev = vhsakmt_dev();
|
||||
struct vhsakmt_ccmd_query_info_rsp* rsp;
|
||||
struct vhsakmt_ccmd_query_info_req req = {
|
||||
.hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)),
|
||||
.type = VHSAKMT_CCMD_QUERY_GET_RUNTIME_CAPS,
|
||||
};
|
||||
|
||||
rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp));
|
||||
if (!rsp) return -ENOMEM;
|
||||
|
||||
vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__);
|
||||
|
||||
*caps_mask = rsp->caps;
|
||||
|
||||
return rsp->ret;
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "hsakmt_virtio_device.h"
|
||||
|
||||
void* vhsakmt_vm_start(void) {
|
||||
void* vm_start = malloc(getpagesize());
|
||||
if (!vm_start) return NULL;
|
||||
|
||||
free(vm_start);
|
||||
return vm_start;
|
||||
}
|
||||
|
||||
int vhsakmt_reserve_va(uint64_t start, uint64_t size) {
|
||||
int32_t protFlags = PROT_NONE;
|
||||
int32_t mapFlags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED;
|
||||
void* va = mmap((void*)start, size, protFlags, mapFlags, -1, 0);
|
||||
if (va == MAP_FAILED) return -ENOMEM;
|
||||
|
||||
if (va != (void*)start) return -ENOMEM;
|
||||
|
||||
madvise(va, size, MADV_DONTFORK);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void vhsakmt_dereserve_va(uint64_t start, uint64_t size) { munmap((void*)start, size); }
|
||||
|
||||
void vhsakmt_set_scratch_area(vhsakmt_device_handle dev, uint32_t node, uint64_t start,
|
||||
uint64_t size) {
|
||||
if (!dev->vhsakmt_nodes || !dev->sys_props) return;
|
||||
if (node >= dev->sys_props->NumNodes) return;
|
||||
|
||||
pthread_mutex_lock(&dev->vhsakmt_mutex);
|
||||
|
||||
if (dev->vhsakmt_nodes[node].scratch_start && dev->vhsakmt_nodes[node].scratch_size) goto out;
|
||||
|
||||
dev->vhsakmt_nodes[node].scratch_start = start;
|
||||
dev->vhsakmt_nodes[node].scratch_size = size;
|
||||
|
||||
out:
|
||||
pthread_mutex_unlock(&dev->vhsakmt_mutex);
|
||||
}
|
||||
|
||||
bool vhsakmt_is_scratch_mem(vhsakmt_device_handle dev, void* addr) {
|
||||
uint32_t i;
|
||||
if (!dev->vhsakmt_nodes || !dev->sys_props) return false;
|
||||
|
||||
for (i = 0; i < dev->sys_props->NumNodes; i++) {
|
||||
if ((uint64_t)addr >= dev->vhsakmt_nodes[i].scratch_start &&
|
||||
(uint64_t)addr <= dev->vhsakmt_nodes[i].scratch_start + dev->vhsakmt_nodes[i].scratch_size)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void vhsakmt_set_vm_area(vhsakmt_device_handle dev, uint64_t start, uint64_t size) {
|
||||
pthread_mutex_lock(&dev->vhsakmt_mutex);
|
||||
if (dev->vm_start && dev->vm_size) goto out;
|
||||
|
||||
dev->vm_start = start;
|
||||
dev->vm_size = size;
|
||||
|
||||
out:
|
||||
pthread_mutex_unlock(&dev->vhsakmt_mutex);
|
||||
}
|
||||
|
||||
bool vhsakmt_is_userptr(vhsakmt_device_handle dev, void* addr) {
|
||||
return !((uint64_t)addr >= dev->vm_start && (uint64_t)addr <= dev->vm_start + dev->vm_size);
|
||||
}
|
||||
|
||||
int vhsakmt_set_node_doorbell(vhsakmt_device_handle dev, uint32_t node, void* doorbell) {
|
||||
if (!dev->vhsakmt_nodes || !dev->sys_props) return -EINVAL;
|
||||
if (node >= dev->sys_props->NumNodes) return -EINVAL;
|
||||
|
||||
pthread_mutex_lock(&dev->vhsakmt_mutex);
|
||||
|
||||
dev->vhsakmt_nodes[node].doorbell_base = doorbell;
|
||||
|
||||
pthread_mutex_unlock(&dev->vhsakmt_mutex);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void* vhsakmt_node_doorbell(vhsakmt_device_handle dev, uint32_t node) {
|
||||
if (!dev->vhsakmt_nodes || !dev->sys_props) return NULL;
|
||||
if (node >= dev->sys_props->NumNodes) return NULL;
|
||||
|
||||
return dev->vhsakmt_nodes[node].doorbell_base;
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/*
|
||||
* Copyright 2013 Red Hat
|
||||
* All Rights Reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef VIRTGPU_DRM_H
|
||||
#define VIRTGPU_DRM_H
|
||||
|
||||
#include "drm.h"
|
||||
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Please note that modifications to all structs defined here are
|
||||
* subject to backwards-compatibility constraints.
|
||||
*
|
||||
* Do not use pointers, use __u64 instead for 32 bit / 64 bit user/kernel
|
||||
* compatibility Keep fields aligned to their size
|
||||
*/
|
||||
|
||||
#define DRM_VIRTGPU_MAP 0x01
|
||||
#define DRM_VIRTGPU_EXECBUFFER 0x02
|
||||
#define DRM_VIRTGPU_GETPARAM 0x03
|
||||
#define DRM_VIRTGPU_RESOURCE_CREATE 0x04
|
||||
#define DRM_VIRTGPU_RESOURCE_INFO 0x05
|
||||
#define DRM_VIRTGPU_TRANSFER_FROM_HOST 0x06
|
||||
#define DRM_VIRTGPU_TRANSFER_TO_HOST 0x07
|
||||
#define DRM_VIRTGPU_WAIT 0x08
|
||||
#define DRM_VIRTGPU_GET_CAPS 0x09
|
||||
#define DRM_VIRTGPU_RESOURCE_CREATE_BLOB 0x0a
|
||||
#define DRM_VIRTGPU_CONTEXT_INIT 0x0b
|
||||
|
||||
#define VIRTGPU_EXECBUF_FENCE_FD_IN 0x01
|
||||
#define VIRTGPU_EXECBUF_FENCE_FD_OUT 0x02
|
||||
#define VIRTGPU_EXECBUF_RING_IDX 0x04
|
||||
#define VIRTGPU_EXECBUF_FLAGS (\
|
||||
VIRTGPU_EXECBUF_FENCE_FD_IN |\
|
||||
VIRTGPU_EXECBUF_FENCE_FD_OUT |\
|
||||
VIRTGPU_EXECBUF_RING_IDX |\
|
||||
0)
|
||||
|
||||
struct drm_virtgpu_map {
|
||||
__u64 offset; /* use for mmap system call */
|
||||
__u32 handle;
|
||||
__u32 pad;
|
||||
};
|
||||
|
||||
#define VIRTGPU_EXECBUF_SYNCOBJ_RESET 0x01
|
||||
#define VIRTGPU_EXECBUF_SYNCOBJ_FLAGS ( \
|
||||
VIRTGPU_EXECBUF_SYNCOBJ_RESET | \
|
||||
0)
|
||||
struct drm_virtgpu_execbuffer_syncobj {
|
||||
__u32 handle;
|
||||
__u32 flags;
|
||||
__u64 point;
|
||||
};
|
||||
|
||||
/* fence_fd is modified on success if VIRTGPU_EXECBUF_FENCE_FD_OUT flag is set. */
|
||||
struct drm_virtgpu_execbuffer {
|
||||
__u32 flags;
|
||||
__u32 size;
|
||||
__u64 command; /* void* */
|
||||
__u64 bo_handles;
|
||||
__u32 num_bo_handles;
|
||||
__s32 fence_fd; /* in/out fence fd (see VIRTGPU_EXECBUF_FENCE_FD_IN/OUT) */
|
||||
__u32 ring_idx; /* command ring index (see VIRTGPU_EXECBUF_RING_IDX) */
|
||||
__u32 syncobj_stride; /* size of @drm_virtgpu_execbuffer_syncobj */
|
||||
__u32 num_in_syncobjs;
|
||||
__u32 num_out_syncobjs;
|
||||
__u64 in_syncobjs;
|
||||
__u64 out_syncobjs;
|
||||
};
|
||||
|
||||
#define VIRTGPU_PARAM_3D_FEATURES 1 /* do we have 3D features in the hw */
|
||||
#define VIRTGPU_PARAM_CAPSET_QUERY_FIX 2 /* do we have the capset fix */
|
||||
#define VIRTGPU_PARAM_RESOURCE_BLOB 3 /* DRM_VIRTGPU_RESOURCE_CREATE_BLOB */
|
||||
#define VIRTGPU_PARAM_HOST_VISIBLE 4 /* Host blob resources are mappable */
|
||||
#define VIRTGPU_PARAM_CROSS_DEVICE 5 /* Cross virtio-device resource sharing */
|
||||
#define VIRTGPU_PARAM_CONTEXT_INIT 6 /* DRM_VIRTGPU_CONTEXT_INIT */
|
||||
#define VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs 7 /* Bitmask of supported capability set ids */
|
||||
#define VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME 8 /* Ability to set debug name from userspace */
|
||||
|
||||
struct drm_virtgpu_getparam {
|
||||
__u64 param;
|
||||
__u64 value;
|
||||
};
|
||||
|
||||
/* NO_BO flags? NO resource flag? */
|
||||
/* resource flag for y_0_top */
|
||||
struct drm_virtgpu_resource_create {
|
||||
__u32 target;
|
||||
__u32 format;
|
||||
__u32 bind;
|
||||
__u32 width;
|
||||
__u32 height;
|
||||
__u32 depth;
|
||||
__u32 array_size;
|
||||
__u32 last_level;
|
||||
__u32 nr_samples;
|
||||
__u32 flags;
|
||||
__u32 bo_handle; /* if this is set - recreate a new resource attached to this bo ? */
|
||||
__u32 res_handle; /* returned by kernel */
|
||||
__u32 size; /* validate transfer in the host */
|
||||
__u32 stride; /* validate transfer in the host */
|
||||
};
|
||||
|
||||
struct drm_virtgpu_resource_info {
|
||||
__u32 bo_handle;
|
||||
__u32 res_handle;
|
||||
__u32 size;
|
||||
__u32 blob_mem;
|
||||
};
|
||||
|
||||
struct drm_virtgpu_3d_box {
|
||||
__u32 x;
|
||||
__u32 y;
|
||||
__u32 z;
|
||||
__u32 w;
|
||||
__u32 h;
|
||||
__u32 d;
|
||||
};
|
||||
|
||||
struct drm_virtgpu_3d_transfer_to_host {
|
||||
__u32 bo_handle;
|
||||
struct drm_virtgpu_3d_box box;
|
||||
__u32 level;
|
||||
__u32 offset;
|
||||
__u32 stride;
|
||||
__u32 layer_stride;
|
||||
};
|
||||
|
||||
struct drm_virtgpu_3d_transfer_from_host {
|
||||
__u32 bo_handle;
|
||||
struct drm_virtgpu_3d_box box;
|
||||
__u32 level;
|
||||
__u32 offset;
|
||||
__u32 stride;
|
||||
__u32 layer_stride;
|
||||
};
|
||||
|
||||
#define VIRTGPU_WAIT_NOWAIT 1 /* like it */
|
||||
struct drm_virtgpu_3d_wait {
|
||||
__u32 handle; /* 0 is an invalid handle */
|
||||
__u32 flags;
|
||||
};
|
||||
|
||||
#define VIRTGPU_DRM_CAPSET_VIRGL 1
|
||||
#define VIRTGPU_DRM_CAPSET_VIRGL2 2
|
||||
#define VIRTGPU_DRM_CAPSET_GFXSTREAM_VULKAN 3
|
||||
#define VIRTGPU_DRM_CAPSET_VENUS 4
|
||||
#define VIRTGPU_DRM_CAPSET_CROSS_DOMAIN 5
|
||||
#define VIRTGPU_DRM_CAPSET_DRM 6
|
||||
struct drm_virtgpu_get_caps {
|
||||
__u32 cap_set_id;
|
||||
__u32 cap_set_ver;
|
||||
__u64 addr;
|
||||
__u32 size;
|
||||
__u32 pad;
|
||||
};
|
||||
|
||||
struct drm_virtgpu_resource_create_blob {
|
||||
#define VIRTGPU_BLOB_MEM_GUEST 0x0001
|
||||
#define VIRTGPU_BLOB_MEM_HOST3D 0x0002
|
||||
#define VIRTGPU_BLOB_MEM_HOST3D_GUEST 0x0003
|
||||
|
||||
#define VIRTGPU_BLOB_FLAG_USE_MAPPABLE 0x0001
|
||||
#define VIRTGPU_BLOB_FLAG_USE_SHAREABLE 0x0002
|
||||
#define VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE 0x0004
|
||||
#define VIRTGPU_BLOB_FLAG_USE_USERPTR 0x0008
|
||||
/* zero is invalid blob_mem */
|
||||
__u32 blob_mem;
|
||||
__u32 blob_flags;
|
||||
__u32 bo_handle;
|
||||
__u32 res_handle;
|
||||
__u64 size;
|
||||
|
||||
/*
|
||||
* for 3D contexts with VIRTGPU_BLOB_MEM_HOST3D_GUEST and
|
||||
* VIRTGPU_BLOB_MEM_HOST3D otherwise, must be zero.
|
||||
*/
|
||||
__u32 pad;
|
||||
__u32 cmd_size;
|
||||
__u64 cmd;
|
||||
__u64 blob_id;
|
||||
__u64 blob_userptr;
|
||||
__s64 offset;
|
||||
};
|
||||
|
||||
#define VIRTGPU_CONTEXT_PARAM_CAPSET_ID 0x0001
|
||||
#define VIRTGPU_CONTEXT_PARAM_NUM_RINGS 0x0002
|
||||
#define VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK 0x0003
|
||||
#define VIRTGPU_CONTEXT_PARAM_DEBUG_NAME 0x0004
|
||||
struct drm_virtgpu_context_set_param {
|
||||
__u64 param;
|
||||
__u64 value;
|
||||
};
|
||||
|
||||
struct drm_virtgpu_context_init {
|
||||
__u32 num_params;
|
||||
__u32 pad;
|
||||
|
||||
/* pointer to drm_virtgpu_context_set_param array */
|
||||
__u64 ctx_set_params;
|
||||
};
|
||||
|
||||
/*
|
||||
* Event code that's given when VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK is in
|
||||
* effect. The event size is sizeof(drm_event), since there is no additional
|
||||
* payload.
|
||||
*/
|
||||
#define VIRTGPU_EVENT_FENCE_SIGNALED 0x90000000
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_MAP \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_MAP, struct drm_virtgpu_map)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_EXECBUFFER \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_EXECBUFFER,\
|
||||
struct drm_virtgpu_execbuffer)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_GETPARAM \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GETPARAM,\
|
||||
struct drm_virtgpu_getparam)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_RESOURCE_CREATE \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_CREATE, \
|
||||
struct drm_virtgpu_resource_create)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_RESOURCE_INFO \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_INFO, \
|
||||
struct drm_virtgpu_resource_info)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_TRANSFER_FROM_HOST \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_FROM_HOST, \
|
||||
struct drm_virtgpu_3d_transfer_from_host)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_TRANSFER_TO_HOST \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_TO_HOST, \
|
||||
struct drm_virtgpu_3d_transfer_to_host)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_WAIT \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_WAIT, \
|
||||
struct drm_virtgpu_3d_wait)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_GET_CAPS \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GET_CAPS, \
|
||||
struct drm_virtgpu_get_caps)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_RESOURCE_CREATE_BLOB \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_CREATE_BLOB, \
|
||||
struct drm_virtgpu_resource_create_blob)
|
||||
|
||||
#define DRM_IOCTL_VIRTGPU_CONTEXT_INIT \
|
||||
DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_CONTEXT_INIT, \
|
||||
struct drm_virtgpu_context_init)
|
||||
|
||||
#if defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
global:
|
||||
vhsaKmtOpenKFD;
|
||||
vhsaKmtCloseKFD;
|
||||
vhsaKmtAllocMemory;
|
||||
vhsaKmtFreeMemory;
|
||||
vhsaKmtMapMemoryToGPUNodes;
|
||||
vhsaKmtUnmapMemoryToGPU;
|
||||
vhsaKmtAvailableMemory;
|
||||
vhsaKmtMapMemoryToGPU;
|
||||
vhsaKmtRegisterMemoryWithFlags;
|
||||
vhsaKmtDeregisterMemory;
|
||||
vhsaKmtGetVersion;
|
||||
vhsaKmtAcquireSystemProperties;
|
||||
vhsaKmtReleaseSystemProperties;
|
||||
vhsaKmtGetNodeProperties;
|
||||
vhsaKmtGetXNACKMode;
|
||||
vhsaKmtRuntimeEnable;
|
||||
vhsaKmtRuntimeDisable;
|
||||
vhsaKmtGetNodeMemoryProperties;
|
||||
vhsaKmtGetNodeCacheProperties;
|
||||
vhsaKmtGetNodeIoLinkProperties;
|
||||
vhsaKmtGetClockCounters;
|
||||
vhsaKmtGetAMDGPUDeviceHandle;
|
||||
vhsaKmtQueryPointerInfo;
|
||||
vhsaKmtGetTileConfig;
|
||||
vhsaKmtCreateEvent;
|
||||
vhsaKmtDestroyEvent;
|
||||
vhsaKmtSetEvent;
|
||||
vhsaKmtResetEvent;
|
||||
vhsaKmtQueryEventState;
|
||||
vhsaKmtWaitOnMultipleEvents;
|
||||
vhsaKmtWaitOnEvent;
|
||||
vhsaKmtWaitOnEvent_Ext;
|
||||
vhsaKmtWaitOnMultipleEvents_Ext;
|
||||
vhsaKmtSetTrapHandler;
|
||||
vhsaKmtCreateQueueExt;
|
||||
vhsaKmtCreateQueue;
|
||||
vhsaKmtDestroyQueue;
|
||||
vhsaKmtRegisterGraphicsHandleToNodes;
|
||||
vhsaKmtGetRuntimeCapabilities;
|
||||
vamdgpu_query_gpu_info;
|
||||
local: *;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <errno.h>
|
||||
#include <libsync.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/mman.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
#include "virtio_gpu.h"
|
||||
|
||||
#define SHMEM_SZ (25 * 0x1000)
|
||||
|
||||
static int set_context(int fd) {
|
||||
struct drm_virtgpu_context_set_param params[] = {
|
||||
{VIRTGPU_CONTEXT_PARAM_CAPSET_ID, VIRGL_RENDERER_CAPSET_HSAKMT},
|
||||
{VIRTGPU_CONTEXT_PARAM_NUM_RINGS, 64},
|
||||
};
|
||||
struct drm_virtgpu_context_init args = {
|
||||
.num_params = ARRAY_SIZE(params),
|
||||
.ctx_set_params = (uintptr_t)(params),
|
||||
};
|
||||
|
||||
return virtio_gpu_ioctl(fd, VIRTGPU_CONTEXT_INIT, &args);
|
||||
}
|
||||
|
||||
int virtio_gpu_map_handle(struct virtio_gpu_device* vgdev, uint32_t handle, uint64_t size,
|
||||
void** addr, void* fixed_map) {
|
||||
struct drm_virtgpu_map args = {
|
||||
.handle = handle,
|
||||
};
|
||||
int r;
|
||||
|
||||
r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_MAP, &args);
|
||||
if (r) return r;
|
||||
|
||||
*addr = mmap(fixed_map, size, PROT_READ | PROT_WRITE, MAP_SHARED | (fixed_map ? MAP_FIXED : 0),
|
||||
vgdev->fd, args.offset);
|
||||
|
||||
if (*addr == MAP_FAILED) return -EINVAL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void virtio_gpu_unmap(void* addr, uint64_t size) { munmap(addr, size); }
|
||||
|
||||
static void virtio_gpu_bo_close(struct virtio_gpu_device* vgdev, uint32_t handle) {
|
||||
struct drm_gem_close args = {
|
||||
.handle = handle,
|
||||
};
|
||||
|
||||
virtio_gpu_ioctl(vgdev->fd, GEM_CLOSE, &args);
|
||||
}
|
||||
|
||||
static int virtio_gpu_shmem_init(struct virtio_gpu_device* vgdev, size_t size) {
|
||||
struct drm_virtgpu_resource_create_blob args = {
|
||||
.blob_mem = VIRTGPU_BLOB_MEM_HOST3D,
|
||||
.blob_flags = VIRTGPU_BLOB_FLAG_USE_MAPPABLE,
|
||||
.size = size,
|
||||
.blob_id = 0,
|
||||
};
|
||||
|
||||
int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_CREATE_BLOB, &args);
|
||||
if (r) return r;
|
||||
|
||||
r = virtio_gpu_map_handle(vgdev, args.bo_handle, size, (void**)&vgdev->shmem, NULL);
|
||||
if (r) {
|
||||
virtio_gpu_bo_close(vgdev, args.bo_handle);
|
||||
return r;
|
||||
}
|
||||
|
||||
vgdev->shmem_handle = args.bo_handle;
|
||||
|
||||
uint32_t offset = vgdev->shmem->base.rsp_mem_offset;
|
||||
vgdev->rsp_mem_len = size - offset;
|
||||
vgdev->rsp_mem = &((uint8_t*)vgdev->shmem)[offset];
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct virtio_gpu_device* virtio_gpu_init(int fd, uint32_t context_id) {
|
||||
struct virtio_gpu_device* vgdev;
|
||||
int r;
|
||||
|
||||
r = set_context(fd);
|
||||
|
||||
if (r) return NULL;
|
||||
|
||||
vgdev = calloc(1, sizeof(*vgdev));
|
||||
if (!vgdev) return NULL;
|
||||
|
||||
vgdev->fd = fd;
|
||||
|
||||
vgdev->reqbuf = calloc(1, SHMEM_SZ);
|
||||
if (!vgdev->reqbuf) {
|
||||
free(vgdev);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
r = virtio_gpu_shmem_init(vgdev, SHMEM_SZ);
|
||||
if (r) {
|
||||
free(vgdev);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pthread_mutex_init(&vgdev->rsp_lock, NULL);
|
||||
pthread_mutex_init(&vgdev->eb_lock, NULL);
|
||||
|
||||
return vgdev;
|
||||
}
|
||||
|
||||
void virtio_gpu_close(struct virtio_gpu_device* vgdev) {
|
||||
virtio_gpu_unmap(vgdev->shmem, SHMEM_SZ);
|
||||
virtio_gpu_bo_close(vgdev, vgdev->shmem_handle);
|
||||
|
||||
pthread_mutex_destroy(&vgdev->rsp_lock);
|
||||
pthread_mutex_destroy(&vgdev->eb_lock);
|
||||
|
||||
close(vgdev->fd);
|
||||
free(vgdev->reqbuf);
|
||||
free(vgdev);
|
||||
}
|
||||
|
||||
void* virtio_gpu_alloc_rsp(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req,
|
||||
uint32_t size) {
|
||||
uint32_t off;
|
||||
|
||||
pthread_mutex_lock(&vgdev->rsp_lock);
|
||||
|
||||
size = VHSA_ALIGN_UP(size, 8);
|
||||
|
||||
if ((vgdev->next_rsp_off + size) >= vgdev->rsp_mem_len) vgdev->next_rsp_off = 0;
|
||||
|
||||
off = vgdev->next_rsp_off;
|
||||
vgdev->next_rsp_off += size;
|
||||
|
||||
pthread_mutex_unlock(&vgdev->rsp_lock);
|
||||
|
||||
req->rsp_off = off;
|
||||
struct virtio_gpu_ccmd_rsp* rsp = (void*)&vgdev->rsp_mem[off];
|
||||
rsp->len = size;
|
||||
|
||||
return rsp;
|
||||
}
|
||||
|
||||
static int virtio_gpu_execbuffer_locked(struct virtio_gpu_device* vgdev, void* cmd,
|
||||
uint32_t cmd_size, uint32_t* handles, uint32_t num_handles,
|
||||
int* fence_fd, int ring_idx, uint32_t num_in_syncobjs,
|
||||
uint32_t num_out_syncobjs,
|
||||
struct drm_virtgpu_execbuffer_syncobj* in_syncobjs,
|
||||
struct drm_virtgpu_execbuffer_syncobj* out_syncobjs,
|
||||
bool in_fence, bool out_fence) {
|
||||
struct drm_virtgpu_execbuffer eb = {
|
||||
.flags = (out_fence ? VIRTGPU_EXECBUF_FENCE_FD_OUT : 0) |
|
||||
(in_fence ? VIRTGPU_EXECBUF_FENCE_FD_IN : 0) | VIRTGPU_EXECBUF_RING_IDX,
|
||||
.size = cmd_size,
|
||||
.command = (uintptr_t)cmd,
|
||||
.bo_handles = (uintptr_t)handles,
|
||||
.num_bo_handles = num_handles,
|
||||
.fence_fd = *fence_fd,
|
||||
.ring_idx = ring_idx,
|
||||
.syncobj_stride = sizeof(struct drm_virtgpu_execbuffer_syncobj),
|
||||
.num_in_syncobjs = num_in_syncobjs,
|
||||
.num_out_syncobjs = num_out_syncobjs,
|
||||
.in_syncobjs = (uintptr_t)in_syncobjs,
|
||||
.out_syncobjs = (uintptr_t)out_syncobjs,
|
||||
};
|
||||
int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_EXECBUFFER, &eb);
|
||||
if (r) return r;
|
||||
|
||||
if (out_fence) *fence_fd = eb.fence_fd;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int virtio_gpu_flush_locked(struct virtio_gpu_device* vgdev, int* fence) {
|
||||
int r;
|
||||
|
||||
if (!vgdev->reqbuf_len) return 0;
|
||||
|
||||
r = virtio_gpu_execbuffer_locked(vgdev, vgdev->reqbuf, vgdev->reqbuf_len, NULL, 0, fence, 0, 0, 0,
|
||||
NULL, NULL, false, !!fence);
|
||||
if (r) return r;
|
||||
|
||||
vgdev->reqbuf_len = 0;
|
||||
vgdev->reqbuf_cnt = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int virtio_gpu_add_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req) {
|
||||
req->seqno = ++vgdev->next_seqno;
|
||||
int r;
|
||||
|
||||
if (vgdev->reqbuf_len + req->len > sizeof(vgdev->reqbuf)) {
|
||||
r = virtio_gpu_flush_locked(vgdev, NULL);
|
||||
if (r) return r;
|
||||
}
|
||||
|
||||
memcpy(&vgdev->reqbuf[vgdev->reqbuf_len], req, req->len);
|
||||
vgdev->reqbuf_len += req->len;
|
||||
vgdev->reqbuf_cnt++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline bool fence_before(uint32_t a, uint32_t b) { return (int32_t)(a - b) < 0; }
|
||||
|
||||
static void virtio_gpu_seqno_sync(struct virtio_gpu_device* vgdev,
|
||||
struct virtio_gpu_ccmd_req* req) {
|
||||
while (fence_before(vgdev->shmem->base.seqno, req->seqno)) sched_yield();
|
||||
}
|
||||
|
||||
int virtio_gpu_exec_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req,
|
||||
bool sync) {
|
||||
int r = 0;
|
||||
int fence;
|
||||
|
||||
pthread_mutex_lock(&vgdev->eb_lock);
|
||||
|
||||
r = virtio_gpu_add_cmd(vgdev, req);
|
||||
|
||||
if (r || !sync) goto out;
|
||||
|
||||
r = virtio_gpu_flush_locked(vgdev, &fence);
|
||||
|
||||
out:
|
||||
pthread_mutex_unlock(&vgdev->eb_lock);
|
||||
if (r) return r;
|
||||
|
||||
if (sync) {
|
||||
sync_wait(fence, -1);
|
||||
close(fence);
|
||||
virtio_gpu_seqno_sync(vgdev, req);
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
int virtio_gpu_create_blob(struct virtio_gpu_device* vgdev,
|
||||
struct drm_virtgpu_resource_create_blob* args) {
|
||||
return virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_CREATE_BLOB, args);
|
||||
}
|
||||
|
||||
int virtio_gpu_destroy_handle(struct virtio_gpu_device* vgdev, uint32_t bo_handle) {
|
||||
struct drm_gem_close args = {
|
||||
.handle = bo_handle,
|
||||
};
|
||||
|
||||
return virtio_gpu_ioctl(vgdev->fd, GEM_CLOSE, &args);
|
||||
}
|
||||
|
||||
int virtio_gpu_res_id(struct virtio_gpu_device* vgdev, uint32_t handle, uint32_t* res_id) {
|
||||
struct drm_virtgpu_resource_info args = {
|
||||
.bo_handle = handle,
|
||||
};
|
||||
int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_INFO, &args);
|
||||
if (r) return r;
|
||||
|
||||
*res_id = args.res_handle;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int virtio_gpu_get_capset(int fd, struct virgl_renderer_capset_hsakmt* caps) {
|
||||
struct drm_virtgpu_get_caps args = {
|
||||
.cap_set_id = VIRGL_RENDERER_CAPSET_HSAKMT,
|
||||
.cap_set_ver = 0,
|
||||
.addr = (uintptr_t)caps,
|
||||
.size = sizeof(*caps),
|
||||
};
|
||||
|
||||
memset(caps, 0, sizeof(*caps));
|
||||
|
||||
return virtio_gpu_ioctl(fd, VIRTGPU_GET_CAPS, &args);
|
||||
}
|
||||
|
||||
int virtio_gpu_kfd_open(void) {
|
||||
drmDevicePtr devices[VHSA_MAX_DEVICES];
|
||||
int num_devices = 0;
|
||||
int i, fd, ret;
|
||||
|
||||
num_devices = drmGetDevices2(0, devices, ARRAY_SIZE(devices));
|
||||
if (num_devices <= 0) return -1;
|
||||
|
||||
for (i = 0; i < num_devices; i++) {
|
||||
fd = open(devices[i]->nodes[DRM_NODE_RENDER], O_RDWR | O_CLOEXEC);
|
||||
if (fd < 0) continue;
|
||||
|
||||
struct virgl_renderer_capset_hsakmt caps;
|
||||
ret = virtio_gpu_get_capset(fd, &caps);
|
||||
if (ret || caps.context_type != VIRTGPU_DRM_CONTEXT_AMDGPU) {
|
||||
close(fd);
|
||||
fd = -1;
|
||||
continue;
|
||||
}
|
||||
|
||||
goto out;
|
||||
}
|
||||
|
||||
out:
|
||||
drmFreeDevices(devices, num_devices);
|
||||
return fd;
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* Copyright 2025 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VIRTIO_GPU_H
|
||||
#define VIRTIO_GPU_H
|
||||
|
||||
#include <pthread.h>
|
||||
#include <stdint.h>
|
||||
#include <xf86drm.h>
|
||||
|
||||
#include "virtgpu_drm.h"
|
||||
|
||||
#define VIRGL_RENDERER_CAPSET_HSAKMT 8
|
||||
#define VIRTGPU_DRM_CONTEXT_AMDGPU 1
|
||||
#define VHSA_MAX_DEVICES 10
|
||||
|
||||
#ifndef ARRAY_SIZE
|
||||
#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
|
||||
#endif
|
||||
|
||||
#define VHSA_ALIGN_UP(x, align) (((uint64_t)(x) + (align)-1) & ~(uint64_t)((align)-1))
|
||||
#define VHSA_ALIGN_DOWN(x, align) ((uint64_t)(x) & ~(uint64_t)((align)-1))
|
||||
|
||||
#define virtio_gpu_ioctl(fd, name, args) \
|
||||
({ \
|
||||
int ret = drmIoctl((fd), DRM_IOCTL_##name, (args)); \
|
||||
ret; \
|
||||
})
|
||||
|
||||
struct virgl_renderer_capset_hsakmt {
|
||||
uint32_t wire_format_version;
|
||||
/* Underlying drm device version: */
|
||||
uint32_t version_major;
|
||||
uint32_t version_minor;
|
||||
uint32_t version_patchlevel;
|
||||
uint32_t context_type;
|
||||
uint32_t pad;
|
||||
};
|
||||
|
||||
struct virtio_gpu_shmem_base {
|
||||
uint32_t seqno;
|
||||
uint32_t rsp_mem_offset;
|
||||
};
|
||||
|
||||
struct virtio_gpu_ccmd_req {
|
||||
uint32_t cmd;
|
||||
uint32_t len;
|
||||
uint32_t seqno;
|
||||
uint32_t rsp_off;
|
||||
};
|
||||
|
||||
struct virtio_gpu_ccmd_rsp {
|
||||
uint32_t len;
|
||||
};
|
||||
|
||||
struct virtio_gpu_shmem {
|
||||
struct virtio_gpu_shmem_base base;
|
||||
uint32_t async_error;
|
||||
uint32_t global_faults;
|
||||
};
|
||||
|
||||
#define vhsakmt_shmem virtio_gpu_shmem
|
||||
#define vhsakmt_ccmd_req virtio_gpu_ccmd_req
|
||||
#define vhsakmt_ccmd_rsp virtio_gpu_ccmd_rsp
|
||||
|
||||
struct virtio_gpu_device {
|
||||
int fd;
|
||||
|
||||
struct virtio_gpu_shmem* shmem;
|
||||
uint32_t shmem_handle;
|
||||
|
||||
uint8_t* rsp_mem;
|
||||
uint32_t rsp_mem_len;
|
||||
uint32_t next_rsp_off;
|
||||
pthread_mutex_t rsp_lock;
|
||||
pthread_mutex_t eb_lock;
|
||||
|
||||
uint32_t next_seqno;
|
||||
uint32_t reqbuf_len;
|
||||
uint32_t reqbuf_cnt;
|
||||
uint8_t* reqbuf;
|
||||
};
|
||||
|
||||
struct virtio_gpu_device* virtio_gpu_init(int fd, uint32_t context_id);
|
||||
void virtio_gpu_close(struct virtio_gpu_device* vgdev);
|
||||
int virtio_gpu_exec_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req,
|
||||
bool sync);
|
||||
void* virtio_gpu_alloc_rsp(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req,
|
||||
uint32_t size);
|
||||
int virtio_gpu_map_handle(struct virtio_gpu_device* vgdev, uint32_t handle, uint64_t size,
|
||||
void** addr, void* fixed_map);
|
||||
void virtio_gpu_unmap(void* addr, uint64_t size);
|
||||
int virtio_gpu_create_blob(struct virtio_gpu_device* vgdev,
|
||||
struct drm_virtgpu_resource_create_blob* args);
|
||||
int virtio_gpu_destroy_handle(struct virtio_gpu_device* vgdev, uint32_t bo_handle);
|
||||
int virtio_gpu_res_id(struct virtio_gpu_device* vgdev, uint32_t handle, uint32_t* res_id);
|
||||
int virtio_gpu_kfd_open(void);
|
||||
|
||||
#endif /* VIRTIO_GPU_H */
|
||||
@@ -302,6 +302,10 @@ target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE elf::elf dl pthread rt )
|
||||
# Link to hsakmt-staticdrm target for static library builds
|
||||
if( BUILD_SHARED_LIBS )
|
||||
target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE hsakmt::hsakmt PkgConfig::drm)
|
||||
if( BUILD_THUNK_VIRTIO )
|
||||
message(STATUS "Building with virtio support")
|
||||
target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE hsakmt_virtio)
|
||||
endif()
|
||||
find_package(rocprofiler-register)
|
||||
if(rocprofiler-register_FOUND)
|
||||
target_compile_definitions(${CORE_RUNTIME_TARGET} PRIVATE HSA_ROCPROFILER_REGISTER=1
|
||||
|
||||
Reference in New Issue
Block a user