diff --git a/projects/rocr-runtime/CMakeLists.txt b/projects/rocr-runtime/CMakeLists.txt index ba8a551507..d375db8b8c 100644 --- a/projects/rocr-runtime/CMakeLists.txt +++ b/projects/rocr-runtime/CMakeLists.txt @@ -108,12 +108,20 @@ if (HSA_DEP_ROCPROFILER_REGISTER) string(APPEND CPACK_RPM_BINARY_PACKAGE_REQUIRES " rocprofiler-register") endif() +if (NOT DEFINED BUILD_THUNK_VIRTIO) + set(BUILD_THUNK_VIRTIO OFF) +endif() + add_rocm_subdir(libhsakmt "${THUNK_DEFINITIONS}") set_target_properties(hsakmt PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/archive" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/lib" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/runtime") +if (BUILD_THUNK_VIRTIO) + add_rocm_subdir(libhsakmt/src/virtio "${THUNK_VIRTIO_DEFINITIONS}") +endif() + if (BUILD_ROCR) add_rocm_subdir(runtime/hsa-runtime "${ROCR_DEFINITIONS}") set_target_properties(hsa-runtime64 PROPERTIES @@ -123,6 +131,9 @@ if (BUILD_ROCR) if (BUILD_SHARED_LIBS) add_dependencies(hsa-runtime64 hsakmt) + if (BUILD_THUNK_VIRTIO) + add_dependencies(hsa-runtime64 hsakmt_virtio) + endif() else() add_dependencies(hsa-runtime64 hsakmt-staticdrm) endif() diff --git a/projects/rocr-runtime/libhsakmt/CMakeLists.txt b/projects/rocr-runtime/libhsakmt/CMakeLists.txt index 8b0ad114af..6849de8721 100644 --- a/projects/rocr-runtime/libhsakmt/CMakeLists.txt +++ b/projects/rocr-runtime/libhsakmt/CMakeLists.txt @@ -211,7 +211,7 @@ install ( TARGETS ${HSAKMT_TARGET} # Install public headers install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/${HSAKMT_TARGET} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - COMPONENT dev PATTERN "linux" EXCLUDE ) + COMPONENT dev PATTERN "linux" EXCLUDE PATTERN "*virtio*" EXCLUDE) # Record our usage data for clients find_package calls. install ( EXPORT ${HSAKMT_TARGET}Targets diff --git a/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt_virtio.h b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt_virtio.h new file mode 100644 index 0000000000..6a2aa6554e --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt_virtio.h @@ -0,0 +1,113 @@ +/* + * Copyright © 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef HSAKMT_VIRTIO_H +#define HSAKMT_VIRTIO_H + +#include "hsakmt/linux/kfd_ioctl.h" +#include "hsakmt/hsakmt.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +HSAKMT_STATUS HSAKMTAPI vhsaKmtOpenKFD(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCloseKFD(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtAllocMemory(HSAuint32 PreferredNode, HSAuint64 SizeInBytes, + HsaMemFlags MemFlags, void** MemoryAddress); +HSAKMT_STATUS HSAKMTAPI vhsaKmtFreeMemory(void* MemoryAddress, HSAuint64 SizeInBytes); +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPUNodes(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU, + HsaMemMapFlags MemMapFlags, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray); +HSAKMT_STATUS HSAKMTAPI vhsaKmtUnmapMemoryToGPU(void* MemoryAddress); +HSAKMT_STATUS HSAKMTAPI vhsaKmtAvailableMemory(HSAuint32 Node, HSAuint64* AvailableBytes); +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPU(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterMemoryWithFlags(void* MemoryAddress, + HSAuint64 MemorySizeInBytes, + HsaMemFlags MemFlags); +HSAKMT_STATUS HSAKMTAPI vhsaKmtDeregisterMemory(void* MemoryAddress); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetVersion(HsaVersionInfo* v); +HSAKMT_STATUS HSAKMTAPI vhsaKmtAcquireSystemProperties(HsaSystemProperties* SystemProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtReleaseSystemProperties(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeProperties(HSAuint32 NodeId, + HsaNodeProperties* NodeProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetXNACKMode(HSAint32* enable); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeEnable(void* rDebug, bool setupTtmp); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeDisable(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAuint32 NumBanks, + HsaMemoryProperties* MemoryProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeCacheProperties(HSAuint32 NodeId, HSAuint32 ProcessorId, + HSAuint32 NumCaches, + HsaCacheProperties* CacheProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks, + HsaIoLinkProperties* IoLinkProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetClockCounters(HSAuint32 NodeId, HsaClockCounters* Counters); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetAMDGPUDeviceHandle(HSAuint32 NodeId, + HsaAMDGPUDeviceHandle* DeviceHandle); +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryPointerInfo(const void* Pointer, HsaPointerInfo* PointerInfo); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig* config); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateEvent(HsaEventDescriptor* EventDesc, _Bool ManualReset, + _Bool IsSignaled, HsaEvent** Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyEvent(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetEvent(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtResetEvent(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryEventState(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent(HsaEvent* Event, HSAuint32 Milliseconds); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent_Ext(HsaEvent* Event, HSAuint32 Milliseconds, + uint64_t* event_age); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents_Ext(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds, + uint64_t* event_age); +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetTrapHandler(HSAuint32 NodeId, void* TrapHandlerBaseAddress, + HSAuint64 TrapHandlerSizeInBytes, + void* TrapBufferBaseAddress, + HSAuint64 TrapBufferSizeInBytes); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueueExt(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, + HSA_QUEUE_PRIORITY Priority, HSAuint32 SdmaEngineId, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueue(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource); +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyQueue(HSA_QUEUEID QueueId); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterGraphicsHandleToNodes( + HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo* GraphicsResourceInfo, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetRuntimeCapabilities(HSAuint32* caps_mask); + +int vamdgpu_query_gpu_info(amdgpu_device_handle dev, void* out); + +#ifdef __cplusplus +} +#endif + +#endif /* HSAKMT_VIRTIO_H */ diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/CMakeLists.txt b/projects/rocr-runtime/libhsakmt/src/virtio/CMakeLists.txt new file mode 100644 index 0000000000..c0c7a23c4c --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/CMakeLists.txt @@ -0,0 +1,88 @@ + +# Copyright 2025 Advanced Micro Devices, Inc. + +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +cmake_minimum_required ( VERSION 3.7 ) + +set (CMAKE_VERBOSE_MAKEFILE ON) + +set ( HSAKMT_VIRTIO "hsakmt_virtio" ) +set ( HSAKMT_VIRTIO_TARGET "${HSAKMT_VIRTIO}" ) + +project ( ${HSAKMT_VIRTIO_TARGET} VERSION 1.0) + +## Compiler flags +set ( HSAKMT_VIRTIO_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden ) +if ( CMAKE_COMPILER_IS_GNUCC ) + set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -Wlogical-op ) +endif () + +set ( HSAKMT_VIRTIO_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/libhsakmt_virtio.ver" ) + +set ( HSAKMT_VIRTIO_LINK_FLAGS "-Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_VIRTIO_LINKER_SCRIPT} -Wl,-z,nodelete") + +if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) + set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -O2 ) +else () + set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -g ) +endif () + +set ( HSAKMT_VIRTIO_SRC "virtio_gpu.c" + "hsakmt_virtio_vm.c" + "hsakmt_virtio_device.c" + "hsakmt_virtio_memory.c" + "hsakmt_virtio_amdgpu.c" + "hsakmt_virtio_events.c" + "hsakmt_virtio_queues.c" + "hsakmt_virtio_topology.c" + "hsakmt_virtio_openclose.c" + "../rbtree.c" ) + +add_library ( ${HSAKMT_VIRTIO_TARGET} STATIC ${HSAKMT_VIRTIO_SRC} ) + +target_sources ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${HSAKMT_VIRTIO_SRC} ) + +target_compile_options ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${HSAKMT_VIRTIO_C_FLAGS} ) + +target_include_directories ( ${HSAKMT_VIRTIO_TARGET} + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/virtio + ${CMAKE_CURRENT_SOURCE_DIR}/../ + ${CMAKE_CURRENT_SOURCE_DIR}/../../include + ${CMAKE_CURRENT_SOURCE_DIR}/include/linux ) + +set_property(TARGET ${HSAKMT_VIRTIO_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_VIRTIO_LINK_FLAGS}) + +find_package ( PkgConfig ) + +## If environment variable DRM_DIR is set, the script +## will pick up the corresponding libraries from that path. +list ( PREPEND CMAKE_PREFIX_PATH "${DRM_DIR}" ) + +pkg_check_modules ( DRM REQUIRED IMPORTED_TARGET libdrm ) +pkg_check_modules ( DRM_AMDGPU REQUIRED IMPORTED_TARGET libdrm_amdgpu ) +target_include_directories ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${DRM_AMDGPU_INCLUDE_DIRS} ) +target_include_directories ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${DRM_INCLUDE_DIRS} ) + +target_link_libraries ( ${HSAKMT_VIRTIO_TARGET} + PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt c ${CMAKE_DL_LIBS} ) diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c new file mode 100644 index 0000000000..4aca737221 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c @@ -0,0 +1,50 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +int vamdgpu_query_gpu_info(amdgpu_device_handle handle, void* out) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GPU_INFO, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + int ret = vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + if (!ret) memcpy(out, &rsp->gpu_info, sizeof(struct amdgpu_gpu_info)); + + return ret; +} + +HSAKMT_STATUS vhsaKmtGetAMDGPUDeviceHandle(HSAuint32 NodeId, HsaAMDGPUDeviceHandle* DeviceHandle) { + CHECK_VIRTIO_KFD_OPEN(); + + return HSAKMT_STATUS_SUCCESS; +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_device.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_device.c new file mode 100644 index 0000000000..4e7aed500a --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_device.c @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt_virtio_device.h" + +int vhsakmt_execbuf_cpu(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, const char* from) { + return virtio_gpu_exec_cmd(dev->vgdev, req, true); +} + +void* vhsakmt_alloc_rsp(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, uint32_t sz) { + return virtio_gpu_alloc_rsp(dev->vgdev, req, sz); +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_device.h b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_device.h new file mode 100644 index 0000000000..ac53fba263 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_device.h @@ -0,0 +1,189 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef HSAKMT_VIRTIO_DEVICE_H +#define HSAKMT_VIRTIO_DEVICE_H + +#include "hsakmt_virtio_proto.h" +#include "rbtree.h" +#include "virtio_gpu.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define vhsakmt_atomic_inc_return(ptr) (atomic_fetch_add((ptr), 1) + 1) +#define vhsakmt_atomic_dec_return(ptr) (atomic_fetch_sub((ptr), 1) - 1) + +#define VHSA_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr)) +#define VHSA_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v)) + +extern int vhsakmt_debug_level; +#define vhsakmt_print(level, fmt, ...) \ + do { \ + if (level <= vhsakmt_debug_level) fprintf(stderr, fmt, ##__VA_ARGS__); \ + } while (0) +#define VHSAKMT_DEBUG_LEVEL_DEFAULT -1 +#define VHSAKMT_DEBUG_LEVEL_ERR 3 +#define VHSAKMT_DEBUG_LEVEL_WARNING 4 +#define VHSAKMT_DEBUG_LEVEL_INFO 6 +#define VHSAKMT_DEBUG_LEVEL_DEBUG 7 +#define vhsa_err(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_ERR, fmt, ##__VA_ARGS__) +#define vhsa_warn(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_WARNING, fmt, ##__VA_ARGS__) +#define vhsa_info(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__) +#define vhsa_debug(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__) + +struct vhsakmt_device; +struct vhsakmt_bo; + +typedef struct vhsakmt_device* vhsakmt_device_handle; +typedef struct vhsakmt_bo* vhsakmt_bo_handle; +typedef rbtree_node_t* bo_entry; + +extern pthread_mutex_t dev_mutex; +extern vhsakmt_device_handle dev_list; + +#define VHSA_BO_KFD_MEM 1 << 0 /* allocated from KFD (hsaKmtAllocMemory) */ +#define VHSA_BO_USERPTR 1 << 1 +#define VHSA_BO_QUEUE_BUFFER 1 << 2 /* allocated from KFD, but used for queue CMD submit */ +#define VHSA_BO_QUEUE_DOORBELL 1 << 3 /* doorbell memory */ +#define VHSA_BO_QUEUE_RW_PTR 1 << 4 /* queue read write ptr, from host map to guest*/ +/* allocated from KFD, but used for AQL queue read write ptr */ +#define VHSA_BO_QUEUE_AQL_RW_PTR 1 << 5 +#define VHSA_BO_CLGL 1 << 6 /* CLGL memory, imported from mesa GL */ +/* allocated from KFD, but is scratch memory, do not need map and unmap in ioctrl */ +#define VHSA_BO_SCRATCH 1 << 7 +#define VHSA_BO_QUEUE 1 << 8 +#define VHSA_BO_EVENT 1 << 9 +#define VHSA_BO_SCRATCH_MAP 1 << 10 + +#define VHSA_SDMA_NONE UINT32_MAX + +#define CHECK_VIRTIO_KFD_OPEN() \ + do { \ + if (dev_list == NULL) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; \ + } while (0) + +struct vhsakmt_node { + HsaNodeProperties node_props; + void* doorbell_base; + uint64_t scratch_start; + uint64_t scratch_size; +}; + +struct vhsakmt_device { + struct virtio_gpu_device* vgdev; + int refcount; + pthread_mutex_t bo_handles_mutex; + rbtree_t bo_rbt; + + struct vhsakmt_bo* shmem_bo; + + uint32_t reqbuf_max; + uint32_t next_blob_id; + + uint64_t vm_start; + uint64_t vm_size; + + pthread_mutex_t vhsakmt_mutex; + struct vhsakmt_node* vhsakmt_nodes; + HsaSystemProperties* sys_props; +}; + +struct vhsakmt_bo { + rbtree_node_t rbtn; + struct vhsakmt_device* dev; + + int refcount; + unsigned size; + void* cpu_addr; + void* host_addr; + HsaMemFlags flags; + uint32_t bo_type; + uint32_t blob_id; + pthread_mutex_t map_mutex; + + union { + struct { + uint32_t handle; + uint32_t res_id; + uint64_t offset; + uint64_t alloc_size; + int map_count; + } real; + }; + + vHsaEvent* event; + uint64_t queue_id; + vhsakmt_bo_handle rw_bo; + void* gl_meta_data; +}; + +/*hsakmt_virtio_memory.c*/ +vhsakmt_bo_handle vhsakmt_entry_to_bo_handle(bo_entry e); +bo_entry vhsakmt_bo_handle_to_entry(vhsakmt_bo_handle bo); + +void vhsakmt_insert_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo, void* addr, uint64_t size); +void vhsakmt_remove_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo); +vhsakmt_bo_handle vhsakmt_find_bo_by_addr(vhsakmt_device_handle dev, void* addr); +void* vhsakmt_gpu_va(vhsakmt_device_handle dev, void* va); + +int vhsakmt_bo_cpu_unmap(vhsakmt_bo_handle bo); +int vhsakmt_bo_cpu_map(vhsakmt_bo_handle bo_handle, void** cpu, void* fixed_cpu); +int vhsakmt_create_mappable_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle); +int vhsakmt_bo_free(vhsakmt_device_handle dev, vhsakmt_bo_handle bo); +int vhsakmt_init_host_blob(vhsakmt_device_handle dev, size_t size, uint32_t blob_type, + uint32_t blob_flag, uint32_t blob_id, uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle); + +/*hsakmt_virtio_openclose.c*/ +vhsakmt_device_handle vhsakmt_dev(void); + +/*hsakmt_virtio_vm.c*/ +void* vhsakmt_vm_start(void); +int vhsakmt_reserve_va(uint64_t start, uint64_t size); +void vhsakmt_dereserve_va(uint64_t start, uint64_t size); +void vhsakmt_set_scratch_area(vhsakmt_device_handle dev, uint32_t node, uint64_t start, + uint64_t size); +void vhsakmt_set_vm_area(vhsakmt_device_handle dev, uint64_t start, uint64_t size); +int vhsakmt_set_node_doorbell(vhsakmt_device_handle dev, uint32_t node, void* doorbell); +void* vhsakmt_node_doorbell(vhsakmt_device_handle dev, uint32_t node); +bool vhsakmt_is_scratch_mem(vhsakmt_device_handle dev, void* addr); +bool vhsakmt_is_userptr(vhsakmt_device_handle dev, void* addr); + +/*hsakmt_virtio_device.c*/ +int vhsakmt_execbuf_cpu(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, const char* from); +void* vhsakmt_alloc_rsp(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, uint32_t sz); + +/*hsakmt_virtio_event.c*/ +void* vhsakmt_event_host_handle(HsaEvent* h); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_events.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_events.c new file mode 100644 index 0000000000..e7ee49cbed --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_events.c @@ -0,0 +1,189 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +int vhsakmt_debug_level; + +void* vhsakmt_event_host_handle(HsaEvent* h) { return (void*)((vHsaEvent*)h)->event_handle; } + +static inline int32_t vhsakmt_event_res_id(HsaEvent* h) { return ((vHsaEvent*)h)->res_id; } + +static inline vhsakmt_bo_handle vhsakmt_event_bo_handle(HsaEvent* h) { + return (vhsakmt_bo_handle)((vHsaEvent*)h)->bo_handle; +} + +static int vhsakmt_create_event_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + vHsaEvent* vevent_handle, vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, 0, blob_id, VHSA_BO_EVENT, + (void*)vevent_handle->event_handle, bo_handle); + if (r) return r; + + (*bo_handle)->event = vevent_handle; + vevent_handle->bo_handle = (uint64_t)(*bo_handle); + vevent_handle->res_id = (*bo_handle)->real.res_id; + vhsakmt_insert_bo(dev, *bo_handle, vevent_handle, size); + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateEvent(HsaEventDescriptor* EventDesc, _Bool ManualReset, + _Bool IsSignaled, HsaEvent** Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + vhsakmt_bo_handle event_bo; + vHsaEvent* e; + int r; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_CREATE, + .create_args.EventDesc = *EventDesc, + .create_args.ManualReset = ManualReset, + .create_args.IsSignaled = IsSignaled, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) return rsp->ret; + + e = calloc(1, sizeof(vHsaEvent)); + if (!e) return -ENOMEM; + + memcpy(e, &rsp->vevent, sizeof(vHsaEvent)); + + r = vhsakmt_create_event_blob_bo(dev, sizeof(vHsaEvent), req.blob_id, e, &event_bo); + if (r) { + free(e); + return -ENOMEM; + } + + *Event = (HsaEvent*)e; + + vhsa_debug( + "%s: event addr: %p, hw123: %lx, %lx, %x, type: %d, id: %x, host handle: 0x%lx, res id: %d\n", + __FUNCTION__, e, e->event.EventData.HWData1, e->event.EventData.HWData2, + e->event.EventData.HWData3, e->event.EventData.EventType, e->event.EventId, e->event_handle, + event_bo->real.res_id); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyEvent(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_bo* bo; + + if (Event == NULL) return HSAKMT_STATUS_SUCCESS; + + bo = vhsakmt_event_bo_handle(Event); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + return vhsakmt_bo_free(dev, bo); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetEvent(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_SET, + .event_hanele = vhsakmt_event_host_handle(Event), + .res_id = vhsakmt_event_res_id(Event), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtResetEvent(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_RESET, + .event_hanele = vhsakmt_event_host_handle(Event), + .res_id = vhsakmt_event_res_id(Event), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryEventState(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_QUERY_STATE, + .event_hanele = vhsakmt_event_host_handle(Event), + .res_id = vhsakmt_event_res_id(Event), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds) { + return HSAKMT_STATUS_ERROR; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent(HsaEvent* Event, HSAuint32 Milliseconds) { + return vhsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent_Ext(HsaEvent* Event, HSAuint32 Milliseconds, + uint64_t* event_age) { + return vhsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents_Ext(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds, + uint64_t* event_age) { + return vhsaKmtWaitOnMultipleEvents(Events, NumEvents, WaitOnAll, Milliseconds); +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_memory.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_memory.c new file mode 100644 index 0000000000..a4afc1696e --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_memory.c @@ -0,0 +1,852 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +#define VHSA_GL_METADATA_MAX_SIZE (0x50) + +vhsakmt_bo_handle vhsakmt_entry_to_bo_handle(bo_entry e) { return (vhsakmt_bo_handle)e; } +bo_entry vhsakmt_bo_handle_to_entry(vhsakmt_bo_handle bo) { return &bo->rbtn; } +static inline bool vhsakmt_is_mem_bo(vhsakmt_bo_handle bo) { return (!bo->queue_id && !bo->event); } + +static bool vhsakmt_mappable(HsaMemFlags flags) { return (!flags.ui32.Scratch); } + +static bool vhsakmt_bo_mappable(vhsakmt_bo_handle bo) { return vhsakmt_mappable(bo->flags); } + +void vhsakmt_insert_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo, void* addr, uint64_t size) { + bo->rbtn.key.addr = (unsigned long)addr; + bo->rbtn.key.size = (unsigned long)size; + + pthread_mutex_lock(&dev->bo_handles_mutex); + hsakmt_rbtree_insert(&dev->bo_rbt, &bo->rbtn); + pthread_mutex_unlock(&dev->bo_handles_mutex); +} + +static void vhsakmt_remove_entry(vhsakmt_device_handle dev, bo_entry entry) { + if (!entry) return; + + pthread_mutex_lock(&dev->bo_handles_mutex); + hsakmt_rbtree_delete(&dev->bo_rbt, entry); + pthread_mutex_unlock(&dev->bo_handles_mutex); +} + +void vhsakmt_remove_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + bo_entry entry = vhsakmt_bo_handle_to_entry(bo); + if (entry->key.addr == 0 && entry->key.size == 0) return; + + vhsakmt_remove_entry(dev, entry); +} + +static bo_entry vhsakmt_rbt_search(vhsakmt_device_handle dev, void* addr) { + vhsakmt_bo_handle bo; + + rbtree_key_t key = rbtree_key((uint64_t)addr, 0); + pthread_mutex_lock(&dev->bo_handles_mutex); + bo_entry n = rbtree_lookup_nearest(&dev->bo_rbt, &key, LKP_ADDR, RIGHT); + pthread_mutex_unlock(&dev->bo_handles_mutex); + if (n) { + bo = vhsakmt_entry_to_bo_handle(n); + if (bo->cpu_addr != addr) return NULL; + return n; + } + + return NULL; +} + +static bo_entry vhsakmt_find_entry_by_addr(vhsakmt_device_handle dev, void* addr) { + return vhsakmt_rbt_search(dev, addr); +} + +vhsakmt_bo_handle vhsakmt_find_bo_by_addr(vhsakmt_device_handle dev, void* addr) { + bo_entry entry = vhsakmt_find_entry_by_addr(dev, addr); + + if (entry) { + vhsakmt_bo_handle bo = vhsakmt_entry_to_bo_handle(entry); + if (!vhsakmt_is_mem_bo(bo)) return NULL; + + return bo; + } + + return NULL; +} + +void* vhsakmt_gpu_va(vhsakmt_device_handle dev, void* va) { + if (!vhsakmt_is_userptr(dev, va)) return va; + + bo_entry entry = vhsakmt_find_entry_by_addr(dev, va); + + if (!entry) return NULL; + + return vhsakmt_entry_to_bo_handle(entry)->host_addr; +} + +int vhsakmt_bo_cpu_map(vhsakmt_bo_handle bo, void** cpu, void* fixed_cpu) { + int r; + + if (!vhsakmt_bo_mappable(bo)) return 0; + + pthread_mutex_lock(&bo->map_mutex); + + if (!bo->cpu_addr) { + r = virtio_gpu_map_handle(bo->dev->vgdev, bo->real.handle, bo->size, cpu, fixed_cpu); + if (r) { + pthread_mutex_unlock(&bo->map_mutex); + return r; + } + bo->cpu_addr = *cpu; + atomic_fetch_add(&bo->real.map_count, 1); + } + pthread_mutex_unlock(&bo->map_mutex); + + return *cpu == MAP_FAILED; +} + +int vhsakmt_bo_cpu_unmap(vhsakmt_bo_handle bo) { + int r = 0; + + if (!vhsakmt_bo_mappable(bo)) return 0; + + pthread_mutex_lock(&bo->map_mutex); + + if (!bo->cpu_addr || bo->real.map_count == 0) { + pthread_mutex_unlock(&bo->map_mutex); + return 0; + } + + if (vhsakmt_atomic_dec_return(&bo->real.map_count) <= 0) { + if (bo->bo_type & VHSA_BO_KFD_MEM) { + virtio_gpu_unmap(bo->cpu_addr, bo->size); + vhsakmt_reserve_va(VHSA_VPTR_TO_UINT64(bo->cpu_addr), bo->size); + bo->cpu_addr = NULL; + } + } + + pthread_mutex_unlock(&bo->map_mutex); + return r; +} + +static int vhsakmt_destroy_handle(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + int r = virtio_gpu_destroy_handle(dev->vgdev, bo->real.handle); + free(bo); + + return r; +} + +int vhsakmt_init_host_blob(vhsakmt_device_handle dev, size_t size, uint32_t blob_type, + uint32_t blob_flag, uint32_t blob_id, uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + vhsakmt_bo_handle bo; + struct drm_virtgpu_resource_create_blob args = { + .blob_mem = blob_type, + .size = size, + .blob_id = blob_id, + .blob_flags = blob_flag, + }; + + r = virtio_gpu_create_blob(dev->vgdev, &args); + if (r) return -EINVAL; + + bo = calloc(1, sizeof(struct vhsakmt_bo)); + if (!bo) { + virtio_gpu_destroy_handle(dev->vgdev, args.bo_handle); + return -ENOMEM; + } + + bo->dev = dev; + bo->size = size; + bo->real.alloc_size = size; + bo->bo_type = bo_type; + bo->host_addr = va_handle; + pthread_mutex_init(&bo->map_mutex, NULL); + atomic_store(&bo->real.map_count, 0); + atomic_store(&bo->refcount, 1); + bo->real.handle = args.bo_handle; + + virtio_gpu_res_id(dev->vgdev, bo->real.handle, &bo->real.res_id); + + *bo_handle = bo; + return 0; +} + +static int vhsakmt_init_userptr_blob(vhsakmt_device_handle dev, void* addr, size_t size, + vhsakmt_bo_handle* bo_handle, uint64_t* offset) { + int r; + struct drm_virtgpu_resource_create_blob args = { + .blob_mem = VIRTGPU_BLOB_MEM_HOST3D_GUEST, + .blob_flags = VIRTGPU_BLOB_FLAG_USE_USERPTR, + .size = size, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + .blob_userptr = (uint64_t)addr, + }; + + r = virtio_gpu_create_blob(dev->vgdev, &args); + if (r < 0) return r; + + vhsakmt_bo_handle userptr = calloc(1, sizeof(struct vhsakmt_bo)); + if (!userptr) { + virtio_gpu_destroy_handle(dev->vgdev, args.bo_handle); + return -ENOMEM; + } + + userptr->dev = dev; + userptr->size = size; + userptr->real.alloc_size = size; + userptr->bo_type = VHSA_BO_USERPTR; + userptr->cpu_addr = addr; + pthread_mutex_init(&userptr->map_mutex, NULL); + atomic_store(&userptr->real.map_count, 0); + atomic_store(&userptr->refcount, 1); + userptr->real.handle = args.bo_handle; + + virtio_gpu_res_id(dev->vgdev, userptr->real.handle, &userptr->real.res_id); + + *bo_handle = userptr; + *offset = args.offset; + return r; +} + +int vhsakmt_create_mappable_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, VIRTGPU_BLOB_FLAG_USE_MAPPABLE, + blob_id, bo_type, va_handle, bo_handle); + if (r) return r; + + r = vhsakmt_bo_cpu_map(*bo_handle, &((*bo_handle)->cpu_addr), va_handle); + if (r) { + free(*bo_handle); + *bo_handle = NULL; + return -EINVAL; + } + + if (va_handle && (va_handle != (*bo_handle)->cpu_addr)) + vhsa_warn("%s: target map: %p != real map: %p\n", __FUNCTION__, va_handle, + (*bo_handle)->cpu_addr); + + vhsakmt_insert_bo(dev, *bo_handle, (*bo_handle)->cpu_addr, (*bo_handle)->size); + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtAllocMemory(HSAuint32 PreferredNode, HSAuint64 SizeInBytes, + HsaMemFlags MemFlags, void** MemoryAddress) { + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + vhsakmt_bo_handle bo; + int r; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_ALLOC, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + .alloc_args = + { + .PreferredNode = PreferredNode, + .SizeInBytes = SizeInBytes, + .MemFlags = MemFlags, + }, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) return rsp->ret; + + if (!rsp->memory_handle) return -ENOMEM; + + r = vhsakmt_init_host_blob(dev, SizeInBytes, VIRTGPU_BLOB_MEM_HOST3D, + vhsakmt_mappable(MemFlags) ? VIRTGPU_BLOB_FLAG_USE_MAPPABLE : 0, + req.blob_id, VHSA_BO_KFD_MEM, (void*)rsp->memory_handle, &bo); + if (r) return r; + + if (!vhsakmt_mappable(MemFlags)) { + bo->cpu_addr = bo->host_addr; + if (MemFlags.ui32.Scratch) { + vhsakmt_set_scratch_area(dev, PreferredNode, (uint64_t)bo->cpu_addr, SizeInBytes); + bo->bo_type |= VHSA_BO_SCRATCH; + } + } else { + r = vhsakmt_bo_cpu_map(bo, &bo->cpu_addr, bo->host_addr); + if (r) { + free(bo); + return -ENOMEM; + } + } + + if (!MemFlags.ui32.Scratch) vhsakmt_insert_bo(dev, bo, bo->cpu_addr, bo->size); + + *MemoryAddress = bo->cpu_addr; + + vhsa_debug("alloc mem addr: %p, host addr: %p, size: %lx, res-id: %d, handble: %d\n", + *MemoryAddress, bo->host_addr, SizeInBytes, bo->real.res_id, bo->real.handle); + + return rsp->ret; +} + +int vhsakmt_bo_free(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + bo_entry entry; + int r; + + if (vhsakmt_atomic_dec_return(&bo->refcount) > 0) return 0; + + entry = vhsakmt_bo_handle_to_entry(bo); + if (entry->key.addr == 0 && entry->key.size == 0) return -EINVAL; + + /* do not free BOs of queue, let them be freed with queue */ + if (bo->bo_type & VHSA_BO_QUEUE_DOORBELL) { + vhsa_err("%s: Try to free VHSA_BO_QUEUE_DOORBELL memory: %p\n", __FUNCTION__, bo->cpu_addr); + return 0; + } + + vhsakmt_remove_bo(dev, bo); + + if (bo->cpu_addr) vhsakmt_bo_cpu_unmap(bo); + + if (bo->event) free(bo->event); + + if (bo->gl_meta_data) free(bo->gl_meta_data); + + pthread_mutex_destroy(&bo->map_mutex); + + r = vhsakmt_destroy_handle(dev, bo); + + return r; +} + +/* Only remove bo in rbtree */ +static void vhsakmt_remove_userptr_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + vhsakmt_remove_bo(dev, bo); + free(bo); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtFreeMemory(void* MemoryAddress, HSAuint64 SizeInBytes) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + vhsa_debug("%s: addr: %p, size: %lx, res_id: %d\n", __FUNCTION__, MemoryAddress, SizeInBytes, + bo->real.res_id); + + return vhsakmt_bo_free(dev, bo); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPUNodes(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU, + HsaMemMapFlags MemMapFlags, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + size_t req_len = + VHSA_ALIGN_UP(sizeof(struct vhsakmt_ccmd_memory_req) + NumberOfNodes * sizeof(*NodeArray), 8); + struct vhsakmt_ccmd_memory_req* req; + struct vhsakmt_ccmd_memory_rsp* rsp; + vhsakmt_bo_handle bo; + + req = (void*)calloc(1, req_len); + if (!req) return -ENOMEM; + req->hdr = VHSAKMT_CCMD(MEMORY, req_len); + req->type = VHSAKMT_CCMD_MEMORY_MAP_TO_GPU_NODES; + req->map_to_GPU_nodes_args.MemorySizeInBytes = MemorySizeInBytes; + req->map_to_GPU_nodes_args.MemMapFlags = MemMapFlags; + req->map_to_GPU_nodes_args.NumberOfNodes = NumberOfNodes; + + bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (bo) { + req->map_to_GPU_nodes_args.MemoryAddress = (uint64_t)bo->host_addr; + if (bo->bo_type & VHSA_BO_USERPTR) vhsakmt_remove_userptr_bo(dev, bo); + } else + req->map_to_GPU_nodes_args.MemoryAddress = (uint64_t)MemoryAddress; + + memcpy(req->payload, NodeArray, NumberOfNodes * sizeof(*NodeArray)); + + rsp = vhsakmt_alloc_rsp(dev, &req->hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) { + free(req); + return -ENOMEM; + } + + vhsakmt_execbuf_cpu(dev, &req->hdr, __FUNCTION__); + + *AlternateVAGPU = rsp->alternate_vagpu; + + vhsa_debug("%s: gva: %p, hva: 0x%lx, size: %lx, AlternateVAGPU: %lx, ret: %d\n", __FUNCTION__, + MemoryAddress, req->map_to_GPU_nodes_args.MemoryAddress, MemorySizeInBytes, + *AlternateVAGPU, rsp->ret); + + free(req); + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtUnmapMemoryToGPU(void* MemoryAddress) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + struct vhsakmt_ccmd_memory_rsp* rsp; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_UNMAP_TO_GPU, + .MemoryAddress = (uint64_t)bo->host_addr, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + vhsa_debug("%s: gva: %p, hva: 0x%lx\n", __FUNCTION__, MemoryAddress, req.MemoryAddress); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtAvailableMemory(HSAuint32 Node, HSAuint64* AvailableBytes) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_AVAIL_MEM, + .Node = Node, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + *AvailableBytes = rsp->available_bytes; + + return rsp->ret; +} + +static int vhsakmt_create_scratch_map_memory(vhsakmt_device_handle dev, void* MemoryAddress, + HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU) { + vhsakmt_bo_handle out; + int r; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + .map_to_GPU_args = + { + .MemoryAddress = (uint64_t)MemoryAddress, + .MemorySizeInBytes = MemorySizeInBytes, + .need_create_bo = true, + }, + }; + + struct vhsakmt_ccmd_memory_rsp* rsp = + vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + if (rsp->ret) return rsp->ret; + + r = vhsakmt_init_host_blob(dev, MemorySizeInBytes, VIRTGPU_BLOB_MEM_HOST3D, 0, req.blob_id, + VHSA_BO_SCRATCH_MAP, NULL, &out); + if (r) return r; + + // TODO: insert scratch bo into rbtree, or insert it in dev nodes. + + out->cpu_addr = MemoryAddress; + out->host_addr = (void*)rsp->memory_handle; + *AlternateVAGPU = rsp->alternate_vagpu; + + vhsa_debug( + "%s: create scratch memory, gva: %p, memory_handle: 0x%p, alternate_vagpu: %p, size: %lx\n", + __FUNCTION__, MemoryAddress, (void*)rsp->memory_handle, (void*)rsp->alternate_vagpu, + MemorySizeInBytes); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPU(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo && vhsakmt_is_scratch_mem(dev, MemoryAddress)) + return vhsakmt_create_scratch_map_memory(dev, MemoryAddress, MemorySizeInBytes, AlternateVAGPU); + + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU, + .map_to_GPU_args = + { + .MemoryAddress = bo ? (uint64_t)bo->host_addr : (uint64_t)MemoryAddress, + .MemorySizeInBytes = MemorySizeInBytes, + }, + }; + + if (bo && (bo->bo_type & VHSA_BO_USERPTR)) vhsakmt_remove_userptr_bo(dev, bo); + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + vhsa_debug("%s: gva: %p, hva: 0x%lx, size: %lx\n", __FUNCTION__, MemoryAddress, req.MemoryAddress, + MemorySizeInBytes); + + *AlternateVAGPU = rsp->alternate_vagpu; + + return rsp->ret; +} + +static int vhsakmt_map_userptr(vhsakmt_device_handle dev, void* addr, size_t size, uint32_t res_id, + uint64_t* userptr_handle) { + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_MAP_USERPTR, + .res_id = res_id, + }; + struct vhsakmt_ccmd_memory_rsp* rsp = + vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + rsp->map_userptr_rsp.userptr_handle = 0; + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + *userptr_handle = rsp->map_userptr_rsp.userptr_handle; + return rsp->ret; +} + +static void* vhsakmt_map_to_gpu(void* addr, size_t size) { + vhsakmt_device_handle dev = vhsakmt_dev(); + size_t offset = (uint64_t)addr % getpagesize(); + size_t map_size = (VHSA_ALIGN_UP(size + offset, getpagesize()) / getpagesize()) * getpagesize(); + uint64_t userptr_offset, userptr_handle = 0; + vhsakmt_bo_handle userptr; + int r; + + vhsa_debug("%s: addr: %p, size: 0x%lx, size + offset: 0x%lx, map_size: 0x%lx\n", __FUNCTION__, + addr, size, size + offset, map_size); + + r = vhsakmt_init_userptr_blob(dev, addr, size, &userptr, &userptr_offset); + if (r < 0) { + vhsa_debug("%s: userptr create failed at address: %p, ret = %d\n", __FUNCTION__, addr, r); + return NULL; + } + + vhsakmt_map_userptr(dev, addr, size, userptr->real.res_id, &userptr_handle); + if (!userptr_handle) { + vhsa_debug("%s: map userptr failed at address: %p, ret = %d\n", __FUNCTION__, addr, r); + vhsakmt_destroy_handle(dev, userptr); + vhsakmt_remove_userptr_bo(dev, userptr); + return NULL; + } + userptr->host_addr = VHSA_UINT64_TO_VPTR(VHSA_VPTR_TO_UINT64(userptr_handle) + offset); + + if (r > 0) { + vhsa_debug("%s: userptr: %p already registered, offset: %lx\n", __FUNCTION__, addr, + userptr_offset); + userptr->host_addr = + VHSA_UINT64_TO_VPTR(VHSA_VPTR_TO_UINT64(userptr->host_addr) + userptr_offset); + } + vhsakmt_insert_bo(dev, userptr, userptr->cpu_addr, userptr->size); + + vhsa_debug("%s: real gva: %p, gva: %p, hva: %p, size: %lx, offset: %" PRIu64 + ", map_size: 0x%lx\n", + __FUNCTION__, addr, userptr->cpu_addr, userptr->host_addr, size, offset, map_size); + return userptr->host_addr; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterMemoryWithFlags(void* MemoryAddress, + HSAuint64 MemorySizeInBytes, + HsaMemFlags MemFlags) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + void* addr; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_REG_MEM_WITH_FLAG, + .reg_mem_with_flag = + { + .MemorySizeInBytes = MemorySizeInBytes, + .MemFlags = MemFlags, + }, + }; + + /* no need to register memory from lihsakmt / not a userptr */ + if (!vhsakmt_is_userptr(dev, MemoryAddress)) return HSAKMT_STATUS_SUCCESS; + + addr = vhsakmt_map_to_gpu(MemoryAddress, MemorySizeInBytes); + if (!addr) { + vhsa_debug("%s: register memory failed, gva: %p, size: %lx\n", __FUNCTION__, MemoryAddress, + MemorySizeInBytes); + return HSAKMT_STATUS_ERROR; + } + + req.reg_mem_with_flag.MemoryAddress = (uint64_t)addr; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + return rsp->ret; +} + +static int vhsakmt_remove_clgl_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + struct vhsakmt_ccmd_memory_rsp* rsp; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_DEREG_MEM, + .res_id = bo->real.res_id, + .MemoryAddress = (uint64_t)bo->cpu_addr, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) vhsa_err("%s: deregister failed clgl memory gva: %p\n", __FUNCTION__, bo->cpu_addr); + + vhsakmt_bo_free(dev, bo); + + vhsa_debug("%s: deregister clgl memory gva: %p, ret: %d\n", __FUNCTION__, bo->cpu_addr, rsp->ret); + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtDeregisterMemory(void* MemoryAddress) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + vhsa_debug("%s: remove userptr %p size: 0x%lx, res id: %d\n", __FUNCTION__, MemoryAddress, + (size_t)bo->size, bo->real.res_id); + + if (bo->bo_type & VHSA_BO_CLGL) + return vhsakmt_remove_clgl_bo(dev, bo); + else { + vhsakmt_remove_bo(dev, bo); + free(bo); + } + + return 0; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryPointerInfo(const void* Pointer, HsaPointerInfo* PointerInfo) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + void* gpu_va = vhsakmt_gpu_va(dev, VHSA_UINT64_TO_VPTR(Pointer)); + if (!gpu_va) return -HSAKMT_STATUS_ERROR; + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_POINTER_INFO, + .pointer = VHSA_VPTR_TO_UINT64(gpu_va), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + + QUERY_PTR_INFO_MAX_MAPPED_NODES * sizeof(uint32_t)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(PointerInfo, &rsp->ptr_info, sizeof(HsaPointerInfo)); + + if (PointerInfo->NMappedNodes && PointerInfo->MappedNodes) { + if (PointerInfo->NMappedNodes > QUERY_PTR_INFO_MAX_MAPPED_NODES) { + PointerInfo->NMappedNodes = QUERY_PTR_INFO_MAX_MAPPED_NODES; + vhsa_debug( + "%s: query pointer: %p info mapped nodes greater than QUERY_PTR_INFO_MAX_MAPPED_NODES\n", + __FUNCTION__, Pointer); + } + + PointerInfo->MappedNodes = calloc(PointerInfo->NMappedNodes, sizeof(uint32_t)); + if (!PointerInfo->MappedNodes) { + PointerInfo->NMappedNodes = 0; + return -HSAKMT_STATUS_NO_MEMORY; + } + memcpy(VHSA_UINT64_TO_VPTR(PointerInfo->MappedNodes), rsp->payload, + PointerInfo->NMappedNodes * sizeof(uint32_t)); + } + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig* config) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + uint8_t* config_cpy_addr = NULL; + struct vhsakmt_ccmd_query_info_rsp* rsp; + unsigned req_len = sizeof(struct vhsakmt_ccmd_query_info_req); + unsigned rsp_len = sizeof(struct vhsakmt_ccmd_query_info_rsp) + + config->NumTileConfigs * sizeof(HSAuint32) + config->NumMacroTileConfigs * sizeof(HSAuint32); + + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, req_len), + .type = VHSAKMT_CCMD_QUERY_TILE_CONFIG, + .tile_config_args.NodeId = NodeId, + .tile_config_args.config = *config, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, rsp_len); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(config, &rsp->tile_config_rsp, sizeof(HsaGpuTileConfig)); + config_cpy_addr = ((uint8_t*)rsp->payload); + memcpy(config->TileConfig, config_cpy_addr, config->NumTileConfigs * sizeof(HSAuint32)); + config_cpy_addr += config->NumTileConfigs * sizeof(HSAuint32); + memcpy(config->MacroTileConfig, config_cpy_addr, config->NumMacroTileConfigs * sizeof(HSAuint32)); + + return rsp->ret; +} + +static int vhsakmt_create_clgl_bo(vhsakmt_device_handle dev, void* addr, size_t size, + uint32_t res_id, uint32_t bo_handle, void* meta_data) { + vhsakmt_bo_handle out = calloc(1, sizeof(struct vhsakmt_bo)); + if (!out) return -ENOMEM; + + out->dev = dev; + out->size = size; + atomic_store(&out->real.map_count, 0); + atomic_store(&out->refcount, 1); + +#ifdef CLGL_EXPORT_RESID + out->real.res_id = GraphicsResourceHandle; +#else + out->real.res_id = res_id; +#endif + + /* GL bo handle from GL context*/ + out->real.handle = bo_handle; + out->bo_type |= VHSA_BO_CLGL; + if (meta_data) out->gl_meta_data = meta_data; + + out->host_addr = addr; + + vhsakmt_insert_bo(dev, out, addr, out->size); + + return 0; +} + +static int vhsakmt_gfxhandle_to_resid(vhsakmt_device_handle dev, uint32_t gfx_handle, + uint32_t* res_id, uint32_t* bo_handle) { + int r = drmPrimeFDToHandle(dev->vgdev->fd, gfx_handle, bo_handle); + if (r) { + vhsa_err("%s: drmPrimeFDToHandle failed for handle: %u\n", __FUNCTION__, gfx_handle); + return r; + } + + virtio_gpu_res_id(dev->vgdev, *bo_handle, res_id); + + vhsa_debug("%s: register praphics handle: handle: %d, bo_handle: %d, res_id: %d\n", __FUNCTION__, + gfx_handle, *bo_handle, *res_id); + + return 0; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterGraphicsHandleToNodes( + HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo* GraphicsResourceInfo, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + uint32_t bo_handle, res_id; + uint64_t meta_data_size = VHSA_GL_METADATA_MAX_SIZE; + unsigned req_len = sizeof(struct vhsakmt_ccmd_gl_inter_req) + NumberOfNodes * sizeof(NodeArray); + struct vhsakmt_ccmd_gl_inter_req* req; + struct vhsakmt_ccmd_gl_inter_rsp* rsp; + int r; + + req = calloc(1, req_len); + if (!req) return -ENOMEM; + + req->hdr = VHSAKMT_CCMD(GL_INTER, req_len); + req->type = VHSAKMT_CCMD_GL_REG_GHD_TO_NODES; + req->reg_ghd_to_nodes.NumberOfNodes = NumberOfNodes; + req->reg_ghd_to_nodes.res_handle = GraphicsResourceHandle; + +#ifdef CLGL_EXPORT_RESID + req->reg_ghd_to_nodes.GraphicsResourceHandle = GraphicsResourceHandle; +#else + r = vhsakmt_gfxhandle_to_resid(dev, GraphicsResourceHandle, &res_id, &bo_handle); + if (r) return r; + + req->reg_ghd_to_nodes.GraphicsResourceHandle = bo_handle; + req->reg_ghd_to_nodes.res_handle = res_id; +#endif + + memcpy(req->payload, NodeArray, NumberOfNodes * sizeof(NodeArray)); + + rsp = + vhsakmt_alloc_rsp(dev, &req->hdr, sizeof(struct vhsakmt_ccmd_gl_inter_rsp) + meta_data_size); + if (!rsp) { + r = -ENOMEM; + goto free_out; + } + + vhsakmt_execbuf_cpu(dev, &req->hdr, __FUNCTION__); + if (rsp->ret) return rsp->ret; + + memcpy(GraphicsResourceInfo, &rsp->info, sizeof(HsaGraphicsResourceInfo)); + if (rsp->info.MetadataSizeInBytes) { + GraphicsResourceInfo->Metadata = calloc(1, GraphicsResourceInfo->MetadataSizeInBytes); + if (!GraphicsResourceInfo->Metadata) { + r = -ENOMEM; + goto free_out; + } + + memcpy(VHSA_UINT64_TO_VPTR(GraphicsResourceInfo->Metadata), rsp->payload, + GraphicsResourceInfo->MetadataSizeInBytes); + } else + GraphicsResourceInfo->Metadata = NULL; + + vhsa_debug("%s: register graphics handle: handle: %ld hva: %p, size: %lx\n", __FUNCTION__, + GraphicsResourceHandle, GraphicsResourceInfo->MemoryAddress, + GraphicsResourceInfo->SizeInBytes); + + r = vhsakmt_create_clgl_bo(dev, GraphicsResourceInfo->MemoryAddress, + GraphicsResourceInfo->SizeInBytes, res_id, bo_handle, + VHSA_UINT64_TO_VPTR(GraphicsResourceInfo->Metadata)); + if (r) goto free_out; + + r = rsp->ret; + +free_out: + /* close exported FD after register or close it when deregistre. Close after register here. */ + close(GraphicsResourceHandle); + free(req); + return r; +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_openclose.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_openclose.c new file mode 100644 index 0000000000..7c8f9c18bb --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_openclose.c @@ -0,0 +1,132 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +pthread_mutex_t dev_mutex = PTHREAD_MUTEX_INITIALIZER; +vhsakmt_device_handle dev_list = NULL; + +vhsakmt_device_handle vhsakmt_dev(void) { return dev_list; } + +static HSAKMT_STATUS vhsakmt_openKFD_cmd(vhsakmt_device_handle dev) { + void* vm_start = vhsakmt_vm_start(); + if (!vm_start) return -HSAKMT_STATUS_NO_MEMORY; + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_OPEN_KFD, + .open_kfd_args = + { + .cur_vm_start = VHSA_VPTR_TO_UINT64(vm_start), + }, + }; + + if (!req.open_kfd_args.cur_vm_start) { + vhsa_err("%s: failed to get current heap start address\n", __FUNCTION__); + return -HSAKMT_STATUS_ERROR; + } + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -HSAKMT_STATUS_NO_MEMORY; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp->open_kfd_rsp.vm_start || !rsp->open_kfd_rsp.vm_size) { + vhsa_err("%s: failed to get KFD VM area\n", __FUNCTION__); + return -HSAKMT_STATUS_ERROR; + } + + vhsakmt_set_vm_area(dev, rsp->open_kfd_rsp.vm_start, rsp->open_kfd_rsp.vm_size); + if (vhsakmt_reserve_va(dev->vm_start, dev->vm_size)) { + vhsa_err("%s: failed to reserve VM area: [%lx-%lx]-0x%lx\n", __FUNCTION__, dev->vm_start, + dev->vm_start + dev->vm_size, dev->vm_size); + return -HSAKMT_STATUS_NO_MEMORY; + } + + vhsa_debug("%s: kfd vm range: [%lx-%lx]-0x%lx\n", __FUNCTION__, dev->vm_start, + dev->vm_start + dev->vm_size, dev->vm_size); + return rsp->ret; +} + +static vhsakmt_device_handle vhsakmt_device_init(void) { + int fd; + vhsakmt_device_handle dev = NULL; + + if (vhsakmt_dev()) return vhsakmt_dev(); + + pthread_mutex_lock(&dev_mutex); + + fd = virtio_gpu_kfd_open(); + if (fd < 0) goto open_failed; + + dev = calloc(1, sizeof(struct vhsakmt_device)); + if (!dev) goto open_failed; + + dev->vgdev = virtio_gpu_init(fd, 0); + if (!dev->vgdev) goto malloc_failed; + + rbtree_init(&dev->bo_rbt); + atomic_store(&dev->next_blob_id, 1); + atomic_store(&dev->refcount, 1); + pthread_mutex_init(&dev->bo_handles_mutex, NULL); + pthread_mutex_init(&dev->vhsakmt_mutex, NULL); + dev_list = dev; + + pthread_mutex_unlock(&dev_mutex); + return dev; + +malloc_failed: + free(dev); + dev = NULL; +open_failed: + pthread_mutex_unlock(&dev_mutex); + return dev; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtOpenKFD(void) { + vhsakmt_device_handle dev; + char* d = getenv("VHSAKMT_DEBUG_LEVEL"); + if (d) vhsakmt_debug_level = atoi(d); + + dev = vhsakmt_device_init(); + if (!dev) return HSAKMT_STATUS_ERROR; + + return vhsakmt_openKFD_cmd(vhsakmt_dev()); +} + +static void vhsakmt_device_destroy(struct vhsakmt_device* dev) { + pthread_mutex_destroy(&dev->bo_handles_mutex); + vhsakmt_dereserve_va(dev->vm_start, dev->vm_size); + + if (dev->sys_props) free(dev->sys_props); + if (dev->vhsakmt_nodes) free(dev->vhsakmt_nodes); + + virtio_gpu_close(dev->vgdev); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCloseKFD(void) { + vhsakmt_device_handle dev = vhsakmt_dev(); + pthread_mutex_lock(&dev_mutex); + if (vhsakmt_atomic_dec_return(&dev->refcount) <= 0) vhsakmt_device_destroy(dev); + pthread_mutex_unlock(&dev_mutex); + return 0; +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_proto.h b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_proto.h new file mode 100644 index 0000000000..80f9494003 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_proto.h @@ -0,0 +1,503 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef VHSAKMT_VIRTIO_PROTO_H +#define VHSAKMT_VIRTIO_PROTO_H + +#include "hsakmt/linux/kfd_ioctl.h" +#include "hsakmt/hsakmt.h" + +#include +#include +#include + +#include "virtio_gpu.h" + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic error "-Wpadded" +#endif + +/* defined in other header file in virglrenderer */ +#define VHSAKMT_DEFINE_CAST(parent, child) \ + static inline struct child* to_##child(struct parent* x) { return (struct child*)x; } + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define VHSAKMT_STATIC_ASSERT_SIZE(t) \ + static_assert(sizeof(struct t) % 8 == 0, "sizeof(struct " #t ") not multiple of 8"); \ + static_assert(_Alignof(struct t) <= 8, "alignof(struct " #t ") too large"); +#else +#define VHSAKMT_STATIC_ASSERT_SIZE(t) +#endif + +enum vhsakmt_ccmd { + VHSAKMT_CCMD_NOP = 1, /* No payload, can be used to sync with host */ + VHSAKMT_CCMD_QUERY_INFO, + VHSAKMT_CCMD_EVENT, + VHSAKMT_CCMD_MEMORY, + VHSAKMT_CCMD_QUEUE, + VHSAKMT_CCMD_GL_INTER, +}; + +typedef struct _vHsaEvent { + HsaEvent event; + uint64_t event_handle; + uint64_t bo_handle; + uint32_t res_id; + uint32_t pad; +} vHsaEvent; +VHSAKMT_STATIC_ASSERT_SIZE(_vHsaEvent) + +struct vhsakmt_event_shmem { + uint32_t trigered_events_num; + uint32_t pad; + HsaEvent trigered_events[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_event_shmem) + +#define VHSAKMT_CCMD(_cmd, _len) \ + ((struct vhsakmt_ccmd_req){ \ + .cmd = VHSAKMT_CCMD_##_cmd, \ + .len = (_len), \ + }) + +struct vhsakmt_ccmd_nop_req { + struct vhsakmt_ccmd_req hdr; +}; + +/* + * VHSAKMT_CCMD_QUERY + */ +enum vhsakmt_ccmd_query_type { + VHSAKMT_CCMD_QUERY_GPU_INFO = 0, + VHSAKMT_CCMD_QUERY_OPEN_KFD, + VHSAKMT_CCMD_QUERY_GET_VER, + VHSAKMT_CCMD_QUERY_REL_SYS_PROP, + VHSAKMT_CCMD_QUERY_GET_SYS_PROP, + VHSAKMT_CCMD_QUERY_GET_NODE_PROP, + VHSAKMT_CCMD_QUERY_GET_XNACK_MODE, + VHSAKMT_CCMD_QUERY_RUN_TIME_ENABLE, + VHSAKMT_CCMD_QUERY_RUN_TIME_DISABLE, + VHSAKMT_CCMD_QUERY_GET_NOD_MEM_PROP, + VHSAKMT_CCMD_QUERY_GET_NOD_CACHE_PROP, + VHSAKMT_CCMD_QUERY_GET_NOD_IO_LINK_PROP, + VHSAKMT_CCMD_QUERY_GET_CLOCK_COUNTERS, + VHSAKMT_CCMD_QUERY_POINTER_INFO, + VHSAKMT_CCMD_QUERY_TILE_CONFIG, + VHSAKMT_CCMD_QUERY_NANO_TIME, + VHSAKMT_CCMD_QUERY_GET_RUNTIME_CAPS, +}; + +#define QUERY_PTR_INFO_MAX_MAPPED_NODES 3 + +typedef struct _query_req_run_time_enable_args { + /* void* rDebug, bypassed by payload */ + uint8_t pad[3]; + uint8_t setupTtmp; + uint32_t __pad; +} query_req_run_time_enable_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_run_time_enable_args) + +typedef struct _query_req_node_mem_prop_args { + uint32_t NodeId; + uint32_t NumBanks; +} query_req_node_mem_prop_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_mem_prop_args) + +typedef struct _query_req_node_cache_prop_args { + uint32_t NodeId; + uint32_t ProcessorId; + uint32_t NumCaches; + uint32_t pad; +} query_req_node_cache_prop_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_cache_prop_args) + +typedef struct _query_req_node_io_link_args { + uint32_t NodeId; + uint32_t NumIoLinks; +} query_req_node_io_link_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_io_link_args) + +typedef struct _query_tile_config { + HsaGpuTileConfig config; + uint32_t NodeId; + uint32_t pad; +} query_tile_config; +VHSAKMT_STATIC_ASSERT_SIZE(_query_tile_config) + +typedef struct _query_open_kfd_args { + uint64_t cur_vm_start; +} query_open_kfd_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_open_kfd_args) + +typedef struct _query_open_kfd_rsp { + uint64_t vm_start; + uint64_t vm_size; +} query_open_kfd_rsp; +VHSAKMT_STATIC_ASSERT_SIZE(_query_open_kfd_rsp) + +typedef struct _query_nano_time_rsp { + uint64_t nano_time; +} query_nano_time_rsp; +VHSAKMT_STATIC_ASSERT_SIZE(_query_nano_time_rsp) + +struct vhsakmt_ccmd_query_info_req { + struct vhsakmt_ccmd_req hdr; + struct drm_amdgpu_info info; + uint32_t type; + uint32_t pad; + union { + uint64_t pointer; + uint32_t NodeID; /* some query API just need node ID */ + query_req_run_time_enable_args run_time_enable_args; + query_req_node_mem_prop_args node_mem_prop_args; + query_req_node_cache_prop_args node_cache_prop_args; + query_req_node_io_link_args node_io_link_args; + query_tile_config tile_config_args; + query_open_kfd_args open_kfd_args; + }; + + uint8_t payload[]; +}; +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_query_info_req) +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_query_info_req) +#define VHSAKMT_CCMD_QUERY_MAX_TILE_CONFIG 128 +#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_MEM_PROP 128 +#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_CACHE_PROP 128 +#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_IO_LINK_PROP 128 + +struct vhsakmt_ccmd_query_info_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + union { + query_open_kfd_rsp open_kfd_rsp; + query_nano_time_rsp nano_time_rsp; + HsaGpuTileConfig tile_config_rsp; + HsaPointerInfo ptr_info; + struct amdgpu_gpu_info gpu_info; + HsaVersionInfo kfd_version; + HsaSystemProperties sys_props; + HsaNodeProperties node_props; + int32_t xnack_mode; + HsaClockCounters clock_counters; + uint32_t caps; + uint64_t pad[9]; + }; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_query_info_rsp) + +/* + * VHSAKMT_CCMD_EVENT + */ +enum vhsakmt_ccmd_event_type { + VHSAKMT_CCMD_EVENT_CREATE, + VHSAKMT_CCMD_EVENT_DESTROY, + VHSAKMT_CCMD_EVENT_SET, + VHSAKMT_CCMD_EVENT_RESET, + VHSAKMT_CCMD_EVENT_QUERY_STATE, + VHSAKMT_CCMD_EVENT_WAIT_ON_MULTI_EVENTS, + + VHSAKMT_CCMD_EVENT_SET_TRAP, + +}; +typedef struct _event_req_create_args { + HsaEventDescriptor EventDesc; + uint8_t ManualReset; + uint8_t IsSignaled; + uint8_t pad[6]; +} event_req_create_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_create_args) + +typedef struct _event_req_wait_args { + HsaEvent Event; + uint32_t Milliseconds; + uint32_t pad; +} event_req_wait_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_args) + +typedef struct _event_req_wait_ext_args { + HsaEvent Event; + uint64_t event_age; + uint32_t Milliseconds; + uint32_t pad; +} event_req_wait_ext_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_ext_args) + +typedef struct _event_req_wait_on_multi_args { + /*HsaEvent* Events[], in playloud*/ + uint32_t NumEvents; + uint32_t Milliseconds; + uint8_t WaitOnAll; + uint8_t pad[7]; +} event_req_wait_on_multi_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_on_multi_args) + +typedef struct _event_req_wait_on_multi_ext_args { + /*HsaEvent* Events[], in playloud*/ + uint32_t NumEvents; + uint32_t Milliseconds; + uint64_t event_age; + uint8_t WaitOnAll; + uint8_t pad[7]; +} event_req_wait_on_multi_ext_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_on_multi_ext_args) + +typedef struct _event_set_trap_handler_args { + uint64_t TrapHandlerBaseAddress; + uint64_t TrapHandlerSizeInBytes; + uint64_t TrapBufferBaseAddress; + uint64_t TrapBufferSizeInBytes; + uint32_t NodeId; + uint32_t pad; +} event_set_trap_handler_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_set_trap_handler_args) + +struct vhsakmt_ccmd_event_req { + struct vhsakmt_ccmd_req hdr; + union { + HsaEvent Event; /* For set, reset, query. */ + HsaEvent* event_hanele; + event_req_wait_args wait_args; + event_req_create_args create_args; + event_req_wait_ext_args wait_ext_args; + event_req_wait_on_multi_args wait_on_multi_args; + event_req_wait_on_multi_ext_args wait_on_multi_ext_args; + event_set_trap_handler_args set_trap_handler_args; + }; + uint32_t type; + uint32_t sync_shmem_res_id; + uint64_t blob_id; + uint32_t res_id; + uint32_t pad; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_event_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_event_req) + +struct vhsakmt_ccmd_event_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + vHsaEvent vevent; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_event_rsp) + +/* + * VHSAKMT_CCMD_MEMORY + */ +enum vhsakmt_ccmd_memory_type { + VHSAKMT_CCMD_MEMORY_ALLOC, + VHSAKMT_CCMD_MEMORY_MAP_TO_GPU_NODES, + VHSAKMT_CCMD_MEMORY_FREE, + VHSAKMT_CCMD_MEMORY_UNMAP_TO_GPU, + VHSAKMT_CCMD_MEMORY_AVAIL_MEM, + VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU, + VHSAKMT_CCMD_MEMORY_REG_MEM_WITH_FLAG, + VHSAKMT_CCMD_MEMORY_DEREG_MEM, + VHSAKMT_CCMD_MEMORY_MAP_USERPTR, +}; + +typedef struct _memory_req_alloc_args { + uint32_t PreferredNode; + HsaMemFlags MemFlags; + uint64_t SizeInBytes; + uint64_t MemoryAddress; +} memory_req_alloc_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_alloc_args) + +typedef struct _memory_req_free_args { + uint64_t MemoryAddress; + uint64_t SizeInBytes; +} memory_req_free_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_free_args) + +typedef struct _memory_req_map_to_GPU_nodes_args { + uint64_t MemoryAddress; + uint64_t MemorySizeInBytes; + uint64_t AlternateVAGPU; + HsaMemMapFlags MemMapFlags; + uint32_t pad; + uint64_t NumberOfNodes; + uint32_t* NodeArray; +} memory_req_map_to_GPU_nodes_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_map_to_GPU_nodes_args) + +typedef struct _memory_map_mem_to_gpu_args { + uint64_t MemoryAddress; + uint64_t MemorySizeInBytes; + uint8_t need_create_bo; + uint8_t pad[7]; +} memory_map_mem_to_gpu_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_map_mem_to_gpu_args) + +typedef struct _memory_reg_mem_with_flag { + uint64_t MemoryAddress; + uint64_t MemorySizeInBytes; + HsaMemFlags MemFlags; + uint32_t pad; +} memory_reg_mem_with_flag; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_reg_mem_with_flag) + +struct vhsakmt_ccmd_memory_req { + struct vhsakmt_ccmd_req hdr; + union { + uint64_t MemoryAddress; + uint32_t Node; + memory_req_alloc_args alloc_args; + memory_req_map_to_GPU_nodes_args map_to_GPU_nodes_args; + memory_req_free_args free_args; + memory_map_mem_to_gpu_args map_to_GPU_args; + memory_reg_mem_with_flag reg_mem_with_flag; + }; + uint64_t blob_id; + uint32_t type; + uint32_t res_id; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_memory_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_memory_req) + +typedef struct _vhsakmt_ccmd_memory_map_userptr_rsp { + uint64_t userptr_handle; + uint32_t npfns; + uint32_t pad; +} vhsakmt_ccmd_memory_map_userptr_rsp; +VHSAKMT_STATIC_ASSERT_SIZE(_vhsakmt_ccmd_memory_map_userptr_rsp) + +struct vhsakmt_ccmd_memory_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + union { + vhsakmt_ccmd_memory_map_userptr_rsp map_userptr_rsp; + uint64_t memory_handle; + uint64_t alternate_vagpu; + uint64_t available_bytes; + }; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_memory_rsp) + +/* + * VHSAKMT_CCMD_QUEUE + */ +enum vhsakmt_ccmd_queue_type { + VHSAKMT_CCMD_QUEUE_CREATE, + VHSAKMT_CCMD_QUEUE_DESTROY, +}; + +typedef struct _vHsaQueueResource { + HsaQueueResource r; + uint64_t host_doorbell; + uint64_t host_doorbell_offset; + uint64_t host_write_offset; + uint64_t host_read_offset; + uint64_t host_rw_handle; + uint64_t queue_handle; +} vHsaQueueResource; +VHSAKMT_STATIC_ASSERT_SIZE(_vHsaQueueResource) + +typedef struct _queue_req_create { + uint32_t NodeId; + HSA_QUEUE_TYPE Type; + uint32_t QueuePercentage; + uint32_t pad; + HSA_QUEUE_PRIORITY Priority; + uint32_t pad1; + uint32_t SdmaEngineId; + uint64_t QueueAddress; + uint64_t QueueSizeInBytes; + HsaEvent* Event; + HsaQueueResource* QueueResource; + uint64_t* Queue_write_ptr_aql; + uint64_t* Queue_read_ptr_aql; +} queue_req_create; +VHSAKMT_STATIC_ASSERT_SIZE(_queue_req_create) + +struct vhsakmt_ccmd_queue_req { + struct vhsakmt_ccmd_req hdr; + union { + HSA_QUEUEID QueueId; + queue_req_create create_queue_args; + }; + uint64_t blob_id; /* For queue create, queue resource */ + uint64_t rw_ptr_blob_id; /* For queue create, r/w ptr memory mapping */ + uint64_t doorbell_blob_id; /* For queue create, doorbell ptr memory mapping */ + uint32_t res_id; + uint32_t type; + uint32_t queue_mem_res_id; + uint32_t pad; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_queue_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_queue_req) + +struct vhsakmt_ccmd_queue_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + vHsaQueueResource vqueue_res; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_queue_rsp) + +/* + * VHSAKMT_CCMD_GL_INTER + */ +enum vhsakmt_ccmd_gl_inter_type { + VHSAKMT_CCMD_GL_REG_GHD_TO_NODES, +}; + +typedef struct _gl_inter_req_reg_ghd_to_nodes { + uint64_t GraphicsResourceHandle; + uint64_t NumberOfNodes; // NodeArray in payload + uint32_t res_handle; + uint32_t pad; +} gl_inter_req_reg_ghd_to_nodes; +VHSAKMT_STATIC_ASSERT_SIZE(_gl_inter_req_reg_ghd_to_nodes) + +struct vhsakmt_ccmd_gl_inter_req { + struct vhsakmt_ccmd_req hdr; + union { + gl_inter_req_reg_ghd_to_nodes reg_ghd_to_nodes; + }; + uint32_t type; + uint32_t pad; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_gl_inter_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_gl_inter_req) + +struct vhsakmt_ccmd_gl_inter_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + union { + HsaGraphicsResourceInfo info; + }; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_gl_inter_rsp) + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +#endif diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_queues.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_queues.c new file mode 100644 index 0000000000..5ab12698a8 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_queues.c @@ -0,0 +1,253 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +static inline uint64_t vhsakmt_doorbell_page_size(void) { return 0x2000; } +static inline uint64_t vhsakmt_queue_page_size(void) { return getpagesize(); } + +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetTrapHandler(HSAuint32 NodeId, void* TrapHandlerBaseAddress, + HSAuint64 TrapHandlerSizeInBytes, + void* TrapBufferBaseAddress, + HSAuint64 TrapBufferSizeInBytes) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_SET_TRAP, + .set_trap_handler_args = + { + .NodeId = NodeId, + .TrapHandlerBaseAddress = (uint64_t)TrapHandlerBaseAddress, + .TrapHandlerSizeInBytes = TrapHandlerSizeInBytes, + .TrapBufferBaseAddress = (uint64_t)TrapBufferBaseAddress, + .TrapBufferSizeInBytes = TrapBufferSizeInBytes, + }, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +static int vhsakmt_find_aql_rw_bo(vhsakmt_device_handle dev, uint64_t aql_ptr, + uint32_t* aql_bo_res_id) { + uint64_t aql_base_ptr = VHSA_ALIGN_DOWN(aql_ptr, getpagesize()); + + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, (void*)aql_base_ptr); + if (!bo) return -EINVAL; + + bo->bo_type |= VHSA_BO_QUEUE_AQL_RW_PTR; + *aql_bo_res_id = bo->real.res_id; + return 0; +} + +static int vhsakmt_create_doorbell_blob_bo(vhsakmt_device_handle dev, uint32_t node, size_t size, + uint32_t blob_id, uint64_t host_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_create_mappable_blob_bo(dev, size, blob_id, VHSA_BO_QUEUE_DOORBELL, + (void*)host_handle, bo_handle); + if (r) return r; + + r = vhsakmt_set_node_doorbell(dev, node, (*bo_handle)->cpu_addr); + + return r; +} + +static int vhsakmt_create_queue_rw_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint64_t host_handle, vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_create_mappable_blob_bo(dev, size, blob_id, VHSA_BO_QUEUE_RW_PTR, NULL, bo_handle); + if (r) return r; + + (*bo_handle)->host_addr = (void*)host_handle; + return r; +} + +static int vhsakmt_create_queue_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint64_t queue_id, vhsakmt_bo_handle rw_bo_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, 0, blob_id, VHSA_BO_QUEUE, NULL, + bo_handle); + if (r) return r; + + vhsakmt_insert_bo(dev, *bo_handle, *bo_handle, (*bo_handle)->size); + + (*bo_handle)->queue_id = queue_id; + (*bo_handle)->rw_bo = rw_bo_handle; + + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueueExt(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, + HSA_QUEUE_PRIORITY Priority, HSAuint32 SdmaEngineId, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle rw_bo_handle = NULL, doorbell_bo, queue_bo, queue_mem_bo; + struct vhsakmt_ccmd_queue_rsp* rsp; + struct vhsakmt_ccmd_queue_req req = { + .hdr = VHSAKMT_CCMD(QUEUE, sizeof(struct vhsakmt_ccmd_queue_req)), + .type = VHSAKMT_CCMD_QUEUE_CREATE, + .create_queue_args = + { + .NodeId = NodeId, + .Type = Type, + .QueuePercentage = QueuePercentage, + .Priority = Priority, + .SdmaEngineId = SdmaEngineId, + .QueueAddress = (uint64_t)QueueAddress, + .QueueSizeInBytes = QueueSizeInBytes, + .Event = Event ? vhsakmt_event_host_handle(Event) : 0, + .Queue_write_ptr_aql = QueueResource->Queue_write_ptr_aql, + .Queue_read_ptr_aql = QueueResource->Queue_read_ptr_aql, + }, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), /* For queue resource */ + .doorbell_blob_id = vhsakmt_node_doorbell(dev, NodeId) + ? 0 + : vhsakmt_atomic_inc_return(&dev->next_blob_id), /* For queue doorbell memory map */ + }; + int r; + + /* Queue ptr memory is allocated by hsakmtallocmemory in host then mapped into guest, but their + * address are not aligned. */ + if (Type == HSA_QUEUE_COMPUTE_AQL) { + r = vhsakmt_find_aql_rw_bo(dev, QueueResource->QueueWptrValue, &req.res_id); + if (r) { + vhsa_debug("%s: can not find the AQL queue R/W BO: %p\n", __FUNCTION__, + QueueResource->Queue_write_ptr_aql); + return HSAKMT_STATUS_NO_MEMORY; + } + + vhsa_debug("%s: create AQL queue, read ptr: %p, write ptr: %p, res id: %d\n", __FUNCTION__, + QueueResource->Queue_read_ptr_aql, QueueResource->Queue_write_ptr_aql, req.res_id); + } else + /* For queue not CP AQL, it use r/w ptr by itself. */ + req.rw_ptr_blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id); + + queue_mem_bo = vhsakmt_find_bo_by_addr(dev, QueueAddress); + if (!queue_mem_bo) { + vhsa_err("%s: can not find the queue memory BO: %p\n", __FUNCTION__, QueueAddress); + return HSAKMT_STATUS_NO_MEMORY; + } + queue_mem_bo->bo_type |= VHSA_BO_QUEUE_AQL_RW_PTR; + req.queue_mem_res_id = queue_mem_bo->real.res_id; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_queue_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) { + vhsa_err("%s: queue create failed, ret: %d", __FUNCTION__, rsp->ret); + return rsp->ret; + } + + /* Map doorbell */ + if (req.doorbell_blob_id) { + r = vhsakmt_create_doorbell_blob_bo( + dev, NodeId, vhsakmt_doorbell_page_size(), req.doorbell_blob_id, + rsp->vqueue_res.host_doorbell - rsp->vqueue_res.host_doorbell_offset, &doorbell_bo); + if (r) { + vhsa_err("%s: doorbell create failed, doorbell: %lx\n", __FUNCTION__, + rsp->vqueue_res.host_doorbell); + return r; + } + vhsa_debug("%s: create doorbell: %p, size: 0x%x\n", __FUNCTION__, doorbell_bo->cpu_addr, + doorbell_bo->size); + } + + QueueResource->Queue_DoorBell_aql = (void*)rsp->vqueue_res.host_doorbell; + vhsa_debug("%s: queue create, Doorbell: %p\n", __FUNCTION__, QueueResource->Queue_DoorBell_aql); + + /* Map R/W pointer. + * For a queue is not a COMPUTE AQL, the R/W PTR not using the input address, + * uses the queue memory allocated by hsakmtallocmemory, a page align address. + */ + if (Type != HSA_QUEUE_COMPUTE_AQL) { + r = vhsakmt_create_queue_rw_blob_bo(dev, vhsakmt_queue_page_size(), req.rw_ptr_blob_id, + rsp->vqueue_res.host_rw_handle, &rw_bo_handle); + if (r) { + vhsa_debug("%s: queue rw ptr create failed, host addr: %p\n", __FUNCTION__, + (void*)rsp->vqueue_res.host_rw_handle); + return r; + } + + QueueResource->Queue_write_ptr_aql = VHSA_UINT64_TO_VPTR( + VHSA_VPTR_TO_UINT64(rw_bo_handle->cpu_addr) + rsp->vqueue_res.host_write_offset); + QueueResource->Queue_read_ptr_aql = VHSA_UINT64_TO_VPTR( + VHSA_VPTR_TO_UINT64(rw_bo_handle->cpu_addr) + rsp->vqueue_res.host_read_offset); + + vhsa_debug("%s: queue create: write ptr gva: %p, read ptr gva: %p, base hva: %lx\n", + __FUNCTION__, QueueResource->Queue_write_ptr_aql, QueueResource->Queue_read_ptr_aql, + rsp->vqueue_res.host_rw_handle); + } + + r = vhsakmt_create_queue_blob_bo(dev, QueueSizeInBytes, req.blob_id, rsp->vqueue_res.r.QueueId, + rw_bo_handle, &queue_bo); + if (r) { + vhsa_err("%s: queue create failed, queue ID: 0x%lx\n", __FUNCTION__, rsp->vqueue_res.r.QueueId); + return r; + } + QueueResource->QueueId = (uint64_t)queue_bo; + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueue(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource) { + return vhsaKmtCreateQueueExt(NodeId, Type, QueuePercentage, Priority, VHSA_SDMA_NONE, + QueueAddress, QueueSizeInBytes, Event, QueueResource); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyQueue(HSA_QUEUEID QueueId) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + int r; + + /* queue ID: vhsakmt_bo_handle -> real queue ID*/ + vhsakmt_bo_handle bo = (vhsakmt_bo_handle)QueueId; + vhsakmt_bo_handle rw_bo = bo->rw_bo; + + r = vhsakmt_bo_free(dev, bo); + if (rw_bo) vhsakmt_bo_free(dev, rw_bo); + + vhsa_debug("%s: queue res id: %d, queue ID: %" PRIu64 ", ret = %d\n", __FUNCTION__, + bo->real.res_id, bo->queue_id, r); + + return r; +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_topology.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_topology.c new file mode 100644 index 0000000000..8c06e26459 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_topology.c @@ -0,0 +1,342 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +static int vhsakmt_set_sys_props(vhsakmt_device_handle dev, HsaSystemProperties* sys_props) { + int r = 0; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + if (dev->sys_props) { + r = 0; + goto out; + } + + dev->sys_props = calloc(1, sizeof(HsaSystemProperties)); + if (!dev->sys_props) { + r = -ENOMEM; + goto out; + } + + memcpy(dev->sys_props, sys_props, sizeof(HsaSystemProperties)); + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); + return r; +} + +static int vhsakmt_set_node_props(vhsakmt_device_handle dev, uint32_t node, + HsaNodeProperties* node_props) { + int r = 0; + if (!dev->sys_props) return -EINVAL; + if (node >= dev->sys_props->NumNodes) return -EINVAL; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + + if (!dev->vhsakmt_nodes) { + dev->vhsakmt_nodes = calloc(dev->sys_props->NumNodes, sizeof(struct vhsakmt_node)); + if (!dev->vhsakmt_nodes) { + r = -ENOMEM; + goto out; + } + } + + memcpy(&dev->vhsakmt_nodes[node].node_props, node_props, sizeof(HsaNodeProperties)); + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetVersion(HsaVersionInfo* v) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_VER, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + memcpy(v, &rsp->kfd_version, sizeof(HsaVersionInfo)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtAcquireSystemProperties(HsaSystemProperties* SystemProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + int r; + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_SYS_PROP, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + memcpy(SystemProperties, &rsp->sys_props, sizeof(HsaSystemProperties)); + + r = vhsakmt_set_sys_props(dev, SystemProperties); + if (r) return r; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtReleaseSystemProperties(void) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_REL_SYS_PROP, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + if (dev->sys_props) { + free(dev->sys_props); + dev->sys_props = NULL; + } + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeProperties(HSAuint32 NodeId, + HsaNodeProperties* NodeProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + int r; + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .NodeID = NodeId, + .type = VHSAKMT_CCMD_QUERY_GET_NODE_PROP, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + memcpy(NodeProperties, &rsp->node_props, sizeof(HsaNodeProperties)); + + r = vhsakmt_set_node_props(dev, NodeId, NodeProperties); + if (r) return r; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetXNACKMode(HSAint32* enable) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_XNACK_MODE, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + memcpy(enable, &rsp->xnack_mode, sizeof(HSAint32)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeEnable(void* rDebug, bool setupTtmp) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .run_time_enable_args.setupTtmp = setupTtmp, + .type = VHSAKMT_CCMD_QUERY_RUN_TIME_ENABLE, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeDisable(void) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_RUN_TIME_DISABLE, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAuint32 NumBanks, + HsaMemoryProperties* MemoryProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_NOD_MEM_PROP, + .node_mem_prop_args.NodeId = NodeId, + .node_mem_prop_args.NumBanks = NumBanks, + }; + + rsp = vhsakmt_alloc_rsp( + dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumBanks * sizeof(HsaMemoryProperties)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(MemoryProperties, rsp->payload, NumBanks * sizeof(HsaMemoryProperties)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeCacheProperties(HSAuint32 NodeId, HSAuint32 ProcessorId, + HSAuint32 NumCaches, + HsaCacheProperties* CacheProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_NOD_CACHE_PROP, + .node_cache_prop_args.NodeId = NodeId, + .node_cache_prop_args.ProcessorId = ProcessorId, + .node_cache_prop_args.NumCaches = NumCaches, + }; + + rsp = vhsakmt_alloc_rsp( + dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumCaches * sizeof(HsaCacheProperties)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(CacheProperties, rsp->payload, NumCaches * sizeof(HsaCacheProperties)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks, + HsaIoLinkProperties* IoLinkProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_NOD_IO_LINK_PROP, + .node_io_link_args.NodeId = NodeId, + .node_io_link_args.NumIoLinks = NumIoLinks, + }; + + rsp = vhsakmt_alloc_rsp( + dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumIoLinks * sizeof(HsaIoLinkProperties)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(IoLinkProperties, rsp->payload, NumIoLinks * sizeof(HsaIoLinkProperties)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetClockCounters(HSAuint32 NodeId, HsaClockCounters* Counters) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_CLOCK_COUNTERS, + .NodeID = NodeId, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(Counters, &rsp->clock_counters, sizeof(HsaClockCounters)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetRuntimeCapabilities(HSAuint32* caps_mask) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_RUNTIME_CAPS, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + *caps_mask = rsp->caps; + + return rsp->ret; +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_vm.c b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_vm.c new file mode 100644 index 0000000000..151b374180 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/hsakmt_virtio_vm.c @@ -0,0 +1,113 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "hsakmt_virtio_device.h" + +void* vhsakmt_vm_start(void) { + void* vm_start = malloc(getpagesize()); + if (!vm_start) return NULL; + + free(vm_start); + return vm_start; +} + +int vhsakmt_reserve_va(uint64_t start, uint64_t size) { + int32_t protFlags = PROT_NONE; + int32_t mapFlags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED; + void* va = mmap((void*)start, size, protFlags, mapFlags, -1, 0); + if (va == MAP_FAILED) return -ENOMEM; + + if (va != (void*)start) return -ENOMEM; + + madvise(va, size, MADV_DONTFORK); + + return 0; +} + +void vhsakmt_dereserve_va(uint64_t start, uint64_t size) { munmap((void*)start, size); } + +void vhsakmt_set_scratch_area(vhsakmt_device_handle dev, uint32_t node, uint64_t start, + uint64_t size) { + if (!dev->vhsakmt_nodes || !dev->sys_props) return; + if (node >= dev->sys_props->NumNodes) return; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + + if (dev->vhsakmt_nodes[node].scratch_start && dev->vhsakmt_nodes[node].scratch_size) goto out; + + dev->vhsakmt_nodes[node].scratch_start = start; + dev->vhsakmt_nodes[node].scratch_size = size; + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); +} + +bool vhsakmt_is_scratch_mem(vhsakmt_device_handle dev, void* addr) { + uint32_t i; + if (!dev->vhsakmt_nodes || !dev->sys_props) return false; + + for (i = 0; i < dev->sys_props->NumNodes; i++) { + if ((uint64_t)addr >= dev->vhsakmt_nodes[i].scratch_start && + (uint64_t)addr <= dev->vhsakmt_nodes[i].scratch_start + dev->vhsakmt_nodes[i].scratch_size) + return true; + } + + return false; +} + +void vhsakmt_set_vm_area(vhsakmt_device_handle dev, uint64_t start, uint64_t size) { + pthread_mutex_lock(&dev->vhsakmt_mutex); + if (dev->vm_start && dev->vm_size) goto out; + + dev->vm_start = start; + dev->vm_size = size; + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); +} + +bool vhsakmt_is_userptr(vhsakmt_device_handle dev, void* addr) { + return !((uint64_t)addr >= dev->vm_start && (uint64_t)addr <= dev->vm_start + dev->vm_size); +} + +int vhsakmt_set_node_doorbell(vhsakmt_device_handle dev, uint32_t node, void* doorbell) { + if (!dev->vhsakmt_nodes || !dev->sys_props) return -EINVAL; + if (node >= dev->sys_props->NumNodes) return -EINVAL; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + + dev->vhsakmt_nodes[node].doorbell_base = doorbell; + + pthread_mutex_unlock(&dev->vhsakmt_mutex); + + return 0; +} + +void* vhsakmt_node_doorbell(vhsakmt_device_handle dev, uint32_t node) { + if (!dev->vhsakmt_nodes || !dev->sys_props) return NULL; + if (node >= dev->sys_props->NumNodes) return NULL; + + return dev->vhsakmt_nodes[node].doorbell_base; +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/include/linux/virtgpu_drm.h b/projects/rocr-runtime/libhsakmt/src/virtio/include/linux/virtgpu_drm.h new file mode 100644 index 0000000000..d7625a551a --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/include/linux/virtgpu_drm.h @@ -0,0 +1,279 @@ +/* + * Copyright 2013 Red Hat + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef VIRTGPU_DRM_H +#define VIRTGPU_DRM_H + +#include "drm.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Please note that modifications to all structs defined here are + * subject to backwards-compatibility constraints. + * + * Do not use pointers, use __u64 instead for 32 bit / 64 bit user/kernel + * compatibility Keep fields aligned to their size + */ + +#define DRM_VIRTGPU_MAP 0x01 +#define DRM_VIRTGPU_EXECBUFFER 0x02 +#define DRM_VIRTGPU_GETPARAM 0x03 +#define DRM_VIRTGPU_RESOURCE_CREATE 0x04 +#define DRM_VIRTGPU_RESOURCE_INFO 0x05 +#define DRM_VIRTGPU_TRANSFER_FROM_HOST 0x06 +#define DRM_VIRTGPU_TRANSFER_TO_HOST 0x07 +#define DRM_VIRTGPU_WAIT 0x08 +#define DRM_VIRTGPU_GET_CAPS 0x09 +#define DRM_VIRTGPU_RESOURCE_CREATE_BLOB 0x0a +#define DRM_VIRTGPU_CONTEXT_INIT 0x0b + +#define VIRTGPU_EXECBUF_FENCE_FD_IN 0x01 +#define VIRTGPU_EXECBUF_FENCE_FD_OUT 0x02 +#define VIRTGPU_EXECBUF_RING_IDX 0x04 +#define VIRTGPU_EXECBUF_FLAGS (\ + VIRTGPU_EXECBUF_FENCE_FD_IN |\ + VIRTGPU_EXECBUF_FENCE_FD_OUT |\ + VIRTGPU_EXECBUF_RING_IDX |\ + 0) + +struct drm_virtgpu_map { + __u64 offset; /* use for mmap system call */ + __u32 handle; + __u32 pad; +}; + +#define VIRTGPU_EXECBUF_SYNCOBJ_RESET 0x01 +#define VIRTGPU_EXECBUF_SYNCOBJ_FLAGS ( \ + VIRTGPU_EXECBUF_SYNCOBJ_RESET | \ + 0) +struct drm_virtgpu_execbuffer_syncobj { + __u32 handle; + __u32 flags; + __u64 point; +}; + +/* fence_fd is modified on success if VIRTGPU_EXECBUF_FENCE_FD_OUT flag is set. */ +struct drm_virtgpu_execbuffer { + __u32 flags; + __u32 size; + __u64 command; /* void* */ + __u64 bo_handles; + __u32 num_bo_handles; + __s32 fence_fd; /* in/out fence fd (see VIRTGPU_EXECBUF_FENCE_FD_IN/OUT) */ + __u32 ring_idx; /* command ring index (see VIRTGPU_EXECBUF_RING_IDX) */ + __u32 syncobj_stride; /* size of @drm_virtgpu_execbuffer_syncobj */ + __u32 num_in_syncobjs; + __u32 num_out_syncobjs; + __u64 in_syncobjs; + __u64 out_syncobjs; +}; + +#define VIRTGPU_PARAM_3D_FEATURES 1 /* do we have 3D features in the hw */ +#define VIRTGPU_PARAM_CAPSET_QUERY_FIX 2 /* do we have the capset fix */ +#define VIRTGPU_PARAM_RESOURCE_BLOB 3 /* DRM_VIRTGPU_RESOURCE_CREATE_BLOB */ +#define VIRTGPU_PARAM_HOST_VISIBLE 4 /* Host blob resources are mappable */ +#define VIRTGPU_PARAM_CROSS_DEVICE 5 /* Cross virtio-device resource sharing */ +#define VIRTGPU_PARAM_CONTEXT_INIT 6 /* DRM_VIRTGPU_CONTEXT_INIT */ +#define VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs 7 /* Bitmask of supported capability set ids */ +#define VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME 8 /* Ability to set debug name from userspace */ + +struct drm_virtgpu_getparam { + __u64 param; + __u64 value; +}; + +/* NO_BO flags? NO resource flag? */ +/* resource flag for y_0_top */ +struct drm_virtgpu_resource_create { + __u32 target; + __u32 format; + __u32 bind; + __u32 width; + __u32 height; + __u32 depth; + __u32 array_size; + __u32 last_level; + __u32 nr_samples; + __u32 flags; + __u32 bo_handle; /* if this is set - recreate a new resource attached to this bo ? */ + __u32 res_handle; /* returned by kernel */ + __u32 size; /* validate transfer in the host */ + __u32 stride; /* validate transfer in the host */ +}; + +struct drm_virtgpu_resource_info { + __u32 bo_handle; + __u32 res_handle; + __u32 size; + __u32 blob_mem; +}; + +struct drm_virtgpu_3d_box { + __u32 x; + __u32 y; + __u32 z; + __u32 w; + __u32 h; + __u32 d; +}; + +struct drm_virtgpu_3d_transfer_to_host { + __u32 bo_handle; + struct drm_virtgpu_3d_box box; + __u32 level; + __u32 offset; + __u32 stride; + __u32 layer_stride; +}; + +struct drm_virtgpu_3d_transfer_from_host { + __u32 bo_handle; + struct drm_virtgpu_3d_box box; + __u32 level; + __u32 offset; + __u32 stride; + __u32 layer_stride; +}; + +#define VIRTGPU_WAIT_NOWAIT 1 /* like it */ +struct drm_virtgpu_3d_wait { + __u32 handle; /* 0 is an invalid handle */ + __u32 flags; +}; + +#define VIRTGPU_DRM_CAPSET_VIRGL 1 +#define VIRTGPU_DRM_CAPSET_VIRGL2 2 +#define VIRTGPU_DRM_CAPSET_GFXSTREAM_VULKAN 3 +#define VIRTGPU_DRM_CAPSET_VENUS 4 +#define VIRTGPU_DRM_CAPSET_CROSS_DOMAIN 5 +#define VIRTGPU_DRM_CAPSET_DRM 6 +struct drm_virtgpu_get_caps { + __u32 cap_set_id; + __u32 cap_set_ver; + __u64 addr; + __u32 size; + __u32 pad; +}; + +struct drm_virtgpu_resource_create_blob { +#define VIRTGPU_BLOB_MEM_GUEST 0x0001 +#define VIRTGPU_BLOB_MEM_HOST3D 0x0002 +#define VIRTGPU_BLOB_MEM_HOST3D_GUEST 0x0003 + +#define VIRTGPU_BLOB_FLAG_USE_MAPPABLE 0x0001 +#define VIRTGPU_BLOB_FLAG_USE_SHAREABLE 0x0002 +#define VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE 0x0004 +#define VIRTGPU_BLOB_FLAG_USE_USERPTR 0x0008 + /* zero is invalid blob_mem */ + __u32 blob_mem; + __u32 blob_flags; + __u32 bo_handle; + __u32 res_handle; + __u64 size; + + /* + * for 3D contexts with VIRTGPU_BLOB_MEM_HOST3D_GUEST and + * VIRTGPU_BLOB_MEM_HOST3D otherwise, must be zero. + */ + __u32 pad; + __u32 cmd_size; + __u64 cmd; + __u64 blob_id; + __u64 blob_userptr; + __s64 offset; +}; + +#define VIRTGPU_CONTEXT_PARAM_CAPSET_ID 0x0001 +#define VIRTGPU_CONTEXT_PARAM_NUM_RINGS 0x0002 +#define VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK 0x0003 +#define VIRTGPU_CONTEXT_PARAM_DEBUG_NAME 0x0004 +struct drm_virtgpu_context_set_param { + __u64 param; + __u64 value; +}; + +struct drm_virtgpu_context_init { + __u32 num_params; + __u32 pad; + + /* pointer to drm_virtgpu_context_set_param array */ + __u64 ctx_set_params; +}; + +/* + * Event code that's given when VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK is in + * effect. The event size is sizeof(drm_event), since there is no additional + * payload. + */ +#define VIRTGPU_EVENT_FENCE_SIGNALED 0x90000000 + +#define DRM_IOCTL_VIRTGPU_MAP \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_MAP, struct drm_virtgpu_map) + +#define DRM_IOCTL_VIRTGPU_EXECBUFFER \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_EXECBUFFER,\ + struct drm_virtgpu_execbuffer) + +#define DRM_IOCTL_VIRTGPU_GETPARAM \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GETPARAM,\ + struct drm_virtgpu_getparam) + +#define DRM_IOCTL_VIRTGPU_RESOURCE_CREATE \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_CREATE, \ + struct drm_virtgpu_resource_create) + +#define DRM_IOCTL_VIRTGPU_RESOURCE_INFO \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_INFO, \ + struct drm_virtgpu_resource_info) + +#define DRM_IOCTL_VIRTGPU_TRANSFER_FROM_HOST \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_FROM_HOST, \ + struct drm_virtgpu_3d_transfer_from_host) + +#define DRM_IOCTL_VIRTGPU_TRANSFER_TO_HOST \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_TO_HOST, \ + struct drm_virtgpu_3d_transfer_to_host) + +#define DRM_IOCTL_VIRTGPU_WAIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_WAIT, \ + struct drm_virtgpu_3d_wait) + +#define DRM_IOCTL_VIRTGPU_GET_CAPS \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GET_CAPS, \ + struct drm_virtgpu_get_caps) + +#define DRM_IOCTL_VIRTGPU_RESOURCE_CREATE_BLOB \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_CREATE_BLOB, \ + struct drm_virtgpu_resource_create_blob) + +#define DRM_IOCTL_VIRTGPU_CONTEXT_INIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_CONTEXT_INIT, \ + struct drm_virtgpu_context_init) + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/libhsakmt_virtio.ver b/projects/rocr-runtime/libhsakmt/src/virtio/libhsakmt_virtio.ver new file mode 100644 index 0000000000..1d3c323e17 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/libhsakmt_virtio.ver @@ -0,0 +1,45 @@ +{ +global: +vhsaKmtOpenKFD; +vhsaKmtCloseKFD; +vhsaKmtAllocMemory; +vhsaKmtFreeMemory; +vhsaKmtMapMemoryToGPUNodes; +vhsaKmtUnmapMemoryToGPU; +vhsaKmtAvailableMemory; +vhsaKmtMapMemoryToGPU; +vhsaKmtRegisterMemoryWithFlags; +vhsaKmtDeregisterMemory; +vhsaKmtGetVersion; +vhsaKmtAcquireSystemProperties; +vhsaKmtReleaseSystemProperties; +vhsaKmtGetNodeProperties; +vhsaKmtGetXNACKMode; +vhsaKmtRuntimeEnable; +vhsaKmtRuntimeDisable; +vhsaKmtGetNodeMemoryProperties; +vhsaKmtGetNodeCacheProperties; +vhsaKmtGetNodeIoLinkProperties; +vhsaKmtGetClockCounters; +vhsaKmtGetAMDGPUDeviceHandle; +vhsaKmtQueryPointerInfo; +vhsaKmtGetTileConfig; +vhsaKmtCreateEvent; +vhsaKmtDestroyEvent; +vhsaKmtSetEvent; +vhsaKmtResetEvent; +vhsaKmtQueryEventState; +vhsaKmtWaitOnMultipleEvents; +vhsaKmtWaitOnEvent; +vhsaKmtWaitOnEvent_Ext; +vhsaKmtWaitOnMultipleEvents_Ext; +vhsaKmtSetTrapHandler; +vhsaKmtCreateQueueExt; +vhsaKmtCreateQueue; +vhsaKmtDestroyQueue; +vhsaKmtRegisterGraphicsHandleToNodes; +vhsaKmtGetRuntimeCapabilities; +vamdgpu_query_gpu_info; +local: *; +}; + diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/virtio_gpu.c b/projects/rocr-runtime/libhsakmt/src/virtio/virtio_gpu.c new file mode 100644 index 0000000000..c3d6191437 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/virtio_gpu.c @@ -0,0 +1,324 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "virtio_gpu.h" + +#define SHMEM_SZ (25 * 0x1000) + +static int set_context(int fd) { + struct drm_virtgpu_context_set_param params[] = { + {VIRTGPU_CONTEXT_PARAM_CAPSET_ID, VIRGL_RENDERER_CAPSET_HSAKMT}, + {VIRTGPU_CONTEXT_PARAM_NUM_RINGS, 64}, + }; + struct drm_virtgpu_context_init args = { + .num_params = ARRAY_SIZE(params), + .ctx_set_params = (uintptr_t)(params), + }; + + return virtio_gpu_ioctl(fd, VIRTGPU_CONTEXT_INIT, &args); +} + +int virtio_gpu_map_handle(struct virtio_gpu_device* vgdev, uint32_t handle, uint64_t size, + void** addr, void* fixed_map) { + struct drm_virtgpu_map args = { + .handle = handle, + }; + int r; + + r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_MAP, &args); + if (r) return r; + + *addr = mmap(fixed_map, size, PROT_READ | PROT_WRITE, MAP_SHARED | (fixed_map ? MAP_FIXED : 0), + vgdev->fd, args.offset); + + if (*addr == MAP_FAILED) return -EINVAL; + + return 0; +} + +void virtio_gpu_unmap(void* addr, uint64_t size) { munmap(addr, size); } + +static void virtio_gpu_bo_close(struct virtio_gpu_device* vgdev, uint32_t handle) { + struct drm_gem_close args = { + .handle = handle, + }; + + virtio_gpu_ioctl(vgdev->fd, GEM_CLOSE, &args); +} + +static int virtio_gpu_shmem_init(struct virtio_gpu_device* vgdev, size_t size) { + struct drm_virtgpu_resource_create_blob args = { + .blob_mem = VIRTGPU_BLOB_MEM_HOST3D, + .blob_flags = VIRTGPU_BLOB_FLAG_USE_MAPPABLE, + .size = size, + .blob_id = 0, + }; + + int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_CREATE_BLOB, &args); + if (r) return r; + + r = virtio_gpu_map_handle(vgdev, args.bo_handle, size, (void**)&vgdev->shmem, NULL); + if (r) { + virtio_gpu_bo_close(vgdev, args.bo_handle); + return r; + } + + vgdev->shmem_handle = args.bo_handle; + + uint32_t offset = vgdev->shmem->base.rsp_mem_offset; + vgdev->rsp_mem_len = size - offset; + vgdev->rsp_mem = &((uint8_t*)vgdev->shmem)[offset]; + + return 0; +} + +struct virtio_gpu_device* virtio_gpu_init(int fd, uint32_t context_id) { + struct virtio_gpu_device* vgdev; + int r; + + r = set_context(fd); + + if (r) return NULL; + + vgdev = calloc(1, sizeof(*vgdev)); + if (!vgdev) return NULL; + + vgdev->fd = fd; + + vgdev->reqbuf = calloc(1, SHMEM_SZ); + if (!vgdev->reqbuf) { + free(vgdev); + return NULL; + } + + r = virtio_gpu_shmem_init(vgdev, SHMEM_SZ); + if (r) { + free(vgdev); + return NULL; + } + + pthread_mutex_init(&vgdev->rsp_lock, NULL); + pthread_mutex_init(&vgdev->eb_lock, NULL); + + return vgdev; +} + +void virtio_gpu_close(struct virtio_gpu_device* vgdev) { + virtio_gpu_unmap(vgdev->shmem, SHMEM_SZ); + virtio_gpu_bo_close(vgdev, vgdev->shmem_handle); + + pthread_mutex_destroy(&vgdev->rsp_lock); + pthread_mutex_destroy(&vgdev->eb_lock); + + close(vgdev->fd); + free(vgdev->reqbuf); + free(vgdev); +} + +void* virtio_gpu_alloc_rsp(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + uint32_t size) { + uint32_t off; + + pthread_mutex_lock(&vgdev->rsp_lock); + + size = VHSA_ALIGN_UP(size, 8); + + if ((vgdev->next_rsp_off + size) >= vgdev->rsp_mem_len) vgdev->next_rsp_off = 0; + + off = vgdev->next_rsp_off; + vgdev->next_rsp_off += size; + + pthread_mutex_unlock(&vgdev->rsp_lock); + + req->rsp_off = off; + struct virtio_gpu_ccmd_rsp* rsp = (void*)&vgdev->rsp_mem[off]; + rsp->len = size; + + return rsp; +} + +static int virtio_gpu_execbuffer_locked(struct virtio_gpu_device* vgdev, void* cmd, + uint32_t cmd_size, uint32_t* handles, uint32_t num_handles, + int* fence_fd, int ring_idx, uint32_t num_in_syncobjs, + uint32_t num_out_syncobjs, + struct drm_virtgpu_execbuffer_syncobj* in_syncobjs, + struct drm_virtgpu_execbuffer_syncobj* out_syncobjs, + bool in_fence, bool out_fence) { + struct drm_virtgpu_execbuffer eb = { + .flags = (out_fence ? VIRTGPU_EXECBUF_FENCE_FD_OUT : 0) | + (in_fence ? VIRTGPU_EXECBUF_FENCE_FD_IN : 0) | VIRTGPU_EXECBUF_RING_IDX, + .size = cmd_size, + .command = (uintptr_t)cmd, + .bo_handles = (uintptr_t)handles, + .num_bo_handles = num_handles, + .fence_fd = *fence_fd, + .ring_idx = ring_idx, + .syncobj_stride = sizeof(struct drm_virtgpu_execbuffer_syncobj), + .num_in_syncobjs = num_in_syncobjs, + .num_out_syncobjs = num_out_syncobjs, + .in_syncobjs = (uintptr_t)in_syncobjs, + .out_syncobjs = (uintptr_t)out_syncobjs, + }; + int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_EXECBUFFER, &eb); + if (r) return r; + + if (out_fence) *fence_fd = eb.fence_fd; + + return 0; +} + +static int virtio_gpu_flush_locked(struct virtio_gpu_device* vgdev, int* fence) { + int r; + + if (!vgdev->reqbuf_len) return 0; + + r = virtio_gpu_execbuffer_locked(vgdev, vgdev->reqbuf, vgdev->reqbuf_len, NULL, 0, fence, 0, 0, 0, + NULL, NULL, false, !!fence); + if (r) return r; + + vgdev->reqbuf_len = 0; + vgdev->reqbuf_cnt = 0; + + return 0; +} + +static int virtio_gpu_add_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req) { + req->seqno = ++vgdev->next_seqno; + int r; + + if (vgdev->reqbuf_len + req->len > sizeof(vgdev->reqbuf)) { + r = virtio_gpu_flush_locked(vgdev, NULL); + if (r) return r; + } + + memcpy(&vgdev->reqbuf[vgdev->reqbuf_len], req, req->len); + vgdev->reqbuf_len += req->len; + vgdev->reqbuf_cnt++; + + return 0; +} + +static inline bool fence_before(uint32_t a, uint32_t b) { return (int32_t)(a - b) < 0; } + +static void virtio_gpu_seqno_sync(struct virtio_gpu_device* vgdev, + struct virtio_gpu_ccmd_req* req) { + while (fence_before(vgdev->shmem->base.seqno, req->seqno)) sched_yield(); +} + +int virtio_gpu_exec_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + bool sync) { + int r = 0; + int fence; + + pthread_mutex_lock(&vgdev->eb_lock); + + r = virtio_gpu_add_cmd(vgdev, req); + + if (r || !sync) goto out; + + r = virtio_gpu_flush_locked(vgdev, &fence); + +out: + pthread_mutex_unlock(&vgdev->eb_lock); + if (r) return r; + + if (sync) { + sync_wait(fence, -1); + close(fence); + virtio_gpu_seqno_sync(vgdev, req); + } + + return r; +} + +int virtio_gpu_create_blob(struct virtio_gpu_device* vgdev, + struct drm_virtgpu_resource_create_blob* args) { + return virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_CREATE_BLOB, args); +} + +int virtio_gpu_destroy_handle(struct virtio_gpu_device* vgdev, uint32_t bo_handle) { + struct drm_gem_close args = { + .handle = bo_handle, + }; + + return virtio_gpu_ioctl(vgdev->fd, GEM_CLOSE, &args); +} + +int virtio_gpu_res_id(struct virtio_gpu_device* vgdev, uint32_t handle, uint32_t* res_id) { + struct drm_virtgpu_resource_info args = { + .bo_handle = handle, + }; + int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_INFO, &args); + if (r) return r; + + *res_id = args.res_handle; + return 0; +} + +static int virtio_gpu_get_capset(int fd, struct virgl_renderer_capset_hsakmt* caps) { + struct drm_virtgpu_get_caps args = { + .cap_set_id = VIRGL_RENDERER_CAPSET_HSAKMT, + .cap_set_ver = 0, + .addr = (uintptr_t)caps, + .size = sizeof(*caps), + }; + + memset(caps, 0, sizeof(*caps)); + + return virtio_gpu_ioctl(fd, VIRTGPU_GET_CAPS, &args); +} + +int virtio_gpu_kfd_open(void) { + drmDevicePtr devices[VHSA_MAX_DEVICES]; + int num_devices = 0; + int i, fd, ret; + + num_devices = drmGetDevices2(0, devices, ARRAY_SIZE(devices)); + if (num_devices <= 0) return -1; + + for (i = 0; i < num_devices; i++) { + fd = open(devices[i]->nodes[DRM_NODE_RENDER], O_RDWR | O_CLOEXEC); + if (fd < 0) continue; + + struct virgl_renderer_capset_hsakmt caps; + ret = virtio_gpu_get_capset(fd, &caps); + if (ret || caps.context_type != VIRTGPU_DRM_CONTEXT_AMDGPU) { + close(fd); + fd = -1; + continue; + } + + goto out; + } + +out: + drmFreeDevices(devices, num_devices); + return fd; +} diff --git a/projects/rocr-runtime/libhsakmt/src/virtio/virtio_gpu.h b/projects/rocr-runtime/libhsakmt/src/virtio/virtio_gpu.h new file mode 100644 index 0000000000..f31333ad6a --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/virtio/virtio_gpu.h @@ -0,0 +1,118 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef VIRTIO_GPU_H +#define VIRTIO_GPU_H + +#include +#include +#include + +#include "virtgpu_drm.h" + +#define VIRGL_RENDERER_CAPSET_HSAKMT 8 +#define VIRTGPU_DRM_CONTEXT_AMDGPU 1 +#define VHSA_MAX_DEVICES 10 + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + +#define VHSA_ALIGN_UP(x, align) (((uint64_t)(x) + (align)-1) & ~(uint64_t)((align)-1)) +#define VHSA_ALIGN_DOWN(x, align) ((uint64_t)(x) & ~(uint64_t)((align)-1)) + +#define virtio_gpu_ioctl(fd, name, args) \ + ({ \ + int ret = drmIoctl((fd), DRM_IOCTL_##name, (args)); \ + ret; \ + }) + +struct virgl_renderer_capset_hsakmt { + uint32_t wire_format_version; + /* Underlying drm device version: */ + uint32_t version_major; + uint32_t version_minor; + uint32_t version_patchlevel; + uint32_t context_type; + uint32_t pad; +}; + +struct virtio_gpu_shmem_base { + uint32_t seqno; + uint32_t rsp_mem_offset; +}; + +struct virtio_gpu_ccmd_req { + uint32_t cmd; + uint32_t len; + uint32_t seqno; + uint32_t rsp_off; +}; + +struct virtio_gpu_ccmd_rsp { + uint32_t len; +}; + +struct virtio_gpu_shmem { + struct virtio_gpu_shmem_base base; + uint32_t async_error; + uint32_t global_faults; +}; + +#define vhsakmt_shmem virtio_gpu_shmem +#define vhsakmt_ccmd_req virtio_gpu_ccmd_req +#define vhsakmt_ccmd_rsp virtio_gpu_ccmd_rsp + +struct virtio_gpu_device { + int fd; + + struct virtio_gpu_shmem* shmem; + uint32_t shmem_handle; + + uint8_t* rsp_mem; + uint32_t rsp_mem_len; + uint32_t next_rsp_off; + pthread_mutex_t rsp_lock; + pthread_mutex_t eb_lock; + + uint32_t next_seqno; + uint32_t reqbuf_len; + uint32_t reqbuf_cnt; + uint8_t* reqbuf; +}; + +struct virtio_gpu_device* virtio_gpu_init(int fd, uint32_t context_id); +void virtio_gpu_close(struct virtio_gpu_device* vgdev); +int virtio_gpu_exec_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + bool sync); +void* virtio_gpu_alloc_rsp(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + uint32_t size); +int virtio_gpu_map_handle(struct virtio_gpu_device* vgdev, uint32_t handle, uint64_t size, + void** addr, void* fixed_map); +void virtio_gpu_unmap(void* addr, uint64_t size); +int virtio_gpu_create_blob(struct virtio_gpu_device* vgdev, + struct drm_virtgpu_resource_create_blob* args); +int virtio_gpu_destroy_handle(struct virtio_gpu_device* vgdev, uint32_t bo_handle); +int virtio_gpu_res_id(struct virtio_gpu_device* vgdev, uint32_t handle, uint32_t* res_id); +int virtio_gpu_kfd_open(void); + +#endif /* VIRTIO_GPU_H */ diff --git a/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt b/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt index d40b8936af..165b34f230 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt +++ b/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt @@ -302,6 +302,10 @@ target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE elf::elf dl pthread rt ) # Link to hsakmt-staticdrm target for static library builds if( BUILD_SHARED_LIBS ) target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE hsakmt::hsakmt PkgConfig::drm) + if( BUILD_THUNK_VIRTIO ) + message(STATUS "Building with virtio support") + target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE hsakmt_virtio) + endif() find_package(rocprofiler-register) if(rocprofiler-register_FOUND) target_compile_definitions(${CORE_RUNTIME_TARGET} PRIVATE HSA_ROCPROFILER_REGISTER=1