From e7cb108a5e9dee5d3938f7338c0727c73e72f3e1 Mon Sep 17 00:00:00 2001 From: Flora Cui Date: Tue, 9 Sep 2025 10:16:57 +0800 Subject: [PATCH] [rocr-runtime] Add support for WSL DXG devices (#854) * rocr/rocdxg: add rocdxg support * rocr/dxg: set flags for dxg env * rocr: ring doorbell for dtif/dxg * rocr/dxg: sdma changes 1. align command size to 64 2. call hsaKmtQueueRingDoorbell 3. disable gcr && hdp flush Signed-off-by: Flora Cui Signed-off-by: Horatio Zhang Signed-off-by: tiancyin Signed-off-by: Longlong Yao --- .../runtime/hsa-runtime/core/inc/runtime.h | 5 + .../hsa-runtime/core/inc/thunk_loader.h | 12 +- .../core/runtime/amd_aql_queue.cpp | 3 +- .../core/runtime/amd_blit_sdma.cpp | 7 +- .../core/runtime/amd_gpu_agent.cpp | 14 +- .../hsa-runtime/core/runtime/amd_topology.cpp | 12 + .../hsa-runtime/core/runtime/runtime.cpp | 8 +- .../hsa-runtime/core/runtime/thunk_loader.cpp | 261 ++++++++++-------- .../runtime/hsa-runtime/core/util/flag.h | 41 ++- 9 files changed, 238 insertions(+), 125 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h index 2655d4f85c..ab6f355c20 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h @@ -455,6 +455,7 @@ class Runtime { } const Flag& flag() const { return flag_; } + Flag& flag() { return flag_; } const ThunkLoader* thunkLoader() const { return thunkLoader_; } @@ -479,6 +480,10 @@ class Runtime { if (version.KernelInterfaceMajorVersion == 1 && version.KernelInterfaceMinorVersion >= 14) kfd_version.supports_event_age = true; + + if (thunkLoader()->IsDXG()) { + kfd_version.supports_event_age = false; + } } void KfdVersion(bool exception_debugging, bool core_dump) { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/thunk_loader.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/thunk_loader.h index 4b8addb0f8..3e2984bb3f 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/thunk_loader.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/thunk_loader.h @@ -43,6 +43,7 @@ #ifndef HSA_RUNTIME_CORE_INC_THUNK_LOADER_H #define HSA_RUNTIME_CORE_INC_THUNK_LOADER_H +#include #include #include "hsakmt/hsakmttypes.h" @@ -363,6 +364,10 @@ class ThunkLoader { void LoadThunkApiTable(); bool CreateThunkInstance(); bool DestroyThunkInstance(); + bool IsDXG() const { return is_dxg_; } + bool IsDTIF() const { return is_dtif_; } + bool IsSharedLibraryLoaded() const { return is_loaded_; } + void* ThunkHandle() const { return thunk_handle; } HSAKMT_DEF(hsaKmtOpenKFD)* HSAKMT_PFN(hsaKmtOpenKFD); HSAKMT_DEF(hsaKmtCloseKFD)* HSAKMT_PFN(hsaKmtCloseKFD); @@ -468,7 +473,12 @@ class ThunkLoader { DRM_DEF(drmCommandWriteRead)* DRM_PFN(drmCommandWriteRead); private: - void *dtif_handle; + std::string whoami(); + void *thunk_handle; + std::string library_name; + bool is_dxg_; + bool is_dtif_; + bool is_loaded_; }; } // namespace core diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index 3e39bfa7db..5d6a95fedc 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -466,7 +466,8 @@ uint64_t AqlQueue::AddWriteIndexRelease(uint64_t value) { } void AqlQueue::StoreRelaxed(hsa_signal_value_t value) { - if (core::Runtime::runtime_singleton_->flag().enable_dtif()) { + if (core::Runtime::runtime_singleton_->thunkLoader()->IsDTIF() || + core::Runtime::runtime_singleton_->thunkLoader()->IsDXG()) { HSAKMT_CALL(hsaKmtQueueRingDoorbell(queue_id_)); } else { // Hardware doorbell supports AQL semantics. diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index aed2ab9a71..c9b411e341 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -353,7 +353,9 @@ hsa_status_t BlitSdma::SubmitCommand(const void* cmd, size_t cmd_size, u const uint32_t total_command_size = total_poll_command_size + cmd_size + sync_command_size + total_timestamp_command_size + interrupt_command_size + flush_cmd_size + total_gang_command_size; const uint32_t pad_size = total_command_size < min_submission_size_ ? - min_submission_size_ - total_command_size : 0; + min_submission_size_ - total_command_size : + core::Runtime::runtime_singleton_->thunkLoader()->IsDXG() ? + AlignUp(total_command_size, 64) - total_command_size : 0; uint64_t curr_index; char* command_addr; @@ -716,6 +718,9 @@ void BlitSdma::UpdateWriteAndDoorbellRegister(uint64_t curr_index, uint6 std::atomic_thread_fence(std::memory_order_release); *reinterpret_cast(queue_resource_.Queue_DoorBell) = new_index; + if (core::Runtime::runtime_singleton_->thunkLoader()->IsDXG()) { + HSAKMT_CALL(hsaKmtQueueRingDoorbell(queue_resource_.QueueId)); + } atomic::Store(&cached_commit_index_, new_index, std::memory_order_release); break; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index b04241a940..c01d6d468a 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -491,6 +491,12 @@ void GpuAgent::InitRegionList() { } void GpuAgent::InitScratchPool() { + + if (!core::Runtime::runtime_singleton_->flag().enable_scratch()) { + scratch_pool_. ~SmallHeap(); + return; + } + scratch_per_thread_ = core::Runtime::runtime_singleton_->flag().scratch_mem_size(); if (scratch_per_thread_ == 0) @@ -723,7 +729,11 @@ core::Blit* GpuAgent::CreateBlitSdma(bool use_xgmi, int rec_eng) { break; case 11: case 12: - sdma = new BlitSdmaV5(); + if (core::Runtime::runtime_singleton_->thunkLoader()->IsDXG()) { + sdma = new BlitSdmaV4(); + } else { + sdma = new BlitSdmaV5(); + } copy_size_override = copy_size_overrides[1]; break; default: @@ -1757,6 +1767,8 @@ hsa_status_t GpuAgent::QueueCreate(size_t size, hsa_queue_type32_t queue_type, u const uint32_t num_cu = properties_.NumFComputeCores / properties_.NumSIMDPerCU; scratch.main_size = scratch.main_size_per_thread * properties_.MaxSlotsScratchCU * scratch.main_lanes_per_wave * num_cu; + scratch.main_size = + (core::Runtime::runtime_singleton_->flag().enable_scratch()) ? scratch.main_size : 0; scratch.main_queue_base = nullptr; scratch.main_queue_process_offset = 0; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp index 5a9a909662..5bef4d30a2 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -283,6 +283,18 @@ void SurfaceGpuList(std::vector& gpu_list, bool xnack_mode, bool enable if (core::Runtime::runtime_singleton_->flag().enable_dtif()) core::g_use_interrupt_wait = false; + if (core::Runtime::runtime_singleton_->thunkLoader()->IsDXG()) { + core::g_use_interrupt_wait = false; + core::Runtime::runtime_singleton_->flag().set_sdma(false, false); + core::Runtime::runtime_singleton_->flag().disable_scratch(); + core::Runtime::runtime_singleton_->flag().disable_image(true); + core::Runtime::runtime_singleton_->flag().disable_xnack(); + core::Runtime::runtime_singleton_->flag().disable_fine_grain_pcie(); + core::Runtime::runtime_singleton_->flag().set_ipc_mode_legacy(false); + core::Runtime::runtime_singleton_->flag().disable_dev_mem_queue_buf(); + core::Runtime::runtime_singleton_->flag().disable_sdma_hdp_flush(); + } + // Instantiate a Gpu device. The IO links // of this node have already been registered assert((node_prop.NumFComputeCores != 0) && "Improper node used for GPU device discovery."); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp index 180063d14a..fc434a8675 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -2336,7 +2336,9 @@ void Runtime::CheckVirtualMemApiSupport() { char* error; fn_amdgpu_device_get_fd = - (int (*)(HsaAMDGPUDeviceHandle device_handle))dlsym(RTLD_DEFAULT, "amdgpu_device_get_fd"); + (int (*)(HsaAMDGPUDeviceHandle device_handle))dlsym( + thunkLoader()->IsDXG() ? thunkLoader()->ThunkHandle() : RTLD_DEFAULT, + "amdgpu_device_get_fd"); if ((error = dlerror()) != NULL) { debug_warning("amdgpu_device_get_fd not available. Please update version of libdrm"); fn_amdgpu_device_get_fd = &fn_amdgpu_device_get_fd_nosupport; @@ -2360,7 +2362,9 @@ void Runtime::InitIPCDmaBufSupport() { char* error; fn_amdgpu_device_get_fd = - (int (*)(HsaAMDGPUDeviceHandle device_handle))dlsym(RTLD_DEFAULT, "amdgpu_device_get_fd"); + (int (*)(HsaAMDGPUDeviceHandle device_handle))dlsym( + thunkLoader()->IsDXG() ? thunkLoader()->ThunkHandle() : RTLD_DEFAULT, + "amdgpu_device_get_fd"); if ((error = dlerror()) != NULL) { debug_warning("amdgpu_device_get_fd not available. Please update version of libdrm"); fn_amdgpu_device_get_fd = &fn_amdgpu_device_get_fd_nosupport; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/thunk_loader.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/thunk_loader.cpp index eea7ade330..cdf54bb15a 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/thunk_loader.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/thunk_loader.cpp @@ -45,337 +45,362 @@ #include #include +#include namespace rocr { namespace core { - ThunkLoader::ThunkLoader() { + std::string ThunkLoader::whoami() { + is_dtif_ = is_dxg_ = false; if (core::Runtime::runtime_singleton_->flag().enable_dtif()) { + is_dtif_ = true; + return "libdtif.so"; + } + + if (core::Runtime::runtime_singleton_->flag().enable_dxg_detection()) { + int fd = open("/dev/dxg", O_RDWR); + if (fd >= 0) { + close(fd); + is_dxg_ = true; + return "librocdxg.so"; + } + } + + return ""; + } + + ThunkLoader::ThunkLoader() + : thunk_handle(NULL), + library_name(whoami()), + is_loaded_(false) { + if (!library_name.empty()) { dlerror(); // Clear any existing error messages - dtif_handle = dlopen("libdtif.so", RTLD_LAZY); - if (dtif_handle == NULL) - fprintf(stderr, "Cannot load libdtif.so, failed:%s\n", dlerror()); - else - debug_print("Load libdtif.so successully!\n"); + thunk_handle = dlopen(library_name.c_str(), RTLD_LAZY); + if (thunk_handle == NULL) { + fprintf(stderr, "Cannot load %s, failed:%s\n", library_name.c_str(), dlerror()); + } else { + debug_print("Load %s successully!\n", library_name.c_str()); + } + is_loaded_ = true; } } ThunkLoader::~ThunkLoader() { - if (core::Runtime::runtime_singleton_->flag().enable_dtif() - && (dtif_handle != NULL)) { - if (dlclose(dtif_handle) != 0) { - fprintf(stderr, "Cannot unload libdtif.so, failed:%s\n", dlerror()); + if (IsSharedLibraryLoaded() + && (thunk_handle != NULL)) { + if (dlclose(thunk_handle) != 0) { + fprintf(stderr, "Cannot unload %s, failed:%s\n", library_name.c_str(), dlerror()); } else { - debug_print("Unload libdtif.so successully!\n"); + debug_print("Unload %s successully!\n", library_name.c_str()); } } } void ThunkLoader::LoadThunkApiTable() { - if (core::Runtime::runtime_singleton_->flag().enable_dtif()) { + if (IsSharedLibraryLoaded()) { dlerror(); // Clear any existing error messages - HSAKMT_PFN(hsaKmtOpenKFD) = (HSAKMT_DEF(hsaKmtOpenKFD)*)dlsym(dtif_handle, "hsaKmtOpenKFD"); + HSAKMT_PFN(hsaKmtOpenKFD) = (HSAKMT_DEF(hsaKmtOpenKFD)*)dlsym(thunk_handle, "hsaKmtOpenKFD"); if (HSAKMT_PFN(hsaKmtOpenKFD) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtCloseKFD) = (HSAKMT_DEF(hsaKmtCloseKFD)*)dlsym(dtif_handle, "hsaKmtCloseKFD"); + HSAKMT_PFN(hsaKmtCloseKFD) = (HSAKMT_DEF(hsaKmtCloseKFD)*)dlsym(thunk_handle, "hsaKmtCloseKFD"); if (HSAKMT_PFN(hsaKmtCloseKFD) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetVersion) = (HSAKMT_DEF(hsaKmtGetVersion)*)dlsym(dtif_handle, "hsaKmtGetVersion"); + HSAKMT_PFN(hsaKmtGetVersion) = (HSAKMT_DEF(hsaKmtGetVersion)*)dlsym(thunk_handle, "hsaKmtGetVersion"); if (HSAKMT_PFN(hsaKmtGetVersion) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtAcquireSystemProperties) = (HSAKMT_DEF(hsaKmtAcquireSystemProperties)*)dlsym(dtif_handle, "hsaKmtAcquireSystemProperties"); + HSAKMT_PFN(hsaKmtAcquireSystemProperties) = (HSAKMT_DEF(hsaKmtAcquireSystemProperties)*)dlsym(thunk_handle, "hsaKmtAcquireSystemProperties"); if (HSAKMT_PFN(hsaKmtAcquireSystemProperties) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtReleaseSystemProperties) = (HSAKMT_DEF(hsaKmtReleaseSystemProperties)*)dlsym(dtif_handle, "hsaKmtReleaseSystemProperties"); + HSAKMT_PFN(hsaKmtReleaseSystemProperties) = (HSAKMT_DEF(hsaKmtReleaseSystemProperties)*)dlsym(thunk_handle, "hsaKmtReleaseSystemProperties"); if (HSAKMT_PFN(hsaKmtReleaseSystemProperties) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetNodeProperties) = (HSAKMT_DEF(hsaKmtGetNodeProperties)*)dlsym(dtif_handle, "hsaKmtGetNodeProperties"); + HSAKMT_PFN(hsaKmtGetNodeProperties) = (HSAKMT_DEF(hsaKmtGetNodeProperties)*)dlsym(thunk_handle, "hsaKmtGetNodeProperties"); if (HSAKMT_PFN(hsaKmtGetNodeProperties) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetNodeMemoryProperties) = (HSAKMT_DEF(hsaKmtGetNodeMemoryProperties)*)dlsym(dtif_handle, "hsaKmtGetNodeMemoryProperties"); + HSAKMT_PFN(hsaKmtGetNodeMemoryProperties) = (HSAKMT_DEF(hsaKmtGetNodeMemoryProperties)*)dlsym(thunk_handle, "hsaKmtGetNodeMemoryProperties"); if (HSAKMT_PFN(hsaKmtGetNodeMemoryProperties) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetNodeCacheProperties) = (HSAKMT_DEF(hsaKmtGetNodeCacheProperties)*)dlsym(dtif_handle, "hsaKmtGetNodeCacheProperties"); + HSAKMT_PFN(hsaKmtGetNodeCacheProperties) = (HSAKMT_DEF(hsaKmtGetNodeCacheProperties)*)dlsym(thunk_handle, "hsaKmtGetNodeCacheProperties"); if (HSAKMT_PFN(hsaKmtGetNodeCacheProperties) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetNodeIoLinkProperties) = (HSAKMT_DEF(hsaKmtGetNodeIoLinkProperties)*)dlsym(dtif_handle, "hsaKmtGetNodeIoLinkProperties"); + HSAKMT_PFN(hsaKmtGetNodeIoLinkProperties) = (HSAKMT_DEF(hsaKmtGetNodeIoLinkProperties)*)dlsym(thunk_handle, "hsaKmtGetNodeIoLinkProperties"); if (HSAKMT_PFN(hsaKmtGetNodeIoLinkProperties) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtCreateEvent) = (HSAKMT_DEF(hsaKmtCreateEvent)*)dlsym(dtif_handle, "hsaKmtCreateEvent"); + HSAKMT_PFN(hsaKmtCreateEvent) = (HSAKMT_DEF(hsaKmtCreateEvent)*)dlsym(thunk_handle, "hsaKmtCreateEvent"); if (HSAKMT_PFN(hsaKmtCreateEvent) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDestroyEvent) = (HSAKMT_DEF(hsaKmtDestroyEvent)*)dlsym(dtif_handle, "hsaKmtDestroyEvent"); + HSAKMT_PFN(hsaKmtDestroyEvent) = (HSAKMT_DEF(hsaKmtDestroyEvent)*)dlsym(thunk_handle, "hsaKmtDestroyEvent"); if (HSAKMT_PFN(hsaKmtDestroyEvent) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSetEvent) = (HSAKMT_DEF(hsaKmtSetEvent)*)dlsym(dtif_handle, "hsaKmtSetEvent"); + HSAKMT_PFN(hsaKmtSetEvent) = (HSAKMT_DEF(hsaKmtSetEvent)*)dlsym(thunk_handle, "hsaKmtSetEvent"); if (HSAKMT_PFN(hsaKmtSetEvent) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtResetEvent) = (HSAKMT_DEF(hsaKmtResetEvent)*)dlsym(dtif_handle, "hsaKmtResetEvent"); + HSAKMT_PFN(hsaKmtResetEvent) = (HSAKMT_DEF(hsaKmtResetEvent)*)dlsym(thunk_handle, "hsaKmtResetEvent"); if (HSAKMT_PFN(hsaKmtResetEvent) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtQueryEventState) = (HSAKMT_DEF(hsaKmtQueryEventState)*)dlsym(dtif_handle, "hsaKmtQueryEventState"); + HSAKMT_PFN(hsaKmtQueryEventState) = (HSAKMT_DEF(hsaKmtQueryEventState)*)dlsym(thunk_handle, "hsaKmtQueryEventState"); if (HSAKMT_PFN(hsaKmtQueryEventState) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtWaitOnEvent) = (HSAKMT_DEF(hsaKmtWaitOnEvent)*)dlsym(dtif_handle, "hsaKmtWaitOnEvent"); + HSAKMT_PFN(hsaKmtWaitOnEvent) = (HSAKMT_DEF(hsaKmtWaitOnEvent)*)dlsym(thunk_handle, "hsaKmtWaitOnEvent"); if (HSAKMT_PFN(hsaKmtWaitOnEvent) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtWaitOnMultipleEvents) = (HSAKMT_DEF(hsaKmtWaitOnMultipleEvents)*)dlsym(dtif_handle, "hsaKmtWaitOnMultipleEvents"); + HSAKMT_PFN(hsaKmtWaitOnMultipleEvents) = (HSAKMT_DEF(hsaKmtWaitOnMultipleEvents)*)dlsym(thunk_handle, "hsaKmtWaitOnMultipleEvents"); if (HSAKMT_PFN(hsaKmtWaitOnMultipleEvents) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtCreateQueue) = (HSAKMT_DEF(hsaKmtCreateQueue)*)dlsym(dtif_handle, "hsaKmtCreateQueue"); + HSAKMT_PFN(hsaKmtCreateQueue) = (HSAKMT_DEF(hsaKmtCreateQueue)*)dlsym(thunk_handle, "hsaKmtCreateQueue"); if (HSAKMT_PFN(hsaKmtCreateQueue) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtCreateQueueExt) = (HSAKMT_DEF(hsaKmtCreateQueueExt)*)dlsym(dtif_handle, "hsaKmtCreateQueueExt"); + HSAKMT_PFN(hsaKmtCreateQueueExt) = (HSAKMT_DEF(hsaKmtCreateQueueExt)*)dlsym(thunk_handle, "hsaKmtCreateQueueExt"); if (HSAKMT_PFN(hsaKmtCreateQueueExt) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtUpdateQueue) = (HSAKMT_DEF(hsaKmtUpdateQueue)*)dlsym(dtif_handle, "hsaKmtUpdateQueue"); + HSAKMT_PFN(hsaKmtUpdateQueue) = (HSAKMT_DEF(hsaKmtUpdateQueue)*)dlsym(thunk_handle, "hsaKmtUpdateQueue"); if (HSAKMT_PFN(hsaKmtUpdateQueue) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDestroyQueue) = (HSAKMT_DEF(hsaKmtDestroyQueue)*)dlsym(dtif_handle, "hsaKmtDestroyQueue"); + HSAKMT_PFN(hsaKmtDestroyQueue) = (HSAKMT_DEF(hsaKmtDestroyQueue)*)dlsym(thunk_handle, "hsaKmtDestroyQueue"); if (HSAKMT_PFN(hsaKmtDestroyQueue) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSetQueueCUMask) = (HSAKMT_DEF(hsaKmtSetQueueCUMask)*)dlsym(dtif_handle, "hsaKmtSetQueueCUMask"); + HSAKMT_PFN(hsaKmtSetQueueCUMask) = (HSAKMT_DEF(hsaKmtSetQueueCUMask)*)dlsym(thunk_handle, "hsaKmtSetQueueCUMask"); if (HSAKMT_PFN(hsaKmtSetQueueCUMask) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSetMemoryPolicy) = (HSAKMT_DEF(hsaKmtSetMemoryPolicy)*)dlsym(dtif_handle, "hsaKmtSetMemoryPolicy"); + HSAKMT_PFN(hsaKmtSetMemoryPolicy) = (HSAKMT_DEF(hsaKmtSetMemoryPolicy)*)dlsym(thunk_handle, "hsaKmtSetMemoryPolicy"); if (HSAKMT_PFN(hsaKmtSetMemoryPolicy) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtAllocMemory) = (HSAKMT_DEF(hsaKmtAllocMemory)*)dlsym(dtif_handle, "hsaKmtAllocMemory"); + HSAKMT_PFN(hsaKmtAllocMemory) = (HSAKMT_DEF(hsaKmtAllocMemory)*)dlsym(thunk_handle, "hsaKmtAllocMemory"); if (HSAKMT_PFN(hsaKmtAllocMemory) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtAllocMemoryAlign) = (HSAKMT_DEF(hsaKmtAllocMemoryAlign)*)dlsym(dtif_handle, "hsaKmtAllocMemoryAlign"); + HSAKMT_PFN(hsaKmtAllocMemoryAlign) = (HSAKMT_DEF(hsaKmtAllocMemoryAlign)*)dlsym(thunk_handle, "hsaKmtAllocMemoryAlign"); if (HSAKMT_PFN(hsaKmtAllocMemoryAlign) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtFreeMemory) = (HSAKMT_DEF(hsaKmtFreeMemory)*)dlsym(dtif_handle, "hsaKmtFreeMemory"); + HSAKMT_PFN(hsaKmtFreeMemory) = (HSAKMT_DEF(hsaKmtFreeMemory)*)dlsym(thunk_handle, "hsaKmtFreeMemory"); if (HSAKMT_PFN(hsaKmtFreeMemory) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtAvailableMemory) = (HSAKMT_DEF(hsaKmtAvailableMemory)*)dlsym(dtif_handle, "hsaKmtAvailableMemory"); + HSAKMT_PFN(hsaKmtAvailableMemory) = (HSAKMT_DEF(hsaKmtAvailableMemory)*)dlsym(thunk_handle, "hsaKmtAvailableMemory"); if (HSAKMT_PFN(hsaKmtAvailableMemory) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRegisterMemory) = (HSAKMT_DEF(hsaKmtRegisterMemory)*)dlsym(dtif_handle, "hsaKmtRegisterMemory"); + HSAKMT_PFN(hsaKmtRegisterMemory) = (HSAKMT_DEF(hsaKmtRegisterMemory)*)dlsym(thunk_handle, "hsaKmtRegisterMemory"); if (HSAKMT_PFN(hsaKmtRegisterMemory) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRegisterMemoryToNodes) = (HSAKMT_DEF(hsaKmtRegisterMemoryToNodes)*)dlsym(dtif_handle, "hsaKmtRegisterMemoryToNodes"); + HSAKMT_PFN(hsaKmtRegisterMemoryToNodes) = (HSAKMT_DEF(hsaKmtRegisterMemoryToNodes)*)dlsym(thunk_handle, "hsaKmtRegisterMemoryToNodes"); if (HSAKMT_PFN(hsaKmtRegisterMemoryToNodes) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRegisterMemoryWithFlags) = (HSAKMT_DEF(hsaKmtRegisterMemoryWithFlags)*)dlsym(dtif_handle, "hsaKmtRegisterMemoryWithFlags"); + HSAKMT_PFN(hsaKmtRegisterMemoryWithFlags) = (HSAKMT_DEF(hsaKmtRegisterMemoryWithFlags)*)dlsym(thunk_handle, "hsaKmtRegisterMemoryWithFlags"); if (HSAKMT_PFN(hsaKmtRegisterMemoryWithFlags) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRegisterGraphicsHandleToNodes) = (HSAKMT_DEF(hsaKmtRegisterGraphicsHandleToNodes)*)dlsym(dtif_handle, "hsaKmtRegisterGraphicsHandleToNodes"); + HSAKMT_PFN(hsaKmtRegisterGraphicsHandleToNodes) = (HSAKMT_DEF(hsaKmtRegisterGraphicsHandleToNodes)*)dlsym(thunk_handle, "hsaKmtRegisterGraphicsHandleToNodes"); if (HSAKMT_PFN(hsaKmtRegisterGraphicsHandleToNodes) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRegisterGraphicsHandleToNodesExt) = (HSAKMT_DEF(hsaKmtRegisterGraphicsHandleToNodesExt)*)dlsym(dtif_handle, "hsaKmtRegisterGraphicsHandleToNodesExt"); + HSAKMT_PFN(hsaKmtRegisterGraphicsHandleToNodesExt) = (HSAKMT_DEF(hsaKmtRegisterGraphicsHandleToNodesExt)*)dlsym(thunk_handle, "hsaKmtRegisterGraphicsHandleToNodesExt"); if (HSAKMT_PFN(hsaKmtRegisterGraphicsHandleToNodesExt) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtShareMemory) = (HSAKMT_DEF(hsaKmtShareMemory)*)dlsym(dtif_handle, "hsaKmtShareMemory"); + HSAKMT_PFN(hsaKmtShareMemory) = (HSAKMT_DEF(hsaKmtShareMemory)*)dlsym(thunk_handle, "hsaKmtShareMemory"); if (HSAKMT_PFN(hsaKmtShareMemory) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRegisterSharedHandle) = (HSAKMT_DEF(hsaKmtRegisterSharedHandle)*)dlsym(dtif_handle, "hsaKmtRegisterSharedHandle"); + HSAKMT_PFN(hsaKmtRegisterSharedHandle) = (HSAKMT_DEF(hsaKmtRegisterSharedHandle)*)dlsym(thunk_handle, "hsaKmtRegisterSharedHandle"); if (HSAKMT_PFN(hsaKmtRegisterSharedHandle) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRegisterSharedHandleToNodes) = (HSAKMT_DEF(hsaKmtRegisterSharedHandleToNodes)*)dlsym(dtif_handle, "hsaKmtRegisterSharedHandleToNodes"); + HSAKMT_PFN(hsaKmtRegisterSharedHandleToNodes) = (HSAKMT_DEF(hsaKmtRegisterSharedHandleToNodes)*)dlsym(thunk_handle, "hsaKmtRegisterSharedHandleToNodes"); if (HSAKMT_PFN(hsaKmtRegisterSharedHandleToNodes) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtProcessVMRead) = (HSAKMT_DEF(hsaKmtProcessVMRead)*)dlsym(dtif_handle, "hsaKmtProcessVMRead"); + HSAKMT_PFN(hsaKmtProcessVMRead) = (HSAKMT_DEF(hsaKmtProcessVMRead)*)dlsym(thunk_handle, "hsaKmtProcessVMRead"); if (HSAKMT_PFN(hsaKmtProcessVMRead) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtProcessVMWrite) = (HSAKMT_DEF(hsaKmtProcessVMWrite)*)dlsym(dtif_handle, "hsaKmtProcessVMWrite"); + HSAKMT_PFN(hsaKmtProcessVMWrite) = (HSAKMT_DEF(hsaKmtProcessVMWrite)*)dlsym(thunk_handle, "hsaKmtProcessVMWrite"); if (HSAKMT_PFN(hsaKmtProcessVMWrite) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDeregisterMemory) = (HSAKMT_DEF(hsaKmtDeregisterMemory)*)dlsym(dtif_handle, "hsaKmtDeregisterMemory"); + HSAKMT_PFN(hsaKmtDeregisterMemory) = (HSAKMT_DEF(hsaKmtDeregisterMemory)*)dlsym(thunk_handle, "hsaKmtDeregisterMemory"); if (HSAKMT_PFN(hsaKmtDeregisterMemory) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtMapMemoryToGPU) = (HSAKMT_DEF(hsaKmtMapMemoryToGPU)*)dlsym(dtif_handle, "hsaKmtMapMemoryToGPU"); + HSAKMT_PFN(hsaKmtMapMemoryToGPU) = (HSAKMT_DEF(hsaKmtMapMemoryToGPU)*)dlsym(thunk_handle, "hsaKmtMapMemoryToGPU"); if (HSAKMT_PFN(hsaKmtMapMemoryToGPU) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtMapMemoryToGPUNodes) = (HSAKMT_DEF(hsaKmtMapMemoryToGPUNodes)*)dlsym(dtif_handle, "hsaKmtMapMemoryToGPUNodes"); + HSAKMT_PFN(hsaKmtMapMemoryToGPUNodes) = (HSAKMT_DEF(hsaKmtMapMemoryToGPUNodes)*)dlsym(thunk_handle, "hsaKmtMapMemoryToGPUNodes"); if (HSAKMT_PFN(hsaKmtMapMemoryToGPUNodes) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtUnmapMemoryToGPU) = (HSAKMT_DEF(hsaKmtUnmapMemoryToGPU)*)dlsym(dtif_handle, "hsaKmtUnmapMemoryToGPU"); + HSAKMT_PFN(hsaKmtUnmapMemoryToGPU) = (HSAKMT_DEF(hsaKmtUnmapMemoryToGPU)*)dlsym(thunk_handle, "hsaKmtUnmapMemoryToGPU"); if (HSAKMT_PFN(hsaKmtUnmapMemoryToGPU) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgRegister) = (HSAKMT_DEF(hsaKmtDbgRegister)*)dlsym(dtif_handle, "hsaKmtDbgRegister"); + HSAKMT_PFN(hsaKmtDbgRegister) = (HSAKMT_DEF(hsaKmtDbgRegister)*)dlsym(thunk_handle, "hsaKmtDbgRegister"); if (HSAKMT_PFN(hsaKmtDbgRegister) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgUnregister) = (HSAKMT_DEF(hsaKmtDbgUnregister)*)dlsym(dtif_handle, "hsaKmtDbgUnregister"); + HSAKMT_PFN(hsaKmtDbgUnregister) = (HSAKMT_DEF(hsaKmtDbgUnregister)*)dlsym(thunk_handle, "hsaKmtDbgUnregister"); if (HSAKMT_PFN(hsaKmtDbgUnregister) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgWavefrontControl) = (HSAKMT_DEF(hsaKmtDbgWavefrontControl)*)dlsym(dtif_handle, "hsaKmtDbgWavefrontControl"); + HSAKMT_PFN(hsaKmtDbgWavefrontControl) = (HSAKMT_DEF(hsaKmtDbgWavefrontControl)*)dlsym(thunk_handle, "hsaKmtDbgWavefrontControl"); if (HSAKMT_PFN(hsaKmtDbgWavefrontControl) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgAddressWatch) = (HSAKMT_DEF(hsaKmtDbgAddressWatch)*)dlsym(dtif_handle, "hsaKmtDbgAddressWatch"); + HSAKMT_PFN(hsaKmtDbgAddressWatch) = (HSAKMT_DEF(hsaKmtDbgAddressWatch)*)dlsym(thunk_handle, "hsaKmtDbgAddressWatch"); if (HSAKMT_PFN(hsaKmtDbgAddressWatch) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgEnable) = (HSAKMT_DEF(hsaKmtDbgEnable)*)dlsym(dtif_handle, "hsaKmtDbgEnable"); + HSAKMT_PFN(hsaKmtDbgEnable) = (HSAKMT_DEF(hsaKmtDbgEnable)*)dlsym(thunk_handle, "hsaKmtDbgEnable"); if (HSAKMT_PFN(hsaKmtDbgEnable) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgDisable) = (HSAKMT_DEF(hsaKmtDbgDisable)*)dlsym(dtif_handle, "hsaKmtDbgDisable"); + HSAKMT_PFN(hsaKmtDbgDisable) = (HSAKMT_DEF(hsaKmtDbgDisable)*)dlsym(thunk_handle, "hsaKmtDbgDisable"); if (HSAKMT_PFN(hsaKmtDbgDisable) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgGetDeviceData) = (HSAKMT_DEF(hsaKmtDbgGetDeviceData)*)dlsym(dtif_handle, "hsaKmtDbgGetDeviceData"); + HSAKMT_PFN(hsaKmtDbgGetDeviceData) = (HSAKMT_DEF(hsaKmtDbgGetDeviceData)*)dlsym(thunk_handle, "hsaKmtDbgGetDeviceData"); if (HSAKMT_PFN(hsaKmtDbgGetDeviceData) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDbgGetQueueData) = (HSAKMT_DEF(hsaKmtDbgGetQueueData)*)dlsym(dtif_handle, "hsaKmtDbgGetQueueData"); + HSAKMT_PFN(hsaKmtDbgGetQueueData) = (HSAKMT_DEF(hsaKmtDbgGetQueueData)*)dlsym(thunk_handle, "hsaKmtDbgGetQueueData"); if (HSAKMT_PFN(hsaKmtDbgGetQueueData) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetClockCounters) = (HSAKMT_DEF(hsaKmtGetClockCounters)*)dlsym(dtif_handle, "hsaKmtGetClockCounters"); + HSAKMT_PFN(hsaKmtGetClockCounters) = (HSAKMT_DEF(hsaKmtGetClockCounters)*)dlsym(thunk_handle, "hsaKmtGetClockCounters"); if (HSAKMT_PFN(hsaKmtGetClockCounters) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcGetCounterProperties) = (HSAKMT_DEF(hsaKmtPmcGetCounterProperties)*)dlsym(dtif_handle, "hsaKmtPmcGetCounterProperties"); + HSAKMT_PFN(hsaKmtPmcGetCounterProperties) = (HSAKMT_DEF(hsaKmtPmcGetCounterProperties)*)dlsym(thunk_handle, "hsaKmtPmcGetCounterProperties"); if (HSAKMT_PFN(hsaKmtPmcGetCounterProperties) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcRegisterTrace) = (HSAKMT_DEF(hsaKmtPmcRegisterTrace)*)dlsym(dtif_handle, "hsaKmtPmcRegisterTrace"); + HSAKMT_PFN(hsaKmtPmcRegisterTrace) = (HSAKMT_DEF(hsaKmtPmcRegisterTrace)*)dlsym(thunk_handle, "hsaKmtPmcRegisterTrace"); if (HSAKMT_PFN(hsaKmtPmcRegisterTrace) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcUnregisterTrace) = (HSAKMT_DEF(hsaKmtPmcUnregisterTrace)*)dlsym(dtif_handle, "hsaKmtPmcUnregisterTrace"); + HSAKMT_PFN(hsaKmtPmcUnregisterTrace) = (HSAKMT_DEF(hsaKmtPmcUnregisterTrace)*)dlsym(thunk_handle, "hsaKmtPmcUnregisterTrace"); if (HSAKMT_PFN(hsaKmtPmcUnregisterTrace) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcAcquireTraceAccess) = (HSAKMT_DEF(hsaKmtPmcAcquireTraceAccess)*)dlsym(dtif_handle, "hsaKmtPmcAcquireTraceAccess"); + HSAKMT_PFN(hsaKmtPmcAcquireTraceAccess) = (HSAKMT_DEF(hsaKmtPmcAcquireTraceAccess)*)dlsym(thunk_handle, "hsaKmtPmcAcquireTraceAccess"); if (HSAKMT_PFN(hsaKmtPmcAcquireTraceAccess) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcReleaseTraceAccess) = (HSAKMT_DEF(hsaKmtPmcReleaseTraceAccess)*)dlsym(dtif_handle, "hsaKmtPmcReleaseTraceAccess"); + HSAKMT_PFN(hsaKmtPmcReleaseTraceAccess) = (HSAKMT_DEF(hsaKmtPmcReleaseTraceAccess)*)dlsym(thunk_handle, "hsaKmtPmcReleaseTraceAccess"); if (HSAKMT_PFN(hsaKmtPmcReleaseTraceAccess) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcStartTrace) = (HSAKMT_DEF(hsaKmtPmcStartTrace)*)dlsym(dtif_handle, "hsaKmtPmcStartTrace"); + HSAKMT_PFN(hsaKmtPmcStartTrace) = (HSAKMT_DEF(hsaKmtPmcStartTrace)*)dlsym(thunk_handle, "hsaKmtPmcStartTrace"); if (HSAKMT_PFN(hsaKmtPmcStartTrace) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcQueryTrace) = (HSAKMT_DEF(hsaKmtPmcQueryTrace)*)dlsym(dtif_handle, "hsaKmtPmcQueryTrace"); + HSAKMT_PFN(hsaKmtPmcQueryTrace) = (HSAKMT_DEF(hsaKmtPmcQueryTrace)*)dlsym(thunk_handle, "hsaKmtPmcQueryTrace"); if (HSAKMT_PFN(hsaKmtPmcQueryTrace) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPmcStopTrace) = (HSAKMT_DEF(hsaKmtPmcStopTrace)*)dlsym(dtif_handle, "hsaKmtPmcStopTrace"); + HSAKMT_PFN(hsaKmtPmcStopTrace) = (HSAKMT_DEF(hsaKmtPmcStopTrace)*)dlsym(thunk_handle, "hsaKmtPmcStopTrace"); if (HSAKMT_PFN(hsaKmtPmcStopTrace) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtMapGraphicHandle) = (HSAKMT_DEF(hsaKmtMapGraphicHandle)*)dlsym(dtif_handle, "hsaKmtMapGraphicHandle"); + HSAKMT_PFN(hsaKmtMapGraphicHandle) = (HSAKMT_DEF(hsaKmtMapGraphicHandle)*)dlsym(thunk_handle, "hsaKmtMapGraphicHandle"); if (HSAKMT_PFN(hsaKmtMapGraphicHandle) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtUnmapGraphicHandle) = (HSAKMT_DEF(hsaKmtUnmapGraphicHandle)*)dlsym(dtif_handle, "hsaKmtUnmapGraphicHandle"); + HSAKMT_PFN(hsaKmtUnmapGraphicHandle) = (HSAKMT_DEF(hsaKmtUnmapGraphicHandle)*)dlsym(thunk_handle, "hsaKmtUnmapGraphicHandle"); if (HSAKMT_PFN(hsaKmtUnmapGraphicHandle) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSetTrapHandler) = (HSAKMT_DEF(hsaKmtSetTrapHandler)*)dlsym(dtif_handle, "hsaKmtSetTrapHandler"); + HSAKMT_PFN(hsaKmtSetTrapHandler) = (HSAKMT_DEF(hsaKmtSetTrapHandler)*)dlsym(thunk_handle, "hsaKmtSetTrapHandler"); if (HSAKMT_PFN(hsaKmtSetTrapHandler) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetTileConfig) = (HSAKMT_DEF(hsaKmtGetTileConfig)*)dlsym(dtif_handle, "hsaKmtGetTileConfig"); + HSAKMT_PFN(hsaKmtGetTileConfig) = (HSAKMT_DEF(hsaKmtGetTileConfig)*)dlsym(thunk_handle, "hsaKmtGetTileConfig"); if (HSAKMT_PFN(hsaKmtGetTileConfig) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtQueryPointerInfo) = (HSAKMT_DEF(hsaKmtQueryPointerInfo)*)dlsym(dtif_handle, "hsaKmtQueryPointerInfo"); + HSAKMT_PFN(hsaKmtQueryPointerInfo) = (HSAKMT_DEF(hsaKmtQueryPointerInfo)*)dlsym(thunk_handle, "hsaKmtQueryPointerInfo"); if (HSAKMT_PFN(hsaKmtQueryPointerInfo) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSetMemoryUserData) = (HSAKMT_DEF(hsaKmtSetMemoryUserData)*)dlsym(dtif_handle, "hsaKmtSetMemoryUserData"); + HSAKMT_PFN(hsaKmtSetMemoryUserData) = (HSAKMT_DEF(hsaKmtSetMemoryUserData)*)dlsym(thunk_handle, "hsaKmtSetMemoryUserData"); if (HSAKMT_PFN(hsaKmtSetMemoryUserData) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetQueueInfo) = (HSAKMT_DEF(hsaKmtGetQueueInfo)*)dlsym(dtif_handle, "hsaKmtGetQueueInfo"); + HSAKMT_PFN(hsaKmtGetQueueInfo) = (HSAKMT_DEF(hsaKmtGetQueueInfo)*)dlsym(thunk_handle, "hsaKmtGetQueueInfo"); if (HSAKMT_PFN(hsaKmtGetQueueInfo) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtAllocQueueGWS) = (HSAKMT_DEF(hsaKmtAllocQueueGWS)*)dlsym(dtif_handle, "hsaKmtAllocQueueGWS"); + HSAKMT_PFN(hsaKmtAllocQueueGWS) = (HSAKMT_DEF(hsaKmtAllocQueueGWS)*)dlsym(thunk_handle, "hsaKmtAllocQueueGWS"); if (HSAKMT_PFN(hsaKmtAllocQueueGWS) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRuntimeEnable) = (HSAKMT_DEF(hsaKmtRuntimeEnable)*)dlsym(dtif_handle, "hsaKmtRuntimeEnable"); + HSAKMT_PFN(hsaKmtRuntimeEnable) = (HSAKMT_DEF(hsaKmtRuntimeEnable)*)dlsym(thunk_handle, "hsaKmtRuntimeEnable"); if (HSAKMT_PFN(hsaKmtRuntimeEnable) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtRuntimeDisable) = (HSAKMT_DEF(hsaKmtRuntimeDisable)*)dlsym(dtif_handle, "hsaKmtRuntimeDisable"); + HSAKMT_PFN(hsaKmtRuntimeDisable) = (HSAKMT_DEF(hsaKmtRuntimeDisable)*)dlsym(thunk_handle, "hsaKmtRuntimeDisable"); if (HSAKMT_PFN(hsaKmtRuntimeDisable) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtCheckRuntimeDebugSupport) = (HSAKMT_DEF(hsaKmtCheckRuntimeDebugSupport)*)dlsym(dtif_handle, "hsaKmtCheckRuntimeDebugSupport"); + HSAKMT_PFN(hsaKmtCheckRuntimeDebugSupport) = (HSAKMT_DEF(hsaKmtCheckRuntimeDebugSupport)*)dlsym(thunk_handle, "hsaKmtCheckRuntimeDebugSupport"); if (HSAKMT_PFN(hsaKmtCheckRuntimeDebugSupport) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetRuntimeCapabilities) = (HSAKMT_DEF(hsaKmtGetRuntimeCapabilities)*)dlsym(dtif_handle, "hsaKmtGetRuntimeCapabilities"); + HSAKMT_PFN(hsaKmtGetRuntimeCapabilities) = (HSAKMT_DEF(hsaKmtGetRuntimeCapabilities)*)dlsym(thunk_handle, "hsaKmtGetRuntimeCapabilities"); if (HSAKMT_PFN(hsaKmtGetRuntimeCapabilities) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtDebugTrapIoctl) = (HSAKMT_DEF(hsaKmtDebugTrapIoctl)*)dlsym(dtif_handle, "hsaKmtDebugTrapIoctl"); + HSAKMT_PFN(hsaKmtDebugTrapIoctl) = (HSAKMT_DEF(hsaKmtDebugTrapIoctl)*)dlsym(thunk_handle, "hsaKmtDebugTrapIoctl"); if (HSAKMT_PFN(hsaKmtDebugTrapIoctl) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSPMAcquire) = (HSAKMT_DEF(hsaKmtSPMAcquire)*)dlsym(dtif_handle, "hsaKmtSPMAcquire"); + HSAKMT_PFN(hsaKmtSPMAcquire) = (HSAKMT_DEF(hsaKmtSPMAcquire)*)dlsym(thunk_handle, "hsaKmtSPMAcquire"); if (HSAKMT_PFN(hsaKmtSPMAcquire) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSPMRelease) = (HSAKMT_DEF(hsaKmtSPMRelease)*)dlsym(dtif_handle, "hsaKmtSPMRelease"); + HSAKMT_PFN(hsaKmtSPMRelease) = (HSAKMT_DEF(hsaKmtSPMRelease)*)dlsym(thunk_handle, "hsaKmtSPMRelease"); if (HSAKMT_PFN(hsaKmtSPMRelease) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSPMSetDestBuffer) = (HSAKMT_DEF(hsaKmtSPMSetDestBuffer)*)dlsym(dtif_handle, "hsaKmtSPMSetDestBuffer"); + HSAKMT_PFN(hsaKmtSPMSetDestBuffer) = (HSAKMT_DEF(hsaKmtSPMSetDestBuffer)*)dlsym(thunk_handle, "hsaKmtSPMSetDestBuffer"); if (HSAKMT_PFN(hsaKmtSPMSetDestBuffer) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSVMSetAttr) = (HSAKMT_DEF(hsaKmtSVMSetAttr)*)dlsym(dtif_handle, "hsaKmtSVMSetAttr"); + HSAKMT_PFN(hsaKmtSVMSetAttr) = (HSAKMT_DEF(hsaKmtSVMSetAttr)*)dlsym(thunk_handle, "hsaKmtSVMSetAttr"); if (HSAKMT_PFN(hsaKmtSVMSetAttr) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSVMGetAttr) = (HSAKMT_DEF(hsaKmtSVMGetAttr)*)dlsym(dtif_handle, "hsaKmtSVMGetAttr"); + HSAKMT_PFN(hsaKmtSVMGetAttr) = (HSAKMT_DEF(hsaKmtSVMGetAttr)*)dlsym(thunk_handle, "hsaKmtSVMGetAttr"); if (HSAKMT_PFN(hsaKmtSVMGetAttr) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtSetXNACKMode) = (HSAKMT_DEF(hsaKmtSetXNACKMode)*)dlsym(dtif_handle, "hsaKmtSetXNACKMode"); + HSAKMT_PFN(hsaKmtSetXNACKMode) = (HSAKMT_DEF(hsaKmtSetXNACKMode)*)dlsym(thunk_handle, "hsaKmtSetXNACKMode"); if (HSAKMT_PFN(hsaKmtSetXNACKMode) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetXNACKMode) = (HSAKMT_DEF(hsaKmtGetXNACKMode)*)dlsym(dtif_handle, "hsaKmtGetXNACKMode"); + HSAKMT_PFN(hsaKmtGetXNACKMode) = (HSAKMT_DEF(hsaKmtGetXNACKMode)*)dlsym(thunk_handle, "hsaKmtGetXNACKMode"); if (HSAKMT_PFN(hsaKmtGetXNACKMode) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtOpenSMI) = (HSAKMT_DEF(hsaKmtOpenSMI)*)dlsym(dtif_handle, "hsaKmtOpenSMI"); + HSAKMT_PFN(hsaKmtOpenSMI) = (HSAKMT_DEF(hsaKmtOpenSMI)*)dlsym(thunk_handle, "hsaKmtOpenSMI"); if (HSAKMT_PFN(hsaKmtOpenSMI) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtExportDMABufHandle) = (HSAKMT_DEF(hsaKmtExportDMABufHandle)*)dlsym(dtif_handle, "hsaKmtExportDMABufHandle"); + HSAKMT_PFN(hsaKmtExportDMABufHandle) = (HSAKMT_DEF(hsaKmtExportDMABufHandle)*)dlsym(thunk_handle, "hsaKmtExportDMABufHandle"); if (HSAKMT_PFN(hsaKmtExportDMABufHandle) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtWaitOnEvent_Ext) = (HSAKMT_DEF(hsaKmtWaitOnEvent_Ext)*)dlsym(dtif_handle, "hsaKmtWaitOnEvent_Ext"); + HSAKMT_PFN(hsaKmtWaitOnEvent_Ext) = (HSAKMT_DEF(hsaKmtWaitOnEvent_Ext)*)dlsym(thunk_handle, "hsaKmtWaitOnEvent_Ext"); if (HSAKMT_PFN(hsaKmtWaitOnEvent_Ext) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtWaitOnMultipleEvents_Ext) = (HSAKMT_DEF(hsaKmtWaitOnMultipleEvents_Ext)*)dlsym(dtif_handle, "hsaKmtWaitOnMultipleEvents_Ext"); + HSAKMT_PFN(hsaKmtWaitOnMultipleEvents_Ext) = (HSAKMT_DEF(hsaKmtWaitOnMultipleEvents_Ext)*)dlsym(thunk_handle, "hsaKmtWaitOnMultipleEvents_Ext"); if (HSAKMT_PFN(hsaKmtWaitOnMultipleEvents_Ext) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtReplaceAsanHeaderPage) = (HSAKMT_DEF(hsaKmtReplaceAsanHeaderPage)*)dlsym(dtif_handle, "hsaKmtReplaceAsanHeaderPage"); + HSAKMT_PFN(hsaKmtReplaceAsanHeaderPage) = (HSAKMT_DEF(hsaKmtReplaceAsanHeaderPage)*)dlsym(thunk_handle, "hsaKmtReplaceAsanHeaderPage"); if (HSAKMT_PFN(hsaKmtReplaceAsanHeaderPage) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtReturnAsanHeaderPage) = (HSAKMT_DEF(hsaKmtReturnAsanHeaderPage)*)dlsym(dtif_handle, "hsaKmtReturnAsanHeaderPage"); + HSAKMT_PFN(hsaKmtReturnAsanHeaderPage) = (HSAKMT_DEF(hsaKmtReturnAsanHeaderPage)*)dlsym(thunk_handle, "hsaKmtReturnAsanHeaderPage"); if (HSAKMT_PFN(hsaKmtReturnAsanHeaderPage) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtGetAMDGPUDeviceHandle) = (HSAKMT_DEF(hsaKmtGetAMDGPUDeviceHandle)*)dlsym(dtif_handle, "hsaKmtGetAMDGPUDeviceHandle"); + HSAKMT_PFN(hsaKmtGetAMDGPUDeviceHandle) = (HSAKMT_DEF(hsaKmtGetAMDGPUDeviceHandle)*)dlsym(thunk_handle, "hsaKmtGetAMDGPUDeviceHandle"); if (HSAKMT_PFN(hsaKmtGetAMDGPUDeviceHandle) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPcSamplingQueryCapabilities) = (HSAKMT_DEF(hsaKmtPcSamplingQueryCapabilities)*)dlsym(dtif_handle, "hsaKmtPcSamplingQueryCapabilities"); + HSAKMT_PFN(hsaKmtPcSamplingQueryCapabilities) = (HSAKMT_DEF(hsaKmtPcSamplingQueryCapabilities)*)dlsym(thunk_handle, "hsaKmtPcSamplingQueryCapabilities"); if (HSAKMT_PFN(hsaKmtPcSamplingQueryCapabilities) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPcSamplingCreate) = (HSAKMT_DEF(hsaKmtPcSamplingCreate)*)dlsym(dtif_handle, "hsaKmtPcSamplingCreate"); + HSAKMT_PFN(hsaKmtPcSamplingCreate) = (HSAKMT_DEF(hsaKmtPcSamplingCreate)*)dlsym(thunk_handle, "hsaKmtPcSamplingCreate"); if (HSAKMT_PFN(hsaKmtPcSamplingCreate) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPcSamplingDestroy) = (HSAKMT_DEF(hsaKmtPcSamplingDestroy)*)dlsym(dtif_handle, "hsaKmtPcSamplingDestroy"); + HSAKMT_PFN(hsaKmtPcSamplingDestroy) = (HSAKMT_DEF(hsaKmtPcSamplingDestroy)*)dlsym(thunk_handle, "hsaKmtPcSamplingDestroy"); if (HSAKMT_PFN(hsaKmtPcSamplingDestroy) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPcSamplingStart) = (HSAKMT_DEF(hsaKmtPcSamplingStart)*)dlsym(dtif_handle, "hsaKmtPcSamplingStart"); + HSAKMT_PFN(hsaKmtPcSamplingStart) = (HSAKMT_DEF(hsaKmtPcSamplingStart)*)dlsym(thunk_handle, "hsaKmtPcSamplingStart"); if (HSAKMT_PFN(hsaKmtPcSamplingStart) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPcSamplingStop) = (HSAKMT_DEF(hsaKmtPcSamplingStop)*)dlsym(dtif_handle, "hsaKmtPcSamplingStop"); + HSAKMT_PFN(hsaKmtPcSamplingStop) = (HSAKMT_DEF(hsaKmtPcSamplingStop)*)dlsym(thunk_handle, "hsaKmtPcSamplingStop"); if (HSAKMT_PFN(hsaKmtPcSamplingStop) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtPcSamplingSupport) = (HSAKMT_DEF(hsaKmtPcSamplingSupport)*)dlsym(dtif_handle, "hsaKmtPcSamplingSupport"); + HSAKMT_PFN(hsaKmtPcSamplingSupport) = (HSAKMT_DEF(hsaKmtPcSamplingSupport)*)dlsym(thunk_handle, "hsaKmtPcSamplingSupport"); if (HSAKMT_PFN(hsaKmtPcSamplingSupport) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtModelEnabled) = (HSAKMT_DEF(hsaKmtModelEnabled)*)dlsym(dtif_handle, "hsaKmtModelEnabled"); + HSAKMT_PFN(hsaKmtModelEnabled) = (HSAKMT_DEF(hsaKmtModelEnabled)*)dlsym(thunk_handle, "hsaKmtModelEnabled"); if (HSAKMT_PFN(hsaKmtModelEnabled) == NULL) goto ERROR; - HSAKMT_PFN(hsaKmtQueueRingDoorbell) = (HSAKMT_DEF(hsaKmtQueueRingDoorbell)*)dlsym(dtif_handle, "hsaKmtQueueRingDoorbell"); + HSAKMT_PFN(hsaKmtQueueRingDoorbell) = (HSAKMT_DEF(hsaKmtQueueRingDoorbell)*)dlsym(thunk_handle, "hsaKmtQueueRingDoorbell"); if (HSAKMT_PFN(hsaKmtQueueRingDoorbell) == NULL) goto ERROR; - DRM_PFN(amdgpu_device_initialize) = (DRM_DEF(amdgpu_device_initialize)*)dlsym(dtif_handle, "amdgpu_device_initialize"); + DRM_PFN(amdgpu_device_initialize) = (DRM_DEF(amdgpu_device_initialize)*)dlsym(thunk_handle, "amdgpu_device_initialize"); if (DRM_PFN(amdgpu_device_initialize) == NULL) goto ERROR; - DRM_PFN(amdgpu_device_deinitialize) = (DRM_DEF(amdgpu_device_deinitialize)*)dlsym(dtif_handle, "amdgpu_device_deinitialize"); + DRM_PFN(amdgpu_device_deinitialize) = (DRM_DEF(amdgpu_device_deinitialize)*)dlsym(thunk_handle, "amdgpu_device_deinitialize"); if (DRM_PFN(amdgpu_device_deinitialize) == NULL) goto ERROR; - DRM_PFN(amdgpu_query_gpu_info) = (DRM_DEF(amdgpu_query_gpu_info)*)dlsym(dtif_handle, "amdgpu_query_gpu_info"); + DRM_PFN(amdgpu_query_gpu_info) = (DRM_DEF(amdgpu_query_gpu_info)*)dlsym(thunk_handle, "amdgpu_query_gpu_info"); if (DRM_PFN(amdgpu_query_gpu_info) == NULL) goto ERROR; - DRM_PFN(amdgpu_bo_cpu_map) = (DRM_DEF(amdgpu_bo_cpu_map)*)dlsym(dtif_handle, "amdgpu_bo_cpu_map"); + DRM_PFN(amdgpu_bo_cpu_map) = (DRM_DEF(amdgpu_bo_cpu_map)*)dlsym(thunk_handle, "amdgpu_bo_cpu_map"); if (DRM_PFN(amdgpu_bo_cpu_map) == NULL) goto ERROR; - DRM_PFN(amdgpu_bo_free) = (DRM_DEF(amdgpu_bo_free)*)dlsym(dtif_handle, "amdgpu_bo_free"); + DRM_PFN(amdgpu_bo_free) = (DRM_DEF(amdgpu_bo_free)*)dlsym(thunk_handle, "amdgpu_bo_free"); if (DRM_PFN(amdgpu_bo_free) == NULL) goto ERROR; - DRM_PFN(amdgpu_bo_export) = (DRM_DEF(amdgpu_bo_export)*)dlsym(dtif_handle, "amdgpu_bo_export"); + DRM_PFN(amdgpu_bo_export) = (DRM_DEF(amdgpu_bo_export)*)dlsym(thunk_handle, "amdgpu_bo_export"); if (DRM_PFN(amdgpu_bo_export) == NULL) goto ERROR; - DRM_PFN(amdgpu_bo_import) = (DRM_DEF(amdgpu_bo_import)*)dlsym(dtif_handle, "amdgpu_bo_import"); + DRM_PFN(amdgpu_bo_import) = (DRM_DEF(amdgpu_bo_import)*)dlsym(thunk_handle, "amdgpu_bo_import"); if (DRM_PFN(amdgpu_bo_import) == NULL) goto ERROR; - DRM_PFN(amdgpu_bo_va_op) = (DRM_DEF(amdgpu_bo_va_op)*)dlsym(dtif_handle, "amdgpu_bo_va_op"); + DRM_PFN(amdgpu_bo_va_op) = (DRM_DEF(amdgpu_bo_va_op)*)dlsym(thunk_handle, "amdgpu_bo_va_op"); if (DRM_PFN(amdgpu_bo_va_op) == NULL) goto ERROR; - DRM_PFN(drmCommandWriteRead) = (DRM_DEF(drmCommandWriteRead)*)dlsym(dtif_handle, "drmCommandWriteRead"); + DRM_PFN(drmCommandWriteRead) = (DRM_DEF(drmCommandWriteRead)*)dlsym(thunk_handle, "drmCommandWriteRead"); if (DRM_PFN(drmCommandWriteRead) == NULL) goto ERROR; debug_print("Load all DTIF APIs OK!\n"); @@ -489,10 +514,10 @@ ERROR: } bool ThunkLoader::CreateThunkInstance() { - if (!core::Runtime::runtime_singleton_->flag().enable_dtif()) + if (!IsDTIF()) return true; - DtifCreateFunc* pfnDtifCreate = (DtifCreateFunc*)dlsym(dtif_handle, "DtifCreate"); + DtifCreateFunc* pfnDtifCreate = (DtifCreateFunc*)dlsym(thunk_handle, "DtifCreate"); if (pfnDtifCreate != NULL) { if (pfnDtifCreate("HSA") != NULL) { debug_print("DtifCreate OK!\n"); @@ -506,13 +531,13 @@ ERROR: } bool ThunkLoader::DestroyThunkInstance() { - if (!core::Runtime::runtime_singleton_->flag().enable_dtif()) + if (!IsDTIF()) return true; - if (dtif_handle == NULL) + if (thunk_handle == NULL) return false; - DtifDestroyFunc* pfnDtifDestroy = (DtifDestroyFunc*)dlsym(dtif_handle, "DtifDestroy"); + DtifDestroyFunc* pfnDtifDestroy = (DtifDestroyFunc*)dlsym(thunk_handle, "DtifDestroy"); if (pfnDtifDestroy != NULL) { pfnDtifDestroy(); debug_print("DtifDestroy OK!\n"); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h index 654db1b82f..a79d4e923b 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h @@ -292,6 +292,10 @@ class Flag { var = os::GetEnvVar("HSA_ENABLE_DTIF"); enable_dtif_ = (var == "1") ? true : false; + // This allows detecting if the dxg driver is loaded. + var = os::GetEnvVar("HSA_ENABLE_DXG_DETECTION"); + enable_dxg_detection_ = (var == "1") ? true : false; + var = os::GetEnvVar("HSA_CO_DMACOPY_SIZE"); co_dmacopy_size_ = var.empty() ? 1024*1024 : atoi(var.c_str()); } @@ -357,12 +361,18 @@ class Flag { bool enable_scratch_alt() const { return enable_scratch_alt_; } + bool enable_scratch() const { return enable_scratch_; } + size_t scratch_single_limit_async() const { return scratch_single_limit_async_; } std::string tools_lib_names() const { return tools_lib_names_; } bool disable_image() const { return disable_image_; } + void disable_image(bool disable) { disable_image_ = disable; } + + void set_ipc_mode_legacy(bool enable) { enable_ipc_mode_legacy_ = enable; } + bool disable_pc_sampling() const { return disable_pc_sampling_; } bool loader_enable_mmap_uri() const { return loader_enable_mmap_uri_; } @@ -417,7 +427,34 @@ class Flag { bool enable_3d_swizzle() const { return enable_3d_swizzle_; } bool enable_dtif() const { return enable_dtif_; } - private: + + bool enable_dxg_detection() const { return enable_dxg_detection_; } + + void set_sdma(bool peer_sdma, bool sdma_gang) { + enable_peer_sdma_ = peer_sdma ? SDMA_ENABLE : SDMA_DISABLE; + enable_sdma_gang_ = sdma_gang ? SDMA_ENABLE : SDMA_DISABLE; + } + + void disable_scratch() { + scratch_single_limit_ = 0; + //scratch_single_limit_async_ = 0; + enable_scratch_async_reclaim_ = false; + enable_scratch_alt_ = false; + enable_scratch_ = false; + scratch_mem_size_ = 0; + no_scratch_reclaim_ = true; + no_scratch_thread_limit_ = true; + } + + void disable_xnack() { xnack_ = XNACK_DISABLE; } + + void disable_fine_grain_pcie() { fine_grain_pcie_ = false; } + + void disable_dev_mem_queue_buf() { dev_mem_queue_buf_ = false; } + + void disable_sdma_hdp_flush() { enable_sdma_hdp_flush_ = false; } + + private: bool check_flat_scratch_; bool enable_vm_fault_message_; bool enable_interrupt_; @@ -451,6 +488,7 @@ class Flag { int async_events_thread_priority_; bool enable_3d_swizzle_ = false; bool enable_dtif_; + bool enable_dxg_detection_; SDMA_OVERRIDE enable_sdma_; SDMA_OVERRIDE enable_peer_sdma_; @@ -468,6 +506,7 @@ class Flag { size_t scratch_single_limit_async_; bool enable_scratch_async_reclaim_; bool enable_scratch_alt_; + bool enable_scratch_ = true; std::string tools_lib_names_; std::string svm_profile_;