From eb3d45d3006880509bc80605ed3c242fa645784a Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Fri, 18 Jul 2025 15:49:18 -0400 Subject: [PATCH 01/13] rocr: Fix warnings --- runtime/hsa-runtime/core/inc/runtime.h | 5 ----- runtime/hsa-runtime/core/runtime/runtime.cpp | 5 +++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index 0cbc742685..c47a5f2d25 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -888,11 +888,6 @@ class Runtime { const hsa_amd_memory_access_desc_t *desc, const size_t desc_cnt); - // Frees runtime memory when the runtime library is unloaded if safe to do so. - // Failure to release the runtime indicates an incorrect application but is - // common (example: calls library routines at process exit). - friend class RuntimeCleanup; - void InitIPCDmaBufSupport(); bool ipc_dmabuf_supported_; int IPCClientImport(uint32_t conn_handle, uint64_t dmabuf_fd_handle, diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index ff52495d45..c2ee15c922 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -3194,10 +3194,11 @@ hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) { if (it->second.use_count > 0) return HSA_STATUS_ERROR_RESOURCE_FREE; - if (it->second.registered) + if (it->second.registered) { if (HSAKMT_CALL(hsaKmtFreeMemory(it->second.os_addr, size)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; - else + } else { if (munmap(it->second.os_addr, size)) return HSA_STATUS_ERROR; + } reserved_address_map_.erase(it); return HSA_STATUS_SUCCESS; From f755981f03b3cd785582c0ff4549bd8f6190ec95 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 10 Jul 2025 14:26:26 -0400 Subject: [PATCH 02/13] kfdtest: Remove gfx940/941 references Support was removed for these eng samples, so remove them from the blacklist, and make sure that we're using 942 for the shader store --- libhsakmt/tests/kfdtest/scripts/kfdtest.exclude | 16 ---------------- libhsakmt/tests/kfdtest/src/KFDASMTest.cpp | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/libhsakmt/tests/kfdtest/scripts/kfdtest.exclude b/libhsakmt/tests/kfdtest/scripts/kfdtest.exclude index eac236ae0b..bfa1a48294 100644 --- a/libhsakmt/tests/kfdtest/scripts/kfdtest.exclude +++ b/libhsakmt/tests/kfdtest/scripts/kfdtest.exclude @@ -325,22 +325,6 @@ FILTER[gfx1036]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X" -FILTER[gfx940]=\ -"$BLACKLIST_ALL_ASICS:"\ -"KFDMemoryTest.LargestSysBufferTest:"\ -"KFDMemoryTest.BigSysBufferStressTest:"\ -"KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest:"\ -"KFDQMTest.QueueLatency" - -FILTER[gfx941]=\ -"$BLACKLIST_ALL_ASICS:"\ -"KFDMemoryTest.LargestSysBufferTest:"\ -"KFDMemoryTest.BigSysBufferStressTest:"\ -"KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest:"\ -"KFDQMTest.QueueLatency" - FILTER[gfx942]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDMemoryTest.LargestSysBufferTest:"\ diff --git a/libhsakmt/tests/kfdtest/src/KFDASMTest.cpp b/libhsakmt/tests/kfdtest/src/KFDASMTest.cpp index e0e6a35ef7..8d5ec579d1 100644 --- a/libhsakmt/tests/kfdtest/src/KFDASMTest.cpp +++ b/libhsakmt/tests/kfdtest/src/KFDASMTest.cpp @@ -43,7 +43,7 @@ static const std::vector TargetList = { 0x090009, 0x09000a, 0x09000c, - 0x090400, + 0x090402, 0x0a0100, 0x0a0101, 0x0a0102, From 6015ad101656457ed674c0334c7f5c5bfe75ba4a Mon Sep 17 00:00:00 2001 From: Shweta Khatri Date: Tue, 13 May 2025 00:04:37 -0400 Subject: [PATCH 03/13] rocr: GFX12 - Enable host trap PC Sampling --- runtime/hsa-runtime/core/inc/amd_gpu_pm4.h | 2 +- .../core/runtime/amd_gpu_agent.cpp | 23 + .../runtime/trap_handler/trap_handler_gfx12.s | 803 +++++++++++++++--- 3 files changed, 715 insertions(+), 113 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_pm4.h b/runtime/hsa-runtime/core/inc/amd_gpu_pm4.h index 2e871067cb..31ed5453a9 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_pm4.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_pm4.h @@ -89,7 +89,7 @@ # define PM4_ACQUIRE_MEM_GCR_CNTL_GLV_INV (1 << 8) # define PM4_ACQUIRE_MEM_GCR_CNTL_GL1_INV (1 << 9) # define PM4_ACQUIRE_MEM_GCR_CNTL_GL2_INV (1 << 14) - +# define PM4_ACQUIRE_MEM_GCR_CNTL_GL2_WB (1 << 15) #define PM4_RELEASE_MEM_DW1_EVENT_INDEX(x) (((x) & 0xF) << 8) # define PM4_RELEASE_MEM_EVENT_INDEX_AQL 0x7 diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 6cb147a306..7273bbd9b2 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -2568,6 +2568,10 @@ hsa_status_t GpuAgent::PcSamplingIterateConfig(hsa_ven_amd_pcs_iterate_configura if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; for (uint32_t i = 0; i < size; i++) { + if ((isa_->GetMajorVersion() == 12 && (isa_->GetMinorVersion() == 0)) && + sampleInfoList[i].method == HSA_PC_SAMPLING_METHOD_KIND_STOCHASTIC_V1) { + continue; + } hsa_ven_amd_pcs_configuration_t hsaPcSampling; if (ConvertHsaKmtPcSamplingInfoToHsa(&sampleInfoList[i], &hsaPcSampling) == HSA_STATUS_SUCCESS && cb(&hsaPcSampling, cb_data) == HSA_STATUS_INFO_BREAK) @@ -2614,6 +2618,10 @@ hsa_status_t GpuAgent::PcSamplingCreateFromId(HsaPcSamplingTraceId ioctlId, if (sampling_method == HSA_VEN_AMD_PCS_METHOD_HOSTTRAP_V1) { pcs_data = &pcs_hosttrap_data_; } else if (sampling_method == HSA_VEN_AMD_PCS_METHOD_STOCHASTIC_V1) { + if (isa_->GetMajorVersion() == 12 && (isa_->GetMinorVersion() == 0)) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + pcs_data = &pcs_stochastic_data_; } else { // Unsupported sampling method @@ -3093,6 +3101,7 @@ hsa_status_t GpuAgent::PcSamplingFlushDeviceBuffers( const uint32_t atomic_ex_cmd_sz = 9; const uint32_t wait_reg_mem_cmd_sz = 7; + const uint32_t acquire_mem_cmd_sz = 8; const uint32_t dma_data_cmd_sz = 7; const uint32_t copy_data_cmd_sz = 6; const uint32_t write_data_cmd_sz = 5; @@ -3225,6 +3234,20 @@ hsa_status_t GpuAgent::PcSamplingFlushDeviceBuffers( cmd_data[i++] = PM4_WAIT_REG_MEM_DW6(PM4_WAIT_REG_MEM_POLL_INTERVAL(4) | PM4_WAIT_REG_MEM_OPTIMIZE_ACE_OFFLOAD_MODE); + // For GFX1200 and GFX1201 only - add an ACQUIRE_MEM packet to flush L2 cache before DMA. + // This ensures that any data written by the trap handler is visible to the DMA engine. + if ((isa_->GetMajorVersion() == 12) && (isa_->GetMinorVersion() == 0)) { + cmd_data[i++] = + PM4_HDR(PM4_HDR_IT_OPCODE_ACQUIRE_MEM, acquire_mem_cmd_sz, isa_->GetMajorVersion()); + cmd_data[i++] = 0; // DW1: COHER_CNTL + cmd_data[i++] = 0; // DW2: COHER_SIZE + cmd_data[i++] = 0; // DW3: COHER_SIZE_HI + cmd_data[i++] = 0; // DW4: COHER_BASE_LO + cmd_data[i++] = 0; // DW5: COHER_BASE_HI + cmd_data[i++] = 4; // DW6: POLL_INTERVAL + cmd_data[i++] = PM4_ACQUIRE_MEM_GCR_CNTL_GL2_WB; // DW7: GCR_CNTL (GL2_WB=1, RANGE=ALL) + } + uint8_t* buffer_temp = buffer[which_buffer]; for (copy_bytes = std::min(to_copy, (uint32_t)CP_DMA_DATA_TRANSFER_CNT_MAX); 0 < to_copy; diff --git a/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler_gfx12.s b/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler_gfx12.s index 735e383dd9..aa2c1dd297 100644 --- a/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler_gfx12.s +++ b/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler_gfx12.s @@ -56,6 +56,7 @@ .set SQ_WAVE_EXCP_FLAG_PRIV_HT_SHIFT , 7 .set SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT , 8 .set SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_SHIFT , 9 +.set SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT , 10 .set SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_SHIFT , 11 .set SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT , 12 @@ -74,6 +75,7 @@ .set SQ_WAVE_STATE_PRIV_HALT_BFE , (SQ_WAVE_STATE_PRIV_HALT_SHIFT | (1 << 16)) .set SQ_WAVE_STATE_PRIV_HALT_SHIFT , 14 .set SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT , 2 + .set TRAP_ID_ABORT , 2 .set TRAP_ID_DEBUGTRAP , 3 .set TTMP6_SAVED_STATUS_HALT_MASK , (1 << TTMP6_SAVED_STATUS_HALT_SHIFT) @@ -87,140 +89,273 @@ .set TTMP11_DEBUG_ENABLED_SHIFT , 23 .set TTMP_PC_HI_SHIFT , 7 -// ABI between first and second level trap handler: -// { ttmp1, ttmp0 } = TrapID[3:0], zeros, PC[47:0] -// ttmp11 = 0[7:0], DebugEnabled[0], 0[15:0], NoScratch[0], 0[5:0] -// ttmp12 = SQ_WAVE_STATE_PRIV -// ttmp14 = TMA[31:0] -// ttmp15 = TMA[63:32] +.set TTMP13_HT_FLAG_BIT , 22 // TTMP13 bit for host‑trap +.set TTMP13_STOCH_FLAG_BIT , 21 // TTMP13 bit for stochastic +.set TTMP13_BUF_FULL_BIT , 31 // TTMP13 bit – buf full mark +.set TTMP8_DISPATCH_ID_MASK , 0X1FFFFFF +// Per-sample data layout within the device buffer. Each sample is 64 bytes. +// These are offsets from the start of a specific sample slot in the device buffer. -trap_entry: - // Clear ttmp3 as it will contain the exception code. - s_mov_b32 ttmp3, 0 +.set SAMPLE_OFF_BYTES_PER_SAMPLE , 0x40 // bytes per sample slot +.set SAMPLE_OFF_PC_HOST , 0x00 // original PC (host only) +.set SAMPLE_OFF_EXEC_LOHI , 0x08 // saved EXEC low/high +.set SAMPLE_OFF_WGID_XY , 0x10 // WG id X / Y +.set SAMPLE_OFF_WGID_Z_WAVE , 0x18 // WG id Z +.set SAMPLE_OFF_TIMESTAMP , 0x30 // 64 bit realtime counter +.set SAMPLE_OFF_HW_ID , 0x20 // HW_ID (values combined from the HW_ID1 + HW_ID2) +.set SAMPLE_OFF_SNAPSHOT_DATA , 0x24 +.set SAMPLE_OFF_CORRELATION , 0x38 // doorbell + dispatch id +.set SAMPLE_OFF_BUF_WRITTEN_VAL , 0x10 // Offset to buf_written_val0/1 in pcs_sampling_data_t +.set SAMPLE_OFF_BUF_SIZE , 0x8 // Offset to buf_size in pcs_sampling_data_t +.set SAMPLE_OFF_DONE_SIG0 , 0x18 // Offset for done_sig0 (hsa_signal_t handle for buffer 0) +.set SAMPLE_OFF_DONE_SIG1 , 0x28 // Offset for done_sig1 (hsa_signal_t handle for buffer 1) +.set SAMPLE_OFF_SIGNAL_VALUE , 0x8 // Offset within signal structure to value field +.set SAMPLE_OFF_EVENT_MAILBOX0 , 0x10 // Offset for event mailbox pointer for buffer 0 +.set SAMPLE_OFF_EVENT_MAILBOX1 , 0x20 // Offset for event mailbox pointer for buffer 1 + +.set WAVE_ID_MASK , 0x1f // Mask to extract Wave ID from TTMP register. +.set BUF_INDEX_MASK , 0x7fffffff // strip bit31 from add_x2 +.set SAMPLE_OFF_BUF_WRITTEN_VAL , 0x10 // Offset to buf_written_val0/1 in pcs_sampling_data_t +.set SAMPLE_INDEX_WIDTH , 31 // The sample index is 63 bits; the high part is 31 bits. + +.set HW_REG_SHADER_HW_ID1 , 0xf817 +.set HW_REG_SHADER_HW_ID2 , 0xf818 +.set HW_REG_SQ_PERF_SNAPSHOT_PC_LO , 0xf80b +.set HW_REG_SQ_PERF_SNAPSHOT_PC_HI , 0xf80c +.set HW_REG_SQ_PERF_SNAPSHOT_DATA1 , 0xf80f +.set HW_REG_SQ_PERF_SNAPSHOT_DATA2 , 0xf810 +.set HW_REG_SQ_PERF_SNAPSHOT_DATA , 0xf81b + + // Macro to store the Correlation ID (Dispatch ID and Doorbell ID) into the current sample slot + // + // Assumes the following registers are set before it is called: + // v[0:1]:Must contain the 64-bit base address of the target sample slot + // ttmp8 :Must contain the dispatch ID in bits [24:0] + // exec :Must be set to 0x1 to ensure operations apply only to lane 0 + // + // Clobbers the following registers: + // v[2:3]:Used for [dispatch_id, doorbell_id] + // ttmp6 :Used as scratch register +.macro STORE_CORRELATION_ID + s_sendmsg_rtn_b32 ttmp6, sendmsg(MSG_RTN_GET_DOORBELL) // Gets current queue's doorbell ID into ttmp6. + s_wait_kmcnt 0 + s_and_b32 ttmp6, ttmp6, DOORBELL_ID_MASK // Mask to get actual doorbell ID. + v_writelane_b32 v3, ttmp6, 0 // Store doorbell ID into high part of v[2:3] (via v3). + s_and_b32 ttmp6, ttmp8, TTMP8_DISPATCH_ID_MASK // Get dispatch ID from ttmp8 into ttmp6 + v_writelane_b32 v2, ttmp6, 0 // Store dispatch ID into low part of v[2:3] (via v2) + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_CORRELATION, scope:SCOPE_SYS // Store {dispatch_id, doorbell_id} into sample slot. + // v[0:1] = sample slot base address. + // v[2] = dispatch_id, v[3] = doorbell_id. +.endm + + // Macro to store the HW_ID registers into the current sample slot + // + // Assumes the following registers are set before it is called: + // v[0:1]: Must contain the 64-bit base address of the target sample slot. + // exec : Must be set to 0x1 to ensure operations apply only to lane 0. + // + // Clobbers the following registers: + // v[2:3]: Used to stage the data for the global store. + // ttmp6 : Used as scratch registers. +.macro STORE_HW_ID + // Current ROCr API determines single dword for HW_ID, while this information is scattered accross two + // dword registers HW_ID1 and HW_ID2 on GFX10+ architectures. + // Thus, we combine values from HW_ID1 and HW_ID2 into a single dword HW_ID with the following layout: + // WAVE_ID[4:0] + // QUEUE_ID[8:5] + // RESERVED [9] + // WGP_ID[13:10] + // SIMD_ID[15:14] + // SA_ID[16] + // ME_ID[17] + // SE_ID[19:18] + // PIPE_ID[21:20] + // RESERVED [22] + // WG_ID[27:23] + // VM_ID[31:28] + + // Note: We don't show DP_RATE and STATE_ID that are useless for compute kernels + // Also, we reduced SE_ID to 2 bits as there's only a maximum of 4 SEs on existing gfx12.0 parts + // Finally, ME_ID is reduced to 1 bit as wavefronts are dispatched from either ME0 or ME1 in gfx12. + // Bits 9 and 22 are reserved for a future use. + + s_getreg_b32 ttmp6, HW_REG_SHADER_HW_ID1 // Put HW_ID1 in ttmp6 + v_and_b32 v2, ttmp6, 0x1feffcff // Mask DP_RATE, SE_ID[2] and SIMD_ID + v_and_b32 v3, ttmp6, 0x300 // Put SIMD_ID into ttmp6[8:9] + v_lshl_or_b32 v2, v3, 6, v2 // Put SIMD_ID into v2[15:14] + s_getreg_b32 ttmp6, HW_REG_SHADER_HW_ID2 // Put HW_ID2 in ttmp6 + v_and_b32 v3, ttmp6, 0xf000000 // v3 = VM_ID in bits 27:24 + v_lshl_or_b32 v2, v3, 4, v2 // Put VM_ID into v2[31:28] + v_and_b32 v3, ttmp6, 0x1f0000 // v3 = WG_ID in bits 20:16 + v_lshl_or_b32 v2, v3, 7, v2 // Put WG_ID in v2[27:23] + v_and_b32 v3, ttmp6, 0x100 // v3 = ME_ID[0] in bit 8 + v_lshl_or_b32 v2, v3, 9, v2 // Put ME_ID in v2[17] + v_and_b32 v3, ttmp6, 0x30 // v3 = PIPE_ID in bits 5:4 + v_lshl_or_b32 v2, v3, 16, v2 // Put PIPE_ID in v2[21:20] + v_and_b32 v3, ttmp6, 0xf // v3 = QUEUE_ID in bits 3:0 + v_lshl_or_b32 v2, v3, 5, v2 // Put QUEUE_ID in v2[8:5] + global_store_b32 v[0:1], v2, off, offset:SAMPLE_OFF_HW_ID, scope:SCOPE_SYS // store HW_ID +.endm + +// ABI (Application Binary Interface) between first and second-level trap handler: +// ttmp0: PC_LO[31:0] (Program Counter Low) +// ttmp1: PC_HI[15:0] (Program Counter High, bits 0-15), TrapID[3:0] (in bits 28-31 of original PC_HI) +// ttmp11: 0[7:0], DebugEnabled[0], 0[15:0], NoScratch[0], 0[5:0] +// ttmp12: SQ_WAVE_STATE_PRIV (Private wave state register value). +// ttmp14: TMA[31:0] - TMA_LO (Trap Memory Argument Low - base address for trap handler data, low 32 bits). +// ttmp15: TTMA[63:32] - TMA_HI (Trap Memory Argument High - base address for trap handler data, high 32 bits). +// For PC Sampling, this points to pcs_hosttrap_data_ or pcs_stochastic_data_ + trap_entry: + + s_mov_b32 ttmp3, 0 + +.check_hosttrap: + + // ttmp[14:15] points to TMA. + // Available: ttmp[2:3], ttmp[4:5], ttmp6, ttmp[10:11] + s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_PRIV) // On gfx12, EXCP_FLAG_PRIV.b7 + s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_HT_SHIFT + s_cbranch_scc0 .check_stochastic + + // It's a Host Trap event. + s_load_b64 ttmp[14:15], ttmp[14:15], 0x0, scope:SCOPE_CU // ttmp[14:15]=*host_trap_buffers + s_bitset1_b32 ttmp13, TTMP13_HT_FLAG_BIT // set bit 22 in TTMP13 + + // Clear the Host Trap flag in the hardware register to acknowledge the event + s_setreg_imm32_b32 hwreg(HW_REG_EXCP_FLAG_PRIV, SQ_WAVE_EXCP_FLAG_PRIV_HT_SHIFT,1), 0 + s_wait_kmcnt 0 // Ensure previous load is complete. + s_branch .profile_trap_handlers + +.check_stochastic: + s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_PRIV) // EXCP_FLAG_PRIV.b10=stochastic_sample_trap + s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT // Test Performance Snapshot bit. + + s_cbranch_scc0 .check_exceptions // If not Stochastic, check for other exceptions. + + s_load_b64 ttmp[14:15], ttmp[14:15], 0x8, scope:SCOPE_CU // ttmp[14:15]=*stoch_trap_buf + s_wait_kmcnt 0 + + s_bitset1_b32 ttmp13, TTMP13_STOCH_FLAG_BIT // set bit 21 in TTMP13 + + s_setreg_imm32_b32 hwreg(HW_REG_EXCP_FLAG_PRIV, SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT,1), 0 // Clear the perf_snapshot flag + s_branch .profile_trap_handlers + + // Check if this is a trap (s_trap instruction) or a hardware exception. + // Extract TrapID from ttmp1 (which contains PC_HI). // Branch if not a trap (an exception instead). - s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE - s_cbranch_scc0 .check_exceptions + s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE // ttmp2 = TrapID + s_cbranch_scc0 .check_exceptions // If TrapID is 0, it's an exception, so branch. // If caused by s_trap then advance PC, then figure out the trap ID: // - if trapID is DEBUGTRAP and debugger is attach, report WAVE_TRAP, // - if trapID is ABORTTRAP, report WAVE_ABORT, // - report WAVE_TRAP for any other trap ID. - s_add_u32 ttmp0, ttmp0, 0x4 - s_addc_u32 ttmp1, ttmp1, 0x0 + s_add_u32 ttmp0, ttmp0, 0x4 // PC_LO += 4 + s_addc_u32 ttmp1, ttmp1, 0x0 // PC_HI += carry. // If llvm.debugtrap and debugger is not attached. - s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP - s_cbranch_scc0 .not_debug_trap + s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP + s_cbranch_scc0 .not_debug_trap - s_bitcmp1_b32 ttmp11, TTMP11_DEBUG_ENABLED_SHIFT - s_cbranch_scc0 .check_exceptions - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 + s_bitcmp1_b32 ttmp11, TTMP11_DEBUG_ENABLED_SHIFT + s_cbranch_scc0 .check_exceptions + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 .not_debug_trap: - s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT - s_cbranch_scc0 .not_abort_trap - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0 - s_branch .check_exceptions + s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT + s_cbranch_scc0 .not_abort_trap + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0 + s_branch .check_exceptions .not_abort_trap: - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 + + s_bitcmp1_b32 ttmp8, TTMP8_DEBUG_FLAG_SHIFT + s_cbranch_scc0 .check_exceptions - // We need to explititly look for all exceptions we want to report to the - // host: - // - EXCP_FLAG_PRIV.XNACK_ERROR (&& EXCP_FLAG_PRIV.MEMVIOL) - // -> WAVE_MEMORY_VIOLATION - // - EXCP_FLAG_PRIV.MEMVIOL (and !EXCP_FLAG_PRIV.XNACK_ERROR) - // -> WAVE_APERTURE_VIOLATION - // - EXCP_FLAG_PRIV.ILLEGAL_INST -> WAVE_ILLEGAL_INSTRUCTION - // - EXCP_FLAG_PRIV.WAVE_START -> WAVE_TRAP - // - EXCP_FLAG_PRIV.WAVE_END && TRAP_CTRL.WAVE_END -> WAVE_TRAP - // - TRAP_CTRL.TRAP_AFTER_INST -> WAVE_TRAP - // - EXCP_FLAG_PRIV.ADDR_WATCH && TRAP_CTL.WATCH -> WAVE_TRAP - // - (EXCP_FLAG_USER[ALU] & TRAP_CTRL[ALU]) != 0 -> WAVE_MATH_ERROR .check_exceptions: - s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_PRIV) - s_getreg_b32 ttmp13, hwreg(HW_REG_TRAP_CTRL) + s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_PRIV) + s_getreg_b32 ttmp13, hwreg(HW_REG_TRAP_CTRL) - s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT - s_cbranch_scc0 .not_memory_violation - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 + s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT + s_cbranch_scc0 .not_memory_violation + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 // Aperture violation requires XNACK_ERROR == 0. - s_branch .not_aperture_violation + s_branch .not_aperture_violation .not_memory_violation: - s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_MEMVIOL_SHIFT - s_cbranch_scc0 .not_aperture_violation - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 + s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_MEMVIOL_SHIFT + s_cbranch_scc0 .not_aperture_violation + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 .not_aperture_violation: - s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT - s_cbranch_scc0 .not_illegal_instruction - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 + s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT + s_cbranch_scc0 .not_illegal_instruction + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 .not_illegal_instruction: - s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT - s_cbranch_scc0 .not_wave_end - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 + s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT + s_cbranch_scc0 .not_wave_end + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 .not_wave_start: - s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_SHIFT - s_cbranch_scc0 .not_wave_end - s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_WAVE_END_SHIFT - s_cbranch_scc0 .not_wave_end - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 + s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_SHIFT + s_cbranch_scc0 .not_wave_end + s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_WAVE_END_SHIFT + s_cbranch_scc0 .not_wave_end + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 .not_wave_end: - s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST - s_cbranch_scc0 .not_trap_after_inst - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 + s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST + s_cbranch_scc0 .not_trap_after_inst + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 .not_trap_after_inst: - s_and_b32 ttmp2, ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK - s_cbranch_scc0 .not_addr_watch - s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_ADDR_WATCH_SHIFT - s_cbranch_scc0 .not_addr_watch - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 + s_and_b32 ttmp2, ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK + s_cbranch_scc0 .not_addr_watch + s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_ADDR_WATCH_SHIFT + s_cbranch_scc0 .not_addr_watch + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 .not_addr_watch: - s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_USER, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SHIFT, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE) - s_and_b32 ttmp13, ttmp13, SQ_WAVE_TRAP_CTRL_MATH_EXCP_MASK - s_and_b32 ttmp2, ttmp2, ttmp13 - s_cbranch_scc0 .not_math_exception - s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MATH_ERROR_M0 + s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_USER, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SHIFT, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE) + s_and_b32 ttmp13, ttmp13, SQ_WAVE_TRAP_CTRL_MATH_EXCP_MASK + s_and_b32 ttmp2, ttmp2, ttmp13 + s_cbranch_scc0 .not_math_exception + s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MATH_ERROR_M0 .not_math_exception: - s_cmp_eq_u32 ttmp3, 0 + s_cmp_eq_u32 ttmp3, 0 // This was not a s_trap we are interested in or an exception, return to // the user code. - s_cbranch_scc1 .exit_trap + s_cbranch_scc1 .exit_trap .send_interrupt: // Fetch doorbell id for our queue. - s_sendmsg_rtn_b32 ttmp2, sendmsg(MSG_RTN_GET_DOORBELL) - s_wait_kmcnt 0 - s_and_b32 ttmp2, ttmp2, DOORBELL_ID_MASK - s_or_b32 ttmp3, ttmp2, ttmp3 + s_sendmsg_rtn_b32 ttmp2, sendmsg(MSG_RTN_GET_DOORBELL) + s_wait_kmcnt 0 + s_and_b32 ttmp2, ttmp2, DOORBELL_ID_MASK + s_or_b32 ttmp3, ttmp2, ttmp3 // Save trap id and halt status in ttmp6. - s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK) - s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE - s_min_u32 ttmp2, ttmp2, 0xF - s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT - s_or_b32 ttmp6, ttmp6, ttmp2 - s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATE_PRIV_HALT_BFE - s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT - s_or_b32 ttmp6, ttmp6, ttmp2 + s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK) + s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE + s_min_u32 ttmp2, ttmp2, 0xF + s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT + s_or_b32 ttmp6, ttmp6, ttmp2 + s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATE_PRIV_HALT_BFE + s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT + s_or_b32 ttmp6, ttmp6, ttmp2 // m0 = interrupt data = (exception_code << DOORBELL_ID_SIZE) | doorbell_id - s_mov_b32 ttmp2, m0 - s_mov_b32 m0, ttmp3 - s_nop 0x0 // Manually inserted wait states - s_sendmsg sendmsg(MSG_INTERRUPT) + s_mov_b32 ttmp2, m0 + s_mov_b32 m0, ttmp3 + s_sendmsg sendmsg(MSG_INTERRUPT) // Wait for the message to go out. - s_wait_kmcnt 0 - s_mov_b32 m0, ttmp2 + s_wait_kmcnt 0 + s_mov_b32 m0, ttmp2 // Parking the wave requires saving the original pc in the preserved ttmps. // Register layout before parking the wave: @@ -234,44 +369,488 @@ trap_entry: // ttmp11: 1st_level_ttmp11[31:23] pc_hi[15:0] 1st_level_ttmp11[6:0] // // Save the PC - s_mov_b32 ttmp10, ttmp0 - s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK - s_lshl_b32 ttmp1, ttmp1, TTMP_PC_HI_SHIFT - s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP_PC_HI_SHIFT) - s_or_b32 ttmp11, ttmp11, ttmp1 + s_mov_b32 ttmp10, ttmp0 + s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK + s_lshl_b32 ttmp1, ttmp1, TTMP_PC_HI_SHIFT + s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP_PC_HI_SHIFT) + s_or_b32 ttmp11, ttmp11, ttmp1 // Park the wave - s_getpc_b64 [ttmp0, ttmp1] - s_add_u32 ttmp0, ttmp0, .parked - . - s_addc_u32 ttmp1, ttmp1, 0x0 + s_getpc_b64 [ttmp0, ttmp1] + s_add_u32 ttmp0, ttmp0, .parked - . + s_addc_u32 ttmp1, ttmp1, 0x0 .halt_wave: // Halt the wavefront upon restoring STATUS below. - s_bitset1_b32 ttmp6, TTMP6_WAVE_STOPPED_SHIFT - s_bitset1_b32 ttmp12, SQ_WAVE_STATE_PRIV_HALT_SHIFT + s_bitset1_b32 ttmp6, TTMP6_WAVE_STOPPED_SHIFT + s_bitset1_b32 ttmp12, SQ_WAVE_STATE_PRIV_HALT_SHIFT // Initialize TTMP registers - s_bitcmp1_b32 ttmp8, TTMP8_DEBUG_FLAG_SHIFT - s_cbranch_scc1 .ttmps_initialized - s_mov_b32 ttmp4, 0 - s_mov_b32 ttmp5, 0 - s_bitset1_b32 ttmp8, TTMP8_DEBUG_FLAG_SHIFT + s_bitcmp1_b32 ttmp8, TTMP8_DEBUG_FLAG_SHIFT + s_cbranch_scc1 .ttmps_initialized + s_mov_b32 ttmp4, 0 + s_mov_b32 ttmp5, 0 + s_bitset1_b32 ttmp8, TTMP8_DEBUG_FLAG_SHIFT .ttmps_initialized: + s_branch .exit_trap + +.profile_trap_handlers: + // Register state at the start of profile_trap_handlers: + // + // ttmp0: PC_LO[31:0] - Contains program counter low bits + // ttmp1: PC_HI[15:0] - Contains program counter high bits + // ttmp2: Contains HW_REG_EXCP_FLAG_PRIV + // ttmp3: Initialized to 0, available for use + // ttmp4: Available - Can be freely used + // ttmp5: Available - Can be freely used + // ttmp6: Initially contains flags - trap ID and halt status - reused after saving + // ttmp7: Contains WGID_Y in high 16 bits, WGID_Z in low 16 bits + // ttmp8: Contains dispatch ID in bits [24:0] and debug flag + // ttmp9: Contains WGID_X + // ttmp10: Available - Used next to save exec_lo + // ttmp11: Contains debug flags - Used next to save exec_hi + // ttmp12: Contains SQ_WAVE_STATE_PRIV + // ttmp13: Contains flag bits for sampling type - HT_FLAG_BIT or STOCH_FLAG_BIT + // ttmp[14:15]: Contains HT or ST buffer base address + // + // v[0:3] contain user shader data that must be preserved/restored + // exec: Contains user's execution mask + s_mov_b64 ttmp[10:11], exec // save exec to ttmp[10:11] + s_mov_b64 exec, 0x1 // turn on lane 0 only + + v_readlane_b32 ttmp2, v0, 0 + v_readlane_b32 ttmp3, v1, 0 // Save out lane 0’s first 2 VGPRs + + // At this point, ttmp[4:5], ttmp6 and v[0:1] are free + // Atomically get current sample slot index and select buffer + // pcs_sampling_data_t.buf_write_val (uint64_t) stores: + // Bit 63: current_buffer_id (0 or 1) + // Bits 62-0: current_sample_index_in_buffer + // v0 = 1 (value to add to the low part of buf_write_val) + // v1 = 0 (value to add to the high part of buf_write_val, bit 63 is buffer selector) + + v_mov_b32 v0, 1 + v_mov_b32 v1, 0 + + global_atomic_add_u64 v[0:1], v1, v[0:1], ttmp[14:15], scope:SCOPE_SYS th:TH_ATOMIC_RETURN + s_wait_loadcnt 0 // Wait for atomic operation to complete and return value + + // At this point, ttmp[4:5] and ttmp6 are free + // v[0:1] (lane 0) now holds the previous value of buf_write_val. + // This previous value gives the slot index for the current sample. + + v_readlane_b32 ttmp6, v1, 0x0 // previous buf_write_val[63:32] + s_lshr_b32 ttmp6, ttmp6, TTMP13_BUF_FULL_BIT // ttmp6 = previous_buffer_id (0 or 1, from bit 63 of original uint64_t) + // This ttmp6 is used to select which buffer's metadata (size, watermark, signal) to use. + // It's also used to calculate the base address of the sample buffer. + s_bitset0_b32 ttmp13, TTMP13_BUF_FULL_BIT // Clear our local buffer full flag for now + + s_cmp_eq_u32 ttmp6, 0 // store off buf_to_use + s_cbranch_scc1 .skip_bufbit_set // into bit31 of ttmp13 + s_bitset1_b32 ttmp13, TTMP13_BUF_FULL_BIT + +.skip_bufbit_set: + // ttmp[2:3]=v[0:1]-backup, ttmp[4:5]=free, ttmp6=buf_to_use (also in ttmp13.b31) + // ttmp[10:11]=EXEC backup. ttmp[14:15]=tma + // v[0:1].lane0=local_entry, v[2:3]=original, EXEC=0x1 + + v_bfe_u32 v1, v1, 0, SAMPLE_INDEX_WIDTH // v[0:1] = new local_entry + // removes bit 31 from v1, returning v1 & 0x7FFFFFFF. + + v_readlane_b32 ttmp5, v1, 0 // ttmp5 = high 31 bits of sample index (if index > 2^32-1). + s_cmp_lg_u32 ttmp5, 0 // Check if sample index is very large (overflowed 32 bits). + + s_cbranch_scc1 .lost_sample // If ttmp5 > 0, index is too large, treat as lost sample. + + s_load_b32 ttmp5, ttmp[14:15], SAMPLE_OFF_BUF_SIZE, scope:SCOPE_CU // ttmp5 = pcs_sampling_data_t.buf_size + v_readlane_b32 ttmp4, v0, 0 // ttmp4 = sample_index_for_current_sample (from v0) + s_wait_kmcnt 0 // Wait for buf_size load. + + s_cmp_ge_u32 ttmp4, ttmp5 // if local_entry >= buf_size + s_cbranch_scc1 .lost_sample // If index >= buf_size, buffer is full, sample is lost. + // This also sets TTMP13_BUF_FULL_BIT implicitly by branching. + + // Register state before calculating the sample buffer address: + // ttmp2 = backup of original shader's v0 + // ttmp3 = backup of original shader's v1 + // ttmp4 = sample_index_for_current_sample (from v0) + // ttmp5 = buf_size + // ttmp6 = buffer_id (0 or 1) + // ttmp[10:11] = original shader's [exec_lo, exec_hi] + // ttmp[14:15] = base_address_of_pcs_sampling_data_t (TMA) + // ttmp13.b31 = buffer_id (0 or 1, same as ttmp6) + // v[0:1].lane0 = sample index value from atomic + // v[2:3] = original user shader's v[2:3] values + // exec = backup of user shader's v[0:1] + s_mov_b64 exec, ttmp[2:3] // stash into EXEC to free up ttmp + + // Calculate the base address of the correct sample buffer (buffer0 or buffer1). + // The buffers are located after the pcs_sampling_data_t struct header. + // Address = (TMA + SAMPLE_OFF_BYTES_PER_SAMPLE) + (buffer_id * buf_size * 64) + s_mul_i32 ttmp2, ttmp5, ttmp6 // low 32 bits + s_mul_hi_u32 ttmp3, ttmp5, ttmp6 // high 32 bits + + // Multiply by 64 bytes per sample slot (shift left by 6 bits) + // This converts from units of samples to units of bytes + s_lshl_b64 ttmp[2:3], ttmp[2:3], 6 + s_add_u32 ttmp2, ttmp2, SAMPLE_OFF_BYTES_PER_SAMPLE + s_addc_u32 ttmp3, ttmp3, 0 + s_add_u32 ttmp4, ttmp14, ttmp2 // ttmp4 = TMA_base_lo + total_offset_lo. This is low part of &bufferX + s_addc_u32 ttmp5, ttmp15, ttmp3 // ttmp5 = TMA_base_hi + total_offset_hi + carry. This is high part of &bufferX + // ttmp[4:5] now correctly points to the base of the selected sample buffer array + + s_bitcmp1_b32 ttmp13, TTMP13_HT_FLAG_BIT // if ttmp13.b22==1, this is hosttrap + s_cbranch_scc1 .fill_sample_ht + s_bitcmp1_b32 ttmp13, TTMP13_STOCH_FLAG_BIT + s_cbranch_scc1 .fill_sample_stoch + + s_mov_b64 ttmp[2:3], exec // Restore user v[0:1] backup to ttmp[2:3] + v_readlane_b32 ttmp4, v2, 0 // Backup user v[2:3] to ttmp[4:5] for restore. + v_readlane_b32 ttmp5, v3, 0 + s_branch .restore_vector_before_exit_trap + +.fill_sample_ht: + // At this point, v[0:1] is local_entry (but v1 is 0) + // v[2:3] is original user-data + // ttmp[2:3] is free + // ttmp[4:5] holds &buffer + // ttmp6 holds buf_to_use + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // [ttmp14:15]=‘tma’, ttmp13.b31 = buf_to_use + // EXEC holds holds backup of original shader’s v[0:1] + + v_readlane_b32 ttmp6, v0, 0 // ttmp6=local_entry + s_mul_i32 ttmp2, ttmp6, SAMPLE_OFF_BYTES_PER_SAMPLE // into buffer for 64B objects + s_mul_hi_u32 ttmp3, ttmp6, SAMPLE_OFF_BYTES_PER_SAMPLE // ttmp[2:3] now holds the offset + s_add_u32 ttmp2, ttmp2, ttmp4 + s_addc_u32 ttmp3, ttmp3, ttmp5 // ttmp[2:3]=&bufferX[local_entry] + v_readlane_b32 ttmp4, v2, 0x0 // ttmp[4:5] now holds backup of + v_readlane_b32 ttmp5, v3, 0x0 // user-data from v[2:3] + v_writelane_b32 v0, ttmp2, 0x0 + v_writelane_b32 v1, ttmp3, 0x0 // v[0:1]=&buffer[local_entry] + + s_sendmsg_rtn_b64 ttmp[2:3], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0 // Wait for timestamp + + // v[0:1] = &buffer[local_entry] + // v[2:3] = free + // ttmp[2:3] holds the thing we want to store + // ttmp[4:5] holds backup of original shaders v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shaders [exec_lo,exec_hi] + // ttmp[14:15]=tma, ttmp13.b31 = buf_to_use + // EXEC holds backup of original shaders v[0:1] + + v_writelane_b32 v2, ttmp2, 0 // bring output data to v[2:3] + v_writelane_b32 v3, ttmp3, 0 + + s_mov_b64 ttmp[2:3], exec // vector stores need EXEC set + s_mov_b64 exec, 1 // so ttmp[2:3] holds it for now + + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_TIMESTAMP, scope:SCOPE_SYS // store out timestamp + + // v[0:1] = &buffer[local_entry] + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=‘tma’, ttmp13.b31 = buf_to_use + // EXEC is 0x1 + + s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK // Clear out extra data from PC_HI + v_writelane_b32 v2, ttmp0, 0 + v_writelane_b32 v3, ttmp1, 0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_PC_HOST, scope:SCOPE_SYS // store out PC + + v_writelane_b32 v2, ttmp10, 0 + v_writelane_b32 v3, ttmp11, 0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_EXEC_LOHI, scope:SCOPE_SYS // store out original EXEC + + // Store Workgroup ID X and Y at offset SAMPLE_OFF_WGID_XY (0x10). + // ttmp9 = WGID_X (from first-level handler). + // ttmp7 contains WGID_Y in high 16 bits. + v_writelane_b32 v2, ttmp9, 0 // wg_id_x + s_bfe_u32 ttmp6, ttmp7, (16<<16) // extract bits 15:0, wg_id_y + v_writelane_b32 v3, ttmp6, 0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_WGID_XY, scope:SCOPE_SYS // store wg_id_x and wg_id_y + + // Store Workgroup ID Z and Wave ID at offset SAMPLE_OFF_WGID_Z_WAVE (0x18). + // ttmp7 contains WGID_Z in low 16 bits. + // ttmp11 contains Wave ID in low 6 bits (from EXEC_hi). + s_bfe_u32 ttmp6, ttmp7, (16|16<<16) // extract bits 31:16, wg_id_z + v_writelane_b32 v2, ttmp6, 0 + v_writelane_b32 v3, ttmp8, 0x0 // wave_in_wg is bits 29:25 + v_lshrrev_b32 v3, 25, v3 // Shift wave_in_wg to 4:0 + v_and_b32 v3, v3, WAVE_ID_MASK // put (ttmp8>>25)&0x1f into v3 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_WGID_Z_WAVE, scope:SCOPE_SYS // store wg_id_z and wave_id + + // v[0:1] = &buffer[local_entry] + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=‘tma’, ttmp13.b31 = buf_to_use + // EXEC is 0x1 + // Get HW_ID1 & 2 with S_GETREG_B32 with size=32 (F8 in upper bits), offset=0, and: + // HW_ID1 = 23 (0x17), HW_ID2 = 24 (0x18) + + STORE_HW_ID + + // The following is still true as we get ready to jump to correlation ID check + // v[0:1] = &buffer[local_entry] + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15=‘tma’, ttmp13.b31 = buf_to_use + // EXEC is 0x1 + + STORE_CORRELATION_ID + // Ensure all stores have completed before returning and incrementing written_val + s_wait_storecnt 0 + + // Still true after returning back from correlation ID check + // v[0:1] = &buffer[local_entry], but we no longer need it + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=‘tma’, ttmp13.b31 = buf_to_use + // EXEC is 0x1 + // + s_branch .ret_from_fill_sample + +.fill_sample_stoch: + // v0 contains local_entry, v1 is free + // v[2:3] is original user-data + // ttmp[2:3] is free + // ttmp[4:5] holds &buffer + // ttmp6 holds buf_to_use + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // [ttmp14:15]=‘tma’, ttmp13.b31 = buf_to_use + // EXEC holds holds backup of original shader’s v[0:1] + + v_readlane_b32 ttmp6, v0, 0x0 // ttmp2=local_entry + s_mul_i32 ttmp2, ttmp6, SAMPLE_OFF_BYTES_PER_SAMPLE // into buffer for 64B objects + s_mul_hi_u32 ttmp3, ttmp6, SAMPLE_OFF_BYTES_PER_SAMPLE // ttmp[2:3] now holds the offset + s_add_u32 ttmp2, ttmp2, ttmp4 + s_addc_u32 ttmp3, ttmp3, ttmp5 // ttmp[2:3]=&bufferX[local_entry] + v_readlane_b32 ttmp4, v2, 0x0 // ttmp[4:5] now holds backup of + v_readlane_b32 ttmp5, v3, 0x0 // user-data from v[2:3] + v_writelane_b32 v0, ttmp2, 0x0 + v_writelane_b32 v1, ttmp3, 0x0 // v[0:1]=&buffer[local_entry] + s_sendmsg_rtn_b64 ttmp[2:3], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0 // Wait for timestamp + + // v[0:1] = &buffer[local_entry] + // v[2:3] = free + // ttmp[2:3] holds the thing we want to store + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=‘tma’, ttmp13.b31 = buf_to_use + // EXEC holds backup of original shader’s v[0:1] + + v_writelane_b32 v2, ttmp2, 0 // bring output data to v[2:3] + v_writelane_b32 v3, ttmp3, 0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_TIMESTAMP, scope:SCOPE_SYS // store out timestamp + + // v[0:1] = &buffer[local_entry] + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=‘tma’, ttmp13.b31 = buf_to_use + // EXEC is 0x1 + v_writelane_b32 v2, ttmp10, 0 + v_writelane_b32 v3, ttmp11, 0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_EXEC_LOHI, scope:SCOPE_SYS // store out original EXEC + v_writelane_b32 v2, ttmp9, 0 // wg_id_x + s_bfe_u32 ttmp6, ttmp7, (0 | (16 << 16)) // extract bits 15:0, wg_id_y + v_writelane_b32 v3, ttmp6, 0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_WGID_XY, scope:SCOPE_SYS // store wg_id_x and wg_id_y + s_bfe_u32 ttmp6, ttmp7, (16|16<<16) // extract bits 31:16, wg_id_z + v_writelane_b32 v2, ttmp6, 0 // put wg_id_z in v2 + v_writelane_b32 v3, ttmp8, 0x0 // wave_in_wg is bits 29:25 + + v_lshrrev_b32 v3, 25, v3 // Shift wave_in_wg to 4:0 + + v_and_b32 v3, v3, WAVE_ID_MASK // put (ttmp8>>25)&0x1f into v3 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_WGID_Z_WAVE, scope:SCOPE_SYS // store wg_id_z and wave_id + + STORE_HW_ID + + //Read SNAPSHOT Data + s_getreg_b32 ttmp6, HW_REG_SQ_PERF_SNAPSHOT_DATA1 + v_writelane_b32 v2, ttmp6, 0x0 + s_getreg_b32 ttmp6, HW_REG_SQ_PERF_SNAPSHOT_DATA2 + v_writelane_b32 v3, ttmp6, 0x0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_SNAPSHOT_DATA + 4, scope:SCOPE_SYS // store snapshot DATA1 and DATA2 + + s_getreg_b32 ttmp2, HW_REG_SQ_PERF_SNAPSHOT_DATA + v_writelane_b32 v2, ttmp2, 0 + global_store_b32 v[0:1], v2, off, offset:SAMPLE_OFF_SNAPSHOT_DATA, scope:SCOPE_SYS // store perf snapshot DATA + + s_getreg_b32 ttmp6, HW_REG_SQ_PERF_SNAPSHOT_PC_LO + v_writelane_b32 v2, ttmp6, 0x0 + s_getreg_b32 ttmp6, HW_REG_SQ_PERF_SNAPSHOT_PC_HI + v_writelane_b32 v3, ttmp6, 0x0 + global_store_b64 v[0:1], v[2:3], off, offset:SAMPLE_OFF_PC_HOST, scope:SCOPE_SYS // store PC_HI:PC_LO + + // The following is still true as we get ready to jump to correlation ID check + // v[0:1] = &buffer[local_entry] + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=tma, ttmp13.b31 tells us buf_to_use + // EXEC is 0x1 + + STORE_CORRELATION_ID + // Ensure all stores have completed before returning and incrementing written_val + s_wait_storecnt 0 + +.ret_from_fill_sample: + // v[0:1] = free + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=‘tma’, ttmp13.b31 tells us buf_to_use + // EXEC is 0x1 + + // Sample data has been written to the device buffer. + // Now, atomically increment the count of written samples for the current buffer. + // This is pcs_sampling_data_t.buf_written_val0 or buf_written_val1. + s_lshr_b32 ttmp6, ttmp13, 31 // ttmp6 is buf_to_use + s_mulk_i32 ttmp6, 0x10 // ttmp6=offset from + // written_val0 to written_val_X + s_add_u32 ttmp14, ttmp14, ttmp6 // now ttmp[14:15] points to base for + s_addc_u32 ttmp15, ttmp15, 0 // buf_written_valX atomic operation + + // Atomically increment the chosen buf_written_val. + // v0 = 0 (value to add - low part), v1 = 1 (value to add - high part, effectively just adding 1 to uint32_t) + + v_mov_b32 v0, 0 // want to atomic increment + v_mov_b32 v1, 1 // buf_written_valX + global_atomic_add_u32 v0, v0, v1, ttmp[14:15], offset:SAMPLE_OFF_BUF_WRITTEN_VAL, scope:SCOPE_SYS th:TH_ATOMIC_RETURN + s_wait_loadcnt 0 + + // v0 = done, v1 = free, v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=buf_written_valX-0x10, EXEC=0x1 + // Check Watermark and Signal Host + + s_mov_b64 exec, ttmp[4:5] // stash user’s v[2:3] in EXEC + s_load_b32 ttmp5, ttmp[14:15], 0x14, scope:SCOPE_CU // load watermark into ttmp5 + v_readlane_b32 ttmp4, v0, 0 // put done into ttmp4 + s_wait_kmcnt 0 // wait for watermark to load + s_cmp_lg_u32 ttmp4, ttmp5 // if done != watermark, exit + s_add_u32 ttmp4, ttmp4, 1 // ttmp4 is now current_sample_count (count_before_inc + 1) + s_cmp_lt_u32 ttmp4, ttmp5 // if (current_sample_count < watermark), don't signal + s_mov_b64 ttmp[4:5], exec // restore user’s v[2:3] + s_mov_b64 exec, 1 + s_cbranch_scc1 .restore_vector_before_exit_trap + +.send_signal: + // v[0:3] = free, ttmp[2:5] = backups of original v[0:3], ttmp6=free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=buf_written_valX-0x10, EXEC=old copy of original shader v[2:3] + // write done-signal and optional interrupt + + // Watermark reached or exceeded. Signal the host. + // Load the hsa_signal_t handle for the current buffer. + // done_sig0 is at offset 0x18. done_sig1 is at 0x28. + // addr = ttmp[14:15] + 0x18 + (buffer_id * 0x10). + // ttmp0 still holds buffer_id * 0x10. + + s_load_b64 ttmp[14:15], ttmp[14:15], SAMPLE_OFF_DONE_SIG0, scope:SCOPE_CU // load done_sig into ttmp[14:15] + s_mov_b64 exec, 1 + s_wait_kmcnt 0 + + v_mov_b32 v0, 0 + v_mov_b32 v1, 0 // value to store into v[0:1] + v_writelane_b32 v2, ttmp14, 0 + v_writelane_b32 v3, ttmp15, 0 // Put signal address into v[2:3] + global_store_b64 v[2:3], v[0:1], off, offset:SAMPLE_OFF_SIGNAL_VALUE, scope:SCOPE_SYS // zero out signal value + + s_load_b32 ttmp6, ttmp[14:15], 0x18, scope:SCOPE_CU // load event_id into ttmp6 + s_load_b64 ttmp[14:15], ttmp[14:15], SAMPLE_OFF_EVENT_MAILBOX0, scope:SCOPE_CU // load event mailbox ptr into 14:15 + s_wait_kmcnt 0 + + s_cmp_eq_u64 ttmp[14:15], 0 // null mailbox means no interrupt + s_cbranch_scc1 .restore_vector_before_exit_trap + s_cmp_eq_u32 ttmp6, 0 // event_id zero means no interrupt + s_cbranch_scc1 .restore_vector_before_exit_trap + v_writelane_b32 v2, ttmp14, 0 + v_writelane_b32 v3, ttmp15, 0 // Put mailbox address into v[2:3] + + s_wait_storecnt 0 + v_writelane_b32 v0, ttmp6, 0x0 // put event_id into v0 + global_store_b32 v[2:3], v0, off, offset:0x0, scope:SCOPE_SYS // Send event ID to the mailbox + s_wait_storecnt 0 + s_mov_b32 ttmp14, m0 // save off m0 + v_readlane_b32 ttmp15, v0, 0 // Put ID into message payload + s_mov_b32 m0, ttmp15 + s_sendmsg sendmsg(MSG_INTERRUPT) // send interrupt message + s_wait_kmcnt 0 + s_mov_b32 m0, ttmp14 // restore m0 + + // v[0:1] = free + // v[2:3] = free + // ttmp[2:3] holds backup of original shader’s v[0:1] + // ttmp[4:5] holds backup of original shader’s v[2:3] + // ttmp6 = free + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=somewhere in tma region, EXEC is junk + +.restore_vector_before_exit_trap: + v_writelane_b32 v2, ttmp4, 0 + v_writelane_b32 v3, ttmp5, 0 + +.lost_sample: + // v0 contains local_entry, v1 is free + // v[2:3] is original user-data + // ttmp[2:3] [local_entry, buf_size] + // ttmp[4:5] = free + // ttmp6=buf_to_use (also in ttmp13.b31) + // ttmp[10:11] holds original shader’s [exec_lo,exec_hi] + // ttmp[14:15]=tma + // EXEC=0x1 + // Restore vector registers before exiting + + s_bitcmp1_b32 ttmp13, TTMP13_STOCH_FLAG_BIT // Check if stochastic sampling + s_cbranch_scc0 .lost_sample_restore // If not, just restore and exit + s_getreg_b32 ttmp6, HW_REG_SQ_PERF_SNAPSHOT_PC_HI // Read PC_HI to release lock + +.lost_sample_restore: + v_writelane_b32 v0, ttmp2, 0 // restore v[0:1] to user data + v_writelane_b32 v1, ttmp3, 0 + s_mov_b64 exec, ttmp[10:11] // restore exec mask .exit_trap: // Restore SQ_WAVE_STATUS. - s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 - s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 - s_setreg_b32 hwreg(HW_REG_STATE_PRIV, 0, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT), ttmp12 - s_lshr_b32 ttmp12, ttmp12, (SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1) - s_setreg_b32 hwreg(HW_REG_STATE_PRIV, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1, 32 - SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT - 1), ttmp12 + s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 + s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 + s_setreg_b32 hwreg(HW_REG_STATE_PRIV, 0, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT), ttmp12 + s_lshr_b32 ttmp12, ttmp12, (SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1) + s_setreg_b32 hwreg(HW_REG_STATE_PRIV, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1, 32 - SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT - 1), ttmp12 - // Return to original (possibly modified) PC. - s_rfe_b64 [ttmp0, ttmp1] + s_rfe_b64 [ttmp0, ttmp1] .parked: - s_trap 0x2 - s_branch .parked + s_trap 0x2 + s_branch .parked // Add s_code_end padding so instruction prefetch always has something to read. .rept (256 - ((. - trap_entry) % 64)) / 4 From 48d3719dba6ca91f597a8179d8b341387ce155e8 Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Tue, 8 Jul 2025 16:06:48 +0800 Subject: [PATCH 04/13] libhsakmt/virtio: add virtio support for libhsakmt This patch adds VirtIO support to the libhsakmt library, enabling communication with AMD GPUs via VirtIO. Details - CMakeLists.txt: Added a new CMakeLists.txt file for the VirtIO component of libhsakmt. - hsakmt_virtio.c/h: Implemented the core VirtIO functionality, including VirtIO GPU device initialization, command execution, and memory management. - virtio_gpu.c/h: Contains the implementation of the VirtIO GPU device, including ioctl handling, shared memory management, and command execution. - hsakmt_virtio_events.c: Implements event handling for VirtIO, such as event creation, destruction, setting, resetting, and querying event states. - hsakmt_virtio_memory.c: Manages memory operations for VirtIO, including memory allocation, freeing, mapping, and unmapping. - hsakmt_virtio_queues.c: Implements queue management for VirtIO, including queue creation, destruction, and updating. - hsakmt_virtio_topology.c: Handles system and node properties for VirtIO. - hsakmt_virtio_vm.c: Manages VM-related operations for VirtIO, such as reserving and dereserving VA space. - include/linux/virtgpu_drm.h: Contains DRM definitions for VirtIO GPU. Key Features - VirtIO GPU Initialization: The library can now initialize a VirtIO GPU device and communicate with it. - Command Execution: Supports executing commands on the VirtIO GPU device. - Memory Management: Provides functions for allocating, freeing, mapping, and unmapping memory for VirtIO operations. - Event Handling: Implements a comprehensive event system for VirtIO. - Queue Management: Allows for creating, destroying, and updating queues on the VirtIO GPU device. - System and Node Properties: Retrieves and manages system and node properties for VirtIO. Signed-off-by: Honglei Huang --- CMakeLists.txt | 11 + libhsakmt/CMakeLists.txt | 2 +- libhsakmt/include/hsakmt/hsakmt_virtio.h | 113 +++ libhsakmt/src/virtio/CMakeLists.txt | 88 ++ libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c | 50 + libhsakmt/src/virtio/hsakmt_virtio_device.c | 33 + libhsakmt/src/virtio/hsakmt_virtio_device.h | 189 ++++ libhsakmt/src/virtio/hsakmt_virtio_events.c | 189 ++++ libhsakmt/src/virtio/hsakmt_virtio_memory.c | 852 ++++++++++++++++++ .../src/virtio/hsakmt_virtio_openclose.c | 132 +++ libhsakmt/src/virtio/hsakmt_virtio_proto.h | 503 +++++++++++ libhsakmt/src/virtio/hsakmt_virtio_queues.c | 253 ++++++ libhsakmt/src/virtio/hsakmt_virtio_topology.c | 342 +++++++ libhsakmt/src/virtio/hsakmt_virtio_vm.c | 113 +++ .../src/virtio/include/linux/virtgpu_drm.h | 279 ++++++ libhsakmt/src/virtio/libhsakmt_virtio.ver | 45 + libhsakmt/src/virtio/virtio_gpu.c | 324 +++++++ libhsakmt/src/virtio/virtio_gpu.h | 118 +++ runtime/hsa-runtime/CMakeLists.txt | 4 + 19 files changed, 3639 insertions(+), 1 deletion(-) create mode 100644 libhsakmt/include/hsakmt/hsakmt_virtio.h create mode 100644 libhsakmt/src/virtio/CMakeLists.txt create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_device.c create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_device.h create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_events.c create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_memory.c create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_openclose.c create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_proto.h create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_queues.c create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_topology.c create mode 100644 libhsakmt/src/virtio/hsakmt_virtio_vm.c create mode 100644 libhsakmt/src/virtio/include/linux/virtgpu_drm.h create mode 100644 libhsakmt/src/virtio/libhsakmt_virtio.ver create mode 100644 libhsakmt/src/virtio/virtio_gpu.c create mode 100644 libhsakmt/src/virtio/virtio_gpu.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ba8a551507..d375db8b8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,12 +108,20 @@ if (HSA_DEP_ROCPROFILER_REGISTER) string(APPEND CPACK_RPM_BINARY_PACKAGE_REQUIRES " rocprofiler-register") endif() +if (NOT DEFINED BUILD_THUNK_VIRTIO) + set(BUILD_THUNK_VIRTIO OFF) +endif() + add_rocm_subdir(libhsakmt "${THUNK_DEFINITIONS}") set_target_properties(hsakmt PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/archive" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/lib" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/libhsakmt/runtime") +if (BUILD_THUNK_VIRTIO) + add_rocm_subdir(libhsakmt/src/virtio "${THUNK_VIRTIO_DEFINITIONS}") +endif() + if (BUILD_ROCR) add_rocm_subdir(runtime/hsa-runtime "${ROCR_DEFINITIONS}") set_target_properties(hsa-runtime64 PROPERTIES @@ -123,6 +131,9 @@ if (BUILD_ROCR) if (BUILD_SHARED_LIBS) add_dependencies(hsa-runtime64 hsakmt) + if (BUILD_THUNK_VIRTIO) + add_dependencies(hsa-runtime64 hsakmt_virtio) + endif() else() add_dependencies(hsa-runtime64 hsakmt-staticdrm) endif() diff --git a/libhsakmt/CMakeLists.txt b/libhsakmt/CMakeLists.txt index 8b0ad114af..6849de8721 100644 --- a/libhsakmt/CMakeLists.txt +++ b/libhsakmt/CMakeLists.txt @@ -211,7 +211,7 @@ install ( TARGETS ${HSAKMT_TARGET} # Install public headers install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/${HSAKMT_TARGET} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - COMPONENT dev PATTERN "linux" EXCLUDE ) + COMPONENT dev PATTERN "linux" EXCLUDE PATTERN "*virtio*" EXCLUDE) # Record our usage data for clients find_package calls. install ( EXPORT ${HSAKMT_TARGET}Targets diff --git a/libhsakmt/include/hsakmt/hsakmt_virtio.h b/libhsakmt/include/hsakmt/hsakmt_virtio.h new file mode 100644 index 0000000000..6a2aa6554e --- /dev/null +++ b/libhsakmt/include/hsakmt/hsakmt_virtio.h @@ -0,0 +1,113 @@ +/* + * Copyright © 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef HSAKMT_VIRTIO_H +#define HSAKMT_VIRTIO_H + +#include "hsakmt/linux/kfd_ioctl.h" +#include "hsakmt/hsakmt.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +HSAKMT_STATUS HSAKMTAPI vhsaKmtOpenKFD(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCloseKFD(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtAllocMemory(HSAuint32 PreferredNode, HSAuint64 SizeInBytes, + HsaMemFlags MemFlags, void** MemoryAddress); +HSAKMT_STATUS HSAKMTAPI vhsaKmtFreeMemory(void* MemoryAddress, HSAuint64 SizeInBytes); +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPUNodes(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU, + HsaMemMapFlags MemMapFlags, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray); +HSAKMT_STATUS HSAKMTAPI vhsaKmtUnmapMemoryToGPU(void* MemoryAddress); +HSAKMT_STATUS HSAKMTAPI vhsaKmtAvailableMemory(HSAuint32 Node, HSAuint64* AvailableBytes); +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPU(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterMemoryWithFlags(void* MemoryAddress, + HSAuint64 MemorySizeInBytes, + HsaMemFlags MemFlags); +HSAKMT_STATUS HSAKMTAPI vhsaKmtDeregisterMemory(void* MemoryAddress); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetVersion(HsaVersionInfo* v); +HSAKMT_STATUS HSAKMTAPI vhsaKmtAcquireSystemProperties(HsaSystemProperties* SystemProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtReleaseSystemProperties(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeProperties(HSAuint32 NodeId, + HsaNodeProperties* NodeProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetXNACKMode(HSAint32* enable); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeEnable(void* rDebug, bool setupTtmp); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeDisable(void); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAuint32 NumBanks, + HsaMemoryProperties* MemoryProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeCacheProperties(HSAuint32 NodeId, HSAuint32 ProcessorId, + HSAuint32 NumCaches, + HsaCacheProperties* CacheProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks, + HsaIoLinkProperties* IoLinkProperties); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetClockCounters(HSAuint32 NodeId, HsaClockCounters* Counters); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetAMDGPUDeviceHandle(HSAuint32 NodeId, + HsaAMDGPUDeviceHandle* DeviceHandle); +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryPointerInfo(const void* Pointer, HsaPointerInfo* PointerInfo); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig* config); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateEvent(HsaEventDescriptor* EventDesc, _Bool ManualReset, + _Bool IsSignaled, HsaEvent** Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyEvent(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetEvent(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtResetEvent(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryEventState(HsaEvent* Event); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent(HsaEvent* Event, HSAuint32 Milliseconds); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent_Ext(HsaEvent* Event, HSAuint32 Milliseconds, + uint64_t* event_age); +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents_Ext(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds, + uint64_t* event_age); +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetTrapHandler(HSAuint32 NodeId, void* TrapHandlerBaseAddress, + HSAuint64 TrapHandlerSizeInBytes, + void* TrapBufferBaseAddress, + HSAuint64 TrapBufferSizeInBytes); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueueExt(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, + HSA_QUEUE_PRIORITY Priority, HSAuint32 SdmaEngineId, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource); +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueue(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource); +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyQueue(HSA_QUEUEID QueueId); +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterGraphicsHandleToNodes( + HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo* GraphicsResourceInfo, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray); +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetRuntimeCapabilities(HSAuint32* caps_mask); + +int vamdgpu_query_gpu_info(amdgpu_device_handle dev, void* out); + +#ifdef __cplusplus +} +#endif + +#endif /* HSAKMT_VIRTIO_H */ diff --git a/libhsakmt/src/virtio/CMakeLists.txt b/libhsakmt/src/virtio/CMakeLists.txt new file mode 100644 index 0000000000..c0c7a23c4c --- /dev/null +++ b/libhsakmt/src/virtio/CMakeLists.txt @@ -0,0 +1,88 @@ + +# Copyright 2025 Advanced Micro Devices, Inc. + +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +cmake_minimum_required ( VERSION 3.7 ) + +set (CMAKE_VERBOSE_MAKEFILE ON) + +set ( HSAKMT_VIRTIO "hsakmt_virtio" ) +set ( HSAKMT_VIRTIO_TARGET "${HSAKMT_VIRTIO}" ) + +project ( ${HSAKMT_VIRTIO_TARGET} VERSION 1.0) + +## Compiler flags +set ( HSAKMT_VIRTIO_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden ) +if ( CMAKE_COMPILER_IS_GNUCC ) + set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -Wlogical-op ) +endif () + +set ( HSAKMT_VIRTIO_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/libhsakmt_virtio.ver" ) + +set ( HSAKMT_VIRTIO_LINK_FLAGS "-Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_VIRTIO_LINKER_SCRIPT} -Wl,-z,nodelete") + +if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) + set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -O2 ) +else () + set ( HSAKMT_VIRTIO_C_FLAGS "${HSAKMT_VIRTIO_C_FLAGS}" -g ) +endif () + +set ( HSAKMT_VIRTIO_SRC "virtio_gpu.c" + "hsakmt_virtio_vm.c" + "hsakmt_virtio_device.c" + "hsakmt_virtio_memory.c" + "hsakmt_virtio_amdgpu.c" + "hsakmt_virtio_events.c" + "hsakmt_virtio_queues.c" + "hsakmt_virtio_topology.c" + "hsakmt_virtio_openclose.c" + "../rbtree.c" ) + +add_library ( ${HSAKMT_VIRTIO_TARGET} STATIC ${HSAKMT_VIRTIO_SRC} ) + +target_sources ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${HSAKMT_VIRTIO_SRC} ) + +target_compile_options ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${HSAKMT_VIRTIO_C_FLAGS} ) + +target_include_directories ( ${HSAKMT_VIRTIO_TARGET} + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/virtio + ${CMAKE_CURRENT_SOURCE_DIR}/../ + ${CMAKE_CURRENT_SOURCE_DIR}/../../include + ${CMAKE_CURRENT_SOURCE_DIR}/include/linux ) + +set_property(TARGET ${HSAKMT_VIRTIO_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_VIRTIO_LINK_FLAGS}) + +find_package ( PkgConfig ) + +## If environment variable DRM_DIR is set, the script +## will pick up the corresponding libraries from that path. +list ( PREPEND CMAKE_PREFIX_PATH "${DRM_DIR}" ) + +pkg_check_modules ( DRM REQUIRED IMPORTED_TARGET libdrm ) +pkg_check_modules ( DRM_AMDGPU REQUIRED IMPORTED_TARGET libdrm_amdgpu ) +target_include_directories ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${DRM_AMDGPU_INCLUDE_DIRS} ) +target_include_directories ( ${HSAKMT_VIRTIO_TARGET} PRIVATE ${DRM_INCLUDE_DIRS} ) + +target_link_libraries ( ${HSAKMT_VIRTIO_TARGET} + PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt c ${CMAKE_DL_LIBS} ) diff --git a/libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c b/libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c new file mode 100644 index 0000000000..4aca737221 --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_amdgpu.c @@ -0,0 +1,50 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +int vamdgpu_query_gpu_info(amdgpu_device_handle handle, void* out) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GPU_INFO, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + int ret = vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + if (!ret) memcpy(out, &rsp->gpu_info, sizeof(struct amdgpu_gpu_info)); + + return ret; +} + +HSAKMT_STATUS vhsaKmtGetAMDGPUDeviceHandle(HSAuint32 NodeId, HsaAMDGPUDeviceHandle* DeviceHandle) { + CHECK_VIRTIO_KFD_OPEN(); + + return HSAKMT_STATUS_SUCCESS; +} diff --git a/libhsakmt/src/virtio/hsakmt_virtio_device.c b/libhsakmt/src/virtio/hsakmt_virtio_device.c new file mode 100644 index 0000000000..4e7aed500a --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_device.c @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt_virtio_device.h" + +int vhsakmt_execbuf_cpu(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, const char* from) { + return virtio_gpu_exec_cmd(dev->vgdev, req, true); +} + +void* vhsakmt_alloc_rsp(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, uint32_t sz) { + return virtio_gpu_alloc_rsp(dev->vgdev, req, sz); +} diff --git a/libhsakmt/src/virtio/hsakmt_virtio_device.h b/libhsakmt/src/virtio/hsakmt_virtio_device.h new file mode 100644 index 0000000000..ac53fba263 --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_device.h @@ -0,0 +1,189 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef HSAKMT_VIRTIO_DEVICE_H +#define HSAKMT_VIRTIO_DEVICE_H + +#include "hsakmt_virtio_proto.h" +#include "rbtree.h" +#include "virtio_gpu.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define vhsakmt_atomic_inc_return(ptr) (atomic_fetch_add((ptr), 1) + 1) +#define vhsakmt_atomic_dec_return(ptr) (atomic_fetch_sub((ptr), 1) - 1) + +#define VHSA_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr)) +#define VHSA_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v)) + +extern int vhsakmt_debug_level; +#define vhsakmt_print(level, fmt, ...) \ + do { \ + if (level <= vhsakmt_debug_level) fprintf(stderr, fmt, ##__VA_ARGS__); \ + } while (0) +#define VHSAKMT_DEBUG_LEVEL_DEFAULT -1 +#define VHSAKMT_DEBUG_LEVEL_ERR 3 +#define VHSAKMT_DEBUG_LEVEL_WARNING 4 +#define VHSAKMT_DEBUG_LEVEL_INFO 6 +#define VHSAKMT_DEBUG_LEVEL_DEBUG 7 +#define vhsa_err(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_ERR, fmt, ##__VA_ARGS__) +#define vhsa_warn(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_WARNING, fmt, ##__VA_ARGS__) +#define vhsa_info(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__) +#define vhsa_debug(fmt, ...) vhsakmt_print(VHSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__) + +struct vhsakmt_device; +struct vhsakmt_bo; + +typedef struct vhsakmt_device* vhsakmt_device_handle; +typedef struct vhsakmt_bo* vhsakmt_bo_handle; +typedef rbtree_node_t* bo_entry; + +extern pthread_mutex_t dev_mutex; +extern vhsakmt_device_handle dev_list; + +#define VHSA_BO_KFD_MEM 1 << 0 /* allocated from KFD (hsaKmtAllocMemory) */ +#define VHSA_BO_USERPTR 1 << 1 +#define VHSA_BO_QUEUE_BUFFER 1 << 2 /* allocated from KFD, but used for queue CMD submit */ +#define VHSA_BO_QUEUE_DOORBELL 1 << 3 /* doorbell memory */ +#define VHSA_BO_QUEUE_RW_PTR 1 << 4 /* queue read write ptr, from host map to guest*/ +/* allocated from KFD, but used for AQL queue read write ptr */ +#define VHSA_BO_QUEUE_AQL_RW_PTR 1 << 5 +#define VHSA_BO_CLGL 1 << 6 /* CLGL memory, imported from mesa GL */ +/* allocated from KFD, but is scratch memory, do not need map and unmap in ioctrl */ +#define VHSA_BO_SCRATCH 1 << 7 +#define VHSA_BO_QUEUE 1 << 8 +#define VHSA_BO_EVENT 1 << 9 +#define VHSA_BO_SCRATCH_MAP 1 << 10 + +#define VHSA_SDMA_NONE UINT32_MAX + +#define CHECK_VIRTIO_KFD_OPEN() \ + do { \ + if (dev_list == NULL) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; \ + } while (0) + +struct vhsakmt_node { + HsaNodeProperties node_props; + void* doorbell_base; + uint64_t scratch_start; + uint64_t scratch_size; +}; + +struct vhsakmt_device { + struct virtio_gpu_device* vgdev; + int refcount; + pthread_mutex_t bo_handles_mutex; + rbtree_t bo_rbt; + + struct vhsakmt_bo* shmem_bo; + + uint32_t reqbuf_max; + uint32_t next_blob_id; + + uint64_t vm_start; + uint64_t vm_size; + + pthread_mutex_t vhsakmt_mutex; + struct vhsakmt_node* vhsakmt_nodes; + HsaSystemProperties* sys_props; +}; + +struct vhsakmt_bo { + rbtree_node_t rbtn; + struct vhsakmt_device* dev; + + int refcount; + unsigned size; + void* cpu_addr; + void* host_addr; + HsaMemFlags flags; + uint32_t bo_type; + uint32_t blob_id; + pthread_mutex_t map_mutex; + + union { + struct { + uint32_t handle; + uint32_t res_id; + uint64_t offset; + uint64_t alloc_size; + int map_count; + } real; + }; + + vHsaEvent* event; + uint64_t queue_id; + vhsakmt_bo_handle rw_bo; + void* gl_meta_data; +}; + +/*hsakmt_virtio_memory.c*/ +vhsakmt_bo_handle vhsakmt_entry_to_bo_handle(bo_entry e); +bo_entry vhsakmt_bo_handle_to_entry(vhsakmt_bo_handle bo); + +void vhsakmt_insert_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo, void* addr, uint64_t size); +void vhsakmt_remove_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo); +vhsakmt_bo_handle vhsakmt_find_bo_by_addr(vhsakmt_device_handle dev, void* addr); +void* vhsakmt_gpu_va(vhsakmt_device_handle dev, void* va); + +int vhsakmt_bo_cpu_unmap(vhsakmt_bo_handle bo); +int vhsakmt_bo_cpu_map(vhsakmt_bo_handle bo_handle, void** cpu, void* fixed_cpu); +int vhsakmt_create_mappable_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle); +int vhsakmt_bo_free(vhsakmt_device_handle dev, vhsakmt_bo_handle bo); +int vhsakmt_init_host_blob(vhsakmt_device_handle dev, size_t size, uint32_t blob_type, + uint32_t blob_flag, uint32_t blob_id, uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle); + +/*hsakmt_virtio_openclose.c*/ +vhsakmt_device_handle vhsakmt_dev(void); + +/*hsakmt_virtio_vm.c*/ +void* vhsakmt_vm_start(void); +int vhsakmt_reserve_va(uint64_t start, uint64_t size); +void vhsakmt_dereserve_va(uint64_t start, uint64_t size); +void vhsakmt_set_scratch_area(vhsakmt_device_handle dev, uint32_t node, uint64_t start, + uint64_t size); +void vhsakmt_set_vm_area(vhsakmt_device_handle dev, uint64_t start, uint64_t size); +int vhsakmt_set_node_doorbell(vhsakmt_device_handle dev, uint32_t node, void* doorbell); +void* vhsakmt_node_doorbell(vhsakmt_device_handle dev, uint32_t node); +bool vhsakmt_is_scratch_mem(vhsakmt_device_handle dev, void* addr); +bool vhsakmt_is_userptr(vhsakmt_device_handle dev, void* addr); + +/*hsakmt_virtio_device.c*/ +int vhsakmt_execbuf_cpu(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, const char* from); +void* vhsakmt_alloc_rsp(vhsakmt_device_handle dev, struct vhsakmt_ccmd_req* req, uint32_t sz); + +/*hsakmt_virtio_event.c*/ +void* vhsakmt_event_host_handle(HsaEvent* h); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/libhsakmt/src/virtio/hsakmt_virtio_events.c b/libhsakmt/src/virtio/hsakmt_virtio_events.c new file mode 100644 index 0000000000..e7ee49cbed --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_events.c @@ -0,0 +1,189 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +int vhsakmt_debug_level; + +void* vhsakmt_event_host_handle(HsaEvent* h) { return (void*)((vHsaEvent*)h)->event_handle; } + +static inline int32_t vhsakmt_event_res_id(HsaEvent* h) { return ((vHsaEvent*)h)->res_id; } + +static inline vhsakmt_bo_handle vhsakmt_event_bo_handle(HsaEvent* h) { + return (vhsakmt_bo_handle)((vHsaEvent*)h)->bo_handle; +} + +static int vhsakmt_create_event_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + vHsaEvent* vevent_handle, vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, 0, blob_id, VHSA_BO_EVENT, + (void*)vevent_handle->event_handle, bo_handle); + if (r) return r; + + (*bo_handle)->event = vevent_handle; + vevent_handle->bo_handle = (uint64_t)(*bo_handle); + vevent_handle->res_id = (*bo_handle)->real.res_id; + vhsakmt_insert_bo(dev, *bo_handle, vevent_handle, size); + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateEvent(HsaEventDescriptor* EventDesc, _Bool ManualReset, + _Bool IsSignaled, HsaEvent** Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + vhsakmt_bo_handle event_bo; + vHsaEvent* e; + int r; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_CREATE, + .create_args.EventDesc = *EventDesc, + .create_args.ManualReset = ManualReset, + .create_args.IsSignaled = IsSignaled, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) return rsp->ret; + + e = calloc(1, sizeof(vHsaEvent)); + if (!e) return -ENOMEM; + + memcpy(e, &rsp->vevent, sizeof(vHsaEvent)); + + r = vhsakmt_create_event_blob_bo(dev, sizeof(vHsaEvent), req.blob_id, e, &event_bo); + if (r) { + free(e); + return -ENOMEM; + } + + *Event = (HsaEvent*)e; + + vhsa_debug( + "%s: event addr: %p, hw123: %lx, %lx, %x, type: %d, id: %x, host handle: 0x%lx, res id: %d\n", + __FUNCTION__, e, e->event.EventData.HWData1, e->event.EventData.HWData2, + e->event.EventData.HWData3, e->event.EventData.EventType, e->event.EventId, e->event_handle, + event_bo->real.res_id); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyEvent(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_bo* bo; + + if (Event == NULL) return HSAKMT_STATUS_SUCCESS; + + bo = vhsakmt_event_bo_handle(Event); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + return vhsakmt_bo_free(dev, bo); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetEvent(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_SET, + .event_hanele = vhsakmt_event_host_handle(Event), + .res_id = vhsakmt_event_res_id(Event), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtResetEvent(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_RESET, + .event_hanele = vhsakmt_event_host_handle(Event), + .res_id = vhsakmt_event_res_id(Event), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryEventState(HsaEvent* Event) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_QUERY_STATE, + .event_hanele = vhsakmt_event_host_handle(Event), + .res_id = vhsakmt_event_res_id(Event), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds) { + return HSAKMT_STATUS_ERROR; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent(HsaEvent* Event, HSAuint32 Milliseconds) { + return vhsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnEvent_Ext(HsaEvent* Event, HSAuint32 Milliseconds, + uint64_t* event_age) { + return vhsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtWaitOnMultipleEvents_Ext(HsaEvent* Events[], HSAuint32 NumEvents, + bool WaitOnAll, HSAuint32 Milliseconds, + uint64_t* event_age) { + return vhsaKmtWaitOnMultipleEvents(Events, NumEvents, WaitOnAll, Milliseconds); +} diff --git a/libhsakmt/src/virtio/hsakmt_virtio_memory.c b/libhsakmt/src/virtio/hsakmt_virtio_memory.c new file mode 100644 index 0000000000..a4afc1696e --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_memory.c @@ -0,0 +1,852 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +#define VHSA_GL_METADATA_MAX_SIZE (0x50) + +vhsakmt_bo_handle vhsakmt_entry_to_bo_handle(bo_entry e) { return (vhsakmt_bo_handle)e; } +bo_entry vhsakmt_bo_handle_to_entry(vhsakmt_bo_handle bo) { return &bo->rbtn; } +static inline bool vhsakmt_is_mem_bo(vhsakmt_bo_handle bo) { return (!bo->queue_id && !bo->event); } + +static bool vhsakmt_mappable(HsaMemFlags flags) { return (!flags.ui32.Scratch); } + +static bool vhsakmt_bo_mappable(vhsakmt_bo_handle bo) { return vhsakmt_mappable(bo->flags); } + +void vhsakmt_insert_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo, void* addr, uint64_t size) { + bo->rbtn.key.addr = (unsigned long)addr; + bo->rbtn.key.size = (unsigned long)size; + + pthread_mutex_lock(&dev->bo_handles_mutex); + hsakmt_rbtree_insert(&dev->bo_rbt, &bo->rbtn); + pthread_mutex_unlock(&dev->bo_handles_mutex); +} + +static void vhsakmt_remove_entry(vhsakmt_device_handle dev, bo_entry entry) { + if (!entry) return; + + pthread_mutex_lock(&dev->bo_handles_mutex); + hsakmt_rbtree_delete(&dev->bo_rbt, entry); + pthread_mutex_unlock(&dev->bo_handles_mutex); +} + +void vhsakmt_remove_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + bo_entry entry = vhsakmt_bo_handle_to_entry(bo); + if (entry->key.addr == 0 && entry->key.size == 0) return; + + vhsakmt_remove_entry(dev, entry); +} + +static bo_entry vhsakmt_rbt_search(vhsakmt_device_handle dev, void* addr) { + vhsakmt_bo_handle bo; + + rbtree_key_t key = rbtree_key((uint64_t)addr, 0); + pthread_mutex_lock(&dev->bo_handles_mutex); + bo_entry n = rbtree_lookup_nearest(&dev->bo_rbt, &key, LKP_ADDR, RIGHT); + pthread_mutex_unlock(&dev->bo_handles_mutex); + if (n) { + bo = vhsakmt_entry_to_bo_handle(n); + if (bo->cpu_addr != addr) return NULL; + return n; + } + + return NULL; +} + +static bo_entry vhsakmt_find_entry_by_addr(vhsakmt_device_handle dev, void* addr) { + return vhsakmt_rbt_search(dev, addr); +} + +vhsakmt_bo_handle vhsakmt_find_bo_by_addr(vhsakmt_device_handle dev, void* addr) { + bo_entry entry = vhsakmt_find_entry_by_addr(dev, addr); + + if (entry) { + vhsakmt_bo_handle bo = vhsakmt_entry_to_bo_handle(entry); + if (!vhsakmt_is_mem_bo(bo)) return NULL; + + return bo; + } + + return NULL; +} + +void* vhsakmt_gpu_va(vhsakmt_device_handle dev, void* va) { + if (!vhsakmt_is_userptr(dev, va)) return va; + + bo_entry entry = vhsakmt_find_entry_by_addr(dev, va); + + if (!entry) return NULL; + + return vhsakmt_entry_to_bo_handle(entry)->host_addr; +} + +int vhsakmt_bo_cpu_map(vhsakmt_bo_handle bo, void** cpu, void* fixed_cpu) { + int r; + + if (!vhsakmt_bo_mappable(bo)) return 0; + + pthread_mutex_lock(&bo->map_mutex); + + if (!bo->cpu_addr) { + r = virtio_gpu_map_handle(bo->dev->vgdev, bo->real.handle, bo->size, cpu, fixed_cpu); + if (r) { + pthread_mutex_unlock(&bo->map_mutex); + return r; + } + bo->cpu_addr = *cpu; + atomic_fetch_add(&bo->real.map_count, 1); + } + pthread_mutex_unlock(&bo->map_mutex); + + return *cpu == MAP_FAILED; +} + +int vhsakmt_bo_cpu_unmap(vhsakmt_bo_handle bo) { + int r = 0; + + if (!vhsakmt_bo_mappable(bo)) return 0; + + pthread_mutex_lock(&bo->map_mutex); + + if (!bo->cpu_addr || bo->real.map_count == 0) { + pthread_mutex_unlock(&bo->map_mutex); + return 0; + } + + if (vhsakmt_atomic_dec_return(&bo->real.map_count) <= 0) { + if (bo->bo_type & VHSA_BO_KFD_MEM) { + virtio_gpu_unmap(bo->cpu_addr, bo->size); + vhsakmt_reserve_va(VHSA_VPTR_TO_UINT64(bo->cpu_addr), bo->size); + bo->cpu_addr = NULL; + } + } + + pthread_mutex_unlock(&bo->map_mutex); + return r; +} + +static int vhsakmt_destroy_handle(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + int r = virtio_gpu_destroy_handle(dev->vgdev, bo->real.handle); + free(bo); + + return r; +} + +int vhsakmt_init_host_blob(vhsakmt_device_handle dev, size_t size, uint32_t blob_type, + uint32_t blob_flag, uint32_t blob_id, uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + vhsakmt_bo_handle bo; + struct drm_virtgpu_resource_create_blob args = { + .blob_mem = blob_type, + .size = size, + .blob_id = blob_id, + .blob_flags = blob_flag, + }; + + r = virtio_gpu_create_blob(dev->vgdev, &args); + if (r) return -EINVAL; + + bo = calloc(1, sizeof(struct vhsakmt_bo)); + if (!bo) { + virtio_gpu_destroy_handle(dev->vgdev, args.bo_handle); + return -ENOMEM; + } + + bo->dev = dev; + bo->size = size; + bo->real.alloc_size = size; + bo->bo_type = bo_type; + bo->host_addr = va_handle; + pthread_mutex_init(&bo->map_mutex, NULL); + atomic_store(&bo->real.map_count, 0); + atomic_store(&bo->refcount, 1); + bo->real.handle = args.bo_handle; + + virtio_gpu_res_id(dev->vgdev, bo->real.handle, &bo->real.res_id); + + *bo_handle = bo; + return 0; +} + +static int vhsakmt_init_userptr_blob(vhsakmt_device_handle dev, void* addr, size_t size, + vhsakmt_bo_handle* bo_handle, uint64_t* offset) { + int r; + struct drm_virtgpu_resource_create_blob args = { + .blob_mem = VIRTGPU_BLOB_MEM_HOST3D_GUEST, + .blob_flags = VIRTGPU_BLOB_FLAG_USE_USERPTR, + .size = size, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + .blob_userptr = (uint64_t)addr, + }; + + r = virtio_gpu_create_blob(dev->vgdev, &args); + if (r < 0) return r; + + vhsakmt_bo_handle userptr = calloc(1, sizeof(struct vhsakmt_bo)); + if (!userptr) { + virtio_gpu_destroy_handle(dev->vgdev, args.bo_handle); + return -ENOMEM; + } + + userptr->dev = dev; + userptr->size = size; + userptr->real.alloc_size = size; + userptr->bo_type = VHSA_BO_USERPTR; + userptr->cpu_addr = addr; + pthread_mutex_init(&userptr->map_mutex, NULL); + atomic_store(&userptr->real.map_count, 0); + atomic_store(&userptr->refcount, 1); + userptr->real.handle = args.bo_handle; + + virtio_gpu_res_id(dev->vgdev, userptr->real.handle, &userptr->real.res_id); + + *bo_handle = userptr; + *offset = args.offset; + return r; +} + +int vhsakmt_create_mappable_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint32_t bo_type, void* va_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, VIRTGPU_BLOB_FLAG_USE_MAPPABLE, + blob_id, bo_type, va_handle, bo_handle); + if (r) return r; + + r = vhsakmt_bo_cpu_map(*bo_handle, &((*bo_handle)->cpu_addr), va_handle); + if (r) { + free(*bo_handle); + *bo_handle = NULL; + return -EINVAL; + } + + if (va_handle && (va_handle != (*bo_handle)->cpu_addr)) + vhsa_warn("%s: target map: %p != real map: %p\n", __FUNCTION__, va_handle, + (*bo_handle)->cpu_addr); + + vhsakmt_insert_bo(dev, *bo_handle, (*bo_handle)->cpu_addr, (*bo_handle)->size); + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtAllocMemory(HSAuint32 PreferredNode, HSAuint64 SizeInBytes, + HsaMemFlags MemFlags, void** MemoryAddress) { + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + vhsakmt_bo_handle bo; + int r; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_ALLOC, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + .alloc_args = + { + .PreferredNode = PreferredNode, + .SizeInBytes = SizeInBytes, + .MemFlags = MemFlags, + }, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) return rsp->ret; + + if (!rsp->memory_handle) return -ENOMEM; + + r = vhsakmt_init_host_blob(dev, SizeInBytes, VIRTGPU_BLOB_MEM_HOST3D, + vhsakmt_mappable(MemFlags) ? VIRTGPU_BLOB_FLAG_USE_MAPPABLE : 0, + req.blob_id, VHSA_BO_KFD_MEM, (void*)rsp->memory_handle, &bo); + if (r) return r; + + if (!vhsakmt_mappable(MemFlags)) { + bo->cpu_addr = bo->host_addr; + if (MemFlags.ui32.Scratch) { + vhsakmt_set_scratch_area(dev, PreferredNode, (uint64_t)bo->cpu_addr, SizeInBytes); + bo->bo_type |= VHSA_BO_SCRATCH; + } + } else { + r = vhsakmt_bo_cpu_map(bo, &bo->cpu_addr, bo->host_addr); + if (r) { + free(bo); + return -ENOMEM; + } + } + + if (!MemFlags.ui32.Scratch) vhsakmt_insert_bo(dev, bo, bo->cpu_addr, bo->size); + + *MemoryAddress = bo->cpu_addr; + + vhsa_debug("alloc mem addr: %p, host addr: %p, size: %lx, res-id: %d, handble: %d\n", + *MemoryAddress, bo->host_addr, SizeInBytes, bo->real.res_id, bo->real.handle); + + return rsp->ret; +} + +int vhsakmt_bo_free(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + bo_entry entry; + int r; + + if (vhsakmt_atomic_dec_return(&bo->refcount) > 0) return 0; + + entry = vhsakmt_bo_handle_to_entry(bo); + if (entry->key.addr == 0 && entry->key.size == 0) return -EINVAL; + + /* do not free BOs of queue, let them be freed with queue */ + if (bo->bo_type & VHSA_BO_QUEUE_DOORBELL) { + vhsa_err("%s: Try to free VHSA_BO_QUEUE_DOORBELL memory: %p\n", __FUNCTION__, bo->cpu_addr); + return 0; + } + + vhsakmt_remove_bo(dev, bo); + + if (bo->cpu_addr) vhsakmt_bo_cpu_unmap(bo); + + if (bo->event) free(bo->event); + + if (bo->gl_meta_data) free(bo->gl_meta_data); + + pthread_mutex_destroy(&bo->map_mutex); + + r = vhsakmt_destroy_handle(dev, bo); + + return r; +} + +/* Only remove bo in rbtree */ +static void vhsakmt_remove_userptr_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + vhsakmt_remove_bo(dev, bo); + free(bo); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtFreeMemory(void* MemoryAddress, HSAuint64 SizeInBytes) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + vhsa_debug("%s: addr: %p, size: %lx, res_id: %d\n", __FUNCTION__, MemoryAddress, SizeInBytes, + bo->real.res_id); + + return vhsakmt_bo_free(dev, bo); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPUNodes(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU, + HsaMemMapFlags MemMapFlags, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + size_t req_len = + VHSA_ALIGN_UP(sizeof(struct vhsakmt_ccmd_memory_req) + NumberOfNodes * sizeof(*NodeArray), 8); + struct vhsakmt_ccmd_memory_req* req; + struct vhsakmt_ccmd_memory_rsp* rsp; + vhsakmt_bo_handle bo; + + req = (void*)calloc(1, req_len); + if (!req) return -ENOMEM; + req->hdr = VHSAKMT_CCMD(MEMORY, req_len); + req->type = VHSAKMT_CCMD_MEMORY_MAP_TO_GPU_NODES; + req->map_to_GPU_nodes_args.MemorySizeInBytes = MemorySizeInBytes; + req->map_to_GPU_nodes_args.MemMapFlags = MemMapFlags; + req->map_to_GPU_nodes_args.NumberOfNodes = NumberOfNodes; + + bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (bo) { + req->map_to_GPU_nodes_args.MemoryAddress = (uint64_t)bo->host_addr; + if (bo->bo_type & VHSA_BO_USERPTR) vhsakmt_remove_userptr_bo(dev, bo); + } else + req->map_to_GPU_nodes_args.MemoryAddress = (uint64_t)MemoryAddress; + + memcpy(req->payload, NodeArray, NumberOfNodes * sizeof(*NodeArray)); + + rsp = vhsakmt_alloc_rsp(dev, &req->hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) { + free(req); + return -ENOMEM; + } + + vhsakmt_execbuf_cpu(dev, &req->hdr, __FUNCTION__); + + *AlternateVAGPU = rsp->alternate_vagpu; + + vhsa_debug("%s: gva: %p, hva: 0x%lx, size: %lx, AlternateVAGPU: %lx, ret: %d\n", __FUNCTION__, + MemoryAddress, req->map_to_GPU_nodes_args.MemoryAddress, MemorySizeInBytes, + *AlternateVAGPU, rsp->ret); + + free(req); + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtUnmapMemoryToGPU(void* MemoryAddress) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + struct vhsakmt_ccmd_memory_rsp* rsp; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_UNMAP_TO_GPU, + .MemoryAddress = (uint64_t)bo->host_addr, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + vhsa_debug("%s: gva: %p, hva: 0x%lx\n", __FUNCTION__, MemoryAddress, req.MemoryAddress); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtAvailableMemory(HSAuint32 Node, HSAuint64* AvailableBytes) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_AVAIL_MEM, + .Node = Node, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + *AvailableBytes = rsp->available_bytes; + + return rsp->ret; +} + +static int vhsakmt_create_scratch_map_memory(vhsakmt_device_handle dev, void* MemoryAddress, + HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU) { + vhsakmt_bo_handle out; + int r; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), + .map_to_GPU_args = + { + .MemoryAddress = (uint64_t)MemoryAddress, + .MemorySizeInBytes = MemorySizeInBytes, + .need_create_bo = true, + }, + }; + + struct vhsakmt_ccmd_memory_rsp* rsp = + vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + if (rsp->ret) return rsp->ret; + + r = vhsakmt_init_host_blob(dev, MemorySizeInBytes, VIRTGPU_BLOB_MEM_HOST3D, 0, req.blob_id, + VHSA_BO_SCRATCH_MAP, NULL, &out); + if (r) return r; + + // TODO: insert scratch bo into rbtree, or insert it in dev nodes. + + out->cpu_addr = MemoryAddress; + out->host_addr = (void*)rsp->memory_handle; + *AlternateVAGPU = rsp->alternate_vagpu; + + vhsa_debug( + "%s: create scratch memory, gva: %p, memory_handle: 0x%p, alternate_vagpu: %p, size: %lx\n", + __FUNCTION__, MemoryAddress, (void*)rsp->memory_handle, (void*)rsp->alternate_vagpu, + MemorySizeInBytes); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtMapMemoryToGPU(void* MemoryAddress, HSAuint64 MemorySizeInBytes, + HSAuint64* AlternateVAGPU) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo && vhsakmt_is_scratch_mem(dev, MemoryAddress)) + return vhsakmt_create_scratch_map_memory(dev, MemoryAddress, MemorySizeInBytes, AlternateVAGPU); + + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU, + .map_to_GPU_args = + { + .MemoryAddress = bo ? (uint64_t)bo->host_addr : (uint64_t)MemoryAddress, + .MemorySizeInBytes = MemorySizeInBytes, + }, + }; + + if (bo && (bo->bo_type & VHSA_BO_USERPTR)) vhsakmt_remove_userptr_bo(dev, bo); + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + vhsa_debug("%s: gva: %p, hva: 0x%lx, size: %lx\n", __FUNCTION__, MemoryAddress, req.MemoryAddress, + MemorySizeInBytes); + + *AlternateVAGPU = rsp->alternate_vagpu; + + return rsp->ret; +} + +static int vhsakmt_map_userptr(vhsakmt_device_handle dev, void* addr, size_t size, uint32_t res_id, + uint64_t* userptr_handle) { + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_MAP_USERPTR, + .res_id = res_id, + }; + struct vhsakmt_ccmd_memory_rsp* rsp = + vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + rsp->map_userptr_rsp.userptr_handle = 0; + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + *userptr_handle = rsp->map_userptr_rsp.userptr_handle; + return rsp->ret; +} + +static void* vhsakmt_map_to_gpu(void* addr, size_t size) { + vhsakmt_device_handle dev = vhsakmt_dev(); + size_t offset = (uint64_t)addr % getpagesize(); + size_t map_size = (VHSA_ALIGN_UP(size + offset, getpagesize()) / getpagesize()) * getpagesize(); + uint64_t userptr_offset, userptr_handle = 0; + vhsakmt_bo_handle userptr; + int r; + + vhsa_debug("%s: addr: %p, size: 0x%lx, size + offset: 0x%lx, map_size: 0x%lx\n", __FUNCTION__, + addr, size, size + offset, map_size); + + r = vhsakmt_init_userptr_blob(dev, addr, size, &userptr, &userptr_offset); + if (r < 0) { + vhsa_debug("%s: userptr create failed at address: %p, ret = %d\n", __FUNCTION__, addr, r); + return NULL; + } + + vhsakmt_map_userptr(dev, addr, size, userptr->real.res_id, &userptr_handle); + if (!userptr_handle) { + vhsa_debug("%s: map userptr failed at address: %p, ret = %d\n", __FUNCTION__, addr, r); + vhsakmt_destroy_handle(dev, userptr); + vhsakmt_remove_userptr_bo(dev, userptr); + return NULL; + } + userptr->host_addr = VHSA_UINT64_TO_VPTR(VHSA_VPTR_TO_UINT64(userptr_handle) + offset); + + if (r > 0) { + vhsa_debug("%s: userptr: %p already registered, offset: %lx\n", __FUNCTION__, addr, + userptr_offset); + userptr->host_addr = + VHSA_UINT64_TO_VPTR(VHSA_VPTR_TO_UINT64(userptr->host_addr) + userptr_offset); + } + vhsakmt_insert_bo(dev, userptr, userptr->cpu_addr, userptr->size); + + vhsa_debug("%s: real gva: %p, gva: %p, hva: %p, size: %lx, offset: %" PRIu64 + ", map_size: 0x%lx\n", + __FUNCTION__, addr, userptr->cpu_addr, userptr->host_addr, size, offset, map_size); + return userptr->host_addr; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterMemoryWithFlags(void* MemoryAddress, + HSAuint64 MemorySizeInBytes, + HsaMemFlags MemFlags) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_memory_rsp* rsp; + void* addr; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_REG_MEM_WITH_FLAG, + .reg_mem_with_flag = + { + .MemorySizeInBytes = MemorySizeInBytes, + .MemFlags = MemFlags, + }, + }; + + /* no need to register memory from lihsakmt / not a userptr */ + if (!vhsakmt_is_userptr(dev, MemoryAddress)) return HSAKMT_STATUS_SUCCESS; + + addr = vhsakmt_map_to_gpu(MemoryAddress, MemorySizeInBytes); + if (!addr) { + vhsa_debug("%s: register memory failed, gva: %p, size: %lx\n", __FUNCTION__, MemoryAddress, + MemorySizeInBytes); + return HSAKMT_STATUS_ERROR; + } + + req.reg_mem_with_flag.MemoryAddress = (uint64_t)addr; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + return rsp->ret; +} + +static int vhsakmt_remove_clgl_bo(vhsakmt_device_handle dev, vhsakmt_bo_handle bo) { + struct vhsakmt_ccmd_memory_rsp* rsp; + struct vhsakmt_ccmd_memory_req req = { + .hdr = VHSAKMT_CCMD(MEMORY, sizeof(struct vhsakmt_ccmd_memory_req)), + .type = VHSAKMT_CCMD_MEMORY_DEREG_MEM, + .res_id = bo->real.res_id, + .MemoryAddress = (uint64_t)bo->cpu_addr, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_memory_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) vhsa_err("%s: deregister failed clgl memory gva: %p\n", __FUNCTION__, bo->cpu_addr); + + vhsakmt_bo_free(dev, bo); + + vhsa_debug("%s: deregister clgl memory gva: %p, ret: %d\n", __FUNCTION__, bo->cpu_addr, rsp->ret); + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtDeregisterMemory(void* MemoryAddress) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, MemoryAddress); + if (!bo) return HSAKMT_STATUS_SUCCESS; + + vhsa_debug("%s: remove userptr %p size: 0x%lx, res id: %d\n", __FUNCTION__, MemoryAddress, + (size_t)bo->size, bo->real.res_id); + + if (bo->bo_type & VHSA_BO_CLGL) + return vhsakmt_remove_clgl_bo(dev, bo); + else { + vhsakmt_remove_bo(dev, bo); + free(bo); + } + + return 0; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtQueryPointerInfo(const void* Pointer, HsaPointerInfo* PointerInfo) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + void* gpu_va = vhsakmt_gpu_va(dev, VHSA_UINT64_TO_VPTR(Pointer)); + if (!gpu_va) return -HSAKMT_STATUS_ERROR; + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_POINTER_INFO, + .pointer = VHSA_VPTR_TO_UINT64(gpu_va), + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + + QUERY_PTR_INFO_MAX_MAPPED_NODES * sizeof(uint32_t)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(PointerInfo, &rsp->ptr_info, sizeof(HsaPointerInfo)); + + if (PointerInfo->NMappedNodes && PointerInfo->MappedNodes) { + if (PointerInfo->NMappedNodes > QUERY_PTR_INFO_MAX_MAPPED_NODES) { + PointerInfo->NMappedNodes = QUERY_PTR_INFO_MAX_MAPPED_NODES; + vhsa_debug( + "%s: query pointer: %p info mapped nodes greater than QUERY_PTR_INFO_MAX_MAPPED_NODES\n", + __FUNCTION__, Pointer); + } + + PointerInfo->MappedNodes = calloc(PointerInfo->NMappedNodes, sizeof(uint32_t)); + if (!PointerInfo->MappedNodes) { + PointerInfo->NMappedNodes = 0; + return -HSAKMT_STATUS_NO_MEMORY; + } + memcpy(VHSA_UINT64_TO_VPTR(PointerInfo->MappedNodes), rsp->payload, + PointerInfo->NMappedNodes * sizeof(uint32_t)); + } + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig* config) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + uint8_t* config_cpy_addr = NULL; + struct vhsakmt_ccmd_query_info_rsp* rsp; + unsigned req_len = sizeof(struct vhsakmt_ccmd_query_info_req); + unsigned rsp_len = sizeof(struct vhsakmt_ccmd_query_info_rsp) + + config->NumTileConfigs * sizeof(HSAuint32) + config->NumMacroTileConfigs * sizeof(HSAuint32); + + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, req_len), + .type = VHSAKMT_CCMD_QUERY_TILE_CONFIG, + .tile_config_args.NodeId = NodeId, + .tile_config_args.config = *config, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, rsp_len); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(config, &rsp->tile_config_rsp, sizeof(HsaGpuTileConfig)); + config_cpy_addr = ((uint8_t*)rsp->payload); + memcpy(config->TileConfig, config_cpy_addr, config->NumTileConfigs * sizeof(HSAuint32)); + config_cpy_addr += config->NumTileConfigs * sizeof(HSAuint32); + memcpy(config->MacroTileConfig, config_cpy_addr, config->NumMacroTileConfigs * sizeof(HSAuint32)); + + return rsp->ret; +} + +static int vhsakmt_create_clgl_bo(vhsakmt_device_handle dev, void* addr, size_t size, + uint32_t res_id, uint32_t bo_handle, void* meta_data) { + vhsakmt_bo_handle out = calloc(1, sizeof(struct vhsakmt_bo)); + if (!out) return -ENOMEM; + + out->dev = dev; + out->size = size; + atomic_store(&out->real.map_count, 0); + atomic_store(&out->refcount, 1); + +#ifdef CLGL_EXPORT_RESID + out->real.res_id = GraphicsResourceHandle; +#else + out->real.res_id = res_id; +#endif + + /* GL bo handle from GL context*/ + out->real.handle = bo_handle; + out->bo_type |= VHSA_BO_CLGL; + if (meta_data) out->gl_meta_data = meta_data; + + out->host_addr = addr; + + vhsakmt_insert_bo(dev, out, addr, out->size); + + return 0; +} + +static int vhsakmt_gfxhandle_to_resid(vhsakmt_device_handle dev, uint32_t gfx_handle, + uint32_t* res_id, uint32_t* bo_handle) { + int r = drmPrimeFDToHandle(dev->vgdev->fd, gfx_handle, bo_handle); + if (r) { + vhsa_err("%s: drmPrimeFDToHandle failed for handle: %u\n", __FUNCTION__, gfx_handle); + return r; + } + + virtio_gpu_res_id(dev->vgdev, *bo_handle, res_id); + + vhsa_debug("%s: register praphics handle: handle: %d, bo_handle: %d, res_id: %d\n", __FUNCTION__, + gfx_handle, *bo_handle, *res_id); + + return 0; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRegisterGraphicsHandleToNodes( + HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo* GraphicsResourceInfo, + HSAuint64 NumberOfNodes, HSAuint32* NodeArray) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + uint32_t bo_handle, res_id; + uint64_t meta_data_size = VHSA_GL_METADATA_MAX_SIZE; + unsigned req_len = sizeof(struct vhsakmt_ccmd_gl_inter_req) + NumberOfNodes * sizeof(NodeArray); + struct vhsakmt_ccmd_gl_inter_req* req; + struct vhsakmt_ccmd_gl_inter_rsp* rsp; + int r; + + req = calloc(1, req_len); + if (!req) return -ENOMEM; + + req->hdr = VHSAKMT_CCMD(GL_INTER, req_len); + req->type = VHSAKMT_CCMD_GL_REG_GHD_TO_NODES; + req->reg_ghd_to_nodes.NumberOfNodes = NumberOfNodes; + req->reg_ghd_to_nodes.res_handle = GraphicsResourceHandle; + +#ifdef CLGL_EXPORT_RESID + req->reg_ghd_to_nodes.GraphicsResourceHandle = GraphicsResourceHandle; +#else + r = vhsakmt_gfxhandle_to_resid(dev, GraphicsResourceHandle, &res_id, &bo_handle); + if (r) return r; + + req->reg_ghd_to_nodes.GraphicsResourceHandle = bo_handle; + req->reg_ghd_to_nodes.res_handle = res_id; +#endif + + memcpy(req->payload, NodeArray, NumberOfNodes * sizeof(NodeArray)); + + rsp = + vhsakmt_alloc_rsp(dev, &req->hdr, sizeof(struct vhsakmt_ccmd_gl_inter_rsp) + meta_data_size); + if (!rsp) { + r = -ENOMEM; + goto free_out; + } + + vhsakmt_execbuf_cpu(dev, &req->hdr, __FUNCTION__); + if (rsp->ret) return rsp->ret; + + memcpy(GraphicsResourceInfo, &rsp->info, sizeof(HsaGraphicsResourceInfo)); + if (rsp->info.MetadataSizeInBytes) { + GraphicsResourceInfo->Metadata = calloc(1, GraphicsResourceInfo->MetadataSizeInBytes); + if (!GraphicsResourceInfo->Metadata) { + r = -ENOMEM; + goto free_out; + } + + memcpy(VHSA_UINT64_TO_VPTR(GraphicsResourceInfo->Metadata), rsp->payload, + GraphicsResourceInfo->MetadataSizeInBytes); + } else + GraphicsResourceInfo->Metadata = NULL; + + vhsa_debug("%s: register graphics handle: handle: %ld hva: %p, size: %lx\n", __FUNCTION__, + GraphicsResourceHandle, GraphicsResourceInfo->MemoryAddress, + GraphicsResourceInfo->SizeInBytes); + + r = vhsakmt_create_clgl_bo(dev, GraphicsResourceInfo->MemoryAddress, + GraphicsResourceInfo->SizeInBytes, res_id, bo_handle, + VHSA_UINT64_TO_VPTR(GraphicsResourceInfo->Metadata)); + if (r) goto free_out; + + r = rsp->ret; + +free_out: + /* close exported FD after register or close it when deregistre. Close after register here. */ + close(GraphicsResourceHandle); + free(req); + return r; +} diff --git a/libhsakmt/src/virtio/hsakmt_virtio_openclose.c b/libhsakmt/src/virtio/hsakmt_virtio_openclose.c new file mode 100644 index 0000000000..7c8f9c18bb --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_openclose.c @@ -0,0 +1,132 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +pthread_mutex_t dev_mutex = PTHREAD_MUTEX_INITIALIZER; +vhsakmt_device_handle dev_list = NULL; + +vhsakmt_device_handle vhsakmt_dev(void) { return dev_list; } + +static HSAKMT_STATUS vhsakmt_openKFD_cmd(vhsakmt_device_handle dev) { + void* vm_start = vhsakmt_vm_start(); + if (!vm_start) return -HSAKMT_STATUS_NO_MEMORY; + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_OPEN_KFD, + .open_kfd_args = + { + .cur_vm_start = VHSA_VPTR_TO_UINT64(vm_start), + }, + }; + + if (!req.open_kfd_args.cur_vm_start) { + vhsa_err("%s: failed to get current heap start address\n", __FUNCTION__); + return -HSAKMT_STATUS_ERROR; + } + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -HSAKMT_STATUS_NO_MEMORY; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp->open_kfd_rsp.vm_start || !rsp->open_kfd_rsp.vm_size) { + vhsa_err("%s: failed to get KFD VM area\n", __FUNCTION__); + return -HSAKMT_STATUS_ERROR; + } + + vhsakmt_set_vm_area(dev, rsp->open_kfd_rsp.vm_start, rsp->open_kfd_rsp.vm_size); + if (vhsakmt_reserve_va(dev->vm_start, dev->vm_size)) { + vhsa_err("%s: failed to reserve VM area: [%lx-%lx]-0x%lx\n", __FUNCTION__, dev->vm_start, + dev->vm_start + dev->vm_size, dev->vm_size); + return -HSAKMT_STATUS_NO_MEMORY; + } + + vhsa_debug("%s: kfd vm range: [%lx-%lx]-0x%lx\n", __FUNCTION__, dev->vm_start, + dev->vm_start + dev->vm_size, dev->vm_size); + return rsp->ret; +} + +static vhsakmt_device_handle vhsakmt_device_init(void) { + int fd; + vhsakmt_device_handle dev = NULL; + + if (vhsakmt_dev()) return vhsakmt_dev(); + + pthread_mutex_lock(&dev_mutex); + + fd = virtio_gpu_kfd_open(); + if (fd < 0) goto open_failed; + + dev = calloc(1, sizeof(struct vhsakmt_device)); + if (!dev) goto open_failed; + + dev->vgdev = virtio_gpu_init(fd, 0); + if (!dev->vgdev) goto malloc_failed; + + rbtree_init(&dev->bo_rbt); + atomic_store(&dev->next_blob_id, 1); + atomic_store(&dev->refcount, 1); + pthread_mutex_init(&dev->bo_handles_mutex, NULL); + pthread_mutex_init(&dev->vhsakmt_mutex, NULL); + dev_list = dev; + + pthread_mutex_unlock(&dev_mutex); + return dev; + +malloc_failed: + free(dev); + dev = NULL; +open_failed: + pthread_mutex_unlock(&dev_mutex); + return dev; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtOpenKFD(void) { + vhsakmt_device_handle dev; + char* d = getenv("VHSAKMT_DEBUG_LEVEL"); + if (d) vhsakmt_debug_level = atoi(d); + + dev = vhsakmt_device_init(); + if (!dev) return HSAKMT_STATUS_ERROR; + + return vhsakmt_openKFD_cmd(vhsakmt_dev()); +} + +static void vhsakmt_device_destroy(struct vhsakmt_device* dev) { + pthread_mutex_destroy(&dev->bo_handles_mutex); + vhsakmt_dereserve_va(dev->vm_start, dev->vm_size); + + if (dev->sys_props) free(dev->sys_props); + if (dev->vhsakmt_nodes) free(dev->vhsakmt_nodes); + + virtio_gpu_close(dev->vgdev); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCloseKFD(void) { + vhsakmt_device_handle dev = vhsakmt_dev(); + pthread_mutex_lock(&dev_mutex); + if (vhsakmt_atomic_dec_return(&dev->refcount) <= 0) vhsakmt_device_destroy(dev); + pthread_mutex_unlock(&dev_mutex); + return 0; +} diff --git a/libhsakmt/src/virtio/hsakmt_virtio_proto.h b/libhsakmt/src/virtio/hsakmt_virtio_proto.h new file mode 100644 index 0000000000..80f9494003 --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_proto.h @@ -0,0 +1,503 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef VHSAKMT_VIRTIO_PROTO_H +#define VHSAKMT_VIRTIO_PROTO_H + +#include "hsakmt/linux/kfd_ioctl.h" +#include "hsakmt/hsakmt.h" + +#include +#include +#include + +#include "virtio_gpu.h" + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic error "-Wpadded" +#endif + +/* defined in other header file in virglrenderer */ +#define VHSAKMT_DEFINE_CAST(parent, child) \ + static inline struct child* to_##child(struct parent* x) { return (struct child*)x; } + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define VHSAKMT_STATIC_ASSERT_SIZE(t) \ + static_assert(sizeof(struct t) % 8 == 0, "sizeof(struct " #t ") not multiple of 8"); \ + static_assert(_Alignof(struct t) <= 8, "alignof(struct " #t ") too large"); +#else +#define VHSAKMT_STATIC_ASSERT_SIZE(t) +#endif + +enum vhsakmt_ccmd { + VHSAKMT_CCMD_NOP = 1, /* No payload, can be used to sync with host */ + VHSAKMT_CCMD_QUERY_INFO, + VHSAKMT_CCMD_EVENT, + VHSAKMT_CCMD_MEMORY, + VHSAKMT_CCMD_QUEUE, + VHSAKMT_CCMD_GL_INTER, +}; + +typedef struct _vHsaEvent { + HsaEvent event; + uint64_t event_handle; + uint64_t bo_handle; + uint32_t res_id; + uint32_t pad; +} vHsaEvent; +VHSAKMT_STATIC_ASSERT_SIZE(_vHsaEvent) + +struct vhsakmt_event_shmem { + uint32_t trigered_events_num; + uint32_t pad; + HsaEvent trigered_events[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_event_shmem) + +#define VHSAKMT_CCMD(_cmd, _len) \ + ((struct vhsakmt_ccmd_req){ \ + .cmd = VHSAKMT_CCMD_##_cmd, \ + .len = (_len), \ + }) + +struct vhsakmt_ccmd_nop_req { + struct vhsakmt_ccmd_req hdr; +}; + +/* + * VHSAKMT_CCMD_QUERY + */ +enum vhsakmt_ccmd_query_type { + VHSAKMT_CCMD_QUERY_GPU_INFO = 0, + VHSAKMT_CCMD_QUERY_OPEN_KFD, + VHSAKMT_CCMD_QUERY_GET_VER, + VHSAKMT_CCMD_QUERY_REL_SYS_PROP, + VHSAKMT_CCMD_QUERY_GET_SYS_PROP, + VHSAKMT_CCMD_QUERY_GET_NODE_PROP, + VHSAKMT_CCMD_QUERY_GET_XNACK_MODE, + VHSAKMT_CCMD_QUERY_RUN_TIME_ENABLE, + VHSAKMT_CCMD_QUERY_RUN_TIME_DISABLE, + VHSAKMT_CCMD_QUERY_GET_NOD_MEM_PROP, + VHSAKMT_CCMD_QUERY_GET_NOD_CACHE_PROP, + VHSAKMT_CCMD_QUERY_GET_NOD_IO_LINK_PROP, + VHSAKMT_CCMD_QUERY_GET_CLOCK_COUNTERS, + VHSAKMT_CCMD_QUERY_POINTER_INFO, + VHSAKMT_CCMD_QUERY_TILE_CONFIG, + VHSAKMT_CCMD_QUERY_NANO_TIME, + VHSAKMT_CCMD_QUERY_GET_RUNTIME_CAPS, +}; + +#define QUERY_PTR_INFO_MAX_MAPPED_NODES 3 + +typedef struct _query_req_run_time_enable_args { + /* void* rDebug, bypassed by payload */ + uint8_t pad[3]; + uint8_t setupTtmp; + uint32_t __pad; +} query_req_run_time_enable_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_run_time_enable_args) + +typedef struct _query_req_node_mem_prop_args { + uint32_t NodeId; + uint32_t NumBanks; +} query_req_node_mem_prop_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_mem_prop_args) + +typedef struct _query_req_node_cache_prop_args { + uint32_t NodeId; + uint32_t ProcessorId; + uint32_t NumCaches; + uint32_t pad; +} query_req_node_cache_prop_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_cache_prop_args) + +typedef struct _query_req_node_io_link_args { + uint32_t NodeId; + uint32_t NumIoLinks; +} query_req_node_io_link_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_req_node_io_link_args) + +typedef struct _query_tile_config { + HsaGpuTileConfig config; + uint32_t NodeId; + uint32_t pad; +} query_tile_config; +VHSAKMT_STATIC_ASSERT_SIZE(_query_tile_config) + +typedef struct _query_open_kfd_args { + uint64_t cur_vm_start; +} query_open_kfd_args; +VHSAKMT_STATIC_ASSERT_SIZE(_query_open_kfd_args) + +typedef struct _query_open_kfd_rsp { + uint64_t vm_start; + uint64_t vm_size; +} query_open_kfd_rsp; +VHSAKMT_STATIC_ASSERT_SIZE(_query_open_kfd_rsp) + +typedef struct _query_nano_time_rsp { + uint64_t nano_time; +} query_nano_time_rsp; +VHSAKMT_STATIC_ASSERT_SIZE(_query_nano_time_rsp) + +struct vhsakmt_ccmd_query_info_req { + struct vhsakmt_ccmd_req hdr; + struct drm_amdgpu_info info; + uint32_t type; + uint32_t pad; + union { + uint64_t pointer; + uint32_t NodeID; /* some query API just need node ID */ + query_req_run_time_enable_args run_time_enable_args; + query_req_node_mem_prop_args node_mem_prop_args; + query_req_node_cache_prop_args node_cache_prop_args; + query_req_node_io_link_args node_io_link_args; + query_tile_config tile_config_args; + query_open_kfd_args open_kfd_args; + }; + + uint8_t payload[]; +}; +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_query_info_req) +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_query_info_req) +#define VHSAKMT_CCMD_QUERY_MAX_TILE_CONFIG 128 +#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_MEM_PROP 128 +#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_CACHE_PROP 128 +#define VHSAKMT_CCMD_QUERY_MAX_GET_NOD_IO_LINK_PROP 128 + +struct vhsakmt_ccmd_query_info_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + union { + query_open_kfd_rsp open_kfd_rsp; + query_nano_time_rsp nano_time_rsp; + HsaGpuTileConfig tile_config_rsp; + HsaPointerInfo ptr_info; + struct amdgpu_gpu_info gpu_info; + HsaVersionInfo kfd_version; + HsaSystemProperties sys_props; + HsaNodeProperties node_props; + int32_t xnack_mode; + HsaClockCounters clock_counters; + uint32_t caps; + uint64_t pad[9]; + }; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_query_info_rsp) + +/* + * VHSAKMT_CCMD_EVENT + */ +enum vhsakmt_ccmd_event_type { + VHSAKMT_CCMD_EVENT_CREATE, + VHSAKMT_CCMD_EVENT_DESTROY, + VHSAKMT_CCMD_EVENT_SET, + VHSAKMT_CCMD_EVENT_RESET, + VHSAKMT_CCMD_EVENT_QUERY_STATE, + VHSAKMT_CCMD_EVENT_WAIT_ON_MULTI_EVENTS, + + VHSAKMT_CCMD_EVENT_SET_TRAP, + +}; +typedef struct _event_req_create_args { + HsaEventDescriptor EventDesc; + uint8_t ManualReset; + uint8_t IsSignaled; + uint8_t pad[6]; +} event_req_create_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_create_args) + +typedef struct _event_req_wait_args { + HsaEvent Event; + uint32_t Milliseconds; + uint32_t pad; +} event_req_wait_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_args) + +typedef struct _event_req_wait_ext_args { + HsaEvent Event; + uint64_t event_age; + uint32_t Milliseconds; + uint32_t pad; +} event_req_wait_ext_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_ext_args) + +typedef struct _event_req_wait_on_multi_args { + /*HsaEvent* Events[], in playloud*/ + uint32_t NumEvents; + uint32_t Milliseconds; + uint8_t WaitOnAll; + uint8_t pad[7]; +} event_req_wait_on_multi_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_on_multi_args) + +typedef struct _event_req_wait_on_multi_ext_args { + /*HsaEvent* Events[], in playloud*/ + uint32_t NumEvents; + uint32_t Milliseconds; + uint64_t event_age; + uint8_t WaitOnAll; + uint8_t pad[7]; +} event_req_wait_on_multi_ext_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_req_wait_on_multi_ext_args) + +typedef struct _event_set_trap_handler_args { + uint64_t TrapHandlerBaseAddress; + uint64_t TrapHandlerSizeInBytes; + uint64_t TrapBufferBaseAddress; + uint64_t TrapBufferSizeInBytes; + uint32_t NodeId; + uint32_t pad; +} event_set_trap_handler_args; +VHSAKMT_STATIC_ASSERT_SIZE(_event_set_trap_handler_args) + +struct vhsakmt_ccmd_event_req { + struct vhsakmt_ccmd_req hdr; + union { + HsaEvent Event; /* For set, reset, query. */ + HsaEvent* event_hanele; + event_req_wait_args wait_args; + event_req_create_args create_args; + event_req_wait_ext_args wait_ext_args; + event_req_wait_on_multi_args wait_on_multi_args; + event_req_wait_on_multi_ext_args wait_on_multi_ext_args; + event_set_trap_handler_args set_trap_handler_args; + }; + uint32_t type; + uint32_t sync_shmem_res_id; + uint64_t blob_id; + uint32_t res_id; + uint32_t pad; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_event_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_event_req) + +struct vhsakmt_ccmd_event_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + vHsaEvent vevent; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_event_rsp) + +/* + * VHSAKMT_CCMD_MEMORY + */ +enum vhsakmt_ccmd_memory_type { + VHSAKMT_CCMD_MEMORY_ALLOC, + VHSAKMT_CCMD_MEMORY_MAP_TO_GPU_NODES, + VHSAKMT_CCMD_MEMORY_FREE, + VHSAKMT_CCMD_MEMORY_UNMAP_TO_GPU, + VHSAKMT_CCMD_MEMORY_AVAIL_MEM, + VHSAKMT_CCMD_MEMORY_MAP_MEM_TO_GPU, + VHSAKMT_CCMD_MEMORY_REG_MEM_WITH_FLAG, + VHSAKMT_CCMD_MEMORY_DEREG_MEM, + VHSAKMT_CCMD_MEMORY_MAP_USERPTR, +}; + +typedef struct _memory_req_alloc_args { + uint32_t PreferredNode; + HsaMemFlags MemFlags; + uint64_t SizeInBytes; + uint64_t MemoryAddress; +} memory_req_alloc_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_alloc_args) + +typedef struct _memory_req_free_args { + uint64_t MemoryAddress; + uint64_t SizeInBytes; +} memory_req_free_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_free_args) + +typedef struct _memory_req_map_to_GPU_nodes_args { + uint64_t MemoryAddress; + uint64_t MemorySizeInBytes; + uint64_t AlternateVAGPU; + HsaMemMapFlags MemMapFlags; + uint32_t pad; + uint64_t NumberOfNodes; + uint32_t* NodeArray; +} memory_req_map_to_GPU_nodes_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_req_map_to_GPU_nodes_args) + +typedef struct _memory_map_mem_to_gpu_args { + uint64_t MemoryAddress; + uint64_t MemorySizeInBytes; + uint8_t need_create_bo; + uint8_t pad[7]; +} memory_map_mem_to_gpu_args; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_map_mem_to_gpu_args) + +typedef struct _memory_reg_mem_with_flag { + uint64_t MemoryAddress; + uint64_t MemorySizeInBytes; + HsaMemFlags MemFlags; + uint32_t pad; +} memory_reg_mem_with_flag; +VHSAKMT_STATIC_ASSERT_SIZE(_memory_reg_mem_with_flag) + +struct vhsakmt_ccmd_memory_req { + struct vhsakmt_ccmd_req hdr; + union { + uint64_t MemoryAddress; + uint32_t Node; + memory_req_alloc_args alloc_args; + memory_req_map_to_GPU_nodes_args map_to_GPU_nodes_args; + memory_req_free_args free_args; + memory_map_mem_to_gpu_args map_to_GPU_args; + memory_reg_mem_with_flag reg_mem_with_flag; + }; + uint64_t blob_id; + uint32_t type; + uint32_t res_id; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_memory_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_memory_req) + +typedef struct _vhsakmt_ccmd_memory_map_userptr_rsp { + uint64_t userptr_handle; + uint32_t npfns; + uint32_t pad; +} vhsakmt_ccmd_memory_map_userptr_rsp; +VHSAKMT_STATIC_ASSERT_SIZE(_vhsakmt_ccmd_memory_map_userptr_rsp) + +struct vhsakmt_ccmd_memory_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + union { + vhsakmt_ccmd_memory_map_userptr_rsp map_userptr_rsp; + uint64_t memory_handle; + uint64_t alternate_vagpu; + uint64_t available_bytes; + }; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_memory_rsp) + +/* + * VHSAKMT_CCMD_QUEUE + */ +enum vhsakmt_ccmd_queue_type { + VHSAKMT_CCMD_QUEUE_CREATE, + VHSAKMT_CCMD_QUEUE_DESTROY, +}; + +typedef struct _vHsaQueueResource { + HsaQueueResource r; + uint64_t host_doorbell; + uint64_t host_doorbell_offset; + uint64_t host_write_offset; + uint64_t host_read_offset; + uint64_t host_rw_handle; + uint64_t queue_handle; +} vHsaQueueResource; +VHSAKMT_STATIC_ASSERT_SIZE(_vHsaQueueResource) + +typedef struct _queue_req_create { + uint32_t NodeId; + HSA_QUEUE_TYPE Type; + uint32_t QueuePercentage; + uint32_t pad; + HSA_QUEUE_PRIORITY Priority; + uint32_t pad1; + uint32_t SdmaEngineId; + uint64_t QueueAddress; + uint64_t QueueSizeInBytes; + HsaEvent* Event; + HsaQueueResource* QueueResource; + uint64_t* Queue_write_ptr_aql; + uint64_t* Queue_read_ptr_aql; +} queue_req_create; +VHSAKMT_STATIC_ASSERT_SIZE(_queue_req_create) + +struct vhsakmt_ccmd_queue_req { + struct vhsakmt_ccmd_req hdr; + union { + HSA_QUEUEID QueueId; + queue_req_create create_queue_args; + }; + uint64_t blob_id; /* For queue create, queue resource */ + uint64_t rw_ptr_blob_id; /* For queue create, r/w ptr memory mapping */ + uint64_t doorbell_blob_id; /* For queue create, doorbell ptr memory mapping */ + uint32_t res_id; + uint32_t type; + uint32_t queue_mem_res_id; + uint32_t pad; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_queue_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_queue_req) + +struct vhsakmt_ccmd_queue_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + vHsaQueueResource vqueue_res; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_queue_rsp) + +/* + * VHSAKMT_CCMD_GL_INTER + */ +enum vhsakmt_ccmd_gl_inter_type { + VHSAKMT_CCMD_GL_REG_GHD_TO_NODES, +}; + +typedef struct _gl_inter_req_reg_ghd_to_nodes { + uint64_t GraphicsResourceHandle; + uint64_t NumberOfNodes; // NodeArray in payload + uint32_t res_handle; + uint32_t pad; +} gl_inter_req_reg_ghd_to_nodes; +VHSAKMT_STATIC_ASSERT_SIZE(_gl_inter_req_reg_ghd_to_nodes) + +struct vhsakmt_ccmd_gl_inter_req { + struct vhsakmt_ccmd_req hdr; + union { + gl_inter_req_reg_ghd_to_nodes reg_ghd_to_nodes; + }; + uint32_t type; + uint32_t pad; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_gl_inter_req) +VHSAKMT_DEFINE_CAST(vhsakmt_ccmd_req, vhsakmt_ccmd_gl_inter_req) + +struct vhsakmt_ccmd_gl_inter_rsp { + struct vhsakmt_ccmd_rsp hdr; + int32_t ret; + union { + HsaGraphicsResourceInfo info; + }; + uint8_t payload[]; +}; +VHSAKMT_STATIC_ASSERT_SIZE(vhsakmt_ccmd_gl_inter_rsp) + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +#endif diff --git a/libhsakmt/src/virtio/hsakmt_virtio_queues.c b/libhsakmt/src/virtio/hsakmt_virtio_queues.c new file mode 100644 index 0000000000..5ab12698a8 --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_queues.c @@ -0,0 +1,253 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +static inline uint64_t vhsakmt_doorbell_page_size(void) { return 0x2000; } +static inline uint64_t vhsakmt_queue_page_size(void) { return getpagesize(); } + +HSAKMT_STATUS HSAKMTAPI vhsaKmtSetTrapHandler(HSAuint32 NodeId, void* TrapHandlerBaseAddress, + HSAuint64 TrapHandlerSizeInBytes, + void* TrapBufferBaseAddress, + HSAuint64 TrapBufferSizeInBytes) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_event_rsp* rsp; + struct vhsakmt_ccmd_event_req req = { + .hdr = VHSAKMT_CCMD(EVENT, sizeof(struct vhsakmt_ccmd_event_req)), + .type = VHSAKMT_CCMD_EVENT_SET_TRAP, + .set_trap_handler_args = + { + .NodeId = NodeId, + .TrapHandlerBaseAddress = (uint64_t)TrapHandlerBaseAddress, + .TrapHandlerSizeInBytes = TrapHandlerSizeInBytes, + .TrapBufferBaseAddress = (uint64_t)TrapBufferBaseAddress, + .TrapBufferSizeInBytes = TrapBufferSizeInBytes, + }, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_event_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + return rsp->ret; +} + +static int vhsakmt_find_aql_rw_bo(vhsakmt_device_handle dev, uint64_t aql_ptr, + uint32_t* aql_bo_res_id) { + uint64_t aql_base_ptr = VHSA_ALIGN_DOWN(aql_ptr, getpagesize()); + + vhsakmt_bo_handle bo = vhsakmt_find_bo_by_addr(dev, (void*)aql_base_ptr); + if (!bo) return -EINVAL; + + bo->bo_type |= VHSA_BO_QUEUE_AQL_RW_PTR; + *aql_bo_res_id = bo->real.res_id; + return 0; +} + +static int vhsakmt_create_doorbell_blob_bo(vhsakmt_device_handle dev, uint32_t node, size_t size, + uint32_t blob_id, uint64_t host_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_create_mappable_blob_bo(dev, size, blob_id, VHSA_BO_QUEUE_DOORBELL, + (void*)host_handle, bo_handle); + if (r) return r; + + r = vhsakmt_set_node_doorbell(dev, node, (*bo_handle)->cpu_addr); + + return r; +} + +static int vhsakmt_create_queue_rw_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint64_t host_handle, vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_create_mappable_blob_bo(dev, size, blob_id, VHSA_BO_QUEUE_RW_PTR, NULL, bo_handle); + if (r) return r; + + (*bo_handle)->host_addr = (void*)host_handle; + return r; +} + +static int vhsakmt_create_queue_blob_bo(vhsakmt_device_handle dev, size_t size, uint32_t blob_id, + uint64_t queue_id, vhsakmt_bo_handle rw_bo_handle, + vhsakmt_bo_handle* bo_handle) { + int r; + + r = vhsakmt_init_host_blob(dev, size, VIRTGPU_BLOB_MEM_HOST3D, 0, blob_id, VHSA_BO_QUEUE, NULL, + bo_handle); + if (r) return r; + + vhsakmt_insert_bo(dev, *bo_handle, *bo_handle, (*bo_handle)->size); + + (*bo_handle)->queue_id = queue_id; + (*bo_handle)->rw_bo = rw_bo_handle; + + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueueExt(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, + HSA_QUEUE_PRIORITY Priority, HSAuint32 SdmaEngineId, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + vhsakmt_bo_handle rw_bo_handle = NULL, doorbell_bo, queue_bo, queue_mem_bo; + struct vhsakmt_ccmd_queue_rsp* rsp; + struct vhsakmt_ccmd_queue_req req = { + .hdr = VHSAKMT_CCMD(QUEUE, sizeof(struct vhsakmt_ccmd_queue_req)), + .type = VHSAKMT_CCMD_QUEUE_CREATE, + .create_queue_args = + { + .NodeId = NodeId, + .Type = Type, + .QueuePercentage = QueuePercentage, + .Priority = Priority, + .SdmaEngineId = SdmaEngineId, + .QueueAddress = (uint64_t)QueueAddress, + .QueueSizeInBytes = QueueSizeInBytes, + .Event = Event ? vhsakmt_event_host_handle(Event) : 0, + .Queue_write_ptr_aql = QueueResource->Queue_write_ptr_aql, + .Queue_read_ptr_aql = QueueResource->Queue_read_ptr_aql, + }, + .blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id), /* For queue resource */ + .doorbell_blob_id = vhsakmt_node_doorbell(dev, NodeId) + ? 0 + : vhsakmt_atomic_inc_return(&dev->next_blob_id), /* For queue doorbell memory map */ + }; + int r; + + /* Queue ptr memory is allocated by hsakmtallocmemory in host then mapped into guest, but their + * address are not aligned. */ + if (Type == HSA_QUEUE_COMPUTE_AQL) { + r = vhsakmt_find_aql_rw_bo(dev, QueueResource->QueueWptrValue, &req.res_id); + if (r) { + vhsa_debug("%s: can not find the AQL queue R/W BO: %p\n", __FUNCTION__, + QueueResource->Queue_write_ptr_aql); + return HSAKMT_STATUS_NO_MEMORY; + } + + vhsa_debug("%s: create AQL queue, read ptr: %p, write ptr: %p, res id: %d\n", __FUNCTION__, + QueueResource->Queue_read_ptr_aql, QueueResource->Queue_write_ptr_aql, req.res_id); + } else + /* For queue not CP AQL, it use r/w ptr by itself. */ + req.rw_ptr_blob_id = vhsakmt_atomic_inc_return(&dev->next_blob_id); + + queue_mem_bo = vhsakmt_find_bo_by_addr(dev, QueueAddress); + if (!queue_mem_bo) { + vhsa_err("%s: can not find the queue memory BO: %p\n", __FUNCTION__, QueueAddress); + return HSAKMT_STATUS_NO_MEMORY; + } + queue_mem_bo->bo_type |= VHSA_BO_QUEUE_AQL_RW_PTR; + req.queue_mem_res_id = queue_mem_bo->real.res_id; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_queue_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (rsp->ret) { + vhsa_err("%s: queue create failed, ret: %d", __FUNCTION__, rsp->ret); + return rsp->ret; + } + + /* Map doorbell */ + if (req.doorbell_blob_id) { + r = vhsakmt_create_doorbell_blob_bo( + dev, NodeId, vhsakmt_doorbell_page_size(), req.doorbell_blob_id, + rsp->vqueue_res.host_doorbell - rsp->vqueue_res.host_doorbell_offset, &doorbell_bo); + if (r) { + vhsa_err("%s: doorbell create failed, doorbell: %lx\n", __FUNCTION__, + rsp->vqueue_res.host_doorbell); + return r; + } + vhsa_debug("%s: create doorbell: %p, size: 0x%x\n", __FUNCTION__, doorbell_bo->cpu_addr, + doorbell_bo->size); + } + + QueueResource->Queue_DoorBell_aql = (void*)rsp->vqueue_res.host_doorbell; + vhsa_debug("%s: queue create, Doorbell: %p\n", __FUNCTION__, QueueResource->Queue_DoorBell_aql); + + /* Map R/W pointer. + * For a queue is not a COMPUTE AQL, the R/W PTR not using the input address, + * uses the queue memory allocated by hsakmtallocmemory, a page align address. + */ + if (Type != HSA_QUEUE_COMPUTE_AQL) { + r = vhsakmt_create_queue_rw_blob_bo(dev, vhsakmt_queue_page_size(), req.rw_ptr_blob_id, + rsp->vqueue_res.host_rw_handle, &rw_bo_handle); + if (r) { + vhsa_debug("%s: queue rw ptr create failed, host addr: %p\n", __FUNCTION__, + (void*)rsp->vqueue_res.host_rw_handle); + return r; + } + + QueueResource->Queue_write_ptr_aql = VHSA_UINT64_TO_VPTR( + VHSA_VPTR_TO_UINT64(rw_bo_handle->cpu_addr) + rsp->vqueue_res.host_write_offset); + QueueResource->Queue_read_ptr_aql = VHSA_UINT64_TO_VPTR( + VHSA_VPTR_TO_UINT64(rw_bo_handle->cpu_addr) + rsp->vqueue_res.host_read_offset); + + vhsa_debug("%s: queue create: write ptr gva: %p, read ptr gva: %p, base hva: %lx\n", + __FUNCTION__, QueueResource->Queue_write_ptr_aql, QueueResource->Queue_read_ptr_aql, + rsp->vqueue_res.host_rw_handle); + } + + r = vhsakmt_create_queue_blob_bo(dev, QueueSizeInBytes, req.blob_id, rsp->vqueue_res.r.QueueId, + rw_bo_handle, &queue_bo); + if (r) { + vhsa_err("%s: queue create failed, queue ID: 0x%lx\n", __FUNCTION__, rsp->vqueue_res.r.QueueId); + return r; + } + QueueResource->QueueId = (uint64_t)queue_bo; + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtCreateQueue(HSAuint32 NodeId, HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority, + void* QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent* Event, HsaQueueResource* QueueResource) { + return vhsaKmtCreateQueueExt(NodeId, Type, QueuePercentage, Priority, VHSA_SDMA_NONE, + QueueAddress, QueueSizeInBytes, Event, QueueResource); +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtDestroyQueue(HSA_QUEUEID QueueId) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + int r; + + /* queue ID: vhsakmt_bo_handle -> real queue ID*/ + vhsakmt_bo_handle bo = (vhsakmt_bo_handle)QueueId; + vhsakmt_bo_handle rw_bo = bo->rw_bo; + + r = vhsakmt_bo_free(dev, bo); + if (rw_bo) vhsakmt_bo_free(dev, rw_bo); + + vhsa_debug("%s: queue res id: %d, queue ID: %" PRIu64 ", ret = %d\n", __FUNCTION__, + bo->real.res_id, bo->queue_id, r); + + return r; +} diff --git a/libhsakmt/src/virtio/hsakmt_virtio_topology.c b/libhsakmt/src/virtio/hsakmt_virtio_topology.c new file mode 100644 index 0000000000..8c06e26459 --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_topology.c @@ -0,0 +1,342 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmt_virtio.h" +#include "hsakmt_virtio_device.h" + +static int vhsakmt_set_sys_props(vhsakmt_device_handle dev, HsaSystemProperties* sys_props) { + int r = 0; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + if (dev->sys_props) { + r = 0; + goto out; + } + + dev->sys_props = calloc(1, sizeof(HsaSystemProperties)); + if (!dev->sys_props) { + r = -ENOMEM; + goto out; + } + + memcpy(dev->sys_props, sys_props, sizeof(HsaSystemProperties)); + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); + return r; +} + +static int vhsakmt_set_node_props(vhsakmt_device_handle dev, uint32_t node, + HsaNodeProperties* node_props) { + int r = 0; + if (!dev->sys_props) return -EINVAL; + if (node >= dev->sys_props->NumNodes) return -EINVAL; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + + if (!dev->vhsakmt_nodes) { + dev->vhsakmt_nodes = calloc(dev->sys_props->NumNodes, sizeof(struct vhsakmt_node)); + if (!dev->vhsakmt_nodes) { + r = -ENOMEM; + goto out; + } + } + + memcpy(&dev->vhsakmt_nodes[node].node_props, node_props, sizeof(HsaNodeProperties)); + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); + return r; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetVersion(HsaVersionInfo* v) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_VER, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + memcpy(v, &rsp->kfd_version, sizeof(HsaVersionInfo)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtAcquireSystemProperties(HsaSystemProperties* SystemProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + int r; + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_SYS_PROP, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + memcpy(SystemProperties, &rsp->sys_props, sizeof(HsaSystemProperties)); + + r = vhsakmt_set_sys_props(dev, SystemProperties); + if (r) return r; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtReleaseSystemProperties(void) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_REL_SYS_PROP, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + if (dev->sys_props) { + free(dev->sys_props); + dev->sys_props = NULL; + } + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeProperties(HSAuint32 NodeId, + HsaNodeProperties* NodeProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + int r; + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .NodeID = NodeId, + .type = VHSAKMT_CCMD_QUERY_GET_NODE_PROP, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + memcpy(NodeProperties, &rsp->node_props, sizeof(HsaNodeProperties)); + + r = vhsakmt_set_node_props(dev, NodeId, NodeProperties); + if (r) return r; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetXNACKMode(HSAint32* enable) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_XNACK_MODE, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + memcpy(enable, &rsp->xnack_mode, sizeof(HSAint32)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeEnable(void* rDebug, bool setupTtmp) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .run_time_enable_args.setupTtmp = setupTtmp, + .type = VHSAKMT_CCMD_QUERY_RUN_TIME_ENABLE, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtRuntimeDisable(void) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_RUN_TIME_DISABLE, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + if (!rsp) return -ENOMEM; + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAuint32 NumBanks, + HsaMemoryProperties* MemoryProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_NOD_MEM_PROP, + .node_mem_prop_args.NodeId = NodeId, + .node_mem_prop_args.NumBanks = NumBanks, + }; + + rsp = vhsakmt_alloc_rsp( + dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumBanks * sizeof(HsaMemoryProperties)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(MemoryProperties, rsp->payload, NumBanks * sizeof(HsaMemoryProperties)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeCacheProperties(HSAuint32 NodeId, HSAuint32 ProcessorId, + HSAuint32 NumCaches, + HsaCacheProperties* CacheProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_NOD_CACHE_PROP, + .node_cache_prop_args.NodeId = NodeId, + .node_cache_prop_args.ProcessorId = ProcessorId, + .node_cache_prop_args.NumCaches = NumCaches, + }; + + rsp = vhsakmt_alloc_rsp( + dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumCaches * sizeof(HsaCacheProperties)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(CacheProperties, rsp->payload, NumCaches * sizeof(HsaCacheProperties)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks, + HsaIoLinkProperties* IoLinkProperties) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_NOD_IO_LINK_PROP, + .node_io_link_args.NodeId = NodeId, + .node_io_link_args.NumIoLinks = NumIoLinks, + }; + + rsp = vhsakmt_alloc_rsp( + dev, &req.hdr, + sizeof(struct vhsakmt_ccmd_query_info_rsp) + NumIoLinks * sizeof(HsaIoLinkProperties)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(IoLinkProperties, rsp->payload, NumIoLinks * sizeof(HsaIoLinkProperties)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetClockCounters(HSAuint32 NodeId, HsaClockCounters* Counters) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_CLOCK_COUNTERS, + .NodeID = NodeId, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + memcpy(Counters, &rsp->clock_counters, sizeof(HsaClockCounters)); + + return rsp->ret; +} + +HSAKMT_STATUS HSAKMTAPI vhsaKmtGetRuntimeCapabilities(HSAuint32* caps_mask) { + CHECK_VIRTIO_KFD_OPEN(); + + vhsakmt_device_handle dev = vhsakmt_dev(); + struct vhsakmt_ccmd_query_info_rsp* rsp; + struct vhsakmt_ccmd_query_info_req req = { + .hdr = VHSAKMT_CCMD(QUERY_INFO, sizeof(struct vhsakmt_ccmd_query_info_req)), + .type = VHSAKMT_CCMD_QUERY_GET_RUNTIME_CAPS, + }; + + rsp = vhsakmt_alloc_rsp(dev, &req.hdr, sizeof(struct vhsakmt_ccmd_query_info_rsp)); + if (!rsp) return -ENOMEM; + + vhsakmt_execbuf_cpu(dev, &req.hdr, __FUNCTION__); + + *caps_mask = rsp->caps; + + return rsp->ret; +} diff --git a/libhsakmt/src/virtio/hsakmt_virtio_vm.c b/libhsakmt/src/virtio/hsakmt_virtio_vm.c new file mode 100644 index 0000000000..151b374180 --- /dev/null +++ b/libhsakmt/src/virtio/hsakmt_virtio_vm.c @@ -0,0 +1,113 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "hsakmt_virtio_device.h" + +void* vhsakmt_vm_start(void) { + void* vm_start = malloc(getpagesize()); + if (!vm_start) return NULL; + + free(vm_start); + return vm_start; +} + +int vhsakmt_reserve_va(uint64_t start, uint64_t size) { + int32_t protFlags = PROT_NONE; + int32_t mapFlags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED; + void* va = mmap((void*)start, size, protFlags, mapFlags, -1, 0); + if (va == MAP_FAILED) return -ENOMEM; + + if (va != (void*)start) return -ENOMEM; + + madvise(va, size, MADV_DONTFORK); + + return 0; +} + +void vhsakmt_dereserve_va(uint64_t start, uint64_t size) { munmap((void*)start, size); } + +void vhsakmt_set_scratch_area(vhsakmt_device_handle dev, uint32_t node, uint64_t start, + uint64_t size) { + if (!dev->vhsakmt_nodes || !dev->sys_props) return; + if (node >= dev->sys_props->NumNodes) return; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + + if (dev->vhsakmt_nodes[node].scratch_start && dev->vhsakmt_nodes[node].scratch_size) goto out; + + dev->vhsakmt_nodes[node].scratch_start = start; + dev->vhsakmt_nodes[node].scratch_size = size; + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); +} + +bool vhsakmt_is_scratch_mem(vhsakmt_device_handle dev, void* addr) { + uint32_t i; + if (!dev->vhsakmt_nodes || !dev->sys_props) return false; + + for (i = 0; i < dev->sys_props->NumNodes; i++) { + if ((uint64_t)addr >= dev->vhsakmt_nodes[i].scratch_start && + (uint64_t)addr <= dev->vhsakmt_nodes[i].scratch_start + dev->vhsakmt_nodes[i].scratch_size) + return true; + } + + return false; +} + +void vhsakmt_set_vm_area(vhsakmt_device_handle dev, uint64_t start, uint64_t size) { + pthread_mutex_lock(&dev->vhsakmt_mutex); + if (dev->vm_start && dev->vm_size) goto out; + + dev->vm_start = start; + dev->vm_size = size; + +out: + pthread_mutex_unlock(&dev->vhsakmt_mutex); +} + +bool vhsakmt_is_userptr(vhsakmt_device_handle dev, void* addr) { + return !((uint64_t)addr >= dev->vm_start && (uint64_t)addr <= dev->vm_start + dev->vm_size); +} + +int vhsakmt_set_node_doorbell(vhsakmt_device_handle dev, uint32_t node, void* doorbell) { + if (!dev->vhsakmt_nodes || !dev->sys_props) return -EINVAL; + if (node >= dev->sys_props->NumNodes) return -EINVAL; + + pthread_mutex_lock(&dev->vhsakmt_mutex); + + dev->vhsakmt_nodes[node].doorbell_base = doorbell; + + pthread_mutex_unlock(&dev->vhsakmt_mutex); + + return 0; +} + +void* vhsakmt_node_doorbell(vhsakmt_device_handle dev, uint32_t node) { + if (!dev->vhsakmt_nodes || !dev->sys_props) return NULL; + if (node >= dev->sys_props->NumNodes) return NULL; + + return dev->vhsakmt_nodes[node].doorbell_base; +} diff --git a/libhsakmt/src/virtio/include/linux/virtgpu_drm.h b/libhsakmt/src/virtio/include/linux/virtgpu_drm.h new file mode 100644 index 0000000000..d7625a551a --- /dev/null +++ b/libhsakmt/src/virtio/include/linux/virtgpu_drm.h @@ -0,0 +1,279 @@ +/* + * Copyright 2013 Red Hat + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef VIRTGPU_DRM_H +#define VIRTGPU_DRM_H + +#include "drm.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Please note that modifications to all structs defined here are + * subject to backwards-compatibility constraints. + * + * Do not use pointers, use __u64 instead for 32 bit / 64 bit user/kernel + * compatibility Keep fields aligned to their size + */ + +#define DRM_VIRTGPU_MAP 0x01 +#define DRM_VIRTGPU_EXECBUFFER 0x02 +#define DRM_VIRTGPU_GETPARAM 0x03 +#define DRM_VIRTGPU_RESOURCE_CREATE 0x04 +#define DRM_VIRTGPU_RESOURCE_INFO 0x05 +#define DRM_VIRTGPU_TRANSFER_FROM_HOST 0x06 +#define DRM_VIRTGPU_TRANSFER_TO_HOST 0x07 +#define DRM_VIRTGPU_WAIT 0x08 +#define DRM_VIRTGPU_GET_CAPS 0x09 +#define DRM_VIRTGPU_RESOURCE_CREATE_BLOB 0x0a +#define DRM_VIRTGPU_CONTEXT_INIT 0x0b + +#define VIRTGPU_EXECBUF_FENCE_FD_IN 0x01 +#define VIRTGPU_EXECBUF_FENCE_FD_OUT 0x02 +#define VIRTGPU_EXECBUF_RING_IDX 0x04 +#define VIRTGPU_EXECBUF_FLAGS (\ + VIRTGPU_EXECBUF_FENCE_FD_IN |\ + VIRTGPU_EXECBUF_FENCE_FD_OUT |\ + VIRTGPU_EXECBUF_RING_IDX |\ + 0) + +struct drm_virtgpu_map { + __u64 offset; /* use for mmap system call */ + __u32 handle; + __u32 pad; +}; + +#define VIRTGPU_EXECBUF_SYNCOBJ_RESET 0x01 +#define VIRTGPU_EXECBUF_SYNCOBJ_FLAGS ( \ + VIRTGPU_EXECBUF_SYNCOBJ_RESET | \ + 0) +struct drm_virtgpu_execbuffer_syncobj { + __u32 handle; + __u32 flags; + __u64 point; +}; + +/* fence_fd is modified on success if VIRTGPU_EXECBUF_FENCE_FD_OUT flag is set. */ +struct drm_virtgpu_execbuffer { + __u32 flags; + __u32 size; + __u64 command; /* void* */ + __u64 bo_handles; + __u32 num_bo_handles; + __s32 fence_fd; /* in/out fence fd (see VIRTGPU_EXECBUF_FENCE_FD_IN/OUT) */ + __u32 ring_idx; /* command ring index (see VIRTGPU_EXECBUF_RING_IDX) */ + __u32 syncobj_stride; /* size of @drm_virtgpu_execbuffer_syncobj */ + __u32 num_in_syncobjs; + __u32 num_out_syncobjs; + __u64 in_syncobjs; + __u64 out_syncobjs; +}; + +#define VIRTGPU_PARAM_3D_FEATURES 1 /* do we have 3D features in the hw */ +#define VIRTGPU_PARAM_CAPSET_QUERY_FIX 2 /* do we have the capset fix */ +#define VIRTGPU_PARAM_RESOURCE_BLOB 3 /* DRM_VIRTGPU_RESOURCE_CREATE_BLOB */ +#define VIRTGPU_PARAM_HOST_VISIBLE 4 /* Host blob resources are mappable */ +#define VIRTGPU_PARAM_CROSS_DEVICE 5 /* Cross virtio-device resource sharing */ +#define VIRTGPU_PARAM_CONTEXT_INIT 6 /* DRM_VIRTGPU_CONTEXT_INIT */ +#define VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs 7 /* Bitmask of supported capability set ids */ +#define VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME 8 /* Ability to set debug name from userspace */ + +struct drm_virtgpu_getparam { + __u64 param; + __u64 value; +}; + +/* NO_BO flags? NO resource flag? */ +/* resource flag for y_0_top */ +struct drm_virtgpu_resource_create { + __u32 target; + __u32 format; + __u32 bind; + __u32 width; + __u32 height; + __u32 depth; + __u32 array_size; + __u32 last_level; + __u32 nr_samples; + __u32 flags; + __u32 bo_handle; /* if this is set - recreate a new resource attached to this bo ? */ + __u32 res_handle; /* returned by kernel */ + __u32 size; /* validate transfer in the host */ + __u32 stride; /* validate transfer in the host */ +}; + +struct drm_virtgpu_resource_info { + __u32 bo_handle; + __u32 res_handle; + __u32 size; + __u32 blob_mem; +}; + +struct drm_virtgpu_3d_box { + __u32 x; + __u32 y; + __u32 z; + __u32 w; + __u32 h; + __u32 d; +}; + +struct drm_virtgpu_3d_transfer_to_host { + __u32 bo_handle; + struct drm_virtgpu_3d_box box; + __u32 level; + __u32 offset; + __u32 stride; + __u32 layer_stride; +}; + +struct drm_virtgpu_3d_transfer_from_host { + __u32 bo_handle; + struct drm_virtgpu_3d_box box; + __u32 level; + __u32 offset; + __u32 stride; + __u32 layer_stride; +}; + +#define VIRTGPU_WAIT_NOWAIT 1 /* like it */ +struct drm_virtgpu_3d_wait { + __u32 handle; /* 0 is an invalid handle */ + __u32 flags; +}; + +#define VIRTGPU_DRM_CAPSET_VIRGL 1 +#define VIRTGPU_DRM_CAPSET_VIRGL2 2 +#define VIRTGPU_DRM_CAPSET_GFXSTREAM_VULKAN 3 +#define VIRTGPU_DRM_CAPSET_VENUS 4 +#define VIRTGPU_DRM_CAPSET_CROSS_DOMAIN 5 +#define VIRTGPU_DRM_CAPSET_DRM 6 +struct drm_virtgpu_get_caps { + __u32 cap_set_id; + __u32 cap_set_ver; + __u64 addr; + __u32 size; + __u32 pad; +}; + +struct drm_virtgpu_resource_create_blob { +#define VIRTGPU_BLOB_MEM_GUEST 0x0001 +#define VIRTGPU_BLOB_MEM_HOST3D 0x0002 +#define VIRTGPU_BLOB_MEM_HOST3D_GUEST 0x0003 + +#define VIRTGPU_BLOB_FLAG_USE_MAPPABLE 0x0001 +#define VIRTGPU_BLOB_FLAG_USE_SHAREABLE 0x0002 +#define VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE 0x0004 +#define VIRTGPU_BLOB_FLAG_USE_USERPTR 0x0008 + /* zero is invalid blob_mem */ + __u32 blob_mem; + __u32 blob_flags; + __u32 bo_handle; + __u32 res_handle; + __u64 size; + + /* + * for 3D contexts with VIRTGPU_BLOB_MEM_HOST3D_GUEST and + * VIRTGPU_BLOB_MEM_HOST3D otherwise, must be zero. + */ + __u32 pad; + __u32 cmd_size; + __u64 cmd; + __u64 blob_id; + __u64 blob_userptr; + __s64 offset; +}; + +#define VIRTGPU_CONTEXT_PARAM_CAPSET_ID 0x0001 +#define VIRTGPU_CONTEXT_PARAM_NUM_RINGS 0x0002 +#define VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK 0x0003 +#define VIRTGPU_CONTEXT_PARAM_DEBUG_NAME 0x0004 +struct drm_virtgpu_context_set_param { + __u64 param; + __u64 value; +}; + +struct drm_virtgpu_context_init { + __u32 num_params; + __u32 pad; + + /* pointer to drm_virtgpu_context_set_param array */ + __u64 ctx_set_params; +}; + +/* + * Event code that's given when VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK is in + * effect. The event size is sizeof(drm_event), since there is no additional + * payload. + */ +#define VIRTGPU_EVENT_FENCE_SIGNALED 0x90000000 + +#define DRM_IOCTL_VIRTGPU_MAP \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_MAP, struct drm_virtgpu_map) + +#define DRM_IOCTL_VIRTGPU_EXECBUFFER \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_EXECBUFFER,\ + struct drm_virtgpu_execbuffer) + +#define DRM_IOCTL_VIRTGPU_GETPARAM \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GETPARAM,\ + struct drm_virtgpu_getparam) + +#define DRM_IOCTL_VIRTGPU_RESOURCE_CREATE \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_CREATE, \ + struct drm_virtgpu_resource_create) + +#define DRM_IOCTL_VIRTGPU_RESOURCE_INFO \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_INFO, \ + struct drm_virtgpu_resource_info) + +#define DRM_IOCTL_VIRTGPU_TRANSFER_FROM_HOST \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_FROM_HOST, \ + struct drm_virtgpu_3d_transfer_from_host) + +#define DRM_IOCTL_VIRTGPU_TRANSFER_TO_HOST \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_TRANSFER_TO_HOST, \ + struct drm_virtgpu_3d_transfer_to_host) + +#define DRM_IOCTL_VIRTGPU_WAIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_WAIT, \ + struct drm_virtgpu_3d_wait) + +#define DRM_IOCTL_VIRTGPU_GET_CAPS \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_GET_CAPS, \ + struct drm_virtgpu_get_caps) + +#define DRM_IOCTL_VIRTGPU_RESOURCE_CREATE_BLOB \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_RESOURCE_CREATE_BLOB, \ + struct drm_virtgpu_resource_create_blob) + +#define DRM_IOCTL_VIRTGPU_CONTEXT_INIT \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_VIRTGPU_CONTEXT_INIT, \ + struct drm_virtgpu_context_init) + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/libhsakmt/src/virtio/libhsakmt_virtio.ver b/libhsakmt/src/virtio/libhsakmt_virtio.ver new file mode 100644 index 0000000000..1d3c323e17 --- /dev/null +++ b/libhsakmt/src/virtio/libhsakmt_virtio.ver @@ -0,0 +1,45 @@ +{ +global: +vhsaKmtOpenKFD; +vhsaKmtCloseKFD; +vhsaKmtAllocMemory; +vhsaKmtFreeMemory; +vhsaKmtMapMemoryToGPUNodes; +vhsaKmtUnmapMemoryToGPU; +vhsaKmtAvailableMemory; +vhsaKmtMapMemoryToGPU; +vhsaKmtRegisterMemoryWithFlags; +vhsaKmtDeregisterMemory; +vhsaKmtGetVersion; +vhsaKmtAcquireSystemProperties; +vhsaKmtReleaseSystemProperties; +vhsaKmtGetNodeProperties; +vhsaKmtGetXNACKMode; +vhsaKmtRuntimeEnable; +vhsaKmtRuntimeDisable; +vhsaKmtGetNodeMemoryProperties; +vhsaKmtGetNodeCacheProperties; +vhsaKmtGetNodeIoLinkProperties; +vhsaKmtGetClockCounters; +vhsaKmtGetAMDGPUDeviceHandle; +vhsaKmtQueryPointerInfo; +vhsaKmtGetTileConfig; +vhsaKmtCreateEvent; +vhsaKmtDestroyEvent; +vhsaKmtSetEvent; +vhsaKmtResetEvent; +vhsaKmtQueryEventState; +vhsaKmtWaitOnMultipleEvents; +vhsaKmtWaitOnEvent; +vhsaKmtWaitOnEvent_Ext; +vhsaKmtWaitOnMultipleEvents_Ext; +vhsaKmtSetTrapHandler; +vhsaKmtCreateQueueExt; +vhsaKmtCreateQueue; +vhsaKmtDestroyQueue; +vhsaKmtRegisterGraphicsHandleToNodes; +vhsaKmtGetRuntimeCapabilities; +vamdgpu_query_gpu_info; +local: *; +}; + diff --git a/libhsakmt/src/virtio/virtio_gpu.c b/libhsakmt/src/virtio/virtio_gpu.c new file mode 100644 index 0000000000..c3d6191437 --- /dev/null +++ b/libhsakmt/src/virtio/virtio_gpu.c @@ -0,0 +1,324 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "virtio_gpu.h" + +#define SHMEM_SZ (25 * 0x1000) + +static int set_context(int fd) { + struct drm_virtgpu_context_set_param params[] = { + {VIRTGPU_CONTEXT_PARAM_CAPSET_ID, VIRGL_RENDERER_CAPSET_HSAKMT}, + {VIRTGPU_CONTEXT_PARAM_NUM_RINGS, 64}, + }; + struct drm_virtgpu_context_init args = { + .num_params = ARRAY_SIZE(params), + .ctx_set_params = (uintptr_t)(params), + }; + + return virtio_gpu_ioctl(fd, VIRTGPU_CONTEXT_INIT, &args); +} + +int virtio_gpu_map_handle(struct virtio_gpu_device* vgdev, uint32_t handle, uint64_t size, + void** addr, void* fixed_map) { + struct drm_virtgpu_map args = { + .handle = handle, + }; + int r; + + r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_MAP, &args); + if (r) return r; + + *addr = mmap(fixed_map, size, PROT_READ | PROT_WRITE, MAP_SHARED | (fixed_map ? MAP_FIXED : 0), + vgdev->fd, args.offset); + + if (*addr == MAP_FAILED) return -EINVAL; + + return 0; +} + +void virtio_gpu_unmap(void* addr, uint64_t size) { munmap(addr, size); } + +static void virtio_gpu_bo_close(struct virtio_gpu_device* vgdev, uint32_t handle) { + struct drm_gem_close args = { + .handle = handle, + }; + + virtio_gpu_ioctl(vgdev->fd, GEM_CLOSE, &args); +} + +static int virtio_gpu_shmem_init(struct virtio_gpu_device* vgdev, size_t size) { + struct drm_virtgpu_resource_create_blob args = { + .blob_mem = VIRTGPU_BLOB_MEM_HOST3D, + .blob_flags = VIRTGPU_BLOB_FLAG_USE_MAPPABLE, + .size = size, + .blob_id = 0, + }; + + int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_CREATE_BLOB, &args); + if (r) return r; + + r = virtio_gpu_map_handle(vgdev, args.bo_handle, size, (void**)&vgdev->shmem, NULL); + if (r) { + virtio_gpu_bo_close(vgdev, args.bo_handle); + return r; + } + + vgdev->shmem_handle = args.bo_handle; + + uint32_t offset = vgdev->shmem->base.rsp_mem_offset; + vgdev->rsp_mem_len = size - offset; + vgdev->rsp_mem = &((uint8_t*)vgdev->shmem)[offset]; + + return 0; +} + +struct virtio_gpu_device* virtio_gpu_init(int fd, uint32_t context_id) { + struct virtio_gpu_device* vgdev; + int r; + + r = set_context(fd); + + if (r) return NULL; + + vgdev = calloc(1, sizeof(*vgdev)); + if (!vgdev) return NULL; + + vgdev->fd = fd; + + vgdev->reqbuf = calloc(1, SHMEM_SZ); + if (!vgdev->reqbuf) { + free(vgdev); + return NULL; + } + + r = virtio_gpu_shmem_init(vgdev, SHMEM_SZ); + if (r) { + free(vgdev); + return NULL; + } + + pthread_mutex_init(&vgdev->rsp_lock, NULL); + pthread_mutex_init(&vgdev->eb_lock, NULL); + + return vgdev; +} + +void virtio_gpu_close(struct virtio_gpu_device* vgdev) { + virtio_gpu_unmap(vgdev->shmem, SHMEM_SZ); + virtio_gpu_bo_close(vgdev, vgdev->shmem_handle); + + pthread_mutex_destroy(&vgdev->rsp_lock); + pthread_mutex_destroy(&vgdev->eb_lock); + + close(vgdev->fd); + free(vgdev->reqbuf); + free(vgdev); +} + +void* virtio_gpu_alloc_rsp(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + uint32_t size) { + uint32_t off; + + pthread_mutex_lock(&vgdev->rsp_lock); + + size = VHSA_ALIGN_UP(size, 8); + + if ((vgdev->next_rsp_off + size) >= vgdev->rsp_mem_len) vgdev->next_rsp_off = 0; + + off = vgdev->next_rsp_off; + vgdev->next_rsp_off += size; + + pthread_mutex_unlock(&vgdev->rsp_lock); + + req->rsp_off = off; + struct virtio_gpu_ccmd_rsp* rsp = (void*)&vgdev->rsp_mem[off]; + rsp->len = size; + + return rsp; +} + +static int virtio_gpu_execbuffer_locked(struct virtio_gpu_device* vgdev, void* cmd, + uint32_t cmd_size, uint32_t* handles, uint32_t num_handles, + int* fence_fd, int ring_idx, uint32_t num_in_syncobjs, + uint32_t num_out_syncobjs, + struct drm_virtgpu_execbuffer_syncobj* in_syncobjs, + struct drm_virtgpu_execbuffer_syncobj* out_syncobjs, + bool in_fence, bool out_fence) { + struct drm_virtgpu_execbuffer eb = { + .flags = (out_fence ? VIRTGPU_EXECBUF_FENCE_FD_OUT : 0) | + (in_fence ? VIRTGPU_EXECBUF_FENCE_FD_IN : 0) | VIRTGPU_EXECBUF_RING_IDX, + .size = cmd_size, + .command = (uintptr_t)cmd, + .bo_handles = (uintptr_t)handles, + .num_bo_handles = num_handles, + .fence_fd = *fence_fd, + .ring_idx = ring_idx, + .syncobj_stride = sizeof(struct drm_virtgpu_execbuffer_syncobj), + .num_in_syncobjs = num_in_syncobjs, + .num_out_syncobjs = num_out_syncobjs, + .in_syncobjs = (uintptr_t)in_syncobjs, + .out_syncobjs = (uintptr_t)out_syncobjs, + }; + int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_EXECBUFFER, &eb); + if (r) return r; + + if (out_fence) *fence_fd = eb.fence_fd; + + return 0; +} + +static int virtio_gpu_flush_locked(struct virtio_gpu_device* vgdev, int* fence) { + int r; + + if (!vgdev->reqbuf_len) return 0; + + r = virtio_gpu_execbuffer_locked(vgdev, vgdev->reqbuf, vgdev->reqbuf_len, NULL, 0, fence, 0, 0, 0, + NULL, NULL, false, !!fence); + if (r) return r; + + vgdev->reqbuf_len = 0; + vgdev->reqbuf_cnt = 0; + + return 0; +} + +static int virtio_gpu_add_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req) { + req->seqno = ++vgdev->next_seqno; + int r; + + if (vgdev->reqbuf_len + req->len > sizeof(vgdev->reqbuf)) { + r = virtio_gpu_flush_locked(vgdev, NULL); + if (r) return r; + } + + memcpy(&vgdev->reqbuf[vgdev->reqbuf_len], req, req->len); + vgdev->reqbuf_len += req->len; + vgdev->reqbuf_cnt++; + + return 0; +} + +static inline bool fence_before(uint32_t a, uint32_t b) { return (int32_t)(a - b) < 0; } + +static void virtio_gpu_seqno_sync(struct virtio_gpu_device* vgdev, + struct virtio_gpu_ccmd_req* req) { + while (fence_before(vgdev->shmem->base.seqno, req->seqno)) sched_yield(); +} + +int virtio_gpu_exec_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + bool sync) { + int r = 0; + int fence; + + pthread_mutex_lock(&vgdev->eb_lock); + + r = virtio_gpu_add_cmd(vgdev, req); + + if (r || !sync) goto out; + + r = virtio_gpu_flush_locked(vgdev, &fence); + +out: + pthread_mutex_unlock(&vgdev->eb_lock); + if (r) return r; + + if (sync) { + sync_wait(fence, -1); + close(fence); + virtio_gpu_seqno_sync(vgdev, req); + } + + return r; +} + +int virtio_gpu_create_blob(struct virtio_gpu_device* vgdev, + struct drm_virtgpu_resource_create_blob* args) { + return virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_CREATE_BLOB, args); +} + +int virtio_gpu_destroy_handle(struct virtio_gpu_device* vgdev, uint32_t bo_handle) { + struct drm_gem_close args = { + .handle = bo_handle, + }; + + return virtio_gpu_ioctl(vgdev->fd, GEM_CLOSE, &args); +} + +int virtio_gpu_res_id(struct virtio_gpu_device* vgdev, uint32_t handle, uint32_t* res_id) { + struct drm_virtgpu_resource_info args = { + .bo_handle = handle, + }; + int r = virtio_gpu_ioctl(vgdev->fd, VIRTGPU_RESOURCE_INFO, &args); + if (r) return r; + + *res_id = args.res_handle; + return 0; +} + +static int virtio_gpu_get_capset(int fd, struct virgl_renderer_capset_hsakmt* caps) { + struct drm_virtgpu_get_caps args = { + .cap_set_id = VIRGL_RENDERER_CAPSET_HSAKMT, + .cap_set_ver = 0, + .addr = (uintptr_t)caps, + .size = sizeof(*caps), + }; + + memset(caps, 0, sizeof(*caps)); + + return virtio_gpu_ioctl(fd, VIRTGPU_GET_CAPS, &args); +} + +int virtio_gpu_kfd_open(void) { + drmDevicePtr devices[VHSA_MAX_DEVICES]; + int num_devices = 0; + int i, fd, ret; + + num_devices = drmGetDevices2(0, devices, ARRAY_SIZE(devices)); + if (num_devices <= 0) return -1; + + for (i = 0; i < num_devices; i++) { + fd = open(devices[i]->nodes[DRM_NODE_RENDER], O_RDWR | O_CLOEXEC); + if (fd < 0) continue; + + struct virgl_renderer_capset_hsakmt caps; + ret = virtio_gpu_get_capset(fd, &caps); + if (ret || caps.context_type != VIRTGPU_DRM_CONTEXT_AMDGPU) { + close(fd); + fd = -1; + continue; + } + + goto out; + } + +out: + drmFreeDevices(devices, num_devices); + return fd; +} diff --git a/libhsakmt/src/virtio/virtio_gpu.h b/libhsakmt/src/virtio/virtio_gpu.h new file mode 100644 index 0000000000..f31333ad6a --- /dev/null +++ b/libhsakmt/src/virtio/virtio_gpu.h @@ -0,0 +1,118 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef VIRTIO_GPU_H +#define VIRTIO_GPU_H + +#include +#include +#include + +#include "virtgpu_drm.h" + +#define VIRGL_RENDERER_CAPSET_HSAKMT 8 +#define VIRTGPU_DRM_CONTEXT_AMDGPU 1 +#define VHSA_MAX_DEVICES 10 + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + +#define VHSA_ALIGN_UP(x, align) (((uint64_t)(x) + (align)-1) & ~(uint64_t)((align)-1)) +#define VHSA_ALIGN_DOWN(x, align) ((uint64_t)(x) & ~(uint64_t)((align)-1)) + +#define virtio_gpu_ioctl(fd, name, args) \ + ({ \ + int ret = drmIoctl((fd), DRM_IOCTL_##name, (args)); \ + ret; \ + }) + +struct virgl_renderer_capset_hsakmt { + uint32_t wire_format_version; + /* Underlying drm device version: */ + uint32_t version_major; + uint32_t version_minor; + uint32_t version_patchlevel; + uint32_t context_type; + uint32_t pad; +}; + +struct virtio_gpu_shmem_base { + uint32_t seqno; + uint32_t rsp_mem_offset; +}; + +struct virtio_gpu_ccmd_req { + uint32_t cmd; + uint32_t len; + uint32_t seqno; + uint32_t rsp_off; +}; + +struct virtio_gpu_ccmd_rsp { + uint32_t len; +}; + +struct virtio_gpu_shmem { + struct virtio_gpu_shmem_base base; + uint32_t async_error; + uint32_t global_faults; +}; + +#define vhsakmt_shmem virtio_gpu_shmem +#define vhsakmt_ccmd_req virtio_gpu_ccmd_req +#define vhsakmt_ccmd_rsp virtio_gpu_ccmd_rsp + +struct virtio_gpu_device { + int fd; + + struct virtio_gpu_shmem* shmem; + uint32_t shmem_handle; + + uint8_t* rsp_mem; + uint32_t rsp_mem_len; + uint32_t next_rsp_off; + pthread_mutex_t rsp_lock; + pthread_mutex_t eb_lock; + + uint32_t next_seqno; + uint32_t reqbuf_len; + uint32_t reqbuf_cnt; + uint8_t* reqbuf; +}; + +struct virtio_gpu_device* virtio_gpu_init(int fd, uint32_t context_id); +void virtio_gpu_close(struct virtio_gpu_device* vgdev); +int virtio_gpu_exec_cmd(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + bool sync); +void* virtio_gpu_alloc_rsp(struct virtio_gpu_device* vgdev, struct virtio_gpu_ccmd_req* req, + uint32_t size); +int virtio_gpu_map_handle(struct virtio_gpu_device* vgdev, uint32_t handle, uint64_t size, + void** addr, void* fixed_map); +void virtio_gpu_unmap(void* addr, uint64_t size); +int virtio_gpu_create_blob(struct virtio_gpu_device* vgdev, + struct drm_virtgpu_resource_create_blob* args); +int virtio_gpu_destroy_handle(struct virtio_gpu_device* vgdev, uint32_t bo_handle); +int virtio_gpu_res_id(struct virtio_gpu_device* vgdev, uint32_t handle, uint32_t* res_id); +int virtio_gpu_kfd_open(void); + +#endif /* VIRTIO_GPU_H */ diff --git a/runtime/hsa-runtime/CMakeLists.txt b/runtime/hsa-runtime/CMakeLists.txt index d40b8936af..165b34f230 100644 --- a/runtime/hsa-runtime/CMakeLists.txt +++ b/runtime/hsa-runtime/CMakeLists.txt @@ -302,6 +302,10 @@ target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE elf::elf dl pthread rt ) # Link to hsakmt-staticdrm target for static library builds if( BUILD_SHARED_LIBS ) target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE hsakmt::hsakmt PkgConfig::drm) + if( BUILD_THUNK_VIRTIO ) + message(STATUS "Building with virtio support") + target_link_libraries ( ${CORE_RUNTIME_TARGET} PRIVATE hsakmt_virtio) + endif() find_package(rocprofiler-register) if(rocprofiler-register_FOUND) target_compile_definitions(${CORE_RUNTIME_TARGET} PRIVATE HSA_ROCPROFILER_REGISTER=1 From d36cb195daa101ed5a59fb23f43d083114825293 Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Tue, 15 Jul 2025 13:37:32 +0800 Subject: [PATCH 05/13] rocr/driver: add virtio driver support for ROCm runtime This commit adds virtio driver support to the ROCm runtime by: 1. Implementing KfdVirtioDriver class that inherits from core::Driver 2. Adding KFD_VIRTIO to DriverType enum 3. Registering virtio driver discovery function in topology 4. Adding virtio driver source files to CMake build The virtio driver implementation provides basic memory management and queue operations for virtualized GPU environments. Some advanced features like PC sampling and SMI are currently not supported. Key changes: - Add new files: amd_kfd_virtio_driver.h/cpp - Update CMakeLists.txt to include virtio driver - Add VIRTIO to DriverType enum in driver.h - Register virtio driver in amd_topology.cpp Signed-off-by: Honglei Huang --- runtime/hsa-runtime/CMakeLists.txt | 5 + .../driver/virtio/amd_kfd_virtio_driver.cpp | 514 ++++++++++++++++++ .../hsa-runtime/core/inc/amd_virtio_driver.h | 124 +++++ runtime/hsa-runtime/core/inc/driver.h | 2 +- .../hsa-runtime/core/runtime/amd_topology.cpp | 15 +- 5 files changed, 657 insertions(+), 3 deletions(-) create mode 100644 runtime/hsa-runtime/core/driver/virtio/amd_kfd_virtio_driver.cpp create mode 100644 runtime/hsa-runtime/core/inc/amd_virtio_driver.h diff --git a/runtime/hsa-runtime/CMakeLists.txt b/runtime/hsa-runtime/CMakeLists.txt index 165b34f230..4c45bd6bec 100644 --- a/runtime/hsa-runtime/CMakeLists.txt +++ b/runtime/hsa-runtime/CMakeLists.txt @@ -204,6 +204,11 @@ set ( SRCS core/driver/driver.cpp libamdhsacode/amd_hsa_code.cpp libamdhsacode/amd_core_dump.cpp ) +if ( BUILD_THUNK_VIRTIO ) + list(APPEND SRCS core/driver/virtio/amd_kfd_virtio_driver.cpp) + target_compile_definitions(hsa-runtime64 PRIVATE HSAKMT_VIRTIO_ENABLED=1) +endif() + target_sources( ${CORE_RUNTIME_TARGET} PRIVATE ${SRCS} ) ## Depend on trap handler target. diff --git a/runtime/hsa-runtime/core/driver/virtio/amd_kfd_virtio_driver.cpp b/runtime/hsa-runtime/core/driver/virtio/amd_kfd_virtio_driver.cpp new file mode 100644 index 0000000000..a6a832b3c6 --- /dev/null +++ b/runtime/hsa-runtime/core/driver/virtio/amd_kfd_virtio_driver.cpp @@ -0,0 +1,514 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#include "core/inc/amd_virtio_driver.h" +#include "hsakmt/hsakmt_virtio.h" + +#include +#include + +#include "core/inc/amd_gpu_agent.h" +#include "core/inc/amd_memory_region.h" +#include "core/inc/runtime.h" + +extern r_debug _amdgpu_r_debug; + +namespace rocr { +namespace AMD { + +KfdVirtioDriver::KfdVirtioDriver(std::string devnode_name) + : core::Driver(core::DriverType::KFD_VIRTIO, std::move(devnode_name)) {} + +hsa_status_t KfdVirtioDriver::DiscoverDriver(std::unique_ptr& driver) { + auto tmp_driver = std::unique_ptr(new KfdVirtioDriver("")); + + if (tmp_driver->Open() == HSA_STATUS_SUCCESS) { + driver = std::move(tmp_driver); + return HSA_STATUS_SUCCESS; + } + + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::Open() { + return vhsaKmtOpenKFD() == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::Close() { + return vhsaKmtCloseKFD() == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::Init() { + HSAKMT_STATUS ret = + vhsaKmtRuntimeEnable(&_amdgpu_r_debug, core::Runtime::runtime_singleton_->flag().debug()); + uint32_t caps_mask = 0; + + if (ret != HSAKMT_STATUS_SUCCESS && ret != HSAKMT_STATUS_NOT_SUPPORTED) return HSA_STATUS_ERROR; + + if (vhsaKmtGetRuntimeCapabilities(&caps_mask) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + core::Runtime::runtime_singleton_->KfdVersion( + ret != HSAKMT_STATUS_NOT_SUPPORTED, + !!(caps_mask & HSA_RUNTIME_ENABLE_CAPS_SUPPORTS_CORE_DUMP_MASK)); + + if (vhsaKmtGetVersion(&version_) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + core::Runtime::runtime_singleton_->KfdVersion(version_); + + if (version_.KernelInterfaceMajorVersion == 1 && version_.KernelInterfaceMinorVersion == 0) + core::g_use_interrupt_wait = false; + + /* Force disable interrupt wait in VIRTIO driver temporarily */ + core::g_use_interrupt_wait = false; + + /* Force disable XNACK in VIRTIO driver temporarily */ + core::Runtime::runtime_singleton_->XnackEnabled(false); + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::ShutDown() { + HSAKMT_STATUS ret = vhsaKmtRuntimeDisable(); + if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + ret = vhsaKmtReleaseSystemProperties(); + + if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + return Close(); +} + +hsa_status_t KfdVirtioDriver::QueryKernelModeDriver(core::DriverQuery query) { + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::GetSystemProperties(HsaSystemProperties& sys_props) const { + if (vhsaKmtAcquireSystemProperties(&sys_props) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::GetNodeProperties(HsaNodeProperties& node_props, + uint32_t node_id) const { + if (vhsaKmtGetNodeProperties(node_id, &node_props) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::GetEdgeProperties(std::vector& io_link_props, + uint32_t node_id) const { + if (vhsaKmtGetNodeIoLinkProperties(node_id, io_link_props.size(), io_link_props.data()) != + HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::GetMemoryProperties( + uint32_t node_id, std::vector& mem_props) const { + if (mem_props.empty()) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + if (vhsaKmtGetNodeMemoryProperties(node_id, mem_props.size(), mem_props.data()) != + HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::GetCacheProperties( + uint32_t node_id, uint32_t processor_id, std::vector& cache_props) const { + if (cache_props.empty()) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + if (vhsaKmtGetNodeCacheProperties(node_id, 0, cache_props.size(), cache_props.data()) != + HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::GetDeviceHandle(uint32_t node_id, void** device_handle) const { + assert(device_handle != nullptr); + + if (vhsaKmtGetAMDGPUDeviceHandle(node_id, device_handle) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::GetClockCounters(uint32_t node_id, + HsaClockCounters* clock_counter) const { + assert(clock_counter != nullptr); + + if (vhsaKmtGetClockCounters(node_id, clock_counter) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::SetTrapHandler(uint32_t node_id, const void* base, uint64_t base_size, + const void* buffer_base, + uint64_t buffer_base_size) const { + if (vhsaKmtSetTrapHandler(node_id, const_cast(base), base_size, + const_cast(buffer_base), + buffer_base_size) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::AllocateMemory(const core::MemoryRegion& mem_region, + core::MemoryRegion::AllocateFlags alloc_flags, + void** mem, size_t size, uint32_t agent_node_id) { + const MemoryRegion& m_region(static_cast(mem_region)); + HsaMemFlags kmt_alloc_flags(m_region.mem_flags()); + HSAKMT_STATUS ret; + + kmt_alloc_flags.ui32.ExecuteAccess = + (alloc_flags & core::MemoryRegion::AllocateExecutable ? 1 : 0); + kmt_alloc_flags.ui32.AQLQueueMemory = + (alloc_flags & core::MemoryRegion::AllocateDoubleMap ? 1 : 0); + + if (m_region.IsSystem() && (alloc_flags & core::MemoryRegion::AllocateNonPaged)) { + kmt_alloc_flags.ui32.NonPaged = 1; + } + + if (!m_region.IsLocalMemory() && (alloc_flags & core::MemoryRegion::AllocateMemoryOnly)) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + // Allocating a memory handle for virtual memory + kmt_alloc_flags.ui32.NoAddress = !!(alloc_flags & core::MemoryRegion::AllocateMemoryOnly); + + // Allocate pseudo fine grain memory + kmt_alloc_flags.ui32.CoarseGrain = + (alloc_flags & core::MemoryRegion::AllocatePCIeRW ? 0 : kmt_alloc_flags.ui32.CoarseGrain); + + kmt_alloc_flags.ui32.NoSubstitute = + (alloc_flags & core::MemoryRegion::AllocatePinned ? 1 : kmt_alloc_flags.ui32.NoSubstitute); + + kmt_alloc_flags.ui32.GTTAccess = + (alloc_flags & core::MemoryRegion::AllocateGTTAccess ? 1 : kmt_alloc_flags.ui32.GTTAccess); + + kmt_alloc_flags.ui32.Uncached = + (alloc_flags & core::MemoryRegion::AllocateUncached ? 1 : kmt_alloc_flags.ui32.Uncached); + + if (m_region.IsLocalMemory()) { + // Allocate physically contiguous memory. AllocateKfdMemory function call + // will fail if this flag is not supported in KFD. + kmt_alloc_flags.ui32.Contiguous = + (alloc_flags & core::MemoryRegion::AllocateContiguous ? 1 + : kmt_alloc_flags.ui32.Contiguous); + } + + //// Only allow using the suballocator for ordinary VRAM. + if (m_region.IsLocalMemory() && !kmt_alloc_flags.ui32.NoAddress) { + bool subAllocEnabled = !core::Runtime::runtime_singleton_->flag().disable_fragment_alloc(); + // Avoid modifying executable or queue allocations. + bool useSubAlloc = subAllocEnabled; + useSubAlloc &= ((alloc_flags & (~core::MemoryRegion::AllocateRestrict)) == 0); + + if (useSubAlloc) { + *mem = m_region.fragment_alloc(size); + + if ((alloc_flags & core::MemoryRegion::AllocateAsan)) { + // TODO: Implement ASAN support for VIRTIO driver + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + return HSA_STATUS_SUCCESS; + } + } + + const uint32_t node_id = (alloc_flags & core::MemoryRegion::AllocateGTTAccess) + ? agent_node_id + : m_region.owner()->node_id(); + + //// Allocate memory. + //// If it fails attempt to release memory from the block allocator and retry. + ret = vhsaKmtAllocMemory(node_id, size, kmt_alloc_flags, mem); + if (ret != HSAKMT_STATUS_SUCCESS) { + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + if (*mem == nullptr) { + m_region.owner()->Trim(); + ret = vhsaKmtAllocMemory(node_id, size, kmt_alloc_flags, mem); + if (ret != HSAKMT_STATUS_SUCCESS) { + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + } + + if (*mem != nullptr) { + if (kmt_alloc_flags.ui32.NoAddress) return HSA_STATUS_SUCCESS; + + // Commit the memory. + // For system memory, on non-restricted allocation, map it to all GPUs. On + // restricted allocation, only CPU is allowed to access by default, so + // no need to map + // For local memory, only map it to the owning GPU. Mapping to other GPU, + // if the access is allowed, is performed on AllowAccess. + HsaMemMapFlags map_flag = m_region.map_flags(); + size_t map_node_count = 1; + const uint32_t owner_node_id = m_region.owner()->node_id(); + const uint32_t* map_node_id = &owner_node_id; + + if (m_region.IsSystem()) { + if ((alloc_flags & core::MemoryRegion::AllocateRestrict) == 0) { + // Map to all GPU agents. + map_node_count = core::Runtime::runtime_singleton_->gpu_ids().size(); + + if (map_node_count == 0) { + // No need to pin since no GPU in the platform. + return HSA_STATUS_SUCCESS; + } + + map_node_id = &core::Runtime::runtime_singleton_->gpu_ids()[0]; + } else { + // No need to pin it for CPU exclusive access. + return HSA_STATUS_SUCCESS; + } + } + + uint64_t alternate_va = 0; + const bool is_resident = + (MakeMemoryResident(*mem, size, &alternate_va, &map_flag, map_node_count, map_node_id) == + HSA_STATUS_SUCCESS); + + const bool require_pinning = + (!m_region.full_profile() || m_region.IsLocalMemory() || m_region.IsScratch()); + + if (require_pinning && !is_resident) { + vhsaKmtFreeMemory(*mem, size); + *mem = nullptr; + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + if ((alloc_flags & core::MemoryRegion::AllocateAsan)) { + // TODO: Implement ASAN support for VIRTIO driver + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + return HSA_STATUS_SUCCESS; + } + + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; +} + +hsa_status_t KfdVirtioDriver::FreeMemory(void* mem, size_t size) { + MakeMemoryUnresident(mem); + return vhsaKmtFreeMemory(mem, size) == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS + : HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::AllocateScratchMemory(uint32_t node_id, uint64_t size, + void** mem) const { + assert(mem != nullptr); + assert(size != 0); + + HsaMemFlags flags = {}; + flags.ui32.Scratch = 1; + flags.ui32.HostAccess = 1; + void* ptr = nullptr; + + HSAKMT_STATUS ret = vhsaKmtAllocMemory(node_id, size, flags, &ptr); + if (ret != HSAKMT_STATUS_SUCCESS || ptr == nullptr) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + + *mem = ptr; + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::RegisterMemory(void* ptr, uint64_t size, + HsaMemFlags mem_flags) const { + assert(ptr != nullptr); + assert(size != 0); + + if (vhsaKmtRegisterMemoryWithFlags(ptr, size, mem_flags) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::DeregisterMemory(void* ptr) const { + if (vhsaKmtDeregisterMemory(ptr) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::AvailableMemory(uint32_t node_id, uint64_t* available_size) const { + assert(available_size != nullptr); + + if (vhsaKmtAvailableMemory(node_id, available_size) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::MakeMemoryResident(const void* mem, size_t size, + uint64_t* alternate_va, + const HsaMemMapFlags* mem_flags, + uint32_t num_nodes, const uint32_t* nodes) const { + assert(mem != nullptr); + assert(size != 0); + + if (mem_flags == nullptr && nodes == nullptr) { + if (vhsaKmtMapMemoryToGPU(const_cast(mem), size, alternate_va) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + } else if (mem_flags != nullptr && nodes != nullptr) { + if (vhsaKmtMapMemoryToGPUNodes(const_cast(mem), size, alternate_va, *mem_flags, + num_nodes, + const_cast(nodes)) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + } else { + debug_print("Invalid memory flags ptr:%p nodes ptr:%p\n", mem_flags, nodes); + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::MakeMemoryUnresident(const void* mem) const { + vhsaKmtUnmapMemoryToGPU(const_cast(mem)); + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::CreateQueue(uint32_t node_id, HSA_QUEUE_TYPE type, uint32_t queue_pct, + HSA_QUEUE_PRIORITY priority, uint32_t sdma_engine_id, + void* queue_addr, uint64_t queue_size_bytes, + HsaEvent* event, HsaQueueResource& queue_resource) const { + if (vhsaKmtCreateQueueExt(node_id, type, queue_pct, priority, sdma_engine_id, queue_addr, + queue_size_bytes, event, &queue_resource) != HSAKMT_STATUS_SUCCESS) + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::DestroyQueue(HSA_QUEUEID queue_id) const { + if (vhsaKmtDestroyQueue(queue_id) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::UpdateQueue(HSA_QUEUEID queue_id, uint32_t queue_percentage, + HSA_QUEUE_PRIORITY priority, void* queue_mem, + uint64_t queue_size, HsaEvent* event) const { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::SetQueueCUMask(HSA_QUEUEID queue_id, uint32_t num_cu_mask, + uint32_t* cu_mask) const { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::AllocQueueGWS(HSA_QUEUEID queue_id, uint32_t num_GWS, + uint32_t* GWS) const { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::ExportDMABuf(void* mem, size_t size, int* dmabuf_fd, size_t* offset) { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::ImportDMABuf(int dmabuf_fd, core::Agent& agent, + core::ShareableHandle& handle) { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::Map(core::ShareableHandle handle, void* mem, size_t offset, + size_t size, hsa_access_permission_t perms) { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::Unmap(core::ShareableHandle handle, void* mem, size_t offset, + size_t size) { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::ReleaseShareableHandle(core::ShareableHandle& handle) { + return HSA_STATUS_ERROR; +} + +hsa_status_t KfdVirtioDriver::GetTileConfig(uint32_t node_id, HsaGpuTileConfig* config) const { + if (vhsaKmtGetTileConfig(node_id, config) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::SPMAcquire(uint32_t node_id) const { return HSA_STATUS_ERROR; } + +hsa_status_t KfdVirtioDriver::SPMRelease(uint32_t node_id) const { return HSA_STATUS_ERROR; } + +hsa_status_t KfdVirtioDriver::SPMSetDestBuffer(uint32_t node_id, uint32_t size, uint32_t* timeout, + uint32_t* size_copied, void* dest, + bool* is_data_loss) const { + return HSA_STATUS_ERROR; +} + + +hsa_status_t KfdVirtioDriver::OpenSMI(uint32_t node_id, int* fd) const { return HSA_STATUS_ERROR; } + +hsa_status_t KfdVirtioDriver::GetWallclockFrequency(uint32_t node_id, uint64_t* frequency) const { + assert(frequency != nullptr); + + amdgpu_gpu_info info; + amdgpu_device_handle handle; + if (GetDeviceHandle(node_id, reinterpret_cast(&handle)) != HSA_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + if (vamdgpu_query_gpu_info(handle, &info) < 0) return HSA_STATUS_ERROR; + + // Reported by libdrm in KHz. + *frequency = uint64_t(info.gpu_counter_freq) * 1000ull; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t KfdVirtioDriver::IsModelEnabled(bool* enable) const { + *enable = false; + return HSA_STATUS_SUCCESS; +} + +} // namespace AMD +} // namespace rocr diff --git a/runtime/hsa-runtime/core/inc/amd_virtio_driver.h b/runtime/hsa-runtime/core/inc/amd_virtio_driver.h new file mode 100644 index 0000000000..fd229e94f6 --- /dev/null +++ b/runtime/hsa-runtime/core/inc/amd_virtio_driver.h @@ -0,0 +1,124 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef HSA_RUNTIME_CORE_INC_AMD_VIRTIO_DRIVER_H_ +#define HSA_RUNTIME_CORE_INC_AMD_VIRTIO_DRIVER_H_ + +#include +#include + +#include "hsakmt/hsakmt.h" + +#include "core/inc/driver.h" +#include "core/inc/memory_region.h" + +namespace rocr { +namespace AMD { + +class KfdVirtioDriver final : public core::Driver { + public: + KfdVirtioDriver(std::string devnode_name); + + static hsa_status_t DiscoverDriver(std::unique_ptr& driver); + + hsa_status_t Init() override; + hsa_status_t ShutDown() override; + hsa_status_t QueryKernelModeDriver(core::DriverQuery query) override; + hsa_status_t Open() override; + hsa_status_t Close() override; + hsa_status_t GetSystemProperties(HsaSystemProperties& sys_props) const override; + hsa_status_t GetNodeProperties(HsaNodeProperties& node_props, uint32_t node_id) const override; + hsa_status_t GetEdgeProperties(std::vector& io_link_props, + uint32_t node_id) const override; + hsa_status_t GetMemoryProperties(uint32_t node_id, + std::vector& mem_props) const override; + hsa_status_t GetCacheProperties(uint32_t node_id, uint32_t processor_id, + std::vector& cache_props) const override; + hsa_status_t GetDeviceHandle(uint32_t node_id, void** device_handle) const; + hsa_status_t GetClockCounters(uint32_t node_id, HsaClockCounters* clock_counter) const; + hsa_status_t SetTrapHandler(uint32_t node_id, const void* base, uint64_t base_size, + const void* buffer_base, uint64_t buffer_base_size) const; + hsa_status_t AllocateMemory(const core::MemoryRegion& mem_region, + core::MemoryRegion::AllocateFlags alloc_flags, void** mem, + size_t size, uint32_t agent_node_id) override; + hsa_status_t FreeMemory(void* mem, size_t size) override; + hsa_status_t AllocateScratchMemory(uint32_t node_id, uint64_t size, void** mem) const; + hsa_status_t RegisterMemory(void* ptr, uint64_t size, HsaMemFlags mem_flags) const override; + hsa_status_t DeregisterMemory(void* ptr) const override; + hsa_status_t AvailableMemory(uint32_t node_id, uint64_t* available_size) const; + hsa_status_t MakeMemoryResident(const void* mem, size_t size, uint64_t* alternate_va, + const HsaMemMapFlags* mem_flags, uint32_t num_nodes, + const uint32_t* nodes) const override; + hsa_status_t MakeMemoryUnresident(const void* mem) const override; + hsa_status_t CreateQueue(uint32_t node_id, HSA_QUEUE_TYPE type, uint32_t queue_pct, + HSA_QUEUE_PRIORITY priority, uint32_t sdma_engine_id, void* queue_addr, + uint64_t queue_size_bytes, HsaEvent* event, + HsaQueueResource& queue_resource) const override; + hsa_status_t DestroyQueue(HSA_QUEUEID queue_id) const override; + hsa_status_t UpdateQueue(HSA_QUEUEID queue_id, uint32_t queue_percentage, + HSA_QUEUE_PRIORITY priority, void* queue_mem, uint64_t queue_size, + HsaEvent* event) const override; + hsa_status_t SetQueueCUMask(HSA_QUEUEID queue_id, uint32_t num_cu_mask, + uint32_t* cu_mask) const override; + hsa_status_t AllocQueueGWS(HSA_QUEUEID queue_id, uint32_t num_GWS, uint32_t* GWS) const override; + hsa_status_t ExportDMABuf(void* mem, size_t size, int* dmabuf_fd, size_t* offset) override; + hsa_status_t ImportDMABuf(int dmabuf_fd, core::Agent& agent, + core::ShareableHandle& handle) override; + hsa_status_t Map(core::ShareableHandle handle, void* mem, size_t offset, size_t size, + hsa_access_permission_t perms) override; + hsa_status_t Unmap(core::ShareableHandle handle, void* mem, size_t offset, size_t size) override; + hsa_status_t ReleaseShareableHandle(core::ShareableHandle& handle) override; + hsa_status_t GetTileConfig(uint32_t node_id, HsaGpuTileConfig* config) const; + hsa_status_t SPMAcquire(uint32_t node_id) const override; + hsa_status_t SPMRelease(uint32_t node_id) const override; + hsa_status_t SPMSetDestBuffer(uint32_t node_id, uint32_t size, uint32_t* timeout, + uint32_t* size_copied, void* dest, + bool* is_data_loss) const override; + hsa_status_t OpenSMI(uint32_t node_id, int* fd) const override; + hsa_status_t GetWallclockFrequency(uint32_t node_id, uint64_t* frequency) const; + hsa_status_t IsModelEnabled(bool* enable) const override; +}; + +} // namespace AMD +} // namespace rocr + +#endif // HSA_RUNTIME_CORE_INC_AMD_VIRTIO_DRIVER_H_ diff --git a/runtime/hsa-runtime/core/inc/driver.h b/runtime/hsa-runtime/core/inc/driver.h index 84d07f4041..c4cdc19f24 100644 --- a/runtime/hsa-runtime/core/inc/driver.h +++ b/runtime/hsa-runtime/core/inc/driver.h @@ -58,7 +58,7 @@ class Queue; enum class DriverQuery { GET_DRIVER_VERSION }; -enum class DriverType { XDNA = 0, KFD, NUM_DRIVER_TYPES }; +enum class DriverType { XDNA = 0, KFD, KFD_VIRTIO, NUM_DRIVER_TYPES }; /// @brief Handle for exported / imported memory. struct ShareableHandle { diff --git a/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/runtime/hsa-runtime/core/runtime/amd_topology.cpp index 55b75591e6..29956d9423 100644 --- a/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -68,6 +68,9 @@ #include "core/inc/amd_memory_region.h" #include "core/inc/runtime.h" #include "core/util/utils.h" +#ifdef HSAKMT_VIRTIO_ENABLED +#include "core/inc/amd_virtio_driver.h" +#endif extern r_debug _amdgpu_r_debug; @@ -78,13 +81,21 @@ namespace { #if _WIN32 constexpr size_t num_drivers = 0; #elif __linux__ -constexpr size_t num_drivers = 2; +constexpr size_t num_drivers = 2 +#ifdef HSAKMT_VIRTIO_ENABLED + + 1 +#endif + ; #endif const std::array&)>, num_drivers> discover_driver_funcs = { #ifdef __linux__ - KfdDriver::DiscoverDriver, XdnaDriver::DiscoverDriver + KfdDriver::DiscoverDriver, + XdnaDriver::DiscoverDriver, +#ifdef HSAKMT_VIRTIO_ENABLED + KfdVirtioDriver::DiscoverDriver, +#endif #endif }; From 20806577ce633ca63f7647fc7f2225f6945631e4 Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Tue, 8 Jul 2025 11:01:05 +0800 Subject: [PATCH 06/13] rocr: support multiple driver types in agent initialization Modify agent initialization to support different driver types, to enable KFD_VIRTIO dirver for CPU and GPU agent here. 1. Add driver_type parameter to CpuAgent and GpuAgent constructors 2. Update topology discovery to handle multiple driver types 3. Fix MakeMemoryResident return value check in VirtioDriver 4. Add helper function IsGPUDriver to check driver types 5. Update agent discovery to iterate through all available drivers This change makes the runtime more flexible by removing hardcoded KFD driver assumptions and properly handling different driver backends. Signed-off-by: Honglei Huang --- runtime/hsa-runtime/core/inc/amd_cpu_agent.h | 5 +- runtime/hsa-runtime/core/inc/amd_gpu_agent.h | 14 ++-- runtime/hsa-runtime/core/inc/driver.h | 9 ++- runtime/hsa-runtime/core/inc/runtime.h | 8 +++ .../core/runtime/amd_cpu_agent.cpp | 7 +- .../core/runtime/amd_gpu_agent.cpp | 4 +- .../hsa-runtime/core/runtime/amd_topology.cpp | 64 ++++++++++--------- 7 files changed, 67 insertions(+), 44 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/amd_cpu_agent.h b/runtime/hsa-runtime/core/inc/amd_cpu_agent.h index 2c3f9afa4d..bfa080cf8c 100644 --- a/runtime/hsa-runtime/core/inc/amd_cpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_cpu_agent.h @@ -51,6 +51,7 @@ #include "core/inc/agent.h" #include "core/inc/queue.h" #include "core/inc/cache.h" +#include "core/inc/driver.h" namespace rocr { namespace AMD { @@ -62,7 +63,9 @@ class CpuAgent : public core::Agent { // @param [in] node Node id. Each CPU in different socket will get distinct // id. // @param [in] node_props Node property. - CpuAgent(HSAuint32 node, const HsaNodeProperties& node_props); + // @param [in] driver_type Driver type. Default is KFD. + CpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, + core::DriverType driver_type = core::DriverType::KFD); // @brief CpuAgent destructor. ~CpuAgent(); diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index f7dd5d26c6..e45a165e97 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -73,10 +73,11 @@ typedef ScratchCache::ScratchInfo ScratchInfo; class GpuAgentInt : public core::Agent { public: // @brief Constructor - GpuAgentInt(uint32_t node_id) - : core::Agent(core::Runtime::runtime_singleton_->AgentDriver( - core::DriverType::KFD), - node_id, core::Agent::DeviceType::kAmdGpuDevice) {} + // @param [in] node_id Node id. + // @param [in] driver_type Driver type. Default is KFD. + GpuAgentInt(uint32_t node_id, core::DriverType driver_type) + : core::Agent(core::Runtime::runtime_singleton_->AgentDriver(driver_type), node_id, + core::Agent::DeviceType::kAmdGpuDevice) {} // @brief Ensure blits are ready (performance hint). virtual void PreloadBlits() {} @@ -231,7 +232,10 @@ class GpuAgent : public GpuAgentInt { // id. // @param [in] node_props Node property. // @param [in] xnack_mode XNACK mode of device. - GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xnack_mode, uint32_t index); + // @param [in] index Index of the GPU device. + // @param [in] driver_type Driver type. Default is KFD. + GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xnack_mode, uint32_t index, + core::DriverType driver_type = core::DriverType::KFD); // @brief GPU agent destructor. ~GpuAgent(); diff --git a/runtime/hsa-runtime/core/inc/driver.h b/runtime/hsa-runtime/core/inc/driver.h index c4cdc19f24..3a4082e24f 100644 --- a/runtime/hsa-runtime/core/inc/driver.h +++ b/runtime/hsa-runtime/core/inc/driver.h @@ -58,7 +58,14 @@ class Queue; enum class DriverQuery { GET_DRIVER_VERSION }; -enum class DriverType { XDNA = 0, KFD, KFD_VIRTIO, NUM_DRIVER_TYPES }; +enum class DriverType { + XDNA = 0, + KFD, +#ifdef HSAKMT_VIRTIO_ENABLED + KFD_VIRTIO, +#endif + NUM_DRIVER_TYPES +}; /// @brief Handle for exported / imported memory. struct ShareableHandle { diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index c47a5f2d25..9ba67f34d3 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -510,6 +510,14 @@ class Runtime { std::vector>& AgentDrivers() { return agent_drivers_; } + static bool IsGPUDriver(DriverType driver_type) { + return driver_type == core::DriverType::KFD +#ifdef HSAKMT_VIRTIO_ENABLED + || driver_type == core::DriverType::KFD_VIRTIO +#endif + ; + } + protected: static void AsyncEventsLoop(void*); static void AsyncIPCSockServerConnLoop(void*); diff --git a/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp index 5cd3cb8a4f..37eda03d85 100644 --- a/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp @@ -54,10 +54,9 @@ namespace rocr { namespace AMD { -CpuAgent::CpuAgent(HSAuint32 node, const HsaNodeProperties &node_props) - : core::Agent( - core::Runtime::runtime_singleton_->AgentDriver(core::DriverType::KFD), - node, kAmdCpuDevice), +CpuAgent::CpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, + core::DriverType driver_type) + : core::Agent(core::Runtime::runtime_singleton_->AgentDriver(driver_type), node, kAmdCpuDevice), properties_(node_props) { InitRegionList(); diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 7273bbd9b2..3b830af294 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -93,8 +93,8 @@ namespace AMD { const uint64_t CP_DMA_DATA_TRANSFER_CNT_MAX = (1 << 26); GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xnack_mode, - uint32_t index) - : GpuAgentInt(node), + uint32_t index, core::DriverType driver_type) + : GpuAgentInt(node, driver_type), properties_(node_props), current_coherency_type_(HSA_AMD_COHERENCY_TYPE_COHERENT), scratch_used_large_(0), diff --git a/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/runtime/hsa-runtime/core/runtime/amd_topology.cpp index 29956d9423..5a9a909662 100644 --- a/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -78,17 +78,14 @@ namespace rocr { namespace AMD { // Anonymous namespace. namespace { -#if _WIN32 -constexpr size_t num_drivers = 0; -#elif __linux__ -constexpr size_t num_drivers = 2 -#ifdef HSAKMT_VIRTIO_ENABLED - + 1 -#endif - ; -#endif -const std::array&)>, num_drivers> +const std::array&)>, +#if _WIN32 + 0 +#elif __linux__ + static_cast(core::DriverType::NUM_DRIVER_TYPES) +#endif + > discover_driver_funcs = { #ifdef __linux__ KfdDriver::DiscoverDriver, @@ -121,14 +118,14 @@ bool InitializeDriver(std::unique_ptr& driver) { return true; } -void DiscoverCpu(HSAuint32 node_id, HsaNodeProperties& node_prop) { - CpuAgent* cpu = new CpuAgent(node_id, node_prop); +void DiscoverCpu(HSAuint32 node_id, HsaNodeProperties& node_prop, core::DriverType driver_type) { + CpuAgent* cpu = new CpuAgent(node_id, node_prop, driver_type); cpu->Enable(); core::Runtime::runtime_singleton_->RegisterAgent(cpu, true); } GpuAgent* DiscoverGpu(HSAuint32 node_id, HsaNodeProperties& node_prop, bool xnack_mode, - bool enabled) { + bool enabled, core::DriverType driver_type) { GpuAgent* gpu = nullptr; if (node_prop.NumFComputeCores == 0) { // Ignore non GPUs. @@ -136,7 +133,7 @@ GpuAgent* DiscoverGpu(HSAuint32 node_id, HsaNodeProperties& node_prop, bool xnac } try { gpu = new GpuAgent(node_id, node_prop, xnack_mode, - core::Runtime::runtime_singleton_->gpu_agents().size()); + core::Runtime::runtime_singleton_->gpu_agents().size(), driver_type); const HsaVersionInfo& kfd_version = core::Runtime::runtime_singleton_->KfdVersion().version; @@ -163,7 +160,7 @@ GpuAgent* DiscoverGpu(HSAuint32 node_id, HsaNodeProperties& node_prop, bool xnac node_prop.Capability.ui32.SRAM_EDCSupport = 1; delete gpu; gpu = new GpuAgent(node_id, node_prop, xnack_mode, - core::Runtime::runtime_singleton_->gpu_agents().size()); + core::Runtime::runtime_singleton_->gpu_agents().size(), driver_type); } } } catch (const hsa_exception& e) { @@ -268,24 +265,29 @@ void SurfaceGpuList(std::vector& gpu_list, bool xnack_mode, bool enable const int32_t invalidIdx = -1; int32_t list_sz = gpu_list.size(); HsaNodeProperties node_prop = {0}; - const auto& gpu_driver = core::Runtime::runtime_singleton_->AgentDriver(core::DriverType::KFD); - for (int32_t idx = 0; idx < list_sz; idx++) { - if (gpu_list[idx] == invalidIdx) { - break; + for (const auto& gpu_driver : core::Runtime::runtime_singleton_->AgentDrivers()) { + if (!core::Runtime::IsGPUDriver(gpu_driver->kernel_driver_type_)) { + continue; } - // Obtain properties of the node - hsa_status_t ret = gpu_driver.GetNodeProperties(node_prop, gpu_list[idx]); - assert(ret == HSA_STATUS_SUCCESS && "Error in getting Node Properties"); + for (int32_t idx = 0; idx < list_sz; idx++) { + if (gpu_list[idx] == invalidIdx) { + break; + } - // disable interrupt signal for DTIF platform - if (core::Runtime::runtime_singleton_->flag().enable_dtif()) - core::g_use_interrupt_wait = false; + // Obtain properties of the node + hsa_status_t ret = gpu_driver->GetNodeProperties(node_prop, gpu_list[idx]); + assert(ret == HSA_STATUS_SUCCESS && "Error in getting Node Properties"); - // Instantiate a Gpu device. The IO links - // of this node have already been registered - assert((node_prop.NumFComputeCores != 0) && "Improper node used for GPU device discovery."); - DiscoverGpu(gpu_list[idx], node_prop, xnack_mode, enabled); + // disable interrupt signal for DTIF platform + if (core::Runtime::runtime_singleton_->flag().enable_dtif()) + core::g_use_interrupt_wait = false; + + // Instantiate a Gpu device. The IO links + // of this node have already been registered + assert((node_prop.NumFComputeCores != 0) && "Improper node used for GPU device discovery."); + DiscoverGpu(gpu_list[idx], node_prop, xnack_mode, enabled, gpu_driver->kernel_driver_type_); + } } } @@ -346,7 +348,7 @@ bool BuildTopology() { /// @todo: Add support for AIEs. // Query if env ROCR_VISIBLE_DEVICES is defined. If defined // determine number and order of GPU devices to be surfaced. - if (filter && driver->kernel_driver_type_ == core::DriverType::KFD) { + if (filter && (core::Runtime::IsGPUDriver(driver->kernel_driver_type_))) { rvdFilter.BuildRvdTokenList(); rvdFilter.BuildDeviceUuidList(node_props_vec); visibleCnt = rvdFilter.BuildUsrDeviceList(); @@ -361,7 +363,7 @@ bool BuildTopology() { for (auto& node_props : node_props_vec) { if (node_props.NumCPUCores) { // Node has CPU cores so instantiate a CPU agent. - DiscoverCpu(node_id, node_props); + DiscoverCpu(node_id, node_props, driver->kernel_driver_type_); } if (node_props.NumNeuralCores) { From 5285c24657c36f6ecb3efa7a7335149b443c1258 Mon Sep 17 00:00:00 2001 From: Tony Gutierrez Date: Thu, 24 Jul 2025 16:33:34 -0700 Subject: [PATCH 07/13] rocr: Remove unused member of GPUAgent The ape1_size_ member was leftover after the removal of KV and is no longer used. Remove it to remove some compiler warnings. Signed-off-by: Tony Gutierrez --- runtime/hsa-runtime/core/inc/amd_gpu_agent.h | 5 +---- runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index e45a165e97..d49c4fdd8a 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -76,7 +76,7 @@ class GpuAgentInt : public core::Agent { // @param [in] node_id Node id. // @param [in] driver_type Driver type. Default is KFD. GpuAgentInt(uint32_t node_id, core::DriverType driver_type) - : core::Agent(core::Runtime::runtime_singleton_->AgentDriver(driver_type), node_id, + : core::Agent(core::Runtime::runtime_singleton_->AgentDriver(driver_type), node_id, core::Agent::DeviceType::kAmdGpuDevice) {} // @brief Ensure blits are ready (performance hint). @@ -725,9 +725,6 @@ class GpuAgent : public GpuAgentInt { // @brief Alternative aperture base address. Only on KV. uintptr_t ape1_base_; - // @brief Alternative aperture size. Only on KV. - size_t ape1_size_; - // @brief Queue with GWS access. struct { lazy_ptr queue_; diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 3b830af294..fdcff3a18f 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -106,7 +106,6 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna memory_max_frequency_(0), enum_index_(index), ape1_base_(0), - ape1_size_(0), pending_copy_req_ref_(0), pending_copy_stat_check_ref_(0), sdma_blit_used_mask_(0), From d3f70910e1569e5a7ca53ae39b7caaa91fc8f3bc Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Mon, 7 Jul 2025 22:12:20 +0000 Subject: [PATCH 08/13] rocr: Remove SDMA code for gfx7 and gfx8 Remove deprecated SDMA code for gfx7 and gfx8 asics --- runtime/hsa-runtime/core/inc/amd_blit_sdma.h | 36 +-- .../core/runtime/amd_blit_sdma.cpp | 250 +++++++----------- .../core/runtime/amd_gpu_agent.cpp | 4 - 3 files changed, 105 insertions(+), 185 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/amd_blit_sdma.h b/runtime/hsa-runtime/core/inc/amd_blit_sdma.h index a5e62ca273..82d9dbeb16 100644 --- a/runtime/hsa-runtime/core/inc/amd_blit_sdma.h +++ b/runtime/hsa-runtime/core/inc/amd_blit_sdma.h @@ -73,11 +73,7 @@ class BlitSdmaBase : public core::Blit { core::Signal& out_signal) = 0; }; -// RingIndexTy: 32/64-bit monotonic ring index, counting in bytes. -// HwIndexMonotonic: true if SDMA HW index is monotonic, false if it wraps at end of ring. -// SizeToCountOffset: value added to size (in bytes) to form SDMA command count field. -template -class BlitSdma : public BlitSdmaBase { +template class BlitSdma : public BlitSdmaBase { public: BlitSdma(); @@ -163,9 +159,9 @@ class BlitSdma : public BlitSdmaBase { /// could be written. NULL if input size is greater than the size of queue /// buffer. - char* AcquireWriteAddress(uint32_t cmd_size, RingIndexTy& curr_index); + char* AcquireWriteAddress(uint32_t cmd_size, uint64_t& curr_index); - void UpdateWriteAndDoorbellRegister(RingIndexTy curr_index, RingIndexTy new_index); + void UpdateWriteAndDoorbellRegister(uint64_t curr_index, uint64_t new_index); /// @brief Updates the Write Register of compute device to the end of /// SDMA packet written into queue buffer. The update to Write Register @@ -178,16 +174,16 @@ class BlitSdma : public BlitSdmaBase { /// @param curr_index Index passed back from AcquireWriteAddress. /// /// @param cmd_size Command packet size in bytes. - void ReleaseWriteAddress(RingIndexTy curr_index, uint32_t cmd_size); + void ReleaseWriteAddress(uint64_t curr_index, uint32_t cmd_size); /// @brief Writes NO-OP words into queue buffer in case writing a command /// causes the queue buffer to wrap. /// /// @param curr_index Index to begin padding from. - void PadRingToEnd(RingIndexTy curr_index); + void PadRingToEnd(uint64_t curr_index); - uint32_t WrapIntoRing(RingIndexTy index); - bool CanWriteUpto(RingIndexTy upto_index); + uint32_t WrapIntoRing(uint64_t index); + bool CanWriteUpto(uint64_t upto_index); /// @brief Build fence command void BuildFenceCommand(char* fence_command_addr, uint32_t* fence, @@ -265,8 +261,8 @@ class BlitSdma : public BlitSdmaBase { HsaQueueResource queue_resource_; // Monotonic ring indices, in bytes, tracking written and submitted commands. - RingIndexTy cached_reserve_index_; - RingIndexTy cached_commit_index_; + uint64_t cached_reserve_index_; + uint64_t cached_commit_index_; static const uint32_t linear_copy_command_size_; @@ -314,21 +310,11 @@ class BlitSdma : public BlitSdmaBase { size_t min_submission_size_; }; -// Ring indices are 32-bit. -// HW ring indices are not monotonic (wrap at end of ring). -// Count fields of SDMA commands are 0-based. -typedef BlitSdma BlitSdmaV2V3; -// Ring indices are 64-bit. -// HW ring indices are monotonic (do not wrap at end of ring). -// Count fields of SDMA commands are 1-based. -typedef BlitSdma BlitSdmaV4; +typedef BlitSdma BlitSdmaV4; -// Ring indices are 64-bit. -// HW ring indices are monotonic (do not wrap at end of ring). -// Count fields of SDMA commands are 1-based. // SDMA is connected to gL2. -typedef BlitSdma BlitSdmaV5; +typedef BlitSdma BlitSdmaV5; } // namespace amd } // namespace rocr diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index 71b46d3447..0875994cbb 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -77,44 +77,33 @@ const size_t BlitSdmaBase::kMaxSingleCopySize = SDMA_PKT_COPY_LINEAR::kMaxSize_; const size_t BlitSdmaBase::kMaxSingleFillSize = SDMA_PKT_CONSTANT_FILL::kMaxSize_; // Initialize size of various sDMA commands use by this module -template -const uint32_t BlitSdma::linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR); +template +const uint32_t BlitSdma::linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR); -template -const uint32_t BlitSdma::fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL); +template +const uint32_t BlitSdma::fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL); -template -const uint32_t BlitSdma::fence_command_size_ = sizeof(SDMA_PKT_FENCE); +template +const uint32_t BlitSdma::fence_command_size_ = sizeof(SDMA_PKT_FENCE); -template -const uint32_t BlitSdma::poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); +template +const uint32_t BlitSdma::poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); -template -const uint32_t BlitSdma::flush_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); +template +const uint32_t BlitSdma::flush_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); -template -const uint32_t BlitSdma::atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC); +template +const uint32_t BlitSdma::atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC); -template -const uint32_t BlitSdma::timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP); +template +const uint32_t BlitSdma::timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP); -template -const uint32_t BlitSdma::trap_command_size_ = sizeof(SDMA_PKT_TRAP); +template const uint32_t BlitSdma::trap_command_size_ = sizeof(SDMA_PKT_TRAP); -template -const uint32_t BlitSdma::gcr_command_size_ = sizeof(SDMA_PKT_GCR); +template const uint32_t BlitSdma::gcr_command_size_ = sizeof(SDMA_PKT_GCR); -template -BlitSdma::BlitSdma() +template +BlitSdma::BlitSdma() : agent_(NULL), queue_start_addr_(NULL), bytes_queued_(0), @@ -129,12 +118,11 @@ BlitSdma::BlitSdma() std::memset(&queue_resource_, 0, sizeof(queue_resource_)); } -template -BlitSdma::~BlitSdma() {} +template BlitSdma::~BlitSdma() {} -template -hsa_status_t BlitSdma::Initialize( - const core::Agent& agent, bool use_xgmi, size_t linear_copy_size_override, int rec_eng) { +template +hsa_status_t BlitSdma::Initialize(const core::Agent& agent, bool use_xgmi, + size_t linear_copy_size_override, int rec_eng) { if (queue_start_addr_ != NULL) { // Already initialized. return HSA_STATUS_SUCCESS; @@ -201,7 +189,7 @@ hsa_status_t BlitSdma: return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } - cached_reserve_index_ = *reinterpret_cast(queue_resource_.Queue_write_ptr); + cached_reserve_index_ = *reinterpret_cast(queue_resource_.Queue_write_ptr); cached_commit_index_ = cached_reserve_index_; if (core::g_use_interrupt_wait) { @@ -218,9 +206,7 @@ hsa_status_t BlitSdma: return HSA_STATUS_SUCCESS; } -template -hsa_status_t BlitSdma::Destroy( - const core::Agent& agent) { +template hsa_status_t BlitSdma::Destroy(const core::Agent& agent) { // Release all allocated resources and reset them to zero. if (queue_resource_.QueueId != 0) { @@ -245,9 +231,8 @@ hsa_status_t BlitSdma: return HSA_STATUS_SUCCESS; } -template -hsa_status_t BlitSdma::SubmitBlockingCommand(const void* cmd, size_t cmd_size, +template +hsa_status_t BlitSdma::SubmitBlockingCommand(const void* cmd, size_t cmd_size, uint64_t size) { ScopedAcquire lock(&lock_); @@ -278,11 +263,11 @@ hsa_status_t BlitSdma -hsa_status_t BlitSdma::SubmitCommand( - const void* cmd, size_t cmd_size, uint64_t size, const std::vector& dep_signals, - core::Signal& out_signal, std::vector& gang_signals) { - +template +hsa_status_t BlitSdma::SubmitCommand(const void* cmd, size_t cmd_size, uint64_t size, + const std::vector& dep_signals, + core::Signal& out_signal, + std::vector& gang_signals) { uint32_t num_poll_command = 0; // Cached copy of dep_signals[i]->LoadRelaxed @@ -355,9 +340,7 @@ hsa_status_t BlitSdma: // Add space for acquire or release Hdp flush command uint32_t flush_cmd_size = 0; if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { - if ((HwIndexMonotonic) && (hdp_flush_support_)) { - flush_cmd_size = flush_command_size_; - } + if (hdp_flush_support_) flush_cmd_size = flush_command_size_; } // Add space for cache flush. @@ -368,7 +351,7 @@ hsa_status_t BlitSdma: const uint32_t pad_size = total_command_size < min_submission_size_ ? min_submission_size_ - total_command_size : 0; - RingIndexTy curr_index; + uint64_t curr_index; char* command_addr; uint64_t prior_bytes, post_bytes; { @@ -426,7 +409,7 @@ hsa_status_t BlitSdma: // Issue a Hdp flush cmd if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { - if ((HwIndexMonotonic) && (hdp_flush_support_)) { + if (hdp_flush_support_) { BuildHdpFlushCommand(command_addr); command_addr += flush_command_size_; bytes_written_[wrapped_index] = prior_bytes; @@ -542,9 +525,8 @@ hsa_status_t BlitSdma: return HSA_STATUS_SUCCESS; } -template -hsa_status_t BlitSdma::SubmitLinearCopyCommand(void* dst, const void* src, size_t size) { +template +hsa_status_t BlitSdma::SubmitLinearCopyCommand(void* dst, const void* src, size_t size) { // Break the copy into multiple copy operation incase the copy size exceeds // the SDMA linear copy limit. const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ : @@ -557,9 +539,8 @@ hsa_status_t BlitSdma -hsa_status_t BlitSdma::SubmitLinearCopyCommand(void* dst, const void* src, size_t size, +template +hsa_status_t BlitSdma::SubmitLinearCopyCommand(void* dst, const void* src, size_t size, std::vector& dep_signals, core::Signal& out_signal, std::vector& gang_signals) { @@ -577,9 +558,8 @@ hsa_status_t BlitSdma -hsa_status_t -BlitSdma::SubmitCopyRectCommand( +template +hsa_status_t BlitSdma::SubmitCopyRectCommand( const hsa_pitched_ptr_t* dst, const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, const hsa_dim3_t* src_offset, const hsa_dim3_t* range, std::vector& dep_signals, core::Signal& out_signal) { @@ -653,9 +633,8 @@ BlitSdma::SubmitCopyRe out_signal, gang_signals); } -template -hsa_status_t BlitSdma::SubmitLinearFillCommand(void* ptr, uint32_t value, size_t count) { +template +hsa_status_t BlitSdma::SubmitLinearFillCommand(void* ptr, uint32_t value, size_t count) { const size_t size = count * sizeof(uint32_t); const uint32_t num_fill_command = (size + kMaxSingleFillSize - 1) / kMaxSingleFillSize; @@ -666,15 +645,12 @@ hsa_status_t BlitSdma -hsa_status_t BlitSdma::EnableProfiling( - bool enable) { +template hsa_status_t BlitSdma::EnableProfiling(bool enable) { return HSA_STATUS_SUCCESS; } -template -char* BlitSdma::AcquireWriteAddress( - uint32_t cmd_size, RingIndexTy& curr_index) { +template +char* BlitSdma::AcquireWriteAddress(uint32_t cmd_size, uint64_t& curr_index) { // Ring is full when all but one byte is written. if (cmd_size >= kQueueSize) { return nullptr; @@ -692,7 +668,7 @@ char* BlitSdma::Acquir } // Check whether the engine has finished using this region. - const RingIndexTy new_index = curr_index + cmd_size; + const uint64_t new_index = curr_index + cmd_size; if (CanWriteUpto(new_index) == false) { // Wait for read index to move and try again. @@ -713,10 +689,8 @@ char* BlitSdma::Acquir return nullptr; } -template -void BlitSdma::UpdateWriteAndDoorbellRegister(RingIndexTy curr_index, - RingIndexTy new_index) { +template +void BlitSdma::UpdateWriteAndDoorbellRegister(uint64_t curr_index, uint64_t new_index) { while (true) { // Make sure that the address before ::curr_index is already released. // Otherwise the CP may read invalid packets. @@ -725,21 +699,19 @@ void BlitSdma(queue_resource_.Queue_read_ptr)) != + while (WrapIntoRing(*reinterpret_cast(queue_resource_.Queue_read_ptr)) != WrapIntoRing(curr_index)) { os::YieldThread(); } } // Update write pointer and doorbell register. - *reinterpret_cast(queue_resource_.Queue_write_ptr) = - (HwIndexMonotonic ? new_index : WrapIntoRing(new_index)); + *reinterpret_cast(queue_resource_.Queue_write_ptr) = new_index; // Ensure write pointer is visible to GPU before doorbell. std::atomic_thread_fence(std::memory_order_release); - *reinterpret_cast(queue_resource_.Queue_DoorBell) = - (HwIndexMonotonic ? new_index : WrapIntoRing(new_index)); + *reinterpret_cast(queue_resource_.Queue_DoorBell) = new_index; atomic::Store(&cached_commit_index_, new_index, std::memory_order_release); break; @@ -750,9 +722,8 @@ void BlitSdma -void BlitSdma::ReleaseWriteAddress( - RingIndexTy curr_index, uint32_t cmd_size) { +template +void BlitSdma::ReleaseWriteAddress(uint64_t curr_index, uint32_t cmd_size) { if (cmd_size > kQueueSize) { assert(false && "cmd_addr is outside the queue buffer range"); return; @@ -761,11 +732,9 @@ void BlitSdma::Release UpdateWriteAndDoorbellRegister(curr_index, curr_index + cmd_size); } -template -void BlitSdma::PadRingToEnd( - RingIndexTy curr_index) { +template void BlitSdma::PadRingToEnd(uint64_t curr_index) { // Reserve region from here to the end of the ring. - RingIndexTy new_index = curr_index + (kQueueSize - WrapIntoRing(curr_index)); + uint64_t new_index = curr_index + (kQueueSize - WrapIntoRing(curr_index)); // Check whether the engine has finished using this region. if (CanWriteUpto(new_index) == false) { @@ -786,37 +755,22 @@ void BlitSdma::PadRing } } -template -uint32_t BlitSdma::WrapIntoRing( - RingIndexTy index) { +template uint32_t BlitSdma::WrapIntoRing(uint64_t index) { return index & (kQueueSize - 1); } -template -bool BlitSdma::CanWriteUpto( - RingIndexTy upto_index) { +template bool BlitSdma::CanWriteUpto(uint64_t upto_index) { // Get/calculate the monotonic read index. - RingIndexTy hw_read_index = *reinterpret_cast(queue_resource_.Queue_read_ptr); - RingIndexTy read_index; - - if (HwIndexMonotonic) { - read_index = hw_read_index; - } else { - // Calculate distance from commit index to HW read index. - // Commit index is always < kQueueSize away from HW read index. - RingIndexTy commit_index = atomic::Load(&cached_commit_index_, std::memory_order_relaxed); - RingIndexTy dist_to_read_index = WrapIntoRing(commit_index - hw_read_index); - read_index = commit_index - dist_to_read_index; - } + uint64_t hw_read_index = *reinterpret_cast(queue_resource_.Queue_read_ptr); // Check whether the read pointer has passed the given index. // At most we can submit (kQueueSize - 1) bytes at a time. - return (upto_index - read_index) < kQueueSize; + return (upto_index - hw_read_index) < kQueueSize; } -template -void BlitSdma::BuildFenceCommand( - char* fence_command_addr, uint32_t* fence, uint32_t fence_value) { +template +void BlitSdma::BuildFenceCommand(char* fence_command_addr, uint32_t* fence, + uint32_t fence_value) { assert(fence_command_addr != NULL); SDMA_PKT_FENCE* packet_addr = reinterpret_cast(fence_command_addr); @@ -836,9 +790,9 @@ void BlitSdma::BuildFe packet_addr->DATA_UNION.data = fence_value; } -template -void BlitSdma::BuildCopyCommand( - char* cmd_addr, uint32_t num_copy_command, void* dst, const void* src, size_t size) { +template +void BlitSdma::BuildCopyCommand(char* cmd_addr, uint32_t num_copy_command, void* dst, + const void* src, size_t size) { size_t cur_size = 0; const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ : kMaxSingleCopySize; @@ -858,9 +812,9 @@ void BlitSdma::BuildCo packet_addr->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR; if (max_copy_size == (1 << 30) -1) - packet_addr->COUNT_UNION.count_ext.count = copy_size + SizeToCountOffset; + packet_addr->COUNT_UNION.count_ext.count = copy_size - 1; /* count is 1-based */ else - packet_addr->COUNT_UNION.count.count = copy_size + SizeToCountOffset; + packet_addr->COUNT_UNION.count.count = copy_size - 1; /* count is 1-based */ packet_addr->SRC_ADDR_LO_UNION.src_addr_31_0 = ptrlow32(cur_src); packet_addr->SRC_ADDR_HI_UNION.src_addr_63_32 = ptrhigh32(cur_src); @@ -881,11 +835,12 @@ Elements are coded by the log2 of the element size in bytes (ie. element 0=1 byt This routine breaks a large rect into tiles that can be handled by hardware. Pitches and offsets must be representable in terms of elements in all tiles of the copy. */ -template -void BlitSdma::BuildCopyRectCommand( - const std::function& append, const hsa_pitched_ptr_t* dst, - const hsa_dim3_t* dst_offset, const hsa_pitched_ptr_t* src, const hsa_dim3_t* src_offset, - const hsa_dim3_t* range) { +template +void BlitSdma::BuildCopyRectCommand(const std::function& append, + const hsa_pitched_ptr_t* dst, + const hsa_dim3_t* dst_offset, + const hsa_pitched_ptr_t* src, + const hsa_dim3_t* src_offset, const hsa_dim3_t* range) { // Returns the index of the first set bit (ie log2 of the largest power of 2 that evenly divides // width), the largest element that perfectly covers width. // width | 16 ensures that we don't return a higher element than is supported and avoids @@ -1029,9 +984,9 @@ void BlitSdma::BuildCo } } -template -void BlitSdma::BuildFillCommand( - char* cmd_addr, uint32_t num_fill_command, void* ptr, uint32_t value, size_t count) { +template +void BlitSdma::BuildFillCommand(char* cmd_addr, uint32_t num_fill_command, void* ptr, + uint32_t value, size_t count) { char* cur_ptr = reinterpret_cast(ptr); const uint32_t maxDwordCount = kMaxSingleFillSize / sizeof(uint32_t); SDMA_PKT_CONSTANT_FILL* packet_addr = reinterpret_cast(cmd_addr); @@ -1050,7 +1005,8 @@ void BlitSdma::BuildFi packet_addr->DATA_UNION.src_data_31_0 = value; - packet_addr->COUNT_UNION.count = (fill_count + SizeToCountOffset) * sizeof(uint32_t); + /* count is 1-based */ + packet_addr->COUNT_UNION.count = (fill_count - 1) * sizeof(uint32_t); packet_addr++; cur_ptr += fill_count * sizeof(uint32_t); @@ -1059,9 +1015,8 @@ void BlitSdma::BuildFi assert(count == 0 && "SDMA fill command count error."); } -template -void BlitSdma::BuildPollCommand( - char* cmd_addr, void* addr, uint32_t reference) { +template +void BlitSdma::BuildPollCommand(char* cmd_addr, void* addr, uint32_t reference) { SDMA_PKT_POLL_REGMEM* packet_addr = reinterpret_cast(cmd_addr); @@ -1081,9 +1036,8 @@ void BlitSdma::BuildPo packet_addr->DW5_UNION.retry_count = 0xfff; // Retry forever. } -template -void BlitSdma::BuildAtomicDecrementCommand(char* cmd_addr, void* addr) { +template +void BlitSdma::BuildAtomicDecrementCommand(char* cmd_addr, void* addr) { SDMA_PKT_ATOMIC* packet_addr = reinterpret_cast(cmd_addr); memset(packet_addr, 0, sizeof(SDMA_PKT_ATOMIC)); @@ -1098,9 +1052,8 @@ void BlitSdmaSRC_DATA_HI_UNION.src_data_63_32 = 0xffffffff; } -template -void BlitSdma::BuildGetGlobalTimestampCommand(char* cmd_addr, void* write_address) { +template +void BlitSdma::BuildGetGlobalTimestampCommand(char* cmd_addr, void* write_address) { SDMA_PKT_TIMESTAMP* packet_addr = reinterpret_cast(cmd_addr); @@ -1113,9 +1066,7 @@ void BlitSdmaADDR_HI_UNION.addr_63_32 = ptrhigh32(write_address); } -template -void BlitSdma::BuildTrapCommand( - char* cmd_addr, uint32_t event_id) { +template void BlitSdma::BuildTrapCommand(char* cmd_addr, uint32_t event_id) { SDMA_PKT_TRAP* packet_addr = reinterpret_cast(cmd_addr); @@ -1125,17 +1076,13 @@ void BlitSdma::BuildTr packet_addr->INT_CONTEXT_UNION.int_ctx = event_id; } -template -void BlitSdma::BuildHdpFlushCommand( - char* cmd_addr) { +template void BlitSdma::BuildHdpFlushCommand(char* cmd_addr) { assert(cmd_addr != NULL); SDMA_PKT_POLL_REGMEM* addr = reinterpret_cast(cmd_addr); memcpy(addr, &hdp_flush_cmd, flush_command_size_); } -template -void BlitSdma::BuildGCRCommand( - char* cmd_addr, bool invalidate) { +template void BlitSdma::BuildGCRCommand(char* cmd_addr, bool invalidate) { assert(cmd_addr != NULL); assert(useGCR && "Unsupported SDMA command - GCR."); SDMA_PKT_GCR* addr = reinterpret_cast(cmd_addr); @@ -1154,25 +1101,16 @@ void BlitSdma::BuildGC addr->WORD2_UNION.GCR_CONTROL_GL2_RANGE = 0; } -template -uint64_t BlitSdma::PendingBytes() { - RingIndexTy commit = atomic::Load(&cached_commit_index_, std::memory_order_acquire); - RingIndexTy hw_read_index = *reinterpret_cast(queue_resource_.Queue_read_ptr); - RingIndexTy read; - if (HwIndexMonotonic) { - read = hw_read_index; - } else { - RingIndexTy dist_to_read_index = WrapIntoRing(commit - hw_read_index); - read = commit - dist_to_read_index; - } +template uint64_t BlitSdma::PendingBytes() { + uint64_t commit = atomic::Load(&cached_commit_index_, std::memory_order_acquire); + uint64_t hw_read_index = *reinterpret_cast(queue_resource_.Queue_read_ptr); - if (commit == read) return 0; - return bytes_queued_ - bytes_written_[WrapIntoRing(read)]; + if (commit == hw_read_index) return 0; + return bytes_queued_ - bytes_written_[WrapIntoRing(hw_read_index)]; } -template class BlitSdma; -template class BlitSdma; -template class BlitSdma; +template class BlitSdma; +template class BlitSdma; } // namespace amd } // namespace rocr diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index fdcff3a18f..6f9fe60e70 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -711,10 +711,6 @@ core::Blit* GpuAgent::CreateBlitSdma(bool use_xgmi, int rec_eng) { const size_t copy_size_overrides[2] = {0x3fffff, 0x3fffffff}; switch (isa_->GetMajorVersion()) { - case 7: - case 8: - sdma = new BlitSdmaV2V3(); - break; case 9: sdma = new BlitSdmaV4(); copy_size_override = (isa_->GetMinorVersion() == 0 && isa_->GetStepping() == 10) ? From 0dec2ab43b26c72b8dacbd3b9e3afcb4bf06dcbe Mon Sep 17 00:00:00 2001 From: "Yat Sin, David" Date: Tue, 8 Jul 2025 21:33:54 -0400 Subject: [PATCH 09/13] Update runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Yat Sin, David --- runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index 0875994cbb..18a027632f 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -340,7 +340,9 @@ hsa_status_t BlitSdma::SubmitCommand(const void* cmd, size_t cmd_size, u // Add space for acquire or release Hdp flush command uint32_t flush_cmd_size = 0; if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) { - if (hdp_flush_support_) flush_cmd_size = flush_command_size_; + if (hdp_flush_support_) { + flush_cmd_size = flush_command_size_; + } } // Add space for cache flush. From ccaac9045b9fb42ce2c57b7e9c6a3608ca342cee Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Tue, 22 Jul 2025 14:54:13 -0400 Subject: [PATCH 10/13] rocr/aie: XdnaDriver::ExportDMABuf implementation --- .../core/driver/xdna/amd_xdna_driver.cpp | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp index 58d7ec9d17..8df55e05e9 100644 --- a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp +++ b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp @@ -332,10 +332,24 @@ hsa_status_t XdnaDriver::AllocQueueGWS(HSA_QUEUEID queue_id, uint32_t num_gws, return HSA_STATUS_ERROR_INVALID_QUEUE; } -hsa_status_t XdnaDriver::ExportDMABuf(void *mem, size_t size, int *dmabuf_fd, - size_t *offset) { - // Not implemented yet. - return HSA_STATUS_ERROR; +hsa_status_t XdnaDriver::ExportDMABuf(void* mem, size_t size, int* dmabuf_fd, size_t* offset) { + auto bo_handle = FindBOHandle(mem); + if (!bo_handle.IsValid()) { + return HSA_STATUS_ERROR_INVALID_ALLOCATION; + } + + drm_prime_handle export_params = {}; + export_params.handle = bo_handle.handle; + export_params.flags = DRM_RDWR; + export_params.fd = -1; + if (ioctl(fd_, DRM_IOCTL_PRIME_HANDLE_TO_FD, &export_params) < 0) { + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + *dmabuf_fd = export_params.fd; + *offset = reinterpret_cast(mem) - reinterpret_cast(bo_handle.vaddr); + + return HSA_STATUS_SUCCESS; } hsa_status_t XdnaDriver::ImportDMABuf(int dmabuf_fd, core::Agent &agent, From f5120bfe688cc14906079dc2537505142dfab8c5 Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Tue, 22 Jul 2025 15:01:39 -0400 Subject: [PATCH 11/13] rocr: DmaBufExport support for other agent types --- runtime/hsa-runtime/core/runtime/runtime.cpp | 29 ++++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index c2ee15c922..edba448c22 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -3078,8 +3078,8 @@ Agent* Runtime::GetSVMPrefetchAgent(void* ptr, size_t size) { return agents_by_node_[prefetch_node][0]; } -hsa_status_t Runtime::DmaBufExport(const void* ptr, size_t size, int* dmabuf, - uint64_t* offset, uint64_t flags) { +hsa_status_t Runtime::DmaBufExport(const void* ptr, size_t size, int* dmabuf, uint64_t* offset, + uint64_t flags) { #ifdef __linux__ ScopedAcquire lock(memory_lock_.shared()); // Lookup containing allocation. @@ -3090,18 +3090,23 @@ hsa_status_t Runtime::DmaBufExport(const void* ptr, size_t size, int* dmabuf, (ptr < reinterpret_cast(mem->first) + mem->second.size)) { // Check size is in bounds. if (uintptr_t(ptr) - uintptr_t(mem->first) + size <= mem->second.size) { - // Check allocation is on GPU - if (mem->second.region->owner()->device_type() != Agent::kAmdGpuDevice) - return HSA_STATUS_ERROR_INVALID_AGENT; + switch (mem->second.region->owner()->device_type()) { + case Agent::kAmdGpuDevice: { + auto* owner = static_cast(mem->second.region->owner()); - rocr::AMD::GpuAgent* owner = - static_cast(mem->second.region->owner()); - - if (flags & HSA_AMD_DMABUF_MAPPING_TYPE_PCIE && - !owner->is_xgmi_cpu_gpu() && - !owner->LargeBarEnabled()) { - return (hsa_status_t)HSA_STATUS_ERROR_NOT_SUPPORTED; + if (flags & HSA_AMD_DMABUF_MAPPING_TYPE_PCIE && !owner->is_xgmi_cpu_gpu() && + !owner->LargeBarEnabled()) { + return static_cast(HSA_STATUS_ERROR_NOT_SUPPORTED); + } + } break; + case Agent::kAmdCpuDevice: + return HSA_STATUS_ERROR_INVALID_AGENT; + case Agent::kAmdAieDevice: + break; + case Agent::kUnknownDevice: + return HSA_STATUS_ERROR_INVALID_AGENT; } + int fd; uint64_t off; hsa_status_t err = mem->second.region->owner()->driver().ExportDMABuf( From b7cd5cc7f19bf0586dbde761e783c2f9b668232d Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Fri, 18 Jul 2025 16:10:27 -0400 Subject: [PATCH 12/13] rocr: Adding conversion function from hsa_amd_vmem_alloc_handle_t to ThunkHandle --- runtime/hsa-runtime/core/inc/runtime.h | 9 ++++++--- runtime/hsa-runtime/core/runtime/runtime.cpp | 12 +++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index 9ba67f34d3..4b7e489d94 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -822,7 +822,6 @@ class Runtime { std::map reserved_address_map_; // Indexed by VA struct MemoryHandle { - MemoryHandle() : region(NULL), size(0), ref_count(0), thunk_handle(NULL), alloc_flag(0) {} MemoryHandle(const MemoryRegion* region, size_t size, uint64_t flags_unused, ThunkHandle thunk_handle, MemoryRegion::AllocateFlags alloc_flag) : region(region), @@ -832,19 +831,23 @@ class Runtime { thunk_handle(thunk_handle), alloc_flag(alloc_flag) {} - static __forceinline hsa_amd_vmem_alloc_handle_t Convert(void* handle) { + static __forceinline hsa_amd_vmem_alloc_handle_t Convert(ThunkHandle handle) { hsa_amd_vmem_alloc_handle_t ret_handle = { static_cast(reinterpret_cast(handle))}; return ret_handle; } + static __forceinline ThunkHandle Convert(hsa_amd_vmem_alloc_handle_t handle) { + return reinterpret_cast(handle.handle); + } + __forceinline core::Agent* agentOwner() const { return region->owner(); } const MemoryRegion* region; size_t size; int ref_count; int use_count; - ThunkHandle thunk_handle; // handle returned by hsaKmtAllocMemory(NoAddress = 1) + ThunkHandle thunk_handle; // handle returned by Driver::Allocate(NoAddress = 1) MemoryRegion::AllocateFlags alloc_flag; }; std::map memory_handle_map_; diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index edba448c22..14b57254bb 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -54,9 +54,7 @@ #include #include #include -#include #include -#include #include "core/inc/runtime.h" #include "core/inc/hsa_table_interface.h" @@ -3219,7 +3217,7 @@ hsa_status_t Runtime::VMemoryHandleCreate(const MemoryRegion* region, size_t siz return HSA_STATUS_ERROR_INVALID_ARGUMENT; ScopedAcquire lock(&memory_lock_); - void *user_mode_driver_handle; + ThunkHandle user_mode_driver_handle; hsa_status_t status = region->Allocate(size, alloc_flags, &user_mode_driver_handle, 0); if (status == HSA_STATUS_SUCCESS) { @@ -3236,7 +3234,7 @@ hsa_status_t Runtime::VMemoryHandleCreate(const MemoryRegion* region, size_t siz hsa_status_t Runtime::VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryOnlyHandle) { ScopedAcquire lock(&memory_lock_); - auto memoryHandleIt = memory_handle_map_.find(reinterpret_cast(memoryOnlyHandle.handle)); + auto memoryHandleIt = memory_handle_map_.find(MemoryHandle::Convert(memoryOnlyHandle)); if (memoryHandleIt == memory_handle_map_.end()) { debug_warning(false && "Can't find memory handle"); @@ -3291,7 +3289,7 @@ hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset, if (reinterpret_cast(va) + size > lowerMappedHandleIt->first) return HSA_STATUS_ERROR_INVALID_ARGUMENT; } - auto memoryHandleIt = memory_handle_map_.find(reinterpret_cast(memoryOnlyHandle.handle)); + auto memoryHandleIt = memory_handle_map_.find(MemoryHandle::Convert(memoryOnlyHandle)); if (memoryHandleIt == memory_handle_map_.end()) { debug_warning(false && "Can't find memory handle"); return HSA_STATUS_ERROR_INVALID_ARGUMENT; @@ -3654,7 +3652,7 @@ hsa_status_t Runtime::VMemoryExportShareableHandle(int* dmabuf_fd, hsa_amd_vmem_alloc_handle_t handle, uint64_t flags) { *dmabuf_fd = -1; - auto memoryHandle = memory_handle_map_.find((void*)handle.handle); + auto memoryHandle = memory_handle_map_.find(MemoryHandle::Convert(handle)); if (memoryHandle == memory_handle_map_.end()) { debug_warning(false && "Can't find memory handle"); return HSA_STATUS_ERROR_INVALID_ALLOCATION; @@ -3750,7 +3748,7 @@ hsa_status_t Runtime::VMemoryRetainAllocHandle(hsa_amd_vmem_alloc_handle_t* mapp hsa_status_t Runtime::VMemoryGetAllocPropertiesFromHandle(hsa_amd_vmem_alloc_handle_t allocHandle, const core::MemoryRegion** mem_region, hsa_amd_memory_type_t* type) { - auto memoryHandleIt = memory_handle_map_.find(reinterpret_cast(allocHandle.handle)); + auto memoryHandleIt = memory_handle_map_.find(MemoryHandle::Convert(allocHandle)); if (memoryHandleIt == memory_handle_map_.end()) return HSA_STATUS_ERROR_INVALID_ALLOCATION; *mem_region = memoryHandleIt->second.region; From 996e8bbfb71310d9a1b641bc954b69138cf85daa Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Fri, 18 Jul 2025 11:33:12 -0500 Subject: [PATCH 13/13] hsakmt: Use udmabuf to allocate system memory This patch uses udmabuf driver to allocate system memory instead of using amdgpu driver for APU. With this function app can account its consumed system memory by cgroup mechanism. This function is enabled by env variable HSA_USE_UDMABUF. Signed-off-by: Xiaogang Chen --- libhsakmt/include/hsakmt/linux/udmabuf.h | 38 ++++ libhsakmt/src/fmm.c | 231 +++++++++++++++++++---- libhsakmt/src/fmm.h | 2 +- libhsakmt/src/globals.c | 1 + libhsakmt/src/libhsakmt.h | 1 + libhsakmt/src/openclose.c | 18 ++ libhsakmt/src/queues.c | 2 +- libhsakmt/src/topology.c | 8 +- 8 files changed, 255 insertions(+), 46 deletions(-) create mode 100644 libhsakmt/include/hsakmt/linux/udmabuf.h diff --git a/libhsakmt/include/hsakmt/linux/udmabuf.h b/libhsakmt/include/hsakmt/linux/udmabuf.h new file mode 100644 index 0000000000..92f1c8311b --- /dev/null +++ b/libhsakmt/include/hsakmt/linux/udmabuf.h @@ -0,0 +1,38 @@ +/* GPL-2.0 WITH Linux-syscall-note */ +/* + * This file was copied from inux-libc-dev package + * This header provides interface to linux kernel udmabuf drver + * Modifications may have been made. + */ +#ifndef _THUNK_UDMABUF_H +#define _THUNK_UDMABUF_H + +#include +#include + +#define UDMABUF_FLAGS_CLOEXEC 0x01 + +struct udmabuf_create { + __u32 memfd; + __u32 flags; + __u64 offset; + __u64 size; +}; + +struct udmabuf_create_item { + __u32 memfd; + __u32 __pad; + __u64 offset; + __u64 size; +}; + +struct udmabuf_create_list { + __u32 flags; + __u32 count; + struct udmabuf_create_item list[]; +}; + +#define UDMABUF_CREATE _IOW('u', 0x42, struct udmabuf_create) +#define UDMABUF_CREATE_LIST _IOW('u', 0x43, struct udmabuf_create_list) + +#endif /* _THUNK_UDMABUF_H */ diff --git a/libhsakmt/src/fmm.c b/libhsakmt/src/fmm.c index 6a6f16dea4..84e18cd42a 100644 --- a/libhsakmt/src/fmm.c +++ b/libhsakmt/src/fmm.c @@ -23,6 +23,7 @@ * DEALINGS IN THE SOFTWARE. */ +#define _GNU_SOURCE #include "libhsakmt.h" #include "fmm.h" #include "hsakmt/hsakmtmodel.h" @@ -44,6 +45,11 @@ #include "rbtree.h" #include +#include +#include +#include +#include "hsakmt/linux/udmabuf.h" + #ifndef MPOL_F_STATIC_NODES /* Bug in numaif.h, this should be defined in there. Definition copied * from linux/mempolicy.h. @@ -148,6 +154,10 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *aper, uint64_t size, uint64_t align); static void reserved_aperture_release(manageable_aperture_t *aper, void *addr, uint64_t size); + +static int bind_mem_to_numa(uint32_t node_id, void *mem, + uint64_t SizeInBytes, HsaMemFlags mflags); + static const manageable_aperture_ops_t reserved_aperture_ops = { reserved_aperture_allocate_aligned, reserved_aperture_release @@ -758,7 +768,7 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, } void *hsakmt_mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align, - uint64_t guard_size, void *aper_base, void *aper_limit) + uint64_t guard_size, void *aper_base, void *aper_limit, int fd) { void *addr, *aligned_addr, *aligned_end, *mapping_end; uint64_t aligned_padded_size; @@ -766,7 +776,7 @@ void *hsakmt_mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t aligned_padded_size = size + guard_size * 2 + (align - PAGE_SIZE); /* Map memory PROT_NONE to alloc address space only */ - addr = mmap(0, aligned_padded_size, PROT_NONE, flags, -1, 0); + addr = mmap(0, aligned_padded_size, PROT_NONE, flags | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) { pr_err("mmap failed: %s\n", strerror(errno)); return NULL; @@ -795,7 +805,7 @@ void *hsakmt_mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t return aligned_addr; /* MAP_FIXED to the aligned address with required prot */ - addr = mmap(aligned_addr, size, prot, flags | MAP_FIXED, -1, 0); + addr = mmap(aligned_addr, size, prot, flags | MAP_FIXED, fd, 0); if (addr == MAP_FAILED) { pr_err("mmap failed: %s\n", strerror(errno)); return NULL; @@ -859,7 +869,7 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, guard_size = (uint64_t)aper->guard_pages * PAGE_SIZE; return hsakmt_mmap_allocate_aligned(PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, - size, align, guard_size, aper->base, aper->limit); + size, align, guard_size, aper->base, aper->limit, -1); } static void mmap_aperture_release(manageable_aperture_t *aper, @@ -1494,7 +1504,7 @@ void *hsakmt_fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t Memor mem = hsakmt_mmap_allocate_aligned(PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, aligned_size, SCRATCH_ALIGN, 0, - 0, (void *)LONG_MAX); + 0, (void *)LONG_MAX, -1); } /* Remember scratch backing aperture for later */ @@ -1603,6 +1613,126 @@ static void *fmm_allocate_va(uint32_t gpu_id, void *address, uint64_t size, return mem; } +/* use udmabuf driver to allocate buf */ +static void* udmabuf_allocation(uint32_t gpu_id, uint32_t node_id, uint64_t size, + manageable_aperture_t *aperture, uint64_t alignment, + HsaMemFlags mflags, vm_object_t** vm_obj) +{ + struct kfd_ioctl_import_dmabuf_args importArgs = {0}; + int memfd, dmabuf_fd; + long long node_size, free_size; + struct udmabuf_create create; + uint64_t alignment_size; + uint32_t numa_node_id; + uint64_t guard_size; + void *mem; + int ret; + + dmabuf_fd = -1; + memfd = -1; + + *vm_obj = NULL; + + memfd = memfd_create("thunk_memfd", MFD_ALLOW_SEALING); + if (memfd == -1) { + pr_debug("running kernel does not support memfd\n"); + return NULL; + } + + if (ftruncate(memfd, size) == -1) { + pr_debug("ftruncate fail\n"); + goto error_release_memfd; + } + pr_debug("PID: %jd; fd: %d; /proc/%jd/fd/%d\n", + (intmax_t) getpid(), memfd, (intmax_t) getpid(), memfd); + + if (fcntl(memfd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW) < 0) { + pr_debug("fcntl fail %s\n", strerror(errno)); + goto error_release_memfd; + } + + alignment_size = PAGE_SIZE << svm.alignment_order; + alignment = alignment ? alignment : aperture->align; + while (alignment < alignment_size && size >= (alignment << 1)) + alignment <<= 1; + + guard_size = (uint64_t)aperture->guard_pages * PAGE_SIZE; + + mem = hsakmt_mmap_allocate_aligned(PROT_WRITE | PROT_READ, MAP_NORESERVE | MAP_SHARED, + size, alignment, guard_size, aperture->base, aperture->limit, memfd); + if (!mem) + goto error_release_memfd; + + /* set madvise flags to HUGEPAGE if allocate more than 2MB */ + if (size >= (2 * 1024 * 1024)) + madvise(mem, size, MADV_HUGEPAGE); + + /* always bind to numa node */ + mflags.ui32.NoSubstitute = 1; + /* Bind to NUMA node */ + /* node_id is gpu id, get closed numa id */ + numa_node_id = hsakmt_get_direct_link_cpu(node_id); + if (bind_mem_to_numa(numa_node_id, mem, size, mflags)) + goto error_release_aperture; + + node_size = numa_node_size(numa_node_id, &free_size); + pr_debug("udmabuf_allocation: numa_node_id %d, node_size %lld, free_size %lld\n", + numa_node_id, node_size, free_size); + /* compare free size at numa_node_id with size */ + if ((uint64_t)free_size < size) { + pr_debug("udmabuf_allocation: has no enough ram on numa_node_id %d, node_size %lld, free_size %lld\n", + numa_node_id, node_size, free_size); + goto error_release_aperture; + } + + create.memfd = memfd; + create.flags = UDMABUF_FLAGS_CLOEXEC; + create.offset = 0; + create.size = size; + dmabuf_fd = ioctl(hsakmt_udmabuf_dev_fd, UDMABUF_CREATE, &create); + + if (dmabuf_fd < 0) { + pr_debug("ioctl UDMABUF_CREATE failed\n"); + goto error_release_aperture; + } + + importArgs.va_addr = (uint64_t)mem; + importArgs.gpu_id = gpu_id; + importArgs.dmabuf_fd = dmabuf_fd; + + ret = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_IMPORT_DMABUF, (void *)&importArgs); + if (ret) { + pr_debug("ioctl AMDKFD_IOC_IMPORT_DMABUF failed\n, ret 0x%x", ret); + goto error_release_dmabuf; + } + + /* Allocate object */ + pthread_mutex_lock(&aperture->fmm_mutex); + *vm_obj = aperture_allocate_object(aperture, mem, importArgs.handle, + size, mflags); + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (*vm_obj == NULL) + goto error_release_dmabuf; + + /* after import udmabuf into kfd driver close dmabuf_fd + * as kfd driver holds the dmabuf + */ + close(dmabuf_fd); + close(memfd); + + return mem; + +error_release_dmabuf: + close(dmabuf_fd); +error_release_aperture: + aperture_release_area(aperture, mem, size); +error_release_memfd: + close(memfd); + + return NULL; +} + void *hsakmt_fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes, uint64_t alignment, HsaMemFlags mflags) { @@ -1653,35 +1783,51 @@ void *hsakmt_fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *addres if (mflags.ui32.Contiguous) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT; - mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, - ioc_flags, alignment, &vm_obj); - - if (mem && vm_obj) { - pthread_mutex_lock(&aperture->fmm_mutex); - /* Store memory allocation flags, not ioc flags */ - vm_obj->mflags = mflags; - hsakmt_gpuid_to_nodeid(gpu_id, &vm_obj->node_id); - pthread_mutex_unlock(&aperture->fmm_mutex); + mem = NULL; + if (hsakmt_udmabuf_dev_fd > 0 && aperture == svm.dgpu_aperture && !hsakmt_is_dgpu + && aperture->ops == &mmap_aperture_ops) { + mem = udmabuf_allocation(gpu_id, node_id, size, aperture, alignment, + mflags, &vm_obj); + pr_debug("udmabuf_allocation mem %p\n", mem); + if (!mem) + pr_debug("udmabuf_allocation allocation fail\n"); } - /* if alloc vram-only not mmap to cpu vm since no va */ - if (mem && !mflags.ui32.NoAddress) { - void *ret = fmm_map_to_cpu(mem, MemorySizeInBytes, + /* env HSA_USE_UDMABUF not set, or not apu, or cannot use udmabuf, + * fall back to use device driver to allocate memory + */ + if (!mem) { + mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, + ioc_flags, alignment, &vm_obj); + + /* if alloc vram-only not mmap to cpu vm since no va */ + if (mem && !mflags.ui32.NoAddress) { + void *ret = fmm_map_to_cpu(mem, MemorySizeInBytes, mflags.ui32.HostAccess, gpu_mem[gpu_mem_id].drm_render_fd, mmap_offset); - if (ret == MAP_FAILED) { - __fmm_release(vm_obj, aperture); - return NULL; - } + if (ret == MAP_FAILED) { + __fmm_release(vm_obj, aperture); + return NULL; + } #ifdef SANITIZER_AMDGPU - if (vm_obj) { - vm_obj->mmap_flags = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; - vm_obj->mmap_fd = gpu_mem[gpu_mem_id].drm_render_fd; - vm_obj->mmap_offset = mmap_offset; - } + if (vm_obj) { + vm_obj->mmap_flags = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; + vm_obj->mmap_fd = gpu_mem[gpu_mem_id].drm_render_fd; + vm_obj->mmap_offset = mmap_offset; + } #endif + } + } + + if (mem && vm_obj) { + pthread_mutex_lock(&aperture->fmm_mutex); + /* Store memory allocation flags, not ioc flags */ + vm_obj->mflags = mflags; + hsakmt_gpuid_to_nodeid(gpu_id, &vm_obj->node_id); + pthread_mutex_unlock(&aperture->fmm_mutex); + } return mem; @@ -1773,7 +1919,7 @@ static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, return mem; } -static int bind_mem_to_numa(uint32_t node_id, void *mem, +static int bind_mem_to_numa(uint32_t numa_node_id, void *mem, uint64_t SizeInBytes, HsaMemFlags mflags) { int mode = MPOL_F_STATIC_NODES; @@ -1782,34 +1928,37 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, long r; pr_debug("%s mem %p flags 0x%x size 0x%lx node_id %d\n", __func__, - mem, mflags.Value, SizeInBytes, node_id); + mem, mflags.Value, SizeInBytes, numa_node_id); - if (mflags.ui32.NoNUMABind) - return 0; - - if (numa_available() == -1) - return 0; + if (mflags.ui32.NoNUMABind || numa_available() == -1) { + /* but need bind to a numa node */ + if (mflags.ui32.NoSubstitute) + return -EFAULT; + else + return 0; + } num_node = numa_max_node() + 1; /* Ignore binding requests to invalid nodes IDs */ - if (node_id >= (unsigned)num_node) { - pr_warn("node_id %d >= num_node %d\n", node_id, num_node); - return 0; + if (numa_node_id >= (unsigned)num_node || numa_node_id == INVALID_NODEID || num_node <= 1) { + pr_warn("numa_node_id is out range: numa_node_id %d, num_node %d\n", numa_node_id, num_node); + if (mflags.ui32.NoSubstitute) + return -EFAULT; + else + return 0; } - if (num_node <= 1) - return 0; - node_mask = numa_bitmask_alloc(num_node); if (!node_mask) return -ENOMEM; #ifdef __PPC64__ - numa_bitmask_setbit(node_mask, node_id * 8); + numa_bitmask_setbit(node_mask, numa_node_id * 8); #else - numa_bitmask_setbit(node_mask, node_id); + numa_bitmask_setbit(node_mask, numa_node_id); #endif + mode |= mflags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; r = mbind(mem, SizeInBytes, mode, node_mask->maskp, num_node + 1, 0); numa_bitmask_free(node_mask); diff --git a/libhsakmt/src/fmm.h b/libhsakmt/src/fmm.h index 29e81ef714..cdecfa93d8 100644 --- a/libhsakmt/src/fmm.h +++ b/libhsakmt/src/fmm.h @@ -100,7 +100,7 @@ HSAKMT_STATUS hsakmt_fmm_map_to_gpu_nodes(void *address, uint64_t size, int hsakmt_open_drm_render_device(int minor); void *hsakmt_mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align, - uint64_t guard_size, void *aper_base, void *aper_limit); + uint64_t guard_size, void *aper_base, void *aper_limit, int fd); extern int (*hsakmt_fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle); #endif /* FMM_H_ */ diff --git a/libhsakmt/src/globals.c b/libhsakmt/src/globals.c index 88cfe57ccb..9a36e6f5cc 100644 --- a/libhsakmt/src/globals.c +++ b/libhsakmt/src/globals.c @@ -28,6 +28,7 @@ // HSAKMT global data int hsakmt_kfd_fd = -1; +int hsakmt_udmabuf_dev_fd = -1; unsigned long hsakmt_kfd_open_count; unsigned long hsakmt_system_properties_count; pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; diff --git a/libhsakmt/src/libhsakmt.h b/libhsakmt/src/libhsakmt.h index 245a621f40..f7387082d7 100644 --- a/libhsakmt/src/libhsakmt.h +++ b/libhsakmt/src/libhsakmt.h @@ -33,6 +33,7 @@ #include extern int hsakmt_kfd_fd; +extern int hsakmt_udmabuf_dev_fd; extern unsigned long hsakmt_kfd_open_count; extern bool hsakmt_forked; extern pthread_mutex_t hsakmt_mutex; diff --git a/libhsakmt/src/openclose.c b/libhsakmt/src/openclose.c index 4b0b21019b..8929072f33 100644 --- a/libhsakmt/src/openclose.c +++ b/libhsakmt/src/openclose.c @@ -46,6 +46,7 @@ int (*hsakmt_fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle); static const char kfd_device_name[] = "/dev/kfd"; +static const char kfd_udmabuf_device_name[] = "/dev/udmabuf"; static pid_t parent_pid = -1; int hsakmt_debug_level; bool hsakmt_forked; @@ -108,6 +109,10 @@ static void clear_after_fork(void) close(hsakmt_kfd_fd); hsakmt_kfd_fd = -1; } + if (hsakmt_udmabuf_dev_fd > 0) { + close(hsakmt_udmabuf_dev_fd); + hsakmt_udmabuf_dev_fd = -1; + } hsakmt_kfd_open_count = 0; parent_pid = -1; hsakmt_forked = false; @@ -152,6 +157,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) HsaSystemProperties sys_props; char *error; char *useSvmStr; + char *useUdmaBuf; pthread_mutex_lock(&hsakmt_mutex); @@ -195,6 +201,18 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (result != HSAKMT_STATUS_SUCCESS) goto kfd_version_failed; + /* check if udmabuf is enabled by env HSA_USE_UDMABUF */ + useUdmaBuf = getenv("HSA_USE_UDMABUF"); + if (useUdmaBuf && atoi(useUdmaBuf)) { + /* open udmabuf device */ + hsakmt_udmabuf_dev_fd = open(kfd_udmabuf_device_name, 0); + if (hsakmt_udmabuf_dev_fd < 0) + pr_debug("running kernel does not support udmabuf\n"); + else + pr_debug("udmabuf is enabled\n"); + } else + pr_debug("udmabuf is not enabled\n"); + useSvmStr = getenv("HSA_USE_SVM"); hsakmt_is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")); if(!hsakmt_use_model) diff --git a/libhsakmt/src/queues.c b/libhsakmt/src/queues.c index 05395fba06..1f95b43b78 100644 --- a/libhsakmt/src/queues.c +++ b/libhsakmt/src/queues.c @@ -541,7 +541,7 @@ static int handle_concrete_asic(struct queue *q, void *addr = hsakmt_mmap_allocate_aligned(PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, size, GPU_HUGE_PAGE_SIZE, 0, - 0, (void *)LONG_MAX); + 0, (void *)LONG_MAX, -1); if (!addr) { pr_err("mmap failed to alloc ctx area size 0x%x: %s\n", size, strerror(errno)); diff --git a/libhsakmt/src/topology.c b/libhsakmt/src/topology.c index 574c3ecb27..0992868f5c 100644 --- a/libhsakmt/src/topology.c +++ b/libhsakmt/src/topology.c @@ -1794,9 +1794,11 @@ static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, node_props_t *node_pro return -1; for (i = 0; i < node_props[gpu_node].node.NumIOLinks; i++) - if (props[i].IoLinkType == HSA_IOLINKTYPE_PCIEXPRESS && - props[i].Weight <= 20) /* >20 is GPU->CPU->GPU */ - return props[i].NodeTo; + if ((props[i].IoLinkType == HSA_IOLINKTYPE_PCIEXPRESS || props[i].IoLinkType == HSA_IOLINK_TYPE_XGMI) && + props[i].Weight <= 20) /* >20 is GPU->CPU->GPU */{ + if (!node_props[props[i].NodeTo].node.KFDGpuID) + return props[i].NodeTo; + } return -1; }