From efef21f4ffefcf09e73188eb38f7f06da58647a9 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 29 Aug 2019 00:21:52 -0400 Subject: [PATCH] add queue snapshot test adds api and test to get newly create queue snapshot per ptraced process. Change-Id: Ife97123a5b930e837ccaa386801145ef23c2cc2c Signed-off-by: Jonathan Kim [ROCm/ROCR-Runtime commit: 8b01a1c4c53408da3c9e87010d460a9218456e3e] --- projects/rocr-runtime/include/hsakmt.h | 21 +++++++++++ .../rocr-runtime/include/linux/kfd_ioctl.h | 24 +++++++++++- projects/rocr-runtime/src/debug.c | 37 ++++++++++++++++++- projects/rocr-runtime/src/libhsakmt.ver | 2 +- .../tests/kfdtest/src/KFDDBGTest.cpp | 34 +++++++++++++++-- 5 files changed, 112 insertions(+), 6 deletions(-) diff --git a/projects/rocr-runtime/include/hsakmt.h b/projects/rocr-runtime/include/hsakmt.h index 79eb5e3446..cc96e492b6 100644 --- a/projects/rocr-runtime/include/hsakmt.h +++ b/projects/rocr-runtime/include/hsakmt.h @@ -823,6 +823,27 @@ hsaKmtQueryDebugEvent( bool *IsNew //OUT ); +/** + Newly created queue snapshot per ptraced process. + + Returns queue snapshot including queue id, gpuid, context save base address, + queue status word, queue address and size, and queue read and write pointer. + + ClearEvents set will clear new queue bit and queue status word bits. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful + */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueSnapshot( + HSAuint32 NodeId, // IN + HSAuint32 Pid, // IN + bool ClearEvents, // IN + void *SnapshotBuf, // IN + HSAuint32 *QssEntries // IN/OUT + ); + /** Set the trap override mask. When debug trap is enabled by hsaKmtEnableDebugTrap() each wave launched has its initial diff --git a/projects/rocr-runtime/include/linux/kfd_ioctl.h b/projects/rocr-runtime/include/linux/kfd_ioctl.h index 4e02aa8158..fa14bd44a3 100644 --- a/projects/rocr-runtime/include/linux/kfd_ioctl.h +++ b/projects/rocr-runtime/include/linux/kfd_ioctl.h @@ -93,6 +93,19 @@ struct kfd_ioctl_get_queue_wave_state_args { __u32 pad; }; +struct kfd_queue_snapshot_entry { + __u64 ring_base_address; + __u64 write_pointer_address; + __u64 read_pointer_address; + __u64 ctx_save_restore_address; + __u32 queue_id; + __u32 gpu_id; + __u32 ring_size; + __u32 queue_type; + __u32 queue_status; + __u32 reserved[19]; +}; + /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ #define KFD_IOC_CACHE_POLICY_COHERENT 0 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 @@ -246,6 +259,14 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 5 +/* KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT: + * ptr: user buffer (IN) + * data1: flags (IN) + * data2: number of queue snapshot entries (IN/OUT) + * data3: unused + */ +#define KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT 6 + /* KFD_IOC_DBG_TRAP_GET_VERSION: * prt: unsused * data1: major version (OUT) @@ -254,6 +275,7 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_GET_VERSION 7 + struct kfd_ioctl_dbg_trap_args { __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ __u32 pid; /* to KFD */ @@ -677,7 +699,7 @@ struct kfd_ioctl_cross_memory_copy_args { AMDKFD_IOWR(0x20, struct kfd_ioctl_ipc_export_handle_args) #define AMDKFD_IOC_DBG_TRAP \ - AMDKFD_IOW(0x21, struct kfd_ioctl_dbg_trap_args) + AMDKFD_IOWR(0x21, struct kfd_ioctl_dbg_trap_args) #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x22, struct kfd_ioctl_cross_memory_copy_args) diff --git a/projects/rocr-runtime/src/debug.c b/projects/rocr-runtime/src/debug.c index 8aa67446ad..92cbaeb186 100644 --- a/projects/rocr-runtime/src/debug.c +++ b/projects/rocr-runtime/src/debug.c @@ -287,7 +287,8 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, if (op == KFD_IOC_DBG_TRAP_NODE_SUSPEND || op == KFD_IOC_DBG_TRAP_NODE_RESUME || - op == KFD_IOC_DBG_TRAP_GET_VERSION) { + op == KFD_IOC_DBG_TRAP_GET_VERSION || + op == KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT) { if (NodeId != INVALID_NODEID) return HSAKMT_STATUS_INVALID_HANDLE; @@ -630,3 +631,37 @@ hsaKmtGetThunkDebugTrapVersionInfo( *Major = KFD_IOCTL_DBG_MAJOR_VERSION; *Minor = KFD_IOCTL_DBG_MINOR_VERSION; } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueSnapshot( + HSAuint32 NodeId, //IN + HSAuint32 Pid, // IN + bool ClearEvents, //IN + void *SnapshotBuf, //IN + HSAuint32 *QssEntries //IN/OUT + ) +{ + HSAKMT_STATUS result; + struct kfd_ioctl_dbg_trap_args argout = {0}; + uint32_t flags = 0; + + if (ClearEvents) + flags |= KFD_DBG_EV_FLAG_CLEAR_STATUS; + + result = debug_trap(NodeId, + KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT, + flags, + *QssEntries, + 0, + Pid, + (HSAuint64)SnapshotBuf, + &argout); + + if (result) + return result; + + *QssEntries = argout.data2; + + return 0; +} diff --git a/projects/rocr-runtime/src/libhsakmt.ver b/projects/rocr-runtime/src/libhsakmt.ver index 3f2e787ac5..fc9564f3ec 100644 --- a/projects/rocr-runtime/src/libhsakmt.ver +++ b/projects/rocr-runtime/src/libhsakmt.ver @@ -61,7 +61,7 @@ hsaKmtEnableDebugTrap; hsaKmtEnableDebugTrapWithPollFd; hsaKmtDisableDebugTrap; hsaKmtQueryDebugEvent; -hsaKmtSetDebugTrapData2; +hsaKmtGetQueueSnapshot; hsaKmtSetWaveLaunchTrapOverride; hsaKmtSetWaveLaunchMode; hsaKmtQueueSuspend; diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDDBGTest.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDDBGTest.cpp index 70b7558c4e..9e7d93cc4c 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDDBGTest.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDDBGTest.cpp @@ -24,6 +24,7 @@ #include "KFDDBGTest.hpp" #include #include +#include "linux/kfd_ioctl.h" #include "KFDQMTest.hpp" #include "PM4Queue.hpp" #include "PM4Packet.hpp" @@ -439,7 +440,7 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - if(!checkDebugVersion(0, 2)) { + if (!checkDebugVersion(0, 2)) { LOG() << "Test disabled due to debug API version mismatch"; goto exit; } @@ -510,13 +511,40 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { ASSERT_EQ(IsNew, true); ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_TRAP); - // suspend queue, query suspended queue and clear pending event + // suspend queue, get snapshot, query suspended queue + // and clear pending event ASSERT_SUCCESS(hsaKmtQueueSuspend(INVALID_PID, 1, queueIds, 10, 0)); syncStatus = dispatch->SyncWithStatus(suspendTimeout); ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); ASSERT_NE(iter[0], result[0]); + struct kfd_queue_snapshot_entry qssBuf[1] = {}; + HSAuint32 QssEntries = 0; + + // get only number of queues and don't update the snapshot buffer + ASSERT_SUCCESS(hsaKmtGetQueueSnapshot(INVALID_NODEID, INVALID_PID, + false, + reinterpret_cast(qssBuf), + &QssEntries)); + + ASSERT_EQ(QssEntries, 1); + ASSERT_EQ(qssBuf[0].ctx_save_restore_address, 0); + ASSERT_EQ(qssBuf[0].ring_base_address, 0); + ASSERT_EQ(qssBuf[0].ring_size, 0); + + // update the snapshot buffer + QssEntries = 1; + ASSERT_SUCCESS(hsaKmtGetQueueSnapshot(INVALID_NODEID, INVALID_PID, + false, + reinterpret_cast(qssBuf), + &QssEntries)); + + ASSERT_EQ(QssEntries, 1); + ASSERT_NE(qssBuf[0].ctx_save_restore_address, 0); + ASSERT_NE(qssBuf[0].ring_base_address, 0); + ASSERT_NE(qssBuf[0].ring_size, 0); + ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, true, &EventReceived, &IsSuspended, &IsNew)); @@ -593,7 +621,7 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - if(!checkDebugVersion(0, 2)) { + if (!checkDebugVersion(0, 2)) { LOG() << "Test disabled due to debug API version mismatch"; goto exit; }