From 5a675921eab0283ccbca3e7f86363ac61e48ea4e Mon Sep 17 00:00:00 2001
From: Jonathan Kim <jonathan.kim@amd.com>
Date: Wed, 11 May 2022 13:34:22 -0400
Subject: [PATCH] kfdtest: add suspend and resume queues operation

Add base debug operations to suspend and resume queues.
Routine will return the number of queues successfully
suspended or resumed.

Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
Change-Id: I8f18317f70464b04231c5cf822e11d545ebfa02a
---
 include/hsakmt.h                |  3 ++-
 src/debug.c                     |  5 +++-
 tests/kfdtest/src/BaseDebug.cpp | 44 +++++++++++++++++++++++++++++----
 tests/kfdtest/src/BaseDebug.hpp |  3 +++
 4 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/include/hsakmt.h b/include/hsakmt.h
index 9d5ef5897f..ce7b7e327c 100644
--- a/include/hsakmt.h
+++ b/include/hsakmt.h
@@ -788,7 +788,8 @@ hsaKmtCheckRuntimeDebugSupport(
  */
 HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl(
     struct kfd_ioctl_dbg_trap_args *arg,
-    HSA_QUEUEID *Queues
+    HSA_QUEUEID *Queues,
+    HSAuint64 *DebugReturn
     );
 
 /**
diff --git a/src/debug.c b/src/debug.c
index b3e312cbc1..d438a2826e 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -514,7 +514,8 @@ free_data:
 }
 
 HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *args,
-					HSA_QUEUEID *Queues)
+					HSA_QUEUEID *Queues,
+					HSAuint64 *DebugReturn)
 {
 	HSAKMT_STATUS result;
 
@@ -533,6 +534,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *arg
 	}
 
 	long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, args);
+	if (DebugReturn)
+		*DebugReturn = err;
 
 	if (args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES &&
 				err >= 0 && err <= args->suspend_queues.num_queues)
diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp
index 1de15f80e6..3b8837c587 100644
--- a/tests/kfdtest/src/BaseDebug.cpp
+++ b/tests/kfdtest/src/BaseDebug.cpp
@@ -67,7 +67,7 @@ HSAKMT_STATUS BaseDebug::Attach(struct kfd_runtime_info *rInfo,
     args.enable.dbg_fd = m_Fd.fd;
     args.enable.exception_mask = exceptionEnable;
 
-    if (hsaKmtDebugTrapIoctl(&args, NULL)) {
+    if (hsaKmtDebugTrapIoctl(&args, NULL, NULL)) {
         close(m_Fd.fd);
         unlink(m_Fd_Name);
         return HSAKMT_STATUS_ERROR;
@@ -87,7 +87,7 @@ void BaseDebug::Detach(void) {
     args.pid = m_Pid;
     args.op = KFD_IOC_DBG_TRAP_DISABLE;
 
-    hsaKmtDebugTrapIoctl(&args, NULL);
+    hsaKmtDebugTrapIoctl(&args, NULL, NULL);
 
     close(m_Fd.fd);
     unlink(m_Fd_Name);
@@ -109,7 +109,7 @@ HSAKMT_STATUS BaseDebug::SendRuntimeEvent(uint64_t exceptions, int gpuId, int qu
     args.send_runtime_event.gpu_id = gpuId;
     args.send_runtime_event.queue_id = queueId;
 
-    return hsaKmtDebugTrapIoctl(&args, NULL);
+    return hsaKmtDebugTrapIoctl(&args, NULL, NULL);
 }
 
 HSAKMT_STATUS BaseDebug::QueryDebugEvent(uint64_t *exceptions,
@@ -134,7 +134,7 @@ HSAKMT_STATUS BaseDebug::QueryDebugEvent(uint64_t *exceptions,
     args.op = KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT;
     args.query_debug_event.exception_mask = *exceptions;
 
-    result = hsaKmtDebugTrapIoctl(&args, NULL);
+    result = hsaKmtDebugTrapIoctl(&args, NULL, NULL);
 
     *exceptions = args.query_debug_event.exception_mask;
 
@@ -157,5 +157,39 @@ void BaseDebug::SetExceptionsEnabled(uint64_t exceptions)
     args.op = KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED;
     args.set_exceptions_enabled.exception_mask = exceptions;
 
-    hsaKmtDebugTrapIoctl(&args, NULL);
+    hsaKmtDebugTrapIoctl(&args, NULL, NULL);
+}
+
+HSAKMT_STATUS BaseDebug::SuspendQueues(unsigned int *numQueues,
+                                       HSA_QUEUEID *queues,
+                                       uint32_t *queueIds,
+                                       uint64_t exceptionsToClear)
+{
+    struct kfd_ioctl_dbg_trap_args args = {0};
+
+    memset(&args, 0x00, sizeof(args));
+
+    args.pid = m_Pid;
+    args.op = KFD_IOC_DBG_TRAP_SUSPEND_QUEUES;
+    args.suspend_queues.num_queues = *numQueues;
+    args.suspend_queues.queue_array_ptr = (uint64_t)queueIds;
+    args.suspend_queues.exception_mask = exceptionsToClear;
+
+    return hsaKmtDebugTrapIoctl(&args, queues, (HSAuint64 *)numQueues);
+}
+
+HSAKMT_STATUS BaseDebug::ResumeQueues(unsigned int *numQueues,
+                                       HSA_QUEUEID *queues,
+                                       uint32_t *queueIds)
+{
+    struct kfd_ioctl_dbg_trap_args args = {0};
+
+    memset(&args, 0x00, sizeof(args));
+
+    args.pid = m_Pid;
+    args.op = KFD_IOC_DBG_TRAP_RESUME_QUEUES;
+    args.resume_queues.num_queues = *numQueues;
+    args.resume_queues.queue_array_ptr = (uint64_t)queueIds;
+
+    return hsaKmtDebugTrapIoctl(&args, queues, (HSAuint64 *)numQueues);
 }
diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp
index f744761b5c..eb3d659297 100644
--- a/tests/kfdtest/src/BaseDebug.hpp
+++ b/tests/kfdtest/src/BaseDebug.hpp
@@ -45,6 +45,9 @@ class BaseDebug {
                                   uint32_t *gpuId, uint32_t *queueId,
                                   int timeoutMsec);
     void SetExceptionsEnabled(uint64_t exceptions);
+    HSAKMT_STATUS SuspendQueues(unsigned int *numQueues, HSA_QUEUEID *queues, uint32_t *queueIds,
+                                uint64_t exceptionsToClear);
+    HSAKMT_STATUS ResumeQueues(unsigned int *numQueues, HSA_QUEUEID *queues, uint32_t *queueIds);
 
  private:
     unsigned int m_Pid;