libhsakmt: update dbg enable trap and add query debug events

Add data out for enable trap to return poll fd to user space.
Add query debug events interface.

Change-Id: Ia4afde1cf167e6aa61d502380a8b329ee89d5f44
Signed-off-by: Jonathan Kim <Jonathan.Kim@amd.com>

include/hsakmt.h

@@ -721,6 +721,8 @@ hsaKmtQueueResume(
 /**
   Enable debug trap for NodeId. If QueueId is INVALID_QUEUEID then
   enable for all queues on NodeId, otherwise enable only for QueueId.
+  Return file descriptor PollFd where on poll wake, fd has readable
+  FIFO data for pending debug events.
 
   When debug trap is enabled the trap handler behavior changes
   depending on architecture of the node and can include the following:
@@ -766,8 +768,18 @@ hsaKmtQueueResume(
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtEnableDebugTrap(
-    HSAuint32   NodeId, //IN
+    HSAuint32	NodeId, //IN
-    HSA_QUEUEID QueueId //IN
+    HSA_QUEUEID	QueueId //IN
+    );
+
+
+/* Similar to EnableDebugTrap with polling fd return*/
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtEnableDebugTrapWithPollFd(
+    HSAuint32	NodeId, //IN
+    HSA_QUEUEID	QueueId, //IN
+    HSAint32	*PollFd //OUT
     );
 
 /**
@@ -787,6 +799,32 @@ hsaKmtDisableDebugTrap(
     HSAuint32 NodeId //IN
     );
 
+
+/**
+  Query pending debug event set by ptrace.
+
+  Can query by target QueueId.  If QueueId is INVALID_QUEUEID, return the
+  first queue id that has a pending event.  Option to clear pending event
+  after query is used by the ClearEvents parameter.
+
+  Pending debug event type will be returned in EventsReceived parameter and is
+  defined by HSA_DEBUG_EVENT_TYPE.  Suspended state of queue is returned in
+  IsSuspended.
+
+  Returns:
+    - HSAKMT_STATUS_SUCCESS if successful
+ */
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtQueryDebugEvent(
+    HSAuint32			NodeId, // IN
+    HSAuint32			Pid, // IN
+    HSAuint32			*QueueId, // IN/OUT
+    bool			ClearEvents, // IN
+    HSA_DEBUG_EVENT_TYPE	*EventsReceived, // OUT
+    bool			*IsSuspended // OUT
+    );
+
 /**
   Set the value to use to initialize the TrapData used when
   initializing trap temp registers for NodeId when debug trap is enabled.

include/hsakmttypes.h

@@ -866,6 +866,18 @@ typedef enum _HSA_EVENTTYPE
     HSA_EVENTTYPE_TYPE_SIZE                  = 0xFFFFFFFF
 } HSA_EVENTTYPE;
 
+
+//
+// Definitions for types of pending debug events
+//
+typedef enum _HSA_DEBUG_EVENT_TYPE
+{
+	HSA_DEBUG_EVENT_TYPE_NONE				= 0,
+	HSA_DEBUG_EVENT_TYPE_TRAP				= 1,
+	HSA_DEBUG_EVENT_TYPE_VMFAULT			= 2,
+	HSA_DEBUG_EVENT_TYPE_TRAP_VMFAULT		= 3
+} HSA_DEBUG_EVENT_TYPE;
+
 typedef HSAuint32  HSA_EVENTID;
 
 //

include/linux/kfd_ioctl.h

@@ -187,11 +187,19 @@ struct kfd_ioctl_dbg_wave_control_args {
 	__u32 buf_size_in_bytes;	/*including gpu_id and buf_size */
 };
 
+/* mapping event types to API spec */
+#define	KFD_DBG_EV_STATUS_TRAP		1
+#define	KFD_DBG_EV_STATUS_VMFAULT	2
+#define	KFD_DBG_EV_STATUS_SUSPENDED	4
+#define	KFD_DBG_EV_FLAG_CLEAR_STATUS	1
+
+#define KFD_INVALID_QUEUEID	0xFFFFFFFFFFFFFFFFULL
+
 /* KFD_IOC_DBG_TRAP_ENABLE:
  * ptr:   unused
  * data1: 0=disable, 1=enable
  * data2: queue ID (for future use)
- * data3: unused
+ * data3: return value for fd
  */
 #define KFD_IOC_DBG_TRAP_ENABLE 0
 
@@ -235,6 +243,14 @@ struct kfd_ioctl_dbg_wave_control_args {
  */
 #define KFD_IOC_DBG_TRAP_NODE_RESUME 5
 
+/* KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT:
+ * ptr: unused
+ * data1: queue id (IN/OUT)
+ * data2: flags (IN)
+ * data3: suspend[2:2], event type [1:0] (OUT)
+ */
+#define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 6
+
 struct kfd_ioctl_dbg_trap_args {
 	__u64 ptr;     /* to KFD -- used for pointer arguments: queue arrays */
 	__u32 pid;     /* to KFD */

src/debug.c

@@ -275,8 +275,8 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId,
 				HSAuint32 data2,
 				HSAuint32 data3,
 				HSAuint32 pid,
-				HSAuint64 pointer
+				HSAuint64 pointer,
-				)
+				struct kfd_ioctl_dbg_trap_args *argout)
 {
 	uint32_t gpu_id;
 	HSAKMT_STATUS result;
@@ -320,6 +320,9 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId,
 
 	long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args);
 
+	if (argout)
+		*argout = args;
+
 	if (err == 0)
 		result = HSAKMT_STATUS_SUCCESS;
 	else
@@ -328,21 +331,45 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId,
 	return result;
 }
 
-HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32   NodeId,
+HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrapWithPollFd(HSAuint32   NodeId,
-					      HSA_QUEUEID QueueId)
+							HSA_QUEUEID QueueId,
+							HSAint32 *PollFd) //OUT
 {
+	int result;
+	struct kfd_ioctl_dbg_trap_args argout = {0};
+
 	if (QueueId != INVALID_QUEUEID)
 		return HSAKMT_STATUS_NOT_SUPPORTED;
 
-	return debug_trap(NodeId,
+	result =  debug_trap(NodeId,
 				KFD_IOC_DBG_TRAP_ENABLE,
 				1,
 				QueueId,
 				0,
 				INVALID_PID,
-				0);
+				0,
+				&argout);
+
+	*PollFd = argout.data3;
+
+	return result;
 }
 
+HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32   NodeId,
+					      HSA_QUEUEID QueueId)
+{
+	HSAint32 PollFd = 0;
+	HSAKMT_STATUS status = hsaKmtEnableDebugTrapWithPollFd(NodeId,
+							       QueueId,
+							       &PollFd);
+
+	if (status == HSAKMT_STATUS_SUCCESS)
+		close(PollFd);
+	return status;
+}
+
+
+
 HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId)
 {
 	return  debug_trap(NodeId,
@@ -351,7 +378,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId)
 			0,
 			0,
 			INVALID_PID,
-			0);
+			0,
+			NULL);
 }
 
 HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId,
@@ -364,7 +392,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId,
 				TrapData1,
 				0,
 				INVALID_PID,
-				0);
+				0,
+				NULL);
 }
 
 HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride(
@@ -381,7 +410,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride(
 				TrapMask,
 				0,
 				INVALID_PID,
-				0);
+				0,
+				NULL);
 }
 
 HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode(
@@ -394,7 +424,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode(
 				0,
 				0,
 				INVALID_PID,
-				0);
+				0,
+				NULL);
 }
 
 /**
@@ -455,7 +486,8 @@ hsaKmtQueueSuspend(
 			NumQueues,
 			GracePeriod,
 			Pid,
-			(HSAuint64)queue_ids_ptr);
+			(HSAuint64)queue_ids_ptr,
+			NULL);
 
 	free(queue_ids_ptr);
 	return result;
@@ -518,7 +550,47 @@ hsaKmtQueueResume(
 			NumQueues,
 			0,
 			Pid,
-			(HSAuint64)queue_ids_ptr);
+			(HSAuint64)queue_ids_ptr,
+			NULL);
 	free(queue_ids_ptr);
 	return result;
 }
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtQueryDebugEvent(
+		HSAuint32		NodeId, //IN
+		HSAuint32		Pid, // IN
+		HSAuint32		*QueueId, // IN/OUT
+		bool			ClearEvents, // IN
+		HSA_DEBUG_EVENT_TYPE	*EventsReceived, // OUT
+		bool			*IsSuspended // OUT
+		)
+{
+	HSAKMT_STATUS result;
+	struct kfd_ioctl_dbg_trap_args argout = {0};
+	uint32_t flags = 0;
+
+	if (ClearEvents)
+		flags |= KFD_DBG_EV_FLAG_CLEAR_STATUS;
+
+	result = debug_trap(NodeId,
+			    KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT,
+			    *QueueId,
+			    flags,
+			    0,
+			    Pid,
+			    0,
+			    &argout);
+
+	if (result)
+		return result;
+
+	*QueueId = argout.data1;
+	*EventsReceived = argout.data3 &
+			(KFD_DBG_EV_STATUS_TRAP | KFD_DBG_EV_STATUS_VMFAULT);
+	*IsSuspended = (argout.data3 & KFD_DBG_EV_STATUS_SUSPENDED) >> 2;
+
+	return result;
+}
+

src/libhsakmt.ver

@@ -58,7 +58,9 @@ hsaKmtQueryPointerInfo;
 hsaKmtSetMemoryUserData;
 hsaKmtGetQueueInfo;
 hsaKmtEnableDebugTrap;
+hsaKmtEnableDebugTrapWithPollFd;
 hsaKmtDisableDebugTrap;
+hsaKmtQueryDebugEvent;
 hsaKmtSetDebugTrapData2;
 hsaKmtSetWaveLaunchTrapOverride;
 hsaKmtSetWaveLaunchMode;