libhsakmt: Update wave suspend/resume API

This is updating to the new suspend and resume API for the
KFD and the thunk.  We now support passing in a list of queues
to suspend, and not just all of the queues for the process.

The kfdtest testcase was also updated so it still compiles.

Change-Id: I71d1b178476bd9df0c311bdedaa6a891528cebcf
Signed-off-by: Philip Cox <Philip.Cox@amd.com>

include/hsakmt.h

@@ -624,54 +624,88 @@ hsaKmtDbgAddressWatch(
 
 /**
    Suspend the execution of a set of queues. A queue that is suspended
-   allows the context save state to be inspected and modified. If a
+   allows the wave context save state to be inspected and modified. If a
    queue is already suspended it remains suspended. A suspended queue
    can be resumed by hsaKmtDbgQueueResume().
 
-   If NoGracePeriod is false then the default grace period used for
-   waiting for waves to complete before context switching is used. If
-   NoGracePeriod is true then no grace period us used and waves are
-   context saved as soon as possible.
-
-   If MemFence is true all queues being suspended will perform a
-   sequentially consistent system scope release that synchronizes with
-   a sequentially consistent system scope acquire performed by this
+   For each node that has a queue suspended, a sequentially consistent
+   system scope release will be performed that synchronizes with a
+   sequentially consistent system scope acquire performed by this
    call. This ensures any memory updates performed by the suspended
    queues are visible to the thread calling this operation.
 
+   Pid is the process that owns the queues that are to be supended or
+   resumed. If the value is -1 then the Pid of the process calling
+   hsaKmtQueueSuspend or hsaKmtQueueResume is used.
+
+   NumQueues is the number of queues that are being requested to
+   suspend or resume.
+
+   Queues is a pointer to an array with NumQueues entries of
+   HSA_QUEUEID. The queues in the list must be for queues that exist
+   for Pid, and can be a mixture of queues for different nodes.
+
+   GracePeriod to wait after initialiating context save before forcing
+   waves to context save. A value of 0 indicates no grace period.
+   It is ignored by hsaKmtQueueResume.
+
+   Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL.
    Returns:
     - HSAKMT_STATUS_SUCCESS if successful.
-
-    - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid.
+    - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid for Pid.
 */
+
 HSAKMT_STATUS
-    HSAKMTAPI
-    hsaKmtNodeSuspend(
-            HSAuint32 Pid,
-            HSAuint32 NodeId,
-            HSAuint32 Flags);
+HSAKMTAPI
+hsaKmtQueueSuspend(
+        HSAuint32    Pid,          // IN
+        HSAuint32    NumQueues,    // IN
+        HSA_QUEUEID *Queues,       // IN
+        HSAuint32    GracePeriod,  // IN
+        HSAuint32    Flags);       // IN
+
 /**
    Resume the execution of a set of queues. If a queue is not
-   suspended by hsaKmtDbgQueueSuspend() then it remains executing.
+   suspended by hsaKmtDbgQueueSuspend() then it remains executing. Any
+   changes to the wave state data will be used when the waves are
+   restored. Changes to the control stack data will have no effect.
 
-   If MemFence is true this call will perform a sequentially
-   consistent system scope release that synchronizes with a
+   For each node that has a queue resumed, a sequentially consistent
+   system scope release will be performed that synchronizes with a
    sequentially consistent system scope acquire performed by all
    queues being resumed. This ensures any memory updates performed by
    the thread calling this operation are visible to the resumed
    queues.
 
-   Returns:
-    - HSAKMT_STATUS_SUCCESS if successful.
+   For each node that has a queue resumed, the instruction cache will
+   be invalidated. This ensures any instruction code updates performed
+   by the thread calling this operation are visible to the resumed
+   queues.
 
+   Pid is the process that owns the queues that are to be supended or
+   resumed. If the value is -1 then the Pid of the process calling
+   hsaKmtQueueSuspend or hsaKmtQueueResume is used.
+
+   NumQueues is the number of queues that are being requested to
+   suspend or resume.
+
+   Queues is a pointer to an array with NumQueues entries of
+   HSA_QUEUEID. The queues in the list must be for queues that exist
+   for Pid, and can be a mixture of queues for different nodes.
+
+   Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL.
+   Returns:
+    - HSAKMT_STATUS_SUCCESS if successful
     - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid.
 */
+
 HSAKMT_STATUS
-    HSAKMTAPI
-    hsaKmtNodeResume(
-            HSAuint32 Pid,
-            HSAuint32 NodeId,
-            HSAuint32 Flags);
+HSAKMTAPI
+hsaKmtQueueResume(
+            HSAuint32   Pid,         // IN
+            HSAuint32   NumQueues,   // IN
+            HSA_QUEUEID *Queues,     // IN
+            HSAuint32   Flags);      // IN
 
 /**
   Enable debug trap for NodeId. If QueueId is INVALID_QUEUEID then

include/hsakmttypes.h

@@ -804,10 +804,11 @@ typedef enum _HSA_DBG_WAVE_LAUNCH_MODE
     HSA_DBG_WAVE_LAUNCH_MODE_NUM
 } HSA_DBG_WAVE_LAUNCH_MODE;
 
+/**
+ *    There are no flags currently defined.
+ */
 typedef enum HSA_DBG_NODE_CONTROL {
-      HSA_DBG_NODE_CONTROL_NO_GRACE_PERIOD = 0x01,
-      HSA_DBG_NODE_CONTROL_MEMORY_FENCE = 0x02,
-      HSA_DBG_NODE_CONTROL_UPDATE_CONTEXT = 0x04,
+    HSA_DBG_NODE_CONTROL_FLAG_MAX = 0x01
 } HSA_DBG_NODE_CONTROL;
 
 

include/linux/kfd_ioctl.h

@@ -188,64 +188,61 @@ struct kfd_ioctl_dbg_wave_control_args {
 };
 
 /* KFD_IOC_DBG_TRAP_ENABLE:
+ * ptr:   unused
  * data1: 0=disable, 1=enable
  * data2: queue ID (for future use)
  * data3: unused
- * data4: unused
  */
 #define KFD_IOC_DBG_TRAP_ENABLE 0
 
 /* KFD_IOC_DBG_TRAP_SET_TRAP_DATA:
+ * ptr:   unused
  * data1: SPI_GDBG_TRAP_DATA0
  * data2: SPI_GDBG_TRAP_DATA1
  * data3: unused
- * data4: unused
  */
 #define KFD_IOC_DBG_TRAP_SET_TRAP_DATA 1
 
 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE:
+ * ptr:   unused
  * data1: override mode: 0=OR, 1=REPLACE
  * data2: mask
  * data3: unused
- * data4: unused
  */
 #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 2
 
 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE:
+ * ptr:   unused
  * data1: 0=normal, 1=halt, 2=kill, 3=singlestep, 4=disable
  * data2: unused
  * data3: unused
- * data4: unused
  */
 #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 3
 
-
-#define KFD__DBG_NODE_SUSPEND_NO_GRACE          0x01
-#define KFD__DBG_NODE_SUSPEND_MEMORY_FENCE      0x02
-#define KFD__DBG_NODE_SUSPEND_UPDATE_CONTEXT    0x04
 /* KFD_IOC_DBG_TRAP_NODE_SUSPEND:
- * data1: pid
- * data2: nodeid
- * data3: flags no_grace=0x01 memory_fence=0x02 update_context=0x04
- * data4: unused
+ * ptr:   pointer to an array of Queues IDs
+ * data1: flags
+ * data2: number of queues
+ * data3: grace period
  */
 #define KFD_IOC_DBG_TRAP_NODE_SUSPEND 4
 
 /* KFD_IOC_DBG_TRAP_NODE_RESUME:
- * data1: pid
- * data2: nodeid
- * data3: flags no_grace=0x01 memory_fence=0x02 update_context=0x04
- * data4: unused
+ * ptr:   pointer to an array of Queues IDs
+ * data1: flags
+ * data2: number of queues
+ * data3: unused
  */
 #define KFD_IOC_DBG_TRAP_NODE_RESUME 5
 
 struct kfd_ioctl_dbg_trap_args {
-	__u32 gpu_id;  /* to KFD */
-	__u32 op;      /* to KFD */
-	__u32 data1;   /* to KFD */
-	__u32 data2;   /* to KFD */
-	__u32 data3;   /* to KFD */
-	__u32 data4;   /* to KFD */
+        __u64 ptr;     /* to KFD -- used for pointer arguments: queue arrays */
+        __u32 pid;     /* to KFD */
+        __u32 gpu_id;  /* to KFD */
+        __u32 op;      /* to KFD */
+        __u32 data1;   /* to KFD */
+        __u32 data2;   /* to KFD */
+        __u32 data3;   /* to KFD */
 };
 
 /* Matching HSA_EVENTTYPE */