Add hsa_amd_queue_set_priority extension function

Controls dispatch and wavefront scheduling arbitration across quees.

Change-Id: I498f4898b544f79b8fb8514bf7e789ca9da29462

runtime/hsa-runtime/core/common/hsa_table_interface.cpp

@@ -1116,3 +1116,9 @@ hsa_status_t hsa_amd_queue_intercept_register(hsa_queue_t* queue,
                                               void* user_data) {
   return amdExtTable->hsa_amd_queue_intercept_register_fn(queue, callback, user_data);
 }
+
+// Mirrors Amd Extension Apis
+hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue,
+                                                hsa_amd_queue_priority_t priority) {
+  return amdExtTable->hsa_amd_queue_set_priority_fn(queue, priority);
+}

runtime/hsa-runtime/core/inc/amd_aql_queue.h

@@ -71,6 +71,9 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo
   /// @brief Queue interfaces
   hsa_status_t Inactivate() override;
 
+  /// @brief Change the scheduling priority of the queue
+  hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) override;
+
   /// @brief Atomically reads the Read index of with Acquire semantics
   ///
   /// @return uint64_t Value of read index
@@ -254,6 +257,12 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo
   std::atomic<uint32_t> dynamicScratchState;
   enum { ERROR_HANDLER_DONE = 1, ERROR_HANDLER_TERMINATE = 2, ERROR_HANDLER_SCRATCH_RETRY = 4 };
 
+  // Queue currently suspended or scheduled
+  bool suspended_;
+
+  // Thunk dispatch and wavefront scheduling priority
+  HSA_QUEUE_PRIORITY priority_;
+
   // Shared event used for queue errors
   static HsaEvent* queue_event_;
 

runtime/hsa-runtime/core/inc/host_queue.h

@@ -59,6 +59,9 @@ class HostQueue : public Queue {
   ~HostQueue();
 
   hsa_status_t Inactivate() override { return HSA_STATUS_SUCCESS; }
+  hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) override {
+    return HSA_STATUS_ERROR_INVALID_QUEUE;
+  }
 
   uint64_t LoadReadIndexAcquire() override {
     return atomic::Load(&amd_queue_.read_dispatch_id,

runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h

@@ -214,6 +214,10 @@ hsa_status_t hsa_amd_ipc_signal_attach(const hsa_amd_ipc_signal_t* handle, hsa_s
 hsa_status_t hsa_amd_register_system_event_handler(
     hsa_amd_system_event_callback_t callback, void* data);
 
+// Mirrors Amd Extension Apis
+hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue,
+                                                hsa_amd_queue_priority_t priority);
+
 }  // end of AMD namespace
 
 #endif  // header guard

runtime/hsa-runtime/core/inc/intercept_queue.h

@@ -70,6 +70,9 @@ class QueueWrapper : public Queue {
   ~QueueWrapper() {}
 
   hsa_status_t Inactivate() override { return wrapped->Inactivate(); }
+  hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) override {
+    return wrapped->SetPriority(priority);
+  }
   uint64_t LoadReadIndexAcquire() override { return wrapped->LoadReadIndexAcquire(); }
   uint64_t LoadReadIndexRelaxed() override { return wrapped->LoadReadIndexRelaxed(); }
   uint64_t LoadWriteIndexRelaxed() override { return wrapped->LoadWriteIndexRelaxed(); }

runtime/hsa-runtime/core/inc/queue.h

@@ -54,6 +54,8 @@
 
 #include "inc/amd_hsa_queue.h"
 
+#include "hsakmt.h"
+
 namespace core {
 struct AqlPacket {
 
@@ -176,6 +178,9 @@ class Queue : public Checked<0xFA3906A679F9DB49>, private LocalQueue {
   /// @return hsa_status_t Status of request
   virtual hsa_status_t Inactivate() = 0;
 
+  /// @brief Change the scheduling priority of the queue
+  virtual hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) = 0;
+
   /// @brief Reads the Read Index of Queue using Acquire semantics
   ///
   /// @return uint64_t Value of Read index

runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp

@@ -93,7 +93,9 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr
       is_kv_queue_(is_kv),
       pm4_ib_buf_(nullptr),
       pm4_ib_size_b_(0x1000),
-      dynamicScratchState(0) {
+      dynamicScratchState(0),
+      suspended_(false),
+      priority_(HSA_QUEUE_PRIORITY_NORMAL) {
   // When queue_full_workaround_ is set to 1, the ring buffer is internally
   // doubled in size. Virtual addresses in the upper half of the ring allocation
   // are mapped to the same set of pages backing the lower half.
@@ -150,8 +152,7 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr
   }
 
   HSAKMT_STATUS kmt_status;
-  kmt_status = hsaKmtCreateQueue(node_id, HSA_QUEUE_COMPUTE_AQL, 100,
-                                 HSA_QUEUE_PRIORITY_NORMAL, ring_buf_,
+  kmt_status = hsaKmtCreateQueue(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, ring_buf_,
                                  ring_buf_alloc_bytes_, NULL, &queue_rsrc);
   if (kmt_status != HSAKMT_STATUS_SUCCESS)
     throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES,
@@ -686,7 +687,8 @@ int AqlQueue::CreateRingBufferFD(const char* ring_buf_shm_path,
 }
 
 void AqlQueue::Suspend() {
-  auto err = hsaKmtUpdateQueue(queue_id_, 0, HSA_QUEUE_PRIORITY_NORMAL, NULL, 0, NULL);
+  suspended_ = true;
+  auto err = hsaKmtUpdateQueue(queue_id_, 0, priority_, NULL, 0, NULL);
   assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtUpdateQueue failed.");
 }
 
@@ -700,6 +702,16 @@ hsa_status_t AqlQueue::Inactivate() {
   return HSA_STATUS_SUCCESS;
 }
 
+hsa_status_t AqlQueue::SetPriority(HSA_QUEUE_PRIORITY priority) {
+  if (suspended_) {
+    return HSA_STATUS_ERROR_INVALID_QUEUE;
+  }
+
+  priority_ = priority;
+  auto err = hsaKmtUpdateQueue(queue_id_, 100, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL);
+  return (err == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR_OUT_OF_RESOURCES);
+}
+
 bool AqlQueue::DynamicScratchHandler(hsa_signal_value_t error_code, void* arg) {
   AqlQueue* queue = (AqlQueue*)arg;
   hsa_status_t errorCode = HSA_STATUS_SUCCESS;

runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp

@@ -380,6 +380,7 @@ void HsaApiTable::UpdateAmdExts() {
   amd_ext_api.hsa_amd_register_system_event_handler_fn = AMD::hsa_amd_register_system_event_handler;
   amd_ext_api.hsa_amd_queue_intercept_create_fn = AMD::hsa_amd_queue_intercept_create;
   amd_ext_api.hsa_amd_queue_intercept_register_fn = AMD::hsa_amd_queue_intercept_register;
+  amd_ext_api.hsa_amd_queue_set_priority_fn = AMD::hsa_amd_queue_set_priority;
 }
 
 class Init {

runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp

@@ -46,6 +46,7 @@
 #include <set>
 #include <utility>
 #include <memory>
+#include <map>
 
 #include "core/inc/runtime.h"
 #include "core/inc/agent.h"
@@ -852,4 +853,28 @@ hsa_status_t hsa_amd_register_system_event_handler(hsa_amd_system_event_callback
   CATCH;
 }
 
+hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue,
+                                                hsa_amd_queue_priority_t priority) {
+  TRY;
+  IS_OPEN();
+  IS_BAD_PTR(queue);
+  core::Queue* cmd_queue = core::Queue::Convert(queue);
+  IS_VALID(cmd_queue);
+
+  static std::map<hsa_amd_queue_priority_t, HSA_QUEUE_PRIORITY> ext_kmt_priomap = {
+      {HSA_AMD_QUEUE_PRIORITY_LOW, HSA_QUEUE_PRIORITY_MINIMUM},
+      {HSA_AMD_QUEUE_PRIORITY_NORMAL, HSA_QUEUE_PRIORITY_NORMAL},
+      {HSA_AMD_QUEUE_PRIORITY_HIGH, HSA_QUEUE_PRIORITY_MAXIMUM},
+  };
+
+  auto priority_it = ext_kmt_priomap.find(priority);
+
+  if (priority_it == ext_kmt_priomap.end()) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  return cmd_queue->SetPriority(priority_it->second);
+  CATCH;
+}
+
 } // end of AMD namespace

runtime/hsa-runtime/hsacore.so.def

@@ -215,6 +215,7 @@ global:
 	hsa_amd_ipc_signal_create;
 	hsa_amd_ipc_signal_attach;
 	hsa_amd_register_system_event_handler;
+	hsa_amd_queue_set_priority;
 
 local:
     *;

runtime/hsa-runtime/inc/hsa_api_trace.h

@@ -171,6 +171,7 @@ struct AmdExtTable {
   decltype(hsa_amd_register_system_event_handler)* hsa_amd_register_system_event_handler_fn;
   decltype(hsa_amd_queue_intercept_create)* hsa_amd_queue_intercept_create_fn;
   decltype(hsa_amd_queue_intercept_register)* hsa_amd_queue_intercept_register_fn;
+  decltype(hsa_amd_queue_set_priority)* hsa_amd_queue_set_priority_fn;
 };
 
 // Table to export HSA Core Runtime Apis

runtime/hsa-runtime/inc/hsa_ext_amd.h

@@ -1697,6 +1697,43 @@ typedef hsa_status_t (*hsa_amd_system_event_callback_t)(const hsa_amd_event_t* e
 hsa_status_t hsa_amd_register_system_event_handler(hsa_amd_system_event_callback_t callback,
                                                    void* data);
 
+/**
+ * @brief Per-queue dispatch and wavefront scheduling priority.
+ */
+typedef enum hsa_amd_queue_priority_s {
+  /*
+  Below normal/high priority compute and all graphics
+  */
+  HSA_AMD_QUEUE_PRIORITY_LOW = 0,
+  /*
+  Above low priority compute, below high priority compute and all graphics
+  */
+  HSA_AMD_QUEUE_PRIORITY_NORMAL = 1,
+  /*
+  Above low/normal priority compute and all graphics
+  */
+  HSA_AMD_QUEUE_PRIORITY_HIGH = 2,
+} hsa_amd_queue_priority_t;
+
+/**
+ * @brief Modifies the dispatch and wavefront scheduling prioirty for a
+ * given compute queue. The default is HSA_AMD_QUEUE_PRIORITY_NORMAL.
+ *
+ * @param[in] queue Compute queue to apply new priority to.
+ *
+ * @param[in] priority Priority to associate with queue.
+ *
+ * @retval HSA_STATUS_SUCCESS if priority was changed successfully.
+ *
+ * @retval HSA_STATUS_ERROR_INVALID_QUEUE if queue is not a valid
+ * compute queue handle.
+ *
+ * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT if priority is not a valid
+ * value from hsa_amd_queue_priority_t.
+ */
+hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue,
+                                                hsa_amd_queue_priority_t priority);
+
 #ifdef __cplusplus
 }  // end extern "C" block
 #endif