diff --git a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index c24c5ec865..7e48e0cbbe 100644 --- a/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -1116,3 +1116,9 @@ hsa_status_t hsa_amd_queue_intercept_register(hsa_queue_t* queue, void* user_data) { return amdExtTable->hsa_amd_queue_intercept_register_fn(queue, callback, user_data); } + +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue, + hsa_amd_queue_priority_t priority) { + return amdExtTable->hsa_amd_queue_set_priority_fn(queue, priority); +} diff --git a/runtime/hsa-runtime/core/inc/amd_aql_queue.h b/runtime/hsa-runtime/core/inc/amd_aql_queue.h index 795928bfb1..801f586608 100644 --- a/runtime/hsa-runtime/core/inc/amd_aql_queue.h +++ b/runtime/hsa-runtime/core/inc/amd_aql_queue.h @@ -71,6 +71,9 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo /// @brief Queue interfaces hsa_status_t Inactivate() override; + /// @brief Change the scheduling priority of the queue + hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) override; + /// @brief Atomically reads the Read index of with Acquire semantics /// /// @return uint64_t Value of read index @@ -254,6 +257,12 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo std::atomic dynamicScratchState; enum { ERROR_HANDLER_DONE = 1, ERROR_HANDLER_TERMINATE = 2, ERROR_HANDLER_SCRATCH_RETRY = 4 }; + // Queue currently suspended or scheduled + bool suspended_; + + // Thunk dispatch and wavefront scheduling priority + HSA_QUEUE_PRIORITY priority_; + // Shared event used for queue errors static HsaEvent* queue_event_; diff --git a/runtime/hsa-runtime/core/inc/host_queue.h b/runtime/hsa-runtime/core/inc/host_queue.h index 20ecd0d683..05dd050a6c 100644 --- a/runtime/hsa-runtime/core/inc/host_queue.h +++ b/runtime/hsa-runtime/core/inc/host_queue.h @@ -59,6 +59,9 @@ class HostQueue : public Queue { ~HostQueue(); hsa_status_t Inactivate() override { return HSA_STATUS_SUCCESS; } + hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) override { + return HSA_STATUS_ERROR_INVALID_QUEUE; + } uint64_t LoadReadIndexAcquire() override { return atomic::Load(&amd_queue_.read_dispatch_id, diff --git a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index 440a397bf2..2d30331d48 100644 --- a/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -214,6 +214,10 @@ hsa_status_t hsa_amd_ipc_signal_attach(const hsa_amd_ipc_signal_t* handle, hsa_s hsa_status_t hsa_amd_register_system_event_handler( hsa_amd_system_event_callback_t callback, void* data); +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue, + hsa_amd_queue_priority_t priority); + } // end of AMD namespace #endif // header guard diff --git a/runtime/hsa-runtime/core/inc/intercept_queue.h b/runtime/hsa-runtime/core/inc/intercept_queue.h index 3f1cd956ef..307e6664b7 100644 --- a/runtime/hsa-runtime/core/inc/intercept_queue.h +++ b/runtime/hsa-runtime/core/inc/intercept_queue.h @@ -70,6 +70,9 @@ class QueueWrapper : public Queue { ~QueueWrapper() {} hsa_status_t Inactivate() override { return wrapped->Inactivate(); } + hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) override { + return wrapped->SetPriority(priority); + } uint64_t LoadReadIndexAcquire() override { return wrapped->LoadReadIndexAcquire(); } uint64_t LoadReadIndexRelaxed() override { return wrapped->LoadReadIndexRelaxed(); } uint64_t LoadWriteIndexRelaxed() override { return wrapped->LoadWriteIndexRelaxed(); } diff --git a/runtime/hsa-runtime/core/inc/queue.h b/runtime/hsa-runtime/core/inc/queue.h index 811206457e..06a6911f15 100644 --- a/runtime/hsa-runtime/core/inc/queue.h +++ b/runtime/hsa-runtime/core/inc/queue.h @@ -54,6 +54,8 @@ #include "inc/amd_hsa_queue.h" +#include "hsakmt.h" + namespace core { struct AqlPacket { @@ -176,6 +178,9 @@ class Queue : public Checked<0xFA3906A679F9DB49>, private LocalQueue { /// @return hsa_status_t Status of request virtual hsa_status_t Inactivate() = 0; + /// @brief Change the scheduling priority of the queue + virtual hsa_status_t SetPriority(HSA_QUEUE_PRIORITY priority) = 0; + /// @brief Reads the Read Index of Queue using Acquire semantics /// /// @return uint64_t Value of Read index diff --git a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index cac25fcf4f..d9ef5d6a66 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -93,7 +93,9 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr is_kv_queue_(is_kv), pm4_ib_buf_(nullptr), pm4_ib_size_b_(0x1000), - dynamicScratchState(0) { + dynamicScratchState(0), + suspended_(false), + priority_(HSA_QUEUE_PRIORITY_NORMAL) { // When queue_full_workaround_ is set to 1, the ring buffer is internally // doubled in size. Virtual addresses in the upper half of the ring allocation // are mapped to the same set of pages backing the lower half. @@ -150,8 +152,7 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr } HSAKMT_STATUS kmt_status; - kmt_status = hsaKmtCreateQueue(node_id, HSA_QUEUE_COMPUTE_AQL, 100, - HSA_QUEUE_PRIORITY_NORMAL, ring_buf_, + kmt_status = hsaKmtCreateQueue(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL, &queue_rsrc); if (kmt_status != HSAKMT_STATUS_SUCCESS) throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES, @@ -686,7 +687,8 @@ int AqlQueue::CreateRingBufferFD(const char* ring_buf_shm_path, } void AqlQueue::Suspend() { - auto err = hsaKmtUpdateQueue(queue_id_, 0, HSA_QUEUE_PRIORITY_NORMAL, NULL, 0, NULL); + suspended_ = true; + auto err = hsaKmtUpdateQueue(queue_id_, 0, priority_, NULL, 0, NULL); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtUpdateQueue failed."); } @@ -700,6 +702,16 @@ hsa_status_t AqlQueue::Inactivate() { return HSA_STATUS_SUCCESS; } +hsa_status_t AqlQueue::SetPriority(HSA_QUEUE_PRIORITY priority) { + if (suspended_) { + return HSA_STATUS_ERROR_INVALID_QUEUE; + } + + priority_ = priority; + auto err = hsaKmtUpdateQueue(queue_id_, 100, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL); + return (err == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR_OUT_OF_RESOURCES); +} + bool AqlQueue::DynamicScratchHandler(hsa_signal_value_t error_code, void* arg) { AqlQueue* queue = (AqlQueue*)arg; hsa_status_t errorCode = HSA_STATUS_SUCCESS; diff --git a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index 3b6f7e037d..75332351e4 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -380,6 +380,7 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_register_system_event_handler_fn = AMD::hsa_amd_register_system_event_handler; amd_ext_api.hsa_amd_queue_intercept_create_fn = AMD::hsa_amd_queue_intercept_create; amd_ext_api.hsa_amd_queue_intercept_register_fn = AMD::hsa_amd_queue_intercept_register; + amd_ext_api.hsa_amd_queue_set_priority_fn = AMD::hsa_amd_queue_set_priority; } class Init { diff --git a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index b0bf5a19d3..1ef79a3b8c 100644 --- a/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include "core/inc/runtime.h" #include "core/inc/agent.h" @@ -852,4 +853,28 @@ hsa_status_t hsa_amd_register_system_event_handler(hsa_amd_system_event_callback CATCH; } +hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue, + hsa_amd_queue_priority_t priority) { + TRY; + IS_OPEN(); + IS_BAD_PTR(queue); + core::Queue* cmd_queue = core::Queue::Convert(queue); + IS_VALID(cmd_queue); + + static std::map ext_kmt_priomap = { + {HSA_AMD_QUEUE_PRIORITY_LOW, HSA_QUEUE_PRIORITY_MINIMUM}, + {HSA_AMD_QUEUE_PRIORITY_NORMAL, HSA_QUEUE_PRIORITY_NORMAL}, + {HSA_AMD_QUEUE_PRIORITY_HIGH, HSA_QUEUE_PRIORITY_MAXIMUM}, + }; + + auto priority_it = ext_kmt_priomap.find(priority); + + if (priority_it == ext_kmt_priomap.end()) { + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + + return cmd_queue->SetPriority(priority_it->second); + CATCH; +} + } // end of AMD namespace diff --git a/runtime/hsa-runtime/hsacore.so.def b/runtime/hsa-runtime/hsacore.so.def index 9299b8de4b..128fd35f28 100644 --- a/runtime/hsa-runtime/hsacore.so.def +++ b/runtime/hsa-runtime/hsacore.so.def @@ -215,6 +215,7 @@ global: hsa_amd_ipc_signal_create; hsa_amd_ipc_signal_attach; hsa_amd_register_system_event_handler; + hsa_amd_queue_set_priority; local: *; diff --git a/runtime/hsa-runtime/inc/hsa_api_trace.h b/runtime/hsa-runtime/inc/hsa_api_trace.h index e25cd49428..c7efec8626 100644 --- a/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -171,6 +171,7 @@ struct AmdExtTable { decltype(hsa_amd_register_system_event_handler)* hsa_amd_register_system_event_handler_fn; decltype(hsa_amd_queue_intercept_create)* hsa_amd_queue_intercept_create_fn; decltype(hsa_amd_queue_intercept_register)* hsa_amd_queue_intercept_register_fn; + decltype(hsa_amd_queue_set_priority)* hsa_amd_queue_set_priority_fn; }; // Table to export HSA Core Runtime Apis diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index ff3903586a..8033050d9b 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -1697,6 +1697,43 @@ typedef hsa_status_t (*hsa_amd_system_event_callback_t)(const hsa_amd_event_t* e hsa_status_t hsa_amd_register_system_event_handler(hsa_amd_system_event_callback_t callback, void* data); +/** + * @brief Per-queue dispatch and wavefront scheduling priority. + */ +typedef enum hsa_amd_queue_priority_s { + /* + Below normal/high priority compute and all graphics + */ + HSA_AMD_QUEUE_PRIORITY_LOW = 0, + /* + Above low priority compute, below high priority compute and all graphics + */ + HSA_AMD_QUEUE_PRIORITY_NORMAL = 1, + /* + Above low/normal priority compute and all graphics + */ + HSA_AMD_QUEUE_PRIORITY_HIGH = 2, +} hsa_amd_queue_priority_t; + +/** + * @brief Modifies the dispatch and wavefront scheduling prioirty for a + * given compute queue. The default is HSA_AMD_QUEUE_PRIORITY_NORMAL. + * + * @param[in] queue Compute queue to apply new priority to. + * + * @param[in] priority Priority to associate with queue. + * + * @retval HSA_STATUS_SUCCESS if priority was changed successfully. + * + * @retval HSA_STATUS_ERROR_INVALID_QUEUE if queue is not a valid + * compute queue handle. + * + * @retval HSA_STATUS_ERROR_INVALID_ARGUMENT if priority is not a valid + * value from hsa_amd_queue_priority_t. + */ +hsa_status_t HSA_API hsa_amd_queue_set_priority(hsa_queue_t* queue, + hsa_amd_queue_priority_t priority); + #ifdef __cplusplus } // end extern "C" block #endif