From ba8e740be435a05e4cd97db73a09868cb40f07eb Mon Sep 17 00:00:00 2001
From: German Andryeyev <German.Andryeyev@amd.com>
Date: Tue, 18 Feb 2025 13:00:01 -0500
Subject: [PATCH] SWDEV-515356 - Make the round-robin queue selection

- Add custom compare to the map of queues, which will help with
 the round-robin selection

Change-Id: Ie67a820bfb1a5b484a1b3edced967eed94228bb8
---
 rocclr/device/rocm/rocdevice.cpp | 13 ++++++++-----
 rocclr/device/rocm/rocdevice.hpp | 14 ++++++++++----
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/rocclr/device/rocm/rocdevice.cpp b/rocclr/device/rocm/rocdevice.cpp
index 9ddf8441f4..17a6a223c9 100644
--- a/rocclr/device/rocm/rocdevice.cpp
+++ b/rocclr/device/rocm/rocdevice.cpp
@@ -2945,16 +2945,18 @@ void Device::getHwEventTime(const amd::Event& event, uint64_t* start, uint64_t*
 hsa_queue_t* Device::getQueueFromPool(const uint qIndex) {
   // Check if queue with refCount 0 is available to use
   if (queuePool_[qIndex].size() < GPU_MAX_HW_QUEUES) {
-    for (auto it = queuePool_[qIndex].begin(); it != queuePool_[qIndex].end(); it++) {
-      if (it->second.refCount == 0) {
-        it->second.refCount++;
+    for (auto& it : queuePool_[qIndex]) {
+      if (it.second.refCount == 0) {
+        it.second.refCount++;
         ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Selected queue refCount: %p (%d)",
-                it->first->base_address, it->second.refCount);
-        return it->first;
+                it.first->base_address, it.second.refCount);
+        return it.first;
       }
     }
   } else {
     if (qIndex < QueuePriority::Total && queuePool_[qIndex].size() > 0) {
+      // Search through all available queues for the lowest counter.
+      // Note: the map is sorted in the allocation order for possible round-robin selection
       typedef decltype(queuePool_)::value_type::const_reference PoolRef;
       auto lowest = std::min_element(
           queuePool_[qIndex].begin(), queuePool_[qIndex].end(),
@@ -2968,6 +2970,7 @@ hsa_queue_t* Device::getQueueFromPool(const uint qIndex) {
   return nullptr;
 }
 
+// ================================================================================================
 hsa_queue_t* Device::acquireQueue(uint32_t queue_size_hint, bool coop_queue,
                                   const std::vector<uint32_t>& cuMask,
                                   amd::CommandQueue::Priority priority) {
diff --git a/rocclr/device/rocm/rocdevice.hpp b/rocclr/device/rocm/rocdevice.hpp
index 4cafb23786..1b53e6227a 100644
--- a/rocclr/device/rocm/rocdevice.hpp
+++ b/rocclr/device/rocm/rocdevice.hpp
@@ -655,12 +655,18 @@ class Device : public NullDevice {
   static address mg_sync_;  //!< MGPU grid launch sync memory (SVM location)
 
   struct QueueInfo {
-    int refCount;
-    void* hostcallBuffer_;
+    int refCount;           //! Reference counter. Shows how many time the queue was shared
+    void* hostcallBuffer_;  //! Host call buffer for the HSA queue
   };
 
+  struct QueueCompare {
+    // Customized queue compare operator to make sure the queues are sorted in the creation order
+    bool operator()(hsa_queue_t* lhs, hsa_queue_t* rhs) const {
+        return (lhs->id < rhs->id) ? true : false;
+    }
+  };
   //! a vector for keeping Pool of HSA queues with low, normal and high priorities for recycling
-  std::vector<std::map<hsa_queue_t*, QueueInfo>> queuePool_;
+  std::vector<std::map<hsa_queue_t*, QueueInfo, QueueCompare>> queuePool_;
 
   //! returns a hsa queue from queuePool with least refCount and updates the refCount as well
   hsa_queue_t* getQueueFromPool(const uint qIndex);
@@ -671,7 +677,7 @@ class Device : public NullDevice {
                             std::vector<LinkAttrType>* link_attr);
 
   //! Pool of HSA queues with custom CU masks
-  std::vector<std::map<hsa_queue_t*, QueueInfo>> queueWithCUMaskPool_;
+  std::vector<std::map<hsa_queue_t*, QueueInfo, QueueCompare>> queueWithCUMaskPool_;
 
   //! Read and Write mask for device<->host
   uint32_t maxSdmaReadMask_;