SWDEV-515356 - Make the round-robin queue selection

- Add custom compare to the map of queues, which will help with
 the round-robin selection

Change-Id: Ie67a820bfb1a5b484a1b3edced967eed94228bb8

rocclr/device/rocm/rocdevice.cpp

@@ -2945,16 +2945,18 @@ void Device::getHwEventTime(const amd::Event& event, uint64_t* start, uint64_t*
 hsa_queue_t* Device::getQueueFromPool(const uint qIndex) {
   // Check if queue with refCount 0 is available to use
   if (queuePool_[qIndex].size() < GPU_MAX_HW_QUEUES) {
-    for (auto it = queuePool_[qIndex].begin(); it != queuePool_[qIndex].end(); it++) {
-      if (it->second.refCount == 0) {
-        it->second.refCount++;
+    for (auto& it : queuePool_[qIndex]) {
+      if (it.second.refCount == 0) {
+        it.second.refCount++;
         ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Selected queue refCount: %p (%d)",
-                it->first->base_address, it->second.refCount);
-        return it->first;
+                it.first->base_address, it.second.refCount);
+        return it.first;
       }
     }
   } else {
     if (qIndex < QueuePriority::Total && queuePool_[qIndex].size() > 0) {
+      // Search through all available queues for the lowest counter.
+      // Note: the map is sorted in the allocation order for possible round-robin selection
       typedef decltype(queuePool_)::value_type::const_reference PoolRef;
       auto lowest = std::min_element(
           queuePool_[qIndex].begin(), queuePool_[qIndex].end(),
@@ -2968,6 +2970,7 @@ hsa_queue_t* Device::getQueueFromPool(const uint qIndex) {
   return nullptr;
 }
 
+// ================================================================================================
 hsa_queue_t* Device::acquireQueue(uint32_t queue_size_hint, bool coop_queue,
                                   const std::vector<uint32_t>& cuMask,
                                   amd::CommandQueue::Priority priority) {

rocclr/device/rocm/rocdevice.hpp

@@ -655,12 +655,18 @@ class Device : public NullDevice {
   static address mg_sync_;  //!< MGPU grid launch sync memory (SVM location)
 
   struct QueueInfo {
-    int refCount;
-    void* hostcallBuffer_;
+    int refCount;           //! Reference counter. Shows how many time the queue was shared
+    void* hostcallBuffer_;  //! Host call buffer for the HSA queue
   };
 
+  struct QueueCompare {
+    // Customized queue compare operator to make sure the queues are sorted in the creation order
+    bool operator()(hsa_queue_t* lhs, hsa_queue_t* rhs) const {
+        return (lhs->id < rhs->id) ? true : false;
+    }
+  };
   //! a vector for keeping Pool of HSA queues with low, normal and high priorities for recycling
-  std::vector<std::map<hsa_queue_t*, QueueInfo>> queuePool_;
+  std::vector<std::map<hsa_queue_t*, QueueInfo, QueueCompare>> queuePool_;
 
   //! returns a hsa queue from queuePool with least refCount and updates the refCount as well
   hsa_queue_t* getQueueFromPool(const uint qIndex);
@@ -671,7 +677,7 @@ class Device : public NullDevice {
                             std::vector<LinkAttrType>* link_attr);
 
   //! Pool of HSA queues with custom CU masks
-  std::vector<std::map<hsa_queue_t*, QueueInfo>> queueWithCUMaskPool_;
+  std::vector<std::map<hsa_queue_t*, QueueInfo, QueueCompare>> queueWithCUMaskPool_;
 
   //! Read and Write mask for device<->host
   uint32_t maxSdmaReadMask_;