Use paged memory for queues on MEC devices

MES devices need GART mappings and therefore need non-paged memory. But
using non-paged memory introduces performance regression where it can
take over 80 ms to see the signal changes if the memory is in the wrong
NUMA node. Currently, we cannot control NUMA affinity when allocating
non-paged memory. Using non-paged memory allocation only on devices that
have MES scheduler

Change-Id: Ib27fb01d75247aa4f2bb2aa4503c6af5a98afda0


[ROCm/ROCR-Runtime commit: c1e836b6ab]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h

@@ -329,6 +329,9 @@ class GpuAgent : public GpuAgentInt {
   // GPU devices.
   __forceinline uint32_t enumeration_index() const { return enum_index_; }
 
+  // @brief returns true if agent uses MES scheduler
+  __forceinline const bool isMES() const { return (isa_->GetMajorVersion() >= 11) ? true : false; };
+
   void Trim() override;
 
   const std::function<void*(size_t size, size_t align, core::MemoryRegion::AllocateFlags flags)>&

projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h

@@ -127,7 +127,7 @@ struct SharedQueue {
 
 class LocalQueue {
  public:
-  LocalQueue() : local_queue_(MemoryRegion::AllocateNonPaged) {}
+  LocalQueue(int mem_flags) : local_queue_(mem_flags) {}
   SharedQueue* queue() const { return local_queue_.shared_object(); }
 
  private:
@@ -144,7 +144,7 @@ All funtions other than Convert and public_handle must be virtual.
 */
 class Queue : public Checked<0xFA3906A679F9DB49>, private LocalQueue {
  public:
-  Queue() : LocalQueue(), amd_queue_(queue()->amd_queue) {
+  Queue(int mem_flags = 0) : LocalQueue(mem_flags), amd_queue_(queue()->amd_queue) {
     queue()->core_queue = this;
     public_handle_ = Convert(this);
   }

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp

@@ -80,7 +80,7 @@ int AqlQueue::rtti_id_ = 0;
 
 AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, ScratchInfo& scratch,
                    core::HsaEventCallback callback, void* err_data, bool is_kv)
-    : Queue(),
+    : Queue(agent->isMES() ? MemoryRegion::AllocateNonPaged : 0),
       LocalSignal(0, false),
       DoorbellSignal(signal()),
       ring_buf_(nullptr),