Place GPU local resources in nearest NUMA node.

For minimal latency we should place command queues and blit code
in the nearest numa node to each GPU.  Add an allocator matching
the current runtime default allocator interface to each GpuAgent
that allocates on the closest numa node as represented by kfd
topology.  Use this allocator for queue ring buffers and blit
objects.

Change-Id: I181127f9c27bafe68976312963146616e3f58369

runtime/hsa-runtime/core/inc/amd_gpu_agent.h

@@ -328,6 +328,13 @@ class GpuAgent : public GpuAgentInt {
 
   void Trim() override;
 
+  const std::function<void*(size_t size, size_t align, core::MemoryRegion::AllocateFlags flags)>&
+  system_allocator() const {
+    return system_allocator_;
+  }
+
+  const std::function<void(void*)>& system_deallocator() const { return system_deallocator_; }
+
  protected:
   static const uint32_t minAqlSize_ = 0x1000;   // 4KB min
   static const uint32_t maxAqlSize_ = 0x20000;  // 8MB max
@@ -492,6 +499,9 @@ class GpuAgent : public GpuAgentInt {
   // @brief Setup GWS accessing queue.
   void InitGWS();
 
+  // @brief Setup NUMA aware system memory allocator.
+  void InitNumaAllocator();
+
   // @brief Register signal for notification when scratch may become available.
   // @p signal is notified by OR'ing with @p value.
   bool AddScratchNotifier(hsa_signal_t signal, hsa_signal_value_t value) {
@@ -532,6 +542,12 @@ class GpuAgent : public GpuAgentInt {
 
   ScratchCache scratch_cache_;
 
+  // System memory allocator in the nearest NUMA node.
+  std::function<void*(size_t size, size_t align, core::MemoryRegion::AllocateFlags flags)>
+      system_allocator_;
+
+  std::function<void(void*)> system_deallocator_;
+
   DISALLOW_COPY_AND_ASSIGN(GpuAgent);
 };
 

runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp

@@ -305,14 +305,12 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr
   }
 
   // Allocate IB for icache flushes.
-  pm4_ib_buf_ = core::Runtime::runtime_singleton_->system_allocator()(
-      pm4_ib_size_b_, 0x1000, core::MemoryRegion::AllocateExecutable);
+  pm4_ib_buf_ =
+      agent_->system_allocator()(pm4_ib_size_b_, 0x1000, core::MemoryRegion::AllocateExecutable);
   if (pm4_ib_buf_ == nullptr)
     throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES, "PM4 IB allocation failed.\n");
 
-  MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() {
-    core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_);
-  });
+  MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() { agent_->system_deallocator()(pm4_ib_buf_); });
 
   // Set initial CU mask
   SetCUMasking(0, nullptr);
@@ -356,7 +354,7 @@ AqlQueue::~AqlQueue() {
       queue_event_ = nullptr;
     }
   }
-  core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_);
+  agent_->system_deallocator()(pm4_ib_buf_);
 }
 
 void AqlQueue::Destroy() {
@@ -676,8 +674,9 @@ void AqlQueue::AllocRegisteredRingBuffer(uint32_t queue_size_pkts) {
     ring_buf_alloc_bytes_ = AlignUp(
         queue_size_pkts * sizeof(core::AqlPacket), 4096);
 
-    ring_buf_ = core::Runtime::runtime_singleton_->system_allocator()(
-        ring_buf_alloc_bytes_, 0x1000, core::MemoryRegion::AllocateExecutable |
+    ring_buf_ = agent_->system_allocator()(
+        ring_buf_alloc_bytes_, 0x1000,
+        core::MemoryRegion::AllocateExecutable |
             (queue_full_workaround_ ? core::MemoryRegion::AllocateDoubleMap : 0));
 
     assert(ring_buf_ != NULL && "AQL queue memory allocation failure");
@@ -699,7 +698,7 @@ void AqlQueue::FreeRegisteredRingBuffer() {
         (void*)(uintptr_t(ring_buf_) + (ring_buf_alloc_bytes_ / 2)));
 #endif
   } else {
-    core::Runtime::runtime_singleton_->system_deallocator()(ring_buf_);
+    agent_->system_deallocator()(ring_buf_);
   }
 
   ring_buf_ = NULL;

runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp

@@ -536,15 +536,14 @@ hsa_status_t BlitKernel::Initialize(const core::Agent& agent) {
     return status;
   }
 
+  const AMD::GpuAgent& gpuAgent = static_cast<const AMD::GpuAgent&>(agent);
   kernarg_async_ = reinterpret_cast<KernelArgs*>(
-      core::Runtime::runtime_singleton_->system_allocator()(
-          queue_->public_handle()->size * AlignUp(sizeof(KernelArgs), 16), 16,
-          core::MemoryRegion::AllocateNoFlags));
+      gpuAgent.system_allocator()(queue_->public_handle()->size * AlignUp(sizeof(KernelArgs), 16),
+                                  16, core::MemoryRegion::AllocateNoFlags));
 
   kernarg_async_mask_ = queue_->public_handle()->size - 1;
 
   // Obtain the number of compute units in the underlying agent.
-  const AMD::GpuAgent& gpuAgent = static_cast<const AMD::GpuAgent&>(agent);
   num_cus_ = gpuAgent.properties().NumFComputeCores / 4;
 
   // Assemble shaders to AQL code objects.
@@ -577,7 +576,7 @@ hsa_status_t BlitKernel::Destroy(const core::Agent& agent) {
   }
 
   if (kernarg_async_ != NULL) {
-    core::Runtime::runtime_singleton_->system_deallocator()(kernarg_async_);
+    gpuAgent.system_deallocator()(kernarg_async_);
   }
 
   if (completion_signal_.handle != 0) {

runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp

@@ -161,8 +161,8 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>:
   }
 
   // Allocate queue buffer.
-  queue_start_addr_ = (char*)core::Runtime::runtime_singleton_->system_allocator()(
-      kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable);
+  queue_start_addr_ =
+      (char*)agent_->system_allocator()(kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable);
 
   if (queue_start_addr_ == NULL) {
     return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
@@ -206,7 +206,7 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>:
 
   if (queue_start_addr_ != NULL) {
     // Release queue buffer.
-    core::Runtime::runtime_singleton_->system_deallocator()(queue_start_addr_);
+    agent_->system_deallocator()(queue_start_addr_);
   }
 
   queue_start_addr_ = NULL;

runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -178,7 +178,7 @@ GpuAgent::~GpuAgent() {
     hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size());
   }
 
-  core::Runtime::runtime_singleton_->system_deallocator()(doorbell_queue_map_);
+  system_deallocator()(doorbell_queue_map_);
 
   if (trap_code_buf_ != NULL) {
     ReleaseShader(trap_code_buf_, trap_code_buf_size_);
@@ -288,8 +288,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar
       (assemble_target == AssembleTarget::AQL ? sizeof(amd_kernel_code_t) : 0);
   code_buf_size = AlignUp(header_size + asic_shader->size, 0x1000);
 
-  code_buf = core::Runtime::runtime_singleton_->system_allocator()(
-      code_buf_size, 0x1000, core::MemoryRegion::AllocateExecutable);
+  code_buf = system_allocator()(code_buf_size, 0x1000, core::MemoryRegion::AllocateExecutable);
   assert(code_buf != NULL && "Code buffer allocation failed");
 
   memset(code_buf, 0, code_buf_size);
@@ -335,7 +334,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar
 }
 
 void GpuAgent::ReleaseShader(void* code_buf, size_t code_buf_size) const {
-  core::Runtime::runtime_singleton_->system_deallocator()(code_buf);
+  system_deallocator()(code_buf);
 }
 
 void GpuAgent::InitRegionList() {
@@ -687,6 +686,7 @@ void GpuAgent::PreloadBlits() {
 
 hsa_status_t GpuAgent::PostToolsInit() {
   // Defer memory allocation until agents have been discovered.
+  InitNumaAllocator();
   InitScratchPool();
   BindTrapHandler();
   InitDma();
@@ -1415,8 +1415,7 @@ void GpuAgent::BindTrapHandler() {
     // The trap handler uses this to retrieve a wave's amd_queue_t*.
     auto doorbell_queue_map_size = MAX_NUM_DOORBELLS * sizeof(amd_queue_t*);
 
-    doorbell_queue_map_ = (amd_queue_t**)core::Runtime::runtime_singleton_->system_allocator()(
-        doorbell_queue_map_size, 0x1000, 0);
+    doorbell_queue_map_ = (amd_queue_t**)system_allocator()(doorbell_queue_map_size, 0x1000, 0);
     assert(doorbell_queue_map_ != NULL && "Doorbell queue map allocation failed");
 
     memset(doorbell_queue_map_, 0, doorbell_queue_map_size);
@@ -1570,5 +1569,37 @@ void GpuAgent::Trim() {
   scratch_cache_.trim(false);
 }
 
+void GpuAgent::InitNumaAllocator() {
+  Agent* nearCpu = nullptr;
+  uint32_t dist = -1u;
+  for (auto cpu : core::Runtime::runtime_singleton_->cpu_agents()) {
+    const core::Runtime::LinkInfo link_info =
+        core::Runtime::runtime_singleton_->GetLinkInfo(node_id(), cpu->node_id());
+    if (link_info.info.numa_distance < dist) {
+      dist = link_info.info.numa_distance;
+      nearCpu = cpu;
+    }
+  }
+
+  for (auto pool : nearCpu->regions()) {
+    if (pool->kernarg()) {
+      system_allocator_ = [pool](size_t size, size_t alignment,
+                                 MemoryRegion::AllocateFlags alloc_flags) -> void* {
+        assert(alignment <= 4096);
+        void* ptr = nullptr;
+        return (HSA_STATUS_SUCCESS ==
+                core::Runtime::runtime_singleton_->AllocateMemory(pool, size, alloc_flags, &ptr))
+            ? ptr
+            : nullptr;
+      };
+
+      system_deallocator_ = [](void* ptr) { core::Runtime::runtime_singleton_->FreeMemory(ptr); };
+
+      return;
+    }
+  }
+  assert(false && "Nearest NUMA node did not have a kernarg pool.");
+}
+
 }  // namespace amd
 }  // namespace rocr