From 5af558f739062569e92b23b5e50f874b00cdc335 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Fri, 10 Sep 2021 21:00:28 -0500 Subject: [PATCH] Place GPU local resources in nearest NUMA node. For minimal latency we should place command queues and blit code in the nearest numa node to each GPU. Add an allocator matching the current runtime default allocator interface to each GpuAgent that allocates on the closest numa node as represented by kfd topology. Use this allocator for queue ring buffers and blit objects. Change-Id: I181127f9c27bafe68976312963146616e3f58369 --- runtime/hsa-runtime/core/inc/amd_gpu_agent.h | 16 +++++++ .../core/runtime/amd_aql_queue.cpp | 17 ++++---- .../core/runtime/amd_blit_kernel.cpp | 9 ++-- .../core/runtime/amd_blit_sdma.cpp | 6 +-- .../core/runtime/amd_gpu_agent.cpp | 43 ++++++++++++++++--- 5 files changed, 68 insertions(+), 23 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index c1b26495c4..8d1544987c 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -328,6 +328,13 @@ class GpuAgent : public GpuAgentInt { void Trim() override; + const std::function& + system_allocator() const { + return system_allocator_; + } + + const std::function& system_deallocator() const { return system_deallocator_; } + protected: static const uint32_t minAqlSize_ = 0x1000; // 4KB min static const uint32_t maxAqlSize_ = 0x20000; // 8MB max @@ -492,6 +499,9 @@ class GpuAgent : public GpuAgentInt { // @brief Setup GWS accessing queue. void InitGWS(); + // @brief Setup NUMA aware system memory allocator. + void InitNumaAllocator(); + // @brief Register signal for notification when scratch may become available. // @p signal is notified by OR'ing with @p value. bool AddScratchNotifier(hsa_signal_t signal, hsa_signal_value_t value) { @@ -532,6 +542,12 @@ class GpuAgent : public GpuAgentInt { ScratchCache scratch_cache_; + // System memory allocator in the nearest NUMA node. + std::function + system_allocator_; + + std::function system_deallocator_; + DISALLOW_COPY_AND_ASSIGN(GpuAgent); }; diff --git a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index 48c7d23d6a..424ec07c86 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -305,14 +305,12 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr } // Allocate IB for icache flushes. - pm4_ib_buf_ = core::Runtime::runtime_singleton_->system_allocator()( - pm4_ib_size_b_, 0x1000, core::MemoryRegion::AllocateExecutable); + pm4_ib_buf_ = + agent_->system_allocator()(pm4_ib_size_b_, 0x1000, core::MemoryRegion::AllocateExecutable); if (pm4_ib_buf_ == nullptr) throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES, "PM4 IB allocation failed.\n"); - MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() { - core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_); - }); + MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() { agent_->system_deallocator()(pm4_ib_buf_); }); // Set initial CU mask SetCUMasking(0, nullptr); @@ -356,7 +354,7 @@ AqlQueue::~AqlQueue() { queue_event_ = nullptr; } } - core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_); + agent_->system_deallocator()(pm4_ib_buf_); } void AqlQueue::Destroy() { @@ -676,8 +674,9 @@ void AqlQueue::AllocRegisteredRingBuffer(uint32_t queue_size_pkts) { ring_buf_alloc_bytes_ = AlignUp( queue_size_pkts * sizeof(core::AqlPacket), 4096); - ring_buf_ = core::Runtime::runtime_singleton_->system_allocator()( - ring_buf_alloc_bytes_, 0x1000, core::MemoryRegion::AllocateExecutable | + ring_buf_ = agent_->system_allocator()( + ring_buf_alloc_bytes_, 0x1000, + core::MemoryRegion::AllocateExecutable | (queue_full_workaround_ ? core::MemoryRegion::AllocateDoubleMap : 0)); assert(ring_buf_ != NULL && "AQL queue memory allocation failure"); @@ -699,7 +698,7 @@ void AqlQueue::FreeRegisteredRingBuffer() { (void*)(uintptr_t(ring_buf_) + (ring_buf_alloc_bytes_ / 2))); #endif } else { - core::Runtime::runtime_singleton_->system_deallocator()(ring_buf_); + agent_->system_deallocator()(ring_buf_); } ring_buf_ = NULL; diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp index 77ea4c7352..2461e05f46 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp @@ -536,15 +536,14 @@ hsa_status_t BlitKernel::Initialize(const core::Agent& agent) { return status; } + const AMD::GpuAgent& gpuAgent = static_cast(agent); kernarg_async_ = reinterpret_cast( - core::Runtime::runtime_singleton_->system_allocator()( - queue_->public_handle()->size * AlignUp(sizeof(KernelArgs), 16), 16, - core::MemoryRegion::AllocateNoFlags)); + gpuAgent.system_allocator()(queue_->public_handle()->size * AlignUp(sizeof(KernelArgs), 16), + 16, core::MemoryRegion::AllocateNoFlags)); kernarg_async_mask_ = queue_->public_handle()->size - 1; // Obtain the number of compute units in the underlying agent. - const AMD::GpuAgent& gpuAgent = static_cast(agent); num_cus_ = gpuAgent.properties().NumFComputeCores / 4; // Assemble shaders to AQL code objects. @@ -577,7 +576,7 @@ hsa_status_t BlitKernel::Destroy(const core::Agent& agent) { } if (kernarg_async_ != NULL) { - core::Runtime::runtime_singleton_->system_deallocator()(kernarg_async_); + gpuAgent.system_deallocator()(kernarg_async_); } if (completion_signal_.handle != 0) { diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index 7dd6900eb0..f35ea8982c 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -161,8 +161,8 @@ hsa_status_t BlitSdma: } // Allocate queue buffer. - queue_start_addr_ = (char*)core::Runtime::runtime_singleton_->system_allocator()( - kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable); + queue_start_addr_ = + (char*)agent_->system_allocator()(kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable); if (queue_start_addr_ == NULL) { return HSA_STATUS_ERROR_OUT_OF_RESOURCES; @@ -206,7 +206,7 @@ hsa_status_t BlitSdma: if (queue_start_addr_ != NULL) { // Release queue buffer. - core::Runtime::runtime_singleton_->system_deallocator()(queue_start_addr_); + agent_->system_deallocator()(queue_start_addr_); } queue_start_addr_ = NULL; diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index b7cad75df0..c0a70837b3 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -178,7 +178,7 @@ GpuAgent::~GpuAgent() { hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size()); } - core::Runtime::runtime_singleton_->system_deallocator()(doorbell_queue_map_); + system_deallocator()(doorbell_queue_map_); if (trap_code_buf_ != NULL) { ReleaseShader(trap_code_buf_, trap_code_buf_size_); @@ -288,8 +288,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar (assemble_target == AssembleTarget::AQL ? sizeof(amd_kernel_code_t) : 0); code_buf_size = AlignUp(header_size + asic_shader->size, 0x1000); - code_buf = core::Runtime::runtime_singleton_->system_allocator()( - code_buf_size, 0x1000, core::MemoryRegion::AllocateExecutable); + code_buf = system_allocator()(code_buf_size, 0x1000, core::MemoryRegion::AllocateExecutable); assert(code_buf != NULL && "Code buffer allocation failed"); memset(code_buf, 0, code_buf_size); @@ -335,7 +334,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar } void GpuAgent::ReleaseShader(void* code_buf, size_t code_buf_size) const { - core::Runtime::runtime_singleton_->system_deallocator()(code_buf); + system_deallocator()(code_buf); } void GpuAgent::InitRegionList() { @@ -687,6 +686,7 @@ void GpuAgent::PreloadBlits() { hsa_status_t GpuAgent::PostToolsInit() { // Defer memory allocation until agents have been discovered. + InitNumaAllocator(); InitScratchPool(); BindTrapHandler(); InitDma(); @@ -1415,8 +1415,7 @@ void GpuAgent::BindTrapHandler() { // The trap handler uses this to retrieve a wave's amd_queue_t*. auto doorbell_queue_map_size = MAX_NUM_DOORBELLS * sizeof(amd_queue_t*); - doorbell_queue_map_ = (amd_queue_t**)core::Runtime::runtime_singleton_->system_allocator()( - doorbell_queue_map_size, 0x1000, 0); + doorbell_queue_map_ = (amd_queue_t**)system_allocator()(doorbell_queue_map_size, 0x1000, 0); assert(doorbell_queue_map_ != NULL && "Doorbell queue map allocation failed"); memset(doorbell_queue_map_, 0, doorbell_queue_map_size); @@ -1570,5 +1569,37 @@ void GpuAgent::Trim() { scratch_cache_.trim(false); } +void GpuAgent::InitNumaAllocator() { + Agent* nearCpu = nullptr; + uint32_t dist = -1u; + for (auto cpu : core::Runtime::runtime_singleton_->cpu_agents()) { + const core::Runtime::LinkInfo link_info = + core::Runtime::runtime_singleton_->GetLinkInfo(node_id(), cpu->node_id()); + if (link_info.info.numa_distance < dist) { + dist = link_info.info.numa_distance; + nearCpu = cpu; + } + } + + for (auto pool : nearCpu->regions()) { + if (pool->kernarg()) { + system_allocator_ = [pool](size_t size, size_t alignment, + MemoryRegion::AllocateFlags alloc_flags) -> void* { + assert(alignment <= 4096); + void* ptr = nullptr; + return (HSA_STATUS_SUCCESS == + core::Runtime::runtime_singleton_->AllocateMemory(pool, size, alloc_flags, &ptr)) + ? ptr + : nullptr; + }; + + system_deallocator_ = [](void* ptr) { core::Runtime::runtime_singleton_->FreeMemory(ptr); }; + + return; + } + } + assert(false && "Nearest NUMA node did not have a kernarg pool."); +} + } // namespace amd } // namespace rocr