diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index c1b26495c4..8d1544987c 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -328,6 +328,13 @@ class GpuAgent : public GpuAgentInt { void Trim() override; + const std::function& + system_allocator() const { + return system_allocator_; + } + + const std::function& system_deallocator() const { return system_deallocator_; } + protected: static const uint32_t minAqlSize_ = 0x1000; // 4KB min static const uint32_t maxAqlSize_ = 0x20000; // 8MB max @@ -492,6 +499,9 @@ class GpuAgent : public GpuAgentInt { // @brief Setup GWS accessing queue. void InitGWS(); + // @brief Setup NUMA aware system memory allocator. + void InitNumaAllocator(); + // @brief Register signal for notification when scratch may become available. // @p signal is notified by OR'ing with @p value. bool AddScratchNotifier(hsa_signal_t signal, hsa_signal_value_t value) { @@ -532,6 +542,12 @@ class GpuAgent : public GpuAgentInt { ScratchCache scratch_cache_; + // System memory allocator in the nearest NUMA node. + std::function + system_allocator_; + + std::function system_deallocator_; + DISALLOW_COPY_AND_ASSIGN(GpuAgent); }; diff --git a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index 48c7d23d6a..424ec07c86 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -305,14 +305,12 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr } // Allocate IB for icache flushes. - pm4_ib_buf_ = core::Runtime::runtime_singleton_->system_allocator()( - pm4_ib_size_b_, 0x1000, core::MemoryRegion::AllocateExecutable); + pm4_ib_buf_ = + agent_->system_allocator()(pm4_ib_size_b_, 0x1000, core::MemoryRegion::AllocateExecutable); if (pm4_ib_buf_ == nullptr) throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES, "PM4 IB allocation failed.\n"); - MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() { - core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_); - }); + MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() { agent_->system_deallocator()(pm4_ib_buf_); }); // Set initial CU mask SetCUMasking(0, nullptr); @@ -356,7 +354,7 @@ AqlQueue::~AqlQueue() { queue_event_ = nullptr; } } - core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_); + agent_->system_deallocator()(pm4_ib_buf_); } void AqlQueue::Destroy() { @@ -676,8 +674,9 @@ void AqlQueue::AllocRegisteredRingBuffer(uint32_t queue_size_pkts) { ring_buf_alloc_bytes_ = AlignUp( queue_size_pkts * sizeof(core::AqlPacket), 4096); - ring_buf_ = core::Runtime::runtime_singleton_->system_allocator()( - ring_buf_alloc_bytes_, 0x1000, core::MemoryRegion::AllocateExecutable | + ring_buf_ = agent_->system_allocator()( + ring_buf_alloc_bytes_, 0x1000, + core::MemoryRegion::AllocateExecutable | (queue_full_workaround_ ? core::MemoryRegion::AllocateDoubleMap : 0)); assert(ring_buf_ != NULL && "AQL queue memory allocation failure"); @@ -699,7 +698,7 @@ void AqlQueue::FreeRegisteredRingBuffer() { (void*)(uintptr_t(ring_buf_) + (ring_buf_alloc_bytes_ / 2))); #endif } else { - core::Runtime::runtime_singleton_->system_deallocator()(ring_buf_); + agent_->system_deallocator()(ring_buf_); } ring_buf_ = NULL; diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp index 77ea4c7352..2461e05f46 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp @@ -536,15 +536,14 @@ hsa_status_t BlitKernel::Initialize(const core::Agent& agent) { return status; } + const AMD::GpuAgent& gpuAgent = static_cast(agent); kernarg_async_ = reinterpret_cast( - core::Runtime::runtime_singleton_->system_allocator()( - queue_->public_handle()->size * AlignUp(sizeof(KernelArgs), 16), 16, - core::MemoryRegion::AllocateNoFlags)); + gpuAgent.system_allocator()(queue_->public_handle()->size * AlignUp(sizeof(KernelArgs), 16), + 16, core::MemoryRegion::AllocateNoFlags)); kernarg_async_mask_ = queue_->public_handle()->size - 1; // Obtain the number of compute units in the underlying agent. - const AMD::GpuAgent& gpuAgent = static_cast(agent); num_cus_ = gpuAgent.properties().NumFComputeCores / 4; // Assemble shaders to AQL code objects. @@ -577,7 +576,7 @@ hsa_status_t BlitKernel::Destroy(const core::Agent& agent) { } if (kernarg_async_ != NULL) { - core::Runtime::runtime_singleton_->system_deallocator()(kernarg_async_); + gpuAgent.system_deallocator()(kernarg_async_); } if (completion_signal_.handle != 0) { diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index 7dd6900eb0..f35ea8982c 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -161,8 +161,8 @@ hsa_status_t BlitSdma: } // Allocate queue buffer. - queue_start_addr_ = (char*)core::Runtime::runtime_singleton_->system_allocator()( - kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable); + queue_start_addr_ = + (char*)agent_->system_allocator()(kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable); if (queue_start_addr_ == NULL) { return HSA_STATUS_ERROR_OUT_OF_RESOURCES; @@ -206,7 +206,7 @@ hsa_status_t BlitSdma: if (queue_start_addr_ != NULL) { // Release queue buffer. - core::Runtime::runtime_singleton_->system_deallocator()(queue_start_addr_); + agent_->system_deallocator()(queue_start_addr_); } queue_start_addr_ = NULL; diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index b7cad75df0..c0a70837b3 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -178,7 +178,7 @@ GpuAgent::~GpuAgent() { hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size()); } - core::Runtime::runtime_singleton_->system_deallocator()(doorbell_queue_map_); + system_deallocator()(doorbell_queue_map_); if (trap_code_buf_ != NULL) { ReleaseShader(trap_code_buf_, trap_code_buf_size_); @@ -288,8 +288,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar (assemble_target == AssembleTarget::AQL ? sizeof(amd_kernel_code_t) : 0); code_buf_size = AlignUp(header_size + asic_shader->size, 0x1000); - code_buf = core::Runtime::runtime_singleton_->system_allocator()( - code_buf_size, 0x1000, core::MemoryRegion::AllocateExecutable); + code_buf = system_allocator()(code_buf_size, 0x1000, core::MemoryRegion::AllocateExecutable); assert(code_buf != NULL && "Code buffer allocation failed"); memset(code_buf, 0, code_buf_size); @@ -335,7 +334,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar } void GpuAgent::ReleaseShader(void* code_buf, size_t code_buf_size) const { - core::Runtime::runtime_singleton_->system_deallocator()(code_buf); + system_deallocator()(code_buf); } void GpuAgent::InitRegionList() { @@ -687,6 +686,7 @@ void GpuAgent::PreloadBlits() { hsa_status_t GpuAgent::PostToolsInit() { // Defer memory allocation until agents have been discovered. + InitNumaAllocator(); InitScratchPool(); BindTrapHandler(); InitDma(); @@ -1415,8 +1415,7 @@ void GpuAgent::BindTrapHandler() { // The trap handler uses this to retrieve a wave's amd_queue_t*. auto doorbell_queue_map_size = MAX_NUM_DOORBELLS * sizeof(amd_queue_t*); - doorbell_queue_map_ = (amd_queue_t**)core::Runtime::runtime_singleton_->system_allocator()( - doorbell_queue_map_size, 0x1000, 0); + doorbell_queue_map_ = (amd_queue_t**)system_allocator()(doorbell_queue_map_size, 0x1000, 0); assert(doorbell_queue_map_ != NULL && "Doorbell queue map allocation failed"); memset(doorbell_queue_map_, 0, doorbell_queue_map_size); @@ -1570,5 +1569,37 @@ void GpuAgent::Trim() { scratch_cache_.trim(false); } +void GpuAgent::InitNumaAllocator() { + Agent* nearCpu = nullptr; + uint32_t dist = -1u; + for (auto cpu : core::Runtime::runtime_singleton_->cpu_agents()) { + const core::Runtime::LinkInfo link_info = + core::Runtime::runtime_singleton_->GetLinkInfo(node_id(), cpu->node_id()); + if (link_info.info.numa_distance < dist) { + dist = link_info.info.numa_distance; + nearCpu = cpu; + } + } + + for (auto pool : nearCpu->regions()) { + if (pool->kernarg()) { + system_allocator_ = [pool](size_t size, size_t alignment, + MemoryRegion::AllocateFlags alloc_flags) -> void* { + assert(alignment <= 4096); + void* ptr = nullptr; + return (HSA_STATUS_SUCCESS == + core::Runtime::runtime_singleton_->AllocateMemory(pool, size, alloc_flags, &ptr)) + ? ptr + : nullptr; + }; + + system_deallocator_ = [](void* ptr) { core::Runtime::runtime_singleton_->FreeMemory(ptr); }; + + return; + } + } + assert(false && "Nearest NUMA node did not have a kernarg pool."); +} + } // namespace amd } // namespace rocr