From 17313ec99d30c937607234ab55d6bf7307b299cf Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Thu, 20 Jun 2024 19:37:04 +0000 Subject: [PATCH] SWDEV-465602 - Refactor kernel arg pool allocation for graphs - Allocate additional argument space to accomodate for kernel node param updates Change-Id: I2d4ea8bddd716f1191f3cbea807920d0248f8c4e --- hipamd/src/hip_graph_internal.cpp | 112 +++++++++++++++--------------- hipamd/src/hip_graph_internal.hpp | 62 ++++++++++------- 2 files changed, 92 insertions(+), 82 deletions(-) diff --git a/hipamd/src/hip_graph_internal.cpp b/hipamd/src/hip_graph_internal.cpp index 3adda8cf31..fd6663a653 100644 --- a/hipamd/src/hip_graph_internal.cpp +++ b/hipamd/src/hip_graph_internal.cpp @@ -237,6 +237,7 @@ void Graph::GetRunList(std::vector>& parallelLists, } } } + bool Graph::TopologicalOrder(std::vector& TopoOrder) { std::queue q; std::unordered_map inDegree; @@ -263,6 +264,7 @@ bool Graph::TopologicalOrder(std::vector& TopoOrder) { } return false; } + Graph* Graph::clone(std::unordered_map& clonedNodes) const { Graph* newGraph = new Graph(device_, this); for (auto entry : vertices_) { @@ -352,6 +354,9 @@ hipError_t GraphExec::Init() { return status; } +//! Chunk size to add to kern arg pool +constexpr uint32_t kKernArgChunkSize = 128 * Ki; +// ================================================================================================ void GetKernelArgSizeForGraph(std::vector>& parallelLists, size_t& kernArgSizeForGraph) { // GPU packet capture is enabled for kernel nodes. Calculate the kernel @@ -361,7 +366,8 @@ void GetKernelArgSizeForGraph(std::vector>& parallelLists, if (node->GetType() == hipGraphNodeTypeKernel) { kernArgSizeForGraph += reinterpret_cast(node)->GetKerArgSize(); } else if (node->GetType() == hipGraphNodeTypeGraph) { - auto& childParallelLists = reinterpret_cast(node)->GetParallelLists(); + auto& childParallelLists = + reinterpret_cast(node)->GetParallelLists(); if (childParallelLists.size() == 1) { GetKernelArgSizeForGraph(childParallelLists, kernArgSizeForGraph); } @@ -370,8 +376,9 @@ void GetKernelArgSizeForGraph(std::vector>& parallelLists, } } -hipError_t AllocKernelArgForGraph(std::vector& topoOrder, hip::Stream* capture_stream, - hip::GraphExec* graphExec) { +// ================================================================================================ +hipError_t AllocKernelArgForGraphNode(std::vector& topoOrder, + hip::Stream* capture_stream, hip::GraphExec* graphExec) { hipError_t status = hipSuccess; for (auto& node : topoOrder) { if (node->GetType() == hipGraphNodeTypeKernel) { @@ -399,7 +406,8 @@ hipError_t AllocKernelArgForGraph(std::vector& topoOrder, hip::Stream if (childParallelLists.size() == 1) { childNode->SetGraphCaptureStatus(true); status = - AllocKernelArgForGraph(childNode->GetChildGraphNodeOrder(), capture_stream, graphExec); + AllocKernelArgForGraphNode(childNode->GetChildGraphNodeOrder(), + capture_stream, graphExec); if (status != hipSuccess) { return status; } @@ -409,28 +417,44 @@ hipError_t AllocKernelArgForGraph(std::vector& topoOrder, hip::Stream return status; } +// ================================================================================================ +hipError_t GraphExec::AllocGraphKernargPool(size_t pool_size) { + hipError_t status = hipSuccess; + assert(pool_size > 0); + address graph_kernarg_base; + auto device = g_devices[ihipGetDevice()]->devices()[0]; + + if (device->info().largeBar_) { + graph_kernarg_base = + reinterpret_cast
(device->deviceLocalAlloc(pool_size)); + device_kernarg_pool_ = true; + } else { + graph_kernarg_base = reinterpret_cast
( + device->hostAlloc(pool_size, 0, + amd::Device::MemorySegment::kKernArg)); + } + + if (graph_kernarg_base == nullptr) { + return hipErrorMemoryAllocation; + } + kernarg_graph_.push_back(KernelArgPoolGraph(graph_kernarg_base, pool_size)); + return status; +} + +// ================================================================================================ hipError_t GraphExec::CaptureAQLPackets() { hipError_t status = hipSuccess; if (parallelLists_.size() == 1) { size_t kernArgSizeForGraph = 0; GetKernelArgSizeForGraph(parallelLists_, kernArgSizeForGraph); auto device = g_devices[ihipGetDevice()]->devices()[0]; - if (kernArgSizeForGraph != 0) { - if (device->info().largeBar_) { - kernarg_pool_graph_ = - reinterpret_cast
(device->deviceLocalAlloc(kernArgSizeForGraph)); - device_kernarg_pool_ = true; - } else { - kernarg_pool_graph_ = reinterpret_cast
( - device->hostAlloc(kernArgSizeForGraph, 0, amd::Device::MemorySegment::kKernArg)); - } - - if (kernarg_pool_graph_ == nullptr) { - return hipErrorMemoryAllocation; - } - kernarg_pool_size_graph_ = kernArgSizeForGraph; + // Add a larger initial pool to accomodate for any updates to kernel args + status = AllocGraphKernargPool(kernArgSizeForGraph + kKernArgChunkSize); + if (status != hipSuccess) { + return status; } - status = AllocKernelArgForGraph(topoOrder_, capture_stream_, this); + + status = AllocKernelArgForGraphNode(topoOrder_, capture_stream_, this); if (status != hipSuccess) { return status; } @@ -442,8 +466,9 @@ hipError_t GraphExec::CaptureAQLPackets() { *device->info().hdpMemFlushCntl = 1u; auto kSentinel = *reinterpret_cast(device->info().hdpMemFlushCntl); } else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback && - kernarg_pool_size_graph_ != 0) { - address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_; + kernarg_graph_.back().kernarg_pool_addr_ != 0) { + address dev_ptr = kernarg_graph_.back().kernarg_pool_addr_ + + kernarg_graph_.back().kernarg_pool_size_; auto kSentinel = *reinterpret_cast(dev_ptr - 1); _mm_sfence(); *(dev_ptr - 1) = kSentinel; @@ -455,46 +480,23 @@ hipError_t GraphExec::CaptureAQLPackets() { return status; } +// ================================================================================================ hipError_t GraphExec::UpdateAQLPacket(hip::GraphKernelNode* node) { + hipError_t status = hipSuccess; if (parallelLists_.size() == 1) { size_t pool_new_usage = 0; - address result = nullptr; - if (!kernarg_graph_.empty()) { - // 1. Allocate memory for the kernel args - size_t kernArgSizeForNode = 0; - kernArgSizeForNode = node->GetKerArgSize(); + address kernArgOffset = nullptr; + kernArgOffset = allocKernArg(node->GetKerArgSize(), + node->GetKernargSegmentAlignment()); - result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_, - node->GetKernargSegmentAlignment()); - pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back(); - } - if (pool_new_usage != 0 && pool_new_usage <= kernarg_graph_size_) { - kernarg_graph_cur_offset_ = pool_new_usage; - } else { - address kernarg_graph; - auto device = g_devices[ihipGetDevice()]->devices()[0]; - if (device->info().largeBar_) { - kernarg_graph = reinterpret_cast
(device->deviceLocalAlloc(kernarg_graph_size_)); - } else { - kernarg_graph = reinterpret_cast
( - device->hostAlloc(kernarg_graph_size_, 0, amd::Device::MemorySegment::kKernArg)); - } - kernarg_graph_.push_back(kernarg_graph); - kernarg_graph_cur_offset_ = 0; - - // 1. Allocate memory for the kernel args - size_t kernArgSizeForNode = 0; - kernArgSizeForNode = node->GetKerArgSize(); - result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_, - node->GetKernargSegmentAlignment()); - const size_t pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back(); - if (pool_new_usage <= kernarg_graph_size_) { - kernarg_graph_cur_offset_ = pool_new_usage; - } + if (kernArgOffset == nullptr ) { + // Allocate new pool for kernarg and get the offset + status = AllocGraphKernargPool(kKernArgChunkSize); + kernArgOffset = allocKernArg(node->GetKerArgSize(), + node->GetKernargSegmentAlignment()); } - // 2. copy kernel args / create new AQL packet - node->CaptureAndFormPacket(capture_stream_, result); + node->CaptureAndFormPacket(capture_stream_, kernArgOffset); } return hipSuccess; } diff --git a/hipamd/src/hip_graph_internal.hpp b/hipamd/src/hip_graph_internal.hpp index 3255724ad1..77c6c6b2ec 100644 --- a/hipamd/src/hip_graph_internal.hpp +++ b/hipamd/src/hip_graph_internal.hpp @@ -546,8 +546,19 @@ struct Graph { }; struct GraphKernelNode; struct GraphExec : public amd::ReferenceCountedObject { + struct KernelArgPoolGraph { + KernelArgPoolGraph(address base_addr, size_t size) + : kernarg_pool_addr_(base_addr), + kernarg_pool_size_(size), + kernarg_pool_offset_(0) + {} + address kernarg_pool_addr_; //! Base address of the kernel arg pool + size_t kernarg_pool_size_; //! Size of the pool + size_t kernarg_pool_offset_; //! Current offset in the kernel arg alloc + }; + std::vector> parallelLists_; - // Topological order of the graph doesn't include nodes embedded as part of the child graph + //! Topological order of the graph doesn't include nodes embedded as part of the child graph std::vector topoOrder_; std::unordered_map> nodeWaitLists_; struct Graph* clonedGraph_; @@ -560,16 +571,13 @@ struct GraphExec : public amd::ReferenceCountedObject { static amd::Monitor graphExecSetLock_; uint64_t flags_ = 0; bool repeatLaunch_ = false; - // Graph Kernel arg vars - bool device_kernarg_pool_ = false; - address kernarg_pool_graph_ = nullptr; - uint32_t kernarg_pool_size_graph_ = 0; - uint32_t kernarg_pool_cur_graph_offset_ = 0; - std::vector
kernarg_graph_; - uint32_t kernarg_graph_cur_offset_ = 0; - uint32_t kernarg_graph_size_ = 128 * Ki; + + bool device_kernarg_pool_ = false; //! Indicate if kernel pool in device mem + std::vector kernarg_graph_; //! Vector of allocated kernarg pool + uint32_t kernarg_graph_cur_offset_ = 0; //! Current offset in kernarg pool + int instantiateDeviceId_ = -1; - bool hasHiddenHeap_ = false; //!< Kernel has hidden heap(device side allocation) + bool hasHiddenHeap_ = false; //!< Hidden heap indicator for Kernel node public: GraphExec(std::vector& topoOrder, std::vector>& lists, @@ -589,21 +597,17 @@ struct GraphExec : public amd::ReferenceCountedObject { } ~GraphExec() { - // new commands are launched for every launch they are destroyed as and when command is - // terminated after it complete execution for (auto stream : parallel_streams_) { if (stream != nullptr) { hip::Stream::Destroy(stream); } } - // Release the kernel arg memory. - auto device = g_devices[ihipGetDevice()]->devices()[0]; + + //! Release the kernel arg pools if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { - if (kernarg_pool_size_graph_ != 0) { - device->hostFree(kernarg_pool_graph_, kernarg_pool_size_graph_); - for (auto& element : kernarg_graph_) { - device->hostFree(element, kernarg_graph_size_); - } + auto device = g_devices[ihipGetDevice()]->devices()[0]; + for (auto& element : kernarg_graph_) { + device->hostFree(element.kernarg_pool_addr_, element.kernarg_pool_size_); } } amd::ScopedLock lock(graphExecSetLock_); @@ -620,25 +624,29 @@ struct GraphExec : public amd::ReferenceCountedObject { } return clonedNode; } - // returns if graph has nodes that require hidden heap/not + //! Check if kernel node has hidden heap bool HasHiddenHeap() const { return hasHiddenHeap_; } - // Graph has nodes that require hidden heap. + //! Graph has nodes that require hidden heap. void SetHiddenHeap() { hasHiddenHeap_ = true; } address allocKernArg(size_t size, size_t alignment) { assert(alignment != 0); address result = nullptr; - result = amd::alignUp(kernarg_pool_graph_ + kernarg_pool_cur_graph_offset_, alignment); - const size_t pool_new_usage = (result + size) - kernarg_pool_graph_; - if (pool_new_usage <= kernarg_pool_size_graph_) { - kernarg_pool_cur_graph_offset_ = pool_new_usage; + result = amd::alignUp(kernarg_graph_.back().kernarg_pool_addr_ + + kernarg_graph_.back().kernarg_pool_offset_, + alignment); + const size_t pool_new_usage = (result + size) - kernarg_graph_.back().kernarg_pool_addr_; + if (pool_new_usage <= kernarg_graph_.back().kernarg_pool_size_) { + kernarg_graph_.back().kernarg_pool_offset_ = pool_new_usage; + } else { + return nullptr; } return result; } - // check executable graphs validity + //! Check executable graphs validity static bool isGraphExecValid(GraphExec* pGraphExec); - + hipError_t AllocGraphKernargPool(size_t pool_size); std::vector& GetNodes() { return topoOrder_; } hip::Stream* GetAvailableStreams() {