diff --git a/hipamd/src/hip_graph.cpp b/hipamd/src/hip_graph.cpp index 3139ae7644..eb31803937 100644 --- a/hipamd/src/hip_graph.cpp +++ b/hipamd/src/hip_graph.cpp @@ -1320,6 +1320,9 @@ hipError_t ihipGraphInstantiate(hip::GraphExec** pGraphExec, hip::Graph* graph, LogPrintfInfo("[hipGraph] graph dump:%s", filename.c_str()); } } + if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { + (*pGraphExec)->SetKernelArgManager(new hip::GraphKernelArgManager()); + } return (*pGraphExec)->Init(); } else { return hipErrorOutOfMemory; diff --git a/hipamd/src/hip_graph_internal.cpp b/hipamd/src/hip_graph_internal.cpp index dce076a7ff..bf1ee627f9 100644 --- a/hipamd/src/hip_graph_internal.cpp +++ b/hipamd/src/hip_graph_internal.cpp @@ -392,8 +392,8 @@ hipError_t AllocKernelArgForGraphNode(std::vector& topoOrder, // From the kernel pool allocate the kern arg size required for the current kernel node. address kernArgOffset = nullptr; if (kernelNode->GetKernargSegmentByteSize()) { - kernArgOffset = graphExec->allocKernArg(kernelNode->GetKernargSegmentByteSize(), - kernelNode->GetKernargSegmentAlignment()); + kernArgOffset = graphExec->kernArgManager_->AllocKernArg( + kernelNode->GetKernargSegmentByteSize(), kernelNode->GetKernargSegmentAlignment()); if (kernArgOffset == nullptr) { return hipErrorMemoryAllocation; } @@ -417,30 +417,6 @@ hipError_t AllocKernelArgForGraphNode(std::vector& topoOrder, return status; } -// ================================================================================================ -hipError_t GraphExec::AllocGraphKernargPool(size_t pool_size) { - hipError_t status = hipSuccess; - assert(pool_size > 0); - address graph_kernarg_base; - auto device = g_devices[ihipGetDevice()]->devices()[0]; - - if (device->info().largeBar_) { - graph_kernarg_base = - reinterpret_cast
(device->deviceLocalAlloc(pool_size)); - device_kernarg_pool_ = true; - } else { - graph_kernarg_base = reinterpret_cast
( - device->hostAlloc(pool_size, 0, - amd::Device::MemorySegment::kKernArg)); - } - - if (graph_kernarg_base == nullptr) { - return hipErrorMemoryAllocation; - } - kernarg_graph_.push_back(KernelArgPoolGraph(graph_kernarg_base, pool_size)); - return status; -} - // ================================================================================================ hipError_t GraphExec::CaptureAQLPackets() { hipError_t status = hipSuccess; @@ -449,33 +425,16 @@ hipError_t GraphExec::CaptureAQLPackets() { GetKernelArgSizeForGraph(parallelLists_, kernArgSizeForGraph); auto device = g_devices[ihipGetDevice()]->devices()[0]; // Add a larger initial pool to accomodate for any updates to kernel args - status = AllocGraphKernargPool(kernArgSizeForGraph + kKernArgChunkSize); - if (status != hipSuccess) { - return status; + bool bStatus = kernArgManager_->AllocGraphKernargPool(kernArgSizeForGraph + kKernArgChunkSize); + if (bStatus != true) { + return hipErrorMemoryAllocation; } status = AllocKernelArgForGraphNode(topoOrder_, capture_stream_, this); if (status != hipSuccess) { return status; } - - if (device_kernarg_pool_) { - auto kernArgImpl = device->settings().kernel_arg_impl_; - - if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) { - *device->info().hdpMemFlushCntl = 1u; - auto kSentinel = *reinterpret_cast(device->info().hdpMemFlushCntl); - } else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback && - kernarg_graph_.back().kernarg_pool_addr_ != 0) { - address dev_ptr = kernarg_graph_.back().kernarg_pool_addr_ + - kernarg_graph_.back().kernarg_pool_size_; - auto kSentinel = *reinterpret_cast(dev_ptr - 1); - _mm_sfence(); - *(dev_ptr - 1) = kSentinel; - _mm_mfence(); - kSentinel = *reinterpret_cast(dev_ptr - 1); - } - } + kernArgManager_->ReadBackOrFlush(); } return status; } @@ -486,16 +445,11 @@ hipError_t GraphExec::UpdateAQLPacket(hip::GraphKernelNode* node) { if (parallelLists_.size() == 1) { size_t pool_new_usage = 0; address kernArgOffset = nullptr; - kernArgOffset = allocKernArg(node->GetKerArgSize(), - node->GetKernargSegmentAlignment()); - - if (kernArgOffset == nullptr ) { - // Allocate new pool for kernarg and get the offset - status = AllocGraphKernargPool(kKernArgChunkSize); - kernArgOffset = allocKernArg(node->GetKerArgSize(), - node->GetKernargSegmentAlignment()); + kernArgOffset = + kernArgManager_->AllocKernArg(node->GetKerArgSize(), node->GetKernargSegmentAlignment()); + if (kernArgOffset == nullptr) { + return hipErrorMemoryAllocation; } - node->CaptureAndFormPacket(capture_stream_, kernArgOffset); } return hipSuccess; @@ -689,6 +643,7 @@ hipError_t GraphExec::Run(hipStream_t stream) { ResetQueueIndex(); return status; } + void ReleaseGraphExec(int deviceId) { // Release all graph exec objects destroyed by user. amd::ScopedLock lock(GraphExecStatusLock_); @@ -705,6 +660,7 @@ void ReleaseGraphExec(int deviceId) { } } } + void ReleaseGraphExec(hip::Stream* stream) { amd::ScopedLock lock(GraphExecStatusLock_); for (auto itr = GraphExecStatus_.begin(); itr != GraphExecStatus_.end();) { @@ -721,4 +677,69 @@ void ReleaseGraphExec(hip::Stream* stream) { } } } + +// ================================================================================================ +bool GraphKernelArgManager::AllocGraphKernargPool(size_t pool_size) { + bool bStatus = true; + assert(pool_size > 0); + address graph_kernarg_base; + auto device = g_devices[ihipGetDevice()]->devices()[0]; + + if (device->info().largeBar_) { + graph_kernarg_base = reinterpret_cast
(device->deviceLocalAlloc(pool_size)); + device_kernarg_pool_ = true; + } else { + graph_kernarg_base = reinterpret_cast
( + device->hostAlloc(pool_size, 0, amd::Device::MemorySegment::kKernArg)); + } + + if (graph_kernarg_base == nullptr) { + return false; + } + kernarg_graph_.push_back(KernelArgPoolGraph(graph_kernarg_base, pool_size)); + return true; +} + +address GraphKernelArgManager::AllocKernArg(size_t size, size_t alignment) { + assert(alignment != 0); + address result = nullptr; + result = amd::alignUp( + kernarg_graph_.back().kernarg_pool_addr_ + kernarg_graph_.back().kernarg_pool_offset_, + alignment); + const size_t pool_new_usage = (result + size) - kernarg_graph_.back().kernarg_pool_addr_; + if (pool_new_usage <= kernarg_graph_.back().kernarg_pool_size_) { + kernarg_graph_.back().kernarg_pool_offset_ = pool_new_usage; + } else { + // If current chunck is full allocate new chunck with same size as current + bool bStatus = AllocGraphKernargPool(kernarg_graph_.back().kernarg_pool_size_); + if (bStatus == false) { + return nullptr; + } else { + // Allocte kernel arg memory from new chunck + return AllocKernArg(size, alignment); + } + } + return result; +} + +void GraphKernelArgManager::ReadBackOrFlush() { + if (device_kernarg_pool_) { + auto device = g_devices[ihipGetDevice()]->devices()[0]; + auto kernArgImpl = device->settings().kernel_arg_impl_; + + if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) { + *device->info().hdpMemFlushCntl = 1u; + auto kSentinel = *reinterpret_cast(device->info().hdpMemFlushCntl); + } else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback && + kernarg_graph_.back().kernarg_pool_addr_ != 0) { + address dev_ptr = + kernarg_graph_.back().kernarg_pool_addr_ + kernarg_graph_.back().kernarg_pool_size_; + auto kSentinel = *reinterpret_cast(dev_ptr - 1); + _mm_sfence(); + *(dev_ptr - 1) = kSentinel; + _mm_mfence(); + kSentinel = *reinterpret_cast(dev_ptr - 1); + } + } +} } // namespace hip diff --git a/hipamd/src/hip_graph_internal.hpp b/hipamd/src/hip_graph_internal.hpp index bd56360e9e..20499a202b 100644 --- a/hipamd/src/hip_graph_internal.hpp +++ b/hipamd/src/hip_graph_internal.hpp @@ -552,18 +552,42 @@ struct Graph { } }; struct GraphKernelNode; -struct GraphExec : public amd::ReferenceCountedObject { +struct GraphKernelArgManager : public amd::ReferenceCountedObject { + public: + GraphKernelArgManager() : ReferenceCountedObject() {} + ~GraphKernelArgManager() { + //! Release the kernel arg pools + auto device = g_devices[ihipGetDevice()]->devices()[0]; + for (auto& element : kernarg_graph_) { + device->hostFree(element.kernarg_pool_addr_, element.kernarg_pool_size_); + } + kernarg_graph_.clear(); + } + + // Allocate kernel arg pool for the given size. + bool AllocGraphKernargPool(size_t pool_size); + + // Allocate kernel args from current chunck for given size and alignment. + // If kernel arg pool is full allocate new chunck and alloc kern args from new pool. + address AllocKernArg(size_t size, size_t alignment); + + // Do HDP flush/When HDP flush register is invalid fallback to Readback + void ReadBackOrFlush(); + + private: struct KernelArgPoolGraph { KernelArgPoolGraph(address base_addr, size_t size) - : kernarg_pool_addr_(base_addr), - kernarg_pool_size_(size), - kernarg_pool_offset_(0) - {} + : kernarg_pool_addr_(base_addr), kernarg_pool_size_(size), kernarg_pool_offset_(0) {} address kernarg_pool_addr_; //! Base address of the kernel arg pool size_t kernarg_pool_size_; //! Size of the pool size_t kernarg_pool_offset_; //! Current offset in the kernel arg alloc }; + bool device_kernarg_pool_ = false; //! Indicate if kernel pool in device mem + std::vector kernarg_graph_; //! Vector of allocated kernarg pool + using KernelArgImpl = device::Settings::KernelArgImpl; +}; +struct GraphExec : public amd::ReferenceCountedObject { std::vector> parallelLists_; //! Topological order of the graph doesn't include nodes embedded as part of the child graph std::vector topoOrder_; @@ -578,11 +602,7 @@ struct GraphExec : public amd::ReferenceCountedObject { static amd::Monitor graphExecSetLock_; uint64_t flags_ = 0; bool repeatLaunch_ = false; - - bool device_kernarg_pool_ = false; //! Indicate if kernel pool in device mem - std::vector kernarg_graph_; //! Vector of allocated kernarg pool - uint32_t kernarg_graph_cur_offset_ = 0; //! Current offset in kernarg pool - + GraphKernelArgManager* kernArgManager_ = nullptr; //!< Kernel Arg manager for graph. int instantiateDeviceId_ = -1; bool hasHiddenHeap_ = false; //!< Hidden heap indicator for Kernel node @@ -609,17 +629,12 @@ struct GraphExec : public amd::ReferenceCountedObject { hip::Stream::Destroy(stream); } } - - //! Release the kernel arg pools - if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { - auto device = g_devices[ihipGetDevice()]->devices()[0]; - for (auto& element : kernarg_graph_) { - device->hostFree(element.kernarg_pool_addr_, element.kernarg_pool_size_); - } - } amd::ScopedLock lock(graphExecSetLock_); graphExecSet_.erase(this); delete clonedGraph_; + if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { + kernArgManager_->release(); + } } Node GetClonedNode(Node node) { @@ -636,24 +651,8 @@ struct GraphExec : public amd::ReferenceCountedObject { //! Graph has nodes that require hidden heap. void SetHiddenHeap() { hasHiddenHeap_ = true; } - address allocKernArg(size_t size, size_t alignment) { - assert(alignment != 0); - address result = nullptr; - result = amd::alignUp(kernarg_graph_.back().kernarg_pool_addr_ + - kernarg_graph_.back().kernarg_pool_offset_, - alignment); - const size_t pool_new_usage = (result + size) - kernarg_graph_.back().kernarg_pool_addr_; - if (pool_new_usage <= kernarg_graph_.back().kernarg_pool_size_) { - kernarg_graph_.back().kernarg_pool_offset_ = pool_new_usage; - } else { - return nullptr; - } - return result; - } - //! Check executable graphs validity static bool isGraphExecValid(GraphExec* pGraphExec); - hipError_t AllocGraphKernargPool(size_t pool_size); std::vector& GetNodes() { return topoOrder_; } hip::Stream* GetAvailableStreams() { @@ -671,7 +670,16 @@ struct GraphExec : public amd::ReferenceCountedObject { // Capture GPU Packets from graph commands hipError_t CaptureAQLPackets(); hipError_t UpdateAQLPacket(hip::GraphKernelNode* node); - using KernelArgImpl = device::Settings::KernelArgImpl; + // Kenrel arg manger is for the entire graph. + // Child graph also shares the same kernel arg manager object. some apps have 100's of + // child graph nodes and each child graph has only one node. + void SetKernelArgManager(GraphKernelArgManager* kernArgManager) { + kernArgManager_ = kernArgManager; + kernArgManager_->retain(); + } + GraphKernelArgManager* GetKernelArgManager() { + return kernArgManager_; + } }; struct ChildGraphNode : public GraphNode {