From 3dc533e1e1652c54970e436b2726ce8dc1b5c29a Mon Sep 17 00:00:00 2001 From: Anusha GodavarthySurya Date: Mon, 8 Jan 2024 17:42:55 +0000 Subject: [PATCH] SWDEV-439628 - hipGraphExecKernelNodeSetParams to update graph kernel node params with graph performance optimizations. During hipGraphExecKernelNodeSetParams kernel function can also be updated. Hence size required for kernel parameters differs from what is allocated during graphInstantiation. So, create new 128KB kernel pool and allocate kernel args from the pool. If the pool is full create new 128KB pool. Release kernel pools when graph exec object is destroyed. Change-Id: I9567946d63400c79cbfd4c5439c654c92557ceae [ROCm/clr commit: e9957151f3b9ad5648b6cf46789e542df834df1a] --- projects/clr/hipamd/src/hip_context.cpp | 6 +- projects/clr/hipamd/src/hip_device.cpp | 8 ++- projects/clr/hipamd/src/hip_graph.cpp | 10 +++- .../clr/hipamd/src/hip_graph_internal.cpp | 55 +++++++++++++++++-- .../clr/hipamd/src/hip_graph_internal.hpp | 34 ++++++++---- projects/clr/hipamd/src/hip_internal.hpp | 4 +- 6 files changed, 90 insertions(+), 27 deletions(-) diff --git a/projects/clr/hipamd/src/hip_context.cpp b/projects/clr/hipamd/src/hip_context.cpp index 47569f0669..4b7f5c8098 100644 --- a/projects/clr/hipamd/src/hip_context.cpp +++ b/projects/clr/hipamd/src/hip_context.cpp @@ -90,7 +90,7 @@ void setCurrentDevice(unsigned int index) { hip::Stream* getStream(hipStream_t stream, bool wait) { if (stream == nullptr) { - return getNullStream(); + return getNullStream(wait); } else { hip::Stream* hip_stream = reinterpret_cast(stream); if (wait && !(hip_stream->Flags() & hipStreamNonBlocking)) { @@ -128,9 +128,9 @@ int getDeviceID(amd::Context& ctx) { } // ================================================================================================ -hip::Stream* getNullStream() { +hip::Stream* getNullStream(bool wait ) { Device* device = getCurrentDevice(); - return device ? device->NullStream() : nullptr; + return device ? device->NullStream(wait) : nullptr; } hipError_t hipInit(unsigned int flags) { diff --git a/projects/clr/hipamd/src/hip_device.cpp b/projects/clr/hipamd/src/hip_device.cpp index 2053461afe..bcf6830f12 100644 --- a/projects/clr/hipamd/src/hip_device.cpp +++ b/projects/clr/hipamd/src/hip_device.cpp @@ -30,7 +30,7 @@ namespace hip { // ================================================================================================ -hip::Stream* Device::NullStream() { +hip::Stream* Device::NullStream(bool wait) { if (null_stream_ == nullptr) { null_stream_ = new Stream(this, Stream::Priority::Normal, 0, true); } @@ -38,8 +38,10 @@ hip::Stream* Device::NullStream() { if (null_stream_ == nullptr) { return nullptr; } - // Wait for all active streams before executing commands on the default - iHipWaitActiveStreams(null_stream_); + if (wait == true) { + // Wait for all active streams before executing commands on the default + iHipWaitActiveStreams(null_stream_); + } return null_stream_; } diff --git a/projects/clr/hipamd/src/hip_graph.cpp b/projects/clr/hipamd/src/hip_graph.cpp index 9e35fe0372..89b3190fdc 100644 --- a/projects/clr/hipamd/src/hip_graph.cpp +++ b/projects/clr/hipamd/src/hip_graph.cpp @@ -1552,7 +1552,15 @@ hipError_t hipGraphExecKernelNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNo if (clonedNode == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - HIP_RETURN(reinterpret_cast(clonedNode)->SetParams(pNodeParams)); + hipError_t status = reinterpret_cast(clonedNode)->SetParams(pNodeParams); + if(status != hipSuccess) { + HIP_RETURN(status); + } + if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { + status = reinterpret_cast(hGraphExec) + ->UpdateAQLPacket(reinterpret_cast(clonedNode)); + } + HIP_RETURN(status); } hipError_t hipGraphChildGraphNodeGetGraph(hipGraphNode_t node, hipGraph_t* pGraph) { diff --git a/projects/clr/hipamd/src/hip_graph_internal.cpp b/projects/clr/hipamd/src/hip_graph_internal.cpp index ee70f8b1b9..f3a5be0880 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.cpp +++ b/projects/clr/hipamd/src/hip_graph_internal.cpp @@ -332,6 +332,9 @@ hipError_t GraphExec::CreateStreams(uint32_t num_streams) { } parallel_streams_.push_back(stream); } + // Don't wait for other streams to finish. + // Capture stream is to capture AQL packet. + capture_stream_ = hip::getNullStream(false); return hipSuccess; } @@ -353,13 +356,10 @@ hipError_t GraphExec::CaptureAQLPackets() { hipError_t status = hipSuccess; if (parallelLists_.size() == 1) { size_t kernArgSizeForGraph = 0; - hip::Stream* stream = nullptr; // GPU packet capture is enabled for kernel nodes. Calculate the kernel // arg size required for all graph kernel nodes to allocate for (const auto& list : parallelLists_) { - stream = GetAvailableStreams(); for (auto& node : list) { - node->SetStream(stream, this); if (node->GetType() == hipGraphNodeTypeKernel) { kernArgSizeForGraph += reinterpret_cast(node)->GetKerArgSize(); } @@ -386,7 +386,6 @@ hipError_t GraphExec::CaptureAQLPackets() { for (auto& node : topoOrder_) { if (node->GetType() == hipGraphNodeTypeKernel) { auto kernelNode = reinterpret_cast(node); - status = node->CreateCommand(node->GetQueue()); // From the kernel pool allocate the kern arg size required for the current kernel node. address kernArgOffset = allocKernArg(kernelNode->GetKernargSegmentByteSize(), kernelNode->GetKernargSegmentAlignment()); @@ -394,7 +393,7 @@ hipError_t GraphExec::CaptureAQLPackets() { return hipErrorMemoryAllocation; } // Form GPU packet capture for the kernel node. - kernelNode->CaptureAndFormPacket(kernArgOffset); + kernelNode->CaptureAndFormPacket(capture_stream_, kernArgOffset) ; } } @@ -408,7 +407,7 @@ hipError_t GraphExec::CaptureAQLPackets() { address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_ - sizeof(int); *dev_ptr = host_val; if (device->info().hdpMemFlushCntl == nullptr) { - amd::Command* command = new amd::Marker(*stream, true); + amd::Command* command = new amd::Marker(*capture_stream_, true); if (command != nullptr) { command->enqueue(); command->release(); @@ -426,6 +425,50 @@ hipError_t GraphExec::CaptureAQLPackets() { return status; } +hipError_t GraphExec::UpdateAQLPacket(hip::GraphKernelNode* node) { + if (parallelLists_.size() == 1) { + size_t pool_new_usage = 0; + address result = nullptr; + if (!kernarg_graph_.empty()) { + // 1. Allocate memory for the kernel args + size_t kernArgSizeForNode = 0; + kernArgSizeForNode = node->GetKerArgSize(); + + result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_, + node->GetKernargSegmentAlignment()); + pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back(); + } + if (pool_new_usage != 0 && pool_new_usage <= kernarg_graph_size_) { + kernarg_graph_cur_offset_ = pool_new_usage; + } else { + address kernarg_graph; + auto device = g_devices[ihipGetDevice()]->devices()[0]; + if (device->info().largeBar_) { + kernarg_graph = reinterpret_cast
(device->deviceLocalAlloc(kernarg_graph_size_)); + } else { + kernarg_graph = reinterpret_cast
( + device->hostAlloc(kernarg_graph_size_, 0, amd::Device::MemorySegment::kKernArg)); + } + kernarg_graph_.push_back(kernarg_graph); + kernarg_graph_cur_offset_ = 0; + + // 1. Allocate memory for the kernel args + size_t kernArgSizeForNode = 0; + kernArgSizeForNode = node->GetKerArgSize(); + result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_, + node->GetKernargSegmentAlignment()); + const size_t pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back(); + if (pool_new_usage <= kernarg_graph_size_) { + kernarg_graph_cur_offset_ = pool_new_usage; + } + } + + // 2. copy kernel args / create new AQL packet + node->CaptureAndFormPacket(capture_stream_, result); + } + return hipSuccess; +} + hipError_t FillCommands(std::vector>& parallelLists, std::unordered_map>& nodeWaitLists, std::vector& topoOrder, Graph* clonedGraph, diff --git a/projects/clr/hipamd/src/hip_graph_internal.hpp b/projects/clr/hipamd/src/hip_graph_internal.hpp index 7f6543cb91..2229b75d8c 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.hpp +++ b/projects/clr/hipamd/src/hip_graph_internal.hpp @@ -543,7 +543,7 @@ struct Graph { graphInstantiated_ = graphInstantiate; } }; - +struct GraphKernelNode; struct GraphExec { std::vector> parallelLists_; // Topological order of the graph doesn't include nodes embedded as part of the child graph @@ -551,6 +551,7 @@ struct GraphExec { std::unordered_map> nodeWaitLists_; struct Graph* clonedGraph_; std::vector parallel_streams_; + hip::Stream* capture_stream_; uint currentQueueIndex_; std::unordered_map clonedNodes_; amd::Command* lastEnqueuedCommand_; @@ -563,6 +564,10 @@ struct GraphExec { address kernarg_pool_graph_ = nullptr; uint32_t kernarg_pool_size_graph_ = 0; uint32_t kernarg_pool_cur_graph_offset_ = 0; + std::vector
kernarg_graph_; + uint32_t kernarg_graph_cur_offset_ = 0; + uint32_t kernarg_graph_size_ = 128 * Ki; + public: GraphExec(std::vector& topoOrder, std::vector>& lists, std::unordered_map>& nodeWaitLists, struct Graph*& clonedGraph, @@ -591,6 +596,9 @@ struct GraphExec { auto device = g_devices[ihipGetDevice()]->devices()[0]; if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { device->hostFree(kernarg_pool_graph_, kernarg_pool_size_graph_); + for (auto& element : kernarg_graph_) { + device->hostFree(element, kernarg_graph_size_); + } } amd::ScopedLock lock(graphExecSetLock_); graphExecSet_.erase(this); @@ -636,6 +644,7 @@ struct GraphExec { hipError_t Run(hipStream_t stream); // Capture GPU Packets from graph commands hipError_t CaptureAQLPackets(); + hipError_t UpdateAQLPacket(hip::GraphKernelNode* node); }; struct ChildGraphNode : public GraphNode { @@ -793,19 +802,20 @@ class GraphKernelNode : public GraphNode { out << "];"; } - void CaptureAndFormPacket(address kernArgOffset) { - for (auto& command : commands_) { - reinterpret_cast(command)->setCapturingState( - true, GetAqlPacket(), kernArgOffset); + void CaptureAndFormPacket(hip::Stream* capture_stream, address kernArgOffset) { + hipError_t status = CreateCommand(capture_stream); + for (auto& command : commands_) { + reinterpret_cast(command)->setCapturingState( + true, GetAqlPacket(), kernArgOffset); - // Enqueue command to capture GPU Packet. The packet is not submitted to the device. - // The packet is stored in gpuPacket_ and submitted during graph launch. - command->submit(*(command->queue())->vdev()); - // Need to ensure if the command is NDRangeKernelCommand if we capture non kernel nodes - SetKernelName(reinterpret_cast(command)->kernel().name()); - command->release(); - } + // Enqueue command to capture GPU Packet. The packet is not submitted to the device. + // The packet is stored in gpuPacket_ and submitted during graph launch. + command->submit(*(command->queue())->vdev()); + // Need to ensure if the command is NDRangeKernelCommand if we capture non kernel nodes + SetKernelName(reinterpret_cast(command)->kernel().name()); + command->release(); } + } std::string GetLabel(hipGraphDebugDotFlags flag) { hipFunction_t func = getFunc(kernelParams_, ihipGetDevice()); diff --git a/projects/clr/hipamd/src/hip_internal.hpp b/projects/clr/hipamd/src/hip_internal.hpp index 88b12d8451..d3d38178fa 100644 --- a/projects/clr/hipamd/src/hip_internal.hpp +++ b/projects/clr/hipamd/src/hip_internal.hpp @@ -487,7 +487,7 @@ public: void setFlags(unsigned int flags) { flags_ = flags; } void Reset(); - hip::Stream* NullStream(); + hip::Stream* NullStream(bool wait = true); Stream* GetNullStream() const {return null_stream_;}; void SetActiveStatus() { @@ -572,7 +572,7 @@ public: /// Get default stream associated with the ROCclr context extern hip::Stream* getNullStream(amd::Context&); /// Get default stream of the thread - extern hip::Stream* getNullStream(); + extern hip::Stream* getNullStream(bool wait = true); /// Get device ID associated with the ROCclr context int getDeviceID(amd::Context& ctx); /// Check if stream is valid