diff --git a/projects/clr/hipamd/src/hip_context.cpp b/projects/clr/hipamd/src/hip_context.cpp index 47569f0669..4b7f5c8098 100644 --- a/projects/clr/hipamd/src/hip_context.cpp +++ b/projects/clr/hipamd/src/hip_context.cpp @@ -90,7 +90,7 @@ void setCurrentDevice(unsigned int index) { hip::Stream* getStream(hipStream_t stream, bool wait) { if (stream == nullptr) { - return getNullStream(); + return getNullStream(wait); } else { hip::Stream* hip_stream = reinterpret_cast(stream); if (wait && !(hip_stream->Flags() & hipStreamNonBlocking)) { @@ -128,9 +128,9 @@ int getDeviceID(amd::Context& ctx) { } // ================================================================================================ -hip::Stream* getNullStream() { +hip::Stream* getNullStream(bool wait ) { Device* device = getCurrentDevice(); - return device ? device->NullStream() : nullptr; + return device ? device->NullStream(wait) : nullptr; } hipError_t hipInit(unsigned int flags) { diff --git a/projects/clr/hipamd/src/hip_device.cpp b/projects/clr/hipamd/src/hip_device.cpp index 2053461afe..bcf6830f12 100644 --- a/projects/clr/hipamd/src/hip_device.cpp +++ b/projects/clr/hipamd/src/hip_device.cpp @@ -30,7 +30,7 @@ namespace hip { // ================================================================================================ -hip::Stream* Device::NullStream() { +hip::Stream* Device::NullStream(bool wait) { if (null_stream_ == nullptr) { null_stream_ = new Stream(this, Stream::Priority::Normal, 0, true); } @@ -38,8 +38,10 @@ hip::Stream* Device::NullStream() { if (null_stream_ == nullptr) { return nullptr; } - // Wait for all active streams before executing commands on the default - iHipWaitActiveStreams(null_stream_); + if (wait == true) { + // Wait for all active streams before executing commands on the default + iHipWaitActiveStreams(null_stream_); + } return null_stream_; } diff --git a/projects/clr/hipamd/src/hip_graph.cpp b/projects/clr/hipamd/src/hip_graph.cpp index 9e35fe0372..89b3190fdc 100644 --- a/projects/clr/hipamd/src/hip_graph.cpp +++ b/projects/clr/hipamd/src/hip_graph.cpp @@ -1552,7 +1552,15 @@ hipError_t hipGraphExecKernelNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNo if (clonedNode == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - HIP_RETURN(reinterpret_cast(clonedNode)->SetParams(pNodeParams)); + hipError_t status = reinterpret_cast(clonedNode)->SetParams(pNodeParams); + if(status != hipSuccess) { + HIP_RETURN(status); + } + if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { + status = reinterpret_cast(hGraphExec) + ->UpdateAQLPacket(reinterpret_cast(clonedNode)); + } + HIP_RETURN(status); } hipError_t hipGraphChildGraphNodeGetGraph(hipGraphNode_t node, hipGraph_t* pGraph) { diff --git a/projects/clr/hipamd/src/hip_graph_internal.cpp b/projects/clr/hipamd/src/hip_graph_internal.cpp index ee70f8b1b9..f3a5be0880 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.cpp +++ b/projects/clr/hipamd/src/hip_graph_internal.cpp @@ -332,6 +332,9 @@ hipError_t GraphExec::CreateStreams(uint32_t num_streams) { } parallel_streams_.push_back(stream); } + // Don't wait for other streams to finish. + // Capture stream is to capture AQL packet. + capture_stream_ = hip::getNullStream(false); return hipSuccess; } @@ -353,13 +356,10 @@ hipError_t GraphExec::CaptureAQLPackets() { hipError_t status = hipSuccess; if (parallelLists_.size() == 1) { size_t kernArgSizeForGraph = 0; - hip::Stream* stream = nullptr; // GPU packet capture is enabled for kernel nodes. Calculate the kernel // arg size required for all graph kernel nodes to allocate for (const auto& list : parallelLists_) { - stream = GetAvailableStreams(); for (auto& node : list) { - node->SetStream(stream, this); if (node->GetType() == hipGraphNodeTypeKernel) { kernArgSizeForGraph += reinterpret_cast(node)->GetKerArgSize(); } @@ -386,7 +386,6 @@ hipError_t GraphExec::CaptureAQLPackets() { for (auto& node : topoOrder_) { if (node->GetType() == hipGraphNodeTypeKernel) { auto kernelNode = reinterpret_cast(node); - status = node->CreateCommand(node->GetQueue()); // From the kernel pool allocate the kern arg size required for the current kernel node. address kernArgOffset = allocKernArg(kernelNode->GetKernargSegmentByteSize(), kernelNode->GetKernargSegmentAlignment()); @@ -394,7 +393,7 @@ hipError_t GraphExec::CaptureAQLPackets() { return hipErrorMemoryAllocation; } // Form GPU packet capture for the kernel node. - kernelNode->CaptureAndFormPacket(kernArgOffset); + kernelNode->CaptureAndFormPacket(capture_stream_, kernArgOffset) ; } } @@ -408,7 +407,7 @@ hipError_t GraphExec::CaptureAQLPackets() { address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_ - sizeof(int); *dev_ptr = host_val; if (device->info().hdpMemFlushCntl == nullptr) { - amd::Command* command = new amd::Marker(*stream, true); + amd::Command* command = new amd::Marker(*capture_stream_, true); if (command != nullptr) { command->enqueue(); command->release(); @@ -426,6 +425,50 @@ hipError_t GraphExec::CaptureAQLPackets() { return status; } +hipError_t GraphExec::UpdateAQLPacket(hip::GraphKernelNode* node) { + if (parallelLists_.size() == 1) { + size_t pool_new_usage = 0; + address result = nullptr; + if (!kernarg_graph_.empty()) { + // 1. Allocate memory for the kernel args + size_t kernArgSizeForNode = 0; + kernArgSizeForNode = node->GetKerArgSize(); + + result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_, + node->GetKernargSegmentAlignment()); + pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back(); + } + if (pool_new_usage != 0 && pool_new_usage <= kernarg_graph_size_) { + kernarg_graph_cur_offset_ = pool_new_usage; + } else { + address kernarg_graph; + auto device = g_devices[ihipGetDevice()]->devices()[0]; + if (device->info().largeBar_) { + kernarg_graph = reinterpret_cast
(device->deviceLocalAlloc(kernarg_graph_size_)); + } else { + kernarg_graph = reinterpret_cast
( + device->hostAlloc(kernarg_graph_size_, 0, amd::Device::MemorySegment::kKernArg)); + } + kernarg_graph_.push_back(kernarg_graph); + kernarg_graph_cur_offset_ = 0; + + // 1. Allocate memory for the kernel args + size_t kernArgSizeForNode = 0; + kernArgSizeForNode = node->GetKerArgSize(); + result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_, + node->GetKernargSegmentAlignment()); + const size_t pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back(); + if (pool_new_usage <= kernarg_graph_size_) { + kernarg_graph_cur_offset_ = pool_new_usage; + } + } + + // 2. copy kernel args / create new AQL packet + node->CaptureAndFormPacket(capture_stream_, result); + } + return hipSuccess; +} + hipError_t FillCommands(std::vector>& parallelLists, std::unordered_map>& nodeWaitLists, std::vector& topoOrder, Graph* clonedGraph, diff --git a/projects/clr/hipamd/src/hip_graph_internal.hpp b/projects/clr/hipamd/src/hip_graph_internal.hpp index 7f6543cb91..2229b75d8c 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.hpp +++ b/projects/clr/hipamd/src/hip_graph_internal.hpp @@ -543,7 +543,7 @@ struct Graph { graphInstantiated_ = graphInstantiate; } }; - +struct GraphKernelNode; struct GraphExec { std::vector> parallelLists_; // Topological order of the graph doesn't include nodes embedded as part of the child graph @@ -551,6 +551,7 @@ struct GraphExec { std::unordered_map> nodeWaitLists_; struct Graph* clonedGraph_; std::vector parallel_streams_; + hip::Stream* capture_stream_; uint currentQueueIndex_; std::unordered_map clonedNodes_; amd::Command* lastEnqueuedCommand_; @@ -563,6 +564,10 @@ struct GraphExec { address kernarg_pool_graph_ = nullptr; uint32_t kernarg_pool_size_graph_ = 0; uint32_t kernarg_pool_cur_graph_offset_ = 0; + std::vector
kernarg_graph_; + uint32_t kernarg_graph_cur_offset_ = 0; + uint32_t kernarg_graph_size_ = 128 * Ki; + public: GraphExec(std::vector& topoOrder, std::vector>& lists, std::unordered_map>& nodeWaitLists, struct Graph*& clonedGraph, @@ -591,6 +596,9 @@ struct GraphExec { auto device = g_devices[ihipGetDevice()]->devices()[0]; if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { device->hostFree(kernarg_pool_graph_, kernarg_pool_size_graph_); + for (auto& element : kernarg_graph_) { + device->hostFree(element, kernarg_graph_size_); + } } amd::ScopedLock lock(graphExecSetLock_); graphExecSet_.erase(this); @@ -636,6 +644,7 @@ struct GraphExec { hipError_t Run(hipStream_t stream); // Capture GPU Packets from graph commands hipError_t CaptureAQLPackets(); + hipError_t UpdateAQLPacket(hip::GraphKernelNode* node); }; struct ChildGraphNode : public GraphNode { @@ -793,19 +802,20 @@ class GraphKernelNode : public GraphNode { out << "];"; } - void CaptureAndFormPacket(address kernArgOffset) { - for (auto& command : commands_) { - reinterpret_cast(command)->setCapturingState( - true, GetAqlPacket(), kernArgOffset); + void CaptureAndFormPacket(hip::Stream* capture_stream, address kernArgOffset) { + hipError_t status = CreateCommand(capture_stream); + for (auto& command : commands_) { + reinterpret_cast(command)->setCapturingState( + true, GetAqlPacket(), kernArgOffset); - // Enqueue command to capture GPU Packet. The packet is not submitted to the device. - // The packet is stored in gpuPacket_ and submitted during graph launch. - command->submit(*(command->queue())->vdev()); - // Need to ensure if the command is NDRangeKernelCommand if we capture non kernel nodes - SetKernelName(reinterpret_cast(command)->kernel().name()); - command->release(); - } + // Enqueue command to capture GPU Packet. The packet is not submitted to the device. + // The packet is stored in gpuPacket_ and submitted during graph launch. + command->submit(*(command->queue())->vdev()); + // Need to ensure if the command is NDRangeKernelCommand if we capture non kernel nodes + SetKernelName(reinterpret_cast(command)->kernel().name()); + command->release(); } + } std::string GetLabel(hipGraphDebugDotFlags flag) { hipFunction_t func = getFunc(kernelParams_, ihipGetDevice()); diff --git a/projects/clr/hipamd/src/hip_internal.hpp b/projects/clr/hipamd/src/hip_internal.hpp index 88b12d8451..d3d38178fa 100644 --- a/projects/clr/hipamd/src/hip_internal.hpp +++ b/projects/clr/hipamd/src/hip_internal.hpp @@ -487,7 +487,7 @@ public: void setFlags(unsigned int flags) { flags_ = flags; } void Reset(); - hip::Stream* NullStream(); + hip::Stream* NullStream(bool wait = true); Stream* GetNullStream() const {return null_stream_;}; void SetActiveStatus() { @@ -572,7 +572,7 @@ public: /// Get default stream associated with the ROCclr context extern hip::Stream* getNullStream(amd::Context&); /// Get default stream of the thread - extern hip::Stream* getNullStream(); + extern hip::Stream* getNullStream(bool wait = true); /// Get device ID associated with the ROCclr context int getDeviceID(amd::Context& ctx); /// Check if stream is valid