diff --git a/projects/clr/hipamd/src/hip_graph.cpp b/projects/clr/hipamd/src/hip_graph.cpp index 7af3b01f61..7c51e5abb6 100644 --- a/projects/clr/hipamd/src/hip_graph.cpp +++ b/projects/clr/hipamd/src/hip_graph.cpp @@ -1530,8 +1530,7 @@ hipError_t ihipGraphInstantiate(hip::GraphExec** pGraphExec, hip::Graph* graph, *pGraphExec = nullptr; return scheduleStatus; } - - if (DEBUG_HIP_GRAPH_DOT_PRINT) { + if (DEBUG_HIP_GRAPH_DOT_PRINT == 1) { static int i = 1; std::string filename = "graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_" + std::to_string(i++); diff --git a/projects/clr/hipamd/src/hip_graph_internal.cpp b/projects/clr/hipamd/src/hip_graph_internal.cpp index 09ce75ff36..17fd391db0 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.cpp +++ b/projects/clr/hipamd/src/hip_graph_internal.cpp @@ -1645,6 +1645,7 @@ hipError_t GraphExec::EnqueueSegment(const Segment& segment, hip::Stream* stream auto& node = segment.nodes[i]; if (DEBUG_HIP_GRAPH_DOT_PRINT) { node->stream_id_ = stream->GetStreamId(); + node->hw_queue_id_ = stream->getQueueID(); } if (!node->GraphCaptureEnabled()) { // Node doesn't support capture - execute individually @@ -1684,12 +1685,16 @@ hipError_t GraphExec::EnqueueSegment(const Segment& segment, hip::Stream* stream if (DEBUG_HIP_GRAPH_DOT_PRINT) { for(int j = i; j < i + packetBatch.nodeRanges.size(); j++) { segment.nodes[j]->stream_id_ = stream->GetStreamId(); + segment.nodes[j]->hw_queue_id_ = stream->getQueueID(); } } // Skip all consecutive captured nodes that belong to this batch i += packetBatch.nodeRanges.size() - 1; // -1 because loop will increment ++batchIndex; } + if (DEBUG_HIP_GRAPH_DOT_PRINT) { + node->hw_queue_id_ = node->GetQueue()->getQueueID(); + } } } @@ -1982,16 +1987,6 @@ hipError_t GraphExec::Run(hip::Stream* launch_stream) { if (last_cmd != nullptr) { last_cmd->release(); } - if (DEBUG_HIP_GRAPH_DOT_PRINT && !graph_dumped_) { - graph_dumped_ = true; - std::string filename = - "graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_launch_1"; - hipError_t status = ihipGraphDebugDotPrint(this, filename.c_str(), 0); - if (status == hipSuccess) { - ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_CODE, "[hipGraph] graph dump:%s", - filename.c_str()); - } - } } else if (max_streams_ == 1 && instantiateDeviceId_ != launch_stream->DeviceId()) { for (int i = 0; i < topoOrder_.size(); i++) { topoOrder_[i]->SetStream(launch_stream); @@ -2010,6 +2005,15 @@ hipError_t GraphExec::Run(hip::Stream* launch_stream) { return hipErrorOutOfMemory; } } + if (DEBUG_HIP_GRAPH_DOT_PRINT == 2 && !graph_dumped_) { + graph_dumped_ = true; + std::string filename = + "graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_launch_1"; + hipError_t status = ihipGraphDebugDotPrint(this, filename.c_str(), 0); + if (status == hipSuccess) { + ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_CODE, "[hipGraph] graph dump:%s", filename.c_str()); + } + } this->retain(); amd::Command* CallbackCommand = new amd::Marker(*launch_stream, kMarkerDisableFlush, {}); // we may not need to flush any caches. diff --git a/projects/clr/hipamd/src/hip_graph_internal.hpp b/projects/clr/hipamd/src/hip_graph_internal.hpp index c0ed09f004..64a13c0012 100644 --- a/projects/clr/hipamd/src/hip_graph_internal.hpp +++ b/projects/clr/hipamd/src/hip_graph_internal.hpp @@ -446,12 +446,27 @@ class GraphNode : public hipGraphNodeDOTAttribute { out << "=\""; out << GetLabel(flag); if (DEBUG_HIP_GRAPH_DOT_PRINT) { - out << "\nStreamId:" << stream_id_; - out << "\nSegmentId:" << segment_id_; - out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false"); + if (DEBUG_HIP_GRAPH_DOT_PRINT >= 2) { + out << "\nStreamId:" << stream_id_; + out << "\nHW Queue:" << hw_queue_id_; + } + if (segment_id_ == -1) { + out << "\nStreamId:" << stream_id_; + out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false"); + } out << "\nDeviceId:" << dev_id_; } out << "\""; + if (DEBUG_HIP_GRAPH_DOT_PRINT) { + // Add color coding based on segment ID for better visualization + if (segment_id_ != -1) { + // Color nodes based on segment ID for better visual grouping + const char* colors[] = {"lightcoral", "lightblue", "lightgreen", "lightyellow", + "lightpink", "lightgray", "lightcyan", "lightsalmon"}; + int color_index = segment_id_ % (sizeof(colors) / sizeof(colors[0])); + out << ",fillcolor=\"" << colors[color_index] << "\",style=\"filled\""; + } + } out << "];"; } void SetDeviceId(int id) { dev_id_ = id; } @@ -473,6 +488,7 @@ class GraphNode : public hipGraphNodeDOTAttribute { size_t inDegree_; //!< count of in coming edges (@todo: remove, it's dependencies_.size()) size_t outDegree_; //!< count of outgoing edges (@todo: remove, it's edges_.size()) int32_t stream_id_ = -1; //! Stream ID on which this node will be executed + int hw_queue_id_ = -1; //! Hardware queue ID on which this node will be executed int32_t segment_id_ = -1; //! Segment ID on which this node will be executed int32_t launch_id_ = -1; //! Launch ID of this node in the entire graph execution sequence static int nextID; @@ -489,7 +505,7 @@ class GraphNode : public hipGraphNodeDOTAttribute { size_t kernargSegmentAlignment_ = 256; //!< Kernel arg segment alignment int dev_id_; //!< Device Id when node is created(dev id from capture stream/current device //!< when explicitly added) - bool wait_ = false; + bool wait_ = false; }; class GraphEventWaitNode : public GraphNode { @@ -733,19 +749,59 @@ class Graph { void GenerateDOT(std::ostream& fout, hipGraphDebugDotFlags flag) { fout << "subgraph cluster_" << GetID() << " {" << std::endl; fout << "label=\"graph_" << GetID() << "\"graph[style=\"dashed\"];\n"; - for (auto node : vertices_) { - node->GenerateDOTNode(GetID(), fout, flag); - } - fout << "\n"; - for (auto& node : vertices_) { - node->GenerateDOTNodeEdges(GetID(), fout, flag); + + // Check if we should group nodes by segments + bool useSegmentClustering = !segments_.empty(); + + if (useSegmentClustering) { + GenerateDOTWithSegments(fout, flag); + } else { + // Original node-by-node generation + for (auto node : vertices_) { + node->GenerateDOTNode(GetID(), fout, flag); + } + fout << "\n"; + for (auto& node : vertices_) { + node->GenerateDOTNodeEdges(GetID(), fout, flag); + } } + fout << "}" << std::endl; for (auto node : vertices_) { node->GenerateDOT(fout, flag); } } + // generate DOT with segment clustering + void GenerateDOTWithSegments(std::ostream& fout, hipGraphDebugDotFlags flag) { + // Generate segment clusters + for (const auto& segment : segments_) { + // if (segment_nodes.find(segment.id) != segment_nodes.end()) { + fout << "subgraph cluster_segment_" << segment.id << " {" << std::endl; + fout << "label=\"Segment " << segment.id; + if (segment.stream_id != -1) { + fout << "\\nStream: " << segment.stream_id; + } + if (segment.dependency_level != -1) { + fout << "\\nLevel: " << segment.dependency_level; + } + fout << "\";" << std::endl; + fout << "style=\"rounded,filled\";" << std::endl; + fout << "fillcolor=\"lightblue\";" << std::endl; + fout << "color=\"blue\";" << std::endl; + for (auto node : segment.nodes) { + node->GenerateDOTNode(GetID(), fout, flag); + } + fout << "}" << std::endl; + } + + fout << "\n"; + // Generate all edges after all nodes are defined + for (auto& node : vertices_) { + node->GenerateDOTNodeEdges(GetID(), fout, flag); + } + } + void* AllocateMemory(size_t size, hip::Stream* stream, void* dptr) const { auto ptr = mem_pool_->AllocateMemory(size, stream, dptr); return ptr; @@ -1187,9 +1243,14 @@ class GraphKernelNode : public GraphNode { out << "=\""; out << GetLabel(flag); if (DEBUG_HIP_GRAPH_DOT_PRINT) { - out << "StreamId:" << stream_id_; - out << "\nSegmentId:" << segment_id_; - out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false"); + if (DEBUG_HIP_GRAPH_DOT_PRINT >= 2) { + out << "\nStreamId:" << stream_id_; + out << "\nHW Queue:" << hw_queue_id_; + } + if (segment_id_ == -1) { + out << "\nStreamId:" << stream_id_; + out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false"); + } out << "\nDeviceId:" << dev_id_; } out << "\""; diff --git a/projects/clr/rocclr/utils/flags.hpp b/projects/clr/rocclr/utils/flags.hpp index b641ef7d79..656c5895da 100644 --- a/projects/clr/rocclr/utils/flags.hpp +++ b/projects/clr/rocclr/utils/flags.hpp @@ -245,8 +245,8 @@ release(cstring, HIPRTC_LINK_OPTIONS_APPEND, "", \ "Set link options needed for hiprtc compilation") \ release(bool, HIP_VMEM_MANAGE_SUPPORT, true, \ "Virtual Memory Management Support") \ -release(bool, DEBUG_HIP_GRAPH_DOT_PRINT, false, \ - "Enable/Disable graph debug dot print dump") \ +release(uint, DEBUG_HIP_GRAPH_DOT_PRINT, 0, \ + "0 = Disable, 1 = Print during Graph Inst, 2 = Print during Graph Launch") \ release(bool, DEBUG_HIP_FORCE_ASYNC_QUEUE, false, \ "Forces grpahs into async queue mode. DEBUG_HIP_FORCE_GRAPH_QUEUES must be 1") \ release(uint, DEBUG_HIP_FORCE_GRAPH_QUEUES, 4, \