SWDEV-549711 - Improve graph DEBUG dot print for segments (#2205)

Co-authored-by: Anusha GodavarthySurya<agodavar@amd.com>

projects/clr/hipamd/src/hip_graph.cpp

@@ -1530,8 +1530,7 @@ hipError_t ihipGraphInstantiate(hip::GraphExec** pGraphExec, hip::Graph* graph,
     *pGraphExec = nullptr;
     return scheduleStatus;
   }
-
-  if (DEBUG_HIP_GRAPH_DOT_PRINT) {
+  if (DEBUG_HIP_GRAPH_DOT_PRINT == 1) {
     static int i = 1;
     std::string filename =
         "graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_" + std::to_string(i++);

projects/clr/hipamd/src/hip_graph_internal.cpp

@@ -1645,6 +1645,7 @@ hipError_t GraphExec::EnqueueSegment(const Segment& segment, hip::Stream* stream
     auto& node = segment.nodes[i];
     if (DEBUG_HIP_GRAPH_DOT_PRINT) {
       node->stream_id_ = stream->GetStreamId();
+      node->hw_queue_id_ = stream->getQueueID();
     }
     if (!node->GraphCaptureEnabled()) {
       // Node doesn't support capture - execute individually
@@ -1684,12 +1685,16 @@ hipError_t GraphExec::EnqueueSegment(const Segment& segment, hip::Stream* stream
         if (DEBUG_HIP_GRAPH_DOT_PRINT) {
           for(int j = i; j < i + packetBatch.nodeRanges.size(); j++) {
             segment.nodes[j]->stream_id_ = stream->GetStreamId();
+            segment.nodes[j]->hw_queue_id_ = stream->getQueueID();
           }
         }
         // Skip all consecutive captured nodes that belong to this batch
         i += packetBatch.nodeRanges.size() - 1;  // -1 because loop will increment
         ++batchIndex;
       }
+      if (DEBUG_HIP_GRAPH_DOT_PRINT) {
+        node->hw_queue_id_ = node->GetQueue()->getQueueID();
+      }
     }
   }
 
@@ -1982,16 +1987,6 @@ hipError_t GraphExec::Run(hip::Stream* launch_stream) {
     if (last_cmd != nullptr) {
       last_cmd->release();
     }
-    if (DEBUG_HIP_GRAPH_DOT_PRINT && !graph_dumped_) {
-      graph_dumped_ = true;
-      std::string filename =
-        "graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_launch_1";
-      hipError_t status = ihipGraphDebugDotPrint(this, filename.c_str(), 0);
-      if (status == hipSuccess) {
-        ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_CODE, "[hipGraph] graph dump:%s",
-                filename.c_str());
-      }
-    }
   } else if (max_streams_ == 1 && instantiateDeviceId_ != launch_stream->DeviceId()) {
     for (int i = 0; i < topoOrder_.size(); i++) {
       topoOrder_[i]->SetStream(launch_stream);
@@ -2010,6 +2005,15 @@ hipError_t GraphExec::Run(hip::Stream* launch_stream) {
       return hipErrorOutOfMemory;
     }
   }
+  if (DEBUG_HIP_GRAPH_DOT_PRINT == 2 && !graph_dumped_) {
+    graph_dumped_ = true;
+    std::string filename =
+        "graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_launch_1";
+    hipError_t status = ihipGraphDebugDotPrint(this, filename.c_str(), 0);
+    if (status == hipSuccess) {
+      ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_CODE, "[hipGraph] graph dump:%s", filename.c_str());
+    }
+  }
   this->retain();
   amd::Command* CallbackCommand = new amd::Marker(*launch_stream, kMarkerDisableFlush, {});
   // we may not need to flush any caches.

projects/clr/hipamd/src/hip_graph_internal.hpp

@@ -446,12 +446,27 @@ class GraphNode : public hipGraphNodeDOTAttribute {
     out << "=\"";
     out << GetLabel(flag);
     if (DEBUG_HIP_GRAPH_DOT_PRINT) {
-      out << "\nStreamId:" << stream_id_;
-      out << "\nSegmentId:" << segment_id_;
-      out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
+      if (DEBUG_HIP_GRAPH_DOT_PRINT >= 2) {
+        out << "\nStreamId:" << stream_id_;
+        out << "\nHW Queue:" << hw_queue_id_;
+      }
+      if (segment_id_ == -1) {
+        out << "\nStreamId:" << stream_id_;
+        out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
+      }
       out << "\nDeviceId:" << dev_id_;
     }
     out << "\"";
+    if (DEBUG_HIP_GRAPH_DOT_PRINT) {
+      // Add color coding based on segment ID for better visualization
+      if (segment_id_ != -1) {
+        // Color nodes based on segment ID for better visual grouping
+        const char* colors[] = {"lightcoral", "lightblue", "lightgreen", "lightyellow",
+                                "lightpink",  "lightgray", "lightcyan",  "lightsalmon"};
+        int color_index = segment_id_ % (sizeof(colors) / sizeof(colors[0]));
+        out << ",fillcolor=\"" << colors[color_index] << "\",style=\"filled\"";
+      }
+    }
     out << "];";
   }
   void SetDeviceId(int id) { dev_id_ = id; }
@@ -473,6 +488,7 @@ class GraphNode : public hipGraphNodeDOTAttribute {
   size_t inDegree_;         //!< count of in coming edges (@todo: remove, it's dependencies_.size())
   size_t outDegree_;        //!< count of outgoing edges (@todo: remove, it's edges_.size())
   int32_t stream_id_ = -1;  //! Stream ID on which this node will be executed
+  int hw_queue_id_ = -1; //! Hardware queue ID on which this node will be executed
   int32_t segment_id_ = -1;  //! Segment ID on which this node will be executed
   int32_t launch_id_ = -1;  //! Launch ID of this node in the entire graph execution sequence
   static int nextID;
@@ -489,7 +505,7 @@ class GraphNode : public hipGraphNodeDOTAttribute {
   size_t kernargSegmentAlignment_ = 256;  //!< Kernel arg segment alignment
   int dev_id_;  //!< Device Id when node is created(dev id from capture stream/current device
                 //!< when explicitly added)
-  bool wait_ = false;                
+  bool wait_ = false;
 };
 
 class GraphEventWaitNode : public GraphNode {
@@ -733,19 +749,59 @@ class Graph {
   void GenerateDOT(std::ostream& fout, hipGraphDebugDotFlags flag) {
     fout << "subgraph cluster_" << GetID() << " {" << std::endl;
     fout << "label=\"graph_" << GetID() << "\"graph[style=\"dashed\"];\n";
-    for (auto node : vertices_) {
-      node->GenerateDOTNode(GetID(), fout, flag);
-    }
-    fout << "\n";
-    for (auto& node : vertices_) {
-      node->GenerateDOTNodeEdges(GetID(), fout, flag);
+
+    // Check if we should group nodes by segments
+    bool useSegmentClustering = !segments_.empty();
+
+    if (useSegmentClustering) {
+      GenerateDOTWithSegments(fout, flag);
+    } else {
+      // Original node-by-node generation
+      for (auto node : vertices_) {
+        node->GenerateDOTNode(GetID(), fout, flag);
+      }
+      fout << "\n";
+      for (auto& node : vertices_) {
+        node->GenerateDOTNodeEdges(GetID(), fout, flag);
+      }
     }
+
     fout << "}" << std::endl;
     for (auto node : vertices_) {
       node->GenerateDOT(fout, flag);
     }
   }
 
+  // generate DOT with segment clustering
+  void GenerateDOTWithSegments(std::ostream& fout, hipGraphDebugDotFlags flag) {
+    // Generate segment clusters
+    for (const auto& segment : segments_) {
+      // if (segment_nodes.find(segment.id) != segment_nodes.end()) {
+      fout << "subgraph cluster_segment_" << segment.id << " {" << std::endl;
+      fout << "label=\"Segment " << segment.id;
+      if (segment.stream_id != -1) {
+        fout << "\\nStream: " << segment.stream_id;
+      }
+      if (segment.dependency_level != -1) {
+        fout << "\\nLevel: " << segment.dependency_level;
+      }
+      fout << "\";" << std::endl;
+      fout << "style=\"rounded,filled\";" << std::endl;
+      fout << "fillcolor=\"lightblue\";" << std::endl;
+      fout << "color=\"blue\";" << std::endl;
+      for (auto node : segment.nodes) {
+        node->GenerateDOTNode(GetID(), fout, flag);
+      }
+      fout << "}" << std::endl;
+    }
+
+    fout << "\n";
+    // Generate all edges after all nodes are defined
+    for (auto& node : vertices_) {
+      node->GenerateDOTNodeEdges(GetID(), fout, flag);
+    }
+  }
+
   void* AllocateMemory(size_t size, hip::Stream* stream, void* dptr) const {
     auto ptr = mem_pool_->AllocateMemory(size, stream, dptr);
     return ptr;
@@ -1187,9 +1243,14 @@ class GraphKernelNode : public GraphNode {
     out << "=\"";
     out << GetLabel(flag);
     if (DEBUG_HIP_GRAPH_DOT_PRINT) {
-      out << "StreamId:" << stream_id_;
-      out << "\nSegmentId:" << segment_id_;
-      out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
+      if (DEBUG_HIP_GRAPH_DOT_PRINT >= 2) {
+        out << "\nStreamId:" << stream_id_;
+        out << "\nHW Queue:" << hw_queue_id_;
+      }
+      if (segment_id_ == -1) {
+        out << "\nStreamId:" << stream_id_;
+        out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
+      }
       out << "\nDeviceId:" << dev_id_;
     }
     out << "\"";

projects/clr/rocclr/utils/flags.hpp

@@ -245,8 +245,8 @@ release(cstring, HIPRTC_LINK_OPTIONS_APPEND, "",                              \
         "Set link options needed for hiprtc compilation")                     \
 release(bool, HIP_VMEM_MANAGE_SUPPORT, true,                                  \
         "Virtual Memory Management Support")                                  \
-release(bool, DEBUG_HIP_GRAPH_DOT_PRINT, false,                               \
-         "Enable/Disable graph debug dot print dump")                         \
+release(uint, DEBUG_HIP_GRAPH_DOT_PRINT, 0,                               \
+        "0 = Disable, 1 = Print during Graph Inst, 2 = Print during Graph Launch") \
 release(bool, DEBUG_HIP_FORCE_ASYNC_QUEUE, false,                             \
         "Forces grpahs into async queue mode. DEBUG_HIP_FORCE_GRAPH_QUEUES must be 1") \
 release(uint, DEBUG_HIP_FORCE_GRAPH_QUEUES, 4,                                \