AI/rocm-systems
2
0
Fork 0
Koodi Ongelmat Pull-pyynnöt Actions Paketit Projektit Julkaisut Wiki Toiminta

SWDEV-549711 - Improve graph DEBUG dot print for segments (#2205)

Selaa lähdekoodia 
Co-authored-by: Anusha GodavarthySurya<agodavar@amd.com>
This commit is contained in:
Godavarthy Surya, Anusha
2026-01-07 14:07:49 +05:30
committed by GitHub
vanhempi 81eed26ec6
commit 1ef6a86ee3
4 muutettua tiedostoa jossa 91 lisäystä ja 27 poistoa
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/hipamd/src/hip_graph.cpp
+1 -2
Näytä tiedosto
@@ -1530,8 +1530,7 @@ hipError_t ihipGraphInstantiate(hip::GraphExec** pGraphExec, hip::Graph* graph,
*pGraphExec = nullptr;
return scheduleStatus;
}
if (DEBUG_HIP_GRAPH_DOT_PRINT) {
if (DEBUG_HIP_GRAPH_DOT_PRINT == 1) {
static int i = 1;
std::string filename =
"graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_" + std::to_string(i++);
projects/clr/hipamd/src/hip_graph_internal.cpp
+14 -10
Näytä tiedosto
@@ -1645,6 +1645,7 @@ hipError_t GraphExec::EnqueueSegment(const Segment& segment, hip::Stream* stream
auto& node = segment.nodes[i];
if (DEBUG_HIP_GRAPH_DOT_PRINT) {
node->stream_id_ = stream->GetStreamId();
node->hw_queue_id_ = stream->getQueueID();
}
if (!node->GraphCaptureEnabled()) {
// Node doesn't support capture - execute individually
@@ -1684,12 +1685,16 @@ hipError_t GraphExec::EnqueueSegment(const Segment& segment, hip::Stream* stream
if (DEBUG_HIP_GRAPH_DOT_PRINT) {
for(int j = i; j < i + packetBatch.nodeRanges.size(); j++) {
segment.nodes[j]->stream_id_ = stream->GetStreamId();
segment.nodes[j]->hw_queue_id_ = stream->getQueueID();
}
}
// Skip all consecutive captured nodes that belong to this batch
i += packetBatch.nodeRanges.size() - 1; // -1 because loop will increment
++batchIndex;
}
if (DEBUG_HIP_GRAPH_DOT_PRINT) {
node->hw_queue_id_ = node->GetQueue()->getQueueID();
}
}
}
@@ -1982,16 +1987,6 @@ hipError_t GraphExec::Run(hip::Stream* launch_stream) {
if (last_cmd != nullptr) {
last_cmd->release();
}
if (DEBUG_HIP_GRAPH_DOT_PRINT && !graph_dumped_) {
graph_dumped_ = true;
std::string filename =
"graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_launch_1";
hipError_t status = ihipGraphDebugDotPrint(this, filename.c_str(), 0);
if (status == hipSuccess) {
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_CODE, "[hipGraph] graph dump:%s",
filename.c_str());
}
}
} else if (max_streams_ == 1 && instantiateDeviceId_ != launch_stream->DeviceId()) {
for (int i = 0; i < topoOrder_.size(); i++) {
topoOrder_[i]->SetStream(launch_stream);
@@ -2010,6 +2005,15 @@ hipError_t GraphExec::Run(hip::Stream* launch_stream) {
return hipErrorOutOfMemory;
}
}
if (DEBUG_HIP_GRAPH_DOT_PRINT == 2 && !graph_dumped_) {
graph_dumped_ = true;
std::string filename =
"graph_" + std::to_string(amd::Os::getProcessId()) + "_dot_print_launch_1";
hipError_t status = ihipGraphDebugDotPrint(this, filename.c_str(), 0);
if (status == hipSuccess) {
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_CODE, "[hipGraph] graph dump:%s", filename.c_str());
}
}
this->retain();
amd::Command* CallbackCommand = new amd::Marker(*launch_stream, kMarkerDisableFlush, {});
// we may not need to flush any caches.
projects/clr/hipamd/src/hip_graph_internal.hpp
+74 -13
Näytä tiedosto
@@ -446,12 +446,27 @@ class GraphNode : public hipGraphNodeDOTAttribute {
out << "=\"";
out << GetLabel(flag);
if (DEBUG_HIP_GRAPH_DOT_PRINT) {
out << "\nStreamId:" << stream_id_;
out << "\nSegmentId:" << segment_id_;
out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
if (DEBUG_HIP_GRAPH_DOT_PRINT >= 2) {
out << "\nStreamId:" << stream_id_;
out << "\nHW Queue:" << hw_queue_id_;
}
if (segment_id_ == -1) {
out << "\nStreamId:" << stream_id_;
out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
}
out << "\nDeviceId:" << dev_id_;
}
out << "\"";
if (DEBUG_HIP_GRAPH_DOT_PRINT) {
// Add color coding based on segment ID for better visualization
if (segment_id_ != -1) {
// Color nodes based on segment ID for better visual grouping
const char* colors[] = {"lightcoral", "lightblue", "lightgreen", "lightyellow",
"lightpink", "lightgray", "lightcyan", "lightsalmon"};
int color_index = segment_id_ % (sizeof(colors) / sizeof(colors[0]));
out << ",fillcolor=\"" << colors[color_index] << "\",style=\"filled\"";
}
}
out << "];";
}
void SetDeviceId(int id) { dev_id_ = id; }
@@ -473,6 +488,7 @@ class GraphNode : public hipGraphNodeDOTAttribute {
size_t inDegree_; //!< count of in coming edges (@todo: remove, it's dependencies_.size())
size_t outDegree_; //!< count of outgoing edges (@todo: remove, it's edges_.size())
int32_t stream_id_ = -1; //! Stream ID on which this node will be executed
int hw_queue_id_ = -1; //! Hardware queue ID on which this node will be executed
int32_t segment_id_ = -1; //! Segment ID on which this node will be executed
int32_t launch_id_ = -1; //! Launch ID of this node in the entire graph execution sequence
static int nextID;
@@ -489,7 +505,7 @@ class GraphNode : public hipGraphNodeDOTAttribute {
size_t kernargSegmentAlignment_ = 256; //!< Kernel arg segment alignment
int dev_id_; //!< Device Id when node is created(dev id from capture stream/current device
//!< when explicitly added)
bool wait_ = false;
bool wait_ = false;
};
class GraphEventWaitNode : public GraphNode {
@@ -733,19 +749,59 @@ class Graph {
void GenerateDOT(std::ostream& fout, hipGraphDebugDotFlags flag) {
fout << "subgraph cluster_" << GetID() << " {" << std::endl;
fout << "label=\"graph_" << GetID() << "\"graph[style=\"dashed\"];\n";
for (auto node : vertices_) {
node->GenerateDOTNode(GetID(), fout, flag);
}
fout << "\n";
for (auto& node : vertices_) {
node->GenerateDOTNodeEdges(GetID(), fout, flag);
// Check if we should group nodes by segments
bool useSegmentClustering = !segments_.empty();
if (useSegmentClustering) {
GenerateDOTWithSegments(fout, flag);
} else {
// Original node-by-node generation
for (auto node : vertices_) {
node->GenerateDOTNode(GetID(), fout, flag);
}
fout << "\n";
for (auto& node : vertices_) {
node->GenerateDOTNodeEdges(GetID(), fout, flag);
}
}
fout << "}" << std::endl;
for (auto node : vertices_) {
node->GenerateDOT(fout, flag);
}
}
// generate DOT with segment clustering
void GenerateDOTWithSegments(std::ostream& fout, hipGraphDebugDotFlags flag) {
// Generate segment clusters
for (const auto& segment : segments_) {
// if (segment_nodes.find(segment.id) != segment_nodes.end()) {
fout << "subgraph cluster_segment_" << segment.id << " {" << std::endl;
fout << "label=\"Segment " << segment.id;
if (segment.stream_id != -1) {
fout << "\\nStream: " << segment.stream_id;
}
if (segment.dependency_level != -1) {
fout << "\\nLevel: " << segment.dependency_level;
}
fout << "\";" << std::endl;
fout << "style=\"rounded,filled\";" << std::endl;
fout << "fillcolor=\"lightblue\";" << std::endl;
fout << "color=\"blue\";" << std::endl;
for (auto node : segment.nodes) {
node->GenerateDOTNode(GetID(), fout, flag);
}
fout << "}" << std::endl;
}
fout << "\n";
// Generate all edges after all nodes are defined
for (auto& node : vertices_) {
node->GenerateDOTNodeEdges(GetID(), fout, flag);
}
}
void* AllocateMemory(size_t size, hip::Stream* stream, void* dptr) const {
auto ptr = mem_pool_->AllocateMemory(size, stream, dptr);
return ptr;
@@ -1187,9 +1243,14 @@ class GraphKernelNode : public GraphNode {
out << "=\"";
out << GetLabel(flag);
if (DEBUG_HIP_GRAPH_DOT_PRINT) {
out << "StreamId:" << stream_id_;
out << "\nSegmentId:" << segment_id_;
out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
if (DEBUG_HIP_GRAPH_DOT_PRINT >= 2) {
out << "\nStreamId:" << stream_id_;
out << "\nHW Queue:" << hw_queue_id_;
}
if (segment_id_ == -1) {
out << "\nStreamId:" << stream_id_;
out << "\nSignalIsRequired: " << ((signal_is_required_) ? "true" : "false");
}
out << "\nDeviceId:" << dev_id_;
}
out << "\"";
projects/clr/rocclr/utils/flags.hpp
+2 -2
Näytä tiedosto
@@ -245,8 +245,8 @@ release(cstring, HIPRTC_LINK_OPTIONS_APPEND, "", \
"Set link options needed for hiprtc compilation") \
release(bool, HIP_VMEM_MANAGE_SUPPORT, true, \
"Virtual Memory Management Support") \
release(bool, DEBUG_HIP_GRAPH_DOT_PRINT, false, \
"Enable/Disable graph debug dot print dump") \
release(uint, DEBUG_HIP_GRAPH_DOT_PRINT, 0, \
"0 = Disable, 1 = Print during Graph Inst, 2 = Print during Graph Launch") \
release(bool, DEBUG_HIP_FORCE_ASYNC_QUEUE, false, \
"Forces grpahs into async queue mode. DEBUG_HIP_FORCE_GRAPH_QUEUES must be 1") \
release(uint, DEBUG_HIP_FORCE_GRAPH_QUEUES, 4, \