AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-460770 - Handle Graph Exec release

Browse Source 
Handle GraphExec instance is destroyed before async launch completes
GraphExec instance is destroyed after async launch completes
GraphExec instance is destroyed without a launch

Change-Id: I45a7c82295fea916c7559bd8f796df710513aea1
This commit is contained in:
Anusha GodavarthySurya
2024-03-28 17:28:41 +00:00
committed by Maneesh Gupta
parent 51e4368723
commit bf4d10ff61
7 changed files with 64 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hipamd/src/hip_graph_internal.cpp
+37 -1
View File
@@ -49,6 +49,9 @@ const char* GetGraphNodeTypeString(uint32_t op) {
};
namespace hip {
std::unordered_map<GraphExec*, std::pair<hip::Stream*, bool>> GraphExecStatus_;
amd::Monitor GraphExecStatusLock_{"Guards graph execution state"};
int GraphNode::nextID = 0;
int Graph::nextID = 0;
std::unordered_set<GraphNode*> GraphNode::nodeSet_;
@@ -586,7 +589,6 @@ hipError_t GraphExec::Run(hipStream_t stream) {
accumulate = new amd::AccumulateCommand(*hip_stream, {}, nullptr, lastCapturedPacket);
}
}
for (int i = 0; i < topoOrder_.size() - 1; i++) {
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE && topoOrder_[i]->GetType() == hipGraphNodeTypeKernel &&
!reinterpret_cast<hip::GraphKernelNode*>(topoOrder_[i])->HasHiddenHeap()) {
@@ -646,7 +648,41 @@ hipError_t GraphExec::Run(hipStream_t stream) {
endCommand->release();
}
}
amd::ScopedLock lock(GraphExecStatusLock_);
GraphExecStatus_[this] = std::make_pair(hip_stream, false);
ResetQueueIndex();
return status;
}
void ReleaseGraphExec(int deviceId) {
// Release all graph exec objects destroyed by user.
amd::ScopedLock lock(GraphExecStatusLock_);
for (auto itr = GraphExecStatus_.begin(); itr != GraphExecStatus_.end();) {
auto pair = itr->second;
if (pair.first->DeviceId() == deviceId) {
if (pair.second == true) {
ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] Release GraphExec");
(itr->first)->release();
}
GraphExecStatus_.erase(itr++);
} else {
itr++;
}
}
}
void ReleaseGraphExec(hip::Stream* stream) {
amd::ScopedLock lock(GraphExecStatusLock_);
for (auto itr = GraphExecStatus_.begin(); itr != GraphExecStatus_.end();) {
auto pair = itr->second;
if (pair.first == stream) {
if (pair.second == true) {
ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] Release GraphExec");
(itr->first)->release();
}
GraphExecStatus_.erase(itr++);
break;
} else {
++itr;
}
}
}
} // namespace hip