Relaxing default timeout limit, add error log (#1052)

Signed-off-by: Tim Hu <timhu102@amd.com>
此提交包含在:
Tim
2024-01-18 15:09:08 -05:00
提交者 GitHub
父節點 c2a073a97d
當前提交 05850e89f2
共有 2 個檔案被更改,包括 7 行新增 和 4 行删除
+6 -3
查看文件
@@ -667,7 +667,7 @@ namespace RcclUnitTesting
for (int i = 0; i < this->numStreamsPerGroup; i++)
streamsToComplete.push_back(this->streams[localRank][i]);
}
int usElapsed = 0;
int usElapsed = 0, timedout = 0;
using namespace std::chrono;
using Clock = std::chrono::high_resolution_clock;
if (this->verbose) INFO("Starting sychronization and timing\n");
@@ -692,7 +692,7 @@ namespace RcclUnitTesting
for (int localRank : localRanksToExecute)
{
ncclCommAbort(this->comms[localRank]);
timeoutUs = -1;
timedout = 1;
}
}
@@ -739,8 +739,11 @@ namespace RcclUnitTesting
}
}
if (timeoutUs == -1)
if (timedout)
{
ERROR("Child %d timed out and exceeded limit %d us in ExecuteCollectives()\n", this->childId, timeoutUs);
return TEST_TIMEOUT;
}
if (this->verbose) INFO("Child %d finishes ExecuteCollectives()\n", this->childId);
return TEST_SUCCESS;