diff --git a/projects/rccl/test/common/EnvVars.cpp b/projects/rccl/test/common/EnvVars.cpp index 42251948b4..b6cc407e5d 100644 --- a/projects/rccl/test/common/EnvVars.cpp +++ b/projects/rccl/test/common/EnvVars.cpp @@ -63,7 +63,7 @@ namespace RcclUnitTesting maxRanksPerGpu = GetEnvVar("UT_MAX_RANKS_PER_GPU", 1); showTiming = GetEnvVar("UT_SHOW_TIMING", 1); useInteractive = GetEnvVar("UT_INTERACTIVE", 0); - timeoutUs = GetEnvVar("UT_TIMEOUT_US" , 200000); + timeoutUs = GetEnvVar("UT_TIMEOUT_US" , 5000000); // Limit number of supported reduction operators to just ncclSum if only allReduce is built #ifdef BUILD_ALLREDUCE_ONLY diff --git a/projects/rccl/test/common/TestBedChild.cpp b/projects/rccl/test/common/TestBedChild.cpp index 38dd0dc0ac..bd8ed50014 100644 --- a/projects/rccl/test/common/TestBedChild.cpp +++ b/projects/rccl/test/common/TestBedChild.cpp @@ -667,7 +667,7 @@ namespace RcclUnitTesting for (int i = 0; i < this->numStreamsPerGroup; i++) streamsToComplete.push_back(this->streams[localRank][i]); } - int usElapsed = 0; + int usElapsed = 0, timedout = 0; using namespace std::chrono; using Clock = std::chrono::high_resolution_clock; if (this->verbose) INFO("Starting sychronization and timing\n"); @@ -692,7 +692,7 @@ namespace RcclUnitTesting for (int localRank : localRanksToExecute) { ncclCommAbort(this->comms[localRank]); - timeoutUs = -1; + timedout = 1; } } @@ -739,8 +739,11 @@ namespace RcclUnitTesting } } - if (timeoutUs == -1) + if (timedout) + { + ERROR("Child %d timed out and exceeded limit %d us in ExecuteCollectives()\n", this->childId, timeoutUs); return TEST_TIMEOUT; + } if (this->verbose) INFO("Child %d finishes ExecuteCollectives()\n", this->childId); return TEST_SUCCESS;