From 0343d9ccacd2b0da6fe02fefdef738d4384ad33b Mon Sep 17 00:00:00 2001
From: Tim <43156029+AtlantaPepsi@users.noreply.github.com>
Date: Thu, 18 Jan 2024 15:09:08 -0500
Subject: [PATCH] Relaxing default timeout limit, add error log (#1052)

Signed-off-by: Tim Hu <timhu102@amd.com>

[ROCm/rccl commit: 05850e89f21d86bc130417d2a05c78b9c968027a]
---
 projects/rccl/test/common/EnvVars.cpp      | 2 +-
 projects/rccl/test/common/TestBedChild.cpp | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/projects/rccl/test/common/EnvVars.cpp b/projects/rccl/test/common/EnvVars.cpp
index 42251948b4..b6cc407e5d 100644
--- a/projects/rccl/test/common/EnvVars.cpp
+++ b/projects/rccl/test/common/EnvVars.cpp
@@ -63,7 +63,7 @@ namespace RcclUnitTesting
     maxRanksPerGpu = GetEnvVar("UT_MAX_RANKS_PER_GPU", 1);
     showTiming     = GetEnvVar("UT_SHOW_TIMING",  1);
     useInteractive = GetEnvVar("UT_INTERACTIVE",  0);
-    timeoutUs      = GetEnvVar("UT_TIMEOUT_US" ,  200000);
+    timeoutUs      = GetEnvVar("UT_TIMEOUT_US" ,  5000000);
 
     // Limit number of supported reduction operators to just ncclSum if only allReduce is built
 #ifdef BUILD_ALLREDUCE_ONLY
diff --git a/projects/rccl/test/common/TestBedChild.cpp b/projects/rccl/test/common/TestBedChild.cpp
index 38dd0dc0ac..bd8ed50014 100644
--- a/projects/rccl/test/common/TestBedChild.cpp
+++ b/projects/rccl/test/common/TestBedChild.cpp
@@ -667,7 +667,7 @@ namespace RcclUnitTesting
       for (int i = 0; i < this->numStreamsPerGroup; i++)
         streamsToComplete.push_back(this->streams[localRank][i]);
     }
-    int usElapsed = 0;
+    int usElapsed = 0, timedout = 0;
     using namespace std::chrono;
     using Clock = std::chrono::high_resolution_clock;
     if (this->verbose) INFO("Starting sychronization and timing\n");
@@ -692,7 +692,7 @@ namespace RcclUnitTesting
       for (int localRank : localRanksToExecute)
       {
         ncclCommAbort(this->comms[localRank]); 
-        timeoutUs = -1;
+        timedout = 1;
       }
     }
 
@@ -739,8 +739,11 @@ namespace RcclUnitTesting
         }
     }
 
-    if (timeoutUs == -1)
+    if (timedout)
+    {
+      ERROR("Child %d timed out and exceeded limit %d us in ExecuteCollectives()\n", this->childId, timeoutUs);
       return TEST_TIMEOUT;
+    }
 
     if (this->verbose) INFO("Child %d finishes ExecuteCollectives()\n", this->childId);
     return TEST_SUCCESS;