Relaxing default timeout limit, add error log (#1052)

Signed-off-by: Tim Hu <timhu102@amd.com>

[ROCm/rccl commit: 05850e89f2]

projects/rccl/test/common/EnvVars.cpp

@@ -63,7 +63,7 @@ namespace RcclUnitTesting
     maxRanksPerGpu = GetEnvVar("UT_MAX_RANKS_PER_GPU", 1);
     showTiming     = GetEnvVar("UT_SHOW_TIMING",  1);
     useInteractive = GetEnvVar("UT_INTERACTIVE",  0);
-    timeoutUs      = GetEnvVar("UT_TIMEOUT_US" ,  200000);
+    timeoutUs      = GetEnvVar("UT_TIMEOUT_US" ,  5000000);
 
     // Limit number of supported reduction operators to just ncclSum if only allReduce is built
 #ifdef BUILD_ALLREDUCE_ONLY

projects/rccl/test/common/TestBedChild.cpp

@@ -667,7 +667,7 @@ namespace RcclUnitTesting
       for (int i = 0; i < this->numStreamsPerGroup; i++)
         streamsToComplete.push_back(this->streams[localRank][i]);
     }
-    int usElapsed = 0;
+    int usElapsed = 0, timedout = 0;
     using namespace std::chrono;
     using Clock = std::chrono::high_resolution_clock;
     if (this->verbose) INFO("Starting sychronization and timing\n");
@@ -692,7 +692,7 @@ namespace RcclUnitTesting
       for (int localRank : localRanksToExecute)
       {
         ncclCommAbort(this->comms[localRank]); 
-        timeoutUs = -1;
+        timedout = 1;
       }
     }
 
@@ -739,8 +739,11 @@ namespace RcclUnitTesting
         }
     }
 
-    if (timeoutUs == -1)
+    if (timedout)
+    {
+      ERROR("Child %d timed out and exceeded limit %d us in ExecuteCollectives()\n", this->childId, timeoutUs);
       return TEST_TIMEOUT;
+    }
 
     if (this->verbose) INFO("Child %d finishes ExecuteCollectives()\n", this->childId);
     return TEST_SUCCESS;