Adding UT_DEBUG_PAUSE to unit tests (#1653)

test/common/EnvVars.cpp

@@ -205,6 +205,7 @@ namespace RcclUnitTesting
     isGfx90 = false;
     getArchInfo(&isGfx90, "gfx90");
 
+    debugPause     = GetEnvVar("UT_DEBUG_PAUSE" , 0);
     showNames      = GetEnvVar("UT_SHOW_NAMES"  , 1);
     minGpus        = GetEnvVar("UT_MIN_GPUS"    , 1);
     maxGpus        = GetEnvVar("UT_MAX_GPUS"    , numDetectedGpus);
@@ -348,6 +349,7 @@ namespace RcclUnitTesting
   {
     std::vector<std::tuple<std::string, int, std::string>> supported =
       {
+        std::make_tuple("UT_DEBUG_PAUSE"      , debugPause    , "Pause for debugger attach"),
         std::make_tuple("UT_SHOW_NAMES"       , showNames     , "Show test case names"),
         std::make_tuple("UT_MIN_GPUS"         , minGpus       , "Minimum number of GPUs to use"),
         std::make_tuple("UT_MAX_GPUS"         , maxGpus       , "Maximum number of GPUs to use"),

test/common/EnvVars.hpp

@@ -18,6 +18,7 @@ namespace RcclUnitTesting
   class EnvVars
   {
   public:
+    bool debugPause;     // Pause for debugger attach              [UT_DEBUG_PAUSE]
     bool showNames;      // List test case names during run        [UT_SHOW_NAMES]
     int  minGpus;        // Set the minimum number of GPUs to use  [UT_MIN_GPUS]
     int  maxGpus;        // Set the maximum number of GPUs to use  [UT_MAX_GPUS]
@@ -30,6 +31,7 @@ namespace RcclUnitTesting
     bool useInteractive; // Run in interactive mode                [UT_INTERACTIVE]
     int  timeoutUs;      // Set timeout for child in microseconds  [UT_TIMEOUT_US]
     bool useMultithreading; // Multi-thread single-process ranks   [UT_MULTITHREAD]
+
     bool isGfx94;        // Detects if architecture is gfx94
     bool isGfx12;        // Detects if architecture is gfx12
     bool isGfx90;        // Detects if architecture is gfx90

test/common/TestBed.cpp

@@ -120,6 +120,19 @@ namespace RcclUnitTesting
       }
     }
 
+    // If debugging is enabled, pause here to allow users to attach debugger
+    if (ev.debugPause) {
+      INFO("============================================================\n");
+      INFO(" Pausing for debug attach: (e.g. sudo rocgdb -p <PID>)\n");
+      INFO("============================================================\n");
+      for (int childId = 0; childId < this->numActiveChildren; ++childId) {
+        INFO(" Child %02d: processID: %d\n", childId, childList[childId]->pid);
+      }
+      INFO("============================================================\n");
+      INFO("<Press enter to continue>\n");
+      scanf("%*c");
+    }
+
     // Determine number of unique GPUs being used.
     std::set<int> unique_devices;
     for (auto a:  this->rankToDeviceMap)