Fix a RCCL initialization to avoid a deadlock (#136)

Also fixes: 

- crash while finalizing rocprof-sys-causal

[ROCm/rocprofiler-systems commit: 26bb604215]

projects/rocprofiler-systems/source/lib/rocprof-sys/library.cpp

@@ -403,16 +403,42 @@ rocprofsys_init_library_hidden()
     ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "\n");
 }
 
+// Initialize RCCL if:
+// - postinit=true - so the code doesn't hang at the initialization stage
+// - get_state() >= State::Init - so the code doesn't throw an exception
+// - rccl_initialized=false - so we don't try to initialize RCCL twice
+// - get_use_rcclp()=true - only if the environment is configured to use RCCL
+static void
+rccl_setup(bool postinit)
+{
+    // Flag used to avoid initializing RCCL twice
+    static bool rccl_initialized = false;
+
+    if(postinit && (get_state() >= State::Init) && !rccl_initialized && get_use_rcclp())
+    {
+        ROCPROFSYS_VERBOSE_F(1, "Setting up RCCLP...\n");
+        rcclp::setup();
+        rccl_initialized = true;
+    }
+}
+
+static void
+rocprofsys_init_library_hidden_with_rccl(bool postinit)
+{
+    rocprofsys_init_library_hidden();
+    rccl_setup(postinit);
+}
+
 //======================================================================================//
 
 extern "C" bool
-rocprofsys_init_tooling_hidden()
+rocprofsys_init_tooling_hidden(bool postinit)
 {
     if(get_env("ROCPROFSYS_MONOCHROME", false, false)) tim::log::monochrome() = true;
 
     if(!tim::get_env("ROCPROFSYS_INIT_TOOLING", true))
     {
-        rocprofsys_init_library_hidden();
+        rocprofsys_init_library_hidden_with_rccl(postinit);
         return false;
     }
 
@@ -422,7 +448,11 @@ rocprofsys_init_tooling_hidden()
     ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "State is %s...\n",
                                          std::to_string(get_state()).c_str());
 
-    if(get_state() != State::PreInit || get_state() == State::Init || _once) return false;
+    if(get_state() != State::PreInit || get_state() == State::Init || _once)
+    {
+        rccl_setup(postinit);
+        return false;
+    }
     _once = true;
 
     ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
@@ -443,7 +473,7 @@ rocprofsys_init_tooling_hidden()
     ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init,
                                          "Calling rocprofsys_init_library()...\n");
 
-    rocprofsys_init_library_hidden();
+    rocprofsys_init_library_hidden_with_rccl(postinit);
 
     ROCPROFSYS_DEBUG_F("\n");
 
@@ -541,12 +571,6 @@ rocprofsys_init_tooling_hidden()
         ompt::setup();
     }
 
-    if(get_use_rcclp())
-    {
-        ROCPROFSYS_VERBOSE_F(1, "Setting up RCCLP...\n");
-        rcclp::setup();
-    }
-
     if(get_use_perfetto())
     {
         ROCPROFSYS_VERBOSE_F(1, "Starting Perfetto...\n");