Support for tracing mutex locking (#52)

* Parallel overhead example with locks

* Support tracing mutex locking + more

- support wrapping pthread_mutex_lock
- support wrapping pthread_mutex_unlock
- support wrapping pthread_mutex_trylock
- get_perfetto_combined_traces setting
- OMNITRACE_TRACE_THREAD_LOCKS option
- ThreadState
- critical trace includes queue id
- enabled/disabled settings in timemory
- fix OMNITRACE_TIMEMORY_COMPONENTS
- fix reading config
- fix setting categories
- applied ThreadState::Internal in various places
- utility::get_filled_array
- utility::get_reserved_vector
- utility::get_thread_index
- fork_gotcha messages about forks
- split out some pthread_gotcha functionality into pthread_create_gotcha
- handle queue id in roctracer callbacks

* Update timemory and PTL submodules

* Misc CMake updates

- Includes fix to omnitrace-static-lib{gcc,stdcxx}

* Misc cleanup to pthread_mutex_gotcha and backtrace

* Fix to duplicate field in module_function json

* Improvement to debug messages

* omnitrace-dl and common improvements

- tweak to delimit
- common::ignore message
- common::join quoting of strings
- omnitrace_set_env ignores if inited and active
- omnitrace_set_mpi ignores if inited and active

* nsync for transpose example

* Fix to thread_deleter<void> functor invoke

* Fix thread state and HIP stream enums

[ROCm/rocprofiler-systems commit: b208047741]

projects/rocprofiler-systems/source/lib/omnitrace-dl/dl.cpp

@@ -122,9 +122,9 @@ const char* _omnitrace_dl_dlopen_descr = "RTLD_LAZY | RTLD_LOCAL";
 /// This class contains function pointers for omnitrace's instrumentation functions
 struct OMNITRACE_HIDDEN_API indirect
 {
-    OMNITRACE_INLINE indirect(std::string omnilib, std::string userlib)
-    : m_omnilib{ find_path(std::move(omnilib)) }
-    , m_userlib{ find_path(std::move(userlib)) }
+    OMNITRACE_INLINE indirect(const std::string& omnilib, const std::string& userlib)
+    : m_omnilib{ find_path(omnilib) }
+    , m_userlib{ find_path(userlib) }
     {
         if(_omnitrace_dl_verbose >= 1)
         {
@@ -218,19 +218,20 @@ struct OMNITRACE_HIDDEN_API indirect
         }
     }
 
-    static OMNITRACE_INLINE std::string find_path(std::string&& _path)
+    static OMNITRACE_INLINE std::string find_path(const std::string& _path)
     {
-        auto _paths =
-            delimit(join(":", get_env("OMNITRACE_PATH", ""),
-                         get_env("LD_LIBRARY_PATH", ""), get_env("LIBRARY_PATH", "")),
-                    ":");
+        auto _paths_search =
+            join(":", get_env("OMNITRACE_PATH", ""), get_env("LD_LIBRARY_PATH", ""),
+                 get_env("LIBRARY_PATH", ""));
+
+        auto _paths = delimit(_paths_search, ":");
 
         auto file_exists = [](const std::string& name) {
             struct stat buffer;
             return (stat(name.c_str(), &buffer) == 0);
         };
 
-        for(auto&& itr : _paths)
+        for(const auto& itr : _paths)
         {
             auto _f = join('/', itr, _path);
             if(file_exists(_f)) return _f;
@@ -269,10 +270,10 @@ get_indirect() OMNITRACE_HIDDEN_API;
 indirect&
 get_indirect()
 {
-    static auto _v =
-        indirect{ get_env("OMNITRACE_LIBRARY", "libomnitrace.so"),
-                  get_env("OMNITRACE_USER_LIBRARY", "libomnitrace-user.so") };
-    return _v;
+    static auto  _libomni = get_env("OMNITRACE_LIBRARY", "libomnitrace.so");
+    static auto  _libuser = get_env("OMNITRACE_USER_LIBRARY", "libomnitrace-user.so");
+    static auto* _v       = new indirect{ _libomni, _libuser };
+    return *_v;
 }
 
 auto&
@@ -340,6 +341,10 @@ bool _omnitrace_dl_fini = (std::atexit([]() {
                                 (::omnitrace::dl::get_thread_status() = false),          \
                                 __VA_ARGS__)
 
+#define OMNITRACE_DL_IGNORE(...)                                                         \
+    ::omnitrace::common::ignore(__FUNCTION__, ::omnitrace::dl::_omnitrace_dl_verbose,    \
+                                __VA_ARGS__)
+
 #define OMNITRACE_DL_INVOKE_STATUS(STATUS, ...)                                          \
     ::omnitrace::common::invoke(__FUNCTION__, ::omnitrace::dl::_omnitrace_dl_verbose,    \
                                 STATUS, __VA_ARGS__)
@@ -464,18 +469,30 @@ extern "C"
 
     void omnitrace_set_env(const char* a, const char* b)
     {
+        if(dl::get_inited() && dl::get_active())
+        {
+            OMNITRACE_DL_IGNORE(2, "already initialized and active", a, b);
+            return;
+        }
         setenv(a, b, 0);
         OMNITRACE_DL_INVOKE(get_indirect().omnitrace_set_env_f, a, b);
     }
 
     void omnitrace_set_mpi(bool a, bool b)
     {
+        if(dl::get_inited() && dl::get_active())
+        {
+            OMNITRACE_DL_IGNORE(2, "already initialized and active", a, b);
+            return;
+        }
         OMNITRACE_DL_INVOKE(get_indirect().omnitrace_set_mpi_f, a, b);
     }
 
     void omnitrace_register_source(const char* file, const char* func, size_t line,
                                    size_t address, const char* source)
     {
+        OMNITRACE_DL_LOG(3, "%s(\"%s\", \"%s\", %zu, %zu, \"%s\")\n", __FUNCTION__, file,
+                         func, line, address, source);
         OMNITRACE_DL_INVOKE(get_indirect().omnitrace_register_source_f, file, func, line,
                             address, source);
     }