Support for tracing mutex locking (#52)

* Parallel overhead example with locks

* Support tracing mutex locking + more

- support wrapping pthread_mutex_lock
- support wrapping pthread_mutex_unlock
- support wrapping pthread_mutex_trylock
- get_perfetto_combined_traces setting
- OMNITRACE_TRACE_THREAD_LOCKS option
- ThreadState
- critical trace includes queue id
- enabled/disabled settings in timemory
- fix OMNITRACE_TIMEMORY_COMPONENTS
- fix reading config
- fix setting categories
- applied ThreadState::Internal in various places
- utility::get_filled_array
- utility::get_reserved_vector
- utility::get_thread_index
- fork_gotcha messages about forks
- split out some pthread_gotcha functionality into pthread_create_gotcha
- handle queue id in roctracer callbacks

* Update timemory and PTL submodules

* Misc CMake updates

- Includes fix to omnitrace-static-lib{gcc,stdcxx}

* Misc cleanup to pthread_mutex_gotcha and backtrace

* Fix to duplicate field in module_function json

* Improvement to debug messages

* omnitrace-dl and common improvements

- tweak to delimit
- common::ignore message
- common::join quoting of strings
- omnitrace_set_env ignores if inited and active
- omnitrace_set_mpi ignores if inited and active

* nsync for transpose example

* Fix to thread_deleter<void> functor invoke

* Fix thread state and HIP stream enums

tests/CMakeLists.txt

@@ -34,6 +34,18 @@ set(_base_environment
     "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
     )
 
+set(_lock_environment
+    "OMNITRACE_USE_SAMPLING=OFF"
+    "OMNITRACE_CRITICAL_TRACE=ON"
+    "OMNITRACE_COLLAPSE_THREADS=ON"
+    "OMNITRACE_TRACE_THREAD_LOCKS=ON"
+    "OMNITRACE_COUT_OUTPUT=ON"
+    "OMNITRACE_TIME_OUTPUT=OFF"
+    "OMNITRACE_FLAT_PROFILE=ON"
+    "OMNITRACE_TIMELINE_PROFILE=OFF"
+    "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
+    )
+
 set(_ompt_environment
     "OMNITRACE_USE_PERFETTO=ON"
     "OMNITRACE_USE_TIMEMORY=ON"
@@ -144,7 +156,7 @@ function(OMNITRACE_ADD_TEST)
             list(APPEND TEST_ENVIRONMENT "OMNITRACE_USE_PID=OFF")
         endif()
 
-        if(NOT SKIP_BASELINE)
+        if(NOT TEST_SKIP_BASELINE)
             add_test(
                 NAME ${TEST_NAME}-baseline
                 COMMAND ${COMMAND_PREFIX} $<TARGET_FILE:${TEST_TARGET}> ${TEST_RUN_ARGS}
@@ -389,7 +401,7 @@ omnitrace_add_test(
     TARGET transpose
     MPI ${TRANSPOSE_USE_MPI}
     NUM_PROCS ${NUM_PROCS}
-    REWRITE_ARGS -e -v 2
+    REWRITE_ARGS -e -v 2 --print-instructions
     RUNTIME_ARGS -e -v 1 --label file line return args
     ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=ON")
 
@@ -410,6 +422,30 @@ omnitrace_add_test(
     RUN_ARGS 10 ${NUM_THREADS} 1000
     ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF")
 
+omnitrace_add_test(
+    SKIP_RUNTIME SKIP_SAMPLING
+    NAME parallel-overhead-locks-timemory
+    TARGET parallel-overhead-locks
+    LABELS "locks"
+    REWRITE_ARGS -e -v 2 --min-instructions=4
+    RUN_ARGS 10 4 1000
+    ENVIRONMENT
+        "${_lock_environment};OMNITRACE_USE_TIMEMORY=ON;OMNITRACE_USE_PERFETTO=OFF"
+    REWRITE_RUN_PASS_REGEX
+        "start_thread (.*) 4 (.*) pthread_mutex_lock (.*) 4000 (.*) pthread_mutex_unlock (.*) 4000"
+    )
+
+omnitrace_add_test(
+    SKIP_BASELINE SKIP_RUNTIME SKIP_SAMPLING
+    NAME parallel-overhead-locks-perfetto
+    TARGET parallel-overhead-locks
+    LABELS "locks"
+    REWRITE_ARGS -e -v 2 --min-instructions=8
+    RUN_ARGS 10 4 1000
+    ENVIRONMENT
+        "${_lock_environment};OMNITRACE_USE_TIMEMORY=OFF;OMNITRACE_USE_PERFETTO=ON"
+    PROPERTIES WILL_FAIL ON)
+
 omnitrace_add_test(
     NAME user-api
     TARGET user-api