Misc updates (#48)

- reworked `add_critical_trace`
- `get_use_thread_sampling` / `"OMNITRACE_USE_THREAD_SAMPLING"` option
- `get_cpu_cid_stack_lock`
- reworked finalization messaging
- significant updates to pthread_gotcha
  - shutdown stability
  - `"start_thread"` entries
- `rocm_smi` stability 
- roctracer_callbacks add critical trace entries on the callback thread
- reworked CPU CID initialization
- thread_sampler stability

[ROCm/rocprofiler-systems commit: 9b25d4b3b5]

projects/rocprofiler-systems/source/lib/omnitrace/include/library.hpp

@@ -52,9 +52,9 @@ namespace omnitrace
 template <critical_trace::Device DevID, critical_trace::Phase PhaseID,
           bool UpdateStack = true>
 inline void
-add_critical_trace(int64_t _tid, size_t _cpu_cid, size_t _gpu_cid, size_t _parent_cid,
-                   int64_t _ts_beg, int64_t _ts_val, size_t _hash, uint16_t _depth,
-                   uint16_t _prio = 0)
+add_critical_trace(int64_t _targ_tid, size_t _cpu_cid, size_t _gpu_cid,
+                   size_t _parent_cid, int64_t _ts_beg, int64_t _ts_val, size_t _hash,
+                   uint16_t _depth, uint16_t _prio = 0)
 {
     // clang-format off
     // these are used to create unique type mutexes
@@ -67,43 +67,58 @@ add_critical_trace(int64_t _tid, size_t _cpu_cid, size_t _gpu_cid, size_t _paren
     static constexpr auto num_mutexes  = max_supported_threads;
     static auto           _update_freq = critical_trace::get_update_frequency();
 
+    auto _self_tid = threading::get_id();
+
     if constexpr(PhaseID != critical_trace::Phase::NONE)
     {
-        // unique lock per thread
-        auto&       _mtx = type_mutex<critical_insert, api::omnitrace, num_mutexes>(_tid);
-        auto_lock_t _lk{ _mtx };
+        auto& _self_mtx =
+            type_mutex<critical_insert, api::omnitrace, num_mutexes>(_self_tid);
 
-        auto& _critical_trace = critical_trace::get(_tid);
+        auto_lock_t _self_lk{ _self_mtx, std::defer_lock };
+
+        // unique lock per thread
+        if(!_self_lk.owns_lock()) _self_lk.lock();
+
+        auto& _critical_trace = critical_trace::get(_self_tid);
         _critical_trace->emplace_back(
-            critical_trace::entry{ _prio, DevID, PhaseID, _depth, _tid, _cpu_cid,
+            critical_trace::entry{ _prio, DevID, PhaseID, _depth, _targ_tid, _cpu_cid,
                                    _gpu_cid, _parent_cid, _ts_beg, _ts_val, _hash });
     }
 
     if constexpr(UpdateStack)
     {
+        auto& _self_mtx = get_cpu_cid_stack_lock(_self_tid);
+        auto& _targ_mtx = get_cpu_cid_stack_lock(_targ_tid);
+
+        auto_lock_t _self_lk{ _self_mtx, std::defer_lock };
+        auto_lock_t _targ_lk{ _targ_mtx, std::defer_lock };
+
         // unique lock per thread
-        auto& _mtx = type_mutex<cpu_cid_stack, api::omnitrace, num_mutexes>(_tid);
+        auto _lock = [&_self_lk, &_targ_lk, _self_tid, _targ_tid]() {
+            if(!_self_lk.owns_lock() && _self_tid != _targ_tid) _self_lk.lock();
+            if(!_targ_lk.owns_lock()) _targ_lk.lock();
+        };
 
         if constexpr(PhaseID == critical_trace::Phase::NONE)
         {
-            auto_lock_t _lk{ _mtx };
-            get_cpu_cid_stack(_tid)->emplace_back(_cpu_cid);
+            _lock();
+            get_cpu_cid_stack(_targ_tid)->emplace_back(_cpu_cid);
         }
         else if constexpr(PhaseID == critical_trace::Phase::BEGIN)
         {
-            auto_lock_t _lk{ _mtx };
-            get_cpu_cid_stack(_tid)->emplace_back(_cpu_cid);
+            _lock();
+            get_cpu_cid_stack(_targ_tid)->emplace_back(_cpu_cid);
         }
         else if constexpr(PhaseID == critical_trace::Phase::END)
         {
-            auto_lock_t _lk{ _mtx };
-            get_cpu_cid_stack(_tid)->pop_back();
+            _lock();
+            get_cpu_cid_stack(_targ_tid)->pop_back();
             if(_gpu_cid == 0 && _cpu_cid % _update_freq == (_update_freq - 1))
-                critical_trace::update(_tid);
+                critical_trace::update(_targ_tid);
         }
     }
 
-    tim::consume_parameters(_tid, _cpu_cid, _gpu_cid, _parent_cid, _ts_beg, _ts_val,
+    tim::consume_parameters(_targ_tid, _cpu_cid, _gpu_cid, _parent_cid, _ts_beg, _ts_val,
                             _hash, _depth, _prio);
 }
 }  // namespace omnitrace