Sampling Tweaks: disable sampling itimer (#95)

* tweak starting sampling on main thread

- rename omnitrace::common::utility to omnitrace::common::path
- tracing::thread_init_sampling does not start sampling on main thread

* Cancel sampling itimers + timemory submodule update

- updates timemory submodule with support for cancelling itimer and SIGRTMIN through SIGRTMAX

[ROCm/rocprofiler-systems commit: 400e5078ed]

projects/rocprofiler-systems/source/lib/common/setup.hpp

@@ -55,7 +55,7 @@ namespace omnitrace
 {
 inline namespace common
 {
-namespace utility
+namespace path
 {
 inline std::string
 find_path(const std::string& _path, int _verbose, std::string _search_paths = {})
@@ -109,15 +109,15 @@ dirname(const std::string& _fname)
         return _fname.substr(0, _fname.find_last_of('/'));
     return std::string{};
 }
-}  // namespace utility
+}  // namespace path
 
-void
+inline void
 setup_environ(int _verbose, const std::string& _search_paths = {},
               std::string _omnilib    = "libomnitrace.so",
               std::string _omnilib_dl = "libomnitrace-dl.so")
 {
-    _omnilib    = utility::find_path(_omnilib, _verbose, _search_paths);
-    _omnilib_dl = utility::find_path(_omnilib_dl, _verbose, _search_paths);
+    _omnilib    = common::path::find_path(_omnilib, _verbose, _search_paths);
+    _omnilib_dl = common::path::find_path(_omnilib_dl, _verbose, _search_paths);
 
     // This environment variable forces the ROCR-Runtime to use polling to wait
     // for signals rather than interrupts. We set this variable to avoid issues with
@@ -161,7 +161,8 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
 #if defined(OMNITRACE_USE_OMPT) && OMNITRACE_USE_OMPT > 0
     std::string _omni_omp_libs = _omnilib_dl;
     const char* _omp_libs      = getenv("OMP_TOOL_LIBRARIES");
-    if(_omp_libs && std::string_view{ _omp_libs }.find(_omnilib_dl) == std::string::npos)
+    if(_omp_libs != nullptr &&
+       std::string_view{ _omp_libs }.find(_omnilib_dl) == std::string::npos)
         _omni_omp_libs = common::join(':', _omp_libs, _omnilib_dl);
     OMNITRACE_SETUP_LOG(_verbose >= 1, "setting OMP_TOOL_LIBRARIES to '%s'\n",
                         _omni_omp_libs.c_str());

projects/rocprofiler-systems/source/lib/omnitrace-dl/dl.cpp

@@ -140,9 +140,9 @@ struct OMNITRACE_HIDDEN_API indirect
 {
     OMNITRACE_INLINE indirect(const std::string& _omnilib, const std::string& _userlib,
                               const std::string& _dllib)
-    : m_omnilib{ utility::find_path(_omnilib, _omnitrace_dl_verbose) }
-    , m_dllib{ utility::find_path(_dllib, _omnitrace_dl_verbose) }
-    , m_userlib{ utility::find_path(_userlib, _omnitrace_dl_verbose) }
+    : m_omnilib{ common::path::find_path(_omnilib, _omnitrace_dl_verbose) }
+    , m_dllib{ common::path::find_path(_dllib, _omnitrace_dl_verbose) }
+    , m_userlib{ common::path::find_path(_userlib, _omnitrace_dl_verbose) }
     {
         if(_omnitrace_dl_verbose >= 1)
         {
@@ -154,8 +154,8 @@ struct OMNITRACE_HIDDEN_API indirect
                     ::basename(_userlib.c_str()), m_userlib.c_str());
         }
 
-        auto _search_paths =
-            common::join(':', utility::dirname(_omnilib), utility::dirname(_dllib));
+        auto _search_paths = common::join(':', common::path::dirname(_omnilib),
+                                          common::path::dirname(_dllib));
         common::setup_environ(_omnitrace_dl_verbose, _search_paths, _omnilib, _dllib);
 
         m_omnihandle = open(m_omnilib);

projects/rocprofiler-systems/source/lib/omnitrace/library.cpp

@@ -78,6 +78,8 @@ auto
 ensure_finalization(bool _static_init = false)
 {
     (void) threading::get_id();
+    (void) utility::get_thread_index();
+
     if(!_static_init)
     {
         OMNITRACE_DEBUG_F("\n");

projects/rocprofiler-systems/source/lib/omnitrace/library/components/backtrace.cpp

@@ -141,7 +141,8 @@ get_backtrace_init(int64_t _tid)
 unique_ptr_t<bool>&
 get_sampler_running(int64_t _tid)
 {
-    static auto& _v = sampler_running_instances::instances();
+    static auto& _v = sampler_running_instances::instances(
+        sampler_running_instances::construct_on_init{}, false);
     return _v.at(_tid);
 }
 }  // namespace
@@ -286,6 +287,8 @@ backtrace::sample(int signum)
             _timestamp_str(_now).c_str(), _diff, _tot);
     }
 
+    if(!*get_sampler_running(0)) return;
+
     m_size       = 0;
     m_tid        = threading::get_id();
     m_ts         = comp::wall_clock::record();
@@ -323,6 +326,7 @@ backtrace::configure(bool _setup, int64_t _tid)
     {
         (void) get_debug_sampling();  // make sure query in sampler does not allocate
         assert(_tid == threading::get_id());
+
         sampling::block_signals(*_signal_types);
         if constexpr(tim::trait::is_available<hw_counters>::value)
         {
@@ -336,7 +340,6 @@ backtrace::configure(bool _setup, int64_t _tid)
         auto _delay     = std::max<double>(1.0e-3, get_sampling_delay());
 
         OMNITRACE_DEBUG("Configuring sampler for thread %lu...\n", _tid);
-        sampler_running_instances::construct(true);
         sampling::sampler_instances::construct("omnitrace", _tid, *_signal_types);
         _sampler->set_signals(*_signal_types);
         _sampler->set_flags(SA_RESTART);
@@ -360,7 +363,7 @@ backtrace::configure(bool _setup, int64_t _tid)
                         _tid, _sampler->get_frequency(units::sec),
                         _sampler->get_rate(units::sec));
 
-        // (void) sampling::sampler_t::get_samplers(_tid);
+        *_running = true;
         backtrace_init_instances::construct();
         get_backtrace_init(_tid)->sample();
         _sampler->configure(false);
@@ -376,8 +379,23 @@ backtrace::configure(bool _setup, int64_t _tid)
             sampling::block_signals(*_signal_types);
         }
 
-        // this propagates to all threads
-        if(_tid == 0) _sampler->ignore(*_signal_types);
+        // remove the timer delivering the signal
+        _sampler->reset(false, *_signal_types);
+
+        if(_tid == 0)
+        {
+            // this propagates to all threads
+            _sampler->ignore(*_signal_types);
+            for(int64_t i = 1; i < OMNITRACE_MAX_THREADS; ++i)
+            {
+                if(sampling::get_sampler(i))
+                {
+                    sampling::get_sampler(i)->reset(false,
+                                                    *sampling::get_signal_types(i));
+                    *get_sampler_running(i) = false;
+                }
+            }
+        }
 
         _sampler->stop();
         _sampler->swap_data();
@@ -389,6 +407,7 @@ backtrace::configure(bool _setup, int64_t _tid)
                 OMNITRACE_DEBUG("HW COUNTER: stopped...\n");
             }
         }
+        OMNITRACE_DEBUG("Sampler destroyed for thread %lu\n", _tid);
     }
 
     return (_signal_types) ? *_signal_types : std::set<int>{};

projects/rocprofiler-systems/source/lib/omnitrace/library/tracing.hpp

@@ -29,6 +29,7 @@
 #include "library/runtime.hpp"
 #include "library/sampling.hpp"
 #include "library/timemory.hpp"
+#include "library/utility.hpp"
 
 namespace omnitrace
 {
@@ -121,9 +122,15 @@ inline void
 thread_init_sampling()
 {
     static thread_local auto _v = []() {
-        auto _use_sampling = get_use_sampling();
-        if(_use_sampling) sampling::setup();
-        return _use_sampling;
+        auto _idx = utility::get_thread_index();
+        // the main thread will initialize sampling when it initializes the tooling
+        if(_idx > 0)
+        {
+            auto _use_sampling = get_use_sampling();
+            if(_use_sampling) sampling::setup();
+            return _use_sampling;
+        }
+        return false;
     }();
     (void) _v;
 }