offset thread ids where possible (#130)

- configure known background threads to start indexing down from TIMEMORY_MAX_THREADS
- invoke sampling::shutdown() instead of just blocking signals where possible

[ROCm/rocprofiler-systems commit: 1343c6722a]

projects/rocprofiler-systems/source/lib/omnitrace/library/components/roctracer.cpp

@@ -122,6 +122,7 @@ roctracer::setup()
     roctracer_is_setup() = true;
 
     OMNITRACE_VERBOSE_F(1, "setting up roctracer...\n");
+    pthread_gotcha::push_enable_sampling_on_child_threads(false);
 
     dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64",
                                find_library_path("libamdhip64.so",
@@ -169,6 +170,8 @@ roctracer::setup()
     for(auto& itr : roctracer_setup_routines())
         itr.second();
 
+    pthread_gotcha::pop_enable_sampling_on_child_threads();
+
     OMNITRACE_VERBOSE_F(1, "roctracer is setup\n");
 }
 

projects/rocprofiler-systems/source/lib/omnitrace/library/process_sampler.cpp

@@ -74,10 +74,11 @@ get_sampler_is_sampling()
 void
 sampler::poll(std::atomic<State>* _state, nsec_t _interval, promise_t* _ready)
 {
-    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
-
+    threading::offset_this_id(true);
     threading::set_thread_name("omni.sampler");
 
+    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
+
     // notify thread started
     if(_ready) _ready->set_value();
 

projects/rocprofiler-systems/source/lib/omnitrace/library/ptl.cpp

@@ -29,6 +29,7 @@
 
 #include <PTL/ThreadPool.hh>
 
+#include <timemory/backends/threading.hpp>
 #include <timemory/utility/declaration.hpp>
 
 namespace omnitrace
@@ -44,6 +45,7 @@ auto _thread_pool_cfg = []() {
     _v.use_tbb      = false;
     _v.verbose      = -1;
     _v.initializer  = []() {
+        threading::offset_this_id(true);
         set_thread_state(ThreadState::Internal);
         sampling::block_signals();
         threading::set_thread_name(

projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.cpp

@@ -232,10 +232,12 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
 
     static thread_local std::once_flag _once{};
     std::call_once(_once, []() {
+        threading::offset_this_id(true);
         if(threading::get_id() != 0)
         {
             sampling::block_signals();
             threading::set_thread_name("roctracer.hsa");
+            sampling::shutdown();
         }
     });
 
@@ -354,8 +356,13 @@ hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
 
     static thread_local std::once_flag _once{};
     std::call_once(_once, []() {
-        sampling::block_signals();
-        threading::set_thread_name("roctracer.hsa");
+        threading::offset_this_id(true);
+        if(threading::get_id() != 0)
+        {
+            sampling::block_signals();
+            threading::set_thread_name("roctracer.hsa");
+            sampling::shutdown();
+        }
     });
 
     auto&& _protect = comp::roctracer::protect_flush_activity();
@@ -795,8 +802,13 @@ hip_activity_callback(const char* begin, const char* end, void*)
 
     static thread_local std::once_flag _once{};
     std::call_once(_once, []() {
-        sampling::block_signals();
-        threading::set_thread_name("roctracer.hip");
+        threading::offset_this_id(true);
+        if(threading::get_id() != 0)
+        {
+            sampling::block_signals();
+            threading::set_thread_name("roctracer.hip");
+            sampling::shutdown();
+        }
     });
 
     auto&& _protect = comp::roctracer::protect_flush_activity();