From ee5fe1c1bdc9a7ffafed5c7a22c7fa9a60ad752d Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Mon, 8 Aug 2022 08:37:37 -0500 Subject: [PATCH] offset thread ids where possible (#130) - configure known background threads to start indexing down from TIMEMORY_MAX_THREADS - invoke sampling::shutdown() instead of just blocking signals where possible [ROCm/rocprofiler-systems commit: 1343c6722a571366b28b60548703a51114353bc0] --- .../rocprofiler-systems/external/timemory | 2 +- .../library/components/roctracer.cpp | 3 +++ .../lib/omnitrace/library/process_sampler.cpp | 5 +++-- .../source/lib/omnitrace/library/ptl.cpp | 2 ++ .../lib/omnitrace/library/roctracer.cpp | 20 +++++++++++++++---- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/projects/rocprofiler-systems/external/timemory b/projects/rocprofiler-systems/external/timemory index 1f4e7602a4..a3283389f6 160000 --- a/projects/rocprofiler-systems/external/timemory +++ b/projects/rocprofiler-systems/external/timemory @@ -1 +1 @@ -Subproject commit 1f4e7602a404ffb18998c5a1c0268de989df2d92 +Subproject commit a3283389f6d8c8d975db250401ca7fbb29043649 diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/components/roctracer.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/components/roctracer.cpp index cbf385a374..9b49cadc7d 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/components/roctracer.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/components/roctracer.cpp @@ -122,6 +122,7 @@ roctracer::setup() roctracer_is_setup() = true; OMNITRACE_VERBOSE_F(1, "setting up roctracer...\n"); + pthread_gotcha::push_enable_sampling_on_child_threads(false); dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64", find_library_path("libamdhip64.so", @@ -169,6 +170,8 @@ roctracer::setup() for(auto& itr : roctracer_setup_routines()) itr.second(); + pthread_gotcha::pop_enable_sampling_on_child_threads(); + OMNITRACE_VERBOSE_F(1, "roctracer is setup\n"); } diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/process_sampler.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/process_sampler.cpp index b1af4d622b..b042e757ef 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/process_sampler.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/process_sampler.cpp @@ -74,10 +74,11 @@ get_sampler_is_sampling() void sampler::poll(std::atomic* _state, nsec_t _interval, promise_t* _ready) { - OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal); - + threading::offset_this_id(true); threading::set_thread_name("omni.sampler"); + OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal); + // notify thread started if(_ready) _ready->set_value(); diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/ptl.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/ptl.cpp index 551d3284a3..ac2e0525f1 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/ptl.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/ptl.cpp @@ -29,6 +29,7 @@ #include +#include #include namespace omnitrace @@ -44,6 +45,7 @@ auto _thread_pool_cfg = []() { _v.use_tbb = false; _v.verbose = -1; _v.initializer = []() { + threading::offset_this_id(true); set_thread_state(ThreadState::Internal); sampling::block_signals(); threading::set_thread_name( diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.cpp index c90ca75b4d..3ac752697b 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.cpp @@ -232,10 +232,12 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* static thread_local std::once_flag _once{}; std::call_once(_once, []() { + threading::offset_this_id(true); if(threading::get_id() != 0) { sampling::block_signals(); threading::set_thread_name("roctracer.hsa"); + sampling::shutdown(); } }); @@ -354,8 +356,13 @@ hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg) static thread_local std::once_flag _once{}; std::call_once(_once, []() { - sampling::block_signals(); - threading::set_thread_name("roctracer.hsa"); + threading::offset_this_id(true); + if(threading::get_id() != 0) + { + sampling::block_signals(); + threading::set_thread_name("roctracer.hsa"); + sampling::shutdown(); + } }); auto&& _protect = comp::roctracer::protect_flush_activity(); @@ -795,8 +802,13 @@ hip_activity_callback(const char* begin, const char* end, void*) static thread_local std::once_flag _once{}; std::call_once(_once, []() { - sampling::block_signals(); - threading::set_thread_name("roctracer.hip"); + threading::offset_this_id(true); + if(threading::get_id() != 0) + { + sampling::block_signals(); + threading::set_thread_name("roctracer.hip"); + sampling::shutdown(); + } }); auto&& _protect = comp::roctracer::protect_flush_activity();