AI/rocm-systems
2
0
Fork 0
Koodi Ongelmat Pull-pyynnöt Actions Paketit Projektit Julkaisut Wiki Toiminta

Support sampling duration, sampling TIDs (#142)

Selaa lähdekoodia 
- Sampling duration config values
  - OMNITRACE_SAMPLING_DURATION
  - OMNITRACE_PROCESS_SAMPLING_DURATION
  - Disables sampling after this time (in seconds) has elapsed 
- Sampling thread-id config values
  - OMNITRACE_SAMPLING_TIDS
  - OMNITRACE_SAMPLING_CPUTIME_TIDS
  - OMNITRACE_SAMPLING_REALTIME_TIDS
  - Allows user to select certain threads for sampling
- Miscellaneous
  - Tweaked the finalization verbosity messages
  - moved sampling-on-child-threads into runtime.hpp and runtime.cpp
  - fixed submodule dyninst header install

[ROCm/rocprofiler-systems commit: e67afd33eb]
This commit is contained in:
Jonathan R. Madsen
2022-08-31 06:29:19 -05:00
committed by GitHub
vanhempi cbdc7cad4b
commit 2ef9dfd002
17 muutettua tiedostoa jossa 390 lisäystä ja 155 poistoa
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocprofiler-systems/source/lib/omnitrace/library/sampling.cpp
+114 -23
Näytä tiedosto
@@ -26,7 +26,6 @@
#include "library/components/backtrace_metrics.hpp"
#include "library/components/backtrace_timestamp.hpp"
#include "library/components/fwd.hpp"
#include "library/components/pthread_gotcha.hpp"
#include "library/config.hpp"
#include "library/debug.hpp"
#include "library/ptl.hpp"
@@ -60,6 +59,8 @@
#include <timemory/variadic.hpp>
#include <array>
#include <chrono>
#include <condition_variable>
#include <cstring>
#include <ctime>
#include <initializer_list>
@@ -161,6 +162,79 @@ get_sampler_running(int64_t _tid)
return _v.at(_tid);
}
auto&
get_duration_cv()
{
static auto _v = std::condition_variable{};
return _v;
}
auto&
get_duration_thread()
{
static auto _v = std::unique_ptr<std::thread>{};
return _v;
}
void
start_duration_thread()
{
static std::mutex _start_mutex{};
std::unique_lock<std::mutex> _start_lk{ _start_mutex, std::defer_lock };
if(!_start_lk.owns_lock()) _start_lk.lock();
if(!get_duration_thread() && config::get_sampling_duration() > 0.0)
{
// we may need to protect against recursion bc of pthread wrapper
static bool _protect = false;
if(_protect) return;
_protect = true;
auto _now = std::chrono::steady_clock::now();
auto _end = _now + std::chrono::nanoseconds{ static_cast<uint64_t>(
config::get_sampling_duration() * units::sec) };
auto _func = [_end]() {
thread_info::init(true);
std::mutex _mutex{};
bool _wait = true;
while(_wait)
{
_wait = false;
std::unique_lock<std::mutex> _lk{ _mutex };
get_duration_cv().wait_until(_lk, _end);
auto _premature = (std::chrono::steady_clock::now() < _end);
auto _finalized = (get_state() == State::Finalized);
if(_premature && !_finalized)
{
// protect against spurious wakeups
OMNITRACE_VERBOSE(
2, "%sSpurious wakeup of sampling duration thread...\n",
tim::log::color::warning());
_wait = true;
}
else if(_finalized)
{
break;
}
else
{
OMNITRACE_VERBOSE(1,
"Sampling duration of %f seconds has elapsed. "
"Shutting down sampling...\n",
config::get_sampling_duration());
shutdown();
}
}
};
OMNITRACE_VERBOSE(1, "Sampling will be disabled after %f seconds...\n",
config::get_sampling_duration());
OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
get_duration_thread() = std::make_unique<std::thread>(_func);
_protect = false;
}
}
std::set<int>
configure(bool _setup, int64_t _tid = threading::get_id())
{
@@ -170,10 +244,24 @@ configure(bool _setup, int64_t _tid = threading::get_id())
bool _is_running = (!_running) ? false : *_running;
auto& _signal_types = sampling::get_signal_types(_tid);
pthread_gotcha::push_enable_sampling_on_child_threads(false);
auto _dtor = scope::destructor{ []() {
pthread_gotcha::pop_enable_sampling_on_child_threads();
} };
OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
auto&& _cpu_tids = get_sampling_cpu_tids();
auto&& _real_tids = get_sampling_real_tids();
auto _erase_tid_signal = [_tid, &_signal_types](auto& _tids, int _signum) {
if(!_tids.empty())
{
if(_tids.count(_tid) == 0)
{
OMNITRACE_VERBOSE(3, "Disabling SIG%i from thread %li\n", _signum, _tid);
_signal_types->erase(_signum);
}
}
};
_erase_tid_signal(_cpu_tids, get_cputime_signal());
_erase_tid_signal(_real_tids, get_realtime_signal());
if(_setup && !_sampler && !_is_running && !_signal_types->empty())
{
@@ -253,6 +341,7 @@ configure(bool _setup, int64_t _tid = threading::get_id())
*_running = true;
sampling::get_sampler_init(_tid)->sample();
start_duration_thread();
_sampler->start();
}
else if(!_setup && _sampler && _is_running)
@@ -265,6 +354,7 @@ configure(bool _setup, int64_t _tid = threading::get_id())
sampling::block_signals(*_signal_types);
}
get_duration_cv().notify_one();
if(_tid == 0)
{
// this propagates to all threads
@@ -278,6 +368,12 @@ configure(bool _setup, int64_t _tid = threading::get_id())
*get_sampler_running(i) = false;
}
}
if(get_duration_thread())
{
get_duration_thread()->join();
get_duration_thread().reset();
}
}
_sampler->stop();
@@ -363,8 +459,8 @@ post_process()
for(size_t i = 0; i < max_supported_threads; ++i)
backtrace_metrics::configure(false, i);
OMNITRACE_VERBOSE(1 || get_debug_sampling(), "Post-processing sampling data...\n");
size_t _total_data = 0;
size_t _total_threads = 0;
for(size_t i = 0; i < max_supported_threads; ++i)
{
auto& _sampler = get_sampler(i);
@@ -398,7 +494,7 @@ post_process()
_sampler->stop();
auto& _raw_data = _sampler->get_data();
OMNITRACE_VERBOSE(0 || get_debug_sampling(),
OMNITRACE_VERBOSE(2 || get_debug_sampling(),
"Sampler data for thread %lu has %zu initial entries...\n", i,
_raw_data.size());
@@ -430,23 +526,27 @@ post_process()
continue;
}
OMNITRACE_VERBOSE(0 || get_debug_sampling(),
OMNITRACE_VERBOSE(2 || get_debug_sampling(),
"Sampler data for thread %lu has %zu valid entries...\n", i,
_raw_data.size());
_total_data += _raw_data.size();
_total_threads += 1;
if(get_use_perfetto()) post_process_perfetto(i, _init, _data);
if(get_use_timemory()) post_process_timemory(i, _init, _data);
}
OMNITRACE_VERBOSE(0 || get_debug_sampling(),
"Post-processing sampling entries completed\n");
OMNITRACE_VERBOSE(3 || get_debug_sampling(), "Destroying samplers...\n");
for(size_t i = 0; i < max_supported_threads; ++i)
{
get_sampler(i).reset();
}
OMNITRACE_VERBOSE(0 || get_debug_sampling(), "Post-processing samplers destroyed\n");
OMNITRACE_VERBOSE(1 || get_debug_sampling(),
"Collected %zu samples from %zu threads...\n", _total_data,
_total_threads);
}
namespace
@@ -535,17 +635,8 @@ post_process_perfetto(int64_t _tid, const bundle_t* _init,
}
};
if(_tid == 0 && config::get_mode() == Mode::Sampling &&
config::get_perfetto_fill_policy() == "discard")
{
_process_perfetto(_data);
}
else
{
pthread_gotcha::push_enable_sampling_on_child_threads(false);
std::thread{ _process_perfetto_wrapper }.join();
pthread_gotcha::pop_enable_sampling_on_child_threads();
}
OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
std::thread{ _process_perfetto_wrapper }.join();
}
void