Rework sampling and colorized logs (#140)
## Overview
This is a significant PR which has 3 very notable characteristics:
1. Omnitrace colorizes most of it's logging
2. Completely reworked the sampling
- Samples now record the current instruction pointers instead of strings
- This _dramatically_ decreases the overhead of taking a sample
- The collection of metrics during a sample are split out into another component, enabling that data collection to be disabled -- which decreases the sampling overhead even further
- When both `OMNITRACE_SAMPLING_CPUTIME` and `OMNITRACE_SAMPLING_REALTIME` are ON:
- `OMNITRACE_SAMPLING_CPUTIME_FREQ` and `OMNITRACE_SAMPLING_REALTIME_FREQ` can be used to individually control the sampling frequency
- `OMNITRACE_SAMPLING_CPUTIME_DELAY` and `OMNITRACE_SAMPLING_REALTIME_DELAY` can be used to individually control the delay time before starting
- Now, omnitrace does not start a real-time sampler on the main thread unless `OMNITRACE_SAMPLING_REALTIME` is ON
- In the future, an `OMNITRACE_SAMPLING_TIDS` (and real-time, cpu-time variants) configuration variable(s) will allow you to select which threads will be sampled
3. Files produced by `omnitrace` exe -- `available-instr.txt`, `instrumented-instr.txt`, etc. -- now no longer has `-instr` suffix and are placed in `instrumentation/` subfolder, i.e. `available-instr.txt` -> instrumentation/available.txt`
- This helped de-clutter the output folder
Most of the other edits were reorganization (e.g. internal namespace changes), cleanup, and splitting up functionality.
## Bug Fixes
There is a bug fix with respect to the HSA callbacks which disabled sampling on child threads when an HSA API call was made
## Details
- created thread_info struct for mapping different thread IDs
- reorganized file structure significantly
- added categories.hpp, concepts.hpp
- moved around name trait definitions
- moved all omnitrace components into `omnitrace::component` namespace
- there was a lot of inconsistency b/t using `tim::component` in some places and `omnitrace::component`
- added macros like OMNITRACE_DECLARE_COMPONENT in lieu of TIMEMORY_DECLARE_COMPONENT
- OMNITRACE_CRITICAL_TRACE_NUM_THREADS -> OMNITRACE_THREAD_POOL_SIZE
- roctracer and critical_trace use same thread pool
- critical_trace functions do not lock anymore bc of thread-local TaskGroup
- added `component::local_category_region` to support using `component::category_region` without explicitly passing in name
- removed `component::omnitrace` (unused)
- migrated KokkosP and OMPT to use `component::local_category_region`
- removed `component::user_region` as a result
- migrated omnitrace_{push,pop}_{trace,region}_hidden to use component::category_region
- removed `component::functors` as a result
- migrated some ppdefs
- `api::omnitrace` -> `project::omnitrace`
- `api::(...)` -> `category::(...)`
- improved recording the execution time of threads
- migrated this functionality out of pthread_create_gotcha and into thread_info
- moved mpi_gotcha, fork_gotcha, exit_gotcha, rcclp into omnitrace::component namespace
- split backtrace up into backtrace, backtrace_metrics, backtrace_timestamp components
- sampling.cpp handles setup and post-processing that was formerly in backtrace
- updated logging to use colors
- `OMNITRACE_COLORIZED_LOG` config variable
- updated docs on JSON output from timemory
- instrumentation info in instrumentation subfolder
- added testing for KokkosP entries
- added testing for ompt entries
- add_critical_trace function defined in critical_trace.hpp
- disable push_thread_state and pop_thread_state when thread state is Disabled or Completed
- add comp::page_rss to main bundle
- thread_data supports std::optional instead of std::unique_ptr
- thread_data supports tim::identity<T> to avoid unique_ptr or optional
- tracing::record_thread_start_time()
- tracing::push_timemory and tracing::pop_timemory are templated on CategoryT
- removed anonymous namespace from omnitrace::utility
- sampling backtrace stores instruction pointers instead of strings
- component::category_region updates
- handle disabled thread state
- handle finalized state
- fewer debug messages
- invoke thread_init()
- invoke thread_init_sampling()
- handle push/pop count based on category
- push/pop count only modified when used
- component::cpu_freq
- components/ensure_storage.hpp
- reworked the pthread_create replacement function
- updated parallel-overhead example to report # of times locked
- OMNITRACE_MAX_UNWIND_DEPTH build option
- update timemory submodule
Этот коммит содержится в:
Jonathan R. Madsen
коммит произвёл
GitHub
GitHub
родитель
a1dcd1bc4b
Коммит
808ea7dfa7
@@ -22,16 +22,19 @@
|
||||
|
||||
#include "library/components/pthread_create_gotcha.hpp"
|
||||
#include "library/components/category_region.hpp"
|
||||
#include "library/components/omnitrace.hpp"
|
||||
#include "library/components/pthread_gotcha.hpp"
|
||||
#include "library/components/roctracer.hpp"
|
||||
#include "library/config.hpp"
|
||||
#include "library/debug.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
#include "library/sampling.hpp"
|
||||
#include "library/state.hpp"
|
||||
#include "library/thread_data.hpp"
|
||||
#include "library/thread_info.hpp"
|
||||
#include "library/utility.hpp"
|
||||
|
||||
#include <timemory/backends/threading.hpp>
|
||||
#include <timemory/components/macros.hpp>
|
||||
#include <timemory/components/timing/wall_clock.hpp>
|
||||
#include <timemory/sampling/allocator.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
@@ -49,13 +52,10 @@ std::set<int>
|
||||
shutdown();
|
||||
} // namespace sampling
|
||||
|
||||
namespace mpl = tim::mpl;
|
||||
|
||||
using bundle_t = tim::lightweight_tuple<comp::wall_clock, comp::roctracer_data>;
|
||||
using wall_pw_t = mpl::piecewise_select<comp::wall_clock>; // only wall-clock
|
||||
using main_pw_t = mpl::piecewise_ignore<comp::wall_clock>; // exclude wall-clock
|
||||
using category_region_t =
|
||||
tim::lightweight_tuple<omnitrace::component::category_region<category::pthread>>;
|
||||
namespace component
|
||||
{
|
||||
using bundle_t = tim::lightweight_tuple<comp::wall_clock, comp::roctracer_data>;
|
||||
using category_region_t = tim::lightweight_tuple<category_region<category::pthread>>;
|
||||
|
||||
namespace
|
||||
{
|
||||
@@ -63,7 +63,7 @@ auto* is_shutdown = new bool{ false }; // intentional data leak
|
||||
auto* bundles = new std::map<int64_t, std::shared_ptr<bundle_t>>{};
|
||||
auto* bundles_mutex = new std::mutex{};
|
||||
auto bundles_dtor = scope::destructor{ []() {
|
||||
omnitrace::pthread_create_gotcha::shutdown();
|
||||
pthread_create_gotcha::shutdown();
|
||||
delete bundles;
|
||||
delete bundles_mutex;
|
||||
bundles = nullptr;
|
||||
@@ -105,12 +105,12 @@ stop_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args)
|
||||
_bundle.key().c_str(), _tid);
|
||||
if(get_use_timemory())
|
||||
{
|
||||
_bundle.stop(wall_pw_t{}); // stop wall-clock so we can get the value
|
||||
auto _wc = *_bundle.get<comp::wall_clock>();
|
||||
_wc.stop();
|
||||
// update roctracer_data
|
||||
_bundle.store(std::plus<double>{},
|
||||
_bundle.get<comp::wall_clock>()->get() * units::sec);
|
||||
// stop all other components including roctracer_data after update
|
||||
_bundle.stop(main_pw_t{});
|
||||
_bundle.store(std::plus<double>{}, _wc.get() * _wc.unit());
|
||||
// stop all
|
||||
_bundle.stop();
|
||||
// exclude popping wall-clock
|
||||
_bundle.pop(_tid);
|
||||
}
|
||||
@@ -154,15 +154,16 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
return m_routine(m_arg);
|
||||
}
|
||||
|
||||
push_thread_state(omnitrace::ThreadState::Internal);
|
||||
push_thread_state(ThreadState::Internal);
|
||||
|
||||
int64_t _tid = -1;
|
||||
void* _ret = nullptr;
|
||||
auto _is_sampling = false;
|
||||
auto _bundle = std::shared_ptr<bundle_t>{};
|
||||
auto _signals = std::set<int>{};
|
||||
auto _coverage = (get_mode() == omnitrace::Mode::Coverage);
|
||||
auto _dtor = [&]() {
|
||||
auto _coverage = (get_mode() == Mode::Coverage);
|
||||
// const auto& _parent_info = thread_info::get(m_parent_tid, LookupTID);
|
||||
auto _dtor = [&]() {
|
||||
set_thread_state(ThreadState::Internal);
|
||||
if(_is_sampling)
|
||||
{
|
||||
@@ -172,11 +173,11 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
|
||||
if(_tid >= 0)
|
||||
{
|
||||
auto _active = (get_state() == omnitrace::State::Active &&
|
||||
auto _active = (get_state() == ::omnitrace::State::Active &&
|
||||
bundles != nullptr && bundles_mutex != nullptr);
|
||||
if(!_active) return;
|
||||
get_execution_time(_tid)->second = comp::wall_clock::record();
|
||||
auto& _thr_bundle = thread_bundle_data_t::instance();
|
||||
thread_info::set_stop(comp::wall_clock::record());
|
||||
auto& _thr_bundle = thread_bundle_data_t::instance();
|
||||
if(_thr_bundle && _thr_bundle->get<comp::wall_clock>() &&
|
||||
_thr_bundle->get<comp::wall_clock>()->get_is_running())
|
||||
_thr_bundle->stop();
|
||||
@@ -185,18 +186,19 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
}
|
||||
};
|
||||
|
||||
auto _active = (get_state() == omnitrace::State::Active && bundles != nullptr &&
|
||||
auto _active = (get_state() == ::omnitrace::State::Active && bundles != nullptr &&
|
||||
bundles_mutex != nullptr);
|
||||
|
||||
if(_active && !_coverage)
|
||||
{
|
||||
_tid = threading::get_id();
|
||||
const auto& _tid_index = thread_info::init();
|
||||
_tid = _tid_index->index_data->internal_value;
|
||||
threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid).c_str());
|
||||
if(!thread_bundle_data_t::instances().at(_tid))
|
||||
{
|
||||
thread_data<omnitrace_thread_bundle_t>::construct(
|
||||
TIMEMORY_JOIN('/', "omnitrace/process", process::get_id(), "thread",
|
||||
threading::get_id()),
|
||||
_tid),
|
||||
quirk::config<quirk::auto_start>{});
|
||||
thread_bundle_data_t::instances().at(_tid)->start();
|
||||
}
|
||||
@@ -207,12 +209,9 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
.first->second;
|
||||
}
|
||||
if(_bundle) start_bundle(*_bundle);
|
||||
get_execution_time(_tid)->first = comp::wall_clock::record();
|
||||
get_cpu_cid_stack(_tid, m_parent_tid);
|
||||
if(m_enable_sampling)
|
||||
{
|
||||
// initialize thread-local statics
|
||||
(void) tim::get_unw_backtrace<12, 1, false>();
|
||||
_is_sampling = true;
|
||||
pthread_gotcha::push_enable_sampling_on_child_threads(false);
|
||||
_signals = sampling::setup();
|
||||
@@ -220,6 +219,10 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
sampling::unblock_signals();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
thread_info::init(true);
|
||||
}
|
||||
|
||||
// notify the wrapper that all internal work is completed
|
||||
if(m_promise) m_promise->set_value();
|
||||
@@ -227,7 +230,7 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
// Internal -> Enabled
|
||||
pop_thread_state();
|
||||
|
||||
push_thread_state(omnitrace::ThreadState::Enabled);
|
||||
push_thread_state(ThreadState::Enabled);
|
||||
|
||||
// execute the original function
|
||||
_ret = m_routine(m_arg);
|
||||
@@ -237,7 +240,7 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
// execute the destructor actions
|
||||
_dtor();
|
||||
|
||||
set_thread_state(omnitrace::ThreadState::Completed);
|
||||
set_thread_state(ThreadState::Completed);
|
||||
|
||||
return _ret;
|
||||
}
|
||||
@@ -324,48 +327,50 @@ int
|
||||
pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
|
||||
void* (*start_routine)(void*), void* arg) const
|
||||
{
|
||||
auto _initial_thread_state = get_thread_state();
|
||||
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
bundle_t _bundle{ "pthread_create" };
|
||||
auto _enable_sampling = pthread_gotcha::sampling_enabled_on_child_threads();
|
||||
auto _coverage = (get_mode() == omnitrace::Mode::Coverage);
|
||||
auto _active = (get_state() == omnitrace::State::Active);
|
||||
int64_t _tid = (_active) ? threading::get_id() : 0;
|
||||
auto _disabled = (get_thread_state() == ThreadState::Disabled);
|
||||
auto _enabled = (get_thread_state() == ThreadState::Enabled);
|
||||
auto _bundle = std::optional<bundle_t>{};
|
||||
|
||||
if(_active)
|
||||
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
|
||||
auto _active = (get_state() == ::omnitrace::State::Active && !_disabled);
|
||||
auto _coverage = (get_mode() == Mode::Coverage);
|
||||
auto _use_sampling = get_use_sampling();
|
||||
auto _sample_child = pthread_gotcha::sampling_enabled_on_child_threads();
|
||||
auto _tid = utility::get_thread_index();
|
||||
auto _use_bundle = (_active && !_coverage);
|
||||
const auto& _info = thread_info::init(!_active || !_sample_child || _disabled);
|
||||
auto _enable_sampling =
|
||||
(!_disabled && _enabled && _sample_child && _use_sampling && !_info->is_offset);
|
||||
|
||||
if(_active && !_disabled && !_info->is_offset)
|
||||
{
|
||||
OMNITRACE_VERBOSE(1, "Creating new thread on PID %i (rank: %i), TID %li\n",
|
||||
process::get_id(), dmp::rank(), _tid);
|
||||
}
|
||||
|
||||
// ensure that cpu cid stack exists on the parent thread if active
|
||||
if(!_coverage && _active) get_cpu_cid_stack();
|
||||
if(_active && !_coverage) get_cpu_cid_stack();
|
||||
|
||||
if(!get_use_sampling() || !_enable_sampling)
|
||||
{
|
||||
auto* _obj = new wrapper(start_routine, arg, _enable_sampling, _tid, nullptr);
|
||||
if(_active && !_coverage && _enable_sampling &&
|
||||
_initial_thread_state == ThreadState::Enabled)
|
||||
start_bundle(_bundle, audit::incoming{}, thread, attr, start_routine, arg);
|
||||
// create the thread
|
||||
auto _ret = (*m_wrappee)(thread, attr, &wrapper::wrap, static_cast<void*>(_obj));
|
||||
if(_active && !_coverage && _enable_sampling &&
|
||||
_initial_thread_state == ThreadState::Enabled)
|
||||
stop_bundle(_bundle, _tid, audit::outgoing{}, _ret);
|
||||
return _ret;
|
||||
}
|
||||
|
||||
// block the signals in entire process
|
||||
OMNITRACE_DEBUG("blocking signals...\n");
|
||||
auto _blocked_signals = get_sampling_signals();
|
||||
tim::sampling::block_signals(_blocked_signals, tim::sampling::sigmask_scope::process);
|
||||
|
||||
start_bundle(_bundle, audit::incoming{}, thread, attr, start_routine, arg);
|
||||
|
||||
// promise set by thread when signal handler is configured
|
||||
set_thread_state(ThreadState::Disabled);
|
||||
auto _blocked = get_sampling_signals();
|
||||
auto _promise = std::promise<void>{};
|
||||
auto _fut = _promise.get_future();
|
||||
auto* _wrap = new wrapper(start_routine, arg, _enable_sampling, _tid, &_promise);
|
||||
set_thread_state(ThreadState::Internal);
|
||||
|
||||
// block the signals in entire process
|
||||
if(_enable_sampling && !_blocked.empty())
|
||||
{
|
||||
OMNITRACE_DEBUG("blocking signals...\n");
|
||||
tim::sampling::block_signals(_blocked, tim::sampling::sigmask_scope::process);
|
||||
}
|
||||
|
||||
if(_use_bundle)
|
||||
{
|
||||
_bundle = bundle_t{ "pthread_create" };
|
||||
start_bundle(*_bundle, audit::incoming{}, thread, attr, start_routine, arg);
|
||||
}
|
||||
|
||||
// create the thread
|
||||
auto _ret = (*m_wrappee)(thread, attr, &wrapper::wrap, static_cast<void*>(_wrap));
|
||||
@@ -374,21 +379,19 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
|
||||
OMNITRACE_DEBUG("waiting for child to signal it is setup...\n");
|
||||
_fut.wait();
|
||||
|
||||
stop_bundle(_bundle, threading::get_id(), audit::outgoing{}, _ret);
|
||||
if(_use_bundle) stop_bundle(*_bundle, threading::get_id(), audit::outgoing{}, _ret);
|
||||
|
||||
// unblock the signals in the entire process
|
||||
OMNITRACE_DEBUG("unblocking signals...\n");
|
||||
tim::sampling::unblock_signals(_blocked_signals,
|
||||
tim::sampling::sigmask_scope::process);
|
||||
if(_enable_sampling && !_blocked.empty())
|
||||
{
|
||||
OMNITRACE_DEBUG("unblocking signals...\n");
|
||||
tim::sampling::unblock_signals(_blocked, tim::sampling::sigmask_scope::process);
|
||||
}
|
||||
|
||||
OMNITRACE_DEBUG("returning success...\n");
|
||||
return _ret;
|
||||
}
|
||||
|
||||
bool
|
||||
pthread_create_gotcha::is_valid_execution_time(int64_t _tid, uint64_t _ts)
|
||||
{
|
||||
return (_ts >= get_execution_time(_tid)->first &&
|
||||
_ts <= get_execution_time(_tid)->second);
|
||||
}
|
||||
} // namespace component
|
||||
} // namespace omnitrace
|
||||
|
||||
TIMEMORY_INITIALIZE_STORAGE(component::roctracer_data)
|
||||
|
||||
Ссылка в новой задаче
Block a user