Address and thread sanitizer fixes (#250)
* Address and thread sanitizer fixes - Fix compilation with clang - Tweak perfetto copy to build tree - Added suppression files to scripts - fix LD_PRELOAD support in omnitrace-causal and omnitrace-sample - use spin_mutex and spin_lock from timemory instead of atomic_mutex and atomic_lock - state uses atomic - fix some memory leaks - tweak testing - mpi tests do not use preload - increase timeout when using sanitizers - add env LD_PRELOAD when using sanitizers * Tweak perfetto build * Update timemory submodule * Update version to 1.8.1 * Update omnitrace-leak.supp * Update timemory submodule - fixed spin_mutex implementation * Remove previously added addr_space->allowTraps(instr_traps) - this appears to cause errors during binary rewrite * causal testing updates - relaxed causal validation on CI systems (to account for hyperthreading decreasing prediction) - improved impact calculation - other general improvements to validate-causal-json.py * Improve fork handling for perfetto - numerous updates changing perfetto:: to ::perfetto:: - added perfetto_fwd.hpp * Updated fork example - user API for validation that stopping/starting perfetto is valid * Misc fixes to perfetto + fork support - tweak regions in fork example - handle disabling tmp files - get rid of stop/start with perfetto before/after fork - fixed sampling support during fork - tweak env of fork test * Fix find_package in build-tree * Fix buildtree export * Fix buildtree export * Restructured ConfigInstall before adding examples * Guard against creating tmp file in sampling when disabled * Fix buildtree package * formatting * exit handlers on child processes - quick exit to avoid perfetto cleanup * Further tweaking of causal tests for reliability - enable PROCESSOR_AFFINITY - decrease to 5 iterations * Further tweaking of causal tests for reliability - disable PROCESSOR_AFFINITY for fast func e2e tests - enabling affinity results in (valid) speedup predictions greater than zero * Fixes to fork handling - use pthread_atfork for redundancy if fork_gotcha fails * cmake formatting * Fix fork init settings + install components - remove dl from PROJECT_BUILD_TARGETS * Testing tweaks - fix mpi-binary-rewrite-run regex when OMNITRACE_VERBOSE set > 1 in env - increase causal e2e iterations to 8 * Fix "Test User API" - test-find-package.sh included dl component * Further tweaks to causal validation - further considerations of variance
This commit is contained in:
Jonathan R. Madsen
committed by
GitHub
GitHub
parent
9a665b5684
commit
846301bcaf
+185
-10
@@ -22,11 +22,31 @@
|
||||
|
||||
#include "perfetto.hpp"
|
||||
#include "config.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
#include "perfetto_fwd.hpp"
|
||||
#include "utility.hpp"
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
namespace perfetto
|
||||
{
|
||||
namespace
|
||||
{
|
||||
auto
|
||||
is_system_backend()
|
||||
{
|
||||
// if get_perfetto_backend() returns 'system' or 'all', this is true
|
||||
return (config::get_perfetto_backend() != "inprocess");
|
||||
}
|
||||
|
||||
auto&
|
||||
get_perfetto_tmp_file(pid_t _pid = process::get_id())
|
||||
{
|
||||
static auto _v = std::unordered_map<pid_t, std::shared_ptr<tmp_file>>{};
|
||||
if(_v.find(_pid) == _v.end()) _v.emplace(_pid, std::shared_ptr<tmp_file>{});
|
||||
return _v.at(_pid);
|
||||
}
|
||||
|
||||
auto&
|
||||
get_config()
|
||||
{
|
||||
@@ -35,11 +55,15 @@ get_config()
|
||||
}
|
||||
|
||||
auto&
|
||||
get_session()
|
||||
get_session(pid_t _pid = process::get_id())
|
||||
{
|
||||
static auto _v = std::unique_ptr<::perfetto::TracingSession>{};
|
||||
return _v;
|
||||
static auto _v =
|
||||
std::unordered_map<pid_t, std::unique_ptr<::perfetto::TracingSession>>{};
|
||||
if(_v.find(_pid) == _v.end())
|
||||
_v.emplace(_pid, std::unique_ptr<::perfetto::TracingSession>{});
|
||||
return _v.at(_pid);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void
|
||||
setup()
|
||||
@@ -73,8 +97,8 @@ setup()
|
||||
|
||||
args.shmem_size_hint_kb = shmem_size_hint;
|
||||
|
||||
if(get_backend() != "inprocess") args.backends |= ::perfetto::kSystemBackend;
|
||||
if(get_backend() != "system") args.backends |= ::perfetto::kInProcessBackend;
|
||||
if(get_perfetto_backend() != "inprocess") args.backends |= ::perfetto::kSystemBackend;
|
||||
if(get_perfetto_backend() != "system") args.backends |= ::perfetto::kInProcessBackend;
|
||||
|
||||
::perfetto::Tracing::Initialize(args);
|
||||
::perfetto::TrackEvent::Register();
|
||||
@@ -83,18 +107,169 @@ setup()
|
||||
void
|
||||
start()
|
||||
{
|
||||
auto& cfg = get_config();
|
||||
if(is_system_backend()) return;
|
||||
|
||||
auto& tracing_session = get_session();
|
||||
tracing_session = ::perfetto::Tracing::NewTrace();
|
||||
tracing_session->Setup(cfg);
|
||||
|
||||
if(!tracing_session) tracing_session = ::perfetto::Tracing::NewTrace();
|
||||
|
||||
tracing_session = ::perfetto::Tracing::NewTrace();
|
||||
auto& _tmp_file = get_perfetto_tmp_file();
|
||||
if(config::get_use_tmp_files())
|
||||
{
|
||||
if(!_tmp_file)
|
||||
{
|
||||
_tmp_file = config::get_tmp_file("perfetto-trace", "proto");
|
||||
_tmp_file->fopen("w+");
|
||||
}
|
||||
else
|
||||
{
|
||||
OMNITRACE_VERBOSE(0, "Resuming perfetto...\n");
|
||||
_tmp_file->fopen("a+");
|
||||
}
|
||||
}
|
||||
|
||||
OMNITRACE_VERBOSE(0, "Setup perfetto...\n");
|
||||
int _fd = (_tmp_file) ? _tmp_file->fd : -1;
|
||||
auto& cfg = get_config();
|
||||
tracing_session->Setup(cfg, _fd);
|
||||
tracing_session->StartBlocking();
|
||||
}
|
||||
|
||||
void
|
||||
stop()
|
||||
{
|
||||
if(is_system_backend()) return;
|
||||
|
||||
auto& tracing_session = get_perfetto_session();
|
||||
|
||||
OMNITRACE_CI_THROW(tracing_session == nullptr, "Null pointer to the tracing session");
|
||||
|
||||
if(tracing_session)
|
||||
{
|
||||
// Make sure the last event is closed
|
||||
OMNITRACE_VERBOSE(2, "Flushing the perfetto trace data...\n");
|
||||
::perfetto::TrackEvent::Flush();
|
||||
tracing_session->FlushBlocking();
|
||||
|
||||
OMNITRACE_VERBOSE(2, "Stopping the perfetto trace session (blocking)...\n");
|
||||
tracing_session->StopBlocking();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
|
||||
{
|
||||
using char_vec_t = std::vector<char>;
|
||||
|
||||
stop();
|
||||
|
||||
auto& tracing_session = get_perfetto_session();
|
||||
if(!tracing_session) return;
|
||||
|
||||
auto _get_session_data = [&tracing_session]() {
|
||||
auto _data = char_vec_t{};
|
||||
auto _tmp_file = get_perfetto_tmp_file();
|
||||
if(_tmp_file && *_tmp_file)
|
||||
{
|
||||
_tmp_file->close();
|
||||
FILE* _fdata = fopen(_tmp_file->filename.c_str(), "rb");
|
||||
fseek(_fdata, 0, SEEK_END);
|
||||
size_t _fnum_elem = ftell(_fdata);
|
||||
fseek(_fdata, 0, SEEK_SET); // same as rewind(f);
|
||||
|
||||
_data.resize(_fnum_elem + 1);
|
||||
auto _fnum_read = fread(_data.data(), sizeof(char), _fnum_elem, _fdata);
|
||||
fclose(_fdata);
|
||||
|
||||
OMNITRACE_CI_THROW(
|
||||
_fnum_read != _fnum_elem,
|
||||
"Error! read %zu elements from perfetto trace file '%s'. Expected %zu\n",
|
||||
_fnum_read, _tmp_file->filename.c_str(), _fnum_elem);
|
||||
}
|
||||
|
||||
return utility::combine(_data,
|
||||
char_vec_t{ tracing_session->ReadTraceBlocking() });
|
||||
};
|
||||
|
||||
auto trace_data = char_vec_t{};
|
||||
#if defined(TIMEMORY_USE_MPI) && TIMEMORY_USE_MPI > 0
|
||||
if(get_perfetto_combined_traces())
|
||||
{
|
||||
using perfetto_mpi_get_t = tim::operation::finalize::mpi_get<char_vec_t, true>;
|
||||
|
||||
auto _trace_data = _get_session_data();
|
||||
auto _rank_data = std::vector<char_vec_t>{};
|
||||
auto _combine = [](char_vec_t& _dst, const char_vec_t& _src) -> char_vec_t& {
|
||||
_dst.reserve(_dst.size() + _src.size());
|
||||
for(auto&& itr : _src)
|
||||
_dst.emplace_back(itr);
|
||||
return _dst;
|
||||
};
|
||||
|
||||
perfetto_mpi_get_t{ get_perfetto_combined_traces(),
|
||||
settings::node_count() }(_rank_data, _trace_data, _combine);
|
||||
for(auto& itr : _rank_data)
|
||||
trace_data =
|
||||
(trace_data.empty()) ? std::move(itr) : _combine(trace_data, itr);
|
||||
}
|
||||
else
|
||||
{
|
||||
trace_data = _get_session_data();
|
||||
}
|
||||
#else
|
||||
trace_data = _get_session_data();
|
||||
#endif
|
||||
|
||||
auto _filename = config::get_perfetto_output_filename();
|
||||
if(!trace_data.empty())
|
||||
{
|
||||
operation::file_output_message<tim::project::omnitrace> _fom{};
|
||||
// Write the trace into a file.
|
||||
if(config::get_verbose() >= 0)
|
||||
_fom(_filename, std::string{ "perfetto" },
|
||||
" (%.2f KB / %.2f MB / %.2f GB)... ",
|
||||
static_cast<double>(trace_data.size()) / units::KB,
|
||||
static_cast<double>(trace_data.size()) / units::MB,
|
||||
static_cast<double>(trace_data.size()) / units::GB);
|
||||
std::ofstream ofs{};
|
||||
if(!filepath::open(ofs, _filename, std::ios::out | std::ios::binary))
|
||||
{
|
||||
_fom.append("Error opening '%s'...", _filename.c_str());
|
||||
_perfetto_output_error = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Write the trace into a file.
|
||||
ofs.write(&trace_data[0], trace_data.size());
|
||||
if(config::get_verbose() >= 0) _fom.append("%s", "Done"); // NOLINT
|
||||
if(_timemory_manager)
|
||||
_timemory_manager->add_file_output("protobuf", "perfetto", _filename);
|
||||
}
|
||||
ofs.close();
|
||||
}
|
||||
else if(dmp::rank() == 0)
|
||||
{
|
||||
OMNITRACE_VERBOSE(
|
||||
0, "perfetto trace data is empty. File '%s' will not be written...\n",
|
||||
_filename.c_str());
|
||||
}
|
||||
|
||||
auto& _tmp_file = get_perfetto_tmp_file();
|
||||
if(_tmp_file)
|
||||
{
|
||||
_tmp_file->close();
|
||||
_tmp_file->remove();
|
||||
_tmp_file.reset();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace perfetto
|
||||
|
||||
std::unique_ptr<::perfetto::TracingSession>&
|
||||
get_perfetto_session()
|
||||
get_perfetto_session(pid_t _pid)
|
||||
{
|
||||
return ::omnitrace::perfetto::get_session();
|
||||
return ::omnitrace::perfetto::get_session(_pid);
|
||||
}
|
||||
} // namespace omnitrace
|
||||
|
||||
|
||||
Reference in New Issue
Block a user