Address and thread sanitizer fixes (#250)
* Address and thread sanitizer fixes - Fix compilation with clang - Tweak perfetto copy to build tree - Added suppression files to scripts - fix LD_PRELOAD support in omnitrace-causal and omnitrace-sample - use spin_mutex and spin_lock from timemory instead of atomic_mutex and atomic_lock - state uses atomic - fix some memory leaks - tweak testing - mpi tests do not use preload - increase timeout when using sanitizers - add env LD_PRELOAD when using sanitizers * Tweak perfetto build * Update timemory submodule * Update version to 1.8.1 * Update omnitrace-leak.supp * Update timemory submodule - fixed spin_mutex implementation * Remove previously added addr_space->allowTraps(instr_traps) - this appears to cause errors during binary rewrite * causal testing updates - relaxed causal validation on CI systems (to account for hyperthreading decreasing prediction) - improved impact calculation - other general improvements to validate-causal-json.py * Improve fork handling for perfetto - numerous updates changing perfetto:: to ::perfetto:: - added perfetto_fwd.hpp * Updated fork example - user API for validation that stopping/starting perfetto is valid * Misc fixes to perfetto + fork support - tweak regions in fork example - handle disabling tmp files - get rid of stop/start with perfetto before/after fork - fixed sampling support during fork - tweak env of fork test * Fix find_package in build-tree * Fix buildtree export * Fix buildtree export * Restructured ConfigInstall before adding examples * Guard against creating tmp file in sampling when disabled * Fix buildtree package * formatting * exit handlers on child processes - quick exit to avoid perfetto cleanup * Further tweaking of causal tests for reliability - enable PROCESSOR_AFFINITY - decrease to 5 iterations * Further tweaking of causal tests for reliability - disable PROCESSOR_AFFINITY for fast func e2e tests - enabling affinity results in (valid) speedup predictions greater than zero * Fixes to fork handling - use pthread_atfork for redundancy if fork_gotcha fails * cmake formatting * Fix fork init settings + install components - remove dl from PROJECT_BUILD_TARGETS * Testing tweaks - fix mpi-binary-rewrite-run regex when OMNITRACE_VERBOSE set > 1 in env - increase causal e2e iterations to 8 * Fix "Test User API" - test-find-package.sh included dl component * Further tweaks to causal validation - further considerations of variance
This commit is contained in:
Jonathan R. Madsen
committed by
GitHub
GitHub
parent
9a665b5684
commit
846301bcaf
@@ -34,6 +34,7 @@
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/gpu.hpp"
|
||||
#include "core/perfetto_fwd.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
#include "core/utility.hpp"
|
||||
#include "library/causal/data.hpp"
|
||||
@@ -42,6 +43,7 @@
|
||||
#include "library/components/exit_gotcha.hpp"
|
||||
#include "library/components/fork_gotcha.hpp"
|
||||
#include "library/components/mpi_gotcha.hpp"
|
||||
#include "library/components/numa_gotcha.hpp"
|
||||
#include "library/components/pthread_gotcha.hpp"
|
||||
#include "library/components/rocprofiler.hpp"
|
||||
#include "library/coverage.hpp"
|
||||
@@ -83,21 +85,6 @@ using namespace omnitrace;
|
||||
|
||||
//======================================================================================//
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
namespace perfetto
|
||||
{
|
||||
// declare this here bc it has a tendency to cause namespace ambiguities
|
||||
void
|
||||
setup();
|
||||
|
||||
void
|
||||
start();
|
||||
} // namespace perfetto
|
||||
} // namespace omnitrace
|
||||
|
||||
//======================================================================================//
|
||||
|
||||
namespace
|
||||
{
|
||||
auto _timemory_manager = tim::manager::instance();
|
||||
@@ -196,13 +183,6 @@ ensure_finalization(bool _static_init = false)
|
||||
return scope::destructor{ []() { omnitrace_finalize_hidden(); } };
|
||||
}
|
||||
|
||||
auto
|
||||
is_system_backend()
|
||||
{
|
||||
// if get_backend() returns 'system' or 'all', this is true
|
||||
return (get_backend() != "inprocess");
|
||||
}
|
||||
|
||||
using Device = critical_trace::Device;
|
||||
using Phase = critical_trace::Phase;
|
||||
|
||||
@@ -552,8 +532,9 @@ omnitrace_init_tooling_hidden()
|
||||
rcclp::setup();
|
||||
}
|
||||
|
||||
if(get_use_perfetto() && !is_system_backend())
|
||||
if(get_use_perfetto())
|
||||
{
|
||||
OMNITRACE_VERBOSE_F(1, "Starting Perfetto...\n");
|
||||
omnitrace::perfetto::start();
|
||||
}
|
||||
|
||||
@@ -761,24 +742,29 @@ omnitrace_finalize_hidden(void)
|
||||
|
||||
OMNITRACE_DEBUG_F("Copying over all timemory hash information to main thread...\n");
|
||||
// copy these over so that all hashes are known
|
||||
auto& _hmain = tim::hash::get_main_hash_ids();
|
||||
auto& _amain = tim::hash::get_main_hash_aliases();
|
||||
auto& _hzero = tracing::get_timemory_hash_ids(0);
|
||||
auto& _azero = tracing::get_timemory_hash_aliases(0);
|
||||
for(size_t i = 1; i < max_supported_threads; ++i)
|
||||
for(size_t i = 0; i < max_supported_threads; ++i)
|
||||
{
|
||||
auto& _hitr = tracing::get_timemory_hash_ids(i);
|
||||
auto& _aitr = tracing::get_timemory_hash_aliases(i);
|
||||
if(_hzero && _hitr)
|
||||
if(_hmain && _hitr)
|
||||
{
|
||||
for(const auto& itr : *_hitr)
|
||||
_hzero->emplace(itr.first, itr.second);
|
||||
_hmain->emplace(itr.first, itr.second);
|
||||
}
|
||||
if(_azero && _aitr)
|
||||
if(_amain && _aitr)
|
||||
{
|
||||
for(auto itr : *_aitr)
|
||||
_azero->emplace(itr.first, itr.second);
|
||||
_amain->emplace(itr.first, itr.second);
|
||||
}
|
||||
}
|
||||
|
||||
if(_hzero && _hmain) *_hzero = *_hmain;
|
||||
if(_azero && _amain) *_azero = *_amain;
|
||||
|
||||
// stop the main bundle which has stats for run
|
||||
if(get_main_bundle())
|
||||
{
|
||||
@@ -833,6 +819,9 @@ omnitrace_finalize_hidden(void)
|
||||
{
|
||||
OMNITRACE_DEBUG_F("Stopping main gotcha...\n");
|
||||
get_init_bundle()->stop();
|
||||
|
||||
pthread_gotcha::shutdown();
|
||||
component::numa_gotcha::shutdown();
|
||||
}
|
||||
|
||||
// stop the gotcha bundle
|
||||
@@ -982,90 +971,11 @@ omnitrace_finalize_hidden(void)
|
||||
}
|
||||
|
||||
bool _perfetto_output_error = false;
|
||||
if(get_use_perfetto() && !is_system_backend())
|
||||
if(get_use_perfetto())
|
||||
{
|
||||
auto& tracing_session = get_perfetto_session();
|
||||
|
||||
OMNITRACE_CI_THROW(tracing_session == nullptr,
|
||||
"Null pointer to the tracing session");
|
||||
|
||||
OMNITRACE_VERBOSE_F(0, "Finalizing perfetto...\n");
|
||||
|
||||
// Make sure the last event is closed for this example.
|
||||
::perfetto::TrackEvent::Flush();
|
||||
tracing_session->FlushBlocking();
|
||||
|
||||
OMNITRACE_VERBOSE_F(3, "Stopping the blocking perfetto trace session...\n");
|
||||
tracing_session->StopBlocking();
|
||||
|
||||
using char_vec_t = std::vector<char>;
|
||||
OMNITRACE_VERBOSE_F(3, "Getting the trace data...\n");
|
||||
|
||||
auto trace_data = char_vec_t{};
|
||||
#if defined(TIMEMORY_USE_MPI) && TIMEMORY_USE_MPI > 0
|
||||
if(get_perfetto_combined_traces())
|
||||
{
|
||||
using perfetto_mpi_get_t =
|
||||
tim::operation::finalize::mpi_get<char_vec_t, true>;
|
||||
|
||||
char_vec_t _trace_data{ tracing_session->ReadTraceBlocking() };
|
||||
std::vector<char_vec_t> _rank_data = {};
|
||||
auto _combine = [](char_vec_t& _dst, const char_vec_t& _src) -> char_vec_t& {
|
||||
_dst.reserve(_dst.size() + _src.size());
|
||||
for(auto&& itr : _src)
|
||||
_dst.emplace_back(itr);
|
||||
return _dst;
|
||||
};
|
||||
|
||||
perfetto_mpi_get_t{ get_perfetto_combined_traces(), settings::node_count() }(
|
||||
_rank_data, _trace_data, _combine);
|
||||
for(auto& itr : _rank_data)
|
||||
trace_data =
|
||||
(trace_data.empty()) ? std::move(itr) : _combine(trace_data, itr);
|
||||
}
|
||||
else
|
||||
{
|
||||
trace_data = tracing_session->ReadTraceBlocking();
|
||||
}
|
||||
#else
|
||||
trace_data = tracing_session->ReadTraceBlocking();
|
||||
#endif
|
||||
|
||||
if(!trace_data.empty())
|
||||
{
|
||||
operation::file_output_message<tim::project::omnitrace> _fom{};
|
||||
// Write the trace into a file.
|
||||
if(get_verbose() >= 0)
|
||||
_fom(get_perfetto_output_filename(), std::string{ "perfetto" },
|
||||
" (%.2f KB / %.2f MB / %.2f GB)... ",
|
||||
static_cast<double>(trace_data.size()) / units::KB,
|
||||
static_cast<double>(trace_data.size()) / units::MB,
|
||||
static_cast<double>(trace_data.size()) / units::GB);
|
||||
std::ofstream ofs{};
|
||||
if(!tim::filepath::open(ofs, get_perfetto_output_filename(),
|
||||
std::ios::out | std::ios::binary))
|
||||
{
|
||||
_fom.append("Error opening '%s'...",
|
||||
get_perfetto_output_filename().c_str());
|
||||
_perfetto_output_error = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Write the trace into a file.
|
||||
ofs.write(&trace_data[0], trace_data.size());
|
||||
if(get_verbose() >= 0) _fom.append("%s", "Done"); // NOLINT
|
||||
if(_timemory_manager)
|
||||
_timemory_manager->add_file_output("protobuf", "perfetto",
|
||||
get_perfetto_output_filename());
|
||||
}
|
||||
ofs.close();
|
||||
}
|
||||
else if(dmp::rank() == 0)
|
||||
{
|
||||
OMNITRACE_VERBOSE_F(0,
|
||||
"trace data is empty. File '%s' will not be written...\n",
|
||||
get_perfetto_output_filename().c_str());
|
||||
}
|
||||
omnitrace::perfetto::post_process(_timemory_manager.get(),
|
||||
_perfetto_output_error);
|
||||
}
|
||||
|
||||
if(_timemory_manager && _timemory_manager != nullptr)
|
||||
|
||||
Reference in New Issue
Block a user