AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved analysis of functions to instrument + MPI support + timemory support (#2)

Browse Source 
* various tweaks
* build updates + cleanup + overlap guard + min addr range
* Library source reorg + miscellaneous tweaks
* Removed unnecessary fwd decls
* Print address range in --print-X pair mode

- hosttrace modifications
  - disable instrumenting functions with overlapping sections or multiple entry points by default (control via --allow-overlapping option)
  - disable instrumenting functions whose address range < 512 bytes unless a loop is present by default (control via --min-address-range option)
  - disable instrumenting functions w/ loops whose address range < 64 bytes (control via --min-loop-address-range)
- Support for wrapping MPI function calls even in binary rewrite mode
  - e.g. use gotcha to wrap MPI functions with hosttrace_push_trace and hosttrace_pop_trace
- New timemory only mode --> HOSTTRACE_USE_TIMEMORY=ON
- New timemory + perfetto mode --> HOSTTRACE_USE_PERFETTO=ON + HOSTTRACE_USE_TIMEMORY=ON
- Full support for all timemory components
- parallel-overhead example for measuring the overhead in a MT-parallelized application with very small instrumentation functions
- improvements to output directories for hosttrace exe
- improvements to output directories for hosttrace library
- new hosttrace options
  - --print-instrumented <type> prints out the instrumented entities and exits
  - --print-available <type> prints out the available instrumentation entities and exits
  - --print-overlapping <type> prints out the overlapping entities and exits
  - NOTE: <type> above refers to the information printed out, e.g. module name vs. function name vs. module and function name, etc.
This commit is contained in:
Jonathan R. Madsen
2021-09-02 11:38:39 -05:00
committed by GitHub
parent f518e09eab
commit 1f15b3070f
15 changed files with 1202 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/library.cpp
+309 -221
View File
@@ -1,103 +1,6 @@
#include <perfetto.h>
#include "library.hpp"
#if defined(NDEBUG)
# undef NDEBUG
#endif
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <fstream>
#include <memory>
#include <string>
#include <sys/types.h>
#include <unistd.h>
#include <utility>
#include <vector>
#include "timemory/api.hpp"
#include "timemory/backends/process.hpp"
#include "timemory/backends/threading.hpp"
#include "timemory/components.hpp"
#include "timemory/config.hpp"
#include "timemory/environment.hpp"
#include "timemory/manager.hpp"
#include "timemory/mpl/apply.hpp"
#include "timemory/operations.hpp"
#include "timemory/settings.hpp"
#include "timemory/storage.hpp"
#include "timemory/variadic.hpp"
#if !defined(JOIN)
# define JOIN(...) tim::mpl::apply<std::string>::join(__VA_ARGS__)
#endif
namespace audit = tim::audit;
namespace comp = tim::component;
namespace quirk = tim::quirk;
struct fork_gotcha : tim::component::base<fork_gotcha, void>
{
using gotcha_data_t = tim::component::gotcha_data;
TIMEMORY_DEFAULT_OBJECT(fork_gotcha)
void audit(const gotcha_data_t& _data, audit::incoming);
void audit(const gotcha_data_t& _data, audit::outgoing, pid_t _pid);
};
struct fork_gotcha_api : tim::concepts::api
{};
using fork_gotcha_t =
tim::component::gotcha<4, tim::component_tuple<fork_gotcha>, fork_gotcha_api>;
using fork_bundle_t =
tim::lightweight_tuple<comp::wall_clock, comp::peak_rss, comp::cpu_clock,
comp::cpu_util, fork_gotcha_t>;
//--------------------------------------------------------------------------------------//
PERFETTO_DEFINE_CATEGORIES(
perfetto::Category("hosttrace").SetDescription("Function trace"));
#if defined(CUSTOM_DATA_SOURCE)
class CustomDataSource : public perfetto::DataSource<CustomDataSource>
{
public:
void OnSetup(const SetupArgs&) override
{
// Use this callback to apply any custom configuration to your data source
// based on the TraceConfig in SetupArgs.
PRINT_HERE("%s", "setup");
}
void OnStart(const StartArgs&) override
{
// This notification can be used to initialize the GPU driver, enable
// counters, etc. StartArgs will contains the DataSourceDescriptor,
// which can be extended.
PRINT_HERE("%s", "start");
}
void OnStop(const StopArgs&) override
{
// Undo any initialization done in OnStart.
PRINT_HERE("%s", "stop");
}
// Data sources can also have per-instance state.
int my_custom_state = 0;
};
PERFETTO_DECLARE_DATA_SOURCE_STATIC_MEMBERS(CustomDataSource);
#endif
extern "C" void
hosttrace_trace_finalize();
namespace
{
bool
get_debug()
{
@@ -105,30 +8,72 @@ get_debug()
return _v;
}
void
setup_fork_gotcha()
State&
get_state()
{
CONDITIONAL_PRINT_HERE(get_debug(), "%s", "configuring gotcha wrapper around fork");
static State _v{ State::PreInit };
return _v;
}
//--------------------------------------------------------------------------------------//
namespace
{
auto
get_use_perfetto()
{
// if using timemory, default to perfetto being off
static auto _default_v = !tim::get_env<bool>("HOSTTRACE_USE_TIMEMORY", false, false);
// explicit env control for using perfetto
static auto _v = tim::get_env<bool>("HOSTTRACE_USE_PERFETTO", _default_v);
return _v;
}
auto
get_use_timemory()
{
// default to opposite of whether perfetto setting
// to use both timemory and perfetto, both HOSTTRACE_USE_TIMEMORY and
// HOSTTRACE_USE_PERFETTO must be true
static auto _v = tim::get_env<bool>("HOSTTRACE_USE_TIMEMORY", !get_use_perfetto());
return _v;
}
bool&
get_use_mpi()
{
// this does not enable anything particularly useful when not using timemory
static bool _v = tim::get_env("HOSTTRACE_USE_MPI", false, get_use_timemory());
return _v;
}
void
setup_gotchas()
{
static bool _initialized = false;
if(_initialized)
return;
_initialized = true;
HOSTTRACE_DEBUG(
"[%s] Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n",
__FUNCTION__);
fork_gotcha_t::get_initializer() = []() {
TIMEMORY_C_GOTCHA(fork_gotcha_t, 0, fork);
};
}
auto&
get_fork_gotcha()
{
static auto _v =
(setup_fork_gotcha(), std::make_unique<fork_bundle_t>(
"hosttrace", quirk::config<quirk::auto_start>{}));
return _v;
mpi_gotcha_t::get_initializer() = []() {
mpi_gotcha_t::template configure<0, int, int*, char***>("MPI_Init");
mpi_gotcha_t::template configure<1, int, int*, char***, int, int*>(
"MPI_Init_thread");
};
}
auto
ensure_finalization()
{
if(get_debug())
fprintf(stderr, "[%s]\n", __FUNCTION__);
HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
return tim::scope::destructor{ []() { hosttrace_trace_finalize(); } };
}
@@ -139,40 +84,30 @@ get_trace_session()
return _session;
}
enum class State : unsigned short
{
PreInit = 0,
Active,
Finalized
};
auto&
get_state()
{
static State _v{ State::PreInit };
return _v;
}
auto&
get_output_filename()
auto
get_perfetto_output_filename()
{
static auto _v = []() {
auto _tmp = tim::get_env<std::string>(
// default name: perfetto-trace.<pid>.proto or perfetto-trace.<rank>.proto
auto _default_fname = tim::settings::compose_output_filename(
JOIN('.', "perfetto-trace", (get_use_mpi()) ? "%rank%" : "%pid%"), "proto");
// have the default display the full path to the output file
return tim::get_env<std::string>(
"HOSTTRACE_OUTPUT_FILE",
JOIN('/', tim::get_env<std::string>("PWD", ".", false),
"hosttrace.perfetto-trace-%pid%"));
auto _replace = [&_tmp](const std::string& _key, auto _val) {
auto _pos = _tmp.find(_key);
if(_pos != std::string::npos)
_tmp.replace(_pos, _key.length(), std::to_string(_val));
};
_replace("%pid%", tim::process::get_id());
_replace("%rank%", tim::mpi::rank());
// backwards compatibility
_replace("%p", tim::process::get_id());
return _tmp;
JOIN('/', tim::get_env<std::string>("PWD", ".", false), _default_fname));
}();
return _v;
auto _tmp = _v;
auto _replace = [&_tmp](const std::string& _key, auto&& _val) {
auto _pos = _tmp.find(_key);
if(_pos != std::string::npos)
_tmp.replace(_pos, _key.length(), std::to_string(_val()));
};
_replace("%pid%", []() { return tim::process::get_id(); });
_replace("%rank%", []() { return tim::mpi::rank(); });
// backwards compatibility
_replace("%p", []() { return tim::process::get_id(); });
return _tmp;
}
auto&
@@ -195,63 +130,185 @@ is_system_backend()
return (get_backend() != "inprocess");
}
auto&
get_timemory_data()
{
static thread_local auto& _v =
hosttrace_timemory_data::instances().at(threading::get_id());
return _v;
}
auto&
get_functors()
{
using functor_t = std::function<void(const char*)>;
static auto _v =
std::pair<functor_t, functor_t>{ [](const char*) {}, [](const char*) {} };
return _v;
}
bool
hosttrace_init_perfetto()
{
if(get_debug())
fprintf(stderr, "[%s]\n", __FUNCTION__);
if(get_state() != State::PreInit)
return false;
HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
// always initialize timemory because gotcha wrappers are always used
tim::settings::flamegraph_output() = false;
tim::settings::file_output() = false;
tim::settings::cout_output() = false;
tim::settings::file_output() = true;
tim::settings::enable_signal_handler() = true;
tim::timemory_init({ "hosttrace" });
tim::settings::collapse_processes() = false;
tim::settings::collapse_threads() = false;
tim::settings::max_thread_bookmarks() = 1;
tim::settings::global_components() = tim::get_env<std::string>(
"HOSTTRACE_COMPONENTS", "wall_clock", get_use_timemory());
auto& _fork_gotcha = get_fork_gotcha();
// enable timestamp directories when perfetto + mpi is activated
if(get_use_perfetto() && get_use_mpi())
tim::settings::time_output() = true;
auto _cmd = tim::read_command_line(tim::process::get_id());
auto _exe = (_cmd.empty()) ? "hosttrace" : _cmd.front();
auto _pos = _exe.find_last_of('/');
if(_pos < _exe.length() - 1)
_exe = _exe.substr(_pos + 1);
tim::timemory_init({ _exe }, "hosttrace-");
if(get_use_timemory())
{
comp::user_global_bundle::global_init();
std::set<int> _comps{};
// convert string into set of enumerations
for(auto&& itr : tim::delimit(tim::settings::global_components()))
_comps.emplace(tim::runtime::enumerate(itr));
if(_comps.size() == 1 && _comps.find(TIMEMORY_WALL_CLOCK) != _comps.end())
{
// using wall_clock directly is lower overhead than using it via user_bundle
bundle_t::get_initializer() = [](bundle_t& _bundle) {
_bundle.initialize<comp::wall_clock>();
};
}
else if(!_comps.empty())
{
// use user_bundle for other than wall-clock
bundle_t::get_initializer() = [](bundle_t& _bundle) {
_bundle.initialize<comp::user_global_bundle>();
};
}
else
{
tim::trait::runtime_enabled<hosttrace>::set(false);
}
}
// always activate gotcha wrappers
auto& _fork_gotcha = get_main_bundle();
_fork_gotcha->start();
assert(_fork_gotcha->get<fork_gotcha_t>()->get_is_running());
// environment settings
auto shmem_size_hint = tim::get_env<size_t>("HOSTTRACE_SHMEM_SIZE_HINT_KB", 40960);
auto buffer_size = tim::get_env<size_t>("HOSTTRACE_BUFFER_SIZE_KB", 1024000);
assert(_fork_gotcha->get<mpi_gotcha_t>()->get_is_running());
perfetto::TracingInitArgs args{};
perfetto::TraceConfig cfg{};
perfetto::protos::gen::TrackEventConfig track_event_cfg{};
auto *buffer_config = cfg.add_buffers();
buffer_config->set_size_kb(buffer_size);
buffer_config->set_fill_policy(perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_DISCARD);
// perfetto initialization
if(get_use_perfetto())
{
// environment settings
auto shmem_size_hint =
tim::get_env<size_t>("HOSTTRACE_SHMEM_SIZE_HINT_KB", 40960);
auto buffer_size = tim::get_env<size_t>("HOSTTRACE_BUFFER_SIZE_KB", 1024000);
auto* ds_cfg = cfg.add_data_sources()->mutable_config();
ds_cfg->set_name("track_event");
ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString());
auto* buffer_config = cfg.add_buffers();
buffer_config->set_size_kb(buffer_size);
buffer_config->set_fill_policy(
perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_DISCARD);
args.shmem_size_hint_kb = shmem_size_hint;
auto* ds_cfg = cfg.add_data_sources()->mutable_config();
ds_cfg->set_name("track_event");
ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString());
if(get_backend() != "inprocess")
args.backends |= perfetto::kSystemBackend;
if(get_backend() != "system")
args.backends |= perfetto::kInProcessBackend;
args.shmem_size_hint_kb = shmem_size_hint;
perfetto::Tracing::Initialize(args);
perfetto::TrackEvent::Register();
if(get_backend() != "inprocess")
args.backends |= perfetto::kSystemBackend;
if(get_backend() != "system")
args.backends |= perfetto::kInProcessBackend;
(void) get_output_filename();
perfetto::Tracing::Initialize(args);
perfetto::TrackEvent::Register();
tim::print_env(std::cerr,
[](const std::string& _v) { return _v.find("HOSTTRACE_") == 0; });
(void) get_perfetto_output_filename();
}
if(!is_system_backend())
// functors for starting and stopping timemory
static auto _push_timemory = [](const char* name) {
auto& _data = get_timemory_data();
// this generates a hash for the raw string array
auto _hash = tim::add_hash_id(tim::string_view_t{ name });
auto* _bundle = _data.allocator.allocate(1);
_data.bundles.emplace_back(_bundle);
_data.allocator.construct(_bundle, _hash);
_bundle->start();
};
static auto _pop_timemory = [](const char* name) {
auto& _data = get_timemory_data();
if(_data.bundles.empty())
{
HOSTTRACE_DEBUG("[%s] skipped %s :: empty bundle stack\n",
"hosttrace_pop_trace", name);
return;
}
_data.bundles.back()->stop();
_data.allocator.destroy(_data.bundles.back());
_data.allocator.deallocate(_data.bundles.back(), 1);
_data.bundles.pop_back();
};
if(get_use_perfetto() && get_use_timemory())
{
// if both are used, then use perfetto overload for calling lambda to launch
// timemory
get_functors().first = [](const char* name) {
TRACE_EVENT_BEGIN("hosttrace", perfetto::StaticString(name),
[&](perfetto::EventContext) { _push_timemory(name); });
};
get_functors().second = [](const char* name) {
TRACE_EVENT_END("hosttrace",
[&](perfetto::EventContext) { _pop_timemory(name); });
};
}
else if(get_use_perfetto())
{
get_functors().first = [](const char* name) {
TRACE_EVENT_BEGIN("hosttrace", perfetto::StaticString(name));
};
get_functors().second = [](const char*) { TRACE_EVENT_END("hosttrace"); };
}
else if(get_use_timemory())
{
get_functors().first = _push_timemory;
get_functors().second = _pop_timemory;
}
if(tim::dmp::rank() == 0)
{
tim::print_env(std::cerr,
[](const std::string& _v) { return _v.find("HOSTTRACE_") == 0; });
}
if(get_use_perfetto() && !is_system_backend())
{
#if defined(CUSTOM_DATA_SOURCE)
// Add the following:
perfetto::DataSourceDescriptor dsd{};
dsd.set_name("com.example.custom_data_source");
CustomDataSource::Register(dsd);
ds_cfg = cfg.add_data_sources()->mutable_config();
auto* ds_cfg = cfg.add_data_sources()->mutable_config();
ds_cfg->set_name("com.example.custom_data_source");
CustomDataSource::Trace([](CustomDataSource::TraceContext ctx) {
auto packet = ctx.NewTracePacket();
@@ -273,87 +330,118 @@ hosttrace_init_perfetto()
// ends the tracing session
static auto _ensure_finalization = ensure_finalization();
puts("");
if(tim::dmp::rank() == 0)
puts("");
return true;
}
} // namespace
//--------------------------------------------------------------------------------------//
extern "C"
{
void hosttrace_push_trace(const char* name)
{
if(get_debug())
fprintf(stderr, "[%s] %s\n", __FUNCTION__, name);
// return if not active
if(get_state() != State::Active && !hosttrace_init_perfetto())
if(get_state() == State::Finalized)
return;
// TRACE_EVENT_BEGIN(
// "hosttrace", perfetto::StaticString(name),
// [&](perfetto::EventContext ctx) { PRINT_HERE("executing %s", name); });
TRACE_EVENT_BEGIN("hosttrace", perfetto::StaticString(name));
if(get_state() != State::Active && !hosttrace_init_perfetto())
{
HOSTTRACE_DEBUG("[%s] %s :: not active and perfetto not initialized\n",
__FUNCTION__, name);
return;
}
else
{
HOSTTRACE_DEBUG("[%s] %s\n", __FUNCTION__, name);
}
get_functors().first(name);
}
void hosttrace_pop_trace(const char* name)
{
if(get_debug())
fprintf(stderr, "[%s] %s\n", __FUNCTION__, name);
// return if not active
if(get_state() != State::Active)
return;
// TRACE_EVENT_END("hosttrace",
// [&](perfetto::EventContext ctx) { PRINT_HERE("executing %s", name); });
TRACE_EVENT_END("hosttrace");
if(get_state() == State::Active)
{
HOSTTRACE_DEBUG("[%s] %s\n", __FUNCTION__, name);
get_functors().second(name);
}
else
{
HOSTTRACE_DEBUG("[%s] %s :: not active\n", __FUNCTION__, name);
}
}
void hosttrace_trace_init(const char*, bool, const char*)
{
if(get_debug())
fprintf(stderr, "[%s]\n", __FUNCTION__);
HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
hosttrace_init_perfetto();
}
void hosttrace_trace_finalize(void)
{
if(get_debug())
fprintf(stderr, "[%s]\n", __FUNCTION__);
// return if not active
if(get_state() != State::Active)
return;
puts("");
HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
if(tim::dmp::rank() == 0)
puts("");
get_state() = State::Finalized;
if(get_fork_gotcha())
if(get_main_bundle())
{
get_fork_gotcha()->stop();
std::cout << *get_fork_gotcha() << std::endl;
get_fork_gotcha().reset();
get_main_bundle()->stop();
int64_t _id = (get_use_mpi()) ? tim::dmp::rank() : tim::process::get_id();
std::stringstream _ss{};
_ss << "[" << __FUNCTION__ << "][" << _id << "] " << *get_main_bundle()
<< "\n";
std::cout << _ss.str();
get_main_bundle().reset();
}
if(!is_system_backend())
// ensure that all the MT instances are flushed
for(auto& itr : hosttrace_timemory_data::instances())
{
while(!itr.bundles.empty())
{
itr.bundles.back()->stop();
itr.bundles.back()->pop();
itr.allocator.destroy(itr.bundles.back());
itr.allocator.deallocate(itr.bundles.back(), 1);
itr.bundles.pop_back();
}
}
if(get_use_perfetto() && !is_system_backend())
{
// Make sure the last event is closed for this example.
perfetto::TrackEvent::Flush();
auto& tracing_session = get_trace_session();
tracing_session->StopBlocking();
std::vector<char> trace_data{ tracing_session->ReadTraceBlocking() };
if(trace_data.empty())
{
fprintf(stderr,
"[%s]> trace data is empty. File '%s' will not be written...\n",
__FUNCTION__, get_output_filename().c_str());
__FUNCTION__, get_perfetto_output_filename().c_str());
return;
}
// Write the trace into a file.
fprintf(stderr, "[%s]> Outputting '%s'. Trace data: %lu bytes...\n",
__FUNCTION__, get_output_filename().c_str(),
__FUNCTION__, get_perfetto_output_filename().c_str(),
(unsigned long) trace_data.size());
std::ofstream output{};
output.open(get_output_filename(), std::ios::out | std::ios::binary);
output.open(get_perfetto_output_filename(), std::ios::out | std::ios::binary);
if(!output)
fprintf(stderr, "[%s]> Error opening '%s'...\n", __FUNCTION__,
get_output_filename().c_str());
get_perfetto_output_filename().c_str());
else
output.write(&trace_data[0], trace_data.size());
output.close();
@@ -364,26 +452,33 @@ extern "C"
void hosttrace_trace_set_env(const char* env_name, const char* env_val)
{
if(get_debug())
fprintf(stderr, "[%s] Setting env: %s=%s\n", __FUNCTION__, env_name, env_val);
HOSTTRACE_DEBUG("[%s] Setting env: %s=%s\n", __FUNCTION__, env_name, env_val);
tim::set_env(env_name, env_val, 0);
}
void hosttrace_trace_set_mpi(bool use, bool attached)
{
HOSTTRACE_DEBUG("[%s] use: %s, attached: %s\n", __FUNCTION__, (use) ? "y" : "n",
(attached) ? "y" : "n");
if(use && !attached)
{
auto& _fork_gotcha = get_main_bundle();
_fork_gotcha->start();
tim::set_env("HOSTTRACE_USE_MPI", "ON", 1);
get_use_mpi() = true;
get_state() = State::DelayedInit;
}
}
}
void
fork_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
std::unique_ptr<hosttrace_bundle_t>&
get_main_bundle()
{
PRINT_HERE("%s",
"Warning! Calling fork() within an OpenMPI application using libfabric "
"may result is segmentation fault");
TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(get_debug(), 16);
}
void
fork_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, pid_t _pid)
{
PRINT_HERE("%s() return PID %i", _data.tool_id.c_str(), (int) _pid);
static auto _v =
(setup_gotchas(), std::make_unique<hosttrace_bundle_t>(
"hosttrace", quirk::config<quirk::auto_start>{}));
return _v;
}
namespace
@@ -393,10 +488,3 @@ namespace
// but static variable in hosttrace_init_perfetto is more likely
auto _ensure_finalization = ensure_finalization();
} // namespace
PERFETTO_TRACK_EVENT_STATIC_STORAGE();
TIMEMORY_INITIALIZE_STORAGE(fork_gotcha)
#if defined(CUSTOM_DATA_SOURCE)
PERFETTO_DEFINE_DATA_SOURCE_STATIC_MEMBERS(CustomDataSource);
#endif