AI/rocm-systems
2
0
Bifurcation 0
Code Tickets Demandes d'ajout Actions Paquets Projets Publications Wiki Activité

tracing NS + category region component + MPI args (#52)

Parcourir la source 
tracing NS + category region component

- made library.cpp impl more broadly available
- support for perfetto args
- MPI wrappers encode args and return type
- new categories / perfetto categories
- omnitrace_library category -> libomnitrace
- omnitrace_dl_library -> libomnitrace-dl
Cette révision appartient à :
Jonathan R. Madsen
2022-06-24 16:08:06 -05:00
révisé par GitHub
Parent 27e4e82376
révision 5105e2c94f
10 fichiers modifiés avec 582 ajouts et 254 suppressions
Télécharger le Fichier Patch Télécharger le Fichier des Différences Développer tous les fichiers Réduire tous les fichiers
source/lib/omnitrace/CMakeLists.txt
+3
Voir le fichier
@@ -72,6 +72,7 @@ set(library_sources
${CMAKE_CURRENT_LIST_DIR}/library/state.cpp
${CMAKE_CURRENT_LIST_DIR}/library/thread_data.cpp
${CMAKE_CURRENT_LIST_DIR}/library/timemory.cpp
${CMAKE_CURRENT_LIST_DIR}/library/tracing.cpp
${CMAKE_CURRENT_LIST_DIR}/library/components/backtrace.cpp
${CMAKE_CURRENT_LIST_DIR}/library/components/fork_gotcha.cpp
${CMAKE_CURRENT_LIST_DIR}/library/components/mpi_gotcha.cpp
@@ -102,9 +103,11 @@ set(library_headers
${CMAKE_CURRENT_LIST_DIR}/library/state.hpp
${CMAKE_CURRENT_LIST_DIR}/library/thread_data.hpp
${CMAKE_CURRENT_LIST_DIR}/library/timemory.hpp
${CMAKE_CURRENT_LIST_DIR}/library/tracing.hpp
${CMAKE_CURRENT_LIST_DIR}/library/utility.hpp
${CMAKE_CURRENT_LIST_DIR}/library/components/fwd.hpp
${CMAKE_CURRENT_LIST_DIR}/library/components/backtrace.hpp
${CMAKE_CURRENT_LIST_DIR}/library/components/category_region.hpp
${CMAKE_CURRENT_LIST_DIR}/library/components/fork_gotcha.hpp
${CMAKE_CURRENT_LIST_DIR}/library/components/functors.hpp
${CMAKE_CURRENT_LIST_DIR}/library/components/mpi_gotcha.hpp
source/lib/omnitrace/library.cpp
+69 -222
Voir le fichier
@@ -41,6 +41,7 @@
#include "library/sampling.hpp"
#include "library/thread_data.hpp"
#include "library/timemory.hpp"
#include "library/tracing.hpp"
#include <timemory/utility/procfs/maps.hpp>
@@ -49,7 +50,6 @@
#include <string_view>
using namespace omnitrace;
using tim::type_list;
//======================================================================================//
@@ -64,9 +64,6 @@ struct user_regions
using omni_functors = omnitrace::component::functors<omni_regions>;
using user_functors = omnitrace::component::functors<user_regions>;
TIMEMORY_DEFINE_NAME_TRAIT("host", omni_functors)
TIMEMORY_DEFINE_NAME_TRAIT("user", user_functors)
TIMEMORY_INVOKE_PREINIT(omni_functors)
TIMEMORY_INVOKE_PREINIT(user_functors)
@@ -74,31 +71,6 @@ TIMEMORY_INVOKE_PREINIT(user_functors)
namespace
{
using interval_data_instances = thread_data<std::vector<bool>>;
auto&
get_interval_data(int64_t _tid = threading::get_id())
{
static auto& _v =
interval_data_instances::instances(interval_data_instances::construct_on_init{});
return _v.at(_tid);
}
auto&
get_timemory_hash_ids(int64_t _tid = threading::get_id())
{
static auto _v = std::array<tim::hash_map_ptr_t, omnitrace::max_supported_threads>{};
return _v.at(_tid);
}
auto&
get_timemory_hash_aliases(int64_t _tid = threading::get_id())
{
static auto _v =
std::array<tim::hash_alias_ptr_t, omnitrace::max_supported_threads>{};
return _v.at(_tid);
}
auto
ensure_finalization(bool _static_init = false)
{
@@ -124,13 +96,6 @@ ensure_finalization(bool _static_init = false)
return scope::destructor{ []() { omnitrace_finalize_hidden(); } };
}
auto&
get_trace_session()
{
static std::unique_ptr<perfetto::TracingSession> _session{};
return _session;
}
auto
is_system_backend()
{
@@ -138,66 +103,28 @@ is_system_backend()
return (get_backend() != "inprocess");
}
auto&
get_instrumentation_bundles(int64_t _tid = threading::get_id())
{
static thread_local auto& _v = instrumentation_bundles::instances().at(_tid);
return _v;
}
auto&
get_finalization_functions()
{
static auto _v = std::vector<std::function<void()>>{};
return _v;
}
using Device = critical_trace::Device;
using Phase = critical_trace::Phase;
} // namespace
//======================================================================================//
///
///
///
//======================================================================================//
namespace
{
auto&
push_count()
{
static std::atomic<size_t> _v{ 0 };
return _v;
}
auto&
pop_count()
{
static std::atomic<size_t> _v{ 0 };
return _v;
}
auto _debug_push = tim::get_env("OMNITRACE_DEBUG_PUSH", false) && !get_debug_env();
auto _debug_pop = tim::get_env("OMNITRACE_DEBUG_POP", false) && !get_debug_env();
auto _debug_user =
tim::get_env("OMNITRACE_DEBUG_USER_REGIONS", false) && !get_debug_env();
} // namespace
//======================================================================================//
extern "C" void
omnitrace_push_trace_hidden(const char* name)
{
++push_count();
++tracing::push_count();
// unconditionally return if finalized
if(get_state() == State::Finalized)
{
OMNITRACE_CONDITIONAL_BASIC_PRINT(
_debug_push, "omnitrace_push_trace(%s) called during finalization\n", name);
tracing::debug_push, "omnitrace_push_trace(%s) called during finalization\n",
name);
return;
}
OMNITRACE_CONDITIONAL_BASIC_PRINT(_debug_push, "omnitrace_push_trace(%s)\n", name);
OMNITRACE_CONDITIONAL_BASIC_PRINT(tracing::debug_push, "omnitrace_push_trace(%s)\n",
name);
// the expectation here is that if the state is not active then the call
// to omnitrace_init_tooling_hidden will activate all the appropriate
@@ -215,7 +142,7 @@ omnitrace_push_trace_hidden(const char* name)
static auto _sample_rate = std::max<size_t>(get_instrumentation_interval(), 1);
static thread_local size_t _sample_idx = 0;
auto& _interval = get_interval_data();
auto& _interval = tracing::get_interval_data();
auto _enabled = (_sample_idx++ % _sample_rate == 0);
_interval->emplace_back(_enabled);
@@ -242,16 +169,17 @@ omnitrace_push_trace_hidden(const char* name)
extern "C" void
omnitrace_pop_trace_hidden(const char* name)
{
++pop_count();
++tracing::pop_count();
OMNITRACE_CONDITIONAL_BASIC_PRINT(_debug_pop, "omnitrace_pop_trace(%s)\n", name);
OMNITRACE_CONDITIONAL_BASIC_PRINT(tracing::debug_pop, "omnitrace_pop_trace(%s)\n",
name);
// only execute when active
if(get_state() == State::Active)
{
OMNITRACE_DEBUG("omnitrace_pop_trace(%s)\n", name);
auto& _interval_data = get_interval_data();
auto& _interval_data = tracing::get_interval_data();
if(!_interval_data->empty())
{
if(_interval_data->back()) omni_functors::stop(name);
@@ -298,11 +226,13 @@ omnitrace_push_region_hidden(const char* name)
if(get_state() == State::Finalized)
{
OMNITRACE_CONDITIONAL_BASIC_PRINT(
_debug_user, "omnitrace_push_region(%s) called during finalization\n", name);
tracing::debug_user, "omnitrace_push_region(%s) called during finalization\n",
name);
return;
}
OMNITRACE_CONDITIONAL_BASIC_PRINT(_debug_push, "omnitrace_push_region(%s)\n", name);
OMNITRACE_CONDITIONAL_BASIC_PRINT(tracing::debug_push, "omnitrace_push_region(%s)\n",
name);
// the expectation here is that if the state is not active then the call
// to omnitrace_init_tooling_hidden will activate all the appropriate
@@ -625,153 +555,65 @@ omnitrace_init_tooling_hidden()
(void) get_perfetto_output_filename();
}
auto _exe = get_exe_name();
auto _hash_ids = tim::get_hash_ids();
auto _hash_aliases = tim::get_hash_aliases();
auto _thread_init = [_exe, _hash_ids, _hash_aliases]() {
static thread_local auto _thread_setup = [_exe]() {
if(threading::get_id() > 0)
threading::set_thread_name(
TIMEMORY_JOIN(" ", "Thread", threading::get_id()).c_str());
thread_data<omnitrace_thread_bundle_t>::construct(
TIMEMORY_JOIN("", _exe, "/thread-", threading::get_id()),
quirk::config<quirk::auto_start>{});
get_interval_data()->reserve(512);
// save the hash maps
get_timemory_hash_ids() = tim::get_hash_ids();
get_timemory_hash_aliases() = tim::get_hash_aliases();
return true;
}();
static thread_local auto _dtor = scope::destructor{ []() {
if(get_state() != State::Finalized)
{
if(get_use_sampling()) sampling::shutdown();
auto& _thr_bundle = thread_data<omnitrace_thread_bundle_t>::instance();
if(_thr_bundle && _thr_bundle->get<comp::wall_clock>() &&
_thr_bundle->get<comp::wall_clock>()->get_is_running())
_thr_bundle->stop();
}
} };
(void) _thread_setup;
(void) _dtor;
};
// separate from _thread_init so that it can be called after the first
// instrumentation on the thread
auto _setup_thread_sampling = []() {
static thread_local auto _v = []() {
auto _use_sampling = get_use_sampling();
if(_use_sampling) sampling::setup();
return _use_sampling;
}();
(void) _v;
};
// functors for starting and stopping timemory omni functors
auto _push_timemory = [](const char* name) {
auto& _data = get_instrumentation_bundles();
// this generates a hash for the raw string array
auto _hash = tim::add_hash_id(tim::string_view_t{ name });
auto* _bundle = _data.allocator.allocate(1);
_data.bundles.emplace_back(_bundle);
_data.allocator.construct(_bundle, _hash);
_bundle->start();
};
auto _pop_timemory = [](const char* name) {
auto _hash = tim::hash::get_hash_id(tim::string_view_t{ name });
auto& _data = get_instrumentation_bundles();
if(_data.bundles.empty())
{
OMNITRACE_DEBUG("[%s] skipped %s :: empty bundle stack\n",
"omnitrace_pop_trace", name);
return;
}
for(size_t i = _data.bundles.size(); i > 0; --i)
{
auto*& _v = _data.bundles.at(i - 1);
if(_v->get_hash() == _hash)
{
_v->stop();
_data.allocator.destroy(_v);
_data.allocator.deallocate(_v, 1);
_data.bundles.erase(_data.bundles.begin() + (i - 1));
break;
}
}
};
// functors for starting and stopping perfetto omni functors
auto _push_perfetto = [](auto _category, const char* name) {
using CategoryT = std::decay_t<decltype(_category)>;
uint64_t _ts = comp::wall_clock::record();
TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name),
_ts, "begin_ns", _ts);
};
auto _pop_perfetto = [](auto _category, const char*) {
using CategoryT = std::decay_t<decltype(_category)>;
uint64_t _ts = comp::wall_clock::record();
TRACE_EVENT_END(trait::name<CategoryT>::value, _ts, "end_ns", _ts);
};
auto _exe = get_exe_name();
if(get_use_perfetto() && get_use_timemory())
{
omni_functors::configure(
[=](const char* name) {
_thread_init();
_push_perfetto(type_list<omni_functors>{}, name);
_push_timemory(name);
_setup_thread_sampling();
[](const char* name) {
tracing::thread_init();
tracing::push_perfetto(category::host{}, name);
tracing::push_timemory(name);
tracing::thread_init_sampling();
},
[=](const char* name) {
_pop_timemory(name);
_pop_perfetto(type_list<omni_functors>{}, name);
[](const char* name) {
tracing::pop_timemory(name);
tracing::pop_perfetto(category::host{}, name);
});
user_functors::configure(
[=](const char* name) {
_thread_init();
_push_perfetto(type_list<user_functors>{}, name);
_push_timemory(name);
[](const char* name) {
tracing::thread_init();
tracing::push_perfetto(category::user{}, name);
tracing::push_timemory(name);
},
[=](const char* name) {
_pop_timemory(name);
_pop_perfetto(type_list<user_functors>{}, name);
[](const char* name) {
tracing::pop_timemory(name);
tracing::pop_perfetto(category::user{}, name);
});
}
else if(get_use_perfetto())
{
omni_functors::configure(
[=](const char* name) {
_thread_init();
_push_perfetto(type_list<omni_functors>{}, name);
_setup_thread_sampling();
[](const char* name) {
tracing::thread_init();
tracing::push_perfetto(category::host{}, name);
tracing::thread_init_sampling();
},
[=](const char* name) { _pop_perfetto(type_list<omni_functors>{}, name); });
[](const char* name) { tracing::pop_perfetto(category::host{}, name); });
user_functors::configure(
[=](const char* name) {
_thread_init();
_push_perfetto(type_list<user_functors>{}, name);
_setup_thread_sampling();
[](const char* name) {
tracing::thread_init();
tracing::push_perfetto(category::user{}, name);
tracing::thread_init_sampling();
},
[=](const char* name) { _pop_perfetto(type_list<user_functors>{}, name); });
[](const char* name) { tracing::pop_perfetto(category::user{}, name); });
}
else if(get_use_timemory())
{
omni_functors::configure(
[=](const char* name) {
_thread_init();
_push_timemory(name);
_setup_thread_sampling();
[](const char* name) {
tracing::thread_init();
tracing::push_timemory(name);
tracing::thread_init_sampling();
},
[=](const char* name) { _pop_timemory(name); });
[](const char* name) { tracing::pop_timemory(name); });
user_functors::configure(
[=](const char* name) {
_thread_init();
_push_timemory(name);
_setup_thread_sampling();
[](const char* name) {
tracing::thread_init();
tracing::push_timemory(name);
tracing::thread_init_sampling();
},
[=](const char* name) { _pop_timemory(name); });
[](const char* name) { tracing::pop_timemory(name); });
}
if(get_use_ompt())
@@ -796,7 +638,7 @@ omnitrace_init_tooling_hidden()
PRINT_HERE("%s", "Trace");
});
#endif
auto& tracing_session = get_trace_session();
auto& tracing_session = tracing::get_trace_session();
tracing_session = perfetto::Tracing::NewTrace();
tracing_session->Setup(cfg);
tracing_session->StartBlocking();
@@ -842,8 +684,8 @@ omnitrace_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _a
// always the first
(void) get_state();
(void) push_count();
(void) pop_count();
(void) tracing::push_count();
(void) tracing::pop_count();
OMNITRACE_CONDITIONAL_THROW(
get_state() >= State::Init &&
@@ -853,7 +695,7 @@ omnitrace_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _a
_mode, std::to_string(_is_binary_rewrite).c_str(), _argv0,
std::to_string(get_state()).c_str());
get_finalization_functions().emplace_back([_argv0]() {
tracing::get_finalization_functions().emplace_back([_argv0]() {
OMNITRACE_CI_THROW(get_state() != State::Active,
"Finalizer function for popping main invoked in non-active "
"state :: state = %s\n",
@@ -914,13 +756,13 @@ omnitrace_finalize_hidden(void)
// some functions called during finalization may alter the push/pop count so we need
// to save them here
auto _push_count = push_count().load();
auto _pop_count = pop_count().load();
auto _push_count = tracing::push_count().load();
auto _pop_count = tracing::pop_count().load();
// e.g. omnitrace_pop_trace("main");
if(_push_count > _pop_count)
{
for(auto& itr : get_finalization_functions())
for(auto& itr : tracing::get_finalization_functions())
{
itr();
++_pop_count;
@@ -955,12 +797,12 @@ omnitrace_finalize_hidden(void)
OMNITRACE_DEBUG_F("Copying over all timemory hash information to main thread...\n");
// copy these over so that all hashes are known
auto& _hzero = get_timemory_hash_ids(0);
auto& _azero = get_timemory_hash_aliases(0);
auto& _hzero = tracing::get_timemory_hash_ids(0);
auto& _azero = tracing::get_timemory_hash_aliases(0);
for(size_t i = 1; i < max_supported_threads; ++i)
{
auto& _hitr = get_timemory_hash_ids(i);
auto& _aitr = get_timemory_hash_aliases(i);
auto& _hitr = tracing::get_timemory_hash_ids(i);
auto& _aitr = tracing::get_timemory_hash_aliases(i);
if(_hzero && _hitr)
{
for(const auto& itr : *_hitr)
@@ -1139,6 +981,11 @@ omnitrace_finalize_hidden(void)
bool _perfetto_output_error = false;
if(get_use_perfetto() && !is_system_backend())
{
auto& tracing_session = tracing::get_trace_session();
OMNITRACE_CI_THROW(tracing_session == nullptr,
"Null pointer to the tracing session");
if(get_verbose() >= 0) fprintf(stderr, "\n");
if(get_verbose() >= 0 || get_debug())
fprintf(stderr, "[%s][%s]|%i> Flushing perfetto...\n", TIMEMORY_PROJECT_NAME,
@@ -1146,9 +993,9 @@ omnitrace_finalize_hidden(void)
// Make sure the last event is closed for this example.
perfetto::TrackEvent::Flush();
tracing_session->FlushBlocking();
auto& tracing_session = get_trace_session();
OMNITRACE_VERBOSE_F(3, "Stopping the blocking perfetto trace sessions...\n");
OMNITRACE_VERBOSE_F(3, "Stopping the blocking perfetto trace session...\n");
tracing_session->StopBlocking();
using char_vec_t = std::vector<char>;
source/lib/omnitrace/library/common.hpp
+3 -2
Voir le fichier
@@ -48,8 +48,9 @@ TIMEMORY_DEFINE_NS_API(api, rocm_smi)
namespace omnitrace
{
namespace api = tim::api; // NOLINT
}
namespace api = tim::api; // NOLINT
namespace category = tim::category; // NOLINT
} // namespace omnitrace
// same sort of functionality as python's " ".join([...])
#if !defined(JOIN)
source/lib/omnitrace/library/components/category_region.hpp
+161
Voir le fichier
@@ -0,0 +1,161 @@
// MIT License
//
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#pragma once
#include "library/config.hpp"
#include "library/defines.hpp"
#include "library/timemory.hpp"
#include "library/tracing.hpp"
#include <timemory/components/gotcha/backends.hpp>
#include <timemory/utility/types.hpp>
namespace omnitrace
{
namespace audit = ::tim::audit;
namespace component
{
// timemory component which calls omnitrace functions
// (used in gotcha wrappers)
template <typename CategoryT>
struct category_region : comp::base<category_region<CategoryT>, void>
{
using gotcha_data_t = tim::component::gotcha_data;
static constexpr auto category_name = trait::name<CategoryT>::value;
static std::string label() { return JOIN('_', "omnitrace", category_name, "region"); }
template <typename... Args>
static void start(const char* name, Args&&...);
template <typename... Args>
static void stop(const char* name, Args&&...);
template <typename... Args>
static void audit(const gotcha_data_t&, audit::incoming, Args&&...);
template <typename... Args>
static void audit(const gotcha_data_t&, audit::outgoing, Args&&...);
};
template <typename CategoryT>
template <typename... Args>
void
category_region<CategoryT>::start(const char* name, Args&&... args)
{
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
// unconditionally return if finalized
if(get_state() == State::Finalized)
{
OMNITRACE_CONDITIONAL_BASIC_PRINT(
tracing::debug_user, "omnitrace_push_region(%s) called during finalization\n",
name);
return;
}
OMNITRACE_CONDITIONAL_BASIC_PRINT(tracing::debug_push, "omnitrace_push_region(%s)\n",
name);
// the expectation here is that if the state is not active then the call
// to omnitrace_init_tooling_hidden will activate all the appropriate
// tooling one time and as it exits set it to active and return true.
if(get_state() != State::Active && !omnitrace_init_tooling_hidden())
{
static auto _debug = get_debug_env() || get_debug_init();
OMNITRACE_CONDITIONAL_BASIC_PRINT(
_debug, "omnitrace_push_region(%s) ignored :: not active. state = %s\n", name,
std::to_string(get_state()).c_str());
return;
}
OMNITRACE_DEBUG("[%s] omnitrace_push_region(%s)\n", category_name, name);
auto _use_timemory = get_use_timemory();
auto _use_perfetto = get_use_perfetto();
if(_use_timemory || _use_perfetto) tracing::thread_init();
if(_use_perfetto)
{
tracing::push_perfetto(CategoryT{}, name, std::forward<Args>(args)...);
}
if(_use_timemory)
{
tracing::push_timemory(name, std::forward<Args>(args)...);
}
if(_use_timemory || _use_perfetto) tracing::thread_init_sampling();
}
template <typename CategoryT>
template <typename... Args>
void
category_region<CategoryT>::stop(const char* name, Args&&... args)
{
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
// only execute when active
if(get_state() == State::Active)
{
OMNITRACE_CONDITIONAL_PRINT(tracing::debug_pop || get_debug(),
"omnitrace_pop_region(%s)\n", name);
if(get_use_timemory())
{
tracing::pop_timemory(name, std::forward<Args>(args)...);
}
if(get_use_perfetto())
{
tracing::pop_perfetto(CategoryT{}, name, std::forward<Args>(args)...);
}
}
else
{
static auto _debug = get_debug_env();
OMNITRACE_CONDITIONAL_BASIC_PRINT(
_debug, "omnitrace_pop_region(%s) ignored :: state = %s\n", name,
std::to_string(get_state()).c_str());
}
}
template <typename CategoryT>
template <typename... Args>
void
category_region<CategoryT>::audit(const gotcha_data_t& _data, audit::incoming,
Args&&... _args)
{
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
start(_data.tool_id.c_str(), "args", JOIN(", ", _args...));
}
template <typename CategoryT>
template <typename... Args>
void
category_region<CategoryT>::audit(const gotcha_data_t& _data, audit::outgoing,
Args&&... _args)
{
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
stop(_data.tool_id.c_str(), "return", JOIN(", ", _args...));
}
} // namespace component
} // namespace omnitrace
source/lib/omnitrace/library/components/fwd.hpp
+24
Voir le fichier
@@ -60,6 +60,30 @@ TIMEMORY_DECLARE_TYPE_TRAIT(name, typename Tp)
} \
}
TIMEMORY_DEFINE_NS_API(category, host)
TIMEMORY_DEFINE_NS_API(category, device)
TIMEMORY_DEFINE_NS_API(category, user)
TIMEMORY_DEFINE_NS_API(category, rocm_smi)
TIMEMORY_DEFINE_NS_API(category, kokkos)
TIMEMORY_DEFINE_NS_API(category, mpi)
TIMEMORY_DEFINE_NS_API(category, ompt)
TIMEMORY_DEFINE_NS_API(category, critical_trace)
TIMEMORY_DEFINE_NS_API(category, host_critical_trace)
TIMEMORY_DEFINE_NS_API(category, device_critical_trace)
TIMEMORY_DEFINE_NAME_TRAIT("host", category::host);
TIMEMORY_DEFINE_NAME_TRAIT("device", category::device);
TIMEMORY_DEFINE_NAME_TRAIT("user", category::user);
TIMEMORY_DEFINE_NAME_TRAIT("rocm_smi", category::rocm_smi);
TIMEMORY_DEFINE_NAME_TRAIT("sampling", category::sampling);
TIMEMORY_DEFINE_NAME_TRAIT("thread_sampling", category::thread_sampling);
TIMEMORY_DEFINE_NAME_TRAIT("kokkos", category::kokkos);
TIMEMORY_DEFINE_NAME_TRAIT("mpi", category::mpi);
TIMEMORY_DEFINE_NAME_TRAIT("ompt", category::ompt);
TIMEMORY_DEFINE_NAME_TRAIT("critical-trace", category::critical_trace);
TIMEMORY_DEFINE_NAME_TRAIT("host-critical-trace", category::host_critical_trace);
TIMEMORY_DEFINE_NAME_TRAIT("device-critical-trace", category::device_critical_trace);
namespace omnitrace
{
namespace component
source/lib/omnitrace/library/components/mpi_gotcha.cpp
+16 -10
Voir le fichier
@@ -22,7 +22,7 @@
#include "library/components/mpi_gotcha.hpp"
#include "library/api.hpp"
#include "library/components/omnitrace.hpp"
#include "library/components/category_region.hpp"
#include "library/config.hpp"
#include "library/debug.hpp"
#include "library/mproc.hpp"
@@ -104,8 +104,10 @@ omnitrace_mpi_set_attr()
static auto _mpi_fini = [](MPI_Comm, int, void*, void*) {
OMNITRACE_DEBUG("MPI Comm attribute finalize\n");
if(mpip_index != std::numeric_limits<uint64_t>::max())
comp::deactivate_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
api::omnitrace>(mpip_index);
comp::deactivate_mpip<
tim::component_tuple<
omnitrace::component::category_region<category::mpi>>,
api::omnitrace>(mpip_index);
omnitrace_finalize_hidden();
return MPI_SUCCESS;
};
@@ -213,8 +215,9 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
OMNITRACE_BASIC_DEBUG_F("%s()\n", _data.tool_id.c_str());
if(mpip_index != std::numeric_limits<uint64_t>::max())
comp::deactivate_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
api::omnitrace>(mpip_index);
comp::deactivate_mpip<
tim::component_tuple<omnitrace::component::category_region<category::mpi>>,
api::omnitrace>(mpip_index);
#if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
tim::mpi::is_initialized_callback() = []() { return false; };
@@ -266,11 +269,14 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
// use env vars OMNITRACE_MPIP_PERMIT_LIST and OMNITRACE_MPIP_REJECT_LIST
// to control the gotcha bindings at runtime
comp::configure_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
api::omnitrace>();
mpip_index =
comp::activate_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
api::omnitrace>();
comp::configure_mpip<
tim::component_tuple<
omnitrace::component::category_region<category::mpi>>,
api::omnitrace>();
mpip_index = comp::activate_mpip<
tim::component_tuple<
omnitrace::component::category_region<category::mpi>>,
api::omnitrace>();
}
auto_lock_t _lk{ type_mutex<mpi_gotcha>() };
source/lib/omnitrace/library/config.cpp
+5 -6
Voir le fichier
@@ -76,7 +76,7 @@ get_setting_name(std::string _v)
[&]() { \
auto _ret = _config->insert<TYPE, TYPE>( \
ENV_NAME, get_setting_name(ENV_NAME), DESCRIPTION, INITIAL_VALUE, \
std::set<std::string>{ "custom", "omnitrace", "omnitrace_library", \
std::set<std::string>{ "custom", "omnitrace", "libomnitrace", \
__VA_ARGS__ }); \
if(!_ret.second) \
OMNITRACE_PRINT("Warning! Duplicate setting: %s / %s\n", \
@@ -84,7 +84,7 @@ get_setting_name(std::string _v)
return _config->find(ENV_NAME)->second; \
}()
// below does not include "omnitrace_library"
// below does not include "libomnitrace"
#define OMNITRACE_CONFIG_EXT_SETTING(TYPE, ENV_NAME, DESCRIPTION, INITIAL_VALUE, ...) \
[&]() { \
auto _ret = _config->insert<TYPE, TYPE>( \
@@ -102,8 +102,7 @@ get_setting_name(std::string _v)
[&]() { \
auto _ret = _config->insert<TYPE, TYPE>( \
ENV_NAME, get_setting_name(ENV_NAME), DESCRIPTION, INITIAL_VALUE, \
std::set<std::string>{ "custom", "omnitrace", "omnitrace_library", \
__VA_ARGS__ }, \
std::set<std::string>{ "custom", "omnitrace", "libomnitrace", __VA_ARGS__ }, \
std::vector<std::string>{ CMD_LINE }); \
if(!_ret.second) \
OMNITRACE_PRINT("Warning! Duplicate setting: %s / %s\n", \
@@ -176,7 +175,7 @@ configure_settings(bool _init)
OMNITRACE_CONFIG_EXT_SETTING(bool, "OMNITRACE_DL_VERBOSE",
"Verbosity within the omnitrace-dl library", false,
"debugging", "omnitrace_dl_library");
"debugging", "libomnitrace-dl");
OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_PERFETTO", "Enable perfetto backend",
_default_perfetto_v, "backend", "perfetto");
@@ -399,7 +398,7 @@ configure_settings(bool _init)
{
auto _categories = itr->second->get_categories();
_categories.emplace("omnitrace");
_categories.emplace("omnitrace_library");
_categories.emplace("libomnitrace");
itr->second->set_categories(_categories);
}
};
source/lib/omnitrace/library/perfetto.hpp
+40 -14
Voir le fichier
@@ -37,6 +37,11 @@
perfetto::Category("rocm_smi").SetDescription("Device-level metrics"), \
perfetto::Category("sampling") \
.SetDescription("Metrics derived from sampling"), \
perfetto::Category("thread_sampling") \
.SetDescription("Metrics derived from background thread sampling"), \
perfetto::Category("mpi").SetDescription("MPI regions"), \
perfetto::Category("kokkos").SetDescription("Kokkos regions"), \
perfetto::Category("ompt").SetDescription("OpenMP Tools regions"), \
perfetto::Category("critical-trace") \
.SetDescription("Combined critical traces"), \
perfetto::Category("host-critical-trace") \
@@ -51,6 +56,11 @@
perfetto::Category("rocm_smi").SetDescription("Device-level metrics"), \
perfetto::Category("sampling") \
.SetDescription("Metrics derived from sampling"), \
perfetto::Category("thread_sampling") \
.SetDescription("Metrics derived from background thread sampling"), \
perfetto::Category("mpi").SetDescription("MPI regions"), \
perfetto::Category("kokkos").SetDescription("Kokkos regions"), \
perfetto::Category("ompt").SetDescription("OpenMP Tools regions"), \
perfetto::Category("critical-trace") \
.SetDescription("Combined critical traces"), \
perfetto::Category("host-critical-trace") \
@@ -136,30 +146,30 @@ struct perfetto_counter_track
const char* _category = nullptr, int64_t _mult = 1,
bool _incr = false)
{
auto& _name_data = get_data().first[_idx];
auto& _track_data = get_data().second[_idx];
std::vector<std::tuple<std::string, const char*, bool>> _missing = {};
if(config::get_is_continuous_integration())
{
for(const auto& itr : get_data().first[_idx])
for(const auto& itr : _name_data)
{
_missing.emplace_back(std::make_tuple(*itr, itr->c_str(), false));
}
}
auto& _name =
get_data().first[_idx].emplace_back(std::make_unique<std::string>(_v));
auto& _name = _name_data.emplace_back(std::make_unique<std::string>(_v));
const char* _unit_name = (_units && strlen(_units) > 0) ? _units : nullptr;
get_data().second[_idx].emplace_back(perfetto::CounterTrack{ _name->c_str() }
.set_unit_name(_unit_name)
.set_category(_category)
.set_unit_multiplier(_mult)
.set_is_incremental(_incr));
_track_data.emplace_back(perfetto::CounterTrack{ _name->c_str() }
.set_unit_name(_unit_name)
.set_category(_category)
.set_unit_multiplier(_mult)
.set_is_incremental(_incr));
if(config::get_is_continuous_integration())
{
for(auto& itr : _missing)
{
for(const auto& ditr : get_data().first.at(_idx))
const char* citr = std::get<1>(itr);
for(const auto& ditr : _name_data)
{
if(*ditr == _v) continue;
const char* citr = std::get<1>(itr);
if(citr == ditr->c_str() && strcmp(citr, ditr->c_str()) == 0)
{
std::get<2>(itr) = true;
@@ -168,9 +178,25 @@ struct perfetto_counter_track
}
if(!std::get<2>(itr))
{
OMNITRACE_THROW("perfetto_counter_track emplace method for '%s' "
"invalidated C-string '%s'\n",
_v.c_str(), std::get<0>(itr).c_str());
std::set<void*> _prev = {};
std::set<void*> _curr = {};
for(const auto& eitr : _missing)
_prev.emplace(
static_cast<void*>(const_cast<char*>(std::get<1>(eitr))));
for(const auto& eitr : _name_data)
_curr.emplace(
static_cast<void*>(const_cast<char*>(eitr->c_str())));
std::stringstream _pss{};
for(auto&& eitr : _prev)
_pss << " " << std::hex << std::setw(12) << std::left << eitr;
std::stringstream _css{};
for(auto&& eitr : _curr)
_css << " " << std::hex << std::setw(12) << std::left << eitr;
OMNITRACE_THROW("perfetto_counter_track emplace method for '%s' (%p) "
"invalidated C-string '%s' (%p).\n%8s: %s\n%8s: %s\n",
_v.c_str(), _name->c_str(), std::get<0>(itr).c_str(),
std::get<0>(itr).c_str(), "previous",
_pss.str().c_str(), "current", _css.str().c_str());
}
}
}
source/lib/omnitrace/library/tracing.cpp
+58
Voir le fichier
@@ -0,0 +1,58 @@
// MIT License
//
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "library/tracing.hpp"
namespace omnitrace
{
namespace tracing
{
std::unique_ptr<perfetto::TracingSession>&
get_trace_session()
{
static auto _session = std::unique_ptr<perfetto::TracingSession>{};
return _session;
}
std::vector<std::function<void()>>&
get_finalization_functions()
{
static auto _v = std::vector<std::function<void()>>{};
return _v;
}
tim::hash_map_ptr_t&
get_timemory_hash_ids(int64_t _tid)
{
static auto _v = std::array<tim::hash_map_ptr_t, omnitrace::max_supported_threads>{};
return _v.at(_tid);
}
tim::hash_alias_ptr_t&
get_timemory_hash_aliases(int64_t _tid)
{
static auto _v =
std::array<tim::hash_alias_ptr_t, omnitrace::max_supported_threads>{};
return _v.at(_tid);
}
} // namespace tracing
} // namespace omnitrace
source/lib/omnitrace/library/tracing.hpp
+203
Voir le fichier
@@ -0,0 +1,203 @@
// MIT License
//
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#pragma once
#include "library/config.hpp"
#include "library/debug.hpp"
#include "library/defines.hpp"
#include "library/perfetto.hpp"
#include "library/runtime.hpp"
#include "library/sampling.hpp"
#include "library/timemory.hpp"
namespace omnitrace
{
namespace tracing
{
using interval_data_instances = thread_data<std::vector<bool>>;
std::unique_ptr<perfetto::TracingSession>&
get_trace_session();
std::vector<std::function<void()>>&
get_finalization_functions();
tim::hash_map_ptr_t&
get_timemory_hash_ids(int64_t _tid = threading::get_id());
tim::hash_alias_ptr_t&
get_timemory_hash_aliases(int64_t _tid = threading::get_id());
namespace
{
bool debug_push = // NOLINT
tim::get_env("OMNITRACE_DEBUG_PUSH", false) && !get_debug_env();
bool debug_pop = // NOLINT
tim::get_env("OMNITRACE_DEBUG_POP", false) && !get_debug_env();
bool debug_user = // NOLINT
tim::get_env("OMNITRACE_DEBUG_USER_REGIONS", false) && !get_debug_env();
} // namespace
inline auto&
get_interval_data(int64_t _tid = threading::get_id())
{
static auto& _v =
interval_data_instances::instances(interval_data_instances::construct_on_init{});
return _v.at(_tid);
}
inline auto&
get_instrumentation_bundles(int64_t _tid = threading::get_id())
{
static thread_local auto& _v = instrumentation_bundles::instances().at(_tid);
return _v;
}
inline auto&
push_count()
{
static std::atomic<size_t> _v{ 0 };
return _v;
}
inline auto&
pop_count()
{
static std::atomic<size_t> _v{ 0 };
return _v;
}
inline void
thread_init()
{
static thread_local auto _thread_setup = []() {
auto _exe = get_exe_name();
if(threading::get_id() > 0)
threading::set_thread_name(JOIN(" ", "Thread", threading::get_id()).c_str());
thread_data<omnitrace_thread_bundle_t>::construct(
JOIN("", _exe, "/thread-", threading::get_id()),
quirk::config<quirk::auto_start>{});
get_interval_data()->reserve(512);
// save the hash maps
get_timemory_hash_ids() = tim::get_hash_ids();
get_timemory_hash_aliases() = tim::get_hash_aliases();
return true;
}();
static thread_local auto _dtor = scope::destructor{ []() {
if(get_state() != State::Finalized)
{
if(get_use_sampling()) sampling::shutdown();
auto& _thr_bundle = thread_data<omnitrace_thread_bundle_t>::instance();
if(_thr_bundle && _thr_bundle->get<comp::wall_clock>() &&
_thr_bundle->get<comp::wall_clock>()->get_is_running())
_thr_bundle->stop();
}
} };
(void) _thread_setup;
(void) _dtor;
}
inline void
thread_init_sampling()
{
static thread_local auto _v = []() {
auto _use_sampling = get_use_sampling();
if(_use_sampling) sampling::setup();
return _use_sampling;
}();
(void) _v;
}
template <typename... Args>
inline void
push_timemory(const char* name, Args&&... args)
{
auto& _data = tracing::get_instrumentation_bundles();
// this generates a hash for the raw string array
auto _hash = tim::add_hash_id(tim::string_view_t{ name });
auto* _bundle = _data.allocator.allocate(1);
_data.bundles.emplace_back(_bundle);
_data.allocator.construct(_bundle, _hash);
_bundle->start(std::forward<Args>(args)...);
}
template <typename... Args>
inline void
pop_timemory(const char* name, Args&&... args)
{
auto _hash = tim::hash::get_hash_id(tim::string_view_t{ name });
auto& _data = tracing::get_instrumentation_bundles();
if(_data.bundles.empty())
{
OMNITRACE_DEBUG("[%s] skipped %s :: empty bundle stack\n", "omnitrace_pop_trace",
name);
return;
}
for(size_t i = _data.bundles.size(); i > 0; --i)
{
auto*& _v = _data.bundles.at(i - 1);
if(_v->get_hash() == _hash)
{
_v->stop(std::forward<Args>(args)...);
_data.allocator.destroy(_v);
_data.allocator.deallocate(_v, 1);
_data.bundles.erase(_data.bundles.begin() + (i - 1));
break;
}
}
}
template <typename CategoryT, typename... Args>
inline void
push_perfetto(CategoryT, const char* name, Args&&... args)
{
uint64_t _ts = comp::wall_clock::record();
TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name), _ts,
std::forward<Args>(args)..., "begin_ns", _ts);
}
template <typename CategoryT, typename... Args>
inline void
pop_perfetto(CategoryT, const char*, Args&&... args)
{
uint64_t _ts = comp::wall_clock::record();
TRACE_EVENT_END(trait::name<CategoryT>::value, _ts, std::forward<Args>(args)...,
"end_ns", _ts);
}
template <typename CategoryT, typename... Args>
inline void
push_perfetto_ts(CategoryT, const char* name, uint64_t _ts, Args&&... args)
{
TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name), _ts,
std::forward<Args>(args)...);
}
template <typename CategoryT, typename... Args>
inline void
pop_perfetto_ts(CategoryT, const char*, uint64_t _ts, Args&&... args)
{
TRACE_EVENT_END(trait::name<CategoryT>::value, _ts, std::forward<Args>(args)...);
}
} // namespace tracing
} // namespace omnitrace