AI/rocm-systems
2
0
Fork 0
Koodi Ongelmat Pull-pyynnöt Actions Paketit Projektit Julkaisut Wiki Toiminta

HIP API args in perfetto + new perfetto categories (#76)

Selaa lähdekoodia 
* HIP API perfetto args + updated perfetto categories

- Support for HIP API args field in perfetto
- PERFETTO_CATEGORIES -> OMNITRACE_PERFETTO_CATEGORIES
- Changed perfetto categories for several trace events and trace counters
- migrated several TRACE_EVENT_* to use omnitrace::tracing::{push,pop}_perfetto_ts(...)

* Tweaked category_region to encode the type of args as well as value

- Affects MPI args field in perfetto

* Improved testing in ubuntu-focal.yml

- "Test Install" step sources setup-env.sh
- "Test Install" step tests python support
- "Test Install" step tests reading ~/.omnitrace.cfg
- Avoid installing boost and tbb libs when building from submodule

* validate-perfetto-proto.py accepts -m / --categories

* Remove reference from category_region typeids

* Tweak opensuse action name

* Tweak the "Test Install" Step of ubuntu-focal
This commit is contained in:
Jonathan R. Madsen
2022-06-29 16:26:02 -05:00
committed by GitHub
vanhempi 9b800c028b
commit 2e1fd5a3c4
12 muutettua tiedostoa jossa 203 lisäystä ja 103 poistoa
Download Patch File Download Diff File Expand all files Collapse all files
source/lib/omnitrace/library/components/backtrace.cpp
+10 -10
Näytä tiedosto
@@ -516,15 +516,15 @@ backtrace::post_process(int64_t _tid)
_last_bt = _bt;
_mean_ts += _ts;
TRACE_COUNTER("sampling",
TRACE_COUNTER("thread_peak_memory",
perfetto_counter_track<perfetto_rusage>::at(_tid, 0), _ts,
_bt->m_mem_peak / units::megabyte);
TRACE_COUNTER("sampling",
TRACE_COUNTER("thread_context_switch",
perfetto_counter_track<perfetto_rusage>::at(_tid, 1), _ts,
_bt->m_ctx_swch);
TRACE_COUNTER("sampling",
TRACE_COUNTER("thread_page_fault",
perfetto_counter_track<perfetto_rusage>::at(_tid, 2), _ts,
_bt->m_page_flt);
@@ -535,7 +535,7 @@ backtrace::post_process(int64_t _tid)
{
if(i < _bt->m_hw_counter.size())
{
TRACE_COUNTER("sampling",
TRACE_COUNTER("hardware_counter",
perfetto_counter_track<hw_counters>::at(_tid, i),
_ts, _bt->m_hw_counter.at(i));
}
@@ -547,15 +547,15 @@ backtrace::post_process(int64_t _tid)
{
auto _ts = pthread_create_gotcha::get_execution_time(_tid)->second;
uint64_t _zero = 0;
TRACE_COUNTER("sampling",
TRACE_COUNTER("thread_peak_memory",
perfetto_counter_track<perfetto_rusage>::at(_tid, 0), _ts,
_zero);
TRACE_COUNTER("sampling",
TRACE_COUNTER("thread_context_switch",
perfetto_counter_track<perfetto_rusage>::at(_tid, 1), _ts,
_zero);
TRACE_COUNTER("sampling",
TRACE_COUNTER("thread_page_fault",
perfetto_counter_track<perfetto_rusage>::at(_tid, 2), _ts,
_zero);
@@ -566,7 +566,7 @@ backtrace::post_process(int64_t _tid)
{
if(i < _last_bt->m_hw_counter.size())
{
TRACE_COUNTER("sampling",
TRACE_COUNTER("hardware_counter",
perfetto_counter_track<hw_counters>::at(_tid, i),
_ts, _zero);
}
@@ -601,10 +601,10 @@ backtrace::post_process(int64_t _tid)
auto _ts = _bt->m_ts;
if(!pthread_create_gotcha::is_valid_execution_time(_tid, _ts)) continue;
TRACE_EVENT_BEGIN("sampling",
TRACE_EVENT_BEGIN("hardware_counter",
perfetto::StaticString{ sitr.first->c_str() },
_last_wall_ts, "begin_ns", _last_wall_ts);
TRACE_EVENT_END("sampling", _ts, "end_ns", _ts);
TRACE_EVENT_END("hardware_counter", _ts, "end_ns", _ts);
}
_last_wall_ts = _bt->m_ts;
}
source/lib/omnitrace/library/components/category_region.hpp
+3 -1
Näytä tiedosto
@@ -145,7 +145,9 @@ category_region<CategoryT>::audit(const gotcha_data_t& _data, audit::incoming,
Args&&... _args)
{
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
start(_data.tool_id.c_str(), "args", JOIN(", ", _args...));
start(_data.tool_id.c_str(), "args",
JOIN(", ",
JOIN('=', tim::try_demangle<std::remove_reference_t<Args>>(), _args)...));
}
template <typename CategoryT>
source/lib/omnitrace/library/components/fwd.hpp
+9 -1
Näytä tiedosto
@@ -61,8 +61,12 @@ TIMEMORY_DECLARE_TYPE_TRAIT(name, typename Tp)
}
TIMEMORY_DEFINE_NS_API(category, host)
TIMEMORY_DEFINE_NS_API(category, device)
TIMEMORY_DEFINE_NS_API(category, user)
TIMEMORY_DEFINE_NS_API(category, device)
TIMEMORY_DEFINE_NS_API(category, device_hip)
TIMEMORY_DEFINE_NS_API(category, device_hsa)
TIMEMORY_DEFINE_NS_API(category, rocm_hip)
TIMEMORY_DEFINE_NS_API(category, rocm_hsa)
TIMEMORY_DEFINE_NS_API(category, rocm_smi)
TIMEMORY_DEFINE_NS_API(category, kokkos)
TIMEMORY_DEFINE_NS_API(category, mpi)
@@ -73,7 +77,11 @@ TIMEMORY_DEFINE_NS_API(category, device_critical_trace)
TIMEMORY_DEFINE_NAME_TRAIT("host", category::host);
TIMEMORY_DEFINE_NAME_TRAIT("device", category::device);
TIMEMORY_DEFINE_NAME_TRAIT("device_hip", category::device_hip);
TIMEMORY_DEFINE_NAME_TRAIT("device_hsa", category::device_hsa);
TIMEMORY_DEFINE_NAME_TRAIT("user", category::user);
TIMEMORY_DEFINE_NAME_TRAIT("rocm_hip", category::rocm_hip);
TIMEMORY_DEFINE_NAME_TRAIT("rocm_hsa", category::rocm_hsa);
TIMEMORY_DEFINE_NAME_TRAIT("rocm_smi", category::rocm_smi);
TIMEMORY_DEFINE_NAME_TRAIT("sampling", category::sampling);
TIMEMORY_DEFINE_NAME_TRAIT("thread_sampling", category::thread_sampling);
source/lib/omnitrace/library/components/rocm_smi.cpp
+5 -4
Näytä tiedosto
@@ -246,10 +246,11 @@ data::post_process(uint32_t _dev_id)
double _temp = itr.m_temp / 1.0e3;
double _power = itr.m_power / 1.0e6;
double _usage = itr.m_mem_usage / static_cast<double>(units::megabyte);
TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 0), _ts, _busy);
TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 1), _ts, _temp);
TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 2), _ts, _power);
TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 3), _ts, _usage);
TRACE_COUNTER("device_busy", counter_track::at(_dev_id, 0), _ts, _busy);
TRACE_COUNTER("device_temp", counter_track::at(_dev_id, 1), _ts, _temp);
TRACE_COUNTER("device_power", counter_track::at(_dev_id, 2), _ts, _power);
TRACE_COUNTER("device_memory_usage", counter_track::at(_dev_id, 3), _ts,
_usage);
}
};
source/lib/omnitrace/library/components/roctracer_callbacks.cpp
+57 -24
Näytä tiedosto
@@ -28,6 +28,7 @@
#include "library/runtime.hpp"
#include "library/sampling.hpp"
#include "library/thread_data.hpp"
#include "library/tracing.hpp"
#include <timemory/backends/cpu.hpp>
#include <timemory/backends/threading.hpp>
@@ -38,6 +39,8 @@
#include <cstdint>
#include <tuple>
#define HIP_PROF_HIP_API_STRING 1
#include <roctracer_ext.h>
#include <roctracer_hcc.h>
#include <roctracer_hip.h>
@@ -45,12 +48,36 @@
#define AMD_INTERNAL_BUILD 1
#include <roctracer_hsa.h>
#if __has_include(<hip/amd_detail/hip_prof_str.h>) || (defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0)
# include <hip/amd_detail/hip_prof_str.h>
# define OMNITRACE_HIP_API_ARGS 1
#else
# define OMNITRACE_HIP_API_ARGS 0
#endif
TIMEMORY_DEFINE_API(roctracer)
namespace omnitrace
{
namespace api = tim::api;
namespace
{
std::string
hip_api_string(hip_api_id_t id, const hip_api_data_t* data)
{
#if OMNITRACE_HIP_API_ARGS > 0
std::string _v = hipApiString(id, data);
if(_v.empty()) return _v;
auto _pbeg = _v.find('(');
if(_pbeg == std::string::npos) return _v;
auto _pend = _v.find_last_of(')');
if(_pend == std::string::npos || _pbeg >= _pend) return _v;
auto _n = (_pend - _pbeg - 1);
return _v.substr(_pbeg + 1, _n);
#else
tim::consume_parameters(id, data);
#endif
}
//
int64_t
get_clock_skew()
{
@@ -275,11 +302,12 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
if(get_use_perfetto())
{
TRACE_EVENT_BEGIN("device", perfetto::StaticString{ _name },
static_cast<uint64_t>(begin_timestamp), "begin_ns",
static_cast<uint64_t>(begin_timestamp));
TRACE_EVENT_END("device", static_cast<uint64_t>(end_timestamp),
"end_ns", static_cast<uint64_t>(end_timestamp));
uint64_t _beg_ts = begin_timestamp;
uint64_t _end_ts = end_timestamp;
tracing::push_perfetto_ts(category::rocm_hsa{}, _name, _beg_ts,
"begin_ns", _beg_ts);
tracing::pop_perfetto_ts(category::rocm_hsa{}, _name, _end_ts,
"end_ns", _end_ts);
}
if(get_use_timemory())
@@ -334,6 +362,9 @@ hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
default: break;
}
OMNITRACE_CI_FAIL(_name == nullptr, "Error! HSA operation type not handled: %u\n",
op);
if(!_name) return;
auto _beg_ns = record->begin_ns + get_clock_skew();
@@ -347,11 +378,10 @@ hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
if(get_use_perfetto())
{
TRACE_EVENT_BEGIN("device", perfetto::StaticString{ *_name },
static_cast<uint64_t>(_beg_ns), "begin_ns",
static_cast<uint64_t>(_beg_ns));
TRACE_EVENT_END("device", static_cast<uint64_t>(_end_ns), "end_ns",
static_cast<uint64_t>(_end_ns));
uint64_t _beg = _beg_ns;
uint64_t _end = _end_ns;
tracing::push_perfetto_ts(category::device_hsa{}, *_name, _beg, "begin_ns", _beg);
tracing::pop_perfetto_ts(category::device_hsa{}, *_name, _end, "end_ns", _end);
}
auto _func = [_beg_ns, _end_ns, _name]() {
@@ -582,11 +612,12 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
if(get_use_perfetto())
{
TRACE_EVENT_BEGIN(
"host", perfetto::StaticString{ op_name }, static_cast<uint64_t>(_ts),
perfetto::Flow::ProcessScoped(_cid), "begin_ns",
static_cast<uint64_t>(_ts), "pcid", _parent_cid, "cid", _cid, "device",
_device_id, "tid", _tid, "depth", _depth, "corr_id", _corr_id);
auto _api_id = static_cast<hip_api_id_t>(cid);
tracing::push_perfetto_ts(
category::rocm_hip{}, op_name, _ts, perfetto::Flow::ProcessScoped(_cid),
"begin_ns", static_cast<uint64_t>(_ts), "pcid", _parent_cid, "cid", _cid,
"device", _device_id, "tid", _tid, "depth", _depth, "corr_id", _corr_id,
"args", hip_api_string(_api_id, data));
}
if(get_use_timemory())
{
@@ -622,8 +653,8 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
if(get_use_perfetto())
{
TRACE_EVENT_END("host", static_cast<uint64_t>(_ts), "end_ns",
static_cast<uint64_t>(_ts));
tracing::pop_perfetto_ts(category::rocm_hip{}, op_name, _ts, "end_ns",
static_cast<uint64_t>(_ts));
}
if(get_use_timemory())
{
@@ -779,14 +810,16 @@ hip_activity_callback(const char* begin, const char* end, void*)
_kernel_names.emplace(_name, tim::demangle(_name));
assert(_end_ns > _beg_ns);
TRACE_EVENT_BEGIN("device",
perfetto::StaticString{ _kernel_names.at(_name).c_str() },
_beg_ns, perfetto::Flow::ProcessScoped(_cid), "begin_ns",
_beg_ns, "corr_id", record->correlation_id, "device",
_devid, "queue", _queid, "op", _op_id_names.at(record->op));
TRACE_EVENT_END("device", _end_ns, "end_ns", _end_ns);
tracing::push_perfetto_ts(
category::device_hip{}, _kernel_names.at(_name).c_str(), _beg_ns,
perfetto::Flow::ProcessScoped(_cid), "begin_ns", _beg_ns, "corr_id",
record->correlation_id, "device", _devid, "queue", _queid, "op",
_op_id_names.at(record->op));
tracing::pop_perfetto_ts(category::device_hip{}, "", _end_ns, "end_ns",
_end_ns);
// for some reason, this is necessary to make sure very last one ends
TRACE_EVENT_END("device", _end_ns, "end_ns", _end_ns);
tracing::pop_perfetto_ts(category::device_hip{}, "", _end_ns, "end_ns",
_end_ns);
}
if(_critical_trace)
source/lib/omnitrace/library/cpu_freq.cpp
+20 -3
Näytä tiedosto
@@ -80,7 +80,24 @@ void init_perfetto_counter_tracks(type_list<Types...>)
(perfetto_counter_track<Types>::init(), ...);
}
} // namespace
} // namespace cpu_freq
} // namespace omnitrace
TIMEMORY_DEFINE_NAME_TRAIT("cpu_freq", omnitrace::cpu_freq::cpu_freq);
TIMEMORY_DEFINE_NAME_TRAIT("process_page_fault", omnitrace::cpu_freq::cpu_page);
TIMEMORY_DEFINE_NAME_TRAIT("process_virtual_memory", omnitrace::cpu_freq::cpu_virt);
TIMEMORY_DEFINE_NAME_TRAIT("process_context_switch",
omnitrace::cpu_freq::cpu_context_switch);
TIMEMORY_DEFINE_NAME_TRAIT("process_page_fault", omnitrace::cpu_freq::cpu_page_fault);
TIMEMORY_DEFINE_NAME_TRAIT("process_user_cpu_time",
omnitrace::cpu_freq::cpu_user_mode_time);
TIMEMORY_DEFINE_NAME_TRAIT("process_kernel_cpu_time",
omnitrace::cpu_freq::cpu_kernel_mode_time);
namespace omnitrace
{
namespace cpu_freq
{
void
setup()
{
@@ -230,7 +247,7 @@ void
write_perfetto_counter_track(Args... _args)
{
using track = perfetto_counter_track<Tp>;
TRACE_COUNTER("sampling", track::at(0, 0), _args...);
TRACE_COUNTER(trait::name<Tp>::value, track::at(0, 0), _args...);
}
} // namespace
@@ -254,11 +271,11 @@ post_process()
uint64_t _ts = std::get<0>(itr);
double _freq = std::get<7>(itr).at(_offset);
if(!pthread_create_gotcha::is_valid_execution_time(0, _ts)) continue;
TRACE_COUNTER("sampling", freq_track::at(_idx, 0), _ts, _freq);
TRACE_COUNTER("cpu_freq", freq_track::at(_idx, 0), _ts, _freq);
}
auto _end_ts = pthread_create_gotcha::get_execution_time(0)->second;
TRACE_COUNTER("sampling", freq_track::at(_idx, 0), _end_ts, 0);
TRACE_COUNTER("cpu_freq", freq_track::at(_idx, 0), _end_ts, 0);
};
auto _process_cpu_rusage = []() {
source/lib/omnitrace/library/perfetto.hpp
+63 -47
Näytä tiedosto
@@ -24,56 +24,72 @@
#include "library/defines.hpp"
#if defined(PERFETTO_CATEGORIES)
# error "PERFETTO_CATEGORIES is already defined. Please include \"" __FILE__ "\" before including any timemory files"
#if defined(OMNITRACE_PERFETTO_CATEGORIES)
# error "OMNITRACE_PERFETTO_CATEGORIES is already defined. Please include \"" __FILE__ "\" before including any timemory files"
#endif
#if !defined(TIMEMORY_USE_PERFETTO)
# include <perfetto.h>
# define PERFETTO_CATEGORIES \
perfetto::Category("host").SetDescription("Host-side function tracing"), \
perfetto::Category("device").SetDescription("Device-side function tracing"), \
perfetto::Category("user").SetDescription("User-defined regions"), \
perfetto::Category("rocm_smi").SetDescription("Device-level metrics"), \
perfetto::Category("sampling") \
.SetDescription("Metrics derived from sampling"), \
perfetto::Category("thread_sampling") \
.SetDescription("Metrics derived from background thread sampling"), \
perfetto::Category("mpi").SetDescription("MPI regions"), \
perfetto::Category("kokkos").SetDescription("Kokkos regions"), \
perfetto::Category("ompt").SetDescription("OpenMP Tools regions"), \
perfetto::Category("critical-trace") \
.SetDescription("Combined critical traces"), \
perfetto::Category("host-critical-trace") \
.SetDescription("Host-side critical traces"), \
perfetto::Category("device-critical-trace") \
.SetDescription("Device-side critical traces")
#define OMNITRACE_PERFETTO_CATEGORIES \
perfetto::Category("host").SetDescription("Host-side function tracing"), \
perfetto::Category("user").SetDescription("User-defined regions"), \
perfetto::Category("device_hip") \
.SetDescription("Device-side functions submitted via HSA API"), \
perfetto::Category("device_hsa") \
.SetDescription("Device-side functions submitted via HIP API"), \
perfetto::Category("rocm_hip").SetDescription("Host-side HIP functions"), \
perfetto::Category("rocm_hsa").SetDescription("Host-side HSA functions"), \
perfetto::Category("device_busy") \
.SetDescription("Busy percentage of a GPU device"), \
perfetto::Category("device_temp") \
.SetDescription("Temperature of GPU device in degC"), \
perfetto::Category("device_power") \
.SetDescription("Power consumption of GPU device in watts"), \
perfetto::Category("device_memory_usage") \
.SetDescription("Memory usage of GPU device in MB"), \
perfetto::Category("thread_peak_memory") \
.SetDescription( \
"Peak memory usage on thread in MB (derived from sampling)"), \
perfetto::Category("thread_context_switch") \
.SetDescription("Context switches on thread (derived from sampling)"), \
perfetto::Category("thread_page_fault") \
.SetDescription("Memory page faults on thread (derived from sampling)"), \
perfetto::Category("hardware_counter") \
.SetDescription("Hardware counter value on thread (derived from sampling)"), \
perfetto::Category("cpu_freq") \
.SetDescription("CPU frequency in MHz (collected in background thread)"), \
perfetto::Category("process_page_fault") \
.SetDescription( \
"Memory page faults in process (collected in background thread)"), \
perfetto::Category("process_virtual_memory") \
.SetDescription("Virtual memory usage in process in MB (collected in " \
"background thread)"), \
perfetto::Category("process_context_switch") \
.SetDescription( \
"Context switches in process (collected in background thread)"), \
perfetto::Category("process_page_fault") \
.SetDescription( \
"Memory page faults in process (collected in background thread)"), \
perfetto::Category("process_user_cpu_time") \
.SetDescription("CPU time of functions executing in user-space in process " \
"in seconds (collected in background thread)"), \
perfetto::Category("process_kernel_cpu_time") \
.SetDescription("CPU time of functions executing in kernel-space in " \
"process in seconds (collected in background thread)"), \
perfetto::Category("mpi").SetDescription("MPI regions"), \
perfetto::Category("kokkos").SetDescription("Kokkos regions"), \
perfetto::Category("ompt").SetDescription("OpenMP Tools regions"), \
perfetto::Category("critical-trace").SetDescription("Combined critical traces"), \
perfetto::Category("host-critical-trace") \
.SetDescription("Host-side critical traces"), \
perfetto::Category("device-critical-trace") \
.SetDescription("Device-side critical traces"), \
perfetto::Category("timemory").SetDescription("Events from the timemory API")
#if defined(TIMEMORY_USE_PERFETTO)
# define TIMEMORY_PERFETTO_CATEGORIES OMNITRACE_PERFETTO_CATEGORIES
# include <timemory/components/perfetto/backends.hpp>
#else
# define PERFETTO_CATEGORIES \
perfetto::Category("host").SetDescription("Host-side function tracing"), \
perfetto::Category("device").SetDescription("Device-side function tracing"), \
perfetto::Category("user").SetDescription("User-defined regions"), \
perfetto::Category("rocm_smi").SetDescription("Device-level metrics"), \
perfetto::Category("sampling") \
.SetDescription("Metrics derived from sampling"), \
perfetto::Category("thread_sampling") \
.SetDescription("Metrics derived from background thread sampling"), \
perfetto::Category("mpi").SetDescription("MPI regions"), \
perfetto::Category("kokkos").SetDescription("Kokkos regions"), \
perfetto::Category("ompt").SetDescription("OpenMP Tools regions"), \
perfetto::Category("critical-trace") \
.SetDescription("Combined critical traces"), \
perfetto::Category("host-critical-trace") \
.SetDescription("Host-side critical traces"), \
perfetto::Category("device-critical-trace") \
.SetDescription("Device-side critical traces"), \
perfetto::Category("timemory") \
.SetDescription("Events from the timemory API")
# define TIMEMORY_PERFETTO_CATEGORIES PERFETTO_CATEGORIES
#endif
#if !defined(TIMEMORY_USE_PERFETTO)
PERFETTO_DEFINE_CATEGORIES(PERFETTO_CATEGORIES);
# include <perfetto.h>
PERFETTO_DEFINE_CATEGORIES(OMNITRACE_PERFETTO_CATEGORIES);
#endif
#include "library/debug.hpp"