AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

roctracer: device and stream tracks (#209)

Browse Source 
* roctracer: use multiple tracks for HIP streams

Use different perfetto tracks for each stream, and set the name of
these tracks to the stream pointer values. Setting the name like this
matches the args in the API traces.
This fixes overlapping work on multiple streams appearing as a call
stack.

* Fix -pedantic

* Run clang-format

* Add option to disable per stream tracks in perfetto

* Updated scheme for roctracer activity + general roctracer fixes

- Per-device tracks
- Handle HSA OPS in ROCm 5.3
  - the changes in ROCm 5.3 were causing HSA ops to get discarded
- Default for OMNITRACE_ROCTRACER_DISCARD_INVALID is now zero
  - i.e. default behavior is to flip beg_ns and end_ns when beg_ns > end_ns

* Flush perfetto at end of hip_activity_callback

- fixes unterminated regions

* GitHub Actions and run-ci script updates

- improve reliability

* Set OMNITRACE_TMPDIR in testing

- files in /tmp get occasionally deleted during CI

Co-authored-by: Gergely Meszaros <gergely@streamhpc.com>
This commit is contained in:
Jonathan R. Madsen
2022-11-16 15:57:27 -06:00
committed by GitHub
parent 2f16e2ecb1
commit 589a729702
13 changed files with 237 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lib/omnitrace/library/config.cpp
+17
View File
@@ -576,6 +576,12 @@ configure_settings(bool _init)
_config->get<bool>("collapse_processes"), "perfetto", "data",
"advanced");
OMNITRACE_CONFIG_SETTING(
bool, "OMNITRACE_PERFETTO_ROCTRACER_PER_STREAM",
"Separate roctracer GPU side traces (copies, kernels) into separate "
"tracks based on the stream they're enqueued into",
true, "perfetto", "roctracer", "rocm", "advanced");
OMNITRACE_CONFIG_SETTING(
std::string, "OMNITRACE_PERFETTO_FILL_POLICY",
"Behavior when perfetto buffer is full. 'discard' will ignore new entries, "
@@ -1518,6 +1524,17 @@ get_use_roctracer()
#endif
}
bool
get_perfetto_roctracer_per_stream()
{
#if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
static auto _v = get_config()->find("OMNITRACE_PERFETTO_ROCTRACER_PER_STREAM");
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
#else
return false;
#endif
}
bool
get_use_rocprofiler()
{
source/lib/omnitrace/library/config.hpp
+3
View File
@@ -266,6 +266,9 @@ get_backend();
std::string
get_perfetto_output_filename();
bool
get_perfetto_roctracer_per_stream() OMNITRACE_HOT;
int64_t
get_critical_trace_count();
source/lib/omnitrace/library/roctracer.cpp
+38 -48
View File
@@ -32,6 +32,7 @@
#include <timemory/backends/cpu.hpp>
#include <timemory/backends/threading.hpp>
#include <timemory/hash/types.hpp>
#include <timemory/utility/types.hpp>
#include <atomic>
@@ -118,7 +119,7 @@ get_roctracer_tid_data()
return _v;
}
using cid_tuple_t = std::tuple<uint64_t, uint64_t, uint32_t>;
using cid_tuple_t = std::tuple<uint64_t, uint64_t, uint32_t, uintptr_t>;
struct cid_data : cid_tuple_t
{
using cid_tuple_t::cid_tuple_t;
@@ -128,10 +129,12 @@ struct cid_data : cid_tuple_t
auto& cid() { return std::get<0>(*this); }
auto& pcid() { return std::get<1>(*this); }
auto& depth() { return std::get<2>(*this); }
auto& queue() { return std::get<3>(*this); }
auto cid() const { return std::get<0>(*this); }
auto pcid() const { return std::get<1>(*this); }
auto depth() const { return std::get<2>(*this); }
auto queue() const { return std::get<3>(*this); }
};
auto&
@@ -344,24 +347,13 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
}
void
hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg)
{
if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
return;
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
static thread_local std::once_flag _once{};
std::call_once(_once, []() {
threading::offset_this_id(true);
if(threading::get_id() != 0)
{
sampling::block_signals();
threading::set_thread_name("roctracer.hsa");
sampling::shutdown();
}
});
auto&& _protect = comp::roctracer::protect_flush_activity();
(void) _protect;
@@ -760,8 +752,8 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
critical_trace::add_hash_id(op_name), _depth);
}
get_roctracer_cid_data(_tid).emplace(_corr_id,
cid_data{ _cid, _parent_cid, _depth });
get_roctracer_cid_data(_tid).emplace(
_corr_id, cid_data{ _cid, _parent_cid, _depth, _queue });
hip_exec_activity_callbacks(_tid);
}
@@ -769,7 +761,8 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
{
hip_exec_activity_callbacks(_tid);
std::tie(_cid, _parent_cid, _depth) = get_roctracer_cid_data(_tid).at(_corr_id);
std::tie(_cid, _parent_cid, _depth, std::ignore) =
get_roctracer_cid_data(_tid).at(_corr_id);
if(get_use_perfetto())
{
@@ -814,24 +807,13 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
// Activity tracing callback
void
hip_activity_callback(const char* begin, const char* end, void*)
hip_activity_callback(const char* begin, const char* end, void* arg)
{
if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
return;
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
static thread_local std::once_flag _once{};
std::call_once(_once, []() {
threading::offset_this_id(true);
if(threading::get_id() != 0)
{
sampling::block_signals();
threading::set_thread_name("roctracer.hip");
sampling::shutdown();
}
});
auto&& _protect = comp::roctracer::protect_flush_activity();
(void) _protect;
@@ -859,8 +841,12 @@ hip_activity_callback(const char* begin, const char* end, void*)
assert(HIP_OP_ID_DISPATCH == 0);
assert(HIP_OP_ID_COPY == 1);
assert(HIP_OP_ID_BARRIER == 2);
assert(record->domain == ACTIVITY_DOMAIN_HIP_OPS);
if(record->domain == ACTIVITY_DOMAIN_HSA_OPS)
{
hsa_activity_callback(record->op, record, arg);
continue;
}
if(record->domain != ACTIVITY_DOMAIN_HIP_OPS) continue;
if(record->op > HIP_OP_ID_BARRIER) continue;
@@ -880,6 +866,7 @@ hip_activity_callback(const char* begin, const char* end, void*)
uint64_t _pcid = 0; // parent corr_id
int32_t _devid = record->device_id; // device id
int64_t _queid = record->queue_id; // queue id
uintptr_t _queue = 0; // Host queue (stream)
auto _laps = _indexes[_corr_id]++; // see note #1
const char* _name = nullptr;
bool _found = false;
@@ -903,7 +890,7 @@ hip_activity_callback(const char* begin, const char* end, void*)
{
auto& _cids = get_roctracer_cid_data(_tid);
if(_cids.find(_corr_id) != _cids.end())
std::tie(_cid, _pcid, _depth) = _cids.at(_corr_id);
std::tie(_cid, _pcid, _depth, _queue) = _cids.at(_corr_id);
else
{
OMNITRACE_VERBOSE_F(3,
@@ -922,7 +909,7 @@ hip_activity_callback(const char* begin, const char* end, void*)
auto _verbose = []() { return get_verbose() >= 0 || get_debug(); };
static size_t _n = 0;
static size_t _nmax =
get_env<size_t>("OMNITRACE_ROCTRACER_DISCARD_INVALID", 10);
get_env<size_t>("OMNITRACE_ROCTRACER_DISCARD_INVALID", 0);
if(_nmax == 0) std::swap(_end_ns, _beg_ns);
OMNITRACE_WARNING_IF_F(
_n < _nmax && _verbose(),
@@ -952,36 +939,36 @@ hip_activity_callback(const char* begin, const char* end, void*)
if(_kernel_names.find(_name) == _kernel_names.end())
_kernel_names.emplace(_name, tim::demangle(_name));
auto _track_desc = [](int32_t _device_id, int64_t _queue_id) {
if(config::get_perfetto_roctracer_per_stream())
return JOIN("", "HIP Activity Device ", _device_id, ", Queue ",
_queue_id);
return JOIN("", "HIP Activity Device ", _device_id);
};
const auto _track = tracing::get_perfetto_track(
category::device_hip{}, _track_desc, _devid,
(get_perfetto_roctracer_per_stream()) ? _queid : 0);
assert(_end_ns >= _beg_ns);
tracing::push_perfetto_ts(
category::device_hip{}, _kernel_names.at(_name).c_str(), _beg_ns,
tracing::push_perfetto_track(
category::device_hip{}, _kernel_names.at(_name).c_str(), _track, _beg_ns,
perfetto::Flow::ProcessScoped(_cid), [&](perfetto::EventContext ctx) {
if(config::get_perfetto_annotations())
{
tracing::add_perfetto_annotation(ctx, "begin_ns", _beg_ns);
tracing::add_perfetto_annotation(ctx, "end_ns", _end_ns);
tracing::add_perfetto_annotation(ctx, "corr_id", _corr_id);
tracing::add_perfetto_annotation(ctx, "device", _devid);
tracing::add_perfetto_annotation(ctx, "queue", _queid);
tracing::add_perfetto_annotation(ctx, "tid", _tid);
tracing::add_perfetto_annotation(
ctx, "stream", JOIN("", "0x", std::hex, _queue));
tracing::add_perfetto_annotation(ctx, "op",
_op_id_names.at(record->op));
}
});
tracing::pop_perfetto_ts(
category::device_hip{}, "", _end_ns, [&](perfetto::EventContext ctx) {
if(config::get_perfetto_annotations())
{
tracing::add_perfetto_annotation(ctx, "end_ns", _end_ns);
}
});
// for some reason, this is necessary to make sure very last one ends
tracing::pop_perfetto_ts(
category::device_hip{}, "", _end_ns, [&](perfetto::EventContext ctx) {
if(config::get_perfetto_annotations())
{
tracing::add_perfetto_annotation(ctx, "end_ns", _end_ns);
}
});
tracing::pop_perfetto_track(category::device_hip{}, "", _track, _end_ns);
}
if(_critical_trace)
@@ -1013,6 +1000,9 @@ hip_activity_callback(const char* begin, const char* end, void*)
_async_ops->emplace_back(std::move(_func));
}
}
// ensures that all the updates are written
if(get_use_perfetto()) ::perfetto::TrackEvent::Flush();
}
bool&
source/lib/omnitrace/library/roctracer.hpp
+1 -1
View File
@@ -57,7 +57,7 @@ void
hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
void
hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg);
hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg);
void
hip_exec_activity_callbacks(int64_t _tid);
source/lib/omnitrace/library/tracing.cpp
+7
View File
@@ -43,6 +43,13 @@ get_perfetto_session()
return _v;
}
std::unordered_map<hash_value_t, std::string>&
get_perfetto_track_uuids()
{
static thread_local auto _v = std::unordered_map<hash_value_t, std::string>{};
return _v;
}
std::vector<std::function<void()>>&
get_finalization_functions()
{
source/lib/omnitrace/library/tracing.hpp
+76
View File
@@ -38,13 +38,22 @@
#include <timemory/hash/types.hpp>
#include <timemory/mpl/type_traits.hpp>
#include <atomic>
#include <functional>
#include <memory>
#include <ratio>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
namespace omnitrace
{
namespace tracing
{
using interval_data_instances = thread_data<std::vector<bool>>;
using hash_value_t = tim::hash_value_t;
perfetto::TraceConfig&
get_perfetto_config();
@@ -52,6 +61,54 @@ get_perfetto_config();
std::unique_ptr<perfetto::TracingSession>&
get_perfetto_session();
template <typename CategoryT, typename... Args>
auto
get_perfetto_category_uuid(Args&&... _args)
{
return tim::hash::get_combined_hash_id(
tim::hash::get_hash_id(JOIN('_', "omnitrace", trait::name<CategoryT>::value)),
std::forward<Args>(_args)...);
}
std::unordered_map<hash_value_t, std::string>&
get_perfetto_track_uuids();
template <typename CategoryT, typename TrackT = ::perfetto::Track, typename FuncT,
typename... Args>
auto
get_perfetto_track(CategoryT, FuncT&& _desc_generator, Args&&... _args)
{
auto _uuid = get_perfetto_category_uuid<CategoryT>(std::forward<Args>(_args)...);
auto& _track_uuids = get_perfetto_track_uuids();
if(_track_uuids.find(_uuid) == _track_uuids.end())
{
const auto _track = TrackT(_uuid);
auto _desc = _track.Serialize();
auto _name = std::forward<FuncT>(_desc_generator)(std::forward<Args>(_args)...);
_desc.set_name(_name);
::perfetto::TrackEvent::SetTrackDescriptor(_track, _desc);
OMNITRACE_VERBOSE_F(4, "[%s] Created %s(%zu) with description: \"%s\"\n",
trait::name<CategoryT>::value, demangle<TrackT>().c_str(),
_uuid, _name.c_str());
_track_uuids.emplace(_uuid, _name);
}
// guard this with ppdefs in addition to runtime check to avoid
// overhead of generating string during releases
#if defined(OMNITRACE_CI) && OMNITRACE_CI > 0
auto _name = std::forward<FuncT>(_desc_generator)(std::forward<Args>(_args)...);
OMNITRACE_CI_THROW(_track_uuids.at(_uuid) != _name,
"Error! Multiple invocations of UUID %zu produced different "
"descriptions: \"%s\" and \"%s\"\n",
_uuid, _track_uuids.at(_uuid).c_str(), _name.c_str());
#endif
return TrackT(_uuid);
}
std::vector<std::function<void()>>&
get_finalization_functions();
@@ -292,5 +349,24 @@ pop_perfetto_ts(CategoryT, const char*, uint64_t _ts, Args&&... args)
TRACE_EVENT_END(trait::name<CategoryT>::value, _ts, std::forward<Args>(args)...);
}
template <typename CategoryT, typename... Args>
inline void
push_perfetto_track(CategoryT, const char* name, perfetto::Track _track, uint64_t _ts,
Args&&... args)
{
TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name), _track,
_ts, std::forward<Args>(args)...);
}
template <typename CategoryT, typename... Args>
inline void
pop_perfetto_track(CategoryT, const char*, perfetto::Track _track, uint64_t _ts,
Args&&... args)
{
TRACE_EVENT_END(trait::name<CategoryT>::value, _track, _ts,
std::forward<Args>(args)...);
}
} // namespace tracing
} // namespace omnitrace