rocDecode API Tracing Support (#49)
* rocDecode API Tracing support * Test bin file added to rocdecode. Need to add validate python methods * Added option to not make rocDecode tests * Added rocdecode and rocprofv3 tests * Added csv test * Address PR comments. Changed tests to use built-in rocstreambit decoder to remove ffmpeg dependancy. Changed cmake option to disbale tests rather than not build them. Tests work locally, but will fail until rocDecode is built with tracing enabled on CI * Add option to avoid building rocdecode tests * Added option to avoid building rocdecode bin file * Merge conflict error * CMake files changed in response to review comments. Attempting to implement callbacks. * Turned off test building for rocdecode * Minor fixes for review comments * Review comments * Updated formatting * Document changes and format.hpp reversion. Need to remove iterate args support for now for later update. * Remove iterate args support * Remove iterate-args * enforce abi versioning in macro if * Fix doc error * removed spaces to fix indentation error --------- Co-authored-by: Madsen, Jonathan <Jonathan.Madsen@amd.com>
このコミットが含まれているのは:
Trowbridge, Ian
GitHub
+124
-5
@@ -397,6 +397,23 @@ struct rccl_api_callback_record_t
|
||||
}
|
||||
};
|
||||
|
||||
struct rocdecode_api_callback_record_t
|
||||
{
|
||||
uint64_t timestamp = 0;
|
||||
rocprofiler_callback_tracing_record_t record = {};
|
||||
rocprofiler_callback_tracing_rocdecode_api_data_t payload = {};
|
||||
callback_arg_array_t args = {};
|
||||
|
||||
template <typename ArchiveT>
|
||||
void save(ArchiveT& ar) const
|
||||
{
|
||||
ar(cereal::make_nvp("timestamp", timestamp));
|
||||
cereal::save(ar, record);
|
||||
ar(cereal::make_nvp("payload", payload));
|
||||
serialize_args(ar, args);
|
||||
}
|
||||
};
|
||||
|
||||
struct ompt_callback_record_t
|
||||
{
|
||||
uint64_t timestamp = 0;
|
||||
@@ -555,6 +572,7 @@ auto kernel_dispatch_cb_records = std::deque<kernel_dispatch_callback_record_
|
||||
auto memory_copy_cb_records = std::deque<memory_copy_callback_record_t>{};
|
||||
auto memory_allocation_cb_records = std::deque<memory_allocation_callback_record_t>{};
|
||||
auto rccl_api_cb_records = std::deque<rccl_api_callback_record_t>{};
|
||||
auto rocdecode_api_cb_records = std::deque<rocdecode_api_callback_record_t>{};
|
||||
auto ompt_cb_records = std::deque<ompt_callback_record_t>{};
|
||||
|
||||
int
|
||||
@@ -824,6 +842,20 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
|
||||
runtime_init_cb_records.emplace_back(
|
||||
runtime_init_callback_record_t{ts, record, *data, std::move(args)});
|
||||
}
|
||||
else if(record.kind == ROCPROFILER_CALLBACK_TRACING_ROCDECODE_API)
|
||||
{
|
||||
auto* data =
|
||||
static_cast<rocprofiler_callback_tracing_rocdecode_api_data_t*>(record.payload);
|
||||
auto args = callback_arg_array_t{};
|
||||
if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
|
||||
rocprofiler_iterate_callback_tracing_kind_operation_args(
|
||||
record, save_args, record.phase, &args);
|
||||
|
||||
static auto _mutex = std::mutex{};
|
||||
auto _lk = std::unique_lock<std::mutex>{_mutex};
|
||||
rocdecode_api_cb_records.emplace_back(
|
||||
rocdecode_api_callback_record_t{ts, record, *data, std::move(args)});
|
||||
}
|
||||
else
|
||||
{
|
||||
throw std::runtime_error{"unsupported callback kind"};
|
||||
@@ -843,8 +875,9 @@ auto scratch_memory_records = std::deque<rocprofiler_buffer_tracing_scratch_memo
|
||||
auto page_migration_records = std::deque<rocprofiler_buffer_tracing_page_migration_record_t>{};
|
||||
auto corr_id_retire_records =
|
||||
std::deque<rocprofiler_buffer_tracing_correlation_id_retirement_record_t>{};
|
||||
auto rccl_api_bf_records = std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>{};
|
||||
auto ompt_bf_records = std::deque<rocprofiler_buffer_tracing_ompt_record_t>{};
|
||||
auto rccl_api_bf_records = std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>{};
|
||||
auto rocdecode_api_bf_records = std::deque<rocprofiler_buffer_tracing_rocdecode_api_record_t>{};
|
||||
auto ompt_bf_records = std::deque<rocprofiler_buffer_tracing_ompt_record_t>{};
|
||||
|
||||
void
|
||||
tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
|
||||
@@ -971,6 +1004,13 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
|
||||
|
||||
runtime_init_bf_records.emplace_back(*record);
|
||||
}
|
||||
else if(header->kind == ROCPROFILER_BUFFER_TRACING_ROCDECODE_API)
|
||||
{
|
||||
auto* record = static_cast<rocprofiler_buffer_tracing_rocdecode_api_record_t*>(
|
||||
header->payload);
|
||||
|
||||
rocdecode_api_bf_records.emplace_back(*record);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw std::runtime_error{
|
||||
@@ -1069,6 +1109,9 @@ rocprofiler_context_id_t kernel_dispatch_buffered_ctx = {0};
|
||||
rocprofiler_context_id_t page_migration_ctx = {0};
|
||||
rocprofiler_context_id_t runtime_init_callback_ctx = {};
|
||||
rocprofiler_context_id_t runtime_init_buffered_ctx = {};
|
||||
rocprofiler_context_id_t rocdecode_api_callback_ctx = {0};
|
||||
rocprofiler_context_id_t rocdecode_api_buffered_ctx = {0};
|
||||
|
||||
// buffers
|
||||
rocprofiler_buffer_id_t runtime_init_buffered_buffer = {};
|
||||
rocprofiler_buffer_id_t hsa_api_buffered_buffer = {};
|
||||
@@ -1082,6 +1125,7 @@ rocprofiler_buffer_id_t counter_collection_buffer = {};
|
||||
rocprofiler_buffer_id_t scratch_memory_buffer = {};
|
||||
rocprofiler_buffer_id_t corr_id_retire_buffer = {};
|
||||
rocprofiler_buffer_id_t rccl_api_buffered_buffer = {};
|
||||
rocprofiler_buffer_id_t rocdecode_api_buffer = {};
|
||||
rocprofiler_buffer_id_t ompt_buffered_buffer = {};
|
||||
|
||||
auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
|
||||
@@ -1107,10 +1151,12 @@ auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
|
||||
{"SCRATCH_MEMORY", &scratch_memory_ctx},
|
||||
{"CORRELATION_ID_RETIREMENT", &corr_id_retire_ctx},
|
||||
{"RCCL_API_BUFFERED", &rccl_api_buffered_ctx},
|
||||
{"ROCDECODE_API_CALLBACK", &rocdecode_api_callback_ctx},
|
||||
{"ROCDECODE_API_BUFFERED", &rocdecode_api_buffered_ctx},
|
||||
{"OMPT_BUFFERED", &ompt_buffered_ctx},
|
||||
};
|
||||
|
||||
auto buffers = std::array<rocprofiler_buffer_id_t*, 13>{&runtime_init_buffered_buffer,
|
||||
auto buffers = std::array<rocprofiler_buffer_id_t*, 14>{&runtime_init_buffered_buffer,
|
||||
&hsa_api_buffered_buffer,
|
||||
&hip_api_buffered_buffer,
|
||||
&marker_api_buffered_buffer,
|
||||
@@ -1122,7 +1168,8 @@ auto buffers = std::array<rocprofiler_buffer_id_t*, 13>{&runtime_init_buffered_b
|
||||
&counter_collection_buffer,
|
||||
&corr_id_retire_buffer,
|
||||
&rccl_api_buffered_buffer,
|
||||
&ompt_buffered_buffer};
|
||||
&ompt_buffered_buffer,
|
||||
&rocdecode_api_buffer};
|
||||
|
||||
auto agents = std::vector<rocprofiler_agent_t>{};
|
||||
auto agents_map = std::unordered_map<rocprofiler_agent_id_t, rocprofiler_agent_t>{};
|
||||
@@ -1288,6 +1335,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
|
||||
nullptr),
|
||||
"rccl api callback tracing service configure");
|
||||
|
||||
ROCPROFILER_CALL(
|
||||
rocprofiler_configure_callback_tracing_service(rocdecode_api_callback_ctx,
|
||||
ROCPROFILER_CALLBACK_TRACING_ROCDECODE_API,
|
||||
nullptr,
|
||||
0,
|
||||
tool_tracing_callback,
|
||||
nullptr),
|
||||
"rocdecode api callback tracing service configure");
|
||||
|
||||
ROCPROFILER_CALL(
|
||||
rocprofiler_configure_callback_tracing_service(ompt_callback_ctx,
|
||||
ROCPROFILER_CALLBACK_TRACING_OMPT,
|
||||
@@ -1408,6 +1464,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
|
||||
&rccl_api_buffered_buffer),
|
||||
"buffer creation");
|
||||
|
||||
ROCPROFILER_CALL(rocprofiler_create_buffer(rocdecode_api_buffered_ctx,
|
||||
buffer_size,
|
||||
watermark,
|
||||
ROCPROFILER_BUFFER_POLICY_LOSSLESS,
|
||||
tool_tracing_buffered,
|
||||
tool_data,
|
||||
&rocdecode_api_buffer),
|
||||
"buffer creation");
|
||||
|
||||
ROCPROFILER_CALL(rocprofiler_create_buffer(ompt_buffered_ctx,
|
||||
buffer_size,
|
||||
watermark,
|
||||
@@ -1532,6 +1597,14 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
|
||||
rccl_api_buffered_buffer),
|
||||
"buffer tracing service for rccl api configure");
|
||||
|
||||
ROCPROFILER_CALL(
|
||||
rocprofiler_configure_buffer_tracing_service(rocdecode_api_buffered_ctx,
|
||||
ROCPROFILER_BUFFER_TRACING_ROCDECODE_API,
|
||||
nullptr,
|
||||
0,
|
||||
rocdecode_api_buffer),
|
||||
"buffer tracing service for rocdecode api configure");
|
||||
|
||||
ROCPROFILER_CALL(
|
||||
rocprofiler_configure_buffer_tracing_service(
|
||||
ompt_buffered_ctx, ROCPROFILER_BUFFER_TRACING_OMPT, nullptr, 0, ompt_buffered_buffer),
|
||||
@@ -1701,7 +1774,8 @@ tool_fini(void* tool_data)
|
||||
<< ", rccl_api_bf_records=" << rccl_api_bf_records.size()
|
||||
<< ", ompt_bf_records=" << ompt_bf_records.size()
|
||||
<< ", counter_collection_value_records=" << counter_collection_bf_records.size()
|
||||
<< "...\n"
|
||||
<< ", rocdecode_api_callback_records=" << rocdecode_api_cb_records.size()
|
||||
<< ", rocdecode_api_bf_records=" << rocdecode_api_bf_records.size() << "...\n"
|
||||
<< std::flush;
|
||||
|
||||
auto* _call_stack = static_cast<call_stack_t*>(tool_data);
|
||||
@@ -1797,6 +1871,7 @@ write_json(call_stack_t* _call_stack)
|
||||
json_ar(cereal::make_nvp("kernel_dispatch", kernel_dispatch_cb_records));
|
||||
json_ar(cereal::make_nvp("memory_copies", memory_copy_cb_records));
|
||||
json_ar(cereal::make_nvp("memory_allocations", memory_allocation_cb_records));
|
||||
json_ar(cereal::make_nvp("rocdecode_api_traces", rocdecode_api_cb_records));
|
||||
} catch(std::exception& e)
|
||||
{
|
||||
std::cerr << "[" << getpid() << "][" << __FUNCTION__
|
||||
@@ -1823,6 +1898,7 @@ write_json(call_stack_t* _call_stack)
|
||||
json_ar(cereal::make_nvp("ompt_traces", ompt_bf_records));
|
||||
json_ar(cereal::make_nvp("retired_correlation_ids", corr_id_retire_records));
|
||||
json_ar(cereal::make_nvp("counter_collection", counter_collection_bf_records));
|
||||
json_ar(cereal::make_nvp("rocdecode_api_traces", rocdecode_api_bf_records));
|
||||
} catch(std::exception& e)
|
||||
{
|
||||
std::cerr << "[" << getpid() << "][" << __FUNCTION__
|
||||
@@ -1894,6 +1970,8 @@ write_perfetto()
|
||||
tids.emplace(itr.thread_id);
|
||||
for(auto itr : ompt_bf_records)
|
||||
tids.emplace(itr.thread_id);
|
||||
for(auto itr : rocdecode_api_bf_records)
|
||||
tids.emplace(itr.thread_id);
|
||||
|
||||
for(auto itr : memory_copy_bf_records)
|
||||
{
|
||||
@@ -2147,6 +2225,47 @@ write_perfetto()
|
||||
itr.end_timestamp);
|
||||
}
|
||||
|
||||
for(auto itr : rocdecode_api_bf_records)
|
||||
{
|
||||
auto name = buffer_names.at(itr.kind, itr.operation);
|
||||
auto& track = thread_tracks.at(itr.thread_id);
|
||||
|
||||
auto _args = callback_arg_array_t{};
|
||||
auto ritr = std::find_if(
|
||||
rocdecode_api_cb_records.begin(),
|
||||
rocdecode_api_cb_records.end(),
|
||||
[&itr](const auto& citr) {
|
||||
return (citr.record.correlation_id.internal == itr.correlation_id.internal &&
|
||||
!citr.args.empty());
|
||||
});
|
||||
if(ritr != rocdecode_api_cb_records.end()) _args = ritr->args;
|
||||
|
||||
TRACE_EVENT_BEGIN(sdk::perfetto_category<sdk::category::rocdecode_api>::name,
|
||||
::perfetto::StaticString(name.data()),
|
||||
track,
|
||||
itr.start_timestamp,
|
||||
::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
|
||||
"begin_ns",
|
||||
itr.start_timestamp,
|
||||
"tid",
|
||||
itr.thread_id,
|
||||
"kind",
|
||||
itr.kind,
|
||||
"operation",
|
||||
itr.operation,
|
||||
"corr_id",
|
||||
itr.correlation_id.internal,
|
||||
[&](::perfetto::EventContext ctx) {
|
||||
for(const auto& aitr : _args)
|
||||
sdk::add_perfetto_annotation(ctx, aitr.first, aitr.second);
|
||||
});
|
||||
TRACE_EVENT_END(sdk::perfetto_category<sdk::category::rocdecode_api>::name,
|
||||
track,
|
||||
itr.end_timestamp,
|
||||
"end_ns",
|
||||
itr.end_timestamp);
|
||||
}
|
||||
|
||||
for(auto itr : ompt_bf_records)
|
||||
{
|
||||
auto name = buffer_names.at(itr.kind, itr.operation);
|
||||
|
||||
新しいイシューから参照
ユーザーをブロックする