AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rocJPEG API Tracing (#73)

Browse Source 
* rocDecode API Tracing support

* Test bin file added to rocdecode. Need to add validate python methods

* Added option to not make rocDecode tests

* Added rocdecode and rocprofv3 tests

* Added csv test

* Address PR comments. Changed tests to use built-in rocstreambit decoder to remove ffmpeg dependancy. Changed cmake option to disbale tests rather than not build them. Tests work locally, but will fail until rocDecode is built with tracing enabled on CI

* Add option to avoid building rocdecode tests

* Added option to avoid building rocdecode bin file

* Support for rocJPEG API Trace

* Added newline to rocjpeg_version.h

* json-tool code added, initial test/bin commit

* Formatting

* Resolved rocjpeg bin test compilation errors

* Tests implemented. Perfetto module currently resulting in errors, so need to retest whenever it is fixed

* Formatting and compilation errors

* Minor fixes

* Copyright year update and minor fixes

* Doc update fix

* Added rocjpeg csv file in data

* Addresses review comments: Updated fixed Findroc.. and uses root directory as a hint, fixed documentation error, changed tables to use _CORE, minor style fixes

* Added rocdecode and rocjpeg to CI

* Removed rocdecode and rocjpeg from CI and added back build tests option

* Updated Cmake Files

* Added rocDecode and rocJPEG to CI

* Remove cmake line added in error

* Temporarily modified tests to pass if rocdecode or rocjpeg tracing are not supported for CI, cmake changes

* Added find_package for test

* Added back use of system rocDecode and rocJPEG, modifies system files to include prefix path

* Updated no-link to include INCLUDE_DIR/roc(decode|jpeg), added comments for tests

* Resolve merge conflicts and formatting

* Added regex find and replace instead of include for CI

* VAAPI package causing errors on Vega20

* Removed system rocjpeg and rocdecode use temporarily until cmake issues resolved

* Removed workflows regex

* Formatting and minor test modification

* Modified test for vega20

* Update rocDecode and rocJPEG cmake and tests

* Changelog

* Fix merge conflict

* Added back if-statements around add-tests since cmake-generator-expressions are resulting in errors when the packages are missing

* Removed if found statements, replaced with TARGET:EXISTS

* Skip json file for rocjpeg and rocdecode tests if not supported

* Add os import

---------

Co-authored-by: Kandula, Venkateshwar reddy <Venkateshwarreddy.Kandula@amd.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
This commit is contained in:
Trowbridge, Ian
2025-02-21 15:43:49 -06:00
committed by GitHub
parent 95e0341266
commit 31fe8858d1
92 changed files with 4614 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/tools/json-tool.cpp
+121 -3
View File
@@ -414,6 +414,23 @@ struct rocdecode_api_callback_record_t
}
};
struct rocjpeg_api_callback_record_t
{
uint64_t timestamp = 0;
rocprofiler_callback_tracing_record_t record = {};
rocprofiler_callback_tracing_rocjpeg_api_data_t payload = {};
callback_arg_array_t args = {};
template <typename ArchiveT>
void save(ArchiveT& ar) const
{
ar(cereal::make_nvp("timestamp", timestamp));
cereal::save(ar, record);
ar(cereal::make_nvp("payload", payload));
serialize_args(ar, args);
}
};
struct ompt_callback_record_t
{
uint64_t timestamp = 0;
@@ -573,6 +590,7 @@ auto memory_copy_cb_records = std::deque<memory_copy_callback_record_t>{}
auto memory_allocation_cb_records = std::deque<memory_allocation_callback_record_t>{};
auto rccl_api_cb_records = std::deque<rccl_api_callback_record_t>{};
auto rocdecode_api_cb_records = std::deque<rocdecode_api_callback_record_t>{};
auto rocjpeg_api_cb_records = std::deque<rocjpeg_api_callback_record_t>{};
auto ompt_cb_records = std::deque<ompt_callback_record_t>{};
int
@@ -856,6 +874,19 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
rocdecode_api_cb_records.emplace_back(
rocdecode_api_callback_record_t{ts, record, *data, std::move(args)});
}
else if(record.kind == ROCPROFILER_CALLBACK_TRACING_ROCJPEG_API)
{
auto* data = static_cast<rocprofiler_callback_tracing_rocjpeg_api_data_t*>(record.payload);
auto args = callback_arg_array_t{};
if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
rocprofiler_iterate_callback_tracing_kind_operation_args(
record, save_args, record.phase, &args);
static auto _mutex = std::mutex{};
auto _lk = std::unique_lock<std::mutex>{_mutex};
rocjpeg_api_cb_records.emplace_back(
rocjpeg_api_callback_record_t{ts, record, *data, std::move(args)});
}
else
{
throw std::runtime_error{"unsupported callback kind"};
@@ -877,6 +908,7 @@ auto corr_id_retire_records =
std::deque<rocprofiler_buffer_tracing_correlation_id_retirement_record_t>{};
auto rccl_api_bf_records = std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>{};
auto rocdecode_api_bf_records = std::deque<rocprofiler_buffer_tracing_rocdecode_api_record_t>{};
auto rocjpeg_api_bf_records = std::deque<rocprofiler_buffer_tracing_rocjpeg_api_record_t>{};
auto ompt_bf_records = std::deque<rocprofiler_buffer_tracing_ompt_record_t>{};
void
@@ -1011,6 +1043,13 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
rocdecode_api_bf_records.emplace_back(*record);
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_ROCJPEG_API)
{
auto* record =
static_cast<rocprofiler_buffer_tracing_rocjpeg_api_record_t*>(header->payload);
rocjpeg_api_bf_records.emplace_back(*record);
}
else
{
throw std::runtime_error{
@@ -1111,6 +1150,8 @@ rocprofiler_context_id_t runtime_init_callback_ctx = {};
rocprofiler_context_id_t runtime_init_buffered_ctx = {};
rocprofiler_context_id_t rocdecode_api_callback_ctx = {0};
rocprofiler_context_id_t rocdecode_api_buffered_ctx = {0};
rocprofiler_context_id_t rocjpeg_api_callback_ctx = {0};
rocprofiler_context_id_t rocjpeg_api_buffered_ctx = {0};
// buffers
rocprofiler_buffer_id_t runtime_init_buffered_buffer = {};
@@ -1126,6 +1167,7 @@ rocprofiler_buffer_id_t scratch_memory_buffer = {};
rocprofiler_buffer_id_t corr_id_retire_buffer = {};
rocprofiler_buffer_id_t rccl_api_buffered_buffer = {};
rocprofiler_buffer_id_t rocdecode_api_buffer = {};
rocprofiler_buffer_id_t rocjpeg_api_buffer = {};
rocprofiler_buffer_id_t ompt_buffered_buffer = {};
auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
@@ -1153,10 +1195,12 @@ auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
{"RCCL_API_BUFFERED", &rccl_api_buffered_ctx},
{"ROCDECODE_API_CALLBACK", &rocdecode_api_callback_ctx},
{"ROCDECODE_API_BUFFERED", &rocdecode_api_buffered_ctx},
{"ROCJPEG_API_CALLBACK", &rocjpeg_api_callback_ctx},
{"ROCJPEG_API_BUFFERED", &rocjpeg_api_buffered_ctx},
{"OMPT_BUFFERED", &ompt_buffered_ctx},
};
auto buffers = std::array<rocprofiler_buffer_id_t*, 14>{&runtime_init_buffered_buffer,
auto buffers = std::array<rocprofiler_buffer_id_t*, 15>{&runtime_init_buffered_buffer,
&hsa_api_buffered_buffer,
&hip_api_buffered_buffer,
&marker_api_buffered_buffer,
@@ -1169,7 +1213,8 @@ auto buffers = std::array<rocprofiler_buffer_id_t*, 14>{&runtime_init_buffered_b
&corr_id_retire_buffer,
&rccl_api_buffered_buffer,
&ompt_buffered_buffer,
&rocdecode_api_buffer};
&rocdecode_api_buffer,
&rocjpeg_api_buffer};
auto agents = std::vector<rocprofiler_agent_t>{};
auto agents_map = std::unordered_map<rocprofiler_agent_id_t, rocprofiler_agent_t>{};
@@ -1344,6 +1389,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
nullptr),
"rocdecode api callback tracing service configure");
ROCPROFILER_CALL(
rocprofiler_configure_callback_tracing_service(rocjpeg_api_callback_ctx,
ROCPROFILER_CALLBACK_TRACING_ROCJPEG_API,
nullptr,
0,
tool_tracing_callback,
nullptr),
"rocjpeg api callback tracing service configure");
ROCPROFILER_CALL(
rocprofiler_configure_callback_tracing_service(ompt_callback_ctx,
ROCPROFILER_CALLBACK_TRACING_OMPT,
@@ -1473,6 +1527,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
&rocdecode_api_buffer),
"buffer creation");
ROCPROFILER_CALL(rocprofiler_create_buffer(rocjpeg_api_buffered_ctx,
buffer_size,
watermark,
ROCPROFILER_BUFFER_POLICY_LOSSLESS,
tool_tracing_buffered,
tool_data,
&rocjpeg_api_buffer),
"buffer creation");
ROCPROFILER_CALL(rocprofiler_create_buffer(ompt_buffered_ctx,
buffer_size,
watermark,
@@ -1605,6 +1668,14 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
rocdecode_api_buffer),
"buffer tracing service for rocdecode api configure");
ROCPROFILER_CALL(
rocprofiler_configure_buffer_tracing_service(rocjpeg_api_buffered_ctx,
ROCPROFILER_BUFFER_TRACING_ROCJPEG_API,
nullptr,
0,
rocjpeg_api_buffer),
"buffer tracing service for rocjpeg api configure");
ROCPROFILER_CALL(
rocprofiler_configure_buffer_tracing_service(
ompt_buffered_ctx, ROCPROFILER_BUFFER_TRACING_OMPT, nullptr, 0, ompt_buffered_buffer),
@@ -1775,7 +1846,9 @@ tool_fini(void* tool_data)
<< ", ompt_bf_records=" << ompt_bf_records.size()
<< ", counter_collection_value_records=" << counter_collection_bf_records.size()
<< ", rocdecode_api_callback_records=" << rocdecode_api_cb_records.size()
<< ", rocdecode_api_bf_records=" << rocdecode_api_bf_records.size() << "...\n"
<< ", rocdecode_api_bf_records=" << rocdecode_api_bf_records.size()
<< ", rocjpeg_api_callback_records=" << rocjpeg_api_cb_records.size()
<< ", rocjpeg_api_bf_records=" << rocjpeg_api_bf_records.size() << "...\n"
<< std::flush;
auto* _call_stack = static_cast<call_stack_t*>(tool_data);
@@ -1872,6 +1945,7 @@ write_json(call_stack_t* _call_stack)
json_ar(cereal::make_nvp("memory_copies", memory_copy_cb_records));
json_ar(cereal::make_nvp("memory_allocations", memory_allocation_cb_records));
json_ar(cereal::make_nvp("rocdecode_api_traces", rocdecode_api_cb_records));
json_ar(cereal::make_nvp("rocjpeg_api_traces", rocjpeg_api_cb_records));
} catch(std::exception& e)
{
std::cerr << "[" << getpid() << "][" << __FUNCTION__
@@ -1899,6 +1973,7 @@ write_json(call_stack_t* _call_stack)
json_ar(cereal::make_nvp("retired_correlation_ids", corr_id_retire_records));
json_ar(cereal::make_nvp("counter_collection", counter_collection_bf_records));
json_ar(cereal::make_nvp("rocdecode_api_traces", rocdecode_api_bf_records));
json_ar(cereal::make_nvp("rocjpeg_api_traces", rocjpeg_api_bf_records));
} catch(std::exception& e)
{
std::cerr << "[" << getpid() << "][" << __FUNCTION__
@@ -1972,6 +2047,8 @@ write_perfetto()
tids.emplace(itr.thread_id);
for(auto itr : rocdecode_api_bf_records)
tids.emplace(itr.thread_id);
for(auto itr : rocjpeg_api_bf_records)
tids.emplace(itr.thread_id);
for(auto itr : memory_copy_bf_records)
{
@@ -2266,6 +2343,47 @@ write_perfetto()
itr.end_timestamp);
}
for(auto itr : rocjpeg_api_bf_records)
{
auto name = buffer_names.at(itr.kind, itr.operation);
auto& track = thread_tracks.at(itr.thread_id);
auto _args = callback_arg_array_t{};
auto ritr = std::find_if(
rocjpeg_api_cb_records.begin(),
rocjpeg_api_cb_records.end(),
[&itr](const auto& citr) {
return (citr.record.correlation_id.internal == itr.correlation_id.internal &&
!citr.args.empty());
});
if(ritr != rocjpeg_api_cb_records.end()) _args = ritr->args;
TRACE_EVENT_BEGIN(sdk::perfetto_category<sdk::category::rocjpeg_api>::name,
::perfetto::StaticString(name.data()),
track,
itr.start_timestamp,
::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
"begin_ns",
itr.start_timestamp,
"tid",
itr.thread_id,
"kind",
itr.kind,
"operation",
itr.operation,
"corr_id",
itr.correlation_id.internal,
[&](::perfetto::EventContext ctx) {
for(const auto& aitr : _args)
sdk::add_perfetto_annotation(ctx, aitr.first, aitr.second);
});
TRACE_EVENT_END(sdk::perfetto_category<sdk::category::rocjpeg_api>::name,
track,
itr.end_timestamp,
"end_ns",
itr.end_timestamp);
}
for(auto itr : ompt_bf_records)
{
auto name = buffer_names.at(itr.kind, itr.operation);