From a4b3a57eccfc0126410b8d134f61062dee554cfb Mon Sep 17 00:00:00 2001 From: Gopesh Bhardwaj Date: Wed, 11 Sep 2024 04:19:36 +0530 Subject: [PATCH] Fix rocprofv3 output filename containing sub-directory (#1062) * Fix -d option broken by hostname * Fix rocprofv3 output filename containing directory * Fix TID handling in Perfetto and OTF2 output * Revert changes which removed hostname * Revise tests/rocprofv3/tracing output filenames - specify an output filename for tests which include a subdirectory --------- Co-authored-by: Jonathan R. Madsen --- .../lib/rocprofiler-sdk-tool/generateOTF2.cpp | 6 + .../rocprofiler-sdk-tool/generatePerfetto.cpp | 6 + .../lib/rocprofiler-sdk-tool/output_file.cpp | 31 ++++- tests/rocprofv3/tracing/CMakeLists.txt | 125 +++++++++--------- tests/rocprofv3/tracing/input_systrace.json | 2 +- tests/rocprofv3/tracing/input_trace.json | 2 +- tests/tools/json-tool.cpp | 4 + 7 files changed, 107 insertions(+), 69 deletions(-) diff --git a/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp b/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp index f615d70f52..1e6a4e5ac3 100644 --- a/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp +++ b/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp @@ -401,11 +401,17 @@ write_otf2(tool_table* tool tids.emplace(itr.thread_id); for(auto itr : *memory_copy_data) + { + tids.emplace(itr.thread_id); agent_thread_ids[itr.thread_id].emplace(itr.dst_agent_id); + } for(auto itr : *kernel_dispatch_data) + { + tids.emplace(itr.thread_id); agent_queue_ids[itr.thread_id][itr.dispatch_info.agent_id].emplace( itr.dispatch_info.queue_id); + } } { diff --git a/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp b/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp index a60a98ead2..6fb76c66dc 100644 --- a/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp +++ b/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp @@ -157,10 +157,16 @@ write_perfetto( tids.emplace(itr.thread_id); for(auto itr : *memory_copy_data) + { + tids.emplace(itr.thread_id); agent_thread_ids[itr.dst_agent_id].emplace(itr.thread_id); + } for(auto itr : *kernel_dispatch_data) + { + tids.emplace(itr.thread_id); agent_queue_ids[itr.dispatch_info.agent_id].emplace(itr.dispatch_info.queue_id); + } } uint64_t nthrn = 0; diff --git a/source/lib/rocprofiler-sdk-tool/output_file.cpp b/source/lib/rocprofiler-sdk-tool/output_file.cpp index 9b76ea70d4..14a1227af0 100644 --- a/source/lib/rocprofiler-sdk-tool/output_file.cpp +++ b/source/lib/rocprofiler-sdk-tool/output_file.cpp @@ -22,6 +22,8 @@ #include "output_file.hpp" #include "config.hpp" + +#include "lib/common/filesystem.hpp" #include "lib/common/logging.hpp" #include @@ -54,12 +56,31 @@ get_output_filename(std::string_view fname, std::string_view ext) auto output_prefix = tool::format(tool::get_config().output_file); if(fs::exists(output_path) && !fs::is_directory(fs::status(output_path))) - throw std::runtime_error{ - fmt::format("ROCPROFILER_OUTPUT_PATH ({}) already exists and is not a directory", - output_path.string())}; - if(!fs::exists(output_path)) fs::create_directories(output_path); + { + ROCP_FATAL << fmt::format( + "ROCPROFILER_OUTPUT_PATH ({}) already exists and is not a directory", + output_path.string()); + } + else if(!fs::exists(output_path)) + { + fs::create_directories(output_path); + } - return tool::format(output_path / fmt::format("{}_{}{}", output_prefix, fname, _ext)); + auto _ofname = tool::format(output_path / fmt::format("{}_{}{}", output_prefix, fname, _ext)); + + // the prefix may contain a subdirectory + if(auto _ofname_path = fs::path{_ofname}.parent_path(); !fs::exists(_ofname_path)) + { + fs::create_directories(_ofname_path); + } + else if(fs::exists(_ofname_path) && !fs::is_directory(fs::status(_ofname_path))) + { + ROCP_FATAL << fmt::format( + "ROCPROFILER_OUTPUT_PATH ({}) already exists and is not a directory", + output_path.string()); + } + + return _ofname; } output_stream_t diff --git a/tests/rocprofv3/tracing/CMakeLists.txt b/tests/rocprofv3/tracing/CMakeLists.txt index 15232a4eab..5b6c7cdff4 100644 --- a/tests/rocprofv3/tracing/CMakeLists.txt +++ b/tests/rocprofv3/tracing/CMakeLists.txt @@ -29,7 +29,7 @@ add_test( COMMAND $ -M --hsa-trace --kernel-trace --memory-copy-trace --marker-trace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o - out --output-format pftrace csv json --log-level ${LOG_LEVEL} -- + cmdl-input/out --output-format pftrace csv json --log-level ${LOG_LEVEL} -- $) set_tests_properties( @@ -50,7 +50,7 @@ add_test( COMMAND $ -i ${CMAKE_CURRENT_BINARY_DIR}/input_trace.json -d - ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace-input-json --log-level ${LOG_LEVEL} -- + ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace --log-level ${LOG_LEVEL} -- $) set_tests_properties( @@ -70,52 +70,53 @@ add_test( NAME rocprofv3-test-trace-validate COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --hsa-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_hsa_api_trace.csv --kernel-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_kernel_trace.csv --memory-copy-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_memory_copy_trace.csv --marker-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_marker_api_trace.csv --agent-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_agent_info.csv - --json-input ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_agent_info.csv + --json-input + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.json --pftrace-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.pftrace) + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.pftrace) add_test( NAME rocprofv3-test-trace-input-json-validate COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --hsa-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_hsa_api_trace.csv --kernel-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_kernel_trace.csv --memory-copy-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_memory_copy_trace.csv --marker-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_marker_api_trace.csv --agent-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_agent_info.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_agent_info.csv --json-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.json --pftrace-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.pftrace) + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.pftrace) set(VALIDATION_FILES - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.pftrace - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.json - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_memory_copy_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_hsa_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_kernel_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_agent_info.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.pftrace - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.json - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_memory_copy_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_hsa_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_kernel_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_marker_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_agent_info.csv) + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.pftrace + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_agent_info.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.pftrace + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_agent_info.csv) set_tests_properties( rocprofv3-test-trace-validate @@ -150,8 +151,8 @@ add_test( NAME rocprofv3-test-systrace-execute COMMAND $ --sys-trace -d - ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -o out --output-format pftrace csv - json -- $) + ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -o cmdl-input/out --output-format + pftrace csv json -- $) set_tests_properties( rocprofv3-test-systrace-execute @@ -171,8 +172,7 @@ add_test( COMMAND $ -i ${CMAKE_CURRENT_BINARY_DIR}/input_systrace.json -d - ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace-input-json -- - $) + ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -- $) set_tests_properties( rocprofv3-test-systrace-input-json-execute @@ -191,55 +191,56 @@ add_test( COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py -k "not test_hsa_api_trace" --hsa-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_hsa_api_trace.csv --kernel-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_kernel_trace.csv --memory-copy-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_memory_copy_trace.csv --marker-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_marker_api_trace.csv --agent-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_agent_info.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_agent_info.csv --json-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.json --pftrace-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.pftrace) + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.pftrace + ) add_test( NAME rocprofv3-test-systrace-input-json-validate COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py -k "not test_hsa_api_trace" --hsa-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_hsa_api_trace.csv --kernel-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_kernel_trace.csv --memory-copy-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_memory_copy_trace.csv --marker-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_marker_api_trace.csv --agent-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_agent_info.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_agent_info.csv --json-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.json --pftrace-input - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.pftrace + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.pftrace ) set(SYS_VALIDATION_FILES - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.pftrace - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.json - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_memory_copy_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_hsa_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_kernel_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_agent_info.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.pftrace - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.json - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_memory_copy_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_hsa_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_kernel_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_marker_api_trace.csv - ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_agent_info.csv) + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.pftrace + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_agent_info.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.pftrace + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.json + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_memory_copy_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_hsa_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_kernel_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_marker_api_trace.csv + ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_agent_info.csv) set_tests_properties( rocprofv3-test-systrace-validate diff --git a/tests/rocprofv3/tracing/input_systrace.json b/tests/rocprofv3/tracing/input_systrace.json index 47e4a07aa1..0cddf54413 100644 --- a/tests/rocprofv3/tracing/input_systrace.json +++ b/tests/rocprofv3/tracing/input_systrace.json @@ -2,7 +2,7 @@ "jobs": [ { "sys_trace": true, - "output_file": "out", + "output_file": "json-input/out", "output_format": [ "csv", "json", diff --git a/tests/rocprofv3/tracing/input_trace.json b/tests/rocprofv3/tracing/input_trace.json index 2c7a2284f4..77b67b41d1 100644 --- a/tests/rocprofv3/tracing/input_trace.json +++ b/tests/rocprofv3/tracing/input_trace.json @@ -5,7 +5,7 @@ "kernel_trace": true, "memory_copy_trace": true, "marker_trace": true, - "output_file": "out", + "output_file": "json-input/out", "output_format": [ "csv", "json", diff --git a/tests/tools/json-tool.cpp b/tests/tools/json-tool.cpp index 65c3cf12a4..affb3228a1 100644 --- a/tests/tools/json-tool.cpp +++ b/tests/tools/json-tool.cpp @@ -1568,13 +1568,17 @@ write_perfetto() for(auto itr : memory_copy_bf_records) { + tids.emplace(itr.thread_id); agent_ids.emplace(itr.dst_agent_id.handle); agent_ids.emplace(itr.src_agent_id.handle); } for(auto itr : kernel_dispatch_bf_records) + { + tids.emplace(itr.thread_id); agent_queue_ids[itr.dispatch_info.agent_id.handle].emplace( itr.dispatch_info.queue_id.handle); + } } auto thread_tracks = std::unordered_map{};