Fix rocprofv3 output filename containing sub-directory (#1062)

* Fix -d option broken by hostname

* Fix rocprofv3 output filename containing directory

* Fix TID handling in Perfetto and OTF2 output

* Revert changes which removed hostname

* Revise tests/rocprofv3/tracing output filenames

- specify an output filename for tests which include a subdirectory

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

[ROCm/rocprofiler-sdk commit: a4b3a57ecc]

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp

@@ -401,11 +401,17 @@ write_otf2(tool_table*                                                      tool
             tids.emplace(itr.thread_id);
 
         for(auto itr : *memory_copy_data)
+        {
+            tids.emplace(itr.thread_id);
             agent_thread_ids[itr.thread_id].emplace(itr.dst_agent_id);
+        }
 
         for(auto itr : *kernel_dispatch_data)
+        {
+            tids.emplace(itr.thread_id);
             agent_queue_ids[itr.thread_id][itr.dispatch_info.agent_id].emplace(
                 itr.dispatch_info.queue_id);
+        }
     }
 
     {

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp

@@ -157,10 +157,16 @@ write_perfetto(
             tids.emplace(itr.thread_id);
 
         for(auto itr : *memory_copy_data)
+        {
+            tids.emplace(itr.thread_id);
             agent_thread_ids[itr.dst_agent_id].emplace(itr.thread_id);
+        }
 
         for(auto itr : *kernel_dispatch_data)
+        {
+            tids.emplace(itr.thread_id);
             agent_queue_ids[itr.dispatch_info.agent_id].emplace(itr.dispatch_info.queue_id);
+        }
     }
 
     uint64_t nthrn = 0;

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/output_file.cpp

@@ -22,6 +22,8 @@
 
 #include "output_file.hpp"
 #include "config.hpp"
+
+#include "lib/common/filesystem.hpp"
 #include "lib/common/logging.hpp"
 
 #include <fmt/core.h>
@@ -54,12 +56,31 @@ get_output_filename(std::string_view fname, std::string_view ext)
     auto output_prefix = tool::format(tool::get_config().output_file);
 
     if(fs::exists(output_path) && !fs::is_directory(fs::status(output_path)))
-        throw std::runtime_error{
-            fmt::format("ROCPROFILER_OUTPUT_PATH ({}) already exists and is not a directory",
-                        output_path.string())};
-    if(!fs::exists(output_path)) fs::create_directories(output_path);
+    {
+        ROCP_FATAL << fmt::format(
+            "ROCPROFILER_OUTPUT_PATH ({}) already exists and is not a directory",
+            output_path.string());
+    }
+    else if(!fs::exists(output_path))
+    {
+        fs::create_directories(output_path);
+    }
 
-    return tool::format(output_path / fmt::format("{}_{}{}", output_prefix, fname, _ext));
+    auto _ofname = tool::format(output_path / fmt::format("{}_{}{}", output_prefix, fname, _ext));
+
+    // the prefix may contain a subdirectory
+    if(auto _ofname_path = fs::path{_ofname}.parent_path(); !fs::exists(_ofname_path))
+    {
+        fs::create_directories(_ofname_path);
+    }
+    else if(fs::exists(_ofname_path) && !fs::is_directory(fs::status(_ofname_path)))
+    {
+        ROCP_FATAL << fmt::format(
+            "ROCPROFILER_OUTPUT_PATH ({}) already exists and is not a directory",
+            output_path.string());
+    }
+
+    return _ofname;
 }
 
 output_stream_t

projects/rocprofiler-sdk/tests/rocprofv3/tracing/CMakeLists.txt

@@ -29,7 +29,7 @@ add_test(
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -M --hsa-trace --kernel-trace
         --memory-copy-trace --marker-trace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o
-        out --output-format pftrace csv json --log-level ${LOG_LEVEL} --
+        cmdl-input/out --output-format pftrace csv json --log-level ${LOG_LEVEL} --
         $<TARGET_FILE:simple-transpose>)
 
 set_tests_properties(
@@ -50,7 +50,7 @@ add_test(
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
         ${CMAKE_CURRENT_BINARY_DIR}/input_trace.json -d
-        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace-input-json --log-level ${LOG_LEVEL} --
+        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace --log-level ${LOG_LEVEL} --
         $<TARGET_FILE:simple-transpose>)
 
 set_tests_properties(
@@ -70,52 +70,53 @@ add_test(
     NAME rocprofv3-test-trace-validate
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --hsa-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_hsa_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_hsa_api_trace.csv
         --kernel-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_kernel_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_kernel_trace.csv
         --memory-copy-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_memory_copy_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_memory_copy_trace.csv
         --marker-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_marker_api_trace.csv
         --agent-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_agent_info.csv
-        --json-input ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.json
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_agent_info.csv
+        --json-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.json
         --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.pftrace)
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.pftrace)
 
 add_test(
     NAME rocprofv3-test-trace-input-json-validate
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --hsa-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_hsa_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_hsa_api_trace.csv
         --kernel-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_kernel_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_kernel_trace.csv
         --memory-copy-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_memory_copy_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_memory_copy_trace.csv
         --marker-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_marker_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_marker_api_trace.csv
         --agent-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_agent_info.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_agent_info.csv
         --json-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.json
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.json
         --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.pftrace)
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.pftrace)
 
 set(VALIDATION_FILES
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.pftrace
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_results.json
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_memory_copy_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_hsa_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_kernel_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_agent_info.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.pftrace
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_results.json
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_memory_copy_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_hsa_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_kernel_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_marker_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace-input-json/out_agent_info.csv)
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.pftrace
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_results.json
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_memory_copy_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_hsa_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_kernel_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_marker_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/cmdl-input/out_agent_info.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.pftrace
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_results.json
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_memory_copy_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_hsa_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_kernel_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_marker_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/json-input/out_agent_info.csv)
 
 set_tests_properties(
     rocprofv3-test-trace-validate
@@ -150,8 +151,8 @@ add_test(
     NAME rocprofv3-test-systrace-execute
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --sys-trace -d
-        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -o out --output-format pftrace csv
-        json -- $<TARGET_FILE:simple-transpose>)
+        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -o cmdl-input/out --output-format
+        pftrace csv json -- $<TARGET_FILE:simple-transpose>)
 
 set_tests_properties(
     rocprofv3-test-systrace-execute
@@ -171,8 +172,7 @@ add_test(
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
         ${CMAKE_CURRENT_BINARY_DIR}/input_systrace.json -d
-        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace-input-json --
-        $<TARGET_FILE:simple-transpose>)
+        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-systrace -- $<TARGET_FILE:simple-transpose>)
 
 set_tests_properties(
     rocprofv3-test-systrace-input-json-execute
@@ -191,55 +191,56 @@ add_test(
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py -k
         "not test_hsa_api_trace" --hsa-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_hsa_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_hsa_api_trace.csv
         --kernel-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_kernel_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_kernel_trace.csv
         --memory-copy-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_memory_copy_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_memory_copy_trace.csv
         --marker-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_marker_api_trace.csv
         --agent-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_agent_info.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_agent_info.csv
         --json-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.json
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.json
         --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.pftrace)
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.pftrace
+    )
 
 add_test(
     NAME rocprofv3-test-systrace-input-json-validate
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py -k
         "not test_hsa_api_trace" --hsa-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_hsa_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_hsa_api_trace.csv
         --kernel-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_kernel_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_kernel_trace.csv
         --memory-copy-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_memory_copy_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_memory_copy_trace.csv
         --marker-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_marker_api_trace.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_marker_api_trace.csv
         --agent-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_agent_info.csv
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_agent_info.csv
         --json-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.json
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.json
         --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.pftrace
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.pftrace
     )
 
 set(SYS_VALIDATION_FILES
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.pftrace
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_results.json
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_memory_copy_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_hsa_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_kernel_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_agent_info.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.pftrace
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_results.json
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_memory_copy_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_hsa_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_kernel_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_marker_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace-input-json/out_agent_info.csv)
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.pftrace
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_results.json
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_memory_copy_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_hsa_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_kernel_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_marker_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/cmdl-input/out_agent_info.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.pftrace
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_results.json
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_memory_copy_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_hsa_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_kernel_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_marker_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/json-input/out_agent_info.csv)
 
 set_tests_properties(
     rocprofv3-test-systrace-validate

projects/rocprofiler-sdk/tests/rocprofv3/tracing/input_systrace.json

@@ -2,7 +2,7 @@
     "jobs": [
         {
             "sys_trace": true,
-            "output_file": "out",
+            "output_file": "json-input/out",
             "output_format": [
                 "csv",
                 "json",

projects/rocprofiler-sdk/tests/rocprofv3/tracing/input_trace.json

@@ -5,7 +5,7 @@
             "kernel_trace": true,
             "memory_copy_trace": true,
             "marker_trace": true,
-            "output_file": "out",
+            "output_file": "json-input/out",
             "output_format": [
                 "csv",
                 "json",

projects/rocprofiler-sdk/tests/tools/json-tool.cpp

@@ -1568,13 +1568,17 @@ write_perfetto()
 
         for(auto itr : memory_copy_bf_records)
         {
+            tids.emplace(itr.thread_id);
             agent_ids.emplace(itr.dst_agent_id.handle);
             agent_ids.emplace(itr.src_agent_id.handle);
         }
 
         for(auto itr : kernel_dispatch_bf_records)
+        {
+            tids.emplace(itr.thread_id);
             agent_queue_ids[itr.dispatch_info.agent_id.handle].emplace(
                 itr.dispatch_info.queue_id.handle);
+        }
     }
 
     auto thread_tracks = std::unordered_map<rocprofiler_thread_id_t, ::perfetto::Track>{};