rocprofv3: summary reports + more JSON metadata (#1029)

* Move include/rocprofiler-sdk/cxx/details/delimit.hpp to tokenize.hpp

* Update docs/how-to/using-rocprofv3.rst

- fix code block indents
- reorder rocprofv3 options, limit them to important options
- add docs for `--runtime-trace`

* Update rocprofv3.py

- parser argument groups
- new `--runtime-trace` option
- new `--summary` option
- new `--summary-per-domain` option
- new `--summary-groups` option
- new `--summary-output-file` option
- new `--summary-units` option

* Update lib/rocprofiler-sdk/hsa/async_copy.cpp

- fix async copy operation names: add "MEMORY_COPY_" prefix

* lib/rocprofiler-sdk-tool: update statistics.{hpp,cpp}

- statistics<>::get_percent function
- stats_entry_t struct
- stats_formatter struct
- percentage struct
- std::to_string(::rocprofiler::tool::percentage)

* lib/rocprofiler-sdk-tool: update domain_type.{hpp,cpp}

- reorder domain_type enum values

* lib/rocprofiler-sdk-tool: update generateCSV.{hpp,cpp}

- separate writing CSV from accumulating statistics
- a lot of functionality was moved to statistics.{hpp,cpp}

* lib/rocprofiler-sdk-tool: update output_file.{hpp,cpp}

- output_stream_t struct
- get_output_stream(...) returns output_stream_t instance

* lib/rocprofiler-sdk-tool: update generateJSON.cpp

- update get_output_stream usage to output_stream_t

* lib/rocprofiler-sdk-tool: update generateOTF2.cpp

- header include order tweak

* lib/rocprofiler-sdk-tool: update buffered_output.hpp

- stats_data_t was renamed to stats_entry_t

* lib/rocprofiler-sdk-tool: update generatePerfetto.cpp

- header include tweak

* lib/rocprofiler-sdk-tool: update tmp_file_buffer.hpp

- emit warning message if write_ring_buffer fails after offloading instead of aborting
- prefer placement new instead of assignment in write_ring_buffer

* lib/rocprofiler-sdk-tool: add generateStats.{hpp,cpp}

- functions for accumulating statistics

* Update tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt

- accommodate tweak to CSV output file name for HIP and HSA traces

* lib/rocprofiler-sdk-tool: update config.{hpp,cpp}

- new config variables
  - stats_summary
  - stats_summary_per_domain
  - summary_output
  - stats_summary_unit_value
  - stats_summary_unit
  - stats_summary_file
  - stats_summary_groups
- support output keys for hostname: %hostname% / %h

* lib/rocprofiler-sdk-tool: update tool.cpp

- support summary output

* Documentation fixes

* Test for summary output

* Update tests/bin/transpose to use more ROCTx

- also support building with the roctracer ROCTx

* Remove roctxMark from OTF2 + fix kernel-rename tests

- following more ROCTx calls in transpose, kernel-rename validation had to be updated

* JSON metadata + JSON summary

- add serialization support for config
- add serialization support for statistics
- additions to json spec
  - rocprofiler-sdk-tool/metadata/config
  - rocprofiler-sdk-tool/metadata/command
  - rocprofiler-sdk-tool/summary
- config output_keys support for NVIDIA %q{<ENV-VAR>} syntax
- config output_keys support keys within keys

* rocprofv3 --summary-groups warning if no domain matches

- emit warning if a regex in for summary groups did not match any domain names

* Compile fix for lib/rocprofiler-sdk-tool/tool.cpp

- get_config().scratch_memory_trace
- pass contributions to write_json

* Update rocprofv3.py to preload rocprofiler-sdk-roctx

- appended to LD_PRELOAD when args.marker_trace is enabled

* Fix ReST link errors about subtitle underline being too short

* Patch tokenization of config::stats_summary_groups

- guard against array values of empty strings

* Tweak rocprofv3 summary test

- input-summary.yaml (used by rocprofv3-test-summary-inp-yaml-execute) only provides one summary group regex

* Disable LD_PRELOAD of librocprofiler-sdk-roctx.so

- this causes problems in the sanitizers, will be addressed in another PR

source/lib/rocprofiler-sdk-tool/generateJSON.cpp

@@ -24,8 +24,10 @@
 #include "config.hpp"
 #include "helper.hpp"
 #include "output_file.hpp"
+#include "statistics.hpp"
 
 #include "lib/common/string_entry.hpp"
+#include "lib/common/utility.hpp"
 
 #include <rocprofiler-sdk/fwd.h>
 #include <rocprofiler-sdk/marker/api_id.h>
@@ -39,6 +41,7 @@ namespace tool
 void
 write_json(tool_table*                                                      tool_functions,
            uint64_t                                                         pid,
+           const domain_stats_vec_t&                                        domain_stats,
            std::vector<rocprofiler_agent_v0_t>                              agent_data,
            std::vector<rocprofiler_tool_counter_info_t>                     counter_data,
            std::deque<rocprofiler_buffer_tracing_hip_api_record_t>*         hip_api_deque,
@@ -52,14 +55,14 @@ write_json(tool_table*                                                      tool
 {
     using JSONOutputArchive = cereal::MinimalJSONOutputArchive;
 
-    constexpr auto json_prec      = 32;
-    constexpr auto json_indent    = JSONOutputArchive::Options::IndentChar::space;
-    auto           json_opts      = JSONOutputArchive::Options{json_prec, json_indent, 1};
-    auto           filename       = std::string_view{"results"};
-    auto [output_stream, cleanup] = get_output_stream(filename, ".json");
+    constexpr auto json_prec   = 32;
+    constexpr auto json_indent = JSONOutputArchive::Options::IndentChar::space;
+    auto           json_opts   = JSONOutputArchive::Options{json_prec, json_indent, 1};
+    auto           filename    = std::string_view{"results"};
+    auto           ofs         = get_output_stream(filename, ".json");
 
     {
-        auto json_ar = JSONOutputArchive{*output_stream, json_opts};
+        auto json_ar = JSONOutputArchive{*ofs.stream, json_opts};
         json_ar.setNextName("rocprofiler-sdk-tool");
         json_ar.startNode();
 
@@ -74,6 +77,29 @@ write_json(tool_table*                                                      tool
             json_ar(cereal::make_nvp("pid", pid));
             json_ar(cereal::make_nvp("init_time", timestamps->app_start_time));
             json_ar(cereal::make_nvp("fini_time", timestamps->app_end_time));
+            json_ar(cereal::make_nvp("config", get_config()));
+            json_ar(cereal::make_nvp("command", common::read_command_line(getpid())));
+            json_ar.finishNode();
+        }
+
+        // summary
+        {
+            json_ar.setNextName("summary");
+            json_ar.startNode();
+            json_ar.makeArray();
+
+            for(const auto& itr : domain_stats)
+            {
+                auto _name = get_domain_column_name(itr.first);
+                json_ar.startNode();
+
+                json_ar(cereal::make_nvp("domain", std::string{_name}));
+                json_ar(cereal::make_nvp("stats", itr.second));
+                // itr.second.serialize(json_ar, 0);
+
+                json_ar.finishNode();
+            }
+
             json_ar.finishNode();
         }
 
@@ -154,8 +180,7 @@ write_json(tool_table*                                                      tool
         json_ar.finishNode();
     }
 
-    *output_stream << std::flush;
-    if(cleanup) cleanup(output_stream);
+    ofs.close();
 }
 
 }  // namespace tool