rocprofv3: refactor and reorganize rocprofiler-sdk-tool library (#1138)

* Add rocprofv3-multi-node.md to source/lib/rocprofiler-sdk-tool * Initial source re-organization - create "output" static library * Update include/rocprofiler-sdk/cxx/serialization.hpp - add GPR count fields to kernel symbol serialization * Add source/scripts/generate-rocpd.py - reads one or more JSON output files from rocprofv3 and writes rocpd SQLite3 database - Note: preliminary implementation * More reorganization b/t lib/rocprofiler-sdk-tool and lib/output * Updates to generate-rocpd.py - add SQL views - option: --absolute-timestamps -> --normalize-timestamps - option: --generic-markers - misc fixes with regards to getting the views working - support marker names * Update generate-rocpd.py - Add --marker-mode option * Update generate-rocpd.py - Improve debugging of bad bulk SQLite statements * Update rocprofv3-multi-node.md - cleanup of proposed SQL schema * lib/output/format_path.{hpp,cpp} - rename format to format_path (in config.hpp and config.cpp) - move format_path functionality to format_path.{hpp,cpp} * Rework lib/output/tmp_file_buffer.{hpp,cpp} * Update output_key.cpp - support %cwd%, %launch_date% * Rework lib/output/buffered_output.hpp * Support csv_output_file constructed via domain_type * Update lib/output/domain_type.{hpp,cpp} - get_domain_trace_file_name - get_domain_stats_file_name * Update lib/rocprofiler-sdk-tool/tool.cpp - tweak headers * Update lib/output/generate*.cpp - remove include of helpers.hpp - CSV uses domain_type for filenames * Update samples/counter_collection/per_dev_serialization.cpp - make wait_on volatile * Remove tool_table from lib/output and lib/rocprofiler-sdk-tool - Also split various structs into their own files - lib/output/agent_info - lib/output/metadata - lib/output/kernel_symbol_info - lib/output/counter_info - Implemented rocprofiler::tool::metadata * Optimize rocprofiler_tool_counter_collection_record_t - reduce the size of the struct from 24784 bytes to 8376 bytes * Introduced output_config - split subset of config (from tools library) into output_config to be able to configure the output generating functions separately from the tool library - this is a significant step towards the output generating functions not relying on static global memory * Stream chunks of data into output instead of loading all info memory * Remove duplicate group_segment_size in rocprofiler_kernel_dispatch_info_t serialization * Adding Q&A to rocprofv3-multi-node.md * Remove all remaining include lib/rocprofiler-sdk-tool from lib/output - migrated a fair amount of code from lib/rocprofiler-sdk-tool/helper.hpp to lib/output * Update Q&A of rocprofv3-multi-node.md * Fix minor compilation errors + minor cleanup * Update hsa/async_copy.cpp - when ROCPROFILER_CI_STRICT_TIMESTAMPS > 0, reduce the active_signal sync wait time * Update profiling_time.hpp - fix log messages for when start/end time is less/greater than enqueue/current CPU time * Fix generate_stats for tool_counter_record_t * Dictionary optimization for generate-rocpd.py --------- Co-authored-by: SrirakshaNag <104580803+SrirakshaNag@users.noreply.github.com>
2024-11-07 01:15:19 -06:00
parent 98858b60ec
commit 5eb8c2658c
59 changed files with 5524 additions and 3294 deletions
@@ -0,0 +1,124 @@
+// MIT License
+//
+// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "output_config.hpp"
+
+namespace rocprofiler
+{
+namespace tool
+{
+output_config
+output_config::load_from_env()
+{
+    auto cfg = output_config{};
+    cfg.parse_env();
+    return cfg;
+}
+
+output_config
+output_config::load_from_env(output_config&& cfg)
+{
+    cfg.parse_env();
+    return cfg;
+}
+
+void
+output_config::parse_env()
+{
+    stats         = common::get_env("ROCPROF_STATS", stats);
+    stats_summary = common::get_env("ROCPROF_STATS_SUMMARY", stats_summary);
+    stats_summary_per_domain =
+        common::get_env("ROCPROF_STATS_SUMMARY_PER_DOMAIN", stats_summary_per_domain);
+    stats_summary_unit = common::get_env("ROCPROF_STATS_SUMMARY_UNITS", stats_summary_unit);
+    stats_summary_file = common::get_env("ROCPROF_STATS_SUMMARY_OUTPUT", stats_summary_file);
+
+    perfetto_backend = common::get_env("ROCPROF_PERFETTO_BACKEND", perfetto_backend);
+    perfetto_buffer_fill_policy =
+        common::get_env("ROCPROF_PERFETTO_BUFFER_FILL_POLICY", perfetto_buffer_fill_policy);
+    perfetto_shmem_size_hint =
+        common::get_env("ROCPROF_PERFETTO_SHMEM_SIZE_HINT_KB", perfetto_shmem_size_hint);
+    perfetto_buffer_size = common::get_env("ROCPROF_PERFETTO_BUFFER_SIZE_KB", perfetto_buffer_size);
+
+    output_path   = common::get_env("ROCPROF_OUTPUT_PATH", output_path);
+    output_file   = common::get_env("ROCPROF_OUTPUT_FILE_NAME", output_file);
+    tmp_directory = common::get_env("ROCPROF_TMPDIR", tmp_directory);
+    kernel_rename = common::get_env("ROCPROF_KERNEL_RENAME", false);
+
+    auto to_upper = [](std::string val) {
+        for(auto& vitr : val)
+            vitr = toupper(vitr);
+        return val;
+    };
+
+    output_format = common::get_env("ROCPROF_OUTPUT_FORMAT", output_format);
+    auto entries  = std::set<std::string>{};
+    for(const auto& itr : sdk::parse::tokenize(output_format, " \t,;:"))
+        entries.emplace(to_upper(itr));
+
+    csv_output     = entries.count("CSV") > 0 || entries.empty();
+    json_output    = entries.count("JSON") > 0;
+    pftrace_output = entries.count("PFTRACE") > 0;
+    otf2_output    = entries.count("OTF2") > 0;
+
+    const auto supported_formats =
+        std::set<std::string_view>{"CSV", "JSON", "PFTRACE", "OTF2", "ROCPD"};
+    for(const auto& itr : entries)
+    {
+        LOG_IF(FATAL, supported_formats.count(itr) == 0)
+            << "Unsupported output format type: " << itr;
+    }
+
+    const auto supported_perfetto_backends = std::set<std::string_view>{"inprocess", "system"};
+    LOG_IF(FATAL, supported_perfetto_backends.count(perfetto_backend) == 0)
+        << "Unsupported perfetto backend type: " << perfetto_backend;
+
+    if(stats_summary_unit == "sec")
+        stats_summary_unit_value = common::units::sec;
+    else if(stats_summary_unit == "msec")
+        stats_summary_unit_value = common::units::msec;
+    else if(stats_summary_unit == "usec")
+        stats_summary_unit_value = common::units::usec;
+    else if(stats_summary_unit == "nsec")
+        stats_summary_unit_value = common::units::nsec;
+    else
+    {
+        ROCP_FATAL << "Unsupported summary units value: " << stats_summary_unit;
+    }
+
+    if(auto _summary_grps = common::get_env("ROCPROF_STATS_SUMMARY_GROUPS", "");
+       !_summary_grps.empty())
+    {
+        stats_summary_groups =
+            sdk::parse::tokenize(_summary_grps, std::vector<std::string_view>{"##@@##"});
+
+        // remove any empty strings (just in case these slipped through)
+        stats_summary_groups.erase(std::remove_if(stats_summary_groups.begin(),
+                                                  stats_summary_groups.end(),
+                                                  [](const auto& itr) { return itr.empty(); }),
+                                   stats_summary_groups.end());
+    }
+
+    // enable summary output if any of these are enabled
+    summary_output = (stats_summary || stats_summary_per_domain || !stats_summary_groups.empty());
+}
+}  // namespace tool
+}  // namespace rocprofiler