diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-avail/generate_config.cpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-avail/generate_config.cpp index 14d68fbbe4..ecce26230f 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-avail/generate_config.cpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-avail/generate_config.cpp @@ -339,8 +339,8 @@ generate_config(std::string _config_file, const std::set& _config_f if(_lomni && !_romni) return true; if(_romni && !_lomni) return false; for(const auto* itr : - { "ROCPROFSYS_CONFIG", "ROCPROFSYS_MODE", "ROCPROFSYS_TRACE", - "ROCPROFSYS_PROFILE", "ROCPROFSYS_USE_SAMPLING", + { "ROCPROFSYS_CONFIG", "ROCPROFSYS_MODE", "ROCPROFSYS_TRACE_CACHED", + "ROCPROFSYS_TRACE", "ROCPROFSYS_PROFILE", "ROCPROFSYS_USE_SAMPLING", "ROCPROFSYS_USE_PROCESS_SAMPLING", "ROCPROFSYS_USE_ROCM", "ROCPROFSYS_USE_AMD_SMI", "ROCPROFSYS_USE_KOKKOSP", "ROCPROFSYS_USE_OMPT", "ROCPROFSYS_USE", "ROCPROFSYS_OUTPUT" }) diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp index 45f95b3683..14bf10b6c3 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp @@ -185,9 +185,10 @@ get_initial_environment() update_env(_env, "ROCPROFSYS_MODE", "causal"); update_env(_env, "ROCPROFSYS_USE_CAUSAL", true); update_env(_env, "ROCPROFSYS_USE_SAMPLING", false); - update_env(_env, "ROCPROFSYS_TRACE", false); + update_env(_env, "ROCPROFSYS_TRACE_CACHED", false); update_env(_env, "ROCPROFSYS_PROFILE", false); update_env(_env, "ROCPROFSYS_USE_PROCESS_SAMPLING", false); + update_env(_env, "ROCPROFSYS_TRACE", false); update_env(_env, "ROCPROFSYS_THREAD_POOL_SIZE", get_env("ROCPROFSYS_THREAD_POOL_SIZE", 0)); update_env(_env, "ROCPROFSYS_LAUNCHER", "rocprof-sys-causal"); diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp index 088b92150f..0b1d763e21 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp @@ -378,6 +378,13 @@ parse_args(int argc, char** argv, std::vector& _env) .action([&](parser_t& p) { update_env(_env, "ROCPROFSYS_TRACE", p.get("trace")); }); + parser + .add_argument({ "--trace-cached" }, + "Generate a detailed trace (perfetto output) from cached data ") + .max_count(1) + .action([&](parser_t& p) { + update_env(_env, "ROCPROFSYS_TRACE_CACHED", p.get("trace-cached")); + }); parser .add_argument( { "-P", "--profile" }, diff --git a/projects/rocprofiler-systems/source/lib/core/CMakeLists.txt b/projects/rocprofiler-systems/source/lib/core/CMakeLists.txt index 8982317304..6657f59c39 100644 --- a/projects/rocprofiler-systems/source/lib/core/CMakeLists.txt +++ b/projects/rocprofiler-systems/source/lib/core/CMakeLists.txt @@ -60,6 +60,7 @@ set(core_headers ${CMAKE_CURRENT_LIST_DIR}/argparse.hpp ${CMAKE_CURRENT_LIST_DIR}/categories.hpp ${CMAKE_CURRENT_LIST_DIR}/common.hpp + ${CMAKE_CURRENT_LIST_DIR}/common_types.hpp ${CMAKE_CURRENT_LIST_DIR}/concepts.hpp ${CMAKE_CURRENT_LIST_DIR}/config.hpp ${CMAKE_CURRENT_LIST_DIR}/constraint.hpp diff --git a/projects/rocprofiler-systems/source/lib/core/common_types.hpp b/projects/rocprofiler-systems/source/lib/core/common_types.hpp new file mode 100644 index 0000000000..ea24a6031f --- /dev/null +++ b/projects/rocprofiler-systems/source/lib/core/common_types.hpp @@ -0,0 +1,100 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include +#include +#include + +namespace rocprofsys +{ +/** + * @brief Structure for function/region argument information + * + * Represents metadata about function or region arguments including + * their position, type, name, and value information. + */ +struct argument_info +{ + uint32_t arg_number = 0; ///< Argument position/index + std::string arg_type = {}; ///< Argument type (e.g., "int", "float*") + std::string arg_name = {}; ///< Argument name + std::string arg_value = {}; ///< Argument value as string +}; + +using function_args_t = std::vector; + +inline std::string +get_args_string(const function_args_t& args) +{ + std::string args_str; + std::for_each(args.begin(), args.end(), [&args_str](const argument_info& arg) { + const auto* delimiter = ";;"; + std::stringstream ss; + ss << arg.arg_number << delimiter << arg.arg_type << delimiter << arg.arg_name + << delimiter << arg.arg_value << delimiter; + args_str.append(ss.str()); + }); + return args_str; +} + +inline function_args_t +process_arguments_string(const std::string& arg_str) +{ + function_args_t args; + const std::string delimiter = ";;"; + + auto split = [](const std::string& str, const std::string& _delimiter) { + std::vector tokens; + size_t start = 0; + size_t end = str.find(_delimiter); + + while(end != std::string::npos) + { + tokens.push_back(str.substr(start, end - start)); + start = end + _delimiter.length(); + end = str.find(_delimiter, start); + } + + return tokens; + }; + + auto tokens = split(arg_str, delimiter); + + // Ensure the number of tokens is a multiple of 4 + if(tokens.size() % 4 != 0) + { + throw std::invalid_argument("Malformed argument string."); + } + + for(auto it = tokens.begin(); it != tokens.end(); it += 4) + { + argument_info arg = { static_cast(std::stoi(*it)), *(it + 1), *(it + 2), + *(it + 3) }; + args.push_back(arg); + } + + return args; +} + +} // namespace rocprofsys diff --git a/projects/rocprofiler-systems/source/lib/core/config.cpp b/projects/rocprofiler-systems/source/lib/core/config.cpp index d607e498c9..7e5808f681 100644 --- a/projects/rocprofiler-systems/source/lib/core/config.cpp +++ b/projects/rocprofiler-systems/source/lib/core/config.cpp @@ -321,6 +321,10 @@ configure_settings(bool _init) ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_USE_ROCPD", "Enable rocpd backend", false, "backend", "rocpd"); + ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_TRACE_CACHED", + "Enable perfetto with trace cache", false, "backend", + "perfetto_caching"); + ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_USE_ROCM", "Enable ROCm API and kernel tracing", true, "backend", "rocm"); @@ -711,6 +715,11 @@ configure_settings(bool _init) "written to a file and re-loaded during finalization", true, "io", "data", "advanced"); + ROCPROFSYS_CONFIG_SETTING(bool, "ROCPROFSYS_MERGE_PERFETTO_FILES", + "Merge Perfetto traces. If not explicitly set, it will " + "default to the value of ROCPROFSYS_COLLAPSE_PROCESSES", + false, "perfetto", "data", "advanced"); + ROCPROFSYS_CONFIG_SETTING( std::string, "ROCPROFSYS_TMPDIR", "Base directory for temporary files", get_env("TMPDIR", "/tmp"), "io", "data", "advanced"); @@ -2370,6 +2379,13 @@ get_use_tmp_files() return static_cast&>(*_v->second).get(); } +bool +get_merge_perfetto_files() +{ + static auto _v = get_config()->find("ROCPROFSYS_MERGE_PERFETTO_FILES"); + return static_cast&>(*_v->second).get(); +} + std::string get_tmpdir() { @@ -2404,6 +2420,51 @@ get_database_absolute_path(std::string_view database_name, std::string_view suff return _val; } +std::string +get_perfetto_output_filename_with_suffix(std::string_view suffix) +{ + static auto _v = get_config()->find("ROCPROFSYS_PERFETTO_FILE"); + auto _val = static_cast&>(*_v->second).get(); + + // If absolute path is provided, return it as-is + if(!_val.empty() && _val.at(0) == '/') return _val; + + auto _pos_dir = _val.find_last_of('/'); + auto _dir = std::string{}; + auto _ext = std::string{ "proto" }; + + if(_pos_dir != std::string::npos) + { + _dir = _val.substr(0, _pos_dir + 1); + _val = _val.substr(_pos_dir + 1); + } + + auto _pos_ext = _val.find_last_of('.'); + if(_pos_ext + 1 < _val.length()) + { + _ext = _val.substr(_pos_ext + 1); + _val = _val.substr(0, _pos_ext); + } + + // Check if explicitly set via environment OR config file + // If explicitly set, don't add suffix; otherwise use provided suffix + bool _explicitly_set = + (_v->second->get_environ_updated() || _v->second->get_config_updated()); + + auto _cfg = settings::compose_filename_config{ + !_explicitly_set && !suffix.empty(), // use_suffix only if not explicitly set + suffix, // suffix value + true, // make_dir + _dir // explicit_path + }; + + _val = settings::compose_output_filename(_val, _ext, _cfg); + if(!_val.empty() && _val.at(0) != '/') + return settings::format(JOIN('/', "%env{PWD}%", _val), get_config()->get_tag()); + + return _val; +} + bool& get_use_rocpd() { @@ -2411,6 +2472,13 @@ get_use_rocpd() return static_cast&>(*_v).get(); } +bool& +get_caching_perfetto() +{ + static auto _v = get_config()->at("ROCPROFSYS_TRACE_CACHED"); + return static_cast&>(*_v).get(); +} + tmp_file::tmp_file(std::string _v) : filename{ std::move(_v) } {} diff --git a/projects/rocprofiler-systems/source/lib/core/config.hpp b/projects/rocprofiler-systems/source/lib/core/config.hpp index 2213160002..ddde939d13 100644 --- a/projects/rocprofiler-systems/source/lib/core/config.hpp +++ b/projects/rocprofiler-systems/source/lib/core/config.hpp @@ -364,9 +364,18 @@ get_tmpdir(); std::string get_database_absolute_path(std::string_view database_name, std::string_view tag); +std::string +get_perfetto_output_filename_with_suffix(std::string_view suffix = ""); + bool& get_use_rocpd() ROCPROFSYS_HOT; +bool& +get_caching_perfetto() ROCPROFSYS_HOT; + +bool +get_merge_perfetto_files(); + struct tmp_file { tmp_file(std::string); diff --git a/projects/rocprofiler-systems/source/lib/core/perfetto.cpp b/projects/rocprofiler-systems/source/lib/core/perfetto.cpp index 3ca5ecc020..800f0a2978 100644 --- a/projects/rocprofiler-systems/source/lib/core/perfetto.cpp +++ b/projects/rocprofiler-systems/source/lib/core/perfetto.cpp @@ -208,6 +208,8 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error) _data = char_vec_t{ tracing_session->ReadTraceBlocking() }; } + tracing_session.reset(); + return _data; }; diff --git a/projects/rocprofiler-systems/source/lib/core/trace_cache/CMakeLists.txt b/projects/rocprofiler-systems/source/lib/core/trace_cache/CMakeLists.txt index bcab341a06..57842f9a2f 100644 --- a/projects/rocprofiler-systems/source/lib/core/trace_cache/CMakeLists.txt +++ b/projects/rocprofiler-systems/source/lib/core/trace_cache/CMakeLists.txt @@ -25,6 +25,7 @@ set(trace_cache_sources ${CMAKE_CURRENT_LIST_DIR}/buffer_storage.cpp ${CMAKE_CURRENT_LIST_DIR}/metadata_registry.cpp ${CMAKE_CURRENT_LIST_DIR}/rocpd_processor.cpp + ${CMAKE_CURRENT_LIST_DIR}/perfetto_processor.cpp ) set(trace_cache_headers @@ -36,6 +37,7 @@ set(trace_cache_headers ${CMAKE_CURRENT_LIST_DIR}/cache_type_traits.hpp ${CMAKE_CURRENT_LIST_DIR}/metadata_registry.hpp ${CMAKE_CURRENT_LIST_DIR}/rocpd_processor.hpp + ${CMAKE_CURRENT_LIST_DIR}/perfetto_processor.hpp ${CMAKE_CURRENT_LIST_DIR}/sample_processor.hpp ${CMAKE_CURRENT_LIST_DIR}/sample_type.hpp ) diff --git a/projects/rocprofiler-systems/source/lib/core/trace_cache/cache_manager.cpp b/projects/rocprofiler-systems/source/lib/core/trace_cache/cache_manager.cpp index db0267ad6d..b9b5ec6701 100644 --- a/projects/rocprofiler-systems/source/lib/core/trace_cache/cache_manager.cpp +++ b/projects/rocprofiler-systems/source/lib/core/trace_cache/cache_manager.cpp @@ -23,6 +23,7 @@ #include "cache_manager.hpp" #include "core/trace_cache/metadata_registry.hpp" +#include "core/trace_cache/perfetto_processor.hpp" #include "core/trace_cache/rocpd_processor.hpp" #include "core/trace_cache/sample_processor.hpp" @@ -33,7 +34,9 @@ #include "library/runtime.hpp" #include +#include #include +#include #include namespace rocprofsys @@ -50,38 +53,128 @@ struct cache_files_t inline bool empty() const { return buff_storage.empty() || metadata.empty(); } }; +struct format_t +{ + bool process_parallel; + bool enabled; + const char* name; +}; + struct enabled_formats_t { - bool rocpd = get_use_rocpd(); + std::vector formats = { { true, get_use_rocpd(), "rocpd" }, + { false, get_caching_perfetto(), "perfetto" } }; void print() const { - constexpr std::pair formats[] = { - { "rocpd", &enabled_formats_t::rocpd }, - }; + if(std::none_of(formats.begin(), formats.end(), + [](const auto& f) { return f.enabled; })) + return; - bool any_enabled = false; - for(const auto& fmt : formats) - any_enabled |= this->*(fmt.second); - - if(!any_enabled) return; - - bool first = true; std::stringstream ss; + bool first = true; for(const auto& fmt : formats) { - if(this->*(fmt.second)) + if(fmt.enabled) { - if(!first && sizeof(formats) > 1) ss << ", "; - ss << fmt.first; + if(!first) ss << ", "; + ss << fmt.name; first = false; } } + ROCPROFSYS_PRINT( "Generating [%s] format(s) with collected data from trace cache. This may " "take a while..\n", ss.str().c_str()); + + if(has_parallel_formats()) + { + std::stringstream parallel_ss; + bool first_parallel = true; + for(const auto& fmt : formats) + { + if(fmt.enabled && fmt.process_parallel) + { + if(!first_parallel) parallel_ss << ", "; + parallel_ss << fmt.name; + first_parallel = false; + } + } + ROCPROFSYS_PRINT(" - Using parallel processing for: %s\n", + parallel_ss.str().c_str()); + } + + if(has_sequential_formats()) + { + std::stringstream sequential_ss; + bool first_sequential = true; + for(const auto& fmt : formats) + { + if(fmt.enabled && !fmt.process_parallel) + { + if(!first_sequential) sequential_ss << ", "; + sequential_ss << fmt.name; + first_sequential = false; + } + } + ROCPROFSYS_PRINT(" - Using sequential processing for: %s\n", + sequential_ss.str().c_str()); + } + } + + bool has_parallel_formats() const + { + return std::any_of(formats.begin(), formats.end(), + [](const auto& f) { return f.enabled && f.process_parallel; }); + } + + bool has_sequential_formats() const + { + return std::any_of(formats.begin(), formats.end(), [](const auto& f) { + return f.enabled && !f.process_parallel; + }); + } + + enabled_formats_t get_parallel_formats() const + { + enabled_formats_t parallel_formats; + parallel_formats.formats.clear(); + for(const auto& fmt : formats) + { + parallel_formats.formats.push_back( + { true, fmt.enabled && fmt.process_parallel, fmt.name }); + } + return parallel_formats; + } + + enabled_formats_t get_sequential_formats() const + { + enabled_formats_t sequential_formats; + sequential_formats.formats.clear(); + for(const auto& fmt : formats) + { + sequential_formats.formats.push_back( + { false, fmt.enabled && !fmt.process_parallel, fmt.name }); + } + return sequential_formats; + } + + bool is_rocpd_enabled() const + { + auto it = std::find_if(formats.begin(), formats.end(), [](const auto& f) { + return std::strcmp(f.name, "rocpd") == 0; + }); + return it != formats.end() && it->enabled; + } + + bool is_perfetto_enabled() const + { + auto it = std::find_if(formats.begin(), formats.end(), [](const auto& f) { + return std::strcmp(f.name, "perfetto") == 0; + }); + return it != formats.end() && it->enabled; } }; @@ -105,7 +198,8 @@ struct processor_config_t struct processor_storage_t { - std::shared_ptr rocpd_processor{ nullptr }; + std::shared_ptr rocpd_processor{ nullptr }; + std::shared_ptr perfetto_processor{ nullptr }; }; using directory_files_t = std::vector; @@ -234,6 +328,84 @@ clear_cache_files(const data::mapped_cache_files_t& _cache_files) } } +void +merge_perfetto_files(const std::vector& perfetto_files, + const std::string& _filename) +{ + if(perfetto_files.empty()) + { + ROCPROFSYS_VERBOSE( + 0, "perfetto trace data is empty. File '%s' will not be written...\n", + _filename.c_str()); + return; + } + + std::vector trace_data; + size_t total_size = 0; + + // Calculate total size for reservation + for(const auto& file : perfetto_files) + { + std::ifstream ifs(file, std::ios::binary | std::ios::ate); + if(ifs) + { + total_size += ifs.tellg(); + } + } + + trace_data.reserve(total_size); + + // Read and concatenate all files + for(const auto& file : perfetto_files) + { + std::ifstream ifs(file, std::ios::binary); + if(!ifs) + { + ROCPROFSYS_VERBOSE(-1, "Error opening '%s'...\n", file.c_str()); + continue; + } + + ifs.seekg(0, std::ios::end); + size_t file_size = ifs.tellg(); + ifs.seekg(0, std::ios::beg); + + size_t current_size = trace_data.size(); + trace_data.resize(current_size + file_size); + + ifs.read(trace_data.data() + current_size, file_size); + } + + if(!trace_data.empty()) + { + operation::file_output_message _fom{}; + // Write the trace into a file. + if(config::get_verbose() >= 0) + _fom(_filename, std::string{ "perfetto" }, + " (%.2f KB / %.2f MB / %.2f GB)... ", + static_cast(trace_data.size()) / units::KB, + static_cast(trace_data.size()) / units::MB, + static_cast(trace_data.size()) / units::GB); + std::ofstream ofs{}; + if(!filepath::open(ofs, _filename, std::ios::out | std::ios::binary)) + { + _fom.append("Error opening '%s'...", _filename.c_str()); + } + else + { + // Write the trace into a file. + ofs.write(trace_data.data(), trace_data.size()); + if(config::get_verbose() >= 0) _fom.append("%s", "Done"); // NOLINT + } + ofs.close(); + } + else + { + ROCPROFSYS_VERBOSE( + 0, "perfetto trace data is empty. File '%s' will not be written...\n", + _filename.c_str()); + } +} + } // namespace filesystem_utils namespace processing_utils @@ -244,13 +416,20 @@ configure_processors(const std::shared_ptr& _type_proc const data::enabled_formats_t& _enabled_formats) { data::processor_storage_t processor_storage; - if(_enabled_formats.rocpd) + if(_enabled_formats.is_rocpd_enabled()) { processor_storage.rocpd_processor = std::make_shared( _processor_config->_metadata_registry, _processor_config->_agent_manager, _processor_config->_pid, _processor_config->_ppid); _type_processing->add_handler(*processor_storage.rocpd_processor); } + if(_enabled_formats.is_perfetto_enabled()) + { + processor_storage.perfetto_processor = std::make_shared( + _processor_config->_metadata_registry, _processor_config->_agent_manager, + _processor_config->_pid, _processor_config->_ppid); + _type_processing->add_handler(*processor_storage.perfetto_processor); + } return processor_storage; } @@ -321,6 +500,37 @@ multithreaded_processing( } } +void +sequential_processing( + const std::vector>& _processor_configs, + const data::enabled_formats_t& _enabled_formats) +{ + for(const auto& processor_config : _processor_configs) + { + process_buffered_storage(processor_config, + utility::get_buffered_storage_filename( + processor_config->_ppid, processor_config->_pid), + _enabled_formats); + } +} + +void +dispatch_processing( + const std::vector>& _processor_configs, + const data::enabled_formats_t& _enabled_formats) +{ + if(_enabled_formats.has_sequential_formats()) + { + auto sequential_formats = _enabled_formats.get_sequential_formats(); + sequential_processing(_processor_configs, sequential_formats); + } + if(_enabled_formats.has_parallel_formats()) + { + auto parallel_formats = _enabled_formats.get_parallel_formats(); + multithreaded_processing(_processor_configs, parallel_formats); + } +} + } // namespace processing_utils cache_manager& @@ -361,7 +571,50 @@ cache_manager::post_process_bulk() getpid(), root_pid, m_metadata, std::make_shared(get_agent_manager_instance().get_agents()))); - processing_utils::multithreaded_processing(processor_configs, enabled_formats); + processing_utils::dispatch_processing(processor_configs, enabled_formats); + + if(enabled_formats.is_perfetto_enabled()) + { + std::vector perfetto_files; + + for(const auto& config : processor_configs) + { + // Check for both naming styles: default (current process) and PID-suffixed + auto filename_default = config::get_perfetto_output_filename(); + auto filename_suffix = config::get_perfetto_output_filename_with_suffix( + std::to_string(config->_pid)); + + if(static_cast(config->_pid) == getpid() && + tim::filepath::exists(filename_default)) + { + perfetto_files.push_back(filename_default); + } + else if(tim::filepath::exists(filename_suffix)) + { + perfetto_files.push_back(filename_suffix); + } + } + + if(config::get_perfetto_combined_traces() && perfetto_files.size() > 1) + { + // Use base filename without suffix for merged output + auto _filename = config::get_perfetto_output_filename(); + filesystem_utils::merge_perfetto_files(perfetto_files, _filename); + } + else if(perfetto_files.size() > 1) + { + ROCPROFSYS_VERBOSE( + 0, + "Generated %zu separate perfetto trace files. " + "Set ROCPROFSYS_PERFETTO_COMBINE_TRACES=ON to merge them.\n", + perfetto_files.size()); + + for(const auto& file : perfetto_files) + { + ROCPROFSYS_VERBOSE(1, " - %s\n", file.c_str()); + } + } + } filesystem_utils::clear_cache_files(cache_files); } diff --git a/projects/rocprofiler-systems/source/lib/core/trace_cache/perfetto_processor.cpp b/projects/rocprofiler-systems/source/lib/core/trace_cache/perfetto_processor.cpp new file mode 100644 index 0000000000..dec39b14bd --- /dev/null +++ b/projects/rocprofiler-systems/source/lib/core/trace_cache/perfetto_processor.cpp @@ -0,0 +1,1231 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "core/trace_cache/perfetto_processor.hpp" +#include "common.hpp" +#include "core/agent_manager.hpp" +#include "core/categories.hpp" +#include "core/common_types.hpp" +#include "core/gpu_metrics.hpp" +#include "core/utility.hpp" +#include "library/tracing.hpp" +#include "perfetto.hpp" +#include "trace_cache/metadata_registry.hpp" +#include "trace_cache/sample_type.hpp" +#include "trace_cache/storage_parser.hpp" + +#include +#include + +#include +#include +#include + +#if ROCPROFSYS_USE_ROCM > 0 +# include "library/rocprofiler-sdk/fwd.hpp" +# include +#endif + +namespace rocprofsys +{ +namespace trace_cache +{ +namespace +{ +struct annotation_entry +{ + const char* key; + std::variant value; +}; + +void +annotate_perfetto(::perfetto::EventContext& ctx, + const std::vector& annotations) +{ + for(const auto& ann : annotations) + { + std::visit( + [&](auto&& val) { tracing::add_perfetto_annotation(ctx, ann.key, val); }, + ann.value); + } +} // close annotate_perfetto + +template +::perfetto::Track +get_track(CategoryT, std::string name, uint64_t hash_arg) +{ + auto _uuid = tracing::get_perfetto_category_uuid(hash_arg); + auto& _track_uuids = tracing::get_perfetto_track_uuids(); + + if(_track_uuids.find(_uuid) == _track_uuids.end()) + { + const auto _track = ::perfetto::Track(_uuid, ::perfetto::ProcessTrack::Current()); + auto _desc = _track.Serialize(); + + _desc.set_name(name); + ::perfetto::TrackEvent::SetTrackDescriptor(_track, _desc); + + _track_uuids.emplace(_uuid, name); + } + return ::perfetto::Track(_uuid, ::perfetto::ProcessTrack::Current()); +} + +using amd_smi_gfx_track = perfetto_counter_track; +using amd_smi_umc_track = perfetto_counter_track; +using amd_smi_mm_track = perfetto_counter_track; +using amd_smi_temp_track = perfetto_counter_track; +using amd_smi_power_track = perfetto_counter_track; +using amd_smi_mem_track = perfetto_counter_track; +using amd_smi_vcn_track = perfetto_counter_track; +using amd_smi_jpeg_track = perfetto_counter_track; +using amd_smi_xgmi_link_width_track = + perfetto_counter_track; +using amd_smi_xgmi_link_speed_track = + perfetto_counter_track; +using amd_smi_xgmi_read_track = perfetto_counter_track; +using amd_smi_xgmi_write_track = + perfetto_counter_track; +using amd_smi_pcie_link_width_track = + perfetto_counter_track; +using amd_smi_pcie_link_speed_track = + perfetto_counter_track; +using amd_smi_pcie_bandwidth_acc_track = + perfetto_counter_track; +using amd_smi_pcie_bandwidth_inst_track = + perfetto_counter_track; + +void +setup_amd_smi_tracks(const uint32_t _device_id, bool is_busy_enabled, + bool is_temp_enabled, bool is_power_enabled, + bool is_mem_usage_enabled) +{ + if(amd_smi_gfx_track::exists(_device_id)) return; + + auto make_track_name = [&](const char* metric) { + return JOIN(" ", "GPU", JOIN("", '[', _device_id, ']'), metric, "(S)"); + }; + + if(is_busy_enabled) + { + amd_smi_gfx_track::emplace(_device_id, make_track_name("GFX Busy"), "%"); + amd_smi_umc_track::emplace(_device_id, make_track_name("UMC Busy"), "%"); + amd_smi_mm_track::emplace(_device_id, make_track_name("MM Busy"), "%"); + } + if(is_temp_enabled) + { + amd_smi_temp_track::emplace(_device_id, make_track_name("Temperature"), "deg C"); + } + if(is_power_enabled) + { + amd_smi_power_track::emplace(_device_id, make_track_name("Power"), "W"); + } + if(is_mem_usage_enabled) + { + amd_smi_mem_track::emplace(_device_id, make_track_name("Memory Usage"), "MB"); + } +} + +template +void +write_sampling_track_data(const struct backtrace_region_sample& _sample, + bool use_annotations) +{ + auto _track_name = _sample.track_name; + auto _thread_id = _sample.thread_id; + auto _main_name = _sample.name; + + auto _track = get_track(Category{}, _track_name, _thread_id); + + auto add_annotations = [&](::perfetto::EventContext& ctx) { + if(!use_annotations) return; + + std::vector annotations = { + { "begin_ns", _sample.start_timestamp }, { "end_ns", _sample.end_timestamp } + }; + + auto _call_stack = _sample.call_stack; + if(!_call_stack.empty()) + { + try + { + auto backtrace = nlohmann::json::parse(_call_stack); + for(const auto& [key, val] : backtrace.items()) + { + annotations.push_back( + { key.c_str(), val.template get() }); + } + } catch(const std::exception& e) + { + ROCPROFSYS_VERBOSE_F(2, "Failed to parse call_stack JSON: %s\n", + e.what()); + } + } + annotate_perfetto(ctx, annotations); + }; + + tracing::push_perfetto_track(Category{}, _main_name.c_str(), _track, + _sample.start_timestamp, add_annotations); + tracing::pop_perfetto_track(Category{}, _main_name.c_str(), _track, + _sample.end_timestamp); +} + +template +void +write_in_time_sample_data(CategoryT, const in_time_sample& _sample, bool use_annotations) +{ + const auto event_metadata = nlohmann::json::parse(_sample.event_metadata); + + const auto _track_name = _sample.track_name; + const auto _timestamp = _sample.timestamp_ns; + + const std::string _name = event_metadata.value("name", ""); + const std::string _event_type = event_metadata.value("event_type", ""); + const std::string _target = event_metadata.value("target", ""); + + const auto _track_uuid = std::hash{}(_track_name); + + auto _track = get_track(CategoryT{}, _track_name, _track_uuid); + auto add_perfetto_annotations = [&](::perfetto::EventContext ctx) { + if(!use_annotations) return; + + annotate_perfetto(ctx, { { "timestamp_ns", _timestamp }, + { "event_type", _event_type }, + { "target", _target } }); + }; + + TRACE_EVENT_INSTANT(trait::name::value, ::perfetto::DynamicString{ _name }, + _track, _timestamp, add_perfetto_annotations); +} + +// Dispatch to write_in_time_sample_data with the correct category type +// based on runtime category_enum_id, using category_type_id mapping from categories.hpp +template +bool +dispatch_in_time_sample(size_t category_enum_id, const in_time_sample& _sample, + bool use_annotations, std::index_sequence) +{ + return ((category_enum_id == Idx + ? (write_in_time_sample_data(category_type_id_t{}, _sample, + use_annotations), + true) + : false) || + ...); +} + +inline bool +dispatch_in_time_sample(size_t category_enum_id, const in_time_sample& _sample, + bool use_annotations) +{ + return dispatch_in_time_sample( + category_enum_id, _sample, use_annotations, + rocprofsys::utility::make_index_sequence_range<1, ROCPROFSYS_CATEGORY_LAST>{}); +} +} // namespace + +perfetto_processor_t::perfetto_processor_t( + const std::shared_ptr& metadata, + const std::shared_ptr& agent_mngr, int pid, int ppid) +: processor_t() +, m_metadata(*metadata) +, m_process_id(pid) +, m_parrent_pid(ppid) +, m_agent_manager(*agent_mngr) +, m_tmp_file(nullptr) +, m_tracing_session(nullptr) +, m_use_annotations(config::get_perfetto_annotations()) +{} + +void +perfetto_processor_t::initialize_perfetto() +{ + static std::once_flag init_flag; + std::call_once(init_flag, []() { + auto args = ::perfetto::TracingInitArgs{}; + args.backends = ::perfetto::kInProcessBackend; + args.shmem_size_hint_kb = config::get_perfetto_shmem_size_hint(); + + ::perfetto::Tracing::Initialize(args); + ::perfetto::TrackEvent::Register(); // Only register once globally! + }); +} + +void +perfetto_processor_t::setup_perfetto() +{ + auto track_event_cfg = ::perfetto::protos::gen::TrackEventConfig{}; + auto& cfg = m_session_config; + + auto perfetto_buffer_size = config::get_perfetto_buffer_size(); + auto flush_period = config::get_perfetto_flush_period(); + + auto _policy = + config::get_perfetto_fill_policy() == "discard" + ? ::perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_DISCARD + : ::perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_RING_BUFFER; + auto* buffer_config = cfg.add_buffers(); + buffer_config->set_size_kb(perfetto_buffer_size); + buffer_config->set_fill_policy(_policy); + + for(const auto& itr : config::get_disabled_categories()) + { + ROCPROFSYS_VERBOSE_F(1, "Disabling perfetto track event category: %s\n", + itr.c_str()); + track_event_cfg.add_disabled_categories(itr); + } + + cfg.set_flush_period_ms(flush_period); + + auto* ds_cfg = cfg.add_data_sources()->mutable_config(); + ds_cfg->set_name("track_event"); // this MUST be track_event + ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString()); +} + +void +perfetto_processor_t::start_session() +{ + if(config::get_perfetto_backend() != "inprocess") return; + + if(!m_tracing_session) + { + m_tracing_session = ::perfetto::Tracing::NewTrace(); + } + + ROCPROFSYS_VERBOSE(2, + "Starting perfetto post-processing session with cached data...\n"); + + int temp_fd = -1; + if(config::get_use_tmp_files()) + { + auto _base = JOIN("-", "cached-perfetto-trace", std::to_string(m_process_id)); + m_tmp_file = config::get_tmp_file(_base, "proto"); + m_tmp_file->open(O_RDWR | O_CREAT | O_TRUNC, 0600); + temp_fd = m_tmp_file->fd; + } + m_tracing_session->Setup(m_session_config, temp_fd); + m_tracing_session->StartBlocking(); +} + +void +perfetto_processor_t::stop_session() +{ + if(!m_tracing_session) return; + + ROCPROFSYS_VERBOSE(2, "Stopping perfetto post-processing session...\n"); + ::perfetto::TrackEvent::Flush(); + m_tracing_session->FlushBlocking(); + m_tracing_session->StopBlocking(); +} + +char_vec_t +perfetto_processor_t::get_session_data() +{ + auto _data = char_vec_t{}; + if(m_tmp_file && *m_tmp_file) + { + m_tmp_file->close(); + FILE* _fdata = ::fopen(m_tmp_file->filename.c_str(), "rb"); + + if(!_fdata) + { + ROCPROFSYS_VERBOSE(-1, + "Error! perfetto temp trace file '%s' could not be read", + m_tmp_file->filename.c_str()); + return char_vec_t{ m_tracing_session->ReadTraceBlocking() }; + } + + ::fseek(_fdata, 0, SEEK_END); + size_t _fnum_elem = ::ftell(_fdata); + ::fseek(_fdata, 0, SEEK_SET); + + _data.resize(_fnum_elem, '\0'); + auto _fnum_read = ::fread(_data.data(), sizeof(char), _fnum_elem, _fdata); + ::fclose(_fdata); + + ROCPROFSYS_CI_THROW( + _fnum_read != _fnum_elem, + "Error! read %zu elements from perfetto trace file '%s'. Expected %zu\n", + _fnum_read, m_tmp_file->filename.c_str(), _fnum_elem); + } + else + { + _data = char_vec_t{ m_tracing_session->ReadTraceBlocking() }; + } + + return _data; +} + +void +perfetto_processor_t::flush(bool& _perfetto_output_error) +{ + if(!m_tracing_session) return; + + stop_session(); + + auto trace_data = char_vec_t{}; + trace_data = get_session_data(); + + // If processing parrent process, use default filename (respects MPI rank/USE_PID + // settings) Otherwise, use PID-based suffix for child process traces + auto _filename = (m_process_id == m_parrent_pid) + ? config::get_perfetto_output_filename() + : config::get_perfetto_output_filename_with_suffix( + std::to_string(m_process_id)); + + if(!trace_data.empty()) + { + operation::file_output_message _fom{}; + // Write the trace into a file. + if(config::get_verbose() >= 0) + _fom(_filename, std::string{ "perfetto" }, + " (%.2f KB / %.2f MB / %.2f GB)... ", + static_cast(trace_data.size()) / units::KB, + static_cast(trace_data.size()) / units::MB, + static_cast(trace_data.size()) / units::GB); + std::ofstream ofs{}; + if(!filepath::open(ofs, _filename, std::ios::out | std::ios::binary)) + { + _fom.append("Error opening '%s'...", _filename.c_str()); + _perfetto_output_error = true; + } + else + { + // Write the trace into a file. + ofs.write(trace_data.data(), trace_data.size()); + if(config::get_verbose() >= 0) _fom.append("%s", "Done"); // NOLINT + } + ofs.close(); + } + else + { + ROCPROFSYS_VERBOSE( + 0, "perfetto trace data is empty. File '%s' will not be written...\n", + _filename.c_str()); + } + + if(m_tmp_file) + { + m_tmp_file->close(); + m_tmp_file->remove(); + m_tmp_file.reset(); + } + + m_tracing_session.reset(); +} + +void +perfetto_processor_t::prepare_for_processing() +{ + initialize_perfetto(); + setup_perfetto(); + start_session(); +} + +void +perfetto_processor_t::finalize_processing() +{ + bool _perfetto_output_error = false; + flush(_perfetto_output_error); + + if(_perfetto_output_error) + { + ROCPROFSYS_WARNING(0, "Perfetto trace generation failed for process: %lu\n", + m_process_id); + } +} + +void +perfetto_processor_t::handle([[maybe_unused]] const kernel_dispatch_sample& _kds) +{ +#if ROCPROFSYS_USE_ROCM > 0 + static auto _track_desc = [](uint64_t _device_id_v, uint64_t _queue_id_v) { + return JOIN("", "GPU Kernel Dispatch [", _device_id_v, "] Queue ", _queue_id_v); + }; + + auto kernel_symbol = m_metadata.get_kernel_symbol(_kds.kernel_id); + auto _agent_device_id = + m_agent_manager.get_agent_by_handle(_kds.agent_id_handle).device_id; + auto _queue_id_handle = _kds.queue_id_handle; + auto _stream_handle = _kds.stream_handle; + auto _corr_id = _kds.correlation_id_internal; + auto _beg_ts = _kds.start_timestamp; + auto _end_ts = _kds.end_timestamp; + + if(!kernel_symbol.has_value()) + { + throw std::runtime_error("Kernel symbol is missing for kernel dispatch"); + } + + auto kernel_name = tim::demangle(kernel_symbol->kernel_name); + + const auto _track = + tracing::get_perfetto_track(category::rocm_kernel_dispatch{}, _track_desc, + _agent_device_id, _queue_id_handle); + + auto add_annotations = [&](::perfetto::EventContext ctx) { + if(!m_use_annotations) return; + + annotate_perfetto( + ctx, + { { "begin_ns", _beg_ts }, + { "end_ns", _end_ts }, + { "corr_id", _corr_id }, + { "stream_id", _stream_handle }, + { "queue", _queue_id_handle }, + { "dispatch_id", _kds.dispatch_id }, + { "kernel_id", _kds.kernel_id }, + { "private_segment_size", _kds.private_segment_size }, + { "group_segment_size", _kds.group_segment_size }, + { "workgroup_size", JOIN("", "(", + JOIN(',', _kds.workgroup_size_x, + _kds.workgroup_size_y, _kds.workgroup_size_z), + ")") }, + { "grid_size", + JOIN("", "(", + JOIN(',', _kds.grid_size_x, _kds.grid_size_y, _kds.grid_size_z), + ")") } }); + }; + + tracing::push_perfetto(category::rocm_kernel_dispatch{}, kernel_name.c_str(), _track, + _beg_ts, ::perfetto::Flow::ProcessScoped(_corr_id), + add_annotations); + + tracing::pop_perfetto(category::rocm_kernel_dispatch{}, kernel_name.c_str(), _track, + _end_ts); +#endif +} + +void +perfetto_processor_t::handle([[maybe_unused]] const memory_copy_sample& _mcs) +{ +#if ROCPROFSYS_USE_ROCM > 0 + auto _corr_id = _mcs.correlation_id_internal; + auto _thrd_id = _mcs.thread_id; + auto _stream_id = _mcs.stream_handle; + auto _beg_ts = _mcs.start_timestamp; + auto _end_ts = _mcs.end_timestamp; + + auto _src_agent_log_node_id = + m_agent_manager.get_agent_by_handle(_mcs.src_agent_id_handle).logical_node_id; + auto _dst_agent_log_node_id = + m_agent_manager.get_agent_by_handle(_mcs.dst_agent_id_handle).logical_node_id; + auto _name = std::string{ m_metadata.get_buffer_name_info().at( + static_cast(_mcs.kind), + static_cast(_mcs.operation)) }; + + auto _track_desc = [](int32_t _device_id_v, rocprofiler_thread_id_t _tid) { + const auto& _tid_v = thread_info::get(_tid, SystemTID); + return JOIN("", "GPU Memory Copy to Agent [", _device_id_v, "] Thread ", + _tid_v->index_data->sequent_value); + }; + + const auto _track = tracing::get_perfetto_track( + category::rocm_memory_copy{}, _track_desc, _dst_agent_log_node_id, _thrd_id); + + auto add_perfetto_annotations = [&](::perfetto::EventContext ctx) { + if(!m_use_annotations) return; + + annotate_perfetto(ctx, { { "begin_ns", _beg_ts }, + { "end_ns", _end_ts }, + { "corr_id", _corr_id }, + { "stream_id", _stream_id }, + { "bytes", _mcs.bytes }, + { "src_agent_id", _src_agent_log_node_id }, + { "dst_agent_id", _dst_agent_log_node_id }, + { "operation", _name }, + { "src_address", _mcs.src_address_value }, + { "dst_address", _mcs.dst_address_value } }); + }; + + tracing::push_perfetto(category::rocm_memory_copy{}, _name.c_str(), _track, _beg_ts, + ::perfetto::Flow::ProcessScoped(_corr_id), + add_perfetto_annotations); + tracing::pop_perfetto(category::rocm_memory_copy{}, "", _track, _end_ts); +#endif +} + +void +perfetto_processor_t::handle([[maybe_unused]] const memory_allocate_sample& _mas) +{ +#if ROCPROFSYS_USE_ROCM > 0 && ROCPROFILER_VERSION >= 600 + auto memop_to_string = + [](rocprofiler_memory_allocation_operation_t op) -> const char* { + switch(op) + { + case ROCPROFILER_MEMORY_ALLOCATION_NONE: return "NONE"; + case ROCPROFILER_MEMORY_ALLOCATION_ALLOCATE: return "ALLOCATE"; + case ROCPROFILER_MEMORY_ALLOCATION_VMEM_ALLOCATE: return "VMEM_ALLOCATE"; + case ROCPROFILER_MEMORY_ALLOCATION_FREE: return "FREE"; + case ROCPROFILER_MEMORY_ALLOCATION_VMEM_FREE: return "VMEM_FREE"; + default: return "UNKNOWN"; + } + }; + + const auto _thrd_id = _mas.thread_id; + const auto _corr_id = _mas.correlation_id_internal; + const auto _stream_id = _mas.stream_handle; + const auto _beg_ts = _mas.start_timestamp; + const auto _end_ts = _mas.end_timestamp; + const auto _addr_val = _mas.address_value; + const auto _alloc_size = _mas.allocation_size; + + const auto invalid_context = ROCPROFILER_CONTEXT_NONE; + if(_mas.agent_id_handle != invalid_context.handle) + { + const auto* operation = memop_to_string( + static_cast(_mas.operation)); + + auto _track_desc = [](int32_t _device_id_v, rocprofiler_thread_id_t _tid) { + const auto& _tid_v = thread_info::get(_tid, SystemTID); + return JOIN("", "GPU Memory Allocation to Agent [", _device_id_v, "] Thread ", + _tid_v->index_data->sequent_value); + }; + + auto _agent_logical_node_id = + m_agent_manager.get_agent_by_handle(_mas.agent_id_handle).logical_node_id; + + const auto _track = + tracing::get_perfetto_track(category::rocm_memory_allocate{}, _track_desc, + _agent_logical_node_id, _thrd_id); + + auto add_perfetto_annotations = [&](::perfetto::EventContext ctx) { + if(!m_use_annotations) return; + + annotate_perfetto(ctx, { { "begin_ns", _beg_ts }, + { "end_ns", _end_ts }, + { "corr_id", _corr_id }, + { "stream_id", _stream_id }, + { "bytes", _alloc_size }, + { "agent_id", _agent_logical_node_id }, + { "address", _addr_val } }); + }; + + tracing::push_perfetto(category::rocm_memory_allocate{}, operation, _track, + _beg_ts, ::perfetto::Flow::ProcessScoped(_corr_id), + add_perfetto_annotations); + tracing::pop_perfetto(category::rocm_memory_allocate{}, "", _track, _end_ts); + } +#endif +} + +void +perfetto_processor_t::handle(const region_sample& _rs) +{ + const auto _corr_id = _rs.correlation_id_internal; + const auto _beg_ts = _rs.start_timestamp; + const auto _end_ts = _rs.end_timestamp; + const auto _category = _rs.category; + const auto _name = _rs.name; + + auto args = process_arguments_string(_rs.args_str); + + auto add_annotations = [&](::perfetto::EventContext ctx) { + if(!m_use_annotations) return; + + std::vector annotations = { { "begin_ns", _beg_ts }, + { "corr_id", _corr_id } }; + for(const auto& arg : args) + { + annotations.push_back({ arg.arg_name.c_str(), arg.arg_value }); + } + + if(!_rs.call_stack.empty()) + { + try + { + auto backtrace = nlohmann::json::parse(_rs.call_stack); + for(const auto& [key, val] : backtrace.items()) + { + annotations.push_back( + { key.c_str(), val.template get() }); + } + } catch(const std::exception& e) + { + ROCPROFSYS_VERBOSE_F(2, "Failed to parse call_stack JSON: %s\n", + e.what()); + } + } + + annotate_perfetto(ctx, annotations); + }; + + tracing::push_perfetto_ts(category::rocm{}, _name.c_str(), _beg_ts, + ::perfetto::Flow::ProcessScoped(_corr_id), add_annotations); + tracing::pop_perfetto_ts(category::rocm{}, _name.c_str(), _end_ts); +} + +void +perfetto_processor_t::handle(const cpu_freq_sample& _cpu_sample) +{ + using process_page_track = perfetto_counter_track; + using process_virt_track = perfetto_counter_track; + using process_peak_track = perfetto_counter_track; + using process_cntx_track = perfetto_counter_track; + using process_flts_track = perfetto_counter_track; + using process_user_track = perfetto_counter_track; + using process_kern_track = perfetto_counter_track; + using cpu_freq_track = perfetto_counter_track; + + struct core_freq_sample + { + size_t id; + float value; + }; + + auto deserialize_freqs = [](const std::vector& buffer) { + std::vector result; + size_t offset = 0; + + while(offset + sizeof(float) + sizeof(size_t) <= buffer.size()) + { + core_freq_sample core_sample; + std::memcpy(&core_sample.id, buffer.data() + offset, sizeof(size_t)); + offset += sizeof(size_t); + std::memcpy(&core_sample.value, buffer.data() + offset, sizeof(float)); + offset += sizeof(float); + result.push_back(core_sample); + } + return result; + }; + + static std::once_flag init_flag; + std::call_once(init_flag, []() { + process_page_track::emplace(0, "CPU Memory Usage (S)", "MB"); + process_virt_track::emplace(0, "CPU Virtual Memory (S)", "MB"); + process_peak_track::emplace(0, "CPU Peak Memory (S)", "MB"); + process_cntx_track::emplace(0, "CPU Context Switches (S)", ""); + process_flts_track::emplace(0, "CPU Page Faults (S)", ""); + process_user_track::emplace(0, "CPU User Time (S)", "sec"); + process_kern_track::emplace(0, "CPU Kernel Time (S)", "sec"); + }); + + auto _ts = _cpu_sample.timestamp; + + TRACE_COUNTER(trait::name::value, + process_page_track::at(0, 0), _ts, + static_cast(_cpu_sample.page_rss) / units::megabyte); + + TRACE_COUNTER(trait::name::value, + process_virt_track::at(0, 0), _ts, + static_cast(_cpu_sample.virt_mem_usage) / units::megabyte); + + TRACE_COUNTER(trait::name::value, + process_peak_track::at(0, 0), _ts, + static_cast(_cpu_sample.peak_rss) / units::megabyte); + + TRACE_COUNTER(trait::name::value, + process_cntx_track::at(0, 0), _ts, + static_cast(_cpu_sample.context_switch_count)); + + TRACE_COUNTER(trait::name::value, + process_flts_track::at(0, 0), _ts, + static_cast(_cpu_sample.page_faults)); + + TRACE_COUNTER(trait::name::value, + process_user_track::at(0, 0), _ts, + static_cast(_cpu_sample.user_mode_time) / units::sec); + + TRACE_COUNTER(trait::name::value, + process_kern_track::at(0, 0), _ts, + static_cast(_cpu_sample.kernel_mode_time) / units::sec); + + auto cpu_freqs = deserialize_freqs(_cpu_sample.freqs); + for(const auto& cpu_data : cpu_freqs) + { + size_t cpu_id = cpu_data.id; + if(!cpu_freq_track::exists(cpu_id)) + { + std::string track_name = "CPU Frequency [" + std::to_string(cpu_id) + "] (S)"; + cpu_freq_track::emplace(cpu_id, track_name, "MHz"); + } + + TRACE_COUNTER(trait::name::value, + cpu_freq_track::at(cpu_id, 0), _ts, + static_cast(cpu_data.value)); + } +} + +void +perfetto_processor_t::handle([[maybe_unused]] const backtrace_region_sample& _bts) +{ + (_bts.category == trait::name::value) + ? write_sampling_track_data(_bts, m_use_annotations) + : write_sampling_track_data(_bts, m_use_annotations); +} + +void +perfetto_processor_t::handle([[maybe_unused]] const pmc_event_with_sample& _pmc) +{ + using counter_collection_track = + perfetto_counter_track; + using thread_cpu_time_track = perfetto_counter_track; + using thread_peak_memory_track = perfetto_counter_track; + using thread_context_switch_track = + perfetto_counter_track; + using thread_page_fault_track = perfetto_counter_track; + using thread_hardware_counter_track = + perfetto_counter_track; + using comm_data_track = perfetto_counter_track; + + m_pmc_track_map = { + { ROCPROFSYS_CATEGORY_ROCM_COUNTER_COLLECTION, + { "Unit Count", [](auto id) { return counter_collection_track::exists(id); }, + [](auto id, auto& n, auto& u) { + counter_collection_track::emplace(id, n, u.c_str()); + }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + counter_collection_track::at(id, idx), ts, val); + } } }, + + { ROCPROFSYS_CATEGORY_THREAD_CPU_TIME, + { "sec", [](auto id) { return thread_cpu_time_track::exists(id); }, + [](auto id, auto& n, auto& u) { + thread_cpu_time_track::emplace(id, n, u.c_str()); + }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + thread_cpu_time_track::at(id, idx), ts, val); + } } }, + + { ROCPROFSYS_CATEGORY_THREAD_PEAK_MEMORY, + { "MB", [](auto id) { return thread_peak_memory_track::exists(id); }, + [](auto id, auto& n, auto& u) { + thread_peak_memory_track::emplace(id, n, u.c_str()); + }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + thread_peak_memory_track::at(id, idx), ts, val); + } } }, + + { ROCPROFSYS_CATEGORY_THREAD_CONTEXT_SWITCH, + { "", [](auto id) { return thread_context_switch_track::exists(id); }, + [](auto id, auto& n, auto& u) { + thread_context_switch_track::emplace(id, n, u.c_str()); + }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + thread_context_switch_track::at(id, idx), ts, val); + } } }, + + { ROCPROFSYS_CATEGORY_THREAD_PAGE_FAULT, + { "", [](auto id) { return thread_page_fault_track::exists(id); }, + [](auto id, auto& n, auto& u) { + thread_page_fault_track::emplace(id, n, u.c_str()); + }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + thread_page_fault_track::at(id, idx), ts, val); + } } }, + + { ROCPROFSYS_CATEGORY_THREAD_HARDWARE_COUNTER, + { "", [](auto id) { return thread_hardware_counter_track::exists(id); }, + [](auto id, auto& n, auto& u) { + thread_hardware_counter_track::emplace(id, n, u.c_str()); + }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + thread_hardware_counter_track::at(id, idx), ts, val); + } } }, + + { ROCPROFSYS_CATEGORY_COMM_DATA, + { "bytes", [](auto id) { return comm_data_track::exists(id); }, + [](auto id, auto& n, auto& u) { comm_data_track::emplace(id, n, u.c_str()); }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + comm_data_track::at(id, idx), ts, val); + } } }, + + { ROCPROFSYS_CATEGORY_MPI, + { "bytes", [](auto id) { return comm_data_track::exists(id); }, + [](auto id, auto& n, auto& u) { comm_data_track::emplace(id, n, u.c_str()); }, + [](auto id, auto idx, auto ts, auto val) { + TRACE_COUNTER(trait::name::value, + comm_data_track::at(id, idx), ts, val); + } } } + }; + + const auto _track_name = _pmc.track_name; + const auto _value = _pmc.value; + const auto _beg_ts = _pmc.timestamp_ns; + const auto _device_id = _pmc.device_id; + + auto track_key = std::hash{}(_track_name + std::to_string(_device_id)); + + auto track_it = m_pmc_track_map.find(_pmc.category_enum_id); + if(track_it != m_pmc_track_map.end()) + { + const auto& track_info = track_it->second; + + if(!track_info.exists_fn(track_key)) + { + track_info.emplace_fn(track_key, _track_name, track_info.default_units); + } + + track_info.trace_fn(track_key, 0, _beg_ts, _value); + } + else + { + ROCPROFSYS_VERBOSE_F(2, + "Unknown PMC event category_enum_id: %zu for track '%s'\n", + _pmc.category_enum_id, _track_name.c_str()); + } +} + +void +perfetto_processor_t::handle([[maybe_unused]] const amd_smi_sample& _amd_smi) +{ + // using amd_smi_gfx_track = perfetto_counter_track; + // using amd_smi_umc_track = perfetto_counter_track; + // using amd_smi_mm_track = perfetto_counter_track; + // using amd_smi_temp_track = perfetto_counter_track; + // using amd_smi_power_track = perfetto_counter_track; + // using amd_smi_mem_track = perfetto_counter_track; + // using amd_smi_vcn_track = perfetto_counter_track; + // using amd_smi_jpeg_track = + // perfetto_counter_track; using + // amd_smi_xgmi_link_width_track = + // perfetto_counter_track; + // using amd_smi_xgmi_link_speed_track = + // perfetto_counter_track; + // using amd_smi_xgmi_read_track = + // perfetto_counter_track; + // using amd_smi_xgmi_write_track = + // perfetto_counter_track; + // using amd_smi_pcie_link_width_track = + // perfetto_counter_track; + // using amd_smi_pcie_link_speed_track = + // perfetto_counter_track; + // using amd_smi_pcie_bandwidth_acc_track = + // perfetto_counter_track; + // using amd_smi_pcie_bandwidth_inst_track = + // perfetto_counter_track; + + // Use the shared gpu_metrics_t from core/gpu_metrics.hpp + using gpu_metrics_t = gpu::gpu_metrics_t; + + using pos = trace_cache::amd_smi_sample::settings_positions; + std::bitset<8> settings_bits(_amd_smi.settings); + bool is_busy_enabled = settings_bits.test(static_cast(pos::busy)); + bool is_temp_enabled = settings_bits.test(static_cast(pos::temp)); + bool is_power_enabled = settings_bits.test(static_cast(pos::power)); + bool is_mem_usage_enabled = settings_bits.test(static_cast(pos::mem_usage)); + bool is_vcn_enabled = settings_bits.test(static_cast(pos::vcn_activity)); + bool is_jpeg_enabled = settings_bits.test(static_cast(pos::jpeg_activity)); + bool is_xgmi_enabled = settings_bits.test(static_cast(pos::xgmi)); + bool is_pcie_enabled = settings_bits.test(static_cast(pos::pcie)); + + auto _ts = _amd_smi.timestamp; + auto _device_id = _amd_smi.device_id; + + // auto setup_tracks = [&]() { + // if(amd_smi_gfx_track::exists(_device_id)) return; + + // auto make_track_name = [&](const char* metric) { + // return JOIN(" ", "GPU", JOIN("", '[', _device_id, ']'), metric, "(S)"); + // }; + + // if(is_busy_enabled) + // { + // amd_smi_gfx_track::emplace(_device_id, make_track_name("GFX Busy"), "%"); + // amd_smi_umc_track::emplace(_device_id, make_track_name("UMC Busy"), "%"); + // amd_smi_mm_track::emplace(_device_id, make_track_name("MM Busy"), "%"); + // } + // if(is_temp_enabled) + // { + // amd_smi_temp_track::emplace(_device_id, make_track_name("Temperature"), + // "deg C"); + // } + // if(is_power_enabled) + // { + // amd_smi_power_track::emplace(_device_id, make_track_name("Power"), "W"); + // } + // if(is_mem_usage_enabled) + // { + // amd_smi_mem_track::emplace(_device_id, make_track_name("Memory Usage"), + // "MB"); + // } + // }; + + // setup_tracks(); + setup_amd_smi_tracks(_device_id, is_busy_enabled, is_temp_enabled, is_power_enabled, + is_mem_usage_enabled); + + if(is_busy_enabled) + { + TRACE_COUNTER("device_busy_gfx", amd_smi_gfx_track::at(_device_id, 0), _ts, + _amd_smi.gfx_activity); + TRACE_COUNTER("device_busy_umc", amd_smi_umc_track::at(_device_id, 0), _ts, + _amd_smi.umc_activity); + TRACE_COUNTER("device_busy_mm", amd_smi_mm_track::at(_device_id, 0), _ts, + _amd_smi.mm_activity); + } + if(is_temp_enabled) + { + TRACE_COUNTER("device_temp", amd_smi_temp_track::at(_device_id, 0), _ts, + _amd_smi.temperature); + } + if(is_power_enabled) + { + TRACE_COUNTER("device_power", amd_smi_power_track::at(_device_id, 0), _ts, + _amd_smi.power); + } + if(is_mem_usage_enabled) + { + double mem_mb = _amd_smi.mem_usage / static_cast(units::megabyte); + TRACE_COUNTER("device_memory_usage", amd_smi_mem_track::at(_device_id, 0), _ts, + mem_mb); + } + + if(!is_vcn_enabled && !is_jpeg_enabled && !is_xgmi_enabled && !is_pcie_enabled) + return; + + gpu_metrics_t gpu_metrics; + gpu::gpu_metrics_capabilities_t capabilities; + gpu::deserialize_gpu_metrics(_amd_smi.gpu_activity, gpu_metrics, is_vcn_enabled, + is_jpeg_enabled, is_xgmi_enabled, is_pcie_enabled, + capabilities); + + // Helper lambda to insert VCN/JPEG activity metrics + auto insert_decode_vector_metrics = [&](auto category, bool _is_enabled, + const std::vector& data, + std::optional _idx = std::nullopt) { + if(!_is_enabled) return; + + using Category = std::decay_t; + + for(size_t i = 0; i < data.size(); ++i) + { + const auto value = data[i]; + if(value == std::numeric_limits::max()) continue; + + std::string track_name; + if(_idx.has_value()) + { + // Per-XCP format + track_name = JOIN( + " ", "GPU", JOIN("", '[', _device_id, ']'), + trait::name::value, + JOIN("", "XCP_", _idx.value(), ": [", (i < 10 ? "0" : ""), i, ']'), + "(S)"); + } + else + { + // Device-level format + track_name = JOIN(" ", "GPU", JOIN("", '[', _device_id, ']'), + trait::name::value, + JOIN("", "[", (i < 10 ? "0" : ""), i, ']'), "(S)"); + } + + auto generate_track_key = [](uint32_t _dev_idx, size_t _xcp_idx, + size_t _clk_idx) { + return (static_cast(_dev_idx) << 16) | + (static_cast(_xcp_idx) << 8) | + static_cast(_clk_idx); + }; + + auto unique_key = generate_track_key(_device_id, _idx.value_or(0), i); + + if constexpr(std::is_same_v) + { + if(!amd_smi_vcn_track::exists(unique_key)) + { + amd_smi_vcn_track::emplace(unique_key, track_name, "%"); + } + TRACE_COUNTER("device_vcn_activity", amd_smi_vcn_track::at(unique_key, 0), + _ts, static_cast(value)); + } + else if constexpr(std::is_same_v) + { + if(!amd_smi_jpeg_track::exists(unique_key)) + { + amd_smi_jpeg_track::emplace(unique_key, track_name, "%"); + } + TRACE_COUNTER("device_jpeg_activity", + amd_smi_jpeg_track::at(unique_key, 0), _ts, + static_cast(value)); + } + } + }; + + auto insert_xgmi_vector_metrics = [&](auto category, bool _is_enabled, + const std::vector& data) { + if(!_is_enabled) return; + + using Category = std::decay_t; + + for(size_t i = 0; i < data.size(); ++i) + { + const auto value = data[i]; + if(value == std::numeric_limits::max()) continue; + + std::string track_name = + JOIN(" ", "GPU", JOIN("", '[', _device_id, ']'), + trait::name::value, JOIN("", "[", i, ']'), "(S)"); + + auto unique_key = (_device_id << 8) | i; + + if constexpr(std::is_same_v) + { + if(!amd_smi_xgmi_read_track::exists(unique_key)) + { + amd_smi_xgmi_read_track::emplace(unique_key, track_name, "bytes"); + } + TRACE_COUNTER("device_xgmi_read_data", + amd_smi_xgmi_read_track::at(unique_key, 0), _ts, + static_cast(value)); + } + else if constexpr(std::is_same_v) + { + if(!amd_smi_xgmi_write_track::exists(unique_key)) + { + amd_smi_xgmi_write_track::emplace(unique_key, track_name, "bytes"); + } + TRACE_COUNTER("device_xgmi_write_data", + amd_smi_xgmi_write_track::at(unique_key, 0), _ts, + static_cast(value)); + } + } + }; + + // Insert VCN activity metrics + if(capabilities.flags.vcn_is_device_level_only) + { + insert_decode_vector_metrics(category::amd_smi_vcn_activity{}, is_vcn_enabled, + gpu_metrics.vcn_activity, std::nullopt); + } + else + { + for(size_t xcp = 0; xcp < gpu_metrics.vcn_busy.size(); ++xcp) + { + insert_decode_vector_metrics(category::amd_smi_vcn_activity{}, is_vcn_enabled, + gpu_metrics.vcn_busy[xcp], xcp); + } + } + + // Insert JPEG activity metrics + if(capabilities.flags.jpeg_is_device_level_only) + { + insert_decode_vector_metrics(category::amd_smi_jpeg_activity{}, is_jpeg_enabled, + gpu_metrics.jpeg_activity, std::nullopt); + } + else + { + for(size_t xcp = 0; xcp < gpu_metrics.jpeg_busy.size(); ++xcp) + { + insert_decode_vector_metrics(category::amd_smi_jpeg_activity{}, + is_jpeg_enabled, gpu_metrics.jpeg_busy[xcp], + xcp); + } + } + + // Insert XGMI metrics + if(is_xgmi_enabled) + { + auto make_track_name = [&](const char* metric) { + return JOIN(" ", "GPU", JOIN("", '[', _device_id, ']'), metric, "(S)"); + }; + + if(!amd_smi_xgmi_link_width_track::exists(_device_id)) + { + amd_smi_xgmi_link_width_track::emplace( + _device_id, make_track_name("XGMI Link Width"), ""); + } + TRACE_COUNTER("device_xgmi_link_width", + amd_smi_xgmi_link_width_track::at(_device_id, 0), _ts, + static_cast(gpu_metrics.xgmi_link_width)); + + if(!amd_smi_xgmi_link_speed_track::exists(_device_id)) + { + amd_smi_xgmi_link_speed_track::emplace( + _device_id, make_track_name("XGMI Link Speed"), "MT/s"); + } + TRACE_COUNTER("device_xgmi_link_speed", + amd_smi_xgmi_link_speed_track::at(_device_id, 0), _ts, + static_cast(gpu_metrics.xgmi_link_speed)); + + insert_xgmi_vector_metrics(category::amd_smi_xgmi_read_data{}, is_xgmi_enabled, + gpu_metrics.xgmi_read_data_acc); + + insert_xgmi_vector_metrics(category::amd_smi_xgmi_write_data{}, is_xgmi_enabled, + gpu_metrics.xgmi_write_data_acc); + } + + // Insert PCIe metrics + if(is_pcie_enabled) + { + auto make_track_name = [&](const char* metric) { + return JOIN(" ", "GPU", JOIN("", '[', _device_id, ']'), metric, "(S)"); + }; + + if(!amd_smi_pcie_link_width_track::exists(_device_id)) + { + amd_smi_pcie_link_width_track::emplace( + _device_id, make_track_name("PCIe Link Width"), ""); + } + TRACE_COUNTER("device_pcie_link_width", + amd_smi_pcie_link_width_track::at(_device_id, 0), _ts, + static_cast(gpu_metrics.pcie_link_width)); + + if(!amd_smi_pcie_link_speed_track::exists(_device_id)) + { + amd_smi_pcie_link_speed_track::emplace( + _device_id, make_track_name("PCIe Link Speed"), "MT/s"); + } + TRACE_COUNTER("device_pcie_link_speed", + amd_smi_pcie_link_speed_track::at(_device_id, 0), _ts, + static_cast(gpu_metrics.pcie_link_speed)); + + if(!amd_smi_pcie_bandwidth_acc_track::exists(_device_id)) + { + amd_smi_pcie_bandwidth_acc_track::emplace( + _device_id, make_track_name("PCIe Bandwidth Acc"), "bytes"); + } + TRACE_COUNTER("device_pcie_bandwidth_acc", + amd_smi_pcie_bandwidth_acc_track::at(_device_id, 0), _ts, + static_cast(gpu_metrics.pcie_bandwidth_acc)); + + if(!amd_smi_pcie_bandwidth_inst_track::exists(_device_id)) + { + amd_smi_pcie_bandwidth_inst_track::emplace( + _device_id, make_track_name("PCIe Bandwidth Inst"), "bytes"); + } + TRACE_COUNTER("device_pcie_bandwidth_inst", + amd_smi_pcie_bandwidth_inst_track::at(_device_id, 0), _ts, + static_cast(gpu_metrics.pcie_bandwidth_inst)); + } +} + +void +perfetto_processor_t::handle([[maybe_unused]] const in_time_sample& _sample) +{ + // Dispatch based on category_enum_id using the category type mapping + if(!dispatch_in_time_sample(_sample.category_enum_id, _sample, m_use_annotations)) + { + ROCPROFSYS_VERBOSE_F( + 2, "Unknown in_time_sample category_enum_id: %zu, using user category\n", + _sample.category_enum_id); + write_in_time_sample_data(category::user{}, _sample, m_use_annotations); + } +} + +} // namespace trace_cache +} // namespace rocprofsys diff --git a/projects/rocprofiler-systems/source/lib/core/trace_cache/perfetto_processor.hpp b/projects/rocprofiler-systems/source/lib/core/trace_cache/perfetto_processor.hpp new file mode 100644 index 0000000000..5ddb2410b6 --- /dev/null +++ b/projects/rocprofiler-systems/source/lib/core/trace_cache/perfetto_processor.hpp @@ -0,0 +1,88 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once +#include "agent_manager.hpp" +#include "config.hpp" +#include "core/perfetto_fwd.hpp" +#include "core/trace_cache/metadata_registry.hpp" +#include "core/trace_cache/sample_processor.hpp" + +#include +#include +#include + +namespace rocprofsys +{ +namespace trace_cache +{ +using char_vec_t = std::vector; + +struct pmc_track_info +{ + const char* default_units; + std::function exists_fn; + std::function emplace_fn; + std::function trace_fn; +}; + +class perfetto_processor_t : public processor_t +{ +public: + perfetto_processor_t(const std::shared_ptr& metadata, + const std::shared_ptr& agent_mngr, int pid, + int ppid); + + void prepare_for_processing(); + void finalize_processing(); + + void handle(const kernel_dispatch_sample& sample); + void handle(const memory_copy_sample& sample); + void handle(const memory_allocate_sample& sample); + void handle(const region_sample& sample); + void handle(const in_time_sample& sample); + void handle(const pmc_event_with_sample& sample); + void handle(const amd_smi_sample& sample); + void handle(const cpu_freq_sample& sample); + void handle(const backtrace_region_sample& sample); + +private: + void initialize_perfetto(); + void setup_perfetto(); + void start_session(); + void stop_session(); + void flush(bool& perfetto_output_error); + char_vec_t get_session_data(); + + metadata_registry& m_metadata; + uint64_t m_process_id; + uint64_t m_parrent_pid; + agent_manager& m_agent_manager; + ::perfetto::TraceConfig m_session_config; + std::shared_ptr m_tmp_file{ nullptr }; + std::unique_ptr<::perfetto::TracingSession> m_tracing_session{ nullptr }; + bool m_use_annotations{ false }; + + std::unordered_map m_pmc_track_map; +}; +} // namespace trace_cache +} // namespace rocprofsys diff --git a/projects/rocprofiler-systems/source/lib/core/trace_cache/rocpd_processor.cpp b/projects/rocprofiler-systems/source/lib/core/trace_cache/rocpd_processor.cpp index 31d5b6118c..d7c0cc4cf1 100644 --- a/projects/rocprofiler-systems/source/lib/core/trace_cache/rocpd_processor.cpp +++ b/projects/rocprofiler-systems/source/lib/core/trace_cache/rocpd_processor.cpp @@ -22,6 +22,7 @@ #include "core/trace_cache/rocpd_processor.hpp" #include "core/agent_manager.hpp" +#include "core/common_types.hpp" #include "core/config.hpp" #include "core/debug.hpp" #include "core/gpu_metrics.hpp" @@ -234,48 +235,6 @@ void rocpd_processor_t::handle([[maybe_unused]] const region_sample& _rs) { #if ROCPROFSYS_USE_ROCM > 0 - static auto parse_args = []([[maybe_unused]] const std::string& arg_str) { - rocprofiler_sdk::function_args_t args; - const std::string delimiter = ";;"; - - auto split = [](const std::string& str, const std::string& _delimiter) { - std::vector tokens; - size_t start = 0; - size_t end = str.find(_delimiter); - - while(end != std::string::npos) - { - tokens.push_back(str.substr(start, end - start)); - start = end + _delimiter.length(); - end = str.find(_delimiter, start); - } - - return tokens; - }; - - if(arg_str.empty()) - { - return args; - } - - auto tokens = split(arg_str, delimiter); - - // Ensure the number of tokens is a multiple of 4 - if(tokens.size() % 4 != 0) - { - throw std::invalid_argument("Malformed argument string."); - } - - for(auto it = tokens.begin(); it != tokens.end(); it += 4) - { - rocprofiler_sdk::argument_info arg = { static_cast(std::stoi(*it)), - *(it + 1), *(it + 2), *(it + 3) }; - args.push_back(arg); - } - - return args; - }; - auto& n_info = node_info::get_instance(); auto process = m_metadata->get_process_info(); auto thread_primary_key = @@ -292,7 +251,7 @@ rocpd_processor_t::handle([[maybe_unused]] const region_sample& _rs) m_data_processor->insert_event(category_primary_key, stack_id, parent_stack_id, correlation_id, _rs.call_stack.c_str()); - auto args = parse_args(_rs.args_str); + auto args = process_arguments_string(_rs.args_str); for(const auto& arg : args) { m_data_processor->insert_args(event_primary_key, arg.arg_number, diff --git a/projects/rocprofiler-systems/source/lib/core/trace_cache/sample_type.hpp b/projects/rocprofiler-systems/source/lib/core/trace_cache/sample_type.hpp index 0b0b3effb0..6e1114cbdb 100644 --- a/projects/rocprofiler-systems/source/lib/core/trace_cache/sample_type.hpp +++ b/projects/rocprofiler-systems/source/lib/core/trace_cache/sample_type.hpp @@ -114,7 +114,7 @@ serialize(uint8_t* buffer, const kernel_dispatch_sample& item) item.correlation_id_internal, item.correlation_id_ancestor, item.private_segment_size, item.group_segment_size, item.workgroup_size_x, item.workgroup_size_y, item.workgroup_size_z, item.grid_size_x, item.grid_size_y, - item.grid_size_z, (uint64_t) item.stream_handle); + item.grid_size_z, static_cast(item.stream_handle)); } template <> @@ -144,7 +144,7 @@ get_size(const kernel_dispatch_sample& item) item.correlation_id_internal, item.correlation_id_ancestor, item.private_segment_size, item.group_segment_size, item.workgroup_size_x, item.workgroup_size_y, item.workgroup_size_z, item.grid_size_x, item.grid_size_y, - item.grid_size_z, (uint64_t) item.stream_handle); + item.grid_size_z, static_cast(item.stream_handle)); } struct memory_copy_sample : cacheable_t @@ -196,7 +196,8 @@ serialize(uint8_t* buffer, const memory_copy_sample& item) item.dst_agent_id_handle, item.src_agent_id_handle, item.kind, item.operation, item.bytes, item.correlation_id_internal, item.correlation_id_ancestor, item.dst_address_value, - item.src_address_value, (uint64_t) item.stream_handle); + item.src_address_value, + static_cast(item.stream_handle)); } template <> @@ -218,11 +219,12 @@ template <> inline size_t get_size(const memory_copy_sample& item) { - return utility::get_size( - item.start_timestamp, item.end_timestamp, item.thread_id, - item.dst_agent_id_handle, item.src_agent_id_handle, item.kind, item.operation, - item.bytes, item.correlation_id_internal, item.correlation_id_ancestor, - item.dst_address_value, item.src_address_value, (uint64_t) item.stream_handle); + return utility::get_size(item.start_timestamp, item.end_timestamp, item.thread_id, + item.dst_agent_id_handle, item.src_agent_id_handle, + item.kind, item.operation, item.bytes, + item.correlation_id_internal, item.correlation_id_ancestor, + item.dst_address_value, item.src_address_value, + static_cast(item.stream_handle)); } struct memory_allocate_sample : cacheable_t @@ -270,7 +272,7 @@ serialize(uint8_t* buffer, const memory_allocate_sample& item) item.agent_id_handle, item.kind, item.operation, item.allocation_size, item.correlation_id_internal, item.correlation_id_ancestor, item.address_value, - (uint64_t) item.stream_handle); + static_cast(item.stream_handle)); } template <> @@ -291,10 +293,11 @@ template <> inline size_t get_size(const memory_allocate_sample& item) { - return utility::get_size( - item.start_timestamp, item.end_timestamp, item.thread_id, item.agent_id_handle, - item.kind, item.operation, item.allocation_size, item.correlation_id_internal, - item.correlation_id_ancestor, item.address_value, (uint64_t) item.stream_handle); + return utility::get_size(item.start_timestamp, item.end_timestamp, item.thread_id, + item.agent_id_handle, item.kind, item.operation, + item.allocation_size, item.correlation_id_internal, + item.correlation_id_ancestor, item.address_value, + static_cast(item.stream_handle)); } struct region_sample : cacheable_t @@ -373,11 +376,12 @@ struct in_time_sample : cacheable_t type_identifier_t::in_time_sample; in_time_sample() = default; - in_time_sample(std::string _track_name, size_t _timestamp_ns, - std::string _event_metadata, size_t _stack_id, size_t _parent_stack_id, - size_t _correlation_id, std::string _call_stack, - std::string _line_info) - : track_name(std::move(_track_name)) + in_time_sample(size_t _category_enum_id, std::string _track_name, + size_t _timestamp_ns, std::string _event_metadata, size_t _stack_id, + size_t _parent_stack_id, size_t _correlation_id, + std::string _call_stack, std::string _line_info) + : category_enum_id(_category_enum_id) + , track_name(std::move(_track_name)) , timestamp_ns(_timestamp_ns) , event_metadata(std::move(_event_metadata)) , stack_id(_stack_id) @@ -387,6 +391,7 @@ struct in_time_sample : cacheable_t , line_info(std::move(_line_info)) {} + size_t category_enum_id; std::string track_name; size_t timestamp_ns; std::string event_metadata; @@ -402,10 +407,11 @@ inline void serialize(uint8_t* buffer, const in_time_sample& item) { utility::store_value( - buffer, std::string_view(item.track_name), (uint64_t) item.timestamp_ns, - std::string_view(item.event_metadata), (uint64_t) item.stack_id, - (uint64_t) item.parent_stack_id, (uint64_t) item.correlation_id, - std::string_view(item.call_stack), std::string_view(item.line_info)); + buffer, std::string_view(item.track_name), + static_cast(item.timestamp_ns), std::string_view(item.event_metadata), + static_cast(item.stack_id), static_cast(item.parent_stack_id), + static_cast(item.correlation_id), std::string_view(item.call_stack), + std::string_view(item.line_info)); } template <> @@ -413,20 +419,22 @@ inline in_time_sample deserialize(uint8_t*& buffer) { in_time_sample item; + size_t category_enum_id; std::string_view track_name_view, event_metadata_view, call_stack_view, line_info_view; uint64_t timestamp_ns, stack_id, parent_stack_id, correlation_id; - utility::parse_value(buffer, track_name_view, timestamp_ns, event_metadata_view, - stack_id, parent_stack_id, correlation_id, call_stack_view, - line_info_view); - item.track_name = std::string(track_name_view); - item.timestamp_ns = timestamp_ns; - item.event_metadata = std::string(event_metadata_view); - item.stack_id = stack_id; - item.parent_stack_id = parent_stack_id; - item.correlation_id = correlation_id; - item.call_stack = std::string(call_stack_view); - item.line_info = std::string(line_info_view); + utility::parse_value(buffer, category_enum_id, track_name_view, timestamp_ns, + event_metadata_view, stack_id, parent_stack_id, correlation_id, + call_stack_view, line_info_view); + item.category_enum_id = category_enum_id; + item.track_name = std::string(track_name_view); + item.timestamp_ns = timestamp_ns; + item.event_metadata = std::string(event_metadata_view); + item.stack_id = stack_id; + item.parent_stack_id = parent_stack_id; + item.correlation_id = correlation_id; + item.call_stack = std::string(call_stack_view); + item.line_info = std::string(line_info_view); return item; } @@ -435,10 +443,11 @@ inline size_t get_size(const in_time_sample& item) { return utility::get_size( - std::string_view(item.track_name), (uint64_t) item.timestamp_ns, - std::string_view(item.event_metadata), (uint64_t) item.stack_id, - (uint64_t) item.parent_stack_id, (uint64_t) item.correlation_id, - std::string_view(item.call_stack), std::string_view(item.line_info)); + item.category_enum_id, std::string_view(item.track_name), + static_cast(item.timestamp_ns), std::string_view(item.event_metadata), + static_cast(item.stack_id), static_cast(item.parent_stack_id), + static_cast(item.correlation_id), std::string_view(item.call_stack), + std::string_view(item.line_info)); } struct pmc_event_with_sample : in_time_sample @@ -447,15 +456,15 @@ struct pmc_event_with_sample : in_time_sample type_identifier_t::pmc_event_with_sample; pmc_event_with_sample() = default; - pmc_event_with_sample(std::string _track_name, size_t _timestamp_ns, - std::string _event_metadata, size_t _stack_id, - size_t _parent_stack_id, size_t _correlation_id, - std::string _call_stack, std::string _line_info, - uint32_t _device_id, uint8_t _device_type, - std::string _pmc_info_name, double _value) - : in_time_sample(std::move(_track_name), _timestamp_ns, std::move(_event_metadata), - _stack_id, _parent_stack_id, _correlation_id, std::move(_call_stack), - std::move(_line_info)) + pmc_event_with_sample(size_t _category_enum_id, std::string _track_name, + size_t _timestamp_ns, std::string _event_metadata, + size_t _stack_id, size_t _parent_stack_id, + size_t _correlation_id, std::string _call_stack, + std::string _line_info, uint32_t _device_id, + uint8_t _device_type, std::string _pmc_info_name, double _value) + : in_time_sample(_category_enum_id, std::move(_track_name), _timestamp_ns, + std::move(_event_metadata), _stack_id, _parent_stack_id, + _correlation_id, std::move(_call_stack), std::move(_line_info)) , device_id(_device_id) , device_type(_device_type) , pmc_info_name(std::move(_pmc_info_name)) @@ -473,12 +482,12 @@ inline void serialize(uint8_t* buffer, const pmc_event_with_sample& item) { utility::store_value( - buffer, std::string_view(item.track_name), (uint64_t) item.timestamp_ns, - std::string_view(item.event_metadata), (uint64_t) item.stack_id, - (uint64_t) item.parent_stack_id, (uint64_t) item.correlation_id, - std::string_view(item.call_stack), std::string_view(item.line_info), - item.device_id, item.device_type, std::string_view(item.pmc_info_name), - item.value); + buffer, item.category_enum_id, std::string_view(item.track_name), + static_cast(item.timestamp_ns), std::string_view(item.event_metadata), + static_cast(item.stack_id), static_cast(item.parent_stack_id), + static_cast(item.correlation_id), std::string_view(item.call_stack), + std::string_view(item.line_info), item.device_id, item.device_type, + std::string_view(item.pmc_info_name), item.value); } template <> @@ -486,22 +495,24 @@ inline pmc_event_with_sample deserialize(uint8_t*& buffer) { pmc_event_with_sample item; + size_t category_enum_id; std::string_view track_name_view, event_metadata_view, call_stack_view, line_info_view, pmc_info_name_view; uint64_t timestamp_ns, stack_id, parent_stack_id, correlation_id; - utility::parse_value(buffer, track_name_view, timestamp_ns, event_metadata_view, - stack_id, parent_stack_id, correlation_id, call_stack_view, - line_info_view, item.device_id, item.device_type, - pmc_info_name_view, item.value); - item.track_name = std::string(track_name_view); - item.timestamp_ns = timestamp_ns; - item.event_metadata = std::string(event_metadata_view); - item.stack_id = stack_id; - item.parent_stack_id = parent_stack_id; - item.correlation_id = correlation_id; - item.call_stack = std::string(call_stack_view); - item.line_info = std::string(line_info_view); - item.pmc_info_name = std::string(pmc_info_name_view); + utility::parse_value(buffer, category_enum_id, track_name_view, timestamp_ns, + event_metadata_view, stack_id, parent_stack_id, correlation_id, + call_stack_view, line_info_view, item.device_id, + item.device_type, pmc_info_name_view, item.value); + item.category_enum_id = category_enum_id; + item.track_name = std::string(track_name_view); + item.timestamp_ns = timestamp_ns; + item.event_metadata = std::string(event_metadata_view); + item.stack_id = stack_id; + item.parent_stack_id = parent_stack_id; + item.correlation_id = correlation_id; + item.call_stack = std::string(call_stack_view); + item.line_info = std::string(line_info_view); + item.pmc_info_name = std::string(pmc_info_name_view); return item; } @@ -510,12 +521,12 @@ inline size_t get_size(const pmc_event_with_sample& item) { return utility::get_size( - std::string_view(item.track_name), (uint64_t) item.timestamp_ns, - std::string_view(item.event_metadata), (uint64_t) item.stack_id, - (uint64_t) item.parent_stack_id, (uint64_t) item.correlation_id, - std::string_view(item.call_stack), std::string_view(item.line_info), - item.device_id, item.device_type, std::string_view(item.pmc_info_name), - item.value); + item.category_enum_id, std::string_view(item.track_name), + static_cast(item.timestamp_ns), std::string_view(item.event_metadata), + static_cast(item.stack_id), static_cast(item.parent_stack_id), + static_cast(item.correlation_id), std::string_view(item.call_stack), + std::string_view(item.line_info), item.device_id, item.device_type, + std::string_view(item.pmc_info_name), item.value); } struct amd_smi_sample : cacheable_t @@ -568,10 +579,10 @@ template <> inline void serialize(uint8_t* buffer, const amd_smi_sample& item) { - utility::store_value(buffer, item.settings, item.device_id, (uint64_t) item.timestamp, - item.gfx_activity, item.umc_activity, item.mm_activity, - item.power, item.temperature, (uint64_t) item.mem_usage, - item.gpu_activity); + utility::store_value( + buffer, item.settings, item.device_id, static_cast(item.timestamp), + item.gfx_activity, item.umc_activity, item.mm_activity, item.power, + item.temperature, static_cast(item.mem_usage), item.gpu_activity); } template <> @@ -592,10 +603,10 @@ template <> inline size_t get_size(const amd_smi_sample& item) { - return utility::get_size(item.settings, item.device_id, (uint64_t) item.timestamp, - item.gfx_activity, item.umc_activity, item.mm_activity, - item.power, item.temperature, (uint64_t) item.mem_usage, - item.gpu_activity); + return utility::get_size( + item.settings, item.device_id, static_cast(item.timestamp), + item.gfx_activity, item.umc_activity, item.mm_activity, item.power, + item.temperature, static_cast(item.mem_usage), item.gpu_activity); } struct cpu_freq_sample : cacheable_t @@ -634,7 +645,7 @@ template <> inline void serialize(uint8_t* buffer, const cpu_freq_sample& item) { - utility::store_value(buffer, (uint64_t) item.timestamp, item.page_rss, + utility::store_value(buffer, static_cast(item.timestamp), item.page_rss, item.virt_mem_usage, item.peak_rss, item.context_switch_count, item.page_faults, item.user_mode_time, item.kernel_mode_time, item.freqs); @@ -657,7 +668,7 @@ template <> inline size_t get_size(const cpu_freq_sample& item) { - return utility::get_size((uint64_t) item.timestamp, item.page_rss, + return utility::get_size(static_cast(item.timestamp), item.page_rss, item.virt_mem_usage, item.peak_rss, item.context_switch_count, item.page_faults, item.user_mode_time, item.kernel_mode_time, item.freqs); diff --git a/projects/rocprofiler-systems/source/lib/core/trace_cache/storage_parser.hpp b/projects/rocprofiler-systems/source/lib/core/trace_cache/storage_parser.hpp index f85698ed9d..20e446be7f 100644 --- a/projects/rocprofiler-systems/source/lib/core/trace_cache/storage_parser.hpp +++ b/projects/rocprofiler-systems/source/lib/core/trace_cache/storage_parser.hpp @@ -150,9 +150,6 @@ public: } ifs.close(); - ROCPROFSYS_DEBUG("File parsing finished. Removing %s from file system.\n", - m_filename.c_str()); - std::remove(m_filename.c_str()); } private: diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/backtrace_metrics.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/backtrace_metrics.cpp index 222ed76354..91d22cdc4b 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/backtrace_metrics.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/backtrace_metrics.cpp @@ -262,9 +262,10 @@ cache_backtrace_metrics_events(const uint32_t device_id, uint64_t timestamp_ns, auto insert_event_and_sample = [&](const char* _track_name, double _value) { trace_cache::get_buffer_storage().store(trace_cache::pmc_event_with_sample{ - _track_name, timestamp_ns, event_metadata, stack_id, parent_stack_id, - correlation_id, call_stack, line_info, device_id, - static_cast(agent_type::CPU), _track_name, _value }); + static_cast(category_enum_id::value), _track_name, + timestamp_ns, event_metadata, stack_id, parent_stack_id, correlation_id, + call_stack, line_info, device_id, static_cast(agent_type::CPU), + _track_name, _value }); }; if constexpr(std::is_same_v) diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/comm_data.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/comm_data.cpp index 25597cb373..01f0c132ad 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/comm_data.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/comm_data.cpp @@ -152,6 +152,7 @@ cache_comm_data_events(const uint32_t device_id, int bytes) const std::string line_info = "{}"; trace_cache::get_buffer_storage().store(trace_cache::pmc_event_with_sample{ + static_cast(category_enum_id::value), track_name.c_str(), timestamp_ns, event_metadata.c_str(), stack_id, parent_stack_id, correlation_id, call_stack.c_str(), line_info.c_str(), device_id, static_cast(agent_type::CPU), track_name.c_str(), diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/kokkosp.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/kokkosp.cpp index 69dd361f8d..d96898ccf3 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/kokkosp.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/kokkosp.cpp @@ -192,6 +192,7 @@ cache_kokkos_event(const char* name, const char* event_type, const char* target, rocprofsys::trace_cache::get_buffer_storage().store( rocprofsys::trace_cache::in_time_sample{ + static_cast(rocprofsys::category_enum_id::value), rocprofsys::trait::name::value, timestamp_ns, event_metadata.dump().c_str(), stack_id, parent_stack_id, correlation_id, call_stack, line_info }); diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp index 05c822e059..d4bd5a5dbb 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk.cpp @@ -24,6 +24,7 @@ #include "api.hpp" #include "common/synchronized.hpp" #include "core/common.hpp" +#include "core/common_types.hpp" #include "core/config.hpp" #include "core/containers/stable_vector.hpp" #include "core/debug.hpp" @@ -594,20 +595,6 @@ cache_memory_allocation(rocprofiler_buffer_tracing_memory_allocation_record_t* r } #endif -std::string -get_args_string(const function_args_t& args) -{ - std::string args_str; - std::for_each(args.begin(), args.end(), [&args_str](const argument_info& arg) { - const auto* delimiter = ";;"; - std::stringstream ss; - ss << arg.arg_number << delimiter << arg.arg_type << delimiter << arg.arg_name - << delimiter << arg.arg_value << delimiter; - args_str.append(ss.str()); - }); - return args_str; -} - template void tool_tracing_callback_start(CategoryT, rocprofiler_callback_tracing_record_t record, diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.cpp index 09e62675c0..0dbceb94cc 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.cpp @@ -121,6 +121,8 @@ counter_event::operator()(const client_data* tool_data, ::perfetto::CounterTrack auto agent = get_agent_manager_instance().get_agent_by_handle(agent_handle); trace_cache::get_buffer_storage().store(trace_cache::pmc_event_with_sample{ + static_cast( + category_enum_id::value), track_name.c_str(), _timing.start, event_metadata.c_str(), stack_id, parent_stack_id, correlation_id, call_stack.c_str(), line_info.c_str(), static_cast(agent.device_id), static_cast(agent.type), @@ -174,8 +176,8 @@ counter_storage::operator()(const counter_event& _event, timing_interval _timing } void -counter_storage::write(counter_storage_type* storage, std::string metric_name, - std::string metric_description) +counter_storage::write(counter_storage_type* storage, const std::string& metric_name, + const std::string& metric_description) { if(!trait::runtime_enabled::get()) { diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.hpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.hpp index 58b31ffcd4..05b65df442 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.hpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/counters.hpp @@ -109,8 +109,8 @@ struct counter_storage void operator()(const counter_event& _event, timing_interval _timing, scope::config _scope = scope::get_default()) const; - static void write(counter_storage_type* storage, std::string metric_name, - std::string metric_description); + static void write(counter_storage_type* storage, const std::string& metric_name, + const std::string& metric_description); }; } // namespace rocprofiler_sdk } // namespace rocprofsys diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/fwd.hpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/fwd.hpp index 3d8ea17dd7..d1950a5e44 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/fwd.hpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/rocprofiler-sdk/fwd.hpp @@ -101,16 +101,6 @@ struct timing_interval rocprofiler_timestamp_t end = 0; }; -struct argument_info -{ - uint32_t arg_number = 0; - std::string arg_type = {}; - std::string arg_name = {}; - std::string arg_value = {}; -}; - -using function_args_t = std::vector; - using agent_counter_info_map_t = std::unordered_map>;