From efe1edd25360d8b13e835193bf3470b7585407ac Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Mon, 27 Jun 2022 23:01:24 -0500 Subject: [PATCH] Fix PID resolution + OMNITRACE_VERSION + fix various configs (#71) * Fix to pid via mpi_gotcha * OMNITRACE_VERSION defines * call perfetto on hsa_activity_callback thread * Test label tweak * Config fixes - Change type of OMNITRACE_DL_VERBOSE - Update OMNITRACE_DL_VERBOSE properly - Add OMNITRACE_ROCM_SMI_DEVICES to deprecated group - Set default_process_suffix * metadata for OMNITRACE_VERSION and OMNITRACE_HIP_VERSION --- .../library/components/mpi_gotcha.cpp | 6 ++-- .../components/roctracer_callbacks.cpp | 17 +++++----- source/lib/omnitrace/library/config.cpp | 32 +++++++++++++++---- source/lib/omnitrace/library/defines.hpp.in | 5 +++ tests/CMakeLists.txt | 3 ++ 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/source/lib/omnitrace/library/components/mpi_gotcha.cpp b/source/lib/omnitrace/library/components/mpi_gotcha.cpp index 3e571fe8be..0e94749b97 100644 --- a/source/lib/omnitrace/library/components/mpi_gotcha.cpp +++ b/source/lib/omnitrace/library/components/mpi_gotcha.cpp @@ -141,6 +141,7 @@ void mpi_gotcha::stop() { OMNITRACE_BASIC_VERBOSE(0, "[pid=%i] Stopping MPI gotcha...\n", process::get_id()); + if(mpip_index == std::numeric_limits::max()) return; update(); } @@ -177,7 +178,8 @@ mpi_gotcha::update() tim::mpi::rank(), _rank); OMNITRACE_BASIC_VERBOSE(0, "[pid=%i] MPI size: %i (%i)\n", process::get_id(), tim::mpi::size(), _size); - last_comm_record = _rank_data; + last_comm_record = _rank_data; + config::get_use_pid() = true; } } @@ -323,7 +325,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) _data.tool_id.c_str(), (int) _retval); } - // if(_comm_entry.updated()) update(); + if(_comm_entry.updated()) update(); } } omnitrace_pop_trace_hidden(_data.tool_id.c_str()); diff --git a/source/lib/omnitrace/library/components/roctracer_callbacks.cpp b/source/lib/omnitrace/library/components/roctracer_callbacks.cpp index 3eace38d95..7ea557d32b 100644 --- a/source/lib/omnitrace/library/components/roctracer_callbacks.cpp +++ b/source/lib/omnitrace/library/components/roctracer_callbacks.cpp @@ -345,15 +345,16 @@ hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg) return _v; }(); + if(get_use_perfetto()) + { + TRACE_EVENT_BEGIN("device", perfetto::StaticString{ *_name }, + static_cast(_beg_ns), "begin_ns", + static_cast(_beg_ns)); + TRACE_EVENT_END("device", static_cast(_end_ns), "end_ns", + static_cast(_end_ns)); + } + auto _func = [_beg_ns, _end_ns, _name]() { - if(get_use_perfetto()) - { - TRACE_EVENT_BEGIN("device", perfetto::StaticString{ *_name }, - static_cast(_beg_ns), "begin_ns", - static_cast(_beg_ns)); - TRACE_EVENT_END("device", static_cast(_end_ns), "end_ns", - static_cast(_end_ns)); - } if(get_use_timemory()) { roctracer_hsa_bundle_t _bundle{ *_name, _scope }; diff --git a/source/lib/omnitrace/library/config.cpp b/source/lib/omnitrace/library/config.cpp index 74d78e0d56..760bea8274 100644 --- a/source/lib/omnitrace/library/config.cpp +++ b/source/lib/omnitrace/library/config.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -150,6 +151,21 @@ configure_settings(bool _init) std::to_string(get_state()).c_str()); } + tim::manager::add_metadata("OMNITRACE_VERSION", OMNITRACE_VERSION_STRING); + tim::manager::add_metadata("OMNITRACE_VERSION_MAJOR", OMNITRACE_VERSION_MAJOR); + tim::manager::add_metadata("OMNITRACE_VERSION_MINOR", OMNITRACE_VERSION_MINOR); + tim::manager::add_metadata("OMNITRACE_VERSION_PATCH", OMNITRACE_VERSION_PATCH); + +#if OMNITRACE_HIP_VERSION > 0 + tim::manager::add_metadata("OMNITRACE_HIP_VERSION", OMNITRACE_HIP_VERSION_STRING); + tim::manager::add_metadata("OMNITRACE_HIP_VERSION_MAJOR", + OMNITRACE_HIP_VERSION_MAJOR); + tim::manager::add_metadata("OMNITRACE_HIP_VERSION_MINOR", + OMNITRACE_HIP_VERSION_MINOR); + tim::manager::add_metadata("OMNITRACE_HIP_VERSION_PATCH", + OMNITRACE_HIP_VERSION_PATCH); +#endif + static auto _config = settings::shared_instance(); // if using timemory, default to perfetto being off @@ -173,8 +189,8 @@ configure_settings(bool _init) "for continuous integration)", false, "debugging"); - OMNITRACE_CONFIG_EXT_SETTING(bool, "OMNITRACE_DL_VERBOSE", - "Verbosity within the omnitrace-dl library", false, + OMNITRACE_CONFIG_EXT_SETTING(int, "OMNITRACE_DL_VERBOSE", + "Verbosity within the omnitrace-dl library", 0, "debugging", "libomnitrace-dl"); OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_PERFETTO", "Enable perfetto backend", @@ -253,7 +269,7 @@ configure_settings(bool _init) OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_ROCM_SMI_DEVICES", "[DEPRECATED] Renamed to OMNITRACE_SAMPLING_GPUS", "all", - "rocm_smi", "rocm", "process_sampling"); + "rocm_smi", "rocm", "process_sampling", "deprecated"); OMNITRACE_CONFIG_SETTING( std::string, "OMNITRACE_SAMPLING_GPUS", @@ -368,8 +384,8 @@ configure_settings(bool _init) OMNITRACE_CONFIG_SETTING( std::string, "OMNITRACE_TIMEMORY_COMPONENTS", - "List of components to collect via timemory (see timemory-avail)", "wall_clock", - "timemory", "component"); + "List of components to collect via timemory (see `omnitrace-avail -C`)", + "wall_clock", "timemory", "component"); OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_OUTPUT_FILE", "Perfetto filename", "", "perfetto", "io", "filename"); @@ -497,12 +513,16 @@ configure_settings(bool _init) settings::suppress_parsing() = true; settings::use_output_suffix() = _config->get("OMNITRACE_USE_PID"); + if(settings::use_output_suffix()) + settings::default_process_suffix() = process::get_id(); #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS) if(tim::dmp::is_initialized()) settings::default_process_suffix() = tim::dmp::rank(); #endif auto _dl_verbose = _config->find("OMNITRACE_DL_VERBOSE"); - tim::set_env(std::string{ _dl_verbose->first }, _dl_verbose->second->as_string(), 0); + if(_dl_verbose->second->get_config_updated()) + tim::set_env(std::string{ _dl_verbose->first }, _dl_verbose->second->as_string(), + 0); #if !defined(TIMEMORY_USE_MPI) || TIMEMORY_USE_MPI == 0 _config->disable("OMNITRACE_PERFETTO_COMBINE_TRACES"); diff --git a/source/lib/omnitrace/library/defines.hpp.in b/source/lib/omnitrace/library/defines.hpp.in index 0166231a83..c9d4c07207 100644 --- a/source/lib/omnitrace/library/defines.hpp.in +++ b/source/lib/omnitrace/library/defines.hpp.in @@ -23,6 +23,11 @@ #pragma once // clang-format off +#define OMNITRACE_VERSION_STRING "@FULL_VERSION_STRING@" +#define OMNITRACE_VERSION_MAJOR @PROJECT_VERSION_MAJOR@ +#define OMNITRACE_VERSION_MINOR @PROJECT_VERSION_MINOR@ +#define OMNITRACE_VERSION_PATCH @PROJECT_VERSION_PATCH@ + #define OMNITRACE_HIP_VERSION_STRING "@OMNITRACE_HIP_VERSION@" #define OMNITRACE_HIP_VERSION_MAJOR @OMNITRACE_HIP_VERSION_MAJOR@ #define OMNITRACE_HIP_VERSION_MINOR @OMNITRACE_HIP_VERSION_MINOR@ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b8f04b3d09..55c794952e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -284,6 +284,9 @@ function(OMNITRACE_ADD_TEST) if("${_TEST}" MATCHES "runtime-instrument") set(_timeout ${TEST_RUNTIME_TIMEOUT}) endif() + if("${_TEST}" MATCHES "binary-rewrite-run") + list(APPEND _labels "binary-rewrite-run") + endif() set(_props) if("${_TEST}" MATCHES "run|baseline") set(_props ${TEST_PROPERTIES})