Fix PID resolution + OMNITRACE_VERSION + fix various configs (#71)

* Fix to pid via mpi_gotcha

* OMNITRACE_VERSION defines

* call perfetto on hsa_activity_callback thread

* Test label tweak

* Config fixes

- Change type of OMNITRACE_DL_VERBOSE
- Update OMNITRACE_DL_VERBOSE properly
- Add OMNITRACE_ROCM_SMI_DEVICES to deprecated group
- Set default_process_suffix

* metadata for OMNITRACE_VERSION and OMNITRACE_HIP_VERSION

source/lib/omnitrace/library/components/mpi_gotcha.cpp

@@ -141,6 +141,7 @@ void
 mpi_gotcha::stop()
 {
     OMNITRACE_BASIC_VERBOSE(0, "[pid=%i] Stopping MPI gotcha...\n", process::get_id());
+    if(mpip_index == std::numeric_limits<uint64_t>::max()) return;
     update();
 }
 
@@ -177,7 +178,8 @@ mpi_gotcha::update()
                                 tim::mpi::rank(), _rank);
         OMNITRACE_BASIC_VERBOSE(0, "[pid=%i] MPI size: %i (%i)\n", process::get_id(),
                                 tim::mpi::size(), _size);
-        last_comm_record = _rank_data;
+        last_comm_record      = _rank_data;
+        config::get_use_pid() = true;
     }
 }
 
@@ -323,7 +325,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
                     _data.tool_id.c_str(), (int) _retval);
             }
 
-            // if(_comm_entry.updated()) update();
+            if(_comm_entry.updated()) update();
         }
     }
     omnitrace_pop_trace_hidden(_data.tool_id.c_str());

source/lib/omnitrace/library/components/roctracer_callbacks.cpp

@@ -345,15 +345,16 @@ hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
         return _v;
     }();
 
+    if(get_use_perfetto())
+    {
+        TRACE_EVENT_BEGIN("device", perfetto::StaticString{ *_name },
+                          static_cast<uint64_t>(_beg_ns), "begin_ns",
+                          static_cast<uint64_t>(_beg_ns));
+        TRACE_EVENT_END("device", static_cast<uint64_t>(_end_ns), "end_ns",
+                        static_cast<uint64_t>(_end_ns));
+    }
+
     auto _func = [_beg_ns, _end_ns, _name]() {
-        if(get_use_perfetto())
-        {
-            TRACE_EVENT_BEGIN("device", perfetto::StaticString{ *_name },
-                              static_cast<uint64_t>(_beg_ns), "begin_ns",
-                              static_cast<uint64_t>(_beg_ns));
-            TRACE_EVENT_END("device", static_cast<uint64_t>(_end_ns), "end_ns",
-                            static_cast<uint64_t>(_end_ns));
-        }
         if(get_use_timemory())
         {
             roctracer_hsa_bundle_t _bundle{ *_name, _scope };

source/lib/omnitrace/library/config.cpp

@@ -31,6 +31,7 @@
 #include <timemory/backends/threading.hpp>
 #include <timemory/environment.hpp>
 #include <timemory/environment/types.hpp>
+#include <timemory/manager.hpp>
 #include <timemory/sampling/allocator.hpp>
 #include <timemory/settings.hpp>
 #include <timemory/settings/types.hpp>
@@ -150,6 +151,21 @@ configure_settings(bool _init)
                                     std::to_string(get_state()).c_str());
     }
 
+    tim::manager::add_metadata("OMNITRACE_VERSION", OMNITRACE_VERSION_STRING);
+    tim::manager::add_metadata("OMNITRACE_VERSION_MAJOR", OMNITRACE_VERSION_MAJOR);
+    tim::manager::add_metadata("OMNITRACE_VERSION_MINOR", OMNITRACE_VERSION_MINOR);
+    tim::manager::add_metadata("OMNITRACE_VERSION_PATCH", OMNITRACE_VERSION_PATCH);
+
+#if OMNITRACE_HIP_VERSION > 0
+    tim::manager::add_metadata("OMNITRACE_HIP_VERSION", OMNITRACE_HIP_VERSION_STRING);
+    tim::manager::add_metadata("OMNITRACE_HIP_VERSION_MAJOR",
+                               OMNITRACE_HIP_VERSION_MAJOR);
+    tim::manager::add_metadata("OMNITRACE_HIP_VERSION_MINOR",
+                               OMNITRACE_HIP_VERSION_MINOR);
+    tim::manager::add_metadata("OMNITRACE_HIP_VERSION_PATCH",
+                               OMNITRACE_HIP_VERSION_PATCH);
+#endif
+
     static auto _config = settings::shared_instance();
 
     // if using timemory, default to perfetto being off
@@ -173,8 +189,8 @@ configure_settings(bool _init)
                              "for continuous integration)",
                              false, "debugging");
 
-    OMNITRACE_CONFIG_EXT_SETTING(bool, "OMNITRACE_DL_VERBOSE",
-                                 "Verbosity within the omnitrace-dl library", false,
+    OMNITRACE_CONFIG_EXT_SETTING(int, "OMNITRACE_DL_VERBOSE",
+                                 "Verbosity within the omnitrace-dl library", 0,
                                  "debugging", "libomnitrace-dl");
 
     OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_PERFETTO", "Enable perfetto backend",
@@ -253,7 +269,7 @@ configure_settings(bool _init)
 
     OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_ROCM_SMI_DEVICES",
                              "[DEPRECATED] Renamed to OMNITRACE_SAMPLING_GPUS", "all",
-                             "rocm_smi", "rocm", "process_sampling");
+                             "rocm_smi", "rocm", "process_sampling", "deprecated");
 
     OMNITRACE_CONFIG_SETTING(
         std::string, "OMNITRACE_SAMPLING_GPUS",
@@ -368,8 +384,8 @@ configure_settings(bool _init)
 
     OMNITRACE_CONFIG_SETTING(
         std::string, "OMNITRACE_TIMEMORY_COMPONENTS",
-        "List of components to collect via timemory (see timemory-avail)", "wall_clock",
-        "timemory", "component");
+        "List of components to collect via timemory (see `omnitrace-avail -C`)",
+        "wall_clock", "timemory", "component");
 
     OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_OUTPUT_FILE", "Perfetto filename",
                              "", "perfetto", "io", "filename");
@@ -497,12 +513,16 @@ configure_settings(bool _init)
 
     settings::suppress_parsing()  = true;
     settings::use_output_suffix() = _config->get<bool>("OMNITRACE_USE_PID");
+    if(settings::use_output_suffix())
+        settings::default_process_suffix() = process::get_id();
 #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
     if(tim::dmp::is_initialized()) settings::default_process_suffix() = tim::dmp::rank();
 #endif
 
     auto _dl_verbose = _config->find("OMNITRACE_DL_VERBOSE");
-    tim::set_env(std::string{ _dl_verbose->first }, _dl_verbose->second->as_string(), 0);
+    if(_dl_verbose->second->get_config_updated())
+        tim::set_env(std::string{ _dl_verbose->first }, _dl_verbose->second->as_string(),
+                     0);
 
 #if !defined(TIMEMORY_USE_MPI) || TIMEMORY_USE_MPI == 0
     _config->disable("OMNITRACE_PERFETTO_COMBINE_TRACES");

source/lib/omnitrace/library/defines.hpp.in

@@ -23,6 +23,11 @@
 #pragma once
 
 // clang-format off
+#define OMNITRACE_VERSION_STRING "@FULL_VERSION_STRING@"
+#define OMNITRACE_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
+#define OMNITRACE_VERSION_MINOR @PROJECT_VERSION_MINOR@
+#define OMNITRACE_VERSION_PATCH @PROJECT_VERSION_PATCH@
+
 #define OMNITRACE_HIP_VERSION_STRING "@OMNITRACE_HIP_VERSION@"
 #define OMNITRACE_HIP_VERSION_MAJOR @OMNITRACE_HIP_VERSION_MAJOR@
 #define OMNITRACE_HIP_VERSION_MINOR @OMNITRACE_HIP_VERSION_MINOR@

tests/CMakeLists.txt

@@ -284,6 +284,9 @@ function(OMNITRACE_ADD_TEST)
             if("${_TEST}" MATCHES "runtime-instrument")
                 set(_timeout ${TEST_RUNTIME_TIMEOUT})
             endif()
+            if("${_TEST}" MATCHES "binary-rewrite-run")
+                list(APPEND _labels "binary-rewrite-run")
+            endif()
             set(_props)
             if("${_TEST}" MATCHES "run|baseline")
                 set(_props ${TEST_PROPERTIES})