exit gotcha + remove DelayedInit state + rocm-smi + cleanup (#110)

* exit gotcha + remove DelayedInit state + cleanup

- exit gotcha which wraps exit, quick_exit, abort
- minor refactor of mpi gotchas
- removed some redundant code in omnitrace_finalized_hidden
- exclude instrumenting functions starting with dlopen and dlsym
- exclude instrumenting exit, quick_exit, and abort functions
- update timemory submodule with support for new gotcha_invoker with (gotcha_data, <function pointer>, args...)

* Improved rocm_smi error handling

[ROCm/rocprofiler-systems commit: 99da25ea80]

projects/rocprofiler-systems/source/bin/omnitrace/details.cpp

@@ -46,7 +46,7 @@ get_whole_function_names()
         "sigandset", "sigdelset", "sigemptyset", "sigfillset", "sighold", "sigisemptyset",
         "sigismember", "sigorset", "sigrelse", "sigvec", "strtok", "strstr", "sbrk",
         "strxfrm", "atexit", "ompt_start_tool", "nanosleep", "cfree", "tolower",
-        "toupper", "fileno", "fileno_unlocked",
+        "toupper", "fileno", "fileno_unlocked", "exit", "quick_exit", "abort",
         // below are functions which never terminate
         "rocr::core::Signal::WaitAny", "rocr::core::Runtime::AsyncEventsLoop",
         "rocr::core::BusyWaitSignal::WaitAcquire",

projects/rocprofiler-systems/source/bin/omnitrace/module_function.cpp

@@ -435,8 +435,8 @@ module_function::is_routine_constrained() const
         "std::(basic_|)(ifstream|ios|istream|ostream|stream))",
         regex_opts);
     static std::regex leading(
-        "^(_|\\.|frame_dummy|transaction clone|virtual "
-        "thunk|non-virtual thunk|\\(|targ|kmp_threadprivate_|Kokkos::Profiling::)",
+        "^(_|\\.|frame_dummy|transaction clone|virtual thunk|non-virtual thunk|"
+        "\\(|targ|kmp_threadprivate_|Kokkos::Profiling::|dlopen|dlsym)",
         regex_opts);
     static std::regex trailing(
         "(_|\\.part\\.[0-9]+|\\.constprop\\.[0-9]+|\\.|\\.[0-9]+)$", regex_opts);

projects/rocprofiler-systems/source/lib/omnitrace/CMakeLists.txt

@@ -75,6 +75,7 @@ set(library_sources
     ${CMAKE_CURRENT_LIST_DIR}/library/timemory.cpp
     ${CMAKE_CURRENT_LIST_DIR}/library/tracing.cpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/backtrace.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/library/components/exit_gotcha.cpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/fork_gotcha.cpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/mpi_gotcha.cpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/omnitrace.cpp
@@ -112,6 +113,7 @@ set(library_headers
     ${CMAKE_CURRENT_LIST_DIR}/library/components/fwd.hpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/backtrace.hpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/category_region.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/library/components/exit_gotcha.hpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/fork_gotcha.hpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/functors.hpp
     ${CMAKE_CURRENT_LIST_DIR}/library/components/mpi_gotcha.hpp

projects/rocprofiler-systems/source/lib/omnitrace/library.cpp

@@ -23,6 +23,7 @@
 #include "library.hpp"
 #include "common/setup.hpp"
 #include "library/api.hpp"
+#include "library/components/exit_gotcha.hpp"
 #include "library/components/fork_gotcha.hpp"
 #include "library/components/functors.hpp"
 #include "library/components/fwd.hpp"
@@ -49,6 +50,7 @@
 #include <timemory/utility/procfs/maps.hpp>
 
 #include <atomic>
+#include <cstdio>
 #include <mutex>
 #include <string_view>
 
@@ -335,8 +337,7 @@ omnitrace_set_mpi_hidden(bool use, bool attached)
     _set_mpi_called       = true;
     config::is_attached() = attached;
 
-    if(use && !attached &&
-       (get_state() == State::PreInit || get_state() == State::DelayedInit))
+    if(use && !attached && get_state() == State::PreInit)
     {
         tim::set_env("OMNITRACE_USE_PID", "ON", 1);
     }
@@ -842,14 +843,15 @@ omnitrace_finalize_hidden(void)
     }
 
     OMNITRACE_DEBUG_F("Stopping and destroying instrumentation bundles...\n");
-    for(auto& itr : instrumentation_bundles::instances())
+    for(size_t i = 0; i < max_supported_threads; ++i)
     {
+        auto& itr = instrumentation_bundles::instances().at(i);
         while(!itr.bundles.empty())
         {
             OMNITRACE_VERBOSE_F(1,
-                                "Warning! instrumentation bundle on thread %li with "
-                                "label '%s' was not stopped.\n",
-                                itr.bundles.back()->tid(),
+                                "Warning! instrumentation bundle on thread %zu (TID=%li) "
+                                "with label '%s' was not stopped.\n",
+                                i, itr.bundles.back()->tid(),
                                 itr.bundles.back()->key().c_str());
             itr.bundles.back()->stop();
             itr.bundles.back()->pop();
@@ -934,20 +936,6 @@ omnitrace_finalize_hidden(void)
         }
     }
 
-    // ensure that all the MT instances are flushed
-    OMNITRACE_VERBOSE_F(3, "Stopping and destroying instrumentation bundles...\n");
-    for(auto& itr : instrumentation_bundles::instances())
-    {
-        while(!itr.bundles.empty())
-        {
-            itr.bundles.back()->stop();
-            itr.bundles.back()->pop();
-            itr.allocator.destroy(itr.bundles.back());
-            itr.allocator.deallocate(itr.bundles.back(), 1);
-            itr.bundles.pop_back();
-        }
-    }
-
     // ensure that all the MT instances are flushed
     if(get_use_sampling())
     {

projects/rocprofiler-systems/source/lib/omnitrace/library/components/exit_gotcha.cpp

@@ -0,0 +1,80 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/exit_gotcha.hpp"
+#include "library/common.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/runtime.hpp"
+#include "library/state.hpp"
+#include "library/timemory.hpp"
+
+#include <timemory/backends/threading.hpp>
+#include <timemory/components/timing/wall_clock.hpp>
+#include <timemory/sampling/allocator.hpp>
+#include <timemory/utility/types.hpp>
+
+#include <cstddef>
+#include <cstdlib>
+
+namespace omnitrace
+{
+void
+exit_gotcha::configure()
+{
+    exit_gotcha_t::get_initializer() = []() {
+        exit_gotcha_t::template configure<0, void>("abort");
+        exit_gotcha_t::template configure<1, void, int>("exit");
+        exit_gotcha_t::template configure<2, void, int>("quick_exit");
+    };
+}
+
+namespace
+{
+template <typename FuncT, typename... Args>
+void
+invoke_exit_gotcha(const exit_gotcha::gotcha_data& _data, FuncT _func, Args... _args)
+{
+    OMNITRACE_VERBOSE(0, "%s called %s(%s)...\n", get_exe_name().c_str(),
+                      _data.tool_id.c_str(), JOIN(", ", _args...).c_str());
+
+    if(get_state() != omnitrace::State::Finalized) omnitrace_finalize_hidden();
+
+    (*_func)(_args...);
+}
+}  // namespace
+
+// exit
+// quick_exit
+void
+exit_gotcha::operator()(const gotcha_data& _data, exit_func_t _func, int _ec) const
+{
+    invoke_exit_gotcha(_data, _func, _ec);
+}
+
+// abort
+void
+exit_gotcha::operator()(const gotcha_data& _data, abort_func_t _func) const
+{
+    invoke_exit_gotcha(_data, _func);
+}
+}  // namespace omnitrace

projects/rocprofiler-systems/source/lib/omnitrace/library/components/exit_gotcha.hpp

@@ -0,0 +1,59 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/defines.hpp"
+#include "library/timemory.hpp"
+
+#include <timemory/components/base.hpp>
+#include <timemory/components/gotcha/backends.hpp>
+
+#include <cstdint>
+#include <cstdlib>
+
+namespace omnitrace
+{
+struct exit_gotcha : tim::component::base<exit_gotcha, void>
+{
+    using gotcha_data  = tim::component::gotcha_data;
+    using exit_func_t  = void (*)(int);
+    using abort_func_t = void (*)();
+
+    TIMEMORY_DEFAULT_OBJECT(exit_gotcha)
+
+    // string id for component
+    static std::string label() { return "exit_gotcha"; }
+
+    // generate the gotcha wrappers
+    static void configure();
+    static void shutdown();
+
+    // exit
+    void operator()(const gotcha_data&, exit_func_t, int) const;
+    // abort
+    void operator()(const gotcha_data&, abort_func_t) const;
+};
+
+using exit_gotcha_t = tim::component::gotcha<3, std::tuple<>, exit_gotcha>;
+}  // namespace omnitrace

projects/rocprofiler-systems/source/lib/omnitrace/library/components/mpi_gotcha.cpp

@@ -22,7 +22,6 @@
 
 #include "library/components/mpi_gotcha.hpp"
 #include "library/api.hpp"
-#include "library/common.hpp"
 #include "library/components/category_region.hpp"
 #include "library/config.hpp"
 #include "library/debug.hpp"
@@ -35,15 +34,10 @@
 #include <cstdint>
 #include <limits>
 #include <thread>
-#include <type_traits>
 #include <unistd.h>
 
 namespace omnitrace
 {
-using mpi_tooltag_t = api::omnitrace;
-using mpi_toolset_t =
-    tim::component_tuple<omnitrace::component::category_region<category::mpi>>;
-
 namespace
 {
 struct comm_rank_data
@@ -110,7 +104,10 @@ omnitrace_mpi_set_attr()
     static auto _mpi_fini = [](MPI_Comm, int, void*, void*) {
         OMNITRACE_DEBUG("MPI Comm attribute finalize\n");
         if(mpip_index != std::numeric_limits<uint64_t>::max())
-            comp::deactivate_mpip<mpi_toolset_t, mpi_tooltag_t>(mpip_index);
+            comp::deactivate_mpip<
+                tim::component_tuple<
+                    omnitrace::component::category_region<category::mpi>>,
+                api::omnitrace>(mpip_index);
         omnitrace_finalize_hidden();
         return MPI_SUCCESS;
     };
@@ -227,7 +224,9 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
     OMNITRACE_BASIC_DEBUG_F("%s()\n", _data.tool_id.c_str());
 
     if(mpip_index != std::numeric_limits<uint64_t>::max())
-        comp::deactivate_mpip<mpi_toolset_t, mpi_tooltag_t>(mpip_index);
+        comp::deactivate_mpip<
+            tim::component_tuple<omnitrace::component::category_region<category::mpi>>,
+            api::omnitrace>(mpip_index);
 
 #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
     tim::mpi::is_initialized_callback() = []() { return false; };
@@ -279,16 +278,14 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
 
             // use env vars OMNITRACE_MPIP_PERMIT_LIST and OMNITRACE_MPIP_REJECT_LIST
             // to control the gotcha bindings at runtime
-            auto _accept = std::set<std::string>{};
-            auto _reject = std::set<std::string>{};
-
-#if defined(OMNITRACE_USE_MPI_HEADERS) && !defined(OMNITRACE_USE_MPI)
-            _reject.emplace("MPI_Comm_rank");
-            _reject.emplace("MPI_Comm_size");
-#endif
-
-            comp::configure_mpip<mpi_toolset_t, mpi_tooltag_t>(_accept, _reject);
-            mpip_index = comp::activate_mpip<mpi_toolset_t, mpi_tooltag_t>();
+            comp::configure_mpip<
+                tim::component_tuple<
+                    omnitrace::component::category_region<category::mpi>>,
+                api::omnitrace>();
+            mpip_index = comp::activate_mpip<
+                tim::component_tuple<
+                    omnitrace::component::category_region<category::mpi>>,
+                api::omnitrace>();
         }
 
         auto_lock_t _lk{ type_mutex<mpi_gotcha>() };

projects/rocprofiler-systems/source/lib/omnitrace/library/components/pthread_mutex_gotcha.cpp

@@ -217,8 +217,8 @@ pthread_mutex_gotcha::operator()(uintptr_t&& _id, const comp::gotcha_data& _data
             _id, get_hashes().at(_data.index), _depth);
     }
 
-    return _ret;
     tim::consume_parameters(_id, _cid, _parent_cid, _depth, _ts);
+    return _ret;
 }
 
 int

projects/rocprofiler-systems/source/lib/omnitrace/library/components/rocm_smi.cpp

@@ -40,6 +40,7 @@
 #include "library/debug.hpp"
 #include "library/gpu.hpp"
 #include "library/perfetto.hpp"
+#include "library/state.hpp"
 
 #include <timemory/backends/threading.hpp>
 #include <timemory/components/timing/backends.hpp>
@@ -52,11 +53,13 @@
 #include <chrono>
 #include <ios>
 #include <sstream>
+#include <stdexcept>
 #include <string>
 #include <sys/resource.h>
 #include <thread>
 
-#define OMNITRACE_ROCM_SMI_CALL(ERROR_CODE) ::omnitrace::rocm_smi::check_error(ERROR_CODE)
+#define OMNITRACE_ROCM_SMI_CALL(ERROR_CODE)                                              \
+    ::omnitrace::rocm_smi::check_error(ERROR_CODE, __FILE__, __LINE__)
 
 namespace omnitrace
 {
@@ -77,15 +80,17 @@ is_initialized()
 }
 
 void
-check_error(rsmi_status_t ec)
+check_error(rsmi_status_t _code, const char* _file, int _line)
 {
-    if(ec == RSMI_STATUS_SUCCESS) return;
+    if(_code == RSMI_STATUS_SUCCESS) return;
     const char* _msg = nullptr;
-    auto        _err = rsmi_status_string(ec, &_msg);
+    auto        _err = rsmi_status_string(_code, &_msg);
     if(_err != RSMI_STATUS_SUCCESS)
-        OMNITRACE_THROW(
-            "rsmi_status_string(%i, ...) failed. No error message available\n", (int) ec);
-    OMNITRACE_THROW("%s", _msg);
+        OMNITRACE_THROW("rsmi_status_string failed. No error message available. "
+                        "Error code %i originated at %s:%i\n",
+                        static_cast<int>(_code), _file, _line);
+    OMNITRACE_THROW("[%s:%i] Error code %i :: %s", _file, _line, static_cast<int>(_code),
+                    _msg);
 }
 
 std::atomic<State>&
@@ -110,15 +115,33 @@ data::sample(uint32_t _dev_id)
     auto _ts = tim::get_clock_real_now<size_t, std::nano>();
     assert(_ts < std::numeric_limits<int64_t>::max());
 
-    if(get_state() != State::Active) return;
+    auto _state = get_state().load();
+
+    if(_state != State::Active) return;
 
     m_dev_id = _dev_id;
     m_ts     = _ts;
 
-    rsmi_dev_busy_percent_get(_dev_id, &m_busy_perc);
-    rsmi_dev_temp_metric_get(_dev_id, RSMI_TEMP_TYPE_EDGE, RSMI_TEMP_CURRENT, &m_temp);
-    rsmi_dev_power_ave_get(_dev_id, 0, &m_power);
-    rsmi_dev_memory_usage_get(_dev_id, RSMI_MEM_TYPE_VRAM, &m_mem_usage);
+#define OMNITRACE_RSMI_GET(FUNCTION, ...)                                                \
+    try                                                                                  \
+    {                                                                                    \
+        OMNITRACE_ROCM_SMI_CALL(FUNCTION(__VA_ARGS__));                                  \
+    } catch(std::runtime_error & _e)                                                     \
+    {                                                                                    \
+        OMNITRACE_VERBOSE_F(                                                             \
+            0, "[%s] Exception: %s. Disabling future samples from rocm-smi...\n",        \
+            #FUNCTION, _e.what());                                                       \
+        get_state().store(State::Disabled);                                              \
+    }
+
+    OMNITRACE_RSMI_GET(rsmi_dev_busy_percent_get, _dev_id, &m_busy_perc);
+    OMNITRACE_RSMI_GET(rsmi_dev_temp_metric_get, _dev_id, RSMI_TEMP_TYPE_EDGE,
+                       RSMI_TEMP_CURRENT, &m_temp);
+    OMNITRACE_RSMI_GET(rsmi_dev_power_ave_get, _dev_id, 0, &m_power);
+    OMNITRACE_RSMI_GET(rsmi_dev_memory_usage_get, _dev_id, RSMI_MEM_TYPE_VRAM,
+                       &m_mem_usage);
+
+#undef OMNITRACE_RSMI_GET
 }
 
 void
@@ -359,17 +382,25 @@ setup()
 
     data::device_list = _devices;
 
-    for(auto itr : _devices)
+    try
     {
-        uint16_t dev_id = 0;
-        OMNITRACE_ROCM_SMI_CALL(rsmi_dev_id_get(itr, &dev_id));
-        // dev_id holds the device ID of device i, upon a successful call
+        for(auto itr : _devices)
+        {
+            uint16_t dev_id = 0;
+            OMNITRACE_ROCM_SMI_CALL(rsmi_dev_id_get(itr, &dev_id));
+            // dev_id holds the device ID of device i, upon a successful call
+        }
+
+        is_initialized() = true;
+
+        data::setup();
+    } catch(std::runtime_error& _e)
+    {
+        OMNITRACE_VERBOSE(0, "Exception thrown when initializing rocm-smi: %s\n",
+                          _e.what());
+        data::device_list = {};
     }
 
-    is_initialized() = true;
-
-    data::setup();
-
     pthread_gotcha::pop_enable_sampling_on_child_threads();
 }
 
@@ -380,9 +411,16 @@ shutdown()
 
     if(!is_initialized()) return;
 
-    if(data::shutdown())
+    try
     {
-        OMNITRACE_ROCM_SMI_CALL(rsmi_shut_down());
+        if(data::shutdown())
+        {
+            OMNITRACE_ROCM_SMI_CALL(rsmi_shut_down());
+        }
+    } catch(std::runtime_error& _e)
+    {
+        OMNITRACE_VERBOSE(0, "Exception thrown when shutting down rocm-smi: %s\n",
+                          _e.what());
     }
 
     is_initialized() = false;
@@ -408,7 +446,8 @@ device_count()
         OMNITRACE_ROCM_SMI_CALL(rsmi_num_monitor_devices(&_num_devices));
     } catch(const std::exception& _e)
     {
-        OMNITRACE_BASIC_PRINT("Exception: %s\n", _e.what());
+        OMNITRACE_BASIC_PRINT("Exception thrown getting the rocm-smi devices: %s\n",
+                              _e.what());
     }
     return _num_devices;
 }

projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp

@@ -350,10 +350,13 @@ configure_settings(bool _init)
                              "cause deadlocks with ROCm-enabled OpenMPI.",
                              false, "backend", "parallelism", "gotcha");
 
-    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_SAMPLING_KEEP_INTERNAL",
-                             "If disabled, omnitrace will attempt to filter out internal "
-                             "routines from the sampling call-stacks",
-                             true, "sampling", "data");
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_SAMPLING_KEEP_INTERNAL",
+        "Configure whether the statistical samples should include call-stack entries "
+        "from internal routines in omnitrace. E.g. when ON, the call-stack will show "
+        "functions like omnitrace_push_trace. If disabled, omnitrace will attempt to "
+        "filter out internal routines from the sampling call-stacks",
+        true, "sampling", "data");
 
     OMNITRACE_CONFIG_SETTING(
         bool, "OMNITRACE_SAMPLING_REALTIME",
@@ -1164,14 +1167,16 @@ is_binary_rewrite()
 bool
 get_debug_env()
 {
-    return (settings_are_configured()) ? get_debug()
-                                       : tim::get_env<bool>("OMNITRACE_DEBUG", false);
+    return (settings_are_configured())
+               ? get_debug()
+               : tim::get_env<bool>("OMNITRACE_DEBUG", false, false);
 }
 
 bool
 get_is_continuous_integration()
 {
-    if(!settings_are_configured()) return tim::get_env<bool>("OMNITRACE_CI", false);
+    if(!settings_are_configured())
+        return tim::get_env<bool>("OMNITRACE_CI", false, false);
     static auto _v = get_config()->find("OMNITRACE_CI");
     return static_cast<tim::tsettings<bool>&>(*_v->second).get();
 }
@@ -1208,7 +1213,7 @@ int
 get_verbose_env()
 {
     return (settings_are_configured()) ? get_verbose()
-                                       : tim::get_env<int>("OMNITRACE_VERBOSE", 0);
+                                       : tim::get_env<int>("OMNITRACE_VERBOSE", 0, false);
 }
 
 int

projects/rocprofiler-systems/source/lib/omnitrace/library/runtime.cpp

@@ -184,6 +184,7 @@ setup_gotchas()
         "Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n");
 
     mpi_gotcha::configure();
+    exit_gotcha::configure();
     fork_gotcha::configure();
     pthread_gotcha::configure();
 }

projects/rocprofiler-systems/source/lib/omnitrace/library/runtime.hpp

@@ -24,6 +24,7 @@
 
 #include "library/api.hpp"
 #include "library/common.hpp"
+#include "library/components/exit_gotcha.hpp"
 #include "library/components/fork_gotcha.hpp"
 #include "library/components/mpi_gotcha.hpp"
 #include "library/components/pthread_gotcha.hpp"
@@ -33,11 +34,11 @@
 #include "library/thread_data.hpp"
 #include "library/timemory.hpp"
 
-#include <memory>
-#include <set>
 #include <timemory/backends/threading.hpp>
 #include <timemory/macros/language.hpp>
 
+#include <memory>
+#include <set>
 #include <string>
 #include <string_view>
 #include <unordered_set>
@@ -49,7 +50,8 @@ using main_bundle_t =
     tim::lightweight_tuple<comp::wall_clock, comp::peak_rss, comp::cpu_clock,
                            comp::cpu_util, pthread_gotcha>;
 
-using gotcha_bundle_t = tim::lightweight_tuple<fork_gotcha_t, mpi_gotcha_t>;
+using gotcha_bundle_t =
+    tim::lightweight_tuple<exit_gotcha_t, fork_gotcha_t, mpi_gotcha_t>;
 
 // bundle of components around each thread
 #if defined(TIMEMORY_RUSAGE_THREAD) && TIMEMORY_RUSAGE_THREAD > 0

projects/rocprofiler-systems/source/lib/omnitrace/library/state.cpp

@@ -31,10 +31,10 @@ to_string(omnitrace::State _v)
 {
     switch(_v)
     {
-        case omnitrace::State::DelayedInit: return "DelayedInit";
         case omnitrace::State::PreInit: return "PreInit";
         case omnitrace::State::Init: return "Init";
         case omnitrace::State::Active: return "Active";
+        case omnitrace::State::Disabled: return "Disabled";
         case omnitrace::State::Finalized: return "Finalized";
     }
     return {};

projects/rocprofiler-systems/source/lib/omnitrace/library/state.hpp

@@ -29,10 +29,10 @@ namespace omnitrace
 // used for specifying the state of omnitrace
 enum class State : unsigned short
 {
-    DelayedInit = 0,
-    PreInit,
+    PreInit = 0,
     Init,
     Active,
+    Disabled,
     Finalized
 };