Critical trace updates (#24)

* Source code restructuring

* Critical trace updates following restructuring

* thread_sampler, timestamps

- thread_sampler
- CPU frequency managed via thread_sampler
- rocm-smi managed via thread_sampler
- Use consistent timestamps for perfetto
- removed hsa_timer_t in favor of wall_clock::record()
- disable KokkosP by default
- re-enable critical-trace testing

* cmake-format

* Fix for defines.hpp.in

* Remove OMNITRACE_ROCM_SMI_FREQ

- thread_sampler freq is set via OMNITRACE_SAMPLING_FREQ w/ max of 1000

* Increase CI Install Dyninst timeout

* Debug macros + omnitrace_init_tooling + config

- new debug macros
- extern "C" omnitrace_init_tooling
- guard get_rocm_smi_devices

* Miscellaneous tweaks

- tweak to transpose
- critical_trace::Device::ANY
- perfetto "critical-trace" category
- OMNITRACE_VERBOSE usage

* Disable key and tid data for HIP API calls

- non-kernels are ignored in activity callback

* critical-trace exe updates

- fix perfetto generation
- improved logging
- improved readability

* timemory submodule update

- lulesh example cmake tweaks

source/lib/CMakeLists.txt

@@ -0,0 +1,139 @@
+# ------------------------------------------------------------------------------#
+#
+# omnitrace interface library
+#
+# ------------------------------------------------------------------------------#
+
+add_library(omnitrace-interface-library INTERFACE)
+add_library(omnitrace::omnitrace-interface-library ALIAS omnitrace-interface-library)
+
+target_include_directories(
+    omnitrace-interface-library INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include
+                                          ${CMAKE_CURRENT_BINARY_DIR}/include)
+target_include_directories(omnitrace-interface-library SYSTEM
+                           INTERFACE ${perfetto_DIR}/sdk)
+
+target_compile_definitions(
+    omnitrace-interface-library
+    INTERFACE OMNITRACE_MAX_THREADS=${OMNITRACE_MAX_THREADS}
+              $<IF:$<BOOL:${OMNITRACE_CUSTOM_DATA_SOURCE}>,CUSTOM_DATA_SOURCE,>)
+
+target_link_libraries(
+    omnitrace-interface-library
+    INTERFACE $<BUILD_INTERFACE:omnitrace::omnitrace-headers>
+              $<BUILD_INTERFACE:omnitrace::omnitrace-threading>
+              $<BUILD_INTERFACE:omnitrace::omnitrace-compile-options>
+              $<BUILD_INTERFACE:omnitrace::omnitrace-hip>
+              $<BUILD_INTERFACE:omnitrace::omnitrace-roctracer>
+              $<BUILD_INTERFACE:omnitrace::omnitrace-rocm-smi>
+              $<BUILD_INTERFACE:omnitrace::omnitrace-mpi>
+              $<BUILD_INTERFACE:omnitrace::omnitrace-ptl>
+              $<BUILD_INTERFACE:timemory::timemory-headers>
+              $<BUILD_INTERFACE:timemory::timemory-gotcha>
+              $<BUILD_INTERFACE:timemory::timemory-cxx-shared>
+              $<IF:$<BOOL:${OMNITRACE_USE_SANITIZER}>,omnitrace::omnitrace-sanitizer,>)
+
+# ------------------------------------------------------------------------------#
+#
+# omnitrace object library
+#
+# ------------------------------------------------------------------------------#
+
+add_library(omnitrace-object-library OBJECT)
+add_library(omnitrace::omnitrace-object-library ALIAS omnitrace-object-library)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/library/defines.hpp.in
+               ${CMAKE_CURRENT_BINARY_DIR}/include/library/defines.hpp @ONLY)
+
+set(library_sources
+    ${CMAKE_CURRENT_LIST_DIR}/src/library.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/config.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/cpu_freq.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/critical_trace.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/kokkosp.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/gpu.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/perfetto.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/ptl.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/sampling.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/state.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/thread_data.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/thread_sampler.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/timemory.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/components/backtrace.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/components/fork_gotcha.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/components/mpi_gotcha.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/components/omnitrace.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/components/pthread_gotcha.cpp
+    ${perfetto_DIR}/sdk/perfetto.cc)
+
+set(library_headers
+    ${CMAKE_CURRENT_LIST_DIR}/include/library.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/api.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/config.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/common.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/cpu_freq.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/critical_trace.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/debug.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/gpu.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/perfetto.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/ptl.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/sampling.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/state.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/thread_data.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/thread_sampler.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/timemory.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/fwd.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/backtrace.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/fork_gotcha.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/mpi_gotcha.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/omnitrace.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/rocm_smi.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/roctracer.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/roctracer_callbacks.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/components/pthread_gotcha.hpp
+    ${perfetto_DIR}/sdk/perfetto.h)
+
+target_sources(omnitrace-object-library PRIVATE ${library_sources} ${library_headers})
+
+if(OMNITRACE_USE_ROCTRACER)
+    target_sources(
+        omnitrace-object-library
+        PRIVATE ${CMAKE_CURRENT_LIST_DIR}/src/library/components/roctracer.cpp
+                ${CMAKE_CURRENT_LIST_DIR}/src/library/components/roctracer_callbacks.cpp)
+endif()
+
+if(OMNITRACE_USE_ROCM_SMI)
+    target_sources(omnitrace-object-library
+                   PRIVATE ${CMAKE_CURRENT_LIST_DIR}/src/library/components/rocm_smi.cpp)
+endif()
+
+target_link_libraries(omnitrace-object-library PRIVATE omnitrace-interface-library)
+
+if(OMNITRACE_DYNINST_API_RT)
+    get_filename_component(OMNITRACE_DYNINST_API_RT_DIR "${OMNITRACE_DYNINST_API_RT}"
+                           DIRECTORY)
+endif()
+
+# ------------------------------------------------------------------------------#
+#
+# omnitrace shared library
+#
+# ------------------------------------------------------------------------------#
+
+add_library(omnitrace-library SHARED $<TARGET_OBJECTS:omnitrace-object-library>)
+add_library(omnitrace::omnitrace-library ALIAS omnitrace-library)
+
+target_link_libraries(omnitrace-library PRIVATE omnitrace-interface-library)
+
+set_target_properties(
+    omnitrace-library
+    PROPERTIES OUTPUT_NAME omnitrace
+               VERSION ${PROJECT_VERSION}
+               SOVERSION ${PROJECT_VERSION_MAJOR}
+               INSTALL_RPATH
+               "\$ORIGIN:\$ORIGIN/timemory/libunwind:\$ORIGIN/dyninst-tpls/libs")
+
+install(
+    TARGETS omnitrace-library
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    OPTIONAL)

source/lib/include/library.hpp

@@ -0,0 +1,108 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+// this always needs to included first
+// clang-format off
+#include "library/perfetto.hpp"
+// clang-format on
+
+#include "library/timemory.hpp"
+#include "library/components/roctracer.hpp"
+#include "library/api.hpp"
+#include "library/components/fork_gotcha.hpp"
+#include "library/components/mpi_gotcha.hpp"
+#include "library/api.hpp"
+#include "library/common.hpp"
+#include "library/state.hpp"
+#include "library/config.hpp"
+#include "library/thread_data.hpp"
+#include "library/ptl.hpp"
+#include "library/debug.hpp"
+#include "library/critical_trace.hpp"
+
+#include <timemory/macros/language.hpp>
+#include <timemory/utility/utility.hpp>
+
+#include <mutex>
+
+namespace omnitrace
+{
+template <critical_trace::Device DevID, critical_trace::Phase PhaseID,
+          bool UpdateStack = true>
+inline void
+add_critical_trace(int64_t _tid, size_t _cpu_cid, size_t _gpu_cid, size_t _parent_cid,
+                   int64_t _ts_beg, int64_t _ts_val, size_t _hash, uint16_t _depth,
+                   uint16_t _prio = 0)
+{
+    // clang-format off
+    // these are used to create unique type mutexes
+    struct critical_insert {};
+    struct cpu_cid_stack {};
+    // clang-format on
+
+    using tim::type_mutex;
+    using auto_lock_t                  = tim::auto_lock_t;
+    static constexpr auto num_mutexes  = max_supported_threads;
+    static auto           _update_freq = critical_trace::get_update_frequency();
+
+    if constexpr(PhaseID != critical_trace::Phase::NONE)
+    {
+        // unique lock per thread
+        auto&       _mtx = type_mutex<critical_insert, api::omnitrace, num_mutexes>(_tid);
+        auto_lock_t _lk{ _mtx };
+
+        auto& _critical_trace = critical_trace::get(_tid);
+        _critical_trace->emplace_back(
+            critical_trace::entry{ _prio, DevID, PhaseID, _depth, _tid, _cpu_cid,
+                                   _gpu_cid, _parent_cid, _ts_beg, _ts_val, _hash });
+    }
+
+    if constexpr(UpdateStack)
+    {
+        // unique lock per thread
+        auto& _mtx = type_mutex<cpu_cid_stack, api::omnitrace, num_mutexes>(_tid);
+
+        if constexpr(PhaseID == critical_trace::Phase::NONE)
+        {
+            auto_lock_t _lk{ _mtx };
+            get_cpu_cid_stack(_tid)->emplace_back(_cpu_cid);
+        }
+        else if constexpr(PhaseID == critical_trace::Phase::BEGIN)
+        {
+            auto_lock_t _lk{ _mtx };
+            get_cpu_cid_stack(_tid)->emplace_back(_cpu_cid);
+        }
+        else if constexpr(PhaseID == critical_trace::Phase::END)
+        {
+            auto_lock_t _lk{ _mtx };
+            get_cpu_cid_stack(_tid)->pop_back();
+            if(_gpu_cid == 0 && _cpu_cid % _update_freq == (_update_freq - 1))
+                critical_trace::update(_tid);
+        }
+    }
+
+    tim::consume_parameters(_tid, _cpu_cid, _gpu_cid, _parent_cid, _ts_beg, _ts_val,
+                            _hash, _depth, _prio);
+}
+}  // namespace omnitrace

source/lib/include/library/api.hpp

@@ -0,0 +1,56 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+
+#include <timemory/compat/macros.h>
+
+// forward decl of the API
+extern "C"
+{
+    /// handles configuration logic
+    void omnitrace_init_library(void) TIMEMORY_VISIBILITY("default");
+
+    /// starts gotcha wrappers
+    void omnitrace_init(const char*, bool, const char*) TIMEMORY_VISIBILITY("default");
+
+    /// shuts down all tooling and generates output
+    void omnitrace_finalize(void) TIMEMORY_VISIBILITY("default");
+
+    /// sets an environment variable
+    void omnitrace_set_env(const char* env_name, const char* env_val)
+        TIMEMORY_VISIBILITY("default");
+
+    /// sets whether MPI should be used
+    void omnitrace_set_mpi(bool use, bool attached) TIMEMORY_VISIBILITY("default");
+
+    /// starts an instrumentation region
+    void omnitrace_push_trace(const char* name) TIMEMORY_VISIBILITY("default");
+
+    /// stops an instrumentation region
+    void omnitrace_pop_trace(const char* name) TIMEMORY_VISIBILITY("default");
+
+    /// used by omnitrace-critical-trace
+    bool omnitrace_init_tooling() TIMEMORY_VISIBILITY("hidden");
+}

source/lib/include/library/common.hpp

@@ -0,0 +1,51 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+
+#include <timemory/api.hpp>
+#include <timemory/backends/dmp.hpp>
+#include <timemory/backends/process.hpp>
+
+#include <cassert>
+#include <cstdint>
+#include <cstdio>
+#include <fstream>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <sys/types.h>
+#include <thread>
+#include <unistd.h>
+#include <utility>
+#include <vector>
+
+TIMEMORY_DEFINE_NS_API(api, omnitrace)
+TIMEMORY_DEFINE_NS_API(api, sampling)
+TIMEMORY_DEFINE_NS_API(api, rocm_smi)
+
+namespace omnitrace
+{
+namespace api = tim::api;  // NOLINT
+}

source/lib/include/library/components/backtrace.hpp

@@ -0,0 +1,120 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/components/fwd.hpp"
+#include "library/defines.hpp"
+#include "library/thread_data.hpp"
+#include "library/timemory.hpp"
+
+#include <timemory/components/base.hpp>
+#include <timemory/components/papi/papi_array.hpp>
+#include <timemory/macros/language.hpp>
+#include <timemory/mpl/concepts.hpp>
+#include <timemory/sampling/sampler.hpp>
+#include <timemory/variadic/types.hpp>
+
+#include <array>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <set>
+#include <vector>
+
+namespace omnitrace
+{
+namespace component
+{
+struct backtrace
+: tim::component::empty_base
+, tim::concepts::component
+{
+    static constexpr size_t num_hw_counters = 8;
+
+    using data_t            = std::array<char[512], 128>;
+    using clock_type        = std::chrono::steady_clock;
+    using time_point_type   = typename clock_type::time_point;
+    using value_type        = void;
+    using hw_counters       = tim::component::papi_array<num_hw_counters>;
+    using hw_counter_data_t = typename hw_counters::value_type;
+    using system_clock      = std::chrono::system_clock;
+    using system_time_point = typename system_clock::time_point;
+
+    static void        preinit();
+    static std::string label();
+    static std::string description();
+
+    backtrace()                 = default;
+    ~backtrace()                = default;
+    backtrace(backtrace&&)      = default;
+    backtrace(const backtrace&) = default;
+
+    backtrace& operator=(const backtrace&) = default;
+    backtrace& operator=(backtrace&&) = default;
+
+    bool operator<(const backtrace& rhs) const;
+
+    static std::set<int>      configure(bool, int64_t _tid = threading::get_id());
+    static void               post_process(int64_t _tid = threading::get_id());
+    static hw_counter_data_t& get_last_hwcounters();
+
+    static void              start();
+    static void              stop();
+    void                     sample(int = -1);
+    bool                     empty() const;
+    size_t                   size() const;
+    std::vector<std::string> get() const;
+    time_point_type          get_timestamp() const;
+    int64_t                  get_thread_cpu_timestamp() const;
+
+private:
+    int64_t           m_tid        = 0;
+    int64_t           m_thr_cpu_ts = 0;
+    int64_t           m_mem_peak   = 0;
+    size_t            m_size       = 0;
+    time_point_type   m_ts         = {};
+    data_t            m_data       = {};
+    hw_counter_data_t m_hw_counter = {};
+};
+}  // namespace component
+}  // namespace omnitrace
+
+#if !defined(OMNITRACE_EXTERN_COMPONENTS) ||                                             \
+    (defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0)
+
+#    include <timemory/operations.hpp>
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_wall_clock>), true,
+    double)
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_cpu_clock>), true,
+    double)
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_fraction>), true,
+    double)
+
+#endif

source/lib/include/library/components/fork_gotcha.hpp

@@ -0,0 +1,52 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/defines.hpp"
+#include "library/timemory.hpp"
+
+namespace omnitrace
+{
+// this is used to wrap fork()
+struct fork_gotcha : comp::base<fork_gotcha, void>
+{
+    using gotcha_data_t = comp::gotcha_data;
+
+    TIMEMORY_DEFAULT_OBJECT(fork_gotcha)
+
+    // string id for component
+    static std::string label() { return "fork_gotcha"; }
+
+    // generate the gotcha wrappers
+    static void configure();
+
+    // this will get called right before fork
+    static void audit(const gotcha_data_t& _data, audit::incoming);
+
+    // this will get called right after fork with the return value
+    static void audit(const gotcha_data_t& _data, audit::outgoing, pid_t _pid);
+};
+
+using fork_gotcha_t = comp::gotcha<4, tim::component_tuple<fork_gotcha>, api::omnitrace>;
+}  // namespace omnitrace

source/lib/include/library/components/fwd.hpp

@@ -0,0 +1,188 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+
+#include <timemory/components/data_tracker/types.hpp>
+#include <timemory/components/macros.hpp>
+#include <timemory/components/user_bundle/types.hpp>
+#include <timemory/enum.h>
+#include <timemory/mpl/concepts.hpp>
+
+TIMEMORY_DECLARE_COMPONENT(roctracer)
+
+namespace omnitrace
+{
+namespace component
+{
+template <typename... Tp>
+using data_tracker = tim::component::data_tracker<Tp...>;
+
+struct omnitrace;
+struct backtrace;
+struct backtrace_wall_clock
+{};
+struct backtrace_cpu_clock
+{};
+struct backtrace_fraction
+{};
+struct backtrace_gpu_busy
+{};
+struct backtrace_gpu_temp
+{};
+struct backtrace_gpu_power
+{};
+struct backtrace_gpu_memory
+{};
+using sampling_wall_clock = data_tracker<double, backtrace_wall_clock>;
+using sampling_cpu_clock  = data_tracker<double, backtrace_cpu_clock>;
+using sampling_percent    = data_tracker<double, backtrace_fraction>;
+using sampling_gpu_busy   = data_tracker<double, backtrace_gpu_busy>;
+using sampling_gpu_temp   = data_tracker<double, backtrace_gpu_temp>;
+using sampling_gpu_power  = data_tracker<double, backtrace_gpu_power>;
+using sampling_gpu_memory = data_tracker<double, backtrace_gpu_memory>;
+using roctracer           = tim::component::roctracer;
+}  // namespace component
+}  // namespace omnitrace
+
+#if !defined(OMNITRACE_USE_ROCTRACER)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, component::roctracer, false_type)
+#endif
+
+#if !defined(TIMEMORY_USE_LIBUNWIND)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::api::sampling, false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::backtrace, false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_wall_clock,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_cpu_clock,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_percent,
+                               false_type)
+#endif
+
+#if !defined(TIMEMORY_USE_LIBUNWIND) || !defined(OMNITRACE_USE_ROCM_SMI)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_busy,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_temp,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_power,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_memory,
+                               false_type)
+#endif
+
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::omnitrace, OMNITRACE_COMPONENT,
+                                 "omnitrace", "omnitrace_component")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::roctracer, OMNITRACE_ROCTRACER,
+                                 "roctracer", "omnitrace_roctracer")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_wall_clock,
+                                 OMNITRACE_SAMPLING_WALL_CLOCK, "sampling_wall_clock", "")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_cpu_clock,
+                                 OMNITRACE_SAMPLING_CPU_CLOCK, "sampling_cpu_clock", "")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_percent,
+                                 OMNITRACE_SAMPLING_PERCENT, "sampling_percent", "")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_busy,
+                                 OMNITRACE_SAMPLING_GPU_BUSY, "sampling_gpu_busy",
+                                 "sampling_gpu_util")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_memory,
+                                 OMNITRACE_SAMPLING_GPU_MEMORY_USAGE,
+                                 "sampling_gpu_memory_usage", "")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_power,
+                                 OMNITRACE_SAMPLING_GPU_POWER, "sampling_gpu_power", "")
+TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_temp,
+                                 OMNITRACE_SAMPLING_GPU_TEMP, "sampling_gpu_temp", "")
+
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::roctracer, "roctracer",
+                                 "High-precision ROCm API and kernel tracing", "")
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_wall_clock,
+                                 "sampling_wall_clock", "Wall-clock timing",
+                                 "Derived from statistical sampling")
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_cpu_clock,
+                                 "sampling_cpu_clock", "CPU-clock timing",
+                                 "Derived from statistical sampling")
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_percent,
+                                 "sampling_percent",
+                                 "Fraction of wall-clock time spent in functions",
+                                 "Derived from statistical sampling")
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_busy,
+                                 "sampling_gpu_busy",
+                                 "GPU Utilization (% busy) via ROCm-SMI",
+                                 "Derived from sampling")
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_memory,
+                                 "sampling_gpu_memory_usage",
+                                 "GPU Memory Usage via ROCm-SMI", "Derived from sampling")
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_power,
+                                 "sampling_gpu_power", "GPU Power Usage via ROCm-SMI",
+                                 "Derived from sampling")
+TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_temp,
+                                 "sampling_gpu_temp", "GPU Temperature via ROCm-SMI",
+                                 "Derived from sampling")
+
+// statistics type
+TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_wall_clock, double)
+TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_cpu_clock, double)
+TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_busy, double)
+TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_temp, double)
+TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_power, double)
+TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_memory, double)
+
+// enable timing units
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category,
+                               omnitrace::component::sampling_wall_clock, true_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category,
+                               omnitrace::component::sampling_cpu_clock, true_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, omnitrace::component::sampling_percent,
+                               true_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units,
+                               omnitrace::component::sampling_wall_clock, true_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units,
+                               omnitrace::component::sampling_cpu_clock, true_type)
+
+// enable percent units
+TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_percent_units,
+                               omnitrace::component::sampling_gpu_busy, true_type)
+
+// enable memory units
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_memory_category,
+                               omnitrace::component::sampling_gpu_memory, true_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_memory_units,
+                               omnitrace::component::sampling_gpu_memory, true_type)
+
+// reporting categories (sum)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_busy,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_temp,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_power,
+                               false_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_memory,
+                               false_type)
+
+// reporting categories (mean)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(report_mean, omnitrace::component::sampling_percent,
+                               false_type)
+
+// reporting categories (stats)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(report_statistics, omnitrace::component::sampling_percent,
+                               false_type)

source/lib/include/library/components/mpi_gotcha.hpp

@@ -0,0 +1,69 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/defines.hpp"
+#include "library/timemory.hpp"
+
+namespace omnitrace
+{
+// this is used to wrap MPI_Init and MPI_Init_thread
+struct mpi_gotcha : comp::base<mpi_gotcha, void>
+{
+    using comm_t        = tim::mpi::comm_t;
+    using gotcha_data_t = comp::gotcha_data;
+
+    TIMEMORY_DEFAULT_OBJECT(mpi_gotcha)
+
+    // string id for component
+    static std::string label() { return "mpi_gotcha"; }
+
+    // generate the gotcha wrappers
+    static void configure();
+
+    // called right before MPI_Init with that functions arguments
+    static void audit(const gotcha_data_t& _data, audit::incoming, int*, char***);
+
+    // called right before MPI_Init_thread with that functions arguments
+    static void audit(const gotcha_data_t& _data, audit::incoming, int*, char***, int,
+                      int*);
+
+    // called right before MPI_Finalize
+    static void audit(const gotcha_data_t& _data, audit::incoming);
+
+    // called right before MPI_Comm_{rank,size} with that functions arguments
+    void audit(const gotcha_data_t& _data, audit::incoming, comm_t, int*);
+
+    // called right after MPI_{Init,Init_thread,Comm_rank,Comm_size} with the return value
+    void audit(const gotcha_data_t& _data, audit::outgoing, int _retval);
+
+private:
+    int* m_rank_ptr = nullptr;
+    int* m_size_ptr = nullptr;
+    int  m_rank     = 0;
+    int  m_size     = 1;
+};
+
+using mpi_gotcha_t = comp::gotcha<5, tim::component_tuple<mpi_gotcha>, api::omnitrace>;
+}  // namespace omnitrace

source/lib/include/library/components/omnitrace.hpp

@@ -0,0 +1,50 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+#include "library/timemory.hpp"
+
+namespace omnitrace
+{
+namespace component
+{
+// timemory component which calls omnitrace functions
+// (used in gotcha wrappers)
+struct omnitrace : comp::base<omnitrace, void>
+{
+    static std::string label() { return "omnitrace"; }
+    void               start();
+    void               stop();
+    void               set_prefix(const char*);
+
+private:
+    const char* m_prefix = nullptr;
+};
+}  // namespace component
+}  // namespace omnitrace
+
+TIMEMORY_METADATA_SPECIALIZATION(
+    omnitrace::component::omnitrace, "omnitrace",
+    "Invokes instrumentation functions 'omnitrace_push_trace' and 'omnitrace_pop_trace'",
+    "Used by gotcha wrappers")

source/lib/include/library/components/pthread_gotcha.hpp

@@ -0,0 +1,70 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/defines.hpp"
+#include "library/timemory.hpp"
+
+#include <future>
+
+namespace omnitrace
+{
+struct pthread_gotcha : tim::component::base<pthread_gotcha, void>
+{
+    struct wrapper
+    {
+        using routine_t = void* (*) (void*);
+        using promise_t = std::promise<void>;
+
+        wrapper(routine_t _routine, void* _arg, bool, promise_t*);
+        void* operator()() const;
+
+        static void* wrap(void* _arg);
+
+    private:
+        bool       m_enable_sampling = false;
+        routine_t  m_routine         = nullptr;
+        void*      m_arg             = nullptr;
+        promise_t* m_promise         = nullptr;
+    };
+
+    TIMEMORY_DEFAULT_OBJECT(pthread_gotcha)
+
+    // string id for component
+    static std::string label() { return "pthread_gotcha"; }
+
+    // generate the gotcha wrappers
+    static void configure();
+    static void shutdown();
+
+    // threads can set this to avoid starting sampling on child threads
+    static bool& enable_sampling_on_child_threads();
+
+    // pthread_create
+    int operator()(pthread_t* thread, const pthread_attr_t* attr,
+                   void* (*start_routine)(void*), void*     arg) const;
+};
+
+using pthread_gotcha_t = tim::component::gotcha<2, std::tuple<>, pthread_gotcha>;
+}  // namespace omnitrace

source/lib/include/library/components/rocm_smi.hpp

@@ -0,0 +1,173 @@
+// Copyright (c) 2018 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// with the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimers.
+//
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimers in the
+// documentation and/or other materials provided with the distribution.
+//
+// * Neither the names of Advanced Micro Devices, Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this Software without specific prior written permission.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
+// THE SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/components/fwd.hpp"
+#include "library/defines.hpp"
+#include "library/thread_data.hpp"
+
+#include <chrono>
+#include <cstdint>
+#include <deque>
+#include <future>
+#include <limits>
+#include <memory>
+#include <ratio>
+#include <thread>
+#include <tuple>
+#include <type_traits>
+
+namespace omnitrace
+{
+namespace rocm_smi
+{
+void
+setup();
+
+void
+config();
+
+void
+sample();
+
+void
+shutdown();
+
+void
+post_process();
+
+void set_state(State);
+
+uint32_t
+device_count();
+
+struct data
+{
+    using msec_t    = std::chrono::milliseconds;
+    using usec_t    = std::chrono::microseconds;
+    using nsec_t    = std::chrono::nanoseconds;
+    using promise_t = std::promise<void>;
+
+    using timestamp_t = int64_t;
+    using power_t     = uint64_t;
+    using busy_perc_t = uint32_t;
+    using mem_usage_t = uint64_t;
+    using temp_t      = int64_t;
+
+    TIMEMORY_DEFAULT_OBJECT(data)
+
+    explicit data(uint32_t _dev_id);
+
+    void sample(uint32_t _dev_id);
+    void print(std::ostream& _os) const;
+
+    static void post_process(uint32_t _dev_id);
+
+    uint32_t    m_dev_id    = std::numeric_limits<uint32_t>::max();
+    timestamp_t m_ts        = 0;
+    busy_perc_t m_busy_perc = 0;
+    temp_t      m_temp      = 0;
+    power_t     m_power     = 0;
+    mem_usage_t m_mem_usage = 0;
+
+    friend std::ostream& operator<<(std::ostream& _os, const data& _v)
+    {
+        _v.print(_os);
+        return _os;
+    }
+
+private:
+    friend void omnitrace::rocm_smi::setup();
+    friend void omnitrace::rocm_smi::config();
+    friend void omnitrace::rocm_smi::sample();
+    friend void omnitrace::rocm_smi::shutdown();
+    friend void omnitrace::rocm_smi::post_process();
+
+    static size_t                        device_count;
+    static std::set<uint32_t>            device_list;
+    static std::unique_ptr<promise_t>    polling_finished;
+    static std::vector<data>&            get_initial();
+    static std::unique_ptr<std::thread>& get_thread();
+    static bool                          setup();
+    static bool                          shutdown();
+};
+
+#if !defined(OMNITRACE_USE_ROCM_SMI)
+inline void
+setup()
+{}
+
+inline void
+config()
+{}
+
+inline void
+sample()
+{}
+
+inline void
+shutdown()
+{}
+
+inline void
+post_process()
+{}
+
+inline void set_state(State) {}
+#endif
+}  // namespace rocm_smi
+}  // namespace omnitrace
+
+#if defined(OMNITRACE_USE_ROCM_SMI)
+#    if !defined(OMNITRACE_EXTERN_COMPONENTS) ||                                         \
+        (defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0)
+
+#        include <timemory/components/base.hpp>
+#        include <timemory/components/data_tracker/components.hpp>
+#        include <timemory/operations.hpp>
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_busy>), true,
+    double)
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_temp>), true,
+    double)
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_power>), true,
+    double)
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_memory>), true,
+    double)
+
+#    endif
+#endif

source/lib/include/library/components/roctracer.hpp

@@ -0,0 +1,104 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/components/fwd.hpp"
+#include "library/defines.hpp"
+
+#include <timemory/api.hpp>
+#include <timemory/components/base.hpp>
+#include <timemory/components/data_tracker/components.hpp>
+#include <timemory/components/macros.hpp>
+#include <timemory/enum.h>
+#include <timemory/macros/os.hpp>
+#include <timemory/mpl/type_traits.hpp>
+#include <timemory/mpl/types.hpp>
+
+namespace tim
+{
+namespace component
+{
+using roctracer_data = data_tracker<double, roctracer>;
+
+struct roctracer
+: base<roctracer, void>
+, private policy::instance_tracker<roctracer, false>
+{
+    using value_type   = void;
+    using base_type    = base<roctracer, void>;
+    using tracker_type = policy::instance_tracker<roctracer, false>;
+
+    TIMEMORY_DEFAULT_OBJECT(roctracer)
+
+    static void preinit();
+    static void global_init() { setup(); }
+    static void global_finalize() { shutdown(); }
+
+    static bool is_setup();
+    static void setup();
+    static void shutdown();
+    static void add_setup(const std::string&, std::function<void()>&&);
+    static void add_shutdown(const std::string&, std::function<void()>&&);
+    static void remove_setup(const std::string&);
+    static void remove_shutdown(const std::string&);
+
+    void start();
+    void stop();
+};
+
+#if !defined(OMNITRACE_USE_ROCTRACER)
+inline void
+roctracer::setup()
+{}
+
+inline void
+roctracer::shutdown()
+{}
+
+inline bool
+roctracer::is_setup()
+{
+    return false;
+}
+#endif
+}  // namespace component
+}  // namespace tim
+
+#if !defined(OMNITRACE_USE_ROCTRACER)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, component::roctracer_data, false_type)
+#endif
+
+TIMEMORY_SET_COMPONENT_API(component::roctracer_data, project::timemory, category::timing,
+                           os::supports_unix)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, component::roctracer_data, true_type)
+TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units, component::roctracer_data, true_type)
+
+#if !defined(OMNITRACE_EXTERN_COMPONENTS) ||                                             \
+    (defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0)
+
+#    include <timemory/operations.hpp>
+
+TIMEMORY_DECLARE_EXTERN_COMPONENT(roctracer, false, void)
+TIMEMORY_DECLARE_EXTERN_COMPONENT(roctracer_data, true, double)
+
+#endif

source/lib/include/library/components/roctracer_callbacks.hpp

@@ -0,0 +1,89 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/components/roctracer.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/dynamic_library.hpp"
+#include "library/perfetto.hpp"
+#include "library/ptl.hpp"
+
+#include <roctracer.h>
+#include <roctracer_ext.h>
+#include <roctracer_hcc.h>
+#include <roctracer_hip.h>
+
+#define AMD_INTERNAL_BUILD 1
+#include <ext/hsa_rt_utils.hpp>
+#include <roctracer_hsa.h>
+
+#include <iostream>
+#include <memory>
+
+// Macro to check ROC-tracer calls status
+#define ROCTRACER_CALL(call)                                                             \
+    do                                                                                   \
+    {                                                                                    \
+        int err = call;                                                                  \
+        if(err != 0)                                                                     \
+        {                                                                                \
+            std::cerr << roctracer_error_string() << " in: " << #call << std::flush;     \
+        }                                                                                \
+    } while(0)
+
+namespace omnitrace
+{
+using roctracer_bundle_t =
+    tim::component_bundle<api::omnitrace, comp::roctracer_data, comp::wall_clock>;
+using roctracer_hsa_bundle_t =
+    tim::component_bundle<api::omnitrace, comp::roctracer_data>;
+using roctracer_functions_t = std::vector<std::pair<std::string, std::function<void()>>>;
+
+// HSA API callback function
+void
+hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
+
+void
+hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg);
+
+void
+hip_exec_activity_callbacks(int64_t _tid);
+
+// HIP API callback function
+void
+hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
+
+// Activity tracing callback
+void
+hip_activity_callback(const char* begin, const char* end, void*);
+
+bool&
+roctracer_is_setup();
+
+roctracer_functions_t&
+roctracer_setup_routines();
+
+roctracer_functions_t&
+roctracer_shutdown_routines();
+}  // namespace omnitrace

source/lib/include/library/config.hpp

@@ -0,0 +1,256 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/api.hpp"
+#include "library/common.hpp"
+#include "library/components/fork_gotcha.hpp"
+#include "library/components/mpi_gotcha.hpp"
+#include "library/components/pthread_gotcha.hpp"
+#include "library/components/roctracer.hpp"
+#include "library/defines.hpp"
+#include "library/state.hpp"
+#include "library/timemory.hpp"
+#include "timemory/macros/language.hpp"
+
+#include <timemory/backends/threading.hpp>
+
+#include <string>
+#include <string_view>
+#include <unordered_set>
+
+namespace omnitrace
+{
+// bundle of components around omnitrace_init and omnitrace_finalize
+using main_bundle_t =
+    tim::lightweight_tuple<comp::wall_clock, comp::peak_rss, comp::cpu_clock,
+                           comp::cpu_util, pthread_gotcha_t>;
+
+using gotcha_bundle_t = tim::lightweight_tuple<fork_gotcha_t, mpi_gotcha_t>;
+
+// bundle of components used in instrumentation
+using instrumentation_bundle_t =
+    tim::component_bundle<api::omnitrace, comp::wall_clock*, comp::user_global_bundle*>;
+
+// allocator for instrumentation_bundle_t
+using bundle_allocator_t = tim::data::ring_buffer_allocator<instrumentation_bundle_t>;
+
+// bundle of components around each thread
+#if defined(TIMEMORY_RUSAGE_THREAD) && TIMEMORY_RUSAGE_THREAD > 0
+using omnitrace_thread_bundle_t =
+    tim::lightweight_tuple<comp::wall_clock, comp::thread_cpu_clock,
+                           comp::thread_cpu_util, comp::peak_rss>;
+#else
+using omnitrace_thread_bundle_t =
+    tim::lightweight_tuple<comp::wall_clock, comp::thread_cpu_clock,
+                           comp::thread_cpu_util>;
+#endif
+
+//
+//      Initialization routines
+//
+inline namespace config
+{
+void
+configure_settings();
+
+void
+print_banner(std::ostream& _os = std::cout);
+
+void
+print_settings(
+    std::ostream&                                                                _os,
+    std::function<bool(const std::string_view&, const std::set<std::string>&)>&& _filter);
+
+void
+print_settings();
+
+std::string&
+get_exe_name();
+
+template <typename Tp>
+bool
+set_setting_value(const std::string& _name, Tp&& _v)
+{
+    auto _instance = tim::settings::shared_instance();
+    auto _setting  = _instance->find(_name);
+    if(_setting == _instance->end()) return false;
+    if(!_setting->second) return false;
+    return _setting->second->set(std::forward<Tp>(_v));
+}
+
+//
+//      User-configurable settings
+//
+std::string
+get_config_file();
+
+Mode
+get_mode();
+
+bool&
+is_attached();
+
+bool&
+is_binary_rewrite();
+
+bool
+get_is_continuous_integration();
+
+bool
+get_debug_env();
+
+bool
+get_debug_init();
+
+bool
+get_debug_finalize();
+
+bool
+get_debug();
+
+bool
+get_debug_tid();
+
+bool
+get_debug_pid();
+
+int
+get_verbose_env();
+
+int
+get_verbose();
+
+bool&
+get_use_perfetto();
+
+bool&
+get_use_timemory();
+
+bool&
+get_use_roctracer();
+
+bool&
+get_use_rocm_smi();
+
+bool&
+get_use_sampling();
+
+bool&
+get_use_pid();
+
+bool&
+get_use_mpip();
+
+bool&
+get_use_critical_trace();
+
+bool
+get_use_kokkosp();
+
+bool
+get_timeline_sampling();
+
+bool
+get_flat_sampling();
+
+bool
+get_roctracer_timeline_profile();
+
+bool
+get_roctracer_flat_profile();
+
+bool
+get_trace_hsa_api();
+
+bool
+get_trace_hsa_activity();
+
+bool
+get_critical_trace_debug();
+
+bool
+get_critical_trace_serialize_names();
+
+size_t
+get_perfetto_shmem_size_hint();
+
+size_t
+get_perfetto_buffer_size();
+
+uint64_t
+get_critical_trace_update_freq();
+
+uint64_t
+get_critical_trace_num_threads();
+
+std::string
+get_trace_hsa_api_types();
+
+std::string&
+get_backend();
+
+// make this visible so omnitrace-avail can call it
+std::string&
+get_perfetto_output_filename();
+
+int64_t
+get_critical_trace_count();
+
+size_t&
+get_instrumentation_interval();
+
+double&
+get_sampling_freq();
+
+double&
+get_sampling_delay();
+
+double&
+get_thread_sampling_freq();
+
+std::string
+get_rocm_smi_devices();
+
+int64_t
+get_critical_trace_per_row();
+}  // namespace config
+
+//
+//      Runtime configuration data
+//
+State&
+get_state();
+
+std::unique_ptr<main_bundle_t>&
+get_main_bundle();
+
+std::unique_ptr<gotcha_bundle_t>&
+get_gotcha_bundle();
+
+std::atomic<uint64_t>&
+get_cpu_cid();
+
+std::unique_ptr<std::vector<uint64_t>>&
+get_cpu_cid_stack(int64_t _tid = threading::get_id());
+}  // namespace omnitrace

source/lib/include/library/cpu_freq.hpp

@@ -0,0 +1,44 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+namespace omnitrace
+{
+namespace cpu_freq
+{
+void
+setup();
+
+void
+config();
+
+void
+sample();
+
+void
+shutdown();
+
+void
+post_process();
+}  // namespace cpu_freq
+}  // namespace omnitrace

source/lib/include/library/critical_trace.hpp

@@ -0,0 +1,244 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/config.hpp"
+#include "library/defines.hpp"
+#include "library/thread_data.hpp"
+
+#include <timemory/hash/types.hpp>
+#include <timemory/tpls/cereal/cereal.hpp>
+#include <timemory/utility/demangle.hpp>
+
+#include <cstdint>
+#include <cstdlib>
+#include <ostream>
+#include <string>
+#include <vector>
+
+namespace omnitrace
+{
+namespace critical_trace
+{
+enum class Device : short
+{
+    NONE = 0,
+    CPU,
+    GPU,
+    ANY,
+};
+
+enum class Phase : short
+{
+    NONE = 0,
+    BEGIN,
+    END,
+    DELTA,
+};
+
+struct entry
+{
+    entry()                 = default;
+    ~entry()                = default;
+    entry(const entry&)     = default;
+    entry(entry&&) noexcept = default;
+    entry& operator=(const entry&) = default;
+    entry& operator=(entry&&) noexcept = default;
+
+    uint16_t priority   = 0;            // priority value (for sorting)
+    Device   device     = Device::CPU;  // which device it executed on
+    Phase    phase      = Phase::NONE;  // start / stop / unspecified
+    uint16_t depth      = 0;            // call-stack depth
+    int64_t  tid        = 0;            // thread id it was registered on
+    uint64_t cpu_cid    = 0;            // CPU correlation id
+    uint64_t gpu_cid    = 0;            // GPU correlation id
+    uint64_t parent_cid = 0;            // parent CPU correlation id
+    int64_t  begin_ns   = 0;            // timestamp of start
+    int64_t  end_ns     = 0;            // timestamp of end
+    size_t   hash       = 0;            // hash for name
+
+    bool operator==(const entry& rhs) const;
+    bool operator!=(const entry& rhs) const { return !(*this == rhs); }
+    bool operator<(const entry& rhs) const;
+    bool operator>(const entry& rhs) const;
+    bool operator<=(const entry& rhs) const { return !(*this > rhs); }
+    bool operator>=(const entry& rhs) const { return !(*this < rhs); }
+
+    entry& operator+=(const entry& rhs);
+
+    size_t  get_hash() const;
+    int64_t get_timestamp() const;
+
+    int64_t get_cost() const;
+
+    bool    is_bounded(const entry& rhs) const;
+    int64_t get_overlap(const entry& rhs) const;
+    int64_t get_independent(const entry& rhs) const;
+
+    int64_t get_overlap(const entry& rhs, int64_t _tid) const;
+    int64_t get_independent(const entry& rhs, int64_t _tid) const;
+    bool    is_bounded(const entry& rhs, int64_t _tid) const;
+
+    void write(std::ostream& _os) const;
+
+    static bool is_delta(const entry&, const std::string_view&);
+
+    friend std::ostream& operator<<(std::ostream& _os, const entry& _v)
+    {
+        _v.write(_os);
+        return _os;
+    }
+    template <typename Archive>
+    void save(Archive& ar, unsigned int) const;
+
+    template <typename Archive>
+    void load(Archive& ar, unsigned int);
+};
+
+template <typename Archive>
+void
+entry::save(Archive& ar, unsigned int) const
+{
+    namespace cereal = tim::cereal;
+    std::string _name{};
+    if(hash > 0) _name = tim::get_hash_identifier(hash);
+    ar(cereal::make_nvp("priority", priority), cereal::make_nvp("device", device),
+       cereal::make_nvp("phase", phase), cereal::make_nvp("depth", depth),
+       cereal::make_nvp("tid", tid), cereal::make_nvp("cpu_cid", cpu_cid),
+       cereal::make_nvp("gpu_cid", gpu_cid), cereal::make_nvp("parent_cid", parent_cid),
+       cereal::make_nvp("begin_ns", begin_ns), cereal::make_nvp("end_ns", end_ns),
+       cereal::make_nvp("hash", hash), cereal::make_nvp("name", _name),
+       cereal::make_nvp("demangled_name", tim::demangle(_name)));
+}
+
+template <typename Archive>
+void
+entry::load(Archive& ar, unsigned int)
+{
+    namespace cereal = tim::cereal;
+    std::string _name{};
+    std::string _demangled_name{};
+    ar(cereal::make_nvp("priority", priority), cereal::make_nvp("device", device),
+       cereal::make_nvp("phase", phase), cereal::make_nvp("depth", depth),
+       cereal::make_nvp("tid", tid), cereal::make_nvp("cpu_cid", cpu_cid),
+       cereal::make_nvp("gpu_cid", gpu_cid), cereal::make_nvp("parent_cid", parent_cid),
+       cereal::make_nvp("begin_ns", begin_ns), cereal::make_nvp("end_ns", end_ns),
+       cereal::make_nvp("hash", hash), cereal::make_nvp("name", _name),
+       cereal::make_nvp("demangled_name", _demangled_name));
+
+    tim::get_hash_ids()->emplace(hash, _name);
+}
+
+struct call_chain : private std::vector<entry>
+{
+    using base_type = std::vector<entry>;
+
+    using base_type::at;
+    using base_type::back;
+    using base_type::begin;
+    using base_type::cbegin;
+    using base_type::cend;
+    using base_type::clear;
+    using base_type::emplace_back;
+    using base_type::empty;
+    using base_type::end;
+    using base_type::erase;
+    using base_type::front;
+    using base_type::pop_back;
+    using base_type::push_back;
+    using base_type::rbegin;
+    using base_type::rend;
+    using base_type::reserve;
+    using base_type::size;
+
+    size_t                          get_hash() const;
+    int64_t                         get_cost(int64_t _tid = -1) const;
+    int64_t                         get_overlap(int64_t _tid = -1) const;
+    int64_t                         get_independent(int64_t _tid = -1) const;
+    static std::vector<call_chain>& get_top_chains();
+
+    bool operator==(const call_chain& rhs) const;
+    bool operator!=(const call_chain& rhs) const { return !(*this == rhs); }
+    friend std::ostream& operator<<(std::ostream& _os, const call_chain& _v)
+    {
+        size_t _n = 0;
+        for(const auto& itr : _v)
+            _os << "    [" << _n++ << "] " << itr << "\n";
+        return _os;
+    }
+
+    template <typename Archive>
+    void serialize(Archive& ar, unsigned int)
+    {
+        namespace cereal = tim::cereal;
+        ar(cereal::make_nvp("call_chain", static_cast<base_type&>(*this)));
+    }
+
+    template <Device DevT>
+    void generate_perfetto(std::set<entry>& _used) const;
+
+    template <bool BoolV = true, typename FuncT>
+    bool query(FuncT&&) const;
+};
+
+template <bool BoolV, typename FuncT>
+bool
+call_chain::query(FuncT&& _func) const
+{
+    for(const auto& itr : *this)
+    {
+        if(std::forward<FuncT>(_func)(itr)) return BoolV;
+    }
+    return !BoolV;
+}
+
+using hash_ids = std::unordered_set<std::string>;
+
+uint64_t
+get_update_frequency();
+
+std::unique_ptr<call_chain>&
+get(int64_t _tid = threading::get_id());
+
+size_t
+add_hash_id(const std::string& _label);
+
+void
+add_hash_id(const hash_ids&);
+
+void
+update(int64_t _tid = threading::get_id());
+
+void
+compute(int64_t _tid = threading::get_id());
+
+std::vector<std::pair<std::string, entry>>
+get_entries(
+    int64_t                                  _ts,
+    const std::function<bool(const entry&)>& _eval = [](const entry&) { return true; });
+
+struct id
+{};
+
+}  // namespace critical_trace
+}  // namespace omnitrace

source/lib/include/library/debug.hpp

@@ -0,0 +1,218 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+
+#include <timemory/api.hpp>
+#include <timemory/backends/dmp.hpp>
+#include <timemory/backends/process.hpp>
+#include <timemory/utility/utility.hpp>
+
+#include <cstdio>
+
+namespace omnitrace
+{
+inline namespace config
+{
+bool
+get_debug();
+
+bool
+get_debug_tid();
+
+bool
+get_debug_pid();
+
+bool
+get_critical_trace_debug();
+}  // namespace config
+}  // namespace omnitrace
+
+#if defined(TIMEMORY_USE_MPI)
+#    define OMNITRACE_PROCESS_IDENTIFIER static_cast<int>(::tim::dmp::rank())
+#elif defined(TIMEMORY_USE_MPI_HEADERS)
+#    define OMNITRACE_PROCESS_IDENTIFIER                                                 \
+        (::tim::dmp::is_initialized()) ? static_cast<int>(::tim::dmp::rank())            \
+                                       : static_cast<int>(::tim::process::get_id())
+#else
+#    define OMNITRACE_PROCESS_IDENTIFIER static_cast<int>(::tim::process::get_id())
+#endif
+
+#define OMNITRACE_CONDITIONAL_PRINT(COND, ...)                                           \
+    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
+       ::omnitrace::config::get_debug_pid())                                             \
+    {                                                                                    \
+        fflush(stderr);                                                                  \
+        tim::auto_lock_t _lk{ tim::type_mutex<decltype(std::cerr)>() };                  \
+        fprintf(stderr, "[omnitrace][%i][%li] ", OMNITRACE_PROCESS_IDENTIFIER,           \
+                tim::threading::get_id());                                               \
+        fprintf(stderr, __VA_ARGS__);                                                    \
+        fflush(stderr);                                                                  \
+    }
+
+#define OMNITRACE_CONDITIONAL_BASIC_PRINT(COND, ...)                                     \
+    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
+       ::omnitrace::config::get_debug_pid())                                             \
+    {                                                                                    \
+        fflush(stderr);                                                                  \
+        tim::auto_lock_t _lk{ tim::type_mutex<decltype(std::cerr)>() };                  \
+        fprintf(stderr, "[omnitrace] ");                                                 \
+        fprintf(stderr, __VA_ARGS__);                                                    \
+        fflush(stderr);                                                                  \
+    }
+
+#define OMNITRACE_CONDITIONAL_PRINT_F(COND, ...)                                         \
+    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
+       ::omnitrace::config::get_debug_pid())                                             \
+    {                                                                                    \
+        fflush(stderr);                                                                  \
+        tim::auto_lock_t _lk{ tim::type_mutex<decltype(std::cerr)>() };                  \
+        fprintf(stderr, "[omnitrace][%i][%li][%s] ", OMNITRACE_PROCESS_IDENTIFIER,       \
+                tim::threading::get_id(), __FUNCTION__);                                 \
+        fprintf(stderr, __VA_ARGS__);                                                    \
+        fflush(stderr);                                                                  \
+    }
+
+#define OMNITRACE_CONDITIONAL_BASIC_PRINT_F(COND, ...)                                   \
+    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
+       ::omnitrace::config::get_debug_pid())                                             \
+    {                                                                                    \
+        fflush(stderr);                                                                  \
+        tim::auto_lock_t _lk{ tim::type_mutex<decltype(std::cerr)>() };                  \
+        fprintf(stderr, "[omnitrace][%s] ", __FUNCTION__);                               \
+        fprintf(stderr, __VA_ARGS__);                                                    \
+        fflush(stderr);                                                                  \
+    }
+
+#define OMNITRACE_CONDITIONAL_THROW(COND, ...)                                           \
+    if(COND)                                                                             \
+    {                                                                                    \
+        char _msg_buffer[2048];                                                          \
+        snprintf(_msg_buffer, 2048, "[omnitrace][%i][%li][%s] ",                         \
+                 OMNITRACE_PROCESS_IDENTIFIER, tim::threading::get_id(), __FUNCTION__);  \
+        auto len = strlen(_msg_buffer);                                                  \
+        snprintf(_msg_buffer + len, 2048 - len, __VA_ARGS__);                            \
+        throw std::runtime_error(_msg_buffer);                                           \
+    }
+
+#define OMNITRACE_CONDITIONAL_BASIC_THROW(COND, ...)                                     \
+    if(COND)                                                                             \
+    {                                                                                    \
+        char _msg_buffer[2048];                                                          \
+        snprintf(_msg_buffer, 2048, "[omnitrace][%s] ", __FUNCTION__);                   \
+        auto len = strlen(_msg_buffer);                                                  \
+        snprintf(_msg_buffer + len, 2048 - len, __VA_ARGS__);                            \
+        throw std::runtime_error(_msg_buffer);                                           \
+    }
+
+#define OMNITRACE_STRINGIZE(...) #__VA_ARGS__
+#define OMNITRACE_ESC(...)       __VA_ARGS__
+
+//--------------------------------------------------------------------------------------//
+//
+//  Debug macros
+//
+//--------------------------------------------------------------------------------------//
+#define OMNITRACE_DEBUG(...)                                                             \
+    OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_debug(), __VA_ARGS__)
+
+#define OMNITRACE_BASIC_DEBUG(...)                                                       \
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(::omnitrace::get_debug_env(), __VA_ARGS__)
+
+#define OMNITRACE_DEBUG_F(...)                                                           \
+    OMNITRACE_CONDITIONAL_PRINT_F(::omnitrace::get_debug(), __VA_ARGS__)
+
+#define OMNITRACE_BASIC_DEBUG_F(...)                                                     \
+    OMNITRACE_CONDITIONAL_BASIC_PRINT_F(::omnitrace::get_debug_env(), __VA_ARGS__)
+
+#define OMNITRACE_CT_DEBUG(...)                                                          \
+    OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_critical_trace_debug(), __VA_ARGS__)
+
+#define OMNITRACE_CT_DEBUG_F(...)                                                        \
+    OMNITRACE_CONDITIONAL_PRINT_F(::omnitrace::get_critical_trace_debug(), __VA_ARGS__)
+
+//--------------------------------------------------------------------------------------//
+//
+//  Verbose macros
+//
+//--------------------------------------------------------------------------------------//
+#define OMNITRACE_VERBOSE(LEVEL, ...)                                                    \
+    OMNITRACE_CONDITIONAL_PRINT(                                                         \
+        ::omnitrace::get_debug() || ::omnitrace::get_verbose() >= LEVEL, __VA_ARGS__)
+
+#define OMNITRACE_BASIC_VERBOSE(LEVEL, ...)                                              \
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(::omnitrace::get_debug_env() ||                    \
+                                          ::omnitrace::get_verbose_env() >= LEVEL,       \
+                                      __VA_ARGS__)
+
+#define OMNITRACE_VERBOSE_F(LEVEL, ...)                                                  \
+    OMNITRACE_CONDITIONAL_PRINT_F(                                                       \
+        ::omnitrace::get_debug() || ::omnitrace::get_verbose() >= LEVEL, __VA_ARGS__)
+
+#define OMNITRACE_BASIC_VERBOSE_F(LEVEL, ...)                                            \
+    OMNITRACE_CONDITIONAL_BASIC_PRINT_F(::omnitrace::get_debug_env() ||                  \
+                                            ::omnitrace::get_verbose_env() >= LEVEL,     \
+                                        __VA_ARGS__)
+
+//--------------------------------------------------------------------------------------//
+//
+//  Basic print macros (basic means it will not provide PID/RANK or TID) and will not
+//  initialize the settings.
+//
+//--------------------------------------------------------------------------------------//
+
+#define OMNITRACE_BASIC_PRINT(...) OMNITRACE_CONDITIONAL_BASIC_PRINT(true, __VA_ARGS__)
+
+#define OMNITRACE_BASIC_PRINT_F(...) OMNITRACE_CONDITIONAL_BASIC_PRINT(true, __VA_ARGS__)
+
+//--------------------------------------------------------------------------------------//
+//
+//  Print macros. Will provide PID/RANK and TID (will initialize settings)
+//
+//--------------------------------------------------------------------------------------//
+
+#define OMNITRACE_PRINT(...) OMNITRACE_CONDITIONAL_PRINT(true, __VA_ARGS__)
+
+#define OMNITRACE_PRINT_F(...) OMNITRACE_CONDITIONAL_PRINT_F(true, __VA_ARGS__)
+
+//--------------------------------------------------------------------------------------//
+//
+//  Throw macros
+//
+//--------------------------------------------------------------------------------------//
+
+#define OMNITRACE_THROW(...) OMNITRACE_CONDITIONAL_THROW(true, __VA_ARGS__)
+
+#define OMNITRACE_BASIC_THROW(...) OMNITRACE_CONDITIONAL_BASIC_THROW(true, __VA_ARGS__)
+
+#include <string>
+
+namespace std
+{
+inline std::string
+to_string(bool _v)
+{
+    return (_v) ? "true" : "false";
+}
+}  // namespace std

source/lib/include/library/defines.hpp.in

@@ -0,0 +1,52 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+// clang-format off
+#define OMNITRACE_HIP_VERSION_STRING "@HIP_VERSION@"
+#define OMNITRACE_HIP_VERSION_MAJOR @HIP_VERSION_MAJOR@
+#define OMNITRACE_HIP_VERSION_MINOR @HIP_VERSION_MINOR@
+#define OMNITRACE_HIP_VERSION_PATCH @HIP_VERSION_PATCH@
+
+#if defined(OMNITRACE_USE_ROCTRACER)
+#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "@roctracer_kfdwrapper_LIBRARY@"
+#else
+#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "/opt/rocm/roctracer/lib/libkfdwrapper64.so"
+#endif
+// clang-format on
+
+#define TIMEMORY_USER_COMPONENT_ENUM                                                     \
+    OMNITRACE_COMPONENT_idx, OMNITRACE_ROCTRACER_idx, OMNITRACE_SAMPLING_WALL_CLOCK_idx, \
+        OMNITRACE_SAMPLING_CPU_CLOCK_idx, OMNITRACE_SAMPLING_PERCENT_idx,                \
+        OMNITRACE_SAMPLING_GPU_POWER_idx, OMNITRACE_SAMPLING_GPU_TEMP_idx,               \
+        OMNITRACE_SAMPLING_GPU_BUSY_idx, OMNITRACE_SAMPLING_GPU_MEMORY_USAGE_idx,
+
+#define OMNITRACE_COMPONENT                 OMNITRACE_COMPONENT_idx
+#define OMNITRACE_ROCTRACER                 OMNITRACE_ROCTRACER_idx
+#define OMNITRACE_SAMPLING_WALL_CLOCK       OMNITRACE_SAMPLING_WALL_CLOCK_idx
+#define OMNITRACE_SAMPLING_CPU_CLOCK        OMNITRACE_SAMPLING_CPU_CLOCK_idx
+#define OMNITRACE_SAMPLING_PERCENT          OMNITRACE_SAMPLING_PERCENT_idx
+#define OMNITRACE_SAMPLING_GPU_POWER        OMNITRACE_SAMPLING_GPU_POWER_idx
+#define OMNITRACE_SAMPLING_GPU_TEMP         OMNITRACE_SAMPLING_GPU_TEMP_idx
+#define OMNITRACE_SAMPLING_GPU_BUSY         OMNITRACE_SAMPLING_GPU_BUSY_idx
+#define OMNITRACE_SAMPLING_GPU_MEMORY_USAGE OMNITRACE_SAMPLING_GPU_MEMORY_USAGE_idx

source/lib/include/library/dynamic_library.hpp

@@ -0,0 +1,70 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/debug.hpp"
+#include "library/defines.hpp"
+
+#include <timemory/environment.hpp>
+
+#include <dlfcn.h>
+#include <string>
+
+namespace omnitrace
+{
+struct dynamic_library
+{
+    dynamic_library()                           = delete;
+    dynamic_library(const dynamic_library&)     = delete;
+    dynamic_library(dynamic_library&&) noexcept = default;
+    dynamic_library& operator=(const dynamic_library&) = delete;
+    dynamic_library& operator=(dynamic_library&&) noexcept = default;
+
+    dynamic_library(const char* _env, const char* _fname,
+                    int _flags = (RTLD_NOW | RTLD_GLOBAL), bool _store = false)
+    : envname{ _env }
+    , filename{ tim::get_env<std::string>(_env, _fname, _store) }
+    , flags{ _flags }
+    {
+        if(!filename.empty())
+        {
+            handle = dlopen(filename.c_str(), flags);
+            if(!handle)
+            {
+                OMNITRACE_DEBUG("%s\n", dlerror());
+            }
+            dlerror();  // Clear any existing error
+        }
+    }
+
+    ~dynamic_library()
+    {
+        if(handle) dlclose(handle);
+    }
+
+    std::string envname  = {};
+    std::string filename = {};
+    int         flags    = 0;
+    void*       handle   = nullptr;
+};
+}  // namespace omnitrace

source/lib/include/library/gpu.hpp

@@ -0,0 +1,32 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+namespace omnitrace
+{
+namespace gpu
+{
+int
+device_count();
+}
+}  // namespace omnitrace

source/lib/include/library/perfetto.hpp

@@ -0,0 +1,140 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+
+#if defined(PERFETTO_CATEGORIES)
+#    error "PERFETTO_CATEGORIES is already defined. Please include \"" __FILE__ "\" before including any timemory files"
+#endif
+
+#if !defined(TIMEMORY_USE_PERFETTO)
+#    include <perfetto.h>
+#    define PERFETTO_CATEGORIES                                                          \
+        perfetto::Category("host").SetDescription("Host-side function tracing"),         \
+            perfetto::Category("device").SetDescription("Device-side function tracing"), \
+            perfetto::Category("rocm_smi").SetDescription("Device-level metrics"),       \
+            perfetto::Category("sampling")                                               \
+                .SetDescription("Metrics derived from sampling"),                        \
+            perfetto::Category("critical-trace")                                         \
+                .SetDescription("Combined critical traces"),                             \
+            perfetto::Category("host-critical-trace")                                    \
+                .SetDescription("Host-side critical traces"),                            \
+            perfetto::Category("device-critical-trace")                                  \
+                .SetDescription("Device-side critical traces")
+#else
+#    define PERFETTO_CATEGORIES                                                          \
+        perfetto::Category("host").SetDescription("Host-side function tracing"),         \
+            perfetto::Category("device").SetDescription("Device-side function tracing"), \
+            perfetto::Category("rocm_smi").SetDescription("Device-level metrics"),       \
+            perfetto::Category("sampling")                                               \
+                .SetDescription("Metrics derived from sampling"),                        \
+            perfetto::Category("critical-trace")                                         \
+                .SetDescription("Combined critical traces"),                             \
+            perfetto::Category("host-critical-trace")                                    \
+                .SetDescription("Host-side critical traces"),                            \
+            perfetto::Category("device-critical-trace")                                  \
+                .SetDescription("Device-side critical traces"),                          \
+            perfetto::Category("timemory")                                               \
+                .SetDescription("Events from the timemory API")
+#    define TIMEMORY_PERFETTO_CATEGORIES PERFETTO_CATEGORIES
+#endif
+
+#if !defined(TIMEMORY_USE_PERFETTO)
+PERFETTO_DEFINE_CATEGORIES(PERFETTO_CATEGORIES);
+#endif
+
+namespace omnitrace
+{
+#if defined(CUSTOM_DATA_SOURCE)
+class CustomDataSource : public perfetto::DataSource<CustomDataSource>
+{
+public:
+    void OnSetup(const SetupArgs&) override
+    {
+        // Use this callback to apply any custom configuration to your data source
+        // based on the TraceConfig in SetupArgs.
+        PRINT_HERE("%s", "setup");
+    }
+
+    void OnStart(const StartArgs&) override
+    {
+        // This notification can be used to initialize the GPU driver, enable
+        // counters, etc. StartArgs will contains the DataSourceDescriptor,
+        // which can be extended.
+        PRINT_HERE("%s", "start");
+    }
+
+    void OnStop(const StopArgs&) override
+    {
+        // Undo any initialization done in OnStart.
+        PRINT_HERE("%s", "stop");
+    }
+
+    // Data sources can also have per-instance state.
+    int my_custom_state = 0;
+};
+
+PERFETTO_DECLARE_DATA_SOURCE_STATIC_MEMBERS(CustomDataSource);
+#endif
+
+template <typename Tp>
+struct perfetto_counter_track
+{
+    using track_map_t = std::map<uint32_t, std::vector<perfetto::CounterTrack>>;
+    using name_map_t  = std::map<uint32_t, std::vector<std::string>>;
+    using data_t      = std::pair<name_map_t, track_map_t>;
+
+    static auto init() { (void) get_data(); }
+
+    static auto exists(size_t _idx, int64_t _n = -1)
+    {
+        bool _v = get_data().second.count(_idx) != 0;
+        if(_n < 0 || !_v) return _v;
+        return static_cast<size_t>(_n) < get_data().second.at(_idx).size();
+    }
+
+    static size_t size(size_t _idx)
+    {
+        bool _v = get_data().second.count(_idx) != 0;
+        if(!_v) return 0;
+        return get_data().second.at(_idx).size();
+    }
+
+    static auto emplace(size_t _idx, const std::string& _v, const char* _units)
+    {
+        get_data().first[_idx].emplace_back(_v);
+        get_data().second[_idx].emplace_back(get_data().first[_idx].back().c_str(),
+                                             _units);
+    }
+
+    static auto& at(size_t _idx, size_t _n) { return get_data().second.at(_idx).at(_n); }
+
+private:
+    static data_t& get_data()
+    {
+        static auto* _v = new data_t{};
+        return *_v;
+    }
+};
+}  // namespace omnitrace

source/lib/include/library/ptl.hpp

@@ -0,0 +1,53 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+
+#include <PTL/PTL.hh>
+
+#include <mutex>
+
+namespace omnitrace
+{
+namespace tasking
+{
+std::mutex&
+get_roctracer_mutex();
+
+PTL::ThreadPool&
+get_roctracer_thread_pool();
+
+PTL::TaskGroup<void>&
+get_roctracer_task_group();
+
+std::mutex&
+get_critical_trace_mutex();
+
+PTL::ThreadPool&
+get_critical_trace_thread_pool();
+
+PTL::TaskGroup<void>&
+get_critical_trace_task_group();
+}  // namespace tasking
+}  // namespace omnitrace

source/lib/include/library/redirect.hpp

@@ -0,0 +1,95 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <iostream>
+#include <ostream>
+#include <sstream>
+#include <streambuf>
+#include <string>
+
+namespace omnitrace
+{
+inline namespace config
+{
+bool
+get_debug();
+
+int
+get_verbose();
+}  // namespace config
+
+struct redirect
+{
+    redirect(std::ostream& _os, std::string _expected)
+    : m_os{ _os }
+    , m_expected{ std::move(_expected) }
+    {
+        if(!get_debug())
+        {
+            // save stream buffer
+            m_strm_buffer = m_os.rdbuf();
+            // redirect to stringstream
+            _os.rdbuf(m_buffer.rdbuf());
+        }
+    }
+
+    ~redirect()
+    {
+        if(!m_strm_buffer) return;
+        // restore stream buffer
+        m_os.rdbuf(m_strm_buffer);
+        auto _v      = m_buffer.str();
+        _v           = replace(m_buffer.str(), '\n');
+        auto _expect = replace(m_expected, '\n');
+        if(_v != _expect)
+        {
+            if(get_verbose() > 0)
+                std::cerr << "[omnitrace::redirect] Expected:\n[omnitrace::redirect]    "
+                          << _expect
+                          << "\n[omnitrace::redirect] Found:\n[omnitrace::redirect]    "
+                          << _v << "\n";
+            if(get_verbose() <= 0 || (&m_os != &std::cerr && &m_os != &std::cout))
+                m_os << m_buffer.str() << std::flush;
+        }
+    }
+
+private:
+    template <typename Tp>
+    static std::string replace(std::string _v, Tp _c, const std::string& _s = " ")
+    {
+        while(true)
+        {
+            auto _pos = _v.find(_c);
+            if(_pos == std::string::npos) break;
+            _v = _v.replace(_pos, 1, _s);
+        }
+        return _v;
+    }
+
+    std::ostream&     m_os;
+    std::string       m_expected = {};
+    std::stringstream m_buffer{};
+    std::streambuf*   m_strm_buffer = nullptr;
+};
+}  // namespace omnitrace

source/lib/include/library/sampling.hpp

@@ -0,0 +1,86 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/components/backtrace.hpp"
+#include "library/components/fwd.hpp"
+#include "library/defines.hpp"
+#include "library/thread_data.hpp"
+#include "library/timemory.hpp"
+
+#include <timemory/macros/language.hpp>
+#include <timemory/sampling/sampler.hpp>
+#include <timemory/variadic/types.hpp>
+
+#include <cstdint>
+#include <memory>
+#include <set>
+
+namespace omnitrace
+{
+namespace sampling
+{
+using component::backtrace;
+using component::backtrace_cpu_clock;   // NOLINT
+using component::backtrace_fraction;    // NOLINT
+using component::backtrace_wall_clock;  // NOLINT
+using component::sampling_cpu_clock;
+using component::sampling_gpu_busy;
+using component::sampling_gpu_memory;
+using component::sampling_gpu_power;
+using component::sampling_gpu_temp;
+using component::sampling_percent;
+using component::sampling_wall_clock;
+
+std::unique_ptr<std::set<int>>&
+get_signal_types(int64_t _tid);
+
+std::set<int>
+setup();
+
+std::set<int>
+shutdown();
+
+void block_signals(std::set<int> = {});
+
+void unblock_signals(std::set<int> = {});
+
+using bundle_t          = tim::lightweight_tuple<backtrace>;
+using sampler_t         = tim::sampling::sampler<bundle_t, tim::sampling::dynamic>;
+using sampler_instances = thread_data<sampler_t, api::sampling>;
+
+std::unique_ptr<sampler_t>&
+get_sampler(int64_t _tid = threading::get_id());
+
+}  // namespace sampling
+}  // namespace omnitrace
+
+TIMEMORY_DEFINE_CONCRETE_TRAIT(prevent_reentry, omnitrace::sampling::sampler_t,
+                               std::true_type)
+
+TIMEMORY_DEFINE_CONCRETE_TRAIT(check_signals, omnitrace::sampling::sampler_t,
+                               std::true_type)
+
+TIMEMORY_DEFINE_CONCRETE_TRAIT(buffer_size, omnitrace::sampling::sampler_t,
+                               TIMEMORY_ESC(std::integral_constant<size_t, 256>))

source/lib/include/library/state.hpp

@@ -0,0 +1,55 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/defines.hpp"
+
+namespace omnitrace
+{
+// used for specifying the state of omnitrace
+enum class State : unsigned short
+{
+    DelayedInit = 0,
+    PreInit,
+    Init,
+    Active,
+    Finalized
+};
+
+enum class Mode : unsigned short
+{
+    Trace = 0,
+    Sampling
+};
+}  // namespace omnitrace
+
+#include <string>
+
+namespace std
+{
+std::string
+to_string(omnitrace::State _v);
+
+std::string
+to_string(omnitrace::Mode _v);
+}  // namespace std

source/lib/include/library/thread_data.hpp

@@ -0,0 +1,126 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/config.hpp"
+#include "library/defines.hpp"
+
+#include <array>
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <type_traits>
+
+#if !defined(OMNITRACE_MAX_THREADS)
+#    define OMNITRACE_MAX_THREADS 1024
+#endif
+
+namespace omnitrace
+{
+static constexpr size_t max_supported_threads = OMNITRACE_MAX_THREADS;
+
+template <typename Tp, typename Tag = void, size_t MaxThreads = max_supported_threads>
+struct thread_data
+{
+    using instance_array_t  = std::array<std::unique_ptr<Tp>, MaxThreads>;
+    using construct_on_init = std::true_type;
+
+    template <typename... Args>
+    static void                 construct(Args&&...);
+    static std::unique_ptr<Tp>& instance();
+    static instance_array_t&    instances();
+    template <typename... Args>
+    static std::unique_ptr<Tp>& instance(construct_on_init, Args&&...);
+    template <typename... Args>
+    static instance_array_t& instances(construct_on_init, Args&&...);
+};
+
+template <typename Tp, typename Tag, size_t MaxThreads>
+template <typename... Args>
+void
+thread_data<Tp, Tag, MaxThreads>::construct(Args&&... _args)
+{
+    // construct outside of lambda to prevent data-race
+    static auto&             _instances = instances();
+    static thread_local bool _v         = [&_args...]() {
+        _instances.at(threading::get_id()) =
+            std::make_unique<Tp>(std::forward<Args>(_args)...);
+        return true;
+    }();
+    (void) _v;
+}
+
+template <typename Tp, typename Tag, size_t MaxThreads>
+std::unique_ptr<Tp>&
+thread_data<Tp, Tag, MaxThreads>::instance()
+{
+    return instances().at(threading::get_id());
+}
+
+template <typename Tp, typename Tag, size_t MaxThreads>
+typename thread_data<Tp, Tag, MaxThreads>::instance_array_t&
+thread_data<Tp, Tag, MaxThreads>::instances()
+{
+    static auto _v = instance_array_t{};
+    return _v;
+}
+
+template <typename Tp, typename Tag, size_t MaxThreads>
+template <typename... Args>
+std::unique_ptr<Tp>&
+thread_data<Tp, Tag, MaxThreads>::instance(construct_on_init, Args&&... _args)
+{
+    construct(std::forward<Args>(_args)...);
+    return instances().at(threading::get_id());
+}
+
+template <typename Tp, typename Tag, size_t MaxThreads>
+template <typename... Args>
+typename thread_data<Tp, Tag, MaxThreads>::instance_array_t&
+thread_data<Tp, Tag, MaxThreads>::instances(construct_on_init, Args&&... _args)
+{
+    static auto _v = [&]() {
+        auto _internal = instance_array_t{};
+        for(size_t i = 0; i < MaxThreads; ++i)
+            _internal.at(i) = std::make_unique<Tp>(std::forward<Args>(_args)...);
+        return _internal;
+    }();
+    return _v;
+}
+
+//--------------------------------------------------------------------------------------//
+
+// there are currently some strange things that happen with
+// vector<instrumentation_bundle_t> so using vector<instrumentation_bundle_t*> and
+// timemory's ring_buffer_allocator to create contiguous memory-page aligned instances of
+// the bundle
+struct instrumentation_bundles
+{
+    using instance_array_t = std::array<instrumentation_bundles, max_supported_threads>;
+
+    bundle_allocator_t                     allocator{};
+    std::vector<instrumentation_bundle_t*> bundles{};
+
+    static instance_array_t& instances();
+};
+}  // namespace omnitrace

source/lib/include/library/thread_sampler.hpp

@@ -0,0 +1,103 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/common.hpp"
+#include "library/components/fwd.hpp"
+#include "library/defines.hpp"
+#include "library/state.hpp"
+#include "library/thread_data.hpp"
+
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <future>
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+namespace omnitrace
+{
+namespace thread_sampler
+{
+struct instance
+{
+    std::function<void()> setup        = []() {};
+    std::function<void()> shutdown     = []() {};
+    std::function<void()> config       = []() {};
+    std::function<void()> sample       = []() {};
+    std::function<void()> post_process = []() {};
+};
+//
+struct sampler
+{
+    using msec_t    = std::chrono::milliseconds;
+    using usec_t    = std::chrono::microseconds;
+    using nsec_t    = std::chrono::nanoseconds;
+    using promise_t = std::promise<void>;
+    using future_t  = std::future<void>;
+    using state_t   = State;
+
+    using timestamp_t = int64_t;
+
+    template <typename Tp                                                      = nsec_t,
+              std::enable_if_t<!std::is_same_v<std::decay_t<Tp>, nsec_t>, int> = 0>
+    static void poll(std::atomic<state_t>* _state, Tp&& _interval, promise_t*);
+
+    static void setup();
+    static void shutdown();
+    static void post_process();
+    static void set_state(state_t);
+    static void poll(std::atomic<state_t>* _state, nsec_t _interval, promise_t*);
+};
+//
+template <
+    typename Tp,
+    std::enable_if_t<!std::is_same_v<std::decay_t<Tp>, std::chrono::nanoseconds>, int>>
+void
+sampler::poll(std::atomic<state_t>* _state, Tp&& _interval, promise_t* _prom)
+{
+    poll(_state, std::chrono::duration_cast<nsec_t>(_interval), _prom);
+}
+//
+inline void
+setup()
+{
+    sampler::setup();
+}
+
+inline void
+shutdown()
+{
+    sampler::shutdown();
+}
+
+inline void
+post_process()
+{
+    sampler::post_process();
+}
+//
+}  // namespace thread_sampler
+}  // namespace omnitrace

source/lib/include/library/timemory.hpp

@@ -0,0 +1,60 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "library/components/fwd.hpp"
+#include "library/defines.hpp"
+
+#include <timemory/api.hpp>
+#include <timemory/backends/mpi.hpp>
+#include <timemory/backends/process.hpp>
+#include <timemory/backends/threading.hpp>
+#include <timemory/components.hpp>
+#include <timemory/components/gotcha/mpip.hpp>
+#include <timemory/config.hpp>
+#include <timemory/environment.hpp>
+#include <timemory/manager.hpp>
+#include <timemory/mpl.hpp>
+#include <timemory/operations.hpp>
+#include <timemory/runtime.hpp>
+#include <timemory/settings.hpp>
+#include <timemory/storage.hpp>
+#include <timemory/variadic.hpp>
+
+namespace omnitrace
+{
+namespace audit     = tim::audit;      // NOLINT
+namespace comp      = tim::component;  // NOLINT
+namespace quirk     = tim::quirk;      // NOLINT
+namespace threading = tim::threading;  // NOLINT
+namespace scope     = tim::scope;      // NOLINT
+namespace dmp       = tim::dmp;        // NOLINT
+namespace process   = tim::process;    // NOLINT
+namespace units     = tim::units;      // NOLINT
+namespace trait     = tim::trait;      // NOLINT
+
+// same sort of functionality as python's " ".join([...])
+#if !defined(JOIN)
+#    define JOIN(...) tim::mpl::apply<std::string>::join(__VA_ARGS__)
+#endif
+}  // namespace omnitrace

source/lib/src/library.cpp

@@ -0,0 +1,839 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library.hpp"
+#include "library/components/fork_gotcha.hpp"
+#include "library/components/mpi_gotcha.hpp"
+#include "library/config.hpp"
+#include "library/critical_trace.hpp"
+#include "library/debug.hpp"
+#include "library/defines.hpp"
+#include "library/gpu.hpp"
+#include "library/sampling.hpp"
+#include "library/thread_data.hpp"
+#include "library/thread_sampler.hpp"
+#include "library/timemory.hpp"
+#include "timemory/mpl/type_traits.hpp"
+
+#include <mutex>
+#include <string_view>
+
+using namespace omnitrace;
+
+namespace
+{
+std::vector<bool>&
+get_interval_data()
+{
+    static thread_local auto _v = std::vector<bool>{};
+    return _v;
+}
+
+auto
+ensure_finalization(bool _static_init = false)
+{
+    auto _main_tid = threading::get_id();
+    (void) _main_tid;
+    if(!_static_init)
+    {
+        OMNITRACE_DEBUG("[%s]\n", __FUNCTION__);
+    }
+    else
+    {
+        OMNITRACE_CONDITIONAL_PRINT(get_debug_env(), "[%s]\n", __FUNCTION__);
+        // This environment variable forces the ROCR-Runtime to use polling to wait
+        // for signals rather than interrupts. We set this variable to avoid issues with
+        // rocm/roctracer hanging when interrupted by the sampler
+        //
+        // see:
+        // https://github.com/ROCm-Developer-Tools/roctracer/issues/22#issuecomment-572814465
+        tim::set_env("HSA_ENABLE_INTERRUPT", "0", 0);
+    }
+    return scope::destructor{ []() { omnitrace_finalize(); } };
+}
+
+auto&
+get_trace_session()
+{
+    static std::unique_ptr<perfetto::TracingSession> _session{};
+    return _session;
+}
+
+auto
+is_system_backend()
+{
+    // if get_backend() returns 'system' or 'all', this is true
+    return (get_backend() != "inprocess");
+}
+
+auto&
+get_instrumentation_bundles()
+{
+    static thread_local auto& _v =
+        instrumentation_bundles::instances().at(threading::get_id());
+    return _v;
+}
+
+auto&
+get_functors()
+{
+    using functor_t = std::function<void(const char*)>;
+    static auto _v =
+        std::pair<functor_t, functor_t>{ [](const char*) {}, [](const char*) {} };
+    return _v;
+}
+
+auto&
+get_cpu_cid_parents()
+{
+    static thread_local auto _v =
+        std::unordered_map<uint64_t, std::tuple<uint64_t, uint16_t>>{};
+    return _v;
+}
+
+using Device = critical_trace::Device;
+using Phase  = critical_trace::Phase;
+}  // namespace
+
+//======================================================================================//
+///
+/// \fn void omnitrace_push_trace(const char* name)
+/// \brief the "start" function for an instrumentation region
+///
+//======================================================================================//
+
+extern "C" void
+omnitrace_push_trace(const char* name)
+{
+    // return if not active
+    if(get_state() == State::Finalized) return;
+
+    if(get_state() != State::Active && !omnitrace_init_tooling())
+    {
+        static auto _debug = get_debug_env();
+        OMNITRACE_CONDITIONAL_BASIC_PRINT_F(_debug, "%s :: not active. state = %s\n",
+                                            name, std::to_string(get_state()).c_str());
+        return;
+    }
+
+    OMNITRACE_DEBUG("[%s] %s\n", __FUNCTION__, name);
+
+    static auto _sample_rate = std::max<size_t>(get_instrumentation_interval(), 1);
+    static thread_local size_t _sample_idx = 0;
+    auto                       _enabled    = (_sample_idx++ % _sample_rate == 0);
+    get_interval_data().emplace_back(_enabled);
+    if(_enabled) get_functors().first(name);
+    if(get_use_critical_trace())
+    {
+        auto     _ts     = comp::wall_clock::record();
+        auto     _cid    = get_cpu_cid()++;
+        uint16_t _depth  = (get_cpu_cid_stack()->empty())
+                               ? get_cpu_cid_stack(0)->size()
+                               : get_cpu_cid_stack()->size() - 1;
+        auto _parent_cid = (get_cpu_cid_stack()->empty()) ? get_cpu_cid_stack(0)->back()
+                                                          : get_cpu_cid_stack()->back();
+        get_cpu_cid_parents().emplace(_cid, std::make_tuple(_parent_cid, _depth));
+        add_critical_trace<Device::CPU, Phase::BEGIN>(
+            threading::get_id(), _cid, 0, _parent_cid, _ts, 0,
+            critical_trace::add_hash_id(name), _depth);
+    }
+}
+
+//======================================================================================//
+///
+/// \fn void omnitrace_pop_trace(const char* name)
+/// \brief the "stop" function for an instrumentation region
+///
+//======================================================================================//
+
+extern "C" void
+omnitrace_pop_trace(const char* name)
+{
+    if(get_state() == State::Active)
+    {
+        OMNITRACE_DEBUG("[%s] %s\n", __FUNCTION__, name);
+        auto& _interval_data = get_interval_data();
+        if(!_interval_data.empty())
+        {
+            if(_interval_data.back()) get_functors().second(name);
+            _interval_data.pop_back();
+        }
+        if(get_use_critical_trace())
+        {
+            if(get_cpu_cid_stack() && !get_cpu_cid_stack()->empty())
+            {
+                auto _cid = get_cpu_cid_stack()->back();
+                if(get_cpu_cid_parents().find(_cid) != get_cpu_cid_parents().end())
+                {
+                    uint64_t _parent_cid          = 0;
+                    uint16_t _depth               = 0;
+                    auto     _ts                  = comp::wall_clock::record();
+                    std::tie(_parent_cid, _depth) = get_cpu_cid_parents().at(_cid);
+                    add_critical_trace<Device::CPU, Phase::END>(
+                        threading::get_id(), _cid, 0, _parent_cid, _ts, _ts,
+                        critical_trace::add_hash_id(name), _depth);
+                }
+            }
+        }
+    }
+    else
+    {
+        static auto _debug = get_debug_env();
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(_debug, "[%s] %s ignored :: not active\n",
+                                          __FUNCTION__, name);
+    }
+}
+
+//======================================================================================//
+///
+/// \fn void omnitrace_set_env(const char* name, const char* env_val)
+/// \brief Sets an initial environment variable
+///
+//======================================================================================//
+
+extern "C" void
+omnitrace_set_env(const char* env_name, const char* env_val)
+{
+    // just search env to avoid initializing the settings
+    OMNITRACE_CONDITIONAL_PRINT(get_debug_env() || get_verbose_env() > 2,
+                                "[%s] Setting env: %s=%s\n", __FUNCTION__, env_name,
+                                env_val);
+
+    tim::set_env(env_name, env_val, 0);
+
+    OMNITRACE_CONDITIONAL_THROW(
+        get_state() >= State::Init &&
+            (config::get_is_continuous_integration() || get_debug_env()),
+        "%s(\"%s\", \"%s\") called after omnitrace was initialized. state = %s",
+        __FUNCTION__, env_name, env_val, std::to_string(get_state()).c_str());
+}
+
+//======================================================================================//
+///
+/// \fn void omnitrace_set_mpi(bool use, bool attached)
+/// \brief Configures whether MPI support should be activated
+///
+//======================================================================================//
+
+namespace
+{
+bool                  _set_mpi_called        = false;
+std::function<void()> _start_gotcha_callback = []() {};
+}  // namespace
+
+extern "C" void
+omnitrace_set_mpi(bool use, bool attached)
+{
+    // just search env to avoid initializing the settings
+    OMNITRACE_CONDITIONAL_PRINT(get_debug_env() || get_verbose_env() > 2,
+                                "[%s] use: %s, attached: %s\n", __FUNCTION__,
+                                (use) ? "y" : "n", (attached) ? "y" : "n");
+
+    _set_mpi_called       = true;
+    config::is_attached() = attached;
+
+    if(use && !attached &&
+       (get_state() == State::PreInit || get_state() == State::DelayedInit))
+    {
+        tim::set_env("OMNITRACE_USE_PID", "ON", 1);
+    }
+    else if(!use)
+    {
+        trait::runtime_enabled<mpi_gotcha_t>::set(false);
+    }
+
+    OMNITRACE_CONDITIONAL_THROW(
+        get_state() >= State::Init &&
+            (config::get_is_continuous_integration() || get_debug_env()),
+        "%s(use=%s, attached=%s) called after omnitrace was initialized. state = %s",
+        __FUNCTION__, std::to_string(use).c_str(), std::to_string(attached).c_str(),
+        std::to_string(get_state()).c_str());
+
+    _start_gotcha_callback();
+}
+
+//======================================================================================//
+
+extern "C" void
+omnitrace_init_library()
+{
+    auto _tid = threading::get_id();
+    (void) _tid;
+
+    auto _mode = get_mode();
+
+    get_state() = State::Init;
+
+    // configure the settings
+    configure_settings();
+
+    auto _debug_init  = get_debug_init();
+    auto _debug_value = get_debug();
+    if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", true);
+    scope::destructor _debug_dtor{ [_debug_value, _debug_init]() {
+        if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", _debug_value);
+    } };
+
+    OMNITRACE_DEBUG("[%s]\n", __FUNCTION__);
+
+    // below will effectively do:
+    //      get_cpu_cid_stack(0)->emplace_back(-1);
+    // plus query some env variables
+    add_critical_trace<Device::CPU, Phase::NONE>(0, -1, 0, 0, 0, 0, 0, 0);
+
+    if(gpu::device_count() == 0 && get_state() != State::Active)
+    {
+        OMNITRACE_DEBUG(
+            "No HIP devices were found: disabling roctracer and rocm_smi...\n");
+        get_use_roctracer() = false;
+        get_use_rocm_smi()  = false;
+    }
+
+    if(_mode == Mode::Sampling)
+    {
+        OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0,
+                                    "Disabling critical trace in %s mode...\n",
+                                    std::to_string(_mode).c_str());
+        get_use_sampling()       = true;
+        get_use_critical_trace() = false;
+    }
+
+    tim::trait::runtime_enabled<comp::roctracer>::set(get_use_roctracer());
+    tim::trait::runtime_enabled<comp::roctracer_data>::set(get_use_roctracer());
+
+    if(get_instrumentation_interval() < 1) get_instrumentation_interval() = 1;
+    get_interval_data().reserve(512);
+
+    if(get_use_kokkosp())
+    {
+        auto _force = 0;
+        if(tim::get_env<std::string>("KOKKOS_PROFILE_LIBRARY") == "libtimemory.so")
+            _force = 1;
+        tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force);
+    }
+
+#if defined(OMNITRACE_USE_ROCTRACER)
+    tim::set_env("HSA_TOOLS_LIB", "libomnitrace.so", 0);
+#endif
+}
+
+//======================================================================================//
+
+extern "C" bool
+omnitrace_init_tooling()
+{
+    static bool _once = false;
+    if(get_state() != State::PreInit || get_state() == State::Init || _once) return false;
+    _once = true;
+
+    OMNITRACE_CONDITIONAL_THROW(
+        get_state() == State::Init,
+        "%s called after omnitrace_init_library() was explicitly called", __FUNCTION__);
+
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_verbose_env() >= 0,
+                                      "Instrumentation mode: %s\n",
+                                      std::to_string(config::get_mode()).c_str());
+
+    if(get_verbose_env() >= 0) print_banner();
+
+    omnitrace_init_library();
+
+    OMNITRACE_DEBUG("[%s]\n", __FUNCTION__);
+
+    auto _dtor = scope::destructor{ []() {
+        if(get_use_sampling())
+        {
+            pthread_gotcha::enable_sampling_on_child_threads() = false;
+            thread_sampler::setup();
+            sampling::setup();
+            pthread_gotcha::enable_sampling_on_child_threads() = true;
+            sampling::unblock_signals();
+        }
+        get_main_bundle()->start();
+        get_state()= State::Active;  // set to active as very last operation
+    } };
+
+    if(get_use_sampling())
+    {
+        pthread_gotcha::enable_sampling_on_child_threads() = false;
+        sampling::block_signals();
+    }
+
+    if(!get_use_timemory() && !get_use_perfetto() && !get_use_sampling() &&
+       !get_use_rocm_smi())
+    {
+        get_state() = State::Finalized;
+        OMNITRACE_DEBUG("[%s] Both perfetto and timemory are disabled. Setting the state "
+                        "to finalized\n",
+                        __FUNCTION__);
+        return false;
+    }
+
+    if(get_use_timemory())
+    {
+        comp::user_global_bundle::global_init();
+        std::set<int> _comps{};
+        // convert string into set of enumerations
+        for(auto&& itr : tim::delimit(tim::settings::global_components()))
+            _comps.emplace(tim::runtime::enumerate(itr));
+        if(_comps.size() == 1 && _comps.find(TIMEMORY_WALL_CLOCK) != _comps.end())
+        {
+            // using wall_clock directly is lower overhead than using it via user_bundle
+            instrumentation_bundle_t::get_initializer() =
+                [](instrumentation_bundle_t& _bundle) {
+                    _bundle.initialize<comp::wall_clock>();
+                };
+        }
+        else if(!_comps.empty())
+        {
+            // use user_bundle for other than wall-clock
+            instrumentation_bundle_t::get_initializer() =
+                [](instrumentation_bundle_t& _bundle) {
+                    _bundle.initialize<comp::user_global_bundle>();
+                };
+        }
+        else
+        {
+            tim::trait::runtime_enabled<api::omnitrace>::set(false);
+        }
+    }
+
+    perfetto::TracingInitArgs               args{};
+    perfetto::TraceConfig                   cfg{};
+    perfetto::protos::gen::TrackEventConfig track_event_cfg{};
+
+    // perfetto initialization
+    if(get_use_perfetto())
+    {
+        // environment settings
+        auto shmem_size_hint = get_perfetto_shmem_size_hint();
+        auto buffer_size     = get_perfetto_buffer_size();
+
+        auto* buffer_config = cfg.add_buffers();
+        buffer_config->set_size_kb(buffer_size);
+        buffer_config->set_fill_policy(
+            perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_DISCARD);
+
+        auto* ds_cfg = cfg.add_data_sources()->mutable_config();
+        ds_cfg->set_name("track_event");
+        ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString());
+
+        args.shmem_size_hint_kb = shmem_size_hint;
+
+        if(get_backend() != "inprocess") args.backends |= perfetto::kSystemBackend;
+        if(get_backend() != "system") args.backends |= perfetto::kInProcessBackend;
+
+        perfetto::Tracing::Initialize(args);
+        perfetto::TrackEvent::Register();
+
+        (void) get_perfetto_output_filename();
+    }
+
+    auto        _exe         = get_exe_name();
+    static auto _thread_init = [_exe]() {
+        static thread_local auto _thread_setup = [_exe]() {
+            if(threading::get_id() > 0)
+                threading::set_thread_name(
+                    TIMEMORY_JOIN(" ", "Thread", threading::get_id()).c_str());
+            thread_data<omnitrace_thread_bundle_t>::construct(
+                TIMEMORY_JOIN("", _exe, "/thread-", threading::get_id()),
+                quirk::config<quirk::auto_start>{});
+            if(get_use_sampling()) sampling::setup();
+        };
+        static thread_local auto _once = std::once_flag{};
+        std::call_once(_once, _thread_setup);
+        static thread_local auto _dtor = scope::destructor{ []() {
+            if(get_use_sampling()) sampling::shutdown();
+            thread_data<omnitrace_thread_bundle_t>::instance()->stop();
+        } };
+        (void) _dtor;
+    };
+
+    // functors for starting and stopping timemory
+    static auto _push_timemory = [](const char* name) {
+        _thread_init();
+        auto& _data = get_instrumentation_bundles();
+        // this generates a hash for the raw string array
+        auto  _hash   = tim::add_hash_id(tim::string_view_t{ name });
+        auto* _bundle = _data.allocator.allocate(1);
+        _data.bundles.emplace_back(_bundle);
+        _data.allocator.construct(_bundle, _hash);
+        _bundle->start();
+    };
+
+    static auto _push_perfetto = [](const char* name) {
+        _thread_init();
+        uint64_t _ts = comp::wall_clock::record();
+        TRACE_EVENT_BEGIN("host", perfetto::StaticString(name), _ts);
+    };
+
+    static auto _pop_timemory = [](const char* name) {
+        auto& _data = get_instrumentation_bundles();
+        if(_data.bundles.empty())
+        {
+            OMNITRACE_DEBUG("[%s] skipped %s :: empty bundle stack\n",
+                            "omnitrace_pop_trace", name);
+            return;
+        }
+        _data.bundles.back()->stop();
+        _data.allocator.destroy(_data.bundles.back());
+        _data.allocator.deallocate(_data.bundles.back(), 1);
+        _data.bundles.pop_back();
+    };
+
+    static auto _pop_perfetto = [](const char*) {
+        uint64_t _ts = comp::wall_clock::record();
+        TRACE_EVENT_END("host", _ts);
+    };
+
+    if(get_use_perfetto() && get_use_timemory())
+    {
+        get_functors().first = [](const char* name) {
+            _push_perfetto(name);
+            _push_timemory(name);
+        };
+        get_functors().second = [](const char* name) {
+            _pop_timemory(name);
+            _pop_perfetto(name);
+        };
+    }
+    else if(get_use_perfetto())
+    {
+        get_functors().first  = _push_perfetto;
+        get_functors().second = _pop_perfetto;
+    }
+    else if(get_use_timemory())
+    {
+        get_functors().first  = _push_timemory;
+        get_functors().second = _pop_timemory;
+    }
+
+    if(get_use_perfetto() && !is_system_backend())
+    {
+#if defined(CUSTOM_DATA_SOURCE)
+        // Add the following:
+        perfetto::DataSourceDescriptor dsd{};
+        dsd.set_name("com.example.custom_data_source");
+        CustomDataSource::Register(dsd);
+        auto* ds_cfg = cfg.add_data_sources()->mutable_config();
+        ds_cfg->set_name("com.example.custom_data_source");
+        CustomDataSource::Trace([](CustomDataSource::TraceContext ctx) {
+            auto packet = ctx.NewTracePacket();
+            packet->set_timestamp(perfetto::TrackEvent::GetTraceTimeNs());
+            packet->set_for_testing()->set_str("Hello world!");
+            PRINT_HERE("%s", "Trace");
+        });
+#endif
+        auto& tracing_session = get_trace_session();
+        tracing_session       = perfetto::Tracing::NewTrace();
+        tracing_session->Setup(cfg);
+        tracing_session->StartBlocking();
+    }
+
+    // if static objects are destroyed in the inverse order of when they are
+    // created this should ensure that finalization is called before perfetto
+    // ends the tracing session
+    static auto _ensure_finalization = ensure_finalization();
+
+    if(dmp::rank() == 0 && get_verbose() >= 0) fprintf(stderr, "\n");
+
+    return true;
+}
+
+//======================================================================================//
+
+extern "C" void
+omnitrace_init(const char* _mode, bool _is_binary_rewrite, const char* _argv0)
+{
+    // always the first
+    std::atexit(&omnitrace_finalize);
+
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(
+        get_debug_env() || get_verbose_env() > 2,
+        "[%s] mode: %s | is binary rewrite: %s | command: %s\n", __FUNCTION__, _mode,
+        (_is_binary_rewrite) ? "y" : "n", _argv0);
+
+    tim::set_env("OMNITRACE_MODE", _mode, 0);
+    config::is_binary_rewrite() = _is_binary_rewrite;
+
+    // default to KokkosP enabled when sampling, otherwise default to off
+    tim::set_env("OMNITRACE_USE_KOKKOSP", (get_mode() == Mode::Sampling) ? "ON" : "OFF",
+                 0);
+
+    if(!_set_mpi_called)
+    {
+        _start_gotcha_callback = []() { get_gotcha_bundle()->start(); };
+    }
+    else
+    {
+        get_gotcha_bundle()->start();
+    }
+}
+
+//======================================================================================//
+
+extern "C" void
+omnitrace_finalize(void)
+{
+    // return if not active
+    if(get_state() != State::Active)
+    {
+        OMNITRACE_DEBUG_F("State = %s. Finalization skipped\n",
+                          std::to_string(get_state()).c_str());
+        return;
+    }
+
+    pthread_gotcha::enable_sampling_on_child_threads() = false;
+
+    auto _debug_init  = get_debug_finalize();
+    auto _debug_value = get_debug();
+    if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", true);
+    scope::destructor _debug_dtor{ [_debug_value, _debug_init]() {
+        if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", _debug_value);
+    } };
+
+    OMNITRACE_DEBUG("[%s]\n", __FUNCTION__);
+
+    auto& _thread_bundle = thread_data<omnitrace_thread_bundle_t>::instance();
+    if(_thread_bundle) _thread_bundle->stop();
+
+    if(dmp::rank() == 0 && get_verbose() >= 0) fprintf(stderr, "\n");
+    if(get_verbose_env() > 0) config::print_settings();
+
+    get_state() = State::Finalized;
+
+    if(get_use_sampling())
+    {
+        OMNITRACE_DEBUG("[%s] Shutting down sampling...\n", __FUNCTION__);
+        sampling::shutdown();
+        sampling::block_signals();
+    }
+
+    OMNITRACE_DEBUG("[%s] Stopping gotcha bundle...\n", __FUNCTION__);
+    // stop the gotcha bundle
+    if(get_gotcha_bundle())
+    {
+        get_gotcha_bundle()->stop();
+        get_gotcha_bundle().reset();
+    }
+
+    pthread_gotcha::shutdown();
+    thread_sampler::shutdown();
+
+    OMNITRACE_DEBUG("[%s] Shutting down roctracer...\n", __FUNCTION__);
+    // ensure that threads running roctracer callbacks shutdown
+    comp::roctracer::shutdown();
+
+    if(dmp::rank() == 0) fprintf(stderr, "\n");
+
+    OMNITRACE_DEBUG("[%s] Stopping main bundle...\n", __FUNCTION__);
+    // stop the main bundle and report the high-level metrics
+    if(get_main_bundle())
+    {
+        get_main_bundle()->stop();
+        std::string _msg = JOIN("", *get_main_bundle());
+        auto        _pos = _msg.find(">>>  ");
+        if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
+        OMNITRACE_PRINT("%s\n", _msg.c_str());
+        get_main_bundle().reset();
+    }
+
+    int _threadpool_verbose = (get_debug()) ? 4 : -1;
+    tasking::get_roctracer_thread_pool().set_verbose(_threadpool_verbose);
+    tasking::get_critical_trace_thread_pool().set_verbose(_threadpool_verbose);
+
+    // join extra thread(s) used by roctracer
+    OMNITRACE_DEBUG("[%s] waiting for all roctracer tasks to complete...\n",
+                    __FUNCTION__);
+    tasking::get_roctracer_task_group().join();
+
+    // print out thread-data if they are not still running
+    // if they are still running (e.g. thread-pool still alive), the
+    // thread-specific data will be wrong if try to stop them from
+    // the main thread.
+    OMNITRACE_DEBUG("[%s] Destroying thread bundle data...\n", __FUNCTION__);
+    for(auto& itr : thread_data<omnitrace_thread_bundle_t>::instances())
+    {
+        if(itr && itr->get<comp::wall_clock>() &&
+           !itr->get<comp::wall_clock>()->get_is_running())
+        {
+            std::string _msg = JOIN("", *itr);
+            auto        _pos = _msg.find(">>>  ");
+            if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
+            OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, "%s\n", _msg.c_str());
+        }
+    }
+
+    // ensure that all the MT instances are flushed
+    OMNITRACE_DEBUG("[%s] Stopping and destroying instrumentation bundles...\n",
+                    __FUNCTION__);
+    for(auto& itr : instrumentation_bundles::instances())
+    {
+        while(!itr.bundles.empty())
+        {
+            itr.bundles.back()->stop();
+            itr.bundles.back()->pop();
+            itr.allocator.destroy(itr.bundles.back());
+            itr.allocator.deallocate(itr.bundles.back(), 1);
+            itr.bundles.pop_back();
+        }
+    }
+
+    // ensure that all the MT instances are flushed
+    if(get_use_sampling())
+    {
+        OMNITRACE_DEBUG("[%s] Post-processing the sampling backtraces...\n",
+                        __FUNCTION__);
+        for(size_t i = 0; i < max_supported_threads; ++i)
+        {
+            sampling::backtrace::post_process(i);
+            sampling::get_sampler(i).reset();
+        }
+    }
+
+    if(get_use_critical_trace() || (get_use_rocm_smi() && get_use_roctracer()))
+    {
+        OMNITRACE_DEBUG("[%s] Generating the critical trace...\n", __FUNCTION__);
+        // increase the thread-pool size
+        tasking::get_critical_trace_thread_pool().initialize_threadpool(
+            get_critical_trace_num_threads());
+
+        for(size_t i = 0; i < max_supported_threads; ++i)
+        {
+            using critical_trace_hash_data =
+                thread_data<critical_trace::hash_ids, critical_trace::id>;
+
+            if(critical_trace_hash_data::instances().at(i))
+                critical_trace::add_hash_id(*critical_trace_hash_data::instances().at(i));
+        }
+
+        for(size_t i = 0; i < max_supported_threads; ++i)
+        {
+            using critical_trace_chain_data = thread_data<critical_trace::call_chain>;
+
+            if(critical_trace_chain_data::instances().at(i))
+                critical_trace::update(i);  // launch update task
+        }
+    }
+
+    thread_sampler::post_process();
+
+    if(get_use_critical_trace())
+    {
+        OMNITRACE_DEBUG("[%s] Generating the critical trace...\n", __FUNCTION__);
+        // increase the thread-pool size
+        tasking::get_critical_trace_thread_pool().initialize_threadpool(
+            get_critical_trace_num_threads());
+
+        // make sure outstanding hash tasks completed before compute
+        OMNITRACE_PRINT("[%s] waiting for all critical trace tasks to complete...\n",
+                        __FUNCTION__);
+        tasking::get_critical_trace_task_group().join();
+
+        // launch compute task
+        OMNITRACE_PRINT("[%s] launching critical trace compute task...\n", __FUNCTION__);
+        critical_trace::compute();
+    }
+
+    tasking::get_critical_trace_task_group().join();
+
+    bool _perfetto_output_error = false;
+    if(get_use_perfetto() && !is_system_backend())
+    {
+        if(get_verbose() >= 0) fprintf(stderr, "\n");
+        if(get_verbose() >= 0 || get_debug())
+            fprintf(stderr, "[%s]|%i> Flushing perfetto...\n", __FUNCTION__, dmp::rank());
+
+        // Make sure the last event is closed for this example.
+        perfetto::TrackEvent::Flush();
+
+        auto& tracing_session = get_trace_session();
+        OMNITRACE_DEBUG("[%s] Stopping the blocking perfetto trace sessions...\n",
+                        __FUNCTION__);
+        tracing_session->StopBlocking();
+
+        OMNITRACE_DEBUG("[%s] Getting the trace data...\n", __FUNCTION__);
+        std::vector<char> trace_data{ tracing_session->ReadTraceBlocking() };
+
+        if(trace_data.empty())
+        {
+            fprintf(stderr,
+                    "[%s]> trace data is empty. File '%s' will not be written...\n",
+                    __FUNCTION__, get_perfetto_output_filename().c_str());
+            return;
+        }
+        // Write the trace into a file.
+        if(get_verbose() >= 0)
+            fprintf(stderr, "[%s]|%i> Outputting '%s' (%.2f KB / %.2f MB / %.2f GB)... ",
+                    __FUNCTION__, dmp::rank(), get_perfetto_output_filename().c_str(),
+                    static_cast<double>(trace_data.size()) / units::KB,
+                    static_cast<double>(trace_data.size()) / units::MB,
+                    static_cast<double>(trace_data.size()) / units::GB);
+        std::ofstream ofs{};
+        if(!tim::filepath::open(ofs, get_perfetto_output_filename(),
+                                std::ios::out | std::ios::binary))
+        {
+            fprintf(stderr, "\n[%s]> Error opening '%s'...\n", __FUNCTION__,
+                    get_perfetto_output_filename().c_str());
+            _perfetto_output_error = true;
+        }
+        else
+        {
+            // Write the trace into a file.
+            if(get_verbose() >= 0) fprintf(stderr, "Done\n");
+            ofs.write(&trace_data[0], trace_data.size());
+        }
+        ofs.close();
+        if(get_verbose() >= 0) fprintf(stderr, "\n");
+    }
+
+    // these should be destroyed before timemory is finalized, especially the
+    // roctracer thread-pool
+    OMNITRACE_DEBUG("[%s] Destroying the thread pools...\n", __FUNCTION__);
+    tasking::get_roctracer_thread_pool().destroy_threadpool();
+    tasking::get_critical_trace_thread_pool().destroy_threadpool();
+
+    if(get_use_sampling())
+        static_cast<tim::tsettings<bool>*>(
+            tim::settings::instance()->find("OMNITRACE_DEBUG")->second.get())
+            ->set(false);
+
+    OMNITRACE_DEBUG("[%s] Finalizing timemory...\n", __FUNCTION__);
+    tim::timemory_finalize();
+    OMNITRACE_DEBUG("[%s] Finalizing timemory... Done\n", __FUNCTION__);
+
+    if(_perfetto_output_error)
+    {
+        OMNITRACE_THROW("Error opening perfetto output file: %s",
+                        get_perfetto_output_filename().c_str());
+    }
+}
+
+//======================================================================================//
+
+namespace
+{
+// if static objects are destroyed randomly (relatively uncommon behavior)
+// this might call finalization before perfetto ends the tracing session
+// but static variable in omnitrace_init_tooling is more likely
+auto _ensure_finalization = ensure_finalization(true);
+}  // namespace

source/lib/src/library/components/backtrace.cpp

@@ -0,0 +1,724 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/fwd.hpp"
+#include "library/components/rocm_smi.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/perfetto.hpp"
+#include "library/ptl.hpp"
+#include "library/sampling.hpp"
+
+#include <timemory/backends/papi.hpp>
+#include <timemory/backends/threading.hpp>
+#include <timemory/components/data_tracker/components.hpp>
+#include <timemory/components/macros.hpp>
+#include <timemory/components/papi/extern.hpp>
+#include <timemory/components/papi/papi_array.hpp>
+#include <timemory/components/papi/papi_vector.hpp>
+#include <timemory/components/timing/backends.hpp>
+#include <timemory/components/trip_count/extern.hpp>
+#include <timemory/macros.hpp>
+#include <timemory/math.hpp>
+#include <timemory/mpl.hpp>
+#include <timemory/mpl/quirks.hpp>
+#include <timemory/mpl/type_traits.hpp>
+#include <timemory/operations.hpp>
+#include <timemory/sampling/allocator.hpp>
+#include <timemory/sampling/sampler.hpp>
+#include <timemory/storage.hpp>
+#include <timemory/units.hpp>
+#include <timemory/utility/backtrace.hpp>
+#include <timemory/utility/demangle.hpp>
+#include <timemory/utility/types.hpp>
+#include <timemory/variadic.hpp>
+
+#include <array>
+#include <cstring>
+#include <ctime>
+#include <initializer_list>
+#include <mutex>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+#include <pthread.h>
+#include <signal.h>
+
+namespace
+{
+template <typename... Tp>
+struct ensure_storage
+{
+    TIMEMORY_DEFAULT_OBJECT(ensure_storage)
+
+    void operator()() const { TIMEMORY_FOLD_EXPRESSION((*this)(tim::type_list<Tp>{})); }
+
+private:
+    template <typename Up, std::enable_if_t<tim::trait::is_available<Up>::value, int> = 0>
+    void operator()(tim::type_list<Up>) const
+    {
+        using namespace tim;
+        static thread_local auto _storage = operation::get_storage<Up>{}();
+        static thread_local auto _tid     = threading::get_id();
+        static thread_local auto _dtor =
+            scope::destructor{ []() { operation::set_storage<Up>{}(nullptr, _tid); } };
+
+        tim::operation::set_storage<Up>{}(_storage, _tid);
+        if(_tid == 0 && !_storage) tim::trait::runtime_enabled<Up>::set(false);
+    }
+
+    template <typename Up,
+              std::enable_if_t<!tim::trait::is_available<Up>::value, long> = 0>
+    void operator()(tim::type_list<Up>) const
+    {
+        tim::trait::runtime_enabled<Up>::set(false);
+    }
+};
+}  // namespace
+
+namespace omnitrace
+{
+namespace component
+{
+using hw_counters               = typename backtrace::hw_counters;
+using signal_type_instances     = thread_data<std::set<int>, api::sampling>;
+using backtrace_init_instances  = thread_data<backtrace, api::sampling>;
+using sampler_running_instances = thread_data<bool, api::sampling>;
+using papi_vector_instances     = thread_data<hw_counters, api::sampling>;
+
+namespace
+{
+std::unique_ptr<hw_counters>&
+get_papi_vector(int64_t _tid)
+{
+    static auto& _v = papi_vector_instances::instances();
+    if(_tid == threading::get_id()) papi_vector_instances::construct();
+    return _v.at(_tid);
+}
+
+std::unique_ptr<backtrace>&
+get_backtrace_init(int64_t _tid)
+{
+    static auto& _v = backtrace_init_instances::instances();
+    return _v.at(_tid);
+}
+
+std::unique_ptr<bool>&
+get_sampler_running(int64_t _tid)
+{
+    static auto& _v = sampler_running_instances::instances();
+    return _v.at(_tid);
+}
+}  // namespace
+
+bool
+backtrace::operator<(const backtrace& rhs) const
+{
+    return (m_ts == rhs.m_ts) ? (m_tid < rhs.m_tid) : (m_ts < rhs.m_ts);
+}
+
+std::vector<std::string>
+backtrace::get() const
+{
+    std::vector<std::string> _v{};
+    _v.reserve(m_size);
+    for(size_t i = 0; i < m_size; ++i)
+        _v.emplace_back(m_data.at(i));
+    return _v;
+}
+
+void
+backtrace::preinit()
+{
+    sampling_wall_clock::label()       = "sampling_wall_clock";
+    sampling_wall_clock::description() = "Wall clock time (via sampling)";
+
+    sampling_cpu_clock::label()       = "sampling_cpu_clock";
+    sampling_cpu_clock::description() = "CPU clock time (via sampling)";
+
+    sampling_percent::label()       = "sampling_percent";
+    sampling_percent::description() = "Percentage of samples";
+
+    sampling_gpu_busy::label()       = "sampling_gpu_busy_percent";
+    sampling_gpu_busy::description() = "Utilization of GPU(s)";
+    sampling_gpu_busy::set_precision(0);
+    sampling_gpu_busy::set_format_flags(sampling_gpu_busy::get_format_flags() &
+                                        std::ios_base::showpoint);
+
+    sampling_gpu_memory::label()       = "sampling_gpu_memory_usage";
+    sampling_gpu_memory::description() = "Memory usage of GPU(s)";
+
+    sampling_gpu_power::label()        = "sampling_gpu_power";
+    sampling_gpu_power::description()  = "Power usage of GPU(s)";
+    sampling_gpu_power::unit()         = units::watt;
+    sampling_gpu_power::display_unit() = "watts";
+    sampling_gpu_power::set_precision(2);
+    sampling_gpu_power::set_format_flags(sampling_gpu_power::get_format_flags());
+
+    sampling_gpu_temp::label()        = "sampling_gpu_temperature";
+    sampling_gpu_temp::description()  = "Temperature of GPU(s)";
+    sampling_gpu_temp::unit()         = 1;
+    sampling_gpu_temp::display_unit() = "degC";
+    sampling_gpu_temp::set_precision(1);
+    sampling_gpu_temp::set_format_flags(sampling_gpu_temp::get_format_flags());
+}
+
+std::string
+backtrace::label()
+{
+    return "backtrace";
+}
+
+std::string
+backtrace::description()
+{
+    return "Records backtrace data";
+}
+
+void
+backtrace::start()
+{}
+
+void
+backtrace::stop()
+{}
+
+bool
+backtrace::empty() const
+{
+    return (m_size == 0);
+}
+
+size_t
+backtrace::size() const
+{
+    return m_size;
+}
+
+backtrace::time_point_type
+backtrace::get_timestamp() const
+{
+    return m_ts;
+}
+
+int64_t
+backtrace::get_thread_cpu_timestamp() const
+{
+    return m_thr_cpu_ts;
+}
+
+void
+backtrace::sample(int signum)
+{
+    static bool _debug = tim::get_env<bool>("OMNITRACE_DEBUG_SAMPLING", get_debug());
+    if(_debug)
+    {
+        static auto _timestamp_str = [](const auto& _tp) {
+            char _repr[64];
+            std::memset(_repr, '\0', sizeof(_repr));
+            std::time_t _value = system_clock::to_time_t(_tp);
+            // alternative: "%c %Z"
+            if(std::strftime(_repr, sizeof(_repr), "%a %b %d %T %Y %Z",
+                             std::localtime(&_value)) > 0)
+                return std::string{ _repr };
+            return std::string{};
+        };
+
+        static thread_local size_t _tot  = 0;
+        static thread_local auto   _last = system_clock::now();
+        auto                       _now  = system_clock::now();
+        auto                       _diff = (_now - _last).count();
+        _last                            = _now;
+        _tot += _diff;
+
+        OMNITRACE_PRINT(
+            "Sample on signal %i taken at %s after interval %zu :: total %zu\n", signum,
+            _timestamp_str(_now).c_str(), _diff, _tot);
+    }
+
+    m_size       = 0;
+    m_tid        = threading::get_id();
+    m_ts         = clock_type::now();
+    m_thr_cpu_ts = tim::get_clock_thread_now<int64_t, std::nano>();
+    m_mem_peak   = tim::get_peak_rss(RUSAGE_THREAD);
+    m_data       = tim::get_unw_backtrace<128, 4, false>();
+    auto* itr    = m_data.begin();
+    for(; itr != m_data.end(); ++itr, ++m_size)
+    {
+        if(strlen(*itr) == 0) break;
+    }
+    std::reverse(m_data.begin(), itr);
+    if(!get_debug())
+    {
+        bool _ignore = false;
+        for(auto& itr : m_data)
+        {
+            if(strlen(itr) == 0) break;
+            if(strncmp(itr, "funlockfile", 11) == 0) _ignore = true;
+            if(_ignore && strlen(itr) > 0)
+            {
+                OMNITRACE_DEBUG("Discarding sample: '%s'...\n", itr);
+                itr[0] = '\0';
+                --m_size;
+            }
+        }
+    }
+
+    if constexpr(tim::trait::is_available<hw_counters>::value)
+    {
+        if(tim::trait::runtime_enabled<hw_counters>::get())
+        {
+            assert(get_papi_vector(m_tid).get() != nullptr);
+            static thread_local auto& _pv         = get_papi_vector(m_tid);
+            auto                      _hw_counter = _pv->record();
+            auto _num_hw_counters = std::min<size_t>(_hw_counter.size(), num_hw_counters);
+            for(size_t i = 0; i < _num_hw_counters; ++i)
+            {
+                auto& _last     = get_last_hwcounters().at(i);
+                auto  itr       = _hw_counter.at(i);
+                m_hw_counter[i] = itr - _last;
+                _last           = itr;
+            }
+        }
+    }
+}
+
+std::set<int>
+backtrace::configure(bool _setup, int64_t _tid)
+{
+    auto& _sampler      = sampling::get_sampler(_tid);
+    auto& _running      = get_sampler_running(_tid);
+    bool  _is_running   = (!_running) ? false : *_running;
+    auto& _signal_types = sampling::get_signal_types(_tid);
+
+    ensure_storage<comp::trip_count, sampling_wall_clock, sampling_cpu_clock, hw_counters,
+                   sampling_percent>{}();
+
+    if(_setup && !_sampler && !_is_running)
+    {
+        assert(_tid == threading::get_id());
+        sampling::block_signals(*_signal_types);
+        if constexpr(tim::trait::is_available<hw_counters>::value)
+        {
+            perfetto_counter_track<hw_counters>::init();
+            OMNITRACE_DEBUG("HW COUNTER: starting...\n");
+            if(get_papi_vector(_tid)) get_papi_vector(_tid)->start();
+        }
+
+        auto _alrm_freq = 1.0 / std::min<double>(get_sampling_freq(), 5.0);
+        auto _prof_freq = 1.0 / get_sampling_freq();
+        auto _delay     = std::max<double>(1.0e-3, get_sampling_delay());
+
+        OMNITRACE_DEBUG("Configuring sampler for thread %lu...\n", _tid);
+        sampler_running_instances::construct(true);
+        backtrace_init_instances::construct();
+        sampling::sampler_instances::construct("omnitrace", _tid, *_signal_types);
+        _sampler->set_signals(*_signal_types);
+        _sampler->set_flags(SA_RESTART);
+        _sampler->set_delay(_delay);
+        _sampler->set_frequency(_prof_freq, { SIGPROF });
+        _sampler->set_frequency(_alrm_freq, { SIGALRM });
+
+        static_assert(tim::trait::buffer_size<sampling::sampler_t>::value > 0,
+                      "Error! Zero buffer size");
+
+        OMNITRACE_CONDITIONAL_THROW(
+            _sampler->get_buffer_size() <= 0,
+            "dynamic sampler requires a positive buffer size: %zu",
+            _sampler->get_buffer_size());
+
+        OMNITRACE_DEBUG("Sampler for thread %lu will be triggered %5.1fx per second "
+                        "(every %5.2e seconds)...\n",
+                        _tid, _sampler->get_frequency(units::sec),
+                        _sampler->get_rate(units::sec));
+
+        // (void) sampling::sampler_t::get_samplers(_tid);
+        get_backtrace_init(_tid)->sample();
+        _sampler->configure(false);
+        _sampler->start();
+    }
+    else if(!_setup && _sampler && _is_running)
+    {
+        OMNITRACE_DEBUG("Destroying sampler for thread %lu...\n", _tid);
+        *_running = false;
+
+        if(_tid == threading::get_id())
+        {
+            sampling::block_signals(*_signal_types);
+        }
+
+        // this propagates to all threads
+        if(_tid == 0) _sampler->ignore(*_signal_types);
+
+        _sampler->stop();
+        _sampler->swap_data();
+        if constexpr(tim::trait::is_available<hw_counters>::value)
+        {
+            if(_tid == threading::get_id())
+            {
+                if(get_papi_vector(_tid)) get_papi_vector(_tid)->stop();
+                OMNITRACE_DEBUG("HW COUNTER: stopped...\n");
+            }
+        }
+    }
+
+    return (_signal_types) ? *_signal_types : std::set<int>{};
+}
+
+backtrace::hw_counter_data_t&
+backtrace::get_last_hwcounters()
+{
+    static thread_local auto _v = hw_counter_data_t{ 0 };
+    return _v;
+}
+
+void
+backtrace::post_process(int64_t _tid)
+{
+    namespace quirk = tim::quirk;
+
+    configure(false, _tid);
+
+    auto& _sampler = sampling::sampler_instances::instances().at(_tid);
+    if(!_sampler)
+    {
+        // this should be relatively common
+        OMNITRACE_DEBUG(
+            "Post-processing sampling entries for thread %lu skipped (no sampler)\n",
+            _tid);
+        return;
+    }
+
+    auto& _init = backtrace_init_instances::instances().at(_tid);
+    if(!_init)
+    {
+        // this is not common
+        OMNITRACE_PRINT(
+            "Post-processing sampling entries for thread %lu skipped (not initialized)\n",
+            _tid);
+        return;
+    }
+
+    // check whether the call-stack entry should be used. -1 means break, 0 means continue
+    auto _use_label = [](const std::string& _lbl, bool _check_internal) -> short {
+        // debugging feature
+        static bool _keep_internal =
+            tim::get_env<bool>("OMNITRACE_SAMPLING_KEEP_INTERNAL", get_debug());
+        const auto _npos = std::string::npos;
+        if(_keep_internal) return 1;
+        if(_lbl.find("omnitrace_init_tooling") != _npos) return -1;
+        if(_lbl.find("omnitrace_push_trace") != _npos) return -1;
+        if(_lbl.find("omnitrace_pop_trace") != _npos) return -1;
+        if(_lbl.find("amd_comgr_") == 0) return -1;
+        if(_check_internal)
+        {
+            if(std::regex_search(
+                   _lbl, std::regex("(14pthread_gotcha7wrapper|default_error_condition)",
+                                    std::regex_constants::optimize)))
+                return 0;
+            else if(std::regex_search(
+                        _lbl, std::regex("(8sampling9backtrace9configure|"
+                                         "8sampling15unblock_signals|pthread_sigmask)",
+                                         std::regex_constants::optimize)))
+                return 0;
+        }
+        return 1;
+    };
+
+    // in the dyninst binary rewrite runtime, instrumented functions are appended with
+    // "_dyninst", i.e. "main" will show up as "main_dyninst" in the backtrace.
+    auto _patch_label = [](std::string _lbl) -> std::string {
+        // debugging feature
+        static bool _keep_suffix =
+            tim::get_env<bool>("OMNITRACE_SAMPLING_KEEP_DYNINST_SUFFIX", get_debug());
+        if(_keep_suffix) return _lbl;
+        const std::string _dyninst{ "_dyninst" };
+        auto              _pos = _lbl.find(_dyninst);
+        if(_pos == std::string::npos) return _lbl;
+        return _lbl.replace(_pos, _dyninst.length(), "");
+    };
+
+    using common_type_t = typename hw_counters::common_type;
+    auto _hw_cnt_labels = (get_papi_vector(_tid))
+                              ? comp::papi_common::get_events<common_type_t>()
+                              : std::vector<int>{};
+
+    auto _process_perfetto_counters = [&](const std::vector<sampling::bundle_t*>& _data) {
+        if(!perfetto_counter_track<comp::peak_rss>::exists(_tid))
+        {
+            auto _thrname = TIMEMORY_JOIN("", "[Thread ", _tid, "] ");
+            auto addendum = [&](const std::string& _v) { return _thrname + _v + " (S)"; };
+            perfetto_counter_track<comp::peak_rss>::emplace(
+                _tid, addendum("Peak Memory Usage"), "MB");
+        }
+
+        if(!perfetto_counter_track<hw_counters>::exists(_tid) &&
+           tim::trait::runtime_enabled<hw_counters>::get())
+        {
+            auto _thrname = TIMEMORY_JOIN("", "[Thread ", _tid, "] ");
+            auto addendum = [&](const std::string& _v) { return _thrname + _v + " (S)"; };
+            for(auto& itr : _hw_cnt_labels)
+            {
+                perfetto_counter_track<hw_counters>::emplace(
+                    _tid, addendum(tim::papi::get_event_info(itr).short_descr), "");
+            }
+        }
+
+        for(const auto& ditr : _data)
+        {
+            const auto* _bt = ditr->get<backtrace>();
+            if(_bt->m_tid != _tid) continue;
+
+            auto _ts = static_cast<uint64_t>(_bt->m_ts.time_since_epoch().count());
+
+            TRACE_COUNTER("sampling", perfetto_counter_track<comp::peak_rss>::at(_tid, 0),
+                          _ts, _bt->m_mem_peak / units::megabyte);
+
+            if(tim::trait::runtime_enabled<hw_counters>::get())
+            {
+                for(size_t i = 0; i < perfetto_counter_track<hw_counters>::size(_tid);
+                    ++i)
+                {
+                    if(i < _bt->m_hw_counter.size())
+                    {
+                        TRACE_COUNTER("sampling",
+                                      perfetto_counter_track<hw_counters>::at(_tid, i),
+                                      _ts, _bt->m_hw_counter.at(i));
+                    }
+                }
+            }
+        }
+    };
+
+    auto _process_perfetto = [&](const std::vector<sampling::bundle_t*>& _data,
+                                 bool                                    _rename) {
+        if(_rename)
+            threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid, "(S)").c_str());
+        time_point_type _last_wall_ts = _init->get_timestamp();
+
+        for(const auto& ditr : _data)
+        {
+            const auto* _bt = ditr->get<backtrace>();
+            if(_bt->m_tid != _tid) continue;
+
+            static std::set<std::string> _static_strings{};
+            std::string                  _last = {};
+            for(const auto& itr : _bt->get())
+            {
+                auto _name = tim::demangle(_patch_label(itr));
+                auto _use =
+                    _use_label(_name, !_last.empty() &&
+                                          (_last == "start_thread" || _last == "clone"));
+                if(_use == -1) break;
+                if(_use == 0) continue;
+                auto sitr = _static_strings.emplace(_name);
+                _last     = *sitr.first;
+                auto _ts  = static_cast<uint64_t>(_bt->m_ts.time_since_epoch().count());
+
+                TRACE_EVENT_BEGIN(
+                    "sampling", perfetto::StaticString{ sitr.first->c_str() },
+                    static_cast<uint64_t>(_last_wall_ts.time_since_epoch().count()));
+                TRACE_EVENT_END("sampling", _ts);
+            }
+            _last_wall_ts = _bt->m_ts;
+        }
+    };
+
+    auto _raw_data = _sampler->get_allocator().get_data();
+    // single sample that is useless (backtrace to unblocking signals)
+    if(_raw_data.size() == 1 && _raw_data.front().size() <= 1) _raw_data.clear();
+
+    std::vector<sampling::bundle_t*> _data{};
+    for(auto& ditr : _raw_data)
+    {
+        _data.reserve(_data.size() + ditr.size());
+        for(auto& ritr : ditr)
+        {
+            auto* _bt = ritr.get<backtrace>();
+            if(!_bt)
+            {
+                OMNITRACE_PRINT(
+                    "Warning! Nullptr to backtrace instance for thread %lu...\n", _tid);
+                continue;
+            }
+            if(_bt->empty()) continue;
+            _data.emplace_back(&ritr);
+        }
+    }
+
+    if(_data.empty()) return;
+
+    OMNITRACE_CONDITIONAL_PRINT(
+        get_verbose() >= 0 || get_debug(),
+        "Post-processing %zu sampling entries for thread %lu...\n", _data.size(), _tid);
+
+    std::sort(_data.begin(), _data.end(),
+              [](const sampling::bundle_t* _lhs, const sampling::bundle_t* _rhs) {
+                  return _lhs->get<backtrace>()->m_ts < _rhs->get<backtrace>()->m_ts;
+              });
+
+    if(get_use_perfetto())
+    {
+        _process_perfetto_counters(_data);
+
+        if(_tid == 0 && get_mode() == Mode::Sampling)
+            _process_perfetto(_data, false);
+        else
+        {
+            auto _v = pthread_gotcha::enable_sampling_on_child_threads();
+            pthread_gotcha::enable_sampling_on_child_threads() = false;
+            std::thread{ _process_perfetto, _data, true }.join();
+            pthread_gotcha::enable_sampling_on_child_threads() = _v;
+        }
+    }
+
+    if(!get_use_timemory()) return;
+
+    std::map<int64_t, std::map<int64_t, int64_t>> _depth_sum = {};
+    auto                                          _scope     = tim::scope::config{};
+    if(get_timeline_sampling()) _scope += scope::timeline{};
+    if(get_flat_sampling()) _scope += scope::flat{};
+
+    time_point_type _last_wall_ts = _init->get_timestamp();
+    int64_t         _last_cpu_ts  = _init->get_thread_cpu_timestamp();
+    for(auto& ditr : _data)
+    {
+        using bundle_t = tim::lightweight_tuple<comp::trip_count, sampling_wall_clock,
+                                                sampling_cpu_clock, hw_counters>;
+
+        auto* _bt = ditr->get<backtrace>();
+
+        double _elapsed_wc = (_bt->m_ts - _last_wall_ts).count();
+        double _elapsed_cc = (_bt->m_thr_cpu_ts - _last_cpu_ts);
+
+        std::vector<bundle_t> _tc{};
+        _tc.reserve(_bt->size());
+
+        // generate the instances of the tuple of components and start them
+        for(const auto& itr : _bt->get())
+        {
+            auto _lbl = _patch_label(itr);
+            auto _use =
+                _use_label(_lbl, !_tc.empty() && (_tc.back().key() == "start_thread" ||
+                                                  _tc.back().key() == "clone"));
+            if(_use == -1) break;
+            if(_use == 0) continue;
+            _tc.emplace_back(tim::string_view_t{ _lbl }, _scope);
+            _tc.back().push(_bt->m_tid);
+            _tc.back().start();
+        }
+
+        // stop the instances and update the values as needed
+        for(size_t i = 0; i < _tc.size(); ++i)
+        {
+            auto&  itr    = _tc.at(_tc.size() - i - 1);
+            size_t _depth = 0;
+            _depth_sum[_bt->m_tid][_depth] += 1;
+            itr.stop();
+            if constexpr(tim::trait::is_available<sampling_wall_clock>::value)
+            {
+                auto* _sc = itr.get<sampling_wall_clock>();
+                if(_sc)
+                {
+                    auto _value = _elapsed_wc / sampling_wall_clock::get_unit();
+                    _sc->set_value(_value);
+                    _sc->set_accum(_value);
+                }
+            }
+            if constexpr(tim::trait::is_available<sampling_cpu_clock>::value)
+            {
+                auto* _cc = itr.get<sampling_cpu_clock>();
+                if(_cc)
+                {
+                    _cc->set_value(_elapsed_cc / sampling_cpu_clock::get_unit());
+                    _cc->set_accum(_elapsed_cc / sampling_cpu_clock::get_unit());
+                }
+            }
+            if constexpr(tim::trait::is_available<hw_counters>::value)
+            {
+                auto* _hw_counter = itr.get<hw_counters>();
+                if(_hw_counter)
+                {
+                    _hw_counter->set_value(_bt->m_hw_counter);
+                    _hw_counter->set_accum(_bt->m_hw_counter);
+                }
+            }
+            itr.pop();
+        }
+        _last_wall_ts = _bt->m_ts;
+        _last_cpu_ts  = _bt->m_thr_cpu_ts;
+    }
+
+    for(auto&& ditr : _data)
+    {
+        using bundle_t =
+            tim::lightweight_tuple<sampling_percent, quirk::config<quirk::tree_scope>>;
+
+        auto* _bt = ditr->get<backtrace>();
+
+        std::vector<bundle_t> _tc{};
+        _tc.reserve(_bt->size());
+
+        // generate the instances of the tuple of components and start them
+        for(const auto& itr : _bt->get())
+        {
+            auto _lbl = _patch_label(itr);
+            auto _use =
+                _use_label(_lbl, !_tc.empty() && _tc.back().key() == "start_thread");
+            if(_use == -1) break;
+            if(_use == 0) continue;
+            _tc.emplace_back(tim::string_view_t{ _lbl });
+            _tc.back().push(_bt->m_tid);
+            _tc.back().start();
+        }
+
+        // stop the instances and update the values as needed
+        for(size_t i = 0; i < _tc.size(); ++i)
+        {
+            auto&  itr    = _tc.at(_tc.size() - i - 1);
+            size_t _depth = 0;
+            double _value = (1.0 / _depth_sum[_bt->m_tid][_depth]) * 100.0;
+            itr.store(std::plus<double>{}, _value);
+            itr.stop();
+            itr.pop();
+        }
+    }
+}
+
+}  // namespace component
+}  // namespace omnitrace
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_wall_clock>), true,
+    double)
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_cpu_clock>), true,
+    double)
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_fraction>), true,
+    double)
+
+TIMEMORY_INITIALIZE_STORAGE(omnitrace::component::backtrace)

source/lib/src/library/components/fork_gotcha.cpp

@@ -0,0 +1,57 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/fork_gotcha.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+
+namespace omnitrace
+{
+void
+fork_gotcha::configure()
+{
+    fork_gotcha_t::get_initializer() = []() {
+        TIMEMORY_C_GOTCHA(fork_gotcha_t, 0, fork);
+    };
+
+    pthread_gotcha_t::get_initializer() = []() {
+        TIMEMORY_C_GOTCHA(pthread_gotcha_t, 0, pthread_create);
+    };
+}
+
+void
+fork_gotcha::audit(const gotcha_data_t&, audit::incoming)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(
+        get_debug_env(),
+        "Warning! Calling fork() within an OpenMPI application using libfabric "
+        "may result is segmentation fault\n");
+    TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(get_debug_env(), 16);
+}
+
+void
+fork_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, pid_t _pid)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "%s() return PID %i\n",
+                                      _data.tool_id.c_str(), (int) _pid);
+}
+}  // namespace omnitrace

source/lib/src/library/components/mpi_gotcha.cpp

@@ -0,0 +1,231 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/mpi_gotcha.hpp"
+#include "library/api.hpp"
+#include "library/components/omnitrace.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "timemory/backends/process.hpp"
+
+#include <thread>
+#include <timemory/backends/mpi.hpp>
+
+namespace omnitrace
+{
+namespace
+{
+uint64_t    mpip_index      = std::numeric_limits<uint64_t>::max();
+std::string mpi_init_string = {};
+
+// this ensures omnitrace_finalize is called before MPI_Finalize
+void
+omnitrace_mpi_set_attr()
+{
+#if defined(TIMEMORY_USE_MPI)
+    static auto _mpi_copy = [](MPI_Comm, int, void*, void*, void*, int*) {
+        return MPI_SUCCESS;
+    };
+    static auto _mpi_fini = [](MPI_Comm, int, void*, void*) {
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(),
+                                          "MPI Comm attribute finalize\n");
+        if(mpip_index != std::numeric_limits<uint64_t>::max())
+            comp::deactivate_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
+                                  api::omnitrace>(mpip_index);
+        omnitrace_finalize();
+        return MPI_SUCCESS;
+    };
+    using copy_func_t = int (*)(MPI_Comm, int, void*, void*, void*, int*);
+    using fini_func_t = int (*)(MPI_Comm, int, void*, void*);
+    int _comm_key     = -1;
+    if(PMPI_Comm_create_keyval(static_cast<copy_func_t>(_mpi_copy),
+                               static_cast<fini_func_t>(_mpi_fini), &_comm_key,
+                               nullptr) == MPI_SUCCESS)
+        PMPI_Comm_set_attr(MPI_COMM_SELF, _comm_key, nullptr);
+#endif
+}
+}  // namespace
+
+void
+mpi_gotcha::configure()
+{
+    mpi_gotcha_t::get_initializer() = []() {
+        mpi_gotcha_t::template configure<0, int, int*, char***>("MPI_Init");
+        mpi_gotcha_t::template configure<1, int, int*, char***, int, int*>(
+            "MPI_Init_thread");
+        mpi_gotcha_t::template configure<2, int>("MPI_Finalize");
+#if defined(OMNITRACE_USE_MPI_HEADERS)
+        mpi_gotcha_t::template configure<3, int, comm_t, int*>("MPI_Comm_rank");
+        mpi_gotcha_t::template configure<4, int, comm_t, int*>("MPI_Comm_size");
+#endif
+    };
+}
+
+void
+mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s(int*, char***)\n",
+                                      __FUNCTION__, _data.tool_id.c_str());
+
+    if(get_state() == ::omnitrace::State::DelayedInit)
+        get_state() = ::omnitrace::State::PreInit;
+
+    omnitrace_push_trace(_data.tool_id.c_str());
+#if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
+    tim::mpi::is_initialized_callback() = []() { return true; };
+    tim::mpi::is_finalized()            = false;
+#endif
+}
+
+void
+mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***, int, int*)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(),
+                                      "[%s] %s(int*, char***, int, int*)\n", __FUNCTION__,
+                                      _data.tool_id.c_str());
+    if(get_state() == ::omnitrace::State::DelayedInit)
+        get_state() = ::omnitrace::State::PreInit;
+
+    omnitrace_push_trace(_data.tool_id.c_str());
+#if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
+    tim::mpi::is_initialized_callback() = []() { return true; };
+    tim::mpi::is_finalized()            = false;
+#endif
+}
+
+void
+mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s()\n", __FUNCTION__,
+                                      _data.tool_id.c_str());
+
+    if(mpip_index != std::numeric_limits<uint64_t>::max())
+        comp::deactivate_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
+                              api::omnitrace>(mpip_index);
+
+#if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
+    tim::mpi::is_initialized_callback() = []() { return false; };
+    tim::mpi::is_finalized()            = true;
+#else
+    omnitrace_finalize();
+#endif
+}
+
+void
+mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, comm_t, int* _val)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s()\n", __FUNCTION__,
+                                      _data.tool_id.c_str());
+
+    omnitrace_push_trace(_data.tool_id.c_str());
+    if(_data.tool_id == "MPI_Comm_rank")
+    {
+        m_rank_ptr = _val;
+    }
+    else if(_data.tool_id == "MPI_Comm_size")
+    {
+        m_size_ptr = _val;
+    }
+    else
+    {
+        OMNITRACE_BASIC_PRINT("[%s] %s(<comm>, %p) :: unexpected function wrapper\n",
+                              __FUNCTION__, _data.tool_id.c_str(), _val);
+    }
+}
+
+void
+mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s() returned %i\n",
+                                      __FUNCTION__, _data.tool_id.c_str(), (int) _retval);
+
+    if(_retval == tim::mpi::success_v && _data.tool_id.find("MPI_Init") == 0)
+    {
+        omnitrace_mpi_set_attr();
+        // omnitrace will set this environement variable to true in binary rewrite mode
+        // when it detects MPI. Hides this env variable from the user to avoid this
+        // being activated unwaringly during runtime instrumentation because that
+        // will result in double instrumenting the MPI functions (unless the MPI functions
+        // were excluded via a regex expression)
+        if(get_use_mpip())
+        {
+            OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env() || get_verbose_env() > 0,
+                                              "[%s] Activating MPI wrappers...\n",
+                                              __FUNCTION__);
+
+            // use env vars OMNITRACE_MPIP_PERMIT_LIST and OMNITRACE_MPIP_REJECT_LIST
+            // to control the gotcha bindings at runtime
+            comp::configure_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
+                                 api::omnitrace>();
+            mpip_index =
+                comp::activate_mpip<tim::component_tuple<omnitrace::component::omnitrace>,
+                                    api::omnitrace>();
+        }
+    }
+    else if(_retval == tim::mpi::success_v && _data.tool_id.find("MPI_Comm_") == 0)
+    {
+        if(_data.tool_id == "MPI_Comm_rank")
+        {
+            if(m_rank_ptr)
+            {
+                m_rank = std::max<int>(*m_rank_ptr, m_rank);
+                tim::mpi::set_rank(m_rank);
+                tim::settings::default_process_suffix() = m_rank;
+                get_perfetto_output_filename().clear();
+                OMNITRACE_CONDITIONAL_BASIC_PRINT(
+                    get_debug() || get_verbose() > 0, "[pid=%i] MPI rank: %i (%i)\n",
+                    process::get_id(), tim::mpi::rank(), m_rank);
+            }
+            else
+            {
+                OMNITRACE_BASIC_PRINT("[%s] %s() returned %i :: nullptr to rank\n",
+                                      __FUNCTION__, _data.tool_id.c_str(), (int) _retval);
+            }
+        }
+        else if(_data.tool_id == "MPI_Comm_size")
+        {
+            if(m_size_ptr)
+            {
+                m_size = std::max<int>(*m_size_ptr, m_size);
+                tim::mpi::set_size(m_size);
+                OMNITRACE_CONDITIONAL_BASIC_PRINT(
+                    get_debug() || get_verbose() > 0, "[pid=%i] MPI size: %i (%i)\n",
+                    process::get_id(), tim::mpi::size(), m_size);
+            }
+            else
+            {
+                OMNITRACE_BASIC_PRINT("[%s] %s() returned %i :: nullptr to size\n",
+                                      __FUNCTION__, _data.tool_id.c_str(), (int) _retval);
+            }
+        }
+        else
+        {
+            OMNITRACE_BASIC_PRINT(
+                "[%s] %s() returned %i :: unexpected function wrapper\n", __FUNCTION__,
+                _data.tool_id.c_str(), (int) _retval);
+        }
+    }
+    omnitrace_pop_trace(_data.tool_id.c_str());
+}
+}  // namespace omnitrace
+
+TIMEMORY_INITIALIZE_STORAGE(omnitrace::mpi_gotcha)

source/lib/src/library/components/omnitrace.cpp

@@ -0,0 +1,50 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/omnitrace.hpp"
+#include "library/api.hpp"
+
+namespace omnitrace
+{
+namespace component
+{
+void
+omnitrace::start()
+{
+    if(m_prefix) omnitrace_push_trace(m_prefix);
+}
+
+void
+omnitrace::stop()
+{
+    if(m_prefix) omnitrace_pop_trace(m_prefix);
+}
+
+void
+omnitrace::set_prefix(const char* _prefix)
+{
+    m_prefix = _prefix;
+}
+}  // namespace component
+}  // namespace omnitrace
+
+TIMEMORY_INITIALIZE_STORAGE(omnitrace::component::omnitrace)

source/lib/src/library/components/pthread_gotcha.cpp

@@ -0,0 +1,242 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/pthread_gotcha.hpp"
+#include "library/components/omnitrace.hpp"
+#include "library/components/roctracer.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/sampling.hpp"
+
+#include <ostream>
+#include <timemory/sampling/allocator.hpp>
+#include <timemory/utility/types.hpp>
+
+#include <pthread.h>
+
+namespace omnitrace
+{
+namespace sampling
+{
+std::set<int>
+setup();
+std::set<int>
+shutdown();
+}  // namespace sampling
+
+namespace mpl = tim::mpl;
+
+using bundle_t  = tim::lightweight_tuple<comp::wall_clock, comp::roctracer_data>;
+using wall_pw_t = mpl::piecewise_select<comp::wall_clock>;  // only wall-clock
+using main_pw_t = mpl::piecewise_ignore<comp::wall_clock>;  // exclude wall-clock
+using omni_pw_t = mpl::piecewise_select<>;
+
+namespace
+{
+std::map<int64_t, std::shared_ptr<bundle_t>> bundles = {};
+std::mutex                                   bundles_mutex{};
+
+inline void
+start_bundle(bundle_t& _bundle)
+{
+    if(comp::roctracer::is_setup())
+    {
+        _bundle.push(main_pw_t{});
+        _bundle.start();
+    }
+    else
+    {
+        _bundle.push(omni_pw_t{});
+        _bundle.start(omni_pw_t{});
+    }
+}
+
+inline void
+stop_bundle(bundle_t& _bundle, int64_t _tid)
+{
+    _bundle.stop(wall_pw_t{});  // stop wall-clock so we can get the value
+    // update roctracer_data
+    _bundle.store(std::plus<double>{},
+                  _bundle.get<comp::wall_clock>()->get() * units::sec);
+    // stop all other components including roctracer_data after update
+    _bundle.stop(main_pw_t{});
+    // exclude popping wall-clock
+    _bundle.pop(main_pw_t{}, _tid);
+}
+}  // namespace
+
+pthread_gotcha::wrapper::wrapper(routine_t _routine, void* _arg, bool _enable_sampling,
+                                 promise_t* _p)
+: m_enable_sampling{ _enable_sampling }
+, m_routine{ _routine }
+, m_arg{ _arg }
+, m_promise{ _p }
+{}
+
+void*
+pthread_gotcha::wrapper::operator()() const
+{
+    std::shared_ptr<bundle_t> _bundle{};
+    std::set<int>             _signals{};
+    auto&   _enable_sampling = pthread_gotcha::enable_sampling_on_child_threads();
+    auto    _active          = (get_state() == omnitrace::State::Active);
+    int64_t _tid             = -1;
+    auto    _is_sampling     = false;
+    auto    _dtor            = scope::destructor{ [&]() {
+        if(_is_sampling)
+        {
+            sampling::block_signals(_signals);
+            sampling::shutdown();
+        }
+
+        if(_bundle)
+        {
+            std::unique_lock<std::mutex> _lk{ bundles_mutex };
+            stop_bundle(*_bundle, _tid);
+            _bundle.reset();
+            bundles.erase(_tid);
+        }
+    } };
+
+    if(m_enable_sampling && _enable_sampling && _active)
+    {
+        _tid = threading::get_id();
+        threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid).c_str());
+        {
+            std::unique_lock<std::mutex> _lk{ bundles_mutex };
+            if(comp::roctracer::is_setup())
+                _bundle =
+                    bundles.emplace(_tid, std::make_shared<bundle_t>("start_thread"))
+                        .first->second;
+        }
+        if(_bundle) start_bundle(*_bundle);
+        _is_sampling     = true;
+        _enable_sampling = false;
+        _signals         = sampling::setup();
+        _enable_sampling = true;
+        sampling::unblock_signals();
+    }
+
+    if(m_promise) m_promise->set_value();
+
+    // execute the original function
+    return m_routine(m_arg);
+}
+
+void*
+pthread_gotcha::wrapper::wrap(void* _arg)
+{
+    if(_arg == nullptr) return nullptr;
+
+    // convert the argument
+    wrapper* _wrapper = static_cast<wrapper*>(_arg);
+
+    // execute the original function
+    return (*_wrapper)();
+}
+
+void
+pthread_gotcha::configure()
+{
+    pthread_gotcha_t::get_initializer() = []() {
+        TIMEMORY_C_GOTCHA(pthread_gotcha_t, 0, pthread_create);
+    };
+}
+
+void
+pthread_gotcha::shutdown()
+{
+    std::unique_lock<std::mutex> _lk{ bundles_mutex };
+    unsigned long                _ndangling = 0;
+    for(auto itr : bundles)
+    {
+        if(itr.second)
+        {
+            stop_bundle(*itr.second, itr.first);
+            ++_ndangling;
+        }
+        itr.second.reset();
+    }
+
+    OMNITRACE_CONDITIONAL_PRINT(
+        (get_verbose() > 0 || get_debug()) && _ndangling > 0,
+        "pthread_gotcha::shutdown() cleaned up %lu dangling bundles\n", _ndangling);
+    bundles.clear();
+}
+
+bool&
+pthread_gotcha::enable_sampling_on_child_threads()
+{
+    static thread_local bool _v = get_use_sampling();
+    return _v;
+}
+
+// pthread_create
+int
+pthread_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
+                           void* (*start_routine)(void*), void*     arg) const
+{
+    bundle_t _bundle{ "pthread_create" };
+    auto     _enable_sampling = enable_sampling_on_child_threads();
+
+    if(!get_use_sampling() || !_enable_sampling)
+    {
+        // if(!get_use_sampling()) start_bundle(_bundle);
+        auto* _obj = new wrapper(start_routine, arg, _enable_sampling, nullptr);
+        // create the thread
+        auto _ret =
+            pthread_create(thread, attr, &wrapper::wrap, static_cast<void*>(_obj));
+        // if(!get_use_sampling()) stop_bundle(_bundle, threading::get_id());
+        return _ret;
+    }
+
+    // block the signals in entire process
+    OMNITRACE_DEBUG("blocking signals...\n");
+    tim::sampling::block_signals({ SIGALRM, SIGPROF },
+                                 tim::sampling::sigmask_scope::process);
+
+    start_bundle(_bundle);
+
+    // promise set by thread when signal handler is configured
+    auto  _promise = std::promise<void>{};
+    auto  _fut     = _promise.get_future();
+    auto* _wrap    = new wrapper(start_routine, arg, _enable_sampling, &_promise);
+
+    // create the thread
+    auto _ret = pthread_create(thread, attr, &wrapper::wrap, static_cast<void*>(_wrap));
+
+    // wait for thread to set promise
+    OMNITRACE_DEBUG("waiting for child to signal it is setup...\n");
+    _fut.wait();
+
+    stop_bundle(_bundle, threading::get_id());
+
+    // unblock the signals in the entire process
+    OMNITRACE_DEBUG("unblocking signals...\n");
+    tim::sampling::unblock_signals({ SIGALRM, SIGPROF },
+                                   tim::sampling::sigmask_scope::process);
+
+    OMNITRACE_DEBUG("returning success...\n");
+    return _ret;
+}
+
+}  // namespace omnitrace

source/lib/src/library/components/rocm_smi.cpp

@@ -0,0 +1,449 @@
+// Copyright (c) 2018 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// with the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimers.
+//
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimers in the
+// documentation and/or other materials provided with the distribution.
+//
+// * Neither the names of Advanced Micro Devices, Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this Software without specific prior written permission.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
+// THE SOFTWARE.
+
+#if defined(NDEBUG)
+#    undef NDEBUG
+#endif
+
+#include "library/components/rocm_smi.hpp"
+#include "library/common.hpp"
+#include "library/components/fwd.hpp"
+#include "library/config.hpp"
+#include "library/critical_trace.hpp"
+#include "library/debug.hpp"
+#include "library/gpu.hpp"
+#include "library/perfetto.hpp"
+
+#include <timemory/backends/threading.hpp>
+#include <timemory/components/timing/backends.hpp>
+#include <timemory/units.hpp>
+#include <timemory/utility/locking.hpp>
+
+#include <rocm_smi/rocm_smi.h>
+
+#include <cassert>
+#include <chrono>
+#include <ios>
+#include <sstream>
+#include <sys/resource.h>
+#include <thread>
+
+#define OMNITRACE_ROCM_SMI_CALL(ERROR_CODE) ::omnitrace::rocm_smi::check_error(ERROR_CODE)
+
+namespace omnitrace
+{
+namespace rocm_smi
+{
+using tim::type_mutex;
+using auto_lock_t       = tim::auto_lock_t;
+using bundle_t          = std::deque<data>;
+using sampler_instances = thread_data<bundle_t, api::rocm_smi>;
+
+namespace
+{
+bool&
+is_initialized()
+{
+    static bool _v = false;
+    return _v;
+}
+
+void
+check_error(rsmi_status_t ec)
+{
+    if(ec == RSMI_STATUS_SUCCESS) return;
+    const char* _msg = nullptr;
+    auto        _err = rsmi_status_string(ec, &_msg);
+    if(_err != RSMI_STATUS_SUCCESS)
+        OMNITRACE_THROW(
+            "rsmi_status_string(%i, ...) failed. No error message available\n", (int) ec);
+    OMNITRACE_THROW("%s", _msg);
+}
+
+std::atomic<State>&
+get_rocm_smi_state()
+{
+    static std::atomic<State> _v{ State::PreInit };
+    return _v;
+}
+}  // namespace
+
+//--------------------------------------------------------------------------------------//
+
+size_t                           data::device_count     = 0;
+std::set<uint32_t>               data::device_list      = {};
+std::unique_ptr<data::promise_t> data::polling_finished = {};
+
+data::data(uint32_t _dev_id) { sample(_dev_id); }
+
+void
+data::sample(uint32_t _dev_id)
+{
+    auto _ts = tim::get_clock_real_now<size_t, std::nano>();
+    assert(_ts < std::numeric_limits<int64_t>::max());
+
+    m_dev_id = _dev_id;
+    m_ts     = _ts;
+
+    rsmi_dev_busy_percent_get(_dev_id, &m_busy_perc);
+    rsmi_dev_temp_metric_get(_dev_id, RSMI_TEMP_TYPE_EDGE, RSMI_TEMP_CURRENT, &m_temp);
+    rsmi_dev_power_ave_get(_dev_id, 0, &m_power);
+    rsmi_dev_memory_usage_get(_dev_id, RSMI_MEM_TYPE_VRAM, &m_mem_usage);
+}
+
+void
+data::print(std::ostream& _os) const
+{
+    std::stringstream _ss{};
+    _ss << "device: " << m_dev_id << ", busy = " << m_busy_perc << "%, temp = " << m_temp
+        << ", power = " << m_power << ", memory usage = " << m_mem_usage;
+    _os << _ss.str();
+}
+
+namespace
+{
+std::vector<std::unique_ptr<bundle_t>*> _bundle_data{};
+}
+
+void
+config()
+{
+    _bundle_data.resize(data::device_count, nullptr);
+    for(size_t i = 0; i < data::device_count; ++i)
+    {
+        if(data::device_list.count(i) > 0)
+        {
+            _bundle_data.at(i) = &sampler_instances::instances().at(i);
+            if(!*_bundle_data.at(i)) *_bundle_data.at(i) = std::make_unique<bundle_t>();
+        }
+    }
+
+    data::get_initial().resize(data::device_count);
+    for(auto itr : data::device_list)
+        data::get_initial().at(itr).sample(itr);
+}
+
+void
+sample()
+{
+    if(get_rocm_smi_state() != State::Active) return;
+
+    for(auto itr : data::device_list)
+    {
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(),
+                                          "Polling rocm-smi for device %u...\n", itr);
+        auto& _data = *_bundle_data.at(itr);
+        if(!_data) continue;
+        _data->emplace_back(data{ itr });
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(), "    %s\n",
+                                          TIMEMORY_JOIN("", _data->back()).c_str());
+    }
+}
+
+void
+set_state(State _v)
+{
+    get_rocm_smi_state().store(_v);
+}
+
+/*
+void
+data::poll(std::atomic<State>* _state, nsec_t _interval, promise_t* _ready)
+{
+    threading::set_thread_name("omni.rocm_smi");
+
+    // notify thread started
+    if(_ready) _ready->set_value();
+
+    std::vector<std::unique_ptr<bundle_t>*> _bundle_data{};
+    _bundle_data.resize(device_count, nullptr);
+    for(size_t i = 0; i < device_count; ++i)
+    {
+        if(device_list.count(i) > 0)
+        {
+            _bundle_data.at(i) = &sampler_instances::instances().at(i);
+            if(!*_bundle_data.at(i)) *_bundle_data.at(i) = std::make_unique<bundle_t>();
+        }
+    }
+
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(
+        get_verbose() > 0 || get_debug(),
+        "Polling rocm-smi for %zu device(s) at an interval of %f seconds...\n",
+        device_list.size(),
+        std::chrono::duration_cast<std::chrono::duration<double>>(_interval).count());
+
+    get_initial().resize(device_count);
+    for(auto itr : device_list)
+        get_initial().at(itr).sample(itr);
+
+    auto _now = std::chrono::steady_clock::now();
+    while(_state && _state->load() != State::Finalized && get_state() != State::Finalized)
+    {
+        std::this_thread::sleep_until(_now);
+        if(_state->load() != State::Active) continue;
+        for(auto itr : device_list)
+        {
+            OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(),
+                                              "Polling rocm-smi for device %u...\n", itr);
+            auto& _data = *_bundle_data.at(itr);
+            if(!_data) continue;
+            _data->emplace_back(data{ itr });
+            OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(), "    %s\n",
+                                              TIMEMORY_JOIN("", _data->back()).c_str());
+        }
+        while(_now < std::chrono::steady_clock::now())
+            _now += _interval;
+    }
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(), "Polling rocm-smi completed...\n");
+
+    if(polling_finished) polling_finished->set_value();
+}
+*/
+
+std::vector<data>&
+data::get_initial()
+{
+    static std::vector<data> _v{};
+    return _v;
+}
+
+bool
+data::setup()
+{
+    perfetto_counter_track<data>::init();
+    set_state(State::PreInit);
+    return true;
+}
+
+bool
+data::shutdown()
+{
+    OMNITRACE_DEBUG("Shutting down rocm-smi...\n");
+    set_state(State::Finalized);
+    return true;
+}
+
+#define GPU_METRIC(COMPONENT, ...)                                                       \
+    if constexpr(tim::trait::is_available<COMPONENT>::value)                             \
+    {                                                                                    \
+        auto* _val = _v.get<COMPONENT>();                                                \
+        if(_val)                                                                         \
+        {                                                                                \
+            _val->set_value(itr.__VA_ARGS__);                                            \
+            _val->set_accum(itr.__VA_ARGS__);                                            \
+        }                                                                                \
+    }
+
+void
+data::post_process(uint32_t _dev_id)
+{
+    OMNITRACE_CONDITIONAL_PRINT(get_debug() || get_verbose() > 0,
+                                "Post-processing rocm-smi data for device %u\n", _dev_id);
+
+    using component::sampling_gpu_busy;
+    using component::sampling_gpu_memory;
+    using component::sampling_gpu_power;
+    using component::sampling_gpu_temp;
+    using bundle_t = tim::lightweight_tuple<sampling_gpu_busy, sampling_gpu_temp,
+                                            sampling_gpu_power, sampling_gpu_memory>;
+
+    if(device_count < _dev_id) return;
+
+    auto& _rocm_smi_v = sampler_instances::instances().at(_dev_id);
+    auto  _rocm_smi   = (_rocm_smi_v) ? *_rocm_smi_v : std::deque<rocm_smi::data>{};
+
+    auto _process_perfetto = [&]() {
+        for(auto& itr : _rocm_smi)
+        {
+            using counter_track = perfetto_counter_track<data>;
+            if(itr.m_dev_id != _dev_id) continue;
+            if(!counter_track::exists(_dev_id))
+            {
+                auto _devname = TIMEMORY_JOIN("", "[GPU ", _dev_id, "] ");
+                auto addendum = [&](const char* _v) {
+                    return _devname + std::string{ _v };
+                };
+                counter_track::emplace(_dev_id, addendum("Busy"), "%");
+                counter_track::emplace(_dev_id, addendum("Temperature"), "deg C");
+                counter_track::emplace(_dev_id, addendum("Power"), "watts");
+                counter_track::emplace(_dev_id, addendum("Memory Usage"), "megabytes");
+            }
+            uint64_t _ts    = itr.m_ts;
+            double   _busy  = itr.m_busy_perc;
+            double   _temp  = itr.m_temp / 1.0e3;
+            double   _power = itr.m_power / 1.0e6;
+            double   _usage = itr.m_mem_usage / static_cast<double>(units::megabyte);
+            TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 0), _ts, _busy);
+            TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 1), _ts, _temp);
+            TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 2), _ts, _power);
+            TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 3), _ts, _usage);
+        }
+    };
+
+    if(get_use_perfetto()) _process_perfetto();
+
+    if(!get_use_timemory()) return;
+
+    for(auto& itr : _rocm_smi)
+    {
+        using entry_t = critical_trace::entry;
+        auto _ts      = itr.m_ts;
+        auto _entries = critical_trace::get_entries(_ts, [](const entry_t& _e) {
+            return _e.device == critical_trace::Device::GPU;
+        });
+
+        std::vector<bundle_t> _tc{};
+        _tc.reserve(_entries.size());
+        for(auto& eitr : _entries)
+        {
+            auto& _v = _tc.emplace_back(eitr.first);
+            _v.push();
+            _v.start();
+            _v.stop();
+
+            GPU_METRIC(sampling_gpu_busy, m_busy_perc)
+            GPU_METRIC(sampling_gpu_temp, m_temp / 1.0e3)  // provided in milli-degree C
+            GPU_METRIC(sampling_gpu_power,
+                       m_power * units::microwatt / static_cast<double>(units::watt))
+            GPU_METRIC(sampling_gpu_memory,
+                       m_mem_usage / static_cast<double>(units::megabyte))
+
+            _v.pop();
+        }
+    }
+}
+
+//--------------------------------------------------------------------------------------//
+
+void
+setup()
+{
+    auto_lock_t _lk{ type_mutex<api::rocm_smi>() };
+
+    if(is_initialized() || !get_use_rocm_smi()) return;
+
+    auto _enable_samp = pthread_gotcha::enable_sampling_on_child_threads();
+    pthread_gotcha::enable_sampling_on_child_threads() = false;
+
+    // assign the data value to determined by rocm-smi
+    data::device_count = device_count();
+
+    auto _devices_v = get_rocm_smi_devices();
+    for(auto& itr : _devices_v)
+        itr = tolower(itr);
+    bool _all_devices = _devices_v.find("all") != std::string::npos || _devices_v.empty();
+    bool _no_devices  = _devices_v.find("none") != std::string::npos;
+
+    std::set<uint32_t> _devices{};
+    if(_all_devices)
+    {
+        for(uint32_t i = 0; i < data::device_count; ++i)
+            _devices.emplace(i);
+    }
+    else if(!_no_devices)
+    {
+        for(auto&& itr : tim::delimit(get_rocm_smi_devices()))
+        {
+            uint32_t idx = std::stoul(itr);
+            if(idx < data::device_count) _devices.emplace(idx);
+        }
+    }
+
+    data::device_list = _devices;
+
+    for(auto itr : _devices)
+    {
+        uint16_t dev_id = 0;
+        OMNITRACE_ROCM_SMI_CALL(rsmi_dev_id_get(itr, &dev_id));
+        // dev_id holds the device ID of device i, upon a successful call
+    }
+
+    is_initialized() = true;
+
+    data::setup();
+
+    pthread_gotcha::enable_sampling_on_child_threads() = _enable_samp;
+}
+
+void
+shutdown()
+{
+    auto_lock_t _lk{ type_mutex<api::rocm_smi>() };
+
+    if(!is_initialized()) return;
+
+    if(data::shutdown())
+    {
+        OMNITRACE_ROCM_SMI_CALL(rsmi_shut_down());
+    }
+
+    is_initialized() = false;
+}
+
+void
+post_process()
+{
+    for(auto itr : data::device_list)
+        data::post_process(itr);
+}
+
+uint32_t
+device_count()
+{
+    uint32_t _num_devices = 0;
+    try
+    {
+        static auto _rsmi_init_once = []() { OMNITRACE_ROCM_SMI_CALL(rsmi_init(0)); };
+        static std::once_flag _once{};
+        std::call_once(_once, _rsmi_init_once);
+
+        OMNITRACE_ROCM_SMI_CALL(rsmi_num_monitor_devices(&_num_devices));
+    } catch(const std::exception& _e)
+    {
+        OMNITRACE_BASIC_PRINT("Exception: %s\n", _e.what());
+    }
+    return _num_devices;
+}
+}  // namespace rocm_smi
+}  // namespace omnitrace
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_busy>), true,
+    double)
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_temp>), true,
+    double)
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_power>), true,
+    double)
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(
+    TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_memory>), true,
+    double)

source/lib/src/library/components/roctracer.cpp

@@ -0,0 +1,328 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/roctracer.hpp"
+#include "library/components/pthread_gotcha.hpp"
+#include "library/components/rocm_smi.hpp"
+#include "library/components/roctracer_callbacks.hpp"
+#include "library/config.hpp"
+#include "library/defines.hpp"
+#include "library/redirect.hpp"
+#include "library/sampling.hpp"
+#include "library/thread_data.hpp"
+
+namespace rocm_smi = omnitrace::rocm_smi;
+using namespace omnitrace;
+
+namespace tim
+{
+namespace component
+{
+void
+roctracer::preinit()
+{
+    roctracer_data::label()       = "roctracer";
+    roctracer_data::description() = "ROCm tracer (activity API)";
+}
+
+void
+roctracer::start()
+{
+    if(tracker_type::start() == 0) setup();
+}
+
+void
+roctracer::stop()
+{
+    if(tracker_type::stop() == 0) shutdown();
+}
+
+bool
+roctracer::is_setup()
+{
+    return roctracer_is_setup();
+}
+
+void
+roctracer::add_setup(const std::string& _lbl, std::function<void()>&& _func)
+{
+    roctracer_setup_routines().emplace_back(_lbl, std::move(_func));
+}
+
+void
+roctracer::add_shutdown(const std::string& _lbl, std::function<void()>&& _func)
+{
+    roctracer_shutdown_routines().emplace_back(_lbl, std::move(_func));
+}
+
+void
+roctracer::remove_setup(const std::string& _lbl)
+{
+    auto& _data = roctracer_setup_routines();
+    for(auto itr = _data.begin(); itr != _data.end(); ++itr)
+    {
+        if(itr->first == _lbl)
+        {
+            _data.erase(itr);
+            break;
+        }
+    }
+}
+
+void
+roctracer::remove_shutdown(const std::string& _lbl)
+{
+    auto& _data = roctracer_setup_routines();
+    for(auto itr = _data.begin(); itr != _data.end(); ++itr)
+    {
+        if(itr->first == _lbl)
+        {
+            _data.erase(itr);
+            break;
+        }
+    }
+}
+
+void
+roctracer::setup()
+{
+    trait::runtime_enabled<roctracer_data>::set(get_use_timemory());
+
+    if(!get_use_roctracer()) return;
+
+    auto_lock_t _lk{ type_mutex<roctracer>() };
+    if(roctracer_is_setup()) return;
+    roctracer_is_setup() = true;
+
+    OMNITRACE_DEBUG("[%s]\n", __FUNCTION__);
+
+#if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR < 4
+    auto _kfdwrapper = dynamic_library{ "OMNITRACE_ROCTRACER_LIBKFDWRAPPER",
+                                        OMNITRACE_ROCTRACER_LIBKFDWRAPPER };
+#endif
+
+    ROCTRACER_CALL(
+        roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, (void*) hip_api_callback));
+
+    if(roctracer_default_pool() == nullptr)
+    {
+        // Allocating tracing pool
+        roctracer_properties_t properties{};
+        memset(&properties, 0, sizeof(roctracer_properties_t));
+        // properties.mode                = 0x1000;
+        properties.buffer_size         = 0x1000;
+        properties.buffer_callback_fun = hip_activity_callback;
+        ROCTRACER_CALL(roctracer_open_pool(&properties));
+    }
+
+#if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR >= 4 &&              \
+    OMNITRACE_HIP_VERSION_MINOR <= 5
+    // HIP 4.5.0 has an invalid warning
+    redirect _rd{ std::cerr, "roctracer_enable_callback(), get_op_end(), invalid domain "
+                             "ID(4)  in: roctracer_enable_callback(hip_api_callback, "
+                             "nullptr)roctracer_enable_activity_expl(), get_op_end(), "
+                             "invalid domain ID(4)  in: roctracer_enable_activity()" };
+#endif
+
+    // Enable API callbacks, all domains
+    ROCTRACER_CALL(roctracer_enable_callback(hip_api_callback, nullptr));
+    // Enable activity tracing, all domains
+    ROCTRACER_CALL(roctracer_enable_activity());
+
+    // callback for HSA
+    for(auto& itr : roctracer_setup_routines())
+        itr.second();
+}
+
+void
+roctracer::shutdown()
+{
+    auto_lock_t _lk{ type_mutex<roctracer>() };
+    if(!roctracer_is_setup()) return;
+    roctracer_is_setup() = false;
+    OMNITRACE_DEBUG("[%s]\n", __FUNCTION__);
+
+    // flush all the activity
+    if(roctracer_default_pool() != nullptr)
+    {
+        OMNITRACE_DEBUG("[%s] roctracer_flush_activity\n", __FUNCTION__);
+        ROCTRACER_CALL(roctracer_flush_activity());
+
+        // flush all buffers
+        OMNITRACE_DEBUG("[%s] roctracer_flush_buf\n", __FUNCTION__);
+        roctracer_flush_buf();
+    }
+
+    OMNITRACE_DEBUG("[%s] executing hip_exec_activity_callbacks\n", __FUNCTION__);
+    // make sure all async operations are executed
+    for(size_t i = 0; i < max_supported_threads; ++i)
+        hip_exec_activity_callbacks(i);
+
+    // callback for hsa
+    OMNITRACE_DEBUG("[%s] executing roctracer_shutdown_routines...\n", __FUNCTION__);
+    for(auto& itr : roctracer_shutdown_routines())
+        itr.second();
+
+#if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR >= 4 &&              \
+    OMNITRACE_HIP_VERSION_MINOR <= 5
+    OMNITRACE_DEBUG("[%s] redirecting roctracer warnings\n", __FUNCTION__);
+    // HIP 4.5.0 has an invalid warning
+    redirect _rd{
+        std::cerr, "roctracer_disable_callback(), get_op_end(), invalid domain ID(4)  "
+                   "in: roctracer_disable_callback()roctracer_disable_activity(), "
+                   "get_op_end(), invalid domain ID(4)  in: roctracer_disable_activity()"
+    };
+#endif
+
+    // Disable tracing and closing the pool
+    OMNITRACE_DEBUG("[%s] roctracer_disable_callback\n", __FUNCTION__);
+    ROCTRACER_CALL(roctracer_disable_callback());
+
+    OMNITRACE_DEBUG("[%s] roctracer_disable_activity\n", __FUNCTION__);
+    ROCTRACER_CALL(roctracer_disable_activity());
+
+    OMNITRACE_DEBUG("[%s] roctracer_close_pool\n", __FUNCTION__);
+    ROCTRACER_CALL(roctracer_close_pool());
+
+    OMNITRACE_DEBUG("[%s] roctracer is shutdown\n", __FUNCTION__);
+}
+}  // namespace component
+}  // namespace tim
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer, false, void)
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer_data, true, double)
+
+// HSA-runtime tool on-load method
+extern "C"
+{
+    bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                const char* const* failed_tool_names) TIMEMORY_VISIBILITY("default");
+    void OnUnload() TIMEMORY_VISIBILITY("default");
+
+    bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                const char* const* failed_tool_names)
+    {
+        pthread_gotcha::enable_sampling_on_child_threads() = false;
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env() || get_verbose_env() > 0,
+                                          "[%s]\n", __FUNCTION__);
+        tim::consume_parameters(table, runtime_version, failed_tool_count,
+                                failed_tool_names);
+
+        auto _setup = [=]() {
+            try
+            {
+                OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1,
+                                                  "[%s] setting up HSA...\n",
+                                                  __FUNCTION__);
+
+                // const char* output_prefix = getenv("ROCP_OUTPUT_DIR");
+                const char* output_prefix = nullptr;
+
+                bool trace_hsa_api = get_trace_hsa_api();
+
+                // Enable HSA API callbacks/activity
+                if(trace_hsa_api)
+                {
+                    std::vector<std::string> hsa_api_vec =
+                        tim::delimit(get_trace_hsa_api_types());
+
+                    // initialize HSA tracing
+                    roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
+
+                    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1,
+                                                      "    HSA-trace(");
+                    if(!hsa_api_vec.empty())
+                    {
+                        for(const auto& itr : hsa_api_vec)
+                        {
+                            uint32_t    cid = HSA_API_ID_NUMBER;
+                            const char* api = itr.c_str();
+                            ROCTRACER_CALL(roctracer_op_code(ACTIVITY_DOMAIN_HSA_API, api,
+                                                             &cid, nullptr));
+                            ROCTRACER_CALL(roctracer_enable_op_callback(
+                                ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));
+
+                            OMNITRACE_CONDITIONAL_BASIC_PRINT(
+                                get_debug() || get_verbose() > 1, " %s", api);
+                        }
+                    }
+                    else
+                    {
+                        ROCTRACER_CALL(roctracer_enable_domain_callback(
+                            ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
+                    }
+                    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1,
+                                                      "\n");
+                }
+
+                bool trace_hsa_activity = get_trace_hsa_activity();
+                // Enable HSA GPU activity
+                if(trace_hsa_activity)
+                {
+                    // initialize HSA tracing
+                    ::roctracer::hsa_ops_properties_t ops_properties{
+                        table,
+                        reinterpret_cast<activity_async_callback_t>(
+                            hsa_activity_callback),
+                        nullptr, output_prefix
+                    };
+                    roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);
+
+                    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1,
+                                                      "    HSA-activity-trace()\n");
+                    ROCTRACER_CALL(roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS,
+                                                                HSA_OP_ID_COPY));
+                }
+            } catch(std::exception& _e)
+            {
+                OMNITRACE_BASIC_PRINT("Exception was thrown in HSA setup: %s\n",
+                                      _e.what());
+            }
+        };
+
+        auto _shutdown = []() {
+            OMNITRACE_DEBUG("[%s] roctracer_disable_domain_callback\n", __FUNCTION__);
+            ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
+
+            OMNITRACE_DEBUG("[%s] roctracer_disable_op_activity\n", __FUNCTION__);
+            ROCTRACER_CALL(
+                roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+        };
+
+        comp::roctracer::add_setup("hsa", std::move(_setup));
+        comp::roctracer::add_shutdown("hsa", std::move(_shutdown));
+
+        rocm_smi::set_state(State::Active);
+        comp::roctracer::setup();
+
+        pthread_gotcha::enable_sampling_on_child_threads() = true;
+        return true;
+    }
+
+    // HSA-runtime on-unload method
+    void OnUnload()
+    {
+        OMNITRACE_DEBUG("[%s]\n", __FUNCTION__);
+        rocm_smi::set_state(State::Finalized);
+        comp::roctracer::shutdown();
+    }
+}

source/lib/src/library/components/roctracer_callbacks.cpp

@@ -0,0 +1,630 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/components/roctracer_callbacks.hpp"
+#include "library.hpp"
+#include "library/config.hpp"
+#include "library/critical_trace.hpp"
+#include "library/sampling.hpp"
+#include "library/thread_data.hpp"
+
+#include <timemory/backends/threading.hpp>
+
+#include <cstdint>
+
+TIMEMORY_DEFINE_API(roctracer)
+namespace omnitrace
+{
+namespace api = tim::api;
+
+std::unordered_set<uint64_t>&
+get_roctracer_kernels()
+{
+    static auto _v = std::unordered_set<uint64_t>{};
+    return _v;
+}
+
+auto&
+get_roctracer_hip_data(int64_t _tid = threading::get_id())
+{
+    using data_t        = std::unordered_map<uint64_t, roctracer_bundle_t>;
+    using thread_data_t = thread_data<data_t, api::roctracer>;
+    static auto& _v     = thread_data_t::instances(thread_data_t::construct_on_init{});
+    return _v.at(_tid);
+}
+
+std::unordered_map<uint64_t, const char*>&
+get_roctracer_key_data()
+{
+    static auto _v = std::unordered_map<uint64_t, const char*>{};
+    return _v;
+}
+
+std::unordered_map<uint64_t, int64_t>&
+get_roctracer_tid_data()
+{
+    static auto _v = std::unordered_map<uint64_t, int64_t>{};
+    return _v;
+}
+
+using cid_tuple_t = std::tuple<uint64_t, uint64_t, uint16_t>;
+std::unordered_map<uint64_t, cid_tuple_t>&
+get_roctracer_cid_data()
+{
+    static auto _v = std::unordered_map<uint64_t, cid_tuple_t>{};
+    return _v;
+}
+
+auto&
+get_hip_activity_callbacks(int64_t _tid = threading::get_id())
+{
+    using thread_data_t = thread_data<std::vector<std::function<void()>>, api::roctracer>;
+    static auto& _v     = thread_data_t::instances(thread_data_t::construct_on_init{});
+    return _v.at(_tid);
+}
+
+using hip_activity_mutex_t = std::decay_t<decltype(get_hip_activity_callbacks())>;
+using key_data_mutex_t     = std::decay_t<decltype(get_roctracer_key_data())>;
+using hip_data_mutex_t     = std::decay_t<decltype(get_roctracer_hip_data())>;
+using cid_data_mutex_t     = std::decay_t<decltype(get_roctracer_cid_data())>;
+
+auto&
+get_hip_activity_mutex(int64_t _tid = threading::get_id())
+{
+    return tim::type_mutex<hip_activity_mutex_t, api::roctracer, max_supported_threads>(
+        _tid);
+}
+
+// HSA API callback function
+void
+hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg)
+{
+    if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
+        return;
+
+    (void) arg;
+    const hsa_api_data_t* data = reinterpret_cast<const hsa_api_data_t*>(callback_data);
+    OMNITRACE_DEBUG("<%-30s id(%u)\tcorrelation_id(%lu) %s>\n",
+                    roctracer_op_string(domain, cid, 0), cid, data->correlation_id,
+                    (data->phase == ACTIVITY_API_PHASE_ENTER) ? "on-enter" : "on-exit");
+
+    static thread_local int64_t begin_timestamp = 0;
+    static auto                 _scope          = []() {
+        auto _v = scope::config{};
+        if(get_roctracer_timeline_profile()) _v += scope::timeline{};
+        if(get_roctracer_flat_profile()) _v += scope::flat{};
+        return _v;
+    }();
+
+    switch(cid)
+    {
+        case HSA_API_ID_hsa_init:
+        case HSA_API_ID_hsa_shut_down:
+        case HSA_API_ID_hsa_agent_get_exception_policies:
+        case HSA_API_ID_hsa_agent_get_info:
+        case HSA_API_ID_hsa_amd_agent_iterate_memory_pools:
+        case HSA_API_ID_hsa_amd_agent_memory_pool_get_info:
+        case HSA_API_ID_hsa_amd_coherency_get_type:
+        case HSA_API_ID_hsa_amd_memory_pool_get_info:
+        case HSA_API_ID_hsa_amd_pointer_info:
+        case HSA_API_ID_hsa_amd_pointer_info_set_userdata:
+        case HSA_API_ID_hsa_amd_profiling_async_copy_enable:
+        case HSA_API_ID_hsa_amd_profiling_get_async_copy_time:
+        case HSA_API_ID_hsa_amd_profiling_get_dispatch_time:
+        case HSA_API_ID_hsa_amd_profiling_set_profiler_enabled:
+        case HSA_API_ID_hsa_cache_get_info:
+        case HSA_API_ID_hsa_code_object_get_info:
+        case HSA_API_ID_hsa_code_object_get_symbol:
+        case HSA_API_ID_hsa_code_object_get_symbol_from_name:
+        case HSA_API_ID_hsa_code_object_reader_create_from_memory:
+        case HSA_API_ID_hsa_code_symbol_get_info:
+        case HSA_API_ID_hsa_executable_create_alt:
+        case HSA_API_ID_hsa_executable_freeze:
+        case HSA_API_ID_hsa_executable_get_info:
+        case HSA_API_ID_hsa_executable_get_symbol:
+        case HSA_API_ID_hsa_executable_get_symbol_by_name:
+        case HSA_API_ID_hsa_executable_symbol_get_info:
+        case HSA_API_ID_hsa_extension_get_name:
+        case HSA_API_ID_hsa_ext_image_data_get_info:
+        case HSA_API_ID_hsa_ext_image_data_get_info_with_layout:
+        case HSA_API_ID_hsa_ext_image_get_capability:
+        case HSA_API_ID_hsa_ext_image_get_capability_with_layout:
+        case HSA_API_ID_hsa_isa_get_exception_policies:
+        case HSA_API_ID_hsa_isa_get_info:
+        case HSA_API_ID_hsa_isa_get_info_alt:
+        case HSA_API_ID_hsa_isa_get_round_method:
+        case HSA_API_ID_hsa_region_get_info:
+        case HSA_API_ID_hsa_system_extension_supported:
+        case HSA_API_ID_hsa_system_get_extension_table:
+        case HSA_API_ID_hsa_system_get_info:
+        case HSA_API_ID_hsa_system_get_major_extension_table:
+        case HSA_API_ID_hsa_wavefront_get_info: break;
+        default:
+        {
+            if(data->phase == ACTIVITY_API_PHASE_ENTER)
+            {
+                begin_timestamp = comp::wall_clock::record();
+            }
+            else
+            {
+                const auto* _name         = roctracer_op_string(domain, cid, 0);
+                const auto  end_timestamp = (cid == HSA_API_ID_hsa_shut_down)
+                                                ? begin_timestamp
+                                                : comp::wall_clock::record();
+
+                if(begin_timestamp > end_timestamp) return;
+
+                if(get_use_perfetto())
+                {
+                    TRACE_EVENT_BEGIN("device", perfetto::StaticString{ _name },
+                                      static_cast<uint64_t>(begin_timestamp));
+                    TRACE_EVENT_END("device", static_cast<uint64_t>(end_timestamp));
+                }
+
+                if(get_use_timemory())
+                {
+                    std::unique_lock<std::mutex> _lk{ tasking::get_roctracer_mutex() };
+                    auto                         _begin_ns = begin_timestamp;
+                    auto                         _end_ns   = end_timestamp;
+                    tasking::get_roctracer_task_group().exec(
+                        [_name, _begin_ns, _end_ns]() {
+                            roctracer_hsa_bundle_t _bundle{ _name, _scope };
+                            _bundle.start()
+                                .store(std::plus<double>{},
+                                       static_cast<double>(_end_ns - _begin_ns))
+                                .stop();
+                        });
+                }
+                // timemory is disabled in this callback because collecting data in this
+                // thread causes strange segmentation faults
+            }
+        }
+    }
+}
+
+void
+hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
+{
+    if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
+        return;
+
+    static const char* copy_op_name     = "hsa_async_copy";
+    static const char* dispatch_op_name = "hsa_dispatch";
+    static const char* barrier_op_name  = "hsa_barrier";
+    const char**       _name            = nullptr;
+
+    static thread_local auto _once = (threading::set_thread_name("omni.roctracer"), true);
+    (void) _once;
+
+    switch(op)
+    {
+        case HSA_OP_ID_DISPATCH: _name = &dispatch_op_name; break;
+        case HSA_OP_ID_COPY: _name = &copy_op_name; break;
+        case HSA_OP_ID_BARRIER: _name = &barrier_op_name; break;
+        default: break;
+    }
+
+    if(!_name) return;
+
+    auto        _begin_ns = record->begin_ns;
+    auto        _end_ns   = record->end_ns;
+    static auto _scope    = []() {
+        auto _v = scope::config{};
+        if(get_roctracer_timeline_profile()) _v += scope::timeline{};
+        if(get_roctracer_flat_profile()) _v += scope::flat{};
+        return _v;
+    }();
+
+    auto _func = [_begin_ns, _end_ns, _name]() {
+        if(get_use_perfetto())
+        {
+            TRACE_EVENT_BEGIN("device", perfetto::StaticString{ *_name },
+                              static_cast<uint64_t>(_begin_ns));
+            TRACE_EVENT_END("device", static_cast<uint64_t>(_end_ns));
+        }
+        if(get_use_timemory())
+        {
+            roctracer_hsa_bundle_t _bundle{ *_name, _scope };
+            _bundle.start()
+                .store(std::plus<double>{}, static_cast<double>(_end_ns - _begin_ns))
+                .stop();
+        }
+    };
+
+    std::unique_lock<std::mutex> _lk{ tasking::get_roctracer_mutex() };
+    tasking::get_roctracer_task_group().exec(_func);
+
+    // timemory is disabled in this callback because collecting data in this thread
+    // causes strange segmentation faults
+    tim::consume_parameters(arg);
+}
+
+void
+hip_exec_activity_callbacks(int64_t _tid)
+{
+    // ROCTRACER_CALL(roctracer_flush_activity());
+    tim::auto_lock_t _lk{ get_hip_activity_mutex(_tid) };
+    auto&            _async_ops = get_hip_activity_callbacks(_tid);
+    for(auto& itr : *_async_ops)
+        itr();
+    _async_ops->clear();
+}
+
+namespace
+{
+thread_local std::unordered_map<size_t, size_t> gpu_cids = {};
+}
+
+// HIP API callback function
+void
+hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg)
+{
+    if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
+        return;
+
+    using Device = critical_trace::Device;
+    using Phase  = critical_trace::Phase;
+
+    const char* op_name = roctracer_op_string(domain, cid, 0);
+    if(op_name == nullptr) op_name = hip_api_name(cid);
+    if(op_name == nullptr) return;
+
+    const hip_api_data_t* data = reinterpret_cast<const hip_api_data_t*>(callback_data);
+    OMNITRACE_DEBUG("<%-30s id(%u)\tcorrelation_id(%lu) %s>\n", op_name, cid,
+                    data->correlation_id,
+                    (data->phase == ACTIVITY_API_PHASE_ENTER) ? "on-enter" : "on-exit");
+
+    switch(cid)
+    {
+        case HIP_API_ID___hipPushCallConfiguration:
+        case HIP_API_ID___hipPopCallConfiguration:
+        case HIP_API_ID_hipDeviceEnablePeerAccess:
+        case HIP_API_ID_hipImportExternalMemory:
+        case HIP_API_ID_hipDestroyExternalMemory: return;
+        default: break;
+    }
+
+    int64_t _ts = comp::wall_clock::record();
+
+    if(data->phase == ACTIVITY_API_PHASE_ENTER)
+    {
+        switch(cid)
+        {
+            case HIP_API_ID_hipLaunchKernel:
+            case HIP_API_ID_hipLaunchCooperativeKernel:
+            {
+                const char* _name =
+                    hipKernelNameRefByPtr(data->args.hipLaunchKernel.function_address,
+                                          data->args.hipLaunchKernel.stream);
+                if(_name != nullptr)
+                {
+                    if(get_use_perfetto() || get_use_timemory() || get_use_rocm_smi())
+                    {
+                        tim::auto_lock_t _lk{ tim::type_mutex<key_data_mutex_t>() };
+                        get_roctracer_key_data().emplace(data->correlation_id, _name);
+                        get_roctracer_tid_data().emplace(data->correlation_id,
+                                                         threading::get_id());
+                    }
+                }
+                break;
+            }
+            case HIP_API_ID_hipModuleLaunchKernel:
+            {
+                const char* _name = hipKernelNameRef(data->args.hipModuleLaunchKernel.f);
+                if(_name != nullptr)
+                {
+                    if(get_use_perfetto() || get_use_timemory() || get_use_rocm_smi())
+                    {
+                        tim::auto_lock_t _lk{ tim::type_mutex<key_data_mutex_t>() };
+                        get_roctracer_key_data().emplace(data->correlation_id, _name);
+                        get_roctracer_tid_data().emplace(data->correlation_id,
+                                                         threading::get_id());
+                    }
+                }
+                break;
+            }
+            default: break;
+        }
+
+        if(get_use_perfetto())
+        {
+            TRACE_EVENT_BEGIN("device", perfetto::StaticString{ op_name },
+                              static_cast<uint64_t>(_ts));
+        }
+        if(get_use_timemory())
+        {
+            auto itr = get_roctracer_hip_data()->emplace(data->correlation_id,
+                                                         roctracer_bundle_t{ op_name });
+            if(itr.second)
+            {
+                itr.first->second.start();
+            }
+            else if(itr.first != get_roctracer_hip_data()->end())
+            {
+                itr.first->second.stop();
+                get_roctracer_hip_data()->erase(itr.first);
+            }
+        }
+        if(get_use_critical_trace() || get_use_rocm_smi())
+        {
+            auto     _cid        = get_cpu_cid()++;
+            uint16_t _depth      = (get_cpu_cid_stack()->empty())
+                                       ? get_cpu_cid_stack(0)->size()
+                                       : get_cpu_cid_stack()->size() - 1;
+            auto     _parent_cid = (get_cpu_cid_stack()->empty())
+                                       ? get_cpu_cid_stack(0)->back()
+                                       : get_cpu_cid_stack()->back();
+            add_critical_trace<Device::CPU, Phase::BEGIN>(
+                threading::get_id(), _cid, data->correlation_id, _parent_cid, _ts, 0,
+                critical_trace::add_hash_id(op_name), _depth);
+            tim::auto_lock_t _lk{ tim::type_mutex<cid_data_mutex_t>() };
+            get_roctracer_cid_data().emplace(data->correlation_id,
+                                             cid_tuple_t{ _cid, _parent_cid, _depth });
+        }
+
+        hip_exec_activity_callbacks(threading::get_id());
+    }
+    else if(data->phase == ACTIVITY_API_PHASE_EXIT)
+    {
+        hip_exec_activity_callbacks(threading::get_id());
+
+        if(get_use_perfetto())
+        {
+            TRACE_EVENT_END("device", static_cast<uint64_t>(_ts));
+        }
+        if(get_use_timemory())
+        {
+            auto _stop = [data](int64_t _tid) {
+                auto& _data = get_roctracer_hip_data(_tid);
+                auto  itr   = _data->find(data->correlation_id);
+                if(itr != get_roctracer_hip_data()->end())
+                {
+                    itr->second.stop();
+                    _data->erase(itr);
+                    return true;
+                }
+                return false;
+            };
+            if(!_stop(threading::get_id()))
+            {
+                for(size_t i = 0; i < max_supported_threads; ++i)
+                {
+                    if(_stop(i)) break;
+                }
+            }
+        }
+        if(get_use_critical_trace() || get_use_rocm_smi())
+        {
+            uint16_t _depth      = 0;
+            uint64_t _cid        = 0;
+            uint64_t _parent_cid = 0;
+            {
+                tim::auto_lock_t _lk{ tim::type_mutex<cid_data_mutex_t>() };
+                std::tie(_cid, _parent_cid, _depth) =
+                    get_roctracer_cid_data().at(data->correlation_id);
+            }
+            add_critical_trace<Device::CPU, Phase::END>(
+                threading::get_id(), _cid, data->correlation_id, _parent_cid, _ts, _ts,
+                critical_trace::add_hash_id(op_name), _depth);
+        }
+    }
+    tim::consume_parameters(arg);
+}
+
+// Activity tracing callback
+void
+hip_activity_callback(const char* begin, const char* end, void*)
+{
+    if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
+        return;
+
+    sampling::block_signals();
+
+    static thread_local auto _once = (threading::set_thread_name("omni.roctracer"), true);
+    (void) _once;
+
+    using Device = critical_trace::Device;
+    using Phase  = critical_trace::Phase;
+
+    if(!trait::runtime_enabled<comp::roctracer>::get()) return;
+    static auto _kernel_names        = std::unordered_map<const char*, std::string>{};
+    static auto _indexes             = std::unordered_map<uint64_t, int>{};
+    const roctracer_record_t* record = reinterpret_cast<const roctracer_record_t*>(begin);
+    const roctracer_record_t* end_record =
+        reinterpret_cast<const roctracer_record_t*>(end);
+
+    OMNITRACE_DEBUG("Activity records:\n");
+
+    auto&& _advance_record = [&record]() {
+        ROCTRACER_CALL(roctracer_next_record(record, &record));
+    };
+
+    while(record < end_record)
+    {
+        // make sure every iteration advances regardless of where return point happens
+        scope::destructor _next_dtor{ _advance_record };
+
+        const char* op_name =
+            roctracer_op_string(record->domain, record->correlation_id, 0);
+        if(op_name == nullptr) op_name = hip_api_name(record->correlation_id);
+
+        switch(record->kind)
+        {
+            case HIP_API_ID_hipLaunchKernel:
+            case HIP_API_ID_hipLaunchCooperativeKernel:
+            case HIP_API_ID_hipModuleLaunchKernel: break;
+            case HIP_API_ID_hipGetLastError: continue;
+            default:
+            {
+                if(op_name != nullptr && strcmp(op_name, "unknown") != 0 &&
+                   strcmp(op_name, "InternalMarker") != 0)
+                {
+                    OMNITRACE_BASIC_VERBOSE_F(2, "[%s] ignoring callback for %s\n",
+                                              __FUNCTION__, op_name);
+                    continue;
+                }
+                break;
+            }
+        }
+
+        auto _dev_id = record->device_id;
+        auto _thr_id = record->thread_id;
+        auto _prc_id = record->process_id;
+        auto _que_id = record->queue_id;
+
+        if(op_name != nullptr)
+        {
+            OMNITRACE_DEBUG(
+                "\t%-30s\tcorrelation_id(%6lu) time_ns(%12lu:%12lu) "
+                "delta_ns(%12lu) device_id(%d) stream_id(%lu) proc_id(%u) thr_id(%u)\n",
+                op_name, record->correlation_id, record->begin_ns, record->end_ns,
+                (record->end_ns - record->begin_ns), _dev_id, _que_id, _prc_id, _thr_id);
+        }
+
+        auto        _begin_ns = record->begin_ns;
+        auto        _end_ns   = record->end_ns;
+        auto        _corr_id  = record->correlation_id;
+        static auto _scope    = []() {
+            auto _v = scope::config{};
+            if(get_roctracer_timeline_profile()) _v += scope::timeline{};
+            if(get_roctracer_flat_profile()) _v += scope::flat{};
+            return _v;
+        }();
+
+        auto& _keys = get_roctracer_key_data();
+        auto& _cids = get_roctracer_cid_data();
+        auto& _tids = get_roctracer_tid_data();
+
+        int16_t     _depth          = 0;                     // depth of kernel launch
+        int64_t     _tid            = 0;                     // thread id
+        uint64_t    _cid            = 0;                     // correlation id
+        uint64_t    _pcid           = 0;                     // parent corr_id
+        auto        _laps           = _indexes[_corr_id]++;  // see note #1
+        const char* _name           = nullptr;
+        bool        _found          = false;
+        bool        _critical_trace = get_use_critical_trace() || get_use_rocm_smi();
+
+        {
+            tim::auto_lock_t _lk{ tim::type_mutex<key_data_mutex_t>() };
+            if(_tids.find(_corr_id) != _tids.end())
+            {
+                _found   = true;
+                _tid     = _tids.at(_corr_id);
+                auto itr = _keys.find(_corr_id);
+                if(itr != _keys.end()) _name = itr->second;
+            }
+        }
+
+        if(_critical_trace)
+        {
+            tim::auto_lock_t _lk{ tim::type_mutex<cid_data_mutex_t>() };
+            if(_cids.find(_corr_id) != _cids.end())
+                std::tie(_cid, _pcid, _depth) = _cids.at(_corr_id);
+            else
+                _critical_trace = false;
+        }
+
+        auto _func = [_critical_trace, _depth, _tid, _cid, _laps, _begin_ns, _end_ns,
+                      _corr_id, _name]() {
+            // NOTE #1: we get two measurements for 1 kernel so we need to
+            // tweak the number of laps for the wall-clock component
+            if(_name != nullptr)
+            {
+                if(get_use_perfetto())
+                {
+                    if(_kernel_names.find(_name) == _kernel_names.end())
+                        _kernel_names.emplace(_name, tim::demangle(_name));
+                    TRACE_EVENT_BEGIN(
+                        "device",
+                        perfetto::StaticString{ _kernel_names.at(_name).c_str() },
+                        static_cast<uint64_t>(_begin_ns));
+                    TRACE_EVENT_END("device", static_cast<uint64_t>(_end_ns));
+                }
+                if(get_use_timemory())
+                {
+                    roctracer_bundle_t _bundle{ _name, _scope };
+                    _bundle.start()
+                        .store(std::plus<double>{},
+                               static_cast<double>(_end_ns - _begin_ns))
+                        .stop()
+                        .get<comp::wall_clock>([&](comp::wall_clock* wc) {
+                            wc->set_value(_end_ns - _begin_ns);
+                            wc->set_accum(_end_ns - _begin_ns);
+                            if(_laps % 2 == 1)
+                            {
+                                // below is a hack bc we get two measurements for 1 kernel
+                                wc->set_laps(0);
+
+                                auto itr = wc->get_iterator();
+                                if(itr && itr->data().get_laps() == 0)
+                                {
+                                    wc->set_is_invalid(true);
+                                    itr->data().set_is_invalid(true);
+                                }
+                            }
+                            return wc;
+                        });
+                    _bundle.pop();
+                }
+                if(_critical_trace)
+                {
+                    auto     _hash = critical_trace::add_hash_id(_name);
+                    uint16_t _prio = _laps + 1;  // priority
+                    add_critical_trace<Device::GPU, Phase::DELTA, false>(
+                        _tid, _cid, _corr_id, _cid, _begin_ns, _end_ns, _hash, _depth + 1,
+                        _prio);
+                }
+            }
+        };
+
+        if(_found)
+        {
+            auto&            _async_ops = get_hip_activity_callbacks(_tid);
+            tim::auto_lock_t _lk{ get_hip_activity_mutex(_tid) };
+            _async_ops->emplace_back(std::move(_func));
+        }
+    }
+}
+
+bool&
+roctracer_is_setup()
+{
+    static bool _v = false;
+    return _v;
+}
+
+using roctracer_functions_t = std::vector<std::pair<std::string, std::function<void()>>>;
+
+roctracer_functions_t&
+roctracer_setup_routines()
+{
+    static auto _v = roctracer_functions_t{};
+    return _v;
+}
+
+roctracer_functions_t&
+roctracer_shutdown_routines()
+{
+    static auto _v = roctracer_functions_t{};
+    return _v;
+}
+}  // namespace omnitrace

source/lib/src/library/config.cpp

@@ -0,0 +1,950 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/defines.hpp"
+#include "library/thread_data.hpp"
+
+#include <timemory/backends/dmp.hpp>
+#include <timemory/backends/mpi.hpp>
+#include <timemory/backends/process.hpp>
+#include <timemory/environment.hpp>
+#include <timemory/settings.hpp>
+#include <timemory/settings/types.hpp>
+#include <timemory/utility/argparse.hpp>
+
+#include <array>
+#include <cstdint>
+#include <cstdlib>
+#include <numeric>
+#include <ostream>
+#include <string>
+
+namespace omnitrace
+{
+using settings = tim::settings;
+
+namespace
+{
+auto
+get_config()
+{
+    static auto _once = (configure_settings(), true);
+    return settings::shared_instance();
+    (void) _once;
+}
+
+std::string
+get_setting_name(std::string _v)
+{
+    static const auto _prefix = tim::string_view_t{ "omnitrace_" };
+    for(auto& itr : _v)
+        itr = tolower(itr);
+    auto _pos = _v.find(_prefix);
+    if(_pos == 0) return _v.substr(_prefix.length());
+    return _v;
+}
+
+#define OMNITRACE_CONFIG_SETTING(TYPE, ENV_NAME, DESCRIPTION, INITIAL_VALUE, ...)        \
+    {                                                                                    \
+        auto _ret = _config->insert<TYPE, TYPE>(                                         \
+            ENV_NAME, get_setting_name(ENV_NAME), DESCRIPTION, INITIAL_VALUE,            \
+            std::set<std::string>{ "custom", "omnitrace", __VA_ARGS__ });                \
+        if(!_ret.second)                                                                 \
+            OMNITRACE_PRINT("Warning! Duplicate setting: %s / %s\n",                     \
+                            get_setting_name(ENV_NAME).c_str(), ENV_NAME);               \
+    }
+}  // namespace
+
+inline namespace config
+{
+void
+configure_settings()
+{
+    static bool _once = false;
+    if(_once) return;
+    _once = true;
+
+    OMNITRACE_CONDITIONAL_THROW(
+        get_state() < State::Init,
+        "config::configure_settings() called before omnitrace_init_library. state = %s",
+        std::to_string(get_state()).c_str());
+
+    OMNITRACE_CONDITIONAL_THROW(
+        get_state() > State::Init,
+        "config::configure_settings() called after omnitrace was initialized. state = %s",
+        std::to_string(get_state()).c_str());
+
+    static auto _config = settings::shared_instance();
+
+    // if using timemory, default to perfetto being off
+    auto _default_perfetto_v =
+        !tim::get_env<bool>("OMNITRACE_USE_TIMEMORY", false, false);
+
+    auto _system_backend = tim::get_env("OMNITRACE_BACKEND_SYSTEM", false, false);
+
+    auto _omnitrace_debug = _config->get<bool>("OMNITRACE_DEBUG");
+    if(_omnitrace_debug) tim::set_env("TIMEMORY_DEBUG_SETTINGS", "1", 0);
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_PERFETTO", "Enable perfetto backend",
+                             _default_perfetto_v, "backend", "perfetto");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_TIMEMORY", "Enable timemory backend",
+                             !_config->get<bool>("OMNITRACE_USE_PERFETTO"), "backend",
+                             "timemory");
+
+#if defined(OMNITRACE_USE_ROCTRACER)
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_ROCTRACER", "Enable ROCM tracing", true,
+                             "backend", "roctracer");
+#endif
+
+#if defined(OMNITRACE_USE_ROCM_SMI)
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_USE_ROCM_SMI",
+        "Enable sampling GPU power, temp, utilization, and memory usage", true, "backend",
+        "rocm-smi");
+
+    OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_ROCM_SMI_DEVICES",
+                             "Devices to query when OMNITRACE_USE_ROCM_SMI=ON", "all",
+                             "backend", "rocm-smi");
+#endif
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_SAMPLING",
+                             "Enable statistical sampling of call-stack", false,
+                             "backend", "sampling");
+
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_USE_PID",
+        "Enable tagging filenames with process identifier (either MPI rank or pid)", true,
+        "io");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_KOKKOSP",
+                             "Enable support for Kokkos Tools", false, "kokkos");
+
+    OMNITRACE_CONFIG_SETTING(size_t, "OMNITRACE_INSTRUMENTATION_INTERVAL",
+                             "Instrumentation only takes measurements once every N "
+                             "function calls (not statistical)",
+                             1, "instrumentation");
+
+    OMNITRACE_CONFIG_SETTING(
+        double, "OMNITRACE_SAMPLING_FREQ",
+        "Number of software interrupts per second when OMNITTRACE_USE_SAMPLING=ON", 10.0,
+        "sampling");
+
+    OMNITRACE_CONFIG_SETTING(
+        double, "OMNITRACE_SAMPLING_DELAY",
+        "Number of seconds to delay activating the statistical sampling", 0.05,
+        "sampling");
+
+    auto _backend = tim::get_env_choice<std::string>(
+        "OMNITRACE_BACKEND",
+        (_system_backend)
+            ? "system"      // if OMNITRACE_BACKEND_SYSTEM is true, default to system.
+            : "inprocess",  // Otherwise, default to inprocess
+        { "inprocess", "system", "all" }, false);
+
+    OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_BACKEND",
+                             "Specify the perfetto backend to activate. Options are: "
+                             "'inprocess', 'system', or 'all'",
+                             _backend, "perfetto");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_CRITICAL_TRACE",
+                             "Enable generation of the critical trace", false, "feature");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_FLAT_SAMPLING",
+                             "Ignore hierarchy in all statistical sampling entries",
+                             _config->get_flat_profile(), "sampling", "data_layout");
+
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_TIMELINE_SAMPLING",
+        "Create unique entries for every sample when statistical sampling is enabled",
+        _config->get_timeline_profile(), "sampling", "data_layout");
+
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_ROCTRACER_FLAT_PROFILE",
+        "Ignore hierarchy in all kernels entries with timemory backend",
+        _config->get_flat_profile(), "roctracer", "data_layout");
+
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_ROCTRACER_TIMELINE_PROFILE",
+        "Create unique entries for every kernel with timemory backend",
+        _config->get_timeline_profile(), "roctracer", "data_layout");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_ROCTRACER_HSA_ACTIVITY",
+                             "Enable HSA activity tracing support", false, "roctracer");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_ROCTRACER_HSA_API",
+                             "Enable HSA API tracing support", false, "roctracer");
+
+    OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_ROCTRACER_HSA_API_TYPES",
+                             "HSA API type to collect", "", "roctracer");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_CRITICAL_TRACE_DEBUG",
+                             "Enable debugging for critical trace", _omnitrace_debug,
+                             "debugging");
+
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_CRITICAL_TRACE_SERIALIZE_NAMES",
+        "Include names in serialization of critical trace (mainly for debugging)",
+        _omnitrace_debug, "debugging");
+
+    OMNITRACE_CONFIG_SETTING(size_t, "OMNITRACE_SHMEM_SIZE_HINT_KB",
+                             "Hint for shared-memory buffer size in perfetto (in KB)",
+                             40960, "perfetto", "data");
+
+    OMNITRACE_CONFIG_SETTING(size_t, "OMNITRACE_BUFFER_SIZE_KB",
+                             "Size of perfetto buffer (in KB)", 1024000, "perfetto",
+                             "data");
+
+    OMNITRACE_CONFIG_SETTING(int64_t, "OMNITRACE_CRITICAL_TRACE_COUNT",
+                             "Number of critical trace to export (0 == all)", 0, "data");
+
+    OMNITRACE_CONFIG_SETTING(uint64_t, "OMNITRACE_CRITICAL_TRACE_BUFFER_COUNT",
+                             "Number of critical trace records to store in thread-local "
+                             "memory before submitting to shared buffer",
+                             2000, "data");
+
+    OMNITRACE_CONFIG_SETTING(
+        uint64_t, "OMNITRACE_CRITICAL_TRACE_NUM_THREADS",
+        "Number of threads to use when generating the critical trace",
+        std::min<uint64_t>(8, std::thread::hardware_concurrency()), "parallelism");
+
+    OMNITRACE_CONFIG_SETTING(
+        int64_t, "OMNITRACE_CRITICAL_TRACE_PER_ROW",
+        "How many critical traces per row in perfetto (0 == all in one row)", 0, "io");
+
+    OMNITRACE_CONFIG_SETTING(
+        std::string, "OMNITRACE_TIMEMORY_COMPONENTS",
+        "List of components to collect via timemory (see timemory-avail)", "wall_clock",
+        "timemory", "component");
+
+    OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_OUTPUT_FILE", "Perfetto filename",
+                             "", "perfetto", "io");
+
+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_SETTINGS_DESC",
+                             "Provide descriptions when printing settings", false,
+                             "debugging");
+
+    _config->get_flamegraph_output()     = false;
+    _config->get_cout_output()           = false;
+    _config->get_file_output()           = true;
+    _config->get_json_output()           = true;
+    _config->get_tree_output()           = true;
+    _config->get_enable_signal_handler() = true;
+    _config->get_collapse_processes()    = false;
+    _config->get_collapse_threads()      = false;
+    _config->get_stack_clearing()        = false;
+    _config->get_time_output()           = true;
+    _config->get_timing_precision()      = 6;
+    _config->get_max_thread_bookmarks()  = 1;
+    _config->get_timing_units()          = "sec";
+    _config->get_memory_units()          = "MB";
+    _config->get_papi_events()           = "PAPI_TOT_CYC, PAPI_TOT_INS";
+
+#if defined(TIMEMORY_USE_PAPI)
+    int _paranoid = 2;
+    {
+        std::ifstream _fparanoid{ "/proc/sys/kernel/perf_event_paranoid" };
+        if(_fparanoid) _fparanoid >> _paranoid;
+    }
+
+    if(_paranoid > 1)
+    {
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(
+            get_verbose_env() >= 0,
+            "/proc/sys/kernel/perf_event_paranoid has a value of %i. "
+            "Disabling PAPI (requires a value <= 1)...\n",
+            _paranoid);
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(
+            get_verbose_env() >= 0,
+            "In order to enable PAPI support, run 'echo N | sudo tee "
+            "/proc/sys/kernel/perf_event_paranoid' where N is < 2\n");
+        tim::trait::runtime_enabled<comp::papi_common>::set(false);
+        tim::trait::runtime_enabled<comp::papi_array_t>::set(false);
+        tim::trait::runtime_enabled<comp::papi_vector>::set(false);
+        tim::trait::runtime_enabled<comp::cpu_roofline_flops>::set(false);
+        tim::trait::runtime_enabled<comp::cpu_roofline_dp_flops>::set(false);
+        tim::trait::runtime_enabled<comp::cpu_roofline_sp_flops>::set(false);
+        _config->get_papi_events() = "";
+    }
+#else
+    _config->get_papi_quiet() = true;
+#endif
+
+    for(auto&& itr :
+        tim::delimit(_config->get<std::string>("OMNITRACE_CONFIG_FILE"), ";:"))
+    {
+        OMNITRACE_CONDITIONAL_BASIC_PRINT(get_verbose_env() > 0,
+                                          "Reading config file %s\n", itr.c_str());
+        _config->read(itr);
+    }
+
+    _config->get_global_components() =
+        _config->get<std::string>("OMNITRACE_TIMEMORY_COMPONENTS");
+
+    // always initialize timemory because gotcha wrappers are always used
+    auto _cmd = tim::read_command_line(process::get_id());
+    auto _exe = (_cmd.empty()) ? "exe" : _cmd.front();
+    auto _pos = _exe.find_last_of('/');
+    if(_pos < _exe.length() - 1) _exe = _exe.substr(_pos + 1);
+    get_exe_name() = _exe;
+
+    scope::get_fields()[scope::flat::value]     = tim::settings::flat_profile();
+    scope::get_fields()[scope::timeline::value] = tim::settings::timeline_profile();
+
+    bool _found_sep = false;
+    for(const auto& itr : _cmd)
+    {
+        if(itr == "--") _found_sep = true;
+    }
+    if(!_found_sep && _cmd.size() > 1) _cmd.insert(_cmd.begin() + 1, "--");
+
+    using argparser_t = tim::argparse::argument_parser;
+    argparser_t _parser{ _exe };
+    tim::timemory_init(_cmd, _parser, "omnitrace-");
+
+    settings::suppress_parsing()  = true;
+    settings::suppress_config()   = true;
+    settings::use_output_suffix() = _config->get<bool>("OMNITRACE_USE_PID");
+#if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
+    if(tim::mpi::is_initialized()) settings::default_process_suffix() = tim::mpi::rank();
+#endif
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_verbose_env() > 0, "configuration complete\n");
+}
+
+void
+print_banner(std::ostream& _os)
+{
+    static const char* _banner = R"banner(
+
+      ______   .___  ___. .__   __.  __  .___________..______          ___       ______  _______
+     /  __  \  |   \/   | |  \ |  | |  | |           ||   _  \        /   \     /      ||   ____|
+    |  |  |  | |  \  /  | |   \|  | |  | `---|  |----`|  |_)  |      /  ^  \   |  ,----'|  |__
+    |  |  |  | |  |\/|  | |  . `  | |  |     |  |     |      /      /  /_\  \  |  |     |   __|
+    |  `--'  | |  |  |  | |  |\   | |  |     |  |     |  |\  \----./  _____  \ |  `----.|  |____
+     \______/  |__|  |__| |__| \__| |__|     |__|     | _| `._____/__/     \__\ \______||_______|
+
+    )banner";
+    _os << _banner << std::endl;
+}
+
+void
+print_settings(
+    std::ostream&                                                                _ros,
+    std::function<bool(const std::string_view&, const std::set<std::string>&)>&& _filter)
+{
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(true, "configuration:\n");
+
+    std::stringstream _os{};
+
+    bool _md = tim::get_env<bool>("OMNITRACE_SETTINGS_DESC_MARKDOWN", false);
+
+    constexpr size_t nfields = 3;
+    using str_array_t        = std::array<std::string, nfields>;
+    std::vector<str_array_t>    _data{};
+    std::array<size_t, nfields> _widths{};
+    _widths.fill(0);
+    for(const auto& itr : *get_config())
+    {
+        if(_filter(itr.first, itr.second->get_categories()))
+        {
+            auto _disp = itr.second->get_display(std::ios::boolalpha);
+            _data.emplace_back(str_array_t{ _disp.at("env_name"), _disp.at("value"),
+                                            _disp.at("description") });
+            for(size_t i = 0; i < nfields; ++i)
+            {
+                size_t _wextra = (_md && i < 2) ? 2 : 0;
+                _widths.at(i)  = std::max<size_t>(_widths.at(i),
+                                                 _data.back().at(i).length() + _wextra);
+            }
+        }
+    }
+
+    std::sort(_data.begin(), _data.end(), [](const auto& lhs, const auto& rhs) {
+        auto _npos = std::string::npos;
+        // OMNITRACE_CONFIG_FILE always first
+        if(lhs.at(0).find("OMNITRACE_CONFIG") != _npos) return true;
+        if(rhs.at(0).find("OMNITRACE_CONFIG") != _npos) return false;
+        // OMNITRACE_USE_* prioritized
+        auto _lhs_use = lhs.at(0).find("OMNITRACE_USE_");
+        auto _rhs_use = rhs.at(0).find("OMNITRACE_USE_");
+        if(_lhs_use != _rhs_use && _lhs_use < _rhs_use) return true;
+        if(_lhs_use != _rhs_use && _lhs_use > _rhs_use) return false;
+        // alphabetical sort
+        return lhs.at(0) < rhs.at(0);
+    });
+
+    bool _print_desc = get_debug() || get_config()->get<bool>("OMNITRACE_SETTINGS_DESC");
+
+    auto tot_width = std::accumulate(_widths.begin(), _widths.end(), 0);
+    if(!_print_desc) tot_width -= _widths.back() + 4;
+
+    size_t _spacer_extra = 9;
+    if(!_md)
+        _spacer_extra += 2;
+    else if(_md && _print_desc)
+        _spacer_extra -= 1;
+    std::stringstream _spacer{};
+    _spacer.fill('-');
+    _spacer << "#" << std::setw(tot_width + _spacer_extra) << ""
+            << "#";
+    _os << _spacer.str() << "\n";
+    // _os << "# api::omnitrace settings:" << std::setw(tot_width - 8) << "#" << "\n";
+    for(const auto& itr : _data)
+    {
+        _os << ((_md) ? "| " : "# ");
+        for(size_t i = 0; i < nfields; ++i)
+        {
+            switch(i)
+            {
+                case 0: _os << std::left; break;
+                case 1: _os << std::left; break;
+                case 2: _os << std::left; break;
+            }
+            if(_md)
+            {
+                std::stringstream _ss{};
+                _ss.setf(_os.flags());
+                std::string _extra = (i < 2) ? "`" : "";
+                _ss << _extra << itr.at(i) << _extra;
+                _os << std::setw(_widths.at(i)) << _ss.str() << " | ";
+                if(!_print_desc && i == 1) break;
+            }
+            else
+            {
+                _os << std::setw(_widths.at(i)) << itr.at(i) << " ";
+                if(!_print_desc && i == 1) break;
+                switch(i)
+                {
+                    case 0: _os << "= "; break;
+                    case 1: _os << "[ "; break;
+                    case 2: _os << "]"; break;
+                }
+            }
+        }
+        _os << ((_md) ? "\n" : "  #\n");
+    }
+    _os << _spacer.str() << "\n";
+
+    _ros << _os.str() << std::flush;
+}
+
+void
+print_settings()
+{
+    if(dmp::rank() > 0) return;
+
+    static std::set<tim::string_view_t> _sample_options = {
+        "OMNITRACE_SAMPLING_FREQ", "OMNITRACE_SAMPLING_DELAY",
+        "OMNITRACE_FLAT_SAMPLING", "OMNITRACE_TIMELINE_SAMPLING",
+        "OMNITRACE_FLAT_SAMPLING", "OMNITRACE_TIMELINE_SAMPLING",
+    };
+    static std::set<tim::string_view_t> _perfetto_options = {
+        "OMNITRACE_OUTPUT_FILE",
+        "OMNITRACE_BACKEND",
+        "OMNITRACE_SHMEM_SIZE_HINT_KB",
+        "OMNITRACE_BUFFER_SIZE_KB",
+    };
+    static std::set<tim::string_view_t> _timemory_options = {
+        "OMNITRACE_ROCTRACER_FLAT_PROFILE", "OMNITRACE_ROCTRACER_TIMELINE_PROFILE"
+    };
+    // generic filter for filtering relevant options
+    auto _is_omnitrace_option = [](const auto& _v, const auto& _c) {
+        if(!get_use_roctracer() && _v.find("OMNITRACE_ROCTRACER_") == 0) return false;
+        if(!get_use_critical_trace() && _v.find("OMNITRACE_CRITICAL_TRACE_") == 0)
+            return false;
+        if(!get_use_perfetto() && _perfetto_options.count(_v) > 0) return false;
+        if(!get_use_timemory() && _timemory_options.count(_v) > 0) return false;
+        if(!get_use_sampling() && _sample_options.count(_v) > 0) return false;
+        const auto npos = std::string::npos;
+        if(_v.find("WIDTH") != npos || _v.find("SEPARATOR_FREQ") != npos ||
+           _v.find("AUTO_OUTPUT") != npos || _v.find("DART_OUTPUT") != npos ||
+           _v.find("FILE_OUTPUT") != npos || _v.find("PLOT_OUTPUT") != npos ||
+           _v.find("FLAMEGRAPH_OUTPUT") != npos)
+            return false;
+        if(!_c.empty())
+        {
+            if(_c.find("omnitrace") != _c.end()) return true;
+            if(_c.find("debugging") != _c.end() && _v.find("DEBUG") != npos) return true;
+            if(_c.find("config") != _c.end()) return true;
+            if(_c.find("dart") != _c.end()) return false;
+            if(_c.find("io") != _c.end() && _v.find("_OUTPUT") != npos) return true;
+            if(_c.find("format") != _c.end()) return true;
+            return false;
+        }
+        return (_v.find("OMNITRACE_") == 0);
+    };
+
+    tim::print_env(std::cerr, [_is_omnitrace_option](const std::string& _v) {
+        return _is_omnitrace_option(_v, std::set<std::string>{});
+    });
+
+    print_settings(std::cerr, _is_omnitrace_option);
+
+    fprintf(stderr, "\n");
+}
+
+std::string&
+get_exe_name()
+{
+    static std::string _v = {};
+    return _v;
+}
+
+std::string
+get_config_file()
+{
+    static auto _v = get_config()->find("OMNITRACE_CONFIG_FILE");
+    return static_cast<tim::tsettings<std::string>&>(*_v->second).get();
+}
+
+Mode
+get_mode()
+{
+    static auto _v = []() {
+        auto _mode = tim::get_env_choice<std::string>("OMNITRACE_MODE", "trace",
+                                                      { "trace", "sampling" });
+        if(_mode == "sampling") return Mode::Sampling;
+        return Mode::Trace;
+    }();
+    return _v;
+}
+
+bool&
+is_attached()
+{
+    static bool _v = false;
+    return _v;
+}
+
+bool&
+is_binary_rewrite()
+{
+    static bool _v = false;
+    return _v;
+}
+
+bool
+get_debug_env()
+{
+    return tim::get_env<bool>("OMNITRACE_DEBUG", false);
+}
+
+bool
+get_is_continuous_integration()
+{
+    return tim::get_env<bool>("OMNITRACE_CI", false);
+}
+
+bool
+get_debug_init()
+{
+    return tim::get_env<bool>("OMNITRACE_DEBUG_INIT", false);
+}
+
+bool
+get_debug_finalize()
+{
+    return tim::get_env<bool>("OMNITRACE_DEBUG_FINALIZE", false);
+}
+
+bool
+get_debug()
+{
+    static auto _v = get_config()->find("OMNITRACE_DEBUG");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+int
+get_verbose_env()
+{
+    return tim::get_env<int>("OMNITRACE_VERBOSE", 0);
+}
+
+int
+get_verbose()
+{
+    static auto _v = get_config()->find("OMNITRACE_VERBOSE");
+    return static_cast<tim::tsettings<int>&>(*_v->second).get();
+}
+
+bool&
+get_use_perfetto()
+{
+    static auto _v = get_config()->find("OMNITRACE_USE_PERFETTO");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool&
+get_use_timemory()
+{
+    static auto _v = get_config()->find("OMNITRACE_USE_TIMEMORY");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool&
+get_use_roctracer()
+{
+#if defined(OMNITRACE_USE_ROCTRACER)
+    static auto _v = get_config()->find("OMNITRACE_USE_ROCTRACER");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+#else
+    static auto _v            = false;
+    return _v;
+#endif
+}
+
+bool&
+get_use_rocm_smi()
+{
+#if defined(OMNITRACE_USE_ROCM_SMI)
+    static auto _v = get_config()->find("OMNITRACE_USE_ROCM_SMI");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+#else
+    static auto _v = false;
+    return _v;
+#endif
+}
+
+bool&
+get_use_sampling()
+{
+#if defined(TIMEMORY_USE_LIBUNWIND)
+    static auto _v = get_config()->find("OMNITRACE_USE_SAMPLING");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+#else
+    OMNITRACE_THROW(
+        "Error! sampling was enabled but omnitrace was not built with libunwind support");
+    static bool _v = false;
+    return _v;
+#endif
+}
+
+bool&
+get_use_pid()
+{
+    static auto _v = get_config()->find("OMNITRACE_USE_PID");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool&
+get_use_mpip()
+{
+    static bool _v = tim::get_env("OMNITRACE_USE_MPIP", false, false);
+    return _v;
+}
+
+bool&
+get_use_critical_trace()
+{
+    static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_use_kokkosp()
+{
+    static auto _v = get_config()->find("OMNITRACE_USE_KOKKOSP");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_critical_trace_debug()
+{
+    static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_DEBUG");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_critical_trace_serialize_names()
+{
+    static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_SERIALIZE_NAMES");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_timeline_sampling()
+{
+    static auto _v = get_config()->find("OMNITRACE_TIMELINE_SAMPLING");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_flat_sampling()
+{
+    static auto _v = get_config()->find("OMNITRACE_FLAT_SAMPLING");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_roctracer_timeline_profile()
+{
+    static auto _v = get_config()->find("OMNITRACE_ROCTRACER_TIMELINE_PROFILE");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_roctracer_flat_profile()
+{
+    static auto _v = get_config()->find("OMNITRACE_ROCTRACER_FLAT_PROFILE");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_trace_hsa_api()
+{
+    static auto _v = get_config()->find("OMNITRACE_ROCTRACER_HSA_API");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+bool
+get_trace_hsa_activity()
+{
+    static auto _v = get_config()->find("OMNITRACE_ROCTRACER_HSA_ACTIVITY");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
+int64_t
+get_critical_trace_per_row()
+{
+    static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_PER_ROW");
+    return static_cast<tim::tsettings<int64_t>&>(*_v->second).get();
+}
+
+size_t
+get_perfetto_shmem_size_hint()
+{
+    static auto _v = get_config()->find("OMNITRACE_SHMEM_SIZE_HINT_KB");
+    return static_cast<tim::tsettings<size_t>&>(*_v->second).get();
+}
+
+size_t
+get_perfetto_buffer_size()
+{
+    static auto _v = get_config()->find("OMNITRACE_BUFFER_SIZE_KB");
+    return static_cast<tim::tsettings<size_t>&>(*_v->second).get();
+}
+
+uint64_t
+get_critical_trace_update_freq()
+{
+    static uint64_t _v =
+        get_config()->get<uint64_t>("OMNITRACE_CRITICAL_TRACE_BUFFER_COUNT");
+    return _v;
+}
+
+uint64_t
+get_critical_trace_num_threads()
+{
+    static uint64_t _v =
+        get_config()->get<uint64_t>("OMNITRACE_CRITICAL_TRACE_NUM_THREADS");
+    return _v;
+}
+
+std::string
+get_trace_hsa_api_types()
+{
+    static std::string _v =
+        get_config()->get<std::string>("OMNITRACE_ROCTRACER_HSA_API_TYPES");
+    return _v;
+}
+
+std::string&
+get_backend()
+{
+    // select inprocess, system, or both (i.e. all)
+    static auto _v = get_config()->find("OMNITRACE_BACKEND");
+    return static_cast<tim::tsettings<std::string>&>(*_v->second).get();
+}
+
+std::string&
+get_perfetto_output_filename()
+{
+    static auto  _v = get_config()->find("OMNITRACE_OUTPUT_FILE");
+    static auto& _t = static_cast<tim::tsettings<std::string>&>(*_v->second);
+    if(_t.get().empty())
+    {
+        // default name: perfetto-trace.<pid>.proto or perfetto-trace.<rank>.proto
+        auto _default_fname =
+            settings::compose_output_filename("perfetto-trace", "proto", get_use_pid());
+        auto _pid_patch = std::string{ "/" } + std::to_string(tim::process::get_id()) +
+                          "-perfetto-trace";
+        auto _dpos = _default_fname.find(_pid_patch);
+        if(_dpos != std::string::npos)
+            _default_fname =
+                _default_fname.replace(_dpos, _pid_patch.length(), "/perfetto-trace");
+        // have the default display the full path to the output file
+        _t.set(tim::get_env<std::string>(
+            "OMNITRACE_OUTPUT_FILE",
+            JOIN('/', tim::get_env<std::string>("PWD", ".", false), _default_fname),
+            false));
+    }
+    return _t.get();
+}
+
+size_t&
+get_instrumentation_interval()
+{
+    static auto _v = get_config()->find("OMNITRACE_INSTRUMENTATION_INTERVAL");
+    return static_cast<tim::tsettings<size_t>&>(*_v->second).get();
+}
+
+double&
+get_sampling_freq()
+{
+    static auto _v = get_config()->find("OMNITRACE_SAMPLING_FREQ");
+    return static_cast<tim::tsettings<double>&>(*_v->second).get();
+}
+
+double&
+get_sampling_delay()
+{
+    static auto _v = get_config()->find("OMNITRACE_SAMPLING_DELAY");
+    return static_cast<tim::tsettings<double>&>(*_v->second).get();
+}
+
+int64_t
+get_critical_trace_count()
+{
+    static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_COUNT");
+    return static_cast<tim::tsettings<int64_t>&>(*_v->second).get();
+}
+
+double&
+get_thread_sampling_freq()
+{
+    static auto _v = std::min<double>(get_sampling_freq(), 1000.0);
+    return _v;
+}
+
+std::string
+get_rocm_smi_devices()
+{
+#if defined(OMNITRACE_USE_ROCM_SMI)
+    static auto _v = get_config()->find("OMNITRACE_ROCM_SMI_DEVICES");
+    return static_cast<tim::tsettings<std::string>&>(*_v->second).get();
+#else
+    return std::string{};
+#endif
+}
+
+bool
+get_debug_tid()
+{
+    static auto _vlist = []() {
+        std::unordered_set<int64_t> _tids{};
+        for(auto itr : tim::delimit<std::vector<int64_t>>(
+                tim::get_env<std::string>("OMNITRACE_DEBUG_TIDS", ""),
+                ",: ", [](const std::string& _v) { return std::stoll(_v); }))
+            _tids.insert(itr);
+        return _tids;
+    }();
+    static thread_local bool _v =
+        _vlist.empty() || _vlist.count(tim::threading::get_id()) > 0;
+    return _v;
+}
+
+bool
+get_debug_pid()
+{
+    static auto _vlist = []() {
+        std::unordered_set<int64_t> _pids{};
+        for(auto itr : tim::delimit<std::vector<int64_t>>(
+                tim::get_env<std::string>("OMNITRACE_DEBUG_PIDS", ""),
+                ",: ", [](const std::string& _v) { return std::stoll(_v); }))
+            _pids.insert(itr);
+        return _pids;
+    }();
+    static bool _v = _vlist.empty() || _vlist.count(tim::process::get_id()) > 0 ||
+                     _vlist.count(dmp::rank()) > 0;
+    return _v;
+}
+}  // namespace config
+
+State&
+get_state()
+{
+    static State _v{ State::PreInit };
+    return _v;
+}
+
+std::atomic<uint64_t>&
+get_cpu_cid()
+{
+    static std::atomic<uint64_t> _v{ 0 };
+    return _v;
+}
+
+std::unique_ptr<std::vector<uint64_t>>&
+get_cpu_cid_stack(int64_t _tid)
+{
+    struct omnitrace_cpu_cid_stack
+    {};
+    using thread_data_t = thread_data<std::vector<uint64_t>, omnitrace_cpu_cid_stack>;
+    static auto&             _v       = thread_data_t::instances();
+    static thread_local auto _v_check = [_tid]() {
+        thread_data_t::construct((_tid > 0) ? *thread_data_t::instances().at(0)
+                                            : std::vector<uint64_t>{});
+        return true;
+    }();
+    return _v.at(_tid);
+    (void) _v_check;
+}
+
+namespace
+{
+void
+setup_gotchas()
+{
+    static bool _initialized = false;
+    if(_initialized) return;
+    _initialized = true;
+
+    OMNITRACE_CONDITIONAL_PRINT(
+        get_debug_env(),
+        "[%s] Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n",
+        __FUNCTION__);
+
+    mpi_gotcha::configure();
+    fork_gotcha::configure();
+    pthread_gotcha::configure();
+}
+}  // namespace
+
+std::unique_ptr<main_bundle_t>&
+get_main_bundle()
+{
+    static auto _v =
+        std::make_unique<main_bundle_t>("omnitrace", quirk::config<quirk::auto_start>{});
+    return _v;
+}
+
+std::unique_ptr<gotcha_bundle_t>&
+get_gotcha_bundle()
+{
+    static auto _v =
+        (setup_gotchas(), std::make_unique<gotcha_bundle_t>(
+                              "omnitrace", quirk::config<quirk::auto_start>{}));
+    return _v;
+}
+}  // namespace omnitrace

source/lib/src/library/cpu_freq.cpp

@@ -0,0 +1,184 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/cpu_freq.hpp"
+#include "library/common.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/defines.hpp"
+#include "library/perfetto.hpp"
+#include "library/timemory.hpp"
+
+#include <cstdlib>
+#include <utility>
+#include <vector>
+
+namespace omnitrace
+{
+namespace cpu_freq
+{
+namespace
+{
+struct cpu_freq
+{};
+using freq_pair_t                                    = std::pair<size_t, double>;
+std::vector<std::deque<freq_pair_t>> cpu_frequencies = {};
+
+struct cpu_mem
+{};
+using cpu_mem_usage_pair_t                     = std::pair<size_t, int64_t>;
+std::deque<cpu_mem_usage_pair_t> cpu_mem_usage = {};
+
+int64_t                        ncpu        = threading::affinity::hw_concurrency();
+std::unique_ptr<std::ifstream> ifs         = {};
+std::vector<size_t>            cpu_mhz_pos = {};
+
+}  // namespace
+
+void
+setup()
+{
+    perfetto_counter_track<cpu_freq>::init();
+    perfetto_counter_track<cpu_mem>::init();
+}
+
+void
+config()
+{
+    auto                _ncpu = threading::affinity::hw_concurrency();
+    std::vector<size_t> _cpu_mhz_pos{};
+    std::ifstream       _ifs{ "/proc/cpuinfo" };
+    if(_ifs)
+    {
+        for(size_t i = 0; i < _ncpu; ++i)
+        {
+            short       _n = 0;
+            std::string _st{};
+            while(_ifs && _ifs.good())
+            {
+                std::string _s{};
+                _ifs >> _s;
+                if(!_ifs.good() || !_ifs) break;
+
+                if(_s == "cpu" || _s == "MHz" || _s == ":")
+                {
+                    ++_n;
+                    _st += _s + " ";
+                }
+                else
+                {
+                    _n  = 0;
+                    _st = {};
+                }
+
+                if(_n == 3)
+                {
+                    size_t _pos = _ifs.tellg();
+                    _cpu_mhz_pos.emplace_back(_pos + 1);
+                    _ifs >> _s;
+                    if(!_ifs.good() || !_ifs) break;
+                    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1,
+                                                      "[%zu] %s %s (pos = %zu)\n", i,
+                                                      _st.c_str(), _s.c_str(), _pos + 1);
+                    break;
+                }
+            }
+        }
+    }
+
+    _ifs.close();
+
+    cpu_frequencies.resize(_ncpu);
+    cpu_mhz_pos = _cpu_mhz_pos;
+    ifs         = std::make_unique<std::ifstream>("/proc/cpuinfo", std::ifstream::binary);
+}
+
+void
+sample()
+{
+    cpu_mem_usage.emplace_back(tim::get_clock_real_now<size_t, std::nano>(),
+                               tim::get_page_rss());
+
+    if(!ifs) return;
+
+    auto _read_cpu_freq = [](size_t _idx) {
+        double _freq = 0;
+        ifs->seekg(cpu_mhz_pos.at(_idx), ifs->beg);
+        (*ifs) >> _freq;
+        return _freq;
+    };
+
+    auto _ts = tim::get_clock_real_now<size_t, std::nano>();
+    for(int64_t i = 0; i < ncpu; ++i)
+        cpu_frequencies.at(i).emplace_back(_ts, _read_cpu_freq(i));
+}
+
+void
+shutdown()
+{}
+
+void
+post_process()
+{
+    OMNITRACE_PRINT("Post-processing %zu cpu freqs and %zu memory usage entries\n",
+                    cpu_frequencies.size(), cpu_mem_usage.size());
+    auto _process_frequencies = [](size_t _idx) {
+        using counter_track = perfetto_counter_track<cpu_freq>;
+        if(!counter_track::exists(_idx))
+        {
+            auto _devname = TIMEMORY_JOIN("", "[CPU ", _idx, "] ");
+            auto addendum = [&](const char* _v) { return _devname + std::string{ _v }; };
+            counter_track::emplace(_idx, addendum("Frequency (S)"), "MHz");
+        }
+
+        for(auto& itr : cpu_frequencies.at(_idx))
+        {
+            uint64_t _ts   = itr.first;
+            double   _freq = itr.second;
+            TRACE_COUNTER("sampling", counter_track::at(_idx, 0), _ts, _freq);
+        }
+    };
+
+    auto _process_cpu_mem_usage = []() {
+        using counter_track = perfetto_counter_track<cpu_mem>;
+        if(!counter_track::exists(0))
+        {
+            auto _devname = TIMEMORY_JOIN("", "[CPU] ");
+            auto addendum = [&](const char* _v) { return _devname + std::string{ _v }; };
+            counter_track::emplace(0, addendum("Memory Usage (S)"), "MB");
+        }
+
+        for(auto& itr : cpu_mem_usage)
+        {
+            uint64_t _ts        = itr.first;
+            double   _mem_usage = itr.second;
+            TRACE_COUNTER("sampling", counter_track::at(0, 0), _ts,
+                          _mem_usage / units::megabyte);
+        }
+    };
+
+    _process_cpu_mem_usage();
+    for(size_t i = 0; i < cpu_frequencies.size(); ++i)
+        _process_frequencies(i);
+}
+}  // namespace cpu_freq
+}  // namespace omnitrace

source/lib/src/library/critical_trace.cpp

@@ -0,0 +1,837 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/critical_trace.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/defines.hpp"
+#include "library/perfetto.hpp"
+#include "library/ptl.hpp"
+
+#include <PTL/ThreadPool.hh>
+#include <timemory/backends/dmp.hpp>
+#include <timemory/backends/threading.hpp>
+#include <timemory/hash/types.hpp>
+#include <timemory/tpls/cereal/cereal/archives/json.hpp>
+#include <timemory/tpls/cereal/cereal/cereal.hpp>
+#include <timemory/utility/macros.hpp>
+#include <timemory/utility/types.hpp>
+#include <timemory/utility/utility.hpp>
+
+#include <cctype>
+#include <cstdint>
+#include <exception>
+#include <iomanip>
+#include <sstream>
+#include <stdexcept>
+#include <utility>
+
+namespace omnitrace
+{
+namespace critical_trace
+{
+namespace
+{
+using call_graph_t              = tim::graph<entry>;
+using call_graph_itr_t          = typename call_graph_t::iterator;
+using call_graph_sibling_itr_t  = typename call_graph_t::sibling_iterator;
+using call_graph_preorder_itr_t = typename call_graph_t::pre_order_iterator;
+
+hash_ids   complete_hash_ids{};
+call_chain complete_call_chain{};
+std::mutex complete_call_mutex{};
+
+void
+update_critical_path(call_chain _chain, int64_t _tid);
+
+void
+compute_critical_trace();
+
+void
+copy_hash_ids()
+{
+    // make copy to avoid parallel iteration issues
+    auto _hash_ids = complete_hash_ids;
+    // ensure all hash ids exist
+    for(const auto& itr : _hash_ids)
+        tim::hash::add_hash_id(itr);
+}
+}  // namespace
+}  // namespace critical_trace
+
+namespace critical_trace
+{
+namespace
+{
+template <typename Arg0, typename Arg1, typename... Args>
+size_t
+get_combined_hash(Arg0&& _zero, Arg1&& _one, Args&&... _args)
+{
+    size_t _hash = tim::hash::get_combined_hash_id(std::forward<Arg0>(_zero),
+                                                   std::forward<Arg1>(_one));
+    if constexpr(sizeof...(_args) == 0)
+    {
+        return _hash;
+    }
+    else
+    {
+        return get_combined_hash(_hash, std::forward<Args>(_args)...);
+    }
+}
+}  // namespace
+
+//--------------------------------------------------------------------------------------//
+//
+//                          ENTRY
+//
+//--------------------------------------------------------------------------------------//
+
+bool
+entry::operator==(const entry& rhs) const
+{
+    return (device == rhs.device && depth == rhs.depth && priority == rhs.priority &&
+            tid == rhs.tid && cpu_cid == rhs.cpu_cid && gpu_cid == rhs.gpu_cid &&
+            hash == rhs.hash);
+}
+
+bool
+entry::operator<(const entry& rhs) const
+{
+    // sort by cpu ids
+    auto _cpu_eq = (cpu_cid == rhs.cpu_cid);
+    if(!_cpu_eq) return (cpu_cid < rhs.cpu_cid);
+
+    // sort by gpu ids
+    if(gpu_cid > 0 && rhs.gpu_cid > 0)
+    {
+        auto _gpu_eq = (gpu_cid == rhs.gpu_cid);
+        if(!_gpu_eq) return (gpu_cid < rhs.gpu_cid);
+    }
+
+    // sort by parent ids
+    auto _par_eq = (parent_cid == rhs.parent_cid);
+    if(!_par_eq) return (parent_cid < rhs.parent_cid);
+
+    // sort by priority
+    auto _prio_eq = (priority == rhs.priority);
+    if(!_prio_eq) return (priority < rhs.priority);
+
+    // sort by timestamp (last resort)
+    return (begin_ns < rhs.begin_ns);
+}
+
+bool
+entry::operator>(const entry& rhs) const
+{
+    return (!(*this < rhs) && begin_ns != rhs.begin_ns && cpu_cid != rhs.cpu_cid &&
+            gpu_cid != rhs.gpu_cid);
+}
+
+entry&
+entry::operator+=(const entry& rhs)
+{
+    if(phase == Phase::BEGIN && rhs.phase == Phase::END)
+    {
+        assert(rhs.end_ns >= begin_ns);
+        end_ns = rhs.end_ns;
+        phase  = Phase::DELTA;
+        return *this;
+    }
+    else
+    {
+        OMNITRACE_CONDITIONAL_PRINT(
+            get_verbose() > 1,
+            "Warning! Incorrect phase. entry::operator+=(entry) is only valid for "
+            "Phase::BEGIN += Phase::END\n");
+    }
+    return *this;
+}
+
+size_t
+entry::get_hash() const
+{
+    return get_combined_hash(hash, static_cast<short>(device), static_cast<short>(phase),
+                             tid, cpu_cid, gpu_cid, priority);
+}
+
+int64_t
+entry::get_timestamp() const
+{
+    switch(phase)
+    {
+        case Phase::BEGIN: return begin_ns;
+        case Phase::END: return end_ns;
+        case Phase::DELTA: return (end_ns - begin_ns);
+        case Phase::NONE: break;
+    }
+    return 0;
+}
+
+int64_t
+entry::get_cost() const
+{
+    switch(phase)
+    {
+        case Phase::DELTA: return (end_ns - begin_ns);
+        default: break;
+    }
+    return 0;
+}
+
+int64_t
+entry::get_overlap(const entry& rhs) const
+{
+    if(begin_ns >= rhs.end_ns || end_ns >= rhs.begin_ns)  // no overlap
+        return 0;
+    else if(begin_ns >= rhs.begin_ns && end_ns <= rhs.end_ns)  // inclusive to rhs
+        return get_cost();
+    else if(begin_ns <= rhs.begin_ns && end_ns >= rhs.end_ns)  // rhs is inclusive
+        return rhs.get_cost();
+    else if(begin_ns <= rhs.begin_ns && end_ns <= rhs.end_ns)  // at beginning
+        return (end_ns - rhs.begin_ns);
+    else if(begin_ns >= rhs.begin_ns && end_ns >= rhs.end_ns)  // at end
+        return (rhs.end_ns - begin_ns);
+    else
+    {
+        OMNITRACE_PRINT("Warning! entry::get_overlap(entry, tid) "
+                        "could not determine the overlap :: %s\n",
+                        JOIN("", *this).c_str());
+    }
+    return 0;
+}
+
+int64_t
+entry::get_overlap(const entry& rhs, int64_t _tid) const
+{
+    if(!is_delta(*this, __FUNCTION__)) return 0;
+    if(!is_delta(rhs, __FUNCTION__)) return 0;
+
+    if(_tid < 0 || (this->tid == _tid && rhs.tid == _tid))  // all threads or same thread
+        return get_overlap(rhs);
+
+    return 0;
+}
+
+int64_t
+entry::get_independent(const entry& rhs) const
+{
+    if(begin_ns >= rhs.end_ns || end_ns >= rhs.begin_ns)  // no overlap
+        return get_cost();
+    else if(begin_ns >= rhs.begin_ns && end_ns <= rhs.end_ns)  // inclusive to rhs
+        return 0;
+    else if(begin_ns <= rhs.begin_ns && end_ns >= rhs.end_ns)  // rhs is inclusive
+        return get_cost() - rhs.get_cost();
+    else if(begin_ns <= rhs.begin_ns && end_ns <= rhs.end_ns)  // at beginning
+        return (rhs.begin_ns - begin_ns);
+    else if(begin_ns >= rhs.begin_ns && end_ns >= rhs.end_ns)  // at end
+        return (end_ns - rhs.end_ns);
+    else
+    {
+        OMNITRACE_PRINT("Warning! entry::get_independent(entry, tid) "
+                        "could not determine the overlap :: %s\n",
+                        JOIN("", *this).c_str());
+    }
+    return 0;
+}
+
+int64_t
+entry::get_independent(const entry& rhs, int64_t _tid) const
+{
+    if(!is_delta(*this, __FUNCTION__)) return 0;
+    if(!is_delta(rhs, __FUNCTION__)) return 0;
+
+    if(_tid < 0 || (this->tid == _tid && rhs.tid == _tid))  // all threads or same thread
+        return get_independent(rhs);
+    else if(this->tid == _tid && rhs.tid != _tid)  // rhs is on different thread
+        return get_cost();
+    return 0;
+}
+
+bool
+entry::is_bounded(const entry& rhs) const
+{
+    // ignores thread
+    return !(begin_ns < rhs.begin_ns || end_ns > rhs.end_ns);
+}
+
+bool
+entry::is_bounded(const entry& rhs, int64_t _tid) const
+{
+    if(this->tid == _tid && rhs.tid == _tid)  // all threads or same thread
+        return !(begin_ns < rhs.begin_ns || end_ns > rhs.end_ns);
+
+    return false;
+}
+
+void
+entry::write(std::ostream& _os) const
+{
+    if(device == Device::GPU)
+        _os << "[GPU][" << cpu_cid << "][" << gpu_cid << "]";
+    else
+        _os << "[CPU][" << cpu_cid << "]";
+    _os << " parent: " << static_cast<int64_t>(parent_cid);
+    _os << ", tid: " << tid;
+    _os << ", depth: " << depth;
+    _os << ", priority: " << priority;
+    if(phase == Phase::DELTA)
+    {
+        std::stringstream _cost{};
+        _cost << std::setprecision(4) << std::scientific << (get_timestamp() / 1.0e9);
+        _os << ", cost: [" << std::setw(8) << _cost.str() << " sec]";
+    }
+    else
+    {
+        _os << ", phase: ";
+        if(phase == Phase::BEGIN)
+            _os << "begin ";
+        else if(phase == Phase::END)
+            _os << "end ";
+        _os << "[" << begin_ns << ":" << end_ns << "]";
+    }
+    _os << ", hash: " << hash << " :: " << tim::demangle(tim::get_hash_identifier(hash));
+}
+
+bool
+entry::is_delta(const entry& _v, const std::string_view& _ctx)
+{
+    if(_v.phase != Phase::DELTA)
+    {
+        OMNITRACE_CT_DEBUG(
+            "Warning! Invalid phase for entry. entry::%s requires Phase::DELTA :: %s\n",
+            _ctx.data(), JOIN("", _v).c_str());
+        return true;
+    }
+    return false;
+}
+
+//--------------------------------------------------------------------------------------//
+//
+//                          CALL CHAIN
+//
+//--------------------------------------------------------------------------------------//
+
+bool
+call_chain::operator==(const call_chain& rhs) const
+{
+    if(size() != rhs.size()) return false;
+    for(size_t i = 0; i < size(); ++i)
+        if(at(i) != rhs.at(i)) return false;
+    return true;
+}
+
+size_t
+call_chain::get_hash() const
+{
+    if(empty()) return 0;
+    int64_t _hash = this->at(0).get_hash();
+    for(size_t i = 1; i < this->size(); ++i)
+        _hash = get_combined_hash(_hash, at(i).get_hash());
+    return _hash;
+}
+
+int64_t
+call_chain::get_cost(int64_t _tid) const
+{
+    int64_t _cost = 0;
+    if(_tid < 0)
+    {
+        for(const auto& itr : *this)
+            _cost += itr.get_cost();
+    }
+    else
+    {
+        for(const auto& itr : *this)
+        {
+            if(itr.tid == _tid) _cost += itr.get_cost();
+        }
+    }
+    return _cost;
+}
+
+int64_t
+call_chain::get_overlap(int64_t _tid) const
+{
+    int64_t _cost = 0;
+    auto    itr   = this->begin();
+    auto    nitr  = ++this->begin();
+    for(; nitr != this->end(); ++nitr, ++itr)
+        _cost += nitr->get_overlap(*itr, _tid);
+    return _cost;
+}
+
+int64_t
+call_chain::get_independent(int64_t _tid) const
+{
+    int64_t _cost = 0;
+    auto    itr   = this->begin();
+    auto    nitr  = ++this->begin();
+    for(; nitr != this->end(); ++nitr, ++itr)
+        _cost += itr->get_independent(*nitr, _tid);
+    return _cost;
+}
+
+std::vector<call_chain>&
+call_chain::get_top_chains()
+{
+    static std::vector<call_chain> _v{};
+    return _v;
+}
+
+template <>
+void
+call_chain::generate_perfetto<Device::NONE>(std::set<entry>& _used) const
+{
+    for(const auto& itr : *this)
+    {
+        if(!_used.emplace(itr).second) continue;
+        if(itr.device == Device::CPU)
+        {
+            TRACE_EVENT_BEGIN("device-critical-trace", "CPU",
+                              static_cast<uint64_t>(itr.begin_ns));
+        }
+        else if(itr.device == Device::GPU)
+        {
+            TRACE_EVENT_BEGIN("device-critical-trace", "GPU",
+                              static_cast<uint64_t>(itr.begin_ns));
+        }
+        TRACE_EVENT_END("device-critical-trace", static_cast<uint64_t>(itr.end_ns));
+    }
+}
+
+template <>
+void
+call_chain::generate_perfetto<Device::CPU>(std::set<entry>& _used) const
+{
+    static std::set<std::string> _static_strings{};
+    static std::mutex            _static_mutex{};
+    for(const auto& itr : *this)
+    {
+        if(!_used.emplace(itr).second) continue;
+        if(itr.device != Device::CPU) continue;
+        std::string _name = tim::demangle(tim::get_hash_identifier(itr.hash));
+        _static_mutex.lock();
+        auto sitr = _static_strings.emplace(_name);
+        _static_mutex.unlock();
+        TRACE_EVENT_BEGIN("host-critical-trace",
+                          perfetto::StaticString{ sitr.first->c_str() },
+                          static_cast<uint64_t>(itr.begin_ns));
+        TRACE_EVENT_END("host-critical-trace", static_cast<uint64_t>(itr.end_ns));
+    }
+}
+
+template <>
+void
+call_chain::generate_perfetto<Device::GPU>(std::set<entry>& _used) const
+{
+    static std::set<std::string> _static_strings{};
+    static std::mutex            _static_mutex{};
+    for(const auto& itr : *this)
+    {
+        if(!_used.emplace(itr).second) continue;
+        if(itr.device != Device::GPU) continue;
+        std::string _name = tim::demangle(tim::get_hash_identifier(itr.hash));
+        _static_mutex.lock();
+        auto sitr = _static_strings.emplace(_name);
+        _static_mutex.unlock();
+        TRACE_EVENT_BEGIN("device-critical-trace",
+                          perfetto::StaticString{ sitr.first->c_str() },
+                          static_cast<uint64_t>(itr.begin_ns));
+        TRACE_EVENT_END("device-critical-trace", static_cast<uint64_t>(itr.end_ns));
+    }
+}
+
+template <>
+void
+call_chain::generate_perfetto<Device::ANY>(std::set<entry>& _used) const
+{
+    static std::set<std::string> _static_strings{};
+    static std::mutex            _static_mutex{};
+    for(const auto& itr : *this)
+    {
+        if(!_used.emplace(itr).second) continue;
+        std::string _name = tim::demangle(tim::get_hash_identifier(itr.hash));
+        _static_mutex.lock();
+        auto sitr = _static_strings.emplace(_name);
+        _static_mutex.unlock();
+        TRACE_EVENT_BEGIN("critical-trace", perfetto::StaticString{ sitr.first->c_str() },
+                          static_cast<uint64_t>(itr.begin_ns));
+        TRACE_EVENT_END("critical-trace", static_cast<uint64_t>(itr.end_ns));
+    }
+}
+
+//--------------------------------------------------------------------------------------//
+//
+//                          FREE FUNCTIONS
+//
+//--------------------------------------------------------------------------------------//
+
+uint64_t
+get_update_frequency()
+{
+    return get_critical_trace_update_freq();
+}
+
+std::unique_ptr<call_chain>&
+get(int64_t _tid)
+{
+    static auto&             _v    = thread_data<call_chain>::instances();
+    static thread_local auto _once = [_tid]() {
+        if(!_v.at(0)) _v.at(0) = std::make_unique<call_chain>();
+        if(!_v.at(_tid)) _v.at(_tid) = std::make_unique<call_chain>();
+        if(_tid > 0) *_v.at(_tid) = *_v.at(0);
+        return true;
+    }();
+    (void) _once;
+    return _v.at(_tid);
+}
+
+void
+add_hash_id(const hash_ids& _labels)
+{
+    std::unique_lock<std::mutex> _lk{ tasking::get_critical_trace_mutex(),
+                                      std::defer_lock };
+    if(!_lk.owns_lock()) _lk.lock();
+    tasking::get_critical_trace_task_group().run([_labels]() {
+        static std::mutex _mtx{};
+        _mtx.lock();
+        for(auto itr : _labels)
+            complete_hash_ids.emplace(std::move(itr));
+        _mtx.unlock();
+    });
+}
+
+size_t
+add_hash_id(const std::string& _label)
+{
+    using critical_trace_hash_data =
+        thread_data<critical_trace::hash_ids, critical_trace::id>;
+
+    auto _hash = tim::hash::add_hash_id(_label);
+    if(get_use_critical_trace() || get_use_rocm_smi())
+    {
+        critical_trace_hash_data::construct();
+        critical_trace_hash_data::instance()->emplace(_label);
+    }
+    return _hash;
+}
+
+void
+update(int64_t _tid)
+{
+    if(!get_use_critical_trace() && !get_use_rocm_smi()) return;
+    std::unique_lock<std::mutex> _lk{ tasking::get_critical_trace_mutex(),
+                                      std::defer_lock };
+    if(!_lk.owns_lock()) _lk.lock();
+    call_chain _data{};
+    std::swap(_data, *critical_trace::get(_tid));
+    tasking::get_critical_trace_task_group().exec(update_critical_path, _data, _tid);
+}
+
+void
+compute(int64_t _tid)
+{
+    update(_tid);
+    tasking::get_critical_trace_task_group().exec(compute_critical_trace);
+}
+
+//--------------------------------------------------------------------------------------//
+//
+//                          HELPER FUNCTIONS
+//
+//--------------------------------------------------------------------------------------//
+
+namespace
+{
+std::string
+get_perf_name(std::string _func)
+{
+    const auto _npos = std::string::npos;
+    auto       _pos  = std::string::npos;
+    while((_pos = _func.find('_')) != _npos)
+        _func = _func.replace(_pos, 1, " ");
+    if(_func.length() > 0) _func.at(0) = std::toupper(_func.at(0));
+    return _func;
+}
+
+void
+save_call_chain_json(const std::string& _fname, const std::string& _label,
+                     const call_chain& _call_chain, bool _msg = false,
+                     std::string _func = {})
+{
+    OMNITRACE_CT_DEBUG("[%s]\n", __FUNCTION__);
+
+    using perfstats_t =
+        tim::lightweight_tuple<comp::wall_clock, comp::peak_rss, comp::page_rss>;
+    perfstats_t _perf{ get_perf_name(__FUNCTION__) };
+    _perf.start();
+
+    auto _save = [&](std::ostream& _os) {
+        namespace cereal = tim::cereal;
+        auto ar = tim::policy::output_archive<cereal::MinimalJSONOutputArchive>::get(_os);
+
+        auto _hash_map = *tim::hash::get_hash_ids();
+        for(auto& itr : _hash_map)
+            itr.second = tim::demangle(itr.second);
+        ar->setNextName("omnitrace");
+        ar->startNode();
+        (*ar)(cereal::make_nvp("hash_map", _hash_map),
+              cereal::make_nvp(_label.c_str(), _call_chain));
+        ar->finishNode();
+    };
+
+    std::ofstream ofs{};
+    if(tim::filepath::open(ofs, _fname))
+    {
+        if(_msg)
+        {
+            if(_func.empty()) _func = __FUNCTION__;
+            OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, "[%s] Outputting '%s'...\n",
+                                        _func.c_str(), _fname.c_str());
+        }
+        std::stringstream oss{};
+        if(_call_chain.size() > 100000)
+        {
+            _save(ofs);
+        }
+        else
+        {
+            _save(oss);
+            ofs << oss.str() << std::endl;
+        }
+    }
+
+    _perf.stop();
+    if(_msg)
+    {
+        OMNITRACE_CT_DEBUG("%s\n", JOIN("", _perf).c_str());
+    }
+}
+
+template <typename Tp, template <typename...> class ContainerT, typename... Args,
+          typename FuncT = bool (*)(const Tp&, const Tp&)>
+inline Tp*
+find(
+    const Tp& _v, ContainerT<Tp, Args...>& _vec,
+    FuncT&& _func = [](const Tp& _lhs, const Tp& _rhs) { return (_lhs == _rhs); })
+{
+    for(auto& itr : _vec)
+    {
+        if(std::forward<FuncT>(_func)(_v, itr)) return &itr;
+    }
+    return nullptr;
+};
+
+template <typename FuncT = bool (*)(const entry&, const entry&)>
+inline entry*
+find(
+    const entry& _v, call_chain& _vec,
+    FuncT&& _func = [](const entry& _lhs, const entry& _rhs) { return (_lhs == _rhs); })
+{
+    return find(_v, reinterpret_cast<std::vector<entry>&>(_vec),
+                std::forward<FuncT>(_func));
+}
+
+void
+squash_critical_path(call_chain& _targ)
+{
+    OMNITRACE_CT_DEBUG("[%s]\n", __FUNCTION__);
+    static auto _strict_equal = [](const entry& _lhs, const entry& _rhs) {
+        auto _same_phase  = (_lhs.phase == _rhs.phase);
+        bool _phase_check = true;
+        if(_same_phase) _phase_check = (_lhs.get_timestamp() == _rhs.get_timestamp());
+        return (_lhs == _rhs && _lhs.parent_cid == _rhs.parent_cid && _phase_check);
+    };
+
+    std::sort(_targ.begin(), _targ.end());
+
+    call_chain _squashed{};
+    for(auto& itr : _targ)
+    {
+        if(itr.phase == Phase::DELTA)
+        {
+            _squashed.emplace_back(itr);
+        }
+        else if(itr.phase == Phase::BEGIN)
+        {
+            if(!find(itr, _squashed, _strict_equal)) _squashed.emplace_back(itr);
+        }
+        else
+        {
+            entry* _match = nullptr;
+            if((_match = find(itr, _squashed)) != nullptr)
+                *_match += itr;
+            else
+                _squashed.emplace_back(itr);
+        }
+    }
+
+    std::swap(_targ, _squashed);
+    std::sort(_targ.begin(), _targ.end());
+}
+
+void
+combine_critical_path(call_chain& _targ, call_chain _chain)
+{
+    OMNITRACE_CT_DEBUG("[%s]\n", __FUNCTION__);
+    call_chain _delta{};
+    call_chain _begin{};
+    call_chain _end{};
+    for(auto& itr : _chain)
+    {
+        if(itr.phase == Phase::DELTA)
+            _delta.emplace_back(itr);
+        else if(itr.phase == Phase::BEGIN)
+            _begin.emplace_back(itr);
+        else if(itr.phase == Phase::END)
+        {
+            entry* _match = nullptr;
+            if((_match = find(itr, _begin)) != nullptr)
+                *_match += itr;
+            else
+                _end.emplace_back(itr);
+        }
+    }
+
+    call_chain _combined{};
+    _combined.reserve(_delta.size() + _begin.size() + _end.size());
+    for(auto& itr : _delta)
+        _combined.emplace_back(itr);
+    for(auto& itr : _begin)
+        _combined.emplace_back(itr);
+    for(auto& itr : _end)
+        _combined.emplace_back(itr);
+    std::sort(_combined.begin(), _combined.end());
+
+    std::unique_lock<std::mutex> _lk{ complete_call_mutex };
+    for(auto& itr : _combined)
+        _targ.emplace_back(itr);
+
+    // squash_critical_path(_targ);
+}
+
+void
+update_critical_path(call_chain _chain, int64_t)
+{
+    OMNITRACE_CT_DEBUG("[%s]\n", __FUNCTION__);
+    try
+    {
+        // remove any data not
+        // auto _diff_tid = [_tid](const entry& _v) { return _v.tid != _tid; };
+        //_chain.erase(std::remove_if(_chain.begin(), _chain.end(), _diff_tid),
+        //             _chain.end());
+        combine_critical_path(complete_call_chain, std::move(_chain));
+    } catch(const std::exception& e)
+    {
+        std::cerr << "Thread exited with exception: " << e.what() << std::endl;
+        TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(true, 32);
+    }
+}
+
+void
+compute_critical_trace()
+{
+    static bool                  _computed = false;
+    std::unique_lock<std::mutex> _lk{ complete_call_mutex };
+
+    if(_computed) return;
+
+    OMNITRACE_CONDITIONAL_PRINT(get_critical_trace_debug() || get_verbose() >= 0,
+                                "[%s] Generating critical trace...\n", __FUNCTION__);
+
+    // ensure all hash ids exist
+    copy_hash_ids();
+
+    using perfstats_t =
+        tim::lightweight_tuple<comp::wall_clock, comp::peak_rss, comp::page_rss>;
+
+    perfstats_t _ct_perf{ JOIN("", "[", __FUNCTION__, "]") };
+    _ct_perf.start();
+
+    try
+    {
+        OMNITRACE_CT_DEBUG("[%s] initial call chain: %zu entries\n", __FUNCTION__,
+                           complete_call_chain.size());
+
+        perfstats_t _perf{ get_perf_name(__FUNCTION__) };
+        _perf.start();
+
+        std::sort(complete_call_chain.begin(), complete_call_chain.end());
+
+        _perf.stop().rekey("Sorting critical trace");
+        OMNITRACE_CT_DEBUG("%s\n", JOIN("", _perf).c_str());
+
+        _perf.reset().start();
+        save_call_chain_json(
+            tim::settings::compose_output_filename("call-chain", ".json"), "call_chain",
+            complete_call_chain, true, __FUNCTION__);
+
+        _perf.stop().rekey("Save call-chain");
+        OMNITRACE_CT_DEBUG("%s\n", JOIN("", _perf).c_str());
+
+    } catch(std::exception& e)
+    {
+        OMNITRACE_PRINT("Thread exited '%s' with exception: %s\n", __FUNCTION__,
+                        e.what());
+        TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(true, 32);
+    }
+
+    _ct_perf.stop();
+    auto _ct_msg = JOIN("", _ct_perf);
+    auto _ct_pos = _ct_msg.find(">>>  ");
+    if(_ct_pos != std::string::npos) _ct_msg = _ct_msg.substr(_ct_pos + 5);
+    OMNITRACE_PRINT("%s\n", _ct_msg.c_str());
+}
+}  // namespace
+
+std::vector<std::pair<std::string, entry>>
+get_entries(int64_t _ts, const std::function<bool(const entry&)>& _eval)
+{
+    copy_hash_ids();
+    auto _func = [_eval, _ts](std::vector<std::pair<std::string, entry>>* _targ,
+                              size_t*                                     _avail) {
+        copy_hash_ids();
+        squash_critical_path(complete_call_chain);
+        *_avail = complete_call_chain.size();
+        std::vector<std::pair<std::string, entry>> _v{};
+        std::sort(complete_call_chain.begin(), complete_call_chain.end());
+        for(const auto& itr : complete_call_chain)
+        {
+            if(itr.phase != Phase::DELTA) continue;
+            if(itr.begin_ns <= _ts && itr.end_ns >= _ts)
+            {
+                if(_eval(itr)) _v.emplace_back(tim::get_hash_identifier(itr.hash), itr);
+            }
+        }
+        *_targ = _v;
+    };
+    size_t                                     _n = 0;
+    std::vector<std::pair<std::string, entry>> _v{};
+    tasking::get_critical_trace_task_group().exec(_func, &_v, &_n);
+    tasking::get_critical_trace_task_group().join();
+    OMNITRACE_DEBUG("critical_trace::%s :: found %zu out of %zu entries at %li...\n",
+                    __FUNCTION__, _v.size(), _n, _ts);
+    return _v;
+}
+
+}  // namespace critical_trace
+}  // namespace omnitrace

source/lib/src/library/gpu.cpp

@@ -0,0 +1,52 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/gpu.hpp"
+
+#if defined(OMNITRACE_USE_ROCM_SMI)
+#    include "library/components/rocm_smi.hpp"
+#elif defined(OMNITRACE_USE_HIP)
+#    if !defined(TIMEMORY_USE_HIP)
+#        define TIMEMORY_USE_HIP 1
+#    endif
+#    include "timemory/components/hip/backends.hpp"
+#endif
+
+namespace omnitrace
+{
+namespace gpu
+{
+int
+device_count()
+{
+#if defined(OMNITRACE_USE_ROCM_SMI)
+    // store as static since calls after rsmi_shutdown will return zero
+    static auto _v = rocm_smi::device_count();
+    return _v;
+#elif defined(OMNITRACE_USE_HIP)
+    return ::tim::hip::device_count();
+#else
+    return 0;
+#endif
+}
+}  // namespace gpu
+}  // namespace omnitrace

source/lib/src/library/kokkosp.cpp

@@ -0,0 +1,322 @@
+// MIT License
+//
+// Copyright (c) 2020, The Regents of the University of California,
+// through Lawrence Berkeley National Laboratory (subject to receipt of any
+// required approvals from the U.S. Dept. of Energy).  All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// used by Kokkos decls
+#if !defined(TIMEMORY_LIBRARY_SOURCE)
+#    define TIMEMORY_LIBRARY_SOURCE 1
+#endif
+
+#include "library/components/omnitrace.hpp"
+#include "library/config.hpp"
+
+#include <timemory/api/kokkosp.hpp>
+
+namespace kokkosp = tim::kokkosp;
+
+//--------------------------------------------------------------------------------------//
+
+namespace tim
+{
+template <>
+inline auto
+invoke_preinit<kokkosp::memory_tracker>(long)
+{
+    kokkosp::memory_tracker::label()       = "kokkos_memory";
+    kokkosp::memory_tracker::description() = "Kokkos Memory tracker";
+}
+}  // namespace tim
+
+//--------------------------------------------------------------------------------------//
+
+namespace
+{
+std::string kokkos_banner =
+    "#---------------------------------------------------------------------------#";
+
+//--------------------------------------------------------------------------------------//
+
+bool enable_kernel_logger = false;
+
+inline void
+add_kernel_logger()
+{
+    static bool _first = true;
+    if(!_first) return;
+    _first         = false;
+    using strvec_t = std::vector<std::string>;
+
+    tim::settings::instance()->insert<bool, bool&>(
+        std::string{ "OMNITRACE_KOKKOS_KERNEL_LOGGER" }, std::string{},
+        std::string{ "Enables kernel logging" }, enable_kernel_logger,
+        strvec_t({ "--omnitrace-kokkos-kernel-logger" }));
+}
+
+inline void
+setup_kernel_logger()
+{
+    if(tim::settings::debug() || tim::settings::verbose() > 3 || enable_kernel_logger)
+    {
+        kokkosp::logger_t::get_initializer() = [](kokkosp::logger_t& _obj) {
+            _obj.initialize<kokkosp::kernel_logger>();
+        };
+    }
+}
+
+}  // namespace
+
+//--------------------------------------------------------------------------------------//
+
+extern "C"
+{
+    void kokkosp_print_help(char*) {}
+
+    void kokkosp_parse_args(int, char**) {}
+
+    void kokkosp_declare_metadata(const char* key, const char* value)
+    {
+        tim::manager::add_metadata(key, value);
+    }
+
+    void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
+                              const uint32_t devInfoCount, void* deviceInfo)
+    {
+        add_kernel_logger();
+
+        tim::consume_parameters(devInfoCount, deviceInfo);
+
+        printf("%s\n", kokkos_banner.c_str());
+        printf("# KokkosP: omnitrace connector (sequence is %d, version: %llu)\n",
+               loadSeq, (unsigned long long) interfaceVer);
+        printf("%s\n", kokkos_banner.c_str());
+
+        setup_kernel_logger();
+
+        tim::trait::runtime_enabled<kokkosp::memory_tracker>::set(
+            omnitrace::config::get_use_timemory());
+    }
+
+    void kokkosp_finalize_library()
+    {
+        printf("%s\n", kokkos_banner.c_str());
+        printf("# KokkosP: Finalization of omnitrace connector. Complete.\n");
+        printf("%s\n", kokkos_banner.c_str());
+
+        kokkosp::cleanup();
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_begin_parallel_for(const char* name, uint32_t devid, uint64_t* kernid)
+    {
+        auto pname =
+            (devid > std::numeric_limits<uint16_t>::max())  // junk device number
+                ? TIMEMORY_JOIN(" ", "[kokkos]", name)
+                : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name);
+        *kernid = kokkosp::get_unique_id();
+        kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid);
+        kokkosp::create_profiler<omnitrace::component::omnitrace>(pname, *kernid);
+        kokkosp::start_profiler<omnitrace::component::omnitrace>(*kernid);
+    }
+
+    void kokkosp_end_parallel_for(uint64_t kernid)
+    {
+        kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid);
+        kokkosp::stop_profiler<omnitrace::component::omnitrace>(kernid);
+        kokkosp::destroy_profiler<omnitrace::component::omnitrace>(kernid);
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_begin_parallel_reduce(const char* name, uint32_t devid, uint64_t* kernid)
+    {
+        auto pname =
+            (devid > std::numeric_limits<uint16_t>::max())  // junk device number
+                ? TIMEMORY_JOIN(" ", "[kokkos]", name)
+                : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name);
+        *kernid = kokkosp::get_unique_id();
+        kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid);
+        kokkosp::create_profiler<omnitrace::component::omnitrace>(pname, *kernid);
+        kokkosp::start_profiler<omnitrace::component::omnitrace>(*kernid);
+    }
+
+    void kokkosp_end_parallel_reduce(uint64_t kernid)
+    {
+        kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid);
+        kokkosp::stop_profiler<omnitrace::component::omnitrace>(kernid);
+        kokkosp::destroy_profiler<omnitrace::component::omnitrace>(kernid);
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_begin_parallel_scan(const char* name, uint32_t devid, uint64_t* kernid)
+    {
+        auto pname =
+            (devid > std::numeric_limits<uint16_t>::max())  // junk device number
+                ? TIMEMORY_JOIN(" ", "[kokkos]", name)
+                : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name);
+        *kernid = kokkosp::get_unique_id();
+        kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid);
+        kokkosp::create_profiler<omnitrace::component::omnitrace>(pname, *kernid);
+        kokkosp::start_profiler<omnitrace::component::omnitrace>(*kernid);
+    }
+
+    void kokkosp_end_parallel_scan(uint64_t kernid)
+    {
+        kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid);
+        kokkosp::stop_profiler<omnitrace::component::omnitrace>(kernid);
+        kokkosp::destroy_profiler<omnitrace::component::omnitrace>(kernid);
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_begin_fence(const char* name, uint32_t devid, uint64_t* kernid)
+    {
+        auto pname =
+            (devid > std::numeric_limits<uint16_t>::max())  // junk device number
+                ? TIMEMORY_JOIN(" ", "[kokkos]", name)
+                : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name);
+        *kernid = kokkosp::get_unique_id();
+        kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid);
+        kokkosp::create_profiler<omnitrace::component::omnitrace>(pname, *kernid);
+        kokkosp::start_profiler<omnitrace::component::omnitrace>(*kernid);
+    }
+
+    void kokkosp_end_fence(uint64_t kernid)
+    {
+        kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid);
+        kokkosp::stop_profiler<omnitrace::component::omnitrace>(kernid);
+        kokkosp::destroy_profiler<omnitrace::component::omnitrace>(kernid);
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_push_profile_region(const char* name)
+    {
+        kokkosp::logger_t{}.mark(1, __FUNCTION__, name);
+        kokkosp::get_profiler_stack<omnitrace::component::omnitrace>().push_back(
+            kokkosp::profiler_t<omnitrace::component::omnitrace>(name));
+        kokkosp::get_profiler_stack<omnitrace::component::omnitrace>().back().start();
+    }
+
+    void kokkosp_pop_profile_region()
+    {
+        kokkosp::logger_t{}.mark(-1, __FUNCTION__);
+        if(kokkosp::get_profiler_stack<omnitrace::component::omnitrace>().empty()) return;
+        kokkosp::get_profiler_stack<omnitrace::component::omnitrace>().back().stop();
+        kokkosp::get_profiler_stack<omnitrace::component::omnitrace>().pop_back();
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_create_profile_section(const char* name, uint32_t* secid)
+    {
+        *secid     = kokkosp::get_unique_id();
+        auto pname = TIMEMORY_JOIN(" ", "[kokkos]", name);
+        kokkosp::create_profiler<omnitrace::component::omnitrace>(pname, *secid);
+    }
+
+    void kokkosp_destroy_profile_section(uint32_t secid)
+    {
+        kokkosp::destroy_profiler<omnitrace::component::omnitrace>(secid);
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_start_profile_section(uint32_t secid)
+    {
+        kokkosp::logger_t{}.mark(1, __FUNCTION__, secid);
+        kokkosp::start_profiler<omnitrace::component::omnitrace>(secid);
+    }
+
+    void kokkosp_stop_profile_section(uint32_t secid)
+    {
+        kokkosp::logger_t{}.mark(-1, __FUNCTION__, secid);
+        kokkosp::start_profiler<omnitrace::component::omnitrace>(secid);
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_allocate_data(const SpaceHandle space, const char* label,
+                               const void* const ptr, const uint64_t size)
+    {
+        kokkosp::logger_t{}.mark(0, __FUNCTION__, space.name, label,
+                                 TIMEMORY_JOIN("", '[', ptr, ']'), size);
+        kokkosp::profiler_alloc_t<>{ TIMEMORY_JOIN(" ", "[kokkos][allocate]", space.name,
+                                                   label) }
+            .store(std::plus<int64_t>{}, size);
+    }
+
+    void kokkosp_deallocate_data(const SpaceHandle space, const char* label,
+                                 const void* const ptr, const uint64_t size)
+    {
+        kokkosp::logger_t{}.mark(0, __FUNCTION__, space.name, label,
+                                 TIMEMORY_JOIN("", '[', ptr, ']'), size);
+        kokkosp::profiler_alloc_t<>{ TIMEMORY_JOIN(" ", "[kokkos][deallocate]",
+                                                   space.name, label) }
+            .store(std::plus<int64_t>{}, size);
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_begin_deep_copy(SpaceHandle dst_handle, const char* dst_name,
+                                 const void* dst_ptr, SpaceHandle src_handle,
+                                 const char* src_name, const void* src_ptr, uint64_t size)
+    {
+        kokkosp::logger_t{}.mark(1, __FUNCTION__, dst_handle.name, dst_name,
+                                 TIMEMORY_JOIN("", '[', dst_ptr, ']'), src_handle.name,
+                                 src_name, TIMEMORY_JOIN("", '[', src_ptr, ']'), size);
+
+        auto name = TIMEMORY_JOIN(" ", "[kokkos][deep_copy]",
+                                  TIMEMORY_JOIN('=', dst_handle.name, dst_name),
+                                  TIMEMORY_JOIN('=', src_handle.name, src_name));
+
+        auto& _data = kokkosp::get_profiler_stack<omnitrace::component::omnitrace>();
+        _data.emplace_back(name);
+        _data.back().audit(dst_handle, dst_name, dst_ptr, src_handle, src_name, src_ptr,
+                           size);
+        _data.back().start();
+        _data.back().store(std::plus<int64_t>{}, size);
+    }
+
+    void kokkosp_end_deep_copy()
+    {
+        kokkosp::logger_t{}.mark(-1, __FUNCTION__);
+        auto& _data = kokkosp::get_profiler_stack<omnitrace::component::omnitrace>();
+        if(_data.empty()) return;
+        _data.back().store(std::minus<int64_t>{}, 0);
+        _data.back().stop();
+        _data.pop_back();
+    }
+
+    //----------------------------------------------------------------------------------//
+
+    void kokkosp_profile_event(const char* name)
+    {
+        kokkosp::profiler_t<omnitrace::component::omnitrace>{}.mark(name);
+    }
+
+    //----------------------------------------------------------------------------------//
+}
+
+TIMEMORY_INITIALIZE_STORAGE(kokkosp::memory_tracker)

source/lib/src/library/perfetto.cpp

@@ -0,0 +1,29 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/perfetto.hpp"
+
+PERFETTO_TRACK_EVENT_STATIC_STORAGE();
+
+#if defined(CUSTOM_DATA_SOURCE)
+PERFETTO_DEFINE_DATA_SOURCE_STATIC_MEMBERS(CustomDataSource);
+#endif

source/lib/src/library/ptl.cpp

@@ -0,0 +1,98 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/ptl.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/defines.hpp"
+#include "library/sampling.hpp"
+
+#include <PTL/ThreadPool.hh>
+#include <timemory/utility/declaration.hpp>
+
+namespace omnitrace
+{
+namespace tasking
+{
+namespace
+{
+auto _thread_pool_cfg = []() {
+    PTL::ThreadPool::Config _v{};
+    _v.init         = true;
+    _v.use_affinity = false;
+    _v.use_tbb      = false;
+    _v.initializer  = []() {
+        sampling::block_signals();
+        threading::set_thread_name(
+            TIMEMORY_JOIN('.', "ptl", PTL::Threading::GetThreadId()).c_str());
+    };
+    _v.finalizer = []() {};
+    _v.priority  = 5;
+    _v.pool_size = 1;
+    return _v;
+}();
+}
+
+std::mutex&
+get_roctracer_mutex()
+{
+    static std::mutex _v{};
+    return _v;
+}
+
+PTL::ThreadPool&
+get_roctracer_thread_pool()
+{
+    static auto _v = PTL::ThreadPool{ _thread_pool_cfg };
+    return _v;
+}
+
+PTL::TaskGroup<void>&
+get_roctracer_task_group()
+{
+    static PTL::TaskGroup<void> _v{ &get_roctracer_thread_pool() };
+    return _v;
+}
+
+std::mutex&
+get_critical_trace_mutex()
+{
+    static std::mutex _v{};
+    return _v;
+}
+
+PTL::ThreadPool&
+get_critical_trace_thread_pool()
+{
+    static auto _v = PTL::ThreadPool{ _thread_pool_cfg };
+    return _v;
+}
+
+PTL::TaskGroup<void>&
+get_critical_trace_task_group()
+{
+    static PTL::TaskGroup<void> _v{ &get_critical_trace_thread_pool() };
+    return _v;
+}
+
+}  // namespace tasking
+}  // namespace omnitrace

source/lib/src/library/sampling.cpp

@@ -0,0 +1,188 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/sampling.hpp"
+#include "library/components/fwd.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/ptl.hpp"
+
+#include <timemory/backends/papi.hpp>
+#include <timemory/backends/threading.hpp>
+#include <timemory/components/data_tracker/components.hpp>
+#include <timemory/components/macros.hpp>
+#include <timemory/components/papi/extern.hpp>
+#include <timemory/components/papi/papi_array.hpp>
+#include <timemory/components/papi/papi_vector.hpp>
+#include <timemory/components/timing/backends.hpp>
+#include <timemory/components/trip_count/extern.hpp>
+#include <timemory/macros.hpp>
+#include <timemory/math.hpp>
+#include <timemory/mpl.hpp>
+#include <timemory/mpl/quirks.hpp>
+#include <timemory/mpl/type_traits.hpp>
+#include <timemory/operations.hpp>
+#include <timemory/sampling/allocator.hpp>
+#include <timemory/sampling/sampler.hpp>
+#include <timemory/storage.hpp>
+#include <timemory/utility/backtrace.hpp>
+#include <timemory/utility/demangle.hpp>
+#include <timemory/utility/types.hpp>
+#include <timemory/variadic.hpp>
+
+#include <array>
+#include <cstring>
+#include <ctime>
+#include <initializer_list>
+#include <mutex>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+#include <pthread.h>
+#include <signal.h>
+
+namespace omnitrace
+{
+namespace sampling
+{
+using bundle_t  = tim::lightweight_tuple<backtrace>;
+using sampler_t = tim::sampling::sampler<bundle_t, tim::sampling::dynamic>;
+}  // namespace sampling
+}  // namespace omnitrace
+
+namespace omnitrace
+{
+namespace sampling
+{
+using hw_counters               = typename component::backtrace::hw_counters;
+using signal_type_instances     = thread_data<std::set<int>, api::sampling>;
+using backtrace_init_instances  = thread_data<backtrace, api::sampling>;
+using sampler_running_instances = thread_data<bool, api::sampling>;
+using papi_vector_instances     = thread_data<hw_counters, api::sampling>;
+
+namespace
+{
+template <typename... Args>
+void
+thread_sigmask(Args... _args)
+{
+    auto _err = pthread_sigmask(_args...);
+    if(_err != 0)
+    {
+        errno = _err;
+        perror("pthread_sigmask");
+        exit(EXIT_FAILURE);
+    }
+}
+
+template <typename Tp>
+sigset_t
+get_signal_set(Tp&& _v)
+{
+    sigset_t _sigset;
+    sigemptyset(&_sigset);
+    for(auto itr : _v)
+        sigaddset(&_sigset, itr);
+    return _sigset;
+}
+
+template <typename Tp>
+std::string
+get_signal_names(Tp&& _v)
+{
+    std::string _sig_names{};
+    for(auto&& itr : _v)
+        _sig_names += std::get<0>(tim::signal_settings::get_info(
+                          static_cast<tim::sys_signal>(itr))) +
+                      " ";
+    return _sig_names.substr(0, _sig_names.length() - 1);
+}
+}  // namespace
+
+std::unique_ptr<std::set<int>>&
+get_signal_types(int64_t _tid)
+{
+    static auto& _v = signal_type_instances::instances();
+    // on the main thread, use both SIGALRM and SIGPROF.
+    // on secondary threads, only use SIGPROF.
+    signal_type_instances::construct((_tid == 0) ? std::set<int>{ SIGALRM, SIGPROF }
+                                                 : std::set<int>{ SIGPROF });
+    return _v.at(_tid);
+}
+
+std::set<int>
+setup()
+{
+    if(!get_use_sampling()) return std::set<int>{};
+    return backtrace::configure(true);
+}
+
+std::set<int>
+shutdown()
+{
+    return backtrace::configure(false);
+}
+
+void
+block_signals(std::set<int> _signals)
+{
+    if(_signals.empty()) _signals = *get_signal_types(threading::get_id());
+    if(_signals.empty())
+    {
+        OMNITRACE_PRINT("No signals to block...\n");
+        return;
+    }
+
+    OMNITRACE_DEBUG("Blocking signals [%s] on thread #%lu...\n",
+                    get_signal_names(_signals).c_str(), threading::get_id());
+
+    sigset_t _v = get_signal_set(_signals);
+    thread_sigmask(SIG_BLOCK, &_v, nullptr);
+}
+
+void
+unblock_signals(std::set<int> _signals)
+{
+    if(_signals.empty()) _signals = *get_signal_types(threading::get_id());
+    if(_signals.empty())
+    {
+        OMNITRACE_PRINT("No signals to unblock...\n");
+        return;
+    }
+
+    OMNITRACE_DEBUG("Unblocking signals [%s] on thread #%lu...\n",
+                    get_signal_names(_signals).c_str(), threading::get_id());
+
+    sigset_t _v = get_signal_set(_signals);
+    thread_sigmask(SIG_UNBLOCK, &_v, nullptr);
+}
+
+std::unique_ptr<sampler_t>&
+get_sampler(int64_t _tid)
+{
+    static auto& _v = sampler_instances::instances();
+    return _v.at(_tid);
+}
+}  // namespace sampling
+}  // namespace omnitrace

source/lib/src/library/state.cpp

@@ -0,0 +1,53 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/state.hpp"
+
+#include <string>
+
+namespace std
+{
+std::string
+to_string(omnitrace::State _v)
+{
+    switch(_v)
+    {
+        case omnitrace::State::DelayedInit: return "DelayedInit";
+        case omnitrace::State::PreInit: return "PreInit";
+        case omnitrace::State::Init: return "Init";
+        case omnitrace::State::Active: return "Active";
+        case omnitrace::State::Finalized: return "Finalized";
+    }
+    return {};
+}
+
+std::string
+to_string(omnitrace::Mode _v)
+{
+    switch(_v)
+    {
+        case omnitrace::Mode::Trace: return "Trace";
+        case omnitrace::Mode::Sampling: return "Sampling";
+    }
+    return {};
+}
+}  // namespace std

source/lib/src/library/thread_data.cpp

@@ -0,0 +1,33 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/thread_data.hpp"
+
+namespace omnitrace
+{
+instrumentation_bundles::instance_array_t&
+instrumentation_bundles::instances()
+{
+    static auto _v = instance_array_t{};
+    return _v;
+}
+}  // namespace omnitrace

source/lib/src/library/thread_sampler.cpp

@@ -0,0 +1,196 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/thread_sampler.hpp"
+#include "library/components/rocm_smi.hpp"
+#include "library/config.hpp"
+#include "library/cpu_freq.hpp"
+#include "library/debug.hpp"
+
+#include <memory>
+#include <vector>
+
+namespace omnitrace
+{
+namespace thread_sampler
+{
+namespace
+{
+using tim::type_mutex;
+using auto_lock_t                                       = tim::auto_lock_t;
+using promise_t                                         = std::promise<void>;
+std::unique_ptr<promise_t>             polling_finished = {};
+std::vector<std::unique_ptr<instance>> instances        = {};
+
+bool&
+is_initialized()
+{
+    static bool _v = false;
+    return _v;
+}
+
+std::unique_ptr<std::thread>&
+get_thread()
+{
+    static std::unique_ptr<std::thread> _v;
+    return _v;
+}
+
+std::atomic<State>&
+get_sampler_state()
+{
+    static std::atomic<State> _v{ State::PreInit };
+    return _v;
+}
+}  // namespace
+
+void
+sampler::poll(std::atomic<State>* _state, nsec_t _interval, promise_t* _ready)
+{
+    threading::set_thread_name("omni.sampler");
+
+    // notify thread started
+    if(_ready) _ready->set_value();
+
+    for(auto& itr : instances)
+        itr->config();
+
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(
+        get_verbose() > 0 || get_debug(),
+        "Thread sampler polling at an interval of %f seconds...\n",
+        std::chrono::duration_cast<std::chrono::duration<double>>(_interval).count());
+
+    auto _now = std::chrono::steady_clock::now();
+    while(_state && _state->load() != State::Finalized && get_state() != State::Finalized)
+    {
+        std::this_thread::sleep_until(_now);
+        if(_state->load() != State::Active) continue;
+        for(auto& itr : instances)
+            itr->sample();
+        while(_now < std::chrono::steady_clock::now())
+            _now += _interval;
+    }
+
+    OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(),
+                                      "Thread sampler polling completed...\n");
+
+    if(polling_finished) polling_finished->set_value();
+}
+
+void
+sampler::setup()
+{
+    OMNITRACE_VERBOSE(1, "Setting up background sampler...\n");
+
+    // shutdown if already running
+    shutdown();
+
+    auto _enable_samp = pthread_gotcha::enable_sampling_on_child_threads();
+    pthread_gotcha::enable_sampling_on_child_threads() = false;
+
+    if(get_use_rocm_smi())
+    {
+        auto& _rocm_smi         = instances.emplace_back(std::make_unique<instance>());
+        _rocm_smi->setup        = []() { rocm_smi::setup(); };
+        _rocm_smi->shutdown     = []() { rocm_smi::shutdown(); };
+        _rocm_smi->post_process = []() { rocm_smi::post_process(); };
+        _rocm_smi->config       = []() { rocm_smi::config(); };
+        _rocm_smi->sample       = []() { rocm_smi::sample(); };
+    }
+
+    auto& _cpu_freq         = instances.emplace_back(std::make_unique<instance>());
+    _cpu_freq->setup        = []() { cpu_freq::setup(); };
+    _cpu_freq->shutdown     = []() { cpu_freq::shutdown(); };
+    _cpu_freq->post_process = []() { cpu_freq::post_process(); };
+    _cpu_freq->config       = []() { cpu_freq::config(); };
+    _cpu_freq->sample       = []() { cpu_freq::sample(); };
+
+    for(auto& itr : instances)
+        itr->setup();
+
+    polling_finished = std::make_unique<promise_t>();
+
+    auto     _freq      = get_thread_sampling_freq();
+    uint64_t _msec_freq = (1.0 / _freq) * 1.0e3;
+
+    promise_t _prom{};
+    auto      _fut   = _prom.get_future();
+    polling_finished = std::make_unique<promise_t>();
+
+    set_state(State::PreInit);
+    get_thread() = std::make_unique<std::thread>(&poll<msec_t>, &get_sampler_state(),
+                                                 msec_t{ _msec_freq }, &_prom);
+
+    _fut.wait();
+
+    pthread_gotcha::enable_sampling_on_child_threads() = _enable_samp;
+    set_state(State::Active);
+}
+
+void
+sampler::shutdown()
+{
+    for(auto& itr : instances)
+        itr->shutdown();
+
+    auto& _thread = get_thread();
+    if(_thread)
+    {
+        OMNITRACE_VERBOSE(1, "Shutting down background sampler...\n");
+        set_state(State::Finalized);
+        if(polling_finished)
+        {
+            auto     _fut  = polling_finished->get_future();
+            uint64_t _freq = (1.0 / get_thread_sampling_freq()) * 1.0e3;
+            _fut.wait_for(msec_t{ 5 * _freq });
+            _thread->join();
+        }
+        else
+        {
+            uint64_t _freq = (1.0 / get_thread_sampling_freq()) * 1.0e3;
+            std::this_thread::sleep_for(msec_t{ 5 * _freq });
+            pthread_cancel(_thread->native_handle());
+            _thread->detach();
+        }
+        _thread          = std::unique_ptr<std::thread>{};
+        polling_finished = std::unique_ptr<promise_t>{};
+    }
+
+    is_initialized() = false;
+}
+
+void
+sampler::post_process()
+{
+    for(auto& itr : instances)
+        itr->post_process();
+
+    instances.clear();
+}
+
+void
+sampler::set_state(state_t _state)
+{
+    get_sampler_state().store(_state);
+}
+}  // namespace thread_sampler
+}  // namespace omnitrace

source/lib/src/library/timemory.cpp

@@ -0,0 +1,27 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/timemory.hpp"
+
+using namespace omnitrace;
+
+TIMEMORY_INITIALIZE_STORAGE(comp::wall_clock, comp::user_global_bundle)