Address and thread sanitizer fixes (#250)

* Address and thread sanitizer fixes

- Fix compilation with clang
- Tweak perfetto copy to build tree
- Added suppression files to scripts
- fix LD_PRELOAD support in omnitrace-causal and omnitrace-sample
- use spin_mutex and spin_lock from timemory instead of atomic_mutex and atomic_lock
- state uses atomic
- fix some memory leaks
- tweak testing
  - mpi tests do not use preload
  - increase timeout when using sanitizers
  - add env LD_PRELOAD when using sanitizers

* Tweak perfetto build

* Update timemory submodule

* Update version to 1.8.1

* Update omnitrace-leak.supp

* Update timemory submodule

- fixed spin_mutex implementation

* Remove previously added addr_space->allowTraps(instr_traps)

- this appears to cause errors during binary rewrite

* causal testing updates

- relaxed causal validation on CI systems (to account for hyperthreading decreasing prediction)
- improved impact calculation
- other general improvements to validate-causal-json.py

* Improve fork handling for perfetto

- numerous updates changing perfetto:: to ::perfetto::
- added perfetto_fwd.hpp

* Updated fork example

- user API for validation that stopping/starting perfetto is valid

* Misc fixes to perfetto + fork support

- tweak regions in fork example
- handle disabling tmp files
- get rid of stop/start with perfetto before/after fork
- fixed sampling support during fork
- tweak env of fork test

* Fix find_package in build-tree

* Fix buildtree export

* Fix buildtree export

* Restructured ConfigInstall before adding examples

* Guard against creating tmp file in sampling when disabled

* Fix buildtree package

* formatting

* exit handlers on child processes

- quick exit to avoid perfetto cleanup

* Further tweaking of causal tests for reliability

- enable PROCESSOR_AFFINITY
- decrease to 5 iterations

* Further tweaking of causal tests for reliability

- disable PROCESSOR_AFFINITY for fast func e2e tests
- enabling affinity results in (valid) speedup predictions greater than zero

* Fixes to fork handling

- use pthread_atfork for redundancy if fork_gotcha fails

* cmake formatting

* Fix fork init settings + install components

- remove dl from PROJECT_BUILD_TARGETS

* Testing tweaks

- fix mpi-binary-rewrite-run regex when OMNITRACE_VERBOSE set > 1 in env
- increase causal e2e iterations to 8

* Fix "Test User API"

- test-find-package.sh included dl component

* Further tweaks to causal validation

- further considerations of variance

[ROCm/rocprofiler-systems commit: 846301bcaf]

projects/rocprofiler-systems/CMakeLists.txt

@@ -391,6 +391,15 @@ install(
     DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}
     COMPONENT setup)
 
+# ------------------------------------------------------------------------------#
+#
+# install
+#
+# ------------------------------------------------------------------------------#
+
+set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME core)
+include(ConfigInstall)
+
 # ------------------------------------------------------------------------------#
 #
 # examples
@@ -398,6 +407,7 @@ install(
 # ------------------------------------------------------------------------------#
 
 if(OMNITRACE_BUILD_EXAMPLES)
+    set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME examples)
     add_subdirectory(examples)
 endif()
 
@@ -408,18 +418,10 @@ endif()
 # ------------------------------------------------------------------------------#
 
 if(OMNITRACE_BUILD_TESTING)
+    set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME testing)
     add_subdirectory(tests)
 endif()
 
-# install the validate-causal-json python script as a utility
-configure_file(
-    ${PROJECT_SOURCE_DIR}/tests/validate-causal-json.py
-    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/omnitrace-causal-print
-    USE_SOURCE_PERMISSIONS COPYONLY)
-
-install(PROGRAMS ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/omnitrace-causal-print
-        DESTINATION ${CMAKE_INSTALL_BINDIR})
-
 # ------------------------------------------------------------------------------#
 #
 # packaging
@@ -427,7 +429,6 @@ install(PROGRAMS ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/omnitrace-causal-
 # ------------------------------------------------------------------------------#
 
 set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME core)
-include(ConfigInstall)
 include(ConfigCPack)
 
 # ------------------------------------------------------------------------------#

projects/rocprofiler-systems/VERSION

@@ -1 +1 @@
-1.8.0
+1.8.1

projects/rocprofiler-systems/cmake/BuildSettings.cmake

@@ -337,8 +337,7 @@ target_link_options(
 
 target_compile_options(
     omnitrace-static-libstdcxx
-    INTERFACE $<$<COMPILE_LANGUAGE:CXX>:$<$<CXX_COMPILER_ID:GNU,Clang>:-static-libstdc++>>
-    )
+    INTERFACE $<$<COMPILE_LANGUAGE:CXX>:$<$<CXX_COMPILER_ID:GNU>:-static-libstdc++>>)
 target_link_options(
     omnitrace-static-libstdcxx INTERFACE
     $<$<COMPILE_LANGUAGE:CXX>:$<$<CXX_COMPILER_ID:GNU,Clang>:-static-libstdc++>>)

projects/rocprofiler-systems/cmake/ConfigInstall.cmake

@@ -17,7 +17,7 @@ install(
 set(PROJECT_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR})
 set(LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR})
-set(PROJECT_BUILD_TARGETS user dl)
+set(PROJECT_BUILD_TARGETS user)
 
 configure_package_config_file(
     ${PROJECT_SOURCE_DIR}/cmake/Templates/${PROJECT_NAME}-config.cmake.in
@@ -39,3 +39,40 @@ install(
     OPTIONAL)
 
 export(PACKAGE ${PROJECT_NAME})
+
+# ------------------------------------------------------------------------------#
+# install the validate-causal-json python script as a utility
+#
+configure_file(
+    ${PROJECT_SOURCE_DIR}/tests/validate-causal-json.py
+    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/omnitrace-causal-print COPYONLY)
+
+install(PROGRAMS ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/omnitrace-causal-print
+        DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+# ------------------------------------------------------------------------------#
+# build tree
+#
+set(_BUILDTREE_EXPORT_DIR
+    "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/cmake/omnitrace")
+
+if(NOT EXISTS "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+    file(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+endif()
+
+if(NOT EXISTS "${_BUILDTREE_EXPORT_DIR}")
+    file(MAKE_DIRECTORY "${_BUILDTREE_EXPORT_DIR}")
+endif()
+
+if(NOT EXISTS "${_BUILDTREE_EXPORT_DIR}/omnitrace-library-targets.cmake")
+    file(TOUCH "${_BUILDTREE_EXPORT_DIR}/omnitrace-library-targets.cmake")
+endif()
+
+export(
+    EXPORT omnitrace-library-targets
+    NAMESPACE omnitrace::
+    FILE "${_BUILDTREE_EXPORT_DIR}/omnitrace-library-targets.cmake")
+
+set(omnitrace_DIR
+    "${_BUILDTREE_EXPORT_DIR}"
+    CACHE PATH "omnitrace" FORCE)

projects/rocprofiler-systems/cmake/MacroUtilities.cmake

@@ -341,7 +341,7 @@ macro(OMNITRACE_ADD_INTERFACE_LIBRARY _TARGET)
         TARGETS ${_TARGET}
         DESTINATION ${CMAKE_INSTALL_LIBDIR}
         COMPONENT core
-        EXPORT ${PROJECT_NAME}-library-depends
+        EXPORT ${PROJECT_NAME}-interface-targets
         OPTIONAL)
     if(NOT "${ARGN}" STREQUAL "")
         set_property(GLOBAL APPEND PROPERTY ${PROJECT_NAME}_CMAKE_INTERFACE_DOC

projects/rocprofiler-systems/cmake/PAPI.cmake

@@ -257,10 +257,12 @@ set(PAPI_pfm_STATIC_LIBRARY
     ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.a
     CACHE FILEPATH "PAPI library" FORCE)
 
-target_include_directories(omnitrace-papi SYSTEM INTERFACE ${PAPI_INCLUDE_DIR})
-target_link_libraries(omnitrace-papi INTERFACE ${PAPI_LIBRARY} ${PAPI_pfm_LIBRARY})
-omnitrace_target_compile_definitions(omnitrace-papi INTERFACE OMNITRACE_USE_PAPI
-                                                              TIMEMORY_USE_PAPI=1)
+target_include_directories(omnitrace-papi SYSTEM
+                           INTERFACE $<BUILD_INTERFACE:${PAPI_INCLUDE_DIR}>)
+target_link_libraries(omnitrace-papi INTERFACE $<BUILD_INTERFACE:${PAPI_LIBRARY}>
+                                               $<BUILD_INTERFACE:${PAPI_pfm_LIBRARY}>)
+omnitrace_target_compile_definitions(
+    omnitrace-papi INTERFACE OMNITRACE_USE_PAPI $<BUILD_INTERFACE:TIMEMORY_USE_PAPI=1>)
 
 install(
     DIRECTORY ${OMNITRACE_PAPI_INSTALL_DIR}/lib/

projects/rocprofiler-systems/cmake/Packages.cmake

@@ -51,9 +51,11 @@ set(OMNITRACE_EXTENSION_LIBRARIES
 
 target_include_directories(
     omnitrace-headers
-    INTERFACE ${PROJECT_BINARY_DIR}/source/lib ${PROJECT_BINARY_DIR}/source/lib/core
-              ${PROJECT_SOURCE_DIR}/source/lib ${PROJECT_SOURCE_DIR}/source/lib/omnitrace
-              ${PROJECT_SOURCE_DIR}/source/lib/omnitrace-user)
+    INTERFACE $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/source/lib>
+              $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/source/lib/core>
+              $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/source/lib>
+              $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/source/lib/omnitrace>
+              $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/source/lib/omnitrace-user>)
 
 # include threading because of rooflines
 target_link_libraries(omnitrace-headers INTERFACE omnitrace::omnitrace-threading)
@@ -630,6 +632,12 @@ set(TIMEMORY_USE_LIBUNWIND
 set(TIMEMORY_USE_VISIBILITY
     OFF
     CACHE BOOL "Enable/disable using visibility decorations")
+set(TIMEMORY_USE_SANITIZER
+    ${OMNITRACE_USE_SANITIZER}
+    CACHE BOOL "Build with -fsanitze=\${OMNITRACE_SANITIZER_TYPE}" FORCE)
+set(TIMEMORY_SANITIZER_TYPE
+    ${OMNITRACE_SANITIZER_TYPE}
+    CACHE STRING "Sanitizer type, e.g. leak, thread, address, memory, etc." FORCE)
 
 if(DEFINED TIMEMORY_BUILD_GOTCHA AND NOT TIMEMORY_BUILD_GOTCHA)
     omnitrace_message(
@@ -788,8 +796,11 @@ if(NOT TARGET PTL::ptl-shared)
                   CMAKE_VISIBILITY_INLINES_HIDDEN)
 endif()
 
-target_sources(omnitrace-ptl INTERFACE $<TARGET_OBJECTS:PTL::ptl-object>)
-target_link_libraries(omnitrace-ptl INTERFACE PTL::ptl-object)
+target_sources(omnitrace-ptl
+               INTERFACE $<BUILD_INTERFACE:$<TARGET_OBJECTS:PTL::ptl-object>>)
+target_include_directories(
+    omnitrace-ptl INTERFACE $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/PTL/source>
+                            $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/external/PTL/source>)
 
 # ----------------------------------------------------------------------------------------#
 #

projects/rocprofiler-systems/cmake/Perfetto.cmake

@@ -99,6 +99,27 @@ if(NOT EXISTS "${OMNITRACE_PERFETTO_SOURCE_DIR}")
                 ${OMNITRACE_PERFETTO_SOURCE_DIR})
 endif()
 
+file(READ ${PROJECT_SOURCE_DIR}/external/perfetto/sdk/perfetto.h _PERFETTO_HEADER)
+
+string(
+    REGEX
+    REPLACE " perfetto::internal::ValidateEventNameType"
+            " ::perfetto::internal::ValidateEventNameType" _PERFETTO_HEADER
+            "${_PERFETTO_HEADER}")
+
+if(OMNITRACE_USE_SANITIZER AND OMNITRACE_SANITIZER_TYPE MATCHES "address")
+    string(REPLACE "__asan_poison_memory_region((a), (s))" "" _PERFETTO_HEADER
+                   "${_PERFETTO_HEADER}")
+    string(REPLACE "__asan_unpoison_memory_region((a), (s))" "" _PERFETTO_HEADER
+                   "${_PERFETTO_HEADER}")
+endif()
+
+file(WRITE ${OMNITRACE_PERFETTO_SOURCE_DIR}/sdk/perfetto.h.tmp "${_PERFETTO_HEADER}")
+
+configure_file(${OMNITRACE_PERFETTO_SOURCE_DIR}/sdk/perfetto.h.tmp
+               ${OMNITRACE_PERFETTO_SOURCE_DIR}/sdk/perfetto.h COPYONLY)
+configure_file(${PROJECT_SOURCE_DIR}/external/perfetto/sdk/perfetto.cc
+               ${OMNITRACE_PERFETTO_SOURCE_DIR}/sdk/perfetto.cc COPYONLY)
 configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/args.gn.in
                ${OMNITRACE_PERFETTO_BINARY_DIR}/args.gn @ONLY)
 
@@ -178,7 +199,7 @@ target_sources(
 target_link_libraries(
     omnitrace-perfetto-library
     PRIVATE omnitrace::omnitrace-threading omnitrace::omnitrace-static-libgcc
-            omnitrace::omnitrace-static-libstdcxx omnitrace::omnitrace-compile-options)
+            omnitrace::omnitrace-static-libstdcxx)
 set_target_properties(
     omnitrace-perfetto-library
     PROPERTIES OUTPUT_NAME perfetto

projects/rocprofiler-systems/examples/fork/CMakeLists.txt

@@ -6,8 +6,9 @@ set(CMAKE_BUILD_TYPE "RelWithDebInfo")
 string(REPLACE " " ";" _FLAGS "${CMAKE_CXX_FLAGS_DEBUG}")
 
 find_package(Threads REQUIRED)
+find_package(omnitrace REQUIRED COMPONENTS user)
 add_executable(fork-example fork.cpp)
-target_link_libraries(fork-example PRIVATE Threads::Threads)
+target_link_libraries(fork-example PRIVATE Threads::Threads omnitrace::omnitrace)
 target_compile_options(fork-example PRIVATE ${_FLAGS})
 
 if(OMNITRACE_INSTALL_EXAMPLES)

projects/rocprofiler-systems/examples/fork/fork.cpp

@@ -1,4 +1,6 @@
 
+#include <omnitrace/user.h>
+
 #include <chrono>
 #include <cstdio>
 #include <cstdlib>
@@ -19,6 +21,7 @@ run(const char* _name, int nchildren)
 {
     for(int i = 0; i < nchildren; ++i)
     {
+        omnitrace_user_push_region("launch_child");
         auto _run = [i, _name]() {
             pid_t _pid = fork();
             if(_pid == 0)
@@ -33,9 +36,11 @@ run(const char* _name, int nchildren)
             }
         };
         std::thread{ _run }.join();
-        //_run();
+        omnitrace_user_pop_region("launch_child");
     }
 
+    omnitrace_user_push_region("wait_for_children");
+
     int   _status   = 0;
     pid_t _wait_pid = 0;
     // parent waits for all the child processes
@@ -64,6 +69,8 @@ run(const char* _name, int nchildren)
             printf("unknown\n");
         }
     }
+
+    omnitrace_user_pop_region("wait_for_children");
     return _status;
 }
 

projects/rocprofiler-systems/examples/transpose/CMakeLists.txt

@@ -56,9 +56,6 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang"
         transpose
         PRIVATE $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>
                 $<TARGET_NAME_IF_EXISTS:hip::host> $<TARGET_NAME_IF_EXISTS:hip::device>)
-elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-    target_link_libraries(
-        transpose PRIVATE $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>)
 else()
     target_compile_options(transpose PRIVATE -W -Wall)
 endif()

projects/rocprofiler-systems/scripts/omnitrace-leak.supp

@@ -0,0 +1,17 @@
+leak:Dyninst::
+leak:dyninst
+leak:tbb::
+leak:boost::
+leak:libdw
+leak:libamd_comgr
+leak:libpfm4
+leak:libhsa
+leak:libamdhip64
+leak:libroctracer64
+leak:librocprofiler64
+leak:__cxx_global_var_init
+leak:tim::timemory_argparse
+leak:tim::argparse::argument_parser::parse_known_args
+leak:std::shared_ptr<std::promise<void> >
+leak:realloc
+leak:Kokkos::

projects/rocprofiler-systems/scripts/omnitrace-thread.supp

@@ -0,0 +1,31 @@
+# false positive
+race:omnitrace::component::pthread_create_gotcha::operator()
+race:pthread_create
+race:tim::impl::storage<*>::_data()
+race:tim::graph<*>::append_child
+
+# unlock of mutex by wrong thread (rsmi_init -> rsmi_shut_down)
+called_from_lib:librocm_smi64.so
+
+# suppress any races originating in Dyninst
+called_from_lib:libcommon.so
+called_from_lib:libdynC_API.so
+called_from_lib:libdynDwarf.so
+called_from_lib:libdynElf.so
+called_from_lib:libdyninstAPI.so
+called_from_lib:libinstructionAPI.so
+called_from_lib:libparseAPI.so
+called_from_lib:libpatchAPI.so
+called_from_lib:libpcontrol.so
+called_from_lib:libstackwalk.so
+called_from_lib:libsymtabAPI.so
+called_from_lib:libtbb.so
+
+# suppress any races originating in some Dyninst function
+race_top:Dyninst::
+race:Dyninst::
+race:libparseAPI.so
+race:boost::
+
+# thread leaks
+thread:*

projects/rocprofiler-systems/scripts/test-find-package.sh

@@ -122,7 +122,7 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-find_package(omnitrace REQUIRED COMPONENTS user dl)
+find_package(omnitrace REQUIRED COMPONENTS user)
 find_package(Threads REQUIRED)
 
 get_target_property(LIBS omnitrace::omnitrace INTERFACE_LINK_LIBRARIES)

projects/rocprofiler-systems/source/bin/omnitrace-causal/impl.cpp

@@ -424,8 +424,18 @@ update_env(std::vector<char*>& _environ, std::string_view _env_var, Tp&& _env_va
                 {
                     auto _val = std::string{ itr }.substr(_key.length());
                     free(itr);
-                    itr = strdup(
-                        join('=', _env_var, join(_join_delim, _env_val, _val)).c_str());
+                    if(_env_var == "LD_PRELOAD")
+                    {
+                        itr =
+                            strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
+                                       .c_str());
+                    }
+                    else
+                    {
+                        itr =
+                            strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
+                                       .c_str());
+                    }
                 }
             }
             else

projects/rocprofiler-systems/source/bin/omnitrace-critical-trace/critical-trace.cpp

@@ -97,9 +97,9 @@ namespace
 void
 init_perfetto()
 {
-    perfetto::TracingInitArgs               args{};
-    perfetto::TraceConfig                   cfg{};
-    perfetto::protos::gen::TrackEventConfig track_event_cfg{};
+    ::perfetto::TracingInitArgs               args{};
+    ::perfetto::TraceConfig                   cfg{};
+    ::perfetto::protos::gen::TrackEventConfig track_event_cfg{};
 
     auto  shmem_size_hint = config::get_perfetto_shmem_size_hint();
     auto  buffer_size     = config::get_perfetto_buffer_size();
@@ -107,19 +107,19 @@ init_perfetto()
 
     buffer_config->set_size_kb(buffer_size);
     buffer_config->set_fill_policy(
-        perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_DISCARD);
+        ::perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_DISCARD);
 
     auto* ds_cfg = cfg.add_data_sources()->mutable_config();
     ds_cfg->set_name("track_event");
     ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString());
 
-    args.backends |= perfetto::kInProcessBackend;
+    args.backends |= ::perfetto::kInProcessBackend;
     args.shmem_size_hint_kb = shmem_size_hint;
 
-    perfetto::Tracing::Initialize(args);
-    perfetto::TrackEvent::Register();
+    ::perfetto::Tracing::Initialize(args);
+    ::perfetto::TrackEvent::Register();
 
-    tracing_session = perfetto::Tracing::NewTrace();
+    tracing_session = ::perfetto::Tracing::NewTrace();
     tracing_session->Setup(cfg);
     tracing_session->StartBlocking();
 }
@@ -128,7 +128,7 @@ void
 fini_perfetto()
 {
     // Make sure the last event is closed for this example.
-    perfetto::TrackEvent::Flush();
+    ::perfetto::TrackEvent::Flush();
 
     OMNITRACE_DEBUG_F("Stopping the blocking perfetto trace sessions...\n");
     tracing_session->StopBlocking();

projects/rocprofiler-systems/source/bin/omnitrace-sample/impl.cpp

@@ -246,7 +246,12 @@ update_env(std::vector<char*>& _environ, std::string_view _env_var, Tp&& _env_va
                 {
                     auto _val = std::string{ itr }.substr(_key.length());
                     free(itr);
-                    itr = strdup(join('=', _env_var, join(":", _env_val, _val)).c_str());
+                    if(_env_var == "LD_PRELOAD")
+                        itr = strdup(
+                            join('=', _env_var, join(":", _val, _env_val)).c_str());
+                    else
+                        itr = strdup(
+                            join('=', _env_var, join(":", _env_val, _val)).c_str());
                 }
             }
             else

projects/rocprofiler-systems/source/bin/omnitrace/internal_libs.cpp

@@ -467,7 +467,7 @@ get_internal_libs_data_impl()
                                         file_internal_include))
                 continue;
 
-            verbprintf(2, "[internal]     parsing module: '%s' (via '%s')...\n",
+            verbprintf(3, "[internal]     parsing module: '%s' (via '%s')...\n",
                        _mname.c_str(), filepath::basename(itr.first));
 
             _data[itr.first].emplace(_mpath, func_set_t{});

projects/rocprofiler-systems/source/bin/omnitrace/omnitrace.cpp

@@ -347,7 +347,7 @@ main(int argc, char** argv)
         if(resolved_mutname != mutname)
         {
             mutname = resolved_mutname;
-            delete _cmdv[0];
+            free(_cmdv[0]);
             copy_str(_cmdv[0], resolved_mutname.c_str());
         }
     }
@@ -1343,6 +1343,8 @@ main(int argc, char** argv)
     addr_space =
         omnitrace_get_address_space(bpatch, _cmdc, _cmdv, binary_rewrite, _pid, mutname);
 
+    // addr_space->allowTraps(instr_traps);
+
     if(!addr_space)
     {
         errprintf(-1, "address space for dynamic instrumentation was not created\n");

projects/rocprofiler-systems/source/lib/core/CMakeLists.txt

@@ -10,7 +10,6 @@ set(core_sources
     ${CMAKE_CURRENT_LIST_DIR}/dynamic_library.cpp
     ${CMAKE_CURRENT_LIST_DIR}/exception.cpp
     ${CMAKE_CURRENT_LIST_DIR}/gpu.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/locking.cpp
     ${CMAKE_CURRENT_LIST_DIR}/mproc.cpp
     ${CMAKE_CURRENT_LIST_DIR}/perfetto.cpp
     ${CMAKE_CURRENT_LIST_DIR}/state.cpp

projects/rocprofiler-systems/source/lib/core/config.cpp

@@ -53,6 +53,7 @@
 #include <atomic>
 #include <csignal>
 #include <cstdint>
+#include <cstdio>
 #include <cstdlib>
 #include <fstream>
 #include <limits>
@@ -280,7 +281,8 @@ configure_settings(bool _init)
         std::string, "OMNITRACE_MODE",
         "Data collection mode. Used to set default values for OMNITRACE_USE_* options. "
         "Typically set by omnitrace binary instrumenter.",
-        std::string{ "trace" }, "backend", "advanced");
+        std::string{ "trace" }, "backend", "advanced")
+        ->set_choices({ "trace", "sampling", "causal", "coverage" });
 
     OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_CI",
                              "Enable some runtime validation checks (typically enabled "
@@ -2164,7 +2166,7 @@ get_trace_hsa_api_types()
 }
 
 std::string&
-get_backend()
+get_perfetto_backend()
 {
     // select inprocess, system, or both (i.e. all)
     static auto _v = get_config()->find("OMNITRACE_PERFETTO_BACKEND");
@@ -2435,7 +2437,7 @@ tmp_file::~tmp_file()
     remove();
 }
 
-void
+bool
 tmp_file::open(std::ios::openmode _mode)
 {
     OMNITRACE_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
@@ -2448,23 +2450,67 @@ tmp_file::open(std::ios::openmode _mode)
     }
 
     stream.open(filename, _mode);
+
+    return (stream.is_open() && stream.good());
 }
 
-void
+bool
+tmp_file::fopen(const char* _mode)
+{
+    OMNITRACE_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
+
+    if(!filepath::exists(filename))
+    {
+        // if the filepath does not exist, open in out mode to create it
+        std::ofstream _ofs{};
+        filepath::open(_ofs, filename);
+    }
+
+    file = filepath::fopen(filename, _mode);
+    if(file) fd = ::fileno(file);
+
+    return (file != nullptr && fd > 0);
+}
+
+bool
 tmp_file::close()
 {
-    if(stream.is_open()) stream.close();
+    if(stream.is_open())
+    {
+        stream.close();
+        return !stream.is_open();
+    }
+    else if(file != nullptr)
+    {
+        auto _ret = fclose(file);
+        if(_ret == 0)
+        {
+            file = nullptr;
+            fd   = -1;
+        }
+        return (_ret == 0);
+    }
+
+    return true;
 }
 
-void
+bool
 tmp_file::remove()
 {
     close();
     if(filepath::exists(filename))
     {
         OMNITRACE_BASIC_VERBOSE(2, "Removing temporary file '%s'...\n", filename.c_str());
-        ::remove(filename.c_str());
+        auto _ret = ::remove(filename.c_str());
+        return (_ret == 0);
     }
+
+    return true;
+}
+
+tmp_file::operator bool() const
+{
+    return (stream.is_open() && stream.good()) || (file != nullptr && fd > 0);
 }
 
 std::shared_ptr<tmp_file>
@@ -2508,7 +2554,6 @@ get_tmp_file(std::string _basename, std::string _ext)
     if(itr != _existing_files.end()) return itr->second;
 
     auto _v = std::make_shared<tmp_file>(_fname);
-    _v->open();
     _existing_files.emplace(_fname, std::move(_v));
     return _existing_files.at(_fname);
 }

projects/rocprofiler-systems/source/lib/core/config.hpp

@@ -294,7 +294,7 @@ std::string
 get_trace_hsa_api_types();
 
 std::string&
-get_backend();
+get_perfetto_backend();
 
 // make this visible so omnitrace-avail can call it
 std::string
@@ -392,14 +392,17 @@ struct tmp_file
     tmp_file(std::string);
     ~tmp_file();
 
-    void open(std::ios::openmode = std::ios::binary | std::ios::in | std::ios::out);
-    void close();
-    void remove();
+    bool open(std::ios::openmode = std::ios::binary | std::ios::in | std::ios::out);
+    bool fopen(const char* = "r+");
+    bool close();
+    bool remove();
 
-    explicit operator bool() const { return stream.is_open() && stream.good(); }
+    explicit operator bool() const;
 
     std::string  filename = {};
     std::fstream stream   = {};
+    FILE*        file     = nullptr;
+    int          fd       = -1;
 };
 
 std::shared_ptr<tmp_file>

projects/rocprofiler-systems/source/lib/core/locking.hpp

@@ -22,6 +22,8 @@
 
 #pragma once
 
+#include <timemory/utility/locking.hpp>
+
 #include <atomic>
 #include <mutex>
 
@@ -29,50 +31,7 @@ namespace omnitrace
 {
 namespace locking
 {
-/// simple mutex which spins on an atomic while trying to lock.
-/// Provided for internal use for when there is low contention
-/// but we want to avoid using pthread mutexes since those
-/// are wrapped by library
-struct atomic_mutex
-{
-    atomic_mutex()  = default;
-    ~atomic_mutex() = default;
-
-    atomic_mutex(const atomic_mutex&)     = delete;
-    atomic_mutex(atomic_mutex&&) noexcept = delete;
-
-    atomic_mutex& operator=(const atomic_mutex&) = delete;
-    atomic_mutex& operator=(atomic_mutex&&) noexcept = delete;
-
-    void lock();
-    void unlock();
-    bool try_lock();
-
-private:
-    std::atomic<int64_t> m_value = {};
-};
-
-struct atomic_lock
-{
-    atomic_lock(atomic_mutex&);
-    atomic_lock(atomic_mutex&, std::defer_lock_t);
-    ~atomic_lock();
-
-    atomic_lock(const atomic_lock&)     = delete;
-    atomic_lock(atomic_lock&&) noexcept = delete;
-
-    atomic_lock& operator=(const atomic_lock&) = delete;
-    atomic_lock& operator=(atomic_lock&&) noexcept = delete;
-
-    bool owns_lock() const;
-
-    void lock();
-    void unlock();
-    bool try_lock();
-
-private:
-    bool          m_owns = false;
-    atomic_mutex& m_mutex;
-};
+using atomic_mutex = ::tim::locking::spin_mutex;
+using atomic_lock  = ::tim::locking::spin_lock;
 }  // namespace locking
 }  // namespace omnitrace

projects/rocprofiler-systems/source/lib/core/perfetto.cpp

@@ -22,11 +22,31 @@
 
 #include "perfetto.hpp"
 #include "config.hpp"
+#include "library/runtime.hpp"
+#include "perfetto_fwd.hpp"
+#include "utility.hpp"
 
 namespace omnitrace
 {
 namespace perfetto
 {
+namespace
+{
+auto
+is_system_backend()
+{
+    // if get_perfetto_backend() returns 'system' or 'all', this is true
+    return (config::get_perfetto_backend() != "inprocess");
+}
+
+auto&
+get_perfetto_tmp_file(pid_t _pid = process::get_id())
+{
+    static auto _v = std::unordered_map<pid_t, std::shared_ptr<tmp_file>>{};
+    if(_v.find(_pid) == _v.end()) _v.emplace(_pid, std::shared_ptr<tmp_file>{});
+    return _v.at(_pid);
+}
+
 auto&
 get_config()
 {
@@ -35,11 +55,15 @@ get_config()
 }
 
 auto&
-get_session()
+get_session(pid_t _pid = process::get_id())
 {
-    static auto _v = std::unique_ptr<::perfetto::TracingSession>{};
-    return _v;
+    static auto _v =
+        std::unordered_map<pid_t, std::unique_ptr<::perfetto::TracingSession>>{};
+    if(_v.find(_pid) == _v.end())
+        _v.emplace(_pid, std::unique_ptr<::perfetto::TracingSession>{});
+    return _v.at(_pid);
 }
+}  // namespace
 
 void
 setup()
@@ -73,8 +97,8 @@ setup()
 
     args.shmem_size_hint_kb = shmem_size_hint;
 
-    if(get_backend() != "inprocess") args.backends |= ::perfetto::kSystemBackend;
-    if(get_backend() != "system") args.backends |= ::perfetto::kInProcessBackend;
+    if(get_perfetto_backend() != "inprocess") args.backends |= ::perfetto::kSystemBackend;
+    if(get_perfetto_backend() != "system") args.backends |= ::perfetto::kInProcessBackend;
 
     ::perfetto::Tracing::Initialize(args);
     ::perfetto::TrackEvent::Register();
@@ -83,18 +107,169 @@ setup()
 void
 start()
 {
-    auto& cfg             = get_config();
+    if(is_system_backend()) return;
+
     auto& tracing_session = get_session();
-    tracing_session       = ::perfetto::Tracing::NewTrace();
-    tracing_session->Setup(cfg);
+
+    if(!tracing_session) tracing_session = ::perfetto::Tracing::NewTrace();
+
+    tracing_session = ::perfetto::Tracing::NewTrace();
+    auto& _tmp_file = get_perfetto_tmp_file();
+    if(config::get_use_tmp_files())
+    {
+        if(!_tmp_file)
+        {
+            _tmp_file = config::get_tmp_file("perfetto-trace", "proto");
+            _tmp_file->fopen("w+");
+        }
+        else
+        {
+            OMNITRACE_VERBOSE(0, "Resuming perfetto...\n");
+            _tmp_file->fopen("a+");
+        }
+    }
+
+    OMNITRACE_VERBOSE(0, "Setup perfetto...\n");
+    int   _fd = (_tmp_file) ? _tmp_file->fd : -1;
+    auto& cfg = get_config();
+    tracing_session->Setup(cfg, _fd);
     tracing_session->StartBlocking();
 }
+
+void
+stop()
+{
+    if(is_system_backend()) return;
+
+    auto& tracing_session = get_perfetto_session();
+
+    OMNITRACE_CI_THROW(tracing_session == nullptr, "Null pointer to the tracing session");
+
+    if(tracing_session)
+    {
+        // Make sure the last event is closed
+        OMNITRACE_VERBOSE(2, "Flushing the perfetto trace data...\n");
+        ::perfetto::TrackEvent::Flush();
+        tracing_session->FlushBlocking();
+
+        OMNITRACE_VERBOSE(2, "Stopping the perfetto trace session (blocking)...\n");
+        tracing_session->StopBlocking();
+    }
+}
+
+void
+post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
+{
+    using char_vec_t = std::vector<char>;
+
+    stop();
+
+    auto& tracing_session = get_perfetto_session();
+    if(!tracing_session) return;
+
+    auto _get_session_data = [&tracing_session]() {
+        auto _data     = char_vec_t{};
+        auto _tmp_file = get_perfetto_tmp_file();
+        if(_tmp_file && *_tmp_file)
+        {
+            _tmp_file->close();
+            FILE* _fdata = fopen(_tmp_file->filename.c_str(), "rb");
+            fseek(_fdata, 0, SEEK_END);
+            size_t _fnum_elem = ftell(_fdata);
+            fseek(_fdata, 0, SEEK_SET);  // same as rewind(f);
+
+            _data.resize(_fnum_elem + 1);
+            auto _fnum_read = fread(_data.data(), sizeof(char), _fnum_elem, _fdata);
+            fclose(_fdata);
+
+            OMNITRACE_CI_THROW(
+                _fnum_read != _fnum_elem,
+                "Error! read %zu elements from perfetto trace file '%s'. Expected %zu\n",
+                _fnum_read, _tmp_file->filename.c_str(), _fnum_elem);
+        }
+
+        return utility::combine(_data,
+                                char_vec_t{ tracing_session->ReadTraceBlocking() });
+    };
+
+    auto trace_data = char_vec_t{};
+#if defined(TIMEMORY_USE_MPI) && TIMEMORY_USE_MPI > 0
+    if(get_perfetto_combined_traces())
+    {
+        using perfetto_mpi_get_t = tim::operation::finalize::mpi_get<char_vec_t, true>;
+
+        auto _trace_data = _get_session_data();
+        auto _rank_data  = std::vector<char_vec_t>{};
+        auto _combine    = [](char_vec_t& _dst, const char_vec_t& _src) -> char_vec_t& {
+            _dst.reserve(_dst.size() + _src.size());
+            for(auto&& itr : _src)
+                _dst.emplace_back(itr);
+            return _dst;
+        };
+
+        perfetto_mpi_get_t{ get_perfetto_combined_traces(),
+                            settings::node_count() }(_rank_data, _trace_data, _combine);
+        for(auto& itr : _rank_data)
+            trace_data =
+                (trace_data.empty()) ? std::move(itr) : _combine(trace_data, itr);
+    }
+    else
+    {
+        trace_data = _get_session_data();
+    }
+#else
+    trace_data = _get_session_data();
+#endif
+
+    auto _filename = config::get_perfetto_output_filename();
+    if(!trace_data.empty())
+    {
+        operation::file_output_message<tim::project::omnitrace> _fom{};
+        // Write the trace into a file.
+        if(config::get_verbose() >= 0)
+            _fom(_filename, std::string{ "perfetto" },
+                 " (%.2f KB / %.2f MB / %.2f GB)... ",
+                 static_cast<double>(trace_data.size()) / units::KB,
+                 static_cast<double>(trace_data.size()) / units::MB,
+                 static_cast<double>(trace_data.size()) / units::GB);
+        std::ofstream ofs{};
+        if(!filepath::open(ofs, _filename, std::ios::out | std::ios::binary))
+        {
+            _fom.append("Error opening '%s'...", _filename.c_str());
+            _perfetto_output_error = true;
+        }
+        else
+        {
+            // Write the trace into a file.
+            ofs.write(&trace_data[0], trace_data.size());
+            if(config::get_verbose() >= 0) _fom.append("%s", "Done");  // NOLINT
+            if(_timemory_manager)
+                _timemory_manager->add_file_output("protobuf", "perfetto", _filename);
+        }
+        ofs.close();
+    }
+    else if(dmp::rank() == 0)
+    {
+        OMNITRACE_VERBOSE(
+            0, "perfetto trace data is empty. File '%s' will not be written...\n",
+            _filename.c_str());
+    }
+
+    auto& _tmp_file = get_perfetto_tmp_file();
+    if(_tmp_file)
+    {
+        _tmp_file->close();
+        _tmp_file->remove();
+        _tmp_file.reset();
+    }
+}
+
 }  // namespace perfetto
 
 std::unique_ptr<::perfetto::TracingSession>&
-get_perfetto_session()
+get_perfetto_session(pid_t _pid)
 {
-    return ::omnitrace::perfetto::get_session();
+    return ::omnitrace::perfetto::get_session(_pid);
 }
 }  // namespace omnitrace
 

projects/rocprofiler-systems/source/lib/core/perfetto.hpp

@@ -34,6 +34,8 @@ PERFETTO_DEFINE_CATEGORIES(OMNITRACE_PERFETTO_CATEGORIES);
 
 #include "debug.hpp"
 
+#include <timemory/process/process.hpp>
+
 #include <map>
 #include <memory>
 #include <string>
@@ -43,8 +45,8 @@ PERFETTO_DEFINE_CATEGORIES(OMNITRACE_PERFETTO_CATEGORIES);
 
 namespace omnitrace
 {
-std::unique_ptr<::perfetto::TracingSession>&
-get_perfetto_session();
+std::unique_ptr<::perfetto::TracingSession>& get_perfetto_session(
+    pid_t = process::get_id());
 
 template <typename Tp>
 struct perfetto_counter_track
@@ -151,5 +153,4 @@ perfetto_counter_track<Tp>::emplace(size_t _idx, const std::string& _v,
     }
     return _index;
 }
-
 }  // namespace omnitrace

projects/rocprofiler-systems/source/lib/core/perfetto_fwd.hpp

@@ -20,83 +20,27 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-#include "locking.hpp"
+#pragma once
+
+namespace tim
+{
+class manager;
+}
 
 namespace omnitrace
 {
-namespace locking
+namespace perfetto
 {
 void
-atomic_mutex::lock()
-{
-    while(!try_lock())
-    {}
-}
+setup();
 
 void
-atomic_mutex::unlock()
-{
-    if((m_value.load() & 1) == 1) ++m_value;
-}
-
-bool
-atomic_mutex::try_lock()
-{
-    auto _targ = m_value.load(std::memory_order_relaxed);
-    if((_targ & 1) == 0)
-    {
-        return (
-            m_value.compare_exchange_strong(_targ, _targ + 1, std::memory_order_relaxed));
-    }
-    return false;
-}
-
-atomic_lock::atomic_lock(atomic_mutex& _v)
-: m_mutex{ _v }
-{
-    lock();
-}
-
-atomic_lock::atomic_lock(atomic_mutex& _v, std::defer_lock_t)
-: m_mutex{ _v }
-{}
-
-atomic_lock::~atomic_lock() { unlock(); }
-
-bool
-atomic_lock::owns_lock() const
-{
-    return m_owns;
-}
+start();
 
 void
-atomic_lock::lock()
-{
-    if(!owns_lock())
-    {
-        m_mutex.lock();
-        m_owns = true;
-    }
-}
+stop();
 
 void
-atomic_lock::unlock()
-{
-    if(owns_lock())
-    {
-        m_mutex.unlock();
-        m_owns = false;
-    }
-}
-
-bool
-atomic_lock::try_lock()
-{
-    if(!owns_lock())
-    {
-        m_owns = m_mutex.try_lock();
-    }
-    return m_owns;
-}
-}  // namespace locking
+post_process(tim::manager*, bool&);
+}  // namespace perfetto
 }  // namespace omnitrace

projects/rocprofiler-systems/source/lib/core/state.cpp

@@ -25,6 +25,7 @@
 #include "debug.hpp"
 #include "utility.hpp"
 
+#include <atomic>
 #include <string>
 
 namespace omnitrace
@@ -34,14 +35,14 @@ namespace
 auto&
 get_state_value()
 {
-    static State _v{ State::PreInit };
+    static auto _v = std::atomic<State>{ State::PreInit };
     return _v;
 }
 
 ThreadState&
 get_thread_state_value()
 {
-    static thread_local ThreadState _v{ ThreadState::Enabled };
+    static thread_local auto _v = ThreadState{ ThreadState::Enabled };
     return _v;
 }
 
@@ -64,7 +65,7 @@ get_thread_state_history(int64_t _idx = utility::get_thread_index())
 State
 get_state()
 {
-    return get_state_value();
+    return get_state_value().load(std::memory_order_relaxed);
 }
 
 ThreadState
@@ -83,8 +84,10 @@ set_state(State _n)
     OMNITRACE_CI_BASIC_THROW(
         _n < get_state(), "State is being assigned to a lesser value :: %s -> %s",
         std::to_string(get_state()).c_str(), std::to_string(_n).c_str());
-    std::swap(get_state_value(), _n);
-    return _n;
+    auto _v = get_state();
+    get_state_value().store(_n, std::memory_order_relaxed);
+    // std::swap(get_state_value(), _n);
+    return _v;
 }
 
 ThreadState

projects/rocprofiler-systems/source/lib/omnitrace-dl/CMakeLists.txt

@@ -47,7 +47,4 @@ omnitrace_strip_target(omnitrace-dl-library)
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/dl.hpp
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/omnitrace)
 
-install(
-    TARGETS omnitrace-dl-library
-    EXPORT omnitrace-library-targets
-    DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(TARGETS omnitrace-dl-library DESTINATION ${CMAKE_INSTALL_LIBDIR})

projects/rocprofiler-systems/source/lib/omnitrace-user/CMakeLists.txt

@@ -13,12 +13,15 @@ set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
 add_library(omnitrace-user-library SHARED)
 add_library(omnitrace::omnitrace-user-library ALIAS omnitrace-user-library)
 
-target_sources(
-    omnitrace-user-library
-    PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/user.cpp
-            ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/user.h
-            ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/causal.h
-            ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/types.h)
+set(_user_headers
+    ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/user.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/types.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/causal.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/categories.h)
+
+set(_user_sources ${CMAKE_CURRENT_SOURCE_DIR}/user.cpp)
+
+target_sources(omnitrace-user-library PRIVATE ${_user_sources} ${_user_headers})
 target_include_directories(
     omnitrace-user-library PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
                                   $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
@@ -36,12 +39,14 @@ set_target_properties(
 
 omnitrace_strip_target(omnitrace-user-library)
 
-install(
-    FILES ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/user.h
-          ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/types.h
-          ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/causal.h
-          ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/categories.h
-    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/omnitrace)
+foreach(_HEADER ${_user_headers})
+    string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/"
+                   "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/" _DEST
+                   "${_HEADER}")
+    configure_file(${_HEADER} ${_DEST} COPYONLY)
+endforeach()
+
+install(FILES ${_user_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/omnitrace)
 
 install(
     TARGETS omnitrace-user-library

projects/rocprofiler-systems/source/lib/omnitrace/library.cpp

@@ -34,6 +34,7 @@
 #include "core/debug.hpp"
 #include "core/defines.hpp"
 #include "core/gpu.hpp"
+#include "core/perfetto_fwd.hpp"
 #include "core/timemory.hpp"
 #include "core/utility.hpp"
 #include "library/causal/data.hpp"
@@ -42,6 +43,7 @@
 #include "library/components/exit_gotcha.hpp"
 #include "library/components/fork_gotcha.hpp"
 #include "library/components/mpi_gotcha.hpp"
+#include "library/components/numa_gotcha.hpp"
 #include "library/components/pthread_gotcha.hpp"
 #include "library/components/rocprofiler.hpp"
 #include "library/coverage.hpp"
@@ -83,21 +85,6 @@ using namespace omnitrace;
 
 //======================================================================================//
 
-namespace omnitrace
-{
-namespace perfetto
-{
-// declare this here bc it has a tendency to cause namespace ambiguities
-void
-setup();
-
-void
-start();
-}  // namespace perfetto
-}  // namespace omnitrace
-
-//======================================================================================//
-
 namespace
 {
 auto _timemory_manager  = tim::manager::instance();
@@ -196,13 +183,6 @@ ensure_finalization(bool _static_init = false)
     return scope::destructor{ []() { omnitrace_finalize_hidden(); } };
 }
 
-auto
-is_system_backend()
-{
-    // if get_backend() returns 'system' or 'all', this is true
-    return (get_backend() != "inprocess");
-}
-
 using Device = critical_trace::Device;
 using Phase  = critical_trace::Phase;
 
@@ -552,8 +532,9 @@ omnitrace_init_tooling_hidden()
         rcclp::setup();
     }
 
-    if(get_use_perfetto() && !is_system_backend())
+    if(get_use_perfetto())
     {
+        OMNITRACE_VERBOSE_F(1, "Starting Perfetto...\n");
         omnitrace::perfetto::start();
     }
 
@@ -761,24 +742,29 @@ omnitrace_finalize_hidden(void)
 
     OMNITRACE_DEBUG_F("Copying over all timemory hash information to main thread...\n");
     // copy these over so that all hashes are known
+    auto& _hmain = tim::hash::get_main_hash_ids();
+    auto& _amain = tim::hash::get_main_hash_aliases();
     auto& _hzero = tracing::get_timemory_hash_ids(0);
     auto& _azero = tracing::get_timemory_hash_aliases(0);
-    for(size_t i = 1; i < max_supported_threads; ++i)
+    for(size_t i = 0; i < max_supported_threads; ++i)
     {
         auto& _hitr = tracing::get_timemory_hash_ids(i);
         auto& _aitr = tracing::get_timemory_hash_aliases(i);
-        if(_hzero && _hitr)
+        if(_hmain && _hitr)
         {
             for(const auto& itr : *_hitr)
-                _hzero->emplace(itr.first, itr.second);
+                _hmain->emplace(itr.first, itr.second);
         }
-        if(_azero && _aitr)
+        if(_amain && _aitr)
         {
             for(auto itr : *_aitr)
-                _azero->emplace(itr.first, itr.second);
+                _amain->emplace(itr.first, itr.second);
         }
     }
 
+    if(_hzero && _hmain) *_hzero = *_hmain;
+    if(_azero && _amain) *_azero = *_amain;
+
     // stop the main bundle which has stats for run
     if(get_main_bundle())
     {
@@ -833,6 +819,9 @@ omnitrace_finalize_hidden(void)
     {
         OMNITRACE_DEBUG_F("Stopping main gotcha...\n");
         get_init_bundle()->stop();
+
+        pthread_gotcha::shutdown();
+        component::numa_gotcha::shutdown();
     }
 
     // stop the gotcha bundle
@@ -982,90 +971,11 @@ omnitrace_finalize_hidden(void)
     }
 
     bool _perfetto_output_error = false;
-    if(get_use_perfetto() && !is_system_backend())
+    if(get_use_perfetto())
     {
-        auto& tracing_session = get_perfetto_session();
-
-        OMNITRACE_CI_THROW(tracing_session == nullptr,
-                           "Null pointer to the tracing session");
-
         OMNITRACE_VERBOSE_F(0, "Finalizing perfetto...\n");
-
-        // Make sure the last event is closed for this example.
-        ::perfetto::TrackEvent::Flush();
-        tracing_session->FlushBlocking();
-
-        OMNITRACE_VERBOSE_F(3, "Stopping the blocking perfetto trace session...\n");
-        tracing_session->StopBlocking();
-
-        using char_vec_t = std::vector<char>;
-        OMNITRACE_VERBOSE_F(3, "Getting the trace data...\n");
-
-        auto trace_data = char_vec_t{};
-#if defined(TIMEMORY_USE_MPI) && TIMEMORY_USE_MPI > 0
-        if(get_perfetto_combined_traces())
-        {
-            using perfetto_mpi_get_t =
-                tim::operation::finalize::mpi_get<char_vec_t, true>;
-
-            char_vec_t              _trace_data{ tracing_session->ReadTraceBlocking() };
-            std::vector<char_vec_t> _rank_data = {};
-            auto _combine = [](char_vec_t& _dst, const char_vec_t& _src) -> char_vec_t& {
-                _dst.reserve(_dst.size() + _src.size());
-                for(auto&& itr : _src)
-                    _dst.emplace_back(itr);
-                return _dst;
-            };
-
-            perfetto_mpi_get_t{ get_perfetto_combined_traces(), settings::node_count() }(
-                _rank_data, _trace_data, _combine);
-            for(auto& itr : _rank_data)
-                trace_data =
-                    (trace_data.empty()) ? std::move(itr) : _combine(trace_data, itr);
-        }
-        else
-        {
-            trace_data = tracing_session->ReadTraceBlocking();
-        }
-#else
-        trace_data = tracing_session->ReadTraceBlocking();
-#endif
-
-        if(!trace_data.empty())
-        {
-            operation::file_output_message<tim::project::omnitrace> _fom{};
-            // Write the trace into a file.
-            if(get_verbose() >= 0)
-                _fom(get_perfetto_output_filename(), std::string{ "perfetto" },
-                     " (%.2f KB / %.2f MB / %.2f GB)... ",
-                     static_cast<double>(trace_data.size()) / units::KB,
-                     static_cast<double>(trace_data.size()) / units::MB,
-                     static_cast<double>(trace_data.size()) / units::GB);
-            std::ofstream ofs{};
-            if(!tim::filepath::open(ofs, get_perfetto_output_filename(),
-                                    std::ios::out | std::ios::binary))
-            {
-                _fom.append("Error opening '%s'...",
-                            get_perfetto_output_filename().c_str());
-                _perfetto_output_error = true;
-            }
-            else
-            {
-                // Write the trace into a file.
-                ofs.write(&trace_data[0], trace_data.size());
-                if(get_verbose() >= 0) _fom.append("%s", "Done");  // NOLINT
-                if(_timemory_manager)
-                    _timemory_manager->add_file_output("protobuf", "perfetto",
-                                                       get_perfetto_output_filename());
-            }
-            ofs.close();
-        }
-        else if(dmp::rank() == 0)
-        {
-            OMNITRACE_VERBOSE_F(0,
-                                "trace data is empty. File '%s' will not be written...\n",
-                                get_perfetto_output_filename().c_str());
-        }
+        omnitrace::perfetto::post_process(_timemory_manager.get(),
+                                          _perfetto_output_error);
     }
 
     if(_timemory_manager && _timemory_manager != nullptr)

projects/rocprofiler-systems/source/lib/omnitrace/library/causal/data.cpp

@@ -926,6 +926,7 @@ start_experimenting()
         }
     }
 
+    delay::setup();
     compute_eligible_lines();
 
     if(get_state() < State::Finalized)

projects/rocprofiler-systems/source/lib/omnitrace/library/causal/delay.cpp

@@ -91,9 +91,16 @@ compute_sleep_for_overhead()
     return _stats.get_mean();
 }
 
-int64_t sleep_for_overhead = compute_sleep_for_overhead();
+int64_t sleep_for_overhead = 0;
 }  // namespace
 
+void
+delay::setup()
+{
+    static std::once_flag _once{};
+    std::call_once(_once, []() { sleep_for_overhead = compute_sleep_for_overhead(); });
+}
+
 void
 delay::process()
 {

projects/rocprofiler-systems/source/lib/omnitrace/library/causal/delay.hpp

@@ -46,6 +46,7 @@ struct delay
 
     OMNITRACE_DEFAULT_OBJECT(delay)
 
+    static void    setup();
     static void    process();
     static void    credit();
     static void    preblock();

projects/rocprofiler-systems/source/lib/omnitrace/library/components/category_region.hpp

@@ -33,6 +33,7 @@
 #include "library/tracing/annotation.hpp"
 
 #include <timemory/components/gotcha/backends.hpp>
+#include <timemory/hash/types.hpp>
 #include <timemory/mpl/concepts.hpp>
 #include <timemory/mpl/types.hpp>
 #include <timemory/utility/types.hpp>
@@ -168,6 +169,9 @@ category_region<CategoryT>::start(std::string_view name, Args&&... args)
         ++tracing::push_count();
     }
 
+    auto _hash = tim::add_hash_id(name);
+    name       = tim::get_hash_identifier_fast(_hash);
+
     if constexpr(_ct_use_causal)
     {
         if constexpr(!is_one_of<CategoryT, causal_throughput_categories_t>::value)
@@ -349,7 +353,7 @@ void
 category_region<CategoryT>::audit(const gotcha_data_t& _data, audit::incoming,
                                   Args&&... _args)
 {
-    start<OptsT...>(_data.tool_id.c_str(), [&](perfetto::EventContext ctx) {
+    start<OptsT...>(_data.tool_id.c_str(), [&](::perfetto::EventContext ctx) {
         if(config::get_perfetto_annotations())
         {
             int64_t _n = 0;
@@ -365,7 +369,7 @@ void
 category_region<CategoryT>::audit(const gotcha_data_t& _data, audit::outgoing,
                                   Args&&... _args)
 {
-    stop<OptsT...>(_data.tool_id.c_str(), [&](perfetto::EventContext ctx) {
+    stop<OptsT...>(_data.tool_id.c_str(), [&](::perfetto::EventContext ctx) {
         if(config::get_perfetto_annotations())
             tracing::add_perfetto_annotation(ctx, "return", JOIN(", ", _args...));
     });
@@ -377,7 +381,7 @@ void
 category_region<CategoryT>::audit(std::string_view _name, audit::incoming,
                                   Args&&... _args)
 {
-    start<OptsT...>(_name.data(), [&](perfetto::EventContext ctx) {
+    start<OptsT...>(_name.data(), [&](::perfetto::EventContext ctx) {
         if(config::get_perfetto_annotations())
         {
             int64_t _n = 0;
@@ -393,7 +397,7 @@ void
 category_region<CategoryT>::audit(std::string_view _name, audit::outgoing,
                                   Args&&... _args)
 {
-    stop<OptsT...>(_name.data(), [&](perfetto::EventContext ctx) {
+    stop<OptsT...>(_name.data(), [&](::perfetto::EventContext ctx) {
         if(config::get_perfetto_annotations())
             tracing::add_perfetto_annotation(ctx, "return", JOIN(", ", _args...));
     });

projects/rocprofiler-systems/source/lib/omnitrace/library/components/fork_gotcha.cpp

@@ -25,6 +25,7 @@
 #include "core/config.hpp"
 #include "core/debug.hpp"
 #include "core/perfetto.hpp"
+#include "core/perfetto_fwd.hpp"
 #include "core/state.hpp"
 #include "library/components/fork_gotcha.hpp"
 #include "library/runtime.hpp"
@@ -32,24 +33,45 @@
 
 #include <timemory/backends/process.hpp>
 #include <timemory/backends/threading.hpp>
+#include <timemory/mpl/types.hpp>
+#include <timemory/process/process.hpp>
 
+#include <cstdlib>
+#include <memory>
+#include <pthread.h>
 #include <unistd.h>
 
 namespace omnitrace
 {
 namespace component
 {
-void
-fork_gotcha::configure()
+namespace
 {
-    fork_gotcha_t::get_initializer() = []() {
-        TIMEMORY_C_GOTCHA(fork_gotcha_t, 0, fork);
-    };
+// these are used to prevent handlers from executing multiple times
+bool prefork_lock         = false;
+bool postfork_parent_lock = false;
+bool postfork_child_lock  = false;
+
+// this does a quick exit (no cleanup) on child processes
+// because perfetto has a tendency to access memory it
+// shouldn't during cleanup
+void
+child_exit(int _ec, void*)
+{
+    std::quick_exit(_ec);
 }
 
 void
-fork_gotcha::audit(const gotcha_data_t&, audit::incoming)
+prefork_setup()
 {
+    if(prefork_lock) return;
+
+    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
+    OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
+
+    if(get_state() < State::Active && !config::settings_are_configured())
+        omnitrace_init_library_hidden();
+
     tim::set_env("OMNITRACE_PRELOAD", "0", 1);
     tim::set_env("OMNITRACE_ROOT_PROCESS", process::get_id(), 0);
     omnitrace_reset_preload_hidden();
@@ -59,39 +81,98 @@ fork_gotcha::audit(const gotcha_data_t&, audit::incoming)
         "Warning! Calling fork() within an OpenMPI application using libfabric "
         "may result is segmentation fault\n");
     TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(get_debug_env(), 16);
+
+    if(config::get_use_sampling()) sampling::block_samples();
+
+    omnitrace::categories::disable_categories(config::get_enabled_categories());
+
+    // prevent re-entry until post-fork routines have been called
+    prefork_lock         = true;
+    postfork_parent_lock = false;
+    postfork_child_lock  = false;
 }
 
 void
-fork_gotcha::audit(const gotcha_data_t&, audit::outgoing, pid_t _pid)
+postfork_parent()
 {
+    if(postfork_parent_lock) return;
+
+    omnitrace::categories::enable_categories(config::get_enabled_categories());
+
+    if(config::get_use_sampling()) sampling::unblock_samples();
+
+    // prevent re-entry until prefork has been called
+    postfork_parent_lock = true;
+    prefork_lock         = false;
+}
+
+void
+postfork_child()
+{
+    if(postfork_child_lock) return;
+
+    OMNITRACE_REQUIRE(is_child_process())
+        << "Error! child process " << process::get_id()
+        << " believes it is the root process " << get_root_process_id() << "\n";
+
+    settings::enabled() = false;
+    settings::verbose() = -127;
+    settings::debug()   = false;
+    omnitrace::sampling::shutdown();
+    omnitrace::categories::shutdown();
+    set_thread_state(::omnitrace::ThreadState::Disabled);
+
+    omnitrace::get_perfetto_session(process::get_parent_id()).release();
+
+    // register these exit handlers to avoid cleaning up resources
+    on_exit(&child_exit, nullptr);
+    std::atexit([]() { child_exit(EXIT_SUCCESS, nullptr); });
+
+    // prevent re-entry until prefork has been called
+    postfork_child_lock = true;
+    prefork_lock        = false;
+}
+}  // namespace
+
+void
+fork_gotcha::configure()
+{
+    fork_gotcha_t::get_initializer() = []() {
+        TIMEMORY_C_GOTCHA(fork_gotcha_t, 0, fork);
+    };
+
+    // registering the pthread_atfork and gotcha means that we might execute twice
+    // handlers twice, hence the locks
+    pthread_atfork(&prefork_setup, &postfork_parent, &postfork_child);
+}
+
+pid_t
+fork_gotcha::operator()(const gotcha_data_t&, pid_t (*_real_fork)()) const
+{
+    prefork_setup();
+
+    auto _pid = (*_real_fork)();
+
     if(_pid != 0)
     {
         OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getppid(),
                                 _pid);
+
+        postfork_parent();
     }
     else
     {
-        OMNITRACE_REQUIRE(is_child_process())
-            << "Error! child process " << process::get_id()
-            << " believes it is the root process " << get_root_process_id() << "\n";
-        settings::enabled() = false;
-        settings::verbose() = -127;
-        settings::debug()   = false;
-        omnitrace::sampling::shutdown();
-        omnitrace::categories::shutdown();
-        omnitrace::get_perfetto_session().release();
-        set_thread_state(::omnitrace::ThreadState::Disabled);
+        postfork_child();
     }
 
     if(!settings::use_output_suffix())
     {
-        settings::use_output_suffix()      = true;
-        settings::default_process_suffix() = process::get_id();
-
         OMNITRACE_BASIC_VERBOSE(
-            0, "call to fork() enables using an output suffix. PID %i will use %i\n",
-            process::get_id(), process::get_id());
+            0, "Application which make calls to fork() should enable using an process "
+               "identifier output suffix (i.e. set OMNITRACE_USE_PID=ON)\n");
     }
+
+    return _pid;
 }
 }  // namespace component
 }  // namespace omnitrace

projects/rocprofiler-systems/source/lib/omnitrace/library/components/fork_gotcha.hpp

@@ -46,10 +46,7 @@ struct fork_gotcha : comp::base<fork_gotcha, void>
     static void configure();
 
     // this will get called right before fork
-    static void audit(const gotcha_data_t& _data, audit::incoming);
-
-    // this will get called right after fork with the return value
-    static void audit(const gotcha_data_t& _data, audit::outgoing, pid_t _pid);
+    pid_t operator()(const gotcha_data_t&, pid_t (*)()) const;
 
     // silence SFINAE disabled for omnitrace::fork_gotcha warnings
     static inline void start() {}
@@ -57,7 +54,6 @@ struct fork_gotcha : comp::base<fork_gotcha, void>
 };
 }  // namespace component
 
-using fork_gotcha_t =
-    comp::gotcha<component::fork_gotcha::gotcha_capacity,
-                 tim::component_tuple<component::fork_gotcha>, project::omnitrace>;
+using fork_gotcha_t = comp::gotcha<component::fork_gotcha::gotcha_capacity, std::tuple<>,
+                                   component::fork_gotcha>;
 }  // namespace omnitrace

projects/rocprofiler-systems/source/lib/omnitrace/library/components/numa_gotcha.cpp

@@ -30,6 +30,7 @@
 #include "library/runtime.hpp"
 
 #include <timemory/backends/threading.hpp>
+#include <timemory/components/macros.hpp>
 #include <timemory/mpl/concepts.hpp>
 #include <timemory/utility/types.hpp>
 
@@ -204,3 +205,5 @@ numa_gotcha::audit(const gotcha_data& _data, audit::outgoing, void* ret)
 }
 }  // namespace component
 }  // namespace omnitrace
+
+TIMEMORY_STORAGE_INITIALIZER(omnitrace::component::numa_gotcha)

projects/rocprofiler-systems/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp

@@ -207,6 +207,8 @@ pthread_create_gotcha::wrapper::operator()() const
                                 _info->index_data->as_string().c_str(),
                                 _parent_info->index_data->as_string().c_str());
         threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid).c_str());
+        auto _manager = tim::manager::instance();
+        if(_manager) _manager->initialize();
         if(!thread_bundle_data_t::instances().at(_tid))
         {
             thread_data<thread_bundle_t>::construct(
@@ -262,12 +264,15 @@ pthread_create_gotcha::wrapper::operator()() const
     // execute the original function
     _ret = m_routine(m_arg);
 
-    pop_thread_state();
+    if(get_state() < ::omnitrace::State::Finalized)
+    {
+        pop_thread_state();
 
-    // execute the destructor actions
-    _dtor();
+        // execute the destructor actions
+        _dtor();
 
-    set_thread_state(ThreadState::Completed);
+        set_thread_state(ThreadState::Completed);
+    }
 
     return _ret;
 }
@@ -281,7 +286,12 @@ pthread_create_gotcha::wrapper::wrap(void* _arg)
     wrapper* _wrapper = static_cast<wrapper*>(_arg);
 
     // execute the original function
-    return (*_wrapper)();
+    void* _ret = (*_wrapper)();
+
+    // eliminate memory leak
+    if(_ret != _arg) delete _wrapper;
+
+    return _ret;
 }
 
 void

projects/rocprofiler-systems/source/lib/omnitrace/library/cpu_freq.cpp

@@ -180,7 +180,7 @@ post_process()
         for(auto& itr : data)
         {
             uint64_t _ts   = std::get<0>(itr);
-            double   _freq = std::get<8>(itr).at(_offset);
+            double   _freq = static_cast<double>(std::get<8>(itr).at(_offset));
             if(!_thread_info->is_valid_time(_ts)) continue;
             write_perfetto_counter_track<category::cpu_freq>(index{ _idx }, _ts, _freq);
         }

projects/rocprofiler-systems/source/lib/omnitrace/library/rocm/hsa_rsrc_factory.cpp

@@ -145,7 +145,8 @@ HsaRsrcFactory::HsaRsrcFactory(bool initialize_hsa)
         CHECK_STATUS("Kern-arg memory pool is not found", HSA_STATUS_ERROR);
 
     // Get AqlProfile API table
-    aqlprofile_api_ = { nullptr };
+    aqlprofile_api_ = {};
+    memset(&aqlprofile_api_, 0, sizeof(aqlprofile_api_));
 #ifdef ROCP_LD_AQLPROFILE
     status = LoadAqlProfileLib(&aqlprofile_api_);
 #else
@@ -156,8 +157,9 @@ HsaRsrcFactory::HsaRsrcFactory(bool initialize_hsa)
     CHECK_STATUS("aqlprofile API table load failed", status);
 
     // Get Loader API table
-    loader_api_ = { nullptr };
-    status      = hsa_api_.hsa_system_get_major_extension_table(
+    loader_api_ = {};
+    memset(&loader_api_, 0, sizeof(loader_api_));
+    status = hsa_api_.hsa_system_get_major_extension_table(
         HSA_EXTENSION_AMD_LOADER, 1, sizeof(loader_api_), &loader_api_);
     CHECK_STATUS("loader API table query failed", status);
 

projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.cpp

@@ -29,6 +29,7 @@
 #include "library/runtime.hpp"
 #include "library/sampling.hpp"
 #include "library/thread_data.hpp"
+#include "library/thread_info.hpp"
 #include "library/tracing.hpp"
 
 #include <timemory/backends/cpu.hpp>
@@ -309,7 +310,7 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
                     uint64_t _beg_ts = begin_timestamp;
                     uint64_t _end_ts = end_timestamp;
                     tracing::push_perfetto_ts(category::rocm_hsa{}, _name, _beg_ts,
-                                              [&](perfetto::EventContext ctx) {
+                                              [&](::perfetto::EventContext ctx) {
                                                   if(config::get_perfetto_annotations())
                                                   {
                                                       tracing::add_perfetto_annotation(
@@ -317,7 +318,7 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
                                                   }
                                               });
                     tracing::pop_perfetto_ts(category::rocm_hsa{}, _name, _end_ts,
-                                             [&](perfetto::EventContext ctx) {
+                                             [&](::perfetto::EventContext ctx) {
                                                  if(config::get_perfetto_annotations())
                                                  {
                                                      tracing::add_perfetto_annotation(
@@ -384,14 +385,14 @@ hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg)
         uint64_t _beg = _beg_ns;
         uint64_t _end = _end_ns;
         tracing::push_perfetto_ts(
-            category::device_hsa{}, *_name, _beg, [&](perfetto::EventContext ctx) {
+            category::device_hsa{}, *_name, _beg, [&](::perfetto::EventContext ctx) {
                 if(config::get_perfetto_annotations())
                 {
                     tracing::add_perfetto_annotation(ctx, "begin_ns", _beg);
                 }
             });
         tracing::pop_perfetto_ts(
-            category::device_hsa{}, *_name, _end, [&](perfetto::EventContext ctx) {
+            category::device_hsa{}, *_name, _end, [&](::perfetto::EventContext ctx) {
                 if(config::get_perfetto_annotations())
                 {
                     tracing::add_perfetto_annotation(ctx, "end_ns", _end);
@@ -702,8 +703,8 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
         {
             auto _api_id = static_cast<hip_api_id_t>(cid);
             tracing::push_perfetto_ts(
-                category::rocm_hip{}, op_name, _ts, perfetto::Flow::ProcessScoped(_cid),
-                [&](perfetto::EventContext ctx) {
+                category::rocm_hip{}, op_name, _ts, ::perfetto::Flow::ProcessScoped(_cid),
+                [&](::perfetto::EventContext ctx) {
                     if(config::get_perfetto_annotations())
                     {
                         tracing::add_perfetto_annotation(ctx, "begin_ns", _ts);
@@ -753,7 +754,7 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
         if(get_use_perfetto())
         {
             tracing::pop_perfetto_ts(
-                category::rocm_hip{}, op_name, _ts, [&](perfetto::EventContext ctx) {
+                category::rocm_hip{}, op_name, _ts, [&](::perfetto::EventContext ctx) {
                     if(config::get_perfetto_annotations())
                     {
                         tracing::add_perfetto_annotation(ctx, "end_ns", _ts);
@@ -939,7 +940,7 @@ hip_activity_callback(const char* begin, const char* end, void* arg)
             assert(_end_ns >= _beg_ns);
             tracing::push_perfetto_track(
                 category::device_hip{}, _kernel_names.at(_name).c_str(), _track, _beg_ns,
-                perfetto::Flow::ProcessScoped(_cid), [&](perfetto::EventContext ctx) {
+                ::perfetto::Flow::ProcessScoped(_cid), [&](::perfetto::EventContext ctx) {
                     if(config::get_perfetto_annotations())
                     {
                         tracing::add_perfetto_annotation(ctx, "begin_ns", _beg_ns);

projects/rocprofiler-systems/source/lib/omnitrace/library/sampling.cpp

@@ -370,7 +370,17 @@ start_duration_thread()
 auto&
 get_offload_file()
 {
-    static auto _v = config::get_tmp_file("sampling");
+    static auto _v = []() {
+        auto _tmp_v = config::get_tmp_file("sampling");
+        if(get_use_tmp_files())
+        {
+            auto _success = _tmp_v->open();
+            OMNITRACE_CI_FAIL(!_success,
+                              "Error opening sampling offload temporary file '%s'\n",
+                              _tmp_v->filename.c_str());
+        }
+        return _tmp_v;
+    }();
     return _v;
 }
 
@@ -853,7 +863,7 @@ post_process()
         if(itr) itr.reset();
     }
 
-    if(get_offload_file())
+    if(get_use_tmp_files() && get_offload_file())
     {
         get_offload_file()->remove();
         get_offload_file().reset();
@@ -917,7 +927,7 @@ post_process_perfetto(int64_t _tid, const bundle_t* _init,
         _thread_info->index_data->sequent_value, _thread_info->index_data->system_value);
 
     tracing::push_perfetto_track(category::sampling{}, "samples [omnitrace]", _track,
-                                 _beg_ns, [&](perfetto::EventContext ctx) {
+                                 _beg_ns, [&](::perfetto::EventContext ctx) {
                                      if(config::get_perfetto_annotations())
                                      {
                                          tracing::add_perfetto_annotation(ctx, "begin_ns",
@@ -1002,7 +1012,7 @@ post_process_perfetto(int64_t _tid, const bundle_t* _init,
                     auto _info = JOIN(':', litr.location, litr.line);
                     tracing::push_perfetto_track(
                         category::sampling{}, _name, _track, _beg,
-                        [&](perfetto::EventContext ctx) {
+                        [&](::perfetto::EventContext ctx) {
                             if(config::get_perfetto_annotations())
                             {
                                 _common_annotate(ctx, (_n == 0 && _ncur == 0) ||
@@ -1023,7 +1033,7 @@ post_process_perfetto(int64_t _tid, const bundle_t* _init,
                 const auto* _name = _static_strings.emplace(iitr.name).first->c_str();
                 tracing::push_perfetto_track(
                     category::sampling{}, _name, _track, _beg,
-                    [&](perfetto::EventContext ctx) {
+                    [&](::perfetto::EventContext ctx) {
                         if(config::get_perfetto_annotations())
                         {
                             _common_annotate(ctx, true);
@@ -1057,7 +1067,7 @@ post_process_perfetto(int64_t _tid, const bundle_t* _init,
     }
 
     tracing::pop_perfetto_track(category::sampling{}, "samples [omnitrace]", _track,
-                                _end_ns, [&](perfetto::EventContext ctx) {
+                                _end_ns, [&](::perfetto::EventContext ctx) {
                                     if(config::get_perfetto_annotations())
                                     {
                                         tracing::add_perfetto_annotation(ctx, "end_ns",

projects/rocprofiler-systems/source/lib/omnitrace/library/tracing.hpp

@@ -108,11 +108,11 @@ pop_perfetto_ts(CategoryT, const char*, uint64_t, Args&&...);
 
 template <typename CategoryT, typename... Args>
 inline void
-push_perfetto_track(CategoryT, const char*, perfetto::Track, uint64_t, Args&&...);
+push_perfetto_track(CategoryT, const char*, ::perfetto::Track, uint64_t, Args&&...);
 
 template <typename CategoryT, typename... Args>
 inline void
-pop_perfetto_track(CategoryT, const char*, perfetto::Track, uint64_t, Args&&...);
+pop_perfetto_track(CategoryT, const char*, ::perfetto::Track, uint64_t, Args&&...);
 
 template <typename CategoryT, typename... Args>
 inline void
@@ -124,7 +124,7 @@ mark_perfetto_ts(CategoryT, const char*, uint64_t, Args&&...);
 
 template <typename CategoryT, typename... Args>
 inline void
-mark_perfetto_track(CategoryT, const char*, perfetto::Track, uint64_t, Args&&...);
+mark_perfetto_track(CategoryT, const char*, ::perfetto::Track, uint64_t, Args&&...);
 
 //
 //  definitions
@@ -336,19 +336,21 @@ push_perfetto(CategoryT, const char* name, Args&&... args)
     if(category_push_disabled<CategoryT>()) return;
 
     if constexpr(sizeof...(Args) == 1 &&
-                 std::is_invocable<Args..., perfetto::EventContext>::value)
+                 std::is_invocable<Args..., ::perfetto::EventContext>::value)
     {
         ++get_tracing_stack<CategoryT>();
         uint64_t _ts = now();
         if(config::get_perfetto_annotations())
         {
-            TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name),
-                              _ts, "begin_ns", _ts, std::forward<Args>(args)...);
+            TRACE_EVENT_BEGIN(trait::name<CategoryT>::value,
+                              ::perfetto::StaticString(name), _ts, "begin_ns", _ts,
+                              std::forward<Args>(args)...);
         }
         else
         {
-            TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name),
-                              _ts, std::forward<Args>(args)...);
+            TRACE_EVENT_BEGIN(trait::name<CategoryT>::value,
+                              ::perfetto::StaticString(name), _ts,
+                              std::forward<Args>(args)...);
         }
     }
     else
@@ -357,7 +359,7 @@ push_perfetto(CategoryT, const char* name, Args&&... args)
         using arg0_type  = concepts::tuple_element_t<0, tuple_type>;
         using arg1_type  = concepts::tuple_element_t<1, tuple_type>;
 
-        if constexpr(std::is_same<arg0_type, perfetto::Track>::value &&
+        if constexpr(std::is_same<arg0_type, ::perfetto::Track>::value &&
                      std::is_same<arg1_type, uint64_t>::value)
         {
             push_perfetto_track(CategoryT{}, name, std::forward<Args>(args)...);
@@ -371,8 +373,8 @@ push_perfetto(CategoryT, const char* name, Args&&... args)
             ++get_tracing_stack<CategoryT>();
             uint64_t _ts = now();
             TRACE_EVENT_BEGIN(
-                trait::name<CategoryT>::value, perfetto::StaticString(name), _ts,
-                std::forward<Args>(args)..., [&](perfetto::EventContext ctx) {
+                trait::name<CategoryT>::value, ::perfetto::StaticString(name), _ts,
+                std::forward<Args>(args)..., [&](::perfetto::EventContext ctx) {
                     if(config::get_perfetto_annotations())
                     {
                         tracing::add_perfetto_annotation(ctx, "begin_ns", _ts);
@@ -390,7 +392,7 @@ pop_perfetto(CategoryT, const char* name, Args&&... args)
     if(tracing_pop_disabled<CategoryT>()) return;
 
     if constexpr(sizeof...(Args) == 1 &&
-                 std::is_invocable<Args..., perfetto::EventContext>::value)
+                 std::is_invocable<Args..., ::perfetto::EventContext>::value)
     {
         // decrement tracing stack
         --get_tracing_stack<CategoryT>();
@@ -412,7 +414,7 @@ pop_perfetto(CategoryT, const char* name, Args&&... args)
         using arg0_type  = concepts::tuple_element_t<0, tuple_type>;
         using arg1_type  = concepts::tuple_element_t<1, tuple_type>;
 
-        if constexpr(std::is_same<arg0_type, perfetto::Track>::value &&
+        if constexpr(std::is_same<arg0_type, ::perfetto::Track>::value &&
                      std::is_same<arg1_type, uint64_t>::value)
         {
             pop_perfetto_track(CategoryT{}, name, std::forward<Args>(args)...);
@@ -427,7 +429,8 @@ pop_perfetto(CategoryT, const char* name, Args&&... args)
             --get_tracing_stack<CategoryT>();
             uint64_t _ts = now();
             TRACE_EVENT_END(trait::name<CategoryT>::value, _ts,
-                            std::forward<Args>(args)..., [&](perfetto::EventContext ctx) {
+                            std::forward<Args>(args)...,
+                            [&](::perfetto::EventContext ctx) {
                                 if(config::get_perfetto_annotations())
                                 {
                                     tracing::add_perfetto_annotation(ctx, "end_ns", _ts);
@@ -447,7 +450,7 @@ push_perfetto_ts(CategoryT, const char* name, uint64_t _ts, Args&&... args)
     if(category_push_disabled<CategoryT>()) return;
 
     ++get_tracing_stack<CategoryT>();
-    TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name), _ts,
+    TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, ::perfetto::StaticString(name), _ts,
                       std::forward<Args>(args)...);
 }
 
@@ -466,20 +469,20 @@ pop_perfetto_ts(CategoryT, const char*, uint64_t _ts, Args&&... args)
 
 template <typename CategoryT, typename... Args>
 inline void
-push_perfetto_track(CategoryT, const char* name, perfetto::Track _track, uint64_t _ts,
+push_perfetto_track(CategoryT, const char* name, ::perfetto::Track _track, uint64_t _ts,
                     Args&&... args)
 {
     // skip if category is disabled
     if(category_push_disabled<CategoryT>()) return;
 
     ++get_tracing_stack<CategoryT>();
-    TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name), _track,
-                      _ts, std::forward<Args>(args)...);
+    TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, ::perfetto::StaticString(name),
+                      _track, _ts, std::forward<Args>(args)...);
 }
 
 template <typename CategoryT, typename... Args>
 inline void
-pop_perfetto_track(CategoryT, const char*, perfetto::Track _track, uint64_t _ts,
+pop_perfetto_track(CategoryT, const char*, ::perfetto::Track _track, uint64_t _ts,
                    Args&&... args)
 {
     // skip if category is disabled and not pushed on this thread
@@ -500,19 +503,19 @@ mark_perfetto(CategoryT, const char* name, Args&&... args)
     if(category_mark_disabled<CategoryT>()) return;
 
     if constexpr(sizeof...(Args) == 1 &&
-                 std::is_invocable<Args..., perfetto::EventContext>::value)
+                 std::is_invocable<Args..., ::perfetto::EventContext>::value)
     {
         uint64_t _ts = now();
         if(config::get_perfetto_annotations())
         {
             TRACE_EVENT_INSTANT(trait::name<CategoryT>::value,
-                                perfetto::StaticString(name), _ts, "ns", _ts,
+                                ::perfetto::StaticString(name), _ts, "ns", _ts,
                                 std::forward<Args>(args)...);
         }
         else
         {
             TRACE_EVENT_INSTANT(trait::name<CategoryT>::value,
-                                perfetto::StaticString(name), _ts,
+                                ::perfetto::StaticString(name), _ts,
                                 std::forward<Args>(args)...);
         }
     }
@@ -522,7 +525,7 @@ mark_perfetto(CategoryT, const char* name, Args&&... args)
         using arg0_type  = concepts::tuple_element_t<0, tuple_type>;
         using arg1_type  = concepts::tuple_element_t<1, tuple_type>;
 
-        if constexpr(std::is_same<arg0_type, perfetto::Track>::value &&
+        if constexpr(std::is_same<arg0_type, ::perfetto::Track>::value &&
                      std::is_same<arg1_type, uint64_t>::value)
         {
             mark_perfetto_track(CategoryT{}, name, std::forward<Args>(args)...);
@@ -535,8 +538,8 @@ mark_perfetto(CategoryT, const char* name, Args&&... args)
         {
             uint64_t _ts = now();
             TRACE_EVENT_INSTANT(
-                trait::name<CategoryT>::value, perfetto::StaticString(name), _ts,
-                std::forward<Args>(args)..., [&](perfetto::EventContext ctx) {
+                trait::name<CategoryT>::value, ::perfetto::StaticString(name), _ts,
+                std::forward<Args>(args)..., [&](::perfetto::EventContext ctx) {
                     if(config::get_perfetto_annotations())
                     {
                         tracing::add_perfetto_annotation(ctx, "ns", _ts);
@@ -553,13 +556,13 @@ mark_perfetto_ts(CategoryT, const char* name, uint64_t _ts, Args&&... args)
     // skip if category is disabled
     if(category_mark_disabled<CategoryT>()) return;
 
-    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, perfetto::StaticString(name), _ts,
-                        std::forward<Args>(args)...);
+    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, ::perfetto::StaticString(name),
+                        _ts, std::forward<Args>(args)...);
 }
 
 template <typename CategoryT, typename... Args>
 inline void
-mark_perfetto_track(CategoryT, const char*, perfetto::Track _track, uint64_t _ts,
+mark_perfetto_track(CategoryT, const char*, ::perfetto::Track _track, uint64_t _ts,
                     Args&&... args)
 {
     // skip if category is disabled

projects/rocprofiler-systems/source/lib/omnitrace/library/tracing/annotation.hpp

@@ -107,30 +107,30 @@ add_perfetto_annotation(
     }
     else if constexpr(std::is_same<value_type, bool>::value)
     {
-        _get_dbg()->set_bool_value(std::forward<Tp>(_val));
+        _get_dbg()->set_bool_value(_val);
     }
     else if constexpr(std::is_enum<value_type>::value)
     {
-        _get_dbg()->set_int_value(static_cast<int64_t>(std::forward<Tp>(_val)));
+        _get_dbg()->set_int_value(static_cast<int64_t>(_val));
     }
     else if constexpr(std::is_floating_point<value_type>::value)
     {
-        _get_dbg()->set_double_value(std::forward<Tp>(_val));
+        _get_dbg()->set_double_value(static_cast<double>(_val));
     }
     else if constexpr(std::is_integral<value_type>::value)
     {
         if constexpr(std::is_unsigned<value_type>::value)
         {
-            _get_dbg()->set_uint_value(std::forward<Tp>(_val));
+            _get_dbg()->set_uint_value(_val);
         }
         else
         {
-            _get_dbg()->set_int_value(std::forward<Tp>(_val));
+            _get_dbg()->set_int_value(_val);
         }
     }
     else if constexpr(std::is_pointer<value_type>::value)
     {
-        _get_dbg()->set_pointer_value(reinterpret_cast<uint64_t>(std::forward<Tp>(_val)));
+        _get_dbg()->set_pointer_value(reinterpret_cast<uint64_t>(_val));
     }
     else if constexpr(concepts::can_stringify<value_type>::value)
     {

projects/rocprofiler-systems/tests/CMakeLists.txt

@@ -253,7 +253,7 @@ if(OMNITRACE_USE_MPI OR OMNITRACE_USE_MPI_HEADERS)
             args
             --min-instructions
             0
-        ENVIRONMENT "${_base_environment}"
+        ENVIRONMENT "${_base_environment};OMNITRACE_VERBOSE=1"
         REWRITE_RUN_PASS_REGEX
             "(/[A-Za-z-]+/perfetto-trace-0.proto).*(/[A-Za-z-]+/wall_clock-0.txt')"
         REWRITE_RUN_FAIL_REGEX
@@ -338,7 +338,7 @@ if(OMNITRACE_USE_MPI OR OMNITRACE_USE_MPI_HEADERS)
             set(_mpip_${_EXAMPLE}_environment "${_mpip_environment}")
         endif()
         omnitrace_add_test(
-            SKIP_RUNTIME SKIP_SAMPLING
+            SKIP_RUNTIME SKIP_SAMPLING SKIP_PRELOAD
             NAME "mpi-${_EXAMPLE}"
             TARGET mpi-${_EXAMPLE}
             MPI ON
@@ -815,7 +815,18 @@ omnitrace_add_test(
     TARGET fork-example
     REWRITE_ARGS -e -v 2 --print-instrumented modules -i 16
     RUNTIME_ARGS -e -v 1 --label file -i 16
-    ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=ON")
+    ENVIRONMENT
+        "${_base_environment};OMNITRACE_CRITICAL_TRACE=ON;OMNITRACE_SAMPLING_FREQ=250;OMNITRACE_SAMPLING_REALTIME=ON"
+    PRELOAD_PASS_REGEX "fork.. called on PID"
+    RUNTIME_PASS_REGEX "fork.. called on PID"
+    REWRITE_RUN_PASS_REGEX "fork.. called on PID"
+    PRELOAD_FAIL_REGEX
+        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
+    RUNTIME_FAIL_REGEX
+        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
+    REWRITE_RUN_FAIL_REGEX
+        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
+    )
 
 # -------------------------------------------------------------------------------------- #
 #
@@ -1068,7 +1079,7 @@ omnitrace_add_causal_test(
     )
 
 set(_causal_common_args
-    "-n 12 -e -s 0 10 20 30 -B $<TARGET_FILE_BASE_NAME:causal-cpu-omni>")
+    "-n 5 -e -s 0 10 20 30 -B $<TARGET_FILE_BASE_NAME:causal-cpu-omni>")
 
 macro(
     causal_e2e_args_and_validation
@@ -1103,49 +1114,49 @@ omnitrace_add_causal_test(
     SKIP_BASELINE
     NAME cpu-omni-slow-func-e2e
     TARGET causal-cpu-omni
-    RUN_ARGS 80 12 432525 250000000
+    RUN_ARGS 80 12 432525 500000000
     CAUSAL_MODE "func"
     CAUSAL_ARGS ${_causal_slow_func_args}
     CAUSAL_VALIDATE_ARGS ${_causal_slow_func_valid}
     CAUSAL_PASS_REGEX
         "Starting causal experiment #1(.*)causal/experiments.json(.*)causal/experiments.coz"
-    )
+    PROPERTIES PROCESSORS 2 PROCESSOR_AFFINITY ON)
 
 omnitrace_add_causal_test(
     SKIP_BASELINE
     NAME cpu-omni-fast-func-e2e
     TARGET causal-cpu-omni
-    RUN_ARGS 80 12 432525 250000000
+    RUN_ARGS 80 12 432525 500000000
     CAUSAL_MODE "func"
     CAUSAL_ARGS ${_causal_fast_func_args}
     CAUSAL_VALIDATE_ARGS ${_causal_fast_func_valid}
     CAUSAL_PASS_REGEX
         "Starting causal experiment #1(.*)causal/experiments.json(.*)causal/experiments.coz"
-    )
+    PROPERTIES PROCESSORS 2 PROCESSOR_AFFINITY OFF)
 
 omnitrace_add_causal_test(
     SKIP_BASELINE
     NAME cpu-omni-line-155-e2e
     TARGET causal-cpu-omni
-    RUN_ARGS 80 12 432525 250000000
+    RUN_ARGS 80 12 432525 500000000
     CAUSAL_MODE "line"
     CAUSAL_ARGS ${_causal_line_155_args}
     CAUSAL_VALIDATE_ARGS ${_causal_line_155_valid}
     CAUSAL_PASS_REGEX
         "Starting causal experiment #1(.*)causal/experiments.json(.*)causal/experiments.coz"
-    )
+    PROPERTIES PROCESSORS 2 PROCESSOR_AFFINITY ON)
 
 omnitrace_add_causal_test(
     SKIP_BASELINE
     NAME cpu-omni-line-165-e2e
     TARGET causal-cpu-omni
-    RUN_ARGS 80 12 432525 250000000
+    RUN_ARGS 80 12 432525 500000000
     CAUSAL_MODE "line"
     CAUSAL_ARGS ${_causal_line_165_args}
     CAUSAL_VALIDATE_ARGS ${_causal_line_165_valid}
     CAUSAL_PASS_REGEX
         "Starting causal experiment #1(.*)causal/experiments.json(.*)causal/experiments.coz"
-    )
+    PROPERTIES PROCESSORS 2 PROCESSOR_AFFINITY OFF)
 
 # -------------------------------------------------------------------------------------- #
 #

projects/rocprofiler-systems/tests/omnitrace-testing.cmake

@@ -257,7 +257,7 @@ endif()
 
 function(OMNITRACE_WRITE_TEST_CONFIG _FILE _ENV)
     set(_ENV_ONLY
-        "OMNITRACE_(MODE|USE_MPIP|DEBUG_SETTINGS|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME)="
+        "OMNITRACE_(MODE|USE_MPIP|DEBUG_SETTINGS|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)="
         )
     set(_FILE_CONTENTS)
     set(_ENV_CONTENTS)
@@ -296,6 +296,43 @@ ${_FILE_CONTENTS}
         PARENT_SCOPE)
 endfunction()
 
+# -------------------------------------------------------------------------------------- #
+# extends the timeout when sanitizers are used due to slowdown
+function(OMNITRACE_ADJUST_TIMEOUT_FOR_SANITIZER _VAR)
+    if(OMNITRACE_USE_SANITIZER)
+        math(EXPR _timeout_v "2 * ${${_VAR}}")
+        set(${_VAR}
+            "${_timeout_v}"
+            PARENT_SCOPE)
+    endif()
+endfunction()
+
+# -------------------------------------------------------------------------------------- #
+# extends the timeout when sanitizers are used due to slowdown
+macro(OMNITRACE_PATCH_SANITIZER_ENVIRONMENT _VAR)
+    if(OMNITRACE_USE_SANITIZER)
+        if(OMNITRACE_USE_SANITIZER)
+            if(OMNITRACE_SANITIZER_TYPE MATCHES "address")
+                if(NOT ASAN_LIBRARY)
+                    omnitrace_message(
+                        FATAL_ERROR
+                        "Please define the realpath to the address sanitizer library in variable ASAN_LIBRARY"
+                        )
+                endif()
+                list(APPEND ${_VAR} "LD_PRELOAD=${ASAN_LIBRARY}")
+            elseif(OMNITRACE_SANITIZER_TYPE MATCHES "thread")
+                if(NOT TSAN_LIBRARY)
+                    omnitrace_message(
+                        FATAL_ERROR
+                        "Please define the realpath to the thread sanitizer library in variable TSAN_LIBRARY"
+                        )
+                endif()
+                list(APPEND ${_VAR} "LD_PRELOAD=${TSAN_LIBRARY}")
+            endif()
+        endif()
+    endif()
+endmacro()
+
 # -------------------------------------------------------------------------------------- #
 
 function(OMNITRACE_ADD_TEST)
@@ -370,7 +407,7 @@ function(OMNITRACE_ADD_TEST)
             list(APPEND TEST_ENVIRONMENT "OMNITRACE_USE_PID=OFF")
         endif()
 
-        if(NOT TEST_SKIP_BASELINE)
+        if(NOT TEST_SKIP_BASELINE AND NOT OMNITRACE_USE_SANITIZER)
             add_test(
                 NAME ${TEST_NAME}-baseline
                 COMMAND ${COMMAND_PREFIX} $<TARGET_FILE:${TEST_TARGET}> ${TEST_RUN_ARGS}
@@ -419,7 +456,7 @@ function(OMNITRACE_ADD_TEST)
                 WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
         endif()
 
-        if(NOT TEST_SKIP_RUNTIME)
+        if(NOT TEST_SKIP_RUNTIME AND NOT OMNITRACE_USE_SANITIZER)
             add_test(
                 NAME ${TEST_NAME}-runtime-instrument
                 COMMAND $<TARGET_FILE:omnitrace-exe> ${TEST_RUNTIME_ARGS} --
@@ -427,7 +464,8 @@ function(OMNITRACE_ADD_TEST)
                 WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
         endif()
 
-        if(TEST_FORCE_SAMPLING OR (NOT TEST_SKIP_RUNTIME AND NOT TEST_SKIP_SAMPLING))
+        if((TEST_FORCE_SAMPLING OR (NOT TEST_SKIP_RUNTIME AND NOT TEST_SKIP_SAMPLING))
+           AND NOT OMNITRACE_USE_SANITIZER)
             add_test(
                 NAME ${TEST_NAME}-runtime-instrument-sampling
                 COMMAND $<TARGET_FILE:omnitrace-exe> -M sampling ${TEST_RUNTIME_ARGS} --
@@ -494,6 +532,12 @@ function(OMNITRACE_ADD_TEST)
                 set(_REGEX_VAR)
             endif()
 
+            if("${_TEST}" MATCHES "binary-rewrite-run|runtime-instrument|preload")
+                omnitrace_patch_sanitizer_environment(_environ)
+            endif()
+
+            omnitrace_adjust_timeout_for_sanitizer(_timeout)
+
             foreach(_TYPE PASS FAIL SKIP)
                 if(_REGEX_VAR)
                     set(_${_TYPE}_REGEX TEST_${_REGEX_VAR}_${_TYPE}_REGEX)
@@ -570,10 +614,18 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
 
         if(NOT "${TEST_CAUSAL_VALIDATE_ARGS}" STREQUAL "")
+            if("$ENV{OMNITRACE_CI}" MATCHES "ON|on|1|true|TRUE"
+               OR "$ENV{CI}" MATCHES "true"
+               OR NOT "$ENV{GITHUB_RUN_ID}" STREQUAL "")
+                set(_VALIDATE_EXTRA "--ci")
+            else()
+                set(_VALIDATE_EXTRA "")
+            endif()
+
             add_test(
                 NAME validate-causal-${TEST_NAME}
                 COMMAND ${CMAKE_CURRENT_LIST_DIR}/validate-causal-json.py
-                        ${TEST_CAUSAL_VALIDATE_ARGS}
+                        ${_VALIDATE_EXTRA} ${TEST_CAUSAL_VALIDATE_ARGS}
                 WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
         endif()
 
@@ -610,6 +662,8 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
 
             set(_timeout ${TEST_CAUSAL_TIMEOUT})
 
+            omnitrace_adjust_timeout_for_sanitizer(_timeout)
+
             if("${_TEST}" MATCHES "validate-causal")
                 set(_timeout ${TEST_CAUSAL_VALIDATE_TIMEOUT})
             endif()
@@ -671,6 +725,8 @@ function(OMNITRACE_ADD_PYTHON_TEST)
         set(TEST_TIMEOUT 120)
     endif()
 
+    omnitrace_adjust_timeout_for_sanitizer(TEST_TIMEOUT)
+
     set(PYTHON_EXECUTABLE "${TEST_PYTHON_EXECUTABLE}")
 
     if(NOT DEFINED TEST_ENVIRONMENT OR "${TEST_ENVIRONMENT}" STREQUAL "")
@@ -853,6 +909,8 @@ function(OMNITRACE_ADD_VALIDATION_TEST)
         set(TEST_TIMEOUT 30)
     endif()
 
+    omnitrace_adjust_timeout_for_sanitizer(TEST_TIMEOUT)
+
     set(PYTHON_EXECUTABLE "${OMNITRACE_VALIDATION_PYTHON}")
 
     list(APPEND TEST_LABELS "validate")

projects/rocprofiler-systems/tests/validate-causal-json.py

@@ -24,6 +24,22 @@ def stddev(_data):
     return _variance**0.5
 
 
+def simpsons_rule(a, b, fa, fb):
+    """Simple numerical integration via Simpson's rule
+
+    https://en.m.wikipedia.org/wiki/Simpson%27s_rule
+    """
+    slope = (fb - fa) / (b - a)
+    # f(x) at midpoint
+    fm = fa + (0.5 * (b - a) * slope)
+
+    factor = (b - a) / 6.0
+    # print(
+    #    f"[{a:8.3f} : {b:8.3f}|{fa:8.3f} : {fb:8.3f}][slope={slope:8.3f}] {factor:8.3f} * ({fa:8.3f} + (4.0 * {fm:8.3f}) + {fb:8.3f})"
+    # )
+    return factor * (fa + (4.0 * fm) + fb)
+
+
 class validation(object):
     def __init__(self, _exp_re, _pp_re, _virt, _expected, _tolerance):
         self.experiment_filter = re.compile(_exp_re)
@@ -40,6 +56,7 @@ class validation(object):
         _prog_speedup,
         _prog_speedup_stddev,
         _base_speedup_stddev,
+        _ci=False,
     ):
         if (
             not re.search(self.experiment_filter, _exp_name)
@@ -49,20 +66,29 @@ class validation(object):
             return None
 
         _tolerance = self.tolerance
-        if _base_speedup_stddev > 2.0 * self.tolerance:
-            sys.stderr.write(
-                f"  [{_exp_name}][{_pp_name}][{_virt_speedup}] base speedup has stddev > 2 * tolerance (+/- {_base_speedup_stddev:.3f}). Relaxing validation...\n"
-            )
+
+        if _ci is True and _virt_speedup > 10:
+            """On GitHub Action servers, you typically only get one core with two hyperthreads.
+            The hyperthreading causes the speedup potential to drop off at higher virtual speedups
+            so we consider
+            """
+            _tolerance += max([_base_speedup_stddev, _prog_speedup_stddev])
+        elif _base_speedup_stddev > self.tolerance:
             _tolerance += math.sqrt(_base_speedup_stddev)
-        elif _prog_speedup_stddev > 2.0 * self.tolerance:
-            sys.stderr.write(
-                f"  [{_exp_name}][{_pp_name}][{_virt_speedup}] program speedup has stddev > 2 * tolerance (+/- {_prog_speedup_stddev:.3f}). Relaxing validation...\n"
-            )
+        elif _prog_speedup_stddev > 1.0:
             _tolerance += math.sqrt(_prog_speedup_stddev)
 
-        return _prog_speedup >= (self.program_speedup - _tolerance) and _prog_speedup <= (
-            self.program_speedup + _tolerance
-        )
+        if _tolerance > self.tolerance:
+            sys.stderr.write(
+                f"    [{_exp_name}][{_pp_name}][{_virt_speedup}] Tolerance adjusted due to stddev or to account for hyperthreading on CI systems ({self.tolerance:.3f} increased to {_tolerance:.3f})...\n"
+            )
+
+        def _compute(_speedup_v, _tolerance_v):
+            return _speedup_v >= (self.program_speedup - _tolerance_v) and _speedup_v <= (
+                self.program_speedup + _tolerance_v
+            )
+
+        return _compute(_prog_speedup, _tolerance)
 
 
 class throughput_point(object):
@@ -203,18 +229,15 @@ class experiment_progress(object):
         self.data = _data
 
     def get_impact(self):
-        speedup_c = [x.compute_speedup() for x in self.data]
-        speedup_v = [x.virtual_speedup() for x in self.data]
+        speedup_c = [float(x.compute_speedup()) for x in self.data]
+        speedup_v = [float(x.virtual_speedup()) for x in self.data]
         impact = []
         for i in range(len(self.data) - 1):
-            x = speedup_v[i + 1] - speedup_v[i]
-            y = [speedup_c[i], speedup_c[i + 1]]
-            y_min = min(y)
-            y_max = max(y)
-            a_low = x * y_min
-            a_upp = 0.5 * x * (y_max - y_min)
-            impact += [a_low + a_upp]
-        # impact = [x.compute_speedup() for x in self.data]
+            impact += [
+                simpsons_rule(
+                    speedup_v[i], speedup_v[i + 1], speedup_c[i], speedup_c[i + 1]
+                )
+            ]
         return [sum(impact), mean(impact), stddev(impact)]
 
     def __len__(self):
@@ -235,16 +258,34 @@ class experiment_progress(object):
         return self.get_impact()[0] < rhs.get_impact()[0]
 
 
-def find_or_insert(_data, _value, _type):
-    if _value not in _data:
-        if _type == "throughput":
-            _data[_value] = throughput_point(_value)
-        elif _type == "latency":
-            _data[_value] = latency_point(_value)
-    return _data[_value]
+def process_samples(data, _data):
+    if not _data:
+        return data
+    for record in _data["omnitrace"]["causal"]["records"]:
+        for samp in record["samples"]:
+            _info = samp["info"]
+            _count = samp["count"]
+            _func = _info["dfunc"]
+            if _func not in data:
+                data[_func] = 0
+            data[_func] += _count
+            for dwarf_entry in _info["dwarf_info"]:
+                _name = "{}:{}".format(dwarf_entry["file"], dwarf_entry["line"])
+                if _name not in data:
+                    data[_name] = 0
+                data[_name] += _count
+    return data
 
 
 def process_data(data, _data, args):
+    def find_or_insert(_data, _value, _type):
+        if _value not in _data:
+            if _type == "throughput":
+                _data[_value] = throughput_point(_value)
+            elif _type == "latency":
+                _data[_value] = latency_point(_value)
+        return _data[_value]
+
     if not _data:
         return data
 
@@ -399,6 +440,14 @@ def main():
         help="Validate speedup: {experiment regex} {progress-point regex} {virtual-speedup} {expected-speedup} {tolerance}",
         default=[],
     )
+    parser.add_argument(
+        "--ci",
+        action="store_true",
+        help="{}. {}".format(
+            "Accept speedup predictions when: (A) virtual speedup > 10 and (B) prediction is within the tolerance after being increased by (0.5 * stddev) and (1.0 * stddev)",
+            "This is primarily used for the CI where the two threads commonly run on 1 CPU core with 2 hyperthreads (causing the speedup potential to drop)",
+        ),
+    )
 
     args = parser.parse_args()
 
@@ -409,17 +458,27 @@ def main():
         args.num_points = num_speedups
 
     data = {}
+    samp = {}
     for inp in args.input:
         with open(inp, "r") as f:
             inp_data = json.load(f)
         data = process_data(data, inp_data, args)
+        samp = process_samples(samp, inp_data)
+
+    print("Samples:")
+    width = max([len(x) for x in samp.keys()])
+    for name, count in sorted(samp.items()):
+        print(f"    {name:{width}} :: {count}")
 
     results = compute_speedups(data, args)
+    print("")
+    print("Experiments:")
     for itr in results:
         if len(itr) < args.num_points:
             continue
         print("")
-        print(f"{itr}")
+        # split each line, indent each line, and join again into single string
+        print("{}".format("\n".join([f"    {x}" for x in f"{itr}".split("\n")])))
 
     sys.stdout.flush()
 
@@ -445,6 +504,7 @@ def main():
                         _prog_speedup,
                         _prog_speedup_stddev,
                         _base_speedup_stddev,
+                        args.ci,
                     )
                     if _v is None:
                         continue
@@ -453,7 +513,7 @@ def main():
                         correct_validations += 1
                     else:
                         sys.stderr.write(
-                            f"  [{_experiment}][{_progresspt}][{_virt_speedup}] failed validation: {_prog_speedup:8.3f} != {vitr.program_speedup} +/- {vitr.tolerance}\n"
+                            f"\n    [{_experiment}][{_progresspt}][{_virt_speedup}] failed validation: {_prog_speedup:8.3f} != {vitr.program_speedup} +/- {vitr.tolerance}\n\n"
                         )
 
     if expected_validations != correct_validations: