CI timeout + line-info in releases (#279)

* Update perfetto args.gn.in

- remove enable_perfetto_tools_trace_to_text (unused)

* core timeout implementation

- requires OMNITRACE_CI=ON
- requires OMNITRACE_CI_TIMEOUT=<sec>
- adds pthread_self and std::this_thread::get_id to thread info
- pthread_create_gotcha stores native handles (pthread_self)

* Testing updates

- improve detection of segfault/failures with PASS_REGEX exists
- add OMNITRACE_CI_TIMEOUT env variable to all tests

* Line-info in releases

- e.g. -g1 + more options to minimize size of debug info

* Fix typo in config exit action message

* OMNITRACE_UNLIKELY around debug/verbose messages

* format fixes

* Overflow tests + capability check

* transpose example update

- link to threads library

* roctracer/rocprofiler update

- in ROCm 5.5.0, cannot include rocprofiler.h and roctracer.h in same file due to conflicting enum defs
- Moved HSA tracing setup/shutdown to component::roctracer

* roctracer update

- fix definition of roctracer::setup when disabled

* Update fork example

- detach threads on main PID
- flush io outputs when printing info

* Update overflow tests

- pass regular expressions
- overflow on PERF_COUNT_SW_CPU_CLOCK event

* fork gotcha update

- use getpid() instead of getppid()

* update fork example

- wait on threads calling fork

* timeout update

- wait on timeout thread to launch before proceeding

[ROCm/rocprofiler-systems commit: 3e2fa69a14]

projects/rocprofiler-systems/.cmake-format.yaml

@@ -80,7 +80,6 @@ parse:
     omnitrace_add_bin_test:
       flags:
       - WILL_FAIL
-      - ADD_INVERSE
       kwargs:
         NAME: '*'
         ARGS: '*'

projects/rocprofiler-systems/cmake/BuildSettings.cmake

@@ -15,6 +15,7 @@ include(MacroUtilities)
 omnitrace_add_option(
     OMNITRACE_BUILD_DEVELOPER "Extra build flags for development like -Werror"
     ${OMNITRACE_BUILD_CI})
+omnitrace_add_option(OMNITRACE_BUILD_RELEASE "Build with minimal debug line info" OFF)
 omnitrace_add_option(OMNITRACE_BUILD_EXTRA_OPTIMIZATIONS "Extra optimization flags" OFF)
 omnitrace_add_option(OMNITRACE_BUILD_LTO "Build with link-time optimization" OFF)
 omnitrace_add_option(OMNITRACE_USE_COMPILE_TIMING
@@ -245,6 +246,15 @@ if(OMNITRACE_BUILD_LINKER)
         $<$<CXX_COMPILER_ID:GNU>:-fuse-ld=${OMNITRACE_BUILD_LINKER}>)
 endif()
 
+# ----------------------------------------------------------------------------------------#
+# release build flags
+#
+if(OMNITRACE_BUILD_RELEASE AND NOT OMNITRACE_BUILD_DEBUG)
+    add_target_flag_if_avail(
+        omnitrace-compile-options "-g1" "-feliminate-unused-debug-symbols"
+        "-gno-column-info" "-gno-variable-location-views" "-gline-tables-only")
+endif()
+
 # ----------------------------------------------------------------------------------------#
 # visibility build flags
 #

projects/rocprofiler-systems/cmake/Templates/args.gn.in

@@ -15,7 +15,6 @@ enable_perfetto_fuzzers = false
 # enable_perfetto_stderr_crash_dump = false
 enable_perfetto_heapprofd = false
 enable_perfetto_tools = false
-enable_perfetto_tools_trace_to_text = false
 enable_perfetto_trace_processor = false
 enable_perfetto_trace_processor_httpd = false
 enable_perfetto_trace_processor_json = false

projects/rocprofiler-systems/examples/fork/fork.cpp

@@ -9,16 +9,22 @@
 #include <sys/wait.h>
 #include <thread>
 #include <unistd.h>
+#include <vector>
 
 void
 print_info(const char* _name)
 {
+    fflush(stdout);
+    fflush(stderr);
     printf("[%s] pid = %i, ppid = %i\n", _name, getpid(), getppid());
+    fflush(stdout);
+    fflush(stderr);
 }
 
 int
 run(const char* _name, int nchildren)
 {
+    auto _threads = std::vector<std::thread>{};
     for(int i = 0; i < nchildren; ++i)
     {
         omnitrace_user_push_region("launch_child");
@@ -35,7 +41,7 @@ run(const char* _name, int nchildren)
                 exit(EXIT_SUCCESS);
             }
         };
-        std::thread{ _run }.join();
+        _threads.emplace_back(_run);
         omnitrace_user_pop_region("launch_child");
     }
 
@@ -70,6 +76,9 @@ run(const char* _name, int nchildren)
         }
     }
 
+    for(auto& itr : _threads)
+        itr.join();
+
     omnitrace_user_pop_region("wait_for_children");
     return _status;
 }

projects/rocprofiler-systems/examples/transpose/CMakeLists.txt

@@ -43,11 +43,13 @@ endif()
 
 option(TRANSPOSE_USE_MPI "Enable MPI support in transpose exe" ${TIMEMORY_USE_MPI})
 
+find_package(Threads REQUIRED)
 if(TRANSPOSE_USE_MPI)
     find_package(MPI REQUIRED)
 endif()
 
 add_executable(transpose transpose.cpp)
+target_link_libraries(transpose PRIVATE Threads::Threads)
 
 if(CMAKE_CXX_COMPILER_ID MATCHES "Clang"
    AND NOT CMAKE_CXX_COMPILER_IS_HIPCC

projects/rocprofiler-systems/scripts/build-release.sh

@@ -197,7 +197,7 @@ if [ ${NJOBS} -gt ${NPROC} ]; then NJOBS=${NPROC}; fi
 
 CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCPACK_GENERATOR=STGZ"
 OMNITRACE_GENERAL_ARGS="-DOMNITRACE_CPACK_SYSTEM_NAME=${DISTRO} -DOMNITRACE_ROCM_VERSION=${ROCM_VERSION} -DOMNITRACE_MAX_THREADS=${MAX_THREADS} -DOMNITRACE_STRIP_LIBRARIES=${STRIP} -DOMNITRACE_INSTALL_PERFETTO_TOOLS=${PERFETTO_TOOLS}"
-OMNITRACE_BUILD_ARGS="-DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_BUILD_EXAMPLES=OFF -DOMNITRACE_BUILD_PAPI=ON -DOMNITRACE_BUILD_LTO=${LTO} -DOMNITRACE_BUILD_HIDDEN_VISIBILITY=${HIDDEN_VIZ} -DOMNITRACE_BUILD_STATIC_LIBGCC=${LIBGCC} -DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${LIBSTDCXX}"
+OMNITRACE_BUILD_ARGS="-DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_BUILD_EXAMPLES=OFF -DOMNITRACE_BUILD_PAPI=ON -DOMNITRACE_BUILD_LTO=${LTO} -DOMNITRACE_BUILD_HIDDEN_VISIBILITY=${HIDDEN_VIZ} -DOMNITRACE_BUILD_STATIC_LIBGCC=${LIBGCC} -DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${LIBSTDCXX} -DOMNITRACE_BUILD_RELEASE=ON"
 OMNITRACE_USE_ARGS="-DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_USE_PAPI=ON"
 TIMEMORY_ARGS="-DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON"
 DYNINST_ARGS="-DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_USE_OpenMP=ON $(echo -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON) -DDYNINST_BOOST_DOWNLOAD_VERSION=${BOOST_VERSION}"

projects/rocprofiler-systems/source/bin/tests/CMakeLists.txt

@@ -1,8 +1,12 @@
+set(OMNITRACE_ABORT_FAIL_REGEX
+    "### ERROR ###|address of faulting memory reference|exiting with non-zero exit code|terminate called after throwing an instance|calling abort.. in |Exit code: [1-9]"
+    CACHE INTERNAL "Regex to catch abnormal exits when a PASS_REGULAR_EXPRESSION is set")
+
 # adds a ctest for executable
 function(OMNITRACE_ADD_BIN_TEST)
     cmake_parse_arguments(
         TEST
-        "ADD_INVERSE" # options
+        "" # options
         "NAME;TARGET;TIMEOUT;WORKING_DIRECTORY" # single value args
         "ARGS;ENVIRONMENT;LABELS;PROPERTIES;PASS_REGEX;FAIL_REGEX;SKIP_REGEX;DEPENDS;COMMAND" # multiple
         # value args
@@ -33,9 +37,14 @@ function(OMNITRACE_ADD_BIN_TEST)
     endif()
 
     # common
-    list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON" "OMNITRACE_CONFIG_FILE="
-         "OMNITRACE_OUTPUT_PATH=${PROJECT_BINARY_DIR}/omnitrace-tests-output"
-         "TWD=${TEST_WORKING_DIRECTORY}")
+    list(
+        APPEND
+        TEST_ENVIRONMENT
+        "OMNITRACE_CI=ON"
+        "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}"
+        "OMNITRACE_CONFIG_FILE="
+        "OMNITRACE_OUTPUT_PATH=${PROJECT_BINARY_DIR}/omnitrace-tests-output"
+        "TWD=${TEST_WORKING_DIRECTORY}")
     # copy for inverse
     set(TEST_ENVIRONMENT_INV "${TEST_ENVIRONMENT}")
 
@@ -43,6 +52,22 @@ function(OMNITRACE_ADD_BIN_TEST)
     list(APPEND TEST_ENVIRONMENT "OMNITRACE_OUTPUT_PREFIX=${TEST_NAME}/")
     list(APPEND TEST_ENVIRONMENT_INV "OMNITRACE_OUTPUT_PREFIX=${TEST_NAME}-inverse/")
 
+    if(NOT "${TEST_PASS_REGEX}" STREQUAL ""
+       AND NOT "${TEST_FAIL_REGEX}" STREQUAL ""
+       AND NOT "${TEST_FAIL_REGEX}" MATCHES "\\|OMNITRACE_ABORT_FAIL_REGEX")
+        omnitrace_message(
+            FATAL_ERROR
+            "${TEST_NAME} has set pass and fail regexes but fail regex does not include '|OMNITRACE_ABORT_FAIL_REGEX'"
+            )
+    endif()
+
+    if("${TEST_FAIL_REGEX}" STREQUAL "")
+        set(TEST_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})")
+    else()
+        string(REPLACE "|OMNITRACE_ABORT_FAIL_REGEX" "|${OMNITRACE_ABORT_FAIL_REGEX}"
+                       TEST_FAIL_REGEX "${TEST_FAIL_REGEX}")
+    endif()
+
     if(TEST_COMMAND)
         add_test(
             NAME ${TEST_NAME}
@@ -66,32 +91,6 @@ function(OMNITRACE_ADD_BIN_TEST)
                        SKIP_REGULAR_EXPRESSION
                        "${TEST_SKIP_REGEX}"
                        ${TEST_PROPERTIES})
-        if(TEST_ADD_INVERSE)
-            add_test(
-                NAME ${TEST_NAME}-inverse
-                COMMAND ${TEST_COMMAND} ${TEST_ARGS}
-                WORKING_DIRECTORY ${TEST_WORKING_DIRECTORY})
-
-            set_tests_properties(
-                ${TEST_NAME}-inverse
-                PROPERTIES ENVIRONMENT
-                           "${TEST_ENVIRONMENT_INV}"
-                           TIMEOUT
-                           ${TEST_TIMEOUT}
-                           DEPENDS
-                           "${TEST_DEPENDS}"
-                           LABELS
-                           "omnitrace-bin;${TEST_LABELS}"
-                           PASS_REGULAR_EXPRESSION
-                           "${TEST_FAIL_REGEX}"
-                           FAIL_REGULAR_EXPRESSION
-                           "${TEST_PASS_REGEX}"
-                           SKIP_REGULAR_EXPRESSION
-                           "${TEST_SKIP_REGEX}"
-                           WILL_FAIL
-                           ON
-                           ${TEST_PROPERTIES})
-        endif()
     elseif(TARGET ${TEST_TARGET})
         add_test(
             NAME ${TEST_NAME}
@@ -115,32 +114,6 @@ function(OMNITRACE_ADD_BIN_TEST)
                        SKIP_REGULAR_EXPRESSION
                        "${TEST_SKIP_REGEX}"
                        ${TEST_PROPERTIES})
-        if(TEST_ADD_INVERSE)
-            add_test(
-                NAME ${TEST_NAME}-inverse
-                COMMAND $<TARGET_FILE:${TEST_TARGET}> ${TEST_ARGS}
-                WORKING_DIRECTORY ${TEST_WORKING_DIRECTORY})
-
-            set_tests_properties(
-                ${TEST_NAME}-inverse
-                PROPERTIES ENVIRONMENT
-                           "${TEST_ENVIRONMENT_INV}"
-                           TIMEOUT
-                           ${TEST_TIMEOUT}
-                           DEPENDS
-                           "${TEST_DEPENDS}"
-                           LABELS
-                           "omnitrace-bin;${TEST_LABELS}"
-                           PASS_REGULAR_EXPRESSION
-                           "${TEST_FAIL_REGEX}"
-                           FAIL_REGULAR_EXPRESSION
-                           "${TEST_PASS_REGEX}"
-                           SKIP_REGULAR_EXPRESSION
-                           "${TEST_SKIP_REGEX}"
-                           WILL_FAIL
-                           ON
-                           ${TEST_PROPERTIES})
-        endif()
     elseif(OMNITRACE_BUILD_TESTING)
         message(FATAL_ERROR "Error! ${TEST_TARGET} does not exist")
     endif()
@@ -189,10 +162,9 @@ omnitrace_add_bin_test(
     TIMEOUT 60
     PASS_REGEX
         ".*available.json.*available.txt.*available.xml.*excluded.json.*excluded.txt.*excluded.xml.*instrumented.json.*instrumented.txt.*instrumented.xml.*overlapping.json.*overlapping.txt.*overlapping.xml.*"
-    FAIL_REGEX "No such file or directory|not found")
+    FAIL_REGEX "No such file or directory|not found|OMNITRACE_ABORT_FAIL_REGEX")
 
 omnitrace_add_bin_test(
-    ADD_INVERSE
     NAME omnitrace-instrument-simulate-lib
     TARGET omnitrace-instrument
     ARGS --print-available functions -v 2 -- $<TARGET_FILE:omnitrace-user-library>
@@ -243,7 +215,7 @@ omnitrace_add_bin_test(
     LABELS "log"
     TIMEOUT 60
     PASS_REGEX "user.log"
-    FAIL_REGEX "No such file or directory|not found")
+    FAIL_REGEX "No such file or directory|not found|OMNITRACE_ABORT_FAIL_REGEX")
 
 omnitrace_add_bin_test(
     NAME omnitrace-avail-help
@@ -268,7 +240,7 @@ omnitrace_add_bin_test(
     ARGS --all --expand-keys
     LABELS "omnitrace-avail"
     TIMEOUT 45
-    FAIL_REGEX "%[a-zA-Z_]%")
+    FAIL_REGEX "%[a-zA-Z_]%|OMNITRACE_ABORT_FAIL_REGEX")
 
 omnitrace_add_bin_test(
     NAME omnitrace-avail-all-only-available-alphabetical
@@ -309,7 +281,7 @@ omnitrace_add_bin_test(
     TIMEOUT 45
     PASS_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE|OUTPUT_PREFIX)"
     FAIL_REGEX
-        "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)"
+        "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)|OMNITRACE_ABORT_FAIL_REGEX"
     )
 
 omnitrace_add_bin_test(
@@ -320,7 +292,7 @@ omnitrace_add_bin_test(
     TIMEOUT 45
     PASS_REGEX
         "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)"
-    FAIL_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE)")
+    FAIL_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE)|OMNITRACE_ABORT_FAIL_REGEX")
 
 omnitrace_add_bin_test(
     NAME omnitrace-avail-regex-negation
@@ -341,7 +313,7 @@ omnitrace_add_bin_test(
     TIMEOUT 45
     PASS_REGEX
         "ENVIRONMENT VARIABLE,[ \n]+OMNITRACE_THREAD_POOL_SIZE,[ \n]+OMNITRACE_USE_PID,[ \n]+"
-    FAIL_REGEX "OMNITRACE_USE_PERFETTO")
+    FAIL_REGEX "OMNITRACE_USE_PERFETTO|OMNITRACE_ABORT_FAIL_REGEX")
 
 string(REPLACE "+" "\\\+" _AVAIL_CFG_PATH
                "${PROJECT_BINARY_DIR}/omnitrace-tests-output/omnitrace-avail/omnitrace-")

projects/rocprofiler-systems/source/lib/core/config.cpp

@@ -1215,7 +1215,7 @@ omnitrace_exit_action(int nsig)
 {
     tim::signals::block_signals(get_sampling_signals(),
                                 tim::signals::sigmask_scope::process);
-    OMNITRACE_BASIC_PRINT("Finalizing afer signal %i :: %s\n", nsig,
+    OMNITRACE_BASIC_PRINT("Finalizing after signal %i :: %s\n", nsig,
                           signal_settings::str(static_cast<sys_signal>(nsig)).c_str());
     auto _handler = get_signal_handler().load();
     if(_handler) (*_handler)();

projects/rocprofiler-systems/source/lib/core/debug.hpp

@@ -222,8 +222,8 @@ as_hex<void*>(void*, size_t);
 //--------------------------------------------------------------------------------------//
 
 #define OMNITRACE_CONDITIONAL_PRINT_COLOR(COLOR, COND, ...)                              \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -236,8 +236,8 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_PRINT_COLOR_F(COLOR, COND, ...)                            \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -259,8 +259,8 @@ as_hex<void*>(void*, size_t);
 //--------------------------------------------------------------------------------------//
 
 #define OMNITRACE_CONDITIONAL_PRINT(COND, ...)                                           \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -273,8 +273,8 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_BASIC_PRINT(COND, ...)                                     \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -287,8 +287,8 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_PRINT_F(COND, ...)                                         \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -302,8 +302,8 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_BASIC_PRINT_F(COND, ...)                                   \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -318,8 +318,8 @@ as_hex<void*>(void*, size_t);
 //--------------------------------------------------------------------------------------//
 
 #define OMNITRACE_CONDITIONAL_WARN(COND, ...)                                            \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -332,8 +332,8 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_BASIC_WARN(COND, ...)                                      \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -346,8 +346,8 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_WARN_F(COND, ...)                                          \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -361,8 +361,8 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_BASIC_WARN_F(COND, ...)                                    \
-    if((COND) && ::omnitrace::config::get_debug_tid() &&                                 \
-       ::omnitrace::config::get_debug_pid())                                             \
+    if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() &&              \
+                          ::omnitrace::config::get_debug_pid()))                         \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::debug::lock _debug_lk{};                                            \
@@ -377,7 +377,7 @@ as_hex<void*>(void*, size_t);
 //--------------------------------------------------------------------------------------//
 
 #define OMNITRACE_CONDITIONAL_THROW_E(COND, TYPE, ...)                                   \
-    if(COND)                                                                             \
+    if(OMNITRACE_UNLIKELY((COND)))                                                       \
     {                                                                                    \
         char _msg_buffer[OMNITRACE_DEBUG_BUFFER_LEN];                                    \
         snprintf(_msg_buffer, OMNITRACE_DEBUG_BUFFER_LEN, "[omnitrace][%i][%li][%s]%s",  \
@@ -391,7 +391,7 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_BASIC_THROW_E(COND, TYPE, ...)                             \
-    if(COND)                                                                             \
+    if(OMNITRACE_UNLIKELY((COND)))                                                       \
     {                                                                                    \
         char _msg_buffer[OMNITRACE_DEBUG_BUFFER_LEN];                                    \
         snprintf(_msg_buffer, OMNITRACE_DEBUG_BUFFER_LEN, "[omnitrace][%i][%s]%s",       \
@@ -428,7 +428,7 @@ as_hex<void*>(void*, size_t);
 //--------------------------------------------------------------------------------------//
 
 #define OMNITRACE_CONDITIONAL_FAILURE(COND, METHOD, ...)                                 \
-    if(COND)                                                                             \
+    if(OMNITRACE_UNLIKELY((COND)))                                                       \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         OMNITRACE_FPRINTF_STDERR_COLOR(fatal);                                           \
@@ -443,7 +443,7 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, METHOD, ...)                           \
-    if(COND)                                                                             \
+    if(OMNITRACE_UNLIKELY((COND)))                                                       \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         OMNITRACE_FPRINTF_STDERR_COLOR(fatal);                                           \
@@ -458,7 +458,7 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_FAILURE_F(COND, METHOD, ...)                               \
-    if(COND)                                                                             \
+    if(OMNITRACE_UNLIKELY((COND)))                                                       \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         OMNITRACE_FPRINTF_STDERR_COLOR(fatal);                                           \
@@ -474,7 +474,7 @@ as_hex<void*>(void*, size_t);
     }
 
 #define OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, METHOD, ...)                         \
-    if(COND)                                                                             \
+    if(OMNITRACE_UNLIKELY((COND)))                                                       \
     {                                                                                    \
         ::omnitrace::debug::flush();                                                     \
         OMNITRACE_FPRINTF_STDERR_COLOR(fatal);                                           \
@@ -621,7 +621,7 @@ as_hex<void*>(void*, size_t);
 
 #define OMNITRACE_WARNING_OR_CI_THROW(LEVEL, ...)                                        \
     {                                                                                    \
-        if(::omnitrace::get_is_continuous_integration())                                 \
+        if(OMNITRACE_UNLIKELY(::omnitrace::get_is_continuous_integration()))             \
         {                                                                                \
             OMNITRACE_CI_THROW(true, __VA_ARGS__);                                       \
         }                                                                                \
@@ -635,7 +635,7 @@ as_hex<void*>(void*, size_t);
 
 #define OMNITRACE_REQUIRE(...) TIMEMORY_REQUIRE(__VA_ARGS__)
 #define OMNITRACE_PREFER(COND)                                                           \
-    (COND)                                           ? ::tim::log::base()                \
+    (OMNITRACE_LIKELY(COND))                         ? ::tim::log::base()                \
     : (::omnitrace::get_is_continuous_integration()) ? TIMEMORY_FATAL                    \
                                                      : TIMEMORY_WARNING
 

projects/rocprofiler-systems/source/lib/omnitrace/CMakeLists.txt

@@ -11,7 +11,7 @@ target_sources(
     omnitrace-object-library
     PRIVATE ${CMAKE_CURRENT_LIST_DIR}/library.cpp ${CMAKE_CURRENT_LIST_DIR}/regions.cpp
             ${CMAKE_CURRENT_LIST_DIR}/progress.cpp ${CMAKE_CURRENT_LIST_DIR}/api.cpp
-            ${CMAKE_CURRENT_LIST_DIR}/api.hpp)
+            ${CMAKE_CURRENT_LIST_DIR}/timeout.cpp ${CMAKE_CURRENT_LIST_DIR}/api.hpp)
 
 add_subdirectory(library)
 

projects/rocprofiler-systems/source/lib/omnitrace/library.cpp

@@ -34,6 +34,7 @@
 #include "core/debug.hpp"
 #include "core/defines.hpp"
 #include "core/gpu.hpp"
+#include "core/locking.hpp"
 #include "core/perfetto_fwd.hpp"
 #include "core/timemory.hpp"
 #include "core/utility.hpp"
@@ -61,22 +62,28 @@
 #include "omnitrace/categories.h"  // in omnitrace-user
 
 #include <timemory/hash/types.hpp>
+#include <timemory/log/logger.hpp>
 #include <timemory/manager/manager.hpp>
 #include <timemory/mpl/type_traits.hpp>
 #include <timemory/operations/types/file_output_message.hpp>
+#include <timemory/process/process.hpp>
 #include <timemory/process/threading.hpp>
 #include <timemory/settings/types.hpp>
 #include <timemory/signals/signal_handlers.hpp>
 #include <timemory/signals/signal_mask.hpp>
 #include <timemory/signals/types.hpp>
+#include <timemory/units.hpp>
 #include <timemory/utility/backtrace.hpp>
 #include <timemory/utility/join.hpp>
 #include <timemory/utility/procfs/maps.hpp>
 
 #include <atomic>
+#include <chrono>
+#include <csignal>
 #include <cstdio>
 #include <cstdlib>
 #include <mutex>
+#include <pthread.h>
 #include <stdexcept>
 #include <string_view>
 #include <utility>
@@ -85,6 +92,15 @@ using namespace omnitrace;
 
 //======================================================================================//
 
+namespace omnitrace
+{
+namespace timeout
+{
+void
+setup() OMNITRACE_INTERNAL_API;
+}
+}  // namespace omnitrace
+
 namespace
 {
 auto _timemory_manager  = tim::manager::instance();
@@ -152,6 +168,8 @@ ensure_finalization(bool _static_init = false)
 
     if(common::get_env("OMNITRACE_MONOCHROME", false)) tim::log::monochrome() = true;
 
+    timeout::setup();
+
     (void) tim::manager::instance();
     (void) tim::settings::shared_instance();
 
@@ -1001,6 +1019,7 @@ omnitrace_finalize_hidden(void)
 
         auto _cfg       = settings::compose_filename_config{};
         _cfg.use_suffix = config::get_use_pid();
+        _cfg.suffix     = settings::default_process_suffix();
         _timemory_manager->write_metadata(settings::get_global_output_prefix(),
                                           "omnitrace", _cfg);
     }

projects/rocprofiler-systems/source/lib/omnitrace/library/components/fork_gotcha.cpp

@@ -155,7 +155,7 @@ fork_gotcha::operator()(const gotcha_data_t&, pid_t (*_real_fork)()) const
 
     if(_pid != 0)
     {
-        OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getppid(),
+        OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getpid(),
                                 _pid);
 
         postfork_parent();

projects/rocprofiler-systems/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp

@@ -138,6 +138,8 @@ stop_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args)
         tim::consume_parameters(_args...);
     }
 }
+
+std::set<pthread_create_gotcha::native_handle_t> native_handles = {};
 }  // namespace
 
 //--------------------------------------------------------------------------------------//
@@ -291,12 +293,20 @@ pthread_create_gotcha::wrapper::wrap(void* _arg)
 {
     if(_arg == nullptr) return nullptr;
 
+    auto _self = pthread_self();
+
     // convert the argument
     wrapper* _wrapper = static_cast<wrapper*>(_arg);
 
+    // store the handle
+    native_handles.emplace(_self);
+
     // execute the original function
     void* _ret = (*_wrapper)();
 
+    // remove the handle
+    if(::pthread_equal(_self, pthread_self()) == 0) native_handles.erase(_self);
+
     // eliminate memory leak
     if(_ret != _arg) delete _wrapper;
 
@@ -378,6 +388,13 @@ pthread_create_gotcha::set_data(wrappee_t _v)
     m_wrappee = _v;
 }
 
+std::set<pthread_create_gotcha::native_handle_t>
+pthread_create_gotcha::get_native_handles()
+{
+    auto _v = native_handles;
+    return _v;
+}
+
 // pthread_create
 int
 pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,

projects/rocprofiler-systems/source/lib/omnitrace/library/components/pthread_create_gotcha.hpp

@@ -32,15 +32,18 @@
 
 namespace omnitrace
 {
+struct pthread_gotcha;
+
 namespace component
 {
 struct pthread_create_gotcha : tim::component::base<pthread_create_gotcha, void>
 {
     static constexpr size_t gotcha_capacity = 1;
 
-    using routine_t = void* (*) (void*);
-    using wrappee_t = int (*)(pthread_t*, const pthread_attr_t*, routine_t, void*);
-    using promise_t = std::shared_ptr<std::promise<void>>;
+    using routine_t       = void* (*) (void*);
+    using wrappee_t       = int (*)(pthread_t*, const pthread_attr_t*, routine_t, void*);
+    using promise_t       = std::shared_ptr<std::promise<void>>;
+    using native_handle_t = std::thread::native_handle_type;
 
     struct wrapper_config
     {
@@ -81,6 +84,10 @@ struct pthread_create_gotcha : tim::component::base<pthread_create_gotcha, void>
     void set_data(wrappee_t);
 
 private:
+    friend struct ::omnitrace::pthread_gotcha;
+
+    static std::set<native_handle_t> get_native_handles();
+
     wrappee_t m_wrappee = &pthread_create;
 };
 

projects/rocprofiler-systems/source/lib/omnitrace/library/components/pthread_gotcha.cpp

@@ -114,4 +114,10 @@ pthread_gotcha::stop()
 {
     get_bundle()->stop();
 }
+
+std::set<pthread_gotcha::native_handle_t>
+pthread_gotcha::get_native_handles()
+{
+    return ::omnitrace::component::pthread_create_gotcha::get_native_handles();
+}
 }  // namespace omnitrace

projects/rocprofiler-systems/source/lib/omnitrace/library/components/pthread_gotcha.hpp

@@ -25,6 +25,7 @@
 #include "core/common.hpp"
 #include "core/defines.hpp"
 #include "core/timemory.hpp"
+#include "library/thread_info.hpp"
 
 #include <cstdint>
 #include <future>
@@ -33,6 +34,8 @@ namespace omnitrace
 {
 struct pthread_gotcha : tim::component::base<pthread_gotcha, void>
 {
+    using native_handle_t = std::thread::native_handle_type;
+
     OMNITRACE_DEFAULT_OBJECT(pthread_gotcha)
 
     // string id for component
@@ -44,5 +47,7 @@ struct pthread_gotcha : tim::component::base<pthread_gotcha, void>
 
     static void start();
     static void stop();
+
+    static std::set<native_handle_t> get_native_handles();
 };
 }  // namespace omnitrace

projects/rocprofiler-systems/source/lib/omnitrace/library/components/roctracer.cpp

@@ -31,6 +31,20 @@
 #include "library/runtime.hpp"
 #include "library/thread_data.hpp"
 
+#include <roctracer.h>
+
+#define HIP_PROF_HIP_API_STRING 1
+
+#include <roctracer_ext.h>
+#include <roctracer_hip.h>
+
+#if OMNITRACE_HIP_VERSION < 50300
+#    include <roctracer_hcc.h>
+#endif
+
+#define AMD_INTERNAL_BUILD 1
+#include <roctracer_hsa.h>
+
 namespace omnitrace
 {
 namespace component
@@ -55,7 +69,7 @@ roctracer::preinit()
 void
 roctracer::start()
 {
-    if(tracker_type::start() == 0) setup();
+    if(tracker_type::start() == 0) setup(nullptr);
 }
 
 void
@@ -111,7 +125,7 @@ roctracer::remove_shutdown(const std::string& _lbl)
 }
 
 void
-roctracer::setup()
+roctracer::setup(void* table, bool on_load_trace)
 {
     if(!get_use_roctracer()) return;
 
@@ -173,6 +187,79 @@ roctracer::setup()
             roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));
     }
 
+    if(table != nullptr)
+    {
+        OMNITRACE_VERBOSE(1 || on_load_trace, "[OnLoad] setting up HSA...\n");
+
+        bool trace_hsa_api = get_trace_hsa_api();
+
+        // Enable HSA API callbacks/activity
+        if(trace_hsa_api)
+        {
+            std::vector<std::string> hsa_api_vec =
+                tim::delimit(get_trace_hsa_api_types());
+
+            // initialize HSA tracing
+            roctracer_set_properties(
+                static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API), (void*) table);
+
+            if(!hsa_api_vec.empty())
+            {
+                for(const auto& itr : hsa_api_vec)
+                {
+                    uint32_t    cid = HSA_API_ID_NUMBER;
+                    const char* api = itr.c_str();
+                    OMNITRACE_ROCTRACER_CALL(roctracer_op_code(
+                        static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API), api,
+                        &cid, nullptr));
+                    OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_callback(
+                        static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API), cid,
+                        hsa_api_callback, nullptr));
+
+                    OMNITRACE_VERBOSE(1 || on_load_trace, "    HSA-trace(%s)", api);
+                }
+            }
+            else
+            {
+                OMNITRACE_VERBOSE(1 || on_load_trace, "    HSA-trace()\n");
+                OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback(
+                    static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API),
+                    hsa_api_callback, nullptr));
+            }
+        }
+
+        bool trace_hsa_activity = get_trace_hsa_activity();
+        // Enable HSA GPU activity
+        if(trace_hsa_activity)
+        {
+#if OMNITRACE_HIP_VERSION < 50300
+            using namespace roctracer;
+            // initialize HSA tracing
+            const char*          output_prefix = nullptr;
+            hsa_ops_properties_t ops_properties{
+                table, reinterpret_cast<activity_async_callback_t>(hsa_activity_callback),
+                nullptr, output_prefix
+            };
+#elif OMNITRACE_HIP_VERSION < 50301
+            hsa_ops_properties_t ops_properties;
+            ops_properties.table        = table;
+            ops_properties.reserved1[0] = reinterpret_cast<void*>(&hsa_activity_callback);
+            ops_properties.reserved1[1] = nullptr;
+            ops_properties.reserved1[2] = nullptr;
+#else
+            hsa_ops_properties_t ops_properties{
+                table, reinterpret_cast<void*>(&hsa_activity_callback), nullptr, nullptr
+            };
+#endif
+            roctracer_set_properties(
+                static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_OPS), &ops_properties);
+
+            OMNITRACE_VERBOSE(1 || on_load_trace, "    HSA-activity-trace()\n");
+            OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_activity(
+                static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_OPS), HSA_OP_ID_COPY));
+        }
+    }
+
     // callback for HSA
     for(auto& itr : roctracer_setup_routines())
         itr.second();
@@ -246,6 +333,24 @@ roctracer::shutdown()
             roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));
     }
 
+    if(get_trace_hsa_api())
+    {
+        OMNITRACE_VERBOSE_F(
+            2,
+            "executing roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HSA_API)...\n");
+        OMNITRACE_ROCTRACER_CALL(
+            roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
+    }
+
+    if(get_trace_hsa_api())
+    {
+        OMNITRACE_VERBOSE_F(
+            2, "executing roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, "
+               "HSA_OP_ID_COPY)...\n");
+        OMNITRACE_ROCTRACER_CALL(
+            roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+    }
+
     if(roctracer_activity_count() == 0)
     {
         OMNITRACE_VERBOSE_F(2, "executing roctracer_flush_activity()...\n");

projects/rocprofiler-systems/source/lib/omnitrace/library/components/roctracer.hpp

@@ -54,11 +54,10 @@ struct roctracer
     OMNITRACE_DEFAULT_OBJECT(roctracer)
 
     static void preinit();
-    static void global_init() { setup(); }
     static void global_finalize() { shutdown(); }
 
     static bool is_setup();
-    static void setup();
+    static void setup(void* hsa_api_table, bool on_load_trace = false);
     static void shutdown();
     static void add_setup(const std::string&, std::function<void()>&&);
     static void add_shutdown(const std::string&, std::function<void()>&&);
@@ -75,7 +74,7 @@ struct roctracer
 
 #if !defined(OMNITRACE_USE_ROCTRACER)
 inline void
-roctracer::setup()
+roctracer::setup(void*, bool)
 {}
 
 inline void

projects/rocprofiler-systems/source/lib/omnitrace/library/rocm.cpp

@@ -45,25 +45,6 @@
 #include <mutex>
 #include <tuple>
 
-#define HIP_PROF_HIP_API_STRING 1
-
-#include <roctracer_ext.h>
-#include <roctracer_hip.h>
-
-#if OMNITRACE_HIP_VERSION < 50300
-#    include <roctracer_hcc.h>
-#endif
-
-#define AMD_INTERNAL_BUILD 1
-#include <roctracer_hsa.h>
-
-#if __has_include(<hip/amd_detail/hip_prof_str.h>) || (defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0)
-#    include <hip/amd_detail/hip_prof_str.h>
-#    define OMNITRACE_HIP_API_ARGS 1
-#else
-#    define OMNITRACE_HIP_API_ARGS 0
-#endif
-
 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
 #    include <rocprofiler.h>
 #endif
@@ -186,113 +167,20 @@ extern "C"
 
         OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
 
-        static auto _setup = [=]() {
-            try
-            {
-                OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
-                                  "[OnLoad] setting up HSA...\n");
-
-                bool trace_hsa_api = get_trace_hsa_api();
-
-                // Enable HSA API callbacks/activity
-                if(trace_hsa_api)
-                {
-                    std::vector<std::string> hsa_api_vec =
-                        tim::delimit(get_trace_hsa_api_types());
-
-                    // initialize HSA tracing
-                    roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
-
-                    if(!hsa_api_vec.empty())
-                    {
-                        for(const auto& itr : hsa_api_vec)
-                        {
-                            uint32_t    cid = HSA_API_ID_NUMBER;
-                            const char* api = itr.c_str();
-                            OMNITRACE_ROCTRACER_CALL(roctracer_op_code(
-                                ACTIVITY_DOMAIN_HSA_API, api, &cid, nullptr));
-                            OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_callback(
-                                ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));
-
-                            OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
-                                              "    HSA-trace(%s)", api);
-                        }
-                    }
-                    else
-                    {
-                        OMNITRACE_VERBOSE(1 || rocm::on_load_trace, "    HSA-trace()\n");
-                        OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback(
-                            ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
-                    }
-                }
-
-                bool trace_hsa_activity = get_trace_hsa_activity();
-                // Enable HSA GPU activity
-                if(trace_hsa_activity)
-                {
-#if OMNITRACE_HIP_VERSION < 50300
-                    using namespace roctracer;
-                    // initialize HSA tracing
-                    const char*          output_prefix = nullptr;
-                    hsa_ops_properties_t ops_properties{
-                        table,
-                        reinterpret_cast<activity_async_callback_t>(
-                            hsa_activity_callback),
-                        nullptr, output_prefix
-                    };
-#elif OMNITRACE_HIP_VERSION < 50301
-                    hsa_ops_properties_t ops_properties;
-                    ops_properties.table = table;
-                    ops_properties.reserved1[0] =
-                        reinterpret_cast<void*>(&hsa_activity_callback);
-                    ops_properties.reserved1[1] = nullptr;
-                    ops_properties.reserved1[2] = nullptr;
-#else
-                    hsa_ops_properties_t ops_properties{
-                        table, reinterpret_cast<void*>(&hsa_activity_callback), nullptr,
-                        nullptr
-                    };
-#endif
-                    roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);
-
-                    OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
-                                      "    HSA-activity-trace()\n");
-                    OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_activity(
-                        ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
-                }
-            } catch(std::exception& _e)
-            {
-                OMNITRACE_BASIC_PRINT("Exception was thrown in HSA setup: %s\n",
-                                      _e.what());
-            }
-        };
-
-        static auto _shutdown = []() {
-            OMNITRACE_DEBUG_F("roctracer_disable_domain_callback\n");
-            OMNITRACE_ROCTRACER_CALL(
-                roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
-
-            OMNITRACE_DEBUG_F("roctracer_disable_op_activity\n");
-            OMNITRACE_ROCTRACER_CALL(
-                roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
-        };
-
 #if OMNITRACE_HIP_VERSION < 50300
         OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
                             "Computing the roctracer clock skew...\n");
         (void) omnitrace::get_clock_skew();
 #endif
 
-        comp::roctracer::add_setup("hsa", _setup);
-        comp::roctracer::add_shutdown("hsa", _shutdown);
+        if(get_use_process_sampling() && get_use_rocm_smi())
+        {
+            OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
+                                "Setting rocm_smi state to active...\n");
+            rocm_smi::set_state(State::Active);
+        }
 
-        OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
-                            "Setting rocm_smi state to active...\n");
-        rocm_smi::set_state(State::Active);
-
-        OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
-                            "Requesting roctracer to setup...\n");
-        comp::roctracer::setup();
+        comp::roctracer::setup(static_cast<void*>(table), rocm::on_load_trace);
 
 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
         bool _force_rocprofiler_init =

projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.cpp

@@ -358,8 +358,10 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
 }
 
 void
-hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg)
+hsa_activity_callback(uint32_t op, const void* vrecord, void* arg)
 {
+    const auto* record = static_cast<const activity_record_t*>(vrecord);
+
     if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
         return;
 

projects/rocprofiler-systems/source/lib/omnitrace/library/roctracer.hpp

@@ -28,8 +28,6 @@
 #include "library/components/roctracer.hpp"
 #include "library/ptl.hpp"
 
-#include <roctracer.h>
-
 #include <iostream>
 #include <memory>
 
@@ -57,7 +55,7 @@ void
 hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
 
 void
-hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg);
+hsa_activity_callback(uint32_t op, const void* record, void* arg);
 
 void
 hip_exec_activity_callbacks(int64_t _tid);

projects/rocprofiler-systems/source/lib/omnitrace/library/thread_info.cpp

@@ -75,21 +75,31 @@ init_index_data(int64_t _tid, bool _offset = false)
     if(!itr)
     {
         threading::offset_this_id(_offset);
-        itr       = thread_index_data{};
+        itr = thread_index_data{};
+
+        OMNITRACE_CONDITIONAL_THROW(itr->internal_value != _tid,
+                                    "Error! thread_info::init_index_data was called for "
+                                    "thread %zi on thread %zi\n",
+                                    _tid, itr->internal_value);
+
         int _verb = 2;
         // if thread created using finalization, bump up the minimum verbosity level
         if(get_state() >= State::Finalized && _offset) _verb += 2;
         if(!config::settings_are_configured())
         {
-            OMNITRACE_BASIC_VERBOSE_F(
-                _verb, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n",
-                itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value);
+            OMNITRACE_BASIC_VERBOSE_F(_verb,
+                                      "Thread %li on PID %i (rank: %i) assigned "
+                                      "omnitrace TID %li (internal: %li)\n",
+                                      itr->system_value, process::get_id(), dmp::rank(),
+                                      itr->sequent_value, itr->internal_value);
         }
         else
         {
-            OMNITRACE_VERBOSE_F(
-                _verb, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n",
-                itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value);
+            OMNITRACE_VERBOSE_F(_verb,
+                                "Thread %li on PID %i (rank: %i) assigned omnitrace TID "
+                                "%li (internal: %li)\n",
+                                itr->system_value, process::get_id(), dmp::rank(),
+                                itr->sequent_value, itr->internal_value);
         }
     }
     return itr;
@@ -191,6 +201,46 @@ thread_info::get()
     return get_info_data(utility::get_thread_index());
 }
 
+const std::optional<thread_info>&
+thread_info::get(native_handle_t& _tid)
+{
+    return get(native_handle_t{ _tid });
+}
+
+const std::optional<thread_info>&
+thread_info::get(native_handle_t&& _tid)
+{
+    const auto& _v = get_info_data();
+    if(_v)
+    {
+        for(const auto& itr : *_v)
+        {
+            if(itr && itr->index_data &&
+               pthread_equal(itr->index_data->pthread_value, _tid) == 0)
+                return itr;
+        }
+    }
+
+    OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
+    return unknown_thread;
+}
+
+const std::optional<thread_info>&
+thread_info::get(std::thread::id _tid)
+{
+    const auto& _v = get_info_data();
+    if(_v)
+    {
+        for(const auto& itr : *_v)
+        {
+            if(itr && itr->index_data && itr->index_data->stl_value == _tid) return itr;
+        }
+    }
+
+    OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
+    return unknown_thread;
+}
+
 const std::optional<thread_info>&
 thread_info::get(int64_t _tid, ThreadIdType _type)
 {
@@ -203,7 +253,8 @@ thread_info::get(int64_t _tid, ThreadIdType _type)
         {
             for(const auto& itr : *_v)
             {
-                if(itr && itr->index_data->system_value == _tid) return itr;
+                if(itr && itr->index_data && itr->index_data->system_value == _tid)
+                    return itr;
             }
         }
     }
@@ -214,10 +265,21 @@ thread_info::get(int64_t _tid, ThreadIdType _type)
         {
             for(const auto& itr : *_v)
             {
-                if(itr && itr->index_data->sequent_value == _tid) return itr;
+                if(itr && itr->index_data && itr->index_data->sequent_value == _tid)
+                    return itr;
             }
         }
     }
+    else if(_type == ThreadIdType::PthreadID)
+    {
+        OMNITRACE_THROW("omnitrace does not support thread_info::get(int64_t, "
+                        "ThreadIdType) with ThreadIdType::PthreadID\n");
+    }
+    else if(_type == ThreadIdType::StlThreadID)
+    {
+        OMNITRACE_THROW("omnitrace does not support thread_info::get(int64_t, "
+                        "ThreadIdType) with ThreadIdType::StlThreadID\n");
+    }
 
     OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
     return unknown_thread;
@@ -302,8 +364,11 @@ thread_info::as_string() const
     std::stringstream _ss{};
     _ss << std::boolalpha << "is_offset=" << is_offset;
     if(index_data)
+    {
         _ss << ", index_data=(" << index_data->internal_value << ", "
-            << index_data->system_value << ", " << index_data->sequent_value << ")";
+            << index_data->system_value << ", " << index_data->sequent_value << ", "
+            << index_data->pthread_value << ", " << index_data->stl_value << ")";
+    }
     if(causal_count) _ss << ", causal count=" << *causal_count;
     _ss << ", lifetime=(" << lifetime.first << ":" << lifetime.second << ")";
     return _ss.str();

projects/rocprofiler-systems/source/lib/omnitrace/library/thread_info.hpp

@@ -24,6 +24,8 @@
 
 #include "core/utility.hpp"
 
+#include <pthread.h>
+#include <thread>
 #include <timemory/backends/threading.hpp>
 
 #include <cstdint>
@@ -34,36 +36,46 @@
 
 namespace omnitrace
 {
-//  InternalTID: zero-based, process-local thread-ID from atomic increment
-//               from user-created threads and omnitrace-created threads.
-//               This value may vary based on threads created by different
-//               backends, e.g., roctracer will create threads
+//  InternalTID:  zero-based, process-local thread-ID from atomic increment
+//                from user-created threads and omnitrace-created threads.
+//                This value may vary based on threads created by different
+//                backends, e.g., roctracer will create threads
 //
-//  SystemTID:   system thread-ID. Should be same value as what is seen
-//               in debugger, etc.
+//  SystemTID:    system thread-ID. Should be same value as what is seen
+//                in debugger, etc.
+//
+//  SequentTID:   zero-based, process-local thread-ID based on the sequence of
+//                user-created threads which are created in-between the
+//                initialization and finalization of omnitrace.
+//                In theory, omnitrace will never increment this value
+//                because of a thread explicitly by omnitrace or
+//                by other of the dependent libraries. Most commonly
+//                used for indexing into omnitrace's thread-local data.
+//
+//  NativeHandle: value of static_cast<int64_t>(pthread_self())
 //
-//  SequentTID:  zero-based, process-local thread-ID based on the sequence of
-//               user-created threads which are created in-between the
-//               initialization and finalization of omnitrace.
-//               In theory, omnitrace will never increment this value
-//               because of a thread explicitly by omnitrace or
-//               by other of the dependent libraries. Most commonly
-//               used for indexing into omnitrace's thread-local data.
 enum ThreadIdType : int
 {
     InternalTID = 0,
     SystemTID   = 1,  // system thread id
     SequentTID  = 2,
+    PthreadID   = 3,
+    StlThreadID = 4,
 };
 
 struct thread_index_data
 {
+    using stl_tid_t    = std::thread::id;
+    using native_tid_t = pthread_t;
+
     // the lookup value is always incremented for each thread
     // the system value is the tid provided by the operating system
     // the internal value is the value which the user expects
-    int64_t internal_value = utility::get_thread_index();
-    int64_t system_value   = tim::threading::get_sys_tid();
-    int64_t sequent_value  = tim::threading::get_id();
+    int64_t      internal_value = utility::get_thread_index();
+    int64_t      system_value   = tim::threading::get_sys_tid();
+    int64_t      sequent_value  = tim::threading::get_id();
+    native_tid_t pthread_value  = ::pthread_self();
+    stl_tid_t    stl_value      = std::this_thread::get_id();
 
     std::string as_string() const;
 };
@@ -74,6 +86,7 @@ struct thread_info
 {
     using index_data_t    = std::optional<thread_index_data>;
     using lifetime_data_t = std::pair<uint64_t, uint64_t>;
+    using native_handle_t = std::thread::native_handle_type;
 
     ~thread_info()                  = default;
     thread_info(const thread_info&) = delete;
@@ -98,7 +111,11 @@ struct thread_info
     static bool                              exists();
     static const std::optional<thread_info>& init(bool _offset = false);
     static const std::optional<thread_info>& get();
+    static const std::optional<thread_info>& get(native_handle_t&);
+    static const std::optional<thread_info>& get(native_handle_t&&);
+    static const std::optional<thread_info>& get(std::thread::id);
     static const std::optional<thread_info>& get(int64_t _tid, ThreadIdType _type);
+    // note: get(native_handle_t) overloaded to & and && to prevent implicit conversion
 
     bool            is_offset    = false;
     const int64_t*  causal_count = nullptr;

projects/rocprofiler-systems/source/lib/omnitrace/timeout.cpp

@@ -0,0 +1,231 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "core/categories.hpp"
+#include "core/config.hpp"
+#include "core/debug.hpp"
+#include "core/locking.hpp"
+#include "core/state.hpp"
+#include "library/components/pthread_gotcha.hpp"
+#include "library/runtime.hpp"
+#include "library/thread_info.hpp"
+
+#include <timemory/log/color.hpp>
+#include <timemory/signals/types.hpp>
+#include <timemory/unwind/backtrace.hpp>
+
+#include <chrono>
+#include <sstream>
+#include <thread>
+
+namespace omnitrace
+{
+namespace timeout
+{
+void
+setup() OMNITRACE_INTERNAL_API;
+
+namespace
+{
+namespace unwind  = ::tim::unwind;
+namespace signals = ::tim::signals;
+namespace log     = ::tim::log;
+
+constexpr auto timeout_signal   = signals::sys_signal::Hangup;
+constexpr auto timeout_signal_v = static_cast<int>(timeout_signal);
+
+auto                  main_thread_native_handle         = pthread_self();
+bool                  ci_timeout_active                 = false;
+auto                  ci_timeout_mutex                  = locking::atomic_mutex{};
+uint64_t              ci_timeout_backtrace_global_count = 1;
+uint64_t              ci_timeout_backtrace_global_done  = 0;
+thread_local uint64_t ci_timeout_backtrace_local_count  = 0;
+
+void
+ci_timeout_backtrace(int)
+{
+    if(ci_timeout_backtrace_local_count >= ci_timeout_backtrace_global_count) return;
+    ++ci_timeout_backtrace_local_count;
+
+    auto _err            = std::stringstream{};
+    auto _cfg            = unwind::detailed_backtrace_config{};
+    _cfg.proc_pid_maps   = false;
+    _cfg.unwind_lineinfo = false;
+    _cfg.force_color     = !log::monochrome();
+
+    unwind::detailed_backtrace<0>(_err, _cfg);
+
+    static auto _mutex = locking::atomic_mutex{};
+    auto        _lk    = locking::atomic_lock{ _mutex };
+    OMNITRACE_PRINT("%s\n", _err.str().c_str());
+
+    ++ci_timeout_backtrace_global_done;
+}
+
+void
+ensure_ci_timeout_backtrace(double             _ci_timeout_seconds,
+                            std::promise<void> _ci_timeout_ready)
+{
+    _ci_timeout_ready.set_value();
+
+    thread_info::init(true);
+    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Disabled);
+
+    auto _factor = 3.0;
+    while(_ci_timeout_seconds <= _factor)
+        _factor /= 1.25;
+
+    uint64_t _ci_timeout_nitr    = 0;
+    int64_t  _ci_timeout_nanosec = (_ci_timeout_seconds - _factor) * units::sec;
+    auto     _ci_timeout_total_count =
+        get_env<uint64_t>("OMNITRACE_CI_TIMEOUT_COUNT", 1, false);
+    const auto root_pid =
+        get_env<pid_t>("OMNITRACE_ROOT_PROCESS", process::get_id(), false);
+
+    while(get_state() < State::Finalized && _ci_timeout_nitr < _ci_timeout_total_count)
+    {
+        // sleep until timeout reached
+        std::this_thread::sleep_for(std::chrono::nanoseconds{ _ci_timeout_nanosec });
+
+        // guard against thread in fork
+        if(process::get_id() != root_pid)
+        {
+            ci_timeout_active = false;
+            setup();
+            return;
+        }
+
+        auto    _tids             = pthread_gotcha::get_native_handles();
+        int64_t _ci_timeout_pause = (_factor * units::sec) / (3 * (_tids.size() + 1));
+        auto    _kill_thread      = [_ci_timeout_pause](auto _handle) {
+            // execute the pthread_kill and wait until ci_timeout_backtrace increments
+            // ci_timeout_backtrace_global_done (or 50 iterations pass) to avoid
+            // the backtraces overlapping output
+            auto _n      = 0;
+            auto _done_v = ci_timeout_backtrace_global_done;
+            if(::pthread_kill(_handle, timeout_signal_v) != 0)
+            {
+                const auto& _info = thread_info::get(_handle);
+                if(_info)
+                {
+                    OMNITRACE_WARNING_F(
+                        0, "pthread_kill(%zu, %i) failed for thread %zi (info: %s)\n",
+                        _handle, timeout_signal_v, _info->index_data->sequent_value,
+                        _info->as_string().c_str());
+                }
+                else
+                {
+                    OMNITRACE_WARNING_F(0,
+                                        "pthread_kill(%zu, %i) failed. executing generic "
+                                        "kill(%i, %i)...\n",
+                                        _handle, timeout_signal_v, process::get_id(),
+                                        timeout_signal_v);
+                }
+
+                ::kill(process::get_id(), timeout_signal_v);
+            }
+
+            // wait until the signal has been delivered
+            while(ci_timeout_backtrace_global_done == _done_v && _n++ < 50)
+                std::this_thread::sleep_for(
+                    std::chrono::nanoseconds{ _ci_timeout_pause });
+        };
+
+        _tids.erase(main_thread_native_handle);
+        OMNITRACE_WARNING_F(-127,
+                            "timeout after %8.3f seconds... Generating backtraces for "
+                            "%zu threads...\n",
+                            _ci_timeout_seconds, _tids.size() + 1);
+
+        for(auto itr : _tids)
+            _kill_thread(itr);
+
+        _kill_thread(main_thread_native_handle);
+
+        ::omnitrace::debug::flush();
+        ::omnitrace::debug::lock _debug_lk{};
+
+        if(++_ci_timeout_nitr >= _ci_timeout_total_count)
+        {
+            // use SIGQUIT because it will generate a core dump
+            ::kill(process::get_id(), SIGQUIT);
+            return;
+        }
+        else
+        {
+            ++ci_timeout_backtrace_global_count;
+        }
+    }
+
+    OMNITRACE_WARNING_F(0, "timeout thread exiting...\n");
+}
+}  // namespace
+
+void
+setup()
+{
+    // make sure there isn't any datarace for ci_timeout_active
+    auto _lk = locking::atomic_lock{ ci_timeout_mutex };
+
+    if(ci_timeout_active) return;
+
+    // in CI mode, if OMNITRACE_CI_TIMEOUT or OMNITRACE_CI_TIMEOUT_OVERRIDE is
+    // set, start a thread that will print out the backtrace for each thread
+    // before the timeout is hit (i.e. killed by CTest) so we can potentially
+    // diagnose where the code is stuck
+    auto _ci = get_env("OMNITRACE_CI", false, false);
+    if(_ci)
+    {
+        // set by CTest
+        auto _ci_timeout_default = get_env("OMNITRACE_CI_TIMEOUT", -1.0, false);
+        // allow override by user
+        auto _ci_timeout_seconds =
+            get_env("OMNITRACE_CI_TIMEOUT_OVERRIDE", _ci_timeout_default, false);
+
+        if(_ci_timeout_seconds > 0.0)
+        {
+            // lock served its purpose after setting to true
+            ci_timeout_active = true;
+            _lk.unlock();
+
+            OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
+            OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
+
+            // enable the signal handler for when the timeout is reached
+            struct sigaction _action = {};
+            sigemptyset(&_action.sa_mask);
+            _action.sa_flags   = SA_RESTART;
+            _action.sa_handler = ci_timeout_backtrace;
+            sigaction(timeout_signal_v, &_action, nullptr);
+
+            // start a background thread that handles waiting for the timeout
+            auto _ci_timeout_ready = std::promise<void>{};
+            auto _ci_timeout_wait  = _ci_timeout_ready.get_future();
+            std::thread{ ensure_ci_timeout_backtrace, _ci_timeout_seconds,
+                         std::move(_ci_timeout_ready) }
+                .detach();
+            _ci_timeout_wait.wait_for(std::chrono::seconds{ 1 });
+        }
+    }
+}
+}  // namespace timeout
+}  // namespace omnitrace

projects/rocprofiler-systems/tests/CMakeLists.txt

@@ -20,5 +20,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-time-window-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-critical-trace-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-attach-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-rccl-tests.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-overflow-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-causal-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-python-tests.cmake)

projects/rocprofiler-systems/tests/omnitrace-capchk.cpp

@@ -0,0 +1,476 @@
+
+
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <initializer_list>
+#include <iomanip>
+#include <iostream>
+#include <linux/capability.h>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <unistd.h>
+#include <vector>
+
+using cap_value_t = int;
+
+struct cap_info
+{
+    const char* name  = nullptr;
+    cap_value_t value = -1;
+};
+
+struct cap_status
+{
+    unsigned long long inherited = 0;
+    unsigned long long permitted = 0;
+    unsigned long long effective = 0;
+    unsigned long long bounding  = 0;
+    unsigned long long ambient   = 0;
+};
+
+#define CAP_INFO_ENTRY(VAL)                                                              \
+    cap_info { #VAL, VAL }
+
+namespace
+{
+std::initializer_list<cap_info> known_capabilities = {
+#if defined(CAP_CHOWN)
+    CAP_INFO_ENTRY(CAP_CHOWN),
+#endif
+
+#if defined(CAP_DAC_OVERRIDE)
+    CAP_INFO_ENTRY(CAP_DAC_OVERRIDE),
+#endif
+
+#if defined(CAP_DAC_READ_SEARCH)
+    CAP_INFO_ENTRY(CAP_DAC_READ_SEARCH),
+#endif
+
+#if defined(CAP_FOWNER)
+    CAP_INFO_ENTRY(CAP_FOWNER),
+#endif
+
+#if defined(CAP_FSETID)
+    CAP_INFO_ENTRY(CAP_FSETID),
+#endif
+
+#if defined(CAP_KILL)
+    CAP_INFO_ENTRY(CAP_KILL),
+#endif
+
+#if defined(CAP_SETGID)
+    CAP_INFO_ENTRY(CAP_SETGID),
+#endif
+
+#if defined(CAP_SETUID)
+    CAP_INFO_ENTRY(CAP_SETUID),
+#endif
+
+#if defined(CAP_SETPCAP)
+    CAP_INFO_ENTRY(CAP_SETPCAP),
+#endif
+
+#if defined(CAP_LINUX_IMMUTABLE)
+    CAP_INFO_ENTRY(CAP_LINUX_IMMUTABLE),
+#endif
+
+#if defined(CAP_NET_BIND_SERVICE)
+    CAP_INFO_ENTRY(CAP_NET_BIND_SERVICE),
+#endif
+
+#if defined(CAP_NET_BROADCAST)
+    CAP_INFO_ENTRY(CAP_NET_BROADCAST),
+#endif
+
+#if defined(CAP_NET_ADMIN)
+    CAP_INFO_ENTRY(CAP_NET_ADMIN),
+#endif
+
+#if defined(CAP_NET_RAW)
+    CAP_INFO_ENTRY(CAP_NET_RAW),
+#endif
+
+#if defined(CAP_IPC_LOCK)
+    CAP_INFO_ENTRY(CAP_IPC_LOCK),
+#endif
+
+#if defined(CAP_IPC_OWNER)
+    CAP_INFO_ENTRY(CAP_IPC_OWNER),
+#endif
+
+#if defined(CAP_SYS_MODULE)
+    CAP_INFO_ENTRY(CAP_SYS_MODULE),
+#endif
+
+#if defined(CAP_SYS_RAWIO)
+    CAP_INFO_ENTRY(CAP_SYS_RAWIO),
+#endif
+
+#if defined(CAP_SYS_CHROOT)
+    CAP_INFO_ENTRY(CAP_SYS_CHROOT),
+#endif
+
+#if defined(CAP_SYS_PTRACE)
+    CAP_INFO_ENTRY(CAP_SYS_PTRACE),
+#endif
+
+#if defined(CAP_SYS_PACCT)
+    CAP_INFO_ENTRY(CAP_SYS_PACCT),
+#endif
+
+#if defined(CAP_SYS_ADMIN)
+    CAP_INFO_ENTRY(CAP_SYS_ADMIN),
+#endif
+
+#if defined(CAP_SYS_BOOT)
+    CAP_INFO_ENTRY(CAP_SYS_BOOT),
+#endif
+
+#if defined(CAP_SYS_NICE)
+    CAP_INFO_ENTRY(CAP_SYS_NICE),
+#endif
+
+#if defined(CAP_SYS_RESOURCE)
+    CAP_INFO_ENTRY(CAP_SYS_RESOURCE),
+#endif
+
+#if defined(CAP_SYS_TIME)
+    CAP_INFO_ENTRY(CAP_SYS_TIME),
+#endif
+
+#if defined(CAP_SYS_TTY_CONFIG)
+    CAP_INFO_ENTRY(CAP_SYS_TTY_CONFIG),
+#endif
+
+#if defined(CAP_MKNOD)
+    CAP_INFO_ENTRY(CAP_MKNOD),
+#endif
+
+#if defined(CAP_LEASE)
+    CAP_INFO_ENTRY(CAP_LEASE),
+#endif
+
+#if defined(CAP_AUDIT_WRITE)
+    CAP_INFO_ENTRY(CAP_AUDIT_WRITE),
+#endif
+
+#if defined(CAP_AUDIT_CONTROL)
+    CAP_INFO_ENTRY(CAP_AUDIT_CONTROL),
+#endif
+
+#if defined(CAP_SETFCAP)
+    CAP_INFO_ENTRY(CAP_SETFCAP),
+#endif
+
+#if defined(CAP_MAC_OVERRIDE)
+    CAP_INFO_ENTRY(CAP_MAC_OVERRIDE),
+#endif
+
+#if defined(CAP_MAC_ADMIN)
+    CAP_INFO_ENTRY(CAP_MAC_ADMIN),
+#endif
+
+#if defined(CAP_SYSLOG)
+    CAP_INFO_ENTRY(CAP_SYSLOG),
+#endif
+
+#if defined(CAP_WAKE_ALARM)
+    CAP_INFO_ENTRY(CAP_WAKE_ALARM),
+#endif
+
+#if defined(CAP_BLOCK_SUSPEND)
+    CAP_INFO_ENTRY(CAP_BLOCK_SUSPEND),
+#endif
+
+#if defined(CAP_AUDIT_READ)
+    CAP_INFO_ENTRY(CAP_AUDIT_READ),
+#endif
+
+#if defined(CAP_PERFMON)
+    CAP_INFO_ENTRY(CAP_PERFMON),
+#endif
+
+#if defined(CAP_BPF)
+    CAP_INFO_ENTRY(CAP_BPF),
+#endif
+
+#if defined(CAP_CHECKPOINT_RESTORE)
+    CAP_INFO_ENTRY(CAP_CHECKPOINT_RESTORE),
+#endif
+
+#if defined(CAP_LAST_CAP)
+    CAP_INFO_ENTRY(CAP_LAST_CAP),
+#endif
+};
+
+auto cap_max_bits_v = []() {
+    unsigned _value = 0;
+    for(const auto& itr : known_capabilities)
+        _value = std::max<unsigned>(_value, itr.value + 1);
+    return _value;
+}();
+
+std::string
+to_lower(std::string&& _s)
+{
+    for(auto& citr : _s)
+        citr = tolower(citr);
+    return _s;
+}
+
+std::string
+to_upper(std::string _s)
+{
+    for(auto& citr : _s)
+        citr = toupper(citr);
+    return _s;
+}
+
+cap_status
+cap_read(pid_t _pid)
+{
+    auto fname = std::string{ "/proc/" } + std::to_string(_pid) + "/status";
+    auto ifs   = std::ifstream{ fname };
+    if(!ifs) return cap_status{};
+
+    auto _lines = std::vector<std::string>{};
+
+    while(ifs && ifs.good())
+    {
+        auto _line = std::string{};
+        std::getline(ifs, _line);
+        if(ifs && ifs.good() && !_line.empty()) _lines.emplace_back(std::move(_line));
+    }
+
+    auto _data = cap_status{};
+    for(const auto& itr : _lines)
+    {
+        auto iss    = std::istringstream{ itr };
+        auto _key   = std::string{};
+        auto _value = std::string{};
+        iss >> _key;
+
+        if(_key.find("Cap") == 0) iss >> _value;
+
+        if(!_value.empty())
+        {
+            auto _key_matches = [&_key](std::string_view _cap_id_str) {
+                return (_key.find(_cap_id_str) == 0);
+            };
+            if(_key_matches("CapInh"))
+                _data.inherited = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapPrm"))
+                _data.permitted = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapEff"))
+                _data.effective = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapBnd"))
+                _data.bounding = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapAmb"))
+                _data.ambient = std::stoull(_value, nullptr, 16);
+        }
+    }
+
+    return _data;
+}
+
+std::string
+cap_name(cap_value_t _v)
+{
+    for(const auto& itr : known_capabilities)
+        if(itr.value == _v) return to_lower(std::string{ itr.name });
+
+    return std::string{};
+}
+
+std::vector<cap_value_t>
+decode(unsigned long long value)
+{
+    auto _data = std::vector<cap_value_t>{};
+    for(unsigned cap = 0; (cap < 64) && ((value >> cap) != 0U); ++cap)
+    {
+        auto _mask = value & (1ULL << cap);
+        if(_mask != 0U)
+        {
+            if(cap >= 0 && cap < cap_max_bits_v) _data.emplace_back(cap);
+        }
+    }
+
+    return _data;
+}
+
+std::vector<cap_value_t>
+decode(const char* arg)
+{
+    return decode(std::strtoull(arg, nullptr, 16));
+}
+/*
+template <typename Tp>
+std::string
+as_hex(Tp _v, size_t _width = 16)
+{
+    std::stringstream _ss;
+    _ss.fill('0');
+    _ss << "0x" << std::hex << std::setw(_width) << _v;
+    return _ss.str();
+}
+
+void run(std::string&& arg)
+{
+    if(arg.find("0x") == 0)
+        arg = arg.substr(2);
+
+    arg.insert(0, "0x");
+    // arg = std::string{ "0x" } + arg;
+
+    std::cout << arg << "=";
+    auto _decoded = decode(arg.c_str());
+    auto _msg     = std::stringstream{};
+    for(auto&& itr : _decoded)
+    {
+        _msg << "," << cap_name(itr);
+    }
+    auto _msg_v = _msg.str();
+    if(!_msg_v.empty())
+    {
+        std::cout << _msg_v.substr(1);
+    }
+    std::cout << "\n";
+}
+
+void
+run(std::string_view _label, unsigned long long arg)
+{
+    std::cout << " " << std::setw(12) << _label << " ";
+    run(as_hex(arg));
+}
+*/
+}  // namespace
+
+int
+main(int argc, char** argv)
+{
+    const auto* _usage = R"usage(
+usage: omnitrace-capchk <capability-name> <capability-set> <pid>
+
+    Description:
+        Simple tool for checking the effective capabilities of a running process
+
+    Arguments:
+        capability-name (string):
+            case-insensitive string matching CAP_* fields defined in `man 7 capabilities`
+
+        capability-set (string; optional):
+            Choices:
+                - effective (default)
+                - permitted
+                - inherited
+                - bounding
+                - ambient
+            See `man 7 capabilities` for more info
+
+        pid (numeric process-identifier; optional):
+            target process identifier for capability query. If not specified, queries the
+            capabilities of the current process (this exe).
+
+    Exit value:
+        0 if the process has the specified capability in the specified set
+        1 if the process does not have the capability
+        2 if the capability name is not supported
+        3 if the capability set name is not supported
+
+    Examples:
+        $ omnitrace-capchk CAP_SYS_ADMIN
+            Check if this exe (self) has CAP_SYS_ADMIN capability in the (default) effective capability set
+
+        $ omnitrace-capchk sys_admin bounding 423032
+            Check if process 423032 has CAP_SYS_ADMIN capability in the bounding capability set
+    )usage";
+
+    std::string capability_name = {};
+    std::string capability_mode = "effective";
+    pid_t       target_pid      = getpid();
+
+    for(int i = 1; i < argc; ++i)
+    {
+        auto arg = std::string_view{ argv[i] };
+        if(arg == "-h" || arg == "--help" || arg == "-?")
+        {
+            std::cout << _usage << "\n";
+            return EXIT_SUCCESS;
+        }
+    }
+
+    if(argc > 1) capability_name = to_lower(std::string{ argv[1] });
+    if(argc > 2) capability_mode = to_lower(std::string{ argv[2] });
+    if(argc > 3)
+    {
+        auto _pid_s = to_lower(std::string{ argv[3] });
+        if(_pid_s != "self") target_pid = std::stoul(argv[3]);
+    }
+
+    if(capability_name.find("cap_") != 0) capability_name.insert(0, "cap_");
+
+    capability_name = to_upper(capability_name);
+
+    const cap_info* _info = nullptr;
+    for(const auto& itr : known_capabilities)
+    {
+        if(capability_name == std::string_view{ itr.name })
+        {
+            _info = &itr;
+            break;
+        }
+    }
+
+    if(!_info)
+    {
+        fprintf(stderr, "Error! invalid capability: %s\n", capability_name.c_str());
+        return EXIT_FAILURE + 1;
+    }
+
+    auto  _status  = cap_read(target_pid);
+    auto* _dataset = &_status.effective;
+
+    /*
+    std::cout << "pid=" << target_pid << ":\n";
+    run("inherited", _status.inherited);
+    run("permitted", _status.permitted);
+    run("effective", _status.effective);
+    run("bounding", _status.bounding);
+    run("ambient", _status.ambient);
+    */
+
+    if(capability_mode == "effective")
+        _dataset = &_status.effective;
+    else if(capability_mode == "permitted")
+        _dataset = &_status.permitted;
+    else if(capability_mode == "inherited")
+        _dataset = &_status.inherited;
+    else if(capability_mode == "bounding")
+        _dataset = &_status.bounding;
+    else if(capability_mode == "ambient")
+        _dataset = &_status.ambient;
+    else
+    {
+        fprintf(stderr, "Error! invalid capability set: %s\n", capability_mode.c_str());
+        return EXIT_FAILURE + 2;
+    }
+
+    auto _ec = EXIT_FAILURE;
+    for(auto&& itr : decode(*_dataset))
+    {
+        if(itr == _info->value)
+        {
+            _ec = EXIT_SUCCESS;
+            break;
+        }
+    }
+
+    std::cout << ((_ec == EXIT_SUCCESS) ? "Found" : "Missing") << " capability "
+              << capability_name << " in " << capability_mode
+              << " capability set. Exit code: " << _ec << ".\n";
+    return _ec;
+}

projects/rocprofiler-systems/tests/omnitrace-config-tests.cmake

@@ -24,9 +24,15 @@ add_test(
 
 set_tests_properties(
     omnitrace-invalid-config
-    PROPERTIES ENVIRONMENT
-               "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/invalid.cfg" TIMEOUT
-               120 LABELS "config" WILL_FAIL ON)
+    PROPERTIES
+        ENVIRONMENT
+        "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/invalid.cfg;OMNITRACE_CI=ON;OMNITRACE_CI_TIMEOUT=120"
+        TIMEOUT
+        120
+        LABELS
+        "config"
+        WILL_FAIL
+        ON)
 
 add_test(
     NAME omnitrace-missing-config
@@ -35,6 +41,12 @@ add_test(
 
 set_tests_properties(
     omnitrace-missing-config
-    PROPERTIES ENVIRONMENT
-               "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/missing.cfg" TIMEOUT
-               120 LABELS "config" WILL_FAIL ON)
+    PROPERTIES
+        ENVIRONMENT
+        "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/missing.cfg;OMNITRACE_CI=ON;OMNITRACE_CI_TIMEOUT=120"
+        TIMEOUT
+        120
+        LABELS
+        "config"
+        WILL_FAIL
+        ON)

projects/rocprofiler-systems/tests/omnitrace-critical-trace-tests.cmake

@@ -35,7 +35,9 @@ set(_parallel_overhead_critical_trace_environ
     "OMNITRACE_CRITICAL_TRACE_DEBUG=ON"
     "OMNITRACE_VERBOSE=4"
     "OMNITRACE_USE_PID=OFF"
-    "OMNITRACE_TIME_OUTPUT=OFF")
+    "OMNITRACE_TIME_OUTPUT=OFF"
+    "OMNITRACE_CI=ON"
+    "OMNITRACE_CI_TIMEOUT=300")
 
 set_tests_properties(
     parallel-overhead-process-critical-trace

projects/rocprofiler-systems/tests/omnitrace-fork-tests.cmake

@@ -14,10 +14,6 @@ omnitrace_add_test(
     SAMPLING_PASS_REGEX "fork.. called on PID"
     RUNTIME_PASS_REGEX "fork.. called on PID"
     REWRITE_RUN_PASS_REGEX "fork.. called on PID"
-    SAMPLING_FAIL_REGEX
-        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
-    RUNTIME_FAIL_REGEX
-        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
-    REWRITE_RUN_FAIL_REGEX
-        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
-    )
+    SAMPLING_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})"
+    RUNTIME_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})"
+    REWRITE_RUN_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})")

projects/rocprofiler-systems/tests/omnitrace-mpi-tests.cmake

@@ -29,7 +29,7 @@ omnitrace_add_test(
     REWRITE_RUN_PASS_REGEX
         "(/[A-Za-z-]+/perfetto-trace-0.proto).*(/[A-Za-z-]+/wall_clock-0.txt')"
     REWRITE_RUN_FAIL_REGEX
-        "(perfetto-trace|trip_count|sampling_percent|sampling_cpu_clock|sampling_wall_clock|wall_clock)-[0-9][0-9]+.(json|txt|proto)"
+        "(perfetto-trace|trip_count|sampling_percent|sampling_cpu_clock|sampling_wall_clock|wall_clock)-[0-9][0-9]+.(json|txt|proto)|OMNITRACE_ABORT_FAIL_REGEX"
     )
 
 omnitrace_add_test(

projects/rocprofiler-systems/tests/omnitrace-overflow-tests.cmake

@@ -0,0 +1,32 @@
+# -------------------------------------------------------------------------------------- #
+#
+# overflow tests
+#
+# -------------------------------------------------------------------------------------- #
+
+set(_overflow_environment
+    "${_base_environment}"
+    "OMNITRACE_VERBOSE=2"
+    "OMNITRACE_SAMPLING_CPUTIME=OFF"
+    "OMNITRACE_SAMPLING_REALTIME=OFF"
+    "OMNITRACE_SAMPLING_OVERFLOW=ON"
+    "OMNITRACE_SAMPLING_OVERFLOW_EVENT=PERF_COUNT_SW_CPU_CLOCK"
+    "OMNITRACE_SAMPLING_OVERFLOW_FREQ=10000"
+    "OMNITRACE_DEBUG_THREADING_GET_ID=ON")
+
+if(omnitrace_perf_event_paranoid LESS_EQUAL 3
+   OR omnitrace_cap_sys_admin EQUAL 0
+   OR omnitrace_cap_perfmon EQUAL 0)
+    omnitrace_add_test(
+        SKIP_BASELINE
+        NAME overflow
+        TARGET parallel-overhead
+        RUN_ARGS 30 2 200
+        REWRITE_ARGS -e -v 2
+        RUNTIME_ARGS -e -v 1
+        ENVIRONMENT "${_overflow_environment}"
+        LABELS "perf;overflow"
+        SAMPLING_PASS_REGEX "sampling_wall_clock.txt"
+        RUNTIME_PASS_REGEX "sampling_wall_clock.txt"
+        REWRITE_RUN_PASS_REGEX "sampling_wall_clock.txt")
+endif()

projects/rocprofiler-systems/tests/omnitrace-python-tests.cmake

@@ -116,7 +116,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
             COMMAND ${OMNITRACE_CAT_COMMAND}
             PYTHON_VERSION ${_VERSION}
             FILE omnitrace-tests-output/python-external-exclude-inefficient/${_VERSION}/trip_count.txt
-            FAIL_REGEX "(\\\|_inefficient).*(\\\|_sum)"
+            FAIL_REGEX "(\\\|_inefficient).*(\\\|_sum)|OMNITRACE_ABORT_FAIL_REGEX"
             DEPENDS python-external-exclude-inefficient-${_VERSION}
             ENVIRONMENT "${_python_environment}")
 
@@ -135,7 +135,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
             PYTHON_VERSION ${_VERSION}
             FILE omnitrace-tests-output/python-builtin-noprofile/${_VERSION}/trip_count.txt
             PASS_REGEX ".(run)..(noprofile.py)."
-            FAIL_REGEX ".(fib|inefficient)..(noprofile.py)."
+            FAIL_REGEX ".(fib|inefficient)..(noprofile.py).|OMNITRACE_ABORT_FAIL_REGEX"
             DEPENDS python-builtin-noprofile-${_VERSION}
             ENVIRONMENT "${_python_environment}")
     else()

projects/rocprofiler-systems/tests/omnitrace-rocm-tests.cmake

@@ -81,5 +81,5 @@ if(OMNITRACE_USE_ROCPROFILER)
             "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_ROCTRACER=OFF;OMNITRACE_ROCM_EVENTS=${OMNITRACE_ROCM_EVENTS_TEST}"
         REWRITE_RUN_PASS_REGEX
             "rocprof-device-0-GRBM_COUNT.txt(.*)rocprof-device-0-GPUBusy.txt(.*)rocprof-device-0-SQ_WAVES.txt(.*)rocprof-device-0-SQ_INSTS_VALU.txt(.*)rocprof-device-0-VALUInsts.txt(.*)rocprof-device-0-TCC_HIT_sum.txt(.*)rocprof-device-0-TA_TA_BUSY_0.txt(.*)rocprof-device-0-TA_TA_BUSY_11.txt"
-        REWRITE_RUN_FAIL_REGEX "roctracer.txt")
+        REWRITE_RUN_FAIL_REGEX "roctracer.txt|OMNITRACE_ABORT_FAIL_REGEX")
 endif()

projects/rocprofiler-systems/tests/omnitrace-testing.cmake

@@ -213,6 +213,40 @@ execute_process(
     RESULT_VARIABLE _mpiexec_oversubscribe
     OUTPUT_QUIET ERROR_QUIET)
 
+set(omnitrace_perf_event_paranoid "4")
+set(omnitrace_cap_sys_admin "1")
+set(omnitrace_cap_perfmon "1")
+
+if(EXISTS "/proc/sys/kernel/perf_event_paranoid")
+    file(STRINGS "/proc/sys/kernel/perf_event_paranoid" omnitrace_perf_event_paranoid
+         LIMIT_COUNT 1)
+endif()
+
+execute_process(
+    COMMAND ${CMAKE_CXX_COMPILER} -O2 -g -std=c++17
+            ${CMAKE_CURRENT_LIST_DIR}/omnitrace-capchk.cpp -o omnitrace-capchk
+    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin
+    RESULT_VARIABLE _capchk_compile
+    OUTPUT_QUIET ERROR_QUIET)
+
+if(_capchk_compile EQUAL 0)
+    execute_process(
+        COMMAND ${PROJECT_BINARY_DIR}/bin/omnitrace-capchk CAP_SYS_ADMIN effective
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+        RESULT_VARIABLE omnitrace_cap_sys_admin
+        OUTPUT_QUIET ERROR_QUIET)
+
+    execute_process(
+        COMMAND ${PROJECT_BINARY_DIR}/bin/omnitrace-capchk CAP_PERFMON effective
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+        RESULT_VARIABLE omnitrace_cap_perfmon
+        OUTPUT_QUIET ERROR_QUIET)
+endif()
+
+omnitrace_message(STATUS "perf_event_paranoid: ${omnitrace_perf_event_paranoid}")
+omnitrace_message(STATUS "CAP_SYS_ADMIN: ${omnitrace_cap_sys_admin}")
+omnitrace_message(STATUS "CAP_PERFMON: ${omnitrace_cap_perfmon}")
+
 if(_mpiexec_oversubscribe EQUAL 0)
     list(APPEND MPIEXEC_EXECUTABLE_ARGS --oversubscribe)
 endif()
@@ -255,9 +289,30 @@ endif()
 
 # -------------------------------------------------------------------------------------- #
 
+macro(OMNITRACE_CHECK_PASS_FAIL_REGEX NAME PASS FAIL)
+    if(NOT "${${PASS}}" STREQUAL ""
+       AND NOT "${${FAIL}}" STREQUAL ""
+       AND NOT "${${FAIL}}" MATCHES "\\|OMNITRACE_ABORT_FAIL_REGEX"
+       AND NOT "${${FAIL}}" MATCHES "${OMNITRACE_ABORT_FAIL_REGEX}")
+        omnitrace_message(
+            FATAL_ERROR
+            "${NAME} has set pass and fail regexes but fail regex does not include '|OMNITRACE_ABORT_FAIL_REGEX'"
+            )
+    endif()
+
+    if("${${FAIL}}" STREQUAL "")
+        set(${FAIL} "(${OMNITRACE_ABORT_FAIL_REGEX})")
+    else()
+        string(REPLACE "|OMNITRACE_ABORT_FAIL_REGEX" "|${OMNITRACE_ABORT_FAIL_REGEX}"
+                       ${FAIL} "${${FAIL}}")
+    endif()
+endmacro()
+
+# -------------------------------------------------------------------------------------- #
+
 function(OMNITRACE_WRITE_TEST_CONFIG _FILE _ENV)
     set(_ENV_ONLY
-        "OMNITRACE_(CI|MODE|USE_MPIP|DEBUG_SETTINGS|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)="
+        "OMNITRACE_(CI|CI_TIMEOUT|MODE|USE_MPIP|DEBUG_[A-Z_]+|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)="
         )
     set(_FILE_CONTENTS)
     set(_ENV_CONTENTS)
@@ -357,9 +412,7 @@ function(OMNITRACE_ADD_TEST)
 
     foreach(_PREFIX SAMPLING RUNTIME REWRITE REWRITE_RUN BASELINE)
         if("${${_PREFIX}_FAIL_REGEX}" STREQUAL "")
-            set(${_PREFIX}_FAIL_REGEX
-                "(### ERROR ###|address of faulting memory reference|exiting with non-zero exit code)"
-                )
+            set(${_PREFIX}_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})")
         endif()
     endforeach()
 
@@ -552,6 +605,10 @@ function(OMNITRACE_ADD_TEST)
                 endif()
             endforeach()
 
+            list(APPEND _environ "OMNITRACE_CI_TIMEOUT=${_timeout}")
+
+            omnitrace_check_pass_fail_regex("${TEST_NAME}-${_TEST}" "${_PASS_REGEX}"
+                                            "${_FAIL_REGEX}")
             if(TEST ${TEST_NAME}-${_TEST})
                 omnitrace_write_test_config(${TEST_NAME}-${_TEST}.cfg _environ)
                 set_tests_properties(
@@ -604,9 +661,7 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
     endif()
 
     if("${TEST_CAUSAL_FAIL_REGEX}" STREQUAL "")
-        set(TEST_CAUSAL_FAIL_REGEX
-            "(### ERROR ###|address of faulting memory reference|exiting with non-zero exit code)"
-            )
+        set(TEST_CAUSAL_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})")
     endif()
 
     if(TARGET ${TEST_TARGET})
@@ -649,10 +704,16 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
         foreach(_TEST baseline causal validate-causal)
 
             if(NOT TEST ${_TEST}-${TEST_NAME})
-                continue()
+                if(NOT TEST ${TEST_NAME}-${_TEST})
+                    continue()
+                else()
+                    set(_NAME "${TEST_NAME}-${_TEST}")
+                endif()
+            else()
+                set(_NAME "${_TEST}-${TEST_NAME}")
             endif()
 
-            set(_prefix "${_TEST}-${TEST_NAME}/")
+            set(_prefix "${_NAME}/")
             set(_labels "${_TEST}" "causal-profiling")
 
             if(TEST_TARGET)
@@ -701,9 +762,11 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
                 endif()
             endforeach()
 
-            omnitrace_write_test_config(${_TEST}-${TEST_NAME}.cfg _environ)
+            list(APPEND _environ "OMNITRACE_CI_TIMEOUT=${_timeout}")
+            omnitrace_write_test_config(${_NAME}.cfg _environ)
+            omnitrace_check_pass_fail_regex("${_NAME}" "${_PASS_REGEX}" "${_FAIL_REGEX}")
             set_tests_properties(
-                ${_TEST}-${TEST_NAME}
+                ${_NAME}
                 PROPERTIES ENVIRONMENT
                            "${_environ}"
                            TIMEOUT
@@ -780,16 +843,10 @@ function(OMNITRACE_ADD_PYTHON_TEST)
             NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}
             COMMAND ${TEST_COMMAND} ${TEST_FILE}
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-        add_test(
-            NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse
-            COMMAND ${TEST_COMMAND} ${TEST_FILE}
-            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif()
 
-    foreach(
-        _TEST
-        ${TEST_NAME}-${TEST_PYTHON_VERSION} ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse
-        ${TEST_NAME}-${TEST_PYTHON_VERSION}-annotated)
+    foreach(_TEST ${TEST_NAME}-${TEST_PYTHON_VERSION}
+                  ${TEST_NAME}-${TEST_PYTHON_VERSION}-annotated)
 
         if(NOT TEST "${_TEST}")
             continue()
@@ -797,24 +854,18 @@ function(OMNITRACE_ADD_PYTHON_TEST)
 
         string(REPLACE "${TEST_NAME}-${TEST_PYTHON_VERSION}" "${TEST_NAME}" _TEST_DIR
                        "${_TEST}")
-        set(_TEST_ENV "${TEST_ENVIRONMENT}"
-                      "OMNITRACE_OUTPUT_PREFIX=${_TEST_DIR}/${TEST_PYTHON_VERSION}/")
+        set(_TEST_ENV
+            "${TEST_ENVIRONMENT}"
+            "OMNITRACE_OUTPUT_PREFIX=${_TEST_DIR}/${TEST_PYTHON_VERSION}/"
+            "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}")
 
         set(_TEST_PROPERTIES "${TEST_PROPERTIES}")
-        if(NOT "${_TEST}" MATCHES "inverse")
-            # assign pass variable to pass regex
-            set(_PASS_REGEX TEST_PASS_REGEX)
-            # assign fail variable to fail regex
-            set(_FAIL_REGEX TEST_FAIL_REGEX)
-        else()
-            # assign pass variable to fail regex
-            set(_PASS_REGEX TEST_FAIL_REGEX)
-            # assign fail variable to pass regex
-            set(_FAIL_REGEX TEST_PASS_REGEX)
-            # set to will fail
-            list(APPEND _TEST_PROPERTIES WILL_FAIL ON)
-        endif()
+        # assign pass variable to pass regex
+        set(_PASS_REGEX TEST_PASS_REGEX)
+        # assign fail variable to fail regex
+        set(_FAIL_REGEX TEST_FAIL_REGEX)
 
+        omnitrace_check_pass_fail_regex("${_TEST}" "${_PASS_REGEX}" "${_FAIL_REGEX}")
         set_tests_properties(
             ${_TEST}
             PROPERTIES ENVIRONMENT
@@ -964,16 +1015,19 @@ function(OMNITRACE_ADD_VALIDATION_TEST)
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif()
 
+    list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}")
+
     foreach(_TEST validate-${TEST_NAME}-timemory validate-${TEST_NAME}-perfetto)
 
         if(NOT TEST "${_TEST}")
             continue()
         endif()
 
+        omnitrace_check_pass_fail_regex("${_TEST}" "TEST_PASS_REGEX" "TEST_FAIL_REGEX")
         set_tests_properties(
             ${_TEST}
             PROPERTIES ENVIRONMENT
-                       "${_TEST_ENV}"
+                       "${TEST_ENVIRONMENT}"
                        TIMEOUT
                        ${TEST_TIMEOUT}
                        LABELS

projects/rocprofiler-systems/tests/omnitrace-time-window-tests.cmake

@@ -24,7 +24,7 @@ omnitrace_add_validation_test(
     PERFETTO_METRIC "host"
     PERFETTO_FILE "perfetto-trace.proto"
     LABELS "time-window"
-    FAIL_REGEX "outer_d"
+    FAIL_REGEX "outer_d|OMNITRACE_ABORT_FAIL_REGEX"
     ARGS -l
          trace-time-window.inst
          outer_a
@@ -49,7 +49,7 @@ omnitrace_add_validation_test(
     PERFETTO_METRIC "host"
     PERFETTO_FILE "perfetto-trace.proto"
     LABELS "time-window"
-    FAIL_REGEX "outer_d"
+    FAIL_REGEX "outer_d|OMNITRACE_ABORT_FAIL_REGEX"
     ARGS -l
          trace-time-window
          outer_a