CI timeout + line-info in releases (#279)

* Update perfetto args.gn.in

- remove enable_perfetto_tools_trace_to_text (unused)

* core timeout implementation

- requires OMNITRACE_CI=ON
- requires OMNITRACE_CI_TIMEOUT=<sec>
- adds pthread_self and std::this_thread::get_id to thread info
- pthread_create_gotcha stores native handles (pthread_self)

* Testing updates

- improve detection of segfault/failures with PASS_REGEX exists
- add OMNITRACE_CI_TIMEOUT env variable to all tests

* Line-info in releases

- e.g. -g1 + more options to minimize size of debug info

* Fix typo in config exit action message

* OMNITRACE_UNLIKELY around debug/verbose messages

* format fixes

* Overflow tests + capability check

* transpose example update

- link to threads library

* roctracer/rocprofiler update

- in ROCm 5.5.0, cannot include rocprofiler.h and roctracer.h in same file due to conflicting enum defs
- Moved HSA tracing setup/shutdown to component::roctracer

* roctracer update

- fix definition of roctracer::setup when disabled

* Update fork example

- detach threads on main PID
- flush io outputs when printing info

* Update overflow tests

- pass regular expressions
- overflow on PERF_COUNT_SW_CPU_CLOCK event

* fork gotcha update

- use getpid() instead of getppid()

* update fork example

- wait on threads calling fork

* timeout update

- wait on timeout thread to launch before proceeding

tests/CMakeLists.txt

@@ -20,5 +20,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-time-window-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-critical-trace-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-attach-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-rccl-tests.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-overflow-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-causal-tests.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-python-tests.cmake)

tests/omnitrace-capchk.cpp

@@ -0,0 +1,476 @@
+
+
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <initializer_list>
+#include <iomanip>
+#include <iostream>
+#include <linux/capability.h>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <unistd.h>
+#include <vector>
+
+using cap_value_t = int;
+
+struct cap_info
+{
+    const char* name  = nullptr;
+    cap_value_t value = -1;
+};
+
+struct cap_status
+{
+    unsigned long long inherited = 0;
+    unsigned long long permitted = 0;
+    unsigned long long effective = 0;
+    unsigned long long bounding  = 0;
+    unsigned long long ambient   = 0;
+};
+
+#define CAP_INFO_ENTRY(VAL)                                                              \
+    cap_info { #VAL, VAL }
+
+namespace
+{
+std::initializer_list<cap_info> known_capabilities = {
+#if defined(CAP_CHOWN)
+    CAP_INFO_ENTRY(CAP_CHOWN),
+#endif
+
+#if defined(CAP_DAC_OVERRIDE)
+    CAP_INFO_ENTRY(CAP_DAC_OVERRIDE),
+#endif
+
+#if defined(CAP_DAC_READ_SEARCH)
+    CAP_INFO_ENTRY(CAP_DAC_READ_SEARCH),
+#endif
+
+#if defined(CAP_FOWNER)
+    CAP_INFO_ENTRY(CAP_FOWNER),
+#endif
+
+#if defined(CAP_FSETID)
+    CAP_INFO_ENTRY(CAP_FSETID),
+#endif
+
+#if defined(CAP_KILL)
+    CAP_INFO_ENTRY(CAP_KILL),
+#endif
+
+#if defined(CAP_SETGID)
+    CAP_INFO_ENTRY(CAP_SETGID),
+#endif
+
+#if defined(CAP_SETUID)
+    CAP_INFO_ENTRY(CAP_SETUID),
+#endif
+
+#if defined(CAP_SETPCAP)
+    CAP_INFO_ENTRY(CAP_SETPCAP),
+#endif
+
+#if defined(CAP_LINUX_IMMUTABLE)
+    CAP_INFO_ENTRY(CAP_LINUX_IMMUTABLE),
+#endif
+
+#if defined(CAP_NET_BIND_SERVICE)
+    CAP_INFO_ENTRY(CAP_NET_BIND_SERVICE),
+#endif
+
+#if defined(CAP_NET_BROADCAST)
+    CAP_INFO_ENTRY(CAP_NET_BROADCAST),
+#endif
+
+#if defined(CAP_NET_ADMIN)
+    CAP_INFO_ENTRY(CAP_NET_ADMIN),
+#endif
+
+#if defined(CAP_NET_RAW)
+    CAP_INFO_ENTRY(CAP_NET_RAW),
+#endif
+
+#if defined(CAP_IPC_LOCK)
+    CAP_INFO_ENTRY(CAP_IPC_LOCK),
+#endif
+
+#if defined(CAP_IPC_OWNER)
+    CAP_INFO_ENTRY(CAP_IPC_OWNER),
+#endif
+
+#if defined(CAP_SYS_MODULE)
+    CAP_INFO_ENTRY(CAP_SYS_MODULE),
+#endif
+
+#if defined(CAP_SYS_RAWIO)
+    CAP_INFO_ENTRY(CAP_SYS_RAWIO),
+#endif
+
+#if defined(CAP_SYS_CHROOT)
+    CAP_INFO_ENTRY(CAP_SYS_CHROOT),
+#endif
+
+#if defined(CAP_SYS_PTRACE)
+    CAP_INFO_ENTRY(CAP_SYS_PTRACE),
+#endif
+
+#if defined(CAP_SYS_PACCT)
+    CAP_INFO_ENTRY(CAP_SYS_PACCT),
+#endif
+
+#if defined(CAP_SYS_ADMIN)
+    CAP_INFO_ENTRY(CAP_SYS_ADMIN),
+#endif
+
+#if defined(CAP_SYS_BOOT)
+    CAP_INFO_ENTRY(CAP_SYS_BOOT),
+#endif
+
+#if defined(CAP_SYS_NICE)
+    CAP_INFO_ENTRY(CAP_SYS_NICE),
+#endif
+
+#if defined(CAP_SYS_RESOURCE)
+    CAP_INFO_ENTRY(CAP_SYS_RESOURCE),
+#endif
+
+#if defined(CAP_SYS_TIME)
+    CAP_INFO_ENTRY(CAP_SYS_TIME),
+#endif
+
+#if defined(CAP_SYS_TTY_CONFIG)
+    CAP_INFO_ENTRY(CAP_SYS_TTY_CONFIG),
+#endif
+
+#if defined(CAP_MKNOD)
+    CAP_INFO_ENTRY(CAP_MKNOD),
+#endif
+
+#if defined(CAP_LEASE)
+    CAP_INFO_ENTRY(CAP_LEASE),
+#endif
+
+#if defined(CAP_AUDIT_WRITE)
+    CAP_INFO_ENTRY(CAP_AUDIT_WRITE),
+#endif
+
+#if defined(CAP_AUDIT_CONTROL)
+    CAP_INFO_ENTRY(CAP_AUDIT_CONTROL),
+#endif
+
+#if defined(CAP_SETFCAP)
+    CAP_INFO_ENTRY(CAP_SETFCAP),
+#endif
+
+#if defined(CAP_MAC_OVERRIDE)
+    CAP_INFO_ENTRY(CAP_MAC_OVERRIDE),
+#endif
+
+#if defined(CAP_MAC_ADMIN)
+    CAP_INFO_ENTRY(CAP_MAC_ADMIN),
+#endif
+
+#if defined(CAP_SYSLOG)
+    CAP_INFO_ENTRY(CAP_SYSLOG),
+#endif
+
+#if defined(CAP_WAKE_ALARM)
+    CAP_INFO_ENTRY(CAP_WAKE_ALARM),
+#endif
+
+#if defined(CAP_BLOCK_SUSPEND)
+    CAP_INFO_ENTRY(CAP_BLOCK_SUSPEND),
+#endif
+
+#if defined(CAP_AUDIT_READ)
+    CAP_INFO_ENTRY(CAP_AUDIT_READ),
+#endif
+
+#if defined(CAP_PERFMON)
+    CAP_INFO_ENTRY(CAP_PERFMON),
+#endif
+
+#if defined(CAP_BPF)
+    CAP_INFO_ENTRY(CAP_BPF),
+#endif
+
+#if defined(CAP_CHECKPOINT_RESTORE)
+    CAP_INFO_ENTRY(CAP_CHECKPOINT_RESTORE),
+#endif
+
+#if defined(CAP_LAST_CAP)
+    CAP_INFO_ENTRY(CAP_LAST_CAP),
+#endif
+};
+
+auto cap_max_bits_v = []() {
+    unsigned _value = 0;
+    for(const auto& itr : known_capabilities)
+        _value = std::max<unsigned>(_value, itr.value + 1);
+    return _value;
+}();
+
+std::string
+to_lower(std::string&& _s)
+{
+    for(auto& citr : _s)
+        citr = tolower(citr);
+    return _s;
+}
+
+std::string
+to_upper(std::string _s)
+{
+    for(auto& citr : _s)
+        citr = toupper(citr);
+    return _s;
+}
+
+cap_status
+cap_read(pid_t _pid)
+{
+    auto fname = std::string{ "/proc/" } + std::to_string(_pid) + "/status";
+    auto ifs   = std::ifstream{ fname };
+    if(!ifs) return cap_status{};
+
+    auto _lines = std::vector<std::string>{};
+
+    while(ifs && ifs.good())
+    {
+        auto _line = std::string{};
+        std::getline(ifs, _line);
+        if(ifs && ifs.good() && !_line.empty()) _lines.emplace_back(std::move(_line));
+    }
+
+    auto _data = cap_status{};
+    for(const auto& itr : _lines)
+    {
+        auto iss    = std::istringstream{ itr };
+        auto _key   = std::string{};
+        auto _value = std::string{};
+        iss >> _key;
+
+        if(_key.find("Cap") == 0) iss >> _value;
+
+        if(!_value.empty())
+        {
+            auto _key_matches = [&_key](std::string_view _cap_id_str) {
+                return (_key.find(_cap_id_str) == 0);
+            };
+            if(_key_matches("CapInh"))
+                _data.inherited = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapPrm"))
+                _data.permitted = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapEff"))
+                _data.effective = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapBnd"))
+                _data.bounding = std::stoull(_value, nullptr, 16);
+            else if(_key_matches("CapAmb"))
+                _data.ambient = std::stoull(_value, nullptr, 16);
+        }
+    }
+
+    return _data;
+}
+
+std::string
+cap_name(cap_value_t _v)
+{
+    for(const auto& itr : known_capabilities)
+        if(itr.value == _v) return to_lower(std::string{ itr.name });
+
+    return std::string{};
+}
+
+std::vector<cap_value_t>
+decode(unsigned long long value)
+{
+    auto _data = std::vector<cap_value_t>{};
+    for(unsigned cap = 0; (cap < 64) && ((value >> cap) != 0U); ++cap)
+    {
+        auto _mask = value & (1ULL << cap);
+        if(_mask != 0U)
+        {
+            if(cap >= 0 && cap < cap_max_bits_v) _data.emplace_back(cap);
+        }
+    }
+
+    return _data;
+}
+
+std::vector<cap_value_t>
+decode(const char* arg)
+{
+    return decode(std::strtoull(arg, nullptr, 16));
+}
+/*
+template <typename Tp>
+std::string
+as_hex(Tp _v, size_t _width = 16)
+{
+    std::stringstream _ss;
+    _ss.fill('0');
+    _ss << "0x" << std::hex << std::setw(_width) << _v;
+    return _ss.str();
+}
+
+void run(std::string&& arg)
+{
+    if(arg.find("0x") == 0)
+        arg = arg.substr(2);
+
+    arg.insert(0, "0x");
+    // arg = std::string{ "0x" } + arg;
+
+    std::cout << arg << "=";
+    auto _decoded = decode(arg.c_str());
+    auto _msg     = std::stringstream{};
+    for(auto&& itr : _decoded)
+    {
+        _msg << "," << cap_name(itr);
+    }
+    auto _msg_v = _msg.str();
+    if(!_msg_v.empty())
+    {
+        std::cout << _msg_v.substr(1);
+    }
+    std::cout << "\n";
+}
+
+void
+run(std::string_view _label, unsigned long long arg)
+{
+    std::cout << " " << std::setw(12) << _label << " ";
+    run(as_hex(arg));
+}
+*/
+}  // namespace
+
+int
+main(int argc, char** argv)
+{
+    const auto* _usage = R"usage(
+usage: omnitrace-capchk <capability-name> <capability-set> <pid>
+
+    Description:
+        Simple tool for checking the effective capabilities of a running process
+
+    Arguments:
+        capability-name (string):
+            case-insensitive string matching CAP_* fields defined in `man 7 capabilities`
+
+        capability-set (string; optional):
+            Choices:
+                - effective (default)
+                - permitted
+                - inherited
+                - bounding
+                - ambient
+            See `man 7 capabilities` for more info
+
+        pid (numeric process-identifier; optional):
+            target process identifier for capability query. If not specified, queries the
+            capabilities of the current process (this exe).
+
+    Exit value:
+        0 if the process has the specified capability in the specified set
+        1 if the process does not have the capability
+        2 if the capability name is not supported
+        3 if the capability set name is not supported
+
+    Examples:
+        $ omnitrace-capchk CAP_SYS_ADMIN
+            Check if this exe (self) has CAP_SYS_ADMIN capability in the (default) effective capability set
+
+        $ omnitrace-capchk sys_admin bounding 423032
+            Check if process 423032 has CAP_SYS_ADMIN capability in the bounding capability set
+    )usage";
+
+    std::string capability_name = {};
+    std::string capability_mode = "effective";
+    pid_t       target_pid      = getpid();
+
+    for(int i = 1; i < argc; ++i)
+    {
+        auto arg = std::string_view{ argv[i] };
+        if(arg == "-h" || arg == "--help" || arg == "-?")
+        {
+            std::cout << _usage << "\n";
+            return EXIT_SUCCESS;
+        }
+    }
+
+    if(argc > 1) capability_name = to_lower(std::string{ argv[1] });
+    if(argc > 2) capability_mode = to_lower(std::string{ argv[2] });
+    if(argc > 3)
+    {
+        auto _pid_s = to_lower(std::string{ argv[3] });
+        if(_pid_s != "self") target_pid = std::stoul(argv[3]);
+    }
+
+    if(capability_name.find("cap_") != 0) capability_name.insert(0, "cap_");
+
+    capability_name = to_upper(capability_name);
+
+    const cap_info* _info = nullptr;
+    for(const auto& itr : known_capabilities)
+    {
+        if(capability_name == std::string_view{ itr.name })
+        {
+            _info = &itr;
+            break;
+        }
+    }
+
+    if(!_info)
+    {
+        fprintf(stderr, "Error! invalid capability: %s\n", capability_name.c_str());
+        return EXIT_FAILURE + 1;
+    }
+
+    auto  _status  = cap_read(target_pid);
+    auto* _dataset = &_status.effective;
+
+    /*
+    std::cout << "pid=" << target_pid << ":\n";
+    run("inherited", _status.inherited);
+    run("permitted", _status.permitted);
+    run("effective", _status.effective);
+    run("bounding", _status.bounding);
+    run("ambient", _status.ambient);
+    */
+
+    if(capability_mode == "effective")
+        _dataset = &_status.effective;
+    else if(capability_mode == "permitted")
+        _dataset = &_status.permitted;
+    else if(capability_mode == "inherited")
+        _dataset = &_status.inherited;
+    else if(capability_mode == "bounding")
+        _dataset = &_status.bounding;
+    else if(capability_mode == "ambient")
+        _dataset = &_status.ambient;
+    else
+    {
+        fprintf(stderr, "Error! invalid capability set: %s\n", capability_mode.c_str());
+        return EXIT_FAILURE + 2;
+    }
+
+    auto _ec = EXIT_FAILURE;
+    for(auto&& itr : decode(*_dataset))
+    {
+        if(itr == _info->value)
+        {
+            _ec = EXIT_SUCCESS;
+            break;
+        }
+    }
+
+    std::cout << ((_ec == EXIT_SUCCESS) ? "Found" : "Missing") << " capability "
+              << capability_name << " in " << capability_mode
+              << " capability set. Exit code: " << _ec << ".\n";
+    return _ec;
+}

tests/omnitrace-config-tests.cmake

@@ -24,9 +24,15 @@ add_test(
 
 set_tests_properties(
     omnitrace-invalid-config
-    PROPERTIES ENVIRONMENT
-               "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/invalid.cfg" TIMEOUT
-               120 LABELS "config" WILL_FAIL ON)
+    PROPERTIES
+        ENVIRONMENT
+        "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/invalid.cfg;OMNITRACE_CI=ON;OMNITRACE_CI_TIMEOUT=120"
+        TIMEOUT
+        120
+        LABELS
+        "config"
+        WILL_FAIL
+        ON)
 
 add_test(
     NAME omnitrace-missing-config
@@ -35,6 +41,12 @@ add_test(
 
 set_tests_properties(
     omnitrace-missing-config
-    PROPERTIES ENVIRONMENT
-               "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/missing.cfg" TIMEOUT
-               120 LABELS "config" WILL_FAIL ON)
+    PROPERTIES
+        ENVIRONMENT
+        "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/missing.cfg;OMNITRACE_CI=ON;OMNITRACE_CI_TIMEOUT=120"
+        TIMEOUT
+        120
+        LABELS
+        "config"
+        WILL_FAIL
+        ON)

tests/omnitrace-critical-trace-tests.cmake

@@ -35,7 +35,9 @@ set(_parallel_overhead_critical_trace_environ
     "OMNITRACE_CRITICAL_TRACE_DEBUG=ON"
     "OMNITRACE_VERBOSE=4"
     "OMNITRACE_USE_PID=OFF"
-    "OMNITRACE_TIME_OUTPUT=OFF")
+    "OMNITRACE_TIME_OUTPUT=OFF"
+    "OMNITRACE_CI=ON"
+    "OMNITRACE_CI_TIMEOUT=300")
 
 set_tests_properties(
     parallel-overhead-process-critical-trace

tests/omnitrace-fork-tests.cmake

@@ -14,10 +14,6 @@ omnitrace_add_test(
     SAMPLING_PASS_REGEX "fork.. called on PID"
     RUNTIME_PASS_REGEX "fork.. called on PID"
     REWRITE_RUN_PASS_REGEX "fork.. called on PID"
-    SAMPLING_FAIL_REGEX
-        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
-    RUNTIME_FAIL_REGEX
-        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
-    REWRITE_RUN_FAIL_REGEX
-        "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])"
-    )
+    SAMPLING_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})"
+    RUNTIME_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})"
+    REWRITE_RUN_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})")

tests/omnitrace-mpi-tests.cmake

@@ -29,7 +29,7 @@ omnitrace_add_test(
     REWRITE_RUN_PASS_REGEX
         "(/[A-Za-z-]+/perfetto-trace-0.proto).*(/[A-Za-z-]+/wall_clock-0.txt')"
     REWRITE_RUN_FAIL_REGEX
-        "(perfetto-trace|trip_count|sampling_percent|sampling_cpu_clock|sampling_wall_clock|wall_clock)-[0-9][0-9]+.(json|txt|proto)"
+        "(perfetto-trace|trip_count|sampling_percent|sampling_cpu_clock|sampling_wall_clock|wall_clock)-[0-9][0-9]+.(json|txt|proto)|OMNITRACE_ABORT_FAIL_REGEX"
     )
 
 omnitrace_add_test(

tests/omnitrace-overflow-tests.cmake

@@ -0,0 +1,32 @@
+# -------------------------------------------------------------------------------------- #
+#
+# overflow tests
+#
+# -------------------------------------------------------------------------------------- #
+
+set(_overflow_environment
+    "${_base_environment}"
+    "OMNITRACE_VERBOSE=2"
+    "OMNITRACE_SAMPLING_CPUTIME=OFF"
+    "OMNITRACE_SAMPLING_REALTIME=OFF"
+    "OMNITRACE_SAMPLING_OVERFLOW=ON"
+    "OMNITRACE_SAMPLING_OVERFLOW_EVENT=PERF_COUNT_SW_CPU_CLOCK"
+    "OMNITRACE_SAMPLING_OVERFLOW_FREQ=10000"
+    "OMNITRACE_DEBUG_THREADING_GET_ID=ON")
+
+if(omnitrace_perf_event_paranoid LESS_EQUAL 3
+   OR omnitrace_cap_sys_admin EQUAL 0
+   OR omnitrace_cap_perfmon EQUAL 0)
+    omnitrace_add_test(
+        SKIP_BASELINE
+        NAME overflow
+        TARGET parallel-overhead
+        RUN_ARGS 30 2 200
+        REWRITE_ARGS -e -v 2
+        RUNTIME_ARGS -e -v 1
+        ENVIRONMENT "${_overflow_environment}"
+        LABELS "perf;overflow"
+        SAMPLING_PASS_REGEX "sampling_wall_clock.txt"
+        RUNTIME_PASS_REGEX "sampling_wall_clock.txt"
+        REWRITE_RUN_PASS_REGEX "sampling_wall_clock.txt")
+endif()

tests/omnitrace-python-tests.cmake

@@ -116,7 +116,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
             COMMAND ${OMNITRACE_CAT_COMMAND}
             PYTHON_VERSION ${_VERSION}
             FILE omnitrace-tests-output/python-external-exclude-inefficient/${_VERSION}/trip_count.txt
-            FAIL_REGEX "(\\\|_inefficient).*(\\\|_sum)"
+            FAIL_REGEX "(\\\|_inefficient).*(\\\|_sum)|OMNITRACE_ABORT_FAIL_REGEX"
             DEPENDS python-external-exclude-inefficient-${_VERSION}
             ENVIRONMENT "${_python_environment}")
 
@@ -135,7 +135,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
             PYTHON_VERSION ${_VERSION}
             FILE omnitrace-tests-output/python-builtin-noprofile/${_VERSION}/trip_count.txt
             PASS_REGEX ".(run)..(noprofile.py)."
-            FAIL_REGEX ".(fib|inefficient)..(noprofile.py)."
+            FAIL_REGEX ".(fib|inefficient)..(noprofile.py).|OMNITRACE_ABORT_FAIL_REGEX"
             DEPENDS python-builtin-noprofile-${_VERSION}
             ENVIRONMENT "${_python_environment}")
     else()

tests/omnitrace-rocm-tests.cmake

@@ -81,5 +81,5 @@ if(OMNITRACE_USE_ROCPROFILER)
             "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_ROCTRACER=OFF;OMNITRACE_ROCM_EVENTS=${OMNITRACE_ROCM_EVENTS_TEST}"
         REWRITE_RUN_PASS_REGEX
             "rocprof-device-0-GRBM_COUNT.txt(.*)rocprof-device-0-GPUBusy.txt(.*)rocprof-device-0-SQ_WAVES.txt(.*)rocprof-device-0-SQ_INSTS_VALU.txt(.*)rocprof-device-0-VALUInsts.txt(.*)rocprof-device-0-TCC_HIT_sum.txt(.*)rocprof-device-0-TA_TA_BUSY_0.txt(.*)rocprof-device-0-TA_TA_BUSY_11.txt"
-        REWRITE_RUN_FAIL_REGEX "roctracer.txt")
+        REWRITE_RUN_FAIL_REGEX "roctracer.txt|OMNITRACE_ABORT_FAIL_REGEX")
 endif()

tests/omnitrace-testing.cmake

@@ -213,6 +213,40 @@ execute_process(
     RESULT_VARIABLE _mpiexec_oversubscribe
     OUTPUT_QUIET ERROR_QUIET)
 
+set(omnitrace_perf_event_paranoid "4")
+set(omnitrace_cap_sys_admin "1")
+set(omnitrace_cap_perfmon "1")
+
+if(EXISTS "/proc/sys/kernel/perf_event_paranoid")
+    file(STRINGS "/proc/sys/kernel/perf_event_paranoid" omnitrace_perf_event_paranoid
+         LIMIT_COUNT 1)
+endif()
+
+execute_process(
+    COMMAND ${CMAKE_CXX_COMPILER} -O2 -g -std=c++17
+            ${CMAKE_CURRENT_LIST_DIR}/omnitrace-capchk.cpp -o omnitrace-capchk
+    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin
+    RESULT_VARIABLE _capchk_compile
+    OUTPUT_QUIET ERROR_QUIET)
+
+if(_capchk_compile EQUAL 0)
+    execute_process(
+        COMMAND ${PROJECT_BINARY_DIR}/bin/omnitrace-capchk CAP_SYS_ADMIN effective
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+        RESULT_VARIABLE omnitrace_cap_sys_admin
+        OUTPUT_QUIET ERROR_QUIET)
+
+    execute_process(
+        COMMAND ${PROJECT_BINARY_DIR}/bin/omnitrace-capchk CAP_PERFMON effective
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+        RESULT_VARIABLE omnitrace_cap_perfmon
+        OUTPUT_QUIET ERROR_QUIET)
+endif()
+
+omnitrace_message(STATUS "perf_event_paranoid: ${omnitrace_perf_event_paranoid}")
+omnitrace_message(STATUS "CAP_SYS_ADMIN: ${omnitrace_cap_sys_admin}")
+omnitrace_message(STATUS "CAP_PERFMON: ${omnitrace_cap_perfmon}")
+
 if(_mpiexec_oversubscribe EQUAL 0)
     list(APPEND MPIEXEC_EXECUTABLE_ARGS --oversubscribe)
 endif()
@@ -255,9 +289,30 @@ endif()
 
 # -------------------------------------------------------------------------------------- #
 
+macro(OMNITRACE_CHECK_PASS_FAIL_REGEX NAME PASS FAIL)
+    if(NOT "${${PASS}}" STREQUAL ""
+       AND NOT "${${FAIL}}" STREQUAL ""
+       AND NOT "${${FAIL}}" MATCHES "\\|OMNITRACE_ABORT_FAIL_REGEX"
+       AND NOT "${${FAIL}}" MATCHES "${OMNITRACE_ABORT_FAIL_REGEX}")
+        omnitrace_message(
+            FATAL_ERROR
+            "${NAME} has set pass and fail regexes but fail regex does not include '|OMNITRACE_ABORT_FAIL_REGEX'"
+            )
+    endif()
+
+    if("${${FAIL}}" STREQUAL "")
+        set(${FAIL} "(${OMNITRACE_ABORT_FAIL_REGEX})")
+    else()
+        string(REPLACE "|OMNITRACE_ABORT_FAIL_REGEX" "|${OMNITRACE_ABORT_FAIL_REGEX}"
+                       ${FAIL} "${${FAIL}}")
+    endif()
+endmacro()
+
+# -------------------------------------------------------------------------------------- #
+
 function(OMNITRACE_WRITE_TEST_CONFIG _FILE _ENV)
     set(_ENV_ONLY
-        "OMNITRACE_(CI|MODE|USE_MPIP|DEBUG_SETTINGS|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)="
+        "OMNITRACE_(CI|CI_TIMEOUT|MODE|USE_MPIP|DEBUG_[A-Z_]+|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)="
         )
     set(_FILE_CONTENTS)
     set(_ENV_CONTENTS)
@@ -357,9 +412,7 @@ function(OMNITRACE_ADD_TEST)
 
     foreach(_PREFIX SAMPLING RUNTIME REWRITE REWRITE_RUN BASELINE)
         if("${${_PREFIX}_FAIL_REGEX}" STREQUAL "")
-            set(${_PREFIX}_FAIL_REGEX
-                "(### ERROR ###|address of faulting memory reference|exiting with non-zero exit code)"
-                )
+            set(${_PREFIX}_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})")
         endif()
     endforeach()
 
@@ -552,6 +605,10 @@ function(OMNITRACE_ADD_TEST)
                 endif()
             endforeach()
 
+            list(APPEND _environ "OMNITRACE_CI_TIMEOUT=${_timeout}")
+
+            omnitrace_check_pass_fail_regex("${TEST_NAME}-${_TEST}" "${_PASS_REGEX}"
+                                            "${_FAIL_REGEX}")
             if(TEST ${TEST_NAME}-${_TEST})
                 omnitrace_write_test_config(${TEST_NAME}-${_TEST}.cfg _environ)
                 set_tests_properties(
@@ -604,9 +661,7 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
     endif()
 
     if("${TEST_CAUSAL_FAIL_REGEX}" STREQUAL "")
-        set(TEST_CAUSAL_FAIL_REGEX
-            "(### ERROR ###|address of faulting memory reference|exiting with non-zero exit code)"
-            )
+        set(TEST_CAUSAL_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})")
     endif()
 
     if(TARGET ${TEST_TARGET})
@@ -649,10 +704,16 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
         foreach(_TEST baseline causal validate-causal)
 
             if(NOT TEST ${_TEST}-${TEST_NAME})
-                continue()
+                if(NOT TEST ${TEST_NAME}-${_TEST})
+                    continue()
+                else()
+                    set(_NAME "${TEST_NAME}-${_TEST}")
+                endif()
+            else()
+                set(_NAME "${_TEST}-${TEST_NAME}")
             endif()
 
-            set(_prefix "${_TEST}-${TEST_NAME}/")
+            set(_prefix "${_NAME}/")
             set(_labels "${_TEST}" "causal-profiling")
 
             if(TEST_TARGET)
@@ -701,9 +762,11 @@ function(OMNITRACE_ADD_CAUSAL_TEST)
                 endif()
             endforeach()
 
-            omnitrace_write_test_config(${_TEST}-${TEST_NAME}.cfg _environ)
+            list(APPEND _environ "OMNITRACE_CI_TIMEOUT=${_timeout}")
+            omnitrace_write_test_config(${_NAME}.cfg _environ)
+            omnitrace_check_pass_fail_regex("${_NAME}" "${_PASS_REGEX}" "${_FAIL_REGEX}")
             set_tests_properties(
-                ${_TEST}-${TEST_NAME}
+                ${_NAME}
                 PROPERTIES ENVIRONMENT
                            "${_environ}"
                            TIMEOUT
@@ -780,16 +843,10 @@ function(OMNITRACE_ADD_PYTHON_TEST)
             NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}
             COMMAND ${TEST_COMMAND} ${TEST_FILE}
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-        add_test(
-            NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse
-            COMMAND ${TEST_COMMAND} ${TEST_FILE}
-            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif()
 
-    foreach(
-        _TEST
-        ${TEST_NAME}-${TEST_PYTHON_VERSION} ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse
-        ${TEST_NAME}-${TEST_PYTHON_VERSION}-annotated)
+    foreach(_TEST ${TEST_NAME}-${TEST_PYTHON_VERSION}
+                  ${TEST_NAME}-${TEST_PYTHON_VERSION}-annotated)
 
         if(NOT TEST "${_TEST}")
             continue()
@@ -797,24 +854,18 @@ function(OMNITRACE_ADD_PYTHON_TEST)
 
         string(REPLACE "${TEST_NAME}-${TEST_PYTHON_VERSION}" "${TEST_NAME}" _TEST_DIR
                        "${_TEST}")
-        set(_TEST_ENV "${TEST_ENVIRONMENT}"
-                      "OMNITRACE_OUTPUT_PREFIX=${_TEST_DIR}/${TEST_PYTHON_VERSION}/")
+        set(_TEST_ENV
+            "${TEST_ENVIRONMENT}"
+            "OMNITRACE_OUTPUT_PREFIX=${_TEST_DIR}/${TEST_PYTHON_VERSION}/"
+            "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}")
 
         set(_TEST_PROPERTIES "${TEST_PROPERTIES}")
-        if(NOT "${_TEST}" MATCHES "inverse")
-            # assign pass variable to pass regex
-            set(_PASS_REGEX TEST_PASS_REGEX)
-            # assign fail variable to fail regex
-            set(_FAIL_REGEX TEST_FAIL_REGEX)
-        else()
-            # assign pass variable to fail regex
-            set(_PASS_REGEX TEST_FAIL_REGEX)
-            # assign fail variable to pass regex
-            set(_FAIL_REGEX TEST_PASS_REGEX)
-            # set to will fail
-            list(APPEND _TEST_PROPERTIES WILL_FAIL ON)
-        endif()
+        # assign pass variable to pass regex
+        set(_PASS_REGEX TEST_PASS_REGEX)
+        # assign fail variable to fail regex
+        set(_FAIL_REGEX TEST_FAIL_REGEX)
 
+        omnitrace_check_pass_fail_regex("${_TEST}" "${_PASS_REGEX}" "${_FAIL_REGEX}")
         set_tests_properties(
             ${_TEST}
             PROPERTIES ENVIRONMENT
@@ -964,16 +1015,19 @@ function(OMNITRACE_ADD_VALIDATION_TEST)
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif()
 
+    list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}")
+
     foreach(_TEST validate-${TEST_NAME}-timemory validate-${TEST_NAME}-perfetto)
 
         if(NOT TEST "${_TEST}")
             continue()
         endif()
 
+        omnitrace_check_pass_fail_regex("${_TEST}" "TEST_PASS_REGEX" "TEST_FAIL_REGEX")
         set_tests_properties(
             ${_TEST}
             PROPERTIES ENVIRONMENT
-                       "${_TEST_ENV}"
+                       "${TEST_ENVIRONMENT}"
                        TIMEOUT
                        ${TEST_TIMEOUT}
                        LABELS

tests/omnitrace-time-window-tests.cmake

@@ -24,7 +24,7 @@ omnitrace_add_validation_test(
     PERFETTO_METRIC "host"
     PERFETTO_FILE "perfetto-trace.proto"
     LABELS "time-window"
-    FAIL_REGEX "outer_d"
+    FAIL_REGEX "outer_d|OMNITRACE_ABORT_FAIL_REGEX"
     ARGS -l
          trace-time-window.inst
          outer_a
@@ -49,7 +49,7 @@ omnitrace_add_validation_test(
     PERFETTO_METRIC "host"
     PERFETTO_FILE "perfetto-trace.proto"
     LABELS "time-window"
-    FAIL_REGEX "outer_d"
+    FAIL_REGEX "outer_d|OMNITRACE_ABORT_FAIL_REGEX"
     ARGS -l
          trace-time-window
          outer_a