Misc fixes before v1.8.0 release (#239)

* Update timemory submodule for OMPT

- Updated OMPT support for OpenMP 5.2

* omnitrace exe cleanup

- fixed "omnitrace --" segfault
- added nullptr checks

* CMake updates

- moved omnitrace-interface-library definition up a directory
- general cleanup
- fixed branch/tag/ref for git submodule checkouts

* Improve shutdown of causal profiling after duration limit

* Fix dyninst minimum version number

* Removed debug print from binary::get_link_map

* Remove use of thread-pool in causal

* Relax causal testing when variance is high

* causal_gotcha utilities for blocking signals

* Tweak to causal example

* Install validate-causal-json as omnitrace-causal-print

* simplify address_multirange

* improve causal line saving

CMakeLists.txt

@@ -387,6 +387,15 @@ if(OMNITRACE_BUILD_TESTING)
     add_subdirectory(tests)
 endif()
 
+# install the validate-causal-json python script as a utility
+configure_file(
+    ${PROJECT_SOURCE_DIR}/tests/validate-causal-json.py
+    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/omnitrace-causal-print
+    USE_SOURCE_PERMISSIONS COPYONLY)
+
+install(PROGRAMS ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/omnitrace-causal-print
+        DESTINATION ${CMAKE_INSTALL_BINDIR})
+
 # ------------------------------------------------------------------------------#
 #
 # packaging

cmake/PAPI.cmake

@@ -11,7 +11,7 @@ omnitrace_checkout_git_submodule(
     RELATIVE_PATH external/papi
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     REPO_URL https://bitbucket.org/icl/papi.git
-    REPO_BRANCH master
+    REPO_BRANCH effd1ef4e0fd4b80e36546791277215a2d6b9eba
     TEST_FILE src/configure)
 
 set(PAPI_LIBPFM_SOVERSION
@@ -39,6 +39,10 @@ if(NOT EXISTS "${OMNITRACE_PAPI_INSTALL_DIR}")
             ${CMAKE_COMMAND} -E touch ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpapi.a
             ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.a
             ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.so)
+    set(_OMNITRACE_PAPI_BUILD_BYPRODUCTS
+        ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpapi.a
+        ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.a
+        ${OMNITRACE_PAPI_INSTALL_DIR}/lib/libpfm.so)
 endif()
 
 omnitrace_add_option(OMNITRACE_PAPI_AUTO_COMPONENTS "Automatically enable components" OFF)
@@ -48,7 +52,6 @@ omnitrace_add_option(OMNITRACE_PAPI_AUTO_COMPONENTS "Automatically enable compon
 set(_OMNITRACE_VALID_PAPI_COMPONENTS
     appio
     bgpm
-    components
     coretemp
     coretemp_freebsd
     cuda
@@ -202,23 +205,23 @@ externalproject_add(
     BUILD_IN_SOURCE 1
     PATCH_COMMAND
         ${CMAKE_COMMAND} -E env CC=${PAPI_C_COMPILER}
-        CFLAGS=-fPIC\ -O3\ -g\ -Wno-stringop-truncation LIBS=-lrt LDFLAGS=-lrt
+        CFLAGS=-fPIC\ -O3\ -Wno-stringop-truncation LIBS=-lrt LDFLAGS=-lrt
         ${OMNITRACE_PAPI_EXTRA_ENV} <SOURCE_DIR>/configure
         --prefix=${OMNITRACE_PAPI_INSTALL_DIR} --with-static-lib=yes --with-shared-lib=no
         --with-perf-events --with-tests=no --with-components=${_OMNITRACE_PAPI_COMPONENTS}
-    CONFIGURE_COMMAND
-        ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -g\ -Wno-stringop-truncation
-        ${OMNITRACE_PAPI_EXTRA_ENV} ${MAKE_EXECUTABLE} static install
-    BUILD_COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -g\ -Wno-stringop-truncation
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -Wno-stringop-truncation
+                      ${OMNITRACE_PAPI_EXTRA_ENV} ${MAKE_EXECUTABLE} static install
+    BUILD_COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -Wno-stringop-truncation
                   ${OMNITRACE_PAPI_EXTRA_ENV} ${MAKE_EXECUTABLE} utils install-utils
-    INSTALL_COMMAND "")
+    INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS "${_OMNITRACE_PAPI_BUILD_BYPRODUCTS}")
 
 # target for re-executing the installation
 add_custom_target(
     omnitrace-papi-install
-    COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -g\ -Wno-stringop-truncation
+    COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -Wno-stringop-truncation
             ${OMNITRACE_PAPI_EXTRA_ENV} ${MAKE_EXECUTABLE} static install
-    COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -g\ -Wno-stringop-truncation
+    COMMAND ${CMAKE_COMMAND} -E env CFLAGS=-fPIC\ -O3\ -Wno-stringop-truncation
             ${OMNITRACE_PAPI_EXTRA_ENV} ${MAKE_EXECUTABLE} utils install-utils
     WORKING_DIRECTORY ${OMNITRACE_PAPI_SOURCE_DIR}/src
     COMMENT "Installing PAPI...")

cmake/Packages.cmake

@@ -519,7 +519,7 @@ omnitrace_checkout_git_submodule(
     RELATIVE_PATH external/perfetto
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     REPO_URL https://android.googlesource.com/platform/external/perfetto
-    REPO_BRANCH v17.0
+    REPO_BRANCH v28.0
     TEST_FILE sdk/perfetto.cc)
 
 include(Perfetto)
@@ -756,7 +756,7 @@ if(NOT TARGET PTL::ptl-shared)
         RELATIVE_PATH external/PTL
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
         REPO_URL https://github.com/jrmadsen/PTL.git
-        REPO_BRANCH master)
+        REPO_BRANCH omnitrace)
 
     set(PTL_BUILD_EXAMPLES OFF)
     set(PTL_USE_TBB OFF)
@@ -943,6 +943,9 @@ if("${CMAKE_BUILD_TYPE}" MATCHES "Release" AND NOT OMNITRACE_BUILD_DEBUG)
     add_target_flag_if_avail(omnitrace-compile-options "-g1")
 endif()
 
+target_compile_definitions(omnitrace-compile-definitions
+                           INTERFACE OMNITRACE_MAX_THREADS=${OMNITRACE_MAX_THREADS})
+
 foreach(_LIB ${OMNITRACE_EXTENSION_LIBRARIES})
     get_target_property(_COMPILE_DEFS ${_LIB} INTERFACE_COMPILE_DEFINITIONS)
     if(_COMPILE_DEFS)

examples/causal/impl.cpp

@@ -20,6 +20,8 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
+#include "causal.hpp"
+
 #include <chrono>
 #include <cmath>
 #include <cstdio>
@@ -90,7 +92,11 @@ cpu_func_impl(int64_t n, int nloop)
     auto _cpu_now = get_clock_cpu_now();
     auto _cpu_end = _cpu_now + n;
     // clang-format off
-    while(get_clock_cpu_now() < _cpu_end) { for(volatile int i = 0; i < nloop; ++i) {} }
+    while(get_clock_cpu_now() < _cpu_end) 
+    { 
+        for(volatile int i = 0; i < nloop; ++i) {} 
+        CAUSAL_PROGRESS_NAMED("cpu_impl"); 
+    }
     // clang-format on
     return V;
 }

source/bin/omnitrace/details.cpp

@@ -567,6 +567,12 @@ process_modules(const std::vector<module_t*>& _app_modules)
 {
     parse_internal_libs_data();
 
+    auto _erase_nullptrs = [](auto& _vec) {
+        _vec.erase(std::remove_if(_vec.begin(), _vec.end(),
+                                  [](const auto* itr) { return (itr == nullptr); }),
+                   _vec.end());
+    };
+
     auto _wc = tim::component::wall_clock{};
     auto _pr = tim::component::peak_rss{};
     _wc.start();
@@ -578,15 +584,22 @@ process_modules(const std::vector<module_t*>& _app_modules)
         if(_module) symtab_data.modules.emplace_back(_module);
     }
 
+    _erase_nullptrs(symtab_data.modules);
+
     verbprintf(0, "Processing %zu modules...\n", symtab_data.modules.size());
 
+    if(symtab_data.modules.empty()) return;
+
     const auto& _data  = get_internal_libs_data();
     auto        _names = std::set<std::string_view>{};
     for(const auto& itr : _data)
     {
-        _names.emplace(itr.first);
-        for(const auto& ditr : itr.second)
-            _names.emplace(ditr.first);
+        if(!itr.first.empty())
+        {
+            _names.emplace(itr.first);
+            for(const auto& ditr : itr.second)
+                _names.emplace(ditr.first);
+        }
     }
 
     for(auto* itr : symtab_data.modules)
@@ -594,6 +607,8 @@ process_modules(const std::vector<module_t*>& _app_modules)
         const auto* _base_name = tim::filepath::basename(itr->fullName());
         auto        _real_name = tim::filepath::realpath(itr->fullName(), nullptr, false);
 
+        if(!_base_name) continue;
+
         if(_names.count(_base_name) == 0 && _names.count(_real_name) == 0)
         {
             verbprintf(2, "Processing symbol table for module '%s'...\n",
@@ -601,13 +616,17 @@ process_modules(const std::vector<module_t*>& _app_modules)
         }
 
         symtab_data.functions.emplace(itr, std::vector<symtab_func_t*>{});
-        itr->getAllFunctions(symtab_data.functions.at(itr));
+        if(!itr->getAllFunctions(symtab_data.functions.at(itr))) continue;
+        _erase_nullptrs(symtab_data.functions.at(itr));
+
         for(auto* fitr : symtab_data.functions.at(itr))
         {
             symtab_data.typed_func_names[tim::demangle(fitr->getName())] = fitr;
 
             symtab_data.symbols.emplace(fitr, std::vector<symtab_symbol_t*>{});
-            fitr->getSymbols(symtab_data.symbols.at(fitr));
+            if(!fitr->getSymbols(symtab_data.symbols.at(fitr))) continue;
+            _erase_nullptrs(symtab_data.symbols.at(fitr));
+
             for(auto* sitr : symtab_data.symbols.at(fitr))
             {
                 symtab_data.mangled_symbol_names[sitr->getMangledName()] = sitr;

source/bin/omnitrace/internal_libs.cpp

@@ -265,9 +265,9 @@ get_library_search_paths_impl()
     }
 
     // search hard-coded system paths
-    for(const char* itr : { "/usr/local/lib", "/usr/share/lib", "/usr/lib", "/usr/lib64",
-                            "/usr/lib/x86_64-linux-gnu", "/lib", "/lib64",
-                            "/lib/x86_64-linux-gnu", "/usr/lib/i386-linux-gnu" })
+    for(const char* itr :
+        { "/usr/local/lib", "/usr/share/lib", "/usr/lib", "/usr/lib64",
+          "/usr/lib/x86_64-linux-gnu", "/lib", "/lib64", "/lib/x86_64-linux-gnu" })
     {
         _emplace_if_exists(itr);
     }

source/bin/omnitrace/omnitrace.cpp

@@ -324,7 +324,7 @@ main(int argc, char** argv)
                     std::regex_replace(argv[j], std::regex{ "(.*)([ \t\n\r]+)$" }, "$1");
                 copy_str(_cmdv[k], _v.c_str());
             }
-            mutname = _cmdv[0];
+            if(_cmdc > 0) mutname = _cmdv[0];
             break;
         }
         else

source/docs/installation.md

@@ -114,7 +114,7 @@ and Dyninst requires TBB), and the CMake option to build the package alongside o
 
 | Third-Party Library | Minimum Version | Required By | CMake Option                              |
 |---------------------|-----------------|-------------|-------------------------------------------|
-| Dyninst             | 10.0            | OmniTrace   | `OMNITRACE_BUILD_DYNINST` (default: OFF)  |
+| Dyninst             | 12.0            | OmniTrace   | `OMNITRACE_BUILD_DYNINST` (default: OFF)  |
 | Libunwind           |                 | OmniTrace   | `OMNITRACE_BUILD_LIBUNWIND` (default: ON) |
 | TBB                 | 2018.6          | Dyninst     | `DYNINST_BUILD_TBB` (default: OFF)        |
 | ElfUtils            | 0.178           | Dyninst     | `DYNINST_BUILD_ELFUTILS` (default: OFF)   |

source/lib/CMakeLists.txt

@@ -8,6 +8,10 @@
 #
 # ----------------------------------------------------------------------------- #
 
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.20)
+    cmake_policy(SET CMP0115 NEW)
+endif()
+
 if(OMNITRACE_USE_ROCPROFILER
    AND rocprofiler_LIBRARY_DIR
    AND ROCmVersion_TRIPLE_VERSION VERSION_LESS 5.2.0
@@ -18,10 +22,59 @@ else()
     set(OMNITRACE_LIB_INSTALL_RPATH "\$ORIGIN:\$ORIGIN/omnitrace")
 endif()
 
+# ------------------------------------------------------------------------------#
+#
+# omnitrace interface library
+#
+# ------------------------------------------------------------------------------#
+
+add_library(omnitrace-interface-library INTERFACE)
+add_library(omnitrace::omnitrace-interface-library ALIAS omnitrace-interface-library)
+
+target_include_directories(
+    omnitrace-interface-library INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}
+                                          ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace)
+
+target_link_libraries(
+    omnitrace-interface-library
+    INTERFACE
+        $<BUILD_INTERFACE:omnitrace::omnitrace-headers>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-threading>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-common-library>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-compile-options>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-compile-definitions>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-perfetto>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-timemory>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-elfutils>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-bfd>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-mpi>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-ptl>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-hip>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-roctracer>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-rocprofiler>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-rocm-smi>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-rccl>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-static-libgcc-optional>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-static-libstdcxx-optional>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-sanitizer>
+        $<BUILD_INTERFACE:$<IF:$<BOOL:${OMNITRACE_BUILD_LTO}>,omnitrace::omnitrace-lto,>>)
+
+# ------------------------------------------------------------------------------#
+#
+# omnitrace internal libraries
+#
+# ------------------------------------------------------------------------------#
+
 add_subdirectory(common)
 add_subdirectory(core)
 add_subdirectory(binary)
 
+# ------------------------------------------------------------------------------#
+#
+# omnitrace exported libraries
+#
+# ------------------------------------------------------------------------------#
+
 add_subdirectory(omnitrace)
 add_subdirectory(omnitrace-dl)
 add_subdirectory(omnitrace-user)

source/lib/binary/address_multirange.cpp

@@ -53,8 +53,8 @@ address_multirange::operator+=(uintptr_t _v)
 {
     *this += std::make_pair(coarse{}, _v);
 
-    for(auto&& itr : m_fine_ranges)
-        if(itr.contains(_v)) return *this;
+    // for(auto&& itr : m_fine_ranges)
+    //    if(itr.contains(_v)) return *this;
 
     m_fine_ranges.emplace(address_range{ _v });
     return *this;
@@ -65,8 +65,8 @@ address_multirange::operator+=(address_range _v)
 {
     *this += std::make_pair(coarse{}, _v);
 
-    for(auto&& itr : m_fine_ranges)
-        if(itr.contains(_v)) return *this;
+    // for(auto&& itr : m_fine_ranges)
+    //    if(itr.contains(_v)) return *this;
 
     m_fine_ranges.emplace(_v);
     return *this;

source/lib/binary/link_map.cpp

@@ -126,7 +126,6 @@ get_link_map(const char* _lib, const std::string& _exclude_linked_by,
 
     for(const auto& itr : _full_chain)
     {
-        std::cout << itr << std::endl;
         if(_excl_chain.find(itr) == _excl_chain.end())
         {
             if(_exclude_re.empty() || !std::regex_search(itr, std::regex{ _exclude_re }))

source/lib/core/CMakeLists.txt

@@ -48,5 +48,22 @@ target_include_directories(omnitrace-core-library BEFORE
 
 target_link_libraries(omnitrace-core-library
                       PRIVATE omnitrace::omnitrace-interface-library)
+target_link_libraries(
+    omnitrace-core-library
+    PRIVATE
+        $<BUILD_INTERFACE:omnitrace::omnitrace-headers>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-threading>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-common-library>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-compile-definitions>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-compile-options>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-perfetto>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-timemory>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-mpi>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-hip>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-rocm-smi>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-static-libgcc-optional>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-static-libstdcxx-optional>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-sanitizer>
+        $<BUILD_INTERFACE:$<IF:$<BOOL:${OMNITRACE_BUILD_LTO}>,omnitrace::omnitrace-lto,>>)
 
 set_target_properties(omnitrace-core-library PROPERTIES OUTPUT_NAME omnitrace-core)

source/lib/omnitrace/CMakeLists.txt

@@ -1,47 +1,3 @@
-# ------------------------------------------------------------------------------#
-#
-# omnitrace interface library
-#
-# ------------------------------------------------------------------------------#
-
-if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.20)
-    cmake_policy(SET CMP0115 NEW)
-endif()
-
-add_library(omnitrace-interface-library INTERFACE)
-add_library(omnitrace::omnitrace-interface-library ALIAS omnitrace-interface-library)
-
-target_include_directories(
-    omnitrace-interface-library INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}
-                                          ${CMAKE_CURRENT_BINARY_DIR})
-
-target_compile_definitions(omnitrace-interface-library
-                           INTERFACE OMNITRACE_MAX_THREADS=${OMNITRACE_MAX_THREADS})
-
-target_link_libraries(
-    omnitrace-interface-library
-    INTERFACE
-        $<BUILD_INTERFACE:omnitrace::omnitrace-headers>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-threading>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-common-library>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-compile-options>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-compile-definitions>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-perfetto>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-timemory>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-elfutils>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-bfd>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-mpi>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-ptl>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-hip>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-roctracer>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-rocprofiler>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-rocm-smi>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-rccl>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-static-libgcc-optional>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-static-libstdcxx-optional>
-        $<BUILD_INTERFACE:omnitrace::omnitrace-sanitizer>
-        $<BUILD_INTERFACE:$<IF:$<BOOL:${OMNITRACE_BUILD_LTO}>,omnitrace::omnitrace-lto,>>)
-
 # ------------------------------------------------------------------------------#
 #
 # omnitrace object library
@@ -69,7 +25,7 @@ endif()
 
 # ------------------------------------------------------------------------------#
 #
-# omnitrace shared library
+# omnitrace static library
 #
 # ------------------------------------------------------------------------------#
 
@@ -79,7 +35,7 @@ add_library(omnitrace::libomnitrace-static ALIAS omnitrace-static-library)
 target_link_libraries(
     omnitrace-static-library
     PRIVATE omnitrace::omnitrace-interface-library omnitrace::omnitrace-core
-            omnitrace::omnitrace-core omnitrace::omnitrace-binary)
+            omnitrace::omnitrace-binary)
 
 set_target_properties(omnitrace-static-library PROPERTIES OUTPUT_NAME omnitrace)
 
@@ -96,7 +52,7 @@ add_library(omnitrace::omnitrace-library ALIAS omnitrace-shared-library)
 target_link_libraries(
     omnitrace-shared-library
     PRIVATE omnitrace::omnitrace-interface-library omnitrace::omnitrace-core
-            omnitrace::omnitrace-core omnitrace::omnitrace-binary)
+            omnitrace::omnitrace-binary)
 
 set_target_properties(
     omnitrace-shared-library

source/lib/omnitrace/library/causal/components/backtrace.cpp

@@ -37,6 +37,7 @@
 #include <timemory/components/timing/backends.hpp>
 #include <timemory/components/timing/wall_clock.hpp>
 #include <timemory/mpl/concepts.hpp>
+#include <timemory/mpl/type_traits.hpp>
 #include <timemory/mpl/types.hpp>
 #include <timemory/process/threading.hpp>
 #include <timemory/units.hpp>
@@ -126,7 +127,8 @@ backtrace::sample(int _sig)
     // update the last sample for backtrace signal(s) even when in use
     static thread_local int64_t _last_sample = 0;
 
-    if(is_in_use())
+    if(is_in_use() ||
+       OMNITRACE_UNLIKELY(!trait::runtime_enabled<causal::component::backtrace>::get()))
     {
         if(_sig == get_realtime_signal()) _last_sample = tracing::now();
         return;

source/lib/omnitrace/library/causal/components/causal_gotcha.cpp

@@ -26,6 +26,7 @@
 #include "library/causal/components/unblocking_gotcha.hpp"
 
 #include <timemory/backends/threading.hpp>
+#include <timemory/signals/signal_mask.hpp>
 #include <timemory/utility/macros.hpp>
 #include <timemory/utility/types.hpp>
 
@@ -40,6 +41,8 @@ namespace component
 {
 namespace
 {
+namespace signals = ::tim::signals;
+
 using bundle_t = tim::lightweight_tuple<blocking_gotcha_t, unblocking_gotcha_t>;
 
 auto&
@@ -50,6 +53,13 @@ get_bundle()
     return _v;
 }
 
+const auto&
+sampling_signals()
+{
+    static auto _v = get_sampling_signals();
+    return _v;
+}
+
 bool is_configured = false;
 }  // namespace
 
@@ -90,6 +100,31 @@ causal_gotcha::stop()
     get_bundle()->stop();
     shutdown();
 }
+
+void
+causal_gotcha::block_signals()
+{
+    signals::block_signals(sampling_signals(), signals::sigmask_scope::thread);
+}
+
+void
+causal_gotcha::unblock_signals()
+{
+    signals::unblock_signals(sampling_signals(), signals::sigmask_scope::thread);
+}
+
+void
+causal_gotcha::remove_signals(sigset_t* _set)
+{
+    for(auto _sig : sampling_signals())
+    {
+        if(sigismember(_set, _sig) != 0) sigdelset(_set, _sig);
+    }
+
+    if(sigismember(_set, SIGSEGV) != 0) sigdelset(_set, SIGSEGV);
+
+    if(sigismember(_set, SIGABRT) != 0) sigdelset(_set, SIGABRT);
+}
 }  // namespace component
 }  // namespace causal
 }  // namespace omnitrace

source/lib/omnitrace/library/causal/components/causal_gotcha.hpp

@@ -48,6 +48,10 @@ struct causal_gotcha : tim::component::base<causal_gotcha, void>
 
     static void start();
     static void stop();
+
+    static void block_signals();
+    static void unblock_signals();
+    static void remove_signals(sigset_t*);
 };
 }  // namespace component
 }  // namespace causal

source/lib/omnitrace/library/causal/data.cpp

@@ -424,7 +424,13 @@ void
 compute_eligible_lines()
 {
     static auto _once = std::once_flag{};
-    std::call_once(_once, compute_eligible_lines_impl);
+    std::call_once(_once, []() {
+        compute_eligible_lines_impl();
+        auto _cfg         = settings::compose_filename_config{};
+        _cfg.subdirectory = "causal/binary-info";
+        _cfg.use_suffix   = config::get_use_pid();
+        save_line_info(_cfg, config::get_verbose());
+    });
 }
 
 void
@@ -435,7 +441,7 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started)  // NOLINT
     using duration_sec_t  = std::chrono::duration<double, std::ratio<1>>;
 
     const auto& _thr_info = thread_info::init(true);
-    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
+    set_thread_state(ThreadState::Disabled);
     OMNITRACE_CONDITIONAL_THROW(!_thr_info->is_offset,
                                 "Error! causal profiling thread should be offset");
 
@@ -919,11 +925,6 @@ start_experimenting()
 
     compute_eligible_lines();
 
-    auto _cfg         = settings::compose_filename_config{};
-    _cfg.subdirectory = "causal/binary-info";
-    _cfg.use_suffix   = config::get_use_pid();
-    save_line_info(_cfg, config::get_verbose());
-
     if(get_state() < State::Finalized)
     {
         OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);

source/lib/omnitrace/library/causal/delay.cpp

@@ -23,6 +23,7 @@
 #include "library/causal/delay.hpp"
 #include "core/state.hpp"
 #include "core/utility.hpp"
+#include "library/causal/components/causal_gotcha.hpp"
 #include "library/causal/experiment.hpp"
 #include "library/runtime.hpp"
 #include "library/thread_data.hpp"
@@ -108,10 +109,12 @@ delay::process()
         }
         else if(get_global() > get_local())
         {
+            ::omnitrace::causal::component::causal_gotcha::block_signals();
             auto _beg = tracing::now();
             std::this_thread::sleep_for(
                 std::chrono::nanoseconds{ get_global() - get_local() });
             get_local() += (tracing::now() - _beg);
+            ::omnitrace::causal::component::causal_gotcha::unblock_signals();
         }
     }
     else

source/lib/omnitrace/library/causal/sampling.cpp

@@ -25,6 +25,7 @@
 #include "core/concepts.hpp"
 #include "core/config.hpp"
 #include "core/debug.hpp"
+#include "core/locking.hpp"
 #include "core/state.hpp"
 #include "core/utility.hpp"
 #include "library/causal/components/backtrace.hpp"
@@ -172,12 +173,10 @@ causal_offload_buffer(int64_t, causal_sampler_buffer_t&& _buf)
 
     if(!_processed.empty())
     {
-        tasking::general::get_task_group().exec([_processed]() {
-            static std::mutex _mutex;
-            auto              _lk = std::scoped_lock<std::mutex>{ _mutex };
-            for(const auto& itr : _processed)
-                add_samples(itr.first, itr.second);
-        });
+        static auto _mutex = locking::atomic_mutex{};
+        auto        _lk    = locking::atomic_lock{ _mutex };
+        for(const auto& itr : _processed)
+            add_samples(itr.first, itr.second);
     }
 }
 
@@ -252,11 +251,16 @@ configure(bool _setup, int64_t _tid)
 
             for(int64_t i = 1; i < OMNITRACE_MAX_THREADS; ++i)
             {
-                if(get_causal_sampler(i)) get_causal_sampler(i)->reset();
+                if(get_causal_sampler(i))
+                {
+                    get_causal_sampler(i)->stop();
+                    get_causal_sampler(i)->reset();
+                }
             }
         }
 
         _causal->stop();
+        _causal->reset();
 
         OMNITRACE_DEBUG("Causal sampler destroyed for thread %lu\n", _tid);
     }
@@ -293,11 +297,13 @@ void
 block_samples()
 {
     trait::runtime_enabled<causal_sampler_t>::set(false);
+    trait::runtime_enabled<causal::component::backtrace>::set(false);
 }
 
 void
 unblock_samples()
 {
+    trait::runtime_enabled<causal::component::backtrace>::set(true);
     trait::runtime_enabled<causal_sampler_t>::set(true);
 }
 
@@ -327,6 +333,8 @@ post_process()
     OMNITRACE_VERBOSE(2 || get_debug_sampling(),
                       "Stopping causal sampling components...\n");
 
+    block_samples();
+
     for(size_t i = 0; i < max_supported_threads; ++i)
     {
         auto& _causal = get_causal_sampler(i);

tests/validate-causal-json.py

@@ -4,6 +4,7 @@ import os
 import re
 import sys
 import json
+import math
 import argparse
 from collections import OrderedDict
 
@@ -31,7 +32,15 @@ class validation(object):
         self.program_speedup = float(_expected)
         self.tolerance = float(_tolerance)
 
-    def validate(self, _exp_name, _pp_name, _virt_speedup, _prog_speedup):
+    def validate(
+        self,
+        _exp_name,
+        _pp_name,
+        _virt_speedup,
+        _prog_speedup,
+        _prog_speedup_stddev,
+        _base_speedup_stddev,
+    ):
         if (
             not re.search(self.experiment_filter, _exp_name)
             or not re.search(self.progress_pt_filter, _pp_name)
@@ -39,9 +48,21 @@ class validation(object):
         ):
             return None
 
-        return _prog_speedup >= (
-            self.program_speedup - self.tolerance
-        ) and _prog_speedup <= (self.program_speedup + self.tolerance)
+        _tolerance = self.tolerance
+        if _base_speedup_stddev > 2.0 * self.tolerance:
+            sys.stderr.write(
+                f"  [{_exp_name}][{_pp_name}][{_virt_speedup}] base speedup has stddev > 2 * tolerance (+/- {_base_speedup_stddev:.3f}). Relaxing validation...\n"
+            )
+            _tolerance += math.sqrt(_base_speedup_stddev)
+        elif _prog_speedup_stddev > 2.0 * self.tolerance:
+            sys.stderr.write(
+                f"  [{_exp_name}][{_pp_name}][{_virt_speedup}] program speedup has stddev > 2 * tolerance (+/- {_prog_speedup_stddev:.3f}). Relaxing validation...\n"
+            )
+            _tolerance += math.sqrt(_prog_speedup_stddev)
+
+        return _prog_speedup >= (self.program_speedup - _tolerance) and _prog_speedup <= (
+            self.program_speedup + _tolerance
+        )
 
 
 class experiment_data(object):
@@ -360,6 +381,8 @@ def main():
         print("")
         print(f"{itr}")
 
+    sys.stdout.flush()
+
     validations = get_validations(args)
 
     expected_validations = len(validations)
@@ -369,12 +392,19 @@ def main():
         for eitr in results:
             _experiment = eitr.data[0].get_name()
             _progresspt = eitr.data[0].prog
+            _base_speedup_stddev = eitr.data[0].compute_speedup_stddev()
             for ditr in eitr.data:
                 _virt_speedup = ditr.virtual_speedup()
                 _prog_speedup = ditr.compute_speedup()
+                _prog_speedup_stddev = ditr.compute_speedup_stddev()
                 for vitr in validations:
                     _v = vitr.validate(
-                        _experiment, _progresspt, _virt_speedup, _prog_speedup
+                        _experiment,
+                        _progresspt,
+                        _virt_speedup,
+                        _prog_speedup,
+                        _prog_speedup_stddev,
+                        _base_speedup_stddev,
                     )
                     if _v is None:
                         continue