Submitting jobs to cdash (#124)

* Submitting jobs to cdash

* Fail on submit

* submit url env

* submit url env

* try passing submit url as arg

* fix submit url

* Updated default URL

* Add submissions for remaining ubuntu focal workflow jobs

* Replace g++ with gcc in dashboard build name

* Add --ctest-args to run-ci.sh

* Add cdash support for bionic, jammy, and opensuse workflows

* Decrease CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE

* OMNITRACE_BUILD_CODECOV option

* Support code coverage in CDash script

* CI dyninst built with debug info

* Update ci-containers

- cron schedule moved 4 hours later to UTC+5

* Update implementation of config::configure_signal_handler

- using lambdas failed to compile with codecov flags

* Add codecov job to ubuntu focal workflow

* Fix support for --ctest-args in run-ci script

* Fix ubuntu workflows

* Fix quotation handling in run-ci script

* git safe directory for codecov

* New MPI examples

* Remove --stop-on-failure

* dynamic_library update

- find_library_path checks procfs maps
- invoke find_library_path with no additional args to resolve to mapped file

* RCCLP uses dynamic_library

* check if file exists for memory_map_files metadata

* Testing updates

- include new mpi examples in tests
- fix test labels
- test critical-trace exe

* Update MPI C examples tests (needed arg)

* Remove try/catch block from critical-trace

* Fix sampling max wait when shutting down

* Fix test env for critical-trace

* Fix settings for critical-trace

- disable time output: data is deterministic
- disable PID suffixes: not multiprocess

* Update critical-trace ctest

* Update critical-trace exe

- throw error if input cannot be opened
- throw error if input has no data

* Update lulesh example with more kokkos tools usage

* Fix tasking issue with critical_trace and roctracer

- were not setting pools to active
- also sync before critical_trace::get_entries

* Increase verbosity of critical-trace tests

* Update code coverage tests

- skip code coverage + preload
- code-coverage python example and test

* Remove duplication omnitrace.initialize function

* Skip python3.6 for ubuntu jammy

* Update MPI examples

- use MPI_Isend and MPI_Irecv
- explicitly use MPI_Bcast

* Update Formatting.cmake

- include C files in examples

* run-ci script does not check return of coverage

* mpi-allreduce link to libm

* Update ctest args in run-ci script

* Update dyninst submodule

- safety improvements in BinaryEdit::openResolvedLibraryName

* capture cmake error for ctest_coverage

source/bin/omnitrace-critical-trace/critical-trace.cpp

@@ -24,6 +24,7 @@
 
 #include "api.hpp"
 #include "library/config.hpp"
+#include "library/debug.hpp"
 #include "library/perfetto.hpp"
 
 #include <timemory/hash/types.hpp>
@@ -57,6 +58,8 @@ main(int argc, char** argv)
     config::set_setting_value<int64_t>("OMNITRACE_THREAD_POOL_SIZE",
                                        std::thread::hardware_concurrency());
     config::set_setting_value("OMNITRACE_CRITICAL_TRACE_SERIALIZE_NAMES", true);
+    config::set_setting_value("OMNITRACE_USE_PID", false);
+    config::set_setting_value("OMNITRACE_TIME_OUTPUT", false);
 
     if(config::get_verbose() >= 0)
     {
@@ -70,8 +73,12 @@ main(int argc, char** argv)
     {
         critical_trace::complete_call_chain = {};
         OMNITRACE_BASIC_PRINT_F("Loading call-chain %s...\n", argv[i]);
-        critical_trace::load_call_chain(argv[i], "call_chain",
-                                        critical_trace::complete_call_chain);
+        if(!critical_trace::load_call_chain(argv[i], "call_chain",
+                                            critical_trace::complete_call_chain))
+        {
+            OMNITRACE_THROW("Error loading '%s'. Data size: %zu\n", argv[i],
+                            critical_trace::complete_call_chain.size());
+        }
         for(const auto& itr : *tim::get_hash_ids())
             critical_trace::complete_hash_ids.emplace(itr.second);
         OMNITRACE_BASIC_PRINT_F("Computing critical trace for %s...\n", argv[i]);
@@ -369,22 +376,28 @@ save_call_chain_json(const std::string& _fname, const std::string& _label,
     }
 }
 
-void
+bool
 load_call_chain(const std::string& _fname, const std::string& _label,
                 call_chain& _call_chain)
 {
+    namespace cereal = tim::cereal;
+
     std::ifstream ifs{};
     ifs.open(_fname);
-    if(ifs && ifs.is_open())
-    {
-        namespace cereal = tim::cereal;
-        auto ar          = tim::policy::input_archive<cereal::JSONInputArchive>::get(ifs);
 
-        ar->setNextName("omnitrace");
-        ar->startNode();
-        (*ar)(cereal::make_nvp(_label.c_str(), _call_chain));
-        ar->finishNode();
-    }
+    OMNITRACE_CONDITIONAL_THROW(!ifs || !ifs.is_open(),
+                                "Error! call-chain file '%s' could not be opened",
+                                _fname.c_str());
+
+    auto ar   = tim::policy::input_archive<cereal::JSONInputArchive>::get(ifs);
+    auto _val = call_chain{};
+    ar->setNextName("omnitrace");
+    ar->startNode();
+    (*ar)(cereal::make_nvp(_label.c_str(), _val));
+    ar->finishNode();
+    auto _success = (_val.empty() == false);
+    if(_success) std::swap(_call_chain, _val);
+    return _success;
 }
 
 auto
@@ -484,6 +497,7 @@ find_children(PTL::ThreadPool& _tp, call_graph_t& _graph, const call_chain& _cha
         return !_data.empty();
     };
 
+    OMNITRACE_CT_DEBUG_F("Checking index at -1...\n");
     if(!_indexed.at(-1).empty())
     {
         OMNITRACE_CT_DEBUG_F("Setting root (line %i)...\n", __LINE__);
@@ -856,7 +870,7 @@ compute_critical_trace()
 
     OMNITRACE_BASIC_PRINT("\n");
 
-    try
+    // try
     {
         PTL::ThreadPool _tp{ get_thread_pool_size(), []() { copy_hash_ids(); }, []() {} };
         _tp.set_verbose(-1);
@@ -970,11 +984,11 @@ compute_critical_trace()
 
         _tg.join();
         _tp.destroy_threadpool();
-    } catch(std::exception& e)
+    }  // catch(std::exception& e)
     {
-        OMNITRACE_BASIC_PRINT("Thread exited '%s' with exception: %s\n", __FUNCTION__,
-                              e.what());
-        TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(true, 32);
+        // OMNITRACE_BASIC_PRINT("Thread exited '%s' with exception: %s\n", __FUNCTION__,
+        //                      e.what());
+        // TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(true, 32);
     }
 
     _report_perf(_ct_perf, __FUNCTION__, "critical trace computation");

source/bin/omnitrace-critical-trace/critical-trace.hpp

@@ -65,7 +65,7 @@ std::mutex complete_call_mutex{};
 void
 update_critical_path(call_chain _chain, int64_t _tid);
 
-void
+bool
 load_call_chain(const std::string& _fname, const std::string& _label,
                 call_chain& _call_chain);
 

source/lib/omnitrace/library.cpp

@@ -818,7 +818,6 @@ omnitrace_finalize_hidden(void)
     if(get_use_critical_trace() || (get_use_rocm_smi() && get_use_roctracer()))
     {
         OMNITRACE_VERBOSE_F(1, "Generating the critical trace...\n");
-
         for(size_t i = 0; i < max_supported_threads; ++i)
         {
             using critical_trace_hash_data =
@@ -856,10 +855,10 @@ omnitrace_finalize_hidden(void)
     if(get_use_critical_trace())
     {
         // launch compute task
-        OMNITRACE_VERBOSE_F(1, "launching critical trace compute task...\n");
+        OMNITRACE_VERBOSE_F(1, "Launching critical trace compute task...\n");
         critical_trace::compute();
 
-        OMNITRACE_VERBOSE_F(1, "Waiting on critical trace tasks...\n");
+        OMNITRACE_VERBOSE_F(1, "Waiting on critical trace computation...\n");
         tasking::join();
     }
 
@@ -980,7 +979,8 @@ omnitrace_finalize_hidden(void)
             for(auto& itr : _maps)
             {
                 auto&& _path = itr.pathname;
-                if(!_path.empty() && _path.at(0) != '[') _libs.emplace(_path);
+                if(!_path.empty() && _path.at(0) != '[' && filepath::exists(_path))
+                    _libs.emplace(_path);
             }
             ar(tim::cereal::make_nvp("memory_maps_files", _libs),
                tim::cereal::make_nvp("memory_maps", _maps));

source/lib/omnitrace/library/config.cpp

@@ -941,6 +941,38 @@ configure_mode_settings()
     }
 }
 
+namespace
+{
+using signal_settings = tim::signals::signal_settings;
+using sys_signal      = tim::signals::sys_signal;
+
+void
+omnitrace_exit_action(int nsig)
+{
+    tim::signals::block_signals(get_sampling_signals(),
+                                tim::signals::sigmask_scope::process);
+    OMNITRACE_BASIC_PRINT("Finalizing afer signal %i :: %s\n", nsig,
+                          signal_settings::str(static_cast<sys_signal>(nsig)).c_str());
+    if(get_state() == State::Active) omnitrace_finalize();
+    kill(process::get_id(), nsig);
+}
+
+void
+omnitrace_trampoline_handler(int _v)
+{
+    if(get_verbose_env() >= 1)
+    {
+        ::omnitrace::debug::flush();
+        ::omnitrace::debug::lock _debug_lk{};
+        OMNITRACE_FPRINTF_STDERR_COLOR(warning);
+        fprintf(::omnitrace::debug::get_file(),
+                "signal %i ignored (OMNITRACE_IGNORE_DYNINST_TRAMPOLINE=ON)\n", _v);
+        ::omnitrace::debug::flush();
+        timemory_print_demangled_backtrace<64>();
+    }
+}
+}  // namespace
+
 void
 configure_signal_handler()
 {
@@ -953,19 +985,8 @@ configure_signal_handler()
 
     if(_config->get_enable_signal_handler())
     {
-        using signal_settings = tim::signals::signal_settings;
-        using sys_signal      = tim::signals::sys_signal;
         tim::signals::disable_signal_detection();
-        auto _exit_action = [](int nsig) {
-            tim::signals::block_signals(get_sampling_signals(),
-                                        tim::signals::sigmask_scope::process);
-            OMNITRACE_BASIC_PRINT(
-                "Finalizing afer signal %i :: %s\n", nsig,
-                signal_settings::str(static_cast<sys_signal>(nsig)).c_str());
-            if(get_state() == State::Active) omnitrace_finalize();
-            kill(process::get_id(), nsig);
-        };
-        signal_settings::set_exit_action(_exit_action);
+        signal_settings::set_exit_action(omnitrace_exit_action);
         signal_settings::check_environment();
         auto default_signals = signal_settings::get_default();
         for(const auto& itr : default_signals)
@@ -978,22 +999,11 @@ configure_signal_handler()
 
     if(_ignore_dyninst_trampoline)
     {
-        using signal_handler_t          = void (*)(int);
-        static auto _trampoline_handler = [](int _v) {
-            if(get_verbose_env() >= 2)
-            {
-                ::omnitrace::debug::flush();
-                ::omnitrace::debug::lock _debug_lk{};
-                OMNITRACE_FPRINTF_STDERR_COLOR(warning);
-                fprintf(::omnitrace::debug::get_file(),
-                        "signal %i ignored (OMNITRACE_IGNORE_DYNINST_TRAMPOLINE=ON)\n",
-                        _v);
-                ::omnitrace::debug::flush();
-                timemory_print_demangled_backtrace<64>();
-            }
-        };
-        ::signal(_dyninst_trampoline_signal,
-                 static_cast<signal_handler_t>(_trampoline_handler));
+        struct sigaction _action;
+        sigemptyset(&_action.sa_mask);
+        _action.sa_flags   = {};
+        _action.sa_handler = omnitrace_trampoline_handler;
+        sigaction(_dyninst_trampoline_signal, &_action, nullptr);
     }
 }
 

source/lib/omnitrace/library/critical_trace.cpp

@@ -904,6 +904,7 @@ std::vector<std::pair<std::string, entry>>
 get_entries(const std::function<bool(const entry&)>& _eval)
 {
     OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
+    tasking::join();
     copy_hash_ids();
     squash_critical_path(complete_call_chain);
     std::sort(complete_call_chain.begin(), complete_call_chain.end());

source/lib/omnitrace/library/dynamic_library.cpp

@@ -28,12 +28,15 @@
 #include <timemory/environment.hpp>
 #include <timemory/utility/delimit.hpp>
 #include <timemory/utility/filepath.hpp>
+#include <timemory/utility/procfs/maps.hpp>
 
 #include <string>
 #include <utility>
 
 namespace omnitrace
 {
+namespace procfs = ::tim::procfs;
+
 std::string
 find_library_path(const std::string& _name, const std::vector<std::string>& _env_vars,
                   const std::vector<std::string>& _hints,
@@ -41,6 +44,13 @@ find_library_path(const std::string& _name, const std::vector<std::string>& _env
 {
     if(_name.find('/') == 0) return _name;
 
+    for(const auto& itr : procfs::get_maps(process::get_id(), true))
+    {
+        auto&& _path = itr.pathname;
+        if(_path.find(_name) != std::string::npos && filepath::exists(_path))
+            return _path;
+    }
+
     auto _paths = std::vector<std::string>{};
     for(const std::string& itr : _env_vars)
     {
@@ -74,6 +84,9 @@ dynamic_library::dynamic_library(std::string _env, std::string _fname, int _flag
 , filename{ std::move(_fname) }
 , flags{ _flags }
 {
+    // check the memory maps
+    filename = find_library_path(filename, {}, {});
+
     if(_query_env)
     {
         auto _env_val = get_env(envname, std::string{}, _store);

source/lib/omnitrace/library/kokkosp.cpp

@@ -161,7 +161,9 @@ extern "C"
                 for(auto& itr : _maps)
                 {
                     auto&& _path = itr.pathname;
-                    if(!_path.empty() && _path.at(0) != '[') _libs.emplace(_path);
+                    if(!_path.empty() && _path.at(0) != '[' &&
+                       omnitrace::filepath::exists(_path))
+                        _libs.emplace(_path);
                 }
                 for(const auto& itr : _libs)
                 {

source/lib/omnitrace/library/ptl.cpp

@@ -137,6 +137,10 @@ join()
         for(size_t i = 0; i < max_supported_threads; ++i)
             roctracer::get_task_group(i).join();
     }
+    else
+    {
+        OMNITRACE_DEBUG_F("roctracer thread-pool is not active...\n");
+    }
 
     if(critical_trace::get_thread_pool_state() == State::Active)
     {
@@ -144,6 +148,10 @@ join()
         for(size_t i = 0; i < max_supported_threads; ++i)
             critical_trace::get_task_group(i).join();
     }
+    else
+    {
+        OMNITRACE_DEBUG_F("critical-trace thread-pool is not active...\n");
+    }
 }
 
 void
@@ -160,6 +168,10 @@ shutdown()
         }
         roctracer::get_thread_pool_state() = State::Finalized;
     }
+    else
+    {
+        OMNITRACE_DEBUG_F("roctracer thread-pool is not active...\n");
+    }
 
     if(critical_trace::get_thread_pool_state() == State::Active)
     {
@@ -172,6 +184,10 @@ shutdown()
         }
         critical_trace::get_thread_pool_state() = State::Finalized;
     }
+    else
+    {
+        OMNITRACE_DEBUG_F("critical-trace thread-pool is not active...\n");
+    }
 
     if(get_thread_pool_state() == State::Active)
     {
@@ -179,6 +195,10 @@ shutdown()
         get_thread_pool().destroy_threadpool();
         get_thread_pool_state() = State::Finalized;
     }
+    else
+    {
+        OMNITRACE_DEBUG_F("thread-pool is not active...\n");
+    }
 }
 
 size_t
@@ -194,7 +214,8 @@ roctracer::get_task_group(int64_t _tid)
     {};
     using thread_data_t = thread_data<PTL::TaskGroup<void>, local>;
     static auto& _v =
-        thread_data_t::instances(construct_on_init{}, &tasking::get_thread_pool());
+        (roctracer::get_thread_pool_state() = State::Active,
+         thread_data_t::instances(construct_on_init{}, &tasking::get_thread_pool()));
     return *_v.at(_tid);
 }
 
@@ -205,7 +226,8 @@ critical_trace::get_task_group(int64_t _tid)
     {};
     using thread_data_t = thread_data<PTL::TaskGroup<void>, local>;
     static auto& _v =
-        thread_data_t::instances(construct_on_init{}, &tasking::get_thread_pool());
+        (critical_trace::get_thread_pool_state() = State::Active,
+         thread_data_t::instances(construct_on_init{}, &tasking::get_thread_pool()));
     return *_v.at(_tid);
 }
 }  // namespace tasking

source/lib/omnitrace/library/rcclp.cpp

@@ -26,6 +26,7 @@
 #include "library/components/category_region.hpp"
 #include "library/components/fwd.hpp"
 #include "library/defines.hpp"
+#include "library/dynamic_library.hpp"
 #include "library/timemory.hpp"
 
 #include <timemory/timemory.hpp>
@@ -42,8 +43,10 @@
 #include <set>
 #include <unordered_map>
 
-static uint64_t global_id      = std::numeric_limits<uint64_t>::max();
-static void*    librccl_handle = nullptr;
+namespace
+{
+uint64_t global_id = std::numeric_limits<uint64_t>::max();
+}
 
 namespace omnitrace
 {
@@ -59,10 +62,9 @@ setup()
     configure();
 
     // make sure the symbols are loaded to be wrapped
-    auto libpath   = tim::get_env<std::string>("OMNITRACE_RCCL_LIBRARY", "librccl.so");
-    librccl_handle = dlopen(libpath.c_str(), RTLD_NOW | RTLD_GLOBAL);
-    if(!librccl_handle) fprintf(stderr, "%s\n", dlerror());
-    dlerror();  // Clear any existing error
+    dynamic_library _librccl{
+        "OMNITRACE_RCCL_LIBRARY", "librccl.so", RTLD_NOW | RTLD_GLOBAL, true, true, true
+    };
 
     auto _use_data = tim::get_env("OMNITRACE_RCCLP_COMM_DATA", get_use_timemory());
     if(!get_use_timemory())
@@ -78,7 +80,6 @@ setup()
 
     component::configure_rcclp();
     global_id = component::activate_rcclp();
-    if(librccl_handle) dlclose(librccl_handle);
 }
 
 void

source/lib/omnitrace/library/sampling.cpp

@@ -421,7 +421,7 @@ configure(bool _setup, int64_t _tid)
             auto _freq =
                 std::max<double>(get_sampling_cpu_freq(), get_sampling_real_freq());
             auto _period = (1.0 / _freq) * units::sec;
-            _period      = std::max<double>(_period, 1.0e9);  // max of 1 second
+            _period      = std::min<double>(_period, 1.0e9);  // max of 1 second
             std::this_thread::sleep_for(
                 std::chrono::nanoseconds{ static_cast<int64_t>(_period) });
 

source/python/libpyomnitrace.cpp

@@ -136,26 +136,6 @@ PYBIND11_MODULE(libpyomnitrace, omni)
         },
         "Initialize omnitrace");
 
-    omni.def(
-        "initialize",
-        [](const std::string& _v) {
-            if(_is_initialized)
-                throw std::runtime_error("Error! omnitrace is already initialized");
-            _is_initialized = true;
-            bool _use_mpi   = false;
-            try
-            {
-                py::module::import("mpi4py");
-                _use_mpi = true;
-            } catch(py::error_already_set& _exc)
-            {
-                if(!_exc.matches(PyExc_ImportError)) throw;
-            }
-            omnitrace_set_mpi(_use_mpi, false);
-            omnitrace_init("trace", false, _v.c_str());
-        },
-        "Initialize omnitrace");
-
     omni.def(
         "finalize",
         []() {