Add RedHat CI and release packaging (#251)

- additional miscellaneous tweaks to workflows and docker scripts, e.g. install perfetto python bindings - improves the stability of MPI finalization - reduces some debug messages within timemory when `OMNITRACE_DEBUG=ON` - fixes issue found in RHEL where libunwind is using mutex and omnitrace was not treating this as an internal mutex call - this may have been affecting the causal profiling slightly (tests seem a bit more stable now) - fix data race in timemory * Add RedHat CI and release packaging - additional miscellaneous tweaks to workflows and docker scripts, e.g. install perfetto python bindings * Fix URL for ROCm packages in redhat workflow * Fix dnf --enable-repo for ROCm perl packages * Dockerfile.rhel and redhat.yml updates - Fix dnf repo for ROCm PERL packages - Disable python in CI (interpreter segfaults) - Exclude parallel-overhead-locks tests due to inclusion of internal locks - This needs to be remedied in the future * Exclude _dl_relocate_static_pie from instrumentation * Testing updates - OMNITRACE_SAMPLING_KEEP_INTERNAL=OFF for parallel-overhead-locks * Fix redhat workflow * redhat.yml update - remove if condition on config/build/test step * Update timemory submodule - tweaks to verbosity messages * Set thread state before unw_step - on Redhat, unw_step calls mutex * Update timemory submodule - verbosity changes - gotcha uses spin_lock/spin_mutex * Remove using gsplit-dwarf unless OMNITRACE_BUILD_NUMBER > 2 * Re-enable parallel-overhead-locks tests in redhat workflow * Always disable timemory manager metadata auto output * testing updates - tweak parallel-overhead-locks-timemory to higher instruction count min - OMNITRACE_SAMPLING_KEEP_INTERNAL=OFF for parallel-overhead-locks-perfetto * Update timemory submodule - quiet realpath queries * omnitrace exe updates - detect text files - improved bin/lib locating * cmake format * test-install.sh and redhat workflow updates - handle testing when ls is script - re-enable python testing on redhat workflow - invoke test-install.sh in redhat workflow * Misc guards for finalization * omnitrace-exe, testing updates - test-install.sh: LS_EXEC -> LS_NAME - handle /usr/bin/ls being script in source/bin/tests - improve locating the binary * Fix mpi_gotcha compile error * omnitrace-exe updates - improve file locating * formatting * Misc fixes - remove -static-libstdc++ for RHEL packaging (rocky-linux doesn't distribute static lib) * omnitrace-exe paths * Replace realpath with absolute - using absolute path to symlink fixes issues with locating libdyninstAPI_RT at runtime * omnitrace exe updates - judicious use of realpath * Update timemory submodule - fix update main hash ids/aliases data race in merge * bin tests update - change working directory of omnitrace-exe-simulate-lib-basename * omnitrace exe updates - Update resolved exe/lib messaging * bin tests update - change working directory of omnitrace-exe-simulate-lib-basename [ROCm/rocprofiler-systems commit: 1688a027d8]
2023-03-07 06:04:19 -06:00
parent 49851b05ae
commit 446fd36a93
30 changed files with 958 additions and 395 deletions
@@ -180,6 +180,8 @@ ensure_finalization(bool _static_init = false)
        OMNITRACE_DEBUG_F("\n");
    }

+    if(_timemory_manager) _timemory_manager->set_write_metadata(-1);
+
    return scope::destructor{ []() { omnitrace_finalize_hidden(); } };
 }

@@ -993,8 +995,6 @@ omnitrace_finalize_hidden(void)
               tim::cereal::make_nvp("memory_maps", _maps));
        });

-        _timemory_manager->set_write_metadata(-1);
-
        OMNITRACE_VERBOSE_F(1, "Finalizing timemory...\n");
        tim::timemory_finalize(_timemory_manager.get());

@@ -117,6 +117,8 @@ backtrace::sample(int _sig)
    }

    ++_protect_flag;
+    // on RedHat, the unw_step within get_unw_signal_frame_stack_raw involves a mutex lock
+    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
    m_index = causal::experiment::get_index();
    m_stack = get_unw_signal_frame_stack_raw<depth, ignore_depth>();

@@ -25,6 +25,7 @@
 #include "core/config.hpp"
 #include "core/debug.hpp"
 #include "core/perfetto.hpp"
+#include "core/state.hpp"
 #include "library/components/ensure_storage.hpp"
 #include "library/ptl.hpp"
 #include "library/runtime.hpp"
@@ -184,6 +185,9 @@ backtrace::size() const
 void
 backtrace::sample(int)
 {
+    // on RedHat, the unw_step within get_unw_stack involves a mutex lock
+    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
+
    using namespace tim::backtrace;
    constexpr bool   with_signal_frame = false;
    constexpr size_t ignore_depth      = 3;
@@ -128,10 +128,17 @@ void
 omnitrace_mpi_set_attr()
 {
 #if defined(TIMEMORY_USE_MPI)
+    auto _blocked = get_sampling_signals();
+    if(!_blocked.empty())
+        tim::signals::block_signals(_blocked, tim::signals::sigmask_scope::process);
+
    int _comm_key = -1;
    if(PMPI_Comm_create_keyval(&omnitrace_mpi_copy, &omnitrace_mpi_fini, &_comm_key,
                               nullptr) == MPI_SUCCESS)
        PMPI_Comm_set_attr(MPI_COMM_SELF, _comm_key, nullptr);
+
+    if(!_blocked.empty())
+        tim::signals::unblock_signals(_blocked, tim::signals::sigmask_scope::process);
 #endif
 }

@@ -263,7 +270,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
    tim::mpi::is_initialized_callback() = []() { return false; };
    tim::mpi::is_finalized()            = true;
 #else
-    if(is_root_process()) omnitrace_finalize_hidden();
+    if(is_root_process() && omnitrace::get_state() < omnitrace::State::Finalized)
+        omnitrace_finalize_hidden();
 #endif
 }

@@ -102,6 +102,13 @@ inline void
 stop_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args)
 {
    if(!get_use_timemory() && !get_use_perfetto()) return;
+
+    auto _main_manager = tim::manager::master_instance();
+    auto _this_manager = tim::manager::instance();
+    if(!_main_manager || !_this_manager || _main_manager->is_finalized() ||
+       _this_manager->is_finalized())
+        return;
+
    OMNITRACE_BASIC_VERBOSE_F(3, "stopping bundle '%s' in thread %li...\n",
                              _bundle.key().c_str(), _tid);
    if(get_use_timemory())