Signal handler backtraces provide line info (#178)

* Signal handler backtraces provide line info

- print backtrace after SIGINT during finalization

* Workflow run-name + jammy rocm CI

* fix jammy matrix indentation

* disable building dyninst in jammy

* Update jammy for rocm

* jammy rocm_agent_enumerator

* Fix rocm install for jammy

* jammy bash

* jammy workflow typo

* revert some changes

* stack-usage + omnitrace-rt symlink + ncclSocketAccept + indiv sigs

- symlink omnitrace-rt in build tree
- exclude ncclSocketAccept
- timemory submodule update accepting individual signal handlers

source/bin/omnitrace/CMakeLists.txt

@@ -30,7 +30,8 @@ target_link_libraries(
             omnitrace::omnitrace-compile-definitions
             omnitrace::omnitrace-sanitizer
             timemory::timemory-headers
-            timemory::timemory-extensions)
+            timemory::timemory-extensions
+            timemory::timemory-core)
 
 set_target_properties(
     omnitrace-exe

source/bin/omnitrace/details.cpp

@@ -61,7 +61,7 @@ get_whole_function_names()
         "ncclCommUserRank", "ncclReduce", "ncclBcast", "ncclBroadcast", "ncclAllReduce",
         "ncclReduceScatter", "ncclAllGather", "ncclGroupStart", "ncclGroupEnd",
         "ncclSend", "ncclRecv", "ncclGather", "ncclScatter", "ncclAllToAll",
-        "ncclAllToAllv"
+        "ncclAllToAllv", "ncclSocketAccept"
     };
 }
 

source/bin/omnitrace/log.cpp

@@ -91,7 +91,8 @@ log_entry::add_log_entry(log_entry&& _v)
 
 void
 print_log_entries(std::ostream& _os, int64_t _count,
-                  std::function<bool(const log_entry&)> _condition, const char* _color,
+                  const std::function<bool(const log_entry&)>& _condition,
+                  const std::function<void()>& _prelude, const char* _color,
                   bool _color_entries)
 {
     size_t i0 = (_count < 0) ? 0 : std::max<int64_t>(log_entries.size() - _count, 0);
@@ -107,6 +108,18 @@ print_log_entries(std::ostream& _os, int64_t _count,
     const char* _end =
         (strlen(_color) > 0 || _color_entries) ? tim::log::color::end() : "";
 
+    if(_prelude)
+    {
+        for(size_t i = i0; i < log_entries.size(); ++i)
+        {
+            if(!_condition || _condition(log_entries.at(i)))
+            {
+                _prelude();
+                break;
+            }
+        }
+    }
+
     // the requested number of log entries
     for(size_t i = i0; i < log_entries.size(); ++i)
     {

source/bin/omnitrace/log.hpp

@@ -39,7 +39,8 @@ struct log_entry;
 
 void
 print_log_entries(std::ostream& = std::cerr, int64_t _count = 10,
-                  std::function<bool(const log_entry&)> _cond = {},
+                  const std::function<bool(const log_entry&)>& _cond    = {},
+                  const std::function<void()>&                 _prelude = {},
                   const char* _color         = tim::log::color::warning(),
                   bool        _color_entries = true);
 

source/bin/omnitrace/omnitrace.cpp

@@ -212,16 +212,24 @@ main(int argc, char** argv)
               sys_signal::SegFault, sys_signal::FileSize, sys_signal::CPUtime })
             signal_settings::enable(itr);
 
-        auto _exit_action = [](int nsig) {
+        static bool _protect     = false;
+        auto        _exit_action = [](int nsig) {
+            if(_protect) return;
+            _protect = true;
             TIMEMORY_PRINTF_FATAL(
                 stderr, "omnitrace exited with signal %i :: %s\n", nsig,
                 signal_settings::str(static_cast<sys_signal>(nsig)).c_str());
 
+            // print the last log entries
             print_log_entries(std::cerr, num_log_entries);
 
-            std::cerr << "\n[omnitrace][exe] Potentially important log entries\n\n";
-
-            print_log_entries(std::cerr, -1, [](const auto& _v) { return _v.forced(); });
+            // print any forced entries
+            print_log_entries(
+                std::cerr, -1, [](const auto& _v) { return _v.forced(); },
+                []() {
+                    tim::log::stream(std::cerr, tim::log::color::info())
+                        << "\n[omnitrace][exe] Potentially important log entries:\n\n";
+                });
 
             TIMEMORY_PRINTF_FATAL(stderr, "\n");
             TIMEMORY_PRINTF_FATAL(
@@ -235,6 +243,7 @@ main(int argc, char** argv)
             log_ofs.reset();
 
             kill(process::get_id(), nsig);
+            _protect = false;
         };
 
         signal_settings::set_exit_action(_exit_action);
@@ -1064,7 +1073,7 @@ main(int argc, char** argv)
         if(!tim::filepath::open(*log_ofs, logfile))
             throw std::runtime_error(JOIN(" ", "Error opening log output file", logfile));
         verbprintf_bare(0, "Done\n%s", ::tim::log::color::end());
-        print_log_entries(*log_ofs, -1, {}, "", false);
+        print_log_entries(*log_ofs, -1, {}, {}, "", false);
     }
 
     //----------------------------------------------------------------------------------//

source/lib/omnitrace/library.cpp

@@ -51,6 +51,8 @@
 #include "library/timemory.hpp"
 #include "library/tracing.hpp"
 
+#include <timemory/signals/signal_handlers.hpp>
+#include <timemory/signals/types.hpp>
 #include <timemory/hash/types.hpp>
 #include <timemory/manager/manager.hpp>
 #include <timemory/operations/types/file_output_message.hpp>
@@ -630,6 +632,9 @@ omnitrace_finalize_hidden(void)
     OMNITRACE_VERBOSE_F(1, "omnitrace_push_trace :: called %zux\n", _push_count);
     OMNITRACE_VERBOSE_F(1, "omnitrace_pop_trace  :: called %zux\n", _pop_count);
 
+    tim::signals::enable_signal_detection({ tim::signals::sys_signal::Interrupt },
+                                          [](int) {});
+
     OMNITRACE_DEBUG_F("Copying over all timemory hash information to main thread...\n");
     // copy these over so that all hashes are known
     auto& _hzero = tracing::get_timemory_hash_ids(0);

source/lib/omnitrace/library/debug.hpp

@@ -30,6 +30,7 @@
 #include <timemory/backends/threading.hpp>
 #include <timemory/log/logger.hpp>
 #include <timemory/mpl/concepts.hpp>
+#include <timemory/signals/signal_handlers.hpp>
 #include <timemory/utility/backtrace.hpp>
 #include <timemory/utility/locking.hpp>
 #include <timemory/utility/utility.hpp>