Handle fork in target application (#191)

* Always print PID in log messages

* omnitrace-dl updates

- omnitrace_preload does not call omnitrace_init or omnitrace_init_tooling
- omnitrace_preload will call omnitrace_set_mpi if OMNITRACE_USE_MPI
  or OMNITRACE_USE_MPIP in the env is true but not call it otherwise
  because doing so either overrides OMNITRACE_USE_PID (when true) or
  disable mpip from initialization (when false) and the MPI
  init can be caught later and override OMNITRACE_USE_PID

* config updates

- set_setting_value sets user update type
- remove volatile from get_settings_configured
- don't override settings::default_process_suffix
- don't kill process in omnitrace_exit_action
- set_state ignores updating state if >= State::Finalized

* Handle state > State::Finalized

* fork gotcha updates

- unsets LD_PRELOAD
- sets OMNITRACE_ROOT_PROCESS
- sets OMNITRACE_CHILD_PROCESS

* libomnitrace library.cpp updates

- basic_bundle for fini metrics
- handle finalization from child process

* sampling updates

- sampling::shutdown handles when child process

* Add example and test using fork

* Update run-ci script to support not submitting

* Tweak test envs

* Update build flags when codecov enabled

* remove unnecessary includes of sampling header

* Replace mpi copy/fini static lambda with free-funcs

* Update codecov job

* Fix OMPT segfaults after finalization

* Miscellaneous updates after rebase

* fixes for causal profiling

* revert some run-ci.sh changes

* Disable storing env in sampling::shutdown

* formatting fix

* Update timemory submodule

- fixed occasional synchronization issues with allocator offloading
- exclude protozero:: from internal samples

* improve root/child process detection

- avoid omnitrace_finalize in MPI when child process
- revert some testing tweaks

source/lib/core/config.cpp

@@ -227,20 +227,18 @@ finalize()
 bool
 settings_are_configured()
 {
-    volatile bool _v = _settings_are_configured();
-    return _v;
+    return _settings_are_configured();
 }
 
 void
 configure_settings(bool _init)
 {
-    volatile bool _v = _settings_are_configured();
-    if(_v) return;
-
     static bool _once = false;
     if(_once) return;
     _once = true;
 
+    if(settings_are_configured()) return;
+
     if(get_is_continuous_integration() && get_state() < State::Init)
     {
         timemory_print_demangled_backtrace<64>();
@@ -2192,9 +2190,11 @@ get_perfetto_output_filename()
         _ext = _val.substr(_pos_ext + 1);
         _val = _val.substr(0, _pos_ext);
     }
-    _val = settings::compose_output_filename(_val, _ext, settings::use_output_suffix(),
-                                             settings::default_process_suffix(), false,
-                                             _dir);
+
+    auto _cfg = settings::compose_filename_config{ settings::use_output_suffix(),
+                                                   settings::default_process_suffix(),
+                                                   false, _dir };
+    _val      = settings::compose_output_filename(_val, _ext, _cfg);
     if(!_val.empty() && _val.at(0) != '/')
         return settings::format(JOIN('/', "%env{PWD}%", _val), get_config()->get_tag());
     return _val;

source/lib/core/config.hpp

@@ -100,11 +100,16 @@ template <typename Tp>
 bool
 set_setting_value(const std::string& _name, Tp&& _v)
 {
+    auto _user_upd = tim::settings::update_type::user;
     auto _instance = tim::settings::shared_instance();
     auto _setting  = _instance->find(_name);
     if(_setting == _instance->end()) return false;
     if(!_setting->second) return false;
-    return _setting->second->set(std::forward<Tp>(_v));
+    auto& itr      = _setting->second;
+    auto  _upd     = itr->set_user_updated();
+    auto  _success = itr->set(std::forward<Tp>(_v), _user_upd);
+    if(!_success) itr->set_updated(_upd);
+    return _success;
 }
 
 template <typename Tp>

source/lib/core/debug.cpp

@@ -25,6 +25,7 @@
 #include "state.hpp"
 
 #include <timemory/log/color.hpp>
+#include <timemory/process/threading.hpp>
 #include <timemory/utility/filepath.hpp>
 
 #include <iomanip>
@@ -43,6 +44,29 @@ struct source_location_history
     size_t                          size = 0;
 };
 
+const std::string&
+get_file_name()
+{
+    static auto _fname = tim::get_env<std::string>("OMNITRACE_LOG_FILE", "");
+    return _fname;
+}
+
+std::atomic<FILE*>&
+get_file_pointer()
+{
+    static auto _v = std::atomic<FILE*>{ []() {
+        const auto&_fname= get_file_name();
+        if(!_fname.empty()) tim::log::monochrome() = true;
+        return (_fname.empty())
+                   ? stderr
+                   : filepath::fopen(
+                         settings::format(_fname, filepath::basename(filepath::realpath(
+                                                      "/proc/self/exe", nullptr, false))),
+                         "w");
+    }() };
+    return _v;
+}
+
 auto&
 get_source_location_history()
 {
@@ -88,11 +112,28 @@ lock::~lock()
 FILE*
 get_file()
 {
-    static FILE* _v = []() {
-        auto&& _fname = tim::get_env<std::string>("OMNITRACE_LOG_FILE", "");
-        if(!_fname.empty()) tim::log::monochrome() = true;
-        return (_fname.empty()) ? stderr : tim::filepath::fopen(_fname, "w");
-    }();
+    return get_file_pointer();
+}
+
+void
+close_file()
+{
+    if(get_file() != stderr)
+    {
+        auto* _file = get_file_pointer().load();
+        get_file_pointer().store(stderr);
+        fclose(_file);
+        // Write the trace into a file.
+        if(get_verbose() >= 0)
+            operation::file_output_message<tim::project::omnitrace>{}(
+                get_file_name(), std::string{ "debug" });
+    }
+}
+
+int64_t
+get_tid()
+{
+    static thread_local auto _v = threading::get_id();
     return _v;
 }
 }  // namespace debug

source/lib/core/debug.hpp

@@ -86,6 +86,12 @@ set_source_location(source_location&&);
 FILE*
 get_file();
 //
+void
+close_file();
+//
+int64_t
+get_tid();
+//
 inline void
 flush()
 {
@@ -168,7 +174,7 @@ as_hex<void*>(void*, size_t);
 #endif
 
 #if !defined(OMNITRACE_DEBUG_THREAD_IDENTIFIER)
-#    define OMNITRACE_DEBUG_THREAD_IDENTIFIER ::tim::threading::get_id()
+#    define OMNITRACE_DEBUG_THREAD_IDENTIFIER ::omnitrace::debug::get_tid()
 #endif
 
 #if !defined(OMNITRACE_SOURCE_LOCATION)
@@ -394,7 +400,6 @@ as_hex<void*>(void*, size_t);
         fprintf(::omnitrace::debug::get_file(), __VA_ARGS__);                            \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::set_state(::omnitrace::State::Finalized);                           \
-        ::tim::signals::disable_signal_detection();                                      \
         timemory_print_demangled_backtrace<64>();                                        \
         METHOD;                                                                          \
     }
@@ -410,7 +415,6 @@ as_hex<void*>(void*, size_t);
         fprintf(::omnitrace::debug::get_file(), __VA_ARGS__);                            \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::set_state(::omnitrace::State::Finalized);                           \
-        ::tim::signals::disable_signal_detection();                                      \
         timemory_print_demangled_backtrace<64>();                                        \
         METHOD;                                                                          \
     }
@@ -427,7 +431,6 @@ as_hex<void*>(void*, size_t);
         fprintf(::omnitrace::debug::get_file(), __VA_ARGS__);                            \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::set_state(::omnitrace::State::Finalized);                           \
-        ::tim::signals::disable_signal_detection();                                      \
         timemory_print_demangled_backtrace<64>();                                        \
         METHOD;                                                                          \
     }
@@ -443,7 +446,6 @@ as_hex<void*>(void*, size_t);
         fprintf(::omnitrace::debug::get_file(), __VA_ARGS__);                            \
         ::omnitrace::debug::flush();                                                     \
         ::omnitrace::set_state(::omnitrace::State::Finalized);                           \
-        ::tim::signals::disable_signal_detection();                                      \
         timemory_print_demangled_backtrace<64>();                                        \
         METHOD;                                                                          \
     }