Handle fork in target application (#191)

* Always print PID in log messages

* omnitrace-dl updates

- omnitrace_preload does not call omnitrace_init or omnitrace_init_tooling
- omnitrace_preload will call omnitrace_set_mpi if OMNITRACE_USE_MPI
  or OMNITRACE_USE_MPIP in the env is true but not call it otherwise
  because doing so either overrides OMNITRACE_USE_PID (when true) or
  disable mpip from initialization (when false) and the MPI
  init can be caught later and override OMNITRACE_USE_PID

* config updates

- set_setting_value sets user update type
- remove volatile from get_settings_configured
- don't override settings::default_process_suffix
- don't kill process in omnitrace_exit_action
- set_state ignores updating state if >= State::Finalized

* Handle state > State::Finalized

* fork gotcha updates

- unsets LD_PRELOAD
- sets OMNITRACE_ROOT_PROCESS
- sets OMNITRACE_CHILD_PROCESS

* libomnitrace library.cpp updates

- basic_bundle for fini metrics
- handle finalization from child process

* sampling updates

- sampling::shutdown handles when child process

* Add example and test using fork

* Update run-ci script to support not submitting

* Tweak test envs

* Update build flags when codecov enabled

* remove unnecessary includes of sampling header

* Replace mpi copy/fini static lambda with free-funcs

* Update codecov job

* Fix OMPT segfaults after finalization

* Miscellaneous updates after rebase

* fixes for causal profiling

* revert some run-ci.sh changes

* Disable storing env in sampling::shutdown

* formatting fix

* Update timemory submodule

- fixed occasional synchronization issues with allocator offloading
- exclude protozero:: from internal samples

* improve root/child process detection

- avoid omnitrace_finalize in MPI when child process
- revert some testing tweaks

source/lib/omnitrace-dl/dl.cpp

@@ -128,6 +128,16 @@ reset_omnitrace_preload()
     }
 }
 
+inline pid_t
+get_omnitrace_root_pid()
+{
+    auto _pid = getpid();
+    setenv("OMNITRACE_ROOT_PROCESS", std::to_string(_pid).c_str(), 0);
+    return get_env("OMNITRACE_ROOT_PROCESS", _pid);
+}
+
+pid_t _omnitrace_root_pid = get_omnitrace_root_pid();
+
 // environment priority:
 //  - OMNITRACE_DL_DEBUG
 //  - OMNITRACE_DL_VERBOSE
@@ -538,7 +548,9 @@ bool _omnitrace_dl_fini = (std::atexit([]() {
     {                                                                                    \
         fflush(stderr);                                                                  \
         OMNITRACE_COMMON_LIBRARY_LOG_START                                               \
-        fprintf(stderr, "[omnitrace][" OMNITRACE_COMMON_LIBRARY_NAME "] " __VA_ARGS__);  \
+        fprintf(stderr, "[omnitrace][" OMNITRACE_COMMON_LIBRARY_NAME "][%i] ",           \
+                getpid());                                                               \
+        fprintf(stderr, __VA_ARGS__);                                                    \
         OMNITRACE_COMMON_LIBRARY_LOG_END                                                 \
         fflush(stderr);                                                                  \
     }
@@ -1040,7 +1052,6 @@ extern "C"
     ompt_start_tool_result_t* ompt_start_tool(unsigned int omp_version,
                                               const char*  runtime_version)
     {
-        if(!omnitrace::common::get_env("OMNITRACE_USE_OMPT", true)) return nullptr;
         return OMNITRACE_DL_INVOKE(get_indirect().ompt_start_tool_f, omp_version,
                                    runtime_version);
     }
@@ -1060,6 +1071,7 @@ bool
 omnitrace_preload()
 {
     auto _preload = get_omnitrace_preload() && get_env("OMNITRACE_ENABLED", true);
+    auto _use_mpi = get_env("OMNITRACE_USE_MPI", get_env("OMNITRACE_USE_MPIP", false));
 
     static bool _once = false;
     if(_once) return _preload;
@@ -1067,7 +1079,7 @@ omnitrace_preload()
 
     if(_preload)
     {
-        // reset_omnitrace_preload();
+        reset_omnitrace_preload();
         omnitrace_preinit_library();
         auto _causal = get_env("OMNITRACE_USE_CAUSAL", false);
         auto _mode   = get_env("OMNITRACE_MODE", (_causal) ? "causal" : "sampling");
@@ -1075,6 +1087,16 @@ omnitrace_preload()
                          ::omnitrace::join(::omnitrace::QuoteStrings{}, ", ", _mode,
                                            false, "omnitrace")
                              .c_str());
+        if(_use_mpi && !(_causal && _mode == "causal"))
+        {
+            // only make this call if true bc otherwise, if
+            // false, it will disable the MPIP component and
+            // we may intercept the MPI init call later.
+            // If _use_mpi defaults to true above, calling this
+            // will override can current env or config value for
+            // OMNITRACE_USE_PID.
+            omnitrace_set_mpi(_use_mpi, false);
+        }
         omnitrace_init(_mode.c_str(), false, nullptr);
         omnitrace_init_tooling();
     }