Address and thread sanitizer fixes (#250)

* Address and thread sanitizer fixes

- Fix compilation with clang
- Tweak perfetto copy to build tree
- Added suppression files to scripts
- fix LD_PRELOAD support in omnitrace-causal and omnitrace-sample
- use spin_mutex and spin_lock from timemory instead of atomic_mutex and atomic_lock
- state uses atomic
- fix some memory leaks
- tweak testing
  - mpi tests do not use preload
  - increase timeout when using sanitizers
  - add env LD_PRELOAD when using sanitizers

* Tweak perfetto build

* Update timemory submodule

* Update version to 1.8.1

* Update omnitrace-leak.supp

* Update timemory submodule

- fixed spin_mutex implementation

* Remove previously added addr_space->allowTraps(instr_traps)

- this appears to cause errors during binary rewrite

* causal testing updates

- relaxed causal validation on CI systems (to account for hyperthreading decreasing prediction)
- improved impact calculation
- other general improvements to validate-causal-json.py

* Improve fork handling for perfetto

- numerous updates changing perfetto:: to ::perfetto::
- added perfetto_fwd.hpp

* Updated fork example

- user API for validation that stopping/starting perfetto is valid

* Misc fixes to perfetto + fork support

- tweak regions in fork example
- handle disabling tmp files
- get rid of stop/start with perfetto before/after fork
- fixed sampling support during fork
- tweak env of fork test

* Fix find_package in build-tree

* Fix buildtree export

* Fix buildtree export

* Restructured ConfigInstall before adding examples

* Guard against creating tmp file in sampling when disabled

* Fix buildtree package

* formatting

* exit handlers on child processes

- quick exit to avoid perfetto cleanup

* Further tweaking of causal tests for reliability

- enable PROCESSOR_AFFINITY
- decrease to 5 iterations

* Further tweaking of causal tests for reliability

- disable PROCESSOR_AFFINITY for fast func e2e tests
- enabling affinity results in (valid) speedup predictions greater than zero

* Fixes to fork handling

- use pthread_atfork for redundancy if fork_gotcha fails

* cmake formatting

* Fix fork init settings + install components

- remove dl from PROJECT_BUILD_TARGETS

* Testing tweaks

- fix mpi-binary-rewrite-run regex when OMNITRACE_VERBOSE set > 1 in env
- increase causal e2e iterations to 8

* Fix "Test User API"

- test-find-package.sh included dl component

* Further tweaks to causal validation

- further considerations of variance

source/lib/omnitrace/library/components/fork_gotcha.cpp

@@ -25,6 +25,7 @@
 #include "core/config.hpp"
 #include "core/debug.hpp"
 #include "core/perfetto.hpp"
+#include "core/perfetto_fwd.hpp"
 #include "core/state.hpp"
 #include "library/components/fork_gotcha.hpp"
 #include "library/runtime.hpp"
@@ -32,24 +33,45 @@
 
 #include <timemory/backends/process.hpp>
 #include <timemory/backends/threading.hpp>
+#include <timemory/mpl/types.hpp>
+#include <timemory/process/process.hpp>
 
+#include <cstdlib>
+#include <memory>
+#include <pthread.h>
 #include <unistd.h>
 
 namespace omnitrace
 {
 namespace component
 {
-void
-fork_gotcha::configure()
+namespace
 {
-    fork_gotcha_t::get_initializer() = []() {
-        TIMEMORY_C_GOTCHA(fork_gotcha_t, 0, fork);
-    };
+// these are used to prevent handlers from executing multiple times
+bool prefork_lock         = false;
+bool postfork_parent_lock = false;
+bool postfork_child_lock  = false;
+
+// this does a quick exit (no cleanup) on child processes
+// because perfetto has a tendency to access memory it
+// shouldn't during cleanup
+void
+child_exit(int _ec, void*)
+{
+    std::quick_exit(_ec);
 }
 
 void
-fork_gotcha::audit(const gotcha_data_t&, audit::incoming)
+prefork_setup()
 {
+    if(prefork_lock) return;
+
+    OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
+    OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
+
+    if(get_state() < State::Active && !config::settings_are_configured())
+        omnitrace_init_library_hidden();
+
     tim::set_env("OMNITRACE_PRELOAD", "0", 1);
     tim::set_env("OMNITRACE_ROOT_PROCESS", process::get_id(), 0);
     omnitrace_reset_preload_hidden();
@@ -59,39 +81,98 @@ fork_gotcha::audit(const gotcha_data_t&, audit::incoming)
         "Warning! Calling fork() within an OpenMPI application using libfabric "
         "may result is segmentation fault\n");
     TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(get_debug_env(), 16);
+
+    if(config::get_use_sampling()) sampling::block_samples();
+
+    omnitrace::categories::disable_categories(config::get_enabled_categories());
+
+    // prevent re-entry until post-fork routines have been called
+    prefork_lock         = true;
+    postfork_parent_lock = false;
+    postfork_child_lock  = false;
 }
 
 void
-fork_gotcha::audit(const gotcha_data_t&, audit::outgoing, pid_t _pid)
+postfork_parent()
 {
+    if(postfork_parent_lock) return;
+
+    omnitrace::categories::enable_categories(config::get_enabled_categories());
+
+    if(config::get_use_sampling()) sampling::unblock_samples();
+
+    // prevent re-entry until prefork has been called
+    postfork_parent_lock = true;
+    prefork_lock         = false;
+}
+
+void
+postfork_child()
+{
+    if(postfork_child_lock) return;
+
+    OMNITRACE_REQUIRE(is_child_process())
+        << "Error! child process " << process::get_id()
+        << " believes it is the root process " << get_root_process_id() << "\n";
+
+    settings::enabled() = false;
+    settings::verbose() = -127;
+    settings::debug()   = false;
+    omnitrace::sampling::shutdown();
+    omnitrace::categories::shutdown();
+    set_thread_state(::omnitrace::ThreadState::Disabled);
+
+    omnitrace::get_perfetto_session(process::get_parent_id()).release();
+
+    // register these exit handlers to avoid cleaning up resources
+    on_exit(&child_exit, nullptr);
+    std::atexit([]() { child_exit(EXIT_SUCCESS, nullptr); });
+
+    // prevent re-entry until prefork has been called
+    postfork_child_lock = true;
+    prefork_lock        = false;
+}
+}  // namespace
+
+void
+fork_gotcha::configure()
+{
+    fork_gotcha_t::get_initializer() = []() {
+        TIMEMORY_C_GOTCHA(fork_gotcha_t, 0, fork);
+    };
+
+    // registering the pthread_atfork and gotcha means that we might execute twice
+    // handlers twice, hence the locks
+    pthread_atfork(&prefork_setup, &postfork_parent, &postfork_child);
+}
+
+pid_t
+fork_gotcha::operator()(const gotcha_data_t&, pid_t (*_real_fork)()) const
+{
+    prefork_setup();
+
+    auto _pid = (*_real_fork)();
+
     if(_pid != 0)
     {
         OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getppid(),
                                 _pid);
+
+        postfork_parent();
     }
     else
     {
-        OMNITRACE_REQUIRE(is_child_process())
-            << "Error! child process " << process::get_id()
-            << " believes it is the root process " << get_root_process_id() << "\n";
-        settings::enabled() = false;
-        settings::verbose() = -127;
-        settings::debug()   = false;
-        omnitrace::sampling::shutdown();
-        omnitrace::categories::shutdown();
-        omnitrace::get_perfetto_session().release();
-        set_thread_state(::omnitrace::ThreadState::Disabled);
+        postfork_child();
     }
 
     if(!settings::use_output_suffix())
     {
-        settings::use_output_suffix()      = true;
-        settings::default_process_suffix() = process::get_id();
-
         OMNITRACE_BASIC_VERBOSE(
-            0, "call to fork() enables using an output suffix. PID %i will use %i\n",
-            process::get_id(), process::get_id());
+            0, "Application which make calls to fork() should enable using an process "
+               "identifier output suffix (i.e. set OMNITRACE_USE_PID=ON)\n");
     }
+
+    return _pid;
 }
 }  // namespace component
 }  // namespace omnitrace