omnitrace-run executable - required for running binary writes (#257)

* omnitrace-run exe - ensure LD_PRELOAD for libomnitrace-dl.so - convert config options into command-line options * Update timemory submodule - updates to tsettings - updates to argparser * common environment update - throw error if get_env<bool> has empty string * config updates - minor tweaks to categories of settings * core lib update - add argparse for common handling of argument parsers * omnitrace-sample update - fix handling of --trace-file (OMNITRACE_PERFETTO_FILE) * omnitrace-run update - updated to use omnitrace::argparse functions * Tests for omnitrace-run * argparse core update - remove choices for --cpu-events and --gpu-events * remove some debugging prints * fix timemory include in argparse.cpp * always provide --hsa-interrupt option * Update source/lib/core/argparse.cpp - fix pedantic warning * Update testing - remove testing args that may not be there in some builds * roctracer/pthread_create fix - disable roctracer_data when roctracer not enabled * omnitrace-causal tweak * omnitrace-instrument: module_function tweak - allow DEFAULT_MODULE and LIBRARY_MODULE * common environment update - support get_env for enums * core: config update - Add "mode" category to OMNITRACE_MODE * Update timemory submodule - remove debug print statement * omnitrace-sample tweak - change var init * omnitrace-run testing update - use --help instead of -? * core: common.hpp - tweak header include style * core: argparser update - add_ld_preload func - launcher and command member variables in parser_data - support launcher * omnitrace-run update - clean up and reworked * libomnitrace-dl updates - require LD_PRELOAD with binary rewrite - dl::InstrumentMode - dl::get_instrumented() - verify_instrumented_preloaded() - omnitrace_set_instrumented(int) - relocated omnitrace_main from main.c to dl.cpp - omnitrace_set_env does not dlopen libomnitrace - omnitrace_set_main(func_ptr) [internal API] - OMNITRACE_HIDDEN_API -> OMNITRACE_INTERNAL_API * Update testing to new LD_PRELOAD requirements * omnitrace-instrument updates - adhere to LD_PRELOAD requirementsa - invoke omnitrace_set_instrumented - binary rewrite does not instrument main - binary rewrite does not instrument call to omnitrace_init - runtime instr does not instrument main - runtime instr does not instrument call to omnitrace_init * Bump to v1.9.0 - LD_PRELOAD requirement necessitates minor version increment * common: environment - fix ambiguous get_env calls * omnitrace-instrument update - fix issue with temporaries * omnitrace-instrument and libomnitrace-dl updates - runtime instrumentation does not work if libomnitrace-dl is preloaded * libomnitrace-dl and libpyomnitrace updates - define dl::InstrumentMode in dl.hpp - handle instrumentation via setprofile libpyomnitrace - do not push trace in omnitrace_init * omnitrace-instrument and libomnitrace-dl updates - move header to dl subdirectory - omnitrace::omnitrace-headers include omnitrace-dl folder - use InstrumentMode in omnitrace-instrument * Update workflows and scripts - Use omnitrace-run on instrumented exes * Update docs - add omnitrace-run to examples of running binary rewritten exes
2023-03-14 19:48:29 -05:00
@@ -38,12 +38,21 @@
 #include "common/invoke.hpp"
 #include "common/join.hpp"
 #include "common/setup.hpp"
-#include "dl.hpp"
+#include "dl/dl.hpp"
 #include "omnitrace/categories.h"
 #include "omnitrace/types.h"

+#include <timemory/utility/filepath.hpp>
+
 #include <cassert>
+#include <chrono>
 #include <gnu/libc-version.h>
+#include <link.h>
+#include <linux/limits.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <thread>
+#include <unistd.h>

 //--------------------------------------------------------------------------------------//

@@ -69,6 +78,8 @@

 //--------------------------------------------------------------------------------------//

+using main_func_t = int (*)(int, char**, char**);
+
 std::ostream&
 operator<<(std::ostream& _os, const SpaceHandle& _handle)
 {
@@ -78,7 +89,7 @@ operator<<(std::ostream& _os, const SpaceHandle& _handle)

 namespace omnitrace
 {
-inline namespace dl
+namespace dl
 {
 namespace
 {
@@ -97,6 +108,16 @@ get_omnitrace_dl_env()
               : get_env("OMNITRACE_DL_VERBOSE", get_omnitrace_env());
 }

+inline bool&
+get_omnitrace_is_preloaded()
+{
+    static bool _v = []() {
+        auto&& _preload_libs = get_env("LD_PRELOAD", std::string{});
+        return (_preload_libs.find("libomnitrace-dl.so") != std::string::npos);
+    }();
+    return _v;
+}
+
 inline bool
 get_omnitrace_preload()
 {
@@ -136,6 +157,12 @@ get_omnitrace_root_pid()
    return get_env("OMNITRACE_ROOT_PROCESS", _pid);
 }

+void
+omnitrace_preinit() OMNITRACE_INTERNAL_API;
+
+void
+omnitrace_postinit(std::string exe = {}) OMNITRACE_INTERNAL_API;
+
 pid_t _omnitrace_root_pid = get_omnitrace_root_pid();

 // environment priority:
@@ -189,7 +216,7 @@ const char* _omnitrace_dl_dlopen_descr = "RTLD_LAZY | RTLD_LOCAL";
 #endif

 /// This class contains function pointers for omnitrace's instrumentation functions
-struct OMNITRACE_HIDDEN_API indirect
+struct OMNITRACE_INTERNAL_API indirect
 {
    OMNITRACE_INLINE indirect(const std::string& _omnilib, const std::string& _userlib,
                              const std::string& _dllib)
@@ -456,7 +483,7 @@ private:
 };

 inline indirect&
-get_indirect() OMNITRACE_HIDDEN_API;
+get_indirect() OMNITRACE_INTERNAL_API;

 indirect&
 get_indirect()
@@ -519,6 +546,13 @@ get_thread_status()
    return _v;
 }

+InstrumentMode&
+get_instrumented()
+{
+    static auto _v = get_env("OMNITRACE_INSTRUMENT_MODE", InstrumentMode::None);
+    return _v;
+}
+
 // ensure finalization is called
 bool _omnitrace_dl_fini = (std::atexit([]() {
                               if(get_active()) omnitrace_finalize();
@@ -555,7 +589,7 @@ bool _omnitrace_dl_fini = (std::atexit([]() {
        fflush(stderr);                                                                  \
    }

-using omnitrace::get_indirect;
+using omnitrace::dl::get_indirect;
 namespace dl = omnitrace::dl;

 extern "C"
@@ -598,6 +632,9 @@ extern "C"
            return;
        }

+        if(dl::get_instrumented() < dl::InstrumentMode::PythonProfile)
+            dl::omnitrace_preinit();
+
        bool _invoked = false;
        OMNITRACE_DL_INVOKE_STATUS(_invoked, get_indirect().omnitrace_init_f, a, b, c);
        if(_invoked)
@@ -605,6 +642,8 @@ extern "C"
            dl::get_active()          = true;
            dl::get_inited()          = true;
            dl::_omnitrace_dl_verbose = dl::get_omnitrace_dl_env();
+            if(dl::get_instrumented() < dl::InstrumentMode::PythonProfile)
+                dl::omnitrace_postinit((c) ? std::string{ c } : std::string{});
        }
    }

@@ -727,8 +766,9 @@ extern "C"
            OMNITRACE_DL_IGNORE(2, "already initialized and active", a, b);
            return;
        }
+        OMNITRACE_DL_LOG(2, "%s(%s, %s)\n", __FUNCTION__, a, b);
        setenv(a, b, 0);
-        OMNITRACE_DL_INVOKE(get_indirect().omnitrace_set_env_f, a, b);
+        // OMNITRACE_DL_INVOKE(get_indirect().omnitrace_set_env_f, a, b);
    }

    void omnitrace_set_mpi(bool a, bool b)
@@ -840,6 +880,22 @@ extern "C"
                                   _annotations, _annotation_count);
    }

+    void omnitrace_set_instrumented(int _mode)
+    {
+        OMNITRACE_DL_LOG(2, "%s(%i)\n", __FUNCTION__, _mode);
+        auto _mode_v = static_cast<dl::InstrumentMode>(_mode);
+        if(_mode_v < dl::InstrumentMode::None || _mode_v >= dl::InstrumentMode::Last)
+        {
+            OMNITRACE_DL_LOG(-127,
+                             "%s(mode=%i) invoked with invalid instrumentation mode. "
+                             "mode should be %i >= mode < %i\n",
+                             __FUNCTION__, _mode,
+                             static_cast<int>(dl::InstrumentMode::None),
+                             static_cast<int>(dl::InstrumentMode::Last));
+        }
+        dl::get_instrumented() = _mode_v;
+    }
+
    //----------------------------------------------------------------------------------//
    //
    //      KokkosP
@@ -1060,18 +1116,153 @@ extern "C"

 namespace omnitrace
 {
-inline namespace dl
+namespace dl
 {
 namespace
 {
 bool
-omnitrace_preload() OMNITRACE_HIDDEN_API;
+omnitrace_preload() OMNITRACE_INTERNAL_API;
+
+std::vector<std::string>
+get_link_map(const char*,
+             std::vector<int>&& = { (RTLD_LAZY | RTLD_NOLOAD) }) OMNITRACE_INTERNAL_API;
+
+const char*
+get_default_mode() OMNITRACE_INTERNAL_API;
+
+void
+verify_instrumented_preloaded() OMNITRACE_INTERNAL_API;
+
+std::vector<std::string>
+get_link_map(const char* _name, std::vector<int>&& _open_modes)
+{
+    void* _handle = nullptr;
+    bool  _noload = false;
+    for(auto _mode : _open_modes)
+    {
+        _handle = dlopen(_name, _mode);
+        _noload = (_mode & RTLD_NOLOAD) == RTLD_NOLOAD;
+        if(_handle) break;
+    }
+
+    auto _chain = std::vector<std::string>{};
+    if(_handle)
+    {
+        struct link_map* _link_map = nullptr;
+        dlinfo(_handle, RTLD_DI_LINKMAP, &_link_map);
+        struct link_map* _next = _link_map->l_next;
+        while(_next)
+        {
+            if(_next->l_name != nullptr && !std::string_view{ _next->l_name }.empty())
+            {
+                _chain.emplace_back(_next->l_name);
+            }
+            _next = _next->l_next;
+        }
+
+        if(_noload == false) dlclose(_handle);
+    }
+    return _chain;
+}
+
+const char*
+get_default_mode()
+{
+    if(get_env("OMNITRACE_USE_CAUSAL", false)) return "causal";
+
+    auto _link_map = get_link_map(nullptr);
+    for(const auto& itr : _link_map)
+    {
+        if(itr.find("libomnitrace-rt.so") != std::string::npos ||
+           itr.find("libdyninstAPI_RT.so") != std::string::npos)
+            return "trace";
+    }
+
+    return "sampling";
+}
+
+void
+omnitrace_preinit()
+{
+    switch(get_instrumented())
+    {
+        case InstrumentMode::None:
+        case InstrumentMode::BinaryRewrite:
+        case InstrumentMode::ProcessCreate:
+        case InstrumentMode::ProcessAttach:
+        {
+            auto _use_mpip = get_env("OMNITRACE_USE_MPIP", false);
+            auto _use_mpi  = get_env("OMNITRACE_USE_MPI", _use_mpip);
+            auto _causal   = get_env("OMNITRACE_USE_CAUSAL", false);
+            auto _mode     = get_env("OMNITRACE_MODE", get_default_mode());
+
+            if(_use_mpi && !(_causal && _mode == "causal"))
+            {
+                // only make this call if true bc otherwise, if
+                // false, it will disable the MPIP component and
+                // we may intercept the MPI init call later.
+                // If _use_mpi defaults to true above, calling this
+                // will override can current env or config value for
+                // OMNITRACE_USE_PID.
+                omnitrace_set_mpi(_use_mpi, dl::get_instrumented() ==
+                                                dl::InstrumentMode::ProcessAttach);
+            }
+            break;
+        }
+        case InstrumentMode::PythonProfile:
+        case InstrumentMode::Last: break;
+    }
+}
+
+void
+omnitrace_postinit(std::string _exe)
+{
+    switch(get_instrumented())
+    {
+        case InstrumentMode::None:
+        case InstrumentMode::BinaryRewrite:
+        case InstrumentMode::ProcessCreate:
+        case InstrumentMode::ProcessAttach:
+        {
+            if(_exe.empty())
+                _exe = tim::filepath::readlink(join('/', "/proc", getpid(), "exe"));
+
+            omnitrace_init_tooling();
+            if(_exe.empty())
+                omnitrace_push_trace("main");
+            else
+                omnitrace_push_trace(basename(_exe.c_str()));
+            break;
+        }
+        case InstrumentMode::PythonProfile:
+        {
+            omnitrace_init_tooling();
+            break;
+        }
+        case InstrumentMode::Last: break;
+    }
+}

 bool
 omnitrace_preload()
 {
-    auto _preload = get_omnitrace_preload() && get_env("OMNITRACE_ENABLED", true);
-    auto _use_mpi = get_env("OMNITRACE_USE_MPI", get_env("OMNITRACE_USE_MPIP", false));
+    auto _preload = get_omnitrace_is_preloaded() && get_omnitrace_preload() &&
+                    get_env("OMNITRACE_ENABLED", true);
+
+    auto _link_map = get_link_map(nullptr);
+    auto _instr_mode =
+        get_env("OMNITRACE_INSTRUMENT_MODE", dl::InstrumentMode::BinaryRewrite);
+    for(const auto& itr : _link_map)
+    {
+        if(itr.find("libomnitrace-rt.so") != std::string::npos ||
+           itr.find("libdyninstAPI_RT.so") != std::string::npos)
+        {
+            omnitrace_set_instrumented(static_cast<int>(_instr_mode));
+            break;
+        }
+    }
+
+    verify_instrumented_preloaded();

    static bool _once = false;
    if(_once) return _preload;
@@ -1081,30 +1272,174 @@ omnitrace_preload()
    {
        reset_omnitrace_preload();
        omnitrace_preinit_library();
-        auto _causal = get_env("OMNITRACE_USE_CAUSAL", false);
-        auto _mode   = get_env("OMNITRACE_MODE", (_causal) ? "causal" : "sampling");
-        OMNITRACE_DL_LOG(1, "[%s] invoking %s(%s)\n", __FUNCTION__, "omnitrace_init",
-                         ::omnitrace::join(::omnitrace::QuoteStrings{}, ", ", _mode,
-                                           false, "omnitrace")
-                             .c_str());
-        if(_use_mpi && !(_causal && _mode == "causal"))
-        {
-            // only make this call if true bc otherwise, if
-            // false, it will disable the MPIP component and
-            // we may intercept the MPI init call later.
-            // If _use_mpi defaults to true above, calling this
-            // will override can current env or config value for
-            // OMNITRACE_USE_PID.
-            omnitrace_set_mpi(_use_mpi, false);
-        }
-        omnitrace_init(_mode.c_str(), false, nullptr);
-        omnitrace_init_tooling();
    }

    return _preload;
 }

-bool _handle_preload = omnitrace::dl::omnitrace_preload();
+void
+verify_instrumented_preloaded()
+{
+    // if preloaded then we are fine
+    if(get_omnitrace_is_preloaded()) return;
+
+    // value returned by get_instrumented is set by either:
+    // - the search of the linked libraries
+    // - via the instrumenter
+    // if binary rewrite or runtime instrumentation, there is an opportunity for
+    // LD_PRELOAD
+    switch(dl::get_instrumented())
+    {
+        case dl::InstrumentMode::None:
+        case dl::InstrumentMode::ProcessAttach:
+        case dl::InstrumentMode::ProcessCreate:
+        {
+            return;
+        }
+        case dl::InstrumentMode::BinaryRewrite:
+        {
+            break;
+        }
+        case dl::InstrumentMode::Last:
+        {
+            throw std::runtime_error(
+                "Invalid instrumentation type: InstrumentMode::Last");
+        }
+    }
+
+    static const char* _notice = R"notice(
+
+        NNNNNNNN        NNNNNNNN     OOOOOOOOO     TTTTTTTTTTTTTTTTTTTTTTTIIIIIIIIII      CCCCCCCCCCCCCEEEEEEEEEEEEEEEEEEEEEE
+        N:::::::N       N::::::N   OO:::::::::OO   T:::::::::::::::::::::TI::::::::I   CCC::::::::::::CE::::::::::::::::::::E
+        N::::::::N      N::::::N OO:::::::::::::OO T:::::::::::::::::::::TI::::::::I CC:::::::::::::::CE::::::::::::::::::::E
+        N:::::::::N     N::::::NO:::::::OOO:::::::OT:::::TT:::::::TT:::::TII::::::IIC:::::CCCCCCCC::::CEE::::::EEEEEEEEE::::E
+        N::::::::::N    N::::::NO::::::O   O::::::OTTTTTT  T:::::T  TTTTTT  I::::I C:::::C       CCCCCC  E:::::E       EEEEEE
+        N:::::::::::N   N::::::NO:::::O     O:::::O        T:::::T          I::::IC:::::C                E:::::E
+        N:::::::N::::N  N::::::NO:::::O     O:::::O        T:::::T          I::::IC:::::C                E::::::EEEEEEEEEE
+        N::::::N N::::N N::::::NO:::::O     O:::::O        T:::::T          I::::IC:::::C                E:::::::::::::::E
+        N::::::N  N::::N:::::::NO:::::O     O:::::O        T:::::T          I::::IC:::::C                E:::::::::::::::E
+        N::::::N   N:::::::::::NO:::::O     O:::::O        T:::::T          I::::IC:::::C                E::::::EEEEEEEEEE
+        N::::::N    N::::::::::NO:::::O     O:::::O        T:::::T          I::::IC:::::C                E:::::E
+        N::::::N     N:::::::::NO::::::O   O::::::O        T:::::T          I::::I C:::::C       CCCCCC  E:::::E       EEEEEE
+        N::::::N      N::::::::NO:::::::OOO:::::::O      TT:::::::TT      II::::::IIC:::::CCCCCCCC::::CEE::::::EEEEEEEE:::::E
+        N::::::N       N:::::::N OO:::::::::::::OO       T:::::::::T      I::::::::I CC:::::::::::::::CE::::::::::::::::::::E
+        N::::::N        N::::::N   OO:::::::::OO         T:::::::::T      I::::::::I   CCC::::::::::::CE::::::::::::::::::::E
+        NNNNNNNN         NNNNNNN     OOOOOOOOO           TTTTTTTTTTT      IIIIIIIIII      CCCCCCCCCCCCCEEEEEEEEEEEEEEEEEEEEEE
+
+                                                     _    _  _____ ______
+                                                    | |  | |/ ____|  ____|
+                                                    | |  | | (___ | |__
+                                                    | |  | |\___ \|  __|
+                                                    | |__| |____) | |____
+                                                     \____/|_____/|______|
+
+                     ____  __  __ _   _ _____ _______ _____            _____ ______      _____  _    _ _   _
+                    / __ \|  \/  | \ | |_   _|__   __|  __ \     /\   / ____|  ____|    |  __ \| |  | | \ | |
+                   | |  | | \  / |  \| | | |    | |  | |__) |   /  \ | |    | |__ ______| |__) | |  | |  \| |
+                   | |  | | |\/| | . ` | | |    | |  |  _  /   / /\ \| |    |  __|______|  _  /| |  | | . ` |
+                   | |__| | |  | | |\  |_| |_   | |  | | \ \  / ____ \ |____| |____     | | \ \| |__| | |\  |
+                    \____/|_|  |_|_| \_|_____|  |_|  |_|  \_\/_/    \_\_____|______|    |_|  \_\\____/|_| \_|
+
+
+    Due to a variety of edge cases we've encountered, OmniTrace now requires that binary rewritten executables and libraries be launched
+    with the 'omnitrace-run' executable.
+
+    In order to launch the executable with 'omnitrace-run', prefix the current command with 'omnitrace-run' and a standalone double hyphen ('--').
+    For MPI applications, place 'omnitrace-run --' after the MPI command.
+    E.g.:
+
+        <EXECUTABLE> <ARGS...>
+        mpirun -n 2 <EXECUTABLE> <ARGS...>
+
+    should be:
+
+        omnitrace-run -- <EXECUTABLE> <ARGS...>
+        mpirun -n 2 omnitrace-run -- <EXECUTABLE> <ARGS...>
+
+    Note: the command-line arguments passed to 'omnitrace-run' (which are specified before the double hyphen) will override configuration variables
+    and/or any configuration values specified to 'omnitrace-instrument' via the '--config' or '--env' options.
+    E.g.:
+
+        $ omnitrace-instrument -o ./sleep.inst --env OMNITRACE_SAMPLING_DELAY=5.0 -- sleep
+        $ echo "OMNITRACE_SAMPLING_FREQ = 500" > omnitrace.cfg
+        $ export OMNITRACE_CONFIG_FILE=omnitrace.cfg
+        $ omnitrace-run --sampling-freq=100 --sampling-delay=1.0 -- ./sleep.inst 10
+
+    In the first command, a default sampling delay of 5 seconds in embedded into the instrumented 'sleep.inst'.
+    In the second command, the sampling frequency will be set to 500 interrupts per second when OmniTrace reads the config file
+    In the fourth command, the sampling frequency and sampling delay are overridden to 100 interrupts per second and 1 second, respectively, when sleep.inst runs
+
+    Thanks for using OmniTrace and happy optimizing!
+    )notice";
+
+    // emit notice
+    std::cerr << _notice << std::endl;
+
+    std::quick_exit(EXIT_FAILURE);
+}
+
+bool        _handle_preload = omnitrace_preload();
+main_func_t main_real       = nullptr;
 }  // namespace
 }  // namespace dl
 }  // namespace omnitrace
+
+extern "C"
+{
+    int  omnitrace_main(int argc, char** argv, char** envp) OMNITRACE_INTERNAL_API;
+    void omnitrace_set_main(main_func_t) OMNITRACE_INTERNAL_API;
+
+    void omnitrace_set_main(main_func_t _main_real)
+    {
+        ::omnitrace::dl::main_real = _main_real;
+    }
+
+    int omnitrace_main(int argc, char** argv, char** envp)
+    {
+        OMNITRACE_DL_LOG(0, "%s\n", __FUNCTION__);
+        using ::omnitrace::common::get_env;
+        using ::omnitrace::dl::get_default_mode;
+
+        // prevent re-entry
+        static int _reentry = 0;
+        if(_reentry > 0) return -1;
+        _reentry = 1;
+
+        if(!::omnitrace::dl::main_real)
+            throw std::runtime_error("[omnitrace][dl] Unsuccessful wrapping of main: "
+                                     "nullptr to real main function");
+
+        if(envp)
+        {
+            size_t _idx = 0;
+            while(envp[_idx] != nullptr)
+            {
+                auto _env_v = std::string_view{ envp[_idx++] };
+                if(_env_v.find("OMNITRACE") != 0 &&
+                   _env_v.find("libomnitrace") == std::string_view::npos)
+                    continue;
+                auto _pos = _env_v.find('=');
+                if(_pos < _env_v.length())
+                {
+                    auto _var = std::string{ _env_v }.substr(0, _pos);
+                    auto _val = std::string{ _env_v }.substr(_pos + 1);
+                    OMNITRACE_DL_LOG(1, "%s(%s, %s)\n", "omnitrace_set_env", _var.c_str(),
+                                     _val.c_str());
+                    setenv(_var.c_str(), _val.c_str(), 0);
+                }
+            }
+        }
+
+        auto _mode = get_env("OMNITRACE_MODE", get_default_mode());
+        omnitrace_init(_mode.c_str(),
+                       dl::get_instrumented() == dl::InstrumentMode::BinaryRewrite,
+                       argv[0]);
+
+        int ret = (*::omnitrace::dl::main_real)(argc, argv, envp);
+
+        omnitrace_pop_trace(basename(argv[0]));
+        omnitrace_finalize();
+
+        return ret;
+    }
+}