omnitrace-run executable - required for running binary writes (#257)

* omnitrace-run exe - ensure LD_PRELOAD for libomnitrace-dl.so - convert config options into command-line options * Update timemory submodule - updates to tsettings - updates to argparser * common environment update - throw error if get_env<bool> has empty string * config updates - minor tweaks to categories of settings * core lib update - add argparse for common handling of argument parsers * omnitrace-sample update - fix handling of --trace-file (OMNITRACE_PERFETTO_FILE) * omnitrace-run update - updated to use omnitrace::argparse functions * Tests for omnitrace-run * argparse core update - remove choices for --cpu-events and --gpu-events * remove some debugging prints * fix timemory include in argparse.cpp * always provide --hsa-interrupt option * Update source/lib/core/argparse.cpp - fix pedantic warning * Update testing - remove testing args that may not be there in some builds * roctracer/pthread_create fix - disable roctracer_data when roctracer not enabled * omnitrace-causal tweak * omnitrace-instrument: module_function tweak - allow DEFAULT_MODULE and LIBRARY_MODULE * common environment update - support get_env for enums * core: config update - Add "mode" category to OMNITRACE_MODE * Update timemory submodule - remove debug print statement * omnitrace-sample tweak - change var init * omnitrace-run testing update - use --help instead of -? * core: common.hpp - tweak header include style * core: argparser update - add_ld_preload func - launcher and command member variables in parser_data - support launcher * omnitrace-run update - clean up and reworked * libomnitrace-dl updates - require LD_PRELOAD with binary rewrite - dl::InstrumentMode - dl::get_instrumented() - verify_instrumented_preloaded() - omnitrace_set_instrumented(int) - relocated omnitrace_main from main.c to dl.cpp - omnitrace_set_env does not dlopen libomnitrace - omnitrace_set_main(func_ptr) [internal API] - OMNITRACE_HIDDEN_API -> OMNITRACE_INTERNAL_API * Update testing to new LD_PRELOAD requirements * omnitrace-instrument updates - adhere to LD_PRELOAD requirementsa - invoke omnitrace_set_instrumented - binary rewrite does not instrument main - binary rewrite does not instrument call to omnitrace_init - runtime instr does not instrument main - runtime instr does not instrument call to omnitrace_init * Bump to v1.9.0 - LD_PRELOAD requirement necessitates minor version increment * common: environment - fix ambiguous get_env calls * omnitrace-instrument update - fix issue with temporaries * omnitrace-instrument and libomnitrace-dl updates - runtime instrumentation does not work if libomnitrace-dl is preloaded * libomnitrace-dl and libpyomnitrace updates - define dl::InstrumentMode in dl.hpp - handle instrumentation via setprofile libpyomnitrace - do not push trace in omnitrace_init * omnitrace-instrument and libomnitrace-dl updates - move header to dl subdirectory - omnitrace::omnitrace-headers include omnitrace-dl folder - use InstrumentMode in omnitrace-instrument * Update workflows and scripts - Use omnitrace-run on instrumented exes * Update docs - add omnitrace-run to examples of running binary rewritten exes
2023-03-14 19:48:29 -05:00
@@ -19,6 +19,7 @@ add_subdirectory(omnitrace-critical-trace)
 add_subdirectory(omnitrace-causal)
 add_subdirectory(omnitrace-sample)
 add_subdirectory(omnitrace-instrument)
+add_subdirectory(omnitrace-run)
 # omnitrace-exe is deprecated
 add_subdirectory(omnitrace-exe)

@@ -282,7 +282,7 @@ print_command(const std::vector<char*>& _argv, std::string_view _prefix)
 std::vector<char*>
 get_initial_environment()
 {
-    std::vector<char*> _env;
+    auto _env = std::vector<char*>{};
    if(environ != nullptr)
    {
        int idx = 0;
@@ -499,7 +499,8 @@ module_function::is_module_constrained() const

    // always instrument these modules
    if(module_name == "DEFAULT_MODULE" || module_name == "LIBRARY_MODULE")
-        return _report("Skipping", "default module", 2);
+        // return _report("Skipping", "default module", 2);
+        return false;

    static std::regex ext_regex{ "\\.(s|S)$", regex_opts };
    static std::regex sys_regex{ "^(s|k|e|w)_[A-Za-z_0-9\\-]+\\.(c|C)$", regex_opts };
@@ -22,6 +22,7 @@

 #include "omnitrace-instrument.hpp"
 #include "common/defines.h"
+#include "dl/dl.hpp"
 #include "fwd.hpp"
 #include "internal_libs.hpp"
 #include "log.hpp"
@@ -56,6 +57,7 @@
 #include <string>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <sys/wait.h>
 #include <thread>
 #include <tuple>
 #include <unistd.h>
@@ -86,6 +88,8 @@ get_default_min_address_range()
 }
 }  // namespace

+using InstrumentMode = ::omnitrace::dl::InstrumentMode;
+
 bool   use_return_info              = false;
 bool   use_args_info                = false;
 bool   use_file_info                = false;
@@ -108,7 +112,7 @@ bool   include_uninstr              = false;
 bool   include_internal_linked_libs = false;
 int    verbose_level   = tim::get_env<int>("OMNITRACE_VERBOSE_INSTRUMENT", 0);
 int    num_log_entries = tim::get_env<int>(
-    "OMNITRACE_LOG_COUNT", tim::get_env<bool>("OMNITRACE_CI", false) ? 20 : -1);
+    "OMNITRACE_LOG_COUNT", tim::get_env<bool>("OMNITRACE_CI", false) ? 20 : 50);
 string_t main_fname         = "main";
 string_t argv0              = {};
 string_t cmdv0              = {};
@@ -1410,8 +1414,27 @@ main(int argc, char** argv)
    //
    //----------------------------------------------------------------------------------//

-    addr_space =
-        omnitrace_get_address_space(bpatch, _cmdc, _cmdv, binary_rewrite, _pid, mutname);
+    // prioritize the user environment arguments
+    auto instr_mode_v     = (binary_rewrite) ? InstrumentMode::BinaryRewrite
+                            : (_pid < 0)     ? InstrumentMode::ProcessCreate
+                                             : InstrumentMode::ProcessAttach;
+    auto instr_mode_v_int = static_cast<int>(instr_mode_v);
+    auto env_vars         = parser.get<strvec_t>("env");
+    env_vars.reserve(env_vars.size() + env_config_variables.size());
+    for(auto&& itr : env_config_variables)
+        env_vars.emplace_back(itr);
+    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MODE", instr_mode));
+    env_vars.emplace_back(
+        TIMEMORY_JOIN('=', "OMNITRACE_INSTRUMENT_MODE", instr_mode_v_int));
+    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MPI_INIT", "OFF"));
+    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MPI_FINALIZE", "OFF"));
+    env_vars.emplace_back(
+        TIMEMORY_JOIN('=', "OMNITRACE_TIMEMORY_COMPONENTS", default_components));
+    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_USE_CODE_COVERAGE",
+                                        (coverage_mode != CODECOV_NONE) ? "ON" : "OFF"));
+
+    addr_space = omnitrace_get_address_space(bpatch, _cmdc, _cmdv, env_vars,
+                                             binary_rewrite, _pid, mutname);

    // addr_space->allowTraps(instr_traps);

@@ -1704,14 +1727,15 @@ main(int argc, char** argv)

    verbprintf(0, "Finding instrumentation functions...\n");

-    auto* init_func    = find_function(app_image, "omnitrace_init");
-    auto* fini_func    = find_function(app_image, "omnitrace_finalize");
-    auto* env_func     = find_function(app_image, "omnitrace_set_env");
-    auto* mpi_func     = find_function(app_image, "omnitrace_set_mpi");
-    auto* entr_trace   = find_function(app_image, "omnitrace_push_trace");
-    auto* exit_trace   = find_function(app_image, "omnitrace_pop_trace");
-    auto* reg_src_func = find_function(app_image, "omnitrace_register_source");
-    auto* reg_cov_func = find_function(app_image, "omnitrace_register_coverage");
+    auto* init_func      = find_function(app_image, "omnitrace_init");
+    auto* fini_func      = find_function(app_image, "omnitrace_finalize");
+    auto* env_func       = find_function(app_image, "omnitrace_set_env");
+    auto* mpi_func       = find_function(app_image, "omnitrace_set_mpi");
+    auto* entr_trace     = find_function(app_image, "omnitrace_push_trace");
+    auto* exit_trace     = find_function(app_image, "omnitrace_pop_trace");
+    auto* reg_src_func   = find_function(app_image, "omnitrace_register_source");
+    auto* reg_cov_func   = find_function(app_image, "omnitrace_register_coverage");
+    auto* set_instr_func = find_function(app_image, "omnitrace_set_instrumented");

    if(!main_func && main_fname == "main") main_func = find_function(app_image, "_main");

@@ -1828,12 +1852,14 @@ main(int argc, char** argv)

    using pair_t = std::pair<procedure_t*, string_t>;

-    for(const auto& itr :
-        { pair_t(entr_trace, "omnitrace_push_trace"),
-          pair_t(exit_trace, "omnitrace_pop_trace"), pair_t(init_func, "omnitrace_init"),
-          pair_t(fini_func, "omnitrace_finalize"), pair_t(env_func, "omnitrace_set_env"),
-          pair_t(reg_src_func, "omnitrace_register_source"),
-          pair_t(reg_cov_func, "omnitrace_register_coverage") })
+    for(const auto& itr : { pair_t{ entr_trace, "omnitrace_push_trace" },
+                            pair_t{ exit_trace, "omnitrace_pop_trace" },
+                            pair_t{ init_func, "omnitrace_init" },
+                            pair_t{ fini_func, "omnitrace_finalize" },
+                            pair_t{ env_func, "omnitrace_set_env" },
+                            pair_t{ set_instr_func, "omnitrace_set_instrumented" },
+                            pair_t{ reg_src_func, "omnitrace_register_source" },
+                            pair_t{ reg_cov_func, "omnitrace_register_coverage" } })
    {
        if(!itr.first)
        {
@@ -1888,19 +1914,20 @@ main(int argc, char** argv)
    if(main_func) main_sign.get();

    auto main_call_args = omnitrace_call_expr(main_sign.get());
-    auto init_call_args = omnitrace_call_expr(instr_mode, binary_rewrite, _init_arg0);
+    auto init_call_args = omnitrace_call_expr(instr_mode, binary_rewrite, "");
    auto fini_call_args = omnitrace_call_expr();
    auto umpi_call_args = omnitrace_call_expr(use_mpi, is_attached);
    auto none_call_args = omnitrace_call_expr();
+    auto set_instr_args = omnitrace_call_expr(instr_mode_v_int);

    verbprintf(2, "Done\n");
    verbprintf(2, "Getting call snippets... ");

-    auto init_call = init_call_args.get(init_func);
-    auto fini_call = fini_call_args.get(fini_func);
-    auto umpi_call = umpi_call_args.get(mpi_func);
-
-    auto main_beg_call = main_call_args.get(entr_trace);
+    auto init_call      = init_call_args.get(init_func);
+    auto fini_call      = fini_call_args.get(fini_func);
+    auto umpi_call      = umpi_call_args.get(mpi_func);
+    auto set_instr_call = set_instr_args.get(set_instr_func);
+    auto main_beg_call  = main_call_args.get(entr_trace);

    verbprintf(2, "Done\n");

@@ -1924,35 +1951,27 @@ main(int argc, char** argv)
    }
    if(_libname.empty()) _libname = "libomnitrace-dl.so";

-    // prioritize the user environment arguments
-    auto env_vars = parser.get<strvec_t>("env");
-    env_vars.reserve(env_vars.size() + env_config_variables.size());
-    for(auto&& itr : env_config_variables)
-        env_vars.emplace_back(itr);
-    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MODE", instr_mode));
-    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MPI_INIT", "OFF"));
-    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MPI_FINALIZE", "OFF"));
+    if(!binary_rewrite && !is_attached) env_vars.clear();
+
    env_vars.emplace_back(
        TIMEMORY_JOIN('=', "OMNITRACE_INIT_ENABLED",
                      (user_start_func && user_stop_func) ? "OFF" : "ON"));
-    env_vars.emplace_back(
-        TIMEMORY_JOIN('=', "OMNITRACE_TIMEMORY_COMPONENTS", default_components));
    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_USE_MPIP",
                                        (binary_rewrite && use_mpi) ? "ON" : "OFF"));
-    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_USE_CODE_COVERAGE",
-                                        (coverage_mode != CODECOV_NONE) ? "ON" : "OFF"));
    if(use_mpi) env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_USE_PID", "ON"));

    for(auto& itr : env_vars)
    {
-        auto p = tim::delimit(itr, "=");
-        if(p.size() != 2)
+        auto _pos = itr.find('=');
+        if(_pos == std::string::npos)
        {
            errprintf(0, "environment variable %s not in form VARIABLE=VALUE\n",
                      itr.c_str());
        }
-        tim::set_env(p.at(0), p.at(1));
-        auto _expr = omnitrace_call_expr(p.at(0), p.at(1));
+        auto _var = itr.substr(0, _pos);
+        auto _val = itr.substr(_pos + 1);
+        tim::set_env(_var, _val);
+        auto _expr = omnitrace_call_expr(_var, _val);
        env_variables.emplace_back(_expr.get(env_func));
    }

@@ -1962,6 +1981,12 @@ main(int argc, char** argv)
    //
    //----------------------------------------------------------------------------------//

+    // call into omnitrace-dl to notify that instrumentation is occurring
+    if(binary_rewrite)
+    {
+        init_names.emplace(init_names.begin(), set_instr_call.get());
+    }
+
    for(const auto& itr : env_variables)
    {
        if(itr) init_names.emplace_back(itr.get());
@@ -1981,8 +2006,9 @@ main(int argc, char** argv)
    }

    if(umpi_call) init_names.emplace_back(umpi_call.get());
-    if(init_call) init_names.emplace_back(init_call.get());
-    if(main_func && main_beg_call) init_names.emplace_back(main_beg_call.get());
+    if(!binary_rewrite && init_call) init_names.emplace_back(init_call.get());
+    if(is_attached && main_func && main_beg_call)
+        init_names.emplace_back(main_beg_call.get());

    for(const auto& itr : end_expr)
        if(itr.second) fini_names.emplace_back(itr.second.get());
@@ -2047,34 +2073,56 @@ main(int argc, char** argv)
    auto _init_sequence = sequence_t{ init_names };
    auto _fini_sequence = sequence_t{ fini_names };

-    if(app_thread && is_attached)
+    if(!is_attached)
    {
-        assert(app_thread != nullptr);
-        verbprintf(1, "Executing initial snippets...\n");
-        for(auto* itr : init_names)
-            app_thread->oneTimeCode(*itr);
-    }
-    else
-    {
-        if(main_entr_points)
-        {
-            verbprintf(1, "Adding main entry snippets...\n");
-            addr_space->insertSnippet(_init_sequence, *main_entr_points);
-            // insert_instr(addr_space, *main_entr_points, _init_sequence, BPatch_entry);
-        }
-        else
-        {
+        auto _insert_init_callbacks = std::function<bool()>{};
+        auto _insert_init_snippets  = std::function<bool()>{};
+
+        _insert_init_snippets = [&]() {
+            if(main_entr_points)
+            {
+                verbprintf(1, "Adding main entry snippets...\n");
+                addr_space->insertSnippet(_init_sequence, *main_entr_points);
+            }
+            else
+            {
+                errprintf(0, "Dyninst error inserting main entry snippets: no main entry "
+                             "points\n");
+                return false;
+            }
+            return true;
+        };
+
+        _insert_init_callbacks = [&]() {
+            verbprintf(1, "Adding main init callbacks...\n");
+            size_t _ninits = 0;
            for(auto* itr : _objs)
            {
+                if(itr->name().find("libomnitrace") != std::string::npos) continue;
                try
                {
-                    itr->insertInitCallback(_init_sequence);
+                    verbprintf(2, "Adding main init callbacks (via %s)...\n",
+                               itr->name().c_str());
+                    if(itr->insertInitCallback(_init_sequence))
+                    {
+                        ++_ninits;
+                    }
                } catch(std::runtime_error& _e)
                {
                    errprintf(0, "Dyninst error inserting init callback: %s\n",
                              _e.what());
                }
            }
+            return (_ninits > 0);
+        };
+
+        if(binary_rewrite)
+        {
+            if(!_insert_init_callbacks()) _insert_init_snippets();
+        }
+        else
+        {
+            if(!_insert_init_snippets()) _insert_init_callbacks();
        }
    }

@@ -2469,6 +2517,12 @@ main(int argc, char** argv)
        }
        else if(!app_thread->isTerminated() && is_attached)
        {
+            bpatch->setDebugParsing(false);
+            bpatch->setDelayedParsing(true);
+            verbprintf(1, "Executing initial snippets...\n");
+            for(auto* itr : init_names)
+                app_thread->oneTimeCode(*itr);
+
            app_thread->continueExecution();
            while(!app_thread->isTerminated())
            {
@@ -2492,7 +2546,7 @@ main(int argc, char** argv)
    delete[] _cmdv;

    verbprintf(0, "End of omnitrace\n");
-    verbprintf(1, "Exit code: %i\n", code);
+    verbprintf((code != 0) ? 0 : 1, "Exit code: %i\n", code);

    if(log_ofs)
    {
@@ -29,6 +29,7 @@
 #include "module_function.hpp"

 #include <timemory/utility/filepath.hpp>
+#include <timemory/utility/join.hpp>

 #include <dlfcn.h>
 #include <ios>
@@ -172,7 +173,8 @@ omnitrace_get_is_executable(std::string_view _cmd, bool _default_v)
 //
 static inline address_space_t*
 omnitrace_get_address_space(patch_pointer_t& _bpatch, int _cmdc, char** _cmdv,
-                            bool _rewrite, int _pid = -1, const std::string& _name = {})
+                            const std::vector<std::string>& _cmdenv, bool _rewrite,
+                            int _pid = -1, const std::string& _name = {})
 {
    address_space_t* mutatee = nullptr;

@@ -196,51 +198,83 @@ omnitrace_get_address_space(patch_pointer_t& _bpatch, int _cmdc, char** _cmdv,
        }
        verbprintf_bare(1, "Done\n");
    }
-    else if(_pid >= 0)
-    {
-        verbprintf(1, "Attaching to process %i... ", _pid);
-        fflush(stderr);
-        char* _cmdv0 = (_cmdc > 0) ? _cmdv[0] : nullptr;
-        mutatee      = _bpatch->processAttach(_cmdv0, _pid);
-        if(!mutatee)
-        {
-            verbprintf(-1, "Failed to connect to process %i\n", (int) _pid);
-            throw std::runtime_error("Failed to attach to process");
-        }
-        verbprintf_bare(1, "Done\n");
-    }
    else
    {
-        if(_cmdc < 1) errprintf(-127, "No command provided");
+        bool _attach = (_pid >= 0);

-        if(is_text_file(_cmdv[0]))
+        // override the current environment create/attach to process, revert environment
+        using strpair_t    = std::pair<std::string, std::string>;
+        auto _imported     = std::vector<strpair_t>{};
+        auto _exported     = std::vector<strpair_t>{};
+        auto _get_env_pair = [](const std::string& _full) {
+            auto _pos = _full.find('=');
+            if(_pos < _full.length())
+                return std::make_pair(_full.substr(0, _pos), _full.substr(_pos + 1));
+            return strpair_t{};
+        };
+
+        if(environ)
        {
-            errprintf(-1,
-                      "'%s' is a text file. OmniTrace only supports instrumenting "
-                      "binary files",
-                      _cmdv[0]);
+            size_t _idx = 0;
+            while(environ[_idx] != nullptr)
+                _imported.emplace_back(_get_env_pair(environ[_idx++]));
        }

-        std::stringstream ss;
-        for(int i = 0; i < _cmdc; ++i)
+        for(const auto& itr : _cmdenv)
        {
-            if(!_cmdv || !_cmdv[i]) continue;
-            ss << " " << _cmdv[i];
+            _exported.emplace_back(_get_env_pair(itr));
        }
-        auto _cmd_msg = ss.str();
-        if(_cmd_msg.length() > 1) _cmd_msg = _cmd_msg.substr(1);

-        char** _environ = environ;
-        verbprintf(1, "Creating process '%s'... ", _cmd_msg.c_str());
-        fflush(stderr);
-        mutatee = _bpatch->processCreate(_cmdv[0], (const char**) _cmdv,
-                                         (const char**) _environ);
-        if(!mutatee)
+        for(const auto& itr : _exported)
        {
-            verbprintf(-1, "Failed to create process: '%s'\n", _cmd_msg.c_str());
-            throw std::runtime_error("Failed to create process");
+            setenv(itr.first.c_str(), itr.second.c_str(), 1);
+            verbprintf(4, "[env] %s=%s\n", itr.first.c_str(), itr.second.c_str());
+        }
+
+        if(_attach)
+        {
+            verbprintf(1, "Attaching to process %i... ", _pid);
+            fflush(stderr);
+            char* _cmdv0 = (_cmdc > 0) ? _cmdv[0] : nullptr;
+            mutatee      = _bpatch->processAttach(_cmdv0, _pid);
+            if(!mutatee)
+            {
+                verbprintf(-1, "Failed to connect to process %i\n", (int) _pid);
+                throw std::runtime_error("Failed to attach to process");
+            }
+            verbprintf_bare(1, "Done\n");
+        }
+        else
+        {
+            if(_cmdc < 1) errprintf(-127, "No command provided");
+
+            if(is_text_file(_cmdv[0]))
+            {
+                errprintf(-1,
+                          "'%s' is a text file. OmniTrace only supports instrumenting "
+                          "binary files",
+                          _cmdv[0]);
+            }
+
+            std::stringstream ss;
+            for(int i = 0; i < _cmdc; ++i)
+            {
+                if(!_cmdv || !_cmdv[i]) continue;
+                ss << " " << _cmdv[i];
+            }
+            auto _cmd_msg = ss.str();
+            if(_cmd_msg.length() > 1) _cmd_msg = _cmd_msg.substr(1);
+
+            verbprintf(1, "Creating process '%s'... ", _cmd_msg.c_str());
+            fflush(stderr);
+            mutatee = _bpatch->processCreate(_cmdv[0], (const char**) _cmdv, nullptr);
+            if(!mutatee)
+            {
+                verbprintf(-1, "Failed to create process: '%s'\n", _cmd_msg.c_str());
+                throw std::runtime_error("Failed to create process");
+            }
+            verbprintf_bare(1, "Done\n");
        }
-        verbprintf_bare(1, "Done\n");
    }

    return mutatee;
@@ -0,0 +1,28 @@
+# ------------------------------------------------------------------------------#
+#
+# omnitrace-run target
+#
+# ------------------------------------------------------------------------------#
+
+add_executable(
+    omnitrace-run
+    ${CMAKE_CURRENT_LIST_DIR}/omnitrace-run.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/omnitrace-run.hpp ${CMAKE_CURRENT_LIST_DIR}/impl.cpp)
+
+target_compile_definitions(omnitrace-run PRIVATE TIMEMORY_CMAKE=1)
+target_include_directories(omnitrace-run PRIVATE ${CMAKE_CURRENT_LIST_DIR})
+target_link_libraries(
+    omnitrace-run
+    PRIVATE omnitrace::omnitrace-compile-definitions omnitrace::omnitrace-headers
+            omnitrace::omnitrace-common-library omnitrace::omnitrace-core
+            omnitrace::omnitrace-sanitizer)
+set_target_properties(
+    omnitrace-run PROPERTIES BUILD_RPATH "\$ORIGIN:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
+                             INSTALL_RPATH "${OMNITRACE_EXE_INSTALL_RPATH}")
+
+omnitrace_strip_target(omnitrace-run)
+
+install(
+    TARGETS omnitrace-run
+    DESTINATION ${CMAKE_INSTALL_BINDIR}
+    OPTIONAL)
@@ -0,0 +1,359 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "omnitrace-run.hpp"
+
+#include "common/defines.h"
+#include "common/delimit.hpp"
+#include "common/environment.hpp"
+#include "common/join.hpp"
+#include "common/setup.hpp"
+#include "core/argparse.hpp"
+#include "core/config.hpp"
+#include "core/state.hpp"
+#include "core/timemory.hpp"
+
+#include <timemory/environment.hpp>
+#include <timemory/environment/types.hpp>
+#include <timemory/log/color.hpp>
+#include <timemory/settings/types.hpp>
+#include <timemory/settings/vsettings.hpp>
+#include <timemory/signals/signal_handlers.hpp>
+#include <timemory/utility/argparse.hpp>
+#include <timemory/utility/console.hpp>
+#include <timemory/utility/filepath.hpp>
+#include <timemory/utility/join.hpp>
+
+#include <array>
+#include <cctype>
+#include <chrono>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <regex>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <sys/wait.h>
+#include <thread>
+#include <unistd.h>
+#include <vector>
+
+namespace color    = ::tim::log::color;
+namespace filepath = ::tim::filepath;  // NOLINT
+namespace console  = ::tim::utility::console;
+namespace argparse = ::tim::argparse;
+namespace signals  = ::tim::signals;
+using settings     = ::omnitrace::settings;
+using namespace ::timemory::join;
+using ::tim::get_env;
+using ::tim::log::stream;
+
+namespace std
+{
+std::string
+to_string(bool _v)
+{
+    return (_v) ? "true" : "false";
+}
+}  // namespace std
+
+namespace
+{
+std::string
+get_internal_libpath(const std::string& _lib)
+{
+    auto _exe = std::string_view{ realpath("/proc/self/exe", nullptr) };
+    auto _pos = _exe.find_last_of('/');
+    auto _dir = std::string{ "./" };
+    if(_pos != std::string_view::npos) _dir = _exe.substr(0, _pos);
+    return omnitrace::common::join("/", _dir, "..", "lib", _lib);
+}
+
+parser_data_t&
+get_initial_environment(parser_data_t& _data)
+{
+    if(environ != nullptr)
+    {
+        int idx = 0;
+        while(environ[idx] != nullptr)
+        {
+            auto* _v = environ[idx++];
+            _data.initial.emplace(_v);
+            _data.current.emplace_back(strdup(_v));
+        }
+    }
+
+    return _data;
+}
+
+int
+get_verbose(parser_data_t& _data)
+{
+    auto& verbose = _data.verbose;
+    verbose       = get_env("OMNITRACE_CAUSAL_VERBOSE",
+                      get_env<int>("OMNITRACE_VERBOSE", verbose, false));
+    auto _debug =
+        get_env("OMNITRACE_CAUSAL_DEBUG", get_env<bool>("OMNITRACE_DEBUG", false, false));
+    if(_debug) verbose += 8;
+    return verbose;
+}
+
+std::string
+get_realpath(const std::string& _v)
+{
+    auto* _tmp = realpath(_v.c_str(), nullptr);
+    auto  _ret = std::string{ _tmp };
+    free(_tmp);
+    return _ret;
+}
+
+auto
+toggle_suppression(std::tuple<bool, bool> _inp)
+{
+    auto _out =
+        std::make_tuple(settings::suppress_config(), settings::suppress_parsing());
+    std::tie(settings::suppress_config(), settings::suppress_parsing()) = _inp;
+    return _out;
+}
+
+// disable suppression when exe loads but store original values for restoration later
+auto initial_suppression = toggle_suppression({ true, true });
+}  // namespace
+
+void
+print_command(const parser_data_t& _data, std::string_view _prefix)
+{
+    auto        verbose = _data.verbose;
+    const auto& _argv   = _data.command;
+    if(verbose >= 1)
+        stream(std::cout, color::info())
+            << _prefix << "Executing '" << join(array_config{ " " }, _argv) << "'...\n";
+
+    std::cerr << color::end() << std::flush;
+}
+
+void
+prepare_command_for_run(char* _exe, parser_data_t& _data)
+{
+    if(!_data.launcher.empty())
+    {
+        bool _injected = false;
+        auto _new_argv = std::vector<char*>{};
+        for(auto* itr : _data.command)
+        {
+            if(!_injected && std::regex_search(itr, std::regex{ _data.launcher }))
+            {
+                _new_argv.emplace_back(_exe);
+                _new_argv.emplace_back(strdup("--"));
+                _injected = true;
+            }
+            _new_argv.emplace_back(itr);
+        }
+
+        if(!_injected)
+        {
+            throw std::runtime_error(
+                join("", "omnitrace-run was unable to match \"", _data.launcher,
+                     "\" to any arguments on the command line: \"",
+                     join(array_config{ " ", "", "" }, _data.command), "\""));
+        }
+
+        std::swap(_data.command, _new_argv);
+    }
+}
+
+void
+prepare_environment_for_run(parser_data_t& _data)
+{
+    if(_data.launcher.empty())
+    {
+        omnitrace::argparse::add_ld_preload(_data);
+    }
+}
+
+void
+print_updated_environment(parser_data_t& _data, std::string_view _prefix)
+{
+    auto _verbose = get_verbose(_data);
+
+    if(_verbose < 0) return;
+
+    auto        _env          = _data.current;
+    const auto& _updated_envs = _data.updated;
+
+    std::sort(_env.begin(), _env.end(), [](auto* _lhs, auto* _rhs) {
+        if(!_lhs) return false;
+        if(!_rhs) return true;
+        return std::string_view{ _lhs } < std::string_view{ _rhs };
+    });
+
+    std::vector<std::string_view> _updates = {};
+    std::vector<std::string_view> _general = {};
+
+    for(auto* itr : _env)
+    {
+        if(itr == nullptr) continue;
+
+        auto _is_omni = (std::string_view{ itr }.find("OMNITRACE") == 0);
+        auto _updated = false;
+        for(const auto& vitr : _updated_envs)
+        {
+            if(std::string_view{ itr }.find(vitr) == 0)
+            {
+                _updated = true;
+                break;
+            }
+        }
+
+        if(_updated)
+            _updates.emplace_back(itr);
+        else if(_verbose >= 1 && _is_omni)
+            _general.emplace_back(itr);
+    }
+
+    if(_general.size() + _updates.size() == 0 || _verbose < 0) return;
+
+    std::cerr << std::endl;
+
+    for(auto& itr : _general)
+        stream(std::cerr, color::source()) << _prefix << itr << "\n";
+    for(auto& itr : _updates)
+        stream(std::cerr, color::source()) << _prefix << itr << "\n";
+
+    std::cerr << color::end() << std::flush;
+}
+
+parser_data_t&
+parse_args(int argc, char** argv, parser_data_t& _parser_data)
+{
+    get_initial_environment(_parser_data);
+
+    bool _do_parse_args = false;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string_view{ argv[i] };
+        if(_arg == "--" || _arg == "-?" || _arg == "-h" || _arg == "--help" ||
+           _arg == "--version")
+            _do_parse_args = true;
+    }
+
+    if(!_do_parse_args && argc > 1 && std::string_view{ argv[1] }.find('-') == 0)
+        _do_parse_args = true;
+
+    if(!_do_parse_args) return parse_command(argc, argv, _parser_data);
+
+    using parser_t     = argparse::argument_parser;
+    using parser_err_t = typename parser_t::result_type;
+
+    toggle_suppression(initial_suppression);
+    omnitrace::argparse::init_parser(_parser_data);
+
+    // no need for backtraces
+    signals::disable_signal_detection(signals::signal_settings::get_enabled());
+
+    auto help_check = [](parser_t& p, int _argc, char** _argv) {
+        std::set<std::string> help_args = { "-h", "--help", "-?" };
+        return (p.exists("help") || _argc == 1 ||
+                (_argc > 1 && help_args.find(_argv[1]) != help_args.end()));
+    };
+
+    const auto* _desc = R"desc(
+    Command line interface to omnitrace configuration.
+    )desc";
+
+    auto parser = parser_t{ basename(argv[0]), _desc };
+
+    parser.on_error([](parser_t&, const parser_err_t& _err) {
+        stream(std::cerr, color::fatal()) << _err << "\n";
+        exit(EXIT_FAILURE);
+    });
+
+    parser.enable_help("", "Usage: omnitrace-run <OPTIONS> -- <COMMAND> <ARGS>");
+    parser.enable_version("omnitrace-run", "v" OMNITRACE_VERSION_STRING,
+                          OMNITRACE_GIT_DESCRIBE, OMNITRACE_GIT_REVISION);
+
+    auto _cols = std::get<0>(console::get_columns());
+    if(_cols > parser.get_help_width() + 8)
+        parser.set_description_width(
+            std::min<int>(_cols - parser.get_help_width() - 8, 120));
+
+    // disable options related to causal profiling
+    _parser_data.processed_groups.emplace("causal");
+
+    omnitrace::argparse::add_core_arguments(parser, _parser_data);
+    omnitrace::argparse::add_extended_arguments(parser, _parser_data);
+
+    auto  _inpv = std::vector<char*>{};
+    auto& _outv = _parser_data.command;
+    bool  _hash = false;
+    for(int i = 0; i < argc; ++i)
+    {
+        if(argv[i] == nullptr)
+        {
+            continue;
+        }
+        else if(_hash)
+        {
+            _outv.emplace_back(strdup(argv[i]));
+        }
+        else if(std::string_view{ argv[i] } == "--")
+        {
+            _hash = true;
+        }
+        else
+        {
+            _inpv.emplace_back(strdup(argv[i]));
+        }
+    }
+
+    auto _cerr = parser.parse_args(_inpv.size(), _inpv.data());
+    if(_cerr)
+    {
+        std::cerr << _cerr.what() << std::endl;
+        exit(EXIT_FAILURE);
+    }
+
+    return _parser_data;
+}
+
+parser_data_t&
+parse_command(int argc, char** argv, parser_data_t& _parser_data)
+{
+    toggle_suppression(initial_suppression);
+    omnitrace::argparse::init_parser(_parser_data);
+
+    // no need for backtraces
+    signals::disable_signal_detection(signals::signal_settings::get_enabled());
+
+    auto& _outv = _parser_data.command;
+    bool  _hash = false;
+    for(int i = 1; i < argc; ++i)
+    {
+        _outv.emplace_back(strdup(argv[i]));
+    }
+
+    return _parser_data;
+}
@@ -0,0 +1,81 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "omnitrace-run.hpp"
+
+#include <timemory/log/color.hpp>
+#include <timemory/log/macros.hpp>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <string_view>
+#include <unistd.h>
+
+namespace
+{
+auto* _getenv_at_load = getenv("TIMEMORY_LIBRARY_CTOR");
+auto  _setenv_at_load = setenv("TIMEMORY_LIBRARY_CTOR", "0", 0);
+}  // namespace
+
+int
+main(int argc, char** argv)
+{
+    if(!_getenv_at_load)
+        unsetenv("TIMEMORY_LIBRARY_CTOR");
+    else
+        setenv("TIMEMORY_LIBRARY_CTOR", _getenv_at_load, 1);
+
+    auto _print_usage = [argv]() {
+        std::cerr << tim::log::color::fatal() << "Usage: " << argv[0]
+                  << " <OPTIONS> -- <COMMAND> <ARGS>" << std::endl;
+    };
+
+    if(argc == 1)
+    {
+        _print_usage();
+        return EXIT_FAILURE;
+    }
+
+    auto _parse_data = parser_data_t{};
+    parse_args(argc, argv, _parse_data);
+    prepare_command_for_run(argv[0], _parse_data);
+    prepare_environment_for_run(_parse_data);
+
+    auto& _argv = _parse_data.command;
+    auto& _envp = _parse_data.current;
+    if(!_argv.empty())
+    {
+        print_updated_environment(_parse_data, "OMNITRACE: ");
+        print_command(_parse_data, "OMNITRACE: ");
+        _argv.emplace_back(nullptr);
+        _envp.emplace_back(nullptr);
+        return execvpe(_argv.front(), _argv.data(), _envp.data());
+    }
+
+    _print_usage();
+    return EXIT_FAILURE;
+}
@@ -0,0 +1,53 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "core/argparse.hpp"
+
+#include <csignal>
+#include <map>
+#include <sched.h>
+#include <set>
+#include <string>
+#include <string_view>
+#include <vector>
+
+using parser_data_t = omnitrace::argparse::parser_data;
+
+void
+print_command(const parser_data_t&, std::string_view);
+
+void
+print_updated_environment(parser_data_t&, std::string_view);
+
+void
+prepare_command_for_run(char*, parser_data_t&);
+
+void
+prepare_environment_for_run(parser_data_t&);
+
+parser_data_t&
+parse_args(int argc, char** argv, parser_data_t&);
+
+parser_data_t&
+parse_command(int argc, char** argv, parser_data_t&);
@@ -117,7 +117,7 @@ print_command(const std::vector<char*>& _argv)
 std::vector<char*>
 get_initial_environment()
 {
-    std::vector<char*> _env;
+    auto _env = std::vector<char*>{};
    if(environ != nullptr)
    {
        int idx = 0;
@@ -497,7 +497,7 @@ parse_args(int argc, char** argv, std::vector<char*>& _env)
        .count(1)
        .dtype("filepath")
        .action([&](parser_t& p) {
-            update_env(_env, "OMNITRACE_PERFETTO_FILE", p.get<int64_t>("trace-file"));
+            update_env(_env, "OMNITRACE_PERFETTO_FILE", p.get<std::string>("trace-file"));
        });
    parser
        .add_argument({ "--trace-buffer-size" },
@@ -634,7 +634,7 @@ parse_args(int argc, char** argv, std::vector<char*>& _env)
        .requires({ "host" })
        .action([&](parser_t& p) {
            update_env(
-                _env, "OMNITRACE_PROCESS_SAMPLING_CPUS",
+                _env, "OMNITRACE_SAMPLING_CPUS",
                join(array_config{ "," }, p.get<std::vector<std::string>>("cpus")));
        });
    parser
@@ -644,7 +644,7 @@ parse_args(int argc, char** argv, std::vector<char*>& _env)
        .requires({ "device" })
        .action([&](parser_t& p) {
            update_env(
-                _env, "OMNITRACE_PROCESS_SAMPLING_GPUS",
+                _env, "OMNITRACE_SAMPLING_GPUS",
                join(array_config{ "," }, p.get<std::vector<std::string>>("gpus")));
        });

@@ -410,3 +410,119 @@ omnitrace_add_bin_test(
    PASS_REGEX
        "OMNITRACE_CONFIG_FILE(.*)OMNITRACE_ENABLED(.*)OMNITRACE_SUPPRESS_CONFIG(.*)OMNITRACE_SUPPRESS_PARSING(.*)OMNITRACE_VERBOSE"
    )
+
+omnitrace_add_bin_test(
+    NAME omnitrace-run-help
+    TARGET omnitrace-run
+    ARGS --help
+    TIMEOUT 45
+    LABELS "omnitrace-run")
+
+file(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/omnitrace-tests-config")
+file(
+    WRITE "${PROJECT_BINARY_DIR}/omnitrace-tests-config/empty.cfg"
+    "
+#
+# empty config file
+#
+")
+
+add_executable(sleeper ${CMAKE_CURRENT_SOURCE_DIR}/sleeper.cpp)
+set_target_properties(
+    sleeper PROPERTIES BUILD_TYPE RelWithDebInfo RUNTIME_OUTPUT_DIRECTORY
+                                                 ${PROJECT_BINARY_DIR}/bin/testing)
+
+omnitrace_add_bin_test(
+    NAME omnitrace-run-args
+    TARGET omnitrace-run
+    ARGS --monochrome
+         --debug=false
+         -v
+         1
+         -c
+         %env{TWD}%/omnitrace-tests-config/empty.cfg
+         -o
+         omnitrace-tests-output
+         omnitrace-run-args-output/
+         -TPHD
+         -S
+         cputime
+         realtime
+         --trace-wait=1.0e-12
+         --trace-duration=5.0
+         --wait=1.0
+         --duration=3.0
+         --trace-file=perfetto-run-args-trace.proto
+         --trace-buffer-size=100
+         --trace-fill-policy=ring_buffer
+         --profile-format
+         console
+         json
+         text
+         --process-freq
+         1000
+         --process-wait
+         0.0
+         --process-duration
+         10
+         --cpus
+         0-4
+         --gpus
+         0
+         -f
+         1000
+         --sampling-wait
+         1.0
+         --sampling-duration
+         10
+         -t
+         0-3
+         --sample-cputime
+         1000
+         1.0
+         0-3
+         --sample-realtime
+         10
+         0.5
+         0-3
+         -I
+         all
+         -E
+         mutex-locks
+         rw-locks
+         spin-locks
+         -C
+         perf::INSTRUCTIONS
+         --inlines
+         --hsa-interrupt
+         0
+         --use-causal=false
+         --use-kokkosp
+         --num-threads-hint=4
+         --sampling-allocator-size=32
+         --ci
+         --dl-verbose=3
+         --perfetto-annotations=off
+         --kokkosp-kernel-logger
+         --kokkosp-name-length-max=1024
+         --kokkosp-prefix="[kokkos]"
+         --tmpdir
+         ${CMAKE_BINARY_DIR}/omnitrace-tests-config/tmpdir
+         --perfetto-backend
+         inprocess
+         --use-pid
+         false
+         --time-output
+         off
+         --thread-pool-size
+         0
+         --timemory-components
+         wall_clock
+         cpu_clock
+         peak_rss
+         page_rss
+         --
+         $<TARGET_FILE:sleeper>
+         5
+    TIMEOUT 45
+    LABELS "omnitrace-run")
@@ -0,0 +1,28 @@
+
+#include <chrono>
+#include <ratio>
+#include <sstream>
+#include <thread>
+
+using clock_type = std::chrono::steady_clock;
+
+int
+main(int argc, char** argv)
+{
+    double _val = 0.0;
+    if(argc > 0)
+    {
+        auto _ss = std::stringstream{};
+        _ss << argv[1];
+        _ss >> _val;
+    }
+
+    intmax_t _nsec = _val * std::nano::den;
+    auto     _end  = clock_type::now() + std::chrono::nanoseconds{ _nsec };
+    while(clock_type::now() < _end)
+    {
+        std::this_thread::sleep_for(std::chrono::nanoseconds{ _nsec / 10 });
+    }
+
+    return 0;
+}