Release 0.0.2 (#14)

* Fixed Dyninst TBB symbolic links + bump to v0.0.2 * hosttrace exe and library updates + submodule updates - Updated dyninst submodule with TBB build ORIGIN rpath - Updated timemory submodule - Dyninst build with shared libs - Dockerfile for building packaging - Disable hidden viz in examples - parallel-overhead max parallelism - query_instr in hosttrace - different file-line info format - full module names - minor fix to MPI support - disable instrumention stack frames by default - disable trap instrumentation by default - updated hosttrace output file dumps - removed cstdlib option - dyninst DebugParsing option - improved instrument_module function - fixed some MPI support - tweaked some testing parameters
2021-09-29 18:16:13 -05:00
@@ -12,7 +12,7 @@ endif()
 project(
    hosttrace
    LANGUAGES C CXX
-    VERSION 0.0.1)
+    VERSION 0.0.2)

 message(
    STATUS
@@ -108,8 +108,9 @@ if(DYNINST_API_RT)
 endif()

 set_target_properties(
-    hosttrace-library PROPERTIES OUTPUT_NAME hosttrace INSTALL_RPATH
-                                                       "\$ORIGIN:${DYNINST_API_RT_DIR}")
+    hosttrace-library
+    PROPERTIES OUTPUT_NAME hosttrace
+               INSTALL_RPATH "\$ORIGIN:\$ORIGIN/dyninst-tpls/libs:${DYNINST_API_RT_DIR}")

 install(
    TARGETS hosttrace-library
@@ -138,9 +139,12 @@ target_link_libraries(

 set_target_properties(
    hosttrace-exe
-    PROPERTIES OUTPUT_NAME hosttrace
-               INSTALL_RPATH_USE_LINK_PATH ON
-               INSTALL_RPATH "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:${DYNINST_API_RT_DIR}")
+    PROPERTIES
+        OUTPUT_NAME hosttrace
+        INSTALL_RPATH_USE_LINK_PATH ON
+        INSTALL_RPATH
+        "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib:${DYNINST_API_RT_DIR}"
+    )

 install(
    TARGETS hosttrace-exe
@@ -78,13 +78,13 @@ if(HOSTTRACE_BUILD_DYNINST)
        OFF
        CACHE BOOL "Build dyninst parseThat executable")
    set(DYNINST_BUILD_SHARED_LIBS
-        OFF
+        ON
        CACHE BOOL "Build shared dyninst libraries")
    set(DYNINST_BUILD_STATIC_LIBS
-        ON
+        OFF
        CACHE BOOL "Build static dyninst libraries")
    set(DYNINST_ENABLE_LTO
-        ON
+        OFF
        CACHE BOOL "Enable LTO for dyninst libraries")

    hosttrace_save_variables(PIC VARIABLES CMAKE_POSITION_INDEPENDENT_CODE)
@@ -0,0 +1,26 @@
+
+FROM ubuntu:20.04
+
+ENV HOME /root
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US
+ENV LC_ALL C
+ENV SHELL /bin/bash
+ENV BASH_ENV /etc/bash.bashrc
+ENV DEBIAN_FRONTEND noninteractive
+
+WORKDIR /tmp
+SHELL [ "/bin/bash", "-c" ]
+
+RUN apt-get update && \
+    apt-get dist-upgrade -y && \
+    apt-get install -y build-essential cmake libnuma-dev wget gnupg2 m4 && \
+    wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \
+    echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | tee /etc/apt/sources.list.d/rocm.list && \
+    apt-get update && \
+    apt-get dist-upgrade -y && \
+    apt-get install -y rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev
+
+ENV LC_ALL C.UTF-8
+WORKDIR /home
+SHELL [ "/bin/bash", "--login", "-c" ]
@@ -2,5 +2,8 @@ cmake_minimum_required(VERSION 3.15 FATAL_ERROR)

 project(hosttrace-dyninst-examples LANGUAGES CXX)

+set(CMAKE_VISIBILITY_INLINES_HIDDEN OFF)
+set(CMAKE_CXX_VISIBILITY_PRESET "default")
+
 add_subdirectory(transpose)
 add_subdirectory(parallel-overhead)
@@ -29,7 +29,7 @@ run(size_t nitr, long n)
 int
 main(int argc, char** argv)
 {
-    size_t nthread = 16;
+    size_t nthread = std::min<size_t>(16, std::thread::hardware_concurrency());
    size_t nitr    = 50000;
    long   nfib    = 10;
    if(argc > 1) nfib = atol(argv[1]);
@@ -122,7 +122,6 @@ static bool binary_rewrite   = 0;
 static bool loop_level_instr = false;
 static bool werror           = false;
 static bool stl_func_instr   = false;
-static bool cstd_func_instr  = false;
 static bool use_mpi          = false;
 static bool is_static_exe    = false;
 static bool use_return_info  = false;
@@ -200,11 +199,16 @@ function_signature
 get_loop_file_line_info(module_t* mutatee_module, procedure_t* f, flow_graph_t* cfGraph,
                        basic_loop_t* loopToInstrument);

+bool
+query_instr(procedure_t* funcToInstr, procedure_loc_t traceLoc,
+            flow_graph_t* cfGraph = nullptr, basic_loop_t* loopToInstrument = nullptr,
+            bool allow_traps = true);
+
 template <typename Tp>
-void
+bool
 insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,
             procedure_loc_t traceLoc, flow_graph_t* cfGraph = nullptr,
-             basic_loop_t* loopToInstrument = nullptr);
+             basic_loop_t* loopToInstrument = nullptr, bool allow_traps = true);

 void
 errorFunc(error_level_t level, int num, const char** params);
@@ -321,19 +325,17 @@ struct function_signature
        {
            if(m_info_end)
            {
-                ss << '/' << "[{" << m_row.first << "," << m_col.first << "}-{"
-                   << m_row.second << "," << m_col.second << "}]";
+                ss << " [{" << m_row.first << "," << m_col.first << "}-{" << m_row.second
+                   << "," << m_col.second << "}]";
            }
            else
            {
                ss << "[{" << m_row.first << "," << m_col.first << "}]";
            }
        }
-        else
-        {
-            if(use_file_info && m_file.length() > 0) ss << '/' << m_file;
-            if(use_line_info && m_row.first > 0) ss << ":" << m_row.first;
-        }
+        if(use_file_info && m_file.length() > 0) ss << " [" << m_file;
+        if(use_line_info && m_row.first > 0) ss << ":" << m_row.first;
+        if(use_file_info && m_file.length() > 0) ss << "]";

        m_signature = ss.str();
        return m_signature;
@@ -369,18 +371,25 @@ struct module_function
    }

    module_function(const string_t& _module, const string_t& _func,
-                    const function_signature& _sign)
+                    const function_signature& _sign, procedure_t* proc)
    : module(_module)
    , function(_func)
    , signature(_sign)
-    {}
+    {
+        if(proc)
+        {
+            std::pair<address_t, address_t> _range{};
+            if(proc->getAddressRange(_range.first, _range.second))
+                address_range = _range.second - _range.first;
+        }
+    }

    module_function(module_t* mod, procedure_t* proc)
    {
        char modname[FUNCNAMELEN];
        char fname[FUNCNAMELEN];

-        mod->getName(modname, FUNCNAMELEN);
+        mod->getFullName(modname, FUNCNAMELEN);
        proc->getName(fname, FUNCNAMELEN);

        module    = modname;
@@ -431,7 +440,7 @@ struct module_function
        };

        // clang-format off
-        ss << std::setw(14) << rhs.address_range << " "
+        ss << std::setw(14) << rhs.address_range << "  "
           << std::setw(w0 + 8) << std::left << _get_str(rhs.module) << " "
           << std::setw(w1 + 8) << std::left << _get_str(rhs.function) << " "
           << std::setw(w2 + 8) << std::left << _get_str(rhs.signature.get());
@@ -71,6 +71,7 @@ extern "C"
    if(get_debug())                                                                      \
    {                                                                                    \
        fprintf(stderr, __VA_ARGS__);                                                    \
+        fflush(stderr);                                                                  \
    }

 //--------------------------------------------------------------------------------------//
@@ -443,8 +443,8 @@ c_stdlib_function_constraint(const std::string& _func)
 inline void
 consume()
 {
-    consume_parameters(initialize_expr, bpatch, use_mpi, stl_func_instr, cstd_func_instr,
-                       werror, loop_level_instr, error_print, binary_rewrite, debug_print,
+    consume_parameters(initialize_expr, bpatch, use_mpi, stl_func_instr, werror,
+                       loop_level_instr, error_print, binary_rewrite, debug_print,
                       expect_error, is_static_exe, available_module_functions,
                       instrumented_module_functions);
 }
@@ -31,6 +31,8 @@
 static bool     is_driver                                         = false;
 static bool     allow_overlapping                                 = false;
 static bool     instr_dynamic_callsites                           = false;
+static bool     instr_traps                                       = false;
+static bool     instr_loop_traps                                  = false;
 static size_t   batch_size                                        = 50;
 static strset_t extra_libs                                        = {};
 static size_t   min_address_range                                 = (1 << 9);  // 512
@@ -54,7 +56,7 @@ static string_t    instr_pop_hash     = "hosttrace_pop_trace_hash";
 static string_t    print_instrumented = {};
 static string_t    print_available    = {};
 static string_t    print_overlapping  = {};
-static std::string modfunc_dump_dir   = "hosttrace-module-functions";
+static std::string modfunc_dump_dir   = {};

 std::string
 get_absolute_exe_filepath(std::string exe_name);
@@ -114,14 +116,14 @@ main(int argc, char** argv)
    bpatch->setTypeChecking(true);
    bpatch->setSaveFPR(true);
    bpatch->setDelayedParsing(true);
-    bpatch->setInstrStackFrames(true);
+    bpatch->setDebugParsing(false);
+    bpatch->setInstrStackFrames(false);
    bpatch->setLivenessAnalysis(false);
    bpatch->setBaseTrampDeletion(false);
    bpatch->setTrampRecursive(false);
    bpatch->setMergeTramp(false);

-    std::set<std::string> dyninst_defs = { "TypeChecking", "SaveFPR", "DelayedParsing",
-                                           "InstrStackFrames" };
+    std::set<std::string> dyninst_defs = { "TypeChecking", "SaveFPR", "DelayedParsing" };

    int _argc = argc;
    int _cmdc = 0;
@@ -329,14 +331,35 @@ main(int argc, char** argv)
    parser
        .add_argument({ "--min-address-range-loop" },
                      "If the address range of a function containing a loop is less than "
-                      "this value, "
-                      "exclude it from instrumentation")
+                      "this value, exclude it from instrumentation")
        .count(1)
        .dtype("size_t")
        .set_default(min_loop_address_range)
        .action([](parser_t& p) {
            min_loop_address_range = p.get<size_t>("min-address-range-loop");
        });
+    parser
+        .add_argument(
+            { "--traps" },
+            "Instrument points which require using a trap. On the x86 architecture, "
+            "because instructions are of variable size, the instruction at a point may "
+            "be too small for Dyninst to replace it with the normal code sequence used "
+            "to call instrumentation. Also, when instrumentation is placed at points "
+            "other than subroutine entry, exit, or call points, traps may be used to "
+            "ensure the instrumentation fits. In this case, Dyninst replaces the "
+            "instruction with a single-byte instruction that generates a trap.")
+        .max_count(1)
+        .dtype("bool")
+        .set_default(instr_traps)
+        .action([](parser_t& p) { instr_traps = p.get<bool>("traps"); });
+    parser
+        .add_argument({ "--loop-traps" },
+                      "Instrument points within a loop which require using a trap (only "
+                      "relevant when --instrument-loops is enabled).")
+        .max_count(1)
+        .dtype("bool")
+        .set_default(instr_loop_traps)
+        .action([](parser_t& p) { instr_loop_traps = p.get<bool>("loop-traps"); });
    parser.add_argument()
        .names({ "--allow-overlapping" })
        .description(
@@ -352,7 +375,6 @@ main(int argc, char** argv)
            "function lists, e.g. {print-dir}/available.txt")
        .count(1)
        .dtype("string")
-        .set_default(modfunc_dump_dir)
        .action([](parser_t& p) { modfunc_dump_dir = p.get<std::string>("print-dir"); });
    parser
        .add_argument(
@@ -397,8 +419,9 @@ main(int argc, char** argv)
        .add_argument({ "--dyninst-options" },
                      "Advanced dyninst options: BPatch::set<OPTION>(bool), e.g. "
                      "bpatch->setTrampRecursive(true)")
-        .choices({ "TypeChecking", "SaveFPR", "DelayedParsing", "InstrStackFrames",
-                   "TrampRecursive", "MergeTramp", "BaseTrampDeletion" });
+        .choices({ "TypeChecking", "SaveFPR", "DebugParsing", "DelayedParsing",
+                   "InstrStackFrames", "TrampRecursive", "MergeTramp",
+                   "BaseTrampDeletion" });

    string_t extra_help = "-- <CMD> <ARGS>";
    auto     err        = parser.parse(_argc, _argv);
@@ -483,8 +506,6 @@ main(int argc, char** argv)

    if(parser.exists("S")) stl_func_instr = true;

-    if(parser.exists("cstdlib")) cstd_func_instr = true;
-
    if(parser.exists("mpi")) use_mpi = true;

    if(parser.exists("p")) _pid = parser.get<int>("p");
@@ -554,6 +575,15 @@ main(int argc, char** argv)
    fini_stub_names = parser.get<strvec_t>("fini-functions");
    auto env_vars   = parser.get<strvec_t>("env");

+    if(modfunc_dump_dir.empty())
+    {
+        auto _exe_base = (binary_rewrite) ? outfile : std::string{ cmdv0 };
+        auto _pos      = _exe_base.find_last_of('/');
+        if(_pos != std::string::npos && _pos + 1 < _exe_base.length())
+            _exe_base = _exe_base.substr(_pos + 1);
+        modfunc_dump_dir = TIMEMORY_JOIN("-", "hosttrace", _exe_base, "output");
+    }
+
    if(verbose_level >= 0) tim::makedir(modfunc_dump_dir);

    //----------------------------------------------------------------------------------//
@@ -622,8 +652,9 @@ main(int argc, char** argv)
    bpatch->setTypeChecking(get_dyninst_option("TypeChecking"));
    bpatch->setSaveFPR(get_dyninst_option("SaveFPR"));
    bpatch->setDelayedParsing(get_dyninst_option("DelayedParsing"));
-    bpatch->setInstrStackFrames(get_dyninst_option("InstrStackFrames"));

+    bpatch->setDebugParsing(get_dyninst_option("DebugParsing"));
+    bpatch->setInstrStackFrames(get_dyninst_option("InstrStackFrames"));
    bpatch->setTrampRecursive(get_dyninst_option("TrampRecursive"));
    bpatch->setMergeTramp(get_dyninst_option("MergeTramp"));
    bpatch->setBaseTrampDeletion(get_dyninst_option("BaseTrampDeletion"));
@@ -764,21 +795,28 @@ main(int argc, char** argv)
            module_function::update_width(itr);

        auto mwid = module_function::get_width().at(0);
-        auto ncol = 240 / std::min<size_t>(mwid, 240);
+        mwid      = std::min<size_t>(mwid, 90);
+        auto ncol = 180 / std::min<size_t>(mwid, 180);
        std::cout << "### MODULES ###\n| ";
        for(size_t i = 0; i < module_names.size(); ++i)
        {
            auto itr = module_names.begin();
            std::advance(itr, i);
-            std::cout << std::setw(mwid) << *itr << " | ";
+            std::string _v = *itr;
+            if(_v.length() >= mwid)
+            {
+                auto _resume = _v.length() - mwid + 15;
+                _v           = _v.substr(0, 12) + "..." + _v.substr(_resume);
+            }
+            std::cout << std::setw(mwid) << _v << " | ";
            if(i % ncol == ncol - 1) std::cout << "\n| ";
        }
        std::cout << '\n' << std::endl;
    }

-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "available.txt"),
+    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "available-instr.txt"),
              available_module_functions, 1);
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "overlapping.txt"),
+    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "overlapping-instr.txt"),
              overlapping_module_functions, 1);

    //----------------------------------------------------------------------------------//
@@ -1065,8 +1103,8 @@ main(int argc, char** argv)
            module_t* _module = _func->getModule();
            if(_module)
            {
-                char moduleName[MUTNAMELEN];
-                _module->getName(moduleName, MUTNAMELEN);
+                char moduleName[FUNCNAMELEN];
+                _module->getFullName(moduleName, FUNCNAMELEN);
                if(strcmp(moduleName, "DEFAULT_MODULE") != 0) _has_debug_info = true;
            }
        }
@@ -1283,16 +1321,26 @@ main(int argc, char** argv)
                   (unsigned long) procedures.size());
        for(auto* itr : procedures)
        {
-            if(itr == main_func) continue;
-
-            char modname[MUTNAMELEN];
+            if(!itr) continue;
+            char modname[FUNCNAMELEN];
            char fname[FUNCNAMELEN];

+            itr->getName(fname, FUNCNAMELEN);
            module_t* mod = itr->getModule();
            if(mod)
-                mod->getName(modname, MUTNAMELEN);
+                mod->getFullName(modname, FUNCNAMELEN);
            else
-                itr->getModuleName(modname, MUTNAMELEN);
+                itr->getModuleName(modname, FUNCNAMELEN);
+
+            if(itr == main_func)
+            {
+                hash_ids.emplace_back(std::hash<string_t>()(main_sign.get()),
+                                      main_sign.get());
+                auto main_mf = module_function(modname, fname, main_sign, itr);
+                available_module_functions.insert(main_mf);
+                instrumented_module_functions.insert(main_mf);
+                continue;
+            }

            if(!itr->isInstrumentable())
            {
@@ -1303,11 +1351,8 @@ main(int argc, char** argv)
            if(std::string{ modname }.find("libdyninst") != std::string::npos) continue;

            if(module_constraint(modname)) continue;
-
            if(!instrument_module(modname)) continue;

-            itr->getName(fname, FUNCNAMELEN);
-
            auto name = get_func_file_line_info(mod, itr);

            if(name.get().empty())
@@ -1317,7 +1362,6 @@ main(int argc, char** argv)
            }

            if(routine_constraint(name.m_name.c_str())) continue;
-
            if(!instrument_entity(name.m_name)) continue;

            if(is_static_exe && has_debug_info && string_t{ fname } == "_fini" &&
@@ -1392,6 +1436,29 @@ main(int argc, char** argv)
                           (unsigned long) min_address_range);
            }

+            bool _entr_success =
+                query_instr(itr, BPatch_entry, nullptr, nullptr, instr_traps);
+            bool _exit_success =
+                query_instr(itr, BPatch_exit, nullptr, nullptr, instr_traps);
+            if(!_entr_success && !_exit_success)
+            {
+                verbprintf(2,
+                           "Skipping function [insert-instr]: %s / %s. Either no entry "
+                           "instrumentation points were found or instrumentation "
+                           "required traps and instrumenting via traps were disabled.\n",
+                           name.m_name.c_str(), name.get().c_str());
+                continue;
+            }
+            else if(_entr_success && !_exit_success)
+            {
+                verbprintf(2,
+                           "Skipping function [insert-instr]: %s / %s. Function can be "
+                           "only partially instrumented: entry = %s, exit = %s\n",
+                           name.m_name.c_str(), name.get().c_str(),
+                           _entr_success ? "y" : "n", _exit_success ? "y" : "n");
+                continue;
+            }
+
            hash_ids.emplace_back(std::hash<string_t>()(name.get()), name.get());
            available_module_functions.insert(module_function(mod, itr));
            instrumented_module_functions.insert(module_function(mod, itr));
@@ -1408,8 +1475,10 @@ main(int argc, char** argv)
                auto _entr       = _trace_entr.get((entr_hash) ? entr_hash : entr_trace);
                auto _exit       = _trace_exit.get((exit_hash) ? exit_hash : exit_trace);

-                insert_instr(addr_space, itr, _entr, BPatch_entry, nullptr, nullptr);
-                insert_instr(addr_space, itr, _exit, BPatch_exit, nullptr, nullptr);
+                insert_instr(addr_space, itr, _entr, BPatch_entry, nullptr, nullptr,
+                             instr_traps);
+                insert_instr(addr_space, itr, _exit, BPatch_exit, nullptr, nullptr,
+                             instr_traps);
            };

            instr_procedure_functions.emplace_back(_f);
@@ -1421,6 +1490,31 @@ main(int argc, char** argv)

                for(auto* litr : basic_loop)
                {
+                    bool _lentr_success =
+                        query_instr(itr, BPatch_entry, cfg, litr, instr_loop_traps);
+                    bool _lexit_success =
+                        query_instr(itr, BPatch_exit, cfg, litr, instr_loop_traps);
+                    if(!_lentr_success && !_lexit_success)
+                    {
+                        verbprintf(
+                            2,
+                            "Skipping function [insert-instr-loop]: %s / %s. Either no "
+                            "entry instrumentation points were found or instrumentation "
+                            "required traps and instrumenting via traps were disabled.\n",
+                            name.m_name.c_str(), name.get().c_str());
+                        continue;
+                    }
+                    else if(_lentr_success && !_lexit_success)
+                    {
+                        verbprintf(
+                            2,
+                            "Skipping function [insert-instr-loop]: %s / %s. Function "
+                            "can be only partially instrumented: entry = %s, exit = %s\n",
+                            name.m_name.c_str(), name.get().c_str(),
+                            _lentr_success ? "y" : "n", _lexit_success ? "y" : "n");
+                        continue;
+                    }
+
                    auto lname  = get_loop_file_line_info(mod, itr, cfg, litr);
                    auto _lname = lname.get();
                    auto _lhash = std::hash<string_t>()(_lname);
@@ -1437,8 +1531,10 @@ main(int argc, char** argv)
                        auto _lexit =
                            _ltrace_exit.get((exit_hash) ? exit_hash : exit_trace);

-                        insert_instr(addr_space, itr, _lentr, BPatch_entry, cfg, litr);
-                        insert_instr(addr_space, itr, _lexit, BPatch_exit, cfg, litr);
+                        insert_instr(addr_space, itr, _lentr, BPatch_entry, cfg, litr,
+                                     instr_loop_traps);
+                        insert_instr(addr_space, itr, _lexit, BPatch_exit, cfg, litr,
+                                     instr_loop_traps);
                    };
                    instr_procedure_functions.emplace_back(_lf);
                }
@@ -1585,11 +1681,11 @@ main(int argc, char** argv)
    bool _dump_and_exit = ((print_available.length() + print_instrumented.length() +
                            print_overlapping.length()) > 0);

-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "available.txt"),
+    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "available-instr.txt"),
              available_module_functions, 0);
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "instrumented.txt"),
+    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "instrumented-instr.txt"),
              instrumented_module_functions, 0);
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "overlapping.txt"),
+    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir, "overlapping-instr.txt"),
              overlapping_module_functions, 0);

    auto _dump_info = [](string_t _mode, const fmodset_t& _modset) {
@@ -1824,12 +1920,15 @@ instrument_module(const string_t& file_name)
    static std::regex sys_regex("^(s|k|e|w)_[A-Za-z_0-9\\-]+\\.(c|C)$", regex_opts);
    static std::regex userlib_regex(
        "^(lib|)(hosttrace|caliper|gotcha|papi|cupti|TAU|likwid|"
-        "profiler|tcmalloc|dyninst|pfm|nvtx|upcxx|pthread|nvperf|hsa)",
+        "profiler|tcmalloc|dyninst|pfm|nvtx|upcxx|pthread|nvperf|hsa|\\.\\./sysdeps/|/"
+        "build/)",
        regex_opts);
-    static std::regex corelib_regex("^lib(rt-|dl-|util-|python)", regex_opts);
+    static std::regex corelib_regex("^lib(c|z|rt|dl|util|zstd|elf|dw|pthread|dyninstAPI_"
+                                    "RT|gcc_s|tbbmalloc|tbbmalloc_proxy)(-|\\.)",
+                                    regex_opts);
    // these are all due to TAU
    static std::regex prefix_regex(
-        "^(_|\\.|RT|Tau|Profiler|Rts|Papi|Py|Comp_xl\\.cpp|Comp_gnu\\.cpp|"
+        "^(_|\\.[a-zA-Z0-9]|RT|Tau|Profiler|Rts|Papi|Py|Comp_xl\\.cpp|Comp_gnu\\.cpp|"
        "UserEvent\\.cpp|FunctionInfo\\.cpp|PthreadLayer\\.cpp|"
        "Comp_intel[0-9]\\.cpp|Tracer\\.cpp)",
        regex_opts);
@@ -1845,13 +1944,6 @@ instrument_module(const string_t& file_name)
        "set[a-z_]+|^get[a-z_]+|^shm[a-z]+|^wc[a-z_]+|brk|^write[a-z]+)\\.c$",
        regex_opts);*/

-    /*if(!cstd_func_instr && c_stdlib_module_constraint(file_name))
-    {
-        verbprintf(3, "Excluding instrumentation [c std library] : '%s'...\n",
-                   file_name.c_str());
-        return false;
-    }*/
-
    if(std::regex_search(file_name, ext_regex))
    {
        return (_report("Excluding", "file extension", 3), false);
@@ -1952,12 +2044,6 @@ instrument_entity(const string_t& function_name)
        return false;
    }

-    /*if(!cstd_func_instr && c_stdlib_function_constraint(function_name))
-    {
-        verbprintf(3, "Excluding function [libc] : '%s'...\n", function_name.c_str());
-        return false;
-    }*/
-
    // don't instrument the functions when key is found anywhere in function name
    if(std::regex_search(function_name, exclude))
    {
@@ -1999,23 +2085,62 @@ instrument_entity(const string_t& function_name)

    bool use = is_include(true) && !is_exclude();
    if(use)
-        verbprintf(2, "Including function [user-regex] : '%s'...\n",
+        verbprintf(2, "Including function [no constraint] : '%s'...\n",
                   function_name.c_str());

    return use;
 }

+//======================================================================================//
+// query_instr -- check whether there are one or more instrumentation points
+//
+bool
+query_instr(procedure_t* funcToInstr, procedure_loc_t traceLoc, flow_graph_t* cfGraph,
+            basic_loop_t* loopToInstrument, bool allow_traps)
+{
+    module_t* module = funcToInstr->getModule();
+    if(!module) return false;
+
+    bpvector_t<point_t*>* _points = nullptr;
+
+    if(cfGraph && loopToInstrument)
+    {
+        if(traceLoc == BPatch_entry)
+            _points = cfGraph->findLoopInstPoints(BPatch_locLoopEntry, loopToInstrument);
+        else if(traceLoc == BPatch_exit)
+            _points = cfGraph->findLoopInstPoints(BPatch_locLoopExit, loopToInstrument);
+    }
+    else
+    {
+        _points = funcToInstr->findPoint(traceLoc);
+    }
+
+    if(_points == nullptr) return false;
+    if(_points->empty()) return false;
+
+    size_t _n = _points->size();
+    for(auto& itr : *_points)
+    {
+        if(!itr)
+            --_n;
+        else if(itr && !allow_traps && itr->usesTrap_NP())
+            --_n;
+    }
+
+    return (_n > 0);
+}
+
 //======================================================================================//
 // insert_instr -- generic insert instrumentation function
 //
 template <typename Tp>
-void
+bool
 insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,
             procedure_loc_t traceLoc, flow_graph_t* cfGraph,
-             basic_loop_t* loopToInstrument)
+             basic_loop_t* loopToInstrument, bool allow_traps)
 {
    module_t* module = funcToInstr->getModule();
-    if(!module || !traceFunc) return;
+    if(!module || !traceFunc) return false;

    bpvector_t<point_t*>* _points = nullptr;
    auto                  _trace  = traceFunc.get();
@@ -2032,8 +2157,8 @@ insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,
        _points = funcToInstr->findPoint(traceLoc);
    }

-    if(_points == nullptr) return;
-    if(_points->empty()) return;
+    if(_points == nullptr) return false;
+    if(_points->empty()) return false;

    /*if(loop_level_instr)
    {
@@ -2056,9 +2181,19 @@ insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,

    // verbprintf(0, "Instrumenting |> [ %s ]\n", name.m_name.c_str());

+    std::set<point_t*> _traps{};
+    if(!allow_traps)
+    {
+        for(auto& itr : *_points)
+        {
+            if(itr && itr->usesTrap_NP()) _traps.insert(itr);
+        }
+    }
+
+    size_t _n = 0;
    for(auto& itr : *_points)
    {
-        if(!itr)
+        if(!itr || _traps.count(itr) > 0)
            continue;
        else if(traceLoc == BPatch_entry)
            mutatee->insertSnippet(*_trace, *itr, BPatch_callBefore, BPatch_firstSnippet);
@@ -2067,8 +2202,12 @@ insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,
        //    BPatch_firstSnippet);
        else
            mutatee->insertSnippet(*_trace, *itr);
+        ++_n;
    }
+
+    return (_n > 0);
 }
+
 //======================================================================================//
 // Constraints for instrumentation. Returns true for those modules that
 // shouldn't be instrumented.
@@ -561,9 +561,11 @@ extern "C"
        {
            auto& _main_bundle = get_main_bundle();
            _main_bundle->start();
+#if defined(TIMEMORY_USE_MPI)
            tim::set_env("HOSTTRACE_USE_MPI", "ON", 1);
            get_use_mpi() = true;
-            get_state()   = State::DelayedInit;
+#endif
+            get_state() = State::DelayedInit;
        }
    }
 }
@@ -17,7 +17,7 @@ if(TARGET transpose)
        NAME transpose-binary-rewrite
        COMMAND
            $<TARGET_FILE:hosttrace-exe> -o $<TARGET_FILE_DIR:transpose>/transpose.inst -v
-            2 -- $<TARGET_FILE:transpose>
+            1 -- $<TARGET_FILE:transpose>
        WORKING_DIRECTORY ${PROJECT_BINARY_DIR})

    add_test(
@@ -27,7 +27,8 @@ if(TARGET transpose)

    add_test(
        NAME transpose-runtime-instrument
-        COMMAND $<TARGET_FILE:hosttrace-exe> -v 2 -- $<TARGET_FILE:transpose>
+        COMMAND $<TARGET_FILE:hosttrace-exe> -v 1 --label file line return args --
+                $<TARGET_FILE:transpose>
        WORKING_DIRECTORY ${PROJECT_BINARY_DIR})

    set_tests_properties(transpose-binary-rewrite-run PROPERTIES DEPENDS
@@ -43,7 +44,8 @@ if(TARGET parallel-overhead)
        NAME parallel-overhead-binary-rewrite
        COMMAND
            $<TARGET_FILE:hosttrace-exe> -o
-            $<TARGET_FILE_DIR:parallel-overhead>/parallel-overhead.inst -v 2 --
+            $<TARGET_FILE_DIR:parallel-overhead>/parallel-overhead.inst -v 1
+            --min-address-range-loop=72 --label file line return args --
            $<TARGET_FILE:parallel-overhead>
        WORKING_DIRECTORY ${PROJECT_BINARY_DIR})

@@ -54,7 +56,7 @@ if(TARGET parallel-overhead)

    add_test(
        NAME parallel-overhead-runtime-instrument
-        COMMAND $<TARGET_FILE:hosttrace-exe> -v 2 -- $<TARGET_FILE:parallel-overhead>
+        COMMAND $<TARGET_FILE:hosttrace-exe> -v 1 -- $<TARGET_FILE:parallel-overhead>
        WORKING_DIRECTORY ${PROJECT_BINARY_DIR})

    set_tests_properties(parallel-overhead-binary-rewrite-run