Code Coverage Support (#46)

* Code-coverage support * Examples update - code-coverage example - tweak transpose and parallel-overhead * Coverage output + testing - config::get_setting value(...) - REGULAR_EXPRESSION -> REGEX in cmake func args - coverage.hpp header - coverage JSON - coverage tests * cmake formatting * Library instrumentation w/o main + more - fixed library instrumentation w/o main - use TIMEMORY_PROJECT_NAME in output messages - removed '--driver' option from omnitrace exe - support coverage in trace mode - OMNITRACE_KOKKOS_KERNEL_LOGGER - support multiple calls to omnitrace_set_env after init if already called - support multiple calls to omnitrace_set_mpi after init if same args - support multiple calls to omnitrace_init if same mode - unique_ptr_t for thread_data which calls finalize when thread_data is destroyed - tweaked openmp tests - improved finalization * Replace CI --output-on-failure with -V * Fix to OMNITRACE_DL_INVOKE * omnitrace-exe and testing updates - omnitrace::omnitrace-timemory interface library - support for configs in omnitrace exe - print-{available,instrumented,...} opts no longer exit w/o --simulate - all tests apply --print-instrumented functions - tweaked coverage tests - print-* options print instructions not address range * Remove OMNITRACE_DEBUG_FINALIZE=ON from CI * Python cmake tweaks * Tweak test ordering * Upload CI artifacts if fail or success * CI Python tweaks - Use OMNITRACE_PYTHON_PREFIX and OMNITRACE_PYTHON_ENVS * CI ELFULTILS_DOWNLOAD_VERSION * test tweaks - labels and more coverage tests * tweak to omnitrace --config handling * Update module/function constraint handling + PP - tweak pre-processor definition handling - removed free-standing module_constraint - remove free-standing routine_constraint - remove module_name.find("omnitrace") module constraint - fully handle the output path of omnitrace *-instr files - get_use_code_coverage config option - print-coverage option - coverage_module_functions * use github.job not github.name * Re-enable HSA_ENABLE_INTERRUPT - remove coverage address report [ROCm/rocprofiler-systems commit: 791375bb24]
2022-04-25 17:00:52 -05:00
parent 28ade7fbb9
commit 72d0a7d08a
58 changed files with 2572 additions and 794 deletions
@@ -19,6 +19,11 @@ parse:
        VARIABLES: '*'
        CONDITION: '*'
    omnitrace_add_test:
+      flags:
+        - SKIP_BASELINE
+        - SKIP_REWRITE
+        - SKIP_RUNTIME
+        - SKIP_SAMPLING
      kwargs:
        NAME: '*'
        TARGET: '*'
@@ -32,6 +37,12 @@ parse:
        ENVIRONMENT: '*'
        LABELS: '*'
        PROPERTIES: '*'
+        RUNTIME_PASS_REGEX: '*'
+        RUNTIME_FAIL_REGEX: '*'
+        REWRITE_PASS_REGEX: '*'
+        REWRITE_FAIL_REGEX: '*'
+        REWRITE_RUN_PASS_REGEX: '*'
+        REWRITE_RUN_FAIL_REGEX: '*'
    omnitrace_target_compile_definitions:
      kwargs:
        PUBLIC: '*'
@@ -51,9 +62,9 @@ parse:
        PROPERTIES: '*'
        ENVIRONMENT: '*'
        WORKING_DIRECTORY: '*'
-        PASS_REGULAR_EXPRESSION: '*'
-        FAIL_REGULAR_EXPRESSION: '*'
-        SKIP_REGULAR_EXPRESSION: '*'
+        PASS_REGEX: '*'
+        FAIL_REGEX: '*'
+        SKIP_REGEX: '*'
    omnitrace_add_python_test:
      flags:
      - STANDALONE
@@ -70,9 +81,9 @@ parse:
        PROPERTIES: '*'
        PYTHON_EXECUTABLE: '*'
        PYTHON_VERSION: '*'
-        PASS_REGULAR_EXPRESSION: '*'
-        FAIL_REGULAR_EXPRESSION: '*'
-        SKIP_REGULAR_EXPRESSION: '*'
+        PASS_REGEX: '*'
+        FAIL_REGEX: '*'
+        SKIP_REGEX: '*'
    rocm_version_message:
      flags:
        - STATUS
@@ -105,6 +116,22 @@ parse:
        PYTHON_VERSION: '*'
        CXX_STANDARD: '*'
        VISIBILITY: '*'
+    omnitrace_directory:
+      flags:
+        - MKDIR
+        - FAIL
+      kwargs:
+        PREFIX: '*'
+        OUTPUT_VARIABLE: '*'
+        WORKING_DIRECTORY: '*'
+        PATHS: '*'
+    omnitrace_check_python_dirs_and_versions:
+      flags:
+        - UNSET
+        - FAIL
+      kwargs:
+        RESULT_VARIABLE: '*'
+        OUTPUT_VARIABLE: '*'
  override_spec: {}
  vartags: []
  proptags: []
@@ -9,7 +9,6 @@ on:
 env:
  BUILD_TYPE: Release
  ELFUTILS_DOWNLOAD_VERSION: 0.183
-  OMNITRACE_DEBUG_FINALIZE: ON
  OMNITRACE_VERBOSE: 1
  OMNITRACE_CI: ON
  GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
@@ -81,8 +80,8 @@ jobs:
          -DOMNITRACE_USE_MPI=${USE_MPI}
          -DOMNITRACE_USE_HIP=OFF
          -DOMNITRACE_USE_PYTHON=ON
-          -DOMNITRACE_PYTHON_VERSIONS="3.6;3.7;3.8;3.9"
-          -DOMNITRACE_PYTHON_ROOT_DIRS="/opt/conda/envs/py3.6;/opt/conda/envs/py3.7;/opt/conda/envs/py3.8;/opt/conda/envs/py3.9"
+          -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
+          -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9"
          -DLULESH_BUILD_KOKKOS=OFF

    - name: Build
@@ -100,8 +99,8 @@ jobs:
        cd build &&
        ldd ./omnitrace &&
        ./omnitrace --help &&
-        ctest -V -N -O omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log &&
-        ctest --output-on-failure --output-log omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure
+        ctest -V -N -O omnitrace-ctest-${{ github.job }}-commands.log &&
+        ctest -V --output-log omnitrace-ctest-${{ github.job }}.log --stop-on-failure

    - name: Test Install
      timeout-minutes: 10
@@ -126,15 +125,18 @@ jobs:
        omnitrace -e -v 1 -- ls

    - name: CTest Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: ctest-log
+        name: ctest-${{ github.job }}-log
        path: |
          build/*.log

    - name: Data Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: data-files
+        name: data-${{ github.job }}-files
        path: |
          build/omnitrace-tests-output/**/*.txt
+          build/omnitrace-tests-output/**/*-instr*.json
@@ -8,7 +8,6 @@ on:

 env:
  BUILD_TYPE: MinSizeRel
-  OMNITRACE_DEBUG_FINALIZE: ON
  OMNITRACE_VERBOSE: 1
  OMNITRACE_CI: ON
  OMNITRACE_OUTPUT_PATH: omnitrace-tests-output
@@ -84,8 +83,8 @@ jobs:
        cd build &&
        ldd ./omnitrace &&
        ./omnitrace --help &&
-        ctest -V -N -O omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log &&
-        ctest --output-on-failure --output-log omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure
+        ctest -V -N -O omnitrace-ctest-${{ github.job }}-commands.log &&
+        ctest -V --output-log omnitrace-ctest-${{ github.job }}.log --stop-on-failure

    - name: Test Install
      timeout-minutes: 10
@@ -108,16 +107,19 @@ jobs:
        omnitrace -e -v 1 -- ls

    - name: CTest Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: ctest-log
+        name: ctest-${{ github.job }}-log
        path: |
          build/*.log

    - name: Data Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: data-files
+        name: data-${{ github.job }}-files
        path: |
          omnitrace-tests-output/**/*.txt
          build/omnitrace-tests-output/**/*.txt
+          build/omnitrace-tests-output/**/*-instr*.json
@@ -9,7 +9,6 @@ on:
 env:
  BUILD_TYPE: RelWithDebInfo
  ELFUTILS_DOWNLOAD_VERSION: 0.183
-  OMNITRACE_DEBUG_FINALIZE: ON
  OMNITRACE_VERBOSE: 1
  OMNITRACE_CI: ON

@@ -57,8 +56,8 @@ jobs:
          -DOMNITRACE_USE_HIP=OFF
          -DOMNITRACE_USE_OMPT=OFF
          -DOMNITRACE_USE_PYTHON=ON
-          -DOMNITRACE_PYTHON_VERSIONS="3.6;3.7;3.8;3.9"
-          -DOMNITRACE_PYTHON_ROOT_DIRS="/opt/conda/envs/py3.6;/opt/conda/envs/py3.7;/opt/conda/envs/py3.8;/opt/conda/envs/py3.9"
+          -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
+          -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9"

    - name: Build
      timeout-minutes: 45
@@ -76,8 +75,8 @@ jobs:
        cd build &&
        ldd ./omnitrace &&
        ./omnitrace --help &&
-        ctest -V -N -O omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log &&
-        ctest --output-on-failure --output-log omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure
+        ctest -V -N -O omnitrace-ctest-${{ github.job }}-commands.log &&
+        ctest -V --output-log omnitrace-ctest-${{ github.job }}.log --stop-on-failure

    - name: Test Install
      timeout-minutes: 10
@@ -102,15 +101,18 @@ jobs:
        omnitrace -e -v 1 -- ls

    - name: CTest Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: ctest-log
+        name: ctest-${{ github.job }}-log
        path: |
          build/*.log

    - name: Data Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: data-files
+        name: data-${{ github.job }}-files
        path: |
          build/omnitrace-tests-output/**/*.txt
+          build/omnitrace-tests-output/**/*-instr*.json
@@ -8,8 +8,7 @@ on:

 env:
  BUILD_TYPE: Release
-  ELFUTILS_DOWNLOAD_VERSION: 0.183
-  OMNITRACE_DEBUG_FINALIZE: ON
+  ELFUTILS_DOWNLOAD_VERSION: 0.186
  OMNITRACE_VERBOSE: 1
  OMNITRACE_CI: ON

@@ -60,6 +59,7 @@ jobs:
          -DDYNINST_BUILD_LIBIBERTY=ON
          -DDYNINST_BUILD_SHARED_LIBS=ON
          -DDYNINST_BUILD_STATIC_LIBS=OFF
+          -DDYNINST_ELFUTILS_DOWNLOAD_VERSION=${{ env.ELFUTILS_DOWNLOAD_VERSION }}

    - name: Build
      timeout-minutes: 45
@@ -74,8 +74,8 @@ jobs:
      timeout-minutes: 45
      working-directory: ${{ github.workspace }}/build
      run:
-        ctest -V -N -O ${{ github.workspace }}/build/omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log &&
-        ctest --output-on-failure --output-log ${{ github.workspace }}/build/omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure
+        ctest -V -N -O ${{ github.workspace }}/build/omnitrace-ctest-${{ github.job }}-commands.log &&
+        ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-${{ github.job }}.log --stop-on-failure

    - name: Test Install
      timeout-minutes: 10
@@ -100,15 +100,18 @@ jobs:
        omnitrace -e -v 1 -- ls

    - name: CTest Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: ctest-log
+        name: ctest-${{ github.job }}-log
        path: |
          ${{ github.workspace }}/build/*.log

    - name: Data Artifacts
+      if: success() || failure()
      uses: actions/upload-artifact@v2
      with:
-        name: data-files
+        name: data-${{ github.job }}-files
        path: |
          ${{ github.workspace }}/build/omnitrace-tests-output/**/*.txt
+          ${{ github.workspace }}/build/omnitrace-tests-output/**/*-instr*.json
@@ -556,4 +556,206 @@ function(omnitrace_custom_compilation)
    endif()
 endfunction()

+function(OMNITRACE_WATCH_FOR_CHANGE _var)
+    list(LENGTH ARGN _NUM_EXTRA_ARGS)
+    if(_NUM_EXTRA_ARGS EQUAL 1)
+        set(_VAR ${ARGN})
+    else()
+        set(_VAR)
+    endif()
+
+    macro(update_var _VAL)
+        if(_VAR)
+            set(${_VAR}
+                ${_VAL}
+                PARENT_SCOPE)
+        endif()
+    endmacro()
+
+    update_var(OFF)
+
+    set(_omnitrace_watch_var_name OMNITRACE_WATCH_VALUE_${_var})
+    if(DEFINED ${_omnitrace_watch_var_name})
+        if("${${_var}}" STREQUAL "${${_omnitrace_watch_var_name}}")
+            return()
+        else()
+            omnitrace_message(
+                STATUS
+                "${_var} changed :: ${${_omnitrace_watch_var_name}} --> ${${_var}}")
+            update_var(ON)
+        endif()
+    else()
+        if(NOT "${${_var}}" STREQUAL "")
+            omnitrace_message(STATUS "${_var} :: ${${_var}}")
+            update_var(ON)
+        endif()
+    endif()
+
+    # store the value for the next run
+    set(${_omnitrace_watch_var_name}
+        "${${_var}}"
+        CACHE INTERNAL "Last value of ${_var}" FORCE)
+endfunction()
+
+function(OMNITRACE_DIRECTORY)
+    cmake_parse_arguments(F "MKDIR;FAIL;FORCE" "PREFIX;OUTPUT_VARIABLE;WORKING_DIRECTORY"
+                          "PATHS" ${ARGN})
+
+    if(F_PREFIX AND NOT IS_ABSOLUTE "${F_PREFIX}")
+        if(F_WORKING_DIRECTORY)
+            omnitrace_message(
+                STATUS
+                "PREFIX was specified as a relative path, using working directory + prefix :: '${F_WORKING_DIRECTORY}/${F_PREFIX}'..."
+                )
+            set(F_PREFIX ${F_WORKING_DIRECTORY}/${F_PREFIX})
+        else()
+            omnitrace_message(
+                FATAL_ERROR
+                "PREFIX was specified but it is not an absolute path: ${F_PREFIX}")
+        endif()
+    endif()
+
+    if(NOT F_WORKING_DIRECTORY)
+        set(F_WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+    endif()
+
+    foreach(_PATH ${F_PREFIX} ${F_PATHS})
+        if(F_PREFIX AND NOT "${_PATH}" STREQUAL "${F_PREFIX}")
+            # if path is relative, set to prefix + path
+            if(NOT IS_ABSOLUTE "${_PATH}")
+                set(_PATH ${F_PREFIX}/${_PATH})
+            endif()
+            list(APPEND _OUTPUT_VAR ${_PATH})
+        elseif(NOT F_PREFIX)
+            list(APPEND _OUTPUT_VAR ${_PATH})
+        endif()
+
+        if(NOT EXISTS "${_PATH}" AND F_FAIL)
+            omnitrace_message(FATAL_ERROR "Directory '${_PATH}' does not exist")
+        elseif(NOT IS_DIRECTORY "${_PATH}" AND F_FAIL)
+            omnitrace_message(FATAL_ERROR "'${_PATH}' exists but is not a directory")
+        elseif(NOT EXISTS "${_PATH}" AND F_MKDIR)
+            execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${_PATH}
+                            WORKING_DIRECTORY ${F_WORKING_DIRECTORY})
+        elseif(
+            EXISTS "${_PATH}"
+            AND NOT IS_DIRECTORY "${_PATH}"
+            AND F_MKDIR)
+            if(F_FORCE)
+                execute_process(COMMAND ${CMAKE_COMMAND} -E rm ${_PATH}
+                                WORKING_DIRECTORY ${F_WORKING_DIRECTORY})
+            endif()
+            execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${_PATH}
+                            WORKING_DIRECTORY ${F_WORKING_DIRECTORY})
+        endif()
+    endforeach()
+
+    if(F_OUTPUT_VARIABLE)
+        set(${F_OUTPUT_VARIABLE}
+            "${_OUTPUT_VAR}"
+            PARENT_SCOPE)
+    endif()
+endfunction()
+
+function(OMNITRACE_CHECK_PYTHON_DIRS_AND_VERSIONS)
+    cmake_parse_arguments(F "FAIL;UNSET" "RESULT_VARIABLE;OUTPUT_VARIABLE" "" ${ARGN})
+
+    list(LENGTH OMNITRACE_PYTHON_VERSIONS _NUM_PYTHON_VERSIONS)
+    list(LENGTH OMNITRACE_PYTHON_ROOT_DIRS _NUM_PYTHON_ROOT_DIRS)
+
+    if(NOT _NUM_PYTHON_VERSIONS EQUAL _NUM_PYTHON_ROOT_DIRS)
+        set(_RET 1)
+    else()
+        set(_RET 0)
+        if(F_OUTPUT_VARIABLE)
+            set(${F_OUTPUT_VARIABLE}
+                ${_NUM_PYTHON_VERSIONS}
+                PARENT_SCOPE)
+        endif()
+    endif()
+
+    if(F_RESULT_VARIABLE)
+        set(${F_RESULT_VARIABLE}
+            ${_RET}
+            PARENT_SCOPE)
+    endif()
+
+    if(NOT ${_RET} EQUAL 0)
+        if(F_FAIL)
+            omnitrace_message(
+                WARNING
+                "Error! Number of python versions  : ${_NUM_PYTHON_VERSIONS}. VERSIONS :: ${OMNITRACE_PYTHON_VERSIONS}"
+                )
+            omnitrace_message(
+                WARNING
+                "Error! Number of python root directories : ${_NUM_PYTHON_ROOT_DIRS}. ROOT DIRS :: ${OMNITRACE_PYTHON_ROOT_DIRS}"
+                )
+            omnitrace_message(
+                FATAL_ERROR
+                "Error! Number of python versions != number of python root directories")
+        elseif(F_UNSET)
+            unset(OMNITRACE_PYTHON_VERSIONS CACHE)
+            unset(OMNITRACE_PYTHON_ROOT_DIRS CACHE)
+            if(F_OUTPUT_VARIABLE)
+                set(${F_OUTPUT_VARIABLE} 0)
+            endif()
+        endif()
+    endif()
+endfunction()
+
+# ----------------------------------------------------------------------------
+# Console scripts
+#
+function(OMNITRACE_PYTHON_CONSOLE_SCRIPT SCRIPT_NAME SCRIPT_SUBMODULE)
+    set(options)
+    set(args VERSION ROOT_DIR)
+    set(kwargs)
+    cmake_parse_arguments(ARG "${options}" "${args}" "${kwargs}" ${ARGN})
+
+    if(ARG_VERSION AND ARG_ROOT_DIR)
+        set(Python3_ROOT_DIR "${ARG_ROOT_DIR}")
+        find_package(Python3 ${ARG_VERSION} EXACT QUIET MODULE COMPONENTS Interpreter)
+        set(PYTHON_EXECUTABLE "${Python3_EXECUTABLE}")
+        configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in
+                       ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION} @ONLY)
+
+        if(CMAKE_INSTALL_PYTHONDIR)
+            install(
+                PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION}
+                DESTINATION ${CMAKE_INSTALL_BINDIR}
+                OPTIONAL)
+        endif()
+
+        if(OMNITRACE_BUILD_TESTING OR OMNITRACE_BUILD_PYTHON)
+            add_test(
+                NAME ${SCRIPT_NAME}-console-script-test-${ARG_VERSION}
+                COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION} --help
+                WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+            set_tests_properties(
+                ${SCRIPT_NAME}-console-script-test-${ARG_VERSION}
+                PROPERTIES LABELS "python;python-${ARG_VERSION};console-script")
+            add_test(
+                NAME ${SCRIPT_NAME}-generic-console-script-test-${ARG_VERSION}
+                COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} --help
+                WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+            set_tests_properties(
+                ${SCRIPT_NAME}-generic-console-script-test-${ARG_VERSION}
+                PROPERTIES ENVIRONMENT "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" LABELS
+                           "python;python-${ARG_VERSION};console-script")
+        endif()
+    else()
+        set(PYTHON_EXECUTABLE "python3")
+
+        configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in
+                       ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} @ONLY)
+
+        if(CMAKE_INSTALL_PYTHONDIR)
+            install(
+                PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}
+                DESTINATION ${CMAKE_INSTALL_BINDIR}
+                OPTIONAL)
+        endif()
+    endif()
+endfunction()
+
 cmake_policy(POP)
@@ -23,6 +23,7 @@ omnitrace_add_interface_library(omnitrace-ptl "Enables PTL support (tasking)")
 omnitrace_add_interface_library(omnitrace-papi "Enable PAPI support")
 omnitrace_add_interface_library(omnitrace-ompt "Enable OMPT support")
 omnitrace_add_interface_library(omnitrace-python "Enables Python support")
+omnitrace_add_interface_library(omnitrace-timemory "Provides timemory libraries")
 omnitrace_add_interface_library(omnitrace-timemory-config
                                "CMake interface library applied to all timemory targets")
 omnitrace_add_interface_library(omnitrace-compile-definitions "Compile definitions")
@@ -526,6 +527,12 @@ if(TARGET omnitrace-papi-build)
    endforeach()
 endif()

+target_link_libraries(
+    omnitrace-timemory
+    INTERFACE $<BUILD_INTERFACE:timemory::timemory-headers>
+              $<BUILD_INTERFACE:timemory::timemory-gotcha>
+              $<BUILD_INTERFACE:timemory::timemory-cxx-static>)
+
 # ----------------------------------------------------------------------------------------#
 #
 # PTL (Parallel Tasking Library) submodule
@@ -7,6 +7,7 @@ set(CMAKE_CXX_VISIBILITY_PRESET "default")

 add_subdirectory(transpose)
 add_subdirectory(parallel-overhead)
+add_subdirectory(code-coverage)
 add_subdirectory(user-api)
 add_subdirectory(openmp)

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
+
+project(omnitrace-code-coverage LANGUAGES CXX)
+
+set(CMAKE_BUILD_TYPE "RelWithDebInfo")
+string(REPLACE " " ";" _FLAGS "${CMAKE_CXX_FLAGS_DEBUG}")
+
+find_package(Threads REQUIRED)
+add_executable(code-coverage code-coverage.cpp)
+target_link_libraries(code-coverage Threads::Threads)
+target_compile_options(code-coverage PRIVATE ${_FLAGS})
+
+if(NOT CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
+    set_target_properties(code-coverage PROPERTIES RUNTIME_OUTPUT_DIRECTORY
+                                                   ${CMAKE_BINARY_DIR})
+endif()
@@ -0,0 +1,88 @@
+
+#include <atomic>
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <thread>
+#include <vector>
+
+#define NOINLINE __attribute__((noinline))
+
+std::atomic<long> total{ 0 };
+
+long
+fib(long n) NOINLINE;
+
+void
+run_real(size_t nitr, long) NOINLINE;
+
+void
+run_fake(size_t nitr, long) NOINLINE;
+
+int
+main(int argc, char** argv)
+{
+    using exec_t = void (*)(size_t, long);
+
+    std::string _name = argv[0];
+    auto        _pos  = _name.find_last_of('/');
+    if(_pos != std::string::npos) _name = _name.substr(_pos + 1);
+
+    size_t nthread = std::min<size_t>(16, std::thread::hardware_concurrency());
+    size_t nitr    = 5000;
+    long   nfib    = 10;
+
+    if(argc > 1) nfib = atol(argv[1]);
+    if(argc > 2) nthread = atol(argv[2]);
+    if(argc > 3) nitr = atol(argv[3]);
+
+    exec_t _exec = &run_real;
+
+    // ensure that compiler cannot optimize run_fake away
+    if(std::getenv("CODE_COVERAGE_USE_FAKE") != nullptr) _exec = &run_fake;
+
+    printf("[%s] Threads: %zu\n[%s] Iterations: %zu\n[%s] fibonacci(%li)...\n",
+           _name.c_str(), nthread, _name.c_str(), nitr, _name.c_str(), nfib);
+
+    std::vector<std::thread> threads{};
+    for(size_t i = 0; i < nthread; ++i)
+    {
+        size_t _nitr = ((i % 2) == 1) ? (nitr - (0.1 * nitr)) : (nitr + (0.1 * nitr));
+        _nitr        = std::max<size_t>(_nitr, 1);
+        threads.emplace_back(_exec, _nitr, nfib);
+    }
+
+    auto _nitr = std::max<size_t>(nitr - 0.25 * nitr, 1);
+    (*_exec)(_nitr, nfib - 0.1 * nfib);
+    for(auto& itr : threads)
+        itr.join();
+
+    printf("[%s] fibonacci(%li) x %lu = %li\n", _name.c_str(), nfib, nthread,
+           total.load());
+
+    return 0;
+}
+
+long
+fib(long n)
+{
+    return (n < 2) ? n : fib(n - 1) + fib(n - 2);
+}
+
+void
+run_real(size_t nitr, long n)
+{
+    long local = 0;
+    for(size_t i = 0; i < nitr; ++i)
+        local += fib(n);
+    total += local;
+}
+
+void
+run_fake(size_t nitr, long n)
+{
+    long local = 0;
+    for(size_t i = 0; i < nitr; ++i)
+        local += fib(n);
+    total += local;
+}
@@ -2,6 +2,7 @@
 #include <atomic>
 #include <cstdio>
 #include <cstdlib>
+#include <string>
 #include <thread>
 #include <vector>

@@ -29,28 +30,36 @@ run(size_t nitr, long n)
 int
 main(int argc, char** argv)
 {
+    std::string _name = argv[0];
+    auto        _pos  = _name.find_last_of('/');
+    if(_pos != std::string::npos) _name = _name.substr(_pos + 1);
+
    size_t nthread = std::min<size_t>(16, std::thread::hardware_concurrency());
    size_t nitr    = 50000;
    long   nfib    = 10;
+
    if(argc > 1) nfib = atol(argv[1]);
    if(argc > 2) nthread = atol(argv[2]);
    if(argc > 3) nitr = atol(argv[3]);

-    printf("[%s] Threads: %zu\n[%s] Iterations: %zu\n[%s] fibonacci(%li)...\n", argv[0],
-           nthread, argv[0], nitr, argv[0], nfib);
+    printf("[%s] Threads: %zu\n[%s] Iterations: %zu\n[%s] fibonacci(%li)...\n",
+           _name.c_str(), nthread, _name.c_str(), nitr, _name.c_str(), nfib);

    std::vector<std::thread> threads{};
    for(size_t i = 0; i < nthread; ++i)
    {
        size_t _nitr = ((i % 2) == 1) ? (nitr - (0.1 * nitr)) : (nitr + (0.1 * nitr));
+        _nitr        = std::max<size_t>(_nitr, 1);
        threads.emplace_back(&run, _nitr, nfib);
    }

-    run(nitr - 0.25 * nitr, nfib - 0.1 * nfib);
+    auto _nitr = std::max<size_t>(nitr - 0.25 * nitr, 1);
+    run(_nitr, nfib - 0.1 * nfib);
    for(auto& itr : threads)
        itr.join();

-    printf("[%s] fibonacci(%li) x %lu = %li\n", argv[0], nfib, nthread, total.load());
+    printf("[%s] fibonacci(%li) x %lu = %li\n", _name.c_str(), nfib, nthread,
+           total.load());

    return 0;
 }
@@ -104,33 +104,36 @@ run(int rank, int tid, hipStream_t stream, int argc, char** argv)
    std::cout << "[" << rank << "][" << tid << "] M: " << M << " N: " << N << std::endl;
    _lk.unlock();

-    size_t size   = sizeof(int) * M * N;
-    int*   matrix = new int[size];
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
    for(size_t i = 0; i < M * N; i++)
-        matrix[i] = rand() % 1002;
+    {
+        inp_matrix[i] = rand() % 1002;
+        out_matrix[i] = 0;
+    }
    int* in  = nullptr;
    int* out = nullptr;

-    std::chrono::high_resolution_clock::time_point t1, t2;
-
    HIP_API_CALL(hipMalloc(&in, size));
    HIP_API_CALL(hipMalloc(&out, size));
-    HIP_API_CALL(hipMemset(in, 0, size));
-    HIP_API_CALL(hipMemset(out, 0, size));
-    HIP_API_CALL(hipMemcpy(in, matrix, size, hipMemcpyHostToDevice));
-    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));

    dim3 grid(M / 32, N / 32, 1);
    dim3 block(32, 32, 1);  // transpose_a

-    t1 = std::chrono::high_resolution_clock::now();
+    auto t1 = std::chrono::high_resolution_clock::now();
    for(size_t i = 0; i < nitr; i++)
    {
        transpose_a<<<grid, block, 0, stream>>>(in, out, M, N);
        check_hip_error();
    }
+    auto t2 = std::chrono::high_resolution_clock::now();
    HIP_API_CALL(hipStreamSynchronize(stream));
-    t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
    double time =
        std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
    float GB = (float) size * nitr * 2 / (1 << 30);
@@ -142,18 +145,15 @@ run(int rank, int tid, hipStream_t stream, int argc, char** argv)
              << std::endl;
    print_lock.unlock();

-    HIP_API_CALL(hipDeviceSynchronize());
-
-    int* out_matrix = new int[size];
-    HIP_API_CALL(hipMemcpy(out_matrix, out, size, hipMemcpyDeviceToHost));
+    HIP_API_CALL(hipStreamSynchronize(stream));

    // cpu_transpose(matrix, out_matrix, M, N);
-    verify(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);

    HIP_API_CALL(hipFree(in));
    HIP_API_CALL(hipFree(out));

-    delete[] matrix;
+    delete[] inp_matrix;
    delete[] out_matrix;
 }

@@ -22,9 +22,9 @@ target_link_libraries(
    omnitrace-exe
    PRIVATE omnitrace::omnitrace-headers
            omnitrace::omnitrace-dyninst
+            omnitrace::omnitrace-timemory
            omnitrace::omnitrace-compile-options
            omnitrace::omnitrace-compile-definitions
-            $<BUILD_INTERFACE:timemory::timemory-headers>
            $<IF:$<BOOL:${OMNITRACE_USE_SANITIZER}>,omnitrace::omnitrace-sanitizer,>)

 set_target_properties(
@@ -20,6 +20,8 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.

+#include "function_signature.hpp"
+#include "fwd.hpp"
 #include "omnitrace.hpp"

 static int expect_error = NO_ERROR;
@@ -673,11 +675,12 @@ get_func_file_line_info(module_t* module, procedure_t* func)
    std::vector<string_t> _params = {};
    if(params)
    {
+        _params.reserve(params->size());
        for(auto* itr : *params)
        {
            string_t _name = itr->getType()->getName();
            if(_name.empty()) _name = itr->getName();
-            _params.push_back(_name);
+            _params.emplace_back(_name);
        }
    }

@@ -699,6 +702,136 @@ get_func_file_line_info(module_t* module, procedure_t* func)
    }
 }

+//======================================================================================//
+//
+//  Gets information (line number, filename, and column number) about
+//  the instrumented loop and formats it properly.
+//
+std::map<basic_block_t*, basic_block_signature>
+get_basic_block_file_line_info(module_t* module, procedure_t* func)
+{
+    std::map<basic_block_t*, basic_block_signature> _data{};
+    if(!func) return _data;
+
+    auto*                        _cfg = func->getCFG();
+    std::set<BPatch_basicBlock*> _basic_blocks{};
+    _cfg->getAllBasicBlocks(_basic_blocks);
+
+    if(_basic_blocks.empty()) return _data;
+
+    char        fname[FUNCNAMELEN + 1];
+    char        mname[FUNCNAMELEN + 1];
+    std::string typeName = {};
+
+    memset(fname, '\0', FUNCNAMELEN + 1);
+    memset(mname, '\0', FUNCNAMELEN + 1);
+
+    module->getName(mname, FUNCNAMELEN);
+    func->getName(fname, FUNCNAMELEN);
+
+    auto* returnType = func->getReturnType();
+
+    if(returnType) typeName = returnType->getName();
+
+    auto*                 params = func->getParams();
+    std::vector<string_t> _params;
+    if(params)
+    {
+        for(auto* itr : *params)
+        {
+            string_t _name = itr->getType()->getName();
+            if(_name.empty()) _name = itr->getName();
+            _params.push_back(_name);
+        }
+    }
+
+    for(auto&& itr : _basic_blocks)
+    {
+        auto base_addr = itr->getStartAddress();
+        auto last_addr = itr->getEndAddress();
+
+        verbprintf(4, "BB: size = %lu: base_addr = %lu, last_addr = %lu\n",
+                   (unsigned long) (last_addr - base_addr), base_addr, last_addr);
+
+        bpvector_t<BPatch_statement> linesBeg{};
+        bpvector_t<BPatch_statement> linesEnd{};
+
+        string_t filename = mname;
+
+        if(module->getSourceLines(base_addr, linesBeg) && !linesBeg.empty())
+        {
+            int row1 = linesBeg.front().lineNumber();
+            int col1 = linesBeg.front().lineOffset();
+
+            verbprintf(4, "size of linesEnd = %lu\n", (unsigned long) linesEnd.size());
+
+            if(module->getSourceLines(last_addr, linesEnd) && !linesEnd.empty())
+            {
+                int row2 = linesEnd.back().lineNumber();
+                int col2 = linesEnd.back().lineOffset();
+
+                if(row2 < row1) std::swap(row1, row2);
+                if(row1 == row2 && col2 < col1) std::swap(col1, col2);
+
+                _data.emplace(itr,
+                              basic_block_signature{
+                                  base_addr, last_addr,
+                                  function_signature(typeName, fname, filename, _params,
+                                                     { row1, row2 }, { col1, col2 }, true,
+                                                     true, true) });
+            }
+            else
+            {
+                _data.emplace(itr,
+                              basic_block_signature{
+                                  base_addr, last_addr,
+                                  function_signature(typeName, fname, filename, _params,
+                                                     { row1, 0 }, { col1, 0 }, true, true,
+                                                     false) });
+            }
+        }
+        else
+        {
+            _data.emplace(itr,
+                          basic_block_signature{
+                              base_addr, last_addr,
+                              function_signature(typeName, fname, filename, _params) });
+        }
+    }
+
+    return _data;
+}
+
+//======================================================================================//
+//
+//  We create a new name that embeds the file and line information in the name
+//
+std::vector<statement_t>
+get_source_code(module_t* module, procedure_t* func)
+{
+    std::vector<statement_t> _lines{};
+    if(!module || !func) return _lines;
+    auto*                        _cfg = func->getCFG();
+    std::set<BPatch_basicBlock*> _basic_blocks{};
+    _cfg->getAllBasicBlocks(_basic_blocks);
+
+    for(auto&& itr : _basic_blocks)
+    {
+        auto _base_addr = itr->getStartAddress();
+        auto _last_addr = itr->getEndAddress();
+        for(decltype(_base_addr) _addr = _base_addr; _addr <= _last_addr; ++_addr)
+        {
+            std::vector<statement_t> _src{};
+            if(module->getSourceLines(_addr, _src))
+            {
+                for(auto&& iitr : _src)
+                    _lines.emplace_back(iitr);
+            }
+        }
+    }
+    return _lines;
+}
+
 //======================================================================================//
 //
 //  Error callback routine.
@@ -59,12 +59,14 @@ function_signature::get(function_signature& sig)
 }

 string_t
-function_signature::get() const
+function_signature::get(bool _all, bool _save) const
 {
+    if(!_all && _save && !m_signature.empty()) return m_signature;
+
    std::stringstream ss;
-    if(use_return_info && !m_return.empty()) ss << m_return << " ";
+    if((_all || use_return_info) && !m_return.empty()) ss << m_return << " ";
    ss << m_name;
-    if(use_args_info) ss << m_params;
+    if(_all || use_args_info) ss << m_params;
    if(m_loop && m_info_beg)
    {
        auto _row_col_str = [](unsigned long _row, unsigned long _col) {
@@ -90,10 +92,52 @@ function_signature::get() const
        else
            errprintf(1, "loop line info is empty!");
    }
-    if(use_file_info && m_file.length() > 0) ss << " [" << m_file;
-    if(use_line_info && m_row.first > 0) ss << ":" << m_row.first;
-    if(use_file_info && m_file.length() > 0) ss << "]";
+    if((_all || use_file_info) && m_file.length() > 0) ss << " [" << m_file;
+    if((_all || use_line_info) && m_row.first > 0) ss << ":" << m_row.first;
+    if((_all || use_file_info) && m_file.length() > 0) ss << "]";

-    m_signature = ss.str();
-    return m_signature;
+    if(_save) m_signature = ss.str();
+    return ss.str();
+}
+
+string_t
+function_signature::get_coverage(bool _basic_block) const
+{
+    std::stringstream ss;
+    if(!m_return.empty()) ss << m_return << " ";
+    ss << m_name << m_params;
+    if(_basic_block && m_loop && m_info_beg)
+    {
+        if(m_file.length() > 0) ss << " [" << m_file << "]";
+        auto _row_col_str = [](unsigned long _row, unsigned long _col) {
+            std::stringstream _ss{};
+            if(_row == 0 && _col == 0) return std::string{};
+            if(_col > 0)
+                _ss << "{" << _row << "," << _col << "}";
+            else
+                _ss << "{" << _row << "}";
+            return _ss.str();
+        };
+
+        auto _rc1 = _row_col_str(m_row.first, m_col.first);
+        auto _rc2 = _row_col_str(m_row.second, m_col.second);
+        if(m_info_end && !_rc1.empty() && !_rc2.empty() && _rc1 != _rc2)
+            ss << " [" << _rc1 << "-" << _rc2 << "]";
+        else if(m_info_end && !_rc1.empty() && !_rc2.empty() && _rc1 == _rc2)
+            ss << " [" << _rc1 << "]";
+        else if(m_info_end && !_rc1.empty() && _rc2.empty())
+            ss << " [" << _rc1 << "]";
+        else if(!m_info_end && !_rc1.empty())
+            ss << " [" << _rc1 << "]";
+        else
+            errprintf(1, "loop line info is empty!");
+    }
+    else
+    {
+        if(m_file.length() > 0) ss << " [" << m_file;
+        if(m_row.first > 0) ss << ":" << m_row.first;
+        if(m_file.length() > 0) ss << "]";
+    }
+
+    return ss.str();
 }
@@ -41,7 +41,8 @@ struct function_signature
                       bool _info_beg = false, bool _info_end = false);

    static string_t get(function_signature& sig);
-    string_t        get() const;
+    string_t        get(bool _all = false, bool _save = true) const;
+    string_t        get_coverage(bool _is_basic_block) const;

    bool             m_loop      = false;
    bool             m_info_beg  = false;
@@ -72,3 +73,12 @@ struct function_signature
        (void) get();
    }
 };
+
+struct basic_block_signature
+{
+    using address_t = Dyninst::Address;
+
+    address_t          start_address = {};
+    address_t          last_address  = {};
+    function_signature signature     = {};
+};
@@ -22,7 +22,6 @@

 #pragma once

-#include <string_view>
 #include <timemory/backends/process.hpp>
 #include <timemory/environment.hpp>
 #include <timemory/mpl/apply.hpp>
@@ -65,6 +64,7 @@
 #include <sstream>
 #include <stdexcept>
 #include <string>
+#include <string_view>
 #include <unistd.h>
 #include <vector>

@@ -78,6 +78,7 @@
 #endif

 struct function_signature;
+struct basic_block_signature;
 struct module_function;

 template <typename Tp>
@@ -105,6 +106,7 @@ using snippet_t             = BPatch_snippet;
 using call_expr_t           = BPatch_funcCallExpr;
 using address_space_t       = BPatch_addressSpace;
 using flow_graph_t          = BPatch_flowGraph;
+using statement_t           = BPatch_statement;
 using basic_block_t         = BPatch_basicBlock;
 using basic_loop_t          = BPatch_basicBlockLoop;
 using procedure_loc_t       = BPatch_procedureLocation;
@@ -125,6 +127,13 @@ using instruction_t         = Dyninst::InstructionAPI::Instruction;
 void
 omnitrace_prefork_callback(thread_t* parent, thread_t* child);

+enum CodeCoverageMode
+{
+    CODECOV_NONE = 0,
+    CODECOV_FUNCTION,
+    CODECOV_BASIC_BLOCK
+};
+
 //======================================================================================//
 //
 //                                  Global Variables
@@ -167,21 +176,22 @@ extern string_t prefer_library;
 //
 //  global variables
 //
-extern patch_pointer_t bpatch;
-extern call_expr_t*    terminate_expr;
-extern snippet_vec_t   init_names;
-extern snippet_vec_t   fini_names;
-extern fmodset_t       available_module_functions;
-extern fmodset_t       instrumented_module_functions;
-extern fmodset_t       overlapping_module_functions;
-extern fmodset_t       excluded_module_functions;
-extern fixed_modset_t  fixed_module_functions;
-extern regexvec_t      func_include;
-extern regexvec_t      func_exclude;
-extern regexvec_t      file_include;
-extern regexvec_t      file_exclude;
-extern regexvec_t      file_restrict;
-extern regexvec_t      func_restrict;
+extern patch_pointer_t  bpatch;
+extern call_expr_t*     terminate_expr;
+extern snippet_vec_t    init_names;
+extern snippet_vec_t    fini_names;
+extern fmodset_t        available_module_functions;
+extern fmodset_t        instrumented_module_functions;
+extern fmodset_t        overlapping_module_functions;
+extern fmodset_t        excluded_module_functions;
+extern fixed_modset_t   fixed_module_functions;
+extern regexvec_t       func_include;
+extern regexvec_t       func_exclude;
+extern regexvec_t       file_include;
+extern regexvec_t       file_exclude;
+extern regexvec_t       file_restrict;
+extern regexvec_t       func_restrict;
+extern CodeCoverageMode coverage_mode;
 //
 //======================================================================================//

@@ -230,10 +240,8 @@ consume_parameters(T&&...)
 extern "C"
 {
    bool are_file_include_exclude_lists_empty();
-    bool instrument_module(const string_t& file_name);
-    bool instrument_entity(const string_t& function_name);
-    bool module_constraint(string_view_t fname);
-    bool routine_constraint(string_view_t fname);
+    bool module_constraint(const char* fname);
+    bool routine_constraint(const char* fname);
 }

 //======================================================================================//
@@ -248,6 +256,12 @@ function_signature
 get_loop_file_line_info(module_t* mutatee_module, procedure_t* f, flow_graph_t* cfGraph,
                        basic_loop_t* loopToInstrument);

+std::map<basic_block_t*, basic_block_signature>
+get_basic_block_file_line_info(module_t* module, procedure_t* func);
+
+std::vector<statement_t>
+get_source_code(module_t* module, procedure_t* func);
+
 std::tuple<size_t, size_t>
 query_instr(procedure_t* funcToInstr, procedure_loc_t traceLoc,
            flow_graph_t* cfGraph = nullptr, basic_loop_t* loopToInstrument = nullptr);
@@ -256,11 +270,21 @@ bool
 query_instr(procedure_t* funcToInstr, procedure_loc_t traceLoc, flow_graph_t* cfGraph,
            basic_loop_t* loopToInstrument, bool allow_traps);

+template <typename Tp>
+bool
+insert_instr(address_space_t* mutatee, const bpvector_t<point_t*>& _points, Tp traceFunc,
+             procedure_loc_t traceLoc, bool allow_traps = instr_traps);
+
 template <typename Tp>
 bool
 insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,
             procedure_loc_t traceLoc, flow_graph_t* cfGraph = nullptr,
-             basic_loop_t* loopToInstrument = nullptr, bool allow_traps = true);
+             basic_loop_t* loopToInstrument = nullptr, bool allow_traps = instr_traps);
+
+template <typename Tp>
+bool
+insert_instr(address_space_t* mutatee, Tp traceFunc, procedure_loc_t traceLoc,
+             basic_block_t* basicBlock, bool allow_traps = instr_traps);

 void
 errorFunc(error_level_t level, int num, const char** params);
@@ -25,6 +25,12 @@
 #include "fwd.hpp"
 #include "module_function.hpp"

+#include <timemory/mpl/policy.hpp>
+#include <timemory/settings.hpp>
+#include <timemory/tpls/cereal/cereal.hpp>
+#include <timemory/utility/delimit.hpp>
+#include <timemory/utility/filepath.hpp>
+
 static inline void
 dump_info(std::ostream& _os, const fmodset_t& _data)
 {
@@ -54,7 +60,7 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext,
    namespace cereal = tim::cereal;
    namespace policy = tim::policy;

-    _oname += "." + _ext;
+    _oname             = tim::settings::compose_output_filename(_oname, _ext);
    auto _handle_error = [&]() {
        std::stringstream _msg{};
        _msg << "[dump_info] Error opening '" << _oname << " for output";
@@ -114,16 +114,28 @@ module_function::write_header(std::ostream& os)

 bool
 module_function::should_instrument() const
+{
+    return should_instrument(false);
+}
+
+bool
+module_function::should_coverage_instrument() const
+{
+    return should_instrument(true);
+}
+
+bool
+module_function::should_instrument(bool coverage) const
 {
    // hard constraints
    if(!is_instrumentable()) return false;
    if(!can_instrument_entry()) return false;
-    if(!can_instrument_exit()) return false;
+    if(!coverage && !can_instrument_exit()) return false;
    if(is_module_constrained()) return false;
    if(is_routine_constrained()) return false;

    // should be before user selection
-    constexpr int absolute_min_instructions = 4;
+    constexpr int absolute_min_instructions = 2;
    if(num_instructions < absolute_min_instructions)
    {
        messages.emplace_back(
@@ -134,8 +146,6 @@ module_function::should_instrument() const

    // user selection
    if(is_user_excluded()) return false;
-    if(is_user_restricted()) return true;
-    if(is_user_included()) return true;

    // should be applied before dynamic-callsite check
    if(is_overlapping_constrained()) return false;
@@ -145,6 +155,10 @@ module_function::should_instrument() const
    // needs to be applied before address range and number of instruction constraints
    if(is_dynamic_callsite_forced()) return true;

+    // user selection
+    if(!file_restrict.empty() || !func_restrict.empty()) return !is_user_restricted();
+    if(is_user_included()) return true;
+
    if(is_address_range_constrained()) return false;
    if(is_num_instructions_constrained()) return false;

@@ -194,7 +208,7 @@ module_function::is_user_restricted() const

    if(!func_restrict.empty())
    {
-        if(check_regex_restrictions(module_name, func_restrict))
+        if(check_regex_restrictions(function_name, func_restrict))
        {
            messages.emplace_back(2, "Forcing", "function", "function-restrict-regex");
            return false;
@@ -282,23 +296,134 @@ module_function::is_overlapping() const
 bool
 module_function::is_module_constrained() const
 {
-    if(!instrument_module(module_name) || module_constraint(module_name.c_str()))
-    {
-        messages.emplace_back(2, "Skipping", "module", "module-constraint");
+    auto regex_opts = std::regex_constants::egrep | std::regex_constants::optimize;
+    auto _report    = [&](const string_t& _action, const string_t& _reason, int _lvl) {
+        messages.emplace_back(_lvl, _action, "module", _reason);
        return true;
-    }
+    };
+
+    if(module_constraint(function_name.c_str())) return true;
+
+    // always instrument these modules
+    if(module_name == "DEFAULT_MODULE" || module_name == "LIBRARY_MODULE")
+        return _report("Skipping", "default module", 2);
+
+    static std::regex ext_regex{ "\\.(s|S)$", regex_opts };
+    static std::regex sys_regex{ "^(s|k|e|w)_[A-Za-z_0-9\\-]+\\.(c|C)$", regex_opts };
+    static std::regex sys_build_regex{ "^(\\.\\./sysdeps/|/build/)", regex_opts };
+    static std::regex dyninst_regex{ "(dyninst|DYNINST|(^|/)RT[[:graph:]]+\\.c$)",
+                                     regex_opts };
+    static std::regex dependlib_regex{ "^(lib|)(omnitrace|pthread|caliper|gotcha|papi|"
+                                       "cupti|TAU|likwid|pfm|nvperf|unwind)",
+                                       regex_opts };
+    static std::regex core_cmod_regex{
+        "^(malloc|(f|)lock|sig|sem)[a-z_]+(|64|_r|_l)\\.c$"
+    };
+    static std::regex core_lib_regex{
+        "^(lib|)(c|dl|dw|pthread|tcmalloc|profiler|"
+        "tbbmalloc|tbbmalloc_proxy|malloc|stdc\\+\\+)(-|\\.)",
+        regex_opts
+    };
+    static std::regex prefix_regex{ "^(_|\\.[a-zA-Z0-9])", regex_opts };
+
+    // file extensions that should not be instrumented
+    if(std::regex_search(module_name, ext_regex))
+        return _report("Excluding", "file extension", 3);
+
+    // system modules that should not be instrumented (wastes time)
+    if(std::regex_search(module_name, sys_regex) ||
+       std::regex_search(module_name, sys_build_regex))
+        return _report("Excluding", "system module", 3);
+
+    // dyninst modules that must not be instrumented
+    if(std::regex_search(module_name, dyninst_regex))
+        return _report("Excluding", "dyninst module", 3);
+
+    // modules used by omnitrace and dependent libraries
+    if(std::regex_search(module_name, core_lib_regex) ||
+       std::regex_search(module_name, core_cmod_regex))
+        return _report("Excluding", "core module", 3);
+
+    // modules used by omnitrace and dependent libraries
+    if(std::regex_search(module_name, dependlib_regex))
+        return _report("Excluding", "dependency module", 3);
+
+    // known set of modules whose starting sequence of characters suggest it should not be
+    // instrumented (wastes time)
+    if(std::regex_search(module_name, prefix_regex))
+        return _report("Excluding", "prefix match", 3);
+
    return false;
 }

 bool
 module_function::is_routine_constrained() const
 {
-    if(!instrument_entity(function_name) || !instrument_entity(signature.get()) ||
-       routine_constraint(function_name) || routine_constraint(signature.get()))
-    {
-        messages.emplace_back(2, "Skipping", "function", "function-constraint");
+    auto regex_opts = std::regex_constants::egrep | std::regex_constants::optimize;
+    auto _report    = [&](const string_t& _action, const string_t& _reason, int _lvl) {
+        messages.emplace_back(_lvl, _action, "function", _reason);
        return true;
+    };
+
+    if(routine_constraint(function_name.c_str())) return true;
+
+    auto npos = std::string::npos;
+    if(function_name.find("omnitrace") != npos)
+    {
+        return _report("Skipping", "omnitrace-function", 1);
    }
+
+    if(function_name.find("FunctionInfo") != npos ||
+       function_name.find("_L_lock") != npos || function_name.find("_L_unlock") != npos)
+    {
+        return _report("Skipping", "function-constraint", 2);
+    }
+
+    static std::regex exclude(
+        "(omnitrace|tim::|N3tim|MPI_Init|MPI_Finalize|dyninst|tm_clones)", regex_opts);
+    static std::regex exclude_cxx(
+        "(std::_Sp_counted_base|std::(use|has)_facet|std::locale|::sentry|^std::_|::_(M|"
+        "S)_|::basic_string[a-zA-Z,<>: ]+::_M_create|::__|::_(Alloc|State)|"
+        "std::(basic_|)(ifstream|ios|istream|ostream|stream))",
+        regex_opts);
+    static std::regex leading("^(_|\\.|frame_dummy|transaction clone|virtual "
+                              "thunk|non-virtual thunk|\\(|targ|kmp_threadprivate_)",
+                              regex_opts);
+    static std::regex trailing(
+        "(_|\\.part\\.[0-9]+|\\.constprop\\.[0-9]+|\\.|\\.[0-9]+)$", regex_opts);
+    static strset_t whole = []() {
+        auto _v   = get_whole_function_names();
+        auto _ret = _v;
+        for(std::string _ext : { "64", "_l", "_r" })
+            for(const auto& itr : _v)
+                _ret.emplace(itr + _ext);
+        return _ret;
+    }();
+
+    // don't instrument the functions when key is found anywhere in function name
+    if(std::regex_search(function_name, exclude) ||
+       std::regex_search(function_name, exclude_cxx))
+    {
+        return _report("Excluding", "critical", 3);
+    }
+
+    if(whole.count(function_name) > 0)
+    {
+        return _report("Excluding", "critical-whole-match", 3);
+    }
+
+    // don't instrument the functions when key is found at the start of the function name
+    if(std::regex_search(function_name, leading))
+    {
+        return _report("Excluding", "recommended-leading-match", 3);
+    }
+
+    // don't instrument the functions when key is found at the end of the function name
+    if(std::regex_search(function_name, trailing))
+    {
+        return _report("Excluding", "recommended-trailing-match", 3);
+    }
+
    return false;
 }

@@ -364,6 +489,8 @@ module_function::is_loop_address_range_constrained() const
 bool
 module_function::is_num_instructions_constrained() const
 {
+    if(!loop_blocks.empty()) return is_loop_num_instructions_constrained();
+
    if(num_instructions < min_instructions)
    {
        messages.emplace_back(2, "Skipping", "function", "min-instructions");
@@ -373,6 +500,20 @@ module_function::is_num_instructions_constrained() const
    return false;
 }

+bool
+module_function::is_loop_num_instructions_constrained() const
+{
+    if(loop_blocks.empty()) return false;
+
+    if(num_instructions < min_loop_instructions)
+    {
+        messages.emplace_back(2, "Skipping", "function", "min-instructions-loop");
+        return true;
+    }
+
+    return false;
+}
+
 bool
 module_function::can_instrument_entry() const
 {
@@ -459,10 +600,8 @@ module_function::operator()(address_space_t* _addr_space, procedure_t* _entr_tra
    auto _entr       = _trace_entr.get(_entr_trace);
    auto _exit       = _trace_exit.get(_exit_trace);

-    if(insert_instr(_addr_space, function, _entr, BPatch_entry, nullptr, nullptr,
-                    instr_traps) &&
-       insert_instr(_addr_space, function, _exit, BPatch_exit, nullptr, nullptr,
-                    instr_traps))
+    if(insert_instr(_addr_space, function, _entr, BPatch_entry) &&
+       insert_instr(_addr_space, function, _exit, BPatch_exit))
    {
        messages.emplace_back(1, "Instrumenting", "function", "no-constraint");
        ++_count.first;
@@ -516,3 +655,92 @@ module_function::operator()(address_space_t* _addr_space, procedure_t* _entr_tra

    return _count;
 }
+
+void
+module_function::register_source(address_space_t* _addr_space, procedure_t* _entr_trace,
+                                 const std::vector<point_t*>& _entr_points) const
+{
+    switch(coverage_mode)
+    {
+        case CODECOV_FUNCTION:
+        {
+            auto _name = signature.get_coverage(false);
+            auto _trace_entr =
+                omnitrace_call_expr(signature.m_file, signature.m_name,
+                                    signature.m_row.first, start_address, _name);
+            auto _entr = _trace_entr.get(_entr_trace);
+
+            if(insert_instr(_addr_space, _entr_points, _entr, BPatch_entry))
+            {
+                messages.emplace_back(1, "Code Coverage", "function", "no-constraint");
+            }
+            break;
+        }
+        case CODECOV_BASIC_BLOCK:
+        {
+            for(auto&& itr : get_basic_block_file_line_info(module, function))
+            {
+                auto  _start_addr = itr.second.start_address;
+                auto& _signature  = itr.second.signature;
+                auto  _name       = _signature.get_coverage(true);
+                auto  _trace_entr =
+                    omnitrace_call_expr(_signature.m_file, _signature.m_name,
+                                        _signature.m_row.first, _start_addr, _name);
+                auto _entr = _trace_entr.get(_entr_trace);
+
+                if(insert_instr(_addr_space, _entr_points, _entr, BPatch_entry))
+                {
+                    messages.emplace_back(1, "Code Coverage", "basic_block",
+                                          "no-constraint");
+                }
+            }
+            break;
+        }
+        case CODECOV_NONE: break;
+    }
+}
+
+std::pair<size_t, size_t>
+module_function::register_coverage(address_space_t* _addr_space,
+                                   procedure_t*     _entr_trace) const
+{
+    std::pair<size_t, size_t> _count = { 0, 0 };
+    switch(coverage_mode)
+    {
+        case CODECOV_FUNCTION:
+        {
+            auto _trace_entr =
+                omnitrace_call_expr(signature.m_file, signature.m_name, start_address);
+            auto _entr = _trace_entr.get(_entr_trace);
+
+            if(insert_instr(_addr_space, function, _entr, BPatch_entry))
+            {
+                messages.emplace_back(1, "Code Coverage", "function", "no-constraint");
+                ++_count.first;
+            }
+            break;
+        }
+        case CODECOV_BASIC_BLOCK:
+        {
+            for(auto&& itr : get_basic_block_file_line_info(module, function))
+            {
+                auto  _start_addr = itr.second.start_address;
+                auto& _signature  = itr.second.signature;
+                auto  _trace_entr = omnitrace_call_expr(_signature.m_file,
+                                                       _signature.m_name, _start_addr);
+                auto  _entr       = _trace_entr.get(_entr_trace);
+
+                if(insert_instr(_addr_space, _entr, BPatch_entry, itr.first))
+                {
+                    ++_count.second;
+                    messages.emplace_back(1, "Code Coverage", "basic_block",
+                                          "no-constraint");
+                }
+            }
+            verbprintf(0, "Basic-block code coverage is not available yet\n");
+            break;
+        }
+        case CODECOV_NONE: break;
+    }
+    return _count;
+}
@@ -46,12 +46,20 @@ struct module_function

    module_function(module_t* mod, procedure_t* proc);

+    // code coverage
+    void register_source(address_space_t* _addr_space, procedure_t* _entr_trace,
+                         const std::vector<point_t*>&) const;
+    std::pair<size_t, size_t> register_coverage(address_space_t* _addr_space,
+                                                procedure_t*     _entr_trace) const;
+
+    // instrumentation
    std::pair<size_t, size_t> operator()(address_space_t* _addr_space,
                                         procedure_t*     _entr_trace,
                                         procedure_t*     _exit_trace) const;

    // applies logic for all "is_*" and "can_*" checks below
    bool should_instrument() const;
+    bool should_coverage_instrument() const;

    // hard constraints
    bool is_instrumentable() const;       // checks whether can instrument
@@ -98,8 +106,10 @@ struct module_function
    bool is_overlapping() const;  // checks if func overlaps

 private:
+    bool is_loop_num_instructions_constrained() const;  // checks loop instr constraint
    bool is_loop_address_range_constrained() const;  // checks loop addr range constraint
    bool contains_dynamic_callsites() const;
+    bool should_instrument(bool _coverage) const;

 public:
    template <typename ArchiveT>
@@ -176,6 +186,7 @@ module_function::serialize(ArchiveT& ar, const unsigned)
        ar.setNextName("heuristics");
        ar.startNode();
        ar(cereal::make_nvp("should_instrument", should_instrument()),
+           cereal::make_nvp("should_coverage_instrument", should_coverage_instrument()),
           cereal::make_nvp("is_instrumentable", is_instrumentable()),
           cereal::make_nvp("can_instrument_entry", can_instrument_entry()),
           cereal::make_nvp("can_instrument_exit", can_instrument_exit()),
@@ -23,12 +23,17 @@
 #include "omnitrace.hpp"
 #include "fwd.hpp"

+#include <timemory/config.hpp>
+#include <timemory/settings.hpp>
+
 #include <algorithm>
 #include <chrono>
 #include <csignal>
 #include <cstdint>
+#include <cstdio>
 #include <cstdlib>
 #include <map>
+#include <regex>
 #include <stdexcept>
 #include <string>
 #include <sys/stat.h>
@@ -62,50 +67,53 @@ string_t prefer_library          = {};
 //
 //  global variables
 //
-patch_pointer_t bpatch                        = {};
-call_expr_t*    terminate_expr                = nullptr;
-snippet_vec_t   init_names                    = {};
-snippet_vec_t   fini_names                    = {};
-fmodset_t       available_module_functions    = {};
-fmodset_t       instrumented_module_functions = {};
-fmodset_t       overlapping_module_functions  = {};
-fmodset_t       excluded_module_functions     = {};
-fixed_modset_t  fixed_module_functions        = {};
-regexvec_t      func_include                  = {};
-regexvec_t      func_exclude                  = {};
-regexvec_t      file_include                  = {};
-regexvec_t      file_exclude                  = {};
-regexvec_t      file_restrict                 = {};
-regexvec_t      func_restrict                 = {};
+patch_pointer_t  bpatch                        = {};
+call_expr_t*     terminate_expr                = nullptr;
+snippet_vec_t    init_names                    = {};
+snippet_vec_t    fini_names                    = {};
+fmodset_t        available_module_functions    = {};
+fmodset_t        instrumented_module_functions = {};
+fmodset_t        coverage_module_functions     = {};
+fmodset_t        overlapping_module_functions  = {};
+fmodset_t        excluded_module_functions     = {};
+fixed_modset_t   fixed_module_functions        = {};
+regexvec_t       func_include                  = {};
+regexvec_t       func_exclude                  = {};
+regexvec_t       file_include                  = {};
+regexvec_t       file_exclude                  = {};
+regexvec_t       file_restrict                 = {};
+regexvec_t       func_restrict                 = {};
+CodeCoverageMode coverage_mode                 = CODECOV_NONE;

 namespace
 {
-bool                                       binary_rewrite         = false;
-bool                                       is_attached            = false;
-bool                                       use_mpi                = false;
-bool                                       is_static_exe          = false;
-bool                                       is_driver              = false;
-bool                                       explicit_dump_and_exit = false;
-size_t                                     batch_size             = 50;
-strset_t                                   extra_libs             = {};
-std::vector<std::pair<uint64_t, string_t>> hash_ids               = {};
-std::map<string_t, bool>                   use_stubs              = {};
-std::map<string_t, procedure_t*>           beg_stubs              = {};
-std::map<string_t, procedure_t*>           end_stubs              = {};
-strvec_t                                   init_stub_names        = {};
-strvec_t                                   fini_stub_names        = {};
-strset_t                                   used_stub_names        = {};
-std::vector<call_expr_pointer_t>           env_variables          = {};
-std::map<string_t, call_expr_pointer_t>    beg_expr               = {};
-std::map<string_t, call_expr_pointer_t>    end_expr               = {};
-const auto                                 npos_v                 = string_t::npos;
-string_t                                   instr_mode             = "trace";
-string_t                                   print_instrumented     = {};
-string_t                                   print_excluded         = {};
-string_t                                   print_available        = {};
-string_t                                   print_overlapping      = {};
-strset_t                                   print_formats          = { "txt", "json" };
-std::string                                modfunc_dump_dir       = {};
+bool                                       binary_rewrite       = false;
+bool                                       is_attached          = false;
+bool                                       use_mpi              = false;
+bool                                       is_static_exe        = false;
+bool                                       simulate             = false;
+size_t                                     batch_size           = 50;
+strset_t                                   extra_libs           = {};
+std::vector<std::pair<uint64_t, string_t>> hash_ids             = {};
+std::map<string_t, bool>                   use_stubs            = {};
+std::map<string_t, procedure_t*>           beg_stubs            = {};
+std::map<string_t, procedure_t*>           end_stubs            = {};
+strvec_t                                   init_stub_names      = {};
+strvec_t                                   fini_stub_names      = {};
+strset_t                                   used_stub_names      = {};
+strvec_t                                   env_config_variables = {};
+std::vector<call_expr_pointer_t>           env_variables        = {};
+std::map<string_t, call_expr_pointer_t>    beg_expr             = {};
+std::map<string_t, call_expr_pointer_t>    end_expr             = {};
+const auto                                 npos_v               = string_t::npos;
+string_t                                   instr_mode           = "trace";
+string_t                                   print_coverage       = {};
+string_t                                   print_instrumented   = {};
+string_t                                   print_excluded       = {};
+string_t                                   print_available      = {};
+string_t                                   print_overlapping    = {};
+strset_t                                   print_formats        = { "txt", "json" };
+std::string                                modfunc_dump_dir     = {};
 auto regex_opts = std::regex_constants::egrep | std::regex_constants::optimize;

 std::string
@@ -176,6 +184,7 @@ main(int argc, char** argv)
    fixed_module_functions = {
        { &available_module_functions, false },
        { &instrumented_module_functions, false },
+        { &coverage_module_functions, false },
        { &excluded_module_functions, false },
        { &overlapping_module_functions, false },
    };
@@ -218,7 +227,9 @@ main(int argc, char** argv)
            int k        = 0;
            for(int j = i + 1; j < argc; ++j, ++k)
            {
-                copy_str(_cmdv[k], argv[j]);
+                auto _v =
+                    std::regex_replace(argv[j], std::regex{ "(.*)([ \t\n\r]+)$" }, "$1");
+                copy_str(_cmdv[k], _v.c_str());
            }
            mutname = _cmdv[0];
            break;
@@ -256,9 +267,9 @@ main(int argc, char** argv)
                  << std::endl;
    }

-    if(_cmdc > 0)
-        std::cout << "\n[omnitrace][exe][command]: " << cmd_string(_cmdc, _cmdv)
-                  << "\n\n";
+    verbprintf(0, "\n");
+    verbprintf(0, "command :: '%s'...\n", cmd_string(_cmdc, _cmdv).c_str());
+    verbprintf(0, "\n");

    if(_cmdc > 0) cmdv0 = _cmdv[0];

@@ -299,7 +310,7 @@ main(int argc, char** argv)
                      "function lists, e.g. available-instr.txt")
        .max_count(1)
        .dtype("bool")
-        .action([](parser_t& p) { explicit_dump_and_exit = p.get<bool>("simulate"); });
+        .action([](parser_t& p) { simulate = p.get<bool>("simulate"); });
    parser
        .add_argument({ "--print-format" },
                      "Output format for diagnostic "
@@ -317,12 +328,14 @@ main(int argc, char** argv)
                      "function lists, e.g. {print-dir}/available-instr.txt")
        .count(1)
        .dtype("string")
-        .action([](parser_t& p) { modfunc_dump_dir = p.get<std::string>("print-dir"); });
+        .action([](parser_t& p) {
+            tim::settings::output_path() = p.get<std::string>("print-dir");
+        });
    parser
        .add_argument(
            { "--print-available" },
            "Print the available entities for instrumentation (functions, modules, or "
-            "module-function pair) to stdout applying regular expressions and exit")
+            "module-function pair) to stdout after applying regular expressions")
        .count(1)
        .choices({ "functions", "modules", "functions+", "pair", "pair+" })
        .action(
@@ -331,18 +344,27 @@ main(int argc, char** argv)
        .add_argument(
            { "--print-instrumented" },
            "Print the instrumented entities (functions, modules, or module-function "
-            "pair) to stdout after applying regular expressions and exit")
+            "pair) to stdout after applying regular expressions")
        .count(1)
        .choices({ "functions", "modules", "functions+", "pair", "pair+" })
        .action([](parser_t& p) {
            print_instrumented = p.get<std::string>("print-instrumented");
        });
+    parser
+        .add_argument({ "--print-coverage" },
+                      "Print the instrumented coverage entities (functions, modules, or "
+                      "module-function "
+                      "pair) to stdout after applying regular expressions")
+        .count(1)
+        .choices({ "functions", "modules", "functions+", "pair", "pair+" })
+        .action(
+            [](parser_t& p) { print_coverage = p.get<std::string>("print-coverage"); });
    parser
        .add_argument({ "--print-excluded" },
                      "Print the entities for instrumentation (functions, modules, or "
                      "module-function "
                      "pair) which are excluded from the instrumentation to stdout after "
-                      "applying regular expressions and exit")
+                      "applying regular expressions")
        .count(1)
        .choices({ "functions", "modules", "functions+", "pair", "pair+" })
        .action(
@@ -352,7 +374,7 @@ main(int argc, char** argv)
            { "--print-overlapping" },
            "Print the entities for instrumentation (functions, modules, or "
            "module-function pair) which overlap other function calls or have multiple "
-            "entry points to stdout applying regular expressions and exit")
+            "entry points to stdout after applying regular expressions")
        .count(1)
        .choices({ "functions", "modules", "functions+", "pair", "pair+" })
        .action([](parser_t& p) {
@@ -391,9 +413,13 @@ main(int argc, char** argv)
                      "Instrumentation mode. 'trace' mode instruments the selected "
                      "functions, 'sampling' mode only instruments the main function to "
                      "start and stop the sampler.")
-        .choices({ "trace", "sampling" })
+        .choices({ "trace", "sampling", "coverage" })
        .count(1)
-        .action([](parser_t& p) { instr_mode = p.get<string_t>("mode"); });
+        .action([](parser_t& p) {
+            instr_mode = p.get<string_t>("mode");
+            if(instr_mode == "coverage" && !p.exists("coverage"))
+                coverage_mode = CODECOV_FUNCTION;
+        });
    if(_cmdc == 0)
    {
        parser
@@ -433,23 +459,6 @@ main(int argc, char** argv)
                      "The primary function to instrument around, e.g. 'main'")
        .count(1)
        .action([](parser_t& p) { main_fname = p.get<string_t>("main-function"); });
-    /*
-    parser
-        .add_argument({ "-s", "--stubs" }, "Instrument with library stubs for LD_PRELOAD")
-        .dtype("boolean")
-        .max_count(1)
-        .action([&inputlib](parser_t& p) {
-            if(p.get<bool>("stubs"))
-            {
-                for(auto& itr : inputlib)
-                    itr += "-stubs";
-            }
-        });
-    */
-    parser.add_argument({ "--driver" }, "Force main or _init/_fini instrumentation")
-        .dtype("boolean")
-        .max_count(1)
-        .action([](parser_t& p) { is_driver = p.get<bool>("driver"); });
    parser
        .add_argument({ "--load" },
                      "Supplemental instrumentation library names w/o extension (e.g. "
@@ -471,6 +480,7 @@ main(int argc, char** argv)
            std::map<std::string, fmodset_t*> module_function_map = {
                { "available_module_functions", &available_module_functions },
                { "instrumented_module_functions", &instrumented_module_functions },
+                { "coverage_module_functions", &coverage_module_functions },
                { "excluded_module_functions", &excluded_module_functions },
                { "overlapping_module_functions", &overlapping_module_functions },
            };
@@ -545,6 +555,12 @@ main(int argc, char** argv)
                    use_line_info = true;
            }
        });
+    parser.add_argument()
+        .names({ "-C", "--config" })
+        .dtype("string")
+        .min_count(1)
+        .description("Read in a configuration file and encode these values as the "
+                     "defaults in the executable");
    parser.add_argument()
        .names({ "-d", "--default-components" })
        .dtype("string")
@@ -626,6 +642,18 @@ main(int argc, char** argv)
        .action([](parser_t& p) {
            min_loop_address_range = p.get<size_t>("min-address-range-loop");
        });
+    parser.add_argument({ "--coverage" }, "Enable recording the code coverage")
+        .max_count(1)
+        .choices({ "none", "function", "basic_block" })
+        .action([](parser_t& p) {
+            auto _v = p.get<std::string>("coverage");
+            if(_v == "function" || _v.empty())
+                coverage_mode = CODECOV_FUNCTION;
+            else if(_v == "basic_block")
+                coverage_mode = CODECOV_BASIC_BLOCK;
+            else
+                coverage_mode = CODECOV_NONE;
+        });
    parser
        .add_argument({ "--dynamic-callsites" },
                      "Force instrumentation if a function has dynamic callsites (e.g. "
@@ -703,6 +731,41 @@ main(int argc, char** argv)
        return -1;
    }

+    if(parser.exists("config"))
+    {
+        struct omnitrace_env_config_s
+        {};
+        auto _configs = parser.get<strvec_t>("config");
+        for(auto&& itr : _configs)
+        {
+            auto _settings = tim::settings::push<omnitrace_env_config_s>();
+            for(auto&& itr : *_settings)
+            {
+                itr.second->set_config_updated(false);
+                itr.second->set_environ_updated(false);
+            }
+            _settings->read(itr);
+            for(auto&& itr : *_settings)
+            {
+                if(itr.second && itr.second->get_config_updated())
+                {
+                    env_config_variables.emplace_back(TIMEMORY_JOIN(
+                        '=', itr.second->get_env_name(), itr.second->as_string()));
+                    verbprintf(1, "Exporting known config value :: %s\n",
+                               env_config_variables.back().c_str());
+                }
+            }
+            for(auto&& itr : _settings->get_unknown_configs())
+            {
+                env_config_variables.emplace_back(
+                    TIMEMORY_JOIN('=', itr.first, itr.second));
+                verbprintf(1, "Exporting unknown config value :: %s\n",
+                           env_config_variables.back().c_str());
+            }
+            tim::settings::pop<omnitrace_env_config_s>();
+        }
+    }
+
    auto _handle_heuristics = [&parser](std::string&& _exists, std::string&& _not_exists,
                                        auto& _field, auto _value, std::string&& _msg,
                                        bool _cond) {
@@ -781,20 +844,17 @@ main(int argc, char** argv)
                   outfile.c_str());
    }

-    if(modfunc_dump_dir.empty())
+    if(binary_rewrite)
    {
-        modfunc_dump_dir = tim::get_env<std::string>("OMNITRACE_OUTPUT_PATH", "");
-        if(modfunc_dump_dir.empty())
-        {
-            auto _exe_base = (binary_rewrite) ? outfile : std::string{ cmdv0 };
-            auto _pos      = _exe_base.find_last_of('/');
-            if(_pos != std::string::npos && _pos + 1 < _exe_base.length())
-                _exe_base = _exe_base.substr(_pos + 1);
-            modfunc_dump_dir = TIMEMORY_JOIN("-", "omnitrace", _exe_base, "output");
-        }
+        auto* _save = _cmdv[0];
+        _cmdv[0]    = const_cast<char*>(outfile.c_str());
+        tim::timemory_init(_cmdc, _cmdv, "omnitrace-");
+        _cmdv[0] = _save;
+    }
+    else
+    {
+        tim::timemory_init(_cmdc, _cmdv, "omnitrace-");
    }
-
-    if(verbose_level >= 0) tim::makedir(modfunc_dump_dir);

    //----------------------------------------------------------------------------------//
    //
@@ -914,9 +974,7 @@ main(int argc, char** argv)
    // for runtime instrumentation, we need to set this before the process gets created
    if(!binary_rewrite)
    {
-#if defined(OMNITRACE_USE_ROCTRACER)
        tim::set_env("HSA_ENABLE_INTERRUPT", "0", 0);
-#endif
        if(_pid >= 0)
        {
            verbprintf(-10, "#-------------------------------------------------------"
@@ -1053,15 +1111,10 @@ main(int argc, char** argv)
        std::cout << '\n' << std::endl;
    }

-    auto _output_prefix = tim::get_env<std::string>("OMNITRACE_OUTPUT_PREFIX", "");
-
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir,
-                            TIMEMORY_JOIN("", _output_prefix, "available-instr")),
-              available_module_functions, 1, werror, "available-instr", print_formats);
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir,
-                            TIMEMORY_JOIN("", _output_prefix, "overlapping-instr")),
-              overlapping_module_functions, 1, werror, "overlapping_module_functions",
+    dump_info("available-instr", available_module_functions, 1, werror, "available-instr",
              print_formats);
+    dump_info("overlapping-instr", overlapping_module_functions, 1, werror,
+              "overlapping_module_functions", print_formats);

    //----------------------------------------------------------------------------------//
    //
@@ -1139,8 +1192,8 @@ main(int argc, char** argv)
    //
    //----------------------------------------------------------------------------------//

-    auto* _mutatee_init   = find_function(app_image, "_init");
-    auto* _mutatee_fini   = find_function(app_image, "_fini");
+    auto* main_init       = find_function(app_image, "_init");
+    auto* main_fini       = find_function(app_image, "_fini");
    auto* main_func       = find_function(app_image, main_fname.c_str());
    auto* mpi_init_func   = find_function(app_image, "MPI_Init", { "MPI_Init_thread" });
    auto* mpi_fini_func   = find_function(app_image, "MPI_Finalize");
@@ -1168,12 +1221,14 @@ main(int argc, char** argv)

    verbprintf(0, "Finding instrumentation functions...\n");

-    auto* entr_trace = find_function(app_image, "omnitrace_push_trace");
-    auto* exit_trace = find_function(app_image, "omnitrace_pop_trace");
-    auto* init_func  = find_function(app_image, "omnitrace_init");
-    auto* fini_func  = find_function(app_image, "omnitrace_finalize");
-    auto* env_func   = find_function(app_image, "omnitrace_set_env");
-    auto* mpi_func   = find_function(app_image, "omnitrace_set_mpi");
+    auto* init_func    = find_function(app_image, "omnitrace_init");
+    auto* fini_func    = find_function(app_image, "omnitrace_finalize");
+    auto* env_func     = find_function(app_image, "omnitrace_set_env");
+    auto* mpi_func     = find_function(app_image, "omnitrace_set_mpi");
+    auto* entr_trace   = find_function(app_image, "omnitrace_push_trace");
+    auto* exit_trace   = find_function(app_image, "omnitrace_pop_trace");
+    auto* reg_src_func = find_function(app_image, "omnitrace_register_source");
+    auto* reg_cov_func = find_function(app_image, "omnitrace_register_coverage");

    if(!main_func && main_fname == "main") main_func = find_function(app_image, "_main");

@@ -1293,36 +1348,42 @@ main(int argc, char** argv)
    //
    //----------------------------------------------------------------------------------//

-    if(!main_func && is_driver)
+    if(!main_func)
    {
-        errprintf(0, "could not find '%s'\n", main_fname.c_str());
-        if(!_mutatee_init || !_mutatee_fini)
+        if(!main_init && !main_fini)
        {
-            errprintf(-1, "could not find '%s' or '%s', aborting\n", "_init", "_fini");
+            errprintf(-1, "could not find '%s', '_init' or '_fini', aborting...\n",
+                      main_fname.c_str());
+        }
+        else if(!main_init)
+        {
+            errprintf(-1, "could not find '%s' or '_init', aborting...\n",
+                      main_fname.c_str());
+        }
+        else if(!main_fini)
+        {
+            errprintf(-1, "could not find '%s' or '_fini', aborting...\n",
+                      main_fname.c_str());
        }
        else
        {
-            errprintf(0, "using '%s' and '%s' in lieu of '%s'...", "_init", "_fini",
-                      main_fname.c_str());
+            verbprintf(0, "using '%s' and '%s' in lieu of '%s'...", "_init", "_fini",
+                       main_fname.c_str());
        }
    }
-    else if(!main_func && !is_driver)
-    {
-        verbprintf(0, "Warning! No main function and is not driver!\n");
-    }

    using pair_t = std::pair<procedure_t*, string_t>;

    for(const auto& itr :
-        { pair_t(main_func, main_fname), pair_t(entr_trace, "omnitrace_push_trace"),
+        { pair_t(entr_trace, "omnitrace_push_trace"),
          pair_t(exit_trace, "omnitrace_pop_trace"), pair_t(init_func, "omnitrace_init"),
-          pair_t(fini_func, "omnitrace_finalize"),
-          pair_t(env_func, "omnitrace_set_env") })
+          pair_t(fini_func, "omnitrace_finalize"), pair_t(env_func, "omnitrace_set_env"),
+          pair_t(reg_src_func, "omnitrace_register_source"),
+          pair_t(reg_cov_func, "omnitrace_register_coverage") })
    {
-        if(itr.first == main_func && !is_driver) continue;
        if(!itr.first)
        {
-            errprintf(-1, "could not find required function :: '%s;\n",
+            errprintf(-1, "could not find required function :: '%s'\n",
                      itr.second.c_str());
        }
    }
@@ -1350,8 +1411,8 @@ main(int argc, char** argv)

    bool has_debug_info = false;
    check_for_debug_info(has_debug_info, main_func);
-    check_for_debug_info(has_debug_info, _mutatee_init);
-    check_for_debug_info(has_debug_info, _mutatee_fini);
+    check_for_debug_info(has_debug_info, main_init);
+    check_for_debug_info(has_debug_info, main_fini);

    //----------------------------------------------------------------------------------//
    //
@@ -1370,18 +1431,15 @@ main(int argc, char** argv)
        main_exit_points = main_func->findPoint(BPatch_exit);
        verbprintf(2, "Done\n");
    }
-    else if(is_driver)
+    else
    {
-        if(_mutatee_init)
-        {
-            verbprintf(2, "Finding init entry...\n");
-            main_entr_points = _mutatee_init->findPoint(BPatch_entry);
-        }
-        if(_mutatee_fini)
-        {
-            verbprintf(2, "Finding fini exit...\n");
-            main_exit_points = _mutatee_fini->findPoint(BPatch_exit);
-        }
+        verbprintf(2, "Finding init entry... ");
+        main_entr_points = main_init->findPoint(BPatch_entry);
+        verbprintf(2, "Done\n");
+
+        verbprintf(2, "Finding fini exit... ");
+        main_exit_points = main_fini->findPoint(BPatch_exit);
+        verbprintf(2, "Done\n");
    }

    //----------------------------------------------------------------------------------//
@@ -1452,9 +1510,12 @@ main(int argc, char** argv)

    // prioritize the user environment arguments
    auto env_vars = parser.get<strvec_t>("env");
+    env_vars.reserve(env_vars.size() + env_config_variables.size());
+    for(auto&& itr : env_config_variables)
+        env_vars.emplace_back(itr);
    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MODE", instr_mode));
-#if defined(OMNITRACE_USE_ROCTRACER)
    env_vars.emplace_back(TIMEMORY_JOIN('=', "HSA_ENABLE_INTERRUPT", "0"));
+#if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
    env_vars.emplace_back(TIMEMORY_JOIN('=', "HSA_TOOLS_LIB", _libname));
 #endif
    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MPI_INIT", "OFF"));
@@ -1467,6 +1528,8 @@ main(int argc, char** argv)
    env_vars.emplace_back(
        TIMEMORY_JOIN('=', "OMNITRACE_USE_MPIP",
                      (binary_rewrite && use_mpi && use_mpip) ? "ON" : "OFF"));
+    env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_USE_CODE_COVERAGE",
+                                        (coverage_mode != CODECOV_NONE) ? "ON" : "OFF"));
    if(use_mpi) env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_USE_PID", "ON"));

    for(auto& itr : env_vars)
@@ -1508,7 +1571,7 @@ main(int argc, char** argv)

    if(umpi_call) init_names.emplace_back(umpi_call.get());
    if(init_call) init_names.emplace_back(init_call.get());
-    if(main_beg_call) init_names.emplace_back(main_beg_call.get());
+    if(main_func && main_beg_call) init_names.emplace_back(main_beg_call.get());

    for(const auto& itr : end_expr)
        if(itr.second) fini_names.emplace_back(itr.second.get());
@@ -1524,14 +1587,20 @@ main(int argc, char** argv)
    {
        for(const auto& itr : available_module_functions)
        {
+            bool _is_not_main = itr.function != main_func && itr.function != main_init &&
+                                itr.function != main_fini;
            if(itr.should_instrument())
            {
-                if(itr.function != main_func && itr.function != _mutatee_init &&
-                   itr.function != _mutatee_fini)
+                if(_is_not_main)
                    _insert_module_function(instrumented_module_functions, itr);
            }
            else
                _insert_module_function(excluded_module_functions, itr);
+            if(coverage_mode != CODECOV_NONE)
+            {
+                if(itr.should_coverage_instrument() && _is_not_main)
+                    _insert_module_function(coverage_module_functions, itr);
+            }
            if(itr.is_overlapping())
                _insert_module_function(overlapping_module_functions, itr);
        }
@@ -1581,55 +1650,103 @@ main(int argc, char** argv)

    verbprintf(2, "Beginning instrumentation loop...\n");
    verbprintf(1, "\n");
-    std::map<std::string, std::pair<size_t, size_t>> _pass_info{};
-    const int                                        _pass_verbose_lvl = 2;
-    for(const auto& itr : instrumented_module_functions)
-    {
-        auto _count = itr(addr_space, entr_trace, exit_trace);
-        _pass_info[itr.module_name].first += _count.first;
-        _pass_info[itr.module_name].second += _count.second;
-
-        auto _report = [](int _lvl, const string_t& _action, const string_t& _type,
-                          const string_t& _reason, const string_t& _name,
-                          const std::string& _extra = {}) {
-            static std::map<std::string, strset_t> already_reported{};
-            auto                                   _key = _type + _action + _reason;
-            if(already_reported[_key].count(_name) == 0)
-            {
-                verbprintf(_lvl, "[%s][%s] %s :: '%s'", _type.c_str(), _action.c_str(),
-                           _reason.c_str(), _name.c_str());
-                if(!_extra.empty()) verbprintf_bare(_lvl, " (%s)", _extra.c_str());
-                verbprintf_bare(_lvl, "...\n");
-                already_reported[_key].insert(_name);
-            }
-        };
-
-        for(const auto& mitr : itr.messages)
-            _report(std::get<0>(mitr), std::get<1>(mitr), std::get<2>(mitr),
-                    std::get<3>(mitr),
-                    std::get<2>(mitr) == "module" ? itr.module_name : itr.function_name);
-    }
-    verbprintf(1, "\n");
-
-    // report the instrumented
-    for(auto& itr : _pass_info)
-    {
-        auto _valid = (verbose_level > _pass_verbose_lvl ||
-                       (itr.second.first + itr.second.second) > 0);
-        if(_valid)
+    auto _report_info = [](int _lvl, const string_t& _action, const string_t& _type,
+                           const string_t& _reason, const string_t& _name,
+                           const std::string& _extra = {}) {
+        static std::map<std::string, strset_t> already_reported{};
+        auto _key = TIMEMORY_JOIN('_', _type, _action, _reason, _name, _extra);
+        if(already_reported[_key].count(_name) == 0)
        {
+            verbprintf(_lvl, "[%s][%s] %s :: '%s'", _type.c_str(), _action.c_str(),
+                       _reason.c_str(), _name.c_str());
+            if(!_extra.empty()) verbprintf_bare(_lvl, " (%s)", _extra.c_str());
+            verbprintf_bare(_lvl, "...\n");
+            already_reported[_key].insert(_name);
+        }
+    };
+
+    if(instr_mode != "coverage")
+    {
+        std::map<std::string, std::pair<size_t, size_t>> _pass_info{};
+        const int                                        _pass_verbose_lvl = 1;
+        for(const auto& itr : instrumented_module_functions)
+        {
+            auto _count = itr(addr_space, entr_trace, exit_trace);
+            _pass_info[itr.module_name].first += _count.first;
+            _pass_info[itr.module_name].second += _count.second;
+
+            for(const auto& mitr : itr.messages)
+                _report_info(std::get<0>(mitr), std::get<1>(mitr), std::get<2>(mitr),
+                             std::get<3>(mitr),
+                             std::get<2>(mitr) == "module" ? itr.module_name
+                                                           : itr.function_name);
+        }
+
+        // report the trace instrumented functions
+        for(auto& itr : _pass_info)
+        {
+            auto _valid = (verbose_level > _pass_verbose_lvl ||
+                           (itr.second.first + itr.second.second) > 0);
+            if(!_valid) continue;
            verbprintf(_pass_verbose_lvl, "%4zu instrumented procedures in %s\n",
                       itr.second.first, itr.first.c_str());
            _valid = (loop_level_instr &&
                      (verbose_level > _pass_verbose_lvl || itr.second.second > 0));
            if(_valid)
            {
-                verbprintf(_pass_verbose_lvl, "%4zu instrumented loop procedures in %s\n",
+                verbprintf(_pass_verbose_lvl, "%4zu instrumented loops in procedure %s\n",
                           itr.second.second, itr.first.c_str());
            }
        }
    }

+    if(coverage_mode != CODECOV_NONE)
+    {
+        std::map<std::string, std::pair<size_t, size_t>> _covr_info{};
+        const int                                        _covr_verbose_lvl = 1;
+        for(const auto& itr : coverage_module_functions)
+        {
+            itr.register_source(addr_space, reg_src_func, *main_entr_points);
+            auto _count = itr.register_coverage(addr_space, reg_cov_func);
+            _covr_info[itr.module_name].first += _count.first;
+            _covr_info[itr.module_name].second += _count.second;
+
+            for(const auto& mitr : itr.messages)
+                _report_info(std::get<0>(mitr), std::get<1>(mitr), std::get<2>(mitr),
+                             std::get<3>(mitr),
+                             std::get<2>(mitr) == "module" ? itr.module_name
+                                                           : itr.function_name);
+        }
+
+        // report the coverage instrumented functions
+        for(auto& itr : _covr_info)
+        {
+            auto _valid = (verbose_level > _covr_verbose_lvl ||
+                           (itr.second.first + itr.second.second) > 0);
+            if(!_valid) continue;
+            switch(coverage_mode)
+            {
+                case CODECOV_NONE:
+                {
+                    break;
+                }
+                case CODECOV_FUNCTION:
+                {
+                    verbprintf(_covr_verbose_lvl, "%4zu coverage functions in %s\n",
+                               itr.second.first, itr.first.c_str());
+                    break;
+                }
+                case CODECOV_BASIC_BLOCK:
+                {
+                    verbprintf(_covr_verbose_lvl, "%4zu coverage basic blocks in %s\n",
+                               itr.second.second, itr.first.c_str());
+                    break;
+                }
+            }
+        }
+    }
+    verbprintf(1, "\n");
+
    if(app_thread)
    {
        verbprintf(1, "Finalizing insertion set...\n");
@@ -1692,26 +1809,17 @@ main(int argc, char** argv)
        _insert_module_function(excluded_module_functions, itr);
    }

-    bool _dump_and_exit = ((print_available.length() + print_instrumented.length() +
-                            print_overlapping.length() + print_excluded.length()) > 0) ||
-                          explicit_dump_and_exit;
-
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir,
-                            TIMEMORY_JOIN("", _output_prefix, "available-instr")),
-              available_module_functions, 0, werror, "available_module_functions",
-              print_formats);
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir,
-                            TIMEMORY_JOIN("", _output_prefix, "instrumented-instr")),
-              instrumented_module_functions, 0, werror, "instrumented_module_functions",
-              print_formats);
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir,
-                            TIMEMORY_JOIN("", _output_prefix, "excluded-instr")),
-              excluded_module_functions, 0, werror, "excluded_module_functions",
-              print_formats);
-    dump_info(TIMEMORY_JOIN('/', modfunc_dump_dir,
-                            TIMEMORY_JOIN("", _output_prefix, "overlapping-instr")),
-              overlapping_module_functions, 0, werror, "overlapping_module_functions",
-              print_formats);
+    dump_info("available-instr", available_module_functions, 0, werror,
+              "available_module_functions", print_formats);
+    dump_info("instrumented-instr", instrumented_module_functions, 0, werror,
+              "instrumented_module_functions", print_formats);
+    dump_info("excluded-instr", excluded_module_functions, 0, werror,
+              "excluded_module_functions", print_formats);
+    if(coverage_mode != CODECOV_NONE)
+        dump_info("coverage-instr", coverage_module_functions, 0, werror,
+                  "coverage_module_functions", print_formats);
+    dump_info("overlapping-instr", overlapping_module_functions, 0, werror,
+              "overlapping_module_functions", print_formats);

    auto _dump_info = [](const std::string& _label, const string_t& _mode,
                         const fmodset_t& _modset) {
@@ -1733,13 +1841,13 @@ main(int argc, char** argv)
        {
            for(const auto& itr : _modset)
                _insert(itr.module_name, TIMEMORY_JOIN("", "[", itr.function_name, "][",
-                                                       itr.address_range, "]"));
+                                                       itr.num_instructions, "]"));
        }
        else if(_mode == "functions+")
        {
            for(const auto& itr : _modset)
                _insert(itr.module_name, TIMEMORY_JOIN("", "[", itr.signature.get(), "][",
-                                                       itr.address_range, "]"));
+                                                       itr.num_instructions, "]"));
        }
        else if(_mode == "pair")
        {
@@ -1748,7 +1856,7 @@ main(int argc, char** argv)
                std::stringstream _ss{};
                _ss << std::boolalpha;
                _ss << "" << itr.module_name << "] --> [" << itr.function_name << "]["
-                    << itr.address_range << "]";
+                    << itr.num_instructions << "]";
                _insert(itr.module_name, _ss.str());
            }
        }
@@ -1759,7 +1867,7 @@ main(int argc, char** argv)
                std::stringstream _ss{};
                _ss << std::boolalpha;
                _ss << "[" << itr.module_name << "] --> [" << itr.signature.get() << "]["
-                    << itr.address_range << "]";
+                    << itr.num_instructions << "]";
                _insert(itr.module_name, _ss.str());
            }
        }
@@ -1785,10 +1893,12 @@ main(int argc, char** argv)
        _dump_info("instrumented", print_instrumented, instrumented_module_functions);
    if(!print_excluded.empty())
        _dump_info("excluded", print_excluded, excluded_module_functions);
+    if(!print_coverage.empty())
+        _dump_info("coverage", print_coverage, coverage_module_functions);
    if(!print_overlapping.empty())
        _dump_info("overlapping", print_overlapping, overlapping_module_functions);

-    if(_dump_and_exit) exit(EXIT_SUCCESS);
+    if(simulate) exit(EXIT_SUCCESS);

    //----------------------------------------------------------------------------------//
    //
@@ -1922,154 +2032,6 @@ main(int argc, char** argv)
    return code;
 }

-//======================================================================================//
-
-bool
-instrument_module(const string_t& file_name)
-{
-    auto _report = [&file_name](const string_t& _action, const string_t& _reason,
-                                int _lvl) {
-        static strset_t already_reported{};
-        if(already_reported.count(file_name) == 0)
-        {
-            verbprintf(_lvl, "%s module [%s] : '%s'...\n", _action.c_str(),
-                       _reason.c_str(), file_name.c_str());
-            already_reported.insert(file_name);
-        }
-    };
-
-    static std::regex ext_regex{ "\\.(s|S)$", regex_opts };
-    static std::regex sys_regex{ "^(s|k|e|w)_[A-Za-z_0-9\\-]+\\.(c|C)$", regex_opts };
-    static std::regex sys_build_regex{ "^(\\.\\./sysdeps/|/build/)", regex_opts };
-    static std::regex dyninst_regex{ "(dyninst|DYNINST|(^|/)RT[[:graph:]]+\\.c$)",
-                                     regex_opts };
-    static std::regex dependlib_regex{ "^(lib|)(omnitrace|pthread|caliper|gotcha|papi|"
-                                       "cupti|TAU|likwid|pfm|nvperf|unwind)",
-                                       regex_opts };
-    static std::regex core_cmod_regex{
-        "^(malloc|(f|)lock|sig|sem)[a-z_]+(|64|_r|_l)\\.c$"
-    };
-    static std::regex core_lib_regex{
-        "^(lib|)(c|dl|dw|pthread|tcmalloc|profiler|"
-        "tbbmalloc|tbbmalloc_proxy|malloc|stdc\\+\\+)(-|\\.)",
-        regex_opts
-    };
-    static std::regex prefix_regex{ "^(_|\\.[a-zA-Z0-9])", regex_opts };
-
-    // file extensions that should not be instrumented
-    if(std::regex_search(file_name, ext_regex))
-    {
-        return (_report("Excluding", "file extension", 3), false);
-    }
-
-    // system modules that should not be instrumented (wastes time)
-    if(std::regex_search(file_name, sys_regex) ||
-       std::regex_search(file_name, sys_build_regex))
-    {
-        return (_report("Excluding", "system module", 3), false);
-    }
-
-    // dyninst modules that must not be instrumented
-    if(std::regex_search(file_name, dyninst_regex))
-    {
-        return (_report("Excluding", "dyninst module", 3), false);
-    }
-
-    // modules used by omnitrace and dependent libraries
-    if(std::regex_search(file_name, core_lib_regex) ||
-       std::regex_search(file_name, core_cmod_regex))
-    {
-        return (_report("Excluding", "core module", 3), false);
-    }
-
-    // modules used by omnitrace and dependent libraries
-    if(std::regex_search(file_name, dependlib_regex))
-    {
-        return (_report("Excluding", "dependency module", 3), false);
-    }
-
-    // known set of modules whose starting sequence of characters suggest it should not be
-    // instrumented (wastes time)
-    if(std::regex_search(file_name, prefix_regex))
-    {
-        return (_report("Excluding", "prefix match", 3), false);
-    }
-
-    _report("Including", "no constraint", 2);
-
-    return true;
-}
-
-//======================================================================================//
-
-bool
-instrument_entity(const string_t& function_name)
-{
-    auto _report = [&function_name](const string_t& _action, const string_t& _reason,
-                                    int _lvl) {
-        static strset_t already_reported{};
-        if(already_reported.count(function_name) == 0)
-        {
-            verbprintf(_lvl, "%s function [%s] : '%s'...\n", _action.c_str(),
-                       _reason.c_str(), function_name.c_str());
-            already_reported.insert(function_name);
-        }
-    };
-
-    static std::regex exclude(
-        "(omnitrace|tim::|N3tim|MPI_Init|MPI_Finalize|dyninst|tm_clones)", regex_opts);
-    static std::regex exclude_cxx(
-        "(std::_Sp_counted_base|std::(use|has)_facet|std::locale|::sentry|^std::_|::_(M|"
-        "S)_|::basic_string[a-zA-Z,<>: ]+::_M_create|::__|::_(Alloc|State)|"
-        "std::(basic_|)(ifstream|ios|istream|ostream|stream))",
-        regex_opts);
-    static std::regex leading("^(_|\\.|frame_dummy|transaction clone|virtual "
-                              "thunk|non-virtual thunk|\\(|targ|kmp_threadprivate_)",
-                              regex_opts);
-    static std::regex trailing(
-        "(_|\\.part\\.[0-9]+|\\.constprop\\.[0-9]+|\\.|\\.[0-9]+)$", regex_opts);
-    static strset_t whole = []() {
-        auto _v   = get_whole_function_names();
-        auto _ret = _v;
-        for(std::string _ext : { "64", "_l", "_r" })
-            for(const auto& itr : _v)
-                _ret.emplace(itr + _ext);
-        return _ret;
-    }();
-
-    // don't instrument the functions when key is found anywhere in function name
-    if(std::regex_search(function_name, exclude) ||
-       std::regex_search(function_name, exclude_cxx))
-    {
-        _report("Excluding", "critical", 3);
-        return false;
-    }
-
-    if(whole.count(function_name) > 0)
-    {
-        _report("Excluding", "critical", 3);
-        return false;
-    }
-
-    // don't instrument the functions when key is found at the start of the function name
-    if(std::regex_search(function_name, leading))
-    {
-        _report("Excluding", "recommended", 3);
-        return false;
-    }
-
-    // don't instrument the functions when key is found at the end of the function name
-    if(std::regex_search(function_name, trailing))
-    {
-        _report("Excluding", "recommended", 3);
-        return false;
-    }
-
-    _report("Including", "no constraint", 2);
-
-    return true;
-}
-
 //======================================================================================//
 // query_instr -- check whether there are one or more instrumentation points
 //
@@ -2151,50 +2113,18 @@ query_instr(procedure_t* funcToInstr, procedure_loc_t traceLoc, flow_graph_t* cf
 // Constraints for instrumentation. Returns true for those modules that
 // shouldn't be instrumented.
 bool
-module_constraint(string_view_t fname)
+module_constraint(const char*)
 {
-    // fname is the name of module/file
-    string_t _fname = string_t{ fname };
-
-    // never instrumentat any module matching omnitrace
-    if(_fname.find("omnitrace") != string_t::npos) return true;
-
-    // always instrument these modules
-    if(_fname == "DEFAULT_MODULE" || _fname == "LIBRARY_MODULE") return false;
-
-    if(instrument_module(_fname)) return false;
-
-    // do not instrument
-    return true;
+    return false;
 }

 //======================================================================================//
 // Constraint for routines. The constraint returns true for those routines that
 // should not be instrumented.
 bool
-routine_constraint(string_view_t fname)
+routine_constraint(const char*)
 {
-    string_t _fname = string_t{ fname };
-    if(_fname.find("omnitrace") != string_t::npos) return true;
-
-    auto npos = std::string::npos;
-    if(_fname.find("FunctionInfo") != npos || _fname.find("_L_lock") != npos ||
-       _fname.find("_L_unlock") != npos)
-        return true;  // Don't instrument
-    else
-    {
-        // Should the routine fname be instrumented?
-        if(instrument_entity(string_t(fname)))
-        {
-            // Yes it should be instrumented. Return false
-            return false;
-        }
-        else
-        {
-            // No. The selective instrumentation file says: don't instrument it
-            return true;
-        }
-    }
+    return false;
 }

 namespace
@@ -282,7 +282,42 @@ omnitrace_fork_callback(thread_t* parent, thread_t* child)
 }
 //
 //======================================================================================//
-// insert_instr -- generic insert instrumentation function
+// insert_instr -- insert instrumentation into a function
+//
+template <typename Tp>
+bool
+insert_instr(address_space_t* mutatee, const bpvector_t<point_t*>& _points, Tp traceFunc,
+             procedure_loc_t traceLoc, bool allow_traps)
+{
+    if(!traceFunc || _points.empty()) return false;
+
+    auto _trace = traceFunc.get();
+    auto _traps = std::set<point_t*>{};
+    if(!allow_traps)
+    {
+        for(const auto& itr : _points)
+        {
+            if(itr && itr->usesTrap_NP()) _traps.insert(itr);
+        }
+    }
+
+    size_t _n = 0;
+    for(const auto& itr : _points)
+    {
+        if(!itr || _traps.count(itr) > 0)
+            continue;
+        else if(traceLoc == BPatch_entry)
+            mutatee->insertSnippet(*_trace, *itr, BPatch_callBefore, BPatch_firstSnippet);
+        else
+            mutatee->insertSnippet(*_trace, *itr);
+        ++_n;
+    }
+
+    return (_n > 0);
+}
+//
+//======================================================================================//
+// insert_instr -- insert instrumentation into loops
 //
 template <typename Tp>
 bool
@@ -311,27 +346,6 @@ insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,
    if(_points == nullptr) return false;
    if(_points->empty()) return false;

-    /*if(loop_level_instr)
-    {
-        flow_graph_t*                     flow = funcToInstr->getCFG();
-        bpvector_t<basic_loop_t*> basicLoop;
-        flow->getOuterLoops(basicLoop);
-        for(auto litr = basicLoop.begin(); litr != basicLoop.end(); ++litr)
-        {
-            bpvector_t<point_t*>* _tmp;
-            if(traceLoc == BPatch_entry)
-                _tmp = cfGraph->findLoopInstPoints(BPatch_locLoopEntry, *litr);
-            else if(traceLoc == BPatch_exit)
-                _tmp = cfGraph->findLoopInstPoints(BPatch_locLoopExit, *litr);
-            if(!_tmp)
-                continue;
-            for(auto& itr : *_tmp)
-                _points->push_back(itr);
-        }
-    }*/
-
-    // verbprintf(0, "Instrumenting |> [ %s ]\n", name.m_name.c_str());
-
    std::set<point_t*> _traps{};
    if(!allow_traps)
    {
@@ -348,9 +362,6 @@ insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,
            continue;
        else if(traceLoc == BPatch_entry)
            mutatee->insertSnippet(*_trace, *itr, BPatch_callBefore, BPatch_firstSnippet);
-        // else if(traceLoc == BPatch_exit)
-        //    mutatee->insertSnippet(*_trace, *itr, BPatch_callAfter,
-        //    BPatch_firstSnippet);
        else
            mutatee->insertSnippet(*_trace, *itr);
        ++_n;
@@ -358,3 +369,46 @@ insert_instr(address_space_t* mutatee, procedure_t* funcToInstr, Tp traceFunc,

    return (_n > 0);
 }
+//
+//======================================================================================//
+// insert_instr -- insert instrumentation into basic blocks
+//
+template <typename Tp>
+bool
+insert_instr(address_space_t* mutatee, Tp traceFunc, procedure_loc_t traceLoc,
+             basic_block_t* basicBlock, bool allow_traps)
+{
+    point_t* _point = nullptr;
+    auto     _trace = traceFunc.get();
+
+    basic_block_t* _bb = basicBlock;
+    switch(traceLoc)
+    {
+        case BPatch_entry: _point = _bb->findEntryPoint(); break;
+        case BPatch_exit: _point = _bb->findExitPoint(); break;
+        default:
+            verbprintf(0, "Warning! trace location type %i not supported\n",
+                       (int) traceLoc);
+            return false;
+    }
+
+    if(_point == nullptr) return false;
+
+    if(!allow_traps && _point->usesTrap_NP()) return false;
+
+    switch(traceLoc)
+    {
+        case BPatch_entry:
+            return (mutatee->insertSnippet(*_trace, *_point, BPatch_callBefore,
+                                           BPatch_firstSnippet) != nullptr);
+        case BPatch_exit: return (mutatee->insertSnippet(*_trace, *_point) != nullptr);
+        default:
+        {
+            verbprintf(0, "Warning! trace location type %i not supported\n",
+                       (int) traceLoc);
+            return false;
+        }
+    }
+
+    return false;
+}
@@ -4,7 +4,7 @@ function(OMNITRACE_ADD_BIN_TEST)
        TEST
        "" # options
        "NAME;TARGET;TIMEOUT;WORKING_DIRECTORY" # single value args
-        "ARGS;ENVIRONMENT;LABELS;PROPERTIES;PASS_REGULAR_EXPRESSION;FAIL_REGULAR_EXPRESSION;SKIP_REGULAR_EXPRESSION;DEPENDS;COMMAND" # multiple
+        "ARGS;ENVIRONMENT;LABELS;PROPERTIES;PASS_REGEX;FAIL_REGEX;SKIP_REGEX;DEPENDS;COMMAND" # multiple
        # value args
        ${ARGN})

@@ -50,11 +50,11 @@ function(OMNITRACE_ADD_BIN_TEST)
                       LABELS
                       "omnitrace-bin;${TEST_LABELS}"
                       PASS_REGULAR_EXPRESSION
-                       "${TEST_PASS_REGULAR_EXPRESSION}"
+                       "${TEST_PASS_REGEX}"
                       FAIL_REGULAR_EXPRESSION
-                       "${TEST_FAIL_REGULAR_EXPRESSION}"
+                       "${TEST_FAIL_REGEX}"
                       SKIP_REGULAR_EXPRESSION
-                       "${TEST_SKIP_REGULAR_EXPRESSION}"
+                       "${TEST_SKIP_REGEX}"
                       ${TEST_PROPERTIES})
    elseif(TARGET ${TEST_TARGET})
        add_test(
@@ -73,11 +73,11 @@ function(OMNITRACE_ADD_BIN_TEST)
                       LABELS
                       "omnitrace-bin;${TEST_LABELS}"
                       PASS_REGULAR_EXPRESSION
-                       "${TEST_PASS_REGULAR_EXPRESSION}"
+                       "${TEST_PASS_REGEX}"
                       FAIL_REGULAR_EXPRESSION
-                       "${TEST_FAIL_REGULAR_EXPRESSION}"
+                       "${TEST_FAIL_REGEX}"
                       SKIP_REGULAR_EXPRESSION
-                       "${TEST_SKIP_REGULAR_EXPRESSION}"
+                       "${TEST_SKIP_REGEX}"
                       ${TEST_PROPERTIES})
    elseif(OMNITRACE_BUILD_TESTING)
        message(FATAL_ERROR "Error! ${TEST_TARGET} does not exist")
@@ -90,7 +90,7 @@ omnitrace_add_bin_test(
    ARGS --help
    LABELS omnitrace-exe
    TIMEOUT 45
-    PASS_REGULAR_EXPRESSION
+    PASS_REGEX
        ".*\\\[omnitrace\\\] Usage:.*\\\[DEBUG OPTIONS\\\].*\\\[MODE OPTIONS\\\].*\\\[LIBRARY OPTIONS\\\].*\\\[SYMBOL SELECTION OPTIONS\\\].*\\\[RUNTIME OPTIONS\\\].*\\\[GRANULARITY OPTIONS\\\].*\\\[DYNINST OPTIONS\\\].*"
    )

@@ -115,7 +115,7 @@ omnitrace_add_bin_test(
    WORKING_DIRECTORY
        ${PROJECT_BINARY_DIR}/omnitrace-tests-output/omnitrace-exe-simulate-ls
    TIMEOUT 60
-    PASS_REGULAR_EXPRESSION
+    PASS_REGEX
        ".*available-instr.json.*available-instr.txt.*available-instr.xml.*excluded-instr.json.*excluded-instr.txt.*excluded-instr.xml.*instrumented-instr.json.*instrumented-instr.txt.*instrumented-instr.xml.*overlapping-instr.json.*overlapping-instr.txt.*overlapping-instr.xml.*"
    )

@@ -125,7 +125,7 @@ omnitrace_add_bin_test(
    ARGS --help
    LABELS omnitrace-avail
    TIMEOUT 45
-    PASS_REGULAR_EXPRESSION
+    PASS_REGEX
        ".*\\\[omnitrace-avail\\\] Usage:.*\\\[CATEGORIES\\\].*\\\[VIEW OPTIONS\\\].*\\\[COLUMN OPTIONS\\\].*\\\[WIDTH OPTIONS\\\].*\\\[OUTPUT OPTIONS\\\].*"
    )

@@ -135,7 +135,7 @@ omnitrace_add_bin_test(
    ARGS -r wall_clock -C --available
    LABELS omnitrace-avail
    TIMEOUT 45
-    PASS_REGULAR_EXPRESSION
+    PASS_REGEX
        "\\\|[-]+\\\|\n\\\|[ ]+COMPONENT[ ]+\\\|\n\\\|[-]+\\\|\n\\\| (wall_clock)[ ]+\\\|\n\\\| (sampling_wall_clock)[ ]+\\\|\n\\\|[-]+\\\|"
    )

@@ -145,8 +145,8 @@ omnitrace_add_bin_test(
    ARGS --categories settings::omnitrace --brief
    LABELS omnitrace-avail
    TIMEOUT 45
-    PASS_REGULAR_EXPRESSION "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE|OUTPUT_PREFIX)"
-    FAIL_REGULAR_EXPRESSION
+    PASS_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE|OUTPUT_PREFIX)"
+    FAIL_REGEX
        "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)"
    )

@@ -156,6 +156,6 @@ omnitrace_add_bin_test(
    ARGS --categories settings::timemory --brief
    LABELS omnitrace-avail
    TIMEOUT 45
-    PASS_REGULAR_EXPRESSION
+    PASS_REGEX
        "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)"
-    FAIL_REGULAR_EXPRESSION "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE)")
+    FAIL_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE)")
@@ -62,7 +62,7 @@ get_thread_index()

 template <typename FuncT, typename... Args>
 auto
-invoke(const char* _name, int _verbose, FuncT&& _func, Args... _args)
+invoke(const char* _name, int _verbose, bool& _toggle, FuncT&& _func, Args... _args)
 {
    if(_func)
    {
@@ -78,6 +78,7 @@ invoke(const char* _name, int _verbose, FuncT&& _func, Args... _args)
        int32_t _lk = get_guard()++;
        if(_lk == 0)
        {
+            _toggle = !_toggle;
            if(_verbose >= 3)
            {
                fflush(stderr);
@@ -32,6 +32,8 @@
 #include "common/invoke.hpp"
 #include "common/join.hpp"

+#include <cassert>
+
 //--------------------------------------------------------------------------------------//

 #define OMNITRACE_DLSYM(VARNAME, HANDLE, FUNCNAME)                                       \
@@ -183,6 +185,11 @@ struct OMNITRACE_HIDDEN_API indirect
        OMNITRACE_DLSYM(omnitrace_pop_trace_f, m_omnihandle, "omnitrace_pop_trace");
        OMNITRACE_DLSYM(omnitrace_push_region_f, m_omnihandle, "omnitrace_push_region");
        OMNITRACE_DLSYM(omnitrace_pop_region_f, m_omnihandle, "omnitrace_pop_region");
+        OMNITRACE_DLSYM(omnitrace_register_source_f, m_omnihandle,
+                        "omnitrace_register_source");
+        OMNITRACE_DLSYM(omnitrace_register_coverage_f, m_omnihandle,
+                        "omnitrace_register_coverage");
+
 #if OMNITRACE_USE_OMPT == 0
        _warn_verbose = 5;
 #endif
@@ -231,16 +238,19 @@ struct OMNITRACE_HIDDEN_API indirect
    }

 public:
-    void (*omnitrace_init_library_f)(void)                   = nullptr;
-    void (*omnitrace_init_f)(const char*, bool, const char*) = nullptr;
-    void (*omnitrace_finalize_f)(void)                       = nullptr;
-    void (*omnitrace_set_env_f)(const char*, const char*)    = nullptr;
-    void (*omnitrace_set_mpi_f)(bool, bool)                  = nullptr;
-    void (*omnitrace_push_trace_f)(const char*)              = nullptr;
-    void (*omnitrace_pop_trace_f)(const char*)               = nullptr;
-    int (*omnitrace_push_region_f)(const char*)              = nullptr;
-    int (*omnitrace_pop_region_f)(const char*)               = nullptr;
-    int (*omnitrace_user_configure_f)(int, void*, void*)     = nullptr;
+    void (*omnitrace_init_library_f)(void)                                  = nullptr;
+    void (*omnitrace_init_f)(const char*, bool, const char*)                = nullptr;
+    void (*omnitrace_finalize_f)(void)                                      = nullptr;
+    void (*omnitrace_set_env_f)(const char*, const char*)                   = nullptr;
+    void (*omnitrace_set_mpi_f)(bool, bool)                                 = nullptr;
+    void (*omnitrace_register_source_f)(const char*, const char*, size_t, size_t,
+                                        const char*)                        = nullptr;
+    void (*omnitrace_register_coverage_f)(const char*, const char*, size_t) = nullptr;
+    void (*omnitrace_push_trace_f)(const char*)                             = nullptr;
+    void (*omnitrace_pop_trace_f)(const char*)                              = nullptr;
+    int (*omnitrace_push_region_f)(const char*)                             = nullptr;
+    int (*omnitrace_pop_region_f)(const char*)                              = nullptr;
+    int (*omnitrace_user_configure_f)(int, void*, void*)                    = nullptr;
    ompt_start_tool_result_t* (*ompt_start_tool_f)(unsigned int, const char*);

 private:
@@ -262,18 +272,32 @@ get_indirect()
    return _v;
 }

+auto&
+get_inited()
+{
+    static bool* _v = new bool{ false };
+    return *_v;
+}
+
+auto&
+get_finied()
+{
+    static bool* _v = new bool{ false };
+    return *_v;
+}
+
 auto&
 get_active()
 {
-    static bool _v = false;
-    return _v;
+    static bool* _v = new bool{ false };
+    return *_v;
 }

 auto&
 get_enabled()
 {
-    static std::atomic<bool> _v{ get_env("OMNITRACE_INIT_ENABLED", true) };
-    return _v;
+    static auto* _v = new std::atomic<bool>{ get_env("OMNITRACE_INIT_ENABLED", true) };
+    return *_v;
 }

 auto&
@@ -283,19 +307,25 @@ get_thread_enabled()
    return _v;
 }

-auto&
-get_count()
-{
-    static std::atomic<int64_t> _v{ 0 };
-    return _v;
-}
-
 auto&
 get_thread_count()
 {
    static thread_local int64_t _v = 0;
    return _v;
 }
+
+auto&
+get_thread_status()
+{
+    static thread_local bool _v = false;
+    return _v;
+}
+
+// ensure finalization is called
+bool _omnitrace_dl_fini = (std::atexit([]() {
+                               if(get_active()) omnitrace_finalize();
+                           }),
+                           true);
 }  // namespace
 }  // namespace dl
 }  // namespace omnitrace
@@ -304,8 +334,21 @@ get_thread_count()

 #define OMNITRACE_DL_INVOKE(...)                                                         \
    ::omnitrace::common::invoke(__FUNCTION__, ::omnitrace::dl::_omnitrace_dl_verbose,    \
+                                (::omnitrace::dl::get_thread_status() = false),          \
                                __VA_ARGS__)

+#define OMNITRACE_DL_INVOKE_STATUS(STATUS, ...)                                          \
+    ::omnitrace::common::invoke(__FUNCTION__, ::omnitrace::dl::_omnitrace_dl_verbose,    \
+                                STATUS, __VA_ARGS__)
+
+#define OMNITRACE_DL_LOG(LEVEL, ...)                                                     \
+    if(::omnitrace::dl::_omnitrace_dl_verbose >= LEVEL)                                  \
+    {                                                                                    \
+        fflush(stderr);                                                                  \
+        fprintf(stderr, "[omnitrace][" OMNITRACE_COMMON_LIBRARY_NAME "] " __VA_ARGS__);  \
+        fflush(stderr);                                                                  \
+    }
+
 using omnitrace::get_indirect;
 namespace dl = omnitrace::dl;

@@ -318,14 +361,50 @@ extern "C"

    void omnitrace_init(const char* a, bool b, const char* c)
    {
-        OMNITRACE_DL_INVOKE(get_indirect().omnitrace_init_f, a, b, c);
-        dl::get_active() = true;
+        if(dl::get_inited() && dl::get_finied())
+        {
+            OMNITRACE_DL_LOG(2, "%s(%s) ignored :: already initialized and finalized\n",
+                             __FUNCTION__, ::omnitrace::join(", ", a, b, c).c_str());
+            return;
+        }
+        else if(dl::get_inited() && dl::get_active())
+        {
+            OMNITRACE_DL_LOG(2, "%s(%s) ignored :: already initialized and active\n",
+                             __FUNCTION__, ::omnitrace::join(", ", a, b, c).c_str());
+            return;
+        }
+
+        bool _invoked = false;
+        OMNITRACE_DL_INVOKE_STATUS(_invoked, get_indirect().omnitrace_init_f, a, b, c);
+        if(_invoked)
+        {
+            dl::get_active() = true;
+            dl::get_inited() = true;
+        }
    }

    void omnitrace_finalize(void)
    {
-        dl::get_active() = false;
-        OMNITRACE_DL_INVOKE(get_indirect().omnitrace_finalize_f);
+        if(dl::get_inited() && dl::get_finied())
+        {
+            OMNITRACE_DL_LOG(2, "%s() ignored :: already initialized and finalized\n",
+                             __FUNCTION__);
+            return;
+        }
+        else if(dl::get_finied() && !dl::get_active())
+        {
+            OMNITRACE_DL_LOG(2, "%s() ignored :: already finalized but not active\n",
+                             __FUNCTION__);
+            return;
+        }
+
+        bool _invoked = false;
+        OMNITRACE_DL_INVOKE_STATUS(_invoked, get_indirect().omnitrace_finalize_f);
+        if(_invoked)
+        {
+            dl::get_active() = false;
+            dl::get_finied() = true;
+        }
    }

    void omnitrace_push_trace(const char* name)
@@ -391,6 +470,19 @@ extern "C"
        OMNITRACE_DL_INVOKE(get_indirect().omnitrace_set_mpi_f, a, b);
    }

+    void omnitrace_register_source(const char* file, const char* func, size_t line,
+                                   size_t address, const char* source)
+    {
+        OMNITRACE_DL_INVOKE(get_indirect().omnitrace_register_source_f, file, func, line,
+                            address, source);
+    }
+
+    void omnitrace_register_coverage(const char* file, const char* func, size_t address)
+    {
+        OMNITRACE_DL_INVOKE(get_indirect().omnitrace_register_coverage_f, file, func,
+                            address);
+    }
+
    int omnitrace_user_start_trace_dl(void)
    {
        dl::get_enabled().store(true);
@@ -64,6 +64,11 @@ extern "C"
    void omnitrace_pop_trace(const char* name) OMNITRACE_PUBLIC_API;
    void omnitrace_push_region(const char*) OMNITRACE_PUBLIC_API;
    void omnitrace_pop_region(const char*) OMNITRACE_PUBLIC_API;
+    void omnitrace_register_source(const char* file, const char* func, size_t line,
+                                   size_t      address,
+                                   const char* source) OMNITRACE_PUBLIC_API;
+    void omnitrace_register_coverage(const char* file, const char* func,
+                                     size_t address) OMNITRACE_PUBLIC_API;

 #if defined(OMNITRACE_DL_SOURCE) && (OMNITRACE_DL_SOURCE > 0)
    int omnitrace_user_start_trace_dl(void) OMNITRACE_HIDDEN_API;
@@ -30,9 +30,7 @@ target_link_libraries(
        $<BUILD_INTERFACE:omnitrace::omnitrace-rocm-smi>
        $<BUILD_INTERFACE:omnitrace::omnitrace-mpi>
        $<BUILD_INTERFACE:omnitrace::omnitrace-ptl>
-        $<BUILD_INTERFACE:timemory::timemory-headers>
-        $<BUILD_INTERFACE:timemory::timemory-gotcha>
-        $<BUILD_INTERFACE:timemory::timemory-cxx-static>
+        $<BUILD_INTERFACE:omnitrace::omnitrace-timemory>
        $<IF:$<BOOL:${OMNITRACE_BUILD_STATIC_LIBGCC}>,$<BUILD_INTERFACE:omnitrace::omnitrace-static-libgcc>,>
        $<IF:$<BOOL:${OMNITRACE_BUILD_STATIC_LIBSTDCXX}>,$<BUILD_INTERFACE:omnitrace::omnitrace-static-libstdcxx>,>
        $<IF:$<BOOL:${OMNITRACE_USE_SANITIZER}>,omnitrace::omnitrace-sanitizer,>)
@@ -53,6 +51,7 @@ set(library_sources
    ${CMAKE_CURRENT_LIST_DIR}/src/library.cpp
    ${CMAKE_CURRENT_LIST_DIR}/src/library/api.cpp
    ${CMAKE_CURRENT_LIST_DIR}/src/library/config.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/library/coverage.cpp
    ${CMAKE_CURRENT_LIST_DIR}/src/library/cpu_freq.cpp
    ${CMAKE_CURRENT_LIST_DIR}/src/library/critical_trace.cpp
    ${CMAKE_CURRENT_LIST_DIR}/src/library/kokkosp.cpp
@@ -79,6 +78,7 @@ set(library_headers
    ${CMAKE_CURRENT_LIST_DIR}/include/library/api.hpp
    ${CMAKE_CURRENT_LIST_DIR}/include/library/config.hpp
    ${CMAKE_CURRENT_LIST_DIR}/include/library/common.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/include/library/coverage.hpp
    ${CMAKE_CURRENT_LIST_DIR}/include/library/cpu_freq.hpp
    ${CMAKE_CURRENT_LIST_DIR}/include/library/critical_trace.hpp
    ${CMAKE_CURRENT_LIST_DIR}/include/library/debug.hpp
@@ -40,6 +40,7 @@
 #include "library/ptl.hpp"
 #include "library/debug.hpp"
 #include "library/critical_trace.hpp"
+#include "library/runtime.hpp"

 #include <timemory/macros/language.hpp>
 #include <timemory/utility/utility.hpp>
@@ -26,6 +26,8 @@

 #include <timemory/compat/macros.h>

+#include <cstddef>
+
 // forward decl of the API
 extern "C"
 {
@@ -57,6 +59,15 @@ extern "C"
    /// stops an instrumentation region (user-defined)
    int omnitrace_pop_region(const char* name) OMNITRACE_PUBLIC_API;

+    /// stores source code information
+    void omnitrace_register_source(const char* file, const char* func, size_t line,
+                                   size_t      address,
+                                   const char* source) OMNITRACE_PUBLIC_API;
+
+    /// increments coverage values
+    void omnitrace_register_coverage(const char* file, const char* func,
+                                     size_t address) OMNITRACE_PUBLIC_API;
+
    // these are the real implementations for internal calling convention
    void omnitrace_init_library_hidden(void) OMNITRACE_HIDDEN_API;
    void omnitrace_init_hidden(const char*, bool, const char*) OMNITRACE_HIDDEN_API;
@@ -68,4 +79,9 @@ extern "C"
    void omnitrace_pop_trace_hidden(const char* name) OMNITRACE_HIDDEN_API;
    void omnitrace_push_region_hidden(const char* name) OMNITRACE_HIDDEN_API;
    void omnitrace_pop_region_hidden(const char* name) OMNITRACE_HIDDEN_API;
+    void omnitrace_register_source_hidden(const char* file, const char* func, size_t line,
+                                          size_t      address,
+                                          const char* source) OMNITRACE_HIDDEN_API;
+    void omnitrace_register_coverage_hidden(const char* file, const char* func,
+                                            size_t address) OMNITRACE_HIDDEN_API;
 }
@@ -31,6 +31,7 @@
 #include "library/common.hpp"
 #include "library/components/fwd.hpp"
 #include "library/defines.hpp"
+#include "library/state.hpp"
 #include "library/thread_data.hpp"

 #include <chrono>
@@ -145,7 +146,7 @@ inline void set_state(State) {}
 }  // namespace rocm_smi
 }  // namespace omnitrace

-#if defined(OMNITRACE_USE_ROCM_SMI)
+#if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
 #    if !defined(OMNITRACE_EXTERN_COMPONENTS) ||                                         \
        (defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0)

@@ -73,6 +73,18 @@ set_setting_value(const std::string& _name, Tp&& _v)
    return _setting->second->set(std::forward<Tp>(_v));
 }

+template <typename Tp>
+std::pair<bool, Tp>
+get_setting_value(const std::string& _name)
+{
+    auto _instance = tim::settings::shared_instance();
+    if(!_instance) return std::make_pair(false, Tp{});
+    auto _setting = _instance->find(_name);
+    if(_setting == _instance->end() || !_setting->second)
+        return std::make_pair(false, Tp{});
+    return _setting->second->get<Tp>();
+}
+
 //
 //      User-configurable settings
 //
@@ -145,9 +157,15 @@ get_use_critical_trace();
 bool
 get_use_kokkosp();

+bool
+get_use_kokkosp_kernel_logger();
+
 bool
 get_use_ompt();

+bool
+get_use_code_coverage();
+
 bool
 get_timeline_sampling();

@@ -0,0 +1,167 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "timemory/mpl/concepts.hpp"
+#include "timemory/tpls/cereal/cereal/cereal.hpp"
+#include <timemory/tpls/cereal/cereal.hpp>
+
+#include <cstddef>
+#include <set>
+#include <string>
+
+#if !defined(OMNITRACE_SERIALIZE)
+#    define OMNITRACE_SERIALIZE(MEMBER_VARIABLE)                                         \
+        ar(::tim::cereal::make_nvp(#MEMBER_VARIABLE, MEMBER_VARIABLE))
+#endif
+
+namespace omnitrace
+{
+namespace coverage
+{
+void
+post_process();
+
+//--------------------------------------------------------------------------------------//
+//
+/// \struct code_coverage
+/// \brief Summary information about the code coverage
+//
+//--------------------------------------------------------------------------------------//
+
+struct code_coverage
+{
+    using int_set_t = std::set<size_t>;
+    using str_set_t = std::set<std::string>;
+
+    enum Category
+    {
+        STANDARD = 0,
+        ADDRESS,
+        MODULE,
+        FUNCTION
+    };
+
+    struct data
+    {
+        int_set_t addresses = {};
+        str_set_t modules   = {};
+        str_set_t functions = {};
+
+        template <typename ArchiveT>
+        void serialize(ArchiveT& ar, const unsigned version);
+    };
+
+    double operator()(Category _c = STANDARD) const;
+    double get(Category _c = STANDARD) const { return (*this)(_c); }
+
+    int_set_t get_uncovered_addresses() const;
+    str_set_t get_uncovered_modules() const;
+    str_set_t get_uncovered_functions() const;
+
+    template <typename ArchiveT>
+    void serialize(ArchiveT& ar, const unsigned version);
+
+    size_t count    = 0;
+    size_t size     = 0;
+    data   covered  = {};
+    data   possible = {};
+};
+//
+template <typename ArchiveT>
+void
+code_coverage::serialize(ArchiveT& ar, const unsigned version)
+{
+    OMNITRACE_SERIALIZE(count);
+    OMNITRACE_SERIALIZE(size);
+    OMNITRACE_SERIALIZE(covered);
+    OMNITRACE_SERIALIZE(possible);
+    if constexpr(tim::concepts::is_output_archive<ArchiveT>::value)
+    {
+        ar.setNextName("coverage");
+        ar.startNode();
+        ar(tim::cereal::make_nvp("total", get(STANDARD)));
+        ar(tim::cereal::make_nvp("addresses", get(ADDRESS)));
+        ar(tim::cereal::make_nvp("modules", get(MODULE)));
+        ar(tim::cereal::make_nvp("functions", get(FUNCTION)));
+        ar.finishNode();
+    }
+    (void) version;
+}
+//
+template <typename ArchiveT>
+void
+code_coverage::data::serialize(ArchiveT& ar, const unsigned version)
+{
+    OMNITRACE_SERIALIZE(addresses);
+    OMNITRACE_SERIALIZE(modules);
+    OMNITRACE_SERIALIZE(functions);
+    (void) version;
+}
+
+//--------------------------------------------------------------------------------------//
+//
+/// \struct coverage_data
+/// \brief Detailed information about the code coverage
+//
+//--------------------------------------------------------------------------------------//
+
+struct coverage_data
+{
+    using data_tuple_t = std::tuple<std::string_view, std::string_view, size_t>;
+
+    template <typename ArchiveT>
+    void serialize(ArchiveT& ar, const unsigned version);
+
+    coverage_data& operator+=(const coverage_data& rhs);
+    bool           operator==(const coverage_data& rhs) const;
+    bool           operator==(const data_tuple_t& rhs) const;
+    bool           operator!=(const coverage_data& rhs) const;
+    bool           operator<(const coverage_data& rhs) const;
+    bool           operator<=(const coverage_data& rhs) const;
+    bool           operator>(const coverage_data& rhs) const;
+    bool           operator>=(const coverage_data& rhs) const;
+
+    size_t      count    = 0;
+    size_t      address  = 0;
+    size_t      line     = 0;
+    std::string module   = {};
+    std::string function = {};
+    std::string source   = {};
+};
+//
+template <typename ArchiveT>
+void
+coverage_data::serialize(ArchiveT& ar, const unsigned version)
+{
+    OMNITRACE_SERIALIZE(count);
+    OMNITRACE_SERIALIZE(line);
+    OMNITRACE_SERIALIZE(address);
+    OMNITRACE_SERIALIZE(module);
+    OMNITRACE_SERIALIZE(function);
+    OMNITRACE_SERIALIZE(source);
+    (void) version;
+}
+//
+}  // namespace coverage
+}  // namespace omnitrace
@@ -217,7 +217,7 @@ using hash_ids = std::unordered_set<std::string>;
 uint64_t
 get_update_frequency();

-std::unique_ptr<call_chain>&
+unique_ptr_t<call_chain>&
 get(int64_t _tid = threading::get_id());

 size_t
@@ -30,8 +30,10 @@
 #include "library/components/roctracer.hpp"
 #include "library/defines.hpp"
 #include "library/state.hpp"
+#include "library/thread_data.hpp"
 #include "library/timemory.hpp"

+#include <memory>
 #include <timemory/backends/threading.hpp>
 #include <timemory/macros/language.hpp>

@@ -48,13 +50,6 @@ using main_bundle_t =

 using gotcha_bundle_t = tim::lightweight_tuple<fork_gotcha_t, mpi_gotcha_t>;

-// bundle of components used in instrumentation
-using instrumentation_bundle_t =
-    tim::component_bundle<api::omnitrace, comp::wall_clock*, comp::user_global_bundle*>;
-
-// allocator for instrumentation_bundle_t
-using bundle_allocator_t = tim::data::ring_buffer_allocator<instrumentation_bundle_t>;
-
 // bundle of components around each thread
 #if defined(TIMEMORY_RUSAGE_THREAD) && TIMEMORY_RUSAGE_THREAD > 0
 using omnitrace_thread_bundle_t =
@@ -75,14 +70,14 @@ get_gotcha_bundle();
 std::atomic<uint64_t>&
 get_cpu_cid();

-std::unique_ptr<std::vector<uint64_t>>&
+unique_ptr_t<std::vector<uint64_t>>&
 get_cpu_cid_stack(int64_t _tid = threading::get_id(), int64_t _parent = 0);

 using cpu_cid_data_t       = std::tuple<uint64_t, uint64_t, uint16_t>;
 using cpu_cid_pair_t       = std::tuple<uint64_t, uint16_t>;
 using cpu_cid_parent_map_t = std::unordered_map<uint64_t, cpu_cid_pair_t>;

-std::unique_ptr<cpu_cid_parent_map_t>&
+unique_ptr_t<cpu_cid_parent_map_t>&
 get_cpu_cid_parents(int64_t _tid = threading::get_id());

 cpu_cid_data_t
@@ -53,7 +53,7 @@ using component::sampling_gpu_temp;
 using component::sampling_percent;
 using component::sampling_wall_clock;

-std::unique_ptr<std::set<int>>&
+unique_ptr_t<std::set<int>>&
 get_signal_types(int64_t _tid);

 std::set<int>
@@ -70,7 +70,7 @@ using bundle_t          = tim::lightweight_tuple<backtrace>;
 using sampler_t         = tim::sampling::sampler<bundle_t, tim::sampling::dynamic>;
 using sampler_instances = thread_data<sampler_t, api::sampling>;

-std::unique_ptr<sampler_t>&
+unique_ptr_t<sampler_t>&
 get_sampler(int64_t _tid = threading::get_id());

 }  // namespace sampling
@@ -39,7 +39,8 @@ enum class State : unsigned short
 enum class Mode : unsigned short
 {
    Trace = 0,
-    Sampling
+    Sampling,
+    Coverage
 };
 }  // namespace omnitrace

@@ -22,8 +22,11 @@

 #pragma once

+#include "library/api.hpp"
+#include "library/common.hpp"
+#include "library/config.hpp"
 #include "library/defines.hpp"
-#include "library/runtime.hpp"
+#include "library/timemory.hpp"

 #include <array>
 #include <cstdint>
@@ -37,20 +40,44 @@

 namespace omnitrace
 {
+// bundle of components used in instrumentation
+using instrumentation_bundle_t =
+    tim::component_bundle<api::omnitrace, comp::wall_clock*, comp::user_global_bundle*>;
+
+// allocator for instrumentation_bundle_t
+using bundle_allocator_t = tim::data::ring_buffer_allocator<instrumentation_bundle_t>;
+
+template <typename Tp>
+struct thread_deleter;
+
+// unique ptr type for omnitrace
+template <typename Tp>
+using unique_ptr_t = std::unique_ptr<Tp, thread_deleter<Tp>>;
+
 static constexpr size_t max_supported_threads = OMNITRACE_MAX_THREADS;

+template <typename Tp>
+struct thread_deleter
+{
+    void operator()(Tp* ptr) const
+    {
+        if(get_state() != State::Finalized) omnitrace_finalize_hidden();
+        delete ptr;
+    }
+};
+
 template <typename Tp, typename Tag = void, size_t MaxThreads = max_supported_threads>
 struct thread_data
 {
-    using instance_array_t  = std::array<std::unique_ptr<Tp>, MaxThreads>;
+    using instance_array_t  = std::array<unique_ptr_t<Tp>, MaxThreads>;
    using construct_on_init = std::true_type;

    template <typename... Args>
-    static void                 construct(Args&&...);
-    static std::unique_ptr<Tp>& instance();
-    static instance_array_t&    instances();
+    static void              construct(Args&&...);
+    static unique_ptr_t<Tp>& instance();
+    static instance_array_t& instances();
    template <typename... Args>
-    static std::unique_ptr<Tp>& instance(construct_on_init, Args&&...);
+    static unique_ptr_t<Tp>& instance(construct_on_init, Args&&...);
    template <typename... Args>
    static instance_array_t& instances(construct_on_init, Args&&...);

@@ -72,14 +99,14 @@ thread_data<Tp, Tag, MaxThreads>::construct(Args&&... _args)
    static auto&             _instances = instances();
    static thread_local bool _v         = [&_args...]() {
        _instances.at(threading::get_id()) =
-            std::make_unique<Tp>(std::forward<Args>(_args)...);
+            unique_ptr_t<Tp>{ new Tp(std::forward<Args>(_args)...) };
        return true;
    }();
    (void) _v;
 }

 template <typename Tp, typename Tag, size_t MaxThreads>
-std::unique_ptr<Tp>&
+unique_ptr_t<Tp>&
 thread_data<Tp, Tag, MaxThreads>::instance()
 {
    return instances().at(threading::get_id());
@@ -95,7 +122,7 @@ thread_data<Tp, Tag, MaxThreads>::instances()

 template <typename Tp, typename Tag, size_t MaxThreads>
 template <typename... Args>
-std::unique_ptr<Tp>&
+unique_ptr_t<Tp>&
 thread_data<Tp, Tag, MaxThreads>::instance(construct_on_init, Args&&... _args)
 {
    construct(std::forward<Args>(_args)...);
@@ -110,7 +137,7 @@ thread_data<Tp, Tag, MaxThreads>::instances(construct_on_init, Args&&... _args)
    static auto _v = [&]() {
        auto _internal = instance_array_t{};
        for(size_t i = 0; i < MaxThreads; ++i)
-            _internal.at(i) = std::make_unique<Tp>(std::forward<Args>(_args)...);
+            _internal.at(i) = unique_ptr_t<Tp>{ new Tp(std::forward<Args>(_args)...) };
        return _internal;
    }();
    return _v;
@@ -28,6 +28,7 @@
 #include "library/components/mpi_gotcha.hpp"
 #include "library/components/pthread_gotcha.hpp"
 #include "library/config.hpp"
+#include "library/coverage.hpp"
 #include "library/critical_trace.hpp"
 #include "library/debug.hpp"
 #include "library/defines.hpp"
@@ -114,9 +115,7 @@ ensure_finalization(bool _static_init = false)
        //
        // see:
        // https://github.com/ROCm-Developer-Tools/roctracer/issues/22#issuecomment-572814465
-#if defined(OMNITRACE_USE_ROCTRACER)
        tim::set_env("HSA_ENABLE_INTERRUPT", "0", 0);
-#endif
    }
    return scope::destructor{ []() { omnitrace_finalize_hidden(); } };
 }
@@ -346,6 +345,13 @@ omnitrace_pop_region_hidden(const char* name)
 extern "C" void
 omnitrace_set_env_hidden(const char* env_name, const char* env_val)
 {
+    struct set_env_s  // NOLINT
+    {};
+    tim::auto_lock_t _lk{ tim::type_mutex<set_env_s>() };
+
+    static auto _set_envs = std::set<std::string_view>{};
+    bool        _success  = _set_envs.emplace(env_name).second;
+
    // just search env to avoid initializing the settings
    OMNITRACE_CONDITIONAL_PRINT_F(get_debug_init() || get_verbose_env() > 2,
                                  "Setting env: %s=%s\n", env_name, env_val);
@@ -353,7 +359,7 @@ omnitrace_set_env_hidden(const char* env_name, const char* env_val)
    tim::set_env(env_name, env_val, 0);

    OMNITRACE_CONDITIONAL_THROW(
-        get_state() >= State::Init &&
+        _success && get_state() >= State::Init &&
            (config::get_is_continuous_integration() || get_debug_init()),
        "omnitrace_set_env(\"%s\", \"%s\") called after omnitrace was initialized. state "
        "= %s",
@@ -375,6 +381,14 @@ std::function<void()> _start_gotcha_callback = []() {};
 extern "C" void
 omnitrace_set_mpi_hidden(bool use, bool attached)
 {
+    static bool _once = false;
+    static auto _args = std::make_pair(use, attached);
+
+    // this function may be called multiple times if multiple libraries are instrumented
+    // we want to guard against multiple calls which with different arguments
+    if(_once && std::tie(_args.first, _args.second) == std::tie(use, attached)) return;
+    _once = true;
+
    // just search env to avoid initializing the settings
    OMNITRACE_CONDITIONAL_PRINT_F(get_debug_init() || get_verbose_env() > 2,
                                  "use: %s, attached: %s\n", (use) ? "y" : "n",
@@ -478,6 +492,20 @@ omnitrace_init_library_hidden()
        get_use_sampling()       = tim::get_env("OMNITRACE_USE_SAMPLING", true);
        get_use_critical_trace() = false;
    }
+    else if(_mode == Mode::Coverage)
+    {
+        for(auto&& itr :
+            { "USE_SAMPLING", "CRITICAL_TRACE", "USE_ROCTRACER", "USE_ROCM_SMI",
+              "USE_PERFETTO", "USE_TIMEMORY", "USE_KOKKOSP", "USE_OMPT" })
+        {
+            auto _name = JOIN('_', "OMNITRACE", itr);
+            if(!config::set_setting_value(_name, false))
+            {
+                OMNITRACE_VERBOSE_F(4, "No configuration setting named '%s'",
+                                    _name.c_str());
+            }
+        }
+    }

    tim::trait::runtime_enabled<comp::roctracer>::set(get_use_roctracer());
    tim::trait::runtime_enabled<comp::roctracer_data>::set(get_use_roctracer() &&
@@ -494,7 +522,7 @@ omnitrace_init_library_hidden()
        tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force);
    }

-#if defined(OMNITRACE_USE_ROCTRACER)
+#if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
    tim::set_env("HSA_TOOLS_LIB", "libomnitrace.so", 0);
 #endif
 }
@@ -814,11 +842,40 @@ omnitrace_init_tooling_hidden()
 extern "C" void
 omnitrace_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _argv0)
 {
+    static int  _total_count = 0;
+    static auto _args = std::make_pair(std::string_view{ _mode }, _is_binary_rewrite);
+
+    auto _count   = _total_count++;
+    auto _mode_sv = std::string_view{ _mode };
+    // this function may be called multiple times if multiple libraries are instrumented
+    // we want to guard against multiple calls which with different arguments
+    if(_count > 0 &&
+       std::tie(_args.first, _args.second) == std::tie(_mode_sv, _is_binary_rewrite))
+        return;
+
+    OMNITRACE_CONDITIONAL_THROW(
+        _count > 0 &&
+            std::tie(_args.first, _args.second) != std::tie(_mode_sv, _is_binary_rewrite),
+        "\nomnitrace_init(...) called multiple times with different arguments for mode "
+        "and/or is_binary_rewrite:"
+        "\n    Invocation #1: omnitrace_init(mode=%-8s, is_binary_rewrite=%-5s, ...)"
+        "\n    Invocation #%i: omnitrace_init(mode=%-8s, is_binary_rewrite=%-5s, ...)",
+        _args.first.data(), std::to_string(_args.second).c_str(), _count + 1, _mode,
+        std::to_string(_is_binary_rewrite).c_str());
+
    // always the first
    (void) get_state();
    (void) push_count();
    (void) pop_count();

+    OMNITRACE_CONDITIONAL_THROW(
+        get_state() >= State::Init &&
+            (config::get_is_continuous_integration() || get_debug_init()),
+        "omnitrace_init(mode=%s, is_binary_rewrite=%s, argv0=%s) called after omnitrace "
+        "was initialized. state = %s",
+        _mode, std::to_string(_is_binary_rewrite).c_str(), _argv0,
+        std::to_string(get_state()).c_str());
+
    get_finalization_functions().emplace_back([_argv0]() {
        OMNITRACE_CI_THROW(get_state() != State::Active,
                           "Finalizer function for popping main invoked in non-active "
@@ -845,6 +902,16 @@ omnitrace_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _a
    tim::set_env("OMNITRACE_MODE", _mode, 0);
    config::is_binary_rewrite() = _is_binary_rewrite;

+    if(get_mode() == Mode::Coverage)
+    {
+        tim::set_env("OMNITRACE_USE_PERFETTO", "OFF", 0);
+        tim::set_env("OMNITRACE_USE_TIMEMORY", "OFF", 0);
+        tim::set_env("OMNITRACE_USE_KOKKOSP", "OFF", 0);
+        tim::set_env("OMNITRACE_USE_SAMPLING", "OFF", 0);
+        tim::set_env("OMNITRACE_USE_ROCTRACER", "OFF", 0);
+        tim::set_env("OMNITRACE_USE_ROCM_SMI", "OFF", 0);
+    }
+
    // set OMNITRACE_USE_SAMPLING to ON by default if mode is sampling
    tim::set_env("OMNITRACE_USE_SAMPLING", (get_mode() == Mode::Sampling) ? "ON" : "OFF",
                 0);
@@ -1086,8 +1153,8 @@ omnitrace_finalize_hidden(void)
    {
        if(get_verbose() >= 0) fprintf(stderr, "\n");
        if(get_verbose() >= 0 || get_debug())
-            fprintf(stderr, "[%s]|%i> Flushing perfetto...\n", OMNITRACE_FUNCTION,
-                    dmp::rank());
+            fprintf(stderr, "[%s][%s]|%i> Flushing perfetto...\n", TIMEMORY_PROJECT_NAME,
+                    OMNITRACE_FUNCTION, dmp::rank());

        // Make sure the last event is closed for this example.
        perfetto::TrackEvent::Flush();
@@ -1108,8 +1175,9 @@ omnitrace_finalize_hidden(void)
        }
        // Write the trace into a file.
        if(get_verbose() >= 0)
-            fprintf(stderr, "[%s]|%i> Outputting '%s' (%.2f KB / %.2f MB / %.2f GB)... ",
-                    OMNITRACE_FUNCTION, dmp::rank(),
+            fprintf(stderr,
+                    "[%s][%s]|%i> Outputting '%s' (%.2f KB / %.2f MB / %.2f GB)... ",
+                    TIMEMORY_PROJECT_NAME, OMNITRACE_FUNCTION, dmp::rank(),
                    get_perfetto_output_filename().c_str(),
                    static_cast<double>(trace_data.size()) / units::KB,
                    static_cast<double>(trace_data.size()) / units::MB,
@@ -1152,6 +1220,8 @@ omnitrace_finalize_hidden(void)
        tasking::get_critical_trace_thread_pool().destroy_threadpool();
    }

+    coverage::post_process();
+
    OMNITRACE_DEBUG_F("Finalizing timemory...\n");
    tim::timemory_finalize();

@@ -95,3 +95,16 @@ omnitrace_set_mpi(bool use, bool attached)
 {
    omnitrace_set_mpi_hidden(use, attached);
 }
+
+extern "C" void
+omnitrace_register_source(const char* file, const char* func, size_t line, size_t address,
+                          const char* source)
+{
+    omnitrace_register_source_hidden(file, func, line, address, source);
+}
+
+extern "C" void
+omnitrace_register_coverage(const char* file, const char* func, size_t address)
+{
+    omnitrace_register_coverage_hidden(file, func, address);
+}
@@ -21,6 +21,7 @@
 // SOFTWARE.

 #include "library/components/fwd.hpp"
+#include "library/components/pthread_gotcha.hpp"
 #include "library/components/rocm_smi.hpp"
 #include "library/config.hpp"
 #include "library/debug.hpp"
@@ -109,7 +110,7 @@ using papi_vector_instances     = thread_data<hw_counters, api::sampling>;

 namespace
 {
-std::unique_ptr<hw_counters>&
+unique_ptr_t<hw_counters>&
 get_papi_vector(int64_t _tid)
 {
    static auto& _v = papi_vector_instances::instances();
@@ -117,14 +118,14 @@ get_papi_vector(int64_t _tid)
    return _v.at(_tid);
 }

-std::unique_ptr<backtrace>&
+unique_ptr_t<backtrace>&
 get_backtrace_init(int64_t _tid)
 {
    static auto& _v = backtrace_init_instances::instances();
    return _v.at(_tid);
 }

-std::unique_ptr<bool>&
+unique_ptr_t<bool>&
 get_sampler_running(int64_t _tid)
 {
    static auto& _v = sampler_running_instances::instances();
@@ -345,7 +346,7 @@ backtrace::configure(bool _setup, int64_t _tid)
        _sampler->set_signals(*_signal_types);
        _sampler->set_flags(SA_RESTART);
        _sampler->set_delay(_delay);
-        _sampler->set_verbose(std::min<size_t>(_sampler->get_verbose(), 1));
+        _sampler->set_verbose(std::min<size_t>(_sampler->get_verbose(), 2));
        _sampler->set_frequency(_prof_freq, { SIGPROF });
        _sampler->set_frequency(_alrm_freq, { SIGALRM });

@@ -73,7 +73,7 @@ mpi_gotcha::configure()
        mpi_gotcha_t::template configure<1, int, int*, char***, int, int*>(
            "MPI_Init_thread");
        mpi_gotcha_t::template configure<2, int>("MPI_Finalize");
-#if defined(OMNITRACE_USE_MPI_HEADERS)
+#if defined(OMNITRACE_USE_MPI_HEADERS) && OMNITRACE_USE_MPI_HEADERS > 0
        mpi_gotcha_t::template configure<3, int, comm_t, int*>("MPI_Comm_rank");
        mpi_gotcha_t::template configure<4, int, comm_t, int*>("MPI_Comm_size");
 #endif
@@ -25,6 +25,7 @@
 #include "library/components/roctracer.hpp"
 #include "library/config.hpp"
 #include "library/debug.hpp"
+#include "library/runtime.hpp"
 #include "library/sampling.hpp"
 #include "library/thread_data.hpp"

@@ -33,6 +33,7 @@
 #include "library/components/rocm_smi.hpp"
 #include "library/common.hpp"
 #include "library/components/fwd.hpp"
+#include "library/components/pthread_gotcha.hpp"
 #include "library/config.hpp"
 #include "library/critical_trace.hpp"
 #include "library/debug.hpp"
@@ -127,7 +128,7 @@ data::print(std::ostream& _os) const

 namespace
 {
-std::vector<std::unique_ptr<bundle_t>*> _bundle_data{};
+std::vector<unique_ptr_t<bundle_t>*> _bundle_data{};
 }

 void
@@ -139,7 +140,8 @@ config()
        if(data::device_list.count(i) > 0)
        {
            _bundle_data.at(i) = &sampler_instances::instances().at(i);
-            if(!*_bundle_data.at(i)) *_bundle_data.at(i) = std::make_unique<bundle_t>();
+            if(!*_bundle_data.at(i))
+                *_bundle_data.at(i) = unique_ptr_t<bundle_t>{ new bundle_t{} };
        }
    }

@@ -96,9 +96,9 @@ get_clock_skew()
            _cpu_ave += _cpu_ts / _n;
            _gpu_ave += _gpu_ts / _n;
        }
-        OMNITRACE_BASIC_VERBOSE(1, "CPU timestamp: %li\n", _cpu_ave);
-        OMNITRACE_BASIC_VERBOSE(1, "HIP timestamp: %li\n", _gpu_ave);
-        OMNITRACE_BASIC_VERBOSE(0, "CPU/HIP timestamp skew: %li (used: %s)\n", _diff,
+        OMNITRACE_BASIC_VERBOSE(2, "CPU timestamp: %li\n", _cpu_ave);
+        OMNITRACE_BASIC_VERBOSE(2, "HIP timestamp: %li\n", _gpu_ave);
+        OMNITRACE_BASIC_VERBOSE(1, "CPU/HIP timestamp skew: %li (used: %s)\n", _diff,
                                _use ? "yes" : "no");
        _diff /= _n;
        return _diff;
@@ -339,8 +339,11 @@ hip_exec_activity_callbacks(int64_t _tid)
    // ROCTRACER_CALL(roctracer_flush_activity());
    tim::auto_lock_t _lk{ get_hip_activity_mutex(_tid) };
    auto&            _async_ops = get_hip_activity_callbacks(_tid);
+    if(!_async_ops) return;
    for(auto& itr : *_async_ops)
-        itr();
+    {
+        if(itr) itr();
+    }
    _async_ops->clear();
 }

@@ -734,7 +737,7 @@ extern "C"
        if(!config::settings_are_configured() && get_state() < State::Active)
            omnitrace_init_tooling_hidden();

-        auto _setup = [=]() {
+        static auto _setup = [=]() {
            try
            {
                OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug() || get_verbose() > 1,
@@ -805,7 +808,7 @@ extern "C"
            }
        };

-        auto _shutdown = []() {
+        static auto _shutdown = []() {
            OMNITRACE_DEBUG_F("roctracer_disable_domain_callback\n");
            ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));

@@ -816,8 +819,8 @@ extern "C"

        (void) get_clock_skew();

-        comp::roctracer::add_setup("hsa", std::move(_setup));
-        comp::roctracer::add_shutdown("hsa", std::move(_shutdown));
+        comp::roctracer::add_setup("hsa", _setup);
+        comp::roctracer::add_shutdown("hsa", _shutdown);

        rocm_smi::set_state(State::Active);
        comp::roctracer::setup();
@@ -832,5 +835,6 @@ extern "C"
        OMNITRACE_DEBUG_F("\n");
        rocm_smi::set_state(State::Finalized);
        comp::roctracer::shutdown();
+        omnitrace_finalize_hidden();
    }
 }
@@ -80,6 +80,20 @@ get_setting_name(std::string _v)
            OMNITRACE_PRINT("Warning! Duplicate setting: %s / %s\n",                     \
                            get_setting_name(ENV_NAME).c_str(), ENV_NAME);               \
    }
+
+// setting + command line option
+#define OMNITRACE_CONFIG_CL_SETTING(TYPE, ENV_NAME, DESCRIPTION, INITIAL_VALUE,          \
+                                    CMD_LINE, ...)                                       \
+    {                                                                                    \
+        auto _ret = _config->insert<TYPE, TYPE>(                                         \
+            ENV_NAME, get_setting_name(ENV_NAME), DESCRIPTION, INITIAL_VALUE,            \
+            std::set<std::string>{ "custom", "omnitrace", "omnitrace-library",           \
+                                   __VA_ARGS__ },                                        \
+            std::vector<std::string>{ CMD_LINE });                                       \
+        if(!_ret.second)                                                                 \
+            OMNITRACE_PRINT("Warning! Duplicate setting: %s / %s\n",                     \
+                            get_setting_name(ENV_NAME).c_str(), ENV_NAME);               \
+    }
 }  // namespace

 inline namespace config
@@ -140,12 +154,12 @@ configure_settings(bool _init)
                             !_config->get<bool>("OMNITRACE_USE_PERFETTO"), "backend",
                             "timemory", "instrumentation", "sampling");

-#if defined(OMNITRACE_USE_ROCTRACER)
+#if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_ROCTRACER", "Enable ROCM tracing", true,
                             "backend", "roctracer", "rocm");
 #endif

-#if defined(OMNITRACE_USE_ROCM_SMI)
+#if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
    OMNITRACE_CONFIG_SETTING(
        bool, "OMNITRACE_USE_ROCM_SMI",
        "Enable sampling GPU power, temp, utilization, and memory usage", true, "backend",
@@ -169,12 +183,20 @@ configure_settings(bool _init)
                             "Enable support for Kokkos Tools", false, "kokkos",
                             "backend");

-#if defined(TIMEMORY_USE_OMPT)
+    OMNITRACE_CONFIG_CL_SETTING(
+        bool, "OMNITRACE_KOKKOS_KERNEL_LOGGER", "Enables kernel logging", false,
+        "--omnitrace-kokkos-kernel-logger", "kokkos", "debugging");
+
+#if defined(OMNITRACE_USE_OMPT) && OMNITRACE_USE_OMPT > 0
    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_OMPT",
-                             "Enable support for OpenMP-Tools", true, "openmp", "ompt",
+                             "Enable support for OpenMP-Tools", false, "openmp", "ompt",
                             "backend");
 #endif

+    OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_CODE_COVERAGE",
+                             "Enable support for code coverage", false, "coverage",
+                             "backend");
+
    OMNITRACE_CONFIG_SETTING(size_t, "OMNITRACE_INSTRUMENTATION_INTERVAL",
                             "Instrumentation only takes measurements once every N "
                             "function calls (not statistical)",
@@ -651,9 +673,12 @@ Mode
 get_mode()
 {
    static auto _v = []() {
-        auto _mode = tim::get_env_choice<std::string>("OMNITRACE_MODE", "trace",
-                                                      { "trace", "sampling" });
-        if(_mode == "sampling") return Mode::Sampling;
+        auto _mode = tim::get_env_choice<std::string>(
+            "OMNITRACE_MODE", "trace", { "trace", "sampling", "coverage" });
+        if(_mode == "sampling")
+            return Mode::Sampling;
+        else if(_mode == "coverage")
+            return Mode::Coverage;
        return Mode::Trace;
    }();
    return _v;
@@ -741,7 +766,7 @@ get_use_timemory()
 bool&
 get_use_roctracer()
 {
-#if defined(OMNITRACE_USE_ROCTRACER)
+#if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
    static auto _v = get_config()->find("OMNITRACE_USE_ROCTRACER");
    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
 #else
@@ -753,7 +778,7 @@ get_use_roctracer()
 bool&
 get_use_rocm_smi()
 {
-#if defined(OMNITRACE_USE_ROCM_SMI)
+#if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
    static auto _v = get_config()->find("OMNITRACE_USE_ROCM_SMI");
    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
 #else
@@ -804,6 +829,13 @@ get_use_kokkosp()
    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
 }

+bool
+get_use_kokkosp_kernel_logger()
+{
+    static auto _v = get_config()->find("OMNITRACE_KOKKOS_KERNEL_LOGGER");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
 bool
 get_use_ompt()
 {
@@ -815,6 +847,13 @@ get_use_ompt()
 #endif
 }

+bool
+get_use_code_coverage()
+{
+    static auto _v = get_config()->find("OMNITRACE_USE_CODE_COVERAGE");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+}
+
 bool
 get_critical_trace_debug()
 {
@@ -1001,7 +1040,7 @@ get_thread_sampling_freq()
 std::string
 get_rocm_smi_devices()
 {
-#if defined(OMNITRACE_USE_ROCM_SMI)
+#if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
    static auto _v = get_config()->find("OMNITRACE_ROCM_SMI_DEVICES");
    return static_cast<tim::tsettings<std::string>&>(*_v->second).get();
 #else
@@ -0,0 +1,424 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "library/coverage.hpp"
+#include "library/api.hpp"
+#include "library/config.hpp"
+#include "library/debug.hpp"
+#include "library/thread_data.hpp"
+
+#include <timemory/backends/threading.hpp>
+#include <timemory/tpls/cereal/cereal.hpp>
+#include <timemory/utility/popen.hpp>
+
+#include <algorithm>
+#include <map>
+#include <mutex>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+
+#define OMNITRACE_SERIALIZE(MEMBER_VARIABLE)                                             \
+    ar(::tim::cereal::make_nvp(#MEMBER_VARIABLE, MEMBER_VARIABLE))
+
+namespace omnitrace
+{
+namespace coverage
+{
+namespace
+{
+template <typename... Tp>
+using uomap_t = std::unordered_map<Tp...>;
+//
+using coverage_thread_data_type =
+    uomap_t<std::string_view, uomap_t<std::string_view, std::map<size_t, size_t>>>;
+//
+template <typename Tp, typename... Args>
+inline std::set<Tp, Args...>
+get_uncovered(const std::set<Tp, Args...>& _covered,
+              const std::set<Tp, Args...>& _possible)
+{
+    std::set<Tp, Args...> _v{};
+    for(auto&& itr : _possible)
+    {
+        if(_covered.count(itr) == 0) _v.emplace(itr);
+    }
+    return _v;
+}
+//
+template <typename Tp, typename... Args>
+inline std::vector<Tp, Args...>
+get_uncovered(const std::vector<Tp, Args...>& _covered,
+              const std::vector<Tp, Args...>& _possible)
+{
+    std::vector<Tp, Args...> _v{};
+    for(auto&& itr : _possible)
+    {
+        if(!std::any_of(_covered.begin(), _covered.end(),
+                        [itr](auto&& _entry) { return _entry == itr; }))
+            _v.emplace_back(itr);
+    }
+    return _v;
+}
+//
+using coverage_thread_data =
+    omnitrace::thread_data<coverage_thread_data_type, code_coverage>;
+//
+auto&
+get_code_coverage()
+{
+    static auto _v = code_coverage{};
+    return _v;
+}
+//
+auto&
+get_coverage_data()
+{
+    static auto _v = std::vector<coverage_data>{};
+    return _v;
+}
+//
+auto&
+get_coverage_count(int64_t _tid = tim::threading::get_id())
+{
+    static auto& _v =
+        coverage_thread_data::instances(coverage_thread_data::construct_on_init{});
+    return _v.at(_tid);
+}
+}  // namespace
+
+//--------------------------------------------------------------------------------------//
+
+double
+code_coverage::operator()(Category _c) const
+{
+    switch(_c)
+    {
+        case STANDARD: return static_cast<double>(count) / static_cast<double>(size);
+        case ADDRESS:
+            return static_cast<double>(covered.addresses.size()) /
+                   static_cast<double>(possible.addresses.size());
+        case MODULE:
+            return static_cast<double>(covered.modules.size()) /
+                   static_cast<double>(possible.modules.size());
+        case FUNCTION:
+            return static_cast<double>(covered.functions.size()) /
+                   static_cast<double>(possible.functions.size());
+    }
+    return 0.0;
+}
+
+code_coverage::int_set_t
+code_coverage::get_uncovered_addresses() const
+{
+    return get_uncovered(covered.addresses, possible.addresses);
+}
+
+code_coverage::str_set_t
+code_coverage::get_uncovered_modules() const
+{
+    return get_uncovered(covered.modules, possible.modules);
+}
+
+code_coverage::str_set_t
+code_coverage::get_uncovered_functions() const
+{
+    return get_uncovered(covered.functions, possible.functions);
+}
+
+//--------------------------------------------------------------------------------------//
+
+coverage_data&
+coverage_data::operator+=(const coverage_data& rhs)
+{
+    count += rhs.count;
+    return *this;
+}
+
+bool
+coverage_data::operator==(const coverage_data& rhs) const
+{
+    return std::tie(module, function, address) ==
+           std::tie(rhs.module, rhs.function, rhs.address);
+}
+
+bool
+coverage_data::operator==(const data_tuple_t& rhs) const
+{
+    return std::tie(module, function, address) ==
+           std::tie(std::get<0>(rhs), std::get<1>(rhs), std::get<2>(rhs));
+}
+
+bool
+coverage_data::operator!=(const coverage_data& rhs) const
+{
+    return !(*this == rhs);
+}
+
+bool
+coverage_data::operator<(const coverage_data& rhs) const
+{
+    if(count != rhs.count) return count < rhs.count;
+    if(module != rhs.module) return module < rhs.module;
+    if(function != rhs.function) return function < rhs.function;
+    if(address != rhs.address) return address < rhs.address;
+    if(line != rhs.line) return line < rhs.line;
+    return source < rhs.source;
+}
+
+bool
+coverage_data::operator<=(const coverage_data& rhs) const
+{
+    return (*this == rhs || *this < rhs);
+}
+
+bool
+coverage_data::operator>(const coverage_data& rhs) const
+{
+    return (*this != rhs && !(*this < rhs));
+}
+
+bool
+coverage_data::operator>=(const coverage_data& rhs) const
+{
+    return !(*this < rhs);
+}
+
+//--------------------------------------------------------------------------------------//
+
+void
+post_process()
+{
+    if(!config::get_use_code_coverage()) return;
+
+    code_coverage& _coverage = get_code_coverage();
+    if(_coverage.size == 0)
+    {
+        OMNITRACE_VERBOSE_F(
+            0,
+            "Warning! Code coverage enabled but no code coverage data is available!\n");
+        return;
+    }
+
+    using data_tuple_t   = coverage_data::data_tuple_t;
+    auto& _coverage_data = get_coverage_data();
+
+    auto _find = [&_coverage_data](data_tuple_t&& _v) {
+        for(auto itr = _coverage_data.begin(); itr != _coverage_data.end(); ++itr)
+        {
+            if(*itr == _v) return std::make_pair(itr, true);
+        }
+        return std::make_pair(_coverage_data.end(), false);
+    };
+
+    auto _data = coverage_thread_data_type{};
+    for(size_t i = 0; i < coverage_thread_data::size(); ++i)
+    {
+        const auto& _thr_data = *get_coverage_count(i);
+        for(const auto& file : _thr_data)
+        {
+            for(const auto& func : file.second)
+            {
+                for(const auto& addr : func.second)
+                {
+                    _data[file.first][func.first][addr.first] += addr.second;
+                    auto&& _v = _find({ file.first, func.first, addr.first });
+                    if(_v.second)
+                    {
+                        _v.first->count += addr.second;
+                    }
+                    else
+                    {
+                        OMNITRACE_VERBOSE_F(
+                            0, "Warning! No matching coverage data for %s :: %s (0x%x)\n",
+                            func.first.data(), file.first.data(),
+                            (unsigned int) addr.first);
+                    }
+                }
+            }
+        }
+    }
+
+    for(const auto& file : _data)
+    {
+        for(const auto& func : file.second)
+        {
+            for(const auto& addr : func.second)
+            {
+                if(addr.second > 0)
+                {
+                    _coverage.count += 1;
+                    _coverage.covered.modules.emplace(file.first);
+                    _coverage.covered.functions.emplace(func.first);
+                    _coverage.covered.addresses.emplace(addr.first);
+                }
+            }
+        }
+    }
+
+    std::sort(_coverage_data.begin(), _coverage_data.end(),
+              std::greater<coverage_data>{});
+
+    {
+        auto _tmp         = std::decay_t<decltype(_coverage_data)>{};
+        auto _find_in_tmp = [&_tmp](const auto& _v) {
+            for(auto itr = _tmp.begin(); itr != _tmp.end(); ++itr)
+            {
+                if(itr->source == _v.source && itr->address != _v.address &&
+                   itr->count == _v.count)
+                    return std::make_pair(itr, true);
+            }
+            return std::make_pair(_tmp.end(), false);
+        };
+        for(auto&& itr : _coverage_data)
+        {
+            if(!_find_in_tmp(itr).second) _tmp.emplace_back(itr);
+        }
+        std::swap(_coverage_data, _tmp);
+    }
+
+    OMNITRACE_VERBOSE(0, "code coverage     :: %6.2f%s\n", _coverage() * 100.0, "%");
+    OMNITRACE_VERBOSE(0, "module coverage   :: %6.2f%s\n",
+                      _coverage(code_coverage::MODULE) * 100.0, "%");
+    OMNITRACE_VERBOSE(0, "function coverage :: %6.2f%s\n",
+                      _coverage(code_coverage::FUNCTION) * 100.0, "%");
+
+    if(get_verbose() >= 0) fprintf(stderr, "\n");
+
+    std::sort(_coverage_data.begin(), _coverage_data.end(),
+              std::greater<coverage_data>{});
+
+    auto _get_setting = [](const std::string& _v) {
+        auto&& _b = config::get_setting_value<bool>(_v);
+        OMNITRACE_CI_THROW(!_b.first, "Error! No configuration setting named '%s'",
+                           _v.c_str());
+        return (_b.first) ? _b.second : true;
+    };
+
+    auto _text_output = _get_setting("OMNITRACE_TEXT_OUTPUT");
+    auto _json_output = _get_setting("OMNITRACE_JSON_OUTPUT");
+
+    if(_text_output)
+    {
+        auto          _fname = tim::settings::compose_output_filename("coverage", ".txt");
+        std::ofstream ofs{};
+        if(tim::filepath::open(ofs, _fname))
+        {
+            if(get_verbose() >= 0)
+                fprintf(stderr, "[%s][coverage]|%i> Outputting '%s'...\n",
+                        TIMEMORY_PROJECT_NAME, dmp::rank(), _fname.c_str());
+            for(auto& itr : _coverage_data)
+            {
+                // if(get_debug() && get_verbose() >= 2)
+                if(true)
+                {
+                    auto _addr = TIMEMORY_JOIN("", "0x", std::hex, itr.address);
+                    ofs << std::setw(8) << itr.count << "  " << std::setw(8) << _addr
+                        << "  " << itr.source << "\n";
+                }
+                else
+                {
+                    ofs << std::setw(8) << itr.count << "  " << itr.source << "\n";
+                }
+            }
+        }
+        else
+        {
+            OMNITRACE_THROW("Error opening coverage output file: %s", _fname.c_str());
+        }
+    }
+
+    if(_json_output)
+    {
+        auto _fname = tim::settings::compose_output_filename("coverage", ".json");
+        std::ofstream ofs{};
+        if(tim::filepath::open(ofs, _fname))
+        {
+            if(get_verbose() >= 0)
+                fprintf(stderr, "[%s][coverage]|%i> Outputting '%s'...\n",
+                        TIMEMORY_PROJECT_NAME, dmp::rank(), _fname.c_str());
+            {
+                namespace cereal = tim::cereal;
+                auto ar =
+                    tim::policy::output_archive<cereal::PrettyJSONOutputArchive>::get(
+                        ofs);
+
+                ar->setNextName("omnitrace");
+                ar->startNode();
+                ar->setNextName("coverage");
+                ar->startNode();
+                (*ar)(cereal::make_nvp("summary", _coverage));
+                (*ar)(cereal::make_nvp("details", _coverage_data));
+                ar->finishNode();
+                ar->finishNode();
+            }
+        }
+        else
+        {
+            OMNITRACE_THROW("Error opening coverage output file: %s", _fname.c_str());
+        }
+    }
+
+    if(get_verbose() >= 0) fprintf(stderr, "\n");
+}
+}  // namespace coverage
+}  // namespace omnitrace
+
+//--------------------------------------------------------------------------------------//
+
+namespace coverage = omnitrace::coverage;
+
+extern "C" void
+omnitrace_register_source_hidden(const char* file, const char* func, size_t line,
+                                 size_t address, const char* source)
+{
+    using coverage_data = coverage::coverage_data;
+
+    OMNITRACE_BASIC_VERBOSE_F(2, "[0x%x] :: %-20s :: %20s:%zu :: %s\n",
+                              (unsigned int) address, func, file, line, source);
+
+    coverage::get_coverage_data().emplace_back(
+        coverage_data{ size_t{ 0 }, address, line, file, func,
+                       (source && strlen(source) > 0) ? source : func });
+
+    coverage::get_code_coverage().size += 1;
+    coverage::get_code_coverage().possible.modules.emplace(file);
+    coverage::get_code_coverage().possible.functions.emplace(func);
+    coverage::get_code_coverage().possible.addresses.emplace(address);
+
+    // initialize
+    for(size_t i = 0; i < coverage::coverage_thread_data::size(); ++i)
+        (*coverage::get_coverage_count(i))[file][func][address] = 0;
+}
+
+//--------------------------------------------------------------------------------------//
+
+extern "C" void
+omnitrace_register_coverage_hidden(const char* file, const char* func, size_t address)
+{
+    OMNITRACE_BASIC_VERBOSE_F(3, "[0x%x] %-20s :: %20s\n", (unsigned int) address, func,
+                              file);
+    coverage::get_coverage_count()->at(file).at(func).at(address) += 1;
+}
+
+//--------------------------------------------------------------------------------------//
@@ -26,6 +26,7 @@
 #include "library/defines.hpp"
 #include "library/perfetto.hpp"
 #include "library/ptl.hpp"
+#include "library/thread_data.hpp"

 #include <PTL/ThreadPool.hh>
 #include <timemory/backends/dmp.hpp>
@@ -491,13 +492,13 @@ get_update_frequency()
    return get_critical_trace_update_freq();
 }

-std::unique_ptr<call_chain>&
+unique_ptr_t<call_chain>&
 get(int64_t _tid)
 {
    static auto&             _v    = thread_data<call_chain>::instances();
    static thread_local auto _once = [_tid]() {
-        if(!_v.at(0)) _v.at(0) = std::make_unique<call_chain>();
-        if(!_v.at(_tid)) _v.at(_tid) = std::make_unique<call_chain>();
+        if(!_v.at(0)) _v.at(0) = unique_ptr_t<call_chain>{ new call_chain{} };
+        if(!_v.at(_tid)) _v.at(_tid) = unique_ptr_t<call_chain>{ new call_chain{} };
        if(_tid > 0) *_v.at(_tid) = *_v.at(0);
        return true;
    }();
@@ -22,9 +22,9 @@

 #include "library/gpu.hpp"

-#if defined(OMNITRACE_USE_ROCM_SMI)
+#if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
 #    include "library/components/rocm_smi.hpp"
-#elif defined(OMNITRACE_USE_HIP)
+#elif defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0
 #    if !defined(TIMEMORY_USE_HIP)
 #        define TIMEMORY_USE_HIP 1
 #    endif
@@ -38,11 +38,11 @@ namespace gpu
 int
 device_count()
 {
-#if defined(OMNITRACE_USE_ROCM_SMI)
+#if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
    // store as static since calls after rsmi_shutdown will return zero
    static auto _v = rocm_smi::device_count();
    return _v;
-#elif defined(OMNITRACE_USE_HIP)
+#elif defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0
    return ::tim::hip::device_count();
 #else
    return 0;
@@ -26,6 +26,7 @@

 #include "library/components/omnitrace.hpp"
 #include "library/config.hpp"
+#include "library/debug.hpp"

 #include <timemory/api/kokkosp.hpp>

@@ -53,26 +54,11 @@ std::string kokkos_banner =

 //--------------------------------------------------------------------------------------//

-bool enable_kernel_logger = false;
-
-inline void
-add_kernel_logger()
-{
-    static bool _first = true;
-    if(!_first) return;
-    _first         = false;
-    using strvec_t = std::vector<std::string>;
-
-    tim::settings::instance()->insert<bool, bool&>(
-        std::string{ "OMNITRACE_KOKKOS_KERNEL_LOGGER" }, std::string{},
-        std::string{ "Enables kernel logging" }, enable_kernel_logger,
-        strvec_t({ "--omnitrace-kokkos-kernel-logger" }));
-}
-
 inline void
 setup_kernel_logger()
 {
-    if(tim::settings::debug() || tim::settings::verbose() > 3 || enable_kernel_logger)
+    if((tim::settings::debug() && tim::settings::verbose() >= 3) ||
+       omnitrace::config::get_use_kokkosp_kernel_logger())
    {
        kokkosp::logger_t::get_initializer() = [](kokkosp::logger_t& _obj) {
            _obj.initialize<kokkosp::kernel_logger>();
@@ -98,28 +84,27 @@ extern "C"
    void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
                              const uint32_t devInfoCount, void* deviceInfo)
    {
-        add_kernel_logger();
-
        tim::consume_parameters(devInfoCount, deviceInfo);

-        printf("%s\n", kokkos_banner.c_str());
-        printf("# KokkosP: omnitrace connector (sequence is %d, version: %llu)\n",
-               loadSeq, (unsigned long long) interfaceVer);
-        printf("%s\n", kokkos_banner.c_str());
+        OMNITRACE_VERBOSE_F(0,
+                            "Initializing connector (sequence is %d, version: %llu)...",
+                            loadSeq, (unsigned long long) interfaceVer);

        setup_kernel_logger();

        tim::trait::runtime_enabled<kokkosp::memory_tracker>::set(
            omnitrace::config::get_use_timemory());
+
+        if(omnitrace::get_verbose() >= 0) fprintf(stderr, "Done\n");
    }

    void kokkosp_finalize_library()
    {
-        printf("%s\n", kokkos_banner.c_str());
-        printf("# KokkosP: Finalization of omnitrace connector. Complete.\n");
-        printf("%s\n", kokkos_banner.c_str());
+        OMNITRACE_VERBOSE_F(0, "Finalizing connector... \n");

        kokkosp::cleanup();
+
+        if(omnitrace::get_verbose() >= 0) fprintf(stderr, "Done\n");
    }

    //----------------------------------------------------------------------------------//
@@ -57,7 +57,7 @@ get_cpu_cid()
    return _v;
 }

-std::unique_ptr<std::vector<uint64_t>>&
+unique_ptr_t<std::vector<uint64_t>>&
 get_cpu_cid_stack(int64_t _tid, int64_t _parent)
 {
    struct omnitrace_cpu_cid_stack
@@ -78,7 +78,7 @@ get_cpu_cid_stack(int64_t _tid, int64_t _parent)
    (void) _v_copy;
 }

-std::unique_ptr<cpu_cid_parent_map_t>&
+unique_ptr_t<cpu_cid_parent_map_t>&
 get_cpu_cid_parents(int64_t _tid)
 {
    struct omnitrace_cpu_cid_stack
@@ -120,7 +120,7 @@ get_signal_names(Tp&& _v)
 }
 }  // namespace

-std::unique_ptr<std::set<int>>&
+unique_ptr_t<std::set<int>>&
 get_signal_types(int64_t _tid)
 {
    static auto& _v = signal_type_instances::instances();
@@ -178,7 +178,7 @@ unblock_signals(std::set<int> _signals)
    thread_sigmask(SIG_UNBLOCK, &_v, nullptr);
 }

-std::unique_ptr<sampler_t>&
+unique_ptr_t<sampler_t>&
 get_sampler(int64_t _tid)
 {
    static auto& _v = sampler_instances::instances();
@@ -47,6 +47,7 @@ to_string(omnitrace::Mode _v)
    {
        case omnitrace::Mode::Trace: return "Trace";
        case omnitrace::Mode::Sampling: return "Sampling";
+        case omnitrace::Mode::Coverage: return "Coverage";
    }
    return {};
 }
@@ -21,6 +21,7 @@
 // SOFTWARE.

 #include "library/thread_sampler.hpp"
+#include "library/components/pthread_gotcha.hpp"
 #include "library/components/rocm_smi.hpp"
 #include "library/config.hpp"
 #include "library/cpu_freq.hpp"
@@ -21,10 +21,14 @@ function(OMNITRACE_CONFIGURE_PYTARGET _TARGET _VERSION)
        ${_TARGET}
        PROPERTIES PREFIX ""
                   OUTPUT_NAME libpyomnitrace
-                   LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace
-                   ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace
-                   RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace
-                   PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace
+                   LIBRARY_OUTPUT_DIRECTORY
+                   ${PROJECT_BINARY_DIR}/lib/python/site-packages/omnitrace
+                   ARCHIVE_OUTPUT_DIRECTORY
+                   ${PROJECT_BINARY_DIR}/lib/python/site-packages/omnitrace
+                   RUNTIME_OUTPUT_DIRECTORY
+                   ${PROJECT_BINARY_DIR}/lib/python/site-packages/omnitrace
+                   PDB_OUTPUT_DIRECTORY
+                   ${PROJECT_BINARY_DIR}/lib/python/site-packages/omnitrace
                   INSTALL_RPATH_USE_LINK_PATH ON
                   ${EXTRA_PROPERTIES})

@@ -84,65 +88,33 @@ target_link_libraries(

 target_compile_definitions(libpyomnitrace-interface INTERFACE OMNITRACE_PYBIND11_SOURCE)

-# ----------------------------------------------------------------------------
-# Console scripts
-#
-function(OMNITRACE_PYTHON_CONSOLE_SCRIPT SCRIPT_NAME SCRIPT_SUBMODULE)
-    set(options)
-    set(args VERSION ROOT_DIR)
-    set(kwargs)
-    cmake_parse_arguments(ARG "${options}" "${args}" "${kwargs}" ${ARGN})
-
-    if(ARG_VERSION AND ARG_ROOT_DIR)
-        set(Python3_ROOT_DIR "${ARG_ROOT_DIR}")
-        find_package(Python3 ${ARG_VERSION} EXACT QUIET MODULE COMPONENTS Interpreter)
-        set(PYTHON_EXECUTABLE "${Python3_EXECUTABLE}")
-        configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in
-                       ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION} @ONLY)
-
-        if(CMAKE_INSTALL_PYTHONDIR)
-            install(
-                PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION}
-                DESTINATION ${CMAKE_INSTALL_BINDIR}
-                OPTIONAL)
-        endif()
-
-        if(OMNITRACE_BUILD_TESTING OR OMNITRACE_BUILD_PYTHON)
-            add_test(
-                NAME ${SCRIPT_NAME}-console-script-test-${ARG_VERSION}
-                COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION} --help
-                WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-            set_tests_properties(
-                ${SCRIPT_NAME}-console-script-test-${ARG_VERSION}
-                PROPERTIES LABELS "python;python-${ARG_VERSION};console-script")
-            add_test(
-                NAME ${SCRIPT_NAME}-generic-console-script-test-${ARG_VERSION}
-                COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} --help
-                WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-            set_tests_properties(
-                ${SCRIPT_NAME}-generic-console-script-test-${ARG_VERSION}
-                PROPERTIES ENVIRONMENT "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" LABELS
-                           "python;python-${ARG_VERSION};console-script")
-        endif()
-    else()
-        set(PYTHON_EXECUTABLE "python3")
-
-        configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in
-                       ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} @ONLY)
-
-        if(CMAKE_INSTALL_PYTHONDIR)
-            install(
-                PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}
-                DESTINATION ${CMAKE_INSTALL_BINDIR}
-                OPTIONAL)
-        endif()
-    endif()
-endfunction()
-
 include(PyBind11Tools)

-# OMNITRACE_PYTHON_ROOT_DIRS=/opt/conda/envs/py36;/opt/conda/envs/py37;/opt/conda/envs/py38;/opt/conda/envs/py39
-# OMNITRACE_PYTHON_VERSIONS=3.6;3.7;3.8;3.9
+omnitrace_watch_for_change(OMNITRACE_PYTHON_ROOT_DIRS _PYTHON_DIRS_CHANGED)
+if(_PYTHON_DIRS_CHANGED)
+    unset(OMNITRACE_PYTHON_VERSION CACHE)
+    unset(OMNITRACE_PYTHON_VERSIONS CACHE)
+else()
+    foreach(_VAR PREFIX ENVS)
+        omnitrace_watch_for_change(OMNITRACE_PYTHON_${_VAR} _CHANGED)
+        if(_CHANGED)
+            unset(OMNITRACE_PYTHON_ROOT_DIRS CACHE)
+            unset(OMNITRACE_PYTHON_VERSIONS CACHE)
+            break()
+        endif()
+    endforeach()
+endif()
+
+if(OMNITRACE_PYTHON_PREFIX AND OMNITRACE_PYTHON_ENVS)
+    omnitrace_directory(
+        FAIL
+        PREFIX ${OMNITRACE_PYTHON_PREFIX}
+        PATHS ${OMNITRACE_PYTHON_ENVS}
+        OUTPUT_VARIABLE _PYTHON_ROOT_DIRS)
+    set(OMNITRACE_PYTHON_ROOT_DIRS
+        "${_PYTHON_ROOT_DIRS}"
+        CACHE INTERNAL "Root directories for python")
+endif()

 if(NOT OMNITRACE_PYTHON_VERSIONS AND OMNITRACE_PYTHON_VERSION)
    set(OMNITRACE_PYTHON_VERSIONS "${OMNITRACE_PYTHON_VERSION}")
@@ -152,6 +124,7 @@ if(NOT OMNITRACE_PYTHON_VERSIONS AND OMNITRACE_PYTHON_VERSION)
            "${_PY_ROOT_DIR}"
            CACHE INTERNAL "" FORCE)
    endif()
+    unset(OMNITRACE_PYTHON_VERSION CACHE)
 elseif(
    NOT OMNITRACE_PYTHON_VERSIONS
    AND NOT OMNITRACE_PYTHON_VERSION
@@ -182,27 +155,16 @@ elseif(
        CACHE INTERNAL "" FORCE)
 endif()

-list(LENGTH OMNITRACE_PYTHON_VERSIONS _NUM_PYTHON_VERSIONS)
-list(LENGTH OMNITRACE_PYTHON_ROOT_DIRS _NUM_PYTHON_ROOT_DIRS)
+omnitrace_watch_for_change(OMNITRACE_PYTHON_ROOT_DIRS)
+omnitrace_watch_for_change(OMNITRACE_PYTHON_VERSIONS)

-if(NOT _NUM_PYTHON_VERSIONS EQUAL _NUM_PYTHON_ROOT_DIRS)
-    omnitrace_message(
-        WARNING
-        "Error! Number of python versions  : ${_NUM_PYTHON_VERSIONS}. VERSIONS :: ${OMNITRACE_PYTHON_VERSIONS}"
-        )
-    omnitrace_message(
-        WARNING
-        "Error! Number of python root directories : ${_NUM_PYTHON_ROOT_DIRS}. ROOT DIRS :: ${OMNITRACE_PYTHON_ROOT_DIRS}"
-        )
-    omnitrace_message(
-        FATAL_ERROR
-        "Error! Number of python versions != number of python root directories")
-endif()
+omnitrace_check_python_dirs_and_versions(FAIL)

 file(GLOB_RECURSE PYTHON_FILES ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/*.py)
 foreach(_IN ${PYTHON_FILES})
    string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/omnitrace"
-                   "${PROJECT_BINARY_DIR}/python/omnitrace" _OUT "${_IN}")
+                   "${PROJECT_BINARY_DIR}/lib/python/site-packages/omnitrace" _OUT
+                   "${_IN}")
    configure_file(${_IN} ${_OUT} @ONLY)
    install(
        FILES ${_OUT}
@@ -212,12 +174,6 @@ endforeach()

 omnitrace_python_console_script("omnitrace-python" "omnitrace")

-execute_process(
-    COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_LIBDIR}/python
-    COMMAND ${CMAKE_COMMAND} -E create_symlink ../../python
-            ${CMAKE_INSTALL_LIBDIR}/python/site-packages
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-
 set(_INDEX 0)
 foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
    # add_library(libpyomnitrace MODULE ${pysources} ${pyheaders})
@@ -237,29 +193,9 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
    math(EXPR _INDEX "${_INDEX} + 1")
 endforeach()

-if(PYTHON_EXECUTABLE AND OFF)
-    configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.py.in
-                   ${PROJECT_BINARY_DIR}/python/setup.py @ONLY)
-    configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.cfg.in
-                   ${PROJECT_BINARY_DIR}/python/setup.cfg @ONLY)
-    configure_file(${CMAKE_CURRENT_LIST_DIR}/pyproject.toml
-                   ${PROJECT_BINARY_DIR}/python/pyproject.toml COPYONLY)
-    execute_process(
-        COMMAND ${PYTHON_EXECUTABLE} setup.py dist_info
-        OUTPUT_VARIABLE _OUT
-        RESULT_VARIABLE _RET
-        ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE
-        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python)
-    set(_DIST_DIR
-        ${PROJECT_BINARY_DIR}/python/${PROJECT_NAME}-${PROJECT_VERSION}.dist-info)
-    if(NOT EXISTS ${_DIST_DIR})
-        set(_DIST_DIR ${PROJECT_BINARY_DIR}/${PROJECT_NAME}.dist-info)
-    endif()
-    if(EXISTS ${_DIST_DIR} AND IS_DIRECTORY ${_DIST_DIR})
-        configure_file(${PROJECT_SOURCE_DIR}/LICENSE ${_DIST_DIR}/LICENSE.txt COPYONLY)
-        install(
-            DIRECTORY ${_DIST_DIR}
-            DESTINATION ${CMAKE_INSTALL_PYTHONDIR}
-            OPTIONAL)
-    endif()
-endif()
+configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.py.in
+               ${PROJECT_BINARY_DIR}/lib/python/site-packages/setup.py @ONLY)
+configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.cfg.in
+               ${PROJECT_BINARY_DIR}/lib/python/site-packages/setup.cfg @ONLY)
+configure_file(${CMAKE_CURRENT_LIST_DIR}/pyproject.toml
+               ${PROJECT_BINARY_DIR}/lib/python/site-packages/pyproject.toml COPYONLY)
@@ -26,7 +26,6 @@ classifiers =
    Operating System :: Unix
    Programming Language :: C++
    Programming Language :: Python :: 3
-    Programming Language :: Python :: @OMNITRACE_PYTHON_VERSION@
    Topic :: Software Development :: Libraries :: Python Modules
    Topic :: Utilities

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 from setuptools import setup

 setup(
@@ -7,5 +9,4 @@ setup(
    author="AMD Research",
    url="@PROJECT_HOMEPAGE_URL@",
    packages=["@PROJECT_NAME@"],
-    python_requires="=@OMNITRACE_PYTHON_VERSION@",
 )
@@ -37,7 +37,6 @@ set(_base_environment
 set(_ompt_environment
    "OMNITRACE_USE_PERFETTO=ON"
    "OMNITRACE_USE_TIMEMORY=ON"
-    "OMNITRACE_USE_SAMPLING=ON"
    "OMNITRACE_TIME_OUTPUT=OFF"
    "OMNITRACE_USE_OMPT=ON"
    "OMNITRACE_CRITICAL_TRACE=OFF"
@@ -80,23 +79,33 @@ set(_python_environment
    "OMNITRACE_USE_PID=OFF"
    "OMNITRACE_TIMEMORY_COMPONENTS=trip_count"
    "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
-    "PYTHONPATH=${CMAKE_BINARY_DIR}/python")
+    "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python/site-packages")

 # -------------------------------------------------------------------------------------- #

 function(OMNITRACE_ADD_TEST)
+    foreach(_PREFIX RUNTIME REWRITE REWRITE_RUN)
+        foreach(_TYPE PASS FAIL SKIP)
+            list(APPEND _REGEX_OPTS "${_PREFIX}_${_TYPE}_REGEX")
+        endforeach()
+    endforeach()
+    set(_KWARGS REWRITE_ARGS RUNTIME_ARGS RUN_ARGS ENVIRONMENT LABELS PROPERTIES
+                ${_REGEX_OPTS})
+
    cmake_parse_arguments(
        TEST
-        "SKIP_REWRITE;SKIP_RUNTIME;SKIP_SAMPLING" # options
+        "SKIP_BASELINE;SKIP_REWRITE;SKIP_RUNTIME;SKIP_SAMPLING" # options
        "NAME;TARGET;MPI;NUM_PROCS;REWRITE_TIMEOUT;RUNTIME_TIMEOUT" # single value args
-        "REWRITE_ARGS;RUNTIME_ARGS;RUN_ARGS;ENVIRONMENT;LABELS;PROPERTIES" # multiple
-                                                                           # value args
+        "${_KWARGS}" # multiple value args
        ${ARGN})

    if("${TEST_MPI}" STREQUAL "")
        set(TEST_MPI OFF)
    endif()

+    list(INSERT TEST_REWRITE_ARGS 0 --print-instrumented functions)
+    list(INSERT TEST_RUNTIME_ARGS 0 --print-instrumented functions)
+
    if(NOT DEFINED TEST_NUM_PROCS)
        set(TEST_NUM_PROCS ${NUM_PROCS})
    endif()
@@ -133,10 +142,12 @@ function(OMNITRACE_ADD_TEST)
            list(APPEND TEST_ENVIRONMENT "OMNITRACE_USE_PID=OFF")
        endif()

-        add_test(
-            NAME ${TEST_NAME}-baseline
-            COMMAND ${COMMAND_PREFIX} $<TARGET_FILE:${TEST_TARGET}> ${TEST_RUN_ARGS}
-            WORKING_DIRECTORY $<TARGET_FILE_DIR:${TEST_TARGET}>)
+        if(NOT SKIP_BASELINE)
+            add_test(
+                NAME ${TEST_NAME}-baseline
+                COMMAND ${COMMAND_PREFIX} $<TARGET_FILE:${TEST_TARGET}> ${TEST_RUN_ARGS}
+                WORKING_DIRECTORY $<TARGET_FILE_DIR:${TEST_TARGET}>)
+        endif()

        if(NOT TEST_SKIP_REWRITE)
            add_test(
@@ -213,6 +224,9 @@ function(OMNITRACE_ADD_TEST)
                 "OMNITRACE_OUTPUT_PREFIX=${_prefix}")
            string(REPLACE "-run" "" _labels "${_TEST}")
            string(REPLACE "-sampling" ";sampling" _labels "${_labels}")
+            if(TEST_TARGET)
+                list(APPEND _labels "${TEST_TARGET}")
+            endif()
            if("${_TEST}" MATCHES "runtime-instrument")
                set(_timeout ${TEST_RUNTIME_TIMEOUT})
            endif()
@@ -223,11 +237,41 @@ function(OMNITRACE_ADD_TEST)
                    list(APPEND _props RUN_SERIAL ON)
                endif()
            endif()
+
+            if("${_TEST}" MATCHES "binary-rewrite-run")
+                set(_REGEX_VAR REWRITE_RUN)
+            elseif("${_TEST}" MATCHES "runtime-instrument")
+                set(_REGEX_VAR RUNTIME)
+            elseif("${_TEST}" MATCHES "binary-rewrite")
+                set(_REGEX_VAR REWRITE)
+            else()
+                set(_REGEX_VAR)
+            endif()
+
+            foreach(_TYPE PASS FAIL SKIP)
+                if(_REGEX_VAR)
+                    set(_${_TYPE}_REGEX TEST_${_REGEX_VAR}_${_TYPE}_REGEX)
+                else()
+                    set(_${_TYPE}_REGEX)
+                endif()
+            endforeach()
+
            if(TEST ${TEST_NAME}-${_TEST})
                set_tests_properties(
                    ${TEST_NAME}-${_TEST}
-                    PROPERTIES ENVIRONMENT "${_environ}" TIMEOUT ${_timeout} LABELS
-                               "${_labels};${TEST_LABELS}" ${_props})
+                    PROPERTIES ENVIRONMENT
+                               "${_environ}"
+                               TIMEOUT
+                               ${_timeout}
+                               LABELS
+                               "${_labels};${TEST_LABELS}"
+                               PASS_REGULAR_EXPRESSION
+                               "${${_PASS_REGEX}}"
+                               FAIL_REGULAR_EXPRESSION
+                               "${${_FAIL_REGEX}}"
+                               SKIP_REGULAR_EXPRESSION
+                               "${${_SKIP_REGEX}}"
+                               ${_props})
            endif()
        endforeach()
    endif()
@@ -244,8 +288,8 @@ function(OMNITRACE_ADD_PYTHON_TEST)
        TEST
        "STANDALONE" # options
        "NAME;FILE;TIMEOUT;PYTHON_EXECUTABLE;PYTHON_VERSION" # single value args
-        "PROFILE_ARGS;RUN_ARGS;ENVIRONMENT;LABELS;PROPERTIES;PASS_REGULAR_EXPRESSION;FAIL_REGULAR_EXPRESSION;SKIP_REGULAR_EXPRESSION;DEPENDS;COMMAND" # multiple
-                                                                                                                                                      # value args
+        "PROFILE_ARGS;RUN_ARGS;ENVIRONMENT;LABELS;PROPERTIES;PASS_REGEX;FAIL_REGEX;SKIP_REGEX;DEPENDS;COMMAND" # multiple
+        # value args
        ${ARGN})

    if(NOT TEST_TIMEOUT)
@@ -302,14 +346,14 @@ function(OMNITRACE_ADD_PYTHON_TEST)
        set(_TEST_PROPERTIES "${TEST_PROPERTIES}")
        if(NOT "${_TEST}" MATCHES "inverse")
            # assign pass variable to pass regex
-            set(_PASS_REGEX TEST_PASS_REGULAR_EXPRESSION)
+            set(_PASS_REGEX TEST_PASS_REGEX)
            # assign fail variable to fail regex
-            set(_FAIL_REGEX TEST_FAIL_REGULAR_EXPRESSION)
+            set(_FAIL_REGEX TEST_FAIL_REGEX)
        else()
            # assign pass variable to fail regex
-            set(_PASS_REGEX TEST_FAIL_REGULAR_EXPRESSION)
+            set(_PASS_REGEX TEST_FAIL_REGEX)
            # assign fail variable to pass regex
-            set(_FAIL_REGEX TEST_PASS_REGULAR_EXPRESSION)
+            set(_FAIL_REGEX TEST_PASS_REGEX)
            # set to will fail
            list(APPEND _TEST_PROPERTIES WILL_FAIL ON)
        endif()
@@ -329,7 +373,7 @@ function(OMNITRACE_ADD_PYTHON_TEST)
                       FAIL_REGULAR_EXPRESSION
                       "${${_FAIL_REGEX}}"
                       SKIP_REGULAR_EXPRESSION
-                       "${TEST_SKIP_REGULAR_EXPRESSION}"
+                       "${TEST_SKIP_REGEX}"
                       REQUIRED_FILES
                       "${TEST_FILE}"
                       ${_TEST_PROPERTIES})
@@ -350,12 +394,12 @@ omnitrace_add_test(
 omnitrace_add_test(
    NAME parallel-overhead
    TARGET parallel-overhead
-    REWRITE_ARGS -e -v 2 --min-address-range-loop=64
+    REWRITE_ARGS -e -v 2 --min-instructions=8
    RUNTIME_ARGS
        -e
        -v
        1
-        --min-address-range-loop=64
+        --min-instructions=8
        --label
        file
        line
@@ -367,23 +411,12 @@ omnitrace_add_test(
 omnitrace_add_test(
    NAME user-api
    TARGET user-api
-    REWRITE_ARGS
-        -e
-        -v
-        2
-        -l
-        --min-address-range=0
-        --min-address-range-loop=0
-        --min-instructions=8
-        -E
-        custom_push_region
+    REWRITE_ARGS -e -v 2 -l --min-instructions=8 -E custom_push_region
    RUNTIME_ARGS
        -e
        -v
        1
        -l
-        --min-address-range=0
-        --min-address-range-loop=0
        --min-instructions=8
        -E
        custom_push_region
@@ -400,6 +433,7 @@ omnitrace_add_test(
    TARGET lulesh
    MPI ${LULESH_USE_MPI}
    NUM_PROCS 8
+    LABELS "kokkos"
    REWRITE_ARGS -e -v 2 --label file line return args
    RUNTIME_ARGS
        -e
@@ -417,10 +451,12 @@ omnitrace_add_test(
        "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=OFF")

 omnitrace_add_test(
+    SKIP_BASELINE
    NAME lulesh-kokkosp
    TARGET lulesh
    MPI ${LULESH_USE_MPI}
    NUM_PROCS 8
+    LABELS "kokkos"
    REWRITE_ARGS -e -v 2
    RUNTIME_ARGS
        -e
@@ -438,10 +474,12 @@ omnitrace_add_test(
        "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON")

 omnitrace_add_test(
+    SKIP_BASELINE SKIP_SAMPLING
    NAME lulesh-perfetto
    TARGET lulesh
    MPI ${LULESH_USE_MPI}
    NUM_PROCS 8
+    LABELS "kokkos"
    REWRITE_ARGS -e -v 2
    RUNTIME_ARGS
        -e
@@ -458,10 +496,12 @@ omnitrace_add_test(
        "${_perfetto_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=OFF")

 omnitrace_add_test(
+    SKIP_SAMPLING
    NAME lulesh-timemory
    TARGET lulesh
    MPI ${LULESH_USE_MPI}
    NUM_PROCS 8
+    LABELS "kokkos"
    REWRITE_ARGS -e -v 2 -l --dynamic-callsites --traps --allow-overlapping
    RUNTIME_ARGS
        -e
@@ -479,22 +519,159 @@ omnitrace_add_test(
        "${_timemory_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=OFF")

 omnitrace_add_test(
+    SKIP_SAMPLING
    NAME openmp-cg
    TARGET openmp-cg
+    LABELS "openmp"
    REWRITE_ARGS -e -v 2 --instrument-loops
    RUNTIME_ARGS -e -v 1 --label return args
    REWRITE_TIMEOUT 180
    RUNTIME_TIMEOUT 360
-    ENVIRONMENT "${_ompt_environment}")
+    ENVIRONMENT "${_ompt_environment};OMNITRACE_USE_SAMPLING=OFF")

 omnitrace_add_test(
+    SKIP_RUNTIME
    NAME openmp-lu
    TARGET openmp-lu
+    LABELS "openmp"
    REWRITE_ARGS -e -v 2 --instrument-loops
-    RUNTIME_ARGS -e -v 1 --label return args
+    RUNTIME_ARGS -e -v 1 --label return args -E ^GOMP
    REWRITE_TIMEOUT 180
    RUNTIME_TIMEOUT 360
-    ENVIRONMENT "${_ompt_environment}")
+    ENVIRONMENT
+        "${_ompt_environment};OMNITRACE_USE_SAMPLING=ON;OMNITRACE_SAMPLING_FREQ=100")
+
+omnitrace_add_test(
+    SKIP_BASELINE SKIP_SAMPLING
+    NAME code-coverage
+    TARGET code-coverage
+    REWRITE_ARGS
+        -e
+        -v
+        2
+        --min-instructions=4
+        -E
+        ^std::
+        -M
+        coverage
+        --coverage
+        function
+    RUNTIME_ARGS
+        -e
+        -v
+        1
+        --min-instructions=4
+        -E
+        ^std::
+        --label
+        file
+        line
+        return
+        args
+        -M
+        coverage
+        --coverage
+        function
+        --module-restrict
+        code.coverage
+    LABELS "coverage;function-coverage"
+    RUN_ARGS 10 ${NUM_THREADS} 1000
+    ENVIRONMENT "${_base_environment}"
+    RUNTIME_PASS_REGEX "(\\\[[0-9]+\\\]) code coverage     ::  66.67%"
+    REWRITE_RUN_PASS_REGEX "(\\\[[0-9]+\\\]) code coverage     ::  66.67%")
+
+omnitrace_add_test(
+    SKIP_BASELINE SKIP_SAMPLING
+    NAME code-coverage-hybrid
+    TARGET code-coverage
+    REWRITE_ARGS -e -v 2 --min-instructions=4 -E ^std:: --coverage function
+    RUNTIME_ARGS
+        -e
+        -v
+        1
+        --min-instructions=4
+        -E
+        ^std::
+        --label
+        file
+        line
+        return
+        args
+        --coverage
+        function
+        --module-restrict
+        code.coverage
+    LABELS "coverage;function-coverage;hybrid-coverage"
+    RUN_ARGS 10 ${NUM_THREADS} 1000
+    ENVIRONMENT "${_base_environment}"
+    RUNTIME_PASS_REGEX "(\\\[[0-9]+\\\]) code coverage     ::  66.67%"
+    REWRITE_RUN_PASS_REGEX "(\\\[[0-9]+\\\]) code coverage     ::  66.67%")
+
+omnitrace_add_test(
+    SKIP_BASELINE SKIP_SAMPLING
+    NAME code-coverage-basic-blocks
+    TARGET code-coverage
+    REWRITE_ARGS
+        -e
+        -v
+        2
+        --min-instructions=4
+        -E
+        ^std::
+        -M
+        coverage
+        --coverage
+        basic_block
+    RUNTIME_ARGS
+        -e
+        -v
+        1
+        --min-instructions=4
+        -E
+        ^std::
+        --label
+        file
+        line
+        return
+        args
+        -M
+        coverage
+        --coverage
+        basic_block
+        --module-restrict
+        code.coverage
+    LABELS "coverage;bb-coverage"
+    RUN_ARGS 10 ${NUM_THREADS} 1000
+    ENVIRONMENT "${_base_environment}"
+    RUNTIME_PASS_REGEX "(\\\[[0-9]+\\\]) function coverage ::  66.67%"
+    REWRITE_RUN_PASS_REGEX "(\\\[[0-9]+\\\]) function coverage ::  66.67%")
+
+omnitrace_add_test(
+    SKIP_BASELINE SKIP_SAMPLING
+    NAME code-coverage-basic-blocks-hybrid
+    TARGET code-coverage
+    REWRITE_ARGS -e -v 2 --min-instructions=4 -E ^std:: --coverage basic_block
+    RUNTIME_ARGS
+        -e
+        -v
+        1
+        --min-instructions=4
+        -E
+        ^std::
+        --label
+        file
+        line
+        return
+        args
+        --coverage
+        basic_block
+        --module-restrict
+        code.coverage
+    LABELS "coverage;bb-coverage;hybrid-coverage"
+    RUN_ARGS 10 ${NUM_THREADS} 1000
+    ENVIRONMENT "${_base_environment}"
+    RUNTIME_PASS_REGEX "(\\\[[0-9]+\\\]) function coverage ::  66.67%"
+    REWRITE_RUN_PASS_REGEX "(\\\[[0-9]+\\\]) function coverage ::  66.67%")

 # -------------------------------------------------------------------------------------- #
 #
@@ -572,7 +749,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
            COMMAND ${OMNITRACE_CAT_COMMAND}
            PYTHON_VERSION ${_VERSION}
            FILE omnitrace-tests-output/python-external/${_VERSION}/trip_count.txt
-            PASS_REGULAR_EXPRESSION
+            PASS_REGEX
                "(\\\[compile\\\]).*(\\\| \\\|0>>> \\\[run\\\]\\\[external.py\\\]).*(\\\| \\\|0>>> \\\|_\\\[fib\\\]\\\[external.py\\\]).*(\\\| \\\|0>>> \\\|_\\\[inefficient\\\]\\\[external.py\\\])"
            DEPENDS python-external-${_VERSION}
            ENVIRONMENT "${_python_environment}")
@@ -582,7 +759,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
            COMMAND ${OMNITRACE_CAT_COMMAND}
            PYTHON_VERSION ${_VERSION}
            FILE omnitrace-tests-output/python-external-exclude-inefficient/${_VERSION}/trip_count.txt
-            FAIL_REGULAR_EXPRESSION "(\\\|_inefficient).*(\\\|_sum)"
+            FAIL_REGEX "(\\\|_inefficient).*(\\\|_sum)"
            DEPENDS python-external-exclude-inefficient-${_VERSION}
            ENVIRONMENT "${_python_environment}")

@@ -591,7 +768,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
            COMMAND ${OMNITRACE_CAT_COMMAND}
            PYTHON_VERSION ${_VERSION}
            FILE omnitrace-tests-output/python-builtin/${_VERSION}/trip_count.txt
-            PASS_REGULAR_EXPRESSION "\\\[inefficient\\\]\\\[builtin.py:14\\\]"
+            PASS_REGEX "\\\[inefficient\\\]\\\[builtin.py:14\\\]"
            DEPENDS python-builtin-${_VERSION}
            ENVIRONMENT "${_python_environment}")

@@ -600,7 +777,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
            COMMAND ${OMNITRACE_CAT_COMMAND}
            PYTHON_VERSION ${_VERSION}
            FILE omnitrace-tests-output/python-source/${_VERSION}/trip_count.txt
-            PASS_REGULAR_EXPRESSION
+            PASS_REGEX
                "(\\\| \\\|0>>> run           \\\|      5).*(\\\| \\\|0>>> \\\|_fib         \\\|     40).*(\\\| \\\|0>>>         \\\|_fib \\\|      5).*(\\\| \\\|0>>> \\\|_inefficient \\\|      5).*(\\\| \\\|0>>>   \\\|__sum      \\\|      5)"
            DEPENDS python-source-${_VERSION}
            ENVIRONMENT "${_python_environment}")