GitHub CI (#11)

* Continuous integration * linux-ci on * fix for parallel-overhead * Updated CMAKE_INSTALL_PREFIX * Updated installed boost libraries * Dyninst updates fixing TBB install * timemory + dyninst submodule updates - fixes some timemory package option handling - fixes dyninst libiberty handling * Update dyninst submodule with libiberty fix * dyninst submodule update with TBB internal build fixes * Updated linux-ci + tests + timemeory + dyninst - updated timemory submodule - update dyninst submodule - delay OnLoad implementation - DYNINST_RT_API handling improvement - CI for ubuntu-bionic in addition to focal - CTest in CI * Update TBB handling in CI * Update dyninst with symLite fix * Update dyninst submodule with ElfUtils-External fix * Dyninst::ElfUtils fix * Modified dyninst submodule TPL install * Update dyninst submodule with improved interface libraries * Fix to Dyninst::ElfUtils in dyninst submodule * Updated CI build matrix + test install * Updated CI * CI stage updates * Tweak * Dyninst updates + RPATH for hosttrace exe - hosttrace will rpath to dyninst - Dyninst will statically link to boost by default - Fixes to double init w/ MPI + runtime instrumentation - minor cleanup to hosttrace - hosttrace help exits with zero - CI updates * Dyninst + CI updates * Removed ELFUTILS_BUILD_STATIC option in Dyninst * Dyninst + visibility + roctracer + tests - Dyninst submodule updates with dynamic pcontrol lib - hosttrace visibility is hidden - improved handling of DYNINST_API_RT in hosttrace - roctracer::tear_down in finalize fixes issues with timemory - throw error if cannot create perfetto output file - roctracer_default_pool() safety guards - resume calling roctracer_close_pool() - fixed working dir of tests - fixed env of tests (cmake and CI) - simplified CI * Removed stray CI if condition * Disable hidden visibility for now * ifdef for roctracer::tear_down + reenable hidden * Better CI environment handling + dyninst packaging * Fix for dyninst-package CI * Fixes for cmake 3.15 issues with aliasing imported lib * Fix to ubuntu-focal-dyninst-package * Dyninst updates for packaging - fixes issues with hard-coded paths to libraries after relocation via package installer * Restrict CI to main and develop branches [ROCm/rocprofiler-systems commit: 35ab6b0110]
2021-09-15 16:45:49 -05:00
@@ -0,0 +1,20 @@
+name: cancel-builds-on-update
+on:
+  workflow_run:
+    workflows: ['linux-ci',  'formatting']
+    types: ['requested']
+
+jobs:
+  cancel-duplicate-workflow-runs:
+    name: "Cancel duplicate workflow runs"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel duplicate workflow runs"
+        with:
+          cancelMode: duplicates
+          cancelFutureDuplicates: true
+          token: ${{ secrets.GITHUB_TOKEN }}
+          sourceRunId: ${{ github.event.workflow_run.id }}
+          notifyPRCancel: true
+          skipEventTypes: '["schedule"]'
@@ -0,0 +1,39 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: formatting
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  formatting:
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        DISTRIB_CODENAME=$(cat /etc/lsb-release | grep DISTRIB_CODENAME | awk -F '=' '{print $NF}')
+        sudo apt-get update
+        sudo apt-get install -y software-properties-common wget curl
+        wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+        echo "deb http://apt.llvm.org/${DISTRIB_CODENAME}/ llvm-toolchain-${DISTRIB_CODENAME}-11 main" | sudo tee -a /etc/apt/sources.list.d/llvm-toolchain.list
+        echo "deb-src http://apt.llvm.org/${DISTRIB_CODENAME}/ llvm-toolchain-${DISTRIB_CODENAME}-11 main" | sudo tee -a /etc/apt/sources.list.d/llvm-toolchain.list
+        sudo apt-get update
+        sudo apt-get install -y clang-format-11
+    - name: clang-format
+      run: |
+        set +e
+        FILES=$(find include src examples -type f | egrep '\.hpp$|\.cpp$|\.cpp\.in$')
+        FORMAT_OUT=$(clang-format-11 -output-replacements-xml ${FILES})
+        RET=$(echo ${FORMAT_OUT} | grep -c '<replacement ')
+        if [ "${RET}" -ne 0 ]; then
+          echo -e "\nError! Code not formatted. Detected ${RET} lines\n"
+          clang-format-11 -i ${FILES}
+          git diff
+          exit ${RET}
+        fi
@@ -0,0 +1,327 @@
+name: linux-ci
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+env:
+  BUILD_TYPE: Release
+  ELFUTILS_DOWNLOAD_VERSION: 0.183
+
+jobs:
+  ubuntu-focal:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        compiler: ['g++']
+        mpi: [ '', 'libmpich-dev mpich', 'libopenmpi-dev openmpi-bin libfabric-dev' ]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Packages
+      run:
+        sudo apt-get update &&
+        sudo apt-get install -y build-essential python3-pip libtbb-dev libboost-{atomic,system,thread,date-time,filesystem,timer}-dev ${{ matrix.compiler }} ${{ matrix.mpi }} &&
+        python3 -m pip install --upgrade pip &&
+        python3 -m pip install 'cmake==3.15.3'
+
+    - name: Configure Env
+      run:
+        echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
+        echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
+        echo "/opt/hosttrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
+        echo "LD_LIBRARY_PATH=/opt/hosttrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+    - name: Configure CMake
+      run:
+        cmake --version &&
+        if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
+        cmake -B ${{ github.workspace }}/build
+          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
+          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
+          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+          -DCMAKE_INSTALL_PREFIX=/opt/hosttrace
+          -DHOSTTRACE_USE_MPI=${USE_MPI}
+          -DHOSTTRACE_USE_ROCTRACER=OFF
+          -DHOSTTRACE_BUILD_DYNINST=ON
+          -DDYNINST_BUILD_ELFUTILS=ON
+          -DDYNINST_BUILD_LIBIBERTY=ON
+          -DDYNINST_BUILD_SHARED_LIBS=ON
+          -DDYNINST_BUILD_STATIC_LIBS=OFF
+
+    - name: Build
+      run:
+        cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
+
+    - name: Install
+      run:
+        cmake --build ${{ github.workspace }}/build --target install --parallel 2
+
+    - name: Test
+      working-directory: ${{ github.workspace }}/build
+      run:
+        ctest -V --output-log ${{ github.workspace }}/build/hosttrace-ctest.log
+
+    - name: Test Install
+      run:
+        hosttrace --help &&
+        hosttrace -- sleep 1 &&
+        hosttrace -o sleep.inst -- sleep &&
+        ./sleep.inst 1 &&
+        rm ./sleep.inst
+
+    - name: Artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: ctest-log
+        path: |
+          ${{ github.workspace }}/build/hosttrace-ctest.log
+
+  ubuntu-bionic:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        compiler: ['g++-7', 'g++-8']
+        mpi: [ '', 'libmpich-dev mpich', 'libopenmpi-dev openmpi-bin libfabric-dev' ]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Packages
+      run:
+        sudo apt-get update &&
+        sudo apt-get install -y build-essential python3-pip ${{ matrix.compiler }} ${{ matrix.mpi }} &&
+        python3 -m pip install --upgrade pip &&
+        python3 -m pip install 'cmake==3.15.3'
+
+    - name: Configure Env
+      run:
+        echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
+        echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
+        echo "/opt/hosttrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
+        echo "LD_LIBRARY_PATH=/opt/hosttrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+    - name: Configure CMake
+      run:
+        cmake --version &&
+        if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
+        cmake -B ${{ github.workspace }}/build
+          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
+          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
+          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+          -DCMAKE_INSTALL_PREFIX=/opt/hosttrace
+          -DHOSTTRACE_USE_MPI=${USE_MPI}
+          -DHOSTTRACE_USE_ROCTRACER=OFF
+          -DHOSTTRACE_BUILD_DYNINST=ON
+          -DDYNINST_BUILD_TBB=ON
+          -DDYNINST_BUILD_BOOST=ON
+          -DDYNINST_BUILD_ELFUTILS=ON
+          -DDYNINST_BUILD_LIBIBERTY=ON
+
+    - name: Build
+      run:
+        cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
+
+    - name: Install
+      run:
+        cmake --build ${{ github.workspace }}/build --target install --parallel 2
+
+    - name: Test
+      working-directory: ${{ github.workspace }}/build
+      run:
+        ctest -V --output-log ${{ github.workspace }}/build/hosttrace-ctest.log
+
+    - name: Test Install
+      run:
+        hosttrace --help &&
+        hosttrace -- sleep 1 &&
+        hosttrace -o sleep.inst -- sleep &&
+        ./sleep.inst 1 &&
+        rm ./sleep.inst
+
+    - name: Artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: ctest-log
+        path: |
+          ${{ github.workspace }}/build/hosttrace-ctest.log
+
+  ubuntu-focal-external:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        compiler: ['g++-7', 'g++-8', 'g++-9', 'g++-10']
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Packages
+      run:
+        sudo apt-get update &&
+        sudo apt-get install -y build-essential python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev ${{ matrix.compiler }} &&
+        sudo python3 -m pip install --upgrade pip &&
+        python3 -m pip install 'cmake==3.15.3'
+
+    - name: Configure Env
+      run:
+        echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
+        echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
+        echo "CMAKE_PREFIX_PATH=/opt/opt/dyninst:/opt/elfutils:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
+        echo "/opt/hosttrace/bin:/opt/dyninst/bin:/opt/elfutils/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
+        echo "LD_LIBRARY_PATH=/opt/hosttrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+    - name: Install ElfUtils
+      run:
+        pushd external &&
+        wget https://sourceware.org/elfutils/ftp/${ELFUTILS_DOWNLOAD_VERSION}/elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
+        tar xjf elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
+        pushd elfutils-${ELFUTILS_DOWNLOAD_VERSION} &&
+        CFLAGS="-g -O2" ./configure --enable-install-elfh --prefix=/opt/elfutils --disable-libdebuginfod --disable-debuginfod &&
+        make -j2 &&
+        make install -j2 &&
+        popd &&
+        rm -rf elfutils*
+
+    - name: Install Dyninst
+      run:
+        cmake --version &&
+        git submodule update --init external/dyninst &&
+        cd external/dyninst &&
+        cmake -B build
+          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
+          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
+          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+          -DCMAKE_INSTALL_PREFIX=/opt/dyninst &&
+        cmake --build build --target all --parallel 2 &&
+        cmake --build build --target install --parallel 2 &&
+        rm -rf build
+
+    - name: Configure CMake
+      run:
+        cmake --version &&
+        cmake -B ${{ github.workspace }}/build
+          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
+          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
+          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+          -DCMAKE_INSTALL_PREFIX=/opt/hosttrace
+          -DHOSTTRACE_USE_MPI=OFF
+          -DHOSTTRACE_USE_ROCTRACER=OFF
+
+    - name: Build
+      run:
+        cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
+
+    - name: Install
+      run:
+        cmake --build ${{ github.workspace }}/build --target install --parallel 2
+
+    - name: Test
+      working-directory: ${{ github.workspace }}/build
+      run:
+        ldd ./hosttrace &&
+        ./hosttrace --help &&
+        ctest -V --output-log ${{ github.workspace }}/build/hosttrace-ctest.log
+
+    - name: Test Install
+      run:
+        ldd $(which hosttrace) &&
+        hosttrace --help &&
+        hosttrace -- sleep 1 &&
+        hosttrace -o sleep.inst -- sleep &&
+        ./sleep.inst 1 &&
+        rm ./sleep.inst
+
+    - name: Artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: ctest-log
+        path: |
+          ${{ github.workspace }}/build/hosttrace-ctest.log
+
+  ubuntu-focal-dyninst-package:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        compiler: ['g++']
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Packages
+      run:
+        sudo apt-get update &&
+        sudo apt-get install -y build-essential python3-pip ${{ matrix.compiler }} &&
+        sudo python3 -m pip install --upgrade pip &&
+        python3 -m pip install 'cmake==3.15.3'
+
+    - name: Configure Env
+      run:
+        echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
+        echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
+        echo "CMAKE_PREFIX_PATH=/opt/opt/dyninst:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
+        echo "/opt/hosttrace/bin:/opt/dyninst/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
+        echo "LD_LIBRARY_PATH=/opt/hosttrace/lib:/opt/dyninst/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+    - name: Install Dyninst
+      run:
+        cmake --version &&
+        git submodule update --init external/dyninst &&
+        cd external/dyninst &&
+        cmake -B build
+          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
+          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
+          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+          -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/external/dyninst-install
+          -DBUILD_TBB=ON
+          -DBUILD_BOOST=ON
+          -DBUILD_ELFUTILS=ON
+          -DBUILD_LIBIBERTY=ON &&
+        cmake --build build --target package --parallel 4 &&
+        mkdir /opt/dyninst &&
+        ./build/Dyninst-*-Linux.sh --prefix=/opt/dyninst --exclude-subdir --skip-license &&
+        rm -rf build
+
+    - name: Configure CMake
+      run:
+        cmake --version &&
+        cmake -B ${{ github.workspace }}/build
+          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
+          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
+          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+          -DCMAKE_INSTALL_PREFIX=/opt/hosttrace
+          -DHOSTTRACE_USE_MPI=OFF
+          -DHOSTTRACE_USE_ROCTRACER=OFF
+
+    - name: Build
+      run:
+        cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
+
+    - name: Install
+      run:
+        cmake --build ${{ github.workspace }}/build --target install --parallel 2
+
+    - name: Test
+      working-directory: ${{ github.workspace }}/build
+      run:
+        ldd ./hosttrace &&
+        ./hosttrace --help &&
+        ctest -V --output-log ${{ github.workspace }}/build/hosttrace-ctest.log
+
+    - name: Test Install
+      run:
+        ldd $(which hosttrace) &&
+        hosttrace --help &&
+        hosttrace -- sleep 1 &&
+        hosttrace -o sleep.inst -- sleep &&
+        ./sleep.inst 1 &&
+        rm ./sleep.inst
+
+    - name: Artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: ctest-log
+        path: |
+          ${{ github.workspace }}/build/hosttrace-ctest.log
@@ -44,6 +44,10 @@ include(Packages)           # finds third-party libraries

 hosttrace_activate_clang_tidy()

+set(CMAKE_C_VISIBILITY_PRESET "hidden")
+set(CMAKE_CXX_VISIBILITY_PRESET "hidden")
+set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
+
 #------------------------------------------------------------------------------#
 #
 #                   hosttrace-library target
@@ -86,7 +90,7 @@ endif()

 set_target_properties(hosttrace-library PROPERTIES
    OUTPUT_NAME     hosttrace
-    INSTALL_RPATH   "\$ORIGIN:${DYNINST_API_RT_DIR}:${CMAKE_INSTALL_RPATH}")
+    INSTALL_RPATH   "\$ORIGIN:${DYNINST_API_RT_DIR}")

 install(
    TARGETS     hosttrace-library
@@ -115,7 +119,8 @@ target_link_libraries(hosttrace-exe PRIVATE

 set_target_properties(hosttrace-exe PROPERTIES
    OUTPUT_NAME                 hosttrace
-    INSTALL_RPATH_USE_LINK_PATH ON)
+    INSTALL_RPATH_USE_LINK_PATH ON
+    INSTALL_RPATH               "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:${DYNINST_API_RT_DIR}")

 install(
    TARGETS     hosttrace-exe
@@ -65,119 +65,126 @@ endif()
 #----------------------------------------------------------------------------------------#

 if(HOSTTRACE_BUILD_DYNINST)
-    if(CMAKE_GENERATOR MATCHES "Ninja")
-        message(FATAL_ERROR "Building dyninst submodule requires the cmake generator to be Unix Makefiles")
-    endif()
-
    checkout_git_submodule(
        RELATIVE_PATH       external/dyninst
        WORKING_DIRECTORY   ${PROJECT_SOURCE_DIR}
        REPO_URL            https://github.com/jrmadsen/dyninst.git
        REPO_BRANCH         hosttrace-submodule)

-    add_subdirectory(external/dyninst)
+    set(DYNINST_OPTION_PREFIX       ON)
+    set(DYNINST_BUILD_DOCS          OFF)
+    set(DYNINST_QUIET_CONFIG        ON  CACHE BOOL "Suppress dyninst cmake messages")
+    set(DYNINST_BUILD_PARSE_THAT    OFF CACHE BOOL "Build dyninst parseThat executable")
+    set(DYNINST_BUILD_SHARED_LIBS   OFF CACHE BOOL "Build shared dyninst libraries")
+    set(DYNINST_BUILD_STATIC_LIBS   ON  CACHE BOOL "Build static dyninst libraries")
+    set(DYNINST_ENABLE_LTO          ON  CACHE BOOL "Enable LTO for dyninst libraries")

-    target_link_libraries(hosttrace-dyninst INTERFACE
-        ${DYNINST_LIBRARIES} ${Boost_LIBRARIES})
-    foreach(_TARG dyninst dyninstAPI instructionAPI symtabAPI parseAPI headers atomic system thread date_time TBB)
-        if(TARGET Dyninst::${_TARG})
-            target_link_libraries(hosttrace-dyninst INTERFACE Dyninst::${_TARG})
-        elseif(TARGET Boost::${_TARG})
-            target_link_libraries(hosttrace-dyninst INTERFACE Boost::${_TARG})
-        elseif(TARGET ${_TARG})
-            target_link_libraries(hosttrace-dyninst INTERFACE ${_TARG})
-        endif()
+    hosttrace_save_variables(PIC CMAKE_POSITION_INDEPENDENT_CODE)
+    set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+    add_subdirectory(external/dyninst)
+    hosttrace_restore_variables(PIC CMAKE_POSITION_INDEPENDENT_CODE)
+
+    add_library(Dyninst::Dyninst INTERFACE IMPORTED)
+    foreach(_LIB common dyninstAPI parseAPI instructionAPI symtabAPI stackwalk Boost TBB)
+        target_link_libraries(Dyninst::Dyninst INTERFACE Dyninst::${_LIB})
    endforeach()
-    target_include_directories(hosttrace-dyninst SYSTEM INTERFACE
-        ${TBB_INCLUDE_DIR}
-        ${Boost_INCLUDE_DIRS}
-        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/dyninst/dyninstAPI/h>
-        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/dyninst/instructionAPI/h>
-        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/dyninst/proccontrol/h>
-        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/dyninst/dataflowAPI/h>
-        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/dyninst/symtabAPI/h>
-        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/dyninst/parseAPI/h>
-        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/dyninst/common/h>
-        $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/external/dyninst/common/h>)
-    target_compile_definitions(hosttrace-dyninst INTERFACE hosttrace_USE_DYNINST)
+
+    target_link_libraries(hosttrace-dyninst INTERFACE Dyninst::Dyninst)
+
+    set(HOSTTRACE_DYNINST_API_RT ${PROJECT_BINARY_DIR}/external/dyninst/dyninstAPI_RT/libdyninstAPI_RT${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+    if(HOSTTRACE_DYNINST_API_RT)
+        target_compile_definitions(hosttrace-dyninst INTERFACE
+            DYNINST_API_RT="${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}:$<TARGET_FILE_DIR:Dyninst::dyninstAPI_RT>:${CMAKE_INSTALL_PREFIX}/lib/$<TARGET_FILE_NAME:Dyninst::dyninstAPI_RT>:$<TARGET_FILE:Dyninst::dyninstAPI_RT>")
+    endif()

 else()
-    find_package(Dyninst ${hosttrace_FIND_QUIETLY} REQUIRED
-        COMPONENTS dyninstAPI parseAPI instructionAPI symtabAPI)
-    set(_BOOST_COMPONENTS atomic system thread date_time)
-    set(hosttrace_BOOST_COMPONENTS "${_BOOST_COMPONENTS}" CACHE STRING
-        "Boost components used by Dyninst in hosttrace")
-    set(Boost_NO_BOOST_CMAKE ON)
-    find_package(Boost QUIET REQUIRED
-        COMPONENTS ${hosttrace_BOOST_COMPONENTS})
+    find_package(Dyninst ${hosttrace_FIND_QUIETLY} REQUIRED COMPONENTS dyninstAPI parseAPI instructionAPI symtabAPI)

-    # some installs of dyninst don't set this properly
-    if(EXISTS "${DYNINST_INCLUDE_DIR}" AND NOT DYNINST_HEADER_DIR)
-        get_filename_component(DYNINST_HEADER_DIR "${DYNINST_INCLUDE_DIR}" REALPATH CACHE)
-    else()
-        find_path(DYNINST_HEADER_DIR
-            NAMES BPatch.h dyninstAPI_RT.h
-            HINTS ${Dyninst_ROOT_DIR} ${Dyninst_DIR} ${Dyninst_DIR}/../../..
-            PATHS ${Dyninst_ROOT_DIR} ${Dyninst_DIR} ${Dyninst_DIR}/../../..
-            PATH_SUFFIXES include)
-    endif()
+    if(TARGET Dyninst::Dyninst)     # updated Dyninst CMake system was found
+        # useful for defining the location of the runtime API
+        find_library(HOSTTRACE_DYNINST_API_RT dyninstAPI_RT
+            HINTS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
+            PATHS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
+            PATH_SUFFIXES lib)

-    # useful for defining the location of the runtime API
-    find_library(DYNINST_API_RT dyninstAPI_RT
-        HINTS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
-        PATHS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
-        PATH_SUFFIXES lib)
+        if(HOSTTRACE_DYNINST_API_RT)
+            target_compile_definitions(hosttrace-dyninst INTERFACE
+                DYNINST_API_RT="${HOSTTRACE_DYNINST_API_RT}")
+        endif()

-    # try to find TBB
-    find_package(TBB QUIET)
+        add_rpath(${Dyninst_LIBRARIES})
+        target_link_libraries(hosttrace-dyninst INTERFACE Dyninst::Dyninst)
+    else()  # updated Dyninst CMake system was not found
+        set(_BOOST_COMPONENTS atomic system thread date_time)
+        set(hosttrace_BOOST_COMPONENTS "${_BOOST_COMPONENTS}" CACHE STRING
+            "Boost components used by Dyninst in hosttrace")
+        set(Boost_NO_BOOST_CMAKE ON)
+        find_package(Boost QUIET REQUIRED
+            COMPONENTS ${hosttrace_BOOST_COMPONENTS})

-    # if fail try to use the Dyninst installed FindTBB.cmake
-    if(NOT TBB_FOUND)
-        list(APPEND CMAKE_MODULE_PATH ${Dyninst_DIR}/Modules)
+        # some installs of dyninst don't set this properly
+        if(EXISTS "${DYNINST_INCLUDE_DIR}" AND NOT DYNINST_HEADER_DIR)
+            get_filename_component(DYNINST_HEADER_DIR "${DYNINST_INCLUDE_DIR}" REALPATH CACHE)
+        else()
+            find_path(DYNINST_HEADER_DIR
+                NAMES BPatch.h dyninstAPI_RT.h
+                HINTS ${Dyninst_ROOT_DIR} ${Dyninst_DIR} ${Dyninst_DIR}/../../..
+                PATHS ${Dyninst_ROOT_DIR} ${Dyninst_DIR} ${Dyninst_DIR}/../../..
+                PATH_SUFFIXES include)
+        endif()
+
+        # useful for defining the location of the runtime API
+        find_library(DYNINST_API_RT dyninstAPI_RT
+            HINTS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
+            PATHS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
+            PATH_SUFFIXES lib)
+
+        # try to find TBB
        find_package(TBB QUIET)
-    endif()

-    if(NOT TBB_FOUND)
-        find_path(TBB_INCLUDE_DIR
-            NAMES tbb/tbb.h
-            PATH_SUFFIXES include)
-    endif()
-
-    if(DYNINST_API_RT)
-        target_compile_definitions(hosttrace-dyninst INTERFACE
-            DYNINST_API_RT="${DYNINST_API_RT}")
-    endif()
-
-    if(Boost_DIR)
-        get_filename_component(Boost_RPATH_DIR "${Boost_DIR}" DIRECTORY)
-        get_filename_component(Boost_RPATH_DIR "${Boost_RPATH_DIR}" DIRECTORY)
-        if(EXISTS "${Boost_RPATH_DIR}" AND IS_DIRECTORY "${Boost_RPATH_DIR}")
-            set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${Boost_RPATH_DIR}")
+        # if fail try to use the Dyninst installed FindTBB.cmake
+        if(NOT TBB_FOUND)
+            list(APPEND CMAKE_MODULE_PATH ${Dyninst_DIR}/Modules)
+            find_package(TBB QUIET)
        endif()
-    endif()

-    add_rpath(${DYNINST_LIBRARIES} ${Boost_LIBRARIES})
-    target_link_libraries(hosttrace-dyninst INTERFACE
-        ${DYNINST_LIBRARIES} ${Boost_LIBRARIES})
-    foreach(_TARG dyninst dyninstAPI instructionAPI symtabAPI parseAPI headers atomic system thread date_time TBB)
-        if(TARGET Dyninst::${_TARG})
-            target_link_libraries(hosttrace-dyninst INTERFACE Dyninst::${_TARG})
-        elseif(TARGET Boost::${_TARG})
-            target_link_libraries(hosttrace-dyninst INTERFACE Boost::${_TARG})
-        elseif(TARGET ${_TARG})
-            target_link_libraries(hosttrace-dyninst INTERFACE ${_TARG})
+        if(NOT TBB_FOUND)
+            find_path(TBB_INCLUDE_DIR
+                NAMES tbb/tbb.h
+                PATH_SUFFIXES include)
        endif()
-    endforeach()
-    target_include_directories(hosttrace-dyninst SYSTEM INTERFACE
-        ${TBB_INCLUDE_DIR}
-        ${Boost_INCLUDE_DIRS}
-        ${DYNINST_HEADER_DIR})
-    target_compile_definitions(hosttrace-dyninst INTERFACE hosttrace_USE_DYNINST)

-    if(DYNINST_API_RT)
-        add_cmake_defines(DYNINST_API_RT VALUE QUOTE DEFAULT)
-    else()
-        add_cmake_defines(DYNINST_API_RT VALUE QUOTE)
+        if(DYNINST_API_RT)
+            target_compile_definitions(hosttrace-dyninst INTERFACE
+                DYNINST_API_RT="${DYNINST_API_RT}")
+        endif()
+
+        if(Boost_DIR)
+            get_filename_component(Boost_RPATH_DIR "${Boost_DIR}" DIRECTORY)
+            get_filename_component(Boost_RPATH_DIR "${Boost_RPATH_DIR}" DIRECTORY)
+            if(EXISTS "${Boost_RPATH_DIR}" AND IS_DIRECTORY "${Boost_RPATH_DIR}")
+                set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${Boost_RPATH_DIR}")
+            endif()
+        endif()
+
+        add_rpath(${DYNINST_LIBRARIES} ${Boost_LIBRARIES})
+        target_link_libraries(hosttrace-dyninst INTERFACE
+            ${DYNINST_LIBRARIES} ${Boost_LIBRARIES})
+        foreach(_TARG dyninst dyninstAPI instructionAPI symtabAPI parseAPI headers atomic system thread date_time TBB)
+            if(TARGET Dyninst::${_TARG})
+                target_link_libraries(hosttrace-dyninst INTERFACE Dyninst::${_TARG})
+            elseif(TARGET Boost::${_TARG})
+                target_link_libraries(hosttrace-dyninst INTERFACE Boost::${_TARG})
+            elseif(TARGET ${_TARG})
+                target_link_libraries(hosttrace-dyninst INTERFACE ${_TARG})
+            endif()
+        endforeach()
+        target_include_directories(hosttrace-dyninst SYSTEM INTERFACE
+            ${TBB_INCLUDE_DIR}
+            ${Boost_INCLUDE_DIRS}
+            ${DYNINST_HEADER_DIR})
+        target_compile_definitions(hosttrace-dyninst INTERFACE hosttrace_USE_DYNINST)
    endif()
 endif()

@@ -9,6 +9,6 @@ add_executable(parallel-overhead parallel-overhead.cpp)
 target_link_libraries(parallel-overhead Threads::Threads)

 if(NOT CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
-    set_target_properties(transpose PROPERTIES
+    set_target_properties(parallel-overhead PROPERTIES
        RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 endif()
@@ -187,7 +187,7 @@ consume_parameters(T&&...)
 extern "C"
 {
    bool are_file_include_exclude_lists_empty();
-    bool process_file_for_instrumentation(const string_t& file_name);
+    bool instrument_module(const string_t& file_name);
    bool instrument_entity(const string_t& function_name);
    bool module_constraint(char* fname);
    bool routine_constraint(const char* fname);
@@ -32,12 +32,18 @@ struct roctracer

    TIMEMORY_DEFAULT_OBJECT(roctracer)

-    static void setup();
-    static void tear_down();
    static void preinit();
    static void global_init() { setup(); }
    static void global_finalize() { tear_down(); }

+    static bool is_setup();
+    static void setup();
+    static void tear_down();
+    static void add_setup(const std::string&, std::function<void()>&&);
+    static void add_tear_down(const std::string&, std::function<void()>&&);
+    static void remove_setup(const std::string&);
+    static void remove_tear_down(const std::string&);
+
    void start();
    void stop();
    void set_prefix(const char* _v) { m_prefix = _v; }
@@ -72,8 +72,31 @@ int
 main(int argc, char** argv)
 {
 #if defined(DYNINST_API_RT)
-    tim::set_env<string_t>("DYNINSTAPI_RT_LIB", DYNINST_API_RT, 0);
+    auto _dyn_api_rt_paths = tim::delimit(DYNINST_API_RT, ":");
+#else
+    auto _dyn_api_rt_paths = std::vector<std::string>{};
 #endif
+    auto _dyn_api_rt_abs = get_absolute_lib_filepath("libdyninstAPI_RT.so");
+    _dyn_api_rt_paths.insert(_dyn_api_rt_paths.begin(), _dyn_api_rt_abs);
+    for(auto&& itr : _dyn_api_rt_paths)
+    {
+        auto file_exists = [](const std::string& _fname) {
+            struct stat _buffer;
+            if(stat(_fname.c_str(), &_buffer) == 0)
+                return (S_ISREG(_buffer.st_mode) != 0 || S_ISLNK(_buffer.st_mode) != 0);
+            return false;
+        };
+        if(file_exists(itr))
+            tim::set_env<string_t>("DYNINSTAPI_RT_LIB", itr, 0);
+        else if(file_exists(TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.so")))
+            tim::set_env<string_t>("DYNINSTAPI_RT_LIB",
+                                   TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.so"), 0);
+        else if(file_exists(TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.a")))
+            tim::set_env<string_t>("DYNINSTAPI_RT_LIB",
+                                   TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.a"), 0);
+    }
+    verbprintf(0, "[hosttrace] DYNINST_API_RT: %s\n",
+               tim::get_env<string_t>("DYNINSTAPI_RT_LIB", "").c_str());

    argv0 = argv[0];

@@ -156,12 +179,12 @@ main(int argc, char** argv)

    if(verbose_level > 1)
    {
-        std::cout << "[original]: " << cmd_string(argc, argv) << std::endl;
-        std::cout << "[cfg-args]: " << cmd_string(_argc, _argv) << std::endl;
+        std::cout << "[hosttrace][original]: " << cmd_string(argc, argv) << std::endl;
+        std::cout << "[hosttrace][cfg-args]: " << cmd_string(_argc, _argv) << std::endl;
    }

    if(_cmdc > 0)
-        std::cout << "\n [command]: " << cmd_string(_cmdc, _cmdv) << "\n\n";
+        std::cout << "\n[hosttrace][command]: " << cmd_string(_cmdc, _cmdv) << "\n\n";

    if(_cmdc > 0)
        cmdv0 = _cmdv[0];
@@ -380,12 +403,6 @@ main(int argc, char** argv)

    string_t extra_help = "-- <CMD> <ARGS>";
    auto     err        = parser.parse(_argc, _argv);
-    if(err)
-    {
-        std::cerr << err << std::endl;
-        parser.print_help(extra_help);
-        return -1;
-    }

    if(parser.exists("h") || parser.exists("help"))
    {
@@ -393,6 +410,13 @@ main(int argc, char** argv)
        return 0;
    }

+    if(err)
+    {
+        std::cerr << err << std::endl;
+        parser.print_help(extra_help);
+        return -1;
+    }
+
    if(parser.exists("e"))
        werror = true;

@@ -892,7 +916,7 @@ main(int argc, char** argv)
        use_mpi = true;

    bool use_mpip = false;
-    if(use_mpi && binary_rewrite)
+    if(use_mpi)
        use_mpip = true;

    //----------------------------------------------------------------------------------//
@@ -1146,8 +1170,9 @@ main(int argc, char** argv)
    auto trace_call_args =
        hosttrace_call_expr("HOSTTRACE_COMPONENTS", default_components);
    auto use_mpi_call_args  = hosttrace_call_expr("HOSTTRACE_USE_MPI", "ON");
-    auto use_mpip_call_args = hosttrace_call_expr("HOSTTRACE_USE_MPIP", "ON");
-    auto none_call_args     = hosttrace_call_expr();
+    auto use_mpip_call_args = hosttrace_call_expr(
+        "HOSTTRACE_USE_MPIP", (binary_rewrite && use_mpi && use_mpip) ? "ON" : "OFF");
+    auto none_call_args = hosttrace_call_expr();

    verbprintf(2, "Done\n");
    verbprintf(2, "Getting call snippets... ");
@@ -1231,7 +1256,7 @@ main(int argc, char** argv)
    if(use_mpi && umpi_call)
        init_names.push_back(umpi_call.get());

-    if(init_call)
+    if(init_call && binary_rewrite)
        init_names.push_back(init_call.get());

    if(binary_rewrite)
@@ -1317,11 +1342,11 @@ main(int argc, char** argv)
            if(std::string{ modname }.find("libdyninst") != std::string::npos)
                continue;

-            if(module_constraint(modname) || !process_file_for_instrumentation(modname))
-            {
-                verbprintf(1, "Skipping constrained module: '%s'\n", modname);
+            if(module_constraint(modname))
+                continue;
+
+            if(!instrument_module(modname))
                continue;
-            }

            itr->getName(fname, FUNCNAMELEN);

@@ -1334,18 +1359,10 @@ main(int argc, char** argv)
            }

            if(routine_constraint(name.m_name.c_str()))
-            {
-                verbprintf(1, "Skipping function [constrained]: %s\n",
-                           name.m_name.c_str());
                continue;
-            }

            if(!instrument_entity(name.m_name))
-            {
-                verbprintf(1, "Skipping function [excluded]: %s / %s\n",
-                           name.m_name.c_str(), name.get().c_str());
                continue;
-            }

            if(is_static_exe && has_debug_info && string_t{ fname } == "_fini" &&
               string_t{ modname } == "DEFAULT_MODULE")
@@ -1816,8 +1833,19 @@ main(int argc, char** argv)
 //======================================================================================//

 bool
-process_file_for_instrumentation(const string_t& file_name)
+instrument_module(const string_t& file_name)
 {
+    auto _report = [&file_name](const string_t& _action, const string_t& _reason,
+                                int _lvl) {
+        static strset_t already_reported{};
+        if(already_reported.count(file_name) == 0)
+        {
+            verbprintf(_lvl, "%s module [%s] : '%s'...\n", _action.c_str(),
+                       _reason.c_str(), file_name.c_str());
+            already_reported.insert(file_name);
+        }
+    };
+
    auto is_include = [&](bool _if_empty) {
        if(file_include.empty())
            return _if_empty;
@@ -1835,28 +1863,22 @@ process_file_for_instrumentation(const string_t& file_name)
        for(auto& itr : file_exclude)
        {
            if(std::regex_search(file_name, itr))
-            {
-                verbprintf(2, "Excluding module [user-regex] : '%s'...\n",
-                           file_name.c_str());
                return true;
-            }
        }
        return false;
    };

-    auto _user_include = is_include(false) && !is_exclude();
+    auto _user_include = is_include(false);
+    auto _user_exclude = is_exclude();

-    if(_user_include)
-    {
-        verbprintf(2, "Including module [user-regex] : '%s'...\n", file_name.c_str());
-        return true;
-    }
+    if(_user_include && !_user_exclude)
+        return (_report("Including", "user-regex", 2), true);

    string_t          ext_str = "\\.(s|S)$";
    static std::regex ext_regex(ext_str, regex_opts);
    static std::regex sys_regex("^(s|k|e|w)_[A-Za-z_0-9\\-]+\\.(c|C)$", regex_opts);
    static std::regex userlib_regex(
-        "^lib(hosttrace|caliper|gotcha|papi|cupti|TAU|likwid|"
+        "^(lib|)(hosttrace|caliper|gotcha|papi|cupti|TAU|likwid|"
        "profiler|tcmalloc|dyninst|pfm|nvtx|upcxx|pthread|nvperf|hsa)",
        regex_opts);
    static std::regex corelib_regex("^lib(rt-|dl-|util-|python)", regex_opts);
@@ -1887,37 +1909,27 @@ process_file_for_instrumentation(const string_t& file_name)

    if(std::regex_search(file_name, ext_regex))
    {
-        verbprintf(3, "Excluding instrumentation [file extension] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "file extension", 3), false);
    }

    if(std::regex_search(file_name, sys_regex))
    {
-        verbprintf(3, "Excluding instrumentation [system library] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "system library", 3), false);
    }

    if(std::regex_search(file_name, corelib_regex))
    {
-        verbprintf(3, "Excluding instrumentation [core library] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "core library", 3), false);
    }

    if(std::regex_search(file_name, userlib_regex))
    {
-        verbprintf(3, "Excluding instrumentation [instr library] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "instrumentation", 3), false);
    }

    if(std::regex_search(file_name, prefix_regex))
    {
-        verbprintf(3, "Excluding instrumentation [prefix match] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "prefix match", 3), false);
    }

    /*if(std::regex_search(file_name, suffix_regex))
@@ -1927,17 +1939,12 @@ process_file_for_instrumentation(const string_t& file_name)
        return false;
    }*/

-    bool use = is_include(true) && !is_exclude();
-    if(use)
-    {
-        static strset_t already_reported;
-        if(already_reported.count(file_name) == 0)
-        {
-            verbprintf(2, "%s |> [ %s ]\n", __FUNCTION__, file_name.c_str());
-            already_reported.insert(file_name);
-        }
-    }
-    return use;
+    if(_user_exclude)
+        return (_report("Excluding", "user-regex", 2), false);
+
+    _report("Including", "no constraint", 2);
+
+    return true;
 }

 //======================================================================================//
@@ -1990,7 +1997,7 @@ instrument_entity(const string_t& function_name)
        "delete|std::allocat|"
        "nvtx|gcov|main\\.cold|TAU|tau|Tau|dyn|RT|dl|sys|pthread|posix|clone|virtual "
        "thunk|non-virtual thunk|transaction "
-        "clone|RtsLayer|DYNINST|PthreadLayer|threaded_func|targ8)",
+        "clone|RtsLayer|DYNINST|PthreadLayer|threaded_func|targ8|PMPI)",
        regex_opts);
    static std::regex trailing("(\\.part\\.[0-9]+|\\.constprop\\.[0-9]+|\\.|\\.[0-9]+)$",
                               regex_opts);
@@ -2140,18 +2147,7 @@ module_constraint(char* fname)
    if(_fname == "DEFAULT_MODULE" || _fname == "LIBRARY_MODULE")
        return false;

-    // auto _valid_file_extension = std::regex_search(
-    //    _fname, std::regex{ "\\.(a|c|f|o|cc|so|cxx|cpp|C|F|CC|f90|F90|so\\.[0-9\\.]+)$",
-    //                        regex_opts });
-
-    auto _valid_file_regex = process_file_for_instrumentation(_fname);
-
-    // if module compiled from C, C++, or Fortran or a library
-    // if(_valid_file_extension && _valid_file_regex)
-    //    return false;
-
-    // apply regex expressions
-    if(_valid_file_regex)
+    if(instrument_module(_fname))
        return false;

    // do not instrument
@@ -2207,7 +2203,7 @@ get_absolute_exe_filepath(std::string exe_name)
            if(file_exists(TIMEMORY_JOIN('/', pitr, exe_name)))
            {
                exe_name = TIMEMORY_JOIN('/', pitr, exe_name);
-                verbprintf(0, "Resolved '%s' to '%s'...\n", _exe_orig.c_str(),
+                verbprintf(0, "[hosttrace] Resolved '%s' to '%s'...\n", _exe_orig.c_str(),
                           exe_name.c_str());
                break;
            }
@@ -2242,7 +2238,7 @@ get_absolute_lib_filepath(std::string lib_name)
            if(file_exists(TIMEMORY_JOIN('/', pitr, lib_name)))
            {
                lib_name = TIMEMORY_JOIN('/', pitr, lib_name);
-                verbprintf(0, "Resolved '%s' to '%s'...\n", _lib_orig.c_str(),
+                verbprintf(0, "[hosttrace] Resolved '%s' to '%s'...\n", _lib_orig.c_str(),
                           lib_name.c_str());
                break;
            }
@@ -167,8 +167,10 @@ get_functors()
 bool
 hosttrace_init_tooling()
 {
-    if(get_state() != State::PreInit)
+    static bool _once = false;
+    if(get_state() != State::PreInit || _once)
        return false;
+    _once = true;

    HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);

@@ -466,6 +468,11 @@ extern "C"

        get_state() = State::Finalized;

+#if defined(HOSTTRACE_USE_ROCTRACER)
+        // ensure that threads running roctracer callbacks shutdown
+        comp::roctracer::tear_down();
+#endif
+
        // stop the main bundle and report the high-level metrics
        if(get_main_bundle())
        {
@@ -506,6 +513,7 @@ extern "C"
            }
        }

+        bool _perfetto_output_error = false;
        if(get_use_perfetto() && !is_system_backend())
        {
            // Make sure the last event is closed for this example.
@@ -535,14 +543,20 @@ extern "C"
            std::ofstream output{};
            output.open(get_perfetto_output_filename(), std::ios::out | std::ios::binary);
            if(!output)
+            {
                fprintf(stderr, "[%s]> Error opening '%s'...\n", __FUNCTION__,
                        get_perfetto_output_filename().c_str());
+                _perfetto_output_error = true;
+            }
            else
                output.write(&trace_data[0], trace_data.size());
            output.close();
        }

        tim::timemory_finalize();
+
+        if(_perfetto_output_error)
+            throw std::runtime_error("Unable to create perfetto output file");
    }

    void hosttrace_trace_set_env(const char* env_name, const char* env_val)
@@ -145,6 +145,14 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
                    TRACE_EVENT_END("device", end_timestamp);
                }

+                /*if(get_use_timemory())
+                {
+                    static auto _scope = scope::flat() + scope::timeline();
+                    roctracer_bundle_t{ _name, _scope }
+                        .start()
+                        .store(end_timestamp - hsa_begin_timestamp)
+                        .stop();
+                }*/
                // timemory is disabled in this callback because collecting data in this
                // thread causes strange segmentation faults
            }
@@ -341,6 +349,22 @@ roctracer_is_setup()
    static bool _v = false;
    return _v;
 }
+
+using roctracer_functions_t = std::vector<std::pair<std::string, std::function<void()>>>;
+
+auto&
+roctracer_setup_routines()
+{
+    static auto _v = roctracer_functions_t{};
+    return _v;
+}
+
+auto&
+roctracer_tear_down_routines()
+{
+    static auto _v = roctracer_functions_t{};
+    return _v;
+}
 }  // namespace

 #if !defined(HOSTTRACE_ROCTRACER_LIBKFDWRAPPER)
@@ -391,6 +415,52 @@ roctracer::preinit()
    roctracer_data::description() = "ROCm tracer (activity API)";
 }

+bool
+roctracer::is_setup()
+{
+    return roctracer_is_setup();
+}
+
+void
+roctracer::add_setup(const std::string& _lbl, std::function<void()>&& _func)
+{
+    roctracer_setup_routines().emplace_back(_lbl, std::move(_func));
+}
+
+void
+roctracer::add_tear_down(const std::string& _lbl, std::function<void()>&& _func)
+{
+    roctracer_tear_down_routines().emplace_back(_lbl, std::move(_func));
+}
+
+void
+roctracer::remove_setup(const std::string& _lbl)
+{
+    auto& _data = roctracer_setup_routines();
+    for(auto itr = _data.begin(); itr != _data.end(); ++itr)
+    {
+        if(itr->first == _lbl)
+        {
+            _data.erase(itr);
+            break;
+        }
+    }
+}
+
+void
+roctracer::remove_tear_down(const std::string& _lbl)
+{
+    auto& _data = roctracer_setup_routines();
+    for(auto itr = _data.begin(); itr != _data.end(); ++itr)
+    {
+        if(itr->first == _lbl)
+        {
+            _data.erase(itr);
+            break;
+        }
+    }
+}
+
 void
 roctracer::setup()
 {
@@ -408,17 +478,25 @@ roctracer::setup()
    auto _kfdwrapper = dynamic_library{ "HOSTTRACE_ROCTRACER_LIBKFDWRAPPER",
                                        HOSTTRACE_ROCTRACER_LIBKFDWRAPPER };

-    // Allocating tracing pool
-    roctracer_properties_t properties{};
-    properties.buffer_size         = 0x1000;
-    properties.buffer_callback_fun = hip_activity_callback;
-    ROCTRACER_CALL(roctracer_open_pool(&properties));
    ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));

+    if(roctracer_default_pool() == nullptr)
+    {
+        // Allocating tracing pool
+        roctracer_properties_t properties{};
+        properties.buffer_size         = 0x1000;
+        properties.buffer_callback_fun = hip_activity_callback;
+        ROCTRACER_CALL(roctracer_open_pool(&properties));
+    }
+
    // Enable API callbacks, all domains
    ROCTRACER_CALL(roctracer_enable_callback(hip_api_callback, nullptr));
    // Enable activity tracing, all domains
    ROCTRACER_CALL(roctracer_enable_activity());
+
+    // callback for HSA
+    for(auto& itr : roctracer_setup_routines())
+        itr.second();
 }

 void
@@ -431,22 +509,21 @@ roctracer::tear_down()
    HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);

    // flush all the activity
-    ROCTRACER_CALL(roctracer_flush_activity());
-
+    if(roctracer_default_pool() != nullptr)
+    {
+        ROCTRACER_CALL(roctracer_flush_activity());
+    }
    // flush all buffers
    roctracer_flush_buf();

-    ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
-
-    ROCTRACER_CALL(
-        roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+    // callback for hsa
+    for(auto& itr : roctracer_tear_down_routines())
+        itr.second();

    // Disable tracing and closing the pool
    ROCTRACER_CALL(roctracer_disable_callback());
    ROCTRACER_CALL(roctracer_disable_activity());
-
-    // closing the pool with HSA enabled causes segfaults
-    // ROCTRACER_CALL(roctracer_close_pool());
+    ROCTRACER_CALL(roctracer_close_pool());
 }

 void
@@ -459,8 +536,6 @@ roctracer::start()
 void
 roctracer::stop()
 {
-    // flush all the activity
-    ROCTRACER_CALL(roctracer_flush_activity());
    if(tracker_type::stop() == 0)
        tear_down();
 }
@@ -471,84 +546,112 @@ TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer, false, void)
 TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer_data, true, double)

 // HSA-runtime tool on-load method
-extern "C" TIMEMORY_VISIBILITY("default") bool OnLoad(
-    HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
-    const char* const* failed_tool_names)
+extern "C"
 {
-    puts(__FUNCTION__);
-    tim::consume_parameters(table, runtime_version, failed_tool_count, failed_tool_names);
+    TIMEMORY_VISIBILITY("default")
+    bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                const char* const* failed_tool_names);
+    TIMEMORY_VISIBILITY("default") void OnUnload();

-    // ONLOAD_TRACE_BEG();
-    // on_exit(exit_handler, nullptr);
-
-    get_hsa_timer() = std::make_unique<hsa_timer_t>(table->core_->hsa_system_get_info_fn);
-
-    // const char* output_prefix = getenv("ROCP_OUTPUT_DIR");
-    const char* output_prefix = nullptr;
-
-    // App begin timestamp begin_ts_file.txt
-    // begin_ts_file_handle = open_output_file(output_prefix, "begin_ts_file.txt");
-    // const timestamp_t app_start_time = timer->timestamp_fn_ns();
-    // fprintf(begin_ts_file_handle, "%lu\n", app_start_time);
-
-    bool trace_hsa_api = tim::get_env("HOSTTRACE_ROCTRACER_HSA_API", true);
-    std::vector<std::string> hsa_api_vec =
-        tim::delimit(tim::get_env<std::string>("HOSTTRACE_ROCTRACER_HSA_API_TYPES", ""));
-
-    // Enable HSA API callbacks/activity
-    if(trace_hsa_api)
+    bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                const char* const* failed_tool_names)
    {
-        // hsa_api_file_handle = open_output_file(output_prefix, "hsa_api_trace.txt");
+        puts(__FUNCTION__);
+        tim::consume_parameters(table, runtime_version, failed_tool_count,
+                                failed_tool_names);

-        // initialize HSA tracing
-        roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
+        // ONLOAD_TRACE_BEG();
+        // on_exit(exit_handler, nullptr);

-        fprintf(stdout, "    HSA-trace(");
-        fflush(stdout);
-        if(!hsa_api_vec.empty())
-        {
-            for(unsigned i = 0; i < hsa_api_vec.size(); ++i)
+        auto _setup = [=]() {
+            get_hsa_timer() =
+                std::make_unique<hsa_timer_t>(table->core_->hsa_system_get_info_fn);
+
+            // const char* output_prefix = getenv("ROCP_OUTPUT_DIR");
+            const char* output_prefix = nullptr;
+
+            // App begin timestamp begin_ts_file.txt
+            // begin_ts_file_handle = open_output_file(output_prefix,
+            // "begin_ts_file.txt"); const timestamp_t app_start_time =
+            // timer->timestamp_fn_ns(); fprintf(begin_ts_file_handle, "%lu\n",
+            // app_start_time);
+
+            bool trace_hsa_api = tim::get_env("HOSTTRACE_ROCTRACER_HSA_API", true);
+            std::vector<std::string> hsa_api_vec = tim::delimit(
+                tim::get_env<std::string>("HOSTTRACE_ROCTRACER_HSA_API_TYPES", ""));
+
+            // Enable HSA API callbacks/activity
+            if(trace_hsa_api)
            {
-                uint32_t    cid = HSA_API_ID_NUMBER;
-                const char* api = hsa_api_vec[i].c_str();
-                ROCTRACER_CALL(
-                    roctracer_op_code(ACTIVITY_DOMAIN_HSA_API, api, &cid, nullptr));
-                ROCTRACER_CALL(roctracer_enable_op_callback(ACTIVITY_DOMAIN_HSA_API, cid,
-                                                            hsa_api_callback, nullptr));
-                printf(" %s", api);
+                // hsa_api_file_handle = open_output_file(output_prefix,
+                // "hsa_api_trace.txt");
+
+                // initialize HSA tracing
+                roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
+
+                fprintf(stdout, "    HSA-trace(");
+                fflush(stdout);
+                if(!hsa_api_vec.empty())
+                {
+                    for(unsigned i = 0; i < hsa_api_vec.size(); ++i)
+                    {
+                        uint32_t    cid = HSA_API_ID_NUMBER;
+                        const char* api = hsa_api_vec[i].c_str();
+                        ROCTRACER_CALL(roctracer_op_code(ACTIVITY_DOMAIN_HSA_API, api,
+                                                         &cid, nullptr));
+                        ROCTRACER_CALL(roctracer_enable_op_callback(
+                            ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));
+                        printf(" %s", api);
+                    }
+                }
+                else
+                {
+                    ROCTRACER_CALL(roctracer_enable_domain_callback(
+                        ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
+                }
+                printf(")\n");
            }
-        }
-        else
-        {
-            ROCTRACER_CALL(roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HSA_API,
-                                                            hsa_api_callback, nullptr));
-        }
-        printf(")\n");
-    }

-    bool trace_hsa_activity = tim::get_env("HOSTTRACE_ROCTRACER_HSA_ACTIVITY", true);
-    // Enable HSA GPU activity
-    if(trace_hsa_activity)
-    {
-        // initialize HSA tracing
-        roctracer::hsa_ops_properties_t ops_properties{
-            table, reinterpret_cast<activity_async_callback_t>(hsa_activity_callback),
-            nullptr, output_prefix
+            bool trace_hsa_activity =
+                tim::get_env("HOSTTRACE_ROCTRACER_HSA_ACTIVITY", true);
+            // Enable HSA GPU activity
+            if(trace_hsa_activity)
+            {
+                // initialize HSA tracing
+                roctracer::hsa_ops_properties_t ops_properties{
+                    table,
+                    reinterpret_cast<activity_async_callback_t>(hsa_activity_callback),
+                    nullptr, output_prefix
+                };
+                roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);
+
+                fprintf(stdout, "    HSA-activity-trace()\n");
+                fflush(stdout);
+                ROCTRACER_CALL(roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS,
+                                                            HSA_OP_ID_COPY));
+            }
        };
-        roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);

-        fprintf(stdout, "    HSA-activity-trace()\n");
-        fflush(stdout);
-        ROCTRACER_CALL(
-            roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+        auto _tear_down = []() {
+            ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
+
+            ROCTRACER_CALL(
+                roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+        };
+
+        if(comp::roctracer::is_setup())
+            _setup();
+
+        comp::roctracer::add_setup("hsa", std::move(_setup));
+        comp::roctracer::add_tear_down("hsa", std::move(_tear_down));
+
+        return true;
    }

-    return true;
-}
-
-// HSA-runtime on-unload method
-extern "C" TIMEMORY_VISIBILITY("default") void OnUnload()
-{
-    puts(__FUNCTION__);
-    // ONLOAD_TRACE("");
+    // HSA-runtime on-unload method
+    void OnUnload()
+    {
+        puts(__FUNCTION__);
+        // ONLOAD_TRACE("");
+    }
 }
@@ -1,21 +1,33 @@

+if(NOT DYNINST_API_RT_DIR AND DYNINST_API_RT)
+    get_filename_component(DYNINST_API_RT_DIR "${DYNINST_API_RT}" DIRECTORY)
+endif()
+
+if(HOSTTRACE_BUILD_DYNINST)
+    set(DYNINST_API_RT_DIR "${PROJECT_BINARY_DIR}/external/dyninst/dyninstAPI_RT:${PROJECT_BINARY_DIR}/external/dyninst/dyninstAPI")
+endif()
+
+set(_test_environment
+    "HOSTTRACE_USE_PERFETTO=ON"
+    "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}")
+
 if(TARGET transpose)
    add_test(
        NAME transpose-binary-rewrite
-        COMMAND $<TARGET_FILE:hosttrace-exe> -o transpose.inst -- ${CMAKE_BINARY_DIR}/transpose/transpose
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests
+        COMMAND $<TARGET_FILE:hosttrace-exe> -o $<TARGET_FILE_DIR:transpose>/transpose.inst -v 2 -- $<TARGET_FILE:transpose>
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
    )

    add_test(
        NAME transpose-binary-rewrite-run
-        COMMAND ./transpose.inst
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests
+        COMMAND $<TARGET_FILE_DIR:transpose>/transpose.inst
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
    )

    add_test(
-        NAME transpose-runtime-instrumentation
-        COMMAND $<TARGET_FILE:hosttrace-exe> -- ${CMAKE_BINARY_DIR}/transpose/transpose
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/tests
+        NAME transpose-runtime-instrument
+        COMMAND $<TARGET_FILE:hosttrace-exe> -v 2 -- $<TARGET_FILE:transpose>
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
    )

    set_tests_properties(
@@ -24,13 +36,47 @@ if(TARGET transpose)
            DEPENDS transpose-binary-rewrite
    )

-    get_filename_component(DYNINST_LIB_DIR "${DYNINST_API_RT}" DIRECTORY CACHE)
    set_tests_properties(
        transpose-binary-rewrite
        transpose-binary-rewrite-run
-        transpose-runtime-instrumentation
+        transpose-runtime-instrument
        PROPERTIES
-            ENVIRONMENT "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}:${DYNINST_LIB_DIR}:$ENV{LD_LIBRARY_PATH}"
+            ENVIRONMENT "${_test_environment}"
+            TIMEOUT 600
+    )
+endif()
+
+if(TARGET parallel-overhead)
+    add_test(
+        NAME parallel-overhead-binary-rewrite
+        COMMAND $<TARGET_FILE:hosttrace-exe> -o $<TARGET_FILE_DIR:parallel-overhead>/parallel-overhead.inst -v 2 -- $<TARGET_FILE:parallel-overhead>
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    )
+
+    add_test(
+        NAME parallel-overhead-binary-rewrite-run
+        COMMAND $<TARGET_FILE_DIR:parallel-overhead>/parallel-overhead.inst
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    )
+
+    add_test(
+        NAME parallel-overhead-runtime-instrument
+        COMMAND $<TARGET_FILE:hosttrace-exe> -v 2 -- $<TARGET_FILE:parallel-overhead>
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    )
+
+    set_tests_properties(
+        parallel-overhead-binary-rewrite-run
+        PROPERTIES
+            DEPENDS parallel-overhead-binary-rewrite
+    )
+
+    set_tests_properties(
+        parallel-overhead-binary-rewrite
+        parallel-overhead-binary-rewrite-run
+        parallel-overhead-runtime-instrument
+        PROPERTIES
+            ENVIRONMENT "${_test_environment}"
            TIMEOUT 600
    )
 endif()