CI and testing updates (#203)

* Python implementation of run-ci.sh * Container workflow update - retry failed container build to combat network failures * cpack workflow update - retry failed base container build to combat network failures * General CI workflow updates - retry failed "Install packages" step to combat network failures * Miscellanous linting fixes * Formatting workflow update - improve regex for source formatting * format user.h * Add new omnitrace-avail tests * Make run-ci.py executable * workflow retry fix - timeout_seconds -> retry_wait_seconds * Fix cmake formatting glob * source formatting * Handle PRs in run-ci.py * Specify timeout_minutes in retry steps * Remove remaining --cmake-args from workflows * CI warnings about using MPICH headers * Remove text=True from run-ci.py - not capturing stdout/sterr so unnecessary * Fix OpenSUSE step label * Update omnitrace-avail-write-config tests - use TWD (Test Working Directory) instead of PWD since PWD might not be build directory * paths-ignore + workflow_dispatch
2022-11-13 10:42:14 -06:00
@@ -3,6 +3,7 @@ run-name: ci-containers

 # nightly build
 on:
+  workflow_dispatch:
  schedule:
    - cron: 0 5 * * *

@@ -54,12 +55,16 @@ jobs:
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build CI Container
-        if: ${{ matrix.rocm-version == 0 }}
        timeout-minutes: 45
-        run: |
-          pushd docker
-          ./build-docker-ci.sh --distro ${{ matrix.distro }} --versions ${{ matrix.version }} --user ${{ secrets.DOCKERHUB_USERNAME }} --push --jobs 2 --elfutils-version 0.186 --boost-version 1.79.0
-          popd
+        uses: nick-fields/retry@v2
+        with:
+          retry_wait_seconds: 60
+          timeout_minutes: 45
+          max_attempts: 3
+          command: |
+            pushd docker
+            ./build-docker-ci.sh --distro ${{ matrix.distro }} --versions ${{ matrix.version }} --user ${{ secrets.DOCKERHUB_USERNAME }} --push --jobs 2 --elfutils-version 0.186 --boost-version 1.79.0
+            popd

  omnitrace-release:
    if: github.repository == 'AMDResearch/omnitrace'
@@ -165,7 +170,12 @@ jobs:

      - name: Build Base Container
        timeout-minutes: 45
-        run: |
-          pushd docker
-          ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }} --user ${{ secrets.DOCKERHUB_USERNAME }} --push
-          popd
+        uses: nick-fields/retry@v2
+        with:
+          retry_wait_seconds: 60
+          timeout_minutes: 45
+          max_attempts: 3
+          command: |
+            pushd docker
+            ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }} --user ${{ secrets.DOCKERHUB_USERNAME }} --push
+            popd
@@ -2,9 +2,14 @@ name: Installer Packaging (CPack)
 run-name: cpack

 on:
+  workflow_dispatch:
  push:
    branches: [main, releases/**]
-    tags: []
+    tags:
+      - v*
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -15,7 +20,7 @@ env:

 jobs:
  release:
-    if: github.repository == 'AMDResearch/omnitrace'
+    if: github.repository == 'AMDResearch/omnitrace' && startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-20.04
    permissions:
      contents: write
@@ -25,11 +30,36 @@ jobs:
        uses: actions/checkout@v3
      - name: Generate Release
        uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
        with:
          draft: True
          generate_release_notes: True

+  release-docs:
+    if: github.repository == 'AMDResearch/omnitrace' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: gh-pages
+      - name: Update Docs
+        uses: s-weigand/setup-conda@v1
+        working-directory: source/docs
+        run: |
+          conda env create -n omnitrace-docs -f environment.yml
+          conda activate omnitrace-docs
+          git config --global user.name 'Jonathan R. Madsen'
+          git config --global user.email 'jrmadsen@users.noreply.github.com'
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
+          git pull --rebase=true origin ${{ github.ref }}
+          ./update-docs.sh
+          git add ../../docs
+          git commit -m "Update HTML (automated)"
+          git push -f
+
  ubuntu:
    runs-on: ubuntu-20.04
    strategy:
@@ -131,10 +161,15 @@ jobs:

      - name: Build Base Container
        timeout-minutes: 20
-        run: |
-          pushd docker
-          ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }}
-          popd
+        uses: nick-fields/retry@v2
+        with:
+          retry_wait_seconds: 120
+          timeout_minutes: 20
+          max_attempts: 5
+          command: |
+            pushd docker
+            ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }}
+            popd

      - name: Build Release
        timeout-minutes: 150
@@ -5,8 +5,14 @@ run-name: formatting
 on:
  push:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'
  pull_request:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -75,7 +81,7 @@ jobs:
    - name: clang-format
      run: |
        set +e
-        FILES=$(find source examples -type f | egrep '\.hpp$|\.cpp$|\.cpp\.in$')
+        FILES=$(find source examples tests -type f | egrep '\.(h|hpp|c|cpp)(|\.in)$')
        FORMAT_OUT=$(clang-format-11 -output-replacements-xml ${FILES})
        RET=$(echo ${FORMAT_OUT} | grep -c '<replacement ')
        if [ "${RET}" -ne 0 ]; then
@@ -4,8 +4,14 @@ run-name: opensuse-15
 on:
  push:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'
  pull_request:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -32,8 +38,13 @@ jobs:

    - name: Install Packages
      timeout-minutes: 10
-      run:
-        for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        command: |
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

    - name: Configure Env
      run:
@@ -42,20 +53,20 @@ jobs:
        echo "/opt/omnitrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
        echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV

-    - name: Configure CMake
+    - name: Configure, Build, and Test
      timeout-minutes: 115
      shell: bash
      run:
        git config --global --add safe.directory ${PWD} &&
        cmake --version &&
-        ./scripts/run-ci.sh -B build
+        python3 ./scripts/run-ci.py -B build
          --name ${{ github.repository_owner }}-${{ github.ref_name }}-opensuse-${{ matrix.os-release }}-${{ matrix.compiler }}-nompi-python
          --build-jobs 2
          --site GitHub
-          --cmake-args
+          --
          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DCMAKE_INSTALL_PREFIX=/opt/omnitrace
          -DOMNITRACE_BUILD_TESTING=ON
          -DOMNITRACE_USE_MPI=OFF
@@ -4,8 +4,14 @@ run-name: ubuntu-bionic
 on:
  push:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'
  pull_request:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -46,17 +52,22 @@ jobs:

    - name: Install Packages
      timeout-minutes: 10
-      run:
-        apt-get update &&
-        apt-get upgrade -y &&
-        apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} &&
-        if [ "${{ matrix.mpi }}" = "mpich" ]; then apt-get install -y libmpich-dev mpich; fi &&
-        if [ "${{ matrix.mpi }}" = "openmpi" ]; then apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
-        python3 -m pip install --upgrade pip &&
-        python3 -m pip install numpy &&
-        python3 -m pip install perfetto &&
-        python3 -m pip install 'cmake==3.16.3' &&
-        for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        command: |
+          apt-get update &&
+          apt-get upgrade -y &&
+          apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} &&
+          if [ "${{ matrix.mpi }}" = "mpich" ]; then apt-get install -y libmpich-dev mpich; fi &&
+          if [ "${{ matrix.mpi }}" = "openmpi" ]; then apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy &&
+          python3 -m pip install perfetto &&
+          python3 -m pip install 'cmake==3.16.3' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

    - name: Install Kokkos
      timeout-minutes: 10
@@ -82,11 +93,11 @@ jobs:
        cmake --version &&
        if [ "${{ matrix.mpi }}" = "nompi" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
        append-tagname() { if [ "${1}" == "ON" ]; then TAG="${TAG}-${2}"; fi; } &&
-        ./scripts/run-ci.sh -B build
+        python3 ./scripts/run-ci.py -B build
          --name ${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-bionic-${{ matrix.compiler }}-${{ matrix.mpi }}-python
          --build-jobs 2
          --site GitHub
-          --cmake-args
+          --
          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
@@ -4,8 +4,14 @@ run-name: ubuntu-focal
 on:
  push:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'
  pull_request:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -62,18 +68,23 @@ jobs:

    - name: Install Packages
      timeout-minutes: 10
-      run:
-        apt-get update &&
-        apt-get install -y software-properties-common &&
-        add-apt-repository -y ppa:ubuntu-toolchain-r/test &&
-        apt-get update &&
-        apt-get upgrade -y &&
-        apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libmpich-dev mpich environment-modules ${{ matrix.compiler }} &&
-        python3 -m pip install --upgrade pip &&
-        python3 -m pip install numpy &&
-        python3 -m pip install perfetto &&
-        python3 -m pip install 'cmake==3.16.3' &&
-        for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        command: |
+          apt-get update &&
+          apt-get install -y software-properties-common &&
+          add-apt-repository -y ppa:ubuntu-toolchain-r/test &&
+          apt-get update &&
+          apt-get upgrade -y &&
+          apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libmpich-dev mpich environment-modules ${{ matrix.compiler }} &&
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy &&
+          python3 -m pip install perfetto &&
+          python3 -m pip install 'cmake==3.16.3' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

    - name: Test Environment Modules
      timeout-minutes: 15
@@ -102,11 +113,11 @@ jobs:
        append-tagname ${{ matrix.static-libgcc }} libgcc &&
        append-tagname ${{ matrix.static-libstdcxx }} libstdcxx &&
        cmake --version &&
-        ./scripts/run-ci.sh -B build
+        python3 ./scripts/run-ci.py -B build
          --name ${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-focal-${{ matrix.compiler }}${TAG}
          --build-jobs 2
          --site GitHub
-          --cmake-args
+          --
          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
@@ -127,6 +138,7 @@ jobs:
          -DOMNITRACE_MAX_THREADS=32
          -DOMNITRACE_DISABLE_EXAMPLES="transpose;rccl"
          -DOMNITRACE_BUILD_NUMBER=${{ github.run_attempt }}
+          -DMPI_HEADERS_ALLOW_MPICH=OFF

    - name: Test Build-Tree Module
      timeout-minutes: 45
@@ -233,16 +245,21 @@ jobs:

    - name: Install Packages
      timeout-minutes: 10
-      run:
-        apt-get update &&
-        apt-get install -y software-properties-common wget gnupg2 &&
-        wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - &&
-        echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
-        apt-get update &&
-        apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
-        python3 -m pip install --upgrade pip &&
-        python3 -m pip install 'cmake==3.21.4' &&
-        for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        command: |
+          apt-get update &&
+          apt-get install -y software-properties-common wget gnupg2 &&
+          wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - &&
+          echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
+          apt-get update &&
+          apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install 'cmake==3.21.4' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

    - name: Install RCCL
      if: ${{ matrix.rocm-version != '4.3' }}
@@ -283,11 +300,11 @@ jobs:
        cmake --version &&
        TAG="-rocm-${{ matrix.rocm-version }}" &&
        TAG="$(echo ${TAG} | sed 's/debian/latest/g')" &&
-        ./scripts/run-ci.sh -B build
+        python3 ./scripts/run-ci.py -B build
          --name ${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-focal-rocm-${{ matrix.compiler }}${TAG}
          --build-jobs 2
          --site GitHub
-          --cmake-args
+          --
          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
@@ -311,7 +328,7 @@ jobs:
          -DOMNITRACE_CI_GPU=OFF
          -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF
          -DOMNITRACE_BUILD_NUMBER=${{ github.run_attempt }}
-          --ctest-args
+          --
          ${{ matrix.ctest-exclude }}

    - name: Install
@@ -392,15 +409,20 @@ jobs:

    - name: Install Packages
      timeout-minutes: 10
-      run:
-        sudo apt-get update &&
-        sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} &&
-        if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
-        if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
-        python3 -m pip install --upgrade pip &&
-        python3 -m pip install numpy &&
-        python3 -m pip install perfetto &&
-        python3 -m pip install 'cmake==3.16.3'
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        command: |
+          sudo apt-get update &&
+          sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} &&
+          if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
+          if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy &&
+          python3 -m pip install perfetto &&
+          python3 -m pip install 'cmake==3.16.3'

    - name: Test Environment Modules
      timeout-minutes: 15
@@ -422,11 +444,11 @@ jobs:
        cmake --version &&
        if [ "${{ matrix.mpi }}" = "nompi" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
        if [ -z "${{ matrix.deps }}" ]; then TAG=""; else TAG="-external-deps"; fi &&
-        ./scripts/run-ci.sh -B ${{ github.workspace }}/build
+        python3 ./scripts/run-ci.py -B ${{ github.workspace }}/build
          --name ${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-focal-${{ matrix.compiler }}-${{ matrix.mpi }}${TAG}
          --build-jobs 2
          --site GitHub
-          --cmake-args
+          --
          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
@@ -448,6 +470,7 @@ jobs:
          -DOMNITRACE_MAX_THREADS=32
          -DOMNITRACE_DISABLE_EXAMPLES="transpose;rccl"
          -DOMNITRACE_BUILD_NUMBER=${{ github.run_attempt }}
+          -DMPI_HEADERS_ALLOW_MPICH=ON

    - name: Test Build-Tree Env
      timeout-minutes: 60
@@ -533,19 +556,22 @@ jobs:

    - name: Install Packages
      timeout-minutes: 10
-      run:
-        apt-get update &&
-        apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules gcc g++ mpich libmpich-dev texinfo &&
-        python3 -m pip install --upgrade pip &&
-        python3 -m pip install numpy &&
-        python3 -m pip install perfetto &&
-        python3 -m pip install 'cmake==3.24.1' &&
-        for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        command: |
+          apt-get update &&
+          apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules gcc g++ mpich libmpich-dev texinfo &&
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy &&
+          python3 -m pip install perfetto &&
+          python3 -m pip install 'cmake==3.24.1' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

    - name: Configure Env
      run:
-        echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
-        echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
        echo "${HOME}/.local/bin" >> $GITHUB_PATH

    - name: Configure, Build, and Test
@@ -554,12 +580,12 @@ jobs:
      run:
        git config --global --add safe.directory ${PWD} &&
        cmake --version &&
-        ./scripts/run-ci.sh -B build
+        python3 ./scripts/run-ci.py -B build
          --name ${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-focal-codecov-mpi-python-ompt-papi
          --build-jobs 2
          --site GitHub
          --coverage
-          --cmake-args
+          --
          -DCMAKE_INSTALL_PREFIX=/opt/omnitrace
          -DOMNITRACE_BUILD_CI=OFF
          -DOMNITRACE_BUILD_TESTING=ON
@@ -4,8 +4,14 @@ run-name: ubuntu-jammy
 on:
  push:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'
  pull_request:
    branches: [ main, develop ]
+    paths-ignore:
+      - '*.md'
+      - 'source/docs/**'

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -65,29 +71,39 @@ jobs:

    - name: Install Packages
      timeout-minutes: 10
-      run:
-        apt-get update &&
-        apt-get install -y software-properties-common &&
-        apt-get upgrade -y &&
-        apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
-        python3 -m pip install --upgrade pip &&
-        python3 -m pip install numpy &&
-        python3 -m pip install 'cmake==3.21.4' &&
-        for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        command: |
+          apt-get update &&
+          apt-get install -y software-properties-common &&
+          apt-get upgrade -y &&
+          apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy &&
+          python3 -m pip install 'cmake==3.21.4' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

    - name: Install ROCm Packages
      timeout-minutes: 10
      if: ${{ matrix.rocm-version != '0.0' }}
-      shell: bash
-      run: |
-        wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
-        echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ jammy main" | tee /etc/apt/sources.list.d/rocm.list
-        apt-get update
-        apt-get install -y {rocm-hip-sdk,roctracer-dev,rocprofiler-dev,rocm-smi-lib,rocminfo}${{ matrix.rocm-version }}.0
-        echo "/opt/rocm/bin" >> $GITHUB_PATH
-        echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV
-        echo "LD_LIBRARY_PATH=/opt/rocm/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-        /opt/rocm/bin/hipcc -O3 -c ./examples/transpose/transpose.cpp -o /tmp/transpose.o
+      uses: nick-fields/retry@v2
+      with:
+        retry_wait_seconds: 30
+        timeout_minutes: 10
+        max_attempts: 5
+        shell: bash
+        command: |
+          wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -
+          echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ jammy main" | tee /etc/apt/sources.list.d/rocm.list
+          apt-get update
+          apt-get install -y {rocm-hip-sdk,roctracer-dev,rocprofiler-dev,rocm-smi-lib,rocminfo}${{ matrix.rocm-version }}.0
+          echo "/opt/rocm/bin" >> $GITHUB_PATH
+          echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=/opt/rocm/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+          /opt/rocm/bin/hipcc -O3 -c ./examples/transpose/transpose.cpp -o /tmp/transpose.o

    - name: Test Environment Modules
      timeout-minutes: 15
@@ -122,11 +138,11 @@ jobs:
        append-tagname ${{ matrix.hidden }} hidden-viz &&
        append-tagname ${{ matrix.static-libgcc }} libgcc &&
        append-tagname ${{ matrix.static-libstdcxx }} libstdcxx &&
-        ./scripts/run-ci.sh -B build
+        python3 ./scripts/run-ci.py -B build
          --name ${{ github.repository_owner }}-${{ github.ref_name }}-ubuntu-jammy-${{ matrix.compiler }}${TAG}
          --build-jobs 2
          --site GitHub
-          --cmake-args
+          --
          -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
          -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
@@ -56,12 +56,14 @@ endforeach()
 if(OMNITRACE_CLANG_FORMAT_EXE
   OR OMNITRACE_BLACK_FORMAT_EXE
   OR OMNITRACE_CMAKE_FORMAT_EXE)
-    file(GLOB_RECURSE sources ${PROJECT_SOURCE_DIR}/source/*.cpp)
+    file(GLOB_RECURSE sources ${PROJECT_SOURCE_DIR}/source/*.cpp
+         ${PROJECT_SOURCE_DIR}/source/*.c)
    file(GLOB_RECURSE headers ${PROJECT_SOURCE_DIR}/source/*.hpp
         ${PROJECT_SOURCE_DIR}/source/*.hpp.in ${PROJECT_SOURCE_DIR}/source/*.h
         ${PROJECT_SOURCE_DIR}/source/*.h.in)
    file(GLOB_RECURSE examples ${PROJECT_SOURCE_DIR}/examples/*.cpp
-         ${PROJECT_SOURCE_DIR}/examples/*.c ${PROJECT_SOURCE_DIR}/examples/*.hpp)
+         ${PROJECT_SOURCE_DIR}/examples/*.c ${PROJECT_SOURCE_DIR}/examples/*.hpp
+         ${PROJECT_SOURCE_DIR}/examples/*.h)
    file(GLOB_RECURSE external ${PROJECT_SOURCE_DIR}/examples/lulesh/external/kokkos/*)
    file(
        GLOB_RECURSE
@@ -234,6 +234,9 @@ endif()
 #
 # ----------------------------------------------------------------------------------------#

+# suppress warning during CI that MPI_HEADERS_ALLOW_MPICH was unused
+set(_OMNITRACE_MPI_HEADERS_ALLOW_MPICH ${MPI_HEADERS_ALLOW_MPICH})
+
 if(OMNITRACE_USE_MPI)
    find_package(MPI ${omnitrace_FIND_QUIETLY} REQUIRED)
    target_link_libraries(omnitrace-mpi INTERFACE MPI::MPI_C MPI::MPI_CXX)
@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+
+
+import os
+import re
+import sys
+import socket
+import shutil
+import argparse
+import multiprocessing
+
+
+def which(cmd, require):
+    v = shutil.which(cmd)
+    if require and v is None:
+        raise RuntimeError(f"{cmd} not found")
+    return v if v is not None else ""
+
+
+def generate_custom(args, cmake_args, ctest_args):
+
+    if not os.path.exists(args.binary_dir):
+        os.makedirs(args.binary_dir)
+
+    NAME = args.name
+    SITE = args.site
+    BUILD_JOBS = args.build_jobs
+    SUBMIT_URL = args.submit_url
+    SOURCE_DIR = os.path.realpath(args.source_dir)
+    BINARY_DIR = os.path.realpath(args.binary_dir)
+    CMAKE_ARGS = " ".join(cmake_args)
+    CTEST_ARGS = " ".join(ctest_args)
+
+    GIT_CMD = which("git", require=True)
+    GCOV_CMD = which("gcov", require=False)
+    CMAKE_CMD = which("cmake", require=True)
+    CTEST_CMD = which("ctest", require=True)
+
+    NAME = re.sub(r"(.*)-([0-9]+)/merge", "PR_\\2_\\1", NAME)
+
+    return f"""
+        set(CTEST_PROJECT_NAME "Omnitrace")
+        set(CTEST_NIGHTLY_START_TIME "05:00:00 UTC")
+
+        set(CTEST_DROP_METHOD "http")
+        set(CTEST_DROP_SITE_CDASH TRUE)
+        set(CTEST_SUBMIT_URL "https://{SUBMIT_URL}")
+
+        set(CTEST_UPDATE_TYPE git)
+        set(CTEST_UPDATE_VERSION_ONLY TRUE)
+        set(CTEST_GIT_INIT_SUBMODULES TRUE)
+
+        set(CTEST_OUTPUT_ON_FAILURE TRUE)
+        set(CTEST_USE_LAUNCHERS TRUE)
+        set(CMAKE_CTEST_ARGUMENTS --output-on-failure {CTEST_ARGS})
+
+        set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS "100")
+        set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "100")
+        set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "51200")
+        set(CTEST_CUSTOM_COVERAGE_EXCLUDE "/usr/.*;.*external/.*;.*examples/.*")
+
+        set(CTEST_SITE "{SITE}")
+        set(CTEST_BUILD_NAME "{NAME}")
+
+        set(CTEST_SOURCE_DIRECTORY {SOURCE_DIR})
+        set(CTEST_BINARY_DIRECTORY {BINARY_DIR})
+
+        set(CTEST_UPDATE_COMMAND {GIT_CMD})
+        set(CTEST_CONFIGURE_COMMAND "{CMAKE_CMD} -B {BINARY_DIR} {SOURCE_DIR} -DOMNITRACE_BUILD_CI=ON {CMAKE_ARGS}")
+        set(CTEST_BUILD_COMMAND "{CMAKE_CMD} --build {BINARY_DIR} --target all --parallel {BUILD_JOBS}")
+        set(CTEST_COVERAGE_COMMAND {GCOV_CMD})
+        """
+
+
+def generate_dashboard_script(args):
+
+    CODECOV = 1 if args.coverage else 0
+    DASHBOARD_MODE = args.mode
+    SOURCE_DIR = os.path.realpath(args.source_dir)
+    BINARY_DIR = os.path.realpath(args.binary_dir)
+
+    _script = """
+
+        include("${CMAKE_CURRENT_LIST_DIR}/CTestCustom.cmake")
+
+        macro(handle_error _message _ret)
+            if(NOT ${${_ret}} EQUAL 0)
+                ctest_submit(PARTS Done RETURN_VALUE _submit_ret)
+                message(FATAL_ERROR "${_message} failed: ${${_ret}}")
+            endif()
+        endmacro()
+        """
+
+    _script += f"""
+        ctest_start({DASHBOARD_MODE})
+        ctest_update(SOURCE "{SOURCE_DIR}")
+        ctest_configure(BUILD "{BINARY_DIR}" RETURN_VALUE _configure_ret)
+        ctest_submit(PARTS Start Update Configure RETURN_VALUE _submit_ret)
+
+        handle_error("Configure" _configure_ret)
+
+        ctest_build(BUILD "{BINARY_DIR}" RETURN_VALUE _build_ret)
+        ctest_submit(PARTS Build RETURN_VALUE _submit_ret)
+
+        handle_error("Build" _build_ret)
+
+        ctest_test(BUILD "{BINARY_DIR}" RETURN_VALUE _test_ret)
+        ctest_submit(PARTS Test RETURN_VALUE _submit_ret)
+
+        if("{CODECOV}" GREATER 0)
+            ctest_coverage(
+                BUILD "{BINARY_DIR}"
+                RETURN_VALUE _coverage_ret
+                CAPTURE_CMAKE_ERROR _coverage_err)
+            ctest_submit(PARTS Coverage RETURN_VALUE _submit_ret)
+        endif()
+
+        handle_error("Testing" _test_ret)
+
+        ctest_submit(PARTS Done RETURN_VALUE _submit_ret)
+        """
+    return _script
+
+
+def parse_cdash_args(args):
+    BUILD_JOBS = multiprocessing.cpu_count()
+    DASHBOARD_MODE = "Continuous"
+    DASHBOARD_STAGES = [
+        "Start",
+        "Update",
+        "Configure",
+        "Build",
+        "Test",
+        "MemCheck",
+        "Coverage",
+        "Submit",
+    ]
+    SOURCE_DIR = os.getcwd()
+    BINARY_DIR = os.path.join(SOURCE_DIR, "build")
+    SITE = socket.gethostname()
+    NAME = None
+    SUBMIT_URL = "my.cdash.org/submit.php?project=Omnitrace"
+    CODECOV = False
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-n", "--name", help="Job name", default=None, type=str, required=True
+    )
+    parser.add_argument("-s", "--site", help="Site name", default=SITE, type=str)
+    parser.add_argument(
+        "-c", "--coverage", help="Enable code coverage", action="store_true"
+    )
+    parser.add_argument(
+        "-j", "--build-jobs", help="Number of build tasks", default=BUILD_JOBS, type=int
+    )
+    parser.add_argument(
+        "-B", "--binary-dir", help="Build directory", default=BINARY_DIR, type=str
+    )
+    parser.add_argument(
+        "-S", "--source-dir", help="Source directory", default=SOURCE_DIR, type=str
+    )
+    parser.add_argument(
+        "-M",
+        "--mode",
+        help="Dashboard mode",
+        default=DASHBOARD_MODE,
+        choices=("Continuous", "Nightly", "Experimental"),
+        type=str,
+    )
+    parser.add_argument(
+        "-T",
+        "--stages",
+        help="Dashboard stages",
+        nargs="+",
+        default=DASHBOARD_STAGES,
+        choices=DASHBOARD_STAGES,
+        type=str,
+    )
+    parser.add_argument(
+        "--submit-url", help="CDash submission site", default=SUBMIT_URL, type=str
+    )
+
+    return parser.parse_args(args)
+
+
+def parse_args(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    index = 0
+    input_args = []
+    ctest_args = []
+    cmake_args = []
+    data = [input_args, cmake_args, ctest_args]
+
+    for itr in args:
+        if itr == "--":
+            index += 1
+            if index > 2:
+                raise RuntimeError("Usage: <options> -- <ctest-args> -- <cdash-args>")
+        else:
+            data[index].append(itr)
+
+    cdash_args = parse_cdash_args(input_args)
+
+    if cdash_args.coverage:
+        cmake_args += ["-DOMNITRACE_BUILD_CODECOV=ON", "-DOMNITRACE_STRIP_LIBRARIES=OFF"]
+
+    return [cdash_args, cmake_args, ctest_args]
+
+
+def run(*args, **kwargs):
+    import subprocess
+
+    return subprocess.run(*args, **kwargs)
+
+
+if __name__ == "__main__":
+
+    args, cmake_args, ctest_args = parse_args()
+
+    if not os.path.exists(args.binary_dir):
+        os.makedirs(args.binary_dir)
+
+    from textwrap import dedent
+
+    _config = dedent(generate_custom(args, cmake_args, ctest_args))
+    _script = dedent(generate_dashboard_script(args))
+
+    with open(os.path.join(args.binary_dir, "CTestCustom.cmake"), "w") as f:
+        f.write(f"{_config}\n")
+
+    with open(os.path.join(args.binary_dir, "dashboard.cmake"), "w") as f:
+        f.write(f"{_script}\n")
+
+    CTEST_CMD = which("ctest", require=True)
+
+    dashboard_args = ["-D"]
+    for itr in args.stages:
+        dashboard_args.append(f"{args.mode}{itr}")
+
+    run(
+        [CTEST_CMD]
+        + dashboard_args
+        + [
+            "-S",
+            os.path.join(args.binary_dir, "dashboard.cmake"),
+            "--output-on-failure",
+            "-V",
+        ]
+        + ctest_args,
+        check=True,
+    )
@@ -34,7 +34,7 @@ function(OMNITRACE_ADD_BIN_TEST)

    # common
    list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON" "OMNITRACE_CONFIG_FILE="
-         "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output")
+         "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output" "TWD=${TEST_WORKING_DIRECTORY}")
    # copy for inverse
    set(TEST_ENVIRONMENT_INV "${TEST_ENVIRONMENT}")

@@ -225,6 +225,25 @@ omnitrace_add_bin_test(
    LABELS "omnitrace-avail"
    TIMEOUT 45)

+omnitrace_add_bin_test(
+    NAME omnitrace-avail-all-expand-keys
+    TARGET omnitrace-avail
+    ARGS --all --expand-keys
+    LABELS "omnitrace-avail"
+    TIMEOUT 45
+    FAIL_REGEX "%[a-zA-Z_]%")
+
+omnitrace_add_bin_test(
+    NAME omnitrace-avail-all-only-available-alphabetical
+    TARGET omnitrace-avail
+    ARGS --all --available --alphabetical --debug --output
+         ${CMAKE_CURRENT_BINARY_DIR}/omnitrace-avail-all-only-available-alphabetical.log
+    LABELS "omnitrace-avail"
+    TIMEOUT 45
+    PROPERTIES
+        ATTACHED_FILES
+        ${CMAKE_CURRENT_BINARY_DIR}/omnitrace-avail-all-only-available-alphabetical.log)
+
 omnitrace_add_bin_test(
    NAME omnitrace-avail-all-csv
    TARGET omnitrace-avail
@@ -288,11 +307,20 @@ omnitrace_add_bin_test(

 string(REPLACE "+" "\\\+" _AVAIL_CFG_PATH
               "${PROJECT_BINARY_DIR}/omnitrace-tests-output/omnitrace-avail/omnitrace-")
+# use of TWD == Test Working Directory (added by function)
 omnitrace_add_bin_test(
    NAME omnitrace-avail-write-config
    TARGET omnitrace-avail
-    ARGS -G %env{PWD}%/omnitrace-tests-output/omnitrace-avail/omnitrace-test.cfg -F txt
-         json xml --force --all
+    ARGS -G
+         %env{TWD}%/omnitrace-tests-output/omnitrace-avail/omnitrace-test.cfg
+         -F
+         txt
+         json
+         xml
+         --force
+         --all
+         -c
+         omnitrace
    TIMEOUT 45
    LABELS "omnitrace-avail"
    PASS_REGEX
@@ -302,7 +330,7 @@ omnitrace_add_bin_test(
 omnitrace_add_bin_test(
    NAME omnitrace-avail-write-config-tweak
    TARGET omnitrace-avail
-    ARGS -G %env{PWD}%/omnitrace-tests-output/omnitrace-avail/omnitrace-tweak.cfg -F txt
+    ARGS -G %env{TWD}%/omnitrace-tests-output/omnitrace-avail/omnitrace-tweak.cfg -F txt
         json xml --force
    TIMEOUT 45
    LABELS "omnitrace-avail"
@@ -310,3 +338,37 @@ omnitrace_add_bin_test(
    PASS_REGEX
        "Outputting JSON configuration file '${_AVAIL_CFG_PATH}tweak\\\.json'(.*)Outputting XML configuration file '${_AVAIL_CFG_PATH}tweak\\\.xml'(.*)Outputting text configuration file '${_AVAIL_CFG_PATH}tweak\\\.cfg'(.*)"
    )
+
+omnitrace_add_bin_test(
+    NAME omnitrace-avail-list-keys
+    TARGET omnitrace-avail
+    ARGS --list-keys --expand-keys
+    TIMEOUT 45
+    LABELS "omnitrace-avail"
+    PASS_REGEX "Output Keys:\n(.*)%argv%(.*)%argv_hash%")
+
+omnitrace_add_bin_test(
+    NAME omnitrace-avail-list-keys-markdown
+    TARGET omnitrace-avail
+    ARGS --list-keys --expand-keys --markdown
+    TIMEOUT 45
+    LABELS "omnitrace-avail;markdown"
+    PASS_REGEX "(.*)`%argv%`(.*)`%argv_hash%`")
+
+omnitrace_add_bin_test(
+    NAME omnitrace-avail-list-categories
+    TARGET omnitrace-avail
+    ARGS --list-categories
+    TIMEOUT 45
+    LABELS "omnitrace-avail"
+    PASS_REGEX " component::(.*) hw_counters::(.*) settings::")
+
+omnitrace_add_bin_test(
+    NAME omnitrace-avail-core-categories
+    TARGET omnitrace-avail
+    ARGS -c core
+    TIMEOUT 45
+    LABELS "omnitrace-avail"
+    PASS_REGEX
+        "OMNITRACE_CONFIG_FILE(.*)OMNITRACE_ENABLED(.*)OMNITRACE_SUPPRESS_CONFIG(.*)OMNITRACE_SUPPRESS_PARSING(.*)OMNITRACE_VERBOSE"
+    )
@@ -26,11 +26,11 @@
 #define OMNITRACE_HIDDEN_API __attribute__((visibility("hidden")));

 #include <dlfcn.h>
+#include <stdbool.h>
 #include <stdio.h>
-#include <unistd.h>
 #include <stdlib.h>
 #include <string.h>
-#include <stdbool.h>
+#include <unistd.h>

 //
 // local type definitions
@@ -52,8 +52,8 @@ omnitrace_main(int, char**, char**) OMNITRACE_HIDDEN_API;

 int
 omnitrace_libc_start_main(int (*)(int, char**, char**), int, char**,
-                     int (*)(int, char**, char**), void (*)(void), void (*)(void),
-                     void*) OMNITRACE_HIDDEN_API;
+                          int (*)(int, char**, char**), void (*)(void), void (*)(void),
+                          void*) OMNITRACE_HIDDEN_API;

 int
 __libc_start_main(int (*)(int, char**, char**), int, char**, int (*)(int, char**, char**),
@@ -109,8 +109,8 @@ omnitrace_main(int argc, char** argv, char** envp)

 int
 omnitrace_libc_start_main(int (*_main)(int, char**, char**), int _argc, char** _argv,
-                  int (*_init)(int, char**, char**), void (*_fini)(void),
-                  void (*_rtld_fini)(void), void* _stack_end)
+                          int (*_init)(int, char**, char**), void (*_fini)(void),
+                          void (*_rtld_fini)(void), void* _stack_end)
 {
    int _preload = omnitrace_preload_library();

@@ -159,5 +159,5 @@ __libc_start_main(int (*_main)(int, char**, char**), int _argc, char** _argv,
                  void (*_rtld_fini)(void), void* _stack_end)
 {
    return omnitrace_libc_start_main(_main, _argc, _argv, _init, _fini, _rtld_fini,
-                                _stack_end);
+                                     _stack_end);
 }
@@ -105,7 +105,8 @@ extern "C"
    /// @fn int omnitrace_user_configure(omnitrace_user_configure_mode_t cfg,
    ///                                  omnitrace_user_callbacks_t new_callbacks,
    ///                                  omnitrace_user_callbacks_t* old_callbacks)
-    /// @param[in] config Specifies how the new callbacks are merged with the old callbacks
+    /// @param[in] config Specifies how the new callbacks are merged with the old
+    /// callbacks
    /// @param[in] new_callbacks An @ref omnitrace_user_callbacks_t instance specifying
    ///            the callbacks which should be invoked by the user API.
    /// @param[out] old_callbacks Pointer to @ref omnitrace_user_callbacks_t which,