rocm-systems/.github/workflows/continuous_integration.yml

name: Continuous Integration

on:
  workflow_dispatch:
  push:
    branches: [ amd-staging, amd-mainline ]
    paths-ignore:
      - '*.md'
      - 'source/docs/**'
      - 'CODEOWNERS'
  pull_request:
    paths-ignore:
      - '*.md'
      - 'source/docs/**'
      - 'CODEOWNERS'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  # TODO(jrmadsen): replace LD_RUNPATH_FLAG, GPU_TARGETS, etc. with internal handling in cmake
  ROCM_PATH: "/opt/rocm"
  GPU_TARGETS: "gfx900 gfx906 gfx908 gfx90a gfx940 gfx941 gfx942 gfx1030 gfx1100 gfx1101 gfx1102"
  PATH: "/usr/bin:$PATH"
  navi3_EXCLUDE_TESTS_REGEX: "^(test-page-migration-(execute|validate)|rocprofv3-test-(execute|validate)-app-abort)$"
  vega20_EXCLUDE_TESTS_REGEX: "^(test-page-migration-(execute|validate)|rocprofv3-test-(execute|validate)-app-abort)$"
  mi200_EXCLUDE_TESTS_REGEX: "^(test-page-migration-(execute|validate)|rocprofv3-test-(execute|validate)-app-abort)$"
  mi300_EXCLUDE_TESTS_REGEX: "^(test-page-migration-(execute|validate)|rocprofv3-test-(execute|validate)-app-abort)$"
  navi3_EXCLUDE_LABEL_REGEX: "^(pc-sampling|openmp-target)$"
  vega20_EXCLUDE_LABEL_REGEX: "^(pc-sampling|openmp-target)$"
  mi200_EXCLUDE_LABEL_REGEX: "^(openmp-target)$"
  mi300_EXCLUDE_LABEL_REGEX: "^(pc-sampling)$"

jobs:
  core-deb:
    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
    strategy:
      fail-fast: false
      matrix:
        runner: ['vega20', 'mi300']
        os: ['ubuntu-22.04']
        build-type: ['RelWithDebInfo']
        ci-flags: ['--linter clang-tidy']

    runs-on: ${{ matrix.runner }}-emu-runner-set

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      CORE_EXT_RUNNER: mi300

    steps:
    - uses: actions/checkout@v4

    - name: Install requirements
      timeout-minutes: 10
      shell: bash
      run: |
        git config --global --add safe.directory '*'
        apt-get update
        apt-get install -y build-essential cmake g++-11 g++-12 python3-pip libdw-dev rccl-dev rccl-unittests
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 --slave /usr/bin/gcov gcov /usr/bin/gcov-12
        python3 -m pip install -r requirements.txt

    - name: List Files
      shell: bash
      run: |
        echo "PATH: ${PATH}"
        echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
        cat /opt/rocm/.info/version
        ls -la

    - name: Enable PC Sampling
      if: ${{ contains(matrix.runner, 'mi200') }}
      shell: bash
      run: |
          echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

    - name: Configure, Build, and Test
      timeout-minutes: 30
      shell: bash
      run:
        python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-core
          --build-jobs 16
          --site ${RUNNER_HOSTNAME}
          --gpu-targets ${{ env.GPU_TARGETS }}
          --run-attempt ${{ github.run_attempt }}
          --
          -DROCPROFILER_DEP_ROCMCORE=ON
          -DROCPROFILER_BUILD_DOCS=OFF
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-sdk
          -DCPACK_GENERATOR='DEB;RPM;TGZ'
          -DCPACK_PACKAGING_INSTALL_PREFIX="$(realpath /opt/rocm)"
          -DPython3_EXECUTABLE=$(which python3)
          --
          -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"

    - name: Install
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 10
      run:
        cmake --build build --target install --parallel 16

    - name: Build Packaging
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 10
      run:
        cmake --build build --target package --parallel 16

    - name: Test Install Build
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 10
      shell: bash
      run: |
        CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-samples samples
        CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-tests tests
        export LD_LIBRARY_PATH=/opt/rocprofiler-sdk/lib:${LD_LIBRARY_PATH}
        cmake --build build-samples --target all --parallel 16
        cmake --build build-tests --target all --parallel 16
        ctest --test-dir build-samples -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
        ctest --test-dir build-tests -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

    - name: Install Packages
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 5
      shell: bash
      run: |
        export PATH=${PATH}:/usr/local/sbin:/usr/sbin:/sbin
        ls -la
        ls -la ./build
        dpkg -i ./build/rocprofiler-sdk-roctx_*.deb
        for i in $(ls -S ./build/rocprofiler-sdk*.deb | egrep -v roctx); do dpkg -i ${i}; done;

    - name: Test Installed Packages
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 20
      shell: bash
      run: |
        CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-samples-deb /opt/rocm/share/rocprofiler-sdk/samples
        CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-tests-deb /opt/rocm/share/rocprofiler-sdk/tests
        cmake --build build-samples-deb --target all --parallel 16
        cmake --build build-tests-deb --target all --parallel 16
        ctest --test-dir build-samples-deb -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
        ctest --test-dir build-tests-deb -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

    - name: Archive production artifacts
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      uses: actions/upload-artifact@v4
      with:
        name: installers-deb
        path: |
          ${{github.workspace}}/build/*.deb
          ${{github.workspace}}/build/*.rpm
          ${{github.workspace}}/build/*.tgz

  core-rpm:
    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
    strategy:
      fail-fast: false
      matrix:
        runner: ['mi300']
        os: ['rhel-emu', 'sles-emu']
        build-type: ['RelWithDebInfo']
        ci-flags: ['--linter clang-tidy']

    runs-on: ${{ matrix.os }}-runner-set

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      CORE_EXT_RUNNER: navi3

    steps:
    - uses: actions/checkout@v4

    - name: Install requirements
      timeout-minutes: 10
      shell: bash
      run: |
        sudo python3 -m pip install 'perfetto<0.9.0'
        sudo python3 -m pip install -r requirements.txt

    - name: List Files
      shell: bash
      run: |
        echo "PATH: ${PATH}"
        echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
        cat /opt/rocm/.info/version
        ls -la

    - name: Enable PC Sampling
      if: ${{ contains(matrix.runner, 'mi200') }}
      shell: bash
      run: |
          echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

    - name: Configure, Build, and Test
      timeout-minutes: 30
      shell: bash
      run:
        sudo LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-core
          --build-jobs 16
          --site ${RUNNER_HOSTNAME}
          --gpu-targets ${{ env.GPU_TARGETS }}
          --run-attempt ${{ github.run_attempt }}
          --
          -DROCPROFILER_DEP_ROCMCORE=ON
          -DROCPROFILER_BUILD_DOCS=OFF
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-sdk
          -DCPACK_GENERATOR='DEB;RPM;TGZ'
          -DCPACK_PACKAGING_INSTALL_PREFIX="$(realpath /opt/rocm)"
          -DPython3_EXECUTABLE=$(which python3)
          --
          -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"

    - name: Install
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 10
      run:
        cmake --build build --target install --parallel 16

    - name: Build Packaging
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 10
      run:
        cmake --build build --target package --parallel 16

    - name: Test Install Build
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 10
      shell: bash
      run: |
        CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-samples samples
        CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-tests tests
        export LD_LIBRARY_PATH=/opt/rocprofiler-sdk/lib:${LD_LIBRARY_PATH}
        cmake --build build-samples --target all --parallel 16
        cmake --build build-tests --target all --parallel 16
        ctest --test-dir build-samples -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
        ctest --test-dir build-tests -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

    - name: Install Packages
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 5
      shell: bash
      run: |
        export PATH=${PATH}:/usr/local/sbin:/usr/sbin:/sbin
        ls -la
        ls -la ./build
        dpkg -i ./build/rocprofiler-sdk-roctx_*.deb
        for i in $(ls -S ./build/rocprofiler-sdk*.deb | egrep -v roctx); do dpkg -i ${i}; done;

    - name: Test Installed Packages
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      timeout-minutes: 20
      shell: bash
      run: |
        CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-samples-deb /opt/rocm/share/rocprofiler-sdk/samples
        CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-tests-deb /opt/rocm/share/rocprofiler-sdk/tests
        cmake --build build-samples-deb --target all --parallel 16
        cmake --build build-tests-deb --target all --parallel 16
        ctest --test-dir build-samples-deb -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
        ctest --test-dir build-tests-deb -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

    - name: Archive production artifacts
      if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
      uses: actions/upload-artifact@v4
      with:
        name: installers-rpm
        path: |
          ${{github.workspace}}/build/*.deb
          ${{github.workspace}}/build/*.rpm
          ${{github.workspace}}/build/*.tgz

  code-coverage:
    if: ${{ contains(github.event_name, 'pull_request') }}
    strategy:
      # fail-fast: false
      matrix:
        runner: ['mi200']
        os: ['ubuntu-22.04']
        build-type: ['Release']

    runs-on: ${{ matrix.runner }}-emu-runner-set

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      GCC_COMPILER_VERSION: 11
      ROCPROFILER_PC_SAMPLING_BETA_ENABLED: 1

    steps:
    - name: Patch Git
      timeout-minutes: 25
      run: |
        apt-get update
        apt-get install -y software-properties-common
        add-apt-repository -y ppa:git-core/ppa
        apt-get update
        apt-get install -y git

    - uses: actions/checkout@v4

    - name: Load Existing XML Code Coverage
      if: github.event_name == 'pull_request'
      id: load-coverage
      uses: actions/cache@v4
      with:
        key: ${{ github.event.pull_request.base.sha }}-codecov
        path: .codecov/**

    - name: Copy Existing XML Code Coverage
      if: github.event_name == 'pull_request'
      shell: bash
      run: |
        if [ -d .codecov ]; then cp -r .codecov .codecov.ref; fi

    - name: Configure Env
      shell: bash
      run: |
        echo "${PATH}:/usr/local/bin:${HOME}/.local/bin" >> $GITHUB_PATH
        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:${HOME}/.local/lib" >> $GITHUB_ENV

    - name: Install requirements
      timeout-minutes: 10
      shell: bash
      run: |
        git config --global --add safe.directory '*'
        apt-get update
        apt-get install -y build-essential cmake python3-pip gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig libdw-dev
        python3 -m pip install -r requirements.txt

    - name: Sync gcov with compilers
      timeout-minutes: 10
      shell: bash
      run:
        apt-get install -y gcc-${{ env.GCC_COMPILER_VERSION }} g++-${{ env.GCC_COMPILER_VERSION }} &&
        update-alternatives
            --install /usr/bin/gcc  gcc  /usr/bin/gcc-${{ env.GCC_COMPILER_VERSION }} 500
            --slave   /usr/bin/g++  g++  /usr/bin/g++-${{ env.GCC_COMPILER_VERSION }}
            --slave   /usr/bin/gcov gcov /usr/bin/gcov-${{ env.GCC_COMPILER_VERSION }}

    - name: List Files
      shell: bash
      run: |
        echo "PATH: ${PATH}"
        echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
        cat /opt/rocm/.info/version
        ls -la

    - name: Enable PC Sampling
      if: ${{ contains(matrix.runner, 'mi200') }}
      shell: bash
      run: |
          echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

    - name: Configure, Build, and Test (Total Code Coverage)
      timeout-minutes: 30
      shell: bash
      run:
        python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-codecov
          --build-jobs 16
          --site ${RUNNER_HOSTNAME}
          --gpu-targets ${{ env.GPU_TARGETS }}
          --coverage all
          --run-attempt ${{ github.run_attempt }}
          --
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DPython3_EXECUTABLE=$(which python3)
          --
          -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"

    - name: Configure, Build, and Test (Tests Code Coverage)
      timeout-minutes: 30
      shell: bash
      run:
        find build -type f | egrep '\.gcda$' | xargs rm &&
        python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-codecov-tests
          --build-jobs 16
          --site ${RUNNER_HOSTNAME}
          --gpu-targets ${{ env.GPU_TARGETS }}
          --coverage tests
          --run-attempt ${{ github.run_attempt }}
          --
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DPython3_EXECUTABLE=$(which python3)
          --
          -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"

    - name: Configure, Build, and Test (Samples Code Coverage)
      timeout-minutes: 30
      shell: bash
      run:
        find build -type f | egrep '\.gcda$' | xargs rm &&
        python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-codecov-samples
          --build-jobs 16
          --site ${RUNNER_HOSTNAME}
          --gpu-targets ${{ env.GPU_TARGETS }}
          --coverage samples
          --run-attempt ${{ github.run_attempt }}
          --
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DPython3_EXECUTABLE=$(which python3)
          --
          -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"

    - name: Save XML Code Coverage
      id: save-coverage
      uses: actions/cache/save@v4
      with:
        key: ${{ github.sha }}-codecov
        path: |
          .codecov/*.xml

    - id: generatereport
      name: Generate Code Coverage Comment
      if: github.event_name == 'pull_request'
      timeout-minutes: 5
      shell: bash
      run: |
        echo "PWD: ${PWD}"
        ls -la

        for i in "all" "tests" "samples"; do
            wkhtmltoimage --enable-local-file-access --quality 70 .codecov/${i}.html .codecov/${i}.png
        done
        ls -la .codecov
        which -a git
        git --version

        ./source/scripts/upload-image-to-github.py --bot --token ${{ secrets.TOKEN }} --files .codecov/{all,tests,samples}.png --output-dir .codecov --name pr-${{ github.event.pull_request.number }}

        echo -e "\n${PWD}:"
        ls -la .

        echo -e "\n.codecov:"
        ls -la .codecov

        get-base-md-contents() { cat .codecov/${1}.png.md; }
        get-full-md-contents() { cat .codecov/${1}.png.md .codecov/${1}.md; }
        cat << EOF > .codecov/report.md
        # Code Coverage Report

        ## Tests Only
        $(get-base-md-contents tests)

        ## Samples Only
        $(get-base-md-contents samples)

        ## Tests + Samples
        $(get-full-md-contents all)

        EOF

        echo 'CODECOVERAGE_REPORT<<EOF' > $GITHUB_OUTPUT
        cat .codecov/report.md >> $GITHUB_OUTPUT
        echo 'EOF' >> $GITHUB_OUTPUT

    - name: Write Code Coverage Comment
      if: github.event_name == 'pull_request'
      timeout-minutes: 5
      uses: actions/github-script@v6
      env:
        COMMENT_BODY: ${{ steps.generatereport.outputs.CODECOVERAGE_REPORT }}
      with:
        github-token: ${{ secrets.TOKEN }}
        script: |
          const commentIdentifier = '<!-- unique-comment-identifier -->'; // Use this to identify your comment
            const commentBody = `
              ${commentIdentifier}
              ` + process.env.COMMENT_BODY;

            // Fetch existing comments
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.payload.pull_request.number,
            });

            // Check for an existing comment
            const existingComment = comments.find(comment => comment.body.includes(commentIdentifier));

            if (existingComment) {
              // Update the existing comment
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existingComment.id,
                body: commentBody,
              });
              core.info(`Updated comment with ID: ${existingComment.id}`);
            } else {
              // Create a new comment
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.payload.pull_request.number,
                body: commentBody,
              });
              core.info('Created a new comment.');
            }

    - name: Archive Code Coverage Data
      uses: actions/upload-artifact@v4
      with:
        name: code-coverage-details
        path: |
          ${{github.workspace}}/.codecov/*

    - name: Verify Test Labels
      timeout-minutes: 5
      shell: bash
      run: |
        pushd build
        #
        # if following fails, there is a test that does not have
        # a label identifying it as sample or test (unit or integration).
        # Recommended labels are:
        #   - samples
        #   - unittests
        #   - integration-tests
        #
        ctest -N -LE 'samples|tests' -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}" -O ctest.mislabeled.log
        grep 'Total Tests: 0' ctest.mislabeled.log
        #
        # if following fails, then there is overlap between the labels.
        # A test cannot both be a sample and (unit/integration) test.
        #
        ctest -N -O ctest.all.log
        ctest -N -O ctest.samples.log -L samples
        ctest -N -O ctest.tests.log -L tests
        NUM_ALL=$(grep 'Total Tests:' ctest.all.log | awk '{print $NF}')
        NUM_SAMPLE=$(grep 'Total Tests:' ctest.samples.log | awk '{print $NF}')
        NUM_TEST=$(grep 'Total Tests:' ctest.tests.log | awk '{print $NF}')
        NUM_SUM=$((${NUM_SAMPLE} + ${NUM_TEST}))
        echo "Total tests: ${NUM_ALL}"
        echo "Total labeled tests: ${NUM_SUM}"
        if [ ${NUM_ALL} != ${NUM_SUM} ]; then
            echo "Test label overlap"
            exit 1
        fi
        popd

  sanitizers:
    strategy:
      fail-fast: false
      matrix:
        runner: ['vega20', 'navi3', 'mi300']
        sanitizer: ['AddressSanitizer', 'ThreadSanitizer', 'LeakSanitizer', 'UndefinedBehaviorSanitizer']
        os: ['ubuntu-22.04']
        build-type: ['RelWithDebInfo']
        exclude:
          - { runner: 'vega20', sanitizer: 'ThreadSanitizer' }
          - { runner: 'vega20', sanitizer: 'AddressSanitizer' }
          - { runner: 'vega20', sanitizer: 'UndefinedBehaviorSanitizer' }
          - { runner: 'mi300', sanitizer: 'AddressSanitizer' }
          - { runner: 'mi300', sanitizer: 'LeakSanitizer' }
          - { runner: 'navi3', sanitizer: 'LeakSanitizer' }
          - { runner: 'navi3', sanitizer: 'ThreadSanitizer' }
          - { runner: 'navi3', sanitizer: 'UndefinedBehaviorSanitizer' }

    if: ${{ contains(github.event_name, 'pull_request') }}
    runs-on: ${{ matrix.runner }}-emu-runner-set

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      GCC_COMPILER_VERSION: 13

    steps:
    - uses: actions/checkout@v4

    - name: Install requirements
      timeout-minutes: 10
      shell: bash
      run: |
        git config --global --add safe.directory '*'
        apt-get update
        apt-get install -y build-essential cmake python3-pip libasan8 libtsan2 software-properties-common
        add-apt-repository ppa:ubuntu-toolchain-r/test
        apt-get update
        apt-get install -y gcc-${{ env.GCC_COMPILER_VERSION }} g++-${{ env.GCC_COMPILER_VERSION }} libdw-dev
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_COMPILER_VERSION }} 100 --slave /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_COMPILER_VERSION }} --slave /usr/bin/gcov gcov /usr/bin/gcov-${{ env.GCC_COMPILER_VERSION }}
        python3 -m pip install -r requirements.txt

    - name: List Files
      shell: bash
      run: |
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
        cat /opt/rocm/.info/version
        ls -la

    - name: Enable PC Sampling
      if: ${{ contains(matrix.runner, 'mi200') }}
      shell: bash
      run: |
          echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

    - name: Configure, Build, and Test
      timeout-minutes: 45
      shell: bash
      run:
        python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-${{ matrix.sanitizer }}
          --build-jobs 16
          --site ${RUNNER_HOSTNAME}
          --gpu-targets ${{ env.GPU_TARGETS }}
          --memcheck ${{ matrix.sanitizer }}
          --run-attempt ${{ github.run_attempt }}
          --
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DCMAKE_INSTALL_PREFIX="${{ env.ROCM_PATH }}"
          -DPython3_EXECUTABLE=$(which python3)
          --
          -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"