name: rocprofiler-sdk Continuous Integration

on:
  workflow_dispatch:
  schedule:
    - cron: '0 7 * * *'
  push:
    branches: [ develop ]
    paths:
      - 'projects/rocprofiler-sdk/**'
      - '.github/workflows/rocprofiler-sdk-continuous_integration.yml'
      - '!**/*.md'
      - '!**/*.rtf'
      - '!**/*.rst'
      - '!**/.markdownlint-ci2.yaml'
      - '!**/.readthedocs.yaml'
      - '!**/.spellcheck.local.yaml'
      - '!**/.wordlist.txt'
      - '!projects/rocprofiler-sdk/CODEOWNERS'
      - '!projects/rocprofiler-sdk/source/docs/**'
  pull_request:
    paths:
      - 'projects/rocprofiler-sdk/**'
      - '.github/workflows/rocprofiler-sdk-continuous_integration.yml'
      - '!**/*.md'
      - '!**/*.rtf'
      - '!**/*.rst'
      - '!**/.markdownlint-ci2.yaml'
      - '!**/.readthedocs.yaml'
      - '!**/.spellcheck.local.yaml'
      - '!**/.wordlist.txt'
      - '!projects/rocprofiler-sdk/CODEOWNERS'
      - '!projects/rocprofiler-sdk/source/docs/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Needed to push/pull cached Docker layers + GHCR images
permissions:
  contents: read
  packages: write

env:
  # TODO(jrmadsen): replace LD_RUNPATH_FLAG, GPU_TARGETS, etc. with internal handling in cmake
  ROCM_PATH: "/opt/rocm"
  ROCM_VERSION: "7.1.1"
  PYTHON_VENV_PATH: "rocprofiler-sdk"
  PYTHON_VENV_ACTIVATE: "rocprofiler-sdk/bin/activate"
  GPU_TARGETS: "gfx906 gfx908 gfx90a gfx942 gfx950 gfx1030 gfx1100 gfx1101 gfx1102 gfx1201"
  PATH: "/usr/bin:$PATH"

  ## No tests should be excluded here except for extreme emergencies; tests should only be disabled in CMake
  ## A task should be assigned directly to fix the issues
  ## Scratch memory tests need to be fixed for ROCm 7.0 release
  navi3_EXCLUDE_TESTS_REGEX: ""
  vega20_EXCLUDE_TESTS_REGEX: ""
  mi200_EXCLUDE_TESTS_REGEX: ""
  mi300_EXCLUDE_TESTS_REGEX: ""
  mi300a_EXCLUDE_TESTS_REGEX: ""
  mi325_EXCLUDE_TESTS_REGEX: ""
  mi3xx_EXCLUDE_TESTS_REGEX: ""
  navi4_EXCLUDE_TESTS_REGEX: ""

  navi3_EXCLUDE_LABEL_REGEX: ""
  vega20_EXCLUDE_LABEL_REGEX: ""
  mi200_EXCLUDE_LABEL_REGEX: ""
  mi300_EXCLUDE_LABEL_REGEX: ""
  mi300a_EXCLUDE_LABEL_REGEX: ""
  mi325_EXCLUDE_LABEL_REGEX: ""
  mi3xx_EXCLUDE_LABEL_REGEX: ""
  navi4_EXCLUDE_LABEL_REGEX: ""

  GLOBAL_CMAKE_OPTIONS: ""
  ENABLE_ROCR_BUILD: "true"
  ENABLE_HIP_CLR_BUILD: "true"

  CI_MODE: ${{ github.event_name == 'schedule' && 'Nightly' || 'Continuous' }}


jobs:
  # -----------------------------------------------------------------------------
  # Ubuntu / DEB job(s)
  # -----------------------------------------------------------------------------
  core-deb:
    name: Core • ${{ matrix.system.gpu }} • ${{ matrix.system.os }}
    strategy:
      fail-fast: false
      matrix:
        system:
          - { gpu: 'navi4', runner: 'rocprofiler-navi4-dind', os: 'ubuntu-22.04', build-type: 'RelWithDebInfo', ci-flags: '--linter clang-tidy', gpu-target: 'gfx120X' }
          - { gpu: 'navi3', runner: 'rocprofiler-navi3-dind', os: 'ubuntu-22.04', build-type: 'RelWithDebInfo', ci-flags: '--linter clang-tidy', gpu-target: 'gfx110X' }
          - { gpu: 'mi325', runner: 'linux-mi325-1gpu-ossci-rocm-frac', os: 'ubuntu-22.04', build-type: 'RelWithDebInfo', ci-flags: '--linter clang-tidy', gpu-target: 'gfx94X' }
    runs-on: ${{ matrix.system.runner }}
    container:
      image: docker.io/rocm/rocprofiler-private:${{ matrix.system.os }}-${{ matrix.system.gpu-target }}-latest
      credentials:
        username: ${{ secrets.ROCPROFILER_AZURE_CI_USER }}
        password: ${{ secrets.ROCPROFILER_AZURE_CI_PASS }}
      env:
        DEBIAN_FRONTEND: noninteractive
      options: --privileged
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      CORE_EXT_RUNNER: mi325
      GPU_RUNNER: ${{ matrix.system.gpu }}
    steps:
      - name: Install Latest Nightly ROCm
        shell: bash
        working-directory: /tmp
        run: |
          tar -xf ${{ env.ROCM_PATH }}-${{ matrix.system.gpu-target }}.tar.gz -C ${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}
          ln -s ${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} ${{ env.ROCM_PATH }}
          echo "ROCm installed to: ${{ env.ROCM_PATH }}"

      - name: Clone ROCProfiler SDK & AQLProfile & ROCProfiler Register & ROCR-Runtime
        uses: actions/checkout@v5
        with:
          sparse-checkout: |
            projects/rocprofiler-sdk
            projects/aqlprofile
            projects/rocprofiler-register
            projects/rocr-runtime
            projects/clr
            projects/hip
          submodules: false
          set-safe-directory: true

      - name: Compute submodule cache key
        id: submods
        shell: bash
        run: |
          git --version
          git config --global --add safe.directory '*'
          git submodule status --recursive | awk '{print $1,$2}' > .git-submodules-status
          echo "hash=$(sha256sum .git-submodules-status | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
          # collect submodule paths for cache 'path'
          git config --file .gitmodules --get-regexp path | awk '{print $2}' > .git-submodule-paths
          { echo "paths<<EOF"; cat .git-submodule-paths; echo "EOF"; } >> "$GITHUB_OUTPUT"

      - name: Restore submodule cache
        uses: actions/cache@v4
        with:
          path: |
            .git/modules
            ${{ steps.submods.outputs.paths }}
          key: submods-${{ runner.os }}-${{ steps.submods.outputs.hash }}
          restore-keys: |
            submods-${{ runner.os }}-
            submods-

      - name: Init/Update submodules
        run: git submodule update --init --recursive --jobs 16

      - name: Clone ROCDecode
        uses: actions/checkout@v5
        with:
          repository: 'ROCm/rocDecode'
          ref: 'release/rocm-rel-7.0'
          set-safe-directory: true
          path: 'rocDecode'

      - name: Clone ROCJPEG
        uses: actions/checkout@v5
        with:
          repository: 'ROCm/rocJPEG'
          ref: 'release/rocm-rel-7.0'
          set-safe-directory: true
          path: 'rocJPEG'

      - name: Install requirements
        timeout-minutes: 10
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          git config --global --add safe.directory '*'
          apt-get update
          apt-get install -y g++-11 g++-12 cmake python3-pip libdw-dev libsqlite3-dev libdrm-dev file autoconf pkg-config rpm libzstd-dev
          update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11
          update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 --slave /usr/bin/gcov gcov /usr/bin/gcov-12
          python3 -m pip install -U --user -r requirements.txt
          rm -rf \
            ${{ env.ROCM_PATH }}/lib/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/lib/cmake/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/share/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/libexec/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}*/lib/python*/site-packages/roctx \
            ${{ env.ROCM_PATH }}*/lib/python*/site-packages/rocpd

      - name: Setup ccache
        uses: hendrikmuhs/ccache-action@63069e3931dedbf3b63792097479563182fe70d1 # v1.2.18
        with:
          key: ccache-${{ matrix.system.os }}-${{ matrix.system.runner }}-${{ matrix.system.gpu }}
          max-size: 2G
          save: true

      - name: Install Missing ROCm Dependencies
        shell: bash
        run: |
          echo -e "Building & Installing ROCDecode..."
          cmake -B build-rocdecode \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }} \
            -DCMAKE_CXX_COMPILER=${{ env.ROCM_PATH }}/bin/amdclang++ \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            ${GITHUB_WORKSPACE}/rocDecode
          cmake --build build-rocdecode --target all --parallel 16
          cmake --build build-rocdecode --target install
          echo -e "ROCDecode Installed Successfully!"
          echo -e "Building & Installing ROCJPEG..."
          cmake -B build-rocjpeg \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }} \
            -DCMAKE_CXX_COMPILER=${{ env.ROCM_PATH }}/bin/amdclang++ \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            ${GITHUB_WORKSPACE}/rocJPEG
          cmake --build build-rocjpeg --target all --parallel 16
          cmake --build build-rocjpeg --target install
          echo -e "ROCJPEG Installed Successfully!"

      - name: Build and Install ROCProfiler-Register
        shell: bash
        working-directory: projects/rocprofiler-register
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install ROCProfiler-Register"
          cmake -B build-rocprofiler-register \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
          cmake --build build-rocprofiler-register --target all --parallel 16
          cmake --build build-rocprofiler-register --target install
          echo "✅ ROCProfiler-Register Installation complete!"

      - name: Build and Install ROCR-Runtime
        if: ${{ env.ENABLE_ROCR_BUILD == 'true' }}
        shell: bash
        working-directory: projects/rocr-runtime
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install ROCR-Runtime..."
          cmake -B build \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }};${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            .
          cmake --build build --target all --parallel 16
          cmake --build build --target install
          echo "✅ ROCR-Runtime Installation complete!"

      - name: Build and Install HIP
        if: ${{ env.ENABLE_HIP_CLR_BUILD == 'true' }}
        shell: bash
        working-directory: projects
        run: |
          export HIP_DIR=$PWD/hip
          export CLR_DIR=$PWD/clr
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install HIP..."
          cd $CLR_DIR
          cmake \
            -DHIP_COMMON_DIR=$HIP_DIR \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DHIP_PLATFORM=amd \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }};${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DHIP_LLVM_ROOT=${{ env.ROCM_PATH }}/lib/llvm \
            -DHIP_CATCH_TEST=0 \
            -DCLR_BUILD_HIP=ON \
            -DCLR_BUILD_OCL=ON \
            -S $CLR_DIR \
            -B build
          cmake --build build --target all --parallel 16
          cmake --build build --target install
          echo "✅ HIP Installation complete!"

      - name: Build and Install Aqlprofile
        shell: bash
        working-directory: projects/aqlprofile
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install Aqlprofile..."
          cmake -B build-aqlprofile \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
          cmake --build build-aqlprofile --target all --parallel 16
          cmake --build build-aqlprofile --target install
          echo "✅ AQLProfile Installation complete!"

      - name: List Files
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          echo "PATH: ${PATH}"
          echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
          which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
          for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
          cat ${{ env.ROCM_PATH }}/.info/version
          ls -la

      - name: Enable PC Sampling
        if: ${{ contains(matrix.system.gpu, 'mi200') || contains(matrix.system.gpu, 'mi300a') }}
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

      - name: Configure, Build, and Test
        timeout-minutes: 30
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH \
          PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$HOME/.local/bin:$PATH \
          python3 ./source/scripts/run-ci.py \
            -B build \
            --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.system.os }}-${{ matrix.system.gpu }}-core \
            --build-jobs 16 \
            --mode ${CI_MODE} \
            --site ${{ matrix.system.runner }} \
            --gpu-targets ${{ env.GPU_TARGETS }} \
            --run-attempt ${{ github.run_attempt }} \
            ${{ matrix.system.ci-flags }} -- \
            -DROCPROFILER_DEP_ROCMCORE=ON \
            -DROCPROFILER_BUILD_DOCS=OFF \
            -DROCPROFILER_BUILD_FMT=OFF \
            -DROCPROFILER_INTERNAL_RCCL_API_TRACE=ON \
            -DCMAKE_BUILD_TYPE=${{ matrix.system.build-type }} \
            -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-sdk \
            -DCPACK_GENERATOR='DEB;RPM;TGZ' \
            -DCPACK_PACKAGING_INSTALL_PREFIX="$(realpath ${{ env.ROCM_PATH }})" \
            -DPython3_EXECUTABLE=$(which python3) \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }};${{ env.ROCM_PATH }}/llvm' \
            ${{ env.GLOBAL_CMAKE_OPTIONS }} -- \
            -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" \
            -E  "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}"

      - name: Install
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 10
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          cmake --build build --target install --parallel 16

      - name: Build Packaging
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 10
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          cmake --build build --target package --parallel 16

      - name: Test Install Build
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 20
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$HOME/.local/bin:$PATH
          CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-samples samples
          CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-tests -DGPU_TARGETS="gfx942" tests
          export LD_LIBRARY_PATH=/opt/rocprofiler-sdk/lib:${LD_LIBRARY_PATH}
          cmake --build build-samples --target all --parallel 16
          cmake --build build-tests --target all --parallel 16
          ctest --test-dir build-samples -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
          ctest --test-dir build-tests   -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

      - name: Install Packages
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 5
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$HOME/.local/bin:$PATH
          export PATH=${PATH}:/usr/local/sbin:/usr/sbin:/sbin
          ls -la
          ls -la ./build
          dpkg --force-all -i ./build/rocprofiler-sdk-roctx_*.deb
          dpkg --force-all -i ./build/rocprofiler-sdk-rocpd_*.deb
          for i in $(ls -S ./build/rocprofiler-sdk*.deb | egrep -v 'roctx|rocpd'); do dpkg --force-all -i ${i}; done;

      - name: Test Installed Packages
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 20
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$HOME/.local/bin:$PATH
          CMAKE_PREFIX_PATH=${{ env.ROCM_PATH }} cmake -B build-samples-deb ${{ env.ROCM_PATH }}/share/rocprofiler-sdk/samples
          CMAKE_PREFIX_PATH=${{ env.ROCM_PATH }} cmake -B build-tests-deb -DGPU_TARGETS="gfx942" ${{ env.ROCM_PATH }}/share/rocprofiler-sdk/tests
          cmake --build build-samples-deb --target all --parallel 16
          cmake --build build-tests-deb --target all --parallel 16
          ctest --test-dir build-samples-deb -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
          ctest --test-dir build-tests-deb   -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

      - name: Archive production artifacts
        if: ${{ github.event_name == 'workflow_dispatch' && contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        uses: actions/upload-artifact@v4
        with:
          name: installers-deb
          path: |
            ${{github.workspace}}/build/*.deb
            ${{github.workspace}}/build/*.rpm
            ${{github.workspace}}/build/*.tgz

  # -----------------------------------------------------------------------------
  # RHEL/SLES (RPM) job(s)
  # -----------------------------------------------------------------------------
  core-rpm:
    name: Core • ${{ matrix.system.gpu }} • ${{ matrix.system.os }}
    strategy:
      fail-fast: false
      matrix:
        system:
          - { os: 'rhel-8.8', runner: 'linux-mi325-1gpu-ossci-rocm-frac', gpu: 'mi325', gpu-target: 'gfx94X', build-type: 'RelWithDebInfo', ci-flags: '' }
          - { os: 'rhel-9.5', runner: 'linux-mi325-1gpu-ossci-rocm-frac', gpu: 'mi325', gpu-target: 'gfx94X', build-type: 'RelWithDebInfo', ci-flags: '' }
          - { os: 'sles-15.6', runner: 'linux-mi325-1gpu-ossci-rocm-frac', gpu: 'mi325', gpu-target: 'gfx94X', build-type: 'RelWithDebInfo', ci-flags: '' }
    runs-on: ${{ matrix.system.runner }}
    container:
      image: docker.io/rocm/rocprofiler-private:${{ matrix.system.os }}-${{ matrix.system.gpu-target }}-latest
      credentials:
        username: ${{ secrets.ROCPROFILER_AZURE_CI_USER }}
        password: ${{ secrets.ROCPROFILER_AZURE_CI_PASS }}
      options: --privileged
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      OS_TYPE: ${{ matrix.system.os }}
      GPU_RUNNER: ${{ matrix.system.gpu }}
    steps:
      - name: Clone ROCProfiler SDK & AQLProfile & ROCProfiler Register & ROCR-Runtime
        uses: actions/checkout@v5
        with:
          sparse-checkout: |
            projects/rocprofiler-sdk
            projects/aqlprofile
            projects/rocprofiler-register
            projects/rocr-runtime
            projects/clr
            projects/hip
          submodules: false
          set-safe-directory: true

      - name: Compute submodule cache key
        id: submods
        shell: bash
        run: |
          git --version
          git config --global --add safe.directory '*'
          git submodule status --recursive | awk '{print $1,$2}' > .git-submodules-status
          echo "hash=$(sha256sum .git-submodules-status | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
          # collect submodule paths for cache 'path'
          git config --file .gitmodules --get-regexp path | awk '{print $2}' > .git-submodule-paths
          { echo "paths<<EOF"; cat .git-submodule-paths; echo "EOF"; } >> "$GITHUB_OUTPUT"

      - name: Restore submodule cache
        uses: actions/cache@v4
        with:
          path: |
            .git/modules
            ${{ steps.submods.outputs.paths }}
          key: submods-${{ runner.os }}-${{ steps.submods.outputs.hash }}
          restore-keys: |
            submods-${{ runner.os }}-
            submods-

      - name: Init/Update submodules
        run: git submodule update --init --recursive --jobs 16

      - name: Install Latest Nightly ROCm using TheRock Tarballs
        shell: bash
        working-directory: /tmp
        run: |
          tar -xf ${{ env.ROCM_PATH }}-${{ matrix.system.gpu-target }}.tar.gz -C ${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}
          ln -s ${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} ${{ env.ROCM_PATH }}
          echo "ROCm installed to: ${{ env.ROCM_PATH }}"

      - name: Install requirements (venv)
        timeout-minutes: 10
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          git config --global --add safe.directory '*'
          python3 -m venv ${{ env.PYTHON_VENV_PATH }}
          source ${{ env.PYTHON_VENV_ACTIVATE }}
          export PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade -r requirements.txt
          rm -rf \
            ${{ env.ROCM_PATH }}/lib/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/lib/cmake/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/share/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/libexec/*rocprofiler-sdk*

      - name: Install Curl for RHEL 8.8
        if: ${{ matrix.system.os == 'rhel-8.8' }}
        run: |
          dnf install -y curl
          ln -s /usr/local/bin/curl /usr/bin/curl

      - name: Setup ccache
        uses: hendrikmuhs/ccache-action@63069e3931dedbf3b63792097479563182fe70d1 # v1.2.18
        with:
          key: ccache-${{ matrix.system.os }}-${{ matrix.system.runner }}-${{ matrix.system.gpu }}
          max-size: 2G
          save: true
          variant: sccache

      - name: Build and Install ROCProfiler-Register
        shell: bash
        working-directory: projects/rocprofiler-register
        run: |
          echo "Install ROCProfiler-Register"
          cmake -B build-rocprofiler-register \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            .
          cmake --build build-rocprofiler-register --target all --parallel 16
          cmake --build build-rocprofiler-register --target install
          echo "✅ ROCProfiler-Register Installation complete!"

      - name: Build and Install ROCR-Runtime
        if: ${{ env.ENABLE_ROCR_BUILD == 'true' }}
        shell: bash
        working-directory: projects/rocr-runtime
        run: |
          python3 -m venv ${{ env.PYTHON_VENV_PATH }}
          source ${{ env.PYTHON_VENV_ACTIVATE }}
          export PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade cmake
          echo "Install ROCR-Runtime..."
          cmake -B build \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }};${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            .
          cmake --build build --target all --parallel 16
          cmake --build build --target install
          echo "✅ ROCR-Runtime Installation complete!"

      - name: Build and Install HIP
        if: ${{ env.ENABLE_HIP_CLR_BUILD == 'true' }}
        shell: bash
        working-directory: projects
        run: |
          export HIP_DIR=$PWD/hip
          export CLR_DIR=$PWD/clr
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install HIP..."
          cd $CLR_DIR
          cmake \
            -DHIP_COMMON_DIR=$HIP_DIR \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DHIP_PLATFORM=amd \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }};${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DHIP_LLVM_ROOT=${{ env.ROCM_PATH }}/lib/llvm \
            -DHIP_CATCH_TEST=0 \
            -DCLR_BUILD_HIP=ON \
            -DCLR_BUILD_OCL=ON \
            -S $CLR_DIR \
            -B build
          cmake --build build --target all --parallel 16
          cmake --build build --target install
          echo "✅ HIP Installation complete!"

      - name: Build and Install Aqlprofile
        shell: bash
        working-directory: projects/aqlprofile
        run: |
          echo "Install Aqlprofile."
          python3 -m venv ${{ env.PYTHON_VENV_PATH }}
          source ${{ env.PYTHON_VENV_ACTIVATE }}
          export PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade cmake
          cmake -B build-aqlprofile \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            .
          cmake --build build-aqlprofile --target all --parallel 16
          cmake --build build-aqlprofile --target install
          echo "✅ AQLProfile Installation complete!"

      - name: Enable PC Sampling
        if: ${{ contains(matrix.system.gpu, 'mi200') || contains(matrix.system.gpu, 'mi300a') }}
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run:
            echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

      - name: List Files
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          source ${{ env.PYTHON_VENV_ACTIVATE }}
          echo "PATH: ${PATH}"
          echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
          which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
          for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
          cat ${{ env.ROCM_PATH }}/.info/version
          ls -la

      - name: Configure, Build, and Test
        timeout-minutes: 30
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          source ${{ env.PYTHON_VENV_ACTIVATE }}
          export PATH=~/.local/bin:/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 ./source/scripts/run-ci.py -B build \
            --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.system.os }}-${{ matrix.system.gpu }}-core \
            --mode ${CI_MODE} \
            --build-jobs 16 \
            --site ${{ matrix.system.runner }} \
            --gpu-targets ${{ env.GPU_TARGETS }} \
            --run-attempt ${{ github.run_attempt }} \
            ${{ matrix.system.ci-flags }} \
            -- \
            -DROCPROFILER_DEP_ROCMCORE=ON \
            -DROCPROFILER_BUILD_DOCS=OFF \
            -DROCPROFILER_BUILD_FMT=OFF \
            -DROCPROFILER_INTERNAL_RCCL_API_TRACE=ON \
            -DCMAKE_BUILD_TYPE=${{ matrix.system.build-type }} \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }};${{ env.ROCM_PATH }}/llvm' \
            -DPython3_EXECUTABLE=$(which python3) \
            ${{ env.GLOBAL_CMAKE_OPTIONS }} \
            -- \
            -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" \
            -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" \

  sanitizers:
    name: ${{ matrix.system.sanitizer }} • ${{ matrix.system.gpu }} • ${{ matrix.system.os }}
    strategy:
      fail-fast: false
      matrix:
        system:
          - { sanitizer: 'AddressSanitizer', os: 'ubuntu-22.04', runner: 'linux-mi325-1gpu-ossci-rocm-frac', gpu: 'mi325', gpu-target: 'gfx94X', build-type: 'RelWithDebInfo' }
          - { sanitizer: 'ThreadSanitizer', os: 'ubuntu-22.04', runner: 'linux-mi325-1gpu-ossci-rocm-frac', gpu: 'mi325', gpu-target: 'gfx94X', build-type: 'RelWithDebInfo' }
          - { sanitizer: 'LeakSanitizer', os: 'ubuntu-22.04', runner: 'linux-mi325-1gpu-ossci-rocm-frac', gpu: 'mi325', gpu-target: 'gfx94X', build-type: 'RelWithDebInfo' }
          - { sanitizer: 'UndefinedBehaviorSanitizer', os: 'ubuntu-22.04', runner: 'linux-mi325-1gpu-ossci-rocm-frac', gpu: 'mi325', gpu-target: 'gfx94X', build-type: 'RelWithDebInfo' }

    if: ${{ contains(github.event_name, 'pull_request') }}
    runs-on: ${{ matrix.system.runner }}
    container:
      image: docker.io/rocm/rocprofiler-private:${{ matrix.system.os }}-${{ matrix.system.gpu-target }}-latest
      credentials:
        username: ${{ secrets.ROCPROFILER_AZURE_CI_USER }}
        password: ${{ secrets.ROCPROFILER_AZURE_CI_PASS }}
      env:
        DEBIAN_FRONTEND: noninteractive
      options: --privileged --cap-add=SYS_PTRACE --security-opt seccomp=unconfined

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      GCC_COMPILER_VERSION: 13
      GPU_RUNNER: ${{ matrix.system.gpu }}

    steps:
    - name: Install Latest Nightly ROCm
      shell: bash
      working-directory: /tmp
      run: |
        ls -lah /opt/
        tar -xf ${{ env.ROCM_PATH }}-${{ matrix.system.gpu-target }}.tar.gz -C ${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}
        ln -s ${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} ${{ env.ROCM_PATH }}
        echo "ROCm installed to: ${{ env.ROCM_PATH }}"

    - name: Clone ROCProfiler SDK & AQLProfile & ROCProfiler Register & ROCR-Runtime
      uses: actions/checkout@v5
      with:
        sparse-checkout: |
          projects/rocprofiler-sdk
          projects/aqlprofile
          projects/rocprofiler-register
          projects/rocr-runtime
          projects/clr
          projects/hip
        submodules: false
        set-safe-directory: true

    - name: Compute submodule cache key
      id: submods
      shell: bash
      run: |
        git --version
        git config --global --add safe.directory '*'
        git submodule status --recursive | awk '{print $1,$2}' > .git-submodules-status
        echo "hash=$(sha256sum .git-submodules-status | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
        # collect submodule paths for cache 'path'
        git config --file .gitmodules --get-regexp path | awk '{print $2}' > .git-submodule-paths
        { echo "paths<<EOF"; cat .git-submodule-paths; echo "EOF"; } >> "$GITHUB_OUTPUT"

    - name: Restore submodule cache
      uses: actions/cache@v4
      with:
        path: |
          .git/modules
          ${{ steps.submods.outputs.paths }}
        key: submods-${{ runner.os }}-${{ steps.submods.outputs.hash }}
        restore-keys: |
          submods-${{ runner.os }}-
          submods-

    - name: Init/Update submodules
      run: git submodule update --init --recursive --jobs 16

    - name: Install requirements
      timeout-minutes: 10
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run: |
        git config --global --add safe.directory '*'
        apt-get update
        apt-get install -y build-essential cmake python3-pip libasan8 libtsan2 software-properties-common clang-15 libdw-dev libsqlite3-dev libdrm-dev file autoconf pkg-config
        add-apt-repository ppa:ubuntu-toolchain-r/test
        apt-get update
        apt-get upgrade -y
        apt-get install -y gcc-${{ env.GCC_COMPILER_VERSION }} g++-${{ env.GCC_COMPILER_VERSION }}
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_COMPILER_VERSION }} 100 --slave /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_COMPILER_VERSION }} --slave /usr/bin/gcov gcov /usr/bin/gcov-${{ env.GCC_COMPILER_VERSION }}
        python3 -m pip install -U --user -r requirements.txt
        rm -rf ${{ env.ROCM_PATH }}/lib/*rocprofiler-sdk* ${{ env.ROCM_PATH }}/lib/cmake/*rocprofiler-sdk* ${{ env.ROCM_PATH }}/share/*rocprofiler-sdk* ${{ env.ROCM_PATH }}/libexec/*rocprofiler-sdk* ${{ env.ROCM_PATH }}*/lib/python*/site-packages/roctx ${{ env.ROCM_PATH }}*/lib/python*/site-packages/rocpd

    - name: List Files
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run: |
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
        cat ${{ env.ROCM_PATH }}/.info/version
        ls -la

    - name: Enable PC Sampling
      if: ${{ contains(matrix.system.gpu, 'mi200') || contains(matrix.system.gpu, 'mi300a') }}
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run: echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

    - name: Setup ccache
      uses: hendrikmuhs/ccache-action@63069e3931dedbf3b63792097479563182fe70d1 # v1.2.18
      with:
        key: ccache-${{ matrix.system.os }}-${{ matrix.system.runner }}-${{ matrix.system.gpu }}-${{ matrix.system.sanitizer}}
        max-size: 2G
        save: true

    - name: Build and Install ROCProfiler-Register
      shell: bash
      working-directory: projects/rocprofiler-register
      run: |
        echo "Install ROCProfiler-Register"
        cmake -B build-rocprofiler-register \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
        cmake --build build-rocprofiler-register --target all --parallel 16
        cmake --build build-rocprofiler-register --target install
        echo "✅ ROCProfiler-Register Installation complete!"

    - name: Build and Install ROCR-Runtime
      if: ${{ env.ENABLE_ROCR_BUILD == 'true' }}
      shell: bash
      working-directory: projects/rocr-runtime
      run: |
        echo "Install ROCR-Runtime..."
        cmake -B build \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }};${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
        cmake --build build --target all --parallel 16
        cmake --build build --target install
        echo "✅ ROCR-Runtime Installation complete!"

    - name: Build and Install HIP
      if: ${{ env.ENABLE_HIP_CLR_BUILD == 'true' }}
      shell: bash
      working-directory: projects
      run: |
        export HIP_DIR=$PWD/hip
        export CLR_DIR=$PWD/clr
        export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
        export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
        echo "Install HIP..."
        cd $CLR_DIR
        cmake \
          -DHIP_COMMON_DIR=$HIP_DIR \
          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
          -DHIP_PLATFORM=amd \
          -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }};${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }}/llvm' \
          -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
          -DHIP_LLVM_ROOT=${{ env.ROCM_PATH }}/lib/llvm \
          -DHIP_CATCH_TEST=0 \
          -DCLR_BUILD_HIP=ON \
          -DCLR_BUILD_OCL=ON \
          -S $CLR_DIR \
          -B build
        cmake --build build --target all --parallel 16
        cmake --build build --target install
        echo "✅ HIP Installation complete!"

    - name: Build and Install Aqlprofile
      shell: bash
      working-directory: projects/aqlprofile
      run: |
        echo "Install Aqlprofile."
        cmake -B build-aqlprofile \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-${{ env.ROCM_VERSION }} \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
        cmake --build build-aqlprofile --target all --parallel 16
        cmake --build build-aqlprofile --target install
        echo "✅ Installation complete!"

    - name: Configure, Build, and Test
      timeout-minutes: 45
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run:
        sudo sysctl -w vm.mmap_rnd_bits=28;
        export PATH="$HOME/.local/bin:$PATH";
        python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.system.os }}-${{ matrix.system.gpu }}-${{ matrix.system.sanitizer }}
          --build-jobs 16
          --site ${{ matrix.system.runner }}
          --gpu-targets ${{ env.GPU_TARGETS }}
          --memcheck ${{ matrix.system.sanitizer }}
          --run-attempt ${{ github.run_attempt }}
          --
          -DROCPROFILER_BUILD_FMT=OFF \
          -DROCPROFILER_INTERNAL_RCCL_API_TRACE=ON \
          -DCMAKE_BUILD_TYPE=${{ matrix.system.build-type }}
          -DCMAKE_INSTALL_PREFIX="${{ env.ROCM_PATH }}"
          -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }};${{ env.ROCM_PATH }}/llvm'
          -DPython3_EXECUTABLE=$(which python3)
          ${{ env.GLOBAL_CMAKE_OPTIONS }}
          --
          -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}"