rocm-systems/.github/workflows/rocprofiler-sdk-continuous_integration.yml

name: rocprofiler-sdk Continuous Integration

on:
  workflow_dispatch:
  schedule:
    - cron: '0 7 * * *'
  push:
    branches: [ develop ]
    paths:
      - 'projects/rocprofiler-sdk/**'
      - '!projects/rocprofiler-sdk/*.md'
      - '!projects/rocprofiler-sdk/**/README.md'
      - '!projects/rocprofiler-sdk/CODEOWNERS'
      - '!projects/rocprofiler-sdk/source/docs/**'
      - '.github/workflows/rocprofiler-sdk-continuous_integration.yml'
  pull_request:
    paths:
      - 'projects/rocprofiler-sdk/**'
      - '!projects/rocprofiler-sdk/*.md'
      - '!projects/rocprofiler-sdk/**/README.md'
      - '!projects/rocprofiler-sdk/CODEOWNERS'
      - '!projects/rocprofiler-sdk/source/docs/**'
      - '.github/workflows/rocprofiler-sdk-continuous_integration.yml'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Needed to push/pull cached Docker layers + GHCR images
permissions:
  contents: read
  packages: write

env:
  # TODO(jrmadsen): replace LD_RUNPATH_FLAG, GPU_TARGETS, etc. with internal handling in cmake
  ROCM_PATH: "/opt/rocm"
  GPU_TARGETS: "gfx900 gfx906 gfx908 gfx90a gfx940 gfx941 gfx942 gfx950 gfx1030 gfx1100 gfx1101 gfx1102 gfx1201"
  PATH: "/usr/bin:$PATH"

  ## No tests should be excluded here except for extreme emergencies; tests should only be disabled in CMake
  ## A task should be assigned directly to fix the issues
  ## Scratch memory tests need to be fixed for ROCm 7.0 release
  navi3_EXCLUDE_TESTS_REGEX: ""
  vega20_EXCLUDE_TESTS_REGEX: ""
  mi200_EXCLUDE_TESTS_REGEX: ""
  mi300_EXCLUDE_TESTS_REGEX: ""
  mi300a_EXCLUDE_TESTS_REGEX: ""
  mi325_EXCLUDE_TESTS_REGEX: ""
  mi3xx_EXCLUDE_TESTS_REGEX: ""
  navi4_EXCLUDE_TESTS_REGEX: ""

  navi3_EXCLUDE_LABEL_REGEX: ""
  vega20_EXCLUDE_LABEL_REGEX: ""
  mi200_EXCLUDE_LABEL_REGEX: ""
  mi300_EXCLUDE_LABEL_REGEX: ""
  mi300a_EXCLUDE_LABEL_REGEX: ""
  mi325_EXCLUDE_LABEL_REGEX: ""
  mi3xx_EXCLUDE_LABEL_REGEX: ""
  navi4_EXCLUDE_LABEL_REGEX: ""

  GLOBAL_CMAKE_OPTIONS: ""
  DISABLE_ROCR_BUILD: "true"

jobs:
  # -----------------------------------------------------------------------------
  # Ubuntu / DEB job(s)
  # -----------------------------------------------------------------------------
  core-deb:
    name: Core • ${{ matrix.system.gpu }} • ${{ matrix.system.os }}
    strategy:
      fail-fast: false
      matrix:
        system:
          - { gpu: 'navi4', runner: 'rocprofiler-navi4-dind', os: 'ubuntu-22.04', build-type: 'RelWithDebInfo', therock-s3: "gfx120X", ci-flags: '--linter clang-tidy', gpu-target: "gfx1201" }
          - { gpu: 'navi3', runner: 'rocprofiler-navi3-dind', os: 'ubuntu-22.04', build-type: 'RelWithDebInfo', therock-s3: "gfx110X", ci-flags: '--linter clang-tidy', gpu-target: "gfx1101" }
          - { gpu: 'mi325', runner: 'linux-mi325-1gpu-ossci-rocm', os: 'ubuntu-22.04', build-type: 'RelWithDebInfo', therock-s3: "gfx94X",  ci-flags: '--linter clang-tidy', gpu-target: "gfx942" }
    runs-on: ${{ matrix.system.runner }}
    container:
      image: docker.io/rocm/rocprofiler-private:${{ matrix.system.os }}-${{ matrix.system.therock-s3 }}-latest
      credentials:
        username: ${{ secrets.ROCPROFILER_AZURE_CI_USER }}
        password: ${{ secrets.ROCPROFILER_AZURE_CI_PASS }}
      env:
        DEBIAN_FRONTEND: noninteractive
      options: --privileged
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      CORE_EXT_RUNNER: mi325
      GPU_RUNNER: ${{ matrix.system.gpu }}
    steps:
      - name: Install Latest Nightly ROCm
        shell: bash
        working-directory: /tmp
        run: |
          tar -xf /opt/rocm-${{ matrix.system.therock-s3 }}.tar.gz -C ${{ env.ROCM_PATH }}-7.0.0
          ln -s ${{ env.ROCM_PATH }}-7.0.0 ${{ env.ROCM_PATH }}
          echo "ROCm installed to: ${{ env.ROCM_PATH }}"

      - name: Clone ROCProfiler SDK & AQLProfile & ROCProfiler Register & ROCR-Runtime
        uses: actions/checkout@v5
        with:
          sparse-checkout: |
            projects/rocprofiler-sdk
            projects/aqlprofile
            projects/rocprofiler-register
            projects/rocr-runtime
          submodules: false
          set-safe-directory: true

      - name: Compute submodule cache key
        id: submods
        shell: bash
        run: |
          git config --global --add safe.directory '*'
          git submodule status --recursive | awk '{print $1,$2}' > .git-submodules-status
          echo "hash=$(sha256sum .git-submodules-status | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
          # collect submodule paths for cache 'path'
          git config --file .gitmodules --get-regexp path | awk '{print $2}' > .git-submodule-paths
          { echo "paths<<EOF"; cat .git-submodule-paths; echo "EOF"; } >> "$GITHUB_OUTPUT"

      - name: Restore submodule cache
        uses: actions/cache@v4
        with:
          path: |
            .git/modules
            ${{ steps.submods.outputs.paths }}
          key: submods-${{ runner.os }}-${{ steps.submods.outputs.hash }}
          restore-keys: |
            submods-${{ runner.os }}-
            submods-

      - name: Init/Update submodules
        run: git submodule update --init --recursive --jobs 16

      - name: Clone ROCDecode
        uses: actions/checkout@v5
        with:
          repository: 'ROCm/rocDecode'
          ref: 'release/rocm-rel-7.0'
          set-safe-directory: true
          path: 'rocDecode'

      - name: Clone ROCJPEG
        uses: actions/checkout@v5
        with:
          repository: 'ROCm/rocJPEG'
          ref: 'release/rocm-rel-7.0'
          set-safe-directory: true
          path: 'rocJPEG'

      - name: Install requirements
        timeout-minutes: 10
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          git config --global --add safe.directory '*'
          apt-get update
          apt-get install -y g++-11 g++-12 cmake python3-pip libdw-dev libsqlite3-dev libdrm-dev file autoconf pkg-config rpm libzstd-dev
          update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11
          update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 --slave /usr/bin/gcov gcov /usr/bin/gcov-12
          python3 -m pip install -U --user -r requirements.txt
          rm -rf \
            ${{ env.ROCM_PATH }}/lib/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/lib/cmake/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/share/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/libexec/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}*/lib/python*/site-packages/roctx \
            ${{ env.ROCM_PATH }}*/lib/python*/site-packages/rocpd

      - name: Setup ccache
        uses: hendrikmuhs/ccache-action@63069e3931dedbf3b63792097479563182fe70d1 # v1.2.18
        with:
          key: ccache-${{ matrix.system.os }}-${{ matrix.system.runner }}-${{ matrix.system.gpu }}
          max-size: 2G
          save: true

      - name: Install Missing ROCm Dependencies
        shell: bash
        run: |
          echo -e "Building & Installing ROCDecode..."
          cmake -B build-rocdecode \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }} \
            -DCMAKE_CXX_COMPILER=${{ env.ROCM_PATH }}/bin/amdclang++ \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            ${GITHUB_WORKSPACE}/rocDecode
          cmake --build build-rocdecode --target all --parallel 16
          cmake --build build-rocdecode --target install
          echo -e "ROCDecode Installed Successfully!"
          echo -e "Building & Installing ROCJPEG..."
          cmake -B build-rocjpeg \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }} \
            -DCMAKE_CXX_COMPILER=${{ env.ROCM_PATH }}/bin/amdclang++ \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            ${GITHUB_WORKSPACE}/rocJPEG
          cmake --build build-rocjpeg --target all --parallel 16
          cmake --build build-rocjpeg --target install
          echo -e "ROCJPEG Installed Successfully!"

      - name: Build and Install ROCProfiler-Register
        shell: bash
        working-directory: projects/rocprofiler-register
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install ROCProfiler-Register"
          cmake -B build-rocprofiler-register \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
          cmake --build build-rocprofiler-register --target all --parallel 16
          cmake --build build-rocprofiler-register --target install
          echo "✅ ROCProfiler-Register Installation complete!"

      - name: Build and Install ROCR-Runtime
        if: ${{ !contains(env.DISABLE_ROCR_BUILD, 'true') }}
        shell: bash
        working-directory: projects/rocr-runtime
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install ROCR-Runtime..."
          cmake -B build \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-7.0.0;${{ env.ROCM_PATH }}-7.0.0/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            .
          cmake --build build --target all --parallel 16
          cmake --build build --target install
          echo "✅ ROCR-Runtime Installation complete!"

      - name: Build and Install Aqlprofile
        shell: bash
        working-directory: projects/aqlprofile
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          echo "Install Aqlprofile..."
          cmake -B build-aqlprofile \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
          cmake --build build-aqlprofile --target all --parallel 16
          cmake --build build-aqlprofile --target install
          echo "✅ AQLProfile Installation complete!"

      - name: List Files
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          echo "PATH: ${PATH}"
          echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
          which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
          for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
          cat /opt/rocm/.info/version
          ls -la

      - name: Enable PC Sampling
        if: ${{ contains(matrix.system.gpu, 'mi200') || contains(matrix.system.gpu, 'mi300a') }}
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

      - name: Configure, Build, and Test
        timeout-minutes: 30
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH \
          PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH \
          python3 ./source/scripts/run-ci.py \
            -B build \
            --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.system.os }}-${{ matrix.system.gpu }}-core \
            --build-jobs 16 \
            --site ${{ matrix.system.runner }} \
            --gpu-targets ${{ env.GPU_TARGETS }} \
            --run-attempt ${{ github.run_attempt }} \
            ${{ matrix.system.ci-flags }} -- \
            -DROCPROFILER_DEP_ROCMCORE=ON \
            -DROCPROFILER_BUILD_DOCS=OFF \
            -DCMAKE_BUILD_TYPE=${{ matrix.system.build-type }} \
            -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-sdk \
            -DCPACK_GENERATOR='DEB;RPM;TGZ' \
            -DCPACK_PACKAGING_INSTALL_PREFIX="$(realpath /opt/rocm)" \
            -DPython3_EXECUTABLE=$(which python3) \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }};${{ env.ROCM_PATH }}/llvm' \
            ${{ env.GLOBAL_CMAKE_OPTIONS }} -- \
            -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" \
            -E  "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}"

      - name: Install
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 10
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          cmake --build build --target install --parallel 16

      - name: Build Packaging
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 10
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          cmake --build build --target package --parallel 16

      - name: Test Install Build
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 20
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-samples samples
          CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-tests -DGPU_TARGETS="gfx942" tests
          export LD_LIBRARY_PATH=/opt/rocprofiler-sdk/lib:${LD_LIBRARY_PATH}
          cmake --build build-samples --target all --parallel 16
          cmake --build build-tests --target all --parallel 16
          ctest --test-dir build-samples -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
          ctest --test-dir build-tests   -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

      - name: Install Packages
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 5
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          export PATH=${PATH}:/usr/local/sbin:/usr/sbin:/sbin
          ls -la
          ls -la ./build
          dpkg --force-all -i ./build/rocprofiler-sdk-roctx_*.deb
          dpkg --force-all -i ./build/rocprofiler-sdk-rocpd_*.deb
          for i in $(ls -S ./build/rocprofiler-sdk*.deb | egrep -v 'roctx|rocpd'); do dpkg --force-all -i ${i}; done;

      - name: Test Installed Packages
        if: ${{ contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        timeout-minutes: 20
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          export LD_LIBRARY_PATH=${{ env.ROCM_PATH }}/lib:${{ env.ROCM_PATH }}/llvm/lib:$LD_LIBRARY_PATH
          export PATH=${{ env.ROCM_PATH }}/bin:${{ env.ROCM_PATH }}/llvm/bin:$PATH
          CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-samples-deb /opt/rocm/share/rocprofiler-sdk/samples
          CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-tests-deb -DGPU_TARGETS="gfx942" /opt/rocm/share/rocprofiler-sdk/tests
          cmake --build build-samples-deb --target all --parallel 16
          cmake --build build-tests-deb --target all --parallel 16
          ctest --test-dir build-samples-deb -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure
          ctest --test-dir build-tests-deb   -LE "${${{ matrix.system.gpu }}_EXCLUDE_LABEL_REGEX}" -E "${${{ matrix.system.gpu }}_EXCLUDE_TESTS_REGEX}" --output-on-failure

      - name: Archive production artifacts
        if: ${{ github.event_name == 'workflow_dispatch' && contains(matrix.system.gpu, env.CORE_EXT_RUNNER) }}
        uses: actions/upload-artifact@v4
        with:
          name: installers-deb
          path: |
            ${{github.workspace}}/build/*.deb
            ${{github.workspace}}/build/*.rpm
            ${{github.workspace}}/build/*.tgz

  # -----------------------------------------------------------------------------
  # RHEL/SLES (RPM) job(s)
  # -----------------------------------------------------------------------------
  core-rpm:
    name: Core • ${{ matrix.runner }} • ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        runner: [ 'mi325' ]
        os: [ 'rhel-8.8', 'rhel-9.5', 'sles-15.6' ]
        build-type: [ 'RelWithDebInfo' ]
        ci-flags: [ '' ]
    runs-on: linux-mi325-1gpu-ossci-rocm
    container:
      image: docker.io/rocm/rocprofiler-private:${{ matrix.os }}-gfx94X-latest
      credentials:
        username: ${{ secrets.ROCPROFILER_AZURE_CI_USER }}
        password: ${{ secrets.ROCPROFILER_AZURE_CI_PASS }}
      options: --privileged
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      OS_TYPE: ${{ matrix.os }}
      GPU_RUNNER: ${{ matrix.runner }}
    steps:
      - name: Clone ROCProfiler SDK & AQLProfile & ROCProfiler Register & ROCR-Runtime
        uses: actions/checkout@v5
        with:
          sparse-checkout: |
            projects/rocprofiler-sdk
            projects/aqlprofile
            projects/rocprofiler-register
            projects/rocr-runtime
          submodules: false
          set-safe-directory: true

      - name: Compute submodule cache key
        id: submods
        shell: bash
        run: |
          git config --global --add safe.directory '*'
          git submodule status --recursive | awk '{print $1,$2}' > .git-submodules-status
          echo "hash=$(sha256sum .git-submodules-status | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
          # collect submodule paths for cache 'path'
          git config --file .gitmodules --get-regexp path | awk '{print $2}' > .git-submodule-paths
          { echo "paths<<EOF"; cat .git-submodule-paths; echo "EOF"; } >> "$GITHUB_OUTPUT"

      - name: Restore submodule cache
        uses: actions/cache@v4
        with:
          path: |
            .git/modules
            ${{ steps.submods.outputs.paths }}
          key: submods-${{ runner.os }}-${{ steps.submods.outputs.hash }}
          restore-keys: |
            submods-${{ runner.os }}-
            submods-

      - name: Init/Update submodules
        run: git submodule update --init --recursive --jobs 16

      - name: Install Latest Nightly ROCm using TheRock Tarballs
        shell: bash
        working-directory: /tmp
        run: |
          tar -xf /opt/rocm-gfx94X.tar.gz -C ${{ env.ROCM_PATH }}-7.0.0
          ln -s ${{ env.ROCM_PATH }}-7.0.0 ${{ env.ROCM_PATH }}
          echo "ROCm installed to: ${{ env.ROCM_PATH }}"

      - name: Install requirements (venv)
        timeout-minutes: 10
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          git config --global --add safe.directory '*'
          python3 -m venv rocprofiler-sdk
          source rocprofiler-sdk/bin/activate
          export PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade -r requirements.txt
          rm -rf \
            ${{ env.ROCM_PATH }}/lib/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/lib/cmake/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/share/*rocprofiler-sdk* \
            ${{ env.ROCM_PATH }}/libexec/*rocprofiler-sdk*

      - name: Install Curl for RHEL 8.8
        if: ${{ matrix.os == 'rhel-8.8' }}
        run: |
          dnf install -y curl
          ln -s /usr/local/bin/curl /usr/bin/curl

      - name: Setup ccache
        uses: hendrikmuhs/ccache-action@63069e3931dedbf3b63792097479563182fe70d1 # v1.2.18
        with:
          key: ccache-${{ matrix.os }}-linux-mi325-1gpu-ossci-rocm-${{ matrix.runner }}
          max-size: 2G
          save: true
          variant: sccache

      - name: Build and Install ROCProfiler-Register
        shell: bash
        working-directory: projects/rocprofiler-register
        run: |
          echo "Install ROCProfiler-Register"
          cmake -B build-rocprofiler-register \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            .
          cmake --build build-rocprofiler-register --target all --parallel 16
          cmake --build build-rocprofiler-register --target install
          echo "✅ ROCProfiler-Register Installation complete!"

      - name: Build and Install ROCR-Runtime
        if: ${{ !contains(env.DISABLE_ROCR_BUILD, 'true') }}
        shell: bash
        working-directory: projects/rocr-runtime
        run: |
          python3 -m venv rocprofiler-sdk
          source rocprofiler-sdk/bin/activate
          export PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade cmake
          echo "Install ROCR-Runtime..."
          cmake -B build \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-7.0.0;${{ env.ROCM_PATH }}-7.0.0/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            .
          cmake --build build --target all --parallel 16
          cmake --build build --target install
          echo "✅ ROCR-Runtime Installation complete!"

      - name: Build and Install Aqlprofile
        shell: bash
        working-directory: projects/aqlprofile
        run: |
          echo "Install Aqlprofile."
          python3 -m venv rocprofiler-sdk
          source rocprofiler-sdk/bin/activate
          export PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade cmake
          cmake -B build-aqlprofile \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/local/bin/sccache \
            .
          cmake --build build-aqlprofile --target all --parallel 16
          cmake --build build-aqlprofile --target install
          echo "✅ AQLProfile Installation complete!"

      - name: Enable PC Sampling
        if: ${{ contains(matrix.runner, 'mi200') || contains(matrix.runner, 'mi300a') }}
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run:
            echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

      - name: List Files
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run: |
          source rocprofiler-sdk/bin/activate
          echo "PATH: ${PATH}"
          echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
          which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
          for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
          cat /opt/rocm/.info/version
          ls -la

      - name: Configure, Build, and Test
        timeout-minutes: 30
        shell: bash
        working-directory: projects/rocprofiler-sdk
        run:
          source rocprofiler-sdk/bin/activate;
          PATH=~/.local/bin:/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
          python3 ./source/scripts/run-ci.py -B build
            --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-core
            --build-jobs 16
            --site 'linux-mi325-1gpu-ossci-rocm'
            --gpu-targets ${{ env.GPU_TARGETS }}
            --run-attempt ${{ github.run_attempt }}
            ${{ matrix.ci-flags }}
            --
            -DROCPROFILER_DEP_ROCMCORE=ON
            -DROCPROFILER_BUILD_DOCS=OFF
            -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }};${{ env.ROCM_PATH }}/llvm'
            -DPython3_EXECUTABLE=$(which python3)
            ${{ env.GLOBAL_CMAKE_OPTIONS }}
            --
            -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
            -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"

  sanitizers:
    name: ${{ matrix.sanitizer }} • ${{ matrix.runner }} • ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        runner: ['mi325']
        sanitizer: ['AddressSanitizer', 'ThreadSanitizer', 'LeakSanitizer', 'UndefinedBehaviorSanitizer']
        os: ['ubuntu-22.04']
        build-type: ['RelWithDebInfo']

    if: ${{ contains(github.event_name, 'pull_request') }}
    runs-on: linux-mi325-1gpu-ossci-rocm
    container:
      image: docker.io/rocm/rocprofiler-private:${{ matrix.os }}-gfx94X-latest
      credentials:
        username: ${{ secrets.ROCPROFILER_AZURE_CI_USER }}
        password: ${{ secrets.ROCPROFILER_AZURE_CI_PASS }}
      env:
        DEBIAN_FRONTEND: noninteractive
      options: --privileged --cap-add=SYS_PTRACE --security-opt seccomp=unconfined

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
      GCC_COMPILER_VERSION: 13
      GPU_RUNNER: ${{ matrix.runner }}

    steps:
    - name: Install Latest Nightly ROCm
      shell: bash
      working-directory: /tmp
      run: |
        ls -lah /opt/
        tar -xf /opt/rocm-gfx94X.tar.gz -C ${{ env.ROCM_PATH }}-7.0.0
        ln -s ${{ env.ROCM_PATH }}-7.0.0 ${{ env.ROCM_PATH }}
        echo "ROCm installed to: ${{ env.ROCM_PATH }}"

    - name: Clone ROCProfiler SDK & AQLProfile & ROCProfiler Register & ROCR-Runtime
      uses: actions/checkout@v5
      with:
        sparse-checkout: |
          projects/rocprofiler-sdk
          projects/aqlprofile
          projects/rocprofiler-register
          projects/rocr-runtime
        submodules: false
        set-safe-directory: true

    - name: Compute submodule cache key
      id: submods
      shell: bash
      run: |
        git config --global --add safe.directory '*'
        git submodule status --recursive | awk '{print $1,$2}' > .git-submodules-status
        echo "hash=$(sha256sum .git-submodules-status | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
        # collect submodule paths for cache 'path'
        git config --file .gitmodules --get-regexp path | awk '{print $2}' > .git-submodule-paths
        { echo "paths<<EOF"; cat .git-submodule-paths; echo "EOF"; } >> "$GITHUB_OUTPUT"

    - name: Restore submodule cache
      uses: actions/cache@v4
      with:
        path: |
          .git/modules
          ${{ steps.submods.outputs.paths }}
        key: submods-${{ runner.os }}-${{ steps.submods.outputs.hash }}
        restore-keys: |
          submods-${{ runner.os }}-
          submods-

    - name: Init/Update submodules
      run: git submodule update --init --recursive --jobs 16

    - name: Install requirements
      timeout-minutes: 10
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run: |
        git config --global --add safe.directory '*'
        apt-get update
        apt-get install -y build-essential cmake python3-pip libasan8 libtsan2 software-properties-common clang-15 libdw-dev libsqlite3-dev libdrm-dev file autoconf pkg-config
        add-apt-repository ppa:ubuntu-toolchain-r/test
        apt-get update
        apt-get upgrade -y
        apt-get install -y gcc-${{ env.GCC_COMPILER_VERSION }} g++-${{ env.GCC_COMPILER_VERSION }}
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_COMPILER_VERSION }} 100 --slave /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_COMPILER_VERSION }} --slave /usr/bin/gcov gcov /usr/bin/gcov-${{ env.GCC_COMPILER_VERSION }}
        python3 -m pip install -U --user -r requirements.txt
        rm -rf ${{ env.ROCM_PATH }}/lib/*rocprofiler-sdk* ${{ env.ROCM_PATH }}/lib/cmake/*rocprofiler-sdk* ${{ env.ROCM_PATH }}/share/*rocprofiler-sdk* ${{ env.ROCM_PATH }}/libexec/*rocprofiler-sdk* ${{ env.ROCM_PATH }}*/lib/python*/site-packages/roctx ${{ env.ROCM_PATH }}*/lib/python*/site-packages/rocpd

    - name: List Files
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run: |
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
        cat /opt/rocm/.info/version
        ls -la

    - name: Enable PC Sampling
      if: ${{ contains(matrix.runner, 'mi200') || contains(matrix.runner, 'mi300a') }}
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run: echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV

    - name: Setup ccache
      uses: hendrikmuhs/ccache-action@63069e3931dedbf3b63792097479563182fe70d1 # v1.2.18
      with:
        key: ccache-${{ matrix.os }}-linux-mi325-1gpu-ossci-rocm-${{ matrix.runner }}-${{ matrix.sanitizer}}
        max-size: 2G
        save: true

    - name: Build and Install ROCProfiler-Register
      shell: bash
      working-directory: projects/rocprofiler-register
      run: |
        echo "Install ROCProfiler-Register"
        cmake -B build-rocprofiler-register \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
        cmake --build build-rocprofiler-register --target all --parallel 16
        cmake --build build-rocprofiler-register --target install
        echo "✅ ROCProfiler-Register Installation complete!"

    - name: Build and Install ROCR-Runtime
      if: ${{ !contains(env.DISABLE_ROCR_BUILD, 'true') }}
      shell: bash
      working-directory: projects/rocr-runtime
      run: |
        echo "Install ROCR-Runtime..."
        cmake -B build \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }}-7.0.0;${{ env.ROCM_PATH }}-7.0.0/llvm' \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
        cmake --build build --target all --parallel 16
        cmake --build build --target install
        echo "✅ ROCR-Runtime Installation complete!"

    - name: Build and Install Aqlprofile
      shell: bash
      working-directory: projects/aqlprofile
      run: |
        echo "Install Aqlprofile."
        cmake -B build-aqlprofile \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
            -DCMAKE_PREFIX_PATH=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_INSTALL_PREFIX=${{ env.ROCM_PATH }}-7.0.0 \
            -DCMAKE_C_COMPILER_LAUNCHER=/usr/bin/ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=/usr/bin/ccache \
            .
        cmake --build build-aqlprofile --target all --parallel 16
        cmake --build build-aqlprofile --target install
        echo "✅ Installation complete!"

    - name: Configure, Build, and Test
      timeout-minutes: 45
      shell: bash
      working-directory: projects/rocprofiler-sdk
      run:
        sudo sysctl -w vm.mmap_rnd_bits=28;
        python3 ./source/scripts/run-ci.py -B build
          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.runner }}-${{ matrix.sanitizer }}
          --build-jobs 16
          --site 'linux-mi325-1gpu-ossci-rocm'
          --gpu-targets ${{ env.GPU_TARGETS }}
          --memcheck ${{ matrix.sanitizer }}
          --run-attempt ${{ github.run_attempt }}
          --
          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
          -DCMAKE_INSTALL_PREFIX="${{ env.ROCM_PATH }}"
          -DCMAKE_PREFIX_PATH='${{ env.ROCM_PATH }};${{ env.ROCM_PATH }}/llvm'
          -DPython3_EXECUTABLE=$(which python3)
          ${{ env.GLOBAL_CMAKE_OPTIONS }}
          --
          -LE "${${{ matrix.runner }}_EXCLUDE_LABEL_REGEX}"
          -E "${${{ matrix.runner }}_EXCLUDE_TESTS_REGEX}"