diff --git a/.github/workflows/rocprofiler-systems-debian.yml b/.github/workflows/rocprofiler-systems-debian.yml new file mode 100644 index 0000000000..0e31ccf318 --- /dev/null +++ b/.github/workflows/rocprofiler-systems-debian.yml @@ -0,0 +1,188 @@ +name: rocprofiler-systems Debian (GCC, Python, ROCm) +run-name: debian + +on: + push: + branches: + - develop + paths: + - '.github/workflows/rocprofiler-systems-debian.yml' + - 'projects/rocprofiler-systems/**' + - '!projects/rocprofiler-systems/*.md' + - '!projects/rocprofiler-systems/docs/**' + - '!projects/rocprofiler-systems/source/docs/**' + - '!projects/rocprofiler-systems/source/python/gui/**' + - '!projects/rocprofiler-systems/docker/**' + - '!projects/rocprofiler-systems/.wordlist.txt' + - '!projects/rocprofiler-systems/CMakePresets.json' + pull_request: + paths: + - '.github/workflows/rocprofiler-systems-debian.yml' + - 'projects/rocprofiler-systems/**' + - '!projects/rocprofiler-systems/*.md' + - '!projects/rocprofiler-systems/docs/**' + - '!projects/rocprofiler-systems/source/docs/**' + - '!projects/rocprofiler-systems/source/python/gui/**' + - '!projects/rocprofiler-systems/docker/**' + - '!projects/rocprofiler-systems/.wordlist.txt' + - '!projects/rocprofiler-systems/CMakePresets.json' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + ROCPROFSYS_CI: ON + ROCPROFSYS_TMPDIR: "%env{PWD}%/testing-tmp" + +jobs: + debian: + runs-on: ubuntu-latest + container: + image: dgaliffiamd/rocprofiler-systems:ci-base-debian-${{ matrix.os-release }} + strategy: + fail-fast: false + matrix: + compiler: ['g++'] + python: ['ON'] + build-type: ['Release'] + build-dyninst: ['ON'] + os-release: ['12'] + rocm-version: ['6.4'] + env: + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + ROCPROFSYS_CI: 'ON' + + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: projects/rocprofiler-systems/ + + - name: Install Packages + timeout-minutes: 25 + uses: nick-fields/retry@v3 + with: + retry_wait_seconds: 30 + timeout_minutes: 25 + max_attempts: 5 + command: | + apt-get update && + apt-get upgrade -y && + apt-get install -y libomp-dev libopenmpi-dev openmpi-bin + + - name: Install ROCm Packages + if: ${{ matrix.rocm-version > 0 }} + timeout-minutes: 115 + shell: bash + working-directory: projects/rocprofiler-systems/ + run: | + ROCM_VERSION=${{ matrix.rocm-version }} + ROCM_MAJOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $1}') + ROCM_MINOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $2}') + ROCM_VERSN=$(( (${ROCM_MAJOR}*10000)+(${ROCM_MINOR}*100) )) + echo "ROCM_MAJOR=${ROCM_MAJOR} ROCM_MINOR=${ROCM_MINOR} ROCM_VERSN=${ROCM_VERSN}" + wget -q https://repo.radeon.com/amdgpu-install/${{ matrix.rocm-version }}/ubuntu/jammy/amdgpu-install_${ROCM_MAJOR}.${ROCM_MINOR}.${ROCM_VERSN}-1_all.deb + apt-get install -y ./amdgpu-install_${ROCM_MAJOR}.${ROCM_MINOR}.${ROCM_VERSN}-1_all.deb + apt-get update + apt-get install -y rocm-dev rocdecode-dev libavformat-dev libavcodec-dev + + - name: Configure, Build and Test + timeout-minutes: 115 + shell: bash + working-directory: projects/rocprofiler-systems/ + env: + GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} && + git config --global --add safe.directory ${PWD} + echo "CMake version: $(cmake --version | head -n 1)" + echo "Compiler version: $(${{ matrix.compiler }} --version | head -n 1)" + echo "ROCm version: ${{ matrix.rocm-version }}" + USE_ROCM=OFF + if [ "${{ matrix.rocm-version }}" != "0.0" ]; then + CMAKE_PREFIX_PATH_ARG="-DCMAKE_PREFIX_PATH=/opt/rocm" + USE_ROCM=ON + else + CMAKE_PREFIX_PATH_ARG="" + fi + TAG="" + python3 ./scripts/run-ci.py -B build \ + --name ${{ github.repository_owner }}-${{ github.ref_name }}-debian-${{matrix.os-release}}-${{ matrix.compiler }}${TAG} \ + --build-jobs 2 \ + --site GitHub \ + -- \ + -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') \ + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} \ + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ + -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-systems \ + -DROCPROFSYS_BUILD_TESTING=ON \ + -DROCPROFSYS_USE_ROCM=$USE_ROCM \ + -DROCPROFSYS_USE_PYTHON=ON \ + -DROCPROFSYS_BUILD_DYNINST=${{ matrix.build-dyninst }} \ + -DROCPROFSYS_BUILD_BOOST=${{ matrix.build-dyninst }} \ + -DROCPROFSYS_BUILD_TBB=${{ matrix.build-dyninst }} \ + -DROCPROFSYS_BUILD_ELFUTILS=${{ matrix.build-dyninst }} \ + -DROCPROFSYS_BUILD_LIBIBERTY=${{ matrix.build-dyninst }} \ + -DROCPROFSYS_PYTHON_PREFIX=/opt/conda/envs \ + -DROCPROFSYS_PYTHON_ENVS="py3.8;py3.9;py3.10;py3.11;py3.12;py3.13" \ + -DROCPROFSYS_MAX_THREADS=64 \ + -DROCPROFSYS_DISABLE_EXAMPLES="transpose;rccl;openmp-target" \ + -DROCPROFSYS_BUILD_NUMBER=1 \ + -DUSE_CLANG_OMP=OFF \ + $CMAKE_PREFIX_PATH_ARG \ + -- \ + -LE "transpose|rccl|videodecode|jpegdecode|network" + + - name: Install + timeout-minutes: 10 + working-directory: projects/rocprofiler-systems/ + run: | + cmake --build build --target install --parallel 2 + + - name: CPack and Install + working-directory: projects/rocprofiler-systems/ + run: | + cd build + cpack -G STGZ + mkdir -p /opt/rocprofiler-systems + ./rocprofiler-systems-*.sh --prefix=/opt/rocprofiler-systems --exclude-subdir --skip-license + + - name: Test Install with Modulefile + working-directory: projects/rocprofiler-systems/ + timeout-minutes: 15 + shell: bash + run: | + set -v + source /usr/share/modules/init/$(basename ${SHELL}) + module use /opt/rocprofiler-systems/share/modulefiles + module avail + module load rocprofiler-systems + ./scripts/test-install.sh --test-rocprof-sys-{instrument,avail,sample,python,rewrite,runtime}=1 + + - name: Test User API + timeout-minutes: 10 + working-directory: projects/rocprofiler-systems/ + run: | + set -v + ./scripts/test-find-package.sh --install-dir /opt/rocprofiler-systems + + - name: CTest Artifacts + if: failure() + continue-on-error: True + uses: actions/upload-artifact@v4 + with: + name: ctest-${{ github.job }}-${{ strategy.job-index }}-log + path: | + projects/rocprofiler-systems/build/*.log + + - name: Data Artifacts + if: failure() + continue-on-error: True + uses: actions/upload-artifact@v4 + with: + name: data-${{ github.job }}-${{ strategy.job-index }}-files + path: | + projects/rocprofiler-systems/build/rocprofsys-tests-config/*.cfg + projects/rocprofiler-systems/build/rocprofsys-tests-output/**/*.txt + projects/rocprofiler-systems/build/rocprofsys-tests-output/**/*-instr*.json diff --git a/projects/rocprofiler-systems/docker/Dockerfile.ubuntu b/projects/rocprofiler-systems/docker/Dockerfile.ubuntu index 709dafead2..4992f055f9 100644 --- a/projects/rocprofiler-systems/docker/Dockerfile.ubuntu +++ b/projects/rocprofiler-systems/docker/Dockerfile.ubuntu @@ -3,20 +3,20 @@ ARG DISTRO ARG VERSION FROM ${DISTRO}:${VERSION} -ENV HOME /root -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US -ENV LC_ALL C -ENV SHELL /bin/bash -ENV BASH_ENV /etc/bash.bashrc -ENV DEBIAN_FRONTEND noninteractive +ENV HOME=/root +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US +ENV LC_ALL=C +ENV SHELL=/bin/bash +ENV BASH_ENV=/etc/bash.bashrc +ENV DEBIAN_FRONTEND=noninteractive WORKDIR /tmp SHELL [ "/bin/bash", "-c" ] ARG EXTRA_PACKAGES="" ARG ROCM_VERSION="0.0" -ENV PATH ${HOME}/.local/bin:${PATH} +ENV PATH=${HOME}/.local/bin:${PATH} RUN apt-get update && \ apt-get dist-upgrade -y && \ @@ -24,11 +24,12 @@ RUN apt-get update && \ build-essential chrpath cmake curl flex gettext git-core gnupg2 iproute2 \ libnuma1 libopenmpi-dev libpapi-dev libpfm4-dev librpm-dev libsqlite3-dev \ libtool libudev1 lsb-release m4 ninja-build python3-pip rpm texinfo wget && \ - OS_VERSION=$(cat /etc/os-release | grep VERSION_ID | sed 's/=/ /'1 | awk '{print $NF}' | sed 's/"//g') && \ - if [ "${OS_VERSION}" == "24.04" ]; then \ - python3 -m pip install --break-system-packages 'cmake==3.21'; \ - else \ + OS_VERSION=$(grep '^VERSION_ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') && \ + OS_ID=$(grep '^ID=' /etc/os-release | cut -d'=' -f2 | tr -d '"') && \ + if [ "${OS_ID}" == "ubuntu" ] && [ "${OS_VERSION}" == "22.04" ]; then \ python3 -m pip install 'cmake==3.21'; \ + else \ + python3 -m pip install --break-system-packages 'cmake==3.21' perfetto; \ fi RUN ROCM_MAJOR=$(echo "${ROCM_VERSION}" | sed 's/\./ /g' | awk '{print $1}') && \ @@ -62,6 +63,6 @@ RUN wget https://github.com/conda-forge/miniforge/releases/latest/download/Minif conda clean -a -y && \ conda init -ENV LC_ALL C.UTF-8 +ENV LC_ALL=C.UTF-8 WORKDIR /home SHELL [ "/bin/bash", "--login", "-c" ] diff --git a/projects/rocprofiler-systems/docker/Dockerfile.ubuntu.ci b/projects/rocprofiler-systems/docker/Dockerfile.ubuntu.ci index 2bdda3b1c3..8ea3ca3512 100644 --- a/projects/rocprofiler-systems/docker/Dockerfile.ubuntu.ci +++ b/projects/rocprofiler-systems/docker/Dockerfile.ubuntu.ci @@ -3,11 +3,11 @@ ARG DISTRO ARG VERSION FROM ${DISTRO}:${VERSION} -ENV HOME /root -ENV LANG C.UTF-8 -ENV SHELL /bin/bash -ENV BASH_ENV /etc/bash.bashrc -ENV DEBIAN_FRONTEND noninteractive +ENV HOME=/root +ENV LANG=C.UTF-8 +ENV SHELL=/bin/bash +ENV BASH_ENV=/etc/bash.bashrc +ENV DEBIAN_FRONTEND=noninteractive WORKDIR /tmp SHELL [ "/bin/bash", "-c" ] @@ -18,10 +18,10 @@ ARG BOOST_DOWNLOAD_VERSION="1.79.0" ARG NJOBS="8" -ENV PATH /usr/local/bin:${PATH} -ENV LIBRARY_PATH /usr/local/lib:/usr/local/lib64:${LIBRARY_PATH} -ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib64:${LD_LIBRARY_PATH} -ENV CMAKE_PREFIX_PATH /usr/local:${CMAKE_PREFIX_PATH} +ENV PATH="/usr/local/bin:${PATH}" +ENV LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:${LIBRARY_PATH}" +ENV LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:${LD_LIBRARY_PATH}" +ENV CMAKE_PREFIX_PATH="/usr/local:${CMAKE_PREFIX_PATH}" RUN apt-get update && \ apt-get dist-upgrade -y && \ @@ -30,11 +30,13 @@ RUN apt-get update && \ gzip iproute2 libiberty-dev libpapi-dev libpfm4-dev libsqlite3-dev libtool \ locales lsb-release m4 ninja-build python3-pip software-properties-common \ texinfo unzip wget vim zip zlib1g-dev && \ - apt-get autoclean && \ - if [ "${OS_VERSION}" == "24.04" ]; then \ - python3 -m pip install --break-system-packages 'cmake==3.21' perfetto \ + apt-get autoclean + +RUN if ! pip install 'cmake==3.21' perfetto 2>&1 | tee pip_output.log | grep -q "externally managed environment"; then \ + echo "pip install succeeded without --break-system-packages"; \ else \ - python3 -m pip install 'cmake==3.21' perfetto; \ + echo "Retrying with --break-system-packages"; \ + pip install --break-system-packages 'cmake==3.21' perfetto; \ fi ARG PYTHON_VERSIONS="6 7 8 9 10 11 12 13" @@ -52,6 +54,6 @@ RUN wget https://github.com/conda-forge/miniforge/releases/latest/download/Minif rm -rf * -ENV LC_ALL C.UTF-8 +ENV LC_ALL=C.UTF-8 WORKDIR /home SHELL [ "/bin/bash", "--login", "-c" ] diff --git a/projects/rocprofiler-systems/docker/build-docker-ci.sh b/projects/rocprofiler-systems/docker/build-docker-ci.sh index e6694351b8..d24c3ae5ee 100755 --- a/projects/rocprofiler-systems/docker/build-docker-ci.sh +++ b/projects/rocprofiler-systems/docker/build-docker-ci.sh @@ -4,9 +4,9 @@ set -e : ${USER:=$(whoami)} : ${DISTRO:=ubuntu} -: ${VERSIONS:=20.04} +: ${VERSIONS:=24.04} : ${NJOBS=$(nproc)} -: ${ELFUTILS_VERSION:=0.186} +: ${ELFUTILS_VERSION:=0.188} : ${BOOST_VERSION:=1.79.0} : ${PYTHON_VERSIONS:="6 7 8 9 10 11 12 13"} : ${PUSH:=0} @@ -38,7 +38,7 @@ usage() echo "" print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } - print_default_option distro "[ubuntu|opensuse|rhel]" "OS distribution" "${DISTRO}" + print_default_option distro "[ubuntu|opensuse|rhel|debian]" "OS distribution" "${DISTRO}" print_default_option versions "[VERSION] [VERSION...]" "Ubuntu, OpenSUSE, or RHEL release" "${VERSIONS}" print_default_option python-versions "[VERSION] [VERSION...]" "Python 3 minor releases" "${PYTHON_VERSIONS}" print_default_option "jobs -j" "[N]" "parallel build jobs" "${NJOBS}" @@ -124,7 +124,11 @@ do shift done -DOCKER_FILE=Dockerfile.${DISTRO}.ci +if [ "${DISTRO}" = "debian" ]; then + DOCKER_FILE=Dockerfile.ubuntu.ci +else + DOCKER_FILE=Dockerfile.${DISTRO}.ci +fi if [ ! -f ${DOCKER_FILE} ]; then cd docker; fi diff --git a/projects/rocprofiler-systems/docker/containers-ci.yml b/projects/rocprofiler-systems/docker/containers-ci.yml index 5b4f19edb7..9b19a75a3a 100644 --- a/projects/rocprofiler-systems/docker/containers-ci.yml +++ b/projects/rocprofiler-systems/docker/containers-ci.yml @@ -6,6 +6,9 @@ matrix: version: "22.04" - distro: "ubuntu" version: "24.04" + # debian + - distro: "debian" + version: "12" # opensuse - distro: "opensuse" version: "15.5"