From 22eaa780eccf3d1fde79bd1532c99ea5951fbd5b Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Fri, 22 Apr 2022 03:05:07 -0500 Subject: [PATCH] Update CI to test multiple python versions (#45) * Update CI to test multiple python versions * Ensure numpy is installed * Handle lulesh with cmake < 3.16 * Fix typo * Bump minimum CMake version to 3.16 - CMake 3.15 has issue with PTL object library * Tweak CI test output --- .github/workflows/ubuntu-bionic.yml | 20 +++++++++++++++++-- .../workflows/ubuntu-focal-external-rocm.yml | 6 ++++-- .github/workflows/ubuntu-focal-external.yml | 8 ++++++-- .github/workflows/ubuntu-focal.yml | 3 ++- CMakeLists.txt | 4 ++-- docker/Dockerfile.ubuntu.ci | 13 ++++++++++-- docker/build-docker-ci.sh | 13 +++++++++++- examples/CMakeLists.txt | 14 ++++++++++--- examples/lulesh/CMakeLists.txt | 4 ++-- examples/parallel-overhead/CMakeLists.txt | 2 +- examples/transpose/CMakeLists.txt | 2 +- examples/user-api/CMakeLists.txt | 2 +- 12 files changed, 71 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ubuntu-bionic.yml b/.github/workflows/ubuntu-bionic.yml index f2e26e9f9d..d3ab21152e 100644 --- a/.github/workflows/ubuntu-bionic.yml +++ b/.github/workflows/ubuntu-bionic.yml @@ -47,7 +47,17 @@ jobs: apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} ${{ matrix.mpi }} && python3 -m pip install --upgrade pip && python3 -m pip install numpy && - python3 -m pip install 'cmake==3.16.3' + python3 -m pip install 'cmake==3.16.3' && + for i in 6 7 8 9; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy; done + + - name: Install Kokkos + timeout-minutes: 5 + run: + cd examples/lulesh/external/kokkos && + cmake -B build -DKokkos_ENABLE_OPENMP=ON -DKokkos_ENABLE_SERIAL=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_STANDARD=17 . && + cmake --build build --target all --parallel 2 && + cmake --build build --target install && + rm -rf build - name: Configure Env run: @@ -71,6 +81,9 @@ jobs: -DOMNITRACE_USE_MPI=${USE_MPI} -DOMNITRACE_USE_HIP=OFF -DOMNITRACE_USE_PYTHON=ON + -DOMNITRACE_PYTHON_VERSIONS="3.6;3.7;3.8;3.9" + -DOMNITRACE_PYTHON_ROOT_DIRS="/opt/conda/envs/py3.6;/opt/conda/envs/py3.7;/opt/conda/envs/py3.8;/opt/conda/envs/py3.9" + -DLULESH_BUILD_KOKKOS=OFF - name: Build timeout-minutes: 45 @@ -85,7 +98,10 @@ jobs: timeout-minutes: 45 run: cd build && - ctest -V --output-log build/omnitrace-ctest-ubuntu-bionic.log --stop-on-failure + ldd ./omnitrace && + ./omnitrace --help && + ctest -V -N -O omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log && + ctest --output-on-failure --output-log omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/.github/workflows/ubuntu-focal-external-rocm.yml b/.github/workflows/ubuntu-focal-external-rocm.yml index 47f1f5380b..0cf5877258 100644 --- a/.github/workflows/ubuntu-focal-external-rocm.yml +++ b/.github/workflows/ubuntu-focal-external-rocm.yml @@ -37,7 +37,8 @@ jobs: apt-get update && apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev-dev libnuma-dev rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev libopenmpi-dev && python3 -m pip install --upgrade pip && - python3 -m pip install 'cmake==3.16.3' + python3 -m pip install 'cmake==3.16.3' && + for i in 6 7 8 9; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy; done - name: Configure Env run: @@ -83,7 +84,8 @@ jobs: cd build && ldd ./omnitrace && ./omnitrace --help && - ctest -V --output-log omnitrace-ctest-ubuntu-focal-external-rocm.log --stop-on-failure + ctest -V -N -O omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log && + ctest --output-on-failure --output-log omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/.github/workflows/ubuntu-focal-external.yml b/.github/workflows/ubuntu-focal-external.yml index 39cd3ee347..d82522a9f0 100644 --- a/.github/workflows/ubuntu-focal-external.yml +++ b/.github/workflows/ubuntu-focal-external.yml @@ -32,7 +32,8 @@ jobs: apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev ${{ matrix.compiler }} && python3 -m pip install --upgrade pip && python3 -m pip install numpy && - python3 -m pip install 'cmake==3.16.3' + python3 -m pip install 'cmake==3.16.3' && + for i in 6 7 8 9; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy; done - name: Configure Env run: @@ -56,6 +57,8 @@ jobs: -DOMNITRACE_USE_HIP=OFF -DOMNITRACE_USE_OMPT=OFF -DOMNITRACE_USE_PYTHON=ON + -DOMNITRACE_PYTHON_VERSIONS="3.6;3.7;3.8;3.9" + -DOMNITRACE_PYTHON_ROOT_DIRS="/opt/conda/envs/py3.6;/opt/conda/envs/py3.7;/opt/conda/envs/py3.8;/opt/conda/envs/py3.9" - name: Build timeout-minutes: 45 @@ -73,7 +76,8 @@ jobs: cd build && ldd ./omnitrace && ./omnitrace --help && - ctest -V --output-log build/omnitrace-ctest-ubuntu-focal-external.log --stop-on-failure + ctest -V -N -O omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log && + ctest --output-on-failure --output-log omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index 5574fc6719..a6e3e6ac6d 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -74,7 +74,8 @@ jobs: timeout-minutes: 45 working-directory: ${{ github.workspace }}/build run: - ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal.log --stop-on-failure + ctest -V -N -O ${{ github.workspace }}/build/omnitrace-ctest-${{ env.GITHUB_JOB }}-commands.log && + ctest --output-on-failure --output-log ${{ github.workspace }}/build/omnitrace-ctest-${{ env.GITHUB_JOB }}.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/CMakeLists.txt b/CMakeLists.txt index 7df2f29ae2..2e4cb8612e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) @@ -45,7 +45,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE if("${CMAKE_BUILD_TYPE}" STREQUAL "") set(CMAKE_BUILD_TYPE - RelWithDebInfo + Release CACHE STRING "Build type" FORCE) else() set(VALID_BUILD_TYPES "Release" "RelWithDebInfo" "Debug" "MinSizeRel") diff --git a/docker/Dockerfile.ubuntu.ci b/docker/Dockerfile.ubuntu.ci index fa47e0d5e9..9eb0db5283 100644 --- a/docker/Dockerfile.ubuntu.ci +++ b/docker/Dockerfile.ubuntu.ci @@ -13,8 +13,9 @@ WORKDIR /tmp SHELL [ "/bin/bash", "-c" ] ARG EXTRA_PACKAGES="" -ARG ELFUTILS_DOWNLOAD_VERSION="0.183" +ARG ELFUTILS_DOWNLOAD_VERSION="0.186" ARG NJOBS="12" +ARG PYTHON_VERSIONS="6 7 8 9" ENV PATH ${HOME}/.local/bin:${PATH} ENV LIBRARY_PATH /usr/local/lib:/usr/local/lib64:${LIBRARY_PATH} @@ -30,13 +31,21 @@ RUN apt-get update && \ apt-get autoclean && \ locale -a && \ cd /tmp/dyninst && \ - cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_BOOST=ON -DBUILD_TBB=ON -DBUILD_ELFUTILS=ON -DBUILD_LIBIBERTY=ON && \ + cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_BOOST=ON -DBUILD_TBB=ON -DBUILD_ELFUTILS=ON -DBUILD_LIBIBERTY=ON -DELFUTILS_DOWNLOAD_VERSION=${ELFUTILS_DOWNLOAD_VERSION} && \ cmake --build build --target all --parallel ${NJOBS} && \ cmake --build build --target install --parallel ${NJOBS} && \ + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ + bash miniconda.sh -b -p /opt/conda && \ + export PATH="/opt/conda/bin:${PATH}" && \ + conda config --set always_yes yes --set changeps1 no && \ + conda update -c defaults -n base conda && \ + for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip numpy; done && \ + conda clean -a -y && \ cd /tmp && \ shopt -s dotglob extglob && \ rm -rf * + ENV LC_ALL C.UTF-8 WORKDIR /home SHELL [ "/bin/bash", "--login", "-c" ] diff --git a/docker/build-docker-ci.sh b/docker/build-docker-ci.sh index f89a1b9f7e..19b40e4499 100755 --- a/docker/build-docker-ci.sh +++ b/docker/build-docker-ci.sh @@ -5,7 +5,8 @@ set -e : ${DISTRO:=ubuntu} : ${VERSIONS:=20.04 18.04} : ${NJOBS=$(nproc)} -: ${ELFUTILS_VERSION:=0.183} +: ${ELFUTILS_VERSION:=0.186} +: ${PUSH:=0} send-error() { @@ -39,6 +40,9 @@ do shift ELFUTILS_VERSION=${1} ;; + "--push") + PUSH=1 + ;; *) send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" ;; @@ -77,4 +81,11 @@ do --build-arg ELFUTILS_DOWNLOAD_VERSION=${ELFUTILS_VERSION} done +if [ "${PUSH}" -gt 0 ]; then + for VERSION in ${VERSIONS} + do + verbose-run docker push jrmadsen/omnitrace-ci:${DISTRO}-${VERSION} + done +fi + verbose-run rm -rf ./dyninst-source diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index e27b43892f..aa96b62628 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(omnitrace-dyninst-examples LANGUAGES CXX) @@ -10,6 +10,14 @@ add_subdirectory(parallel-overhead) add_subdirectory(user-api) add_subdirectory(openmp) -option(BUILD_SHARED_LIBS "Build dynamic libraries" ON) +if(CMAKE_VERSION VERSION_LESS 3.16 AND (NOT DEFINED LULESH_BUILD_KOKKOS + OR LULESH_BUILD_KOKKOS)) + message( + AUTHOR_WARNING + "lulesh example cannot build Kokkos (requires CMake 3.16+, current version is ${CMAKE_VERSION})" + ) +else() + option(BUILD_SHARED_LIBS "Build dynamic libraries" ON) -add_subdirectory(lulesh) + add_subdirectory(lulesh) +endif() diff --git a/examples/lulesh/CMakeLists.txt b/examples/lulesh/CMakeLists.txt index cf629dd509..d3310c9465 100644 --- a/examples/lulesh/CMakeLists.txt +++ b/examples/lulesh/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(lulesh LANGUAGES C CXX) @@ -19,7 +19,7 @@ set(CMAKE_CXX_EXTENSIONS OFF) if("${CMAKE_BUILD_TYPE}" STREQUAL "") set(CMAKE_BUILD_TYPE - "RelWithDebInfo" + "Release" CACHE STRING "CMake build type" FORCE) endif() diff --git a/examples/parallel-overhead/CMakeLists.txt b/examples/parallel-overhead/CMakeLists.txt index 3acd578282..d28346e70e 100644 --- a/examples/parallel-overhead/CMakeLists.txt +++ b/examples/parallel-overhead/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(omnitrace-parallel-overhead LANGUAGES CXX) diff --git a/examples/transpose/CMakeLists.txt b/examples/transpose/CMakeLists.txt index 2434fe37bb..0df4e9f460 100644 --- a/examples/transpose/CMakeLists.txt +++ b/examples/transpose/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(omnitrace-transpose LANGUAGES CXX) diff --git a/examples/user-api/CMakeLists.txt b/examples/user-api/CMakeLists.txt index 8c22cfbdf3..7442edf48e 100644 --- a/examples/user-api/CMakeLists.txt +++ b/examples/user-api/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(omnitrace-user-api LANGUAGES CXX)