AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Python 3.11 support + update RedHat CPack (#254)

Browse Source 
* Fixes for Python 3.11

* Add python 3.11 to scripts

- also tweak to to{upper,lower} bash functions

* Fix PAPI RPM packaging in RedHat

- fix error from #!/usr/bin/python in papi_hl_output_writer.py
  - requires either python2 or python3 instead of python

* cpack updates

- only generate STGZ for RedHat
- support `--generators` arg in build-release.sh
- support 7z, zip, and other zip generators
- fix build-release.sh with `--mpi`
- support setting CONDA_ROOT

* Support rhel/fedora/centos in omnitrace-install.py

* RedHat status badge

* Fix support for Python 3.11 + tweak ubuntu ci

- Remove installing clang and mpich in Ubuntu CI container
- Fallback on conda-forge for Python 3.11
- Enable entrypoint-rhel.sh for RHEL CI
- Pull latest container by default

* Update ElfUtils and PAPI builds

- quieter build output
- disable-nls for ElfUtils
- use -s flag for make

* Development Guide Docs

[ROCm/rocprofiler-systems commit: 83f9ed8696]
This commit is contained in:
Jonathan R. Madsen
2023-03-08 00:19:29 -06:00
committed by GitHub
parent 446fd36a93
commit e70d684c98
22 changed files with 570 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocprofiler-systems/docker/Dockerfile.opensuse
+3 -2
View File
@@ -31,14 +31,15 @@ RUN if [ "${ROCM_VERSION}" != "0.0" ]; then \
zypper clean --all; \
fi
ARG PYTHON_VERSIONS="6 7 8 9 10"
ARG PYTHON_VERSIONS="6 7 8 9 10 11"
RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \
bash miniconda.sh -b -p /opt/conda && \
export PATH="/opt/conda/bin:${PATH}" && \
conda config --set always_yes yes --set changeps1 no && \
conda update -c defaults -n base conda && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults -c conda-forge python=3.${i} pip; done && \
for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \
conda clean -a -y && \
conda init
projects/rocprofiler-systems/docker/Dockerfile.opensuse.ci
+3 -3
View File
@@ -33,16 +33,16 @@ RUN cd /tmp/dyninst && \
shopt -s dotglob extglob && \
rm -rf *
ARG PYTHON_VERSIONS="6 7 8 9 10"
ARG PYTHON_VERSIONS="6 7 8 9 10 11"
RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \
bash miniconda.sh -b -p /opt/conda && \
export PATH="/opt/conda/bin:${PATH}" && \
conda config --set always_yes yes --set changeps1 no && \
conda update -c defaults -n base conda && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip numpy; done && \
conda clean -a -y && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults -c conda-forge python=3.${i} pip numpy; done && \
for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \
conda clean -a -y && \
cd /tmp && \
shopt -s dotglob extglob && \
rm -rf *
projects/rocprofiler-systems/docker/Dockerfile.rhel
+5 -2
View File
@@ -34,14 +34,15 @@ RUN if [ "${ROCM_VERSION}" != "0.0" ]; then \
yum clean all; \
fi
ARG PYTHON_VERSIONS="6 7 8 9 10"
ARG PYTHON_VERSIONS="6 7 8 9 10 11"
RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \
bash miniconda.sh -b -p /opt/conda && \
export PATH="/opt/conda/bin:${PATH}" && \
conda config --set always_yes yes --set changeps1 no && \
conda update -c defaults -n base conda && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults -c conda-forge python=3.${i} pip; done && \
for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \
conda clean -a -y && \
conda init
@@ -50,3 +51,5 @@ RUN if [ "${ROCM_VERSION}" != "0.0" ]; then ln -sf /opt/rocm-${ROCM_VERSION}* /o
WORKDIR /home
ENV LC_ALL C.UTF-8
SHELL [ "/bin/bash", "--login", "-c" ]
COPY ./entrypoint-rhel.sh /docker-entrypoint.sh
ENTRYPOINT [ "/docker-entrypoint.sh" ]
projects/rocprofiler-systems/docker/Dockerfile.rhel.ci
+3 -3
View File
@@ -33,16 +33,16 @@ RUN cd /tmp/dyninst && \
shopt -s dotglob extglob && \
rm -rf *
ARG PYTHON_VERSIONS="6 7 8 9 10"
ARG PYTHON_VERSIONS="6 7 8 9 10 11"
RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \
bash miniconda.sh -b -p /opt/conda && \
export PATH="/opt/conda/bin:${PATH}" && \
conda config --set always_yes yes --set changeps1 no && \
conda update -c defaults -n base conda && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip numpy; done && \
conda clean -a -y && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults -c conda-forge python=3.${i} pip numpy; done && \
for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \
conda clean -a -y && \
cd /tmp && \
shopt -s dotglob extglob && \
rm -rf *
projects/rocprofiler-systems/docker/Dockerfile.ubuntu
+3 -2
View File
@@ -18,7 +18,7 @@ ARG EXTRA_PACKAGES=""
ARG ROCM_REPO_VERSION="debian"
ARG ROCM_VERSION="0.0"
ARG ROCM_REPO_DIST="ubuntu"
ARG PYTHON_VERSIONS="6 7 8 9 10"
ARG PYTHON_VERSIONS="6 7 8 9 10 11"
ENV PATH ${HOME}/.local/bin:${PATH}
RUN apt-get update && \
@@ -48,7 +48,8 @@ RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -
export PATH="/opt/conda/bin:${PATH}" && \
conda config --set always_yes yes --set changeps1 no && \
conda update -c defaults -n base conda && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults -c conda-forge python=3.${i} pip; done && \
for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \
conda clean -a -y && \
conda init
projects/rocprofiler-systems/docker/Dockerfile.ubuntu.ci
+4 -4
View File
@@ -16,7 +16,7 @@ ARG EXTRA_PACKAGES=""
ARG ELFUTILS_DOWNLOAD_VERSION="0.186"
ARG BOOST_DOWNLOAD_VERSION="1.79.0"
ARG NJOBS="12"
ARG PYTHON_VERSIONS="6 7 8 9 10"
ARG PYTHON_VERSIONS="6 7 8 9 10 11"
ENV PATH /usr/local/bin:${PATH}
ENV LIBRARY_PATH /usr/local/lib:/usr/local/lib64:${LIBRARY_PATH}
@@ -27,7 +27,7 @@ COPY ./dyninst-source /tmp/dyninst
RUN apt-get update && \
apt-get dist-upgrade -y && \
apt-get install -y autoconf autotools-dev bash-completion build-essential bzip2 clang cmake curl environment-modules git-core gnupg2 gzip libiberty-dev libmpich-dev libpapi-dev libpfm4-dev libtool locales lsb-release m4 mpich python3-pip unzip wget zip zlib1g-dev && \
apt-get install -y autoconf autotools-dev bash-completion build-essential bzip2 cmake curl environment-modules git-core gnupg2 gzip libiberty-dev libpapi-dev libpfm4-dev libtool locales lsb-release m4 python3-pip unzip wget zip zlib1g-dev && \
python3 -m pip install 'cmake==3.18.4' && \
apt-get autoclean && \
locale -a && \
@@ -40,9 +40,9 @@ RUN apt-get update && \
export PATH="/opt/conda/bin:${PATH}" && \
conda config --set always_yes yes --set changeps1 no && \
conda update -c defaults -n base conda && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip numpy; done && \
conda clean -a -y && \
for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults -c conda-forge python=3.${i} pip numpy; done && \
for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \
conda clean -a -y && \
cd /tmp && \
shopt -s dotglob extglob && \
rm -rf *
projects/rocprofiler-systems/docker/build-docker-ci.sh
+17 -2
View File
@@ -8,7 +8,9 @@ set -e
: ${NJOBS=$(nproc)}
: ${ELFUTILS_VERSION:=0.186}
: ${BOOST_VERSION:=1.79.0}
: ${PYTHON_VERSIONS:="6 7 8 9 10 11"}
: ${PUSH:=0}
: ${PULL:=--pull}
verbose-run()
{
@@ -18,12 +20,12 @@ verbose-run()
tolower()
{
echo "$@" | awk -F '\|~\|' '{print tolower($1)}';
echo "$@" | awk -F '\\|~\\|' '{print tolower($1)}';
}
toupper()
{
echo "$@" | awk -F '\|~\|' '{print toupper($1)}';
echo "$@" | awk -F '\\|~\\|' '{print toupper($1)}';
}
usage()
@@ -32,11 +34,13 @@ usage()
echo "Options:"
print_option "help -h" "" "This message"
print_option "push" "" "Push the container to DockerHub when completed"
print_option "no-pull" "" "Do not pull down most recent base container"
echo ""
print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; }
print_default_option distro "[ubuntu|opensuse|rhel]" "OS distribution" "${DISTRO}"
print_default_option versions "[VERSION] [VERSION...]" "Ubuntu, OpenSUSE, or RHEL release" "${VERSIONS}"
print_default_option python-versions "[VERSION] [VERSION...]" "Python 3 minor releases" "${PYTHON_VERSIONS}"
print_default_option "jobs -j" "[N]" "parallel build jobs" "${NJOBS}"
print_default_option elfutils-version "[0.183..0.186]" "ElfUtils version" "${ELFUTILS_VERSION}"
print_default_option boost-version "[1.67.0..1.79.0]" "Boost version" "${BOOST_VERSION}"
@@ -75,6 +79,11 @@ do
VERSIONS=${1}
last() { VERSIONS="${VERSIONS} ${1}"; }
;;
"--python-versions")
shift
PYTHON_VERSIONS=${1}
last() { PYTHON_VERSIONS="${PYTHON_VERSIONS} ${1}"; }
;;
--jobs|-j)
shift
NJOBS=${1}
@@ -99,6 +108,10 @@ do
PUSH=1
reset-last
;;
"--no-pull")
PULL=""
reset-last
;;
--*)
reset-last
last ${1}
@@ -137,11 +150,13 @@ fi
for VERSION in ${VERSIONS}
do
verbose-run docker build . \
${PULL} \
-f ${DOCKER_FILE} \
--tag ${USER}/omnitrace:ci-base-${DISTRO}-${VERSION} \
--build-arg DISTRO=${DISTRO_IMAGE} \
--build-arg VERSION=${VERSION} \
--build-arg NJOBS=${NJOBS} \
--build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" \
--build-arg ELFUTILS_DOWNLOAD_VERSION=${ELFUTILS_VERSION} \
--build-arg BOOST_DOWNLOAD_VERSION=${BOOST_VERSION}
done
projects/rocprofiler-systems/docker/build-docker-release.sh
+1 -1
View File
@@ -101,7 +101,7 @@ reset-last
: ${VERSIONS:=20.04 18.04}
: ${ROCM_VERSIONS:=5.0 4.5 4.3}
: ${MPI:=0}
: ${PYTHON_VERSIONS:="6 7 8 9 10"}
: ${PYTHON_VERSIONS:="6 7 8 9 10 11"}
: ${RETRY:=3}
n=0
projects/rocprofiler-systems/docker/build-docker.sh
+13 -6
View File
@@ -4,21 +4,22 @@
: ${ROCM_VERSIONS:="5.0"}
: ${DISTRO:=ubuntu}
: ${VERSIONS:=20.04}
: ${PYTHON_VERSIONS:="6 7 8 9 10"}
: ${PYTHON_VERSIONS:="6 7 8 9 10 11"}
: ${BUILD_CI:=""}
: ${PUSH:=0}
: ${PULL:=--pull}
: ${RETRY:=3}
set -e
tolower()
{
echo "$@" | awk -F '\|~\|' '{print tolower($1)}';
echo "$@" | awk -F '\\|~\\|' '{print tolower($1)}';
}
toupper()
{
echo "$@" | awk -F '\|~\|' '{print toupper($1)}';
echo "$@" | awk -F '\\|~\\|' '{print toupper($1)}';
}
usage()
@@ -26,6 +27,7 @@ usage()
print_option() { printf " --%-20s %-24s %s\n" "${1}" "${2}" "${3}"; }
echo "Options:"
print_option "help -h" "" "This message"
print_option "no-pull" "" "Do not pull down most recent base container"
echo ""
print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; }
@@ -118,6 +120,11 @@ do
;;
--push)
PUSH=1
reset-last
;;
--no-pull)
PULL=""
reset-last
;;
--retry|-r)
shift
@@ -184,7 +191,7 @@ do
*)
;;
esac
verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
verbose-build docker build . ${PULL} -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
elif [ "${DISTRO}" = "rhel" ]; then
if [ -z "${VERSION_MINOR}" ]; then
send-error "Please provide a major and minor version of the OS. Supported: >= 8.7, <= 9.1"
@@ -216,7 +223,7 @@ do
# use Rocky Linux as a base image for RHEL builds
DISTRO_BASE_IMAGE=rockylinux
verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_BASE_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
verbose-build docker build . ${PULL} -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_BASE_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
elif [ "${DISTRO}" = "opensuse" ]; then
case "${VERSION}" in
15.*)
@@ -253,7 +260,7 @@ do
;;
esac
PERL_REPO="SLE_${VERSION_MAJOR}_SP${VERSION_MINOR}"
verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PERL_REPO=${PERL_REPO} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
verbose-build docker build . ${PULL} -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PERL_REPO=${PERL_REPO} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
fi
if [ "${PUSH}" -ne 0 ]; then
docker push ${CONTAINER}
projects/rocprofiler-systems/docker/entrypoint-rhel.sh
+7 -4
View File
@@ -1,10 +1,13 @@
#!/bin/bash
#!/bin/bash -l
source /etc/profile.d/modules.sh
module load mpi
if [ -f /etc/profile.d/modules.sh ]; then
source /etc/profile.d/modules.sh
module load mpi &> /dev/null
endif
if [ -z "${1}" ]; then
exec bash
: ${SHELL:=/bin/bash}
exec ${SHELL}
else
set -e
eval $@
projects/rocprofiler-systems/docker/test-docker-release.sh
+2 -2
View File
@@ -13,12 +13,12 @@ set -e
tolower()
{
echo "$@" | awk -F '\|~\|' '{print tolower($1)}';
echo "$@" | awk -F '\\|~\\|' '{print tolower($1)}';
}
toupper()
{
echo "$@" | awk -F '\|~\|' '{print toupper($1)}';
echo "$@" | awk -F '\\|~\\|' '{print toupper($1)}';
}
usage()