Support for Ubuntu 22.04 and ROCm 5.3 (#48)

* Testing and CI support for Ubuntu 22.04 * Fixes for ROCm - Jammy does not have ROCm installers * Name, timeout, and python updates - renamed ubuntu-jammy-external.yml to ubuntu-jammy.yml - increased all 5 minute timeouts to 10 minutes - include python 3.10 in testing * Update dyninst to remove interposed definition of _r_debug * Rebuild Dyninst + test install script * Revert container change * git safe directory * pushd -> cd * fix MPI include * Fix testing step * OMPI_ALLOW_RUN_AS_ROOT * Test script changes * Fix mismatched malloc / delete[] * Jammy workflow tweaks * CPack tweak for boost deb deps * pthread_mutex_gotcha config returns when not enabled * fix echoing config in CI * USE_CLANG_OMP - option to disable using LLVM OpenMP when building OpenMP test executables - Jammy workflow sets USE_CLANG_OMP=OFF * Dyninst submodule boost download - updated containers workflow to include jammy - updated workflow to use ci * Updates to workflows + replace test-install.sh - test-install.sh in this branch was replaced with one in main branch * Expand jammy test-install.sh args * Fix openmp-cg-sampling-duration test * update timemory submodule - use-after-free violation in popen::pclose * revert some tweaks to sampling-duration test * Fix env of test-install.sh * cmake format * jammy bash * CPack install for jammy * formatting workflow action version bump * Update timemory submodule - libunwind submodule via timemory sets SOVERSION to 99 to avoid ABI conflicts with v8 * Fix help menu for omnitrace-sample * Support other boolean forms in test-install.sh * Update docker files and build-docker.sh - consolidated cases in build-docker.sh - support rocm version of 0.0 (no rocm install) - support rocm v5.3 - updated centos handling * update opensuse actions/checkout version * Tweaks to ubuntu-focal testing - actions/checkout@v3 - use test-install script * update cpack - ubuntu 22.04 - rocm 5.3 - rename os matrix field to os-version - remove CI_ROCM_VERSION (no longer necessary) - remove default-rocm-version matrix field (no longer necessary) - CentOS packaging * fix argparsing and omnitrace-sample tests in install-tests.sh * focal rocm test install workflow fix * Fix omnitrace-sample build * Dockerfile.centos + build-docker.sh updates * Update actions/upload-artifact version * Dockerfile.ubuntu: install rocm-device-libs * Refactor cpack * fix cpack if quotes * Dockerfile.ubuntu rocm < 5 installs rocm-dev * build-release.sh defaults to boost version 1.79.0
2022-10-17 12:54:26 -05:00
parent a3439d5bf2
commit ede6007f9b
28 changed files with 543 additions and 313 deletions
@@ -12,21 +12,38 @@ SHELL [ "/bin/bash", "-c" ]

 ENV PATH /usr/local/bin:${PATH}

+RUN if [ -n "$(cat /etc/os-release | grep 'VERSION="8"')" ]; then \
+        dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos && \
+        dnf -y distro-sync; \
+    fi
+
 RUN yum update -y && \
-    yum groupinstall -y "Development Tools" && \
-    yum install -y centos-release-scl && \
-    yum install -y epel-release && \
-    yum install -y devtoolset-9 python3-pip openmpi3-devel zlib-devel numactl-devel papi-devel dpkg-devel dpkg-dev wget curl && \
+    yum groupinstall -y "Development Tools"
+
+ARG TOOLSET_VERSION=9
+
+RUN yum install -y epel-release && \
+    if [ -n "$(cat /etc/os-release | grep 'VERSION="8"')" ]; then \
+        yum install -y centos-release-stream && \
+        yum install -y gcc-toolset-${TOOLSET_VERSION} openmpi-devel; \
+    else \
+        yum install -y centos-release-scl && \
+        yum install -y devtoolset-${TOOLSET_VERSION} openmpi3-devel dpkg-dev; \
+    fi; \
+    yum install -y python3-pip zlib-devel numactl-devel papi-devel dpkg-devel wget curl && \
    python3 -m pip install 'cmake==3.21.4'

+ARG ROCM_VERSION=0.0
 ARG AMDGPU_RPM=21.40.2/rhel/7.9/amdgpu-install-21.40.2.40502-1.el7.noarch.rpm
 # ARG AMDGPU_RPM=latest/rhel/7.9/amdgpu-install-21.50.50000-1.el7.noarch.rpm

-RUN yum install -y https://repo.radeon.com/amdgpu-install/${AMDGPU_RPM} && \
-    amdgpu-install --usecase=rocm,hip,hiplibsdk --no-dkms --skip-broken -y && \
-    yum install -y rocm-hip-sdk rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev && \
-    yum update -y && \
-    yum clean all
+RUN if [ "${ROCM_VERSION}" != "0.0" ]; then \
+        yum install -y https://repo.radeon.com/amdgpu-install/${AMDGPU_RPM} && \
+        amdgpu-install --usecase=rocm,hip,hiplibsdk --no-dkms --skip-broken -y && \
+        yum install -y rocm-hip-sdk rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev && \
+        yum update -y && \
+        yum clean all; \
+    fi

 ARG PYTHON_VERSIONS="6 7 8 9 10"

@@ -39,7 +56,7 @@ RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -
    conda clean -a -y && \
    conda init

-RUN ln -s /opt/rocm-* /opt/rocm
+RUN if [ "${ROCM_VERSION}" != "0.0" ]; then ln -sf /opt/rocm-${ROCM_VERSION}* /opt/rocm; fi

 WORKDIR /home
 SHELL [ "/bin/bash", "--login", "-c" ]
@@ -18,15 +18,18 @@ RUN zypper update -y && \
    zypper install -y python3-pip openmpi3-devel gcc-c++ git libnuma-devel dpkg-devel rpm-build wget curl binutils-gold && \
    python3 -m pip install 'cmake==3.21.4'

-# ARG AMDGPU_RPM=21.40.2/sle/15/amdgpu-install-21.40.2.40502-1.noarch.rpm
+ARG ROCM_VERSION=0.0
 ARG AMDGPU_RPM=latest/sle/15/amdgpu-install-21.50.50000-1.noarch.rpm
+# ARG AMDGPU_RPM=21.40.2/sle/15/amdgpu-install-21.40.2.40502-1.noarch.rpm

-RUN zypper --no-gpg-checks install -y https://repo.radeon.com/amdgpu-install/${AMDGPU_RPM} && \
-    zypper addrepo https://download.opensuse.org/repositories/devel:languages:perl/SLE_15/devel:languages:perl.repo && \
-    zypper --non-interactive --gpg-auto-import-keys refresh && \
-    amdgpu-install --usecase=rocm,hip,hiplibsdk --no-dkms -y && \
-    zypper install -y rocm-hip-sdk rocm-smi-lib roctracer-dev rocprofiler-dev rccl-devel && \
-    zypper clean --all
+RUN if [ "${ROCM_VERSION}" != "0.0" ]; then \
+        zypper --no-gpg-checks install -y https://repo.radeon.com/amdgpu-install/${AMDGPU_RPM} && \
+        zypper addrepo https://download.opensuse.org/repositories/devel:languages:perl/SLE_15/devel:languages:perl.repo && \
+        zypper --non-interactive --gpg-auto-import-keys refresh && \
+        amdgpu-install --usecase=rocm,hip,hiplibsdk --no-dkms -y && \
+        zypper install -y rocm-hip-sdk rocm-smi-lib roctracer-dev rocprofiler-dev rccl-devel && \
+        zypper clean --all; \
+    fi

 ARG PYTHON_VERSIONS="6 7 8 9 10"

@@ -16,20 +16,32 @@ SHELL [ "/bin/bash", "-c" ]

 ARG EXTRA_PACKAGES=""
 ARG ROCM_REPO_VERSION="debian"
+ARG ROCM_VERSION="0.0"
 ARG ROCM_REPO_DIST="ubuntu"
 ARG PYTHON_VERSIONS="6 7 8 9 10"
 ENV PATH ${HOME}/.local/bin:${PATH}

 RUN apt-get update && \
    apt-get dist-upgrade -y && \
-    apt-get install -y build-essential cmake libnuma1 wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev1 libopenmpi-dev rpm librpm-dev curl && \
-    python3 -m pip install 'cmake==3.21.4' && \
-    wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \
-    echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${ROCM_REPO_VERSION}/ ${ROCM_REPO_DIST} main" | tee /etc/apt/sources.list.d/rocm.list && \
-    apt-get update && \
-    apt-get dist-upgrade -y && \
-    apt-get install -y rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${EXTRA_PACKAGES} && \
-    apt-get autoclean
+    apt-get install -y build-essential cmake libnuma1 wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev1 libopenmpi-dev rpm librpm-dev curl apt-utils && \
+    python3 -m pip install 'cmake==3.21.4'
+
+RUN if [ "${ROCM_VERSION}" != "0.0" ]; then \
+        if [ -d /etc/apt/trusted.gpg.d ]; then \
+            wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/trusted.gpg.d/rocm.gpg; \
+        else \
+            wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add -; \
+        fi && \
+        echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${ROCM_REPO_VERSION}/ ${ROCM_REPO_DIST} main" | tee /etc/apt/sources.list.d/rocm.list && \
+        apt-get update && \
+        apt-get dist-upgrade -y && \
+        apt-get install -y hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev && \
+        apt-get install -y hip-base hip-runtime-amd hip-dev && \
+        apt-get install -y rocm-llvm rocm-core rocm-smi-lib rocm-device-libs && \
+        apt-get install -y roctracer-dev rocprofiler-dev rccl-dev ${EXTRA_PACKAGES} && \
+        if [ "$(echo ${ROCM_VERSION} | awk -F '.' '{print $1}')" -lt "5" ]; then apt-get install -y rocm-dev; fi && \
+        apt-get autoclean; \
+    fi

 RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \
    bash miniconda.sh -b -p /opt/conda && \
@@ -14,10 +14,11 @@ SHELL [ "/bin/bash", "-c" ]

 ARG EXTRA_PACKAGES=""
 ARG ELFUTILS_DOWNLOAD_VERSION="0.186"
+ARG BOOST_DOWNLOAD_VERSION="1.79.0"
 ARG NJOBS="12"
 ARG PYTHON_VERSIONS="6 7 8 9 10"

-ENV PATH ${HOME}/.local/bin:${PATH}
+ENV PATH /usr/local/bin:${PATH}
 ENV LIBRARY_PATH /usr/local/lib:/usr/local/lib64:${LIBRARY_PATH}
 ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib64:${LD_LIBRARY_PATH}
 ENV CMAKE_PREFIX_PATH /usr/local:${CMAKE_PREFIX_PATH}
@@ -31,7 +32,7 @@ RUN apt-get update && \
    apt-get autoclean && \
    locale -a && \
    cd /tmp/dyninst && \
-    cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_BOOST=ON -DBUILD_TBB=ON -DBUILD_ELFUTILS=ON -DBUILD_LIBIBERTY=ON -DELFUTILS_DOWNLOAD_VERSION=${ELFUTILS_DOWNLOAD_VERSION} && \
+    cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_BOOST=ON -DBUILD_TBB=ON -DBUILD_ELFUTILS=ON -DBUILD_LIBIBERTY=ON -DELFUTILS_DOWNLOAD_VERSION=${ELFUTILS_DOWNLOAD_VERSION} -DBOOST_DOWNLOAD_VERSION=${BOOST_DOWNLOAD_VERSION} && \
    cmake --build build --target all --parallel ${NJOBS} && \
    cmake --build build --target install --parallel ${NJOBS} && \
    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \
@@ -7,6 +7,7 @@ set -e
 : ${VERSIONS:=20.04}
 : ${NJOBS=$(nproc)}
 : ${ELFUTILS_VERSION:=0.186}
+: ${BOOST_VERSION:=1.79.0}
 : ${PUSH:=0}

 verbose-run()
@@ -38,6 +39,7 @@ usage()
    print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}"
    print_default_option "jobs -j" "[N]" "parallel build jobs" "${NJOBS}"
    print_default_option elfutils-version "[0.183..0.186]" "ElfUtils version" "${ELFUTILS_VERSION}"
+    print_default_option boost-version "[1.67.0..1.79.0]" "Boost version" "${BOOST_VERSION}"
    print_default_option user "[USERNAME]" "DockerHub username" "${USER}"
 }

@@ -83,6 +85,11 @@ do
            ELFUTILS_VERSION=${1}
            reset-last
            ;;
+        "--boost-version")
+            shift
+            BOOST_VERSION=${1}
+            reset-last
+            ;;
        --user|-u)
            shift
            USER=${1}
@@ -131,7 +138,8 @@ do
        --build-arg DISTRO=${DISTRO_IMAGE} \
        --build-arg VERSION=${VERSION} \
        --build-arg NJOBS=${NJOBS} \
-        --build-arg ELFUTILS_DOWNLOAD_VERSION=${ELFUTILS_VERSION}
+        --build-arg ELFUTILS_DOWNLOAD_VERSION=${ELFUTILS_VERSION} \
+        --build-arg BOOST_DOWNLOAD_VERSION=${BOOST_VERSION}
 done

 if [ "${PUSH}" -gt 0 ]; then
@@ -111,12 +111,12 @@ if [ ! -f ${DOCKER_FILE} ]; then send-error "File \"${DOCKER_FILE}\" not found";

 for VERSION in ${VERSIONS}
 do
-    for i in ${ROCM_VERSIONS}
+    for ROCM_VERSION in ${ROCM_VERSIONS}
    do
-        CONTAINER=${USER}/omnitrace:release-base-${DISTRO}-${VERSION}-rocm-${i}
-        ROCM_MAJOR=$(echo ${i} | sed 's/\./ /g' | awk '{print $1}')
-        ROCM_MINOR=$(echo ${i} | sed 's/\./ /g' | awk '{print $2}')
-        ROCM_PATCH=$(echo ${i} | sed 's/\./ /g' | awk '{print $3}')
+        CONTAINER=${USER}/omnitrace:release-base-${DISTRO}-${VERSION}-rocm-${ROCM_VERSION}
+        ROCM_MAJOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $1}')
+        ROCM_MINOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $2}')
+        ROCM_PATCH=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $3}')
        if [ -n "${ROCM_PATCH}" ]; then
            ROCM_VERSN=$(( (${ROCM_MAJOR}*10000)+(${ROCM_MINOR}*100)+(${ROCM_PATCH}) ))
            ROCM_SEP="."
@@ -126,46 +126,73 @@ do
        fi
        if [ "${DISTRO}" = "ubuntu" ]; then
            ROCM_REPO_DIST="ubuntu"
-            ROCM_REPO_VERSION=${i}
-            case "${i}" in
+            ROCM_REPO_VERSION=${ROCM_VERSION}
+            case "${ROCM_VERSION}" in
                4.1* | 4.0*)
                    ROCM_REPO_DIST="xenial"
                    ;;
+                5.3*)
+                    case "${VERSION}" in
+                        22.04)
+                            ROCM_REPO_DIST="jammy"
+                            ;;
+                        20.04)
+                            ROCM_REPO_DIST="focal"
+                            ;;
+                        18.04)
+                            ROCM_REPO_DIST="bionic"
+                            ;;
+                        *)
+                            ;;
+                    esac
+                    ;;
                *)
                    ;;
            esac
-            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
+            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
        elif [ "${DISTRO}" = "centos" ]; then
            case "${VERSION}" in
                7)
                    RPM_PATH=7.9
                    RPM_TAG=".el7"
+                    TOOLSET_VERSION=9
                    ;;
                8)
-                    RPM_PATH=8.5
-                    RPM_TAG=".el7"
+                    RPM_PATH=8.4
+                    RPM_TAG=".el8"
+                    TOOLSET_VERSION=11
+                    ;;
+                9)
+                    RPM_PATH=9.0
+                    RPM_TAG=".el9"
+                    TOOLSET_VERSION=11
                    ;;
                *)
-                    send-error "Invalid centos version ${VERSION}. Supported: 7, 8"
+                    send-error "Invalid centos version ${VERSION}. Supported: 7, 8, 9"
            esac
-            case "${i}" in
-                5.0*)
-                    ROCM_RPM=latest/rhel/${RPM_PATH}/amdgpu-install-21.50.50000-1${RPM_TAG}.noarch.rpm
+            case "${ROCM_VERSION}" in
+                5.3 | 5.3.*)
+                    ROCM_RPM=${ROCM_VERSION}/rhel/${RPM_PATH}/amdgpu-install-${ROCM_MAJOR}.${ROCM_MINOR}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm
                    ;;
-                4.5 | 4.5.2)
-                    ROCM_RPM=21.40.2/rhel/${RPM_PATH}/amdgpu-install-21.40.2.40502-1${RPM_TAG}.noarch.rpm
+                5.2 | 5.2.*)
+                    ROCM_RPM=22.20${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-22.20.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm
                    ;;
-                4.5.1)
-                    ROCM_RPM=21.40.1/rhel/${RPM_PATH}/amdgpu-install-21.40.1.40501-1${RPM_TAG}.noarch.rpm
+                5.1 | 5.1.*)
+                    ROCM_RPM=22.10${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-22.10${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm
                    ;;
-                4.5.0)
-                    ROCM_RPM=21.40/rhel/${RPM_PATH}/amdgpu-install-21.40.1.40501-1${RPM_TAG}.noarch.rpm
+                5.0 | 5.0.*)
+                    ROCM_RPM=21.50${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-21.50${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm
+                    ;;
+                4.5 | 4.5.*)
+                    ROCM_RPM=21.40${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-21.40${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1.noarch.rpm
+                    ;;
+                0.0)
                    ;;
                *)
-                    send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${i}"
+                    send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${ROCM_VERSION}"
                    ;;
            esac
-            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
+            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg TOOLSET_VERSION=${TOOLSET_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
        elif [ "${DISTRO}" = "opensuse" ]; then
            case "${VERSION}" in
                15.*)
@@ -176,14 +203,14 @@ do
                    send-error "Invalid opensuse version ${VERSION}. Supported: 15.x"
                    ;;
            esac
-            case "${i}" in
+            case "${ROCM_VERSION}" in
+                5.3 | 5.3.*)
+                    ROCM_RPM=${ROCM_VERSION}/sle/${VERSION}/amdgpu-install-${ROCM_MAJOR}.${ROCM_MINOR}.${ROCM_VERSN}-1.noarch.rpm
+                    ;;
                5.2 | 5.2.*)
                    ROCM_RPM=22.20${ROCM_SEP}${ROCM_PATCH}/sle/${VERSION}/amdgpu-install-22.20.${ROCM_VERSN}-1.noarch.rpm
                    ;;
-                5.1 | 5.1.0)
-                    ROCM_RPM=22.10/sle/15/amdgpu-install-22.10.50100-1.noarch.rpm
-                    ;;
-                5.1.*)
+                5.1 | 5.1.*)
                    ROCM_RPM=22.10${ROCM_SEP}${ROCM_PATCH}/sle/15/amdgpu-install-22.10${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1.noarch.rpm
                    ;;
                5.0 | 5.0.*)
@@ -192,11 +219,13 @@ do
                4.5 | 4.5.*)
                    ROCM_RPM=21.40${ROCM_SEP}${ROCM_PATCH}/sle/15/amdgpu-install-21.40${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1.noarch.rpm
                    ;;
+                0.0)
+                    ;;
                *)
-                    send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${i}"
+                    send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${ROCM_VERSION}"
                ;;
            esac
-            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
+            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
        fi
        if [ "${PUSH}" -ne 0 ]; then
            docker push ${CONTAINER}