From 4db6ba3d28f5f16612e37c0066e8e73620e8ec45 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Thu, 21 Apr 2022 21:36:07 -0500 Subject: [PATCH] Multiple python versions (#42) * Support multiple Python versions in single build * RPATH + Split up config into config and runtime * pybind11 submodule * Docker build updates --- .cmake-format.yaml | 27 ++ .gitmodules | 3 + CMakeLists.txt | 2 +- cmake/ConfigCPack.cmake | 43 +- cmake/Packages.cmake | 6 +- docker/Dockerfile.centos | 13 +- docker/Dockerfile.opensuse | 13 +- docker/Dockerfile.ubuntu | 16 +- docker/build-docker-release.sh | 63 +-- docker/build-docker.sh | 14 +- examples/python/builtin.py | 46 ++- examples/python/external.py | 46 ++- examples/python/pyproject.toml | 5 + examples/python/source.py | 79 +++- external/pybind11 | 1 + scripts/build-release.sh | 211 ++++++++-- source/bin/CMakeLists.txt | 6 + source/bin/omnitrace-avail/CMakeLists.txt | 9 +- .../omnitrace-critical-trace/CMakeLists.txt | 9 +- source/bin/omnitrace/CMakeLists.txt | 11 +- source/docs/python.md | 6 +- source/lib/CMakeLists.txt | 13 + source/lib/omnitrace/CMakeLists.txt | 5 +- .../lib/omnitrace/include/library/config.hpp | 57 +-- .../lib/omnitrace/include/library/runtime.hpp | 94 +++++ .../omnitrace/include/library/thread_data.hpp | 2 +- source/lib/omnitrace/src/library/config.cpp | 107 +---- source/lib/omnitrace/src/library/runtime.cpp | 146 +++++++ source/python/CMakeLists.txt | 184 +++++++-- source/python/cmake/ConfigPython.cmake | 383 +----------------- .../cmake/Modules/FindPyBind11Python.cmake | 183 +++++++++ source/python/cmake/PyBind11Tools.cmake | 301 ++++++++++++++ source/python/omnitrace/CMakeLists.txt | 51 --- source/python/omnitrace/__init__.py | 2 +- source/python/omnitrace/__main__.py | 2 +- source/python/omnitrace/profiler.py | 2 +- tests/CMakeLists.txt | 263 ++++++------ 37 files changed, 1554 insertions(+), 870 deletions(-) create mode 100644 examples/python/pyproject.toml create mode 160000 external/pybind11 create mode 100644 source/lib/omnitrace/include/library/runtime.hpp create mode 100644 source/lib/omnitrace/src/library/runtime.cpp create mode 100644 source/python/cmake/Modules/FindPyBind11Python.cmake create mode 100644 source/python/cmake/PyBind11Tools.cmake delete mode 100644 source/python/omnitrace/CMakeLists.txt diff --git a/.cmake-format.yaml b/.cmake-format.yaml index d4dfc49456..ab649c7bb0 100644 --- a/.cmake-format.yaml +++ b/.cmake-format.yaml @@ -68,6 +68,8 @@ parse: DEPENDS: '*' COMMAND: '*' PROPERTIES: '*' + PYTHON_EXECUTABLE: '*' + PYTHON_VERSION: '*' PASS_REGULAR_EXPRESSION: '*' FAIL_REGULAR_EXPRESSION: '*' SKIP_REGULAR_EXPRESSION: '*' @@ -78,6 +80,31 @@ parse: - SEND_ERROR - FATAL_ERROR - AUTHOR_WARNING + omnitrace_find_python: + flags: + - REQUIRED + - QUIET + kwargs: + VERSION: '*' + ROOT_DIR: '*' + COMPONENTS: '*' + omnitrace_python_console_script: + kwargs: + VERSION: '*' + ROOT_DIR: '*' + omnitrace_pybind11_add_module: + flags: + - MODULE + - SHARED + - EXCLUDE_FROM_ALL + - NO_EXTRAS + - SYSTEM + - THIN_LTO + - LTO + kwargs: + PYTHON_VERSION: '*' + CXX_STANDARD: '*' + VISIBILITY: '*' override_spec: {} vartags: [] proptags: [] diff --git a/.gitmodules b/.gitmodules index f1fe150ee9..69f9de16c6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "external/papi"] path = external/papi url = https://bitbucket.org/icl/papi.git +[submodule "external/pybind11"] + path = external/pybind11 + url = https://github.com/jrmadsen/pybind11.git diff --git a/CMakeLists.txt b/CMakeLists.txt index f2cd97563a..7df2f29ae2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,7 @@ message( "[${PROJECT_NAME}] version ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}" ) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${PROJECT_SOURCE_DIR}/cmake/Modules - ${CMAKE_MODULE_PATH}) + ${PROJECT_SOURCE_DIR}/source/python/cmake ${CMAKE_MODULE_PATH}) set(BUILD_SHARED_LIBS ON CACHE BOOL "Build shared libraries") diff --git a/cmake/ConfigCPack.cmake b/cmake/ConfigCPack.cmake index 2f4b169494..71c734da66 100644 --- a/cmake/ConfigCPack.cmake +++ b/cmake/ConfigCPack.cmake @@ -11,13 +11,34 @@ if(DYNINST_BUILD_ELFUTILS AND DYNINST_ELFUTILS_DOWNLOAD_VERSION) endforeach() endif() -if(EXISTS /etc/lsb-release AND NOT IS_DIRECTORY /etc/lsb-release) - file(READ /etc/lsb-release _LSB_RELEASE) - if(_LSB_RELEASE) - string(REGEX REPLACE "DISTRIB_ID=(.*)\nDISTRIB_RELEASE=(.*)\nDISTRIB_CODENAME=.*" - "\\1-\\2" _SYSTEM_NAME "${_LSB_RELEASE}") +function(omnitrace_parse_release) + if(EXISTS /etc/lsb-release AND NOT IS_DIRECTORY /etc/lsb-release) + file(READ /etc/lsb-release _LSB_RELEASE) + if(_LSB_RELEASE) + string(REGEX + REPLACE "DISTRIB_ID=(.*)\nDISTRIB_RELEASE=(.*)\nDISTRIB_CODENAME=.*" + "\\1-\\2" _SYSTEM_NAME "${_LSB_RELEASE}") + endif() + elseif(EXISTS /etc/os-release AND NOT IS_DIRECTORY /etc/os-release) + file(READ /etc/os-release _OS_RELEASE) + if(_OS_RELEASE) + string(REPLACE "\"" "" _OS_RELEASE "${_OS_RELEASE}") + string(REPLACE "-" " " _OS_RELEASE "${_OS_RELEASE}") + string(REGEX REPLACE "NAME=.*\nVERSION=([0-9\.]+).*\nID=([a-z]+).*" "\\2-\\1" + _SYSTEM_NAME "${_OS_RELEASE}") + endif() endif() -endif() + string(TOLOWER "${_SYSTEM_NAME}" _SYSTEM_NAME) + if(NOT _SYSTEM_NAME) + set(_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}") + endif() + set(_SYSTEM_NAME + "${_SYSTEM_NAME}" + PARENT_SCOPE) +endfunction() + +# parse either /etc/lsb-release or /etc/os-release +omnitrace_parse_release() if(NOT _SYSTEM_NAME) set(_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}") @@ -86,9 +107,13 @@ if(OMNITRACE_USE_MPI) endif() if(OMNITRACE_USE_PYTHON) - string(REPLACE "." "" OMNITRACE_CPACK_PYTHON_VERSION "PY${OMNITRACE_PYTHON_VERSION}") - set(OMNITRACE_CPACK_PACKAGE_SUFFIX - "${OMNITRACE_CPACK_PACKAGE_SUFFIX}-${OMNITRACE_CPACK_PYTHON_VERSION}") + set(_OMNITRACE_PYTHON_NAME "Python3") + foreach(_VER ${OMNITRACE_PYTHON_VERSIONS}) + if("${_VER}" VERSION_LESS 3.0.0) + set(_OMNITRACE_PYTHON_NAME "Python") + endif() + endforeach() + set(OMNITRACE_CPACK_PACKAGE_SUFFIX "${OMNITRACE_CPACK_PACKAGE_SUFFIX}-Python3") endif() set(CPACK_PACKAGE_FILE_NAME diff --git a/cmake/Packages.cmake b/cmake/Packages.cmake index 6c7b156b02..b839ade4e4 100644 --- a/cmake/Packages.cmake +++ b/cmake/Packages.cmake @@ -584,11 +584,15 @@ omnitrace_add_feature(CMAKE_CXX_FLAGS "C++ compiler flags") # # ----------------------------------------------------------------------------------------# +set(OMNITRACE_INSTALL_PYTHONDIR + "${CMAKE_INSTALL_LIBDIR}/python/site-packages" + CACHE STRING "Installation prefix for python") +set(CMAKE_INSTALL_PYTHONDIR ${OMNITRACE_INSTALL_PYTHONDIR}) + if(OMNITRACE_USE_PYTHON) if(OMNITRACE_USE_PYTHON AND NOT OMNITRACE_BUILD_PYTHON) find_package(pybind11 REQUIRED) endif() - list(INSERT CMAKE_MODULE_PATH 0 ${PROJECT_SOURCE_DIR}/source/python/cmake) include(ConfigPython) endif() diff --git a/docker/Dockerfile.centos b/docker/Dockerfile.centos index 4581b3d1fe..99ba35ca29 100644 --- a/docker/Dockerfile.centos +++ b/docker/Dockerfile.centos @@ -16,7 +16,7 @@ RUN yum update -y && \ yum groupinstall -y "Development Tools" && \ yum install -y centos-release-scl && \ yum install -y epel-release && \ - yum install -y devtoolset-9 python3-pip openmpi3-devel zlib-devel numactl-devel papi-devel dpkg-devel dpkg-dev && \ + yum install -y devtoolset-9 python3-pip openmpi3-devel zlib-devel numactl-devel papi-devel dpkg-devel dpkg-dev wget && \ python3 -m pip install 'cmake==3.18.4' ARG AMDGPU_RPM=21.40.2/rhel/7.9/amdgpu-install-21.40.2.40502-1.el7.noarch.rpm @@ -28,6 +28,17 @@ RUN yum install -y https://repo.radeon.com/amdgpu-install/${AMDGPU_RPM} && \ yum update -y && \ yum clean all +ARG PYTHON_VERSIONS="6 7 8 9" + +RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ + bash miniconda.sh -b -p /opt/conda && \ + export PATH="/opt/conda/bin:${PATH}" && \ + conda config --set always_yes yes --set changeps1 no && \ + conda update -c defaults -n base conda && \ + for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \ + conda clean -a -y && \ + conda init + RUN ln -s /opt/rocm-* /opt/rocm WORKDIR /home diff --git a/docker/Dockerfile.opensuse b/docker/Dockerfile.opensuse index 64e0f94e5d..68739bc7bd 100644 --- a/docker/Dockerfile.opensuse +++ b/docker/Dockerfile.opensuse @@ -15,7 +15,7 @@ ENV PATH /usr/local/bin:${PATH} RUN zypper update -y && \ zypper dist-upgrade -y && \ zypper install -y -t pattern devel_basis && \ - zypper install -y python3-pip openmpi3-devel gcc-c++ git libnuma-devel dpkg-devel rpm-build && \ + zypper install -y python3-pip openmpi3-devel gcc-c++ git libnuma-devel dpkg-devel rpm-build wget && \ python3 -m pip install 'cmake==3.18.4' # ARG AMDGPU_RPM=21.40.2/sle/15/amdgpu-install-21.40.2.40502-1.noarch.rpm @@ -28,5 +28,16 @@ RUN zypper --no-gpg-checks install -y https://repo.radeon.com/amdgpu-install/${A zypper install -y rocm-hip-sdk roctracer-dev rocm-smi-lib rocprofiler-dev && \ zypper clean --all +ARG PYTHON_VERSIONS="6 7 8 9" + +RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ + bash miniconda.sh -b -p /opt/conda && \ + export PATH="/opt/conda/bin:${PATH}" && \ + conda config --set always_yes yes --set changeps1 no && \ + conda update -c defaults -n base conda && \ + for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \ + conda clean -a -y && \ + conda init + WORKDIR /home SHELL [ "/bin/bash", "--login", "-c" ] diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 07e335293b..158f19b820 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -17,18 +17,28 @@ SHELL [ "/bin/bash", "-c" ] ARG EXTRA_PACKAGES="" ARG ROCM_REPO_VERSION="debian" ARG ROCM_REPO_DIST="ubuntu" - +ARG PYTHON_VERSIONS="6 7 8 9" ENV PATH ${HOME}/.local/bin:${PATH} RUN apt-get update && \ apt-get dist-upgrade -y && \ - apt-get install -y build-essential cmake libnuma-dev wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev-dev && \ + apt-get install -y build-essential cmake libnuma-dev wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release libpapi-dev libpfm4-dev libudev-dev libopenmpi-dev rpm librpm-dev && \ python3 -m pip install 'cmake==3.18.4' && \ wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \ echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${ROCM_REPO_VERSION}/ ${ROCM_REPO_DIST} main" | tee /etc/apt/sources.list.d/rocm.list && \ apt-get update && \ apt-get dist-upgrade -y && \ - apt-get install -y rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${EXTRA_PACKAGES} + apt-get install -y rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${EXTRA_PACKAGES} && \ + apt-get autoclean + +RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ + bash miniconda.sh -b -p /opt/conda && \ + export PATH="/opt/conda/bin:${PATH}" && \ + conda config --set always_yes yes --set changeps1 no && \ + conda update -c defaults -n base conda && \ + for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \ + conda clean -a -y && \ + conda init ENV LC_ALL C.UTF-8 WORKDIR /home diff --git a/docker/build-docker-release.sh b/docker/build-docker-release.sh index 0e1163f0f7..68fea514cd 100755 --- a/docker/build-docker-release.sh +++ b/docker/build-docker-release.sh @@ -1,27 +1,16 @@ #!/usr/bin/env bash if [ ! -f CMakeLists.txt ]; then - echo "Error! Execute script from source directory" - exit 1 + if [ ! -f ../CMakeLists.txt ]; then + echo "Error! Execute script from source directory" + exit 1 + else + cd .. + fi fi set -e -build-release() -{ - CONTAINER=$1 - OS=$2 - ROCM_VERSION=$3 - CODE_VERSION=$4 - MPI=$4 - docker run -it --rm -v ${PWD}:/home/omnitrace --env DISTRO=${OS} --env ROCM_VERSION=${ROCM_VERSION} --env VERSION=${CODE_VERSION} --env MPI=${MPI} ${CONTAINER} /home/omnitrace/scripts/build-release.sh -} - -: ${DISTRO:=ubuntu} -: ${VERSIONS:=20.04 18.04} -: ${ROCM_VERSIONS:=5.0 4.5 4.3} -: ${MPI:=0} - send-error() { echo -e "\nError: ${@}" @@ -34,6 +23,25 @@ verbose-run() eval $@ } +build-release() +{ + CONTAINER=$1 + OS=$2 + ROCM_VERSION=$3 + CODE_VERSION=$4 + shift + shift + shift + shift + verbose-run docker run -it --rm -v ${PWD}:/home/omnitrace --env DISTRO=${OS} --env ROCM_VERSION=${ROCM_VERSION} --env VERSION=${CODE_VERSION} --env PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" ${CONTAINER} /home/omnitrace/scripts/build-release.sh ${@} +} + +: ${DISTRO:=ubuntu} +: ${VERSIONS:=20.04 18.04} +: ${ROCM_VERSIONS:=5.0 4.5 4.3} +: ${MPI:=0} +: ${PYTHON_VERSIONS:=6 7 8 9} + n=0 while [[ $# -gt 0 ]] do @@ -50,16 +58,17 @@ do shift ROCM_VERSIONS=${1} ;; + "--python-versions") + shift + PYTHON_VERSIONS=${1} + ;; + "--") + shift + SCRIPT_ARGS=${@} + break + ;; *) - if [ "${n}" -eq 0 ]; then - DISTRO=${1} - elif [ "${n}" -eq 1 ]; then - VERSIONS=${1} - elif [ "${n}" -eq 2 ]; then - ROCM_VERSIONS=${1} - else - send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" - fi + send-error "Unsupported argument at position $((${n} + 1)) :: ${1}" ;; esac n=$((${n} + 1)) @@ -73,6 +82,6 @@ do TAG=${DISTRO}-${VERSION} for ROCM_VERSION in ${ROCM_VERSIONS} do - build-release jrmadsen/omnitrace-${TAG}-rocm-${ROCM_VERSION} ${DISTRO}-${VERSION} ${ROCM_VERSION} ${CODE_VERSION} ${MPI} + build-release jrmadsen/omnitrace-${TAG}-rocm-${ROCM_VERSION} ${DISTRO}-${VERSION} ${ROCM_VERSION} ${CODE_VERSION} ${SCRIPT_ARGS} done done diff --git a/docker/build-docker.sh b/docker/build-docker.sh index dc8aa364f5..7ed25f8866 100755 --- a/docker/build-docker.sh +++ b/docker/build-docker.sh @@ -3,6 +3,7 @@ : ${ROCM_VERSIONS:="5.0 4.5 4.3"} : ${DISTRO:=ubuntu} : ${VERSIONS:=20.04 18.04} +: ${PYTHON_VERSIONS:="6 7 8 9"} : ${CI:=""} set -e @@ -35,6 +36,10 @@ do shift ROCM_VERSIONS=${1} ;; + "--python-versions") + shift + PYTHON_VERSIONS=${1} + ;; *) if [ "${n}" -eq 0 ]; then DISTRO=${1} @@ -65,17 +70,16 @@ do ROCM_REPO_DIST="ubuntu" ROCM_REPO_VERSION=${i} case "${i}" in - 5.0*) + 5.1*) ROCM_REPO_VERSION="debian" ;; 4.1* | 4.0*) ROCM_REPO_DIST="xenial" ;; *) - send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${i}" ;; esac - verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} + verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" elif [ "${DISTRO}" = "centos" ]; then case "${VERSION}" in 7) @@ -106,7 +110,7 @@ do send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${i}" ;; esac - verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} + verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" elif [ "${DISTRO}" = "opensuse" ]; then case "${VERSION}" in 15.*) @@ -134,7 +138,7 @@ do send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${i}" ;; esac - verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} + verbose-run docker build . -f ${DOCKER_FILE} --tag jrmadsen/omnitrace-${DISTRO}-${VERSION}-rocm-${i} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" fi done done diff --git a/examples/python/builtin.py b/examples/python/builtin.py index c0bd45ba54..7b3a045099 100755 --- a/examples/python/builtin.py +++ b/examples/python/builtin.py @@ -1,7 +1,10 @@ #!@PYTHON_EXECUTABLE@ +import os import sys -import numpy as np +import random + +_prefix = "" def fib(n): @@ -9,28 +12,41 @@ def fib(n): def inefficient(n): - print(f"inefficient: {n}") + print(f"[{_prefix}] ... running inefficient({n})") a = 0 for i in range(n): a += i for j in range(n): a += j - arr = np.random.rand(a * n * n * n) - sum = arr.sum() - print(f"sum: {sum}") - return sum + _len = a * n * n * n + _arr = [random.random() for _ in range(_len)] + _sum = sum(_arr) + print(f"[{_prefix}] ... sum of {_len} random elements: {_sum}") + return _sum @profile -def run(nfib): - ret = 0 - ret += fib(nfib) - ret += inefficient(nfib) - return ret +def run(n): + _ret = 0 + _ret += fib(n) + _ret += inefficient(n) + return _ret if __name__ == "__main__": - nfib = int(sys.argv[1]) if len(sys.argv) > 1 else 20 - for i in range(5): - ans = run(nfib) - print(f"[{i}] fibonacci({nfib}) = {ans}") + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-n", "--num-iterations", help="Number", type=int, default=3 + ) + parser.add_argument( + "-v", "--value", help="Starting value", type=int, default=20 + ) + args = parser.parse_args() + + _prefix = os.path.basename(__file__) + print(f"[{_prefix}] Executing {args.num_iterations} iterations...\n") + for i in range(args.num_iterations): + ans = run(args.value) + print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans}\n") diff --git a/examples/python/external.py b/examples/python/external.py index 5b306c918f..bbaab0d1d5 100755 --- a/examples/python/external.py +++ b/examples/python/external.py @@ -1,7 +1,10 @@ #!@PYTHON_EXECUTABLE@ +import os import sys -import numpy as np +import random + +_prefix = "" def fib(n): @@ -9,27 +12,40 @@ def fib(n): def inefficient(n): - print(f"inefficient: {n}") + print(f"[{_prefix}] ... running inefficient({n})") a = 0 for i in range(n): a += i for j in range(n): a += j - arr = np.random.rand(a * n * n * n) - sum = arr.sum() - print(f"sum: {sum}") - return sum + _len = a * n * n * n + _arr = [random.random() for _ in range(_len)] + _sum = sum(_arr) + print(f"[{_prefix}] ... sum of {_len} random elements: {_sum}") + return _sum -def run(nfib): - ret = 0 - ret += fib(nfib) - ret += inefficient(nfib) - return ret +def run(n): + _ret = 0 + _ret += fib(n) + _ret += inefficient(n) + return _ret if __name__ == "__main__": - nfib = int(sys.argv[1]) if len(sys.argv) > 1 else 20 - for i in range(5): - ans = run(nfib) - print(f"[{i}] fibonacci({nfib}) = {ans}") + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-n", "--num-iterations", help="Number", type=int, default=3 + ) + parser.add_argument( + "-v", "--value", help="Starting value", type=int, default=20 + ) + args = parser.parse_args() + + _prefix = os.path.basename(__file__) + print(f"[{_prefix}] Executing {args.num_iterations} iterations...\n") + for i in range(args.num_iterations): + ans = run(args.value) + print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans}\n") diff --git a/examples/python/pyproject.toml b/examples/python/pyproject.toml new file mode 100644 index 0000000000..cbd70117ab --- /dev/null +++ b/examples/python/pyproject.toml @@ -0,0 +1,5 @@ + +[tool.black] +line-length = 80 +target-version = ['py38'] +include = '\.py' diff --git a/examples/python/source.py b/examples/python/source.py index 85b495fd75..57e159070d 100755 --- a/examples/python/source.py +++ b/examples/python/source.py @@ -1,36 +1,73 @@ #!@PYTHON_EXECUTABLE@ +import os import sys -import numpy as np import omnitrace +_prefix = "" + + def fib(n): return n if n < 2 else (fib(n - 1) + fib(n - 2)) -def inefficient(n): - print(f"inefficient: {n}") - a = 0 - for i in range(n): - a += i - for j in range(n): - a += j - arr = np.random.rand(a * n * n * n) - sum = arr.sum() - print(f"sum: {sum}") - return sum +try: + import numpy as np + + def inefficient(n): + print(f"[{_prefix}] ... running inefficient({n})") + a = 0 + for i in range(n): + a += i + for j in range(n): + a += j + _len = a * n * n * n + _ret = np.random.rand(_len).sum() + print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}") + return _ret + +except ImportError: + import random + + def _sum(arr): + return sum(arr) + + def inefficient(n): + print(f"[{_prefix}] ... running inefficient({n})") + a = 0 + for i in range(n): + a += i + for j in range(n): + a += j + _len = a * n * n * n + _arr = [random.random() for _ in range(_len)] + _ret = _sum(_arr) + print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}") + return _ret @omnitrace.profile() -def run(nfib): - ret = 0 - ret += fib(nfib) - ret += inefficient(nfib) - return ret +def run(n): + _ret = 0 + _ret += fib(n) + _ret += inefficient(n) + return _ret if __name__ == "__main__": - nfib = int(sys.argv[1]) if len(sys.argv) > 1 else 20 - for i in range(5): - ans = run(nfib) - print(f"[{i}] fibonacci({nfib}) = {ans}") + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-n", "--num-iterations", help="Number", type=int, default=3 + ) + parser.add_argument( + "-v", "--value", help="Starting value", type=int, default=20 + ) + args = parser.parse_args() + + _prefix = os.path.basename(__file__) + print(f"[{_prefix}] Executing {args.num_iterations} iterations...\n") + for i in range(args.num_iterations): + ans = run(args.value) + print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans}\n") diff --git a/external/pybind11 b/external/pybind11 new file mode 160000 index 0000000000..ad0de0f5a6 --- /dev/null +++ b/external/pybind11 @@ -0,0 +1 @@ +Subproject commit ad0de0f5a6bebbebbeb7f8f2f15c0c1430f34268 diff --git a/scripts/build-release.sh b/scripts/build-release.sh index c8cf171f93..8a2bb8a44a 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -1,28 +1,31 @@ #!/bin/bash -e : ${EXTRA_ARGS:=""} -: ${EXTRA_TAGS:=""} : ${BUILD_DIR:=build-release} : ${VERSION:=0.0.4} : ${ROCM_VERSION:=4.5.0} -: ${NJOBS:=8} +: ${NJOBS:=12} : ${DISTRO:=""} -: ${LTO:="ON"} +: ${LTO:="OFF"} +: ${PYTHON_VERSIONS:="6 7 8 9"} +: ${GENERATORS:="STGZ DEB RPM"} +: ${MPI_IMPL:="openmpi"} if [ -z "${DISTRO}" ]; then - DISTRO=$(lsb_release -i | awk '{print $NF}')-$(lsb_release -r | awk '{print $NF}') + if [ -f /etc/os-release ]; then + DISTRO=$(cat /etc/os-release | egrep "^ID=" | sed 's/"/ /g' | sed 's/=/ /g' | sed 's/-/ /g' | awk '{print $2}')-$(cat /etc/os-release | egrep "^VERSION=" | sed 's/"/ /g' | sed 's/=/ /g' | awk '{print $2}') + else + DISTRO=$(lsb_release -i | awk '{print $NF}')-$(lsb_release -r | awk '{print $NF}') + fi fi -STANDARD_ARGS="-DCPACK_GENERATOR=STGZ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DOMNITRACE_MAX_THREADS=2048 -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_BUILD_EXAMPLES=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_CPACK_SYSTEM_NAME=${DISTRO} -DOMNITRACE_ROCM_VERSION=${ROCM_VERSION} -DOMNITRACE_BUILD_LTO=${LTO} -DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON" -STANDARD_ARGS="${STANDARD_ARGS} -DOMNITRACE_BUILD_DYNINST=ON $(echo -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON)" -if [ -n "${EXTRA_ARGS}" ]; then - STANDARD_ARGS="${STANDARD_ARGS} ${EXTRA_ARGS}" -fi - -PACKAGE_BASE_TAG=omnitrace-${VERSION}-${DISTRO} -if [ -n "${EXTRA_TAGS}" ]; then - PACKAGE_BASE_TAG="${PACKAGE_BASE_TAG}-${EXTRA_TAGS}" -fi +CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCPACK_GENERATOR=STGZ" +OMNITRACE_GENERAL_ARGS="-DOMNITRACE_CPACK_SYSTEM_NAME=${DISTRO} -DOMNITRACE_ROCM_VERSION=${ROCM_VERSION} -DOMNITRACE_MAX_THREADS=2048 " +OMNITRACE_BUILD_ARGS="-DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_BUILD_EXAMPLES=OFF -DOMNITRACE_BUILD_PAPI=ON -DOMNITRACE_BUILD_LTO=${LTO}" +OMNITRACE_USE_ARGS="-DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_USE_PAPI=OFF" +TIMEMORY_ARGS="-DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON" +DYNINST_ARGS="${STANDARD_ARGS} -DOMNITRACE_BUILD_DYNINST=ON $(echo -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON)" +STANDARD_ARGS="${CMAKE_ARGS} ${OMNITRACE_GENERAL_ARGS} ${OMNITRACE_USE_ARGS} ${OMNITRACE_BUILD_ARGS} ${TIMEMORY_ARGS} ${DYNINST_ARGS} ${EXTRA_ARGS}" SCRIPT_DIR=$(realpath $(dirname ${BASH_SOURCE[0]})) cd $(dirname ${SCRIPT_DIR}) @@ -30,25 +33,177 @@ echo -e "Working directory: $(pwd)" umask 0000 -build-and-package() +verbose-run() +{ + echo -e "\n##### Executing \"${@}\"... #####\n" + eval $@ +} + +copy-installer() +{ + TYPE=${1} + shift + DEST=${1} + shift + if [ -z "${1}" ]; then + echo "Warning! No cpack installer for ${TYPE} generator" + return + else + for i in ${@} + do + verbose-run cp ${i} ${DEST}/ + done + fi +} + +tolower() +{ + echo "$@" | awk -F '\|~\|' '{print tolower($1)}'; +} + +build-and-package-base() { local DIR=${1} shift - cmake -B ${BUILD_DIR}/${DIR} -DCMAKE_INSTALL_PREFIX=${BUILD_DIR}/${DIR}/install-release ${STANDARD_ARGS} $@ . - cmake --build ${BUILD_DIR}/${DIR} --target all --parallel ${NJOBS} + verbose-run cmake -B ${BUILD_DIR}/${DIR} -DCMAKE_INSTALL_PREFIX=${BUILD_DIR}/${DIR}/install-release ${STANDARD_ARGS} $@ . + verbose-run cmake --build ${BUILD_DIR}/${DIR} --target all --parallel ${NJOBS} pushd ${BUILD_DIR}/${DIR} - rm -f *.sh *.deb *.rpm - cpack -G STGZ - cpack -G DEB -D CPACK_PACKAGING_INSTALL_PREFIX=/opt/omnitrace - cpack -G RPM -D CPACK_PACKAGING_INSTALL_PREFIX=/opt/omnitrace + verbose-run rm -f *.sh *.deb *.rpm popd - cp ${BUILD_DIR}/${DIR}/omnitrace-${VERSION}-*.sh ${BUILD_DIR}/ - cp ${BUILD_DIR}/${DIR}/omnitrace_${VERSION}-*.deb ${BUILD_DIR}/ - cp ${BUILD_DIR}/${DIR}/omnitrace-${VERSION}-*.rpm ${BUILD_DIR}/ + for i in ${GENERATORS} + do + pushd ${BUILD_DIR}/${DIR} + case "${i}" in + STGZ) + verbose-run cpack -G STGZ + EXT="sh" + SEP="-" + DEST="stgz" + ;; + DEB) + verbose-run cpack -G DEB -D CPACK_PACKAGING_INSTALL_PREFIX=/opt/omnitrace + EXT="deb" + SEP="_" + DEST="deb" + ;; + DEB | RPM) + verbose-run cpack -G RPM -D CPACK_PACKAGING_INSTALL_PREFIX=/opt/omnitrace + EXT="rpm" + SEP="-" + DEST="rpm" + ;; + *) + echo "Unsupported cpack generator: ${i}" + continue + ;; + esac + popd + verbose-run mkdir -p ${BUILD_DIR}/${DEST} + verbose-run copy-installer ${i} ${BUILD_DIR}/${DEST} ${BUILD_DIR}/${DIR}/omnitrace${SEP}${VERSION}-*.${EXT} + done } -build-and-package ${DISTRO}-core -DDYNINST_USE_OpenMP=OFF -DOMNITRACE_USE_HIP=OFF -# build-and-package ${DISTRO}-rocm-${ROCM_VERSION} -DOMNITRACE_USE_HIP=ON -DDYNINST_USE_OpenMP=ON -# build-and-package ${DISTRO}-rocm-${ROCM_VERSION}-papi -DOMNITRACE_USE_HIP=ON -DDYNINST_USE_OpenMP=ON -DOMNITRACE_USE_PAPI=ON +build-and-package-python() +{ + local DIR=${1} + shift + for i in ${PYTHON_VERSIONS} + do + conda activate py3.${i} + _PYTHON_VERS="${_PYTHON_VERS}3.${i};" + _PYTHON_DIRS="${_PYTHON_DIRS}$(dirname $(dirname $(which python)));" + conda deactivate + done + build-and-package-base ${DIR}-python $@ -DOMNITRACE_USE_PYTHON=ON -DOMNITRACE_BUILD_PYTHON=ON -DOMNITRACE_PYTHON_VERSIONS=\"${_PYTHON_VERS}\" -DOMNITRACE_PYTHON_ROOT_DIRS=\"${_PYTHON_DIRS}\" +} -# build-and-package ${DISTRO}-rocm-${ROCM_VERSION}-papi-openmpi -DOMNITRACE_USE_HIP=ON -DDYNINST_USE_OpenMP=ON -DOMNITRACE_USE_PAPI=ON -DOMNITRACE_USE_MPI=ON +build-and-package() +{ + local VAL=${1} + shift + if [ "${VAL}" -eq 1 ]; then + build-and-package-base ${@} + elif [ "${VAL}" -eq 2 ]; then + build-and-package-python ${@} + elif [ "${VAL}" -eq 3 ]; then + build-and-package-base ${@} + build-and-package-python ${@} + else + echo -e "Skipping build/package for ${1}" + fi +} + +: ${WITH_CORE:=0} +: ${WITH_MPI:=0} +: ${WITH_ROCM:=0} +: ${WITH_ROCM_MPI:=0} + +usage() +{ + print_option() { printf " --%-10s %-24s %s\n" "${1}" "${2}" "${3}"; } + echo "Options:" + python_info="(Use '-python' to build w/o python, use '+python' to python build with python)" + print_option core "[+nopython] [+python]" "Core ${python_info}" + print_option mpi "[+nopython] [+python]" "MPI ${python_info}" + print_option rocm "[+nopython] [+python]" "ROCm ${python_info}" + print_option rocm-mpi "[+nopython] [+python]" "ROCm + MPI ${python_info}" + print_option mpi-impl "[openmpi|mpich]" "MPI implementation" +} + +while [[ $# -gt 0 ]] +do + ARG=${1} + shift + VAL=0 + + while [[ $# -gt 0 ]] + do + if [ "$1" = "-python" ]; then + VAL=$(( ${VAL} + 1 )) + shift + elif [ "$1" = "+python" ]; then + VAL=$(( ${VAL} + 2 )) + shift + else + break + fi + done + + case "${ARG}" in + ? | -h | --help) + usage + exit 0 + ;; + --core) + WITH_CORE=${VAL} + ;; + --mpi) + WITH_MPI=${VAL} + ;; + --rocm) + WITH_ROCM=${VAL} + ;; + --rocm-mpi) + WITH_ROCM_MPI=${VAL} + ;; + --mpi-impl) + MPI_IMPL=${1} + shift + ;; + *) + echo -e "Error! Unknown option : ${ARG}" + usage + exit 1 + ;; + esac +done + +if [ -d /opt/conda/bin ]; then + export PATH=${PATH}:/opt/conda/bin + source activate +fi + +build-and-package ${WITH_CORE} ${DISTRO}-core -DDYNINST_USE_OpenMP=OFF -DOMNITRACE_USE_HIP=OFF +build-and-package ${WITH_MPI} ${DISTRO}-${MPI_IMPL} -DOMNITRACE_USE_HIP=ON -DDYNINST_USE_OpenMP=ON +build-and-package ${WITH_ROCM} ${DISTRO}-rocm-${ROCM_VERSION} -DOMNITRACE_USE_HIP=ON -DDYNINST_USE_OpenMP=ON +build-and-package ${WITH_ROCM_MPI} ${DISTRO}-rocm-${ROCM_VERSION}-${MPI_IMPL} -DOMNITRACE_USE_HIP=ON -DDYNINST_USE_OpenMP=ON -DOMNITRACE_USE_MPI=ON diff --git a/source/bin/CMakeLists.txt b/source/bin/CMakeLists.txt index e3fdff5d0b..0c1cf69cf9 100644 --- a/source/bin/CMakeLists.txt +++ b/source/bin/CMakeLists.txt @@ -1,3 +1,9 @@ +# executable RPATH +set(OMNITRACE_EXE_INSTALL_RPATH + "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/omnitrace:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/timemory/libunwind:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib" + ) + +# executables add_subdirectory(omnitrace-avail) add_subdirectory(omnitrace-critical-trace) add_subdirectory(omnitrace) diff --git a/source/bin/omnitrace-avail/CMakeLists.txt b/source/bin/omnitrace-avail/CMakeLists.txt index 392544929d..658ddd6bfe 100644 --- a/source/bin/omnitrace-avail/CMakeLists.txt +++ b/source/bin/omnitrace-avail/CMakeLists.txt @@ -14,12 +14,9 @@ target_compile_definitions(omnitrace-avail PRIVATE OMNITRACE_EXTERN_COMPONENTS=0 target_link_libraries(omnitrace-avail PRIVATE omnitrace::omnitrace-interface-library) set_target_properties( omnitrace-avail - PROPERTIES - BUILD_RPATH "\$ORIGIN:${PROJECT_BINARY_DIR}:${CMAKE_BINARY_DIR}" - INSTALL_RPATH_USE_LINK_PATH ON - INSTALL_RPATH - "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/timemory/libunwind:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib" - ) + PROPERTIES BUILD_RPATH "\$ORIGIN:${PROJECT_BINARY_DIR}:${CMAKE_BINARY_DIR}" + INSTALL_RPATH_USE_LINK_PATH ON + INSTALL_RPATH "${OMNITRACE_EXE_INSTALL_RPATH}") install( TARGETS omnitrace-avail diff --git a/source/bin/omnitrace-critical-trace/CMakeLists.txt b/source/bin/omnitrace-critical-trace/CMakeLists.txt index fe56a7badc..2af302503f 100644 --- a/source/bin/omnitrace-critical-trace/CMakeLists.txt +++ b/source/bin/omnitrace-critical-trace/CMakeLists.txt @@ -17,12 +17,9 @@ target_link_libraries(omnitrace-critical-trace PRIVATE omnitrace::omnitrace-interface-library) set_target_properties( omnitrace-critical-trace - PROPERTIES - BUILD_RPATH "\$ORIGIN:${PROJECT_BINARY_DIR}:${CMAKE_BINARY_DIR}" - INSTALL_RPATH_USE_LINK_PATH ON - INSTALL_RPATH - "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/timemory/libunwind:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib" - ) + PROPERTIES BUILD_RPATH "\$ORIGIN:${PROJECT_BINARY_DIR}:${CMAKE_BINARY_DIR}" + INSTALL_RPATH_USE_LINK_PATH ON + INSTALL_RPATH "${OMNITRACE_EXE_INSTALL_RPATH}") install( TARGETS omnitrace-critical-trace diff --git a/source/bin/omnitrace/CMakeLists.txt b/source/bin/omnitrace/CMakeLists.txt index 9a4f283fbb..db874dc091 100644 --- a/source/bin/omnitrace/CMakeLists.txt +++ b/source/bin/omnitrace/CMakeLists.txt @@ -28,13 +28,10 @@ target_link_libraries( set_target_properties( omnitrace-exe - PROPERTIES - OUTPUT_NAME omnitrace - BUILD_RPATH "\$ORIGIN:${PROJECT_BINARY_DIR}:${CMAKE_BINARY_DIR}" - INSTALL_RPATH_USE_LINK_PATH ON - INSTALL_RPATH - "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/timemory/libunwind:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib" - ) + PROPERTIES OUTPUT_NAME omnitrace + BUILD_RPATH "\$ORIGIN:${PROJECT_BINARY_DIR}:${CMAKE_BINARY_DIR}" + INSTALL_RPATH_USE_LINK_PATH ON + INSTALL_RPATH "${OMNITRACE_EXE_INSTALL_RPATH}") if(CMAKE_BUILD_TYPE MATCHES "^(DEBUG|Debug)") string(REPLACE " " ";" _FLAGS "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") diff --git a/source/docs/python.md b/source/docs/python.md index bed1e9966f..8dc20f6ec3 100644 --- a/source/docs/python.md +++ b/source/docs/python.md @@ -7,7 +7,11 @@ ``` [Omnitrace](https://github.com/AMDResearch/omnitrace) supports profiling Python code at the source-level and/or the script-level. -Python support is enabled via the `OMNITRACE_USE_PYTHON` and `OMNITRACE_PYTHON_VERSION=.` CMake options. +Python support is enabled via the `OMNITRACE_USE_PYTHON` and the `OMNITRACE_PYTHON_VERSIONS=".` CMake options. +Alternatively, to build multiple python versions, use `OMNITRACE_PYTHON_VERSIONS=".;[.]"`, +and `OMNITRACE_PYTHON_ROOT_DIRS="/path/to/version;[/path/to/version]"` instead of `OMNITRACE_PYTHON_VERSION`. +When building multiple Python versions, the length of the `OMNITRACE_PYTHON_VERSIONS` and `OMNITRACE_PYTHON_ROOT_DIRS` lists must +be the same size. > ***When using omnitrace for Python, the Python interpreter major and minor version (e.g. 3.7) must match the interpreter major and minor version*** > ***used when compiling the Python bindings, i.e. when building omnitrace, a `libpyomnitrace.----.so` will be generated*** diff --git a/source/lib/CMakeLists.txt b/source/lib/CMakeLists.txt index 8df9b66904..ce499b4f4a 100644 --- a/source/lib/CMakeLists.txt +++ b/source/lib/CMakeLists.txt @@ -8,6 +8,19 @@ # # ----------------------------------------------------------------------------- # +string(REPLACE ":" ";" _INSTALL_RPATH "${CMAKE_INSTALL_RPATH}") +if(_INSTALL_RPATH) + list(REMOVE_DUPLICATES _INSTALL_RPATH) +endif() +string(REPLACE ";" ":" _INSTALL_RPATH "${_INSTALL_RPATH}") +string( + REPLACE + "::" + ":" + OMNITRACE_LIB_INSTALL_RPATH + "\$ORIGIN:\$ORIGIN/omnitrace:\$ORIGIN/timemory/libunwind:\$ORIGIN/dyninst-tpls/libs:${_INSTALL_RPATH}" + ) + add_subdirectory(common) add_subdirectory(omnitrace) add_subdirectory(omnitrace-dl) diff --git a/source/lib/omnitrace/CMakeLists.txt b/source/lib/omnitrace/CMakeLists.txt index 1f29b9e8d1..12fe86ab6e 100644 --- a/source/lib/omnitrace/CMakeLists.txt +++ b/source/lib/omnitrace/CMakeLists.txt @@ -59,6 +59,7 @@ set(library_sources ${CMAKE_CURRENT_LIST_DIR}/src/library/ompt.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/perfetto.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/ptl.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/library/runtime.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/sampling.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/state.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/thread_data.cpp @@ -84,6 +85,7 @@ set(library_headers ${CMAKE_CURRENT_LIST_DIR}/include/library/ompt.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/perfetto.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/ptl.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/library/runtime.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/sampling.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/state.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/thread_data.hpp @@ -139,8 +141,7 @@ set_target_properties( PROPERTIES OUTPUT_NAME omnitrace VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR} - INSTALL_RPATH - "\$ORIGIN:\$ORIGIN/timemory/libunwind:\$ORIGIN/dyninst-tpls/libs") + INSTALL_RPATH "${OMNITRACE_LIB_INSTALL_RPATH}") omnitrace_strip_target(omnitrace-library) diff --git a/source/lib/omnitrace/include/library/config.hpp b/source/lib/omnitrace/include/library/config.hpp index fc103a1fe3..11b4ad63f9 100644 --- a/source/lib/omnitrace/include/library/config.hpp +++ b/source/lib/omnitrace/include/library/config.hpp @@ -24,10 +24,6 @@ #include "library/api.hpp" #include "library/common.hpp" -#include "library/components/fork_gotcha.hpp" -#include "library/components/mpi_gotcha.hpp" -#include "library/components/pthread_gotcha.hpp" -#include "library/components/roctracer.hpp" #include "library/defines.hpp" #include "library/state.hpp" #include "library/timemory.hpp" @@ -41,38 +37,13 @@ namespace omnitrace { -// bundle of components around omnitrace_init and omnitrace_finalize -using main_bundle_t = - tim::lightweight_tuple; - -using gotcha_bundle_t = tim::lightweight_tuple; - -// bundle of components used in instrumentation -using instrumentation_bundle_t = - tim::component_bundle; - -// allocator for instrumentation_bundle_t -using bundle_allocator_t = tim::data::ring_buffer_allocator; - -// bundle of components around each thread -#if defined(TIMEMORY_RUSAGE_THREAD) && TIMEMORY_RUSAGE_THREAD > 0 -using omnitrace_thread_bundle_t = - tim::lightweight_tuple; -#else -using omnitrace_thread_bundle_t = - tim::lightweight_tuple; -#endif - // // Initialization routines // inline namespace config { void -configure_settings(); +configure_settings(bool _init = true); void print_banner(std::ostream& _os = std::cout); @@ -253,30 +224,4 @@ get_state(); /// returns old state State set_state(State); - -std::unique_ptr& -get_main_bundle(); - -std::unique_ptr& -get_gotcha_bundle(); - -std::atomic& -get_cpu_cid(); - -std::unique_ptr>& -get_cpu_cid_stack(int64_t _tid = threading::get_id(), int64_t _parent = 0); - -using cpu_cid_data_t = std::tuple; -using cpu_cid_pair_t = std::tuple; -using cpu_cid_parent_map_t = std::unordered_map; - -std::unique_ptr& -get_cpu_cid_parents(int64_t _tid = threading::get_id()); - -cpu_cid_data_t -create_cpu_cid_entry(int64_t _tid = threading::get_id()); - -cpu_cid_pair_t -get_cpu_cid_entry(uint64_t _cid, int64_t _tid = threading::get_id()); - } // namespace omnitrace diff --git a/source/lib/omnitrace/include/library/runtime.hpp b/source/lib/omnitrace/include/library/runtime.hpp new file mode 100644 index 0000000000..7ce7897610 --- /dev/null +++ b/source/lib/omnitrace/include/library/runtime.hpp @@ -0,0 +1,94 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include "library/api.hpp" +#include "library/common.hpp" +#include "library/components/fork_gotcha.hpp" +#include "library/components/mpi_gotcha.hpp" +#include "library/components/pthread_gotcha.hpp" +#include "library/components/roctracer.hpp" +#include "library/defines.hpp" +#include "library/state.hpp" +#include "library/timemory.hpp" + +#include +#include + +#include +#include +#include + +namespace omnitrace +{ +// bundle of components around omnitrace_init and omnitrace_finalize +using main_bundle_t = + tim::lightweight_tuple; + +using gotcha_bundle_t = tim::lightweight_tuple; + +// bundle of components used in instrumentation +using instrumentation_bundle_t = + tim::component_bundle; + +// allocator for instrumentation_bundle_t +using bundle_allocator_t = tim::data::ring_buffer_allocator; + +// bundle of components around each thread +#if defined(TIMEMORY_RUSAGE_THREAD) && TIMEMORY_RUSAGE_THREAD > 0 +using omnitrace_thread_bundle_t = + tim::lightweight_tuple; +#else +using omnitrace_thread_bundle_t = + tim::lightweight_tuple; +#endif + +std::unique_ptr& +get_main_bundle(); + +std::unique_ptr& +get_gotcha_bundle(); + +std::atomic& +get_cpu_cid(); + +std::unique_ptr>& +get_cpu_cid_stack(int64_t _tid = threading::get_id(), int64_t _parent = 0); + +using cpu_cid_data_t = std::tuple; +using cpu_cid_pair_t = std::tuple; +using cpu_cid_parent_map_t = std::unordered_map; + +std::unique_ptr& +get_cpu_cid_parents(int64_t _tid = threading::get_id()); + +cpu_cid_data_t +create_cpu_cid_entry(int64_t _tid = threading::get_id()); + +cpu_cid_pair_t +get_cpu_cid_entry(uint64_t _cid, int64_t _tid = threading::get_id()); + +} // namespace omnitrace diff --git a/source/lib/omnitrace/include/library/thread_data.hpp b/source/lib/omnitrace/include/library/thread_data.hpp index ec4b12ba7e..954a6e93c1 100644 --- a/source/lib/omnitrace/include/library/thread_data.hpp +++ b/source/lib/omnitrace/include/library/thread_data.hpp @@ -22,8 +22,8 @@ #pragma once -#include "library/config.hpp" #include "library/defines.hpp" +#include "library/runtime.hpp" #include #include diff --git a/source/lib/omnitrace/src/library/config.cpp b/source/lib/omnitrace/src/library/config.cpp index 3b6fe7e156..d927484d1c 100644 --- a/source/lib/omnitrace/src/library/config.cpp +++ b/source/lib/omnitrace/src/library/config.cpp @@ -21,10 +21,8 @@ // SOFTWARE. #include "library/config.hpp" -#include "library/api.hpp" #include "library/debug.hpp" #include "library/defines.hpp" -#include "library/thread_data.hpp" #include #include @@ -87,7 +85,7 @@ get_setting_name(std::string _v) inline namespace config { void -configure_settings() +configure_settings(bool _init) { static bool _once = false; if(_once) return; @@ -372,9 +370,12 @@ configure_settings() } if(!_found_sep && _cmd.size() > 1) _cmd.insert(_cmd.begin() + 1, "--"); - using argparser_t = tim::argparse::argument_parser; - argparser_t _parser{ _exe }; - tim::timemory_init(_cmd, _parser, "omnitrace-"); + if(_init) + { + using argparser_t = tim::argparse::argument_parser; + argparser_t _parser{ _exe }; + tim::timemory_init(_cmd, _parser, "omnitrace-"); + } settings::suppress_parsing() = true; settings::suppress_config() = true; @@ -1040,98 +1041,4 @@ set_state(State _n) get_state() = _n; return _o; } - -std::atomic& -get_cpu_cid() -{ - static std::atomic _v{ 0 }; - return _v; -} - -std::unique_ptr>& -get_cpu_cid_stack(int64_t _tid, int64_t _parent) -{ - struct omnitrace_cpu_cid_stack - {}; - using thread_data_t = thread_data, omnitrace_cpu_cid_stack>; - static auto& _v = thread_data_t::instances(); - static thread_local auto _v_copy = [_tid, _parent]() { - auto _parent_tid = _parent; - // if tid != parent and there is not a valid pointer for the provided parent - // thread id set it to zero since that will always be valid - if(_tid != _parent_tid && !_v.at(_parent_tid)) _parent_tid = 0; - // copy over the thread ids from the parent if tid != parent - thread_data_t::construct((_tid != _parent_tid) ? *(_v.at(_parent_tid)) - : std::vector{}); - return true; - }(); - return _v.at(_tid); - (void) _v_copy; -} - -std::unique_ptr& -get_cpu_cid_parents(int64_t _tid) -{ - struct omnitrace_cpu_cid_stack - {}; - using thread_data_t = thread_data; - static auto& _v = thread_data_t::instances(thread_data_t::construct_on_init{}, - cpu_cid_parent_map_t{}); - return _v.at(_tid); -} - -std::tuple -create_cpu_cid_entry(int64_t _tid) -{ - auto&& _cid = get_cpu_cid()++; - auto&& _parent_cid = (get_cpu_cid_stack(_tid)->empty()) ? get_cpu_cid_stack(0)->back() - : get_cpu_cid_stack()->back(); - uint16_t&& _depth = (get_cpu_cid_stack(_tid)->empty()) - ? get_cpu_cid_stack(0)->size() - : get_cpu_cid_stack()->size() - 1; - get_cpu_cid_parents(_tid)->emplace(_cid, std::make_tuple(_parent_cid, _depth)); - return std::make_tuple(_cid, _parent_cid, _depth); -} - -cpu_cid_pair_t -get_cpu_cid_entry(uint64_t _cid, int64_t _tid) -{ - return get_cpu_cid_parents(_tid)->at(_cid); -} - -namespace -{ -void -setup_gotchas() -{ - static bool _initialized = false; - if(_initialized) return; - _initialized = true; - - OMNITRACE_CONDITIONAL_PRINT_F( - get_debug_env(), - "Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n"); - - mpi_gotcha::configure(); - fork_gotcha::configure(); - pthread_gotcha::configure(); -} -} // namespace - -std::unique_ptr& -get_main_bundle() -{ - static auto _v = - std::make_unique("omnitrace", quirk::config{}); - return _v; -} - -std::unique_ptr& -get_gotcha_bundle() -{ - static auto _v = - (setup_gotchas(), std::make_unique( - "omnitrace", quirk::config{})); - return _v; -} } // namespace omnitrace diff --git a/source/lib/omnitrace/src/library/runtime.cpp b/source/lib/omnitrace/src/library/runtime.cpp new file mode 100644 index 0000000000..cada5d8bf4 --- /dev/null +++ b/source/lib/omnitrace/src/library/runtime.cpp @@ -0,0 +1,146 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "library/runtime.hpp" +#include "library/api.hpp" +#include "library/config.hpp" +#include "library/debug.hpp" +#include "library/defines.hpp" +#include "library/thread_data.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace omnitrace +{ +std::atomic& +get_cpu_cid() +{ + static std::atomic _v{ 0 }; + return _v; +} + +std::unique_ptr>& +get_cpu_cid_stack(int64_t _tid, int64_t _parent) +{ + struct omnitrace_cpu_cid_stack + {}; + using thread_data_t = thread_data, omnitrace_cpu_cid_stack>; + static auto& _v = thread_data_t::instances(); + static thread_local auto _v_copy = [_tid, _parent]() { + auto _parent_tid = _parent; + // if tid != parent and there is not a valid pointer for the provided parent + // thread id set it to zero since that will always be valid + if(_tid != _parent_tid && !_v.at(_parent_tid)) _parent_tid = 0; + // copy over the thread ids from the parent if tid != parent + thread_data_t::construct((_tid != _parent_tid) ? *(_v.at(_parent_tid)) + : std::vector{}); + return true; + }(); + return _v.at(_tid); + (void) _v_copy; +} + +std::unique_ptr& +get_cpu_cid_parents(int64_t _tid) +{ + struct omnitrace_cpu_cid_stack + {}; + using thread_data_t = thread_data; + static auto& _v = thread_data_t::instances(thread_data_t::construct_on_init{}, + cpu_cid_parent_map_t{}); + return _v.at(_tid); +} + +std::tuple +create_cpu_cid_entry(int64_t _tid) +{ + auto&& _cid = get_cpu_cid()++; + auto&& _parent_cid = (get_cpu_cid_stack(_tid)->empty()) ? get_cpu_cid_stack(0)->back() + : get_cpu_cid_stack()->back(); + uint16_t&& _depth = (get_cpu_cid_stack(_tid)->empty()) + ? get_cpu_cid_stack(0)->size() + : get_cpu_cid_stack()->size() - 1; + get_cpu_cid_parents(_tid)->emplace(_cid, std::make_tuple(_parent_cid, _depth)); + return std::make_tuple(_cid, _parent_cid, _depth); +} + +cpu_cid_pair_t +get_cpu_cid_entry(uint64_t _cid, int64_t _tid) +{ + return get_cpu_cid_parents(_tid)->at(_cid); +} + +namespace +{ +void +setup_gotchas() +{ + static bool _initialized = false; + if(_initialized) return; + _initialized = true; + + OMNITRACE_CONDITIONAL_PRINT_F( + get_debug_env(), + "Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n"); + + mpi_gotcha::configure(); + fork_gotcha::configure(); + pthread_gotcha::configure(); +} +} // namespace + +std::unique_ptr& +get_main_bundle() +{ + static auto _v = + std::make_unique("omnitrace", quirk::config{}); + return _v; +} + +std::unique_ptr& +get_gotcha_bundle() +{ + static auto _v = + (setup_gotchas(), std::make_unique( + "omnitrace", quirk::config{})); + return _v; +} +} // namespace omnitrace diff --git a/source/python/CMakeLists.txt b/source/python/CMakeLists.txt index bd53d811fb..b9f3e28195 100644 --- a/source/python/CMakeLists.txt +++ b/source/python/CMakeLists.txt @@ -12,31 +12,23 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME python) # ######################################################################################## -function(OMNITRACE_CONFIGURE_PYTARGET _TARGET) +function(OMNITRACE_CONFIGURE_PYTARGET _TARGET _VERSION) add_library(omnitrace::${_TARGET} ALIAS ${_TARGET}) target_link_libraries(${_TARGET} PRIVATE libpyomnitrace-interface) - set(_SUBDIR ${ARGN}) - if(_SUBDIR) - set(_SUBDIR "/${_SUBDIR}") - endif() - set_target_properties( ${_TARGET} PROPERTIES PREFIX "" - SUFFIX "${PYTHON_MODULE_EXTENSION}" - LIBRARY_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} - ARCHIVE_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} - RUNTIME_OUTPUT_DIRECTORY - ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} - PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} + OUTPUT_NAME libpyomnitrace + LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace + ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace + RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace + PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace INSTALL_RPATH_USE_LINK_PATH ON ${EXTRA_PROPERTIES}) - set(_PYLIB ${CMAKE_INSTALL_PYTHONDIR}/omnitrace${_SUBDIR}) + set(_PYLIB ${CMAKE_INSTALL_PYTHONDIR}/omnitrace) if(NOT IS_ABSOLUTE "${_PYLIB}") set(_PYLIB ${CMAKE_INSTALL_PREFIX}/${_PYLIB}) endif() @@ -57,12 +49,8 @@ function(OMNITRACE_CONFIGURE_PYTARGET _TARGET) install( TARGETS ${_TARGET} - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/omnitrace${_SUBDIR} + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/omnitrace OPTIONAL) - - if(NOT "${_TARGET}" STREQUAL "libpyomnitrace") - add_dependencies(libpyomnitrace ${_TARGET}) - endif() endfunction() # ######################################################################################## @@ -96,12 +84,160 @@ target_link_libraries( target_compile_definitions(libpyomnitrace-interface INTERFACE OMNITRACE_PYBIND11_SOURCE) -add_library(libpyomnitrace MODULE ${pysources} ${pyheaders}) -omnitrace_configure_pytarget(libpyomnitrace) +# ---------------------------------------------------------------------------- +# Console scripts +# +function(OMNITRACE_PYTHON_CONSOLE_SCRIPT SCRIPT_NAME SCRIPT_SUBMODULE) + set(options) + set(args VERSION ROOT_DIR) + set(kwargs) + cmake_parse_arguments(ARG "${options}" "${args}" "${kwargs}" ${ARGN}) -add_subdirectory(omnitrace) + if(ARG_VERSION AND ARG_ROOT_DIR) + set(Python3_ROOT_DIR "${ARG_ROOT_DIR}") + find_package(Python3 ${ARG_VERSION} EXACT QUIET MODULE COMPONENTS Interpreter) + set(PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") + configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in + ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION} @ONLY) -if(PYTHON_EXECUTABLE) + if(CMAKE_INSTALL_PYTHONDIR) + install( + PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION} + DESTINATION ${CMAKE_INSTALL_BINDIR} + OPTIONAL) + endif() + + if(OMNITRACE_BUILD_TESTING OR OMNITRACE_BUILD_PYTHON) + add_test( + NAME ${SCRIPT_NAME}-console-script-test-${ARG_VERSION} + COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME}-${ARG_VERSION} --help + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties( + ${SCRIPT_NAME}-console-script-test-${ARG_VERSION} + PROPERTIES LABELS "python;python-${ARG_VERSION};console-script") + add_test( + NAME ${SCRIPT_NAME}-generic-console-script-test-${ARG_VERSION} + COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} --help + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties( + ${SCRIPT_NAME}-generic-console-script-test-${ARG_VERSION} + PROPERTIES ENVIRONMENT "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" LABELS + "python;python-${ARG_VERSION};console-script") + endif() + else() + set(PYTHON_EXECUTABLE "python3") + + configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in + ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} @ONLY) + + if(CMAKE_INSTALL_PYTHONDIR) + install( + PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} + DESTINATION ${CMAKE_INSTALL_BINDIR} + OPTIONAL) + endif() + endif() +endfunction() + +include(PyBind11Tools) + +# OMNITRACE_PYTHON_ROOT_DIRS=/opt/conda/envs/py36;/opt/conda/envs/py37;/opt/conda/envs/py38;/opt/conda/envs/py39 +# OMNITRACE_PYTHON_VERSIONS=3.6;3.7;3.8;3.9 + +if(NOT OMNITRACE_PYTHON_VERSIONS AND OMNITRACE_PYTHON_VERSION) + set(OMNITRACE_PYTHON_VERSIONS "${OMNITRACE_PYTHON_VERSION}") + if(NOT OMNITRACE_PYTHON_ROOT_DIRS) + omnitrace_find_python(_PY VERSION ${OMNITRACE_PYTHON_VERSION}) + set(OMNITRACE_PYTHON_ROOT_DIRS + "${_PY_ROOT_DIR}" + CACHE INTERNAL "" FORCE) + endif() +elseif( + NOT OMNITRACE_PYTHON_VERSIONS + AND NOT OMNITRACE_PYTHON_VERSION + AND OMNITRACE_PYTHON_ROOT_DIRS) + set(_PY_VERSIONS) + foreach(_DIR ${OMNITRACE_PYTHON_ROOT_DIRS}) + omnitrace_find_python(_PY ROOT_DIR ${_DIR}) + if(NOT _PY_FOUND) + continue() + endif() + if(NOT "${_PY_VERSION}" IN_LIST _PY_VERSIONS) + list(APPEND _PY_VERSIONS "${_PY_VERSION}") + endif() + endforeach() + set(OMNITRACE_PYTHON_VERSIONS + "${_PY_VERSIONS}" + CACHE INTERNAL "" FORCE) +elseif( + NOT OMNITRACE_PYTHON_VERSIONS + AND NOT OMNITRACE_PYTHON_VERSION + AND NOT OMNITRACE_PYTHON_ROOT_DIRS) + omnitrace_find_python(_PY REQUIRED) + set(OMNITRACE_PYTHON_ROOT_DIRS + "${_PY_ROOT_DIR}" + CACHE INTERNAL "" FORCE) + set(OMNITRACE_PYTHON_VERSIONS + "${_PY_VERSION}" + CACHE INTERNAL "" FORCE) +endif() + +list(LENGTH OMNITRACE_PYTHON_VERSIONS _NUM_PYTHON_VERSIONS) +list(LENGTH OMNITRACE_PYTHON_ROOT_DIRS _NUM_PYTHON_ROOT_DIRS) + +if(NOT _NUM_PYTHON_VERSIONS EQUAL _NUM_PYTHON_ROOT_DIRS) + omnitrace_message( + WARNING + "Error! Number of python versions : ${_NUM_PYTHON_VERSIONS}. VERSIONS :: ${OMNITRACE_PYTHON_VERSIONS}" + ) + omnitrace_message( + WARNING + "Error! Number of python root directories : ${_NUM_PYTHON_ROOT_DIRS}. ROOT DIRS :: ${OMNITRACE_PYTHON_ROOT_DIRS}" + ) + omnitrace_message( + FATAL_ERROR + "Error! Number of python versions != number of python root directories") +endif() + +file(GLOB_RECURSE PYTHON_FILES ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/*.py) +foreach(_IN ${PYTHON_FILES}) + string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/omnitrace" + "${PROJECT_BINARY_DIR}/python/omnitrace" _OUT "${_IN}") + configure_file(${_IN} ${_OUT} @ONLY) + install( + FILES ${_OUT} + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/omnitrace + OPTIONAL) +endforeach() + +omnitrace_python_console_script("omnitrace-python" "omnitrace") + +execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_LIBDIR}/python + COMMAND ${CMAKE_COMMAND} -E create_symlink ../../python + ${CMAKE_INSTALL_LIBDIR}/python/site-packages + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + +set(_INDEX 0) +foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS}) + # add_library(libpyomnitrace MODULE ${pysources} ${pyheaders}) + list(GET OMNITRACE_PYTHON_ROOT_DIRS ${_INDEX} Python3_ROOT_DIR) + omnitrace_pybind11_add_module( + libpyomnitrace-${_VERSION} MODULE + PYTHON_VERSION ${_VERSION} + VISIBILITY "hidden" ${pysources} ${pyheaders}) + omnitrace_configure_pytarget(libpyomnitrace-${_VERSION} ${_VERSION}) + + if(OMNITRACE_USE_PYTHON) + omnitrace_python_console_script( + "omnitrace-python" "omnitrace" + VERSION ${_VERSION} + ROOT_DIR "${Python3_ROOT_DIR}") + endif() + math(EXPR _INDEX "${_INDEX} + 1") +endforeach() + +if(PYTHON_EXECUTABLE AND OFF) configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.py.in ${PROJECT_BINARY_DIR}/python/setup.py @ONLY) configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.cfg.in diff --git a/source/python/cmake/ConfigPython.cmake b/source/python/cmake/ConfigPython.cmake index 2ec89c895e..06bcc64e98 100644 --- a/source/python/cmake/ConfigPython.cmake +++ b/source/python/cmake/ConfigPython.cmake @@ -40,8 +40,9 @@ set_property(CACHE Python3_FIND_IMPLEMENTATIONS PROPERTY STRINGS # can edit them interactively. This disables support for multiple version/component # requirements. set(Python3_ARTIFACTS_INTERACTIVE - ON - CACHE BOOL "Create CMake cache entries so that users can edit them interactively") + OFF + CACHE BOOL "Create CMake cache entries so that users can edit them interactively" + FORCE) # if("${Python3_USE_STATIC_LIBS}" STREQUAL "ANY") set(Python3_USE_STATIC_LIBS "OFF" CACHE # STRING "If ON, only static libs; if OFF, only shared libs; if ANY, shared then static") @@ -60,158 +61,7 @@ foreach(_VAR FIND_STRATEGY FIND_VIRTUALENV FIND_FRAMEWORK FIND_IMPLEMENTATIONS endforeach() # display version -omnitrace_add_feature(OMNITRACE_PYTHON_VERSION "Python version for omnitrace" DOC) - -# search hint -if(PYTHON_ROOT_DIR AND NOT Python3_ROOT_DIR) - set(Python3_ROOT_DIR ${PYTHON_ROOT_DIR}) -endif() - -# legacy specification of interpreter -if(PYTHON_EXECUTABLE AND NOT Python3_EXECUTABLE) - set(Python3_EXECUTABLE - "${PYTHON_EXECUTABLE}" - CACHE FILEPATH "Path to Python3 interpreter") -endif() - -# default python types to search for -set(Python_ADDITIONAL_VERSIONS - "3.9;3.8;3.7;3.6" - CACHE STRING "Python versions supported by omnitrace") - -# override types to search for -if(OMNITRACE_PYTHON_VERSION) - set(Python_ADDITIONAL_VERSIONS - ${OMNITRACE_PYTHON_VERSION} - CACHE STRING "Python versions supported by omnitrace" FORCE) -elseif(PYBIND11_PYTHON_VERSION) - set(Python_ADDITIONAL_VERSIONS - ${PYBIND11_PYTHON_VERSION} - CACHE STRING "Python versions supported by omnitrace") -endif() - -# unset the version strings -if(_PYVERSION_LAST AND (OMNITRACE_PYTHON_VERSION VERSION_LESS _PYVERSION_LAST - OR OMNITRACE_PYTHON_VERSION VERSION_GREATER _PYVERSION_LAST)) - unset(OMNITRACE_PYTHON_VERSION CACHE) - unset(PYBIND11_PYTHON_VERSION CACHE) - unset(CMAKE_INSTALL_PYTHONDIR CACHE) -endif() - -# if OMNITRACE_PYTHON_VERSION specified, set to desired python version -set(_PYVERSION ${OMNITRACE_PYTHON_VERSION}) - -# if OMNITRACE_PYTHON_VERSION is not set but PYBIND11_PYTHON_VERSION is -if("${_PYVERSION}" STREQUAL "" AND PYBIND11_PYTHON_VERSION) - set(_PYVERSION ${PYBIND11_PYTHON_VERSION}) -endif() - -# basically just used to get Python3_SITEARCH for installation -find_package(Python3 ${_PYVERSION} MODULE ${OMNITRACE_FIND_REQUIREMENT} - COMPONENTS Interpreter Development) - -# executable -set(PYTHON_EXECUTABLE - "${Python3_EXECUTABLE}" - CACHE FILEPATH "Set via Python3_EXECUTABLE (omnitrace)" FORCE) -# includes -if(Python3_INCLUDE_DIR AND NOT Python3_INCLUDE_DIRS) - set(Python3_INCLUDE_DIRS ${Python3_INCLUDE_DIR}) -endif() -if(Python3_INCLUDE_DIRS) - set(PYTHON_INCLUDE_DIR - "${Python3_INCLUDE_DIRS}" - CACHE PATH "Set via Python3_INCLUDE_DIR (omnitrace)" FORCE) - set(PYTHON_INCLUDE_DIRS - "${Python3_INCLUDE_DIRS}" - CACHE PATH "Set via Python3_INCLUDE_DIRS (omnitrace)" FORCE) -endif() -# libraries -set(PYTHON_LIBRARY_DEBUG - "${Python3_LIBRARY_DEBUG}" - CACHE FILEPATH "Set via Python3_LIBRARY_DEBUG (omnitrace)" FORCE) -set(PYTHON_LIBRARY_RELEASE - "${Python3_LIBRARY_RELEASE}" - CACHE FILEPATH "Set via Python3_LIBRARY_DEBUG (omnitrace)" FORCE) -if(Python3_LIBRARY_RELEASE) - set(PYTHON_LIBRARY - "${Python3_LIBRARY_RELEASE}" - CACHE FILEPATH "Set via Python3_LIBRARY (omnitrace)" FORCE) - set(PYTHON_LIBRARIES - "${Python3_LIBRARY_RELEASE}" - CACHE FILEPATH "Set via Python3_LIBRARIES (omnitrace)" FORCE) -else(Python3_LIBRARY_DEBUG) - set(PYTHON_LIBRARY - "${Python3_LIBRARY_DEBUG}" - CACHE FILEPATH "Set via Python3_LIBRARY (omnitrace)" FORCE) - set(PYTHON_LIBRARIES - "${Python3_LIBRARY_DEBUG}" - CACHE FILEPATH "Set via Python3_LIBRARIES (omnitrace)" FORCE) -endif() -set(PYTHON_LIBRARY_DIRS - "${Python3_LIBRARY_DIRS}" - CACHE PATH "Set via Python3_LIBRARY_DIRS (omnitrace)" FORCE) -set(PYTHON_LINK_OPTIONS - "${Python3_LINK_OPTIONS}" - CACHE STRING "Set via Python3_LINK_OPTIONS (omnitrace)" FORCE) - -# module -set(PYTHON_MODULE_EXTENSION - "${Python3_MODULE_EXTENSION}" - CACHE STRING "Set via Python3_MODULE_EXTENSION (omnitrace)" FORCE) -set(PYTHON_MODULE_PREFIX - "${Python3_MODULE_PREFIX}" - CACHE STRING "Set via Python3_MODULE_PREFIX (omnitrace)" FORCE) - -# version -set(PYTHON_VERSION - "${Python3_VERSION}" - CACHE STRING "Set via Python3_VERSION (omnitrace)" FORCE) -set(PYTHON_VERSION_MAJOR - "${Python3_VERSION_MAJOR}" - CACHE STRING "Set via Python3_VERSION_MAJOR (omnitrace)" FORCE) -set(PYTHON_VERSION_MINOR - "${Python3_VERSION_MINOR}" - CACHE STRING "Set via Python3_VERSION_MINOR (omnitrace)" FORCE) - -# find_package -set(PythonInterp_FOUND ${Python3_Interpreter_FOUND}) -set(PythonLibs_FOUND ${Python3_Development_FOUND}) - -# set OMNITRACE_PYTHON_VERSION if we have the python version -if(PYTHON_VERSION_STRING) - set(OMNITRACE_PYTHON_VERSION - "${PYTHON_VERSION_STRING}" - CACHE STRING "Python version for omnitrace") -endif() - -# if either not found, disable -if(NOT Python3_FOUND) - set(OMNITRACE_USE_PYTHON OFF) - set(OMNITRACE_BUILD_PYTHON OFF) - omnitrace_inform_empty_interface(omnitrace-python "Python embedded interpreter") - omnitrace_inform_empty_interface(omnitrace-plotting "Python plotting from C++") -else() - set(OMNITRACE_PYTHON_VERSION - "${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}" - CACHE STRING "Python version for omnitrace") - omnitrace_add_feature(PYTHON_EXECUTABLE "Python executable") -endif() - -# C++ standard -if(NOT MSVC) - if(NOT "${PYBIND11_CPP_STANDARD}" STREQUAL "-std=c++${CMAKE_CXX_STANDARD}") - set(PYBIND11_CPP_STANDARD - -std=c++${CMAKE_CXX_STANDARD} - CACHE STRING "PyBind11 CXX standard" FORCE) - endif() -else() - if(NOT "${PYBIND11_CPP_STANDARD}" STREQUAL "/std:c++${CMAKE_CXX_STANDARD}") - set(PYBIND11_CPP_STANDARD - /std:c++${CMAKE_CXX_STANDARD} - CACHE STRING "PyBind11 CXX standard" FORCE) - endif() -endif() +omnitrace_add_feature(OMNITRACE_PYTHON_VERSIONS "Python version for omnitrace" DOC) option(PYBIND11_INSTALL "Enable Pybind11 installation" OFF) @@ -220,38 +70,17 @@ if(OMNITRACE_BUILD_PYTHON AND NOT TARGET pybind11) omnitrace_checkout_git_submodule( RECURSIVE RELATIVE_PATH external/pybind11 - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/external/timemory + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} REPO_URL https://github.com/jrmadsen/pybind11.git - REPO_BRANCH master) + REPO_BRANCH omnitrace) - # add PyBind11 to project omnitrace_save_variables(IPO VARIABLES - # CMAKE_INTERPROCEDURAL_OPTIMIZATION) - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) - add_subdirectory(${PROJECT_SOURCE_DIR}/external/timemory/external/pybind11) - # omnitrace_restore_variables(IPO VARIABLES CMAKE_INTERPROCEDURAL_OPTIMIZATION) -endif() - -if(NOT PYBIND11_PYTHON_VERSION) - unset(PYBIND11_PYTHON_VERSION CACHE) - execute_process( - COMMAND - ${PYTHON_EXECUTABLE} -c - "import sys; print('{}.{}'.format(sys.version_info[0], sys.version_info[1]))" - OUTPUT_VARIABLE PYTHON_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) - set(PYBIND11_PYTHON_VERSION - "${PYTHON_VERSION}" - CACHE STRING "Python version") -endif() - -omnitrace_add_feature(PYBIND11_CPP_STANDARD "PyBind11 C++ standard") -omnitrace_add_feature(PYBIND11_PYTHON_VERSION "PyBind11 Python version") - -if(NOT "${OMNITRACE_PYTHON_VERSION}" MATCHES "${PYBIND11_PYTHON_VERSION}*") - message(STATUS "OMNITRACE_PYTHON_VERSION is set to ${OMNITRACE_PYTHON_VERSION}") - message(STATUS "PYBIND11_PYTHON_VERSION is set to ${PYBIND11_PYTHON_VERSION}") - message( - FATAL_ERROR "Mismatched 'OMNITRACE_PYTHON_VERSION' and 'PYBIND11_PYTHON_VERSION'") + if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) + endif() + set(PYBIND11_NOPYTHON ON) + omnitrace_save_variables(IPO VARIABLES CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_subdirectory(${PROJECT_SOURCE_DIR}/external/pybind11) + omnitrace_restore_variables(IPO VARIABLES CMAKE_INTERPROCEDURAL_OPTIMIZATION) endif() execute_process( @@ -261,189 +90,3 @@ execute_process( OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) string(REPLACE " " " " OMNITRACE_INSTALL_DATE "${OMNITRACE_INSTALL_DATE}") - -if(SKBUILD) - set(OMNITRACE_INSTALL_PYTHON - "prefix" - CACHE STRING "SKBUILD forced python install type" FORCE) -else() - set(OMNITRACE_INSTALL_PYTHON - "lib" - CACHE STRING "Installation type for python (prefix, lib, or global)") -endif() - -omnitrace_add_feature(OMNITRACE_INSTALL_PYTHON - "Installation type for python (prefix, lib, or global)") - -if(SKBUILD OR "${OMNITRACE_INSTALL_PYTHON}" STREQUAL "prefix") - set(CMAKE_INSTALL_PYTHONDIR - ${CMAKE_INSTALL_PREFIX} - CACHE PATH "Installation directory for python") -elseif(SPACK_BUILD OR "${OMNITRACE_INSTALL_PYTHON}" STREQUAL "lib") - set(CMAKE_INSTALL_PYTHONDIR - lib/python${PYBIND11_PYTHON_VERSION}/site-packages - CACHE PATH "Installation directory for python") -else() - string(REPLACE "\\" "/" Python3_SITEARCH "${Python3_SITEARCH}") - set(CMAKE_INSTALL_PYTHONDIR ${Python3_SITEARCH}) - omnitrace_add_feature(Python3_SITEARCH - "site-packages directory of python installation") - set(_REMOVE OFF) - # make the directory if it doesn't exist - if(NOT EXISTS ${Python3_SITEARCH}/omnitrace) - set(_REMOVE ON) - execute_process( - COMMAND ${CMAKE_COMMAND} -E make_directory ${Python3_SITEARCH}/omnitrace - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - ERROR_QUIET) - endif() - # figure out if we can install to Python3_SITEARCH - execute_process( - COMMAND ${CMAKE_COMMAND} -E touch ${Python3_SITEARCH}/omnitrace/__init__.py - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - ERROR_VARIABLE ERR_MSG - RESULT_VARIABLE ERR_CODE) - # remove the directory if we created it - if(_REMOVE) - execute_process( - COMMAND ${CMAKE_COMMAND} -E remove_directory ${Python3_SITEARCH}/omnitrace - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - ERROR_QUIET) - endif() - # check the error code of the touch command - if(ERR_CODE) - if("${OMNITRACE_INSTALL_PYTHON}" STREQUAL "global") - message( - FATAL_ERROR - "omnitrace could not install python files to ${Python3_SITEARCH} (not writable):\n${ERR_MSG}" - ) - endif() - # get the python directory name, e.g. 'python3.6' from - # '/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6' - get_filename_component(PYDIR "${Python3_STDLIB}" NAME) - omnitrace_add_feature(Python3_STDLIB - "standard-library directory of python installation") - # Should not be CMAKE_INSTALL_LIBDIR! Python won't look in a lib64 folder - set(CMAKE_INSTALL_PYTHONDIR lib/${PYDIR}/site-packages) - endif() -endif() - -if(OMNITRACE_BUILD_PYTHON OR pybind11_FOUND) - set(_PYBIND11_INCLUDE_DIRS) - foreach(_TARG pybind11 pybind11::pybind11 pybind11::module) - if(TARGET ${_TARG}) - get_target_property(_INCLUDE_DIR ${_TARG} INTERFACE_INCLUDE_DIRECTORIES) - if(_INCLUDE_DIR) - list(APPEND _PYBIND11_INCLUDE_DIRS ${_INCLUDE_DIR}) - endif() - endif() - endforeach() - if(_PYBIND11_INCLUDE_DIRS) - list(REMOVE_DUPLICATES _PYBIND11_INCLUDE_DIRS) - endif() - omnitrace_target_compile_definitions(omnitrace-python INTERFACE OMNITRACE_USE_PYTHON) - if(NOT APPLE) - target_link_libraries(omnitrace-python INTERFACE ${PYTHON_LIBRARIES}) - endif() - if(PYTHON_INCLUDE_DIRS) - target_include_directories(omnitrace-python SYSTEM - INTERFACE ${PYTHON_INCLUDE_DIRS}) - endif() - if(PYBIND11_INCLUDE_DIRS) - target_include_directories(omnitrace-python SYSTEM - INTERFACE ${PYBIND11_INCLUDE_DIRS}) - endif() - if(PYBIND11_INCLUDE_DIR) - target_include_directories(omnitrace-python SYSTEM - INTERFACE $) - endif() - if(_PYBIND11_INCLUDE_DIRS) - target_include_directories(omnitrace-python SYSTEM - INTERFACE $) - endif() -endif() - -if(APPLE) - if(CMAKE_VERSION VERSION_LESS 3.18) - target_link_libraries(omnitrace-python INTERFACE "-undefined dynamic_lookup") - else() - target_link_libraries( - omnitrace-python - INTERFACE "$<$:-undefined dynamic_lookup>") - endif() -endif() - -if(WIN32) - # Windows produces: - # - # CMake Warning (dev) at tests/test-python-install-import.cmake:3 (SET): Syntax error - # in cmake code at - # C:/projects/omnitrace/build-omnitrace/tests/test-python-install-import.cmake:3 when - # parsing string C:\Python36-x64\Lib\site-packages Invalid escape sequence \P - string(REPLACE "\\" "/" INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR}") -else() - set(INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR}") -endif() - -configure_file( - ${PROJECT_SOURCE_DIR}/external/timemory/cmake/Templates/test-python-install-import.cmake.in - ${PROJECT_BINARY_DIR}/tests/test-python-install-import.cmake - @ONLY) -unset(INSTALL_PYTHONDIR) - -omnitrace_add_feature(CMAKE_INSTALL_PYTHONDIR - "Installation prefix of the python bindings") - -set(_PYVERSION_LAST - "${OMNITRACE_PYTHON_VERSION}" - CACHE INTERNAL "Last version" FORCE) - -find_package(PythonInterp ${OMNITRACE_PYTHON_VERSION} EXACT REQUIRED) -find_package(PythonLibs ${OMNITRACE_PYTHON_VERSION} EXACT REQUIRED) -# find_package(PythonExtensions REQUIRED) - -if("${PYTHON_MODULE_EXTENSION}" STREQUAL "") - execute_process( - COMMAND - "${Python3_EXECUTABLE}" "-c" " -from distutils import sysconfig as s;import sys;import struct; -print('.'.join(str(v) for v in sys.version_info)); -print(sys.prefix); -print(s.get_python_inc(plat_specific=True)); -print(s.get_python_lib(plat_specific=True)); -print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO')); -print(hasattr(sys, 'gettotalrefcount')+0); -print(struct.calcsize('@P')); -print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION')); -print(s.get_config_var('LIBDIR') or ''); -print(s.get_config_var('MULTIARCH') or ''); -" - RESULT_VARIABLE _PYTHON_SUCCESS - OUTPUT_VARIABLE _PYTHON_VALUES - ERROR_VARIABLE _PYTHON_ERROR_VALUE) - - if(_PYTHON_SUCCESS MATCHES 0) - # Convert the process output into a list - if(WIN32) - string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES}) - endif() - string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES}) - string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES}) - list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST) - list(GET _PYTHON_VALUES 1 PYTHON_PREFIX) - list(GET _PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) - list(GET _PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) - list(GET _PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) - list(GET _PYTHON_VALUES 5 PYTHON_IS_DEBUG) - list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) - list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) - list(GET _PYTHON_VALUES 8 PYTHON_LIBDIR) - list(GET _PYTHON_VALUES 9 PYTHON_MULTIARCH) - else() - message(WARNING "${_PYTHON_ERROR_VALUE}") - endif() - - if("${PYTHON_MODULE_EXTENSION}" STREQUAL "") - message(WARNING "Python module extension is empty!") - endif() -endif() diff --git a/source/python/cmake/Modules/FindPyBind11Python.cmake b/source/python/cmake/Modules/FindPyBind11Python.cmake new file mode 100644 index 0000000000..cee46fe16d --- /dev/null +++ b/source/python/cmake/Modules/FindPyBind11Python.cmake @@ -0,0 +1,183 @@ +# * Find python libraries This module finds the libraries corresponding to the Python +# interpreter FindPythonInterp provides. This code sets the following variables: +# +# PYTHONLIBS_FOUND - have the Python libs been found PYTHON_PREFIX - path to the +# Python installation PYTHON_LIBRARIES - path to the python library +# PYTHON_INCLUDE_DIRS - path to where Python.h is found PYTHON_MODULE_EXTENSION - +# lib extension, e.g. '.so' or '.pyd' PYTHON_MODULE_PREFIX - lib name prefix: usually an +# empty string PYTHON_SITE_PACKAGES - path to installation site-packages +# PYTHON_IS_DEBUG - whether the Python interpreter is a debug build +# +# Thanks to talljimbo for the patch adding the 'LDVERSION' config variable usage. + +# ============================================================================= +# Copyright 2001-2009 Kitware, Inc. Copyright 2012 Continuum Analytics, Inc. +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, this list of +# conditions and the following disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# * Neither the names of Kitware, Inc., the Insight Software Consortium, nor the names of +# their contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ============================================================================= + +# Checking for the extension makes sure that `LibsNew` was found and not just `Libs`. + +set(_find_quiet) +set(_find_required) +set(_find_exact) + +if(PyBind11Python_FIND_QUIETLY) + set(_find_quiet QUIET) +endif() + +if(PyBind11Python_FIND_REQUIRED) + set(_find_required REQUIRED) +endif() + +if(PyBind11Python_FIND_VERSION) + set(_find_exact EXACT) +endif() + +if(NOT DEFINED PyBind11Python_PYTHON) + set(PyBind11Python_PYTHON Python3) + if(PyBind11Python_FIND_VERSION AND "${PyBind11Python_FIND_VERSION}" VERSION_LESS 3.0) + set(PyBind11Python_PYTHON Python2) + endif() +endif() + +if(NOT PyBind11Python_COMPONENTS) + set(PyBind11Python_COMPONENTS Interpreter Development) +endif() + +# Use the Python interpreter to find the libs. +find_package(${PyBind11Python_PYTHON} ${PyBind11Python_FIND_VERSION} ${_find_exact} MODULE + ${_find_required} ${_find_quiet} COMPONENTS ${PyBind11Python_COMPONENTS}) + +# According to +# https://stackoverflow.com/questions/646518/python-how-to-detect-debug-interpreter +# testing whether sys has the gettotalrefcount function is a reliable, cross-platform way +# to detect a CPython debug interpreter. +# +# The library suffix is from the config var LDVERSION sometimes, otherwise VERSION. +# VERSION will typically be like "2.7" on unix, and "27" on windows. +execute_process( + COMMAND + "${${PyBind11Python_PYTHON}_EXECUTABLE}" "-c" + " +import sys;import struct; +import sysconfig as s +USE_SYSCONFIG = sys.version_info >= (3, 10) +if not USE_SYSCONFIG: + from distutils import sysconfig as ds +print('.'.join(str(v) for v in sys.version_info)); +print(sys.prefix); +if USE_SYSCONFIG: + scheme = s.get_default_scheme() + if scheme == 'posix_local': + # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/ + scheme = 'posix_prefix' + print(s.get_path('platinclude', scheme)) + print(s.get_path('platlib')) +else: + print(ds.get_python_inc(plat_specific=True)); + print(ds.get_python_lib(plat_specific=True)); +print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO')); +print(hasattr(sys, 'gettotalrefcount')+0); +print(struct.calcsize('@P')); +print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION')); +print(s.get_config_var('LIBDIR') or ''); +print(s.get_config_var('MULTIARCH') or ''); +" + RESULT_VARIABLE _PYTHON_SUCCESS + OUTPUT_VARIABLE _PYTHON_VALUES + ERROR_VARIABLE _PYTHON_ERROR_VALUE) + +if(NOT _PYTHON_SUCCESS MATCHES 0) + if(PyBind11Python_FIND_REQUIRED) + message(FATAL_ERROR "Python config failure:\n${_PYTHON_ERROR_VALUE}") + endif() + set(PyBind11Python_FOUND FALSE) + return() +endif() + +# Convert the process output into a list +if(WIN32) + string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES}) +endif() +string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES}) +string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES}) +list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST) +list(GET _PYTHON_VALUES 1 PYTHON_PREFIX) +list(GET _PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) +list(GET _PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) +list(GET _PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) +list(GET _PYTHON_VALUES 5 PYTHON_IS_DEBUG) +list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) +list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) +list(GET _PYTHON_VALUES 8 PYTHON_LIBDIR) +list(GET _PYTHON_VALUES 9 PYTHON_MULTIARCH) + +# Make sure the Python has the same pointer-size as the chosen compiler Skip if +# CMAKE_SIZEOF_VOID_P is not defined +if(CMAKE_SIZEOF_VOID_P AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL + "${CMAKE_SIZEOF_VOID_P}")) + if(PyBind11Python_FIND_REQUIRED) + math(EXPR _PYTHON_BITS "${PYTHON_SIZEOF_VOID_P} * 8") + math(EXPR _CMAKE_BITS "${CMAKE_SIZEOF_VOID_P} * 8") + message(FATAL_ERROR "Python config failure: Python is ${_PYTHON_BITS}-bit, " + "chosen compiler is ${_CMAKE_BITS}-bit") + endif() + set(PyBind11Python_FOUND FALSE) + return() +endif() + +# The built-in FindPython didn't always give the version numbers +string(REGEX REPLACE "\\." ";" _PYTHON_VERSION_LIST ${_PYTHON_VERSION_LIST}) +list(GET _PYTHON_VERSION_LIST 0 PYTHON_VERSION_MAJOR) +list(GET _PYTHON_VERSION_LIST 1 PYTHON_VERSION_MINOR) +list(GET _PYTHON_VERSION_LIST 2 PYTHON_VERSION_PATCH) +set(PYTHON_VERSION + "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}.${PYTHON_VERSION_PATCH}") + +# Make sure all directory separators are '/' +string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX "${PYTHON_PREFIX}") +string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}") +string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES "${PYTHON_SITE_PACKAGES}") + +# We use PYTHON_INCLUDE_DIR, PYTHON_LIBRARY and PYTHON_DEBUG_LIBRARY for the cache entries +# because they are meant to specify the location of a single library. We now set the +# variables listed by the documentation for this module. +set(PYTHON_EXECUTABLE "${${PyBind11Python_PYTHON}_EXECUTABLE}") +set(PYTHON_INCLUDE_DIRS "${${PyBind11Python_PYTHON}_INCLUDE_DIRS}") +set(PYTHON_LIBRARIES "${${PyBind11Python_PYTHON}_LIBRARIES}") +if(NOT PYTHON_DEBUG_LIBRARY) + set(PYTHON_DEBUG_LIBRARY "") +endif() +set(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}") + +# find_package_message(PyBind11Python "Found PyBind11Python: ${PYTHON_LIBRARIES}" +# "${PYTHON_EXECUTABLE}${PYTHON_VERSION_STRING}") + +if(NOT PYTHON_MODULE_PREFIX) + set(PYTHON_MODULE_PREFIX "") +endif() diff --git a/source/python/cmake/PyBind11Tools.cmake b/source/python/cmake/PyBind11Tools.cmake new file mode 100644 index 0000000000..b555e29ad4 --- /dev/null +++ b/source/python/cmake/PyBind11Tools.cmake @@ -0,0 +1,301 @@ +# +function(OMNITRACE_FIND_PYTHON _VAR) + set(options REQUIRED QUIET) + set(args VERSION ROOT_DIR) + set(kwargs COMPONENTS) + cmake_parse_arguments(ARG "${options}" "${args}" "${kwargs}" ${ARGN}) + + if(ARG_QUIET) + set(_QUIET "QUIET") + endif() + + if(ARG_VERSION) + set(_EXACT "EXACT") + endif() + + if(ARG_REQUIRED) + set(_FIND_REQUIREMENT "REQUIRED") + endif() + + if(NOT ARG_COMPONENTS) + set(ARG_COMPONENTS Interpreter Development) + endif() + + set(Python3_ROOT_DIR "${ARG_ROOT_DIR}") + set(Python3_FIND_STRATEGY "LOCATION") + set(Python3_FIND_VIRTUALENV "FIRST") + set(Python3_ARTIFACTS_INTERACTIVE OFF) + + find_package(Python3 ${ARG_VERSION} ${_EXACT} ${_QUIET} MODULE ${_FIND_REQUIREMENT} + COMPONENTS ${ARG_COMPONENTS}) + + set(${_VAR}_FOUND + "${Python3_FOUND}" + PARENT_SCOPE) + if(NOT Python3_FOUND) + set(${_VAR}_EXECUTABLE + "" + PARENT_SCOPE) + set(${_VAR}_ROOT_DIR + "" + PARENT_SCOPE) + set(${_VAR}_VERSION + "" + PARENT_SCOPE) + return() + else() + set(${_VAR}_EXECUTABLE + "${Python3_EXECUTABLE}" + PARENT_SCOPE) + execute_process( + COMMAND + "${Python3_EXECUTABLE}" "-c" + "import sys; print('.'.join(str(v) for v in [sys.version_info[0], sys.version_info[1]])); print(sys.prefix);" + RESULT_VARIABLE _PYTHON_SUCCESS + OUTPUT_VARIABLE _PYTHON_VALUES + ERROR_VARIABLE _PYTHON_ERROR_VALUE) + + if(_PYTHON_SUCCESS MATCHES 0) + # Convert the process output into a list + string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES}) + string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES}) + list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST) + list(GET _PYTHON_VALUES 1 _PYTHON_PREFIX) + set(${_VAR}_ROOT_DIR + "${_PYTHON_PREFIX}" + PARENT_SCOPE) + set(${_VAR}_VERSION + "${_PYTHON_VERSION_LIST}" + PARENT_SCOPE) + else() + omnitrace_message(WARNING "${_PYTHON_ERROR_VALUE}") + endif() + endif() +endfunction() +# +# Internal: find the appropriate link time optimization flags for this compiler +function(_OMNITRACE_PYBIND11_ADD_LTO_FLAGS target_name prefer_thin_lto) + # Checks whether the given CXX/linker flags can compile and link a cxx file. cxxflags + # and linkerflags are lists of flags to use. The result variable is a unique variable + # name for each set of flags: the compilation result will be cached base on the result + # variable. If the flags work, sets them in cxxflags_out/linkerflags_out internal + # cache variables (in addition to ${result}). + function(_PYBIND11_RETURN_IF_CXX_AND_LINKER_FLAGS_WORK result cxxflags linkerflags + cxxflags_out linkerflags_out) + include(CheckCXXCompilerFlag) + set(CMAKE_REQUIRED_LIBRARIES ${linkerflags}) + check_cxx_compiler_flag("${cxxflags}" ${result}) + if(${result}) + set(${cxxflags_out} + "${cxxflags}" + CACHE INTERNAL "" FORCE) + set(${linkerflags_out} + "${linkerflags}" + CACHE INTERNAL "" FORCE) + endif() + endfunction() + + if(NOT DEFINED PYBIND11_LTO_CXX_FLAGS) + set(PYBIND11_LTO_CXX_FLAGS + "" + CACHE INTERNAL "") + set(PYBIND11_LTO_LINKER_FLAGS + "" + CACHE INTERNAL "") + + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(cxx_append "") + set(linker_append "") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT APPLE) + # Clang Gold plugin does not support -Os; append -O3 to MinSizeRel builds + # to override it + set(linker_append ";$<$:-O3>") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + set(cxx_append ";-fno-fat-lto-objects") + endif() + + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND prefer_thin_lto) + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_FLTO_THIN "-flto=thin${cxx_append}" "-flto=thin${linker_append}" + PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() + + if(NOT HAS_FLTO_THIN) + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_FLTO "-flto${cxx_append}" "-flto${linker_append}" + PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") + # Intel equivalent to LTO is called IPO + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS + PYBIND11_LTO_LINKER_FLAGS) + elseif(MSVC) + # cmake only interprets libraries as linker flags when they start with a - + # (otherwise it converts /LTCG to \LTCG as if it was a Windows path). Luckily + # MSVC supports passing flags with - instead of /, even if it is a bit + # non-standard: + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_MSVC_GL_LTCG "/GL" "-LTCG" PYBIND11_LTO_CXX_FLAGS + PYBIND11_LTO_LINKER_FLAGS) + endif() + + if(PYBIND11_LTO_CXX_FLAGS) + omnitrace_message(STATUS "${target_name} :: LTO enabled") + else() + omnitrace_message( + STATUS + "${target_name} :: LTO disabled (not supported by the compiler and/or linker)" + ) + endif() + endif() + + # Enable LTO flags if found, except for Debug builds + if(PYBIND11_LTO_CXX_FLAGS) + target_compile_options( + ${target_name} PRIVATE "$<$>:${PYBIND11_LTO_CXX_FLAGS}>") + endif() + if(PYBIND11_LTO_LINKER_FLAGS) + target_link_libraries( + ${target_name} + PRIVATE "$<$>:${PYBIND11_LTO_LINKER_FLAGS}>") + endif() +endfunction() +# +function(OMNITRACE_PYBIND11_ADD_MODULE target_name) + set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS SYSTEM THIN_LTO LTO) + set(args PYTHON_VERSION VISIBILITY CXX_STANDARD) + set(kwargs) + cmake_parse_arguments(ARG "${options}" "${args}" "${kwargs}" ${ARGN}) + + if(ARG_MODULE AND ARG_SHARED) + omnitrace_message(FATAL_ERROR "Can't be both MODULE and SHARED") + elseif(ARG_SHARED) + set(lib_type SHARED) + else() + set(lib_type MODULE) + endif() + if(NOT ARG_VISIBILITY) + set(ARG_VISIBILITY "hidden") + endif() + if(ARG_PYTHON_VERSION) + set(PythonLibsNew_FIND_REQUIRED ON) + endif() + if(NOT ARG_CXX_STANDARD AND CMAKE_CXX_STANDARD) + set(ARG_CXX_STANDARD ${CMAKE_CXX_STANDARD}) + elseif(NOT ARG_CXX_STANDARD) + set(ARG_CXX_STANDARD 11) + endif() + if(ARG_EXCLUDE_FROM_ALL) + set(exclude_from_all EXCLUDE_FROM_ALL) + endif() + + add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS}) + + target_link_libraries(${target_name} PRIVATE pybind11::module) + + if(ARG_SYSTEM) + set(inc_isystem SYSTEM) + endif() + + list(INSERT CMAKE_MODULE_PATH 0 "${PROJECT_SOURCE_DIR}/source/python/cmake/Modules") + find_package(PyBind11Python ${ARG_PYTHON_VERSION}) + + target_include_directories( + ${target_name} ${inc_isystem} + PRIVATE ${PYBIND11_INCLUDE_DIR} # from project CMakeLists.txt + PRIVATE ${pybind11_INCLUDE_DIR} # from pybind11Config + PRIVATE ${Python3_INCLUDE_DIRS}) + + # Python debug libraries expose slightly different objects + # https://docs.python.org/3.6/c-api/intro.html#debugging-builds + # https://stackoverflow.com/questions/39161202/how-to-work-around-missing-pymodule-create2-in-amd64-win-python35-d-lib + if(PYTHON_IS_DEBUG) + target_compile_definitions(${target_name} PRIVATE Py_DEBUG) + endif() + + # The prefix and extension are provided by FindPythonLibsNew.cmake + set_target_properties(${target_name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}") + set_target_properties(${target_name} PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}") + + # -fvisibility=hidden is required to allow multiple modules compiled against different + # pybind versions to work properly, and for some features (e.g. py::module_local). We + # force it on everything inside the `pybind11` namespace; also turning it on for a + # pybind module compilation here avoids potential warnings or issues from having mixed + # hidden/non-hidden types. + set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET + "${ARG_VISIBILITY}") + set_target_properties(${target_name} PROPERTIES CUDA_VISIBILITY_PRESET + "${ARG_VISIBILITY}") + + if(WIN32 OR CYGWIN) + # Link against the Python shared library on Windows + target_link_libraries(${target_name} PRIVATE ${Python3_LIBRARIES}) + elseif(APPLE) + # It's quite common to have multiple copies of the same Python version installed + # on one's system. E.g.: one copy from the OS and another copy that's statically + # linked into an application like Blender or Maya. If we link our plugin library + # against the OS Python here and import it into Blender or Maya later on, this + # will cause segfaults when multiple conflicting Python instances are active at + # the same time (even when they are of the same version). + + # Windows is not affected by this issue since it handles DLL imports differently. + # The solution for Linux and Mac OS is simple: we just don't link against the + # Python library. The resulting shared library will have missing symbols, but + # that's perfectly fine -- they will be resolved at import time. + + target_link_libraries(${target_name} PRIVATE "-undefined dynamic_lookup") + + if(ARG_SHARED) + # Suppress CMake >= 3.0 warning for shared libraries + set_target_properties(${target_name} PROPERTIES MACOSX_RPATH ON) + endif() + endif() + + # Make sure C++11/14 are enabled + set_target_properties(${target_name} PROPERTIES CXX_STANDARD ${ARG_CXX_STANDARD} + CXX_STANDARD_REQUIRED ON) + + if(ARG_NO_EXTRAS) + return() + endif() + + if(ARG_LTO OR ARG_THIN_LTO) + _omnitrace_pybind11_add_lto_flags(${target_name} ${ARG_THIN_LTO}) + endif() + + if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Strip unnecessary sections of the binary on Linux/Mac OS + if(CMAKE_STRIP) + if(APPLE) + add_custom_command( + TARGET ${target_name} + POST_BUILD + COMMAND ${CMAKE_STRIP} -x $) + else() + add_custom_command( + TARGET ${target_name} + POST_BUILD + COMMAND ${CMAKE_STRIP} $) + endif() + endif() + endif() + + if(MSVC) + # /MP enables multithreaded builds (relevant when there are many files), /bigobj + # is needed for bigger binding projects due to the limit to 64k addressable + # sections + target_compile_options(${target_name} PRIVATE /bigobj) + if(CMAKE_VERSION VERSION_LESS 3.11) + target_compile_options(${target_name} PRIVATE $<$>:/MP>) + else() + # Only set these options for C++ files. This is important so that, for + # instance, projects that include other types of source files like CUDA .cu + # files don't get these options propagated to nvcc since that would cause the + # build to fail. + target_compile_options( + ${target_name} + PRIVATE $<$>:$<$:/MP>>) + endif() + endif() +endfunction() diff --git a/source/python/omnitrace/CMakeLists.txt b/source/python/omnitrace/CMakeLists.txt deleted file mode 100644 index d322f3b0da..0000000000 --- a/source/python/omnitrace/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -# ######################################################################################## -# -# omnitrace (Python) -# -# ######################################################################################## - -file(GLOB_RECURSE PYTHON_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.py) -foreach(_IN ${PYTHON_FILES}) - string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/python/omnitrace" - _OUT "${_IN}") - configure_file(${_IN} ${_OUT} @ONLY) - install( - FILES ${_OUT} - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/omnitrace - OPTIONAL) -endforeach() - -# ---------------------------------------------------------------------------- -# Console scripts -# -function(OMNITRACE_PYTHON_CONSOLE_SCRIPT SCRIPT_NAME SCRIPT_SUBMODULE) - - configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in - ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} @ONLY) - - if(CMAKE_INSTALL_PYTHONDIR) - install( - PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} - DESTINATION ${CMAKE_INSTALL_BINDIR} - OPTIONAL) - endif() - - if(OMNITRACE_BUILD_TESTING OR OMNITRACE_BUILD_PYTHON) - add_test( - NAME ${SCRIPT_NAME}-console-script-test - COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} --help - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - set_tests_properties( - ${SCRIPT_NAME}-console-script-test - PROPERTIES ENVIRONMENT - "PYTHONPATH=${PROJECT_BINARY_DIR}/python:$ENV{PYTHONPATH}") - endif() -endfunction() - -if(NOT PYTHON_EXECUTABLE) - set(PYTHON_EXECUTABLE "python${OMNITRACE_PYTHON_VERSION}") -endif() - -if(OMNITRACE_USE_PYTHON) - omnitrace_python_console_script("omnitrace-python" "omnitrace") -endif() diff --git a/source/python/omnitrace/__init__.py b/source/python/omnitrace/__init__.py index adb39ed087..e1684702c5 100644 --- a/source/python/omnitrace/__init__.py +++ b/source/python/omnitrace/__init__.py @@ -1,4 +1,4 @@ -#!@PYTHON_EXECUTABLE@ +#!/usr/bin/env python@_VERSION@ # MIT License # # Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. diff --git a/source/python/omnitrace/__main__.py b/source/python/omnitrace/__main__.py index 77fe00a2ec..272d635772 100644 --- a/source/python/omnitrace/__main__.py +++ b/source/python/omnitrace/__main__.py @@ -1,4 +1,4 @@ -#!@PYTHON_EXECUTABLE@ +#!/usr/bin/env python@_VERSION@ # MIT License # # Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. diff --git a/source/python/omnitrace/profiler.py b/source/python/omnitrace/profiler.py index f1ecd5f787..3357334c48 100644 --- a/source/python/omnitrace/profiler.py +++ b/source/python/omnitrace/profiler.py @@ -1,4 +1,4 @@ -#!@PYTHON_EXECUTABLE@ +#!/usr/bin/env python@_VERSION@ # MIT License # # Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 88480975f4..c2094bbe75 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -76,10 +76,11 @@ set(_python_environment "OMNITRACE_USE_TIMEMORY=ON" "OMNITRACE_USE_SAMPLING=OFF" "OMNITRACE_TIME_OUTPUT=OFF" + "OMNITRACE_TREE_OUTPUT=OFF" "OMNITRACE_USE_PID=OFF" "OMNITRACE_TIMEMORY_COMPONENTS=trip_count" "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" - "PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}") + "PYTHONPATH=${CMAKE_BINARY_DIR}/python") # -------------------------------------------------------------------------------------- # @@ -242,7 +243,7 @@ function(OMNITRACE_ADD_PYTHON_TEST) cmake_parse_arguments( TEST "STANDALONE" # options - "NAME;FILE;TIMEOUT" # single value args + "NAME;FILE;TIMEOUT;PYTHON_EXECUTABLE;PYTHON_VERSION" # single value args "PROFILE_ARGS;RUN_ARGS;ENVIRONMENT;LABELS;PROPERTIES;PASS_REGULAR_EXPRESSION;FAIL_REGULAR_EXPRESSION;SKIP_REGULAR_EXPRESSION;DEPENDS;COMMAND" # multiple # value args ${ARGN}) @@ -251,64 +252,70 @@ function(OMNITRACE_ADD_PYTHON_TEST) set(TEST_TIMEOUT 120) endif() + set(PYTHON_EXECUTABLE "${TEST_PYTHON_EXECUTABLE}") + if(NOT DEFINED TEST_ENVIRONMENT OR "${TEST_ENVIRONMENT}" STREQUAL "") set(TEST_ENVIRONMENT "${_python_environment}") endif() - list(APPEND TEST_LABELS "python") - list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON" - "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output" - "OMNITRACE_OUTPUT_PREFIX=${TEST_NAME}/") + list(APPEND TEST_LABELS "python" "python-${TEST_PYTHON_VERSION}") if(NOT TEST_COMMAND) + list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON" + "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output" + "OMNITRACE_OUTPUT_PREFIX=${TEST_NAME}/${TEST_PYTHON_VERSION}/") get_filename_component(_TEST_FILE "${TEST_FILE}" NAME) - configure_file(${TEST_FILE} ${CMAKE_BINARY_DIR}/${_TEST_FILE} @ONLY) + set(_TEST_FILE + ${CMAKE_BINARY_DIR}/python/tests/${TEST_PYTHON_VERSION}/${_TEST_FILE}) + configure_file(${TEST_FILE} ${_TEST_FILE} @ONLY) if(TEST_STANDALONE) add_test( - NAME ${TEST_NAME} - COMMAND ${PYTHON_EXECUTABLE} ${_TEST_FILE} ${TEST_RUN_ARGS} + NAME ${TEST_NAME}-${TEST_PYTHON_VERSION} + COMMAND ${TEST_PYTHON_EXECUTABLE} ${_TEST_FILE} ${TEST_RUN_ARGS} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) else() add_test( - NAME ${TEST_NAME} - COMMAND ${PYTHON_EXECUTABLE} -m omnitrace ${TEST_PROFILE_ARGS} -- + NAME ${TEST_NAME}-${TEST_PYTHON_VERSION} + COMMAND ${TEST_PYTHON_EXECUTABLE} -m omnitrace ${TEST_PROFILE_ARGS} -- ${_TEST_FILE} ${TEST_RUN_ARGS} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) endif() else() + list(APPEND TEST_LABELS "python-check" "python-${TEST_PYTHON_VERSION}-check") add_test( - NAME ${TEST_NAME} + NAME ${TEST_NAME}-${TEST_PYTHON_VERSION} COMMAND ${TEST_COMMAND} ${TEST_FILE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test( - NAME ${TEST_NAME}-inverse + NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse COMMAND ${TEST_COMMAND} ${TEST_FILE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) endif() - set_tests_properties( - ${TEST_NAME} - PROPERTIES ENVIRONMENT - "${TEST_ENVIRONMENT}" - TIMEOUT - ${TEST_TIMEOUT} - LABELS - "${TEST_LABELS}" - DEPENDS - "${TEST_DEPENDS}" - PASS_REGULAR_EXPRESSION - "${TEST_PASS_REGULAR_EXPRESSION}" - FAIL_REGULAR_EXPRESSION - "${TEST_FAIL_REGULAR_EXPRESSION}" - SKIP_REGULAR_EXPRESSION - "${TEST_SKIP_REGULAR_EXPRESSION}" - REQUIRED_FILES - "${TEST_FILE}" - ${TEST_PROPERTIES}) + foreach(_TEST ${TEST_NAME}-${TEST_PYTHON_VERSION} + ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse) + + if(NOT TEST "${_TEST}") + continue() + endif() + + set(_TEST_PROPERTIES "${TEST_PROPERTIES}") + if(NOT "${_TEST}" MATCHES "inverse") + # assign pass variable to pass regex + set(_PASS_REGEX TEST_PASS_REGULAR_EXPRESSION) + # assign fail variable to fail regex + set(_FAIL_REGEX TEST_FAIL_REGULAR_EXPRESSION) + else() + # assign pass variable to fail regex + set(_PASS_REGEX TEST_FAIL_REGULAR_EXPRESSION) + # assign fail variable to pass regex + set(_FAIL_REGEX TEST_PASS_REGULAR_EXPRESSION) + # set to will fail + list(APPEND _TEST_PROPERTIES WILL_FAIL ON) + endif() - if(TEST ${TEST_NAME}-inverse) set_tests_properties( - ${TEST_NAME}-inverse + ${_TEST} PROPERTIES ENVIRONMENT "${TEST_ENVIRONMENT}" TIMEOUT @@ -318,15 +325,15 @@ function(OMNITRACE_ADD_PYTHON_TEST) DEPENDS "${TEST_DEPENDS}" PASS_REGULAR_EXPRESSION - "${TEST_FAIL_REGULAR_EXPRESSION}" + "${${_PASS_REGEX}}" FAIL_REGULAR_EXPRESSION - "${TEST_PASS_REGULAR_EXPRESSION}" - WILL_FAIL - ON + "${${_FAIL_REGEX}}" + SKIP_REGULAR_EXPRESSION + "${TEST_SKIP_REGULAR_EXPRESSION}" REQUIRED_FILES "${TEST_FILE}" - ${TEST_PROPERTIES}) - endif() + ${_TEST_PROPERTIES}) + endforeach() endfunction() # -------------------------------------------------------------------------------------- # @@ -495,89 +502,113 @@ omnitrace_add_test( # # -------------------------------------------------------------------------------------- # -omnitrace_add_python_test( - NAME python-external - FILE ${CMAKE_SOURCE_DIR}/examples/python/external.py - PROFILE_ARGS -f - RUN_ARGS 10 - ENVIRONMENT "${_python_environment}") +set(_INDEX 0) +foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS}) + list(GET OMNITRACE_PYTHON_ROOT_DIRS ${_INDEX} _PYTHON_ROOT_DIR) -omnitrace_add_python_test( - NAME python-external-exclude-inefficient - FILE ${CMAKE_SOURCE_DIR}/examples/python/external.py - PROFILE_ARGS -E "^inefficient$" - RUN_ARGS 10 - ENVIRONMENT "${_python_environment}") + omnitrace_find_python( + _PYTHON + ROOT_DIR "${_PYTHON_ROOT_DIR}" + COMPONENTS Interpreter) -omnitrace_add_python_test( - NAME python-builtin - FILE ${CMAKE_SOURCE_DIR}/examples/python/builtin.py - PROFILE_ARGS -b -l -f - RUN_ARGS 10 - ENVIRONMENT "${_python_environment}") + # ---------------------------------------------------------------------------------- # + # python tests + # ---------------------------------------------------------------------------------- # + omnitrace_add_python_test( + NAME python-external + PYTHON_EXECUTABLE ${_PYTHON_EXECUTABLE} + PYTHON_VERSION ${_VERSION} + FILE ${CMAKE_SOURCE_DIR}/examples/python/external.py + PROFILE_ARGS -f + RUN_ARGS -v 10 -n 5 + ENVIRONMENT "${_python_environment}") -omnitrace_add_python_test( - STANDALONE - NAME python-source - FILE ${CMAKE_SOURCE_DIR}/examples/python/source.py - RUN_ARGS 5 - ENVIRONMENT "${_python_environment}") + omnitrace_add_python_test( + NAME python-external-exclude-inefficient + PYTHON_EXECUTABLE ${_PYTHON_EXECUTABLE} + PYTHON_VERSION ${_VERSION} + FILE ${CMAKE_SOURCE_DIR}/examples/python/external.py + PROFILE_ARGS -E "^inefficient$" + RUN_ARGS -v 10 -n 5 + ENVIRONMENT "${_python_environment}") -# -------------------------------------------------------------------------------------- # -# -# python output tests -# -# -------------------------------------------------------------------------------------- # + omnitrace_add_python_test( + NAME python-builtin + PYTHON_EXECUTABLE ${_PYTHON_EXECUTABLE} + PYTHON_VERSION ${_VERSION} + FILE ${CMAKE_SOURCE_DIR}/examples/python/builtin.py + PROFILE_ARGS -b -l -f + RUN_ARGS -v 10 -n 5 + ENVIRONMENT "${_python_environment}") -if(CMAKE_VERSION VERSION_LESS "3.18.0") - find_program( - OMNITRACE_CAT_EXE - NAMES cat - PATH_SUFFIXES bin) - if(OMNITRACE_CAT_EXE) - set(OMNITRACE_CAT_COMMAND ${OMNITRACE_CAT_EXE}) + omnitrace_add_python_test( + STANDALONE + NAME python-source + PYTHON_EXECUTABLE ${_PYTHON_EXECUTABLE} + PYTHON_VERSION ${_VERSION} + FILE ${CMAKE_SOURCE_DIR}/examples/python/source.py + RUN_ARGS -v 5 -n 5 + ENVIRONMENT "${_python_environment}") + + # ---------------------------------------------------------------------------------- # + # python output tests + # ---------------------------------------------------------------------------------- # + + if(CMAKE_VERSION VERSION_LESS "3.18.0") + find_program( + OMNITRACE_CAT_EXE + NAMES cat + PATH_SUFFIXES bin) + if(OMNITRACE_CAT_EXE) + set(OMNITRACE_CAT_COMMAND ${OMNITRACE_CAT_EXE}) + endif() + else() + set(OMNITRACE_CAT_COMMAND ${CMAKE_COMMAND} -E cat) endif() -else() - set(OMNITRACE_CAT_COMMAND ${CMAKE_COMMAND} -E cat) -endif() -if(OMNITRACE_CAT_COMMAND) - omnitrace_add_python_test( - NAME python-external-check - COMMAND ${OMNITRACE_CAT_COMMAND} - FILE omnitrace-tests-output/python-external/trip_count.txt - PASS_REGULAR_EXPRESSION - "(\\\[compile\\\]).*(\\\| \\\|0>>> \\\[run\\\]\\\[external.py\\\]).*(\\\| \\\|0>>> \\\|_\\\[fib\\\]\\\[external.py\\\]).*(\\\| \\\|0>>> \\\|_\\\[inefficient\\\]\\\[external.py\\\])" - DEPENDS python-external - ENVIRONMENT "${_python_environment}") + if(OMNITRACE_CAT_COMMAND) + omnitrace_add_python_test( + NAME python-external-check + COMMAND ${OMNITRACE_CAT_COMMAND} + PYTHON_VERSION ${_VERSION} + FILE omnitrace-tests-output/python-external/${_VERSION}/trip_count.txt + PASS_REGULAR_EXPRESSION + "(\\\[compile\\\]).*(\\\| \\\|0>>> \\\[run\\\]\\\[external.py\\\]).*(\\\| \\\|0>>> \\\|_\\\[fib\\\]\\\[external.py\\\]).*(\\\| \\\|0>>> \\\|_\\\[inefficient\\\]\\\[external.py\\\])" + DEPENDS python-external-${_VERSION} + ENVIRONMENT "${_python_environment}") - omnitrace_add_python_test( - NAME python-external-exclude-inefficient-check - COMMAND ${OMNITRACE_CAT_COMMAND} - FILE omnitrace-tests-output/python-external-exclude-inefficient/trip_count.txt - FAIL_REGULAR_EXPRESSION "(\\\|_inefficient).*(\\\|_sum)" - DEPENDS python-external-exclude-inefficient - ENVIRONMENT "${_python_environment}") + omnitrace_add_python_test( + NAME python-external-exclude-inefficient-check + COMMAND ${OMNITRACE_CAT_COMMAND} + PYTHON_VERSION ${_VERSION} + FILE omnitrace-tests-output/python-external-exclude-inefficient/${_VERSION}/trip_count.txt + FAIL_REGULAR_EXPRESSION "(\\\|_inefficient).*(\\\|_sum)" + DEPENDS python-external-exclude-inefficient-${_VERSION} + ENVIRONMENT "${_python_environment}") - omnitrace_add_python_test( - NAME python-builtin-check - COMMAND ${OMNITRACE_CAT_COMMAND} - FILE omnitrace-tests-output/python-builtin/trip_count.txt - PASS_REGULAR_EXPRESSION "\\\[inefficient\\\]\\\[builtin.py:11\\\]" - DEPENDS python-builtin - ENVIRONMENT "${_python_environment}") + omnitrace_add_python_test( + NAME python-builtin-check + COMMAND ${OMNITRACE_CAT_COMMAND} + PYTHON_VERSION ${_VERSION} + FILE omnitrace-tests-output/python-builtin/${_VERSION}/trip_count.txt + PASS_REGULAR_EXPRESSION "\\\[inefficient\\\]\\\[builtin.py:14\\\]" + DEPENDS python-builtin-${_VERSION} + ENVIRONMENT "${_python_environment}") - omnitrace_add_python_test( - NAME python-source-check - COMMAND ${OMNITRACE_CAT_COMMAND} - FILE omnitrace-tests-output/python-source/trip_count.txt - PASS_REGULAR_EXPRESSION - "(\\\| \\\|0>>> run \\\| 5).*(\\\| \\\|0>>> \\\|_fib \\\| 40).*(\\\| \\\|0>>> \\\|_fib \\\| 5).*(\\\| \\\|0>>> \\\|_inefficient \\\| 5).*(\\\| \\\|0>>> \\\|__sum \\\| 5)" - DEPENDS python-source - ENVIRONMENT "${_python_environment}") -else() - omnitrace_message( - WARNING - "Neither 'cat' nor 'cmake -E cat' are available. Python source checks are disabled" - ) -endif() + omnitrace_add_python_test( + NAME python-source-check + COMMAND ${OMNITRACE_CAT_COMMAND} + PYTHON_VERSION ${_VERSION} + FILE omnitrace-tests-output/python-source/${_VERSION}/trip_count.txt + PASS_REGULAR_EXPRESSION + "(\\\| \\\|0>>> run \\\| 5).*(\\\| \\\|0>>> \\\|_fib \\\| 40).*(\\\| \\\|0>>> \\\|_fib \\\| 5).*(\\\| \\\|0>>> \\\|_inefficient \\\| 5).*(\\\| \\\|0>>> \\\|__sum \\\| 5)" + DEPENDS python-source-${_VERSION} + ENVIRONMENT "${_python_environment}") + else() + omnitrace_message( + WARNING + "Neither 'cat' nor 'cmake -E cat' are available. Python source checks are disabled" + ) + endif() + math(EXPR _INDEX "${_INDEX} + 1") +endforeach()