From afa3edebab18a160bb820945a816624339c38e7b Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 5 Apr 2022 00:24:34 -0500 Subject: [PATCH] Python support (#37) * Initial python support * Add python testing * Increase timeout for bin tests * cmake-format * Valid build types + testing + formatting + more - Enforce valid build types - Fix to numpy install - Increase testing timeout - Fix to cmake format glob - Fix to backtrace verbose * Disable stripping libraries by default * omnitrace exe updates - new '--print-instructions' option - changed format of instructions in JSON - remove no-save-fpr tests * Default to strip libraries when release build --- .cmake-format.yaml | 12 + .github/workflows/ubuntu-bionic.yml | 4 +- .../ubuntu-focal-dyninst-package.yml | 127 ----- .../workflows/ubuntu-focal-external-rocm.yml | 6 +- .github/workflows/ubuntu-focal-external.yml | 10 +- .github/workflows/ubuntu-focal.yml | 4 +- CMakeLists.txt | 29 +- README.md | 3 +- cmake/BuildSettings.cmake | 20 + cmake/ConfigCPack.cmake | 6 + cmake/Formatting.cmake | 11 +- cmake/MacroUtilities.cmake | 30 ++ cmake/Packages.cmake | 68 +++ cmake/Templates/console-script.in | 17 + cmake/Templates/setup-env.sh.in | 2 + examples/python/builtin.py | 36 ++ examples/python/external.py | 35 ++ examples/python/source.py | 38 ++ source/CMakeLists.txt | 4 + source/bin/omnitrace/fwd.hpp | 1 + source/bin/omnitrace/module_function.cpp | 6 +- source/bin/omnitrace/module_function.hpp | 34 +- source/bin/omnitrace/omnitrace.cpp | 7 + source/bin/tests/CMakeLists.txt | 24 +- source/lib/omnitrace-dl/CMakeLists.txt | 20 +- source/lib/omnitrace-user/CMakeLists.txt | 2 + source/lib/omnitrace/CMakeLists.txt | 29 +- source/lib/omnitrace/src/library.cpp | 8 +- .../src/library/components/backtrace.cpp | 3 +- source/lib/omnitrace/src/library/config.cpp | 4 +- source/python/CMakeLists.txt | 129 +++++ source/python/cmake/ConfigPython.cmake | 449 ++++++++++++++++++ source/python/libpyomnitrace.cpp | 447 +++++++++++++++++ source/python/libpyomnitrace.hpp | 56 +++ source/python/omnitrace/CMakeLists.txt | 51 ++ source/python/omnitrace/__init__.py | 67 +++ source/python/omnitrace/__main__.py | 362 ++++++++++++++ source/python/omnitrace/profiler.py | 328 +++++++++++++ source/python/pyproject.toml | 12 + source/python/setup.cfg.in | 36 ++ source/python/setup.py.in | 11 + tests/CMakeLists.txt | 133 ++++-- 42 files changed, 2442 insertions(+), 239 deletions(-) delete mode 100644 .github/workflows/ubuntu-focal-dyninst-package.yml create mode 100755 cmake/Templates/console-script.in create mode 100755 examples/python/builtin.py create mode 100755 examples/python/external.py create mode 100755 examples/python/source.py create mode 100644 source/python/CMakeLists.txt create mode 100644 source/python/cmake/ConfigPython.cmake create mode 100644 source/python/libpyomnitrace.cpp create mode 100644 source/python/libpyomnitrace.hpp create mode 100644 source/python/omnitrace/CMakeLists.txt create mode 100644 source/python/omnitrace/__init__.py create mode 100644 source/python/omnitrace/__main__.py create mode 100644 source/python/omnitrace/profiler.py create mode 100644 source/python/pyproject.toml create mode 100644 source/python/setup.cfg.in create mode 100644 source/python/setup.py.in diff --git a/.cmake-format.yaml b/.cmake-format.yaml index 6ff3370974..d319f4c3de 100644 --- a/.cmake-format.yaml +++ b/.cmake-format.yaml @@ -54,6 +54,18 @@ parse: PASS_REGULAR_EXPRESSION: '*' FAIL_REGULAR_EXPRESSION: '*' SKIP_REGULAR_EXPRESSION: '*' + omnitrace_add_python_test: + flags: + - STANDALONE + kwargs: + NAME: '*' + FILE: '*' + TIMEOUT: '*' + PROFILE_ARGS: '*' + RUN_ARGS: '*' + ENVIRONMENT: '*' + LABELS: '*' + PROPERTIES: '*' rocm_version_message: flags: - STATUS diff --git a/.github/workflows/ubuntu-bionic.yml b/.github/workflows/ubuntu-bionic.yml index 8acacbdf05..f2e26e9f9d 100644 --- a/.github/workflows/ubuntu-bionic.yml +++ b/.github/workflows/ubuntu-bionic.yml @@ -46,6 +46,7 @@ jobs: apt-get upgrade -y && apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} ${{ matrix.mpi }} && python3 -m pip install --upgrade pip && + python3 -m pip install numpy && python3 -m pip install 'cmake==3.16.3' - name: Configure Env @@ -69,6 +70,7 @@ jobs: -DOMNITRACE_BUILD_DYNINST=OFF -DOMNITRACE_USE_MPI=${USE_MPI} -DOMNITRACE_USE_HIP=OFF + -DOMNITRACE_USE_PYTHON=ON - name: Build timeout-minutes: 45 @@ -80,7 +82,7 @@ jobs: cmake --build build --target install --parallel 2 - name: Test - timeout-minutes: 30 + timeout-minutes: 45 run: cd build && ctest -V --output-log build/omnitrace-ctest-ubuntu-bionic.log --stop-on-failure diff --git a/.github/workflows/ubuntu-focal-dyninst-package.yml b/.github/workflows/ubuntu-focal-dyninst-package.yml deleted file mode 100644 index 28b976be04..0000000000 --- a/.github/workflows/ubuntu-focal-dyninst-package.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: ubuntu-focal-dyninst-package - -on: - push: - branches: [ main, develop ] - pull_request: - branches: [ main, develop ] - -env: - BUILD_TYPE: Release - ELFUTILS_DOWNLOAD_VERSION: 0.183 - OMNITRACE_DEBUG_FINALIZE: ON - OMNITRACE_VERBOSE: 1 - OMNITRACE_CI: ON - -jobs: - ubuntu-focal-dyninst-package: - runs-on: ubuntu-20.04 - strategy: - matrix: - compiler: ['g++'] - - steps: - - uses: actions/checkout@v2 - - - name: Install Packages - timeout-minutes: 5 - run: - sudo apt-get update && - sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev libopenmpi-dev openmpi-bin libfabric-dev ${{ matrix.compiler }} && - sudo python3 -m pip install --upgrade pip && - python3 -m pip install 'cmake==3.16.3' - - - name: Configure Env - run: - echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && - echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV && - echo "CMAKE_PREFIX_PATH=/opt/dyninst:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV && - echo "/opt/omnitrace/bin:/opt/dyninst/bin:${HOME}/.local/bin" >> $GITHUB_PATH && - echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - - - name: Install Dyninst - timeout-minutes: 25 - run: - cmake --version && - git submodule update --init external/dyninst && - cd external/dyninst && - cmake -B build - -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} - -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/external/dyninst-install - -DBUILD_TBB=ON - -DBUILD_BOOST=ON - -DBUILD_ELFUTILS=ON - -DBUILD_LIBIBERTY=ON && - cmake --build build --target package --parallel 4 && - mkdir /opt/dyninst && - ./build/Dyninst-*-Linux.sh --prefix=/opt/dyninst --exclude-subdir --skip-license && - rm -rf build - - - name: Configure CMake - timeout-minutes: 5 - run: - cmake --version && - cmake -B ${{ github.workspace }}/build - -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') - -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} - -DCMAKE_INSTALL_PREFIX=/opt/omnitrace - -DOMNITRACE_BUILD_TESTING=ON - -DOMNITRACE_USE_MPI=OFF - -DOMNITRACE_USE_MPI_HEADERS=ON - -DOMNITRACE_USE_HIP=OFF - - - name: Build - timeout-minutes: 45 - run: - cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1 - - - name: Install - run: - cmake --build ${{ github.workspace }}/build --target install --parallel 2 - - - name: Test - timeout-minutes: 30 - working-directory: ${{ github.workspace }}/build - run: - ldd ./omnitrace && - ./omnitrace --help && - ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal-dyninst-package.log --stop-on-failure - - - name: Test Install - timeout-minutes: 10 - run: | - set -v - export OMNITRACE_DEBUG=ON - which omnitrace-avail - ldd $(which omnitrace-avail) - omnitrace-avail --help - omnitrace-avail -a - which omnitrace-critical-trace - ldd $(which omnitrace-critical-trace) - which omnitrace - ldd $(which omnitrace) - omnitrace --help - omnitrace -e -v 1 -o ls.inst --simulate -- ls - for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done - omnitrace -e -v 1 -o ls.inst -- ls - ./ls.inst - omnitrace -e -v 1 --simulate -- ls - for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done - omnitrace -e -v 1 -- ls - - - name: CTest Artifacts - uses: actions/upload-artifact@v2 - with: - name: ctest-log - path: | - ${{ github.workspace }}/build/*.log - - - name: Data Artifacts - uses: actions/upload-artifact@v2 - with: - name: data-files - path: | - ${{ github.workspace }}/build/omnitrace-tests-output/**/*.txt diff --git a/.github/workflows/ubuntu-focal-external-rocm.yml b/.github/workflows/ubuntu-focal-external-rocm.yml index a178aac196..3b1b73555f 100644 --- a/.github/workflows/ubuntu-focal-external-rocm.yml +++ b/.github/workflows/ubuntu-focal-external-rocm.yml @@ -7,7 +7,7 @@ on: branches: [ main, develop ] env: - BUILD_TYPE: Release + BUILD_TYPE: MinSizeRel OMNITRACE_DEBUG_FINALIZE: ON OMNITRACE_VERBOSE: 1 OMNITRACE_CI: ON @@ -54,7 +54,7 @@ jobs: cmake -B build -DCMAKE_C_COMPILER=${CC} -DCMAKE_CXX_COMPILER=${CXX} - -DCMAKE_BUILD_TYPE=MinSizeRel + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_INSTALL_PREFIX=/opt/omnitrace -DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_BUILD_DEVELOPER=ON @@ -77,7 +77,7 @@ jobs: cmake --build build --target install --parallel 2 - name: Test - timeout-minutes: 30 + timeout-minutes: 45 run: cd build && ldd ./omnitrace && diff --git a/.github/workflows/ubuntu-focal-external.yml b/.github/workflows/ubuntu-focal-external.yml index 3877985ad8..718fb2a6bf 100644 --- a/.github/workflows/ubuntu-focal-external.yml +++ b/.github/workflows/ubuntu-focal-external.yml @@ -7,13 +7,11 @@ on: branches: [ main, develop ] env: - BUILD_TYPE: Release + BUILD_TYPE: RelWithDebInfo ELFUTILS_DOWNLOAD_VERSION: 0.183 OMNITRACE_DEBUG_FINALIZE: ON OMNITRACE_VERBOSE: 1 OMNITRACE_CI: ON - CONTAINER_NAME: 'container-${GITHUB_SHA}' - CONTAINER_CMD: 'docker exec "${CONTAINER_NAME}"' jobs: ubuntu-focal-external: @@ -33,6 +31,7 @@ jobs: apt-get update && apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev ${{ matrix.compiler }} && python3 -m pip install --upgrade pip && + python3 -m pip install numpy && python3 -m pip install 'cmake==3.16.3' - name: Configure Env @@ -49,11 +48,12 @@ jobs: cmake -B build -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} - -DCMAKE_BUILD_TYPE=RelWithDebInfo + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_INSTALL_PREFIX=/opt/omnitrace -DOMNITRACE_BUILD_TESTING=ON -DOMNITRACE_USE_MPI=OFF -DOMNITRACE_USE_HIP=OFF + -DOMNITRACE_USE_PYTHON=ON - name: Build timeout-minutes: 45 @@ -66,7 +66,7 @@ jobs: cmake --build build --target install --parallel 2 - name: Test - timeout-minutes: 30 + timeout-minutes: 45 run: cd build && ldd ./omnitrace && diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index 033c4c9614..e1f5d1fc21 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -30,6 +30,7 @@ jobs: sudo apt-get update && sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libtbb-dev libboost-{atomic,system,thread,date-time,filesystem,timer}-dev clang libomp-dev ${{ matrix.compiler }} ${{ matrix.mpi }} && python3 -m pip install --upgrade pip && + python3 -m pip install numpy && python3 -m pip install 'cmake==3.16.3' - name: Configure Env @@ -53,6 +54,7 @@ jobs: -DOMNITRACE_BUILD_DYNINST=ON -DOMNITRACE_USE_MPI=${USE_MPI} -DOMNITRACE_USE_HIP=OFF + -DOMNITRACE_USE_PYTHON=ON -DDYNINST_BUILD_ELFUTILS=ON -DDYNINST_BUILD_LIBIBERTY=ON -DDYNINST_BUILD_SHARED_LIBS=ON @@ -68,7 +70,7 @@ jobs: cmake --build ${{ github.workspace }}/build --target install --parallel 2 - name: Test - timeout-minutes: 30 + timeout-minutes: 45 working-directory: ${{ github.workspace }}/build run: ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal.log --stop-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index 60e141d22c..cffcbd8062 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,19 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "") set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Build type" FORCE) +else() + set(VALID_BUILD_TYPES "Release" "RelWithDebInfo" "Debug" "MinSizeRel") + if(NOT "${CMAKE_BUILD_TYPE}" IN_LIST VALID_BUILD_TYPES) + string(REPLACE ";" ", " _VALID_BUILD_TYPES "${VALID_BUILD_TYPES}") + message( + FATAL_ERROR + "Invalid CMAKE_BUILD_TYPE :: ${CMAKE_BUILD_TYPE}. Valid build types are: ${_VALID_BUILD_TYPES}" + ) + endif() +endif() +set(_STRIP_LIBRARIES_DEFAULT OFF) +if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") + set(_STRIP_LIBRARIES_DEFAULT ON) endif() include(GNUInstallDirs) # install directories @@ -82,14 +95,24 @@ omnitrace_add_option( omnitrace_add_option(OMNITRACE_USE_MPI_HEADERS "Enable wrapping MPI functions w/o enabling MPI dependency" OFF) omnitrace_add_option(OMNITRACE_USE_OMPT "Enable OpenMP tools support" OFF) +omnitrace_add_option(OMNITRACE_USE_PYTHON "Enable Python support" OFF) omnitrace_add_option(OMNITRACE_BUILD_DYNINST "Build dyninst from submodule" OFF) omnitrace_add_option(OMNITRACE_BUILD_EXAMPLES "Enable building the examples" OFF) omnitrace_add_option(OMNITRACE_BUILD_TESTING "Enable building the testing suite" OFF) -omnitrace_add_option(OMNITRACE_CUSTOM_DATA_SOURCE "Enable custom data source" OFF) -omnitrace_add_option(OMNITRACE_BUILD_HIDDEN_VISIBILITY - "Build with hidden visibility (disable for Debug builds)" ON) +omnitrace_add_option(OMNITRACE_CUSTOM_DATA_SOURCE "Enable custom data source" OFF + ADVANCED) +omnitrace_add_option( + OMNITRACE_BUILD_HIDDEN_VISIBILITY + "Build with hidden visibility (disable for Debug builds)" ON ADVANCED) omnitrace_add_option(OMNITRACE_BUILD_CI "Enable internal asserts, etc." OFF ADVANCED NO_FEATURE) +omnitrace_add_option(OMNITRACE_STRIP_LIBRARIES "Strip the libraries" + ${_STRIP_LIBRARIES_DEFAULT} ADVANCED) + +if(OMNITRACE_USE_PYTHON) + omnitrace_add_option(OMNITRACE_BUILD_PYTHON + "Build python bindings with internal pybind11" ON) +endif() if(NOT "$ENV{OMNITRACE_CI}" STREQUAL "") message( diff --git a/README.md b/README.md index 0241aeed7b..ee2e330859 100755 --- a/README.md +++ b/README.md @@ -2,11 +2,10 @@ [![Ubuntu 18.04 (GCC 7, 8, MPICH)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-bionic.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-bionic.yml) [![Ubuntu 20.04 (GCC 7, 8, 9, 10)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-external.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-external.yml) -[![Ubuntu 20.04 (GCC 9, external Dyninst)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-dyninst-package.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-dyninst-package.yml) [![Ubuntu 20.04 (GCC 9, MPICH, OpenMPI)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal.yml) [![Ubuntu 20.04 (GCC 9, MPICH, OpenMPI, ROCm 4.3, 4.5, 5.0)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-external-rocm.yml/badge.svg)](https://github.com/AMDResearch/omnitrace/actions/workflows/ubuntu-focal-external-rocm.yml) -Omnitrace is an AMD research project and should not be treated as an offical part of the ROCm software stack. +Omnitrace is an AMD research project and should not be treated as an offical part of the ROCm software stack. The documentation for omnitrace is available at [amdresearch.github.io/omnitrace](https://amdresearch.github.io/omnitrace/). ## Using Omnitrace Executable diff --git a/cmake/BuildSettings.cmake b/cmake/BuildSettings.cmake index 4b6ef852f8..92b1166480 100644 --- a/cmake/BuildSettings.cmake +++ b/cmake/BuildSettings.cmake @@ -21,6 +21,15 @@ omnitrace_add_option(OMNITRACE_USE_COMPILE_TIMING omnitrace_add_option(OMNITRACE_USE_COVERAGE "Build with code-coverage flags" OFF) omnitrace_add_option(OMNITRACE_USE_SANITIZER "Build with -fsanitze=\${OMNITRACE_SANITIZER_TYPE}" OFF) +omnitrace_add_option(OMNITRACE_BUILD_STATIC_LIBGCC + "Build with -static-libgcc if possible" OFF) +omnitrace_add_option(OMNITRACE_BUILD_STATIC_LIBSTDCXX + "Build with -static-libstdc++ if possible" OFF) + +omnitrace_add_interface_library(omnitrace-static-libgcc + "Link to static version of libgcc") +omnitrace_add_interface_library(omnitrace-static-libstdcxx + "Link to static version of libstdc++") target_compile_definitions(omnitrace-compile-options INTERFACE $<$:DEBUG>) @@ -285,6 +294,17 @@ if(MSVC) add_flag_if_avail("/DEBUG") endif() +# ----------------------------------------------------------------------------------------# +# static lib flags +# +target_compile_options( + omnitrace-static-libgcc + INTERFACE $<$:$<$:-static-libgcc>> + $<$:$<$:-static-libgcc>>) +target_compile_options( + omnitrace-static-libstdcxx + INTERFACE $<$:$<$:-static-libstdc++>>) + # ----------------------------------------------------------------------------------------# # user customization # diff --git a/cmake/ConfigCPack.cmake b/cmake/ConfigCPack.cmake index d02e2acad4..737139e0df 100644 --- a/cmake/ConfigCPack.cmake +++ b/cmake/ConfigCPack.cmake @@ -85,6 +85,12 @@ if(OMNITRACE_USE_MPI) "${OMNITRACE_CPACK_PACKAGE_SUFFIX}-${OMNITRACE_MPI_IMPL_UPPER}") endif() +if(OMNITRACE_USE_PYTHON) + string(REPLACE "." "" OMNITRACE_CPACK_PYTHON_VERSION "PY${OMNITRACE_PYTHON_VERSION}") + set(OMNITRACE_CPACK_PACKAGE_SUFFIX + "${OMNITRACE_CPACK_PACKAGE_SUFFIX}-${OMNITRACE_CPACK_PYTHON_VERSION}") +endif() + set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${OMNITRACE_VERSION}-${OMNITRACE_CPACK_SYSTEM_NAME}${OMNITRACE_CPACK_PACKAGE_SUFFIX}" ) diff --git a/cmake/Formatting.cmake b/cmake/Formatting.cmake index 3838981c0a..b199739ca8 100644 --- a/cmake/Formatting.cmake +++ b/cmake/Formatting.cmake @@ -50,9 +50,14 @@ if(OMNITRACE_CLANG_FORMAT_EXE) file(GLOB_RECURSE examples ${PROJECT_SOURCE_DIR}/examples/*.cpp ${PROJECT_SOURCE_DIR}/examples/*.hpp) file(GLOB_RECURSE external ${PROJECT_SOURCE_DIR}/examples/lulesh/external/kokkos/*) - file(GLOB_RECURSE cmake_files ${PROJECT_SOURCE_DIR}/source/*CMakeLists.txt - ${PROJECT_SOURCE_DIR}/examples/*CMakeLists.txt - ${PROJECT_SOURCE_DIR}/tests/*CMakeLists.txt ${PROJECT_SOURCE_DIR}/cmake/*.cmake) + file( + GLOB_RECURSE + cmake_files + ${PROJECT_SOURCE_DIR}/source/*CMakeLists.txt + ${PROJECT_SOURCE_DIR}/examples/*CMakeLists.txt + ${PROJECT_SOURCE_DIR}/tests/*CMakeLists.txt + ${PROJECT_SOURCE_DIR}/cmake/*.cmake + ${PROJECT_SOURCE_DIR}/source/*.cmake) list(APPEND cmake_files ${PROJECT_SOURCE_DIR}/CMakeLists.txt) if(external) list(REMOVE_ITEM examples ${external}) diff --git a/cmake/MacroUtilities.cmake b/cmake/MacroUtilities.cmake index 9cf92cd573..65a93c2a2c 100644 --- a/cmake/MacroUtilities.cmake +++ b/cmake/MacroUtilities.cmake @@ -107,6 +107,36 @@ function(OMNITRACE_CAPITALIZE str var) PARENT_SCOPE) endfunction() +# ------------------------------------------------------------------------------# +# function omnitrace_strip_target() +# +# Creates a target which runs ctest but depends on all the tests being built. +# +function(OMNITRACE_STRIP_TARGET _TARGET) + if(CMAKE_STRIP AND OMNITRACE_STRIP_LIBRARIES) + add_custom_command( + TARGET ${_TARGET} + POST_BUILD + COMMAND + ${CMAKE_STRIP} --keep-symbol="omnitrace_init" + --keep-symbol="omnitrace_finalize" --keep-symbol="omnitrace_push_trace" + --keep-symbol="omnitrace_pop_trace" --keep-symbol="omnitrace_push_region" + --keep-symbol="omnitrace_pop_region" --keep-symbol="omnitrace_set_env" + --keep-symbol="omnitrace_set_mpi" --keep-symbol="omnitrace_user_configure" + --keep-symbol="omnitrace_user_get_callbacks" + --keep-symbol="omnitrace_user_error_string" + --keep-symbol="omnitrace_user_start_trace" + --keep-symbol="omnitrace_user_stop_trace" + --keep-symbol="omnitrace_user_start_thread_trace" + --keep-symbol="omnitrace_user_stop_thread_trace" + --keep-symbol="omnitrace_user_push_region" + --keep-symbol="omnitrace_user_pop_region" --keep-symbol="ompt_start_tool" + ${ARGN} $ + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Stripping ${_TARGET}...") + endif() +endfunction() + # ------------------------------------------------------------------------------# # function add_omnitrace_test_target() # diff --git a/cmake/Packages.cmake b/cmake/Packages.cmake index 0f43bf076b..c77982bdd5 100644 --- a/cmake/Packages.cmake +++ b/cmake/Packages.cmake @@ -20,6 +20,7 @@ omnitrace_add_interface_library(omnitrace-rocm-smi "Provides flags and libraries for rocm-smi") omnitrace_add_interface_library(omnitrace-mpi "Provides MPI or MPI headers") omnitrace_add_interface_library(omnitrace-ptl "Enables PTL support (tasking)") +omnitrace_add_interface_library(omnitrace-python "Enables Python support") target_include_directories(omnitrace-headers INTERFACE ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include) @@ -303,6 +304,34 @@ else() endif() endif() +# ----------------------------------------------------------------------------------------# +# +# Modify CMAKE_C_FLAGS and CMAKE_CXX_FLAGS with -static-libgcc and -static-libstdc++ +# +# ----------------------------------------------------------------------------------------# + +if(OMNITRACE_BUILD_STATIC_LIBGCC) + if(CMAKE_C_COMPILER_ID MATCHES "GNU") + omnitrace_save_variables(STATIC_LIBGCC_C VARIABLES CMAKE_C_FLAGS) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -static-libgcc") + endif() + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + omnitrace_save_variables(STATIC_LIBGCC_CXX VARIABLES CMAKE_CXX_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc") + else() + set(OMNITRACE_BUILD_STATIC_LIBGCC OFF) + endif() +endif() + +if(OMNITRACE_BUILD_STATIC_LIBSTDCXX) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + omnitrace_save_variables(STATIC_LIBSTDCXX_CXX VARIABLES CMAKE_CXX_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++") + else() + set(OMNITRACE_BUILD_STATIC_LIBSTDCXX OFF) + endif() +endif() + # ----------------------------------------------------------------------------------------# # # Perfetto @@ -492,3 +521,42 @@ endif() target_sources(omnitrace-ptl INTERFACE $) target_link_libraries(omnitrace-ptl INTERFACE PTL::ptl-object) + +# ----------------------------------------------------------------------------------------# +# +# Restore the CMAKE_C_FLAGS and CMAKE_CXX_FLAGS in the inverse order +# +# ----------------------------------------------------------------------------------------# + +if(OMNITRACE_BUILD_STATIC_LIBSTDCXX) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + omnitrace_restore_variables(STATIC_LIBSTDCXX_CXX VARIABLES CMAKE_CXX_FLAGS) + endif() +endif() + +if(OMNITRACE_BUILD_STATIC_LIBGCC) + if(CMAKE_C_COMPILER_ID MATCHES "GNU") + omnitrace_restore_variables(STATIC_LIBGCC_C VARIABLES CMAKE_C_FLAGS) + endif() + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + omnitrace_restore_variables(STATIC_LIBGCC_CXX VARIABLES CMAKE_CXX_FLAGS) + endif() +endif() + +omnitrace_add_feature(CMAKE_C_FLAGS "C compiler flags") +omnitrace_add_feature(CMAKE_CXX_FLAGS "C++ compiler flags") + +# ----------------------------------------------------------------------------------------# +# +# Python +# +# ----------------------------------------------------------------------------------------# + +if(OMNITRACE_USE_PYTHON) + if(OMNITRACE_USE_PYTHON AND NOT OMNITRACE_BUILD_PYTHON) + find_package(pybind11 REQUIRED) + endif() + + list(INSERT CMAKE_MODULE_PATH 0 ${PROJECT_SOURCE_DIR}/source/python/cmake) + include(ConfigPython) +endif() diff --git a/cmake/Templates/console-script.in b/cmake/Templates/console-script.in new file mode 100755 index 0000000000..146da14f0d --- /dev/null +++ b/cmake/Templates/console-script.in @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +export PYTHONPATH=$(cd $(dirname ${BASH_SOURCE[0]})/../@CMAKE_INSTALL_PYTHONDIR@ && pwd):${PYTHONPATH} + +: ${PYTHON_EXECUTABLE:=@PYTHON_EXECUTABLE@} + +if [ ! -f ${PYTHON_EXECUTABLE} ]; then PYTHON_EXECUTABLE=$(basename ${PYTHON_EXECUTABLE}); fi + +set -e + +run-script() +{ + echo -e "\n##### omnitrace :: executing '${@}'... #####\n" + eval $@ +} + +run-script ${PYTHON_EXECUTABLE} -m @SCRIPT_SUBMODULE@ $@ diff --git a/cmake/Templates/setup-env.sh.in b/cmake/Templates/setup-env.sh.in index 47ef3ebcb3..de32706c5a 100644 --- a/cmake/Templates/setup-env.sh.in +++ b/cmake/Templates/setup-env.sh.in @@ -10,6 +10,8 @@ fi PATH=${BASEDIR}/bin:${PATH} LD_LIBRARY_PATH=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@:${LD_LIBRARY_PATH} +PYTHONPATH=${BASE_DIR}/@CMAKE_INSTALL_PYTHONDIR@:${PYTHONPATH} export PATH export LD_LIBRARY_PATH +export PYTHONPATH diff --git a/examples/python/builtin.py b/examples/python/builtin.py new file mode 100755 index 0000000000..c0bd45ba54 --- /dev/null +++ b/examples/python/builtin.py @@ -0,0 +1,36 @@ +#!@PYTHON_EXECUTABLE@ + +import sys +import numpy as np + + +def fib(n): + return n if n < 2 else (fib(n - 1) + fib(n - 2)) + + +def inefficient(n): + print(f"inefficient: {n}") + a = 0 + for i in range(n): + a += i + for j in range(n): + a += j + arr = np.random.rand(a * n * n * n) + sum = arr.sum() + print(f"sum: {sum}") + return sum + + +@profile +def run(nfib): + ret = 0 + ret += fib(nfib) + ret += inefficient(nfib) + return ret + + +if __name__ == "__main__": + nfib = int(sys.argv[1]) if len(sys.argv) > 1 else 20 + for i in range(5): + ans = run(nfib) + print(f"[{i}] fibonacci({nfib}) = {ans}") diff --git a/examples/python/external.py b/examples/python/external.py new file mode 100755 index 0000000000..5b306c918f --- /dev/null +++ b/examples/python/external.py @@ -0,0 +1,35 @@ +#!@PYTHON_EXECUTABLE@ + +import sys +import numpy as np + + +def fib(n): + return n if n < 2 else (fib(n - 1) + fib(n - 2)) + + +def inefficient(n): + print(f"inefficient: {n}") + a = 0 + for i in range(n): + a += i + for j in range(n): + a += j + arr = np.random.rand(a * n * n * n) + sum = arr.sum() + print(f"sum: {sum}") + return sum + + +def run(nfib): + ret = 0 + ret += fib(nfib) + ret += inefficient(nfib) + return ret + + +if __name__ == "__main__": + nfib = int(sys.argv[1]) if len(sys.argv) > 1 else 20 + for i in range(5): + ans = run(nfib) + print(f"[{i}] fibonacci({nfib}) = {ans}") diff --git a/examples/python/source.py b/examples/python/source.py new file mode 100755 index 0000000000..276efebccc --- /dev/null +++ b/examples/python/source.py @@ -0,0 +1,38 @@ +#!@PYTHON_EXECUTABLE@ + +import sys +import numpy as np +import omnitrace + +def fib(n): + return n if n < 2 else (fib(n - 1) + fib(n - 2)) + + +def inefficient(n): + print(f"inefficient: {n}") + a = 0 + for i in range(n): + a += i + for j in range(n): + a += j + arr = np.random.rand(a * n * n * n) + sum = arr.sum() + print(f"sum: {sum}") + return sum + + +@omnitrace.profile() +def run(nfib): + ret = 0 + ret += fib(nfib) + ret += inefficient(nfib) + return ret + + +if __name__ == "__main__": + omnitrace.initialize(__file__) + nfib = int(sys.argv[1]) if len(sys.argv) > 1 else 20 + for i in range(5): + ans = run(nfib) + print(f"[{i}] fibonacci({nfib}) = {ans}") + omnitrace.finalize() diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 0a3c81ac5d..84eb12cdda 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -13,6 +13,10 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) add_subdirectory(lib) add_subdirectory(bin) +if(OMNITRACE_USE_PYTHON) + add_subdirectory(python) +endif() + if(OMNITRACE_BUILD_DEVELOPER) add_custom_target(omnitrace-precommit) foreach(_TARGET format-omnitrace-source format-omnitrace-cmake format-timemory-source diff --git a/source/bin/omnitrace/fwd.hpp b/source/bin/omnitrace/fwd.hpp index 58785515db..d67b2cb079 100644 --- a/source/bin/omnitrace/fwd.hpp +++ b/source/bin/omnitrace/fwd.hpp @@ -153,6 +153,7 @@ extern size_t min_instructions; // extern bool werror; extern bool debug_print; +extern bool instr_print; extern int verbose_level; // // string settings diff --git a/source/bin/omnitrace/module_function.cpp b/source/bin/omnitrace/module_function.cpp index eedf6a9462..a4d89da3ef 100644 --- a/source/bin/omnitrace/module_function.cpp +++ b/source/bin/omnitrace/module_function.cpp @@ -60,14 +60,14 @@ module_function::module_function(module_t* mod, procedure_t* proc) flow_graph->getOuterLoops(loop_blocks); } + instructions.reserve(basic_blocks.size()); for(const auto& itr : basic_blocks) { std::vector _instructions{}; itr->getInstructions(_instructions); num_instructions += _instructions.size(); - instructions.reserve(instructions.size() + _instructions.size()); - for(auto&& iitr : _instructions) - instructions.emplace_back(iitr); + if(debug_print || verbose_level > 3 || instr_print) + instructions.emplace_back(std::move(_instructions)); } char modname[FUNCNAMELEN]; diff --git a/source/bin/omnitrace/module_function.hpp b/source/bin/omnitrace/module_function.hpp index c94149b8c5..460882079f 100644 --- a/source/bin/omnitrace/module_function.hpp +++ b/source/bin/omnitrace/module_function.hpp @@ -77,18 +77,18 @@ struct module_function bool is_address_range_constrained() const; // checks address range constraint bool is_num_instructions_constrained() const; // check # instructions constraint - size_t start_address = 0; - uint64_t address_range = 0; - uint64_t num_instructions = 0; - module_t* module = nullptr; - procedure_t* function = nullptr; - flow_graph_t* flow_graph = nullptr; - string_t module_name = {}; - string_t function_name = {}; - function_signature signature = {}; - basic_block_set_t basic_blocks = {}; - basic_loop_vec_t loop_blocks = {}; - std::vector instructions = {}; + size_t start_address = 0; + uint64_t address_range = 0; + uint64_t num_instructions = 0; + module_t* module = nullptr; + procedure_t* function = nullptr; + flow_graph_t* flow_graph = nullptr; + string_t module_name = {}; + string_t function_name = {}; + function_signature signature = {}; + basic_block_set_t basic_blocks = {}; + basic_loop_vec_t loop_blocks = {}; + std::vector> instructions = {}; using str_msg_t = std::tuple; using str_msg_vec_t = std::vector; @@ -198,13 +198,17 @@ module_function::serialize(ArchiveT& ar, const unsigned) ar.finishNode(); // instructions can inflate JSON size so only output when verbosity is increased // above default - if(verbose_level > 0) + if(debug_print || verbose_level > 3 || instr_print) { - std::vector _instructions{}; + std::vector> _instructions{}; _instructions.reserve(instructions.size()); for(auto&& itr : instructions) { - _instructions.emplace_back(itr.format()); + std::vector _subinstr{}; + _subinstr.reserve(itr.size()); + for(auto&& iitr : itr) + _subinstr.emplace_back(iitr.format()); + _instructions.emplace_back(std::move(_subinstr)); } ar(cereal::make_nvp("instructions", _instructions)); } diff --git a/source/bin/omnitrace/omnitrace.cpp b/source/bin/omnitrace/omnitrace.cpp index 6527d84544..603e4c742d 100644 --- a/source/bin/omnitrace/omnitrace.cpp +++ b/source/bin/omnitrace/omnitrace.cpp @@ -51,6 +51,7 @@ size_t min_loop_address_range = (1 << 8); // 256 size_t min_instructions = (1 << 6); // 64 bool werror = false; bool debug_print = false; +bool instr_print = false; int verbose_level = tim::get_env("OMNITRACE_VERBOSE_INSTRUMENT", 0); string_t main_fname = "main"; string_t argv0 = {}; @@ -356,6 +357,12 @@ main(int argc, char** argv) .action([](parser_t& p) { print_overlapping = p.get("print-overlapping"); }); + parser + .add_argument( + { "--print-instructions" }, + "Print the instructions for each basic-block in the JSON/XML outputs") + .max_count(1) + .action([](parser_t& p) { instr_print = p.get("print-instructions"); }); parser.add_argument({ "" }, ""); parser.add_argument({ "[MODE OPTIONS]" }, ""); diff --git a/source/bin/tests/CMakeLists.txt b/source/bin/tests/CMakeLists.txt index f9f7ef8e6b..3fa98fb1ee 100644 --- a/source/bin/tests/CMakeLists.txt +++ b/source/bin/tests/CMakeLists.txt @@ -89,7 +89,7 @@ omnitrace_add_bin_test( TARGET omnitrace-exe ARGS --help LABELS omnitrace-exe - TIMEOUT 15 + TIMEOUT 45 PASS_REGULAR_EXPRESSION ".*\\\[omnitrace\\\] Usage:.*\\\[DEBUG OPTIONS\\\].*\\\[MODE OPTIONS\\\].*\\\[LIBRARY OPTIONS\\\].*\\\[SYMBOL SELECTION OPTIONS\\\].*\\\[RUNTIME OPTIONS\\\].*\\\[GRANULARITY OPTIONS\\\].*\\\[DYNINST OPTIONS\\\].*" ) @@ -97,8 +97,16 @@ omnitrace_add_bin_test( omnitrace_add_bin_test( NAME omnitrace-exe-simulate-ls TARGET omnitrace-exe - ARGS --simulate --print-format json txt xml -- ls - TIMEOUT 60) + ARGS --simulate + --print-format + json + txt + xml + -v + 1 + -- + ls + TIMEOUT 120) omnitrace_add_bin_test( NAME omnitrace-exe-simulate-ls-check @@ -106,7 +114,7 @@ omnitrace_add_bin_test( COMMAND ls WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/omnitrace-tests-output/omnitrace-exe-simulate-ls - TIMEOUT 30 + TIMEOUT 60 PASS_REGULAR_EXPRESSION ".*available-instr.json.*available-instr.txt.*available-instr.xml.*excluded-instr.json.*excluded-instr.txt.*excluded-instr.xml.*instrumented-instr.json.*instrumented-instr.txt.*instrumented-instr.xml.*overlapping-instr.json.*overlapping-instr.txt.*overlapping-instr.xml.*" ) @@ -116,7 +124,7 @@ omnitrace_add_bin_test( TARGET omnitrace-avail ARGS --help LABELS omnitrace-avail - TIMEOUT 15 + TIMEOUT 45 PASS_REGULAR_EXPRESSION ".*\\\[omnitrace-avail\\\] Usage:.*\\\[CATEGORIES\\\].*\\\[VIEW OPTIONS\\\].*\\\[COLUMN OPTIONS\\\].*\\\[WIDTH OPTIONS\\\].*\\\[OUTPUT OPTIONS\\\].*" ) @@ -126,7 +134,7 @@ omnitrace_add_bin_test( TARGET omnitrace-avail ARGS -r wall_clock -C --available LABELS omnitrace-avail - TIMEOUT 15 + TIMEOUT 45 PASS_REGULAR_EXPRESSION "\\\|[-]+\\\|\n\\\|[ ]+COMPONENT[ ]+\\\|\n\\\|[-]+\\\|\n\\\| (wall_clock)[ ]+\\\|\n\\\| (sampling_wall_clock)[ ]+\\\|\n\\\|[-]+\\\|" ) @@ -136,7 +144,7 @@ omnitrace_add_bin_test( TARGET omnitrace-avail ARGS --categories settings::omnitrace --brief LABELS omnitrace-avail - TIMEOUT 15 + TIMEOUT 45 PASS_REGULAR_EXPRESSION "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE|OUTPUT_PREFIX)" FAIL_REGULAR_EXPRESSION "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)" @@ -147,7 +155,7 @@ omnitrace_add_bin_test( TARGET omnitrace-avail ARGS --categories settings::timemory --brief LABELS omnitrace-avail - TIMEOUT 15 + TIMEOUT 45 PASS_REGULAR_EXPRESSION "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)" FAIL_REGULAR_EXPRESSION "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE)") diff --git a/source/lib/omnitrace-dl/CMakeLists.txt b/source/lib/omnitrace-dl/CMakeLists.txt index 15c2a12294..a37c9d53b7 100644 --- a/source/lib/omnitrace-dl/CMakeLists.txt +++ b/source/lib/omnitrace-dl/CMakeLists.txt @@ -11,11 +11,14 @@ set(BUILD_RPATH_USE_ORIGIN ON) add_library(omnitrace-dl-library SHARED) add_library(omnitrace::omnitrace-dl-library ALIAS omnitrace-dl-library) -target_sources(omnitrace-dl-library PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/dl.cpp) -target_link_libraries(omnitrace-dl-library PRIVATE ${dl_LIBRARY} - omnitrace::common-library) -target_include_directories(omnitrace-dl-library - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../omnitrace-user) +target_sources(omnitrace-dl-library PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/dl.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/dl.hpp) +target_link_libraries(omnitrace-dl-library + PUBLIC ${dl_LIBRARY} $) +target_include_directories( + omnitrace-dl-library + PUBLIC $ + $) check_cxx_compiler_flag("-fno-exceptions" omnitrace_dl_library_fno_exceptions) check_cxx_compiler_flag("-ftls-model=local-dynamic" @@ -32,18 +35,21 @@ if(OMNITRACE_BUILD_DEVELOPER) endif() omnitrace_target_compile_definitions( - omnitrace-dl-library PRIVATE OMNITRACE_USE_OMPT=$) + omnitrace-dl-library + PUBLIC $>) set_target_properties( omnitrace-dl-library PROPERTIES OUTPUT_NAME omnitrace-dl - CXX_VISIBILITY_PRESET "protected" + CXX_VISIBILITY_PRESET "internal" VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR} POSITION_INDEPENDENT_CODE ON BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN") +omnitrace_strip_target(omnitrace-dl-library) + install( TARGETS omnitrace-dl-library EXPORT omnitrace-dl-library-targets diff --git a/source/lib/omnitrace-user/CMakeLists.txt b/source/lib/omnitrace-user/CMakeLists.txt index 7362fe7bd5..ff9bd747f0 100644 --- a/source/lib/omnitrace-user/CMakeLists.txt +++ b/source/lib/omnitrace-user/CMakeLists.txt @@ -43,6 +43,8 @@ set_target_properties( BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN") +omnitrace_strip_target(omnitrace-user-library) + install( TARGETS omnitrace-user-library EXPORT omnitrace-user-library-targets diff --git a/source/lib/omnitrace/CMakeLists.txt b/source/lib/omnitrace/CMakeLists.txt index 453f8f502b..1f29b9e8d1 100644 --- a/source/lib/omnitrace/CMakeLists.txt +++ b/source/lib/omnitrace/CMakeLists.txt @@ -20,18 +20,21 @@ target_compile_definitions( target_link_libraries( omnitrace-interface-library - INTERFACE $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $ - $,omnitrace::omnitrace-sanitizer,>) + INTERFACE + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $,$,> + $,$,> + $,omnitrace::omnitrace-sanitizer,>) # ------------------------------------------------------------------------------# # @@ -139,6 +142,8 @@ set_target_properties( INSTALL_RPATH "\$ORIGIN:\$ORIGIN/timemory/libunwind:\$ORIGIN/dyninst-tpls/libs") +omnitrace_strip_target(omnitrace-library) + install( TARGETS omnitrace-library DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/source/lib/omnitrace/src/library.cpp b/source/lib/omnitrace/src/library.cpp index d0783137e7..5ca8fc5363 100644 --- a/source/lib/omnitrace/src/library.cpp +++ b/source/lib/omnitrace/src/library.cpp @@ -981,14 +981,10 @@ omnitrace_finalize_hidden(void) auto _pos = _msg.find(">>> "); if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5); OMNITRACE_PRINT("%s\n", _msg.c_str()); - get_main_bundle().reset(); + OMNITRACE_DEBUG_F("Resetting main bundle...\n"); + get_main_bundle()->reset(); } - int _threadpool_verbose = (get_debug()) ? 4 : -1; - tasking::get_roctracer_thread_pool().set_verbose(_threadpool_verbose); - if(get_use_critical_trace()) - tasking::get_critical_trace_thread_pool().set_verbose(_threadpool_verbose); - // join extra thread(s) used by roctracer OMNITRACE_DEBUG_F("waiting for all roctracer tasks to complete...\n"); tasking::get_roctracer_task_group().join(); diff --git a/source/lib/omnitrace/src/library/components/backtrace.cpp b/source/lib/omnitrace/src/library/components/backtrace.cpp index 90f25bd7c8..c63e8a817d 100644 --- a/source/lib/omnitrace/src/library/components/backtrace.cpp +++ b/source/lib/omnitrace/src/library/components/backtrace.cpp @@ -413,7 +413,8 @@ backtrace::post_process(int64_t _tid) if(!_sampler) { // this should be relatively common - OMNITRACE_DEBUG( + OMNITRACE_CONDITIONAL_PRINT( + get_debug() && get_verbose() > 1, "Post-processing sampling entries for thread %lu skipped (no sampler)\n", _tid); return; diff --git a/source/lib/omnitrace/src/library/config.cpp b/source/lib/omnitrace/src/library/config.cpp index c01221af78..71ae9b9eb3 100644 --- a/source/lib/omnitrace/src/library/config.cpp +++ b/source/lib/omnitrace/src/library/config.cpp @@ -354,7 +354,9 @@ configure_settings() _config->get("OMNITRACE_TIMEMORY_COMPONENTS"); // always initialize timemory because gotcha wrappers are always used - auto _cmd = tim::read_command_line(process::get_id()); + auto _cmd = tim::read_command_line(process::get_id()); + auto _cmd_env = tim::get_env("OMNITRACE_COMMAND_LINE", ""); + if(!_cmd_env.empty()) _cmd = tim::delimit(_cmd_env, " "); auto _exe = (_cmd.empty()) ? "exe" : _cmd.front(); auto _pos = _exe.find_last_of('/'); if(_pos < _exe.length() - 1) _exe = _exe.substr(_pos + 1); diff --git a/source/python/CMakeLists.txt b/source/python/CMakeLists.txt new file mode 100644 index 0000000000..bd53d811fb --- /dev/null +++ b/source/python/CMakeLists.txt @@ -0,0 +1,129 @@ +# ######################################################################################## +# +# omnitrace (Python) +# +# ######################################################################################## + +# if set, will screw up loading library +unset(CMAKE_DEBUG_POSTFIX) +set(CMAKE_CXX_CLANG_TIDY) +set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME python) + +# ######################################################################################## + +function(OMNITRACE_CONFIGURE_PYTARGET _TARGET) + + add_library(omnitrace::${_TARGET} ALIAS ${_TARGET}) + target_link_libraries(${_TARGET} PRIVATE libpyomnitrace-interface) + + set(_SUBDIR ${ARGN}) + if(_SUBDIR) + set(_SUBDIR "/${_SUBDIR}") + endif() + + set_target_properties( + ${_TARGET} + PROPERTIES PREFIX "" + SUFFIX "${PYTHON_MODULE_EXTENSION}" + LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} + ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} + RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} + PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python/omnitrace${_SUBDIR} + INSTALL_RPATH_USE_LINK_PATH ON + ${EXTRA_PROPERTIES}) + + set(_PYLIB ${CMAKE_INSTALL_PYTHONDIR}/omnitrace${_SUBDIR}) + if(NOT IS_ABSOLUTE "${_PYLIB}") + set(_PYLIB ${CMAKE_INSTALL_PREFIX}/${_PYLIB}) + endif() + + if(SKBUILD) + set(LIB_RELPATH ../../../..) + else() + file(RELATIVE_PATH LIB_RELPATH "${_PYLIB}" + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") + endif() + + set_target_properties( + ${_TARGET} + PROPERTIES + INSTALL_RPATH + "\$ORIGIN:\$ORIGIN/${LIB_RELPATH}:\$ORIGIN/../../../..:${CMAKE_INSTALL_RPATH}" + ) + + install( + TARGETS ${_TARGET} + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/omnitrace${_SUBDIR} + OPTIONAL) + + if(NOT "${_TARGET}" STREQUAL "libpyomnitrace") + add_dependencies(libpyomnitrace ${_TARGET}) + endif() +endfunction() + +# ######################################################################################## + +add_library(omnitrace-python-compile-options INTERFACE) +add_library(omnitrace::omnitrace-python-compile-options ALIAS + omnitrace-python-compile-options) +add_cxx_flag_if_avail("-frtti" omnitrace-python-compile-options) +add_cxx_flag_if_avail("-Wno-unused-value" omnitrace-python-compile-options) +add_cxx_flag_if_avail("-Wno-range-loop-analysis" omnitrace-python-compile-options) +add_cxx_flag_if_avail("-ftls-model=global-dynamic" omnitrace-python-compile-options) +add_cxx_flag_if_avail("-Wno-deprecated-declarations" omnitrace-python-compile-options) +add_cxx_flag_if_avail("-Wno-unused-but-set-parameter" omnitrace-python-compile-options) + +file(GLOB pyheaders ${CMAKE_CURRENT_LIST_DIR}/libpyomnitrace*.hpp) +set(pysources ${CMAKE_CURRENT_LIST_DIR}/libpyomnitrace.cpp) + +set(pybind_libs pybind11::module) + +add_library(libpyomnitrace-interface INTERFACE) +target_link_libraries( + libpyomnitrace-interface + INTERFACE pybind11::module + timemory::timemory-headers + omnitrace::omnitrace-headers + omnitrace::omnitrace-compile-options + omnitrace::omnitrace-lto + omnitrace::omnitrace-dl-library + omnitrace::omnitrace-python + omnitrace::omnitrace-python-compile-options) + +target_compile_definitions(libpyomnitrace-interface INTERFACE OMNITRACE_PYBIND11_SOURCE) + +add_library(libpyomnitrace MODULE ${pysources} ${pyheaders}) +omnitrace_configure_pytarget(libpyomnitrace) + +add_subdirectory(omnitrace) + +if(PYTHON_EXECUTABLE) + configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.py.in + ${PROJECT_BINARY_DIR}/python/setup.py @ONLY) + configure_file(${CMAKE_CURRENT_LIST_DIR}/setup.cfg.in + ${PROJECT_BINARY_DIR}/python/setup.cfg @ONLY) + configure_file(${CMAKE_CURRENT_LIST_DIR}/pyproject.toml + ${PROJECT_BINARY_DIR}/python/pyproject.toml COPYONLY) + execute_process( + COMMAND ${PYTHON_EXECUTABLE} setup.py dist_info + OUTPUT_VARIABLE _OUT + RESULT_VARIABLE _RET + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python) + set(_DIST_DIR + ${PROJECT_BINARY_DIR}/python/${PROJECT_NAME}-${PROJECT_VERSION}.dist-info) + if(NOT EXISTS ${_DIST_DIR}) + set(_DIST_DIR ${PROJECT_BINARY_DIR}/${PROJECT_NAME}.dist-info) + endif() + if(EXISTS ${_DIST_DIR} AND IS_DIRECTORY ${_DIST_DIR}) + configure_file(${PROJECT_SOURCE_DIR}/LICENSE ${_DIST_DIR}/LICENSE.txt COPYONLY) + install( + DIRECTORY ${_DIST_DIR} + DESTINATION ${CMAKE_INSTALL_PYTHONDIR} + OPTIONAL) + endif() +endif() diff --git a/source/python/cmake/ConfigPython.cmake b/source/python/cmake/ConfigPython.cmake new file mode 100644 index 0000000000..2ec89c895e --- /dev/null +++ b/source/python/cmake/ConfigPython.cmake @@ -0,0 +1,449 @@ +# Python configuration +# + +# include guard +include_guard(DIRECTORY) + +# Stops lookup as soon as a version satisfying version constraints is found. +set(Python3_FIND_STRATEGY + "LOCATION" + CACHE STRING + "Stops lookup as soon as a version satisfying version constraints is found") + +# virtual environment is used before any other standard paths to look-up for the +# interpreter +set(Python3_FIND_VIRTUALENV + "FIRST" + CACHE STRING "Virtual environment is used before any other standard paths") +set_property(CACHE Python3_FIND_VIRTUALENV PROPERTY STRINGS "FIRST;LAST;NEVER") + +if(APPLE) + set(Python3_FIND_FRAMEWORK + "LAST" + CACHE STRING + "Order of preference between Apple-style and unix-style package components") + set_property(CACHE Python3_FIND_FRAMEWORK PROPERTY STRINGS "FIRST;LAST;NEVER") +endif() + +# PyPy does not support embedding the interpreter +set(Python3_FIND_IMPLEMENTATIONS + "CPython" + CACHE STRING "Different implementations which will be searched.") +set_property(CACHE Python3_FIND_IMPLEMENTATIONS PROPERTY STRINGS + "CPython;IronPython;PyPy") + +# variable is a 3-tuple specifying, in order, pydebug (d), pymalloc (m) and unicode (u) +# set(Python3_FIND_ABI "OFF" "OFF" "OFF" CACHE STRING "variable is a 3-tuple specifying +# pydebug (d), pymalloc (m) and unicode (u)") + +# Create CMake cache entries for the above artifact specification variables so that users +# can edit them interactively. This disables support for multiple version/component +# requirements. +set(Python3_ARTIFACTS_INTERACTIVE + ON + CACHE BOOL "Create CMake cache entries so that users can edit them interactively") + +# if("${Python3_USE_STATIC_LIBS}" STREQUAL "ANY") set(Python3_USE_STATIC_LIBS "OFF" CACHE +# STRING "If ON, only static libs; if OFF, only shared libs; if ANY, shared then static") +# set_property(CACHE Python3_USE_STATIC_LIBS PROPERTY STRINGS "ON;OFF;ANY") else() +# unset(Python3_USE_STATIC_LIBS) endif() + +foreach(_VAR FIND_STRATEGY FIND_VIRTUALENV FIND_FRAMEWORK FIND_IMPLEMENTATIONS + ARTIFACTS_INTERACTIVE) + if(DEFINED Python3_${_VAR}) + set(Python_${_VAR} + "${Python3_${_VAR}}" + CACHE STRING "Set via Python3_${_VAR} setting (omnitrace)") + mark_as_advanced(Python_${_VAR}) + mark_as_advanced(Python3_${_VAR}) + endif() +endforeach() + +# display version +omnitrace_add_feature(OMNITRACE_PYTHON_VERSION "Python version for omnitrace" DOC) + +# search hint +if(PYTHON_ROOT_DIR AND NOT Python3_ROOT_DIR) + set(Python3_ROOT_DIR ${PYTHON_ROOT_DIR}) +endif() + +# legacy specification of interpreter +if(PYTHON_EXECUTABLE AND NOT Python3_EXECUTABLE) + set(Python3_EXECUTABLE + "${PYTHON_EXECUTABLE}" + CACHE FILEPATH "Path to Python3 interpreter") +endif() + +# default python types to search for +set(Python_ADDITIONAL_VERSIONS + "3.9;3.8;3.7;3.6" + CACHE STRING "Python versions supported by omnitrace") + +# override types to search for +if(OMNITRACE_PYTHON_VERSION) + set(Python_ADDITIONAL_VERSIONS + ${OMNITRACE_PYTHON_VERSION} + CACHE STRING "Python versions supported by omnitrace" FORCE) +elseif(PYBIND11_PYTHON_VERSION) + set(Python_ADDITIONAL_VERSIONS + ${PYBIND11_PYTHON_VERSION} + CACHE STRING "Python versions supported by omnitrace") +endif() + +# unset the version strings +if(_PYVERSION_LAST AND (OMNITRACE_PYTHON_VERSION VERSION_LESS _PYVERSION_LAST + OR OMNITRACE_PYTHON_VERSION VERSION_GREATER _PYVERSION_LAST)) + unset(OMNITRACE_PYTHON_VERSION CACHE) + unset(PYBIND11_PYTHON_VERSION CACHE) + unset(CMAKE_INSTALL_PYTHONDIR CACHE) +endif() + +# if OMNITRACE_PYTHON_VERSION specified, set to desired python version +set(_PYVERSION ${OMNITRACE_PYTHON_VERSION}) + +# if OMNITRACE_PYTHON_VERSION is not set but PYBIND11_PYTHON_VERSION is +if("${_PYVERSION}" STREQUAL "" AND PYBIND11_PYTHON_VERSION) + set(_PYVERSION ${PYBIND11_PYTHON_VERSION}) +endif() + +# basically just used to get Python3_SITEARCH for installation +find_package(Python3 ${_PYVERSION} MODULE ${OMNITRACE_FIND_REQUIREMENT} + COMPONENTS Interpreter Development) + +# executable +set(PYTHON_EXECUTABLE + "${Python3_EXECUTABLE}" + CACHE FILEPATH "Set via Python3_EXECUTABLE (omnitrace)" FORCE) +# includes +if(Python3_INCLUDE_DIR AND NOT Python3_INCLUDE_DIRS) + set(Python3_INCLUDE_DIRS ${Python3_INCLUDE_DIR}) +endif() +if(Python3_INCLUDE_DIRS) + set(PYTHON_INCLUDE_DIR + "${Python3_INCLUDE_DIRS}" + CACHE PATH "Set via Python3_INCLUDE_DIR (omnitrace)" FORCE) + set(PYTHON_INCLUDE_DIRS + "${Python3_INCLUDE_DIRS}" + CACHE PATH "Set via Python3_INCLUDE_DIRS (omnitrace)" FORCE) +endif() +# libraries +set(PYTHON_LIBRARY_DEBUG + "${Python3_LIBRARY_DEBUG}" + CACHE FILEPATH "Set via Python3_LIBRARY_DEBUG (omnitrace)" FORCE) +set(PYTHON_LIBRARY_RELEASE + "${Python3_LIBRARY_RELEASE}" + CACHE FILEPATH "Set via Python3_LIBRARY_DEBUG (omnitrace)" FORCE) +if(Python3_LIBRARY_RELEASE) + set(PYTHON_LIBRARY + "${Python3_LIBRARY_RELEASE}" + CACHE FILEPATH "Set via Python3_LIBRARY (omnitrace)" FORCE) + set(PYTHON_LIBRARIES + "${Python3_LIBRARY_RELEASE}" + CACHE FILEPATH "Set via Python3_LIBRARIES (omnitrace)" FORCE) +else(Python3_LIBRARY_DEBUG) + set(PYTHON_LIBRARY + "${Python3_LIBRARY_DEBUG}" + CACHE FILEPATH "Set via Python3_LIBRARY (omnitrace)" FORCE) + set(PYTHON_LIBRARIES + "${Python3_LIBRARY_DEBUG}" + CACHE FILEPATH "Set via Python3_LIBRARIES (omnitrace)" FORCE) +endif() +set(PYTHON_LIBRARY_DIRS + "${Python3_LIBRARY_DIRS}" + CACHE PATH "Set via Python3_LIBRARY_DIRS (omnitrace)" FORCE) +set(PYTHON_LINK_OPTIONS + "${Python3_LINK_OPTIONS}" + CACHE STRING "Set via Python3_LINK_OPTIONS (omnitrace)" FORCE) + +# module +set(PYTHON_MODULE_EXTENSION + "${Python3_MODULE_EXTENSION}" + CACHE STRING "Set via Python3_MODULE_EXTENSION (omnitrace)" FORCE) +set(PYTHON_MODULE_PREFIX + "${Python3_MODULE_PREFIX}" + CACHE STRING "Set via Python3_MODULE_PREFIX (omnitrace)" FORCE) + +# version +set(PYTHON_VERSION + "${Python3_VERSION}" + CACHE STRING "Set via Python3_VERSION (omnitrace)" FORCE) +set(PYTHON_VERSION_MAJOR + "${Python3_VERSION_MAJOR}" + CACHE STRING "Set via Python3_VERSION_MAJOR (omnitrace)" FORCE) +set(PYTHON_VERSION_MINOR + "${Python3_VERSION_MINOR}" + CACHE STRING "Set via Python3_VERSION_MINOR (omnitrace)" FORCE) + +# find_package +set(PythonInterp_FOUND ${Python3_Interpreter_FOUND}) +set(PythonLibs_FOUND ${Python3_Development_FOUND}) + +# set OMNITRACE_PYTHON_VERSION if we have the python version +if(PYTHON_VERSION_STRING) + set(OMNITRACE_PYTHON_VERSION + "${PYTHON_VERSION_STRING}" + CACHE STRING "Python version for omnitrace") +endif() + +# if either not found, disable +if(NOT Python3_FOUND) + set(OMNITRACE_USE_PYTHON OFF) + set(OMNITRACE_BUILD_PYTHON OFF) + omnitrace_inform_empty_interface(omnitrace-python "Python embedded interpreter") + omnitrace_inform_empty_interface(omnitrace-plotting "Python plotting from C++") +else() + set(OMNITRACE_PYTHON_VERSION + "${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}" + CACHE STRING "Python version for omnitrace") + omnitrace_add_feature(PYTHON_EXECUTABLE "Python executable") +endif() + +# C++ standard +if(NOT MSVC) + if(NOT "${PYBIND11_CPP_STANDARD}" STREQUAL "-std=c++${CMAKE_CXX_STANDARD}") + set(PYBIND11_CPP_STANDARD + -std=c++${CMAKE_CXX_STANDARD} + CACHE STRING "PyBind11 CXX standard" FORCE) + endif() +else() + if(NOT "${PYBIND11_CPP_STANDARD}" STREQUAL "/std:c++${CMAKE_CXX_STANDARD}") + set(PYBIND11_CPP_STANDARD + /std:c++${CMAKE_CXX_STANDARD} + CACHE STRING "PyBind11 CXX standard" FORCE) + endif() +endif() + +option(PYBIND11_INSTALL "Enable Pybind11 installation" OFF) + +if(OMNITRACE_BUILD_PYTHON AND NOT TARGET pybind11) + # checkout PyBind11 if not checked out + omnitrace_checkout_git_submodule( + RECURSIVE + RELATIVE_PATH external/pybind11 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/external/timemory + REPO_URL https://github.com/jrmadsen/pybind11.git + REPO_BRANCH master) + + # add PyBind11 to project omnitrace_save_variables(IPO VARIABLES + # CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) + add_subdirectory(${PROJECT_SOURCE_DIR}/external/timemory/external/pybind11) + # omnitrace_restore_variables(IPO VARIABLES CMAKE_INTERPROCEDURAL_OPTIMIZATION) +endif() + +if(NOT PYBIND11_PYTHON_VERSION) + unset(PYBIND11_PYTHON_VERSION CACHE) + execute_process( + COMMAND + ${PYTHON_EXECUTABLE} -c + "import sys; print('{}.{}'.format(sys.version_info[0], sys.version_info[1]))" + OUTPUT_VARIABLE PYTHON_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + set(PYBIND11_PYTHON_VERSION + "${PYTHON_VERSION}" + CACHE STRING "Python version") +endif() + +omnitrace_add_feature(PYBIND11_CPP_STANDARD "PyBind11 C++ standard") +omnitrace_add_feature(PYBIND11_PYTHON_VERSION "PyBind11 Python version") + +if(NOT "${OMNITRACE_PYTHON_VERSION}" MATCHES "${PYBIND11_PYTHON_VERSION}*") + message(STATUS "OMNITRACE_PYTHON_VERSION is set to ${OMNITRACE_PYTHON_VERSION}") + message(STATUS "PYBIND11_PYTHON_VERSION is set to ${PYBIND11_PYTHON_VERSION}") + message( + FATAL_ERROR "Mismatched 'OMNITRACE_PYTHON_VERSION' and 'PYBIND11_PYTHON_VERSION'") +endif() + +execute_process( + COMMAND ${PYTHON_EXECUTABLE} -c + "import time ; print('{} {}'.format(time.ctime(), time.tzname[0]))" + OUTPUT_VARIABLE OMNITRACE_INSTALL_DATE + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + +string(REPLACE " " " " OMNITRACE_INSTALL_DATE "${OMNITRACE_INSTALL_DATE}") + +if(SKBUILD) + set(OMNITRACE_INSTALL_PYTHON + "prefix" + CACHE STRING "SKBUILD forced python install type" FORCE) +else() + set(OMNITRACE_INSTALL_PYTHON + "lib" + CACHE STRING "Installation type for python (prefix, lib, or global)") +endif() + +omnitrace_add_feature(OMNITRACE_INSTALL_PYTHON + "Installation type for python (prefix, lib, or global)") + +if(SKBUILD OR "${OMNITRACE_INSTALL_PYTHON}" STREQUAL "prefix") + set(CMAKE_INSTALL_PYTHONDIR + ${CMAKE_INSTALL_PREFIX} + CACHE PATH "Installation directory for python") +elseif(SPACK_BUILD OR "${OMNITRACE_INSTALL_PYTHON}" STREQUAL "lib") + set(CMAKE_INSTALL_PYTHONDIR + lib/python${PYBIND11_PYTHON_VERSION}/site-packages + CACHE PATH "Installation directory for python") +else() + string(REPLACE "\\" "/" Python3_SITEARCH "${Python3_SITEARCH}") + set(CMAKE_INSTALL_PYTHONDIR ${Python3_SITEARCH}) + omnitrace_add_feature(Python3_SITEARCH + "site-packages directory of python installation") + set(_REMOVE OFF) + # make the directory if it doesn't exist + if(NOT EXISTS ${Python3_SITEARCH}/omnitrace) + set(_REMOVE ON) + execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${Python3_SITEARCH}/omnitrace + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + ERROR_QUIET) + endif() + # figure out if we can install to Python3_SITEARCH + execute_process( + COMMAND ${CMAKE_COMMAND} -E touch ${Python3_SITEARCH}/omnitrace/__init__.py + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + ERROR_VARIABLE ERR_MSG + RESULT_VARIABLE ERR_CODE) + # remove the directory if we created it + if(_REMOVE) + execute_process( + COMMAND ${CMAKE_COMMAND} -E remove_directory ${Python3_SITEARCH}/omnitrace + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + ERROR_QUIET) + endif() + # check the error code of the touch command + if(ERR_CODE) + if("${OMNITRACE_INSTALL_PYTHON}" STREQUAL "global") + message( + FATAL_ERROR + "omnitrace could not install python files to ${Python3_SITEARCH} (not writable):\n${ERR_MSG}" + ) + endif() + # get the python directory name, e.g. 'python3.6' from + # '/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6' + get_filename_component(PYDIR "${Python3_STDLIB}" NAME) + omnitrace_add_feature(Python3_STDLIB + "standard-library directory of python installation") + # Should not be CMAKE_INSTALL_LIBDIR! Python won't look in a lib64 folder + set(CMAKE_INSTALL_PYTHONDIR lib/${PYDIR}/site-packages) + endif() +endif() + +if(OMNITRACE_BUILD_PYTHON OR pybind11_FOUND) + set(_PYBIND11_INCLUDE_DIRS) + foreach(_TARG pybind11 pybind11::pybind11 pybind11::module) + if(TARGET ${_TARG}) + get_target_property(_INCLUDE_DIR ${_TARG} INTERFACE_INCLUDE_DIRECTORIES) + if(_INCLUDE_DIR) + list(APPEND _PYBIND11_INCLUDE_DIRS ${_INCLUDE_DIR}) + endif() + endif() + endforeach() + if(_PYBIND11_INCLUDE_DIRS) + list(REMOVE_DUPLICATES _PYBIND11_INCLUDE_DIRS) + endif() + omnitrace_target_compile_definitions(omnitrace-python INTERFACE OMNITRACE_USE_PYTHON) + if(NOT APPLE) + target_link_libraries(omnitrace-python INTERFACE ${PYTHON_LIBRARIES}) + endif() + if(PYTHON_INCLUDE_DIRS) + target_include_directories(omnitrace-python SYSTEM + INTERFACE ${PYTHON_INCLUDE_DIRS}) + endif() + if(PYBIND11_INCLUDE_DIRS) + target_include_directories(omnitrace-python SYSTEM + INTERFACE ${PYBIND11_INCLUDE_DIRS}) + endif() + if(PYBIND11_INCLUDE_DIR) + target_include_directories(omnitrace-python SYSTEM + INTERFACE $) + endif() + if(_PYBIND11_INCLUDE_DIRS) + target_include_directories(omnitrace-python SYSTEM + INTERFACE $) + endif() +endif() + +if(APPLE) + if(CMAKE_VERSION VERSION_LESS 3.18) + target_link_libraries(omnitrace-python INTERFACE "-undefined dynamic_lookup") + else() + target_link_libraries( + omnitrace-python + INTERFACE "$<$:-undefined dynamic_lookup>") + endif() +endif() + +if(WIN32) + # Windows produces: + # + # CMake Warning (dev) at tests/test-python-install-import.cmake:3 (SET): Syntax error + # in cmake code at + # C:/projects/omnitrace/build-omnitrace/tests/test-python-install-import.cmake:3 when + # parsing string C:\Python36-x64\Lib\site-packages Invalid escape sequence \P + string(REPLACE "\\" "/" INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR}") +else() + set(INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR}") +endif() + +configure_file( + ${PROJECT_SOURCE_DIR}/external/timemory/cmake/Templates/test-python-install-import.cmake.in + ${PROJECT_BINARY_DIR}/tests/test-python-install-import.cmake + @ONLY) +unset(INSTALL_PYTHONDIR) + +omnitrace_add_feature(CMAKE_INSTALL_PYTHONDIR + "Installation prefix of the python bindings") + +set(_PYVERSION_LAST + "${OMNITRACE_PYTHON_VERSION}" + CACHE INTERNAL "Last version" FORCE) + +find_package(PythonInterp ${OMNITRACE_PYTHON_VERSION} EXACT REQUIRED) +find_package(PythonLibs ${OMNITRACE_PYTHON_VERSION} EXACT REQUIRED) +# find_package(PythonExtensions REQUIRED) + +if("${PYTHON_MODULE_EXTENSION}" STREQUAL "") + execute_process( + COMMAND + "${Python3_EXECUTABLE}" "-c" " +from distutils import sysconfig as s;import sys;import struct; +print('.'.join(str(v) for v in sys.version_info)); +print(sys.prefix); +print(s.get_python_inc(plat_specific=True)); +print(s.get_python_lib(plat_specific=True)); +print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO')); +print(hasattr(sys, 'gettotalrefcount')+0); +print(struct.calcsize('@P')); +print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION')); +print(s.get_config_var('LIBDIR') or ''); +print(s.get_config_var('MULTIARCH') or ''); +" + RESULT_VARIABLE _PYTHON_SUCCESS + OUTPUT_VARIABLE _PYTHON_VALUES + ERROR_VARIABLE _PYTHON_ERROR_VALUE) + + if(_PYTHON_SUCCESS MATCHES 0) + # Convert the process output into a list + if(WIN32) + string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES}) + endif() + string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES}) + string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES}) + list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST) + list(GET _PYTHON_VALUES 1 PYTHON_PREFIX) + list(GET _PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) + list(GET _PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) + list(GET _PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) + list(GET _PYTHON_VALUES 5 PYTHON_IS_DEBUG) + list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) + list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) + list(GET _PYTHON_VALUES 8 PYTHON_LIBDIR) + list(GET _PYTHON_VALUES 9 PYTHON_MULTIARCH) + else() + message(WARNING "${_PYTHON_ERROR_VALUE}") + endif() + + if("${PYTHON_MODULE_EXTENSION}" STREQUAL "") + message(WARNING "Python module extension is empty!") + endif() +endif() diff --git a/source/python/libpyomnitrace.cpp b/source/python/libpyomnitrace.cpp new file mode 100644 index 0000000000..9160bd3c7b --- /dev/null +++ b/source/python/libpyomnitrace.cpp @@ -0,0 +1,447 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "libpyomnitrace.hpp" +#include "dl.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pyomnitrace +{ +namespace pyprofile +{ +py::module +generate(py::module& _pymod); +} +} // namespace pyomnitrace + +PYBIND11_MODULE(libpyomnitrace, omni) +{ + using namespace pyomnitrace; + py::doc("omnitrace profiler for python"); + pyprofile::generate(omni); + + omni.def( + "initialize", + [](const std::string& _v) { + omnitrace_set_mpi(false, false); + omnitrace_init("trace", false, _v.c_str()); + }, + "Initialize omnitrace"); + omni.def( + "finalize", []() { omnitrace_finalize(); }, "Initialize omnitrace"); +} + +//======================================================================================// +// +namespace pyomnitrace +{ +namespace pyprofile +{ +// +using profiler_t = std::pair, std::function>; +using profiler_vec_t = std::vector; +using profiler_label_map_t = std::unordered_map; +using profiler_index_map_t = std::unordered_map; +using strset_t = std::unordered_set; +// +struct config +{ + bool is_running = false; + bool trace_c = false; + bool include_internal = false; + bool include_args = false; + bool include_line = false; + bool include_filename = false; + bool full_filepath = false; + int32_t ignore_stack_depth = 0; + int32_t base_stack_depth = -1; + std::string base_module_path = {}; + strset_t include_functions = {}; + strset_t include_filenames = {}; + strset_t exclude_functions = { "^(FILE|FUNC|LINE)$", + "^get_fcode$", + "^_(_exit__|handle_fromlist|shutdown|get_sep)$", + "^is(function|class)$", + "^basename$", + "^<.*>$" }; + strset_t exclude_filenames = { + "(__init__|__main__|functools|encoder|decoder|_pylab_helpers|threading).py$", + "^<.*>$" + }; + profiler_index_map_t records = {}; + int32_t verbose = 0; +}; +// +inline config& +get_config() +{ + static auto* _instance = new config{}; + static thread_local auto* _tl_instance = []() { + static std::atomic _count{ 0 }; + auto _cnt = _count++; + if(_cnt == 0) return _instance; + + auto* _tmp = new config{}; + _tmp->is_running = _instance->is_running; + _tmp->trace_c = _instance->trace_c; + _tmp->include_internal = _instance->include_internal; + _tmp->include_args = _instance->include_args; + _tmp->include_line = _instance->include_line; + _tmp->include_filename = _instance->include_filename; + _tmp->full_filepath = _instance->full_filepath; + _tmp->base_module_path = _instance->base_module_path; + _tmp->include_functions = _instance->include_functions; + _tmp->include_filenames = _instance->include_filenames; + _tmp->exclude_functions = _instance->exclude_functions; + _tmp->exclude_filenames = _instance->exclude_filenames; + _tmp->verbose = _instance->verbose; + return _tmp; + }(); + return *_tl_instance; +} +// +int32_t +get_depth(PyFrameObject* frame) +{ + return (frame->f_back) ? (get_depth(frame->f_back) + 1) : 0; +} +// +void +profiler_function(py::object pframe, const char* swhat, py::object arg) +{ + static thread_local auto& _config = get_config(); + static thread_local auto _disable = false; + + if(_disable) return; + + _disable = true; + tim::scope::destructor _dtor{ []() { _disable= false; } }; + (void) _dtor; + + if(pframe.is_none() || pframe.ptr() == nullptr) return; + + static auto _omnitrace_path = _config.base_module_path; + + auto* frame = reinterpret_cast(pframe.ptr()); + + int what = (strcmp(swhat, "call") == 0) ? PyTrace_CALL + : (strcmp(swhat, "c_call") == 0) ? PyTrace_C_CALL + : (strcmp(swhat, "return") == 0) ? PyTrace_RETURN + : (strcmp(swhat, "c_return") == 0) ? PyTrace_C_RETURN + : -1; + + // only support PyTrace_{CALL,C_CALL,RETURN,C_RETURN} + if(what < 0) + { + if(_config.verbose > 2) + TIMEMORY_PRINT_HERE("%s :: %s", + "Ignoring what != {CALL,C_CALL,RETURN,C_RETURN}", swhat); + return; + } + + // if PyTrace_C_{CALL,RETURN} is not enabled + if(!_config.trace_c && (what == PyTrace_C_CALL || what == PyTrace_C_RETURN)) + { + if(_config.verbose > 2) + TIMEMORY_PRINT_HERE("%s :: %s", "Ignoring C call/return", swhat); + return; + } + + // get the function name + auto _get_funcname = [&]() -> std::string { + return py::cast(frame->f_code->co_name); + }; + + // get the filename + auto _get_filename = [&]() -> std::string { + return py::cast(frame->f_code->co_filename); + }; + + // get the basename of the filename + auto _get_basename = [&](const std::string& _fullpath) { + if(_fullpath.find('/') != std::string::npos) + return _fullpath.substr(_fullpath.find_last_of('/') + 1); + return _fullpath; + }; + + // get the arguments + auto _get_args = [&]() { + auto inspect = py::module::import("inspect"); + try + { + return py::cast( + inspect.attr("formatargvalues")(*inspect.attr("getargvalues")(pframe))); + } catch(py::error_already_set& _exc) + { + TIMEMORY_CONDITIONAL_PRINT_HERE(_config.verbose > 1, "Error! %s", + _exc.what()); + if(!_exc.matches(PyExc_AttributeError)) throw; + } + return std::string{}; + }; + + // get the final label + auto _get_label = [&](auto& _func, auto& _filename, auto& _fullpath) { + auto _bracket = _config.include_filename; + if(_bracket) _func.insert(0, "["); + // append the arguments + if(_config.include_args) _func.append(_get_args()); + if(_bracket) _func.append("]"); + // append the filename + if(_config.include_filename) + { + if(_config.full_filepath) + _func.append(TIMEMORY_JOIN("", '[', std::move(_fullpath))); + else + _func.append(TIMEMORY_JOIN("", '[', std::move(_filename))); + } + // append the line number + if(_config.include_line && _config.include_filename) + _func.append(TIMEMORY_JOIN("", ':', frame->f_lineno, ']')); + else if(_config.include_line) + _func.append(TIMEMORY_JOIN("", ':', frame->f_lineno)); + else if(_config.include_filename) + _func += "]"; + return _func; + }; + + auto _find_matching = [](const strset_t& _expr, const std::string& _name) { + const auto _rconstants = + std::regex_constants::egrep | std::regex_constants::optimize; + for(const auto& itr : _expr) + { + if(std::regex_search(_name, std::regex(itr, _rconstants))) return true; + } + return false; + }; + + auto& _only_funcs = _config.include_functions; + auto& _skip_funcs = _config.exclude_functions; + auto _func = _get_funcname(); + + if(!_only_funcs.empty() && !_find_matching(_only_funcs, _func)) + { + if(_config.verbose > 1) + TIMEMORY_PRINT_HERE("Skipping non-included function: %s", _func.c_str()); + return; + } + + if(_find_matching(_skip_funcs, _func)) + { + if(_config.verbose > 1) + TIMEMORY_PRINT_HERE("Skipping designated function: '%s'", _func.c_str()); + return; + } + + auto& _only_files = _config.include_filenames; + auto& _skip_files = _config.exclude_filenames; + auto _full = _get_filename(); + auto _file = _get_basename(_full); + + if(!_config.include_internal && + strncmp(_full.c_str(), _omnitrace_path.c_str(), _omnitrace_path.length()) == 0) + { + if(_config.verbose > 2) + TIMEMORY_PRINT_HERE("Skipping internal function: %s", _func.c_str()); + return; + } + + if(!_only_files.empty() && !_find_matching(_only_files, _full)) + { + if(_config.verbose > 2) + TIMEMORY_PRINT_HERE("Skipping non-included file: %s", _full.c_str()); + return; + } + + if(_find_matching(_skip_files, _full)) + { + if(_config.verbose > 2) + TIMEMORY_PRINT_HERE("Skipping non-included file: %s", _full.c_str()); + return; + } + + TIMEMORY_CONDITIONAL_PRINT_HERE(_config.verbose > 3, "%8s | %s%s | %s | %s", swhat, + _func.c_str(), _get_args().c_str(), _file.c_str(), + _full.c_str()); + + auto _label = _get_label(_func, _file, _full); + if(_label.empty()) return; + + static thread_local strset_t _labels{}; + const auto& _label_ref = *_labels.emplace(_label).first; + + // get the depth of the frame + // auto _fdepth = get_depth(frame); + static thread_local int32_t _depth_tracker = 0; + auto _fdepth = _depth_tracker; + switch(what) + { + case PyTrace_CALL: + case PyTrace_C_CALL: _fdepth = _depth_tracker++; break; + case PyTrace_RETURN: + case PyTrace_C_RETURN: _fdepth = --_depth_tracker; break; + } + + // start function + auto _profiler_call = [&]() { + auto& _entry = _config.records[_fdepth][_label]; + _entry.emplace_back( + [&_label_ref]() { omnitrace_push_region(_label_ref.c_str()); }, + [&_label_ref]() { omnitrace_pop_region(_label_ref.c_str()); }); + _entry.back().first(); + }; + + // stop function + auto _profiler_return = [&]() { + auto fitr = _config.records.find(_fdepth); + if(fitr == _config.records.end()) return; + auto litr = fitr->second.find(_label); + if(litr == fitr->second.end()) return; + if(litr->second.empty()) return; + litr->second.back().second(); + litr->second.pop_back(); + }; + + // process what + switch(what) + { + case PyTrace_CALL: + case PyTrace_C_CALL: _profiler_call(); break; + case PyTrace_RETURN: + case PyTrace_C_RETURN: _profiler_return(); break; + default: break; + } + + // don't do anything with arg + tim::consume_parameters(arg); +} +// +py::module +generate(py::module& _pymod) +{ + py::module _prof = _pymod.def_submodule("profiler", "Profiling functions"); + + auto _init = []() { + try + { + auto _file = + py::module::import("omnitrace").attr("__file__").cast(); + if(_file.find('/') != std::string::npos) + _file = _file.substr(0, _file.find_last_of('/')); + get_config().base_module_path = _file; + } catch(py::cast_error& e) + { + std::cerr << "[profiler_init]> " << e.what() << std::endl; + } + if(get_config().is_running) return; + get_config().records.clear(); + get_config().base_stack_depth = -1; + get_config().is_running = true; + }; + + auto _fini = []() { + if(!get_config().is_running) return; + get_config().is_running = false; + get_config().base_stack_depth = -1; + get_config().records.clear(); + }; + + _prof.def("profiler_function", &profiler_function, "Profiling function"); + _prof.def("profiler_init", _init, "Initialize the profiler"); + _prof.def("profiler_finalize", _fini, "Finalize the profiler"); + + py::class_ _pyconfig(_prof, "config", "Profiler configuration"); + +#define CONFIGURATION_PROPERTY(NAME, TYPE, DOC, ...) \ + _pyconfig.def_property_static( \ + NAME, [](py::object&&) { return __VA_ARGS__; }, \ + [](py::object&&, TYPE val) { __VA_ARGS__ = val; }, DOC); + + CONFIGURATION_PROPERTY("_is_running", bool, "Profiler is currently running", + get_config().is_running) + CONFIGURATION_PROPERTY("trace_c", bool, "Enable tracing C functions", + get_config().trace_c) + CONFIGURATION_PROPERTY("include_internal", bool, "Include functions within timemory", + get_config().include_internal) + CONFIGURATION_PROPERTY("include_args", bool, "Encode the function arguments", + get_config().include_args) + CONFIGURATION_PROPERTY("include_line", bool, "Encode the function line number", + get_config().include_line) + CONFIGURATION_PROPERTY("include_filename", bool, + "Encode the function filename (see also: full_filepath)", + get_config().include_filename) + CONFIGURATION_PROPERTY("full_filepath", bool, + "Display the full filepath (instead of file basename)", + get_config().full_filepath) + CONFIGURATION_PROPERTY("verbosity", int32_t, "Verbosity of the logging", + get_config().verbose) + + static auto _get_strset = [](const strset_t& _targ) { + auto _out = py::list{}; + for(auto itr : _targ) + _out.append(itr); + return _out; + }; + + static auto _set_strset = [](const py::list& _inp, strset_t& _targ) { + for(const auto& itr : _inp) + _targ.insert(itr.cast()); + }; + +#define CONFIGURATION_PROPERTY_LAMBDA(NAME, DOC, GET, SET) \ + _pyconfig.def_property_static(NAME, GET, SET, DOC); +#define CONFIGURATION_STRSET(NAME, DOC, ...) \ + { \ + auto GET = [](py::object&&) { return _get_strset(__VA_ARGS__); }; \ + auto SET = [](py::object&&, const py::list& val) { \ + _set_strset(val, __VA_ARGS__); \ + }; \ + CONFIGURATION_PROPERTY_LAMBDA(NAME, DOC, GET, SET) \ + } + + CONFIGURATION_STRSET("only_functions", "Function regexes to collect exclusively", + get_config().include_functions) + CONFIGURATION_STRSET("only_filenames", "Filename regexes to collect exclusively", + get_config().include_filenames) + CONFIGURATION_STRSET("skip_functions", "Function regexes to filter out of collection", + get_config().exclude_functions) + CONFIGURATION_STRSET("skip_filenames", "Filename regexes to filter out of collection", + get_config().exclude_filenames) + + return _prof; +} +} // namespace pyprofile +} // namespace pyomnitrace +// +//======================================================================================// diff --git a/source/python/libpyomnitrace.hpp b/source/python/libpyomnitrace.hpp new file mode 100644 index 0000000000..353a0695bc --- /dev/null +++ b/source/python/libpyomnitrace.hpp @@ -0,0 +1,56 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pyomnitrace +{ +namespace py = pybind11; +using namespace std::placeholders; // for _1, _2, _3... +using namespace py::literals; +} // namespace pyomnitrace diff --git a/source/python/omnitrace/CMakeLists.txt b/source/python/omnitrace/CMakeLists.txt new file mode 100644 index 0000000000..d322f3b0da --- /dev/null +++ b/source/python/omnitrace/CMakeLists.txt @@ -0,0 +1,51 @@ +# ######################################################################################## +# +# omnitrace (Python) +# +# ######################################################################################## + +file(GLOB_RECURSE PYTHON_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.py) +foreach(_IN ${PYTHON_FILES}) + string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/python/omnitrace" + _OUT "${_IN}") + configure_file(${_IN} ${_OUT} @ONLY) + install( + FILES ${_OUT} + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/omnitrace + OPTIONAL) +endforeach() + +# ---------------------------------------------------------------------------- +# Console scripts +# +function(OMNITRACE_PYTHON_CONSOLE_SCRIPT SCRIPT_NAME SCRIPT_SUBMODULE) + + configure_file(${PROJECT_SOURCE_DIR}/cmake/Templates/console-script.in + ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} @ONLY) + + if(CMAKE_INSTALL_PYTHONDIR) + install( + PROGRAMS ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} + DESTINATION ${CMAKE_INSTALL_BINDIR} + OPTIONAL) + endif() + + if(OMNITRACE_BUILD_TESTING OR OMNITRACE_BUILD_PYTHON) + add_test( + NAME ${SCRIPT_NAME}-console-script-test + COMMAND ${PROJECT_BINARY_DIR}/bin/${SCRIPT_NAME} --help + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties( + ${SCRIPT_NAME}-console-script-test + PROPERTIES ENVIRONMENT + "PYTHONPATH=${PROJECT_BINARY_DIR}/python:$ENV{PYTHONPATH}") + endif() +endfunction() + +if(NOT PYTHON_EXECUTABLE) + set(PYTHON_EXECUTABLE "python${OMNITRACE_PYTHON_VERSION}") +endif() + +if(OMNITRACE_USE_PYTHON) + omnitrace_python_console_script("omnitrace-python" "omnitrace") +endif() diff --git a/source/python/omnitrace/__init__.py b/source/python/omnitrace/__init__.py new file mode 100644 index 0000000000..5a7c0ac73d --- /dev/null +++ b/source/python/omnitrace/__init__.py @@ -0,0 +1,67 @@ +#!@PYTHON_EXECUTABLE@ +# MIT License +# +# Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import + +__author__ = "AMD Research" +__copyright__ = "Copyright 2022, Advanced Micro Devices, Inc." +__license__ = "MIT" +__version__ = "@PROJECT_VERSION@" +__maintainer__ = "AMD Research" +__status__ = "Development" + +""" +This submodule imports the timemory Python function profiler +""" + +try: + from .profiler import Profiler, FakeProfiler + from .libpyomnitrace.profiler import ( + profiler_function, + profiler_init, + profiler_finalize, + ) + from .libpyomnitrace import initialize + from .libpyomnitrace import finalize + from .libpyomnitrace.profiler import config as Config + + config = Config + profile = Profiler + noprofile = FakeProfiler + + __all__ = [ + "initialize", + "finalize", + "Profiler", + "Config", + "FakeProfiler", + "profiler_function", + "profiler_init", + "profiler_finalize", + "config", + "profile", + "noprofile", + ] + +except Exception as e: + print("{}".format(e)) diff --git a/source/python/omnitrace/__main__.py b/source/python/omnitrace/__main__.py new file mode 100644 index 0000000000..3818b47538 --- /dev/null +++ b/source/python/omnitrace/__main__.py @@ -0,0 +1,362 @@ +#!@PYTHON_EXECUTABLE@ +# MIT License +# +# Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import + +__author__ = "AMD Research" +__copyright__ = "Copyright 2022, Advanced Micro Devices, Inc." +__license__ = "MIT" +__version__ = "@PROJECT_VERSION@" +__maintainer__ = "AMD Research" +__status__ = "Development" + +""" @file __main__.py +Command line execution for profiler +""" + +import os +import sys +import argparse +import traceback + +PY3 = sys.version_info[0] == 3 + + +# Python 3.x compatibility utils: execfile +try: + execfile +except NameError: + # Python 3.x doesn't have 'execfile' builtin + import builtins + + exec_ = getattr(builtins, "exec") + + def execfile(filename, globals=None, locals=None): + with open(filename, "rb") as f: + exec_(compile(f.read(), filename, "exec"), globals, locals) + + +def find_script(script_name): + """Find the script. + + If the input is not a file, then $PATH will be searched. + """ + if os.path.isfile(script_name): + return script_name + path = os.getenv("PATH", os.defpath).split(os.pathsep) + for dir in path: + if dir == "": + continue + fn = os.path.join(dir, script_name) + if os.path.isfile(fn): + return fn + + sys.stderr.write("Could not find script %s\n" % script_name) + raise SystemExit(1) + + +def parse_args(args=None): + """Parse the arguments""" + + if args is None: + args = sys.argv[1:] + + from .libpyomnitrace.profiler import config as _profiler_config + + def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ("yes", "true", "t", "y", "1"): + return True + elif v.lower() in ("no", "false", "f", "n", "0"): + return False + else: + raise argparse.ArgumentTypeError("Boolean value expected.") + + parser = argparse.ArgumentParser(add_help=True) + parser.add_argument( + "-c", + "--config", + default=None, + type=str, + help="Omnitrace configuration file", + ) + parser.add_argument( + "-b", + "--builtin", + action="store_true", + help="Put 'profile' in the builtins. Use 'profile.enable()' and " + "'profile.disable()' in your code to turn it on and off, or " + "'@profile' to decorate a single function, or 'with profile:' " + "to profile a single section of code.", + ) + parser.add_argument( + "-s", + "--setup", + default=None, + help="Code to execute before the code to profile", + ) + parser.add_argument( + "--trace-c", + type=str2bool, + nargs="?", + const=True, + default=_profiler_config.trace_c, + help="Enable profiling C functions", + ) + parser.add_argument( + "-a", + "--include-args", + type=str2bool, + nargs="?", + const=True, + default=_profiler_config.include_args, + help="Encode the argument values", + ) + parser.add_argument( + "-l", + "--include-line", + type=str2bool, + nargs="?", + const=True, + default=_profiler_config.include_line, + help="Encode the function line number", + ) + parser.add_argument( + "-f", + "--include-file", + type=str2bool, + nargs="?", + const=True, + default=_profiler_config.include_filename, + help="Encode the function filename", + ) + parser.add_argument( + "-F", + "--full-filepath", + type=str2bool, + nargs="?", + const=True, + default=_profiler_config.full_filepath, + help="Encode the full function filename (instead of basename)", + ) + parser.add_argument( + "--skip-funcs", + type=str, + nargs="+", + default=_profiler_config.skip_functions, + help="Filter out any entries with these function names", + ) + parser.add_argument( + "--skip-files", + type=str, + nargs="+", + default=_profiler_config.skip_filenames, + help="Filter out any entries from these files", + ) + parser.add_argument( + "--only-funcs", + type=str, + nargs="+", + default=_profiler_config.only_functions, + help="Select only entries with these function names", + ) + parser.add_argument( + "--only-files", + type=str, + nargs="+", + default=_profiler_config.only_filenames, + help="Select only entries from these files", + ) + parser.add_argument( + "-v", + "--verbosity", + type=int, + default=_profiler_config.verbosity, + help="Logging verbosity", + ) + + return parser.parse_args(args) + + +def get_value(env_var, default_value, dtype, arg=None): + if arg is not None: + return dtype(arg) + else: + val = os.environ.get(env_var) + if val is None: + os.environ[env_var] = "{}".format(default_value) + return dtype(default_value) + else: + return dtype(val) + + +def run(prof, cmd): + if len(cmd) == 0: + return + + progname = cmd[0] + sys.path.insert(0, os.path.dirname(progname)) + with open(progname, "rb") as fp: + code = compile(fp.read(), progname, "exec") + + import __main__ + + dict = __main__.__dict__ + print("code: {} {}".format(type(code).__name__, code)) + globs = { + "__file__": progname, + "__name__": "__main__", + "__package__": None, + "__cached__": None, + **dict, + } + + prof.runctx(code, globs, None) + + +def main(): + """Main function""" + + opts = None + argv = None + if "--" in sys.argv: + _idx = sys.argv.index("--") + _argv = sys.argv[(_idx + 1) :] + opts = parse_args(sys.argv[1:_idx]) + argv = _argv + else: + if "-h" in sys.argv or "--help" in sys.argv: + opts = parse_args() + else: + argv = sys.argv[1:] + opts = parse_args([]) + if len(argv) == 0 or not os.path.isfile(argv[0]): + raise RuntimeError( + "Could not determine input script. Use '--' before " + "the script and its arguments to ensure correct parsing. \nE.g. " + "python -m omnitrace -- ./script.py" + ) + + if len(argv) > 1: + if argv[0] == "-m": + argv = argv[1:] + elif argv[0] == "-c": + argv[0] = os.path.basename(sys.executable) + else: + while len(argv) > 1 and argv[0].startswith("-"): + argv = argv[1:] + if os.path.exists(argv[0]): + break + + if argv: + os.environ["OMNITRACE_COMMAND_LINE"] = " ".join(argv) + + if opts.config is not None: + os.environ["OMNITRACE_CONFIG_FILE"] = ":".join( + [os.environ.get("OMNITRACE_CONFIG_FILE", ""), opts.config] + ) + + from .libpyomnitrace import initialize + + if os.path.isfile(argv[0]): + argv[0] = os.path.realpath(argv[0]) + + initialize(argv[0]) + + from .libpyomnitrace.profiler import config as _profiler_config + + _profiler_config.trace_c = opts.trace_c + _profiler_config.include_args = opts.include_args + _profiler_config.include_line = opts.include_line + _profiler_config.include_filename = opts.include_file + _profiler_config.full_filepath = opts.full_filepath + _profiler_config.skip_functions = opts.skip_funcs + _profiler_config.skip_filenames = opts.skip_files + _profiler_config.only_functions = opts.only_funcs + _profiler_config.only_filenames = opts.only_files + _profiler_config.verbosity = opts.verbosity + + print("[omnitrace]> profiling: {}".format(argv)) + + sys.argv[:] = argv + if opts.setup is not None: + # Run some setup code outside of the profiler. This is good for large + # imports. + setup_file = find_script(opts.setup) + __file__ = setup_file + __name__ = "__main__" + # Make sure the script's directory is on sys.path + sys.path.insert(0, os.path.dirname(setup_file)) + ns = locals() + execfile(setup_file, ns, ns) + + from . import Profiler, FakeProfiler + + script_file = find_script(sys.argv[0]) + __file__ = script_file + __name__ = "__main__" + # Make sure the script's directory is on sys.path + sys.path.insert(0, os.path.dirname(script_file)) + + prof = Profiler() + fake = FakeProfiler() + + if PY3: + import builtins + else: + import __builtin__ as builtins + + builtins.__dict__["profile"] = prof + builtins.__dict__["noprofile"] = fake + builtins.__dict__["trace"] = prof + builtins.__dict__["notrace"] = fake + + try: + try: + if not opts.builtin: + prof.start() + execfile_ = execfile + ns = locals() + if opts.builtin: + execfile(script_file, ns, ns) + else: + prof.runctx("execfile_(%r, globals())" % (script_file,), ns, ns) + except (KeyboardInterrupt, SystemExit): + pass + finally: + if not opts.builtin: + prof.stop() + del prof + del fake + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback.print_exception(exc_type, exc_value, exc_traceback, limit=10) + print("Exception - {}".format(e)) + + +if __name__ == "__main__": + main() + from .libpyomnitrace import finalize + + finalize() diff --git a/source/python/omnitrace/profiler.py b/source/python/omnitrace/profiler.py new file mode 100644 index 0000000000..aa6f9a8ce8 --- /dev/null +++ b/source/python/omnitrace/profiler.py @@ -0,0 +1,328 @@ +#!@PYTHON_EXECUTABLE@ +# MIT License +# +# Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import + +__author__ = "AMD Research" +__copyright__ = "Copyright 2022, Advanced Micro Devices, Inc." +__license__ = "MIT" +__version__ = "@PROJECT_VERSION@" +__maintainer__ = "AMD Research" +__status__ = "Development" + +import sys +import threading +from functools import wraps + +from . import libpyomnitrace +from .libpyomnitrace.profiler import ( + profiler_function as _profiler_function, +) +from .libpyomnitrace.profiler import config as _profiler_config +from .libpyomnitrace.profiler import profiler_init as _profiler_init +from .libpyomnitrace.profiler import profiler_finalize as _profiler_fini + + +__all__ = ["profile", "config", "Profiler", "FakeProfiler", "Config"] + + +# +def _default_functor(): + return True + + +# +PY3 = sys.version_info[0] == 3 +PY35 = PY3 and sys.version_info[1] >= 5 + +# exec (from https://bitbucket.org/gutworth/six/): +if PY3: + import builtins + + exec_ = getattr(builtins, "exec") + del builtins +else: + + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + +config = _profiler_config +Config = _profiler_config + + +# +class Profiler: + """Provides decorators and context-manager for the omnitrace profilers""" + + global _default_functor + + # static variable + _conditional_functor = _default_functor + + # ---------------------------------------------------------------------------------- # + # + @staticmethod + def condition(functor): + """Assign a function evaluating whether to enable the profiler""" + Profiler._conditional_functor = functor + + # ---------------------------------------------------------------------------------- # + # + @staticmethod + def is_enabled(): + """Checks whether the profiler is enabled""" + + try: + return Profiler._conditional_functor() + except Exception: + pass + return False + + # ---------------------------------------------------------------------------------- # + # + def __init__(self, **kwargs): + """ """ + + self._original_function = ( + sys.getprofile() if sys.getprofile() != _profiler_function else None + ) + self._unset = 0 + self._use = ( + not _profiler_config._is_running and Profiler.is_enabled() is True + ) + self.debug = kwargs["debug"] if "debug" in kwargs else False + + # ---------------------------------------------------------------------------------- # + # + def __del__(self): + """Make sure the profiler stops""" + + self.stop() + sys.setprofile(self._original_function) + + # ---------------------------------------------------------------------------------- # + # + def configure(self): + """Initialize, configure the bundle, store original profiler function""" + + _profiler_init() + + # store original + if self.debug: + sys.stderr.write("setting profile function...\n") + if sys.getprofile() != _profiler_function: + self._original_function = sys.getprofile() + + if self.debug: + sys.stderr.write("Tracer configured...\n") + + # ---------------------------------------------------------------------------------- # + # + def update(self): + """Updates whether the profiler is already running based on whether the tracer + is not already running, is enabled, and the function is not already set + """ + + self._use = ( + not _profiler_config._is_running + and Profiler.is_enabled() is True + and sys.getprofile() == self._original_function + ) + + # ---------------------------------------------------------------------------------- # + # + def start(self): + """Start the profiler explicitly""" + + self.update() + if self._use: + if self.debug: + sys.stderr.write("Profiler starting...\n") + self.configure() + sys.setprofile(_profiler_function) + threading.setprofile(_profiler_function) + if self.debug: + sys.stderr.write("Profiler started...\n") + + self._unset = self._unset + 1 + return self._unset + + # ---------------------------------------------------------------------------------- # + # + def stop(self): + """Stop the profiler explicitly""" + + self._unset = self._unset - 1 + if self._unset == 0: + if self.debug: + sys.stderr.write("Profiler stopping...\n") + sys.setprofile(self._original_function) + _profiler_fini() + if self.debug: + sys.stderr.write("Profiler stopped...\n") + + return self._unset + + # ---------------------------------------------------------------------------------- # + # + def __call__(self, func): + """Decorator""" + + @wraps(func) + def function_wrapper(*args, **kwargs): + # store whether this tracer started + self.start() + # execute the wrapped function + result = func(*args, **kwargs) + # unset the profiler if this wrapper set it + self.stop() + # return the result of the wrapped function + return result + + return function_wrapper + + # ---------------------------------------------------------------------------------- # + # + def __enter__(self, *args, **kwargs): + """Context manager start function""" + + self.start() + + # ---------------------------------------------------------------------------------- # + # + def __exit__(self, exec_type, exec_value, exec_tb): + """Context manager stop function""" + + self.stop() + + if ( + exec_type is not None + and exec_value is not None + and exec_tb is not None + ): + import traceback + + traceback.print_exception(exec_type, exec_value, exec_tb, limit=5) + + # ---------------------------------------------------------------------------------- # + # + def run(self, cmd): + """Execute and profile a command""" + + import __main__ + + dict = __main__.__dict__ + if isinstance(cmd, str): + return self.runctx(cmd, dict, dict) + else: + return self.runctx(" ".join(cmd), dict, dict) + + # ---------------------------------------------------------------------------------- # + # + def runctx(self, cmd, globals, locals): + """Profile a context""" + + try: + self.start() + exec_(cmd, globals, locals) + finally: + self.stop() + + return self + + # ---------------------------------------------------------------------------------- # + # + def runcall(self, func, *args, **kw): + """Profile a single function call""" + + try: + self.start() + return func(*args, **kw) + finally: + self.stop() + + +profile = Profiler + + +class FakeProfiler: + """Provides dummy decorators and context-manager for the omnitrace profiler""" + + # ---------------------------------------------------------------------------------- # + # + @staticmethod + def condition(functor): + pass + + # ---------------------------------------------------------------------------------- # + # + @staticmethod + def is_enabled(): + return False + + # ---------------------------------------------------------------------------------- # + # + def __init__(self, *args, **kwargs): + """ """ + pass + + # ---------------------------------------------------------------------------------- # + # + def __call__(self, func): + """Decorator""" + + @wraps(func) + def function_wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return function_wrapper + + # ---------------------------------------------------------------------------------- # + # + def __enter__(self, *args, **kwargs): + """Context manager begin""" + pass + + # ---------------------------------------------------------------------------------- # + # + def __exit__(self, exec_type, exec_value, exec_tb): + """Context manager end""" + + import traceback + + if ( + exec_type is not None + and exec_value is not None + and exec_tb is not None + ): + traceback.print_exception(exec_type, exec_value, exec_tb, limit=5) diff --git a/source/python/pyproject.toml b/source/python/pyproject.toml new file mode 100644 index 0000000000..12ae7d7d63 --- /dev/null +++ b/source/python/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = [ + "setuptools >= 40.0.4", + "setuptools_scm >= 2.0.0", + "wheel >= 0.29.0", +] +build-backend = 'setuptools.build_meta' + +[tool.black] +line-length = 80 +target-version = ['py38'] +include = '\.py' diff --git a/source/python/setup.cfg.in b/source/python/setup.cfg.in new file mode 100644 index 0000000000..c66294de91 --- /dev/null +++ b/source/python/setup.cfg.in @@ -0,0 +1,36 @@ +[metadata] +name = @PROJECT_NAME@ +url = @PROJECT_HOMEPAGE_URL@ +download_url = @PROJECT_HOMEPAGE_URL@.git +maintainer = AMD Research +license = MIT +description = @PROJECT_DESCRIPTION@ +keywords = + performance + profiling + sampling + hardware counters + timing + memory + gpu + hip + rocm +classifiers = + Development Status :: 4 - Beta + Environment :: GPU + Intended Audience :: Developers + Intended Audience :: Science/Research + Natural Language :: English + License :: OSI Approved :: MIT License + Operating System :: POSIX :: Linux + Operating System :: Unix + Programming Language :: C++ + Programming Language :: Python :: 3 + Programming Language :: Python :: @OMNITRACE_PYTHON_VERSION@ + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Utilities + +[options] +packages = @PROJECT_NAME@ +zip_safe = false +include_package_data = false diff --git a/source/python/setup.py.in b/source/python/setup.py.in new file mode 100644 index 0000000000..da0b69f05a --- /dev/null +++ b/source/python/setup.py.in @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name="@PROJECT_NAME@", + version="@PROJECT_VERSION@", + description="@PROJECT_DESCRIPTION@", + author="AMD Research", + url="@PROJECT_HOMEPAGE_URL@", + packages=["@PROJECT_NAME@"], + python_requires="=@OMNITRACE_PYTHON_VERSION@", +) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9d60d84823..2e56c40fef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -236,6 +236,77 @@ endfunction() # -------------------------------------------------------------------------------------- # +function(OMNITRACE_ADD_PYTHON_TEST) + if(NOT OMNITRACE_USE_PYTHON) + return() + endif() + + cmake_parse_arguments( + TEST + "STANDALONE" # options + "NAME;FILE;TIMEOUT" # single value args + "PROFILE_ARGS;RUN_ARGS;ENVIRONMENT;LABELS;PROPERTIES" # multiple value args + ${ARGN}) + + if(NOT TEST_TIMEOUT) + set(TEST_TIMEOUT 120) + endif() + + if(NOT DEFINED TEST_ENVIRONMENT OR "${TEST_ENVIRONMENT}" STREQUAL "") + set(TEST_ENVIRONMENT "${_test_environment}") + endif() + + list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON") + list(APPEND TEST_ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}") + list(APPEND TEST_LABELS "python") + + get_filename_component(_TEST_FILE "${TEST_FILE}" NAME) + configure_file(${TEST_FILE} ${CMAKE_BINARY_DIR}/${_TEST_FILE} @ONLY) + + if(NOT TEST_STANDALONE) + + endif() + + if(TEST_STANDALONE) + add_test( + NAME ${TEST_NAME}-run + COMMAND ${PYTHON_EXECUTABLE} ${_TEST_FILE} ${TEST_RUN_ARGS} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + else() + add_test( + NAME ${TEST_NAME}-run + COMMAND ${PYTHON_EXECUTABLE} -m omnitrace ${TEST_PROFILE_ARGS} -- + ${_TEST_FILE} ${TEST_RUN_ARGS} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + endif() + + foreach(_TEST run) + string(REGEX REPLACE "-run(-|/)" "\\1" _prefix "${TEST_NAME}-${_TEST}/") + set(_environ "${TEST_ENVIRONMENT}") + set(_labels "${_TEST}") + set(_timeout ${TEST_TIMEOUT}) + list(APPEND _environ "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output" + "OMNITRACE_OUTPUT_PREFIX=${_prefix}") + string(REPLACE "-run" "" _labels "${_TEST}") + string(REPLACE "-sampling" ";sampling" _labels "${_labels}") + set(_props) + if("${_TEST}" MATCHES "run|baseline") + set(_props ${TEST_PROPERTIES}) + if(NOT "RUN_SERIAL" IN_LIST _props) + list(APPEND _props RUN_SERIAL ON) + endif() + endif() + if(TEST ${TEST_NAME}-${_TEST}) + set_tests_properties( + ${TEST_NAME}-${_TEST} + PROPERTIES ENVIRONMENT "${_environ}" TIMEOUT ${_timeout} LABELS + "${_labels};${TEST_LABELS}" ${_props}) + endif() + endforeach() +endfunction() + +# -------------------------------------------------------------------------------------- # + omnitrace_add_test( NAME transpose TARGET transpose @@ -245,26 +316,6 @@ omnitrace_add_test( RUNTIME_ARGS -e -v 1 --label file line return args ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=ON") -omnitrace_add_test( - NAME transpose-no-save-fpr - TARGET transpose - MPI ${TRANSPOSE_USE_MPI} - NUM_PROCS ${NUM_PROCS} - REWRITE_ARGS -e -v 2 --dyninst-options DelayedParsing TypeChecking - RUNTIME_ARGS - -e - -v - 1 - --label - file - line - return - args - --dyninst-options - DelayedParsing - TypeChecking - ENVIRONMENT "${_fast_environment}") - omnitrace_add_test( NAME parallel-overhead TARGET parallel-overhead @@ -313,27 +364,6 @@ omnitrace_add_test( RUN_ARGS 10 ${NUM_THREADS} 1000 ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF") -omnitrace_add_test( - NAME parallel-overhead-no-save-fpr - TARGET parallel-overhead - REWRITE_ARGS -e -v 2 --min-address-range-loop=32 --dyninst-options DelayedParsing - TypeChecking - RUNTIME_ARGS - -e - -v - 1 - --min-address-range-loop=32 - --label - file - line - return - args - --dyninst-options - DelayedParsing - TypeChecking - RUN_ARGS 10 ${NUM_THREADS} 1000 - ENVIRONMENT "${_fast_environment}") - omnitrace_add_test( NAME lulesh TARGET lulesh @@ -434,3 +464,24 @@ omnitrace_add_test( REWRITE_TIMEOUT 180 RUNTIME_TIMEOUT 360 ENVIRONMENT "${_ompt_environment}") + +omnitrace_add_python_test( + NAME python-external + FILE ${CMAKE_SOURCE_DIR}/examples/python/external.py + PROFILE_ARGS -l -f + RUN_ARGS 10 + ENVIRONMENT "${_base_environment};OMNITRACE_USE_PID=OFF") + +omnitrace_add_python_test( + NAME python-builtin + FILE ${CMAKE_SOURCE_DIR}/examples/python/builtin.py + PROFILE_ARGS -b -l -f + RUN_ARGS 10 + ENVIRONMENT "${_base_environment};OMNITRACE_USE_PID=OFF") + +omnitrace_add_python_test( + STANDALONE + NAME python-source + FILE ${CMAKE_SOURCE_DIR}/examples/python/source.py + RUN_ARGS 5 + ENVIRONMENT "${_base_environment};OMNITRACE_USE_PID=OFF")