From e2c30bd438bfc3f9574abe5639e2c87edf6db2c2 Mon Sep 17 00:00:00 2001 From: Gopesh Bhardwaj Date: Tue, 9 Apr 2024 17:54:40 +0530 Subject: [PATCH] adding pandas and pytest to rquirements.txt (#748) * adding pandas and pytest to rquirements.txt * setting up requrements.txt * Update requirements - formatting packages - remove packages not directly used by rocprofiler-sdk * Update cmake formatting, linting, and options - if BUILD_CI -> force BUILD_DEVELOPER and BUILD_WERROR - support python installed clang-format and python installed clang-tidy * Update build.sh - split into install-deps.sh and install-apt-deps.sh * Improve code coverage --------- Co-authored-by: Jonathan R. Madsen [ROCm/rocprofiler-sdk commit: e2d8ccad4b4cf84e77a6b286bd2d27ef28b1a920] --- .../.github/workflows/ci_pc_sampling.yml | 2 - .../workflows/continuous_integration.yml | 4 -- .../.github/workflows/rerun.yml | 10 ----- .../cmake/rocprofiler_build_settings.cmake | 4 ++ .../cmake/rocprofiler_formatting.cmake | 43 +++++++++++++++++-- .../cmake/rocprofiler_linting.cmake | 37 +++++++++++++--- .../cmake/rocprofiler_options.cmake | 16 +++---- projects/rocprofiler-sdk/requirements.txt | 9 ++++ .../rocprofiler-sdk/source/docs/rocprofv3.md | 6 +++ .../lib/rocprofiler-sdk-tool/CMakeLists.txt | 5 --- .../source/lib/rocprofiler-sdk-tool/tool.cpp | 7 ++- .../lib/rocprofiler-sdk/registration.cpp | 13 ++++++ .../rocprofiler-sdk/source/scripts/build.sh | 21 ++++----- .../source/scripts/install-apt-deps.sh | 7 +++ .../source/scripts/install-deps.sh | 8 ++++ 15 files changed, 137 insertions(+), 55 deletions(-) create mode 100755 projects/rocprofiler-sdk/source/scripts/install-apt-deps.sh create mode 100755 projects/rocprofiler-sdk/source/scripts/install-deps.sh diff --git a/projects/rocprofiler-sdk/.github/workflows/ci_pc_sampling.yml b/projects/rocprofiler-sdk/.github/workflows/ci_pc_sampling.yml index 5ae7b9adbb..d83f206f33 100644 --- a/projects/rocprofiler-sdk/.github/workflows/ci_pc_sampling.yml +++ b/projects/rocprofiler-sdk/.github/workflows/ci_pc_sampling.yml @@ -58,8 +58,6 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 python3 -m pip install -r requirements.txt - python3 -m pip install pytest - python3 -m pip install 'cmake>=3.22.0' - name: List Files shell: bash diff --git a/projects/rocprofiler-sdk/.github/workflows/continuous_integration.yml b/projects/rocprofiler-sdk/.github/workflows/continuous_integration.yml index b1408c21bd..8018340c8b 100644 --- a/projects/rocprofiler-sdk/.github/workflows/continuous_integration.yml +++ b/projects/rocprofiler-sdk/.github/workflows/continuous_integration.yml @@ -54,8 +54,6 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 python3 -m pip install -r requirements.txt - python3 -m pip install pytest - python3 -m pip install 'cmake>=3.22.0' - name: List Files shell: bash @@ -202,7 +200,6 @@ jobs: apt-get update apt-get install -y cmake python3-pip gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig libdw-dev python3 -m pip install -r requirements.txt - python3 -m pip install pytest pycobertura - name: Configure, Build, and Test (Total Code Coverage) timeout-minutes: 30 @@ -385,7 +382,6 @@ jobs: apt-get update apt-get install -y cmake python3-pip libasan8 libtsan2 software-properties-common python3 -m pip install -r requirements.txt - python3 -m pip install pytest add-apt-repository ppa:ubuntu-toolchain-r/test apt-get update apt-get install -y g++-13 libdw-dev diff --git a/projects/rocprofiler-sdk/.github/workflows/rerun.yml b/projects/rocprofiler-sdk/.github/workflows/rerun.yml index abf514a3c6..0c47a0b153 100644 --- a/projects/rocprofiler-sdk/.github/workflows/rerun.yml +++ b/projects/rocprofiler-sdk/.github/workflows/rerun.yml @@ -60,8 +60,6 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 python3 -m pip install -r requirements.txt - python3 -m pip install pytest - python3 -m pip install 'cmake>=3.22.0' - name: List Files shell: bash @@ -161,8 +159,6 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 python3 -m pip install -r requirements.txt - python3 -m pip install pytest - python3 -m pip install 'cmake>=3.22.0' - name: List Files shell: bash @@ -262,8 +258,6 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 python3 -m pip install -r requirements.txt - python3 -m pip install pytest - python3 -m pip install 'cmake>=3.22.0' - name: List Files shell: bash @@ -363,8 +357,6 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 python3 -m pip install -r requirements.txt - python3 -m pip install pytest - python3 -m pip install 'cmake>=3.22.0' - name: List Files shell: bash @@ -464,8 +456,6 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 python3 -m pip install -r requirements.txt - python3 -m pip install pytest - python3 -m pip install 'cmake>=3.22.0' - name: List Files shell: bash diff --git a/projects/rocprofiler-sdk/cmake/rocprofiler_build_settings.cmake b/projects/rocprofiler-sdk/cmake/rocprofiler_build_settings.cmake index d10644e041..8b6b16716a 100644 --- a/projects/rocprofiler-sdk/cmake/rocprofiler_build_settings.cmake +++ b/projects/rocprofiler-sdk/cmake/rocprofiler_build_settings.cmake @@ -18,6 +18,10 @@ if(ROCPROFILER_BUILD_CI) INTERFACE ROCPROFILER_CI) endif() +if(ROCPROFILER_BUILD_CODECOV) + target_link_libraries(rocprofiler-build-flags INTERFACE gcov) +endif() + # ----------------------------------------------------------------------------------------# # dynamic linking and runtime libraries # diff --git a/projects/rocprofiler-sdk/cmake/rocprofiler_formatting.cmake b/projects/rocprofiler-sdk/cmake/rocprofiler_formatting.cmake index fa68f575f4..9ee635296c 100644 --- a/projects/rocprofiler-sdk/cmake/rocprofiler_formatting.cmake +++ b/projects/rocprofiler-sdk/cmake/rocprofiler_formatting.cmake @@ -13,9 +13,46 @@ include_guard(DIRECTORY) -find_program(ROCPROFILER_CLANG_FORMAT_EXE NAMES clang-format-11 clang-format-mp-11) -find_program(ROCPROFILER_CMAKE_FORMAT_EXE NAMES cmake-format) -find_program(ROCPROFILER_BLACK_FORMAT_EXE NAMES black) +if(ROCPROFILER_BUILD_DEVELOPER) + set(_FMT_REQUIRED REQUIRED) +else() + set(_FMT_REQUIRED) +endif() + +if(NOT ROCPROFILE_CLANG_FORMAT_EXE AND EXISTS $ENV{HOME}/.local/bin/clang-format) + execute_process( + COMMAND $ENV{HOME}/.local/bin/clang-format --version + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + OUTPUT_VARIABLE _CLANG_FMT_OUT + RESULT_VARIABLE _CLANG_FMT_RET + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(_CLANG_FMT_RET EQUAL 0) + if("${_CLANG_FMT_OUT}" MATCHES "version 11\\.([0-9]+)\\.([0-9]+)") + set(ROCPROFILER_CLANG_FORMAT_EXE + "$ENV{HOME}/.local/bin/clang-format" + CACHE FILEPATH "clang-format exe") + endif() + endif() +endif() + +find_program( + ROCPROFILER_CLANG_FORMAT_EXE ${_FMT_REQUIRED} + NAMES clang-format-11 clang-format-mp-11 clang-format + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) +find_program( + ROCPROFILER_CMAKE_FORMAT_EXE ${_FMT_REQUIRED} + NAMES cmake-format + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) +find_program( + ROCPROFILER_BLACK_FORMAT_EXE ${_FMT_REQUIRED} + NAMES black + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) add_custom_target(format-rocprofiler) if(NOT TARGET format) diff --git a/projects/rocprofiler-sdk/cmake/rocprofiler_linting.cmake b/projects/rocprofiler-sdk/cmake/rocprofiler_linting.cmake index 9e0103722c..777c3e79a5 100644 --- a/projects/rocprofiler-sdk/cmake/rocprofiler_linting.cmake +++ b/projects/rocprofiler-sdk/cmake/rocprofiler_linting.cmake @@ -6,8 +6,31 @@ include_guard(GLOBAL) # # ----------------------------------------------------------------------------------------# +if(ROCPROFILER_BUILD_DEVELOPER) + set(_TIDY_REQUIRED REQUIRED) +else() + set(_TIDY_REQUIRED) +endif() + +if(NOT ROCPROFILE_CLANG_TIDY_EXE AND EXISTS $ENV{HOME}/.local/bin/clang-tidy) + execute_process( + COMMAND $ENV{HOME}/.local/bin/clang-tidy --version + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + OUTPUT_VARIABLE _CLANG_TIDY_OUT + RESULT_VARIABLE _CLANG_TIDY_RET + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + + if(_CLANG_TIDY_RET EQUAL 0) + if("${_CLANG_TIDY_OUT}" MATCHES "version 1[5-9]\\.([0-9]+)\\.([0-9]+)") + set(ROCPROFILER_CLANG_TIDY_EXE + "$ENV{HOME}/.local/bin/clang-tidy" + CACHE FILEPATH "clang-tidy exe") + endif() + endif() +endif() + find_program( - ROCPROFILER_CLANG_TIDY_COMMAND + ROCPROFILER_CLANG_TIDY_EXE ${_TIDY_REQUIRED} NAMES clang-tidy-18 clang-tidy-17 clang-tidy-16 @@ -16,22 +39,24 @@ find_program( clang-tidy-13 clang-tidy-12 clang-tidy-11 - clang-tidy) + clang-tidy + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) macro(ROCPROFILER_ACTIVATE_CLANG_TIDY) if(ROCPROFILER_ENABLE_CLANG_TIDY) - if(NOT ROCPROFILER_CLANG_TIDY_COMMAND) + if(NOT ROCPROFILER_CLANG_TIDY_EXE) message( FATAL_ERROR "ROCPROFILER_ENABLE_CLANG_TIDY is ON but clang-tidy is not found!") endif() - rocprofiler_add_feature(ROCPROFILER_CLANG_TIDY_COMMAND + rocprofiler_add_feature(ROCPROFILER_CLANG_TIDY_EXE "path to clang-tidy executable") set(CMAKE_CXX_CLANG_TIDY - ${ROCPROFILER_CLANG_TIDY_COMMAND} - -header-filter=${PROJECT_SOURCE_DIR}/source/.* + ${ROCPROFILER_CLANG_TIDY_EXE} -header-filter=${PROJECT_SOURCE_DIR}/source/.* --warnings-as-errors=*,-misc-header-include-cycle) # Create a preprocessor definition that depends on .clang-tidy content so the diff --git a/projects/rocprofiler-sdk/cmake/rocprofiler_options.cmake b/projects/rocprofiler-sdk/cmake/rocprofiler_options.cmake index 9a08845052..b49d9130a2 100644 --- a/projects/rocprofiler-sdk/cmake/rocprofiler_options.cmake +++ b/projects/rocprofiler-sdk/cmake/rocprofiler_options.cmake @@ -80,14 +80,14 @@ rocprofiler_add_option( "Regenerate the counter parser (requires bison and flex)" OFF ADVANCED) # In the future, we will do this even with clang-tidy enabled -if(ROCPROFILER_BUILD_CI AND NOT ROCPROFILER_BUILD_WERROR) - message(STATUS "Forcing ROCPROFILER_BUILD_WERROR=ON because ROCPROFILER_BUILD_CI=ON") - set(ROCPROFILER_BUILD_WERROR - ON - CACHE BOOL - "Any compiler warnings are errors (forced due ROCPROFILER_BUILD_CI=ON)" - FORCE) -endif() +foreach(_OPT ROCPROFILER_BUILD_DEVELOPER ROCPROFILER_BUILD_WERROR) + if(ROCPROFILER_BUILD_CI AND NOT ${_OPT}) + message(AUTHOR_WARNING "Forcing ${_OPT}=ON because ROCPROFILER_BUILD_CI=ON") + set(${_OPT} + ON + CACHE BOOL "forced due ROCPROFILER_BUILD_CI=ON" FORCE) + endif() +endforeach() set(ROCPROFILER_BUILD_TYPES "Release" "RelWithDebInfo" "Debug" "MinSizeRel" "Coverage") diff --git a/projects/rocprofiler-sdk/requirements.txt b/projects/rocprofiler-sdk/requirements.txt index e69de29bb2..169854cc50 100644 --- a/projects/rocprofiler-sdk/requirements.txt +++ b/projects/rocprofiler-sdk/requirements.txt @@ -0,0 +1,9 @@ +black +clang-format>=11.0.0,<12.0.0 +clang-tidy>=15.0.0,<18.0.0 +cmake>=3.21.0 +cmake-format +pandas +pycobertura +pytest +pyyaml diff --git a/projects/rocprofiler-sdk/source/docs/rocprofv3.md b/projects/rocprofiler-sdk/source/docs/rocprofv3.md index fbb2c5f14d..f38e0e8d4d 100644 --- a/projects/rocprofiler-sdk/source/docs/rocprofv3.md +++ b/projects/rocprofiler-sdk/source/docs/rocprofv3.md @@ -437,3 +437,9 @@ To run the built samples, `cd` into the `` mentioned in the bui ```bash ctest -V ``` + +**Note:** Running few of these tests will require pandas and pytest to be installed first. + +```bash +/usr/local/bin/python -m pip install -r requirements.txt +``` diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/CMakeLists.txt b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/CMakeLists.txt index 315a4aa2da..0f75238a29 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/CMakeLists.txt +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/CMakeLists.txt @@ -20,11 +20,6 @@ target_link_libraries( rocprofiler::rocprofiler-build-flags rocprofiler::rocprofiler-memcheck rocprofiler::rocprofiler-common-library) -if(ROCPROFILER_BUILD_CODECOV) - target_compile_definitions(rocprofiler-sdk-tool PRIVATE ROCPROFILER_CODECOV=1) - target_link_libraries(rocprofiler-sdk-tool PRIVATE gcov) -endif() - set_target_properties( rocprofiler-sdk-tool PROPERTIES LIBRARY_OUTPUT_DIRECTORY diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/tool.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/tool.cpp index 5812ca3836..b64dbf46bd 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/tool.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/tool.cpp @@ -56,7 +56,7 @@ #include #include -#if defined(ROCPROFILER_CODECOV) && ROCPROFILER_CODECOV > 0 +#if defined(CODECOV) && CODECOV > 0 extern "C" { extern void __gcov_dump(void); @@ -1731,7 +1731,7 @@ generate_output(buffer_type_t buffer_type) } // namespace void -tool_fini(void* tool_data) +tool_fini(void* /*tool_data*/) { client_identifier = nullptr; client_finalizer = nullptr; @@ -1802,8 +1802,7 @@ tool_fini(void* tool_data) destructors = nullptr; } - (void) (tool_data); -#if defined(ROCPROFILER_CODECOV) && ROCPROFILER_CODECOV > 0 +#if defined(CODECOV) && CODECOV > 0 __gcov_dump(); #endif } diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp index ebf558d692..f9b1d0ebaa 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp @@ -73,6 +73,11 @@ extern "C" { extern rocprofiler_tool_configure_result_t* rocprofiler_configure(uint32_t, const char*, uint32_t, rocprofiler_client_id_t*); + +#if defined(CODECOV) && CODECOV > 0 +extern void +__gcov_dump(void); +#endif } namespace rocprofiler @@ -569,6 +574,10 @@ initialize() void finalize() { +#if defined(CODECOV) && CODECOV > 0 + if(get_fini_status() > 0) __gcov_dump(); +#endif + if(get_fini_status() != 0) { ROCP_INFO << "ignoring finalization request (value=" << get_fini_status() << ")"; @@ -599,6 +608,10 @@ finalize() internal_threading::finalize(); set_fini_status(1); }); + +#if defined(CODECOV) && CODECOV > 0 + __gcov_dump(); +#endif } } // namespace registration } // namespace rocprofiler diff --git a/projects/rocprofiler-sdk/source/scripts/build.sh b/projects/rocprofiler-sdk/source/scripts/build.sh index 2d5621fdc2..94db892c2d 100755 --- a/projects/rocprofiler-sdk/source/scripts/build.sh +++ b/projects/rocprofiler-sdk/source/scripts/build.sh @@ -1,20 +1,15 @@ #!/bin/bash -sudo apt-get update -sudo apt-get install -y cmake clang-tidy-15 g++-11 g++-12 python3-pip libdw-dev -sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 10 -sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 -sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 -#python3 -m pip install -r requirements.txt -python3 -m pip install pytest pandas pyyaml -python3 -m pip install 'cmake>=3.22.0' +SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}) && pwd) +ROCPROFILER_SDK_PATH="$(cd ${SCRIPT_PATH}/../.. && pwd)" -ROCPROFILER_SDK_PATH="$(pwd)/$(dirname ${BASH_SOURCE[0]})/../.." - -cd ${ROCPROFILER_SDK_PATH} +${SCRIPT_PATH}/install-deps.sh echo -e "Redirecting to location: $ROCPROFILER_SDK_PATH" +cd ${ROCPROFILER_SDK_PATH} -cmake -B build -DROCPROFILER_BUILD_CI=ON -DROCPROFILER_BUILD_TESTS=ON -DROCPROFILER_BUILD_SAMPLES=ON -DROCPROFILER_ENABLE_CLANG_TIDY=ON $* +echo -e "Configuring rocprofiler-sdk: ${ROCPROFILER_SDK_PATH}/build" +cmake -B build -DROCPROFILER_BUILD_{CI,TESTS,SAMPLES}=ON -DROCPROFILER_ENABLE_CLANG_TIDY=ON "${@}" + +echo -e "Building rocprofiler-sdk: ${ROCPROFILER_SDK_PATH}/build" cmake --build build --target all --parallel $(nproc) -#cd -- diff --git a/projects/rocprofiler-sdk/source/scripts/install-apt-deps.sh b/projects/rocprofiler-sdk/source/scripts/install-apt-deps.sh new file mode 100755 index 0000000000..e768a1d402 --- /dev/null +++ b/projects/rocprofiler-sdk/source/scripts/install-apt-deps.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +apt-get update +apt-get install -y cmake clang-tidy-15 g++-11 g++-12 python3-pip libdw-dev +update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 10 +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11 +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12 diff --git a/projects/rocprofiler-sdk/source/scripts/install-deps.sh b/projects/rocprofiler-sdk/source/scripts/install-deps.sh new file mode 100755 index 0000000000..8656141d43 --- /dev/null +++ b/projects/rocprofiler-sdk/source/scripts/install-deps.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +ROCPROFILER_SDK_PATH=$(cd $(dirname ${BASH_SOURCE[0]})/../.. && pwd) + +# echo "ROCPROFILER_SDK_PATH: ${ROCPROFILER_SDK_PATH}" + +echo -e "Installing dependencies from: ${ROCPROFILER_SDK_PATH}/requirements.txt" +python3 -m pip install --user -r ${ROCPROFILER_SDK_PATH}/requirements.txt