adding pandas and pytest to rquirements.txt (#748)

* adding pandas and pytest to rquirements.txt * setting up requrements.txt * Update requirements - formatting packages - remove packages not directly used by rocprofiler-sdk * Update cmake formatting, linting, and options - if BUILD_CI -> force BUILD_DEVELOPER and BUILD_WERROR - support python installed clang-format and python installed clang-tidy * Update build.sh - split into install-deps.sh and install-apt-deps.sh * Improve code coverage --------- Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com> [ROCm/rocprofiler-sdk commit: e2d8ccad4b]
2024-04-09 17:54:40 +05:30
commit e2c30bd438
@@ -58,8 +58,6 @@ jobs:
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
-        python3 -m pip install 'cmake>=3.22.0'

    - name: List Files
      shell: bash
@@ -54,8 +54,6 @@ jobs:
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
-        python3 -m pip install 'cmake>=3.22.0'

    - name: List Files
      shell: bash
@@ -202,7 +200,6 @@ jobs:
        apt-get update
        apt-get install -y cmake python3-pip gcovr wkhtmltopdf xvfb xfonts-base xfonts-75dpi xfonts-100dpi xfonts-utils xfonts-encodings libfontconfig libdw-dev
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest pycobertura

    - name: Configure, Build, and Test (Total Code Coverage)
      timeout-minutes: 30
@@ -385,7 +382,6 @@ jobs:
        apt-get update
        apt-get install -y cmake python3-pip libasan8 libtsan2 software-properties-common
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
        add-apt-repository ppa:ubuntu-toolchain-r/test
        apt-get update
        apt-get install -y g++-13 libdw-dev
@@ -60,8 +60,6 @@ jobs:
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
-        python3 -m pip install 'cmake>=3.22.0'

    - name: List Files
      shell: bash
@@ -161,8 +159,6 @@ jobs:
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
-        python3 -m pip install 'cmake>=3.22.0'

    - name: List Files
      shell: bash
@@ -262,8 +258,6 @@ jobs:
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
-        python3 -m pip install 'cmake>=3.22.0'

    - name: List Files
      shell: bash
@@ -363,8 +357,6 @@ jobs:
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
-        python3 -m pip install 'cmake>=3.22.0'

    - name: List Files
      shell: bash
@@ -464,8 +456,6 @@ jobs:
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
        python3 -m pip install -r requirements.txt
-        python3 -m pip install pytest
-        python3 -m pip install 'cmake>=3.22.0'

    - name: List Files
      shell: bash
@@ -18,6 +18,10 @@ if(ROCPROFILER_BUILD_CI)
                                           INTERFACE ROCPROFILER_CI)
 endif()

+if(ROCPROFILER_BUILD_CODECOV)
+    target_link_libraries(rocprofiler-build-flags INTERFACE gcov)
+endif()
+
 # ----------------------------------------------------------------------------------------#
 # dynamic linking and runtime libraries
 #
@@ -13,9 +13,46 @@

 include_guard(DIRECTORY)

-find_program(ROCPROFILER_CLANG_FORMAT_EXE NAMES clang-format-11 clang-format-mp-11)
-find_program(ROCPROFILER_CMAKE_FORMAT_EXE NAMES cmake-format)
-find_program(ROCPROFILER_BLACK_FORMAT_EXE NAMES black)
+if(ROCPROFILER_BUILD_DEVELOPER)
+    set(_FMT_REQUIRED REQUIRED)
+else()
+    set(_FMT_REQUIRED)
+endif()
+
+if(NOT ROCPROFILE_CLANG_FORMAT_EXE AND EXISTS $ENV{HOME}/.local/bin/clang-format)
+    execute_process(
+        COMMAND $ENV{HOME}/.local/bin/clang-format --version
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+        OUTPUT_VARIABLE _CLANG_FMT_OUT
+        RESULT_VARIABLE _CLANG_FMT_RET
+        OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+    if(_CLANG_FMT_RET EQUAL 0)
+        if("${_CLANG_FMT_OUT}" MATCHES "version 11\\.([0-9]+)\\.([0-9]+)")
+            set(ROCPROFILER_CLANG_FORMAT_EXE
+                "$ENV{HOME}/.local/bin/clang-format"
+                CACHE FILEPATH "clang-format exe")
+        endif()
+    endif()
+endif()
+
+find_program(
+    ROCPROFILER_CLANG_FORMAT_EXE ${_FMT_REQUIRED}
+    NAMES clang-format-11 clang-format-mp-11 clang-format
+    PATHS $ENV{HOME}/.local
+    HINTS $ENV{HOME}/.local
+    PATH_SUFFIXES bin)
+find_program(
+    ROCPROFILER_CMAKE_FORMAT_EXE ${_FMT_REQUIRED}
+    NAMES cmake-format
+    PATHS $ENV{HOME}/.local
+    HINTS $ENV{HOME}/.local
+    PATH_SUFFIXES bin)
+find_program(
+    ROCPROFILER_BLACK_FORMAT_EXE ${_FMT_REQUIRED}
+    NAMES black
+    PATHS $ENV{HOME}/.local
+    HINTS $ENV{HOME}/.local
+    PATH_SUFFIXES bin)

 add_custom_target(format-rocprofiler)
 if(NOT TARGET format)
@@ -6,8 +6,31 @@ include_guard(GLOBAL)
 #
 # ----------------------------------------------------------------------------------------#

+if(ROCPROFILER_BUILD_DEVELOPER)
+    set(_TIDY_REQUIRED REQUIRED)
+else()
+    set(_TIDY_REQUIRED)
+endif()
+
+if(NOT ROCPROFILE_CLANG_TIDY_EXE AND EXISTS $ENV{HOME}/.local/bin/clang-tidy)
+    execute_process(
+        COMMAND $ENV{HOME}/.local/bin/clang-tidy --version
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+        OUTPUT_VARIABLE _CLANG_TIDY_OUT
+        RESULT_VARIABLE _CLANG_TIDY_RET
+        OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+
+    if(_CLANG_TIDY_RET EQUAL 0)
+        if("${_CLANG_TIDY_OUT}" MATCHES "version 1[5-9]\\.([0-9]+)\\.([0-9]+)")
+            set(ROCPROFILER_CLANG_TIDY_EXE
+                "$ENV{HOME}/.local/bin/clang-tidy"
+                CACHE FILEPATH "clang-tidy exe")
+        endif()
+    endif()
+endif()
+
 find_program(
-    ROCPROFILER_CLANG_TIDY_COMMAND
+    ROCPROFILER_CLANG_TIDY_EXE ${_TIDY_REQUIRED}
    NAMES clang-tidy-18
          clang-tidy-17
          clang-tidy-16
@@ -16,22 +39,24 @@ find_program(
          clang-tidy-13
          clang-tidy-12
          clang-tidy-11
-          clang-tidy)
+          clang-tidy
+    PATHS $ENV{HOME}/.local
+    HINTS $ENV{HOME}/.local
+    PATH_SUFFIXES bin)

 macro(ROCPROFILER_ACTIVATE_CLANG_TIDY)
    if(ROCPROFILER_ENABLE_CLANG_TIDY)
-        if(NOT ROCPROFILER_CLANG_TIDY_COMMAND)
+        if(NOT ROCPROFILER_CLANG_TIDY_EXE)
            message(
                FATAL_ERROR
                    "ROCPROFILER_ENABLE_CLANG_TIDY is ON but clang-tidy is not found!")
        endif()

-        rocprofiler_add_feature(ROCPROFILER_CLANG_TIDY_COMMAND
+        rocprofiler_add_feature(ROCPROFILER_CLANG_TIDY_EXE
                                "path to clang-tidy executable")

        set(CMAKE_CXX_CLANG_TIDY
-            ${ROCPROFILER_CLANG_TIDY_COMMAND}
-            -header-filter=${PROJECT_SOURCE_DIR}/source/.*
+            ${ROCPROFILER_CLANG_TIDY_EXE} -header-filter=${PROJECT_SOURCE_DIR}/source/.*
            --warnings-as-errors=*,-misc-header-include-cycle)

        # Create a preprocessor definition that depends on .clang-tidy content so the
@@ -80,14 +80,14 @@ rocprofiler_add_option(
    "Regenerate the counter parser (requires bison and flex)" OFF ADVANCED)

 # In the future, we will do this even with clang-tidy enabled
-if(ROCPROFILER_BUILD_CI AND NOT ROCPROFILER_BUILD_WERROR)
-    message(STATUS "Forcing ROCPROFILER_BUILD_WERROR=ON because ROCPROFILER_BUILD_CI=ON")
-    set(ROCPROFILER_BUILD_WERROR
-        ON
-        CACHE BOOL
-              "Any compiler warnings are errors (forced due ROCPROFILER_BUILD_CI=ON)"
-              FORCE)
-endif()
+foreach(_OPT ROCPROFILER_BUILD_DEVELOPER ROCPROFILER_BUILD_WERROR)
+    if(ROCPROFILER_BUILD_CI AND NOT ${_OPT})
+        message(AUTHOR_WARNING "Forcing ${_OPT}=ON because ROCPROFILER_BUILD_CI=ON")
+        set(${_OPT}
+            ON
+            CACHE BOOL "forced due ROCPROFILER_BUILD_CI=ON" FORCE)
+    endif()
+endforeach()

 set(ROCPROFILER_BUILD_TYPES "Release" "RelWithDebInfo" "Debug" "MinSizeRel" "Coverage")

@@ -0,0 +1,9 @@
+black
+clang-format>=11.0.0,<12.0.0
+clang-tidy>=15.0.0,<18.0.0
+cmake>=3.21.0
+cmake-format
+pandas
+pycobertura
+pytest
+pyyaml
@@ -437,3 +437,9 @@ To run the built samples, `cd` into the `<build directory>` mentioned in the bui
 ```bash
 ctest -V
 ```
+
+**Note:** Running few of these tests will require pandas and pytest to be installed first.
+
+```bash
+/usr/local/bin/python -m pip install -r requirements.txt
+```
@@ -20,11 +20,6 @@ target_link_libraries(
            rocprofiler::rocprofiler-build-flags rocprofiler::rocprofiler-memcheck
            rocprofiler::rocprofiler-common-library)

-if(ROCPROFILER_BUILD_CODECOV)
-    target_compile_definitions(rocprofiler-sdk-tool PRIVATE ROCPROFILER_CODECOV=1)
-    target_link_libraries(rocprofiler-sdk-tool PRIVATE gcov)
-endif()
-
 set_target_properties(
    rocprofiler-sdk-tool
    PROPERTIES LIBRARY_OUTPUT_DIRECTORY
@@ -56,7 +56,7 @@
 #include <unordered_set>
 #include <vector>

-#if defined(ROCPROFILER_CODECOV) && ROCPROFILER_CODECOV > 0
+#if defined(CODECOV) && CODECOV > 0
 extern "C" {
 extern void
 __gcov_dump(void);
@@ -1731,7 +1731,7 @@ generate_output(buffer_type_t buffer_type)
 }  // namespace

 void
-tool_fini(void* tool_data)
+tool_fini(void* /*tool_data*/)
 {
    client_identifier = nullptr;
    client_finalizer  = nullptr;
@@ -1802,8 +1802,7 @@ tool_fini(void* tool_data)
        destructors = nullptr;
    }

-    (void) (tool_data);
-#if defined(ROCPROFILER_CODECOV) && ROCPROFILER_CODECOV > 0
+#if defined(CODECOV) && CODECOV > 0
    __gcov_dump();
 #endif
 }
@@ -73,6 +73,11 @@ extern "C" {

 extern rocprofiler_tool_configure_result_t*
 rocprofiler_configure(uint32_t, const char*, uint32_t, rocprofiler_client_id_t*);
+
+#if defined(CODECOV) && CODECOV > 0
+extern void
+__gcov_dump(void);
+#endif
 }

 namespace rocprofiler
@@ -569,6 +574,10 @@ initialize()
 void
 finalize()
 {
+#if defined(CODECOV) && CODECOV > 0
+    if(get_fini_status() > 0) __gcov_dump();
+#endif
+
    if(get_fini_status() != 0)
    {
        ROCP_INFO << "ignoring finalization request (value=" << get_fini_status() << ")";
@@ -599,6 +608,10 @@ finalize()
        internal_threading::finalize();
        set_fini_status(1);
    });
+
+#if defined(CODECOV) && CODECOV > 0
+    __gcov_dump();
+#endif
 }
 }  // namespace registration
 }  // namespace rocprofiler
@@ -1,20 +1,15 @@
 #!/bin/bash

-sudo apt-get update
-sudo apt-get install -y cmake clang-tidy-15 g++-11 g++-12 python3-pip libdw-dev
-sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 10
-sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
-sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
-#python3 -m pip install -r requirements.txt
-python3 -m pip install pytest pandas pyyaml
-python3 -m pip install 'cmake>=3.22.0'
+SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}) && pwd)
+ROCPROFILER_SDK_PATH="$(cd ${SCRIPT_PATH}/../.. && pwd)"

-ROCPROFILER_SDK_PATH="$(pwd)/$(dirname ${BASH_SOURCE[0]})/../.."
-
-cd ${ROCPROFILER_SDK_PATH}
+${SCRIPT_PATH}/install-deps.sh

 echo -e "Redirecting to location: $ROCPROFILER_SDK_PATH"
+cd ${ROCPROFILER_SDK_PATH}

-cmake -B build -DROCPROFILER_BUILD_CI=ON -DROCPROFILER_BUILD_TESTS=ON -DROCPROFILER_BUILD_SAMPLES=ON -DROCPROFILER_ENABLE_CLANG_TIDY=ON $*
+echo -e "Configuring rocprofiler-sdk: ${ROCPROFILER_SDK_PATH}/build"
+cmake -B build -DROCPROFILER_BUILD_{CI,TESTS,SAMPLES}=ON -DROCPROFILER_ENABLE_CLANG_TIDY=ON "${@}"
+
+echo -e "Building rocprofiler-sdk: ${ROCPROFILER_SDK_PATH}/build"
 cmake --build build --target all --parallel $(nproc)
-#cd --
@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+apt-get update
+apt-get install -y cmake clang-tidy-15 g++-11 g++-12 python3-pip libdw-dev
+update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 10
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 --slave /usr/bin/g++ g++ /usr/bin/g++-11
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 20 --slave /usr/bin/g++ g++ /usr/bin/g++-12
@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+ROCPROFILER_SDK_PATH=$(cd $(dirname ${BASH_SOURCE[0]})/../.. && pwd)
+
+# echo "ROCPROFILER_SDK_PATH: ${ROCPROFILER_SDK_PATH}"
+
+echo -e "Installing dependencies from: ${ROCPROFILER_SDK_PATH}/requirements.txt"
+python3 -m pip install --user -r ${ROCPROFILER_SDK_PATH}/requirements.txt