From d4fe3f1cc38a7af763c79cbb7b82b6ba98c02955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Mestre?= Date: Wed, 10 Dec 2025 16:53:47 +0000 Subject: [PATCH] [hip-tests] Update API coverage report generator (#1932) * [hip-tests] Update API coverage report generator Updates the HIP API coverage tool. It now takes extra arguments for the location of the catch test folder and for the working directory. This avoids issues where the output of the executable is dependent on the path where it is being executed from. Also updates CmakeLists.txt to integrate seamlessly with the hip-tests project and avoid using commands which rely on relative paths. * Remove double new line * Remove Cmake option to generate coverage Removes Cmake option to generate coverage. Instead, explicitly removes the gen_coverage target from all (this is already the default but doing it explicitly prevents confusion). --- projects/hip-tests/catch/CMakeLists.txt | 23 ++++++------ .../hip-tests/utils/coverage/CMakeLists.txt | 14 +++++--- projects/hip-tests/utils/coverage/hipAPI.cpp | 4 +-- projects/hip-tests/utils/coverage/hipAPI.h | 1 - .../utils/coverage/hipAPICoverageUtils.cpp | 3 -- .../hip-tests/utils/coverage/hipAPIGroup.cpp | 4 +-- .../hip-tests/utils/coverage/mainCoverage.cpp | 35 ++++++++++++++----- .../utils/coverage/reportGenerators.cpp | 14 ++++---- .../utils/coverage/reportGenerators.h | 6 ++-- 9 files changed, 64 insertions(+), 40 deletions(-) diff --git a/projects/hip-tests/catch/CMakeLists.txt b/projects/hip-tests/catch/CMakeLists.txt index b5595f1bbe..e6e114e27a 100644 --- a/projects/hip-tests/catch/CMakeLists.txt +++ b/projects/hip-tests/catch/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) -project(hiptests LANGUAGES C CXX HIP) +project(hiptests LANGUAGES C CXX HIP) option(ENABLE_ADDRESS_SANITIZER "Option to enable ASAN build" OFF) option(BUILD_SHARED_LIBS "Option for testing shared libraries" ON) @@ -8,7 +8,6 @@ option(BUILD_UNIT_TESTS "Build unit tests" ON) option(BUILD_PERF_TESTS "Build perf tests" OFF) option(BUILD_STRESS_TESTS "Build stress tests" OFF) option(TEST_CLOCK_CYCLE "Option to use clock64" OFF) -option(GENERATE_COVERATE "Generate coverage" OFF) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -326,14 +325,18 @@ if(BUILD_STRESS_TESTS) add_subdirectory(stress ${CATCH_BUILD_DIR}/stress) endif() -if(GENERATE_COVERATE) - add_custom_target(gen_coverage - COMMAND ${CMAKE_COMMAND} -B build/ - COMMAND ${CMAKE_COMMAND} --build build/ - COMMAND ./build/generateHipAPICoverage ${HIP_PATH}/include - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../utils/coverage - COMMENT "Generating Test Coverage Report") -endif() +add_custom_target(gen_coverage + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/utils/coverage + COMMAND ${CMAKE_COMMAND} -B ${CMAKE_CURRENT_BINARY_DIR}/utils/coverage -S ${CMAKE_CURRENT_SOURCE_DIR}/../utils/coverage + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_CURRENT_BINARY_DIR}/utils/coverage --target clean-cov-report + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_CURRENT_BINARY_DIR}/utils/coverage + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/utils/coverage/generateHipAPICoverage ${HIP_PATH}/include ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/utils/coverage/ + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating Test Coverage Report" + USES_TERMINAL + VERBATIM +) +set_property(TARGET gen_coverage PROPERTY EXCLUDE_FROM_ALL TRUE) # packaging the tests add_subdirectory(packaging) diff --git a/projects/hip-tests/utils/coverage/CMakeLists.txt b/projects/hip-tests/utils/coverage/CMakeLists.txt index 70561f702d..252e6e4af4 100644 --- a/projects/hip-tests/utils/coverage/CMakeLists.txt +++ b/projects/hip-tests/utils/coverage/CMakeLists.txt @@ -31,10 +31,16 @@ set(SRC hipAPICoverageUtils.cpp) add_executable(generateHipAPICoverage ${SRC}) +target_compile_options(generateHipAPICoverage PRIVATE -Wall -Wextra -Wvla -Werror -Wno-deprecated -Wno-option-ignored) + +add_custom_command(TARGET generateHipAPICoverage POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_LIST_DIR}/coverageReportHTML ${CMAKE_CURRENT_BINARY_DIR}/coverageReportHTML +) add_custom_target(clean-cov-report - COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_LIST_DIR}/*.xml - COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_LIST_DIR}/coverageReportHTML/testModules - COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_LIST_DIR}/coverageReportHTML/testAPIs - COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_LIST_DIR}/coverageReportHTML/*.html + COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_BINARY_DIR}/*.xml + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/coverageReportHTML/testModules + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/coverageReportHTML/testAPIs + COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_BINARY_DIR}/coverageReportHTML/*.html ) diff --git a/projects/hip-tests/utils/coverage/hipAPI.cpp b/projects/hip-tests/utils/coverage/hipAPI.cpp index 5fa1c96798..b6b81c34e6 100644 --- a/projects/hip-tests/utils/coverage/hipAPI.cpp +++ b/projects/hip-tests/utils/coverage/hipAPI.cpp @@ -48,9 +48,9 @@ bool operator<(const HipAPI& l_hip_api, const HipAPI& r_hip_api) { HipAPI::HipAPI(std::string api_name, bool deprecated_flag, std::string api_group_name, std::string file_restriction) : api_name{api_name}, + file_restriction{file_restriction}, deprecated{deprecated_flag}, - api_group_name{api_group_name}, - file_restriction{file_restriction} { + api_group_name{api_group_name} { test_cases.clear(); } diff --git a/projects/hip-tests/utils/coverage/hipAPI.h b/projects/hip-tests/utils/coverage/hipAPI.h index ffedf0ae44..72a435abab 100644 --- a/projects/hip-tests/utils/coverage/hipAPI.h +++ b/projects/hip-tests/utils/coverage/hipAPI.h @@ -76,7 +76,6 @@ class HipAPI { private: std::string api_name; std::string file_restriction; - int number_of_calls; bool deprecated; std::string api_group_name; std::vector file_occurrences; diff --git a/projects/hip-tests/utils/coverage/hipAPICoverageUtils.cpp b/projects/hip-tests/utils/coverage/hipAPICoverageUtils.cpp index ea55351046..112fc54aa1 100644 --- a/projects/hip-tests/utils/coverage/hipAPICoverageUtils.cpp +++ b/projects/hip-tests/utils/coverage/hipAPICoverageUtils.cpp @@ -337,16 +337,13 @@ std::vector extractDeviceAPIs(std::string& apis_list_file, main group that shall be considered is HIP API. Before that group is defined, lines of code shall not be considered. */ - int line_number{0}; bool group_start{false}; - bool device_groups_start{false}; std::string restriction{""}; std::string file_restriction_definition{"File restriction: "}; std::string device_groups_definition{"Device groups: ("}; std::vector device_groups; while (std::getline(apis_list_file_handler, line)) { - ++line_number; if (line.find("[") != std::string::npos) { std::string group_name = line.substr(0, line.rfind(" ")); diff --git a/projects/hip-tests/utils/coverage/hipAPIGroup.cpp b/projects/hip-tests/utils/coverage/hipAPIGroup.cpp index 5ec5174874..14a7e0cc9b 100644 --- a/projects/hip-tests/utils/coverage/hipAPIGroup.cpp +++ b/projects/hip-tests/utils/coverage/hipAPIGroup.cpp @@ -28,9 +28,9 @@ bool operator==(const HipAPIGroup& l_hip_api_group, const HipAPIGroup& r_hip_api HipAPIGroup::HipAPIGroup(std::string group_name, std::vector& hip_apis) : group_name{group_name}, + total_number_of_apis{0}, number_of_api_calls{0}, percentage_of_called_apis{0.f}, - total_number_of_apis{0}, number_of_test_cases{0} { std::vector test_cases; for (auto const& hip_api : hip_apis) { @@ -85,7 +85,7 @@ int HipAPIGroup::getNumberOfDeprecatedAPIs() const { return deprecated_apis.size float HipAPIGroup::getPercentageOfCalledAPIs() const { return percentage_of_called_apis; } bool HipAPIGroup::isDeprecated() const { - return (deprecated_apis.size() == total_number_of_apis) ? true : false; + return (static_cast(deprecated_apis.size()) == total_number_of_apis) ? true : false; } std::string HipAPIGroup::getBasicStatsXML() const { diff --git a/projects/hip-tests/utils/coverage/mainCoverage.cpp b/projects/hip-tests/utils/coverage/mainCoverage.cpp index 89f7b61652..e07633e3df 100644 --- a/projects/hip-tests/utils/coverage/mainCoverage.cpp +++ b/projects/hip-tests/utils/coverage/mainCoverage.cpp @@ -23,14 +23,33 @@ THE SOFTWARE. #include "reportGenerators.h" int main(int argc, char** argv) { - if (argc != 2) { - std::cout << "Please provide the path to the cloned HIP/include/ directory as an argument! " - "Only one argument supported." + if (argc == 2 && std::string(argv[1]) == "--help") { + std::cout << "Usage: " << argv[0] << " [work-dir]\n\n" + << "Arguments:\n\n" + << " hip-include-dir Path to the directory containing HIP headers.\n" + << " Example: /projects/hip/include\n" + << " catch-test-dir Path to the directory containing HIP Catch2 tests.\n" + << " Example: /projects/hip-tests/catch2\n" + << " work-dir (Optional) Path to the working directory.\n" + << " Defaults to the current directory.\n" << std::endl; - std::cout << "\tExample: ./generateHipAPICoverage /workspace/user1/HIP/include/" << std::endl; - return -1; + return 0; } + + if (argc < 3 || argc > 4) { + std::cerr << "Error: Invalid number of arguments. Use --help for usage information.\n"; + return 1; + } + std::string hip_include_path = argv[1]; + std::string tests_root_directory{argv[2]}; + + std::string work_directory = "."; + + if (argc == 4) { + work_directory = argv[3]; + } + /* Relative paths to all needed files, as it is expected that the application is called from the HIP/tests/catch/coverage directory. @@ -38,7 +57,7 @@ int main(int argc, char** argv) { std::string hip_api_header_file{ findAbsolutePathOfFile(hip_include_path + "/hip/hip_runtime_api.h")}; std::string hip_rtc_header_file{findAbsolutePathOfFile(hip_include_path + "/hip/hiprtc.h")}; - std::string tests_root_directory{findAbsolutePathOfFile("../../catch")}; + std::string device_api_file{"device_api_list.txt"}; std::vector api_group_names; @@ -79,10 +98,10 @@ int main(int argc, char** argv) { } std::cout << "Generating XML report files." << std::endl; - generateXMLReportFiles(hip_apis, hip_api_groups); + generateXMLReportFiles(hip_apis, hip_api_groups, work_directory); std::cout << "Generating HTML report files." << std::endl; generateHTMLReportFiles(hip_apis, hip_api_groups, tests_root_directory, hip_api_header_file, - hip_rtc_header_file); + hip_rtc_header_file, work_directory); return 0; } diff --git a/projects/hip-tests/utils/coverage/reportGenerators.cpp b/projects/hip-tests/utils/coverage/reportGenerators.cpp index f086aee536..9b82c761e8 100644 --- a/projects/hip-tests/utils/coverage/reportGenerators.cpp +++ b/projects/hip-tests/utils/coverage/reportGenerators.cpp @@ -45,8 +45,8 @@ float BasicAPIStats::getLowCoverageLimit() const { return 40.f; } float BasicAPIStats::getMediumCoverageLimit() const { return 80.f; } -void generateXMLReportFiles(std::vector& hip_apis, - std::vector& hip_api_groups) { +void generateXMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, + const std::string& work_directory) { BasicAPIStats basic_stats{hip_api_groups}; std::cout << "Total number of HIP API calls: " << basic_stats.total_number_of_api_calls @@ -67,7 +67,7 @@ void generateXMLReportFiles(std::vector& hip_apis, for the additional 3rd party library that implements XML file CRUD operations. */ std::fstream coverage_report; - std::string report_file_name{"CoverageReport.xml"}; + std::string report_file_name{work_directory + "/CoverageReport.xml"}; coverage_report.open(report_file_name, std::ios::out); time_t now{time(nullptr)}; @@ -120,15 +120,15 @@ void generateXMLReportFiles(std::vector& hip_apis, void generateHTMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, std::string tests_root_directory, std::string hipApiHeaderFile, - std::string hip_rtc_header_file) { + std::string hip_rtc_header_file, const std::string& work_directory) { BasicAPIStats basic_stats{hip_api_groups}; std::fstream coverage_report; // Main HTML report file. - std::string report_file_name{"./coverageReportHTML/CoverageReport.html"}; + std::string report_file_name{work_directory + "/coverageReportHTML/CoverageReport.html"}; // Directories used to store generated HTML files. - std::string test_modules_directory{"./coverageReportHTML/testModules"}; - std::string test_apis_directory{"./coverageReportHTML/testAPIs"}; + std::string test_modules_directory{work_directory + "/coverageReportHTML/testModules"}; + std::string test_apis_directory{work_directory + "/coverageReportHTML/testAPIs"}; std::filesystem::create_directories(test_modules_directory); std::filesystem::create_directories(test_apis_directory); diff --git a/projects/hip-tests/utils/coverage/reportGenerators.h b/projects/hip-tests/utils/coverage/reportGenerators.h index ccd1500980..15986f8d2b 100644 --- a/projects/hip-tests/utils/coverage/reportGenerators.h +++ b/projects/hip-tests/utils/coverage/reportGenerators.h @@ -36,9 +36,9 @@ class BasicAPIStats { float getMediumCoverageLimit() const; }; -void generateXMLReportFiles(std::vector& hip_apis, - std::vector& hip_api_groups); +void generateXMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, + const std::string& work_directory); void generateHTMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, std::string tests_root_directory, std::string hipApiHeaderFile, - std::string hip_rtc_header_file); + std::string hip_rtc_header_file, const std::string& work_directory);