diff --git a/utils/coverage/Makefile b/utils/coverage/Makefile new file mode 100644 index 0000000000..ee590bb186 --- /dev/null +++ b/utils/coverage/Makefile @@ -0,0 +1,38 @@ +# Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +CC=g++ +CPPFLAGS=-std=c++17 +SRC=mainCoverage.cpp hipAPI.cpp hipAPIGroup.cpp reportGenerators.cpp hipAPICoverageUtils.cpp +OBJ=generateHipAPICoverage + +default_target: all +.PHONY : default_target + +all: ${SRC} + ${CC} ${CPPFLAGS} $^ -o ${OBJ} + +clean: + rm ${OBJ} + rm *.xml + rm -r ./coverageReportHTML/testModules + rm -r ./coverageReportHTML/testAPIs + rm ./coverageReportHTML/*.html +.PHONY : clean diff --git a/utils/coverage/coverageReportHTML/resources/amber.png b/utils/coverage/coverageReportHTML/resources/amber.png new file mode 100644 index 0000000000..2cab170d83 Binary files /dev/null and b/utils/coverage/coverageReportHTML/resources/amber.png differ diff --git a/utils/coverage/coverageReportHTML/resources/coverage.css b/utils/coverage/coverageReportHTML/resources/coverage.css new file mode 100644 index 0000000000..67a8a40b76 --- /dev/null +++ b/utils/coverage/coverageReportHTML/resources/coverage.css @@ -0,0 +1,227 @@ +/* All views: initial background and text color */ +body +{ + color: #000000; + background-color: #FFFFFF; +} + +/* All views: standard link format*/ +a:link +{ + color: #284FA8; + text-decoration: underline; +} + +/* All views: standard link - activated format */ +a:active +{ + color: #FF0040; + text-decoration: underline; +} + +/* All views: main title format */ +td.title +{ + text-align: center; + padding-bottom: 10px; + font-family: sans-serif; + font-size: 20pt; + font-style: italic; + font-weight: bold; +} + +/* All views: header item format */ +td.headerItem +{ + text-align: right; + padding-right: 6px; + font-family: sans-serif; + font-weight: bold; + vertical-align: top; + white-space: nowrap; +} + +/* All views: header item value format */ +td.headerValue +{ + text-align: left; + color: #284FA8; + font-family: sans-serif; + font-weight: bold; +} + +/* All views: header item coverage table heading */ +td.headerCovTableHead +{ + text-align: center; + padding-right: 6px; + padding-left: 6px; + padding-bottom: 0px; + font-family: sans-serif; + font-size: 80%; + white-space: nowrap; +} + +/* All views: header item coverage table entry */ +td.headerCovTableEntry +{ + text-align: center; + color: #284FA8; + font-family: sans-serif; + font-weight: bold; + white-space: nowrap; + padding-left: 6px; + padding-right: 6px; + background-color: #DAE7FE; +} + +/* All views: header item coverage table entry for high coverage rate */ +td.headerCovTableEntryHi +{ + text-align: center; + color: #000000; + font-family: sans-serif; + font-weight: bold; + white-space: nowrap; + padding-left: 12px; + padding-right: 4px; + background-color: #77DD77; +} + +/* All views: header item coverage table entry for medium coverage rate */ +td.headerCovTableEntryMed +{ + text-align: center; + color: #000000; + font-family: sans-serif; + font-weight: bold; + white-space: nowrap; + padding-left: 12px; + padding-right: 4px; + background-color: #E9EC6B; +} + +/* All views: header item coverage table entry for ow coverage rate */ +td.headerCovTableEntryLo +{ + text-align: center; + color: #000000; + font-family: sans-serif; + font-weight: bold; + white-space: nowrap; + padding-left: 12px; + padding-right: 4px; + background-color: #E66360; +} + +/* All views: color of horizontal ruler */ +td.ruler +{ + background-color: #6688D4; +} + +/* All views: version string format */ +td.versionInfo +{ + text-align: center; + padding-top: 2px; + font-family: sans-serif; + font-style: italic; +} + +/* Directory view/File view (all)/Test case descriptions: + table headline format */ +td.tableHead +{ + text-align: center; + color: #FFFFFF; + background-color: #6688D4; + font-family: sans-serif; + font-weight: bold; + white-space: nowrap; + padding-left: 4px; + padding-right: 4px; + word-break: break-all; +} + +/* Directory view/File view (all): filename entry format */ +td.coverFile +{ + text-align: left; + padding-left: 10px; + padding-right: 20px; + color: #284FA8; + background-color: #DAE7FE; + font-family: monospace; +} + +/* Directory view/File view (all): bar-graph entry format*/ +td.coverBar +{ + padding-left: 10px; + padding-right: 10px; + background-color: #DAE7FE; +} + +/* Directory view/File view (all): bar-graph outline color */ +td.coverBarOutline +{ + background-color: #000000; +} + +/* Directory view/File view (all): line count entry for files with + high coverage rate */ +td.coverNumHi +{ + text-align: center; + padding-left: 10px; + padding-right: 10px; + background-color: #77DD77; + white-space: nowrap; + font-family: sans-serif; +} + +/* Directory view/File view (all): line count entry for files with + medium coverage rate */ +td.coverNumMed +{ + text-align: center; + padding-left: 10px; + padding-right: 10px; + background-color: #E9EC6B; + white-space: nowrap; + font-family: sans-serif; +} + +/* Directory view/File view (all): line count entry for files with + low coverage rate */ +td.coverNumLo +{ + text-align: center; + padding-left: 10px; + padding-right: 10px; + background-color: #E66360; + white-space: nowrap; + font-family: sans-serif; +} + +/* File view (all): "show/hide details" link format */ +a.detail:link +{ + color: #B8D0FF; + font-size:80%; +} + +/* File view (all): "show/hide details" link - visited format */ +a.detail:visited +{ + color: #B8D0FF; + font-size:80%; +} + +/* File view (all): "show/hide details" link - activated format */ +a.detail:active +{ + color: #FFFFFF; + font-size:80%; +} diff --git a/utils/coverage/coverageReportHTML/resources/emerald.png b/utils/coverage/coverageReportHTML/resources/emerald.png new file mode 100644 index 0000000000..38ad4f4068 Binary files /dev/null and b/utils/coverage/coverageReportHTML/resources/emerald.png differ diff --git a/utils/coverage/coverageReportHTML/resources/glass.png b/utils/coverage/coverageReportHTML/resources/glass.png new file mode 100644 index 0000000000..e1abc00680 Binary files /dev/null and b/utils/coverage/coverageReportHTML/resources/glass.png differ diff --git a/utils/coverage/coverageReportHTML/resources/ruby.png b/utils/coverage/coverageReportHTML/resources/ruby.png new file mode 100644 index 0000000000..991b6d4ec9 Binary files /dev/null and b/utils/coverage/coverageReportHTML/resources/ruby.png differ diff --git a/utils/coverage/coverageReportHTML/resources/snow.png b/utils/coverage/coverageReportHTML/resources/snow.png new file mode 100644 index 0000000000..2cdae107fc Binary files /dev/null and b/utils/coverage/coverageReportHTML/resources/snow.png differ diff --git a/utils/coverage/coverageReportHTML/resources/updown.png b/utils/coverage/coverageReportHTML/resources/updown.png new file mode 100644 index 0000000000..aa56a238b3 Binary files /dev/null and b/utils/coverage/coverageReportHTML/resources/updown.png differ diff --git a/utils/coverage/hipAPI.cpp b/utils/coverage/hipAPI.cpp new file mode 100644 index 0000000000..39ef99e4a2 --- /dev/null +++ b/utils/coverage/hipAPI.cpp @@ -0,0 +1,222 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "hipAPI.h" + +FileOccurrence::FileOccurrence(std::string file_name, int line_number): + file_name{file_name}, line_number{line_number} {} + +TestCaseOccurrence::TestCaseOccurrence(std::string test_case_name, std::string file_name, int line_number): + FileOccurrence{file_name, line_number}, test_case_name{test_case_name} {} + +bool operator==(const HipAPI& l_hip_api, const HipAPI& r_hip_api) { + return l_hip_api.api_name == r_hip_api.api_name; +} + +bool operator<(const HipAPI& l_hip_api, const HipAPI& r_hip_api) { + return l_hip_api.api_name < r_hip_api.api_name; +} + +HipAPI::HipAPI(std::string api_name, bool deprecated_flag, std::string api_group_name): + api_name{api_name}, deprecated{deprecated_flag}, api_group_name{api_group_name} {} + +std::string HipAPI::getName() const { + return api_name; +} + +std::string HipAPI::getGroupName() const { + return api_group_name; +} + +int HipAPI::getNumberOfCalls() const { + return file_occurrences.size(); +} + +int HipAPI::getNumberOfTestCases() const { + return test_cases.size(); +} + +void HipAPI::addFileOccurrence(FileOccurrence file_occurrence) { + file_occurrences.push_back(file_occurrence); +} + +void HipAPI::addTestCase(TestCaseOccurrence test_case) { + test_cases.push_back(test_case); +} + +bool HipAPI::isDeprecated() const +{ + return deprecated; +} + +std::string HipAPI::getBasicStatsXML() const +{ + std::stringstream xml_node; + xml_node << "\t\t\n"; + + if (!deprecated) { + xml_node << "\t\t\t" << api_name << "\n"; + } else { + xml_node << "\t\t\t" << "[DEPRECATED] " << api_name << "\n"; + } + + if (!file_occurrences.empty()) { + xml_node << "\t\t\t" << file_occurrences.size() << "\n"; + xml_node << "\t\t\t\n"; + for (auto const& file_occurrence: file_occurrences) { + xml_node << "\t\t\t\t" << file_occurrence.file_name << ":" << file_occurrence.line_number << "\n"; + } + xml_node << "\t\t\t\n"; + } + + xml_node << "\t\t\n"; + return xml_node.str(); +} + +std::string HipAPI::createHTMLReport() const { + std::stringstream html_report; + std::string one_tab{"\n\t"}; + std::string two_tabs{"\n\t\t"}; + std::string three_tabs{"\n\t\t\t"}; + std::string four_tabs{"\n\t\t\t\t"}; + std::string five_tabs{"\n\t\t\t\t\t"}; + std::string six_tabs{"\n\t\t\t\t\t\t"}; + + html_report << ""; + html_report << "" << one_tab << ""; + html_report << one_tab << "" << api_name << " Coverage report" << one_tab << "" << one_tab<< ""; + html_report << one_tab << "" << one_tab << ""; + html_report << two_tabs << ""; + html_report << two_tabs << "\n"; + html_report << two_tabs << "" << three_tabs << ""; + html_report << two_tabs << ""; + + html_report << two_tabs << "\n"; + html_report << one_tab << "
" << api_name << " Coverage report
" << four_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << four_tabs << "
"; + html_report << six_tabs << "
Group:" << api_group_name << "
Calls within test source files:" << file_occurrences.size() << "
Number of test cases:" << test_cases.size() << "
"; + html_report << three_tabs << "
"; + + html_report << one_tab << "
"; + // Add info about test cases + html_report << one_tab << ""; + if (!test_cases.empty()) { + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + + for (auto const& test_case: test_cases) { + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + } + } else { + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + } + html_report << one_tab << "



Test case IDTest case occurrence in fileLine number
" << test_case.test_case_name << "" << test_case.file_name << "" << test_case.line_number << "

There are no test cases detected within doxygen comments.
"; + + html_report << one_tab << "
"; + html_report << one_tab << ""; + html_report << two_tabs << ""; + html_report << one_tab << "
"; + + // Add info about API occurrences in the test files. + html_report << one_tab << ""; + if (!file_occurrences.empty()) { + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + + for (auto const& file_occurrence: file_occurrences) { + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + } + } else { + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + } + html_report << one_tab << "


API occurrence in fileLine number
" << file_occurrence.file_name << "" << file_occurrence.line_number << "

There are no occurrences within test source files.
"; + html_report << one_tab << "
"; + + html_report << one_tab << "
"; + html_report << one_tab << ""; + html_report << two_tabs << ""; + + time_t now{time(nullptr)}; + std::string date{asctime(gmtime(&now))}; + html_report << two_tabs << ""; + html_report << one_tab << "
Generated: " << date; + html_report << two_tabs << " UTC
"; + html_report << one_tab << "
"; + html_report << "\n\n"; + + return html_report.str(); +} diff --git a/utils/coverage/hipAPI.h b/utils/coverage/hipAPI.h new file mode 100644 index 0000000000..b14d3b7c6e --- /dev/null +++ b/utils/coverage/hipAPI.h @@ -0,0 +1,75 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include +#include + +/* +Helper class used to store information in what file has HIP API been detected, +and on what line of code in that file. +*/ +class FileOccurrence { + public: + std::string file_name; + int line_number; + FileOccurrence(std::string file_name, int line_number); +}; + +/* +Helper class used to store information in what file has the API Test Case been detected, +and on what line of code in that file. +*/ +class TestCaseOccurrence : public FileOccurrence { + public: + std::string test_case_name; + TestCaseOccurrence(std::string test_case_name, std::string file_name, int line_number); +}; + +/* +Class used to store infromation about each HIP API. All information +is related to the API name, number of calls from test .cc files, +and its status of deprecation. +*/ +class HipAPI { + friend bool operator==(const HipAPI& l_hip_api, const HipAPI& r_hip_api); + friend bool operator<(const HipAPI& l_hip_api, const HipAPI& r_hip_api); + + public: + HipAPI(std::string api_name, bool deprecated_flag, std::string api_group_name); + std::string getName() const; + std::string getGroupName() const; + int getNumberOfCalls() const; + int getNumberOfTestCases() const; + void addFileOccurrence(FileOccurrence file_occurence); + void addTestCase(TestCaseOccurrence test_case); + bool isDeprecated() const; + std::string getBasicStatsXML() const; + std::string createHTMLReport() const; + private: + std::string api_name; + int number_of_calls; + bool deprecated; + std::string api_group_name; + std::vector file_occurrences; + std::vector test_cases; +}; diff --git a/utils/coverage/hipAPICoverageUtils.cpp b/utils/coverage/hipAPICoverageUtils.cpp new file mode 100644 index 0000000000..022815768c --- /dev/null +++ b/utils/coverage/hipAPICoverageUtils.cpp @@ -0,0 +1,284 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "hipAPICoverageUtils.h" + +/* +Used to find all HIP API occurrences within the passed test .cc files. +Occurrence is detected when the HIP API is called in several scenarios. + - API Call with assertion, e.g. REQUIRE(hipAPI()); + - API call with assignment, e.g. result = hipAPI(); + - API call with parameter, e.g. ASSERT_EQUAL(hipSuccess, hipGetDevice(&deviceId)); +*/ +void findAPICallInFile(HipAPI& hip_api, std::string test_module_file) { + std::fstream test_module_file_handler; + test_module_file_handler.open(test_module_file); + + int line_number{0}; + std::string line; + /* + Closed brackets might be in another line, if the API + call is multiline. + */ + std::string api_call_with_assert{"(" + hip_api.getName() + "("}; + std::string api_call_with_assignment{"= " + hip_api.getName() + "("}; + std::string api_call_with_parameter{", " + hip_api.getName() + "("}; + + while (std::getline(test_module_file_handler, line)) { + ++line_number; + if ((line.find(api_call_with_assert) != std::string::npos) || + (line.find(api_call_with_assignment) != std::string::npos) || + (line.find(api_call_with_parameter) != std::string::npos)) { + hip_api.addFileOccurrence(FileOccurrence(test_module_file, line_number)); + } + } + + test_module_file_handler.close(); +} + +/* +Used to find all HIP API test cases within the passed test .cc files. +Matching test case is detected when the HIP API in defined within doxygen comment. +*/ +void findAPITestCaseInFile(HipAPI& hip_api, std::string test_module_file) { + std::fstream test_module_file_handler; + test_module_file_handler.open(test_module_file); + + int line_number{0}; + std::string line; + + std::string add_group_definition{"@addtogroup"}; + std::string ref_test_case{"@ref"}; + std::string test_case_definition{"TEST_CASE("}; + std::string current_api_name{"None"}; + std::string test_case{"None"}; + + while (std::getline(test_module_file_handler, line)) { + ++line_number; + if (line.find(add_group_definition) != std::string::npos) { + current_api_name = line.substr(line.find(add_group_definition) + 1); + current_api_name = current_api_name.substr(current_api_name.rfind(" ") + 1); + } + + if (hip_api.getName() != current_api_name) { + continue; + } + + if (line.find(ref_test_case) != std::string::npos) { + test_case = line.substr(line.rfind(" ") + 1); + hip_api.addTestCase(TestCaseOccurrence{test_case, test_module_file, line_number}); + continue; + } + + if (line.find(test_case_definition) != std::string::npos) { + test_case = line.substr(line.find("\"") + 1); + test_case = test_case.substr(0, test_case.find("\"")); + hip_api.addTestCase(TestCaseOccurrence{test_case, test_module_file, line_number}); + } + } + + test_module_file_handler.close(); +} + +/* +Used to iterate through all passed test .cc files and search for passed +HIP API instance. This instance shall be used to update occurrences. +*/ +void searchForAPI(HipAPI& hip_api, std::vector& test_module_files) { + std::cout << "Searching for " << hip_api.getName() << " in test module files." << std::endl; + for (auto const& test_module_file: test_module_files) { + findAPICallInFile(hip_api, test_module_file); + findAPITestCaseInFile(hip_api, test_module_file); + } +} + +/* +Used to extract all HIP APIs from the passed header file. +*/ +std::vector extractHipAPIs(std::string& hip_api_header_file, + std::vector& api_group_names, bool start_groups) { + std::fstream hip_header_file_handler; + hip_header_file_handler.open(hip_api_header_file); + + std::string line; + std::vector hip_apis; + + /* + If the HIP API is deprecated, it will be marked with + the following deprecation line in code. + */ + std::string deprecated_line{"DEPRECATED("}; + /* + Each HIP API has prefix hip in the name. Groups are marked with @defgroup, and the + main group that shall be considered is HIP API. Before that group is defined, lines + of code shall not be considered. + */ + std::string hip_api_prefix{"hip"}; + std::string group_definition{"@defgroup"}; + std::string add_group_definition{"@addtogroup"}; + std::string start_of_api_groups{"HIP API"}; + std::string end_of_api_groups{"doxygen end HIP API"}; + std::string end_group_definition{"@}"}; + + bool deprecated_flag{false}; + bool api_group_names_start{start_groups}; + std::vector api_group_names_tracker{api_group_names}; + int line_number{0}; + + while (std::getline(hip_header_file_handler, line)) { + ++line_number; + + // Declarations of the HIP APIs start after the HIP API group has been defined. + if ((line.find(group_definition) != std::string::npos || line.find(add_group_definition) != std::string::npos) + && line.find(start_of_api_groups) != std::string::npos) { + api_group_names_start = true; + continue; + } + + // If the API Groups have not started yet, go to the next file line. + if (!api_group_names_start) { + continue; + } + + // If the end of HIP API group has been detected, break the loop. + if (line.find(end_of_api_groups) != std::string::npos) { + break; + } + + /* + If the API is deprecated, raise a flag and go to the + next line where the API is declared. + */ + if (line.find(deprecated_line) != std::string::npos) { + std::getline(hip_header_file_handler, line); + ++line_number; + deprecated_flag = true; + } else { + deprecated_flag = false; + } + + if (line.find(group_definition) != std::string::npos) { + std::string group_name = line.substr(line.find(group_definition) + group_definition.length() + 1); + group_name = group_name.substr(group_name.find(' ') + 1); + api_group_names.push_back(group_name); + api_group_names_tracker.push_back(group_name); + } + else if (line.find(add_group_definition) != std::string::npos) + { + std::string group_name = line.substr(line.find(add_group_definition) + add_group_definition.length() + 1); + group_name = group_name.substr(group_name.find(' ') + 1); + api_group_names.push_back(group_name); + api_group_names_tracker.push_back(group_name); + } + + /* + Possible case is that there are nested groups. While api_group_names + vector contains all detected groups, api_group_names_tracker is responsible + to track the last defined group, because of the nested cases. + */ + if (line.find(end_group_definition) != std::string::npos) { + api_group_names_tracker.pop_back(); + } + + /* + The line which contains HIP API declaration looks like: + hipError_t hipMalloc(void** ptr, size_t size); + The name of HIP API is found by following steps: + - Take the substring from the start to the first open bracket. + - Extract the name from that substring by finding the last "hip". + Avoid comments. + */ + if (line.find(hip_api_prefix) != std::string::npos && + line.find("(") != std::string::npos && + line.find(" ") != 0 && + line.find(" *") != 0) { + std::string api_name_no_brackets{line.substr(0, line.find("("))}; + /* + If there is no hip substring, then there is no valid API in that line. + */ + if (api_name_no_brackets.rfind(hip_api_prefix) == std::string::npos) { + continue; + } + + /* + Extract the substring that starts from the last hip substring to the + end of that string (until the open bracket from the original string). + Remove all spaces if they exist in the parsed string, e.g., + hipError_t hipDeviceSetLimit ( enum hipLimit_t limit, size_t value );. + */ + std::string api_name{api_name_no_brackets.substr(api_name_no_brackets.rfind(hip_api_prefix))}; + api_name.erase(std::remove(api_name.begin(), api_name.end(), ' '), api_name.end()); + + if (!api_group_names_tracker.empty()) { + /* + If the API is not present in the global list of HIP APIs, add it. + */ + HipAPI hip_api{api_name, deprecated_flag, api_group_names_tracker.back()}; + if(std::find(hip_apis.begin(), hip_apis.end(), hip_api) == hip_apis.end()) + { + hip_apis.push_back(hip_api); + } + } else { + std::cout << "[SKIP_FROM_COV] Group not detected for \"" << api_name << "\" in file \"" + << hip_api_header_file << "\", line " << line_number << std::endl; + } + } + } + + hip_header_file_handler.close(); + return hip_apis; +} + +/* +Used to extract test .cc files from the passed tests root directory. +Goes through all subdirectories and searches for .cc and .hh files for +HIP API invocations. +Implements BFS algorithm to avoid recursion. +*/ +std::vector extractTestModuleFiles(std::string& tests_root_directory) +{ + std::vector directory_queue; + directory_queue.push_back(tests_root_directory); + std::vector test_module_files; + + while (!directory_queue.empty()) { + std::string processed_entry = directory_queue.back(); + directory_queue.pop_back(); + for (const auto& entry: std::filesystem::directory_iterator(processed_entry)) { + if (std::filesystem::is_directory(entry.path())) { + directory_queue.push_back(entry.path()); + } else { + if (entry.path().string().find(".cc") != std::string::npos || + entry.path().string().find(".hh") != std::string::npos) { + test_module_files.push_back(entry.path()); + } + } + } + } + + return test_module_files; +} + +std::string findAbsolutePathOfFile(std::string file_path) +{ + return std::filesystem::canonical(std::filesystem::absolute(file_path)); +} diff --git a/utils/coverage/hipAPICoverageUtils.h b/utils/coverage/hipAPICoverageUtils.h new file mode 100644 index 0000000000..68454fd863 --- /dev/null +++ b/utils/coverage/hipAPICoverageUtils.h @@ -0,0 +1,32 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include +#include "hipAPIGroup.h" + +void findAPICallInFile(HipAPI& hip_api, std::string test_module_file); +void findAPITestCaseInFile(HipAPI& hip_api, std::string test_module_file); +void searchForAPI(HipAPI& hip_api, std::vector& test_module_files); +std::vector extractHipAPIs(std::string& hip_api_header_file, std::vector& api_group_names, bool start_groups); +std::vector extractTestModuleFiles(std::string& tests_root_directory); +std::string findAbsolutePathOfFile(std::string file_path); diff --git a/utils/coverage/hipAPIGroup.cpp b/utils/coverage/hipAPIGroup.cpp new file mode 100644 index 0000000000..b24e2f1baa --- /dev/null +++ b/utils/coverage/hipAPIGroup.cpp @@ -0,0 +1,335 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "hipAPIGroup.h" + +bool operator==(const HipAPIGroup& l_hip_api_group, const HipAPIGroup& r_hip_api_group) { + return l_hip_api_group.group_name == r_hip_api_group.group_name; +} + +HipAPIGroup::HipAPIGroup(std::string group_name, std::vector& hip_apis): + group_name{group_name}, number_of_api_calls{0}, percentage_of_called_apis{0.f}, + total_number_of_apis{0}, number_of_test_cases{0} +{ + for (auto const& hip_api: hip_apis) { + if (hip_api.getGroupName() != group_name) { + continue; + } + + if (hip_api.isDeprecated()) { + deprecated_apis.push_back(hip_api); + } else { + if (hip_api.getNumberOfCalls()) { + called_apis.push_back(hip_api); + } else { + not_called_apis.push_back(hip_api); + } + } + + number_of_api_calls += hip_api.getNumberOfCalls(); + number_of_test_cases += hip_api.getNumberOfTestCases(); + } + + total_number_of_apis = called_apis.size() + not_called_apis.size() + deprecated_apis.size(); + if (not_called_apis.empty()) { + percentage_of_called_apis = 100.f; + } else { + percentage_of_called_apis = 100.f * called_apis.size() / (total_number_of_apis - deprecated_apis.size()); + } +} + +std::string HipAPIGroup::getName() const { + return group_name; +} + +int HipAPIGroup::getTotalNumberOfAPIs() const { + return total_number_of_apis; +} + +int HipAPIGroup::getTotalNumberOfCalls() const { + return number_of_api_calls; +} + +int HipAPIGroup::getTotalNumberOfTestCases() const { + return number_of_test_cases; +} + +int HipAPIGroup::getNumberOfCalledAPIs() const { + return called_apis.size(); +} + +int HipAPIGroup::getNumberOfNotCalledAPIs() const { + return not_called_apis.size(); +} + +int HipAPIGroup::getNumberOfDeprecatedAPIs() const { + return deprecated_apis.size(); +} + +float HipAPIGroup::getPercentageOfCalledAPIs() const { + return percentage_of_called_apis; +} + +std::string HipAPIGroup::getBasicStatsXML() const { + std::stringstream xml_node; + + std::string tag_name; + std::transform(group_name.begin(), group_name.end(), std::back_inserter(tag_name), ::toupper); + std::replace(tag_name.begin(), tag_name.end(), ' ', '-'); + + xml_node << "\n<" << tag_name << ">"; + + xml_node << "\n\t"; + xml_node << "\n\t\t" << total_number_of_apis << ""; + xml_node << "\n\t\t" << number_of_api_calls << ""; + xml_node << "\n\t\t" << called_apis.size() << ""; + xml_node << "\n\t\t" << not_called_apis.size() << ""; + xml_node << "\n\t\t" << deprecated_apis.size() << ""; + xml_node << "\n\t\t" << percentage_of_called_apis << "%"; + xml_node << "\n\t"; + + if (!called_apis.empty()) { + xml_node << "\n\t\n"; + for (auto const& hip_api: called_apis) { + xml_node << hip_api.getBasicStatsXML(); + } + xml_node << "\t"; + } + + if (!not_called_apis.empty()) { + xml_node << "\n\t\n"; + for (auto const& hip_api: not_called_apis) { + xml_node << hip_api.getBasicStatsXML(); + } + xml_node << "\t"; + } + + if (!deprecated_apis.empty()) { + xml_node << "\n\t\n"; + for (auto const& hip_api: deprecated_apis) { + xml_node << hip_api.getBasicStatsXML(); + } + xml_node << "\t"; + } + + xml_node << "\n"; + return xml_node.str(); +} + +std::string HipAPIGroup::getBasicStatsHTML() const +{ + std::stringstream html_object; + std::string two_tabs{"\n\t\t"}; + std::string three_tabs{"\n\t\t\t"}; + std::string four_tabs{"\n\t\t\t\t"}; + std::string five_tabs{"\n\t\t\t\t\t"}; + + // Determine font class from coverage.css and image for color bar. + std::string font_class; + std::string color_bar; + + if (percentage_of_called_apis < 40.f) { + font_class = "coverNumLo"; + color_bar = "resources/ruby.png"; + } else if (percentage_of_called_apis < 80.f) { + font_class = "coverNumMed"; + color_bar = "resources/amber.png"; + } else { + font_class = "coverNumHi"; + color_bar = "resources/emerald.png"; + } + + html_object << two_tabs << ""; + html_object << three_tabs << "" << group_name << ""; + html_object << three_tabs << "" << total_number_of_apis << ""; + html_object << three_tabs << "" << number_of_api_calls << ""; + html_object << three_tabs << "" << number_of_test_cases << ""; + html_object << three_tabs << "" << called_apis.size() << ""; + html_object << three_tabs << "" << not_called_apis.size() << ""; + html_object << three_tabs << "" << deprecated_apis.size() << ""; + html_object << three_tabs << "" << std::fixed << std::setprecision(2) << percentage_of_called_apis << "%"; + + html_object << three_tabs << ""; + html_object << four_tabs << ""; + html_object << five_tabs << "
\"""; + html_object << "\""
"; + html_object << four_tabs << ""; + html_object << three_tabs << ""; + + return html_object.str(); +} + +std::string HipAPIGroup::createHTMLReport() const +{ + std::stringstream html_report; + std::string one_tab{"\n\t"}; + std::string two_tabs{"\n\t\t"}; + std::string three_tabs{"\n\t\t\t"}; + std::string four_tabs{"\n\t\t\t\t"}; + std::string five_tabs{"\n\t\t\t\t\t"}; + std::string six_tabs{"\n\t\t\t\t\t\t"}; + + html_report << ""; + html_report << "" << one_tab << ""; + html_report << one_tab << "" << group_name << " Coverage report"; + html_report << one_tab << "" << one_tab<< ""; + html_report << one_tab << "" << one_tab << ""; + html_report << two_tabs << ""; + html_report << two_tabs << "\n"; + html_report << two_tabs << "" << three_tabs << ""; + html_report << two_tabs << ""; + + html_report << two_tabs << "\n"; + html_report << one_tab << "
" << group_name << " Coverage report
" << four_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + // Determine font class from coverage.css file based on coverage percentage. + std::string font_class; + if (percentage_of_called_apis < 40.f) { + font_class = "headerCovTableEntryLo"; + } else if (percentage_of_called_apis < 80.f) { + font_class = "headerCovTableEntryMed"; + } else { + font_class = "headerCovTableEntryHi"; + } + + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + + html_report << five_tabs << ""; + html_report << four_tabs << "
Value
Total number of detected HIP APIs:" << total_number_of_apis << "
HIP API calls within test source files:" << number_of_api_calls << "
Total number of test cases:" << number_of_test_cases << "
HIP APIs that are called at least once:" << called_apis.size() << "
HIP APIs that are not called at all:" << not_called_apis.size() << "
HIP APIs that are marked as deprecated:" << deprecated_apis.size() << "
Test coverage by implemented tests for the HIP APIs:" << std::fixed << std::setprecision(2) << percentage_of_called_apis << "%
"; + html_report << three_tabs << "
"; + + // Add info about Test module APIs. + html_report << one_tab << "
"; + html_report << one_tab << ""; + html_report << two_tabs << ""; + html_report << three_tabs << ""; + html_report << three_tabs << ""; + html_report << two_tabs << ""; + html_report << two_tabs << ""; + + html_report << three_tabs << ""; + + html_report << three_tabs << ""; + + html_report << three_tabs << ""; + + html_report << two_tabs << ""; + + html_report << one_tab << "

"; + html_report << three_tabs << "
"; + html_report << four_tabs << ""; + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + for (auto const& hip_api: called_apis) { + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + } + + html_report << four_tabs << "
Called APIs
" << hip_api.getName() << "
"; + html_report << three_tabs << "		"; + html_report << four_tabs << ""; + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + for (auto const& hip_api: not_called_apis) { + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + } + html_report << four_tabs << "
Not called APIs
" << hip_api.getName() << "
"; + html_report << three_tabs << "		"; + html_report << four_tabs << ""; + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + for (auto const& hip_api: deprecated_apis) { + html_report << five_tabs << ""; + html_report << six_tabs << ""; + html_report << five_tabs << ""; + } + html_report << four_tabs << "
Deprecated APIs
" << hip_api.getName() << "
"; + html_report << three_tabs << "
"; + html_report << one_tab << "
"; + html_report << one_tab << "
"; + + html_report << one_tab << ""; + html_report << two_tabs << ""; + + time_t now{time(nullptr)}; + std::string date{asctime(gmtime(&now))}; + html_report << two_tabs << ""; + html_report << one_tab << "
Generated: " << date; + html_report << two_tabs << " UTC
"; + html_report << one_tab << "
"; + html_report << "\n\n"; + + return html_report.str(); +} diff --git a/utils/coverage/hipAPIGroup.h b/utils/coverage/hipAPIGroup.h new file mode 100644 index 0000000000..c9b07b8656 --- /dev/null +++ b/utils/coverage/hipAPIGroup.h @@ -0,0 +1,53 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "hipAPI.h" +#include +#include + +class HipAPIGroup{ + friend bool operator==(const HipAPIGroup& l_hip_api_group, const HipAPIGroup& r_hip_api_group); + + public: + HipAPIGroup(std::string group_name, std::vector& hip_apis); + std::string getName() const; + int getTotalNumberOfAPIs() const; + int getTotalNumberOfCalls() const; + int getTotalNumberOfTestCases() const; + int getNumberOfCalledAPIs() const; + int getNumberOfNotCalledAPIs() const; + int getNumberOfDeprecatedAPIs() const; + float getPercentageOfCalledAPIs() const; + std::string getBasicStatsXML() const; + std::string getBasicStatsHTML() const; + std::string createHTMLReport() const; + private: + std::string group_name; + int total_number_of_apis; + int number_of_api_calls; + float percentage_of_called_apis; + int number_of_test_cases; + std::string parent_group_name; + std::vector called_apis; + std::vector not_called_apis; + std::vector deprecated_apis; +}; diff --git a/utils/coverage/mainCoverage.cpp b/utils/coverage/mainCoverage.cpp new file mode 100644 index 0000000000..65de5890a8 --- /dev/null +++ b/utils/coverage/mainCoverage.cpp @@ -0,0 +1,74 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "reportGenerators.h" + +int main(int argc, char** argv) +{ + if (argc != 2) { + std::cout << "Please provide the path to the cloned HIP/include/ directory as an argument! Only one argument supported." << std::endl; + std::cout << "\tExample: ./generateHipAPICoverage /workspace/user1/HIP/include/" << std::endl; + return -1; + } + std::string hip_include_path = argv[1]; + /* + Relative paths to all needed files, as it is expected that the application + is called from the HIP/tests/catch/coverage directory. + */ + std::string hip_api_header_file{findAbsolutePathOfFile(hip_include_path + "/hip/hip_runtime_api.h")}; + std::string hip_rtc_header_file{findAbsolutePathOfFile(hip_include_path + "/hip/hiprtc.h")}; + std::string tests_root_directory{findAbsolutePathOfFile("../../catch")}; + + std::vector api_group_names; + // Extract all HIP API declarations from the HIP API header file. + std::vector hip_apis{extractHipAPIs(hip_api_header_file, api_group_names, false)}; + std::cout << "Number of detected HIP APIs from " << hip_api_header_file << ": " << hip_apis.size() << std::endl; + + std::vector hip_rtc_apis{extractHipAPIs(hip_rtc_header_file, api_group_names, true)}; + std::cout << "Number of detected HIP APIs from " << hip_rtc_header_file << ": " << hip_rtc_apis.size() << std::endl; + hip_apis.insert(hip_apis.end(), hip_rtc_apis.begin(), hip_rtc_apis.end()); + + // Extract all test module .cc files that shall be used for API searching. + std::cout << "Searching for HIP API calls in source files within " << tests_root_directory << "." << std::endl; + std::vector test_module_files{extractTestModuleFiles(tests_root_directory)}; + + // Search for each HIP API in the extracted test .cc files. + for(HipAPI& hip_api: hip_apis) { + searchForAPI(hip_api, test_module_files); + } + + std::vector hip_api_groups; + for (auto const& api_group_name: api_group_names) { + HipAPIGroup hip_api_group{api_group_name, hip_apis}; + // Avoid having duplicated groups. + if (std::find(hip_api_groups.begin(), hip_api_groups.end(), hip_api_group) == hip_api_groups.end()) { + hip_api_groups.push_back(hip_api_group); + } + } + + std::cout << "Generating XML report files." << std::endl; + generateXMLReportFiles(hip_apis, hip_api_groups); + std::cout << "Generating HTML report files." << std::endl; + generateHTMLReportFiles(hip_apis, hip_api_groups, tests_root_directory, hip_api_header_file, hip_rtc_header_file); + + return 0; +} diff --git a/utils/coverage/reportGenerators.cpp b/utils/coverage/reportGenerators.cpp new file mode 100644 index 0000000000..7b74282929 --- /dev/null +++ b/utils/coverage/reportGenerators.cpp @@ -0,0 +1,307 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "reportGenerators.h" + +BasicAPIStats::BasicAPIStats(std::vector& hip_api_groups): + number_of_called_apis{0}, number_of_not_called_apis{0}, + number_of_deprecated_apis{0}, total_number_of_api_calls{0}, + total_number_of_test_cases{0} +{ + for (auto const& hip_api_group: hip_api_groups) { + number_of_called_apis += hip_api_group.getNumberOfCalledAPIs(); + number_of_not_called_apis += hip_api_group.getNumberOfNotCalledAPIs(); + number_of_deprecated_apis += hip_api_group.getNumberOfDeprecatedAPIs(); + total_number_of_api_calls += hip_api_group.getTotalNumberOfCalls(); + total_number_of_test_cases += hip_api_group.getTotalNumberOfTestCases(); + } + total_number_of_apis = number_of_called_apis + number_of_not_called_apis + number_of_deprecated_apis; + tests_coverage_percentage = 100.f * number_of_called_apis / (number_of_called_apis + number_of_not_called_apis); +} + +float BasicAPIStats::getLowCoverageLimit() const { + return 40.f; +} + +float BasicAPIStats::getMediumCoverageLimit() const { + return 80.f; +} + +void generateXMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups) { + BasicAPIStats basic_stats{hip_api_groups}; + + std::cout << "Total number of HIP API calls: " << basic_stats.total_number_of_api_calls << std::endl; + std::cout << "Number of the HIP APIs that are called at least once: " << basic_stats.number_of_called_apis << std::endl; + std::cout << "Number of the HIP APIs that are not called at all: " << basic_stats.number_of_not_called_apis << std::endl; + std::cout << "Number of the HIP APIs that are marked as deprecated: " << basic_stats.number_of_deprecated_apis << std::endl; + std::cout << "Test coverage by implemented tests, for the HIP APIs that are not marked as deprecated: "; + std::cout << basic_stats.tests_coverage_percentage << "%" << std::endl; + + /* + Generate XML file that contains relevant information about test coverage. + The XML file is created using raw handling of XML files, as there is no need + for the additional 3rd party library that implements XML file CRUD operations. + */ + std::fstream coverage_report; + std::string report_file_name{"CoverageReport.xml"}; + coverage_report.open(report_file_name, std::ios::out); + + time_t now{time(nullptr)}; + std::string date{asctime(gmtime(&now))}; + coverage_report << "" << date << "\n"; + + coverage_report << "\n"; + + coverage_report << "\t\n\t\tTotal number of detected HIP APIs."; + coverage_report << "\n\t\t" << hip_apis.size() << "\n\t\n"; + + coverage_report << "\t\n\t\tTotal number of HIP API calls within test source files."; + coverage_report << "\n\t\t" << basic_stats.total_number_of_api_calls << "\n\t\n"; + + coverage_report << "\t\n\t\tNumber of the HIP APIs that are called at least once."; + coverage_report << "\n\t\t" << basic_stats.number_of_called_apis << "\n\t\n"; + + coverage_report << "\t\n\t\tNumber of the HIP APIs that are not called at all."; + coverage_report << "\n\t\t" << basic_stats.number_of_not_called_apis << "\n\t\n"; + + coverage_report << "\t\n\t\tNumber of the HIP APIs that are marked as deprecated."; + coverage_report << "\n\t\t" << basic_stats.number_of_deprecated_apis << "\n\t\n"; + + coverage_report << "\t\n\t\tTest coverage by implemented tests for the HIP APIs that are not marked as deprecated."; + coverage_report << "\n\t\t" << basic_stats.tests_coverage_percentage << "%\n\t"; + + coverage_report << "\n"; + + for (auto const& hip_api_group: hip_api_groups) { + coverage_report << hip_api_group.getBasicStatsHTML(); + } + + coverage_report.close(); + std::cout << "Generated XML report file " << findAbsolutePathOfFile(report_file_name) << std::endl; +} + +void generateHTMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, + std::string tests_root_directory, std::string hipApiHeaderFile, std::string hip_rtc_header_file) { + BasicAPIStats basic_stats{hip_api_groups}; + + std::fstream coverage_report; + // Main HTML report file. + std::string report_file_name{"./coverageReportHTML/CoverageReport.html"}; + // Directories used to store generated HTML files. + std::string test_modules_directory{"./coverageReportHTML/testModules"}; + std::string test_apis_directory{"./coverageReportHTML/testAPIs"}; + std::filesystem::create_directories(test_modules_directory); + std::filesystem::create_directories(test_apis_directory); + + coverage_report.open(report_file_name, std::ios::out); + + // Helper strings with tabs and newlines for better HTML formatting. + std::string one_tab{"\n\t"}; + std::string two_tabs{"\n\t\t"}; + std::string three_tabs{"\n\t\t\t"}; + std::string four_tabs{"\n\t\t\t\t"}; + std::string five_tabs{"\n\t\t\t\t\t"}; + std::string six_tabs{"\n\t\t\t\t\t\t"}; + + /* + Create HTML file which contains report from coverage. There is no need for + 3rd party HTML libraries as the HTML report file is pretty simple and only + consists of tables and appropriate data. + It is better to open CoverageReport.html file in browser and view page + source, as it is much more clear. + */ + coverage_report << ""; + coverage_report << "" << one_tab << ""; + coverage_report << one_tab << "HIP API Coverage report"; + coverage_report << one_tab << "" << one_tab << ""; + coverage_report << one_tab << "" << one_tab << ""; + coverage_report << two_tabs << ""; + coverage_report << two_tabs << "\n"; + coverage_report << two_tabs << "" << three_tabs << ""; + coverage_report << two_tabs << ""; + + coverage_report << two_tabs << "\n"; + coverage_report << one_tab << "
HIP API Coverage report
" << four_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + coverage_report << five_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + coverage_report << five_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + coverage_report << five_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + coverage_report << five_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + coverage_report << five_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + coverage_report << five_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + // Based on the tests coverage percentage, pick a color for displaying it. + std::string font_class; + if (basic_stats.tests_coverage_percentage < basic_stats.getLowCoverageLimit()) { + font_class = "headerCovTableEntryLo"; + } + else if (basic_stats.tests_coverage_percentage < basic_stats.getMediumCoverageLimit()) { + font_class = "headerCovTableEntryMed"; + } + else { + font_class = "headerCovTableEntryHi"; + } + + coverage_report << five_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << six_tabs << ""; + coverage_report << five_tabs << ""; + + coverage_report << five_tabs << ""; + coverage_report << four_tabs << "
Catch2 tests location:" << tests_root_directory << "Value
Source files included:" << hipApiHeaderFile << "Total number of detected HIP APIs:" << basic_stats.total_number_of_apis << "
" << hip_rtc_header_file << "HIP API calls within test source files:" << basic_stats.total_number_of_api_calls << "
Total number of test cases:" << basic_stats.total_number_of_test_cases << "
HIP APIs that are called at least once:" << basic_stats.number_of_called_apis << "
HIP APIs that are not called at all:" << basic_stats.number_of_not_called_apis << "
HIP APIs that are marked as deprecated:" << basic_stats.number_of_deprecated_apis << "
Test coverage by implemented tests for the HIP APIs:" << + std::fixed << std::setprecision(2) << basic_stats.tests_coverage_percentage << "%
"; + coverage_report << three_tabs << "
"; + + // Add info about HIP API Groups. + coverage_report << one_tab << "
"; + coverage_report << one_tab << ""; + coverage_report << two_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << two_tabs << ""; + + coverage_report << two_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << two_tabs << ""; + + /* + Get basic stats for each API Group in HTML format and append it to the main HTML. + Create an HTML page for each API Group for more detailed information, as they are + used as hyperlinks from the main HTML page. + */ + for (auto const& hip_api_group: hip_api_groups) { + coverage_report << hip_api_group.getBasicStatsHTML(); + + std::fstream coverage_module_report; + std::string report_module_file_name{test_modules_directory + "/" + hip_api_group.getName() + ".html"}; + coverage_module_report.open(report_module_file_name, std::ios::out); + coverage_module_report << hip_api_group.createHTMLReport(); + coverage_module_report.close(); + } + + coverage_report << two_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << three_tabs << ""; + coverage_report << two_tabs << ""; + + coverage_report << one_tab << "

ModuleHIP APIsHIP API CallsTest casesCalled APIsNot called APIsDeprecated APIsCoverage
" << basic_stats.total_number_of_apis << "" << basic_stats.total_number_of_api_calls << "" << basic_stats.total_number_of_test_cases << "" << basic_stats.number_of_called_apis << "" << basic_stats.number_of_not_called_apis << "" << basic_stats.number_of_deprecated_apis << "" << + std::fixed << std::setprecision(2) << basic_stats.tests_coverage_percentage << "%
"; + coverage_report << one_tab << "
"; + coverage_report << one_tab << "
"; + + coverage_report << one_tab << ""; + coverage_report << two_tabs << ""; + + time_t now{time(nullptr)}; + std::string date{asctime(gmtime(&now))}; + coverage_report << two_tabs << ""; + coverage_report << one_tab << "
Generated: " << date; + coverage_report << two_tabs << " UTC
"; + coverage_report << one_tab << "
"; + coverage_report << "\n\n"; + + coverage_report.close(); + + // Create HTML report for each API, as they are used as hyperlinks from Groups HTML. + for (auto const& hip_api: hip_apis) { + std::fstream coverage_api_report; + std::string report_api_file_name{test_apis_directory + "/" + hip_api.getName() + ".html"}; + coverage_api_report.open(report_api_file_name, std::ios::out); + coverage_api_report << hip_api.createHTMLReport(); + coverage_api_report.close(); + } + + std::cout << "Generated HTML report file " << findAbsolutePathOfFile(report_file_name) << std::endl; +} diff --git a/utils/coverage/reportGenerators.h b/utils/coverage/reportGenerators.h new file mode 100644 index 0000000000..60bb69a2a9 --- /dev/null +++ b/utils/coverage/reportGenerators.h @@ -0,0 +1,41 @@ +/* +Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "hipAPICoverageUtils.h" + +class BasicAPIStats { + public: + int number_of_called_apis; + int number_of_not_called_apis; + int number_of_deprecated_apis; + int total_number_of_api_calls; + int total_number_of_test_cases; + int total_number_of_apis; + float tests_coverage_percentage; + BasicAPIStats(std::vector& hip_api_groups); + float getLowCoverageLimit() const; + float getMediumCoverageLimit() const; +}; + +void generateXMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups); +void generateHTMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, + std::string tests_root_directory, std::string hipApiHeaderFile, std::string hip_rtc_header_file);