/* Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "reportGenerators.h" BasicAPIStats::BasicAPIStats(std::vector& hip_api_groups) : number_of_called_apis{0}, number_of_not_called_apis{0}, number_of_deprecated_apis{0}, total_number_of_api_calls{0}, total_number_of_test_cases{0} { for (auto const& hip_api_group : hip_api_groups) { number_of_called_apis += hip_api_group.getNumberOfCalledAPIs(); number_of_not_called_apis += hip_api_group.getNumberOfNotCalledAPIs(); number_of_deprecated_apis += hip_api_group.getNumberOfDeprecatedAPIs(); total_number_of_api_calls += hip_api_group.getTotalNumberOfCalls(); total_number_of_test_cases += hip_api_group.getTotalNumberOfTestCases(); } total_number_of_apis = number_of_called_apis + number_of_not_called_apis + number_of_deprecated_apis; tests_coverage_percentage = 100.f * number_of_called_apis / (number_of_called_apis + number_of_not_called_apis); } float BasicAPIStats::getLowCoverageLimit() const { return 40.f; } float BasicAPIStats::getMediumCoverageLimit() const { return 80.f; } void generateXMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, const std::string& work_directory) { BasicAPIStats basic_stats{hip_api_groups}; std::cout << "Total number of HIP API calls: " << basic_stats.total_number_of_api_calls << std::endl; std::cout << "Number of the HIP APIs that are called at least once: " << basic_stats.number_of_called_apis << std::endl; std::cout << "Number of the HIP APIs that are not called at all: " << basic_stats.number_of_not_called_apis << std::endl; std::cout << "Number of the HIP APIs that are marked as deprecated: " << basic_stats.number_of_deprecated_apis << std::endl; std::cout << "Test coverage by implemented tests, for the HIP APIs that are not marked as deprecated: "; std::cout << basic_stats.tests_coverage_percentage << "%" << std::endl; /* Generate XML file that contains relevant information about test coverage. The XML file is created using raw handling of XML files, as there is no need for the additional 3rd party library that implements XML file CRUD operations. */ std::fstream coverage_report; std::string report_file_name{work_directory + "/CoverageReport.xml"}; coverage_report.open(report_file_name, std::ios::out); time_t now{time(nullptr)}; std::string date{asctime(gmtime(&now))}; coverage_report << "" << date << "\n"; coverage_report << "\n"; coverage_report << "\t\n\t\tTotal number of detected HIP " "APIs."; coverage_report << "\n\t\t" << hip_apis.size() << "\n\t\n"; coverage_report << "\t\n\t\tTotal number of HIP API " "calls within test source files."; coverage_report << "\n\t\t" << basic_stats.total_number_of_api_calls << "\n\t\n"; coverage_report << "\t\n\t\tNumber of the HIP APIs that are called at " "least once."; coverage_report << "\n\t\t" << basic_stats.number_of_called_apis << "\n\t\n"; coverage_report << "\t\n\t\tNumber of the HIP APIs that are not " "called at all."; coverage_report << "\n\t\t" << basic_stats.number_of_not_called_apis << "\n\t\n"; coverage_report << "\t\n\t\tNumber of the HIP APIs that are marked " "as deprecated."; coverage_report << "\n\t\t" << basic_stats.number_of_deprecated_apis << "\n\t\n"; coverage_report << "\t\n\t\tTest coverage by implemented tests " "for the HIP APIs that are not marked as deprecated."; coverage_report << "\n\t\t" << basic_stats.tests_coverage_percentage << "%\n\t"; coverage_report << "\n"; for (auto const& hip_api_group : hip_api_groups) { coverage_report << hip_api_group.getBasicStatsHTML(); } coverage_report.close(); std::cout << "Generated XML report file " << findAbsolutePathOfFile(report_file_name) << std::endl; } void generateHTMLReportFiles(std::vector& hip_apis, std::vector& hip_api_groups, std::string tests_root_directory, std::string hipApiHeaderFile, std::string hip_rtc_header_file, const std::string& work_directory) { BasicAPIStats basic_stats{hip_api_groups}; std::fstream coverage_report; // Main HTML report file. std::string report_file_name{work_directory + "/coverageReportHTML/CoverageReport.html"}; // Directories used to store generated HTML files. std::string test_modules_directory{work_directory + "/coverageReportHTML/testModules"}; std::string test_apis_directory{work_directory + "/coverageReportHTML/testAPIs"}; std::filesystem::create_directories(test_modules_directory); std::filesystem::create_directories(test_apis_directory); coverage_report.open(report_file_name, std::ios::out); // Helper strings with tabs and newlines for better HTML formatting. std::string one_tab{"\n\t"}; std::string two_tabs{"\n\t\t"}; std::string three_tabs{"\n\t\t\t"}; std::string four_tabs{"\n\t\t\t\t"}; std::string five_tabs{"\n\t\t\t\t\t"}; std::string six_tabs{"\n\t\t\t\t\t\t"}; /* Create HTML file which contains report from coverage. There is no need for 3rd party HTML libraries as the HTML report file is pretty simple and only consists of tables and appropriate data. It is better to open CoverageReport.html file in browser and view page source, as it is much more clear. */ coverage_report << ""; coverage_report << "" << one_tab << ""; coverage_report << one_tab << "HIP API Coverage report"; coverage_report << one_tab << "" << one_tab << ""; coverage_report << one_tab << "" << one_tab << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << "\n"; coverage_report << two_tabs << "" << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << "\n"; coverage_report << one_tab << "
HIP API Coverage report
" << four_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; coverage_report << five_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; coverage_report << five_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; coverage_report << five_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; coverage_report << five_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; coverage_report << five_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; coverage_report << five_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; // Based on the tests coverage percentage, pick a color for displaying it. std::string font_class; if (basic_stats.tests_coverage_percentage < basic_stats.getLowCoverageLimit()) { font_class = "headerCovTableEntryLo"; } else if (basic_stats.tests_coverage_percentage < basic_stats.getMediumCoverageLimit()) { font_class = "headerCovTableEntryMed"; } else { font_class = "headerCovTableEntryHi"; } coverage_report << five_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << six_tabs << ""; coverage_report << five_tabs << ""; coverage_report << five_tabs << ""; coverage_report << four_tabs << "
Catch2 tests location:" << tests_root_directory << "Value
Source files included:" << hipApiHeaderFile << "Total number of detected HIP APIs:" << basic_stats.total_number_of_apis << "
" << hip_rtc_header_file << "HIP API calls within test source files:" << basic_stats.total_number_of_api_calls << "
Total number of test cases:" << basic_stats.total_number_of_test_cases << "
HIP APIs that are called at least once:" << basic_stats.number_of_called_apis << "
HIP APIs that are not called at all:" << basic_stats.number_of_not_called_apis << "
HIP APIs that are marked as deprecated:" << basic_stats.number_of_deprecated_apis << "
Test coverage by implemented tests for the HIP APIs:" << std::fixed << std::setprecision(2) << basic_stats.tests_coverage_percentage << "%
"; coverage_report << three_tabs << "
"; // Add info about HIP API Groups. coverage_report << one_tab << "
"; coverage_report << one_tab << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; /* Get basic stats for each API Group in HTML format and append it to the main HTML. Create an HTML page for each API Group for more detailed information, as they are used as hyperlinks from the main HTML page. */ for (auto const& hip_api_group : hip_api_groups) { if (hip_api_group.getTotalNumberOfAPIs() == 0) { std::cout << "[SKIP_FROM_COV] Couldn't detect any valid HIP API within module " << hip_api_group.getName() << " hence skipping its HTML generation" << std::endl; continue; } coverage_report << hip_api_group.getBasicStatsHTML(); std::fstream coverage_module_report; std::string report_module_file_name{test_modules_directory + "/" + hip_api_group.getName() + ".html"}; coverage_module_report.open(report_module_file_name, std::ios::out); coverage_module_report << hip_api_group.createHTMLReport(); coverage_module_report.close(); } coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << one_tab << "

ModuleHIP APIsHIP API CallsTest casesCalled APIsNot called APIsDeprecated APIsCoverage
" << basic_stats.total_number_of_apis << "" << basic_stats.total_number_of_api_calls << "" << basic_stats.total_number_of_test_cases << "" << basic_stats.number_of_called_apis << "" << basic_stats.number_of_not_called_apis << "" << basic_stats.number_of_deprecated_apis << "" << std::fixed << std::setprecision(2) << basic_stats.tests_coverage_percentage << "%
"; coverage_report << one_tab << "
"; coverage_report << one_tab << "
"; coverage_report << one_tab << ""; coverage_report << two_tabs << ""; coverage_report << one_tab << "
"; coverage_report << one_tab << "
"; coverage_report << one_tab << "
"; coverage_report << one_tab << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << two_tabs << ""; coverage_report << three_tabs << ""; coverage_report << three_tabs << ""; coverage_report << two_tabs << ""; coverage_report << one_tab << "
Color legend
Module is deprecated
Percentage of called APIs within a module is less than 40%
Percentage of called APIs within a module is between " "40% and 80%
Percentage of called APIs within a module is larger " "than 80%
"; coverage_report << one_tab << "
"; coverage_report << one_tab << "
"; coverage_report << one_tab << ""; coverage_report << two_tabs << ""; time_t now{time(nullptr)}; std::string date{asctime(gmtime(&now))}; coverage_report << two_tabs << ""; coverage_report << one_tab << "
Generated: " << date; coverage_report << two_tabs << " UTC
"; coverage_report << one_tab << "
"; coverage_report << "\n\n"; coverage_report.close(); // Create HTML report for each API, as they are used as hyperlinks from Groups HTML. for (auto const& hip_api : hip_apis) { std::fstream coverage_api_report; std::string report_api_file_name{test_apis_directory + "/" + hip_api.getName() + ".html"}; coverage_api_report.open(report_api_file_name, std::ios::out); coverage_api_report << hip_api.createHTMLReport(); coverage_api_report.close(); } std::cout << "Generated HTML report file " << findAbsolutePathOfFile(report_file_name) << std::endl; }