From 4e6ca773fa5b11be2a522d684d3e8dd25907b580 Mon Sep 17 00:00:00 2001 From: Chris Kitching Date: Thu, 26 Oct 2017 04:25:05 +0100 Subject: [PATCH] Decouple the statistics system from the code translation The original implementation had the statistics system woken very tightly into things like PPCallbacks, with counters duplicated in two places, and all the output code duplicated. This made it very difficult to alter the structure of the program without breaking the statistics system. Since the planned approach for solving the remaining preprocessor bugs needs the introduction of a custom FrontendAction, and such a restructure was incompatible with the way the statistics system was set up, this rewrite was required. 'tis rather simpler now, mind you :D This commit also fixes an issue where some stats were counted twice, and allows `-print-stats` to operate independently of `-stat-output`, allowing you to print stats to a file without printing them to a terminal (or vice-versa). [ROCm/clr commit: dd5a60054aae32ff4cf5eb315cec082186446ebc] --- .../clr/hipamd/hipify-clang/src/CUDA2HipMap.h | 10 +- .../clr/hipamd/hipify-clang/src/Cuda2Hip.cpp | 369 ++---------------- .../hipamd/hipify-clang/src/Statistics.cpp | 227 +++++++++++ .../clr/hipamd/hipify-clang/src/Statistics.h | 174 +++++++++ projects/clr/hipamd/hipify-clang/src/Types.h | 47 --- 5 files changed, 440 insertions(+), 387 deletions(-) create mode 100644 projects/clr/hipamd/hipify-clang/src/Statistics.cpp create mode 100644 projects/clr/hipamd/hipify-clang/src/Statistics.h delete mode 100644 projects/clr/hipamd/hipify-clang/src/Types.h diff --git a/projects/clr/hipamd/hipify-clang/src/CUDA2HipMap.h b/projects/clr/hipamd/hipify-clang/src/CUDA2HipMap.h index ecef9ce2b5..605acf7aac 100644 --- a/projects/clr/hipamd/hipify-clang/src/CUDA2HipMap.h +++ b/projects/clr/hipamd/hipify-clang/src/CUDA2HipMap.h @@ -4,15 +4,7 @@ #include #include -#include "Types.h" - -// TODO: This shouldn't really be here. More restructuring needed... -struct hipCounter { - llvm::StringRef hipName; - ConvTypes countType; - ApiTypes countApiType; - bool unsupported; -}; +#include "Statistics.h" #define HIP_UNSUPPORTED true diff --git a/projects/clr/hipamd/hipify-clang/src/Cuda2Hip.cpp b/projects/clr/hipamd/hipify-clang/src/Cuda2Hip.cpp index df3bfaf040..ae83d7f794 100644 --- a/projects/clr/hipamd/hipify-clang/src/Cuda2Hip.cpp +++ b/projects/clr/hipamd/hipify-clang/src/Cuda2Hip.cpp @@ -53,7 +53,6 @@ THE SOFTWARE. #include #include "CUDA2HipMap.h" -#include "Types.h" #include "LLVMCompat.h" #include "StringUtils.h" @@ -64,16 +63,6 @@ using namespace llvm; #define DEBUG_TYPE "cuda2hip" -const char *counterNames[CONV_LAST] = { - "version", "init", "device", "mem", "kern", "coord_func", "math_func", - "special_func", "stream", "event", "occupancy", "ctx", "peer", "module", - "cache", "exec", "err", "def", "tex", "gl", "graphics", - "surface", "jit", "d3d9", "d3d10", "d3d11", "vdpau", "egl", - "thread", "other", "include", "include_cuda_main_header", "type", "literal", - "numeric_literal"}; - -const char *apiNames[API_LAST] = { - "CUDA Driver API", "CUDA RT API", "CUBLAS API"}; // Set up the command line options static cl::OptionCategory ToolTemplateCategory("CUDA to HIP source translator options"); @@ -115,24 +104,10 @@ static cl::opt Examine("examine", static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage); -uint64_t countRepsTotal[CONV_LAST] = { 0 }; -uint64_t countApiRepsTotal[API_LAST] = { 0 }; -uint64_t countRepsTotalUnsupported[CONV_LAST] = { 0 }; -uint64_t countApiRepsTotalUnsupported[API_LAST] = { 0 }; -std::map cuda2hipConvertedTotal; -std::map cuda2hipUnconvertedTotal; - class Cuda2Hip { public: Cuda2Hip(Replacements& R, const std::string &srcFileName) : Replace(R), mainFileName(srcFileName) {} - uint64_t countReps[CONV_LAST] = { 0 }; - uint64_t countApiReps[API_LAST] = { 0 }; - uint64_t countRepsUnsupported[CONV_LAST] = { 0 }; - uint64_t countApiRepsUnsupported[API_LAST] = { 0 }; - std::map cuda2hipConverted; - std::map cuda2hipUnconverted; - std::set LOCs; enum msgTypes { HIPIFY_ERROR = 0, @@ -154,14 +129,15 @@ protected: virtual void insertReplacement(const Replacement &rep, const FullSourceLoc &fullSL) { llcompat::insertReplacement(Replace, rep); if (PrintStats) { - LOCs.insert(fullSL.getExpansionLineNumber()); + rep.getLength(); + Statistics::current().lineTouched(fullSL.getExpansionLineNumber()); + Statistics::current().bytesChanged(rep.getLength()); } } void insertHipHeaders(Cuda2Hip *owner, const SourceManager &SM) { - if (owner->countReps[CONV_INCLUDE_CUDA_MAIN_H] == 0 && countReps[CONV_INCLUDE_CUDA_MAIN_H] == 0 && Replace.size() > 0) { + if (Replace.size() > 0) { std::string repName = "#include "; - hipCounter counter = { repName, CONV_INCLUDE_CUDA_MAIN_H, API_RUNTIME }; - updateCounters(counter, repName); + Statistics::current().incrementCounter({repName, ConvTypes::CONV_INCLUDE_CUDA_MAIN_H, ApiTypes::API_RUNTIME}, "#include "); SourceLocation sl = SM.getLocForStartOfFile(SM.getMainFileID()); FullSourceLoc fullSL(sl, SM); Replacement Rep(SM, sl, 0, repName + "\n"); @@ -174,31 +150,6 @@ protected: llvm::errs() << "[HIPIFY] " << getMsgType(msgType) << ": " << mainFileName << ":" << fullSL.getExpansionLineNumber() << ":" << fullSL.getExpansionColumnNumber() << ": " << message << "\n"; } - virtual void updateCounters(const hipCounter &counter, const std::string &cudaName) { - if (!PrintStats) { - return; - } - - std::map& map = cuda2hipConverted; - std::map& mapTotal = cuda2hipConvertedTotal; - if (counter.unsupported) { - map = cuda2hipUnconverted; - mapTotal = cuda2hipUnconvertedTotal; - countRepsUnsupported[counter.countType]++; - countRepsTotalUnsupported[counter.countType]++; - countApiRepsUnsupported[counter.countApiType]++; - countApiRepsTotalUnsupported[counter.countApiType]++; - } else { - countReps[counter.countType]++; - countRepsTotal[counter.countType]++; - countApiReps[counter.countApiType]++; - countApiRepsTotal[counter.countApiType]++; - } - - map[cudaName]++; - mapTotal[cudaName]++; - } - void processString(StringRef s, SourceManager &SM, SourceLocation start) { size_t begin = 0; while ((begin = s.find("cu", begin)) != StringRef::npos) { @@ -207,8 +158,8 @@ protected: const auto found = CUDA_RENAMES_MAP().find(name); if (found != CUDA_RENAMES_MAP().end()) { StringRef repName = found->second.hipName; - hipCounter counter = {"", CONV_LITERAL, API_RUNTIME, found->second.unsupported}; - updateCounters(counter, name.str()); + hipCounter counter = {"[string literal]", ConvTypes::CONV_LITERAL, ApiTypes::API_RUNTIME, found->second.unsupported}; + Statistics::current().incrementCounter(counter, name.str()); if (!counter.unsupported) { SourceLocation sl = start.getLocWithOffset(begin + 1); Replacement Rep(SM, sl, name.size(), repName); @@ -266,7 +217,7 @@ public: return; } - updateCounters(found->second, file_name.str()); + Statistics::current().incrementCounter(found->second, file_name.str()); if (found->second.unsupported) { // An unsupported CUDA header? Oh dear. Print a warning. printHipifyMessage(*_sm, hash_loc, "Unsupported CUDA header used: " + file_name.str()); @@ -312,7 +263,7 @@ public: // So it's an identifier, but not CUDA? Boring. return; } - updateCounters(found->second, name.str()); + Statistics::current().incrementCounter(found->second, name.str()); SourceLocation sl = t.getLocation(); if (found->second.unsupported) { @@ -408,7 +359,7 @@ private: } const hipCounter& hipCtr = found->second; - updateCounters(found->second, name); + Statistics::current().incrementCounter(hipCtr, name); if (hipCtr.unsupported) { return true; // Silently fail when you find an unsupported member. @@ -416,7 +367,6 @@ private: } size_t length = name.size(); - updateCounters(found->second, name); Replacement Rep(*SM, sl, length, hipCtr.hipName); FullSourceLoc fullSL(sl, *SM); insertReplacement(Rep, fullSL); @@ -517,8 +467,8 @@ private: Replacement Rep(*SM, launchStart, length, OS.str()); FullSourceLoc fullSL(launchStart, *SM); insertReplacement(Rep, fullSL); - hipCounter counter = {"hipLaunchKernelGGL", CONV_KERN, API_RUNTIME}; - updateCounters(counter, refName.str()); + hipCounter counter = {"hipLaunchKernelGGL", ConvTypes::CONV_KERN, ApiTypes::API_RUNTIME}; + Statistics::current().incrementCounter(counter, refName.str()); return true; } return false; @@ -542,7 +492,7 @@ private: // TODO: Make a lookup table just for builtins to improve performance. const auto found = CUDA_IDENTIFIER_MAP.find(name); if (found != CUDA_IDENTIFIER_MAP.end()) { - updateCounters(found->second, name.str()); + Statistics::current().incrementCounter(found->second, name.str()); if (!found->second.unsupported) { StringRef repName = found->second.hipName; Replacement Rep(*SM, sl, name.size(), repName); @@ -569,7 +519,7 @@ private: // TODO: Make a lookup table just for enum values to improve performance. const auto found = CUDA_IDENTIFIER_MAP.find(name); if (found != CUDA_IDENTIFIER_MAP.end()) { - updateCounters(found->second, name.str()); + Statistics::current().incrementCounter(found->second, name.str()); if (!found->second.unsupported) { StringRef repName = found->second.hipName; Replacement Rep(*SM, sl, name.size(), repName); @@ -665,8 +615,8 @@ private: Replacement Rep(*SM, slStart, repLength, repName); FullSourceLoc fullSL(slStart, *SM); insertReplacement(Rep, fullSL); - hipCounter counter = { "HIP_DYNAMIC_SHARED", CONV_MEM, API_RUNTIME }; - updateCounters(counter, refName.str()); + hipCounter counter = { "HIP_DYNAMIC_SHARED", ConvTypes::CONV_MEM, ApiTypes::API_RUNTIME }; + Statistics::current().incrementCounter(counter, refName.str()); } } return true; @@ -787,223 +737,6 @@ void addAllMatchers(ast_matchers::MatchFinder &Finder, Cuda2HipCallback *Callbac ); } -/** - * Print a named stat value to both the terminal and the CSV file. - */ -template -void printStat(std::ofstream &csv, const std::string& name, T value) { - llvm::outs() << " " << name << ": " << value << "\n"; - csv << name << ";" << value << "\n"; -} - -int64_t printStats(const std::string &csvFile, const std::string &srcFile, - HipifyPPCallbacks &PPCallbacks, Cuda2HipCallback &Callback, - uint64_t replacedBytes, uint64_t totalBytes, unsigned totalLines, - const std::chrono::steady_clock::time_point &start) { - std::ofstream csv(csvFile, std::ios::app); - int64_t sum = 0, sum_interm = 0; - std::string str; - const std::string hipify_info = "[HIPIFY] info: ", separator = ";"; - for (int i = 0; i < CONV_LAST; i++) { - sum += Callback.countReps[i] + PPCallbacks.countReps[i]; - } - int64_t sum_unsupported = 0; - for (int i = 0; i < CONV_LAST; i++) { - sum_unsupported += Callback.countRepsUnsupported[i] + PPCallbacks.countRepsUnsupported[i]; - } - - if (sum > 0 || sum_unsupported > 0) { - str = "file \'" + srcFile + "\' statistics:\n"; - llvm::outs() << "\n" << hipify_info << str; - csv << "\n" << str; - - size_t changedLines = Callback.LOCs.size() + PPCallbacks.LOCs.size(); - - printStat(csv, "CONVERTED refs count", sum); - printStat(csv, "UNCONVERTED refs count", sum_unsupported); - printStat(csv, "CONVERSION %", 100 - std::lround(double(sum_unsupported * 100) / double(sum + sum_unsupported))); - printStat(csv, "REPLACED bytes", replacedBytes); - printStat(csv, "TOTAL bytes", totalBytes); - printStat(csv, "CHANGED lines of code", changedLines); - printStat(csv, "TOTAL lines of code", totalLines); - - if (totalBytes > 0) { - printStat(csv, "CODE CHANGED (in bytes) %", std::lround(double(replacedBytes * 100) / double(totalBytes))); - } - - if (totalLines > 0) { - printStat(csv, "CODE CHANGED (in lines) %", std::lround(double(changedLines * 100) / double(totalLines))); - } - - typedef std::chrono::duration duration; - duration elapsed = std::chrono::steady_clock::now() - start; - std::stringstream stream; - stream << std::fixed << std::setprecision(2) << elapsed.count() / 1000; - printStat(csv, "TIME ELAPSED s", stream.str()); - } - - if (sum > 0) { - llvm::outs() << hipify_info << "CONVERTED refs by type:\n"; - csv << "\nCUDA ref type" << separator << "Count\n"; - for (int i = 0; i < CONV_LAST; i++) { - sum_interm = Callback.countReps[i] + PPCallbacks.countReps[i]; - if (0 == sum_interm) { - continue; - } - printStat(csv, counterNames[i], sum_interm); - } - llvm::outs() << hipify_info << "CONVERTED refs by API:\n"; - csv << "\nCUDA API" << separator << "Count\n"; - for (int i = 0; i < API_LAST; i++) { - printStat(csv, apiNames[i], Callback.countApiReps[i] + PPCallbacks.countApiReps[i]); - } - for (const auto & it : PPCallbacks.cuda2hipConverted) { - const auto found = Callback.cuda2hipConverted.find(it.first); - if (found == Callback.cuda2hipConverted.end()) { - Callback.cuda2hipConverted.insert(std::pair(it.first, 1)); - } else { - found->second += it.second; - } - } - llvm::outs() << hipify_info << "CONVERTED refs by names:\n"; - csv << "\nCUDA ref name" << separator << "Count\n"; - for (const auto & it : Callback.cuda2hipConverted) { - printStat(csv, it.first, it.second); - } - } - - if (sum_unsupported > 0) { - str = "UNCONVERTED refs by type:"; - llvm::outs() << hipify_info << str << "\n"; - csv << "\nUNCONVERTED CUDA ref type" << separator << "Count\n"; - for (int i = 0; i < CONV_LAST; i++) { - sum_interm = Callback.countRepsUnsupported[i] + PPCallbacks.countRepsUnsupported[i]; - if (0 == sum_interm) { - continue; - } - printStat(csv, counterNames[i], sum_interm); - } - llvm::outs() << hipify_info << "UNCONVERTED refs by API:\n"; - csv << "\nUNCONVERTED CUDA API" << separator << "Count\n"; - for (int i = 0; i < API_LAST; i++) { - printStat(csv, apiNames[i], Callback.countApiRepsUnsupported[i] + PPCallbacks.countApiRepsUnsupported[i]); - } - for (const auto & it : PPCallbacks.cuda2hipUnconverted) { - const auto found = Callback.cuda2hipUnconverted.find(it.first); - if (found == Callback.cuda2hipUnconverted.end()) { - Callback.cuda2hipUnconverted.insert(std::pair(it.first, 1)); - } else { - found->second += it.second; - } - } - - llvm::outs() << hipify_info << "UNCONVERTED refs by names:\n"; - csv << "\nUNCONVERTED CUDA ref name" << separator << "Count\n"; - for (const auto & it : Callback.cuda2hipUnconverted) { - printStat(csv, it.first, it.second); - } - } - - csv.close(); - return sum; -} - -void printAllStats(const std::string &csvFile, int64_t totalFiles, int64_t convertedFiles, - uint64_t replacedBytes, uint64_t totalBytes, unsigned changedLines, unsigned totalLines, - const std::chrono::steady_clock::time_point &start) { - std::ofstream csv(csvFile, std::ios::app); - int64_t sum = 0, sum_interm = 0; - std::string str; - const std::string hipify_info = "[HIPIFY] info: ", separator = ";"; - for (int i = 0; i < CONV_LAST; i++) { - sum += countRepsTotal[i]; - } - int64_t sum_unsupported = 0; - for (int i = 0; i < CONV_LAST; i++) { - sum_unsupported += countRepsTotalUnsupported[i]; - } - - if (sum > 0 || sum_unsupported > 0) { - str = "TOTAL statistics:\n"; - llvm::outs() << "\n" << hipify_info << str; - csv << "\n" << str; - printStat(csv, "CONVERTED files", convertedFiles); - printStat(csv, "PROCESSED files", totalFiles); - printStat(csv, "CONVERTED refs count", sum); - printStat(csv, "UNCONVERTED refs count", sum_unsupported); - printStat(csv, "CONVERSION %", 100 - std::lround(double(sum_unsupported * 100) / double(sum + sum_unsupported))); - printStat(csv, "REPLACED bytes", replacedBytes); - printStat(csv, "TOTAL bytes", totalBytes); - printStat(csv, "CHANGED lines of code", changedLines); - printStat(csv, "TOTAL lines of code", totalLines); - if (totalBytes > 0) { - printStat(csv, "CODE CHANGED (in bytes) %", std::lround(double(replacedBytes * 100) / double(totalBytes))); - } - if (totalLines > 0) { - printStat(csv, "CODE CHANGED (in lines) %", std::lround(double(changedLines * 100) / double(totalLines))); - } - - typedef std::chrono::duration duration; - duration elapsed = std::chrono::steady_clock::now() - start; - std::stringstream stream; - stream << std::fixed << std::setprecision(2) << elapsed.count() / 1000; - printStat(csv, "TIME ELAPSED s", stream.str()); - } - - if (sum > 0) { - llvm::outs() << hipify_info << "CONVERTED refs by type:\n"; - csv << "\nCUDA ref type" << separator << "Count\n"; - for (int i = 0; i < CONV_LAST; i++) { - sum_interm = countRepsTotal[i]; - if (0 == sum_interm) { - continue; - } - - printStat(csv, counterNames[i], sum_interm); - } - - llvm::outs() << hipify_info << "CONVERTED refs by API:\n"; - csv << "\nCUDA API" << separator << "Count\n"; - for (int i = 0; i < API_LAST; i++) { - printStat(csv, apiNames[i], countApiRepsTotal[i]); - } - - llvm::outs() << hipify_info << "CONVERTED refs by names:\n"; - csv << "\nCUDA ref name" << separator << "Count\n"; - for (const auto & it : cuda2hipConvertedTotal) { - printStat(csv, it.first, it.second); - } - } - - if (sum_unsupported > 0) { - str = "UNCONVERTED refs by type:"; - llvm::outs() << hipify_info << str << "\n"; - csv << "\nUNCONVERTED CUDA ref type" << separator << "Count\n"; - for (int i = 0; i < CONV_LAST; i++) { - sum_interm = countRepsTotalUnsupported[i]; - if (0 == sum_interm) { - continue; - } - - printStat(csv, counterNames[i], sum_interm); - } - - llvm::outs() << hipify_info << "UNCONVERTED refs by API:\n"; - csv << "\nUNCONVERTED CUDA API" << separator << "Count\n"; - for (int i = 0; i < API_LAST; i++) { - printStat(csv, apiNames[i], countApiRepsTotalUnsupported[i]); - } - - llvm::outs() << hipify_info << "UNCONVERTED refs by names:\n"; - csv << "\nUNCONVERTED CUDA ref name" << separator << "Count\n"; - for (const auto & it : cuda2hipUnconvertedTotal) { - printStat(csv, it.first, it.second); - } - } - - csv.close(); -} - void copyFile(const std::string& src, const std::string& dst) { std::ifstream source(src, std::ios::binary); std::ofstream dest(dst, std::ios::binary); @@ -1011,9 +744,6 @@ void copyFile(const std::string& src, const std::string& dst) { } int main(int argc, const char **argv) { - auto start = std::chrono::steady_clock::now(); - auto begin = start; - llcompat::PrintStackTraceOnErrorSignal(); CommonOptionsParser OptionsParser(argc, argv, ToolTemplateCategory, llvm::cl::OneOrMore); @@ -1023,6 +753,7 @@ int main(int argc, const char **argv) { llvm::errs() << "[HIPIFY] conflict: -o and multiple source files are specified.\n"; return 1; } + if (NoOutput) { if (Inplace) { llvm::errs() << "[HIPIFY] conflict: both -no-output and -inplace options are specified.\n"; @@ -1033,24 +764,23 @@ int main(int argc, const char **argv) { return 1; } } + if (Examine) { NoOutput = PrintStats = true; } + int Result = 0; - std::string csv; + + // Arguments for the Statistics print routines. + std::unique_ptr csv = nullptr; + llvm::raw_ostream* statPrint = nullptr; if (!OutputStatsFilename.empty()) { - csv = OutputStatsFilename; - } else { - csv = "hipify_stats.csv"; + csv = std::unique_ptr(new std::ofstream(OutputStatsFilename, std::ios_base::trunc)); } - size_t filesTranslated = fileSources.size(); - uint64_t repBytesTotal = 0; - uint64_t bytesTotal = 0; - unsigned changedLinesTotal = 0; - unsigned linesTotal = 0; - if (PrintStats && filesTranslated > 1) { - std::remove(csv.c_str()); + if (PrintStats) { + statPrint = &llvm::errs(); } + for (const auto & src : fileSources) { if (dst.empty()) { if (Inplace) { @@ -1070,6 +800,9 @@ int main(int argc, const char **argv) { // Should we fail for some reason, we'll just leak this file and not corrupt the input. copyFile(src, tmpFile); + // Initialise the statistics counters for this file. + Statistics::setActive(src); + // RefactoringTool operates on the file in-place. Giving it the output path is no good, // because that'll break relative includes, and we don't want to overwrite the input file. // So what we do is operate on a copy, which we then move to the output. @@ -1101,24 +834,8 @@ int main(int argc, const char **argv) { TextDiagnosticPrinter DiagnosticPrinter(llvm::errs(), &*DiagOpts); DiagnosticsEngine Diagnostics(IntrusiveRefCntPtr(new DiagnosticIDs()), &*DiagOpts, &DiagnosticPrinter, false); - uint64_t repBytes = 0; - uint64_t bytes = 0; - unsigned lines = 0; SourceManager SM(Diagnostics, Tool.getFiles()); - if (PrintStats) { - DEBUG(dbgs() << "Replacements collected by the tool:\n"); - Replacements& replacements = llcompat::getReplacements(Tool, tmpFile); - for (const auto &replacement : replacements) { - DEBUG(dbgs() << replacement.toString() << "\n"); - repBytes += replacement.getLength(); - } - std::ifstream src_file(dst, std::ios::binary | std::ios::ate); - src_file.clear(); - src_file.seekg(0); - lines = std::count(std::istreambuf_iterator(src_file), std::istreambuf_iterator(), '\n'); - bytes = src_file.tellg(); - } Rewriter Rewrite(SM, DefaultLangOptions); if (!Tool.applyAllReplacements(Rewrite)) { DEBUG(dbgs() << "Skipped some replacements.\n"); @@ -1131,26 +848,16 @@ int main(int argc, const char **argv) { } else { remove(tmpFile.c_str()); } - if (PrintStats) { - if (fileSources.size() == 1) { - if (OutputStatsFilename.empty()) { - csv = dst + ".csv"; - } - std::remove(csv.c_str()); - } - if (0 == printStats(csv, src, *PPCallbacks, Callback, repBytes, bytes, lines, start)) { - filesTranslated--; - } - start = std::chrono::steady_clock::now(); - repBytesTotal += repBytes; - bytesTotal += bytes; - changedLinesTotal += PPCallbacks->LOCs.size() + Callback.LOCs.size(); - linesTotal += lines; - } + + Statistics::current().markCompletion(); + Statistics::current().print(csv.get(), statPrint); + dst.clear(); } - if (PrintStats && fileSources.size() > 1) { - printAllStats(csv, fileSources.size(), filesTranslated, repBytesTotal, bytesTotal, changedLinesTotal, linesTotal, begin); + + if (fileSources.size() > 1) { + Statistics::printAggregate(csv.get(), statPrint); } + return Result; } diff --git a/projects/clr/hipamd/hipify-clang/src/Statistics.cpp b/projects/clr/hipamd/hipify-clang/src/Statistics.cpp new file mode 100644 index 0000000000..00b8a66f11 --- /dev/null +++ b/projects/clr/hipamd/hipify-clang/src/Statistics.cpp @@ -0,0 +1,227 @@ +#include "Statistics.h" +#include +#include +#include + + +const char *counterNames[NUM_CONV_TYPES] = { + "version", "init", "device", "mem", "kern", "coord_func", "math_func", + "special_func", "stream", "event", "occupancy", "ctx", "peer", "module", + "cache", "exec", "err", "def", "tex", "gl", "graphics", + "surface", "jit", "d3d9", "d3d10", "d3d11", "vdpau", "egl", + "thread", "other", "include", "include_cuda_main_header", "type", "literal", + "numeric_literal" +}; + +const char *apiNames[NUM_API_TYPES] = { + "CUDA Driver API", "CUDA RT API", "CUBLAS API" +}; + +namespace { + +template +void conditionalPrint(ST *stream1, + ST2* stream2, + const std::string& s1, + const std::string& s2) { + if (stream1) { + *stream1 << s1; + } + + if (stream2) { + *stream2 << s2; + } +} + + +/** + * Print a named stat value to both the terminal and the CSV file. + */ +template +void printStat(std::ostream *csv, llvm::raw_ostream* printOut, const std::string &name, T value) { + if (printOut) { + *printOut << " " << name << ": " << value << "\n"; + } + + if (csv) { + *csv << name << ";" << value << "\n"; + } +} + + +} // Anonymous namespace + +void StatCounter::incrementCounter(const hipCounter& counter, std::string name) { + counters[name]++; + apiCounters[(int) counter.countApiType]++; + convTypeCounters[(int) counter.countType]++; +} + +void StatCounter::add(const StatCounter& other) { + for (const auto& p : other.counters) { + counters[p.first] += p.second; + } + + for (int i = 0; i < NUM_API_TYPES; i++) { + apiCounters[i] += other.apiCounters[i]; + } + + for (int i = 0; i < NUM_CONV_TYPES; i++) { + convTypeCounters[i] += other.convTypeCounters[i]; + } +} + +int StatCounter::getConvSum() { + int acc = 0; + for (const int& i : convTypeCounters) { + acc += i; + } + + return acc; +} + +void StatCounter::print(std::ostream* csv, llvm::raw_ostream* printOut, std::string prefix) { + conditionalPrint(csv, printOut, "\nCUDA ref type;Count\n", "[HIPIFY] info: " + prefix + " refs by type:\n"); + for (int i = 0; i < NUM_CONV_TYPES; i++) { + if (convTypeCounters[i] > 0) { + printStat(csv, printOut, counterNames[i], convTypeCounters[i]); + } + } + + conditionalPrint(csv, printOut, "\nCUDA API;Count\n", "[HIPIFY] info: " + prefix + " refs by API:\n"); + for (int i = 0; i < NUM_API_TYPES; i++) { + printStat(csv, printOut, apiNames[i], apiCounters[i]); + } + + conditionalPrint(csv, printOut, "\nCUDA ref name;Count\n", "[HIPIFY] info: " + prefix + " refs by names:\n"); + for (const auto &it : counters) { + printStat(csv, printOut, it.first, it.second); + } +} + + +Statistics::Statistics(std::string name): fileName(name) { + // Compute the total bytes/lines in the input file. + std::ifstream src_file(name, std::ios::binary | std::ios::ate); + src_file.clear(); + src_file.seekg(0); + totalLines = (int) std::count(std::istreambuf_iterator(src_file), std::istreambuf_iterator(), '\n'); + totalBytes = (int) src_file.tellg(); + + // Mark the start time... + startTime = chr::steady_clock::now(); +}; + + +///////// Counter update routines ////////// + +void Statistics::incrementCounter(const hipCounter &counter, std::string name) { + if (counter.unsupported) { + unsupported.incrementCounter(counter, name); + } else { + supported.incrementCounter(counter, name); + } +} + +void Statistics::add(const Statistics &other) { + supported.add(other.supported); + unsupported.add(other.unsupported); + totalBytes += other.totalBytes; + totalLines += other.totalLines; + touchedBytes += other.touchedBytes; +} + +void Statistics::lineTouched(int lineNumber) { + touchedLines.insert(lineNumber); +} +void Statistics::bytesChanged(int bytes) { + touchedBytes += bytes; +} +void Statistics::markCompletion() { + completionTime = chr::steady_clock::now(); +} + + +///////// Output functions ////////// + +void Statistics::print(std::ostream* csv, llvm::raw_ostream* printOut, bool skipHeader) { + if (!skipHeader) { + std::string str = "file \'" + fileName + "\' statistics:\n"; + conditionalPrint(csv, printOut, "\n" + str, "\n[HIPIFY] info: " + str); + } + + size_t changedLines = touchedLines.size(); + + // Total number of (un)supported refs that were converted. + int supportedSum = supported.getConvSum(); + int unsupportedSum = unsupported.getConvSum(); + + printStat(csv, printOut, "CONVERTED refs count", supportedSum); + printStat(csv, printOut, "UNCONVERTED refs count", unsupportedSum); + printStat(csv, printOut, "CONVERSION %", 100 - std::lround(double(unsupportedSum * 100) / double(supportedSum + unsupportedSum))); + printStat(csv, printOut, "REPLACED bytes", touchedBytes); + printStat(csv, printOut, "TOTAL bytes", totalBytes); + printStat(csv, printOut, "CHANGED lines of code", changedLines); + printStat(csv, printOut, "TOTAL lines of code", totalLines); + + if (totalBytes > 0) { + printStat(csv, printOut, "CODE CHANGED (in bytes) %", std::lround(double(touchedBytes * 100) / double(totalBytes))); + } + + if (totalLines > 0) { + printStat(csv, printOut, "CODE CHANGED (in lines) %", std::lround(double(changedLines * 100) / double(totalLines))); + } + + typedef std::chrono::duration duration; + duration elapsed = completionTime - startTime; + std::stringstream stream; + stream << std::fixed << std::setprecision(2) << elapsed.count() / 1000; + printStat(csv, printOut, "TIME ELAPSED s", stream.str()); + + supported.print(csv, printOut, "CONVERTED"); + unsupported.print(csv, printOut, "UNCONVERTED"); +} + +void Statistics::printAggregate(std::ostream *csv, llvm::raw_ostream* printOut) { + Statistics globalStats = getAggregate(); + + conditionalPrint(csv, printOut, "\nTOTAL statistics:\n", "\n[HIPIFY] info: TOTAL statistics:\n"); + + // A file is considered "converted" if we made any changes to it. + int convertedFiles = 0; + for (const auto& p : stats) { + if (!p.second.touchedLines.empty()) { + convertedFiles++; + } + } + + printStat(csv, printOut, "CONVERTED files", convertedFiles); + printStat(csv, printOut, "PROCESSED files", stats.size()); + + globalStats.print(csv, printOut); +} + +//// Static state management //// + +Statistics Statistics::getAggregate() { + Statistics globalStats("global"); + + for (const auto& p : stats) { + globalStats.add(p.second); + } + + return globalStats; +} + +Statistics& Statistics::current() { + assert(Statistics::currentStatistics); + return *Statistics::currentStatistics; +} + +void Statistics::setActive(std::string name) { + stats.emplace(std::make_pair(name, Statistics{name})); + Statistics::currentStatistics = &stats.at(name); +} + +std::map Statistics::stats = {}; +Statistics* Statistics::currentStatistics = nullptr; diff --git a/projects/clr/hipamd/hipify-clang/src/Statistics.h b/projects/clr/hipamd/hipify-clang/src/Statistics.h new file mode 100644 index 0000000000..da4e296db0 --- /dev/null +++ b/projects/clr/hipamd/hipify-clang/src/Statistics.h @@ -0,0 +1,174 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace chr = std::chrono; + +enum ConvTypes { + CONV_VERSION = 0, + CONV_INIT, + CONV_DEVICE, + CONV_MEM, + CONV_KERN, + CONV_COORD_FUNC, + CONV_MATH_FUNC, + CONV_SPECIAL_FUNC, + CONV_STREAM, + CONV_EVENT, + CONV_OCCUPANCY, + CONV_CONTEXT, + CONV_PEER, + CONV_MODULE, + CONV_CACHE, + CONV_EXEC, + CONV_ERROR, + CONV_DEF, + CONV_TEX, + CONV_GL, + CONV_GRAPHICS, + CONV_SURFACE, + CONV_JIT, + CONV_D3D9, + CONV_D3D10, + CONV_D3D11, + CONV_VDPAU, + CONV_EGL, + CONV_THREAD, + CONV_OTHER, + CONV_INCLUDE, + CONV_INCLUDE_CUDA_MAIN_H, + CONV_TYPE, + CONV_LITERAL, + CONV_NUMERIC_LITERAL, + CONV_LAST +}; +constexpr int NUM_CONV_TYPES = (int) ConvTypes::CONV_LAST; + +enum ApiTypes { + API_DRIVER = 0, + API_RUNTIME, + API_BLAS, + API_LAST +}; +constexpr int NUM_API_TYPES = (int) ApiTypes::API_LAST; + +// The names of various fields in in the statistics reports. +extern const char *counterNames[NUM_CONV_TYPES]; +extern const char *apiNames[NUM_API_TYPES]; + + +struct hipCounter { + llvm::StringRef hipName; + ConvTypes countType; + ApiTypes countApiType; + bool unsupported; +}; + + +/** + * Tracks a set of named counters, as well as counters for each of the type enums defined above. + */ +class StatCounter { +private: + // Each thing we track is either "supported" or "unsupported"... + std::map counters; + + int apiCounters[NUM_API_TYPES] = {}; + int convTypeCounters[NUM_CONV_TYPES] = {}; + +public: + void incrementCounter(const hipCounter& counter, std::string name); + + /** + * Add the counters from `other` onto the counters of this object. + */ + void add(const StatCounter& other); + + int getConvSum(); + + void print(std::ostream* csv, llvm::raw_ostream* printOut, std::string prefix); +}; + +/** + * Tracks the statistics for a single input file. + */ +class Statistics { + StatCounter supported; + StatCounter unsupported; + + std::string fileName; + + std::set touchedLines = {}; + int touchedBytes = 0; + + int totalLines = 0; + int totalBytes = 0; + + chr::steady_clock::time_point startTime; + chr::steady_clock::time_point completionTime; + +public: + Statistics(std::string name); + + void incrementCounter(const hipCounter &counter, std::string name); + + /** + * Add the counters from `other` onto the counters of this object. + */ + void add(const Statistics &other); + + void lineTouched(int lineNumber); + void bytesChanged(int bytes); + + /** + * Set the completion timestamp to now. + */ + void markCompletion(); + + /////// Output functions /////// + +public: + /** + * Pretty-print the statistics stored in this object. + * + * @param csv Pointer to an output stream for the CSV to write. If null, no CSV is written + * @param printOut Pointer to an output stream to print human-readable textual stats to. If null, no + * such stats are produced. + */ + void print(std::ostream* csv, llvm::raw_ostream* printOut, bool skipHeader = false); + + /// Print aggregated statistics for all registered counters. + static void printAggregate(std::ostream *csv, llvm::raw_ostream* printOut); + + /////// Static nonsense /////// + + // The Statistics for each input file. + static std::map stats; + + // The Statistics objects for the currently-being-processed input file. + static Statistics* currentStatistics; + + /** + * Aggregate statistics over all entries in `stats` and return the resulting Statistics object. + */ + static Statistics getAggregate(); + + /** + * Convenient global entry point for updating the "active" Statistics. Since we operate single-threadedly + * processing one file at a time, this allows us to simply expose the stats for the current file globally, + * simplifying things. + */ + static Statistics& current(); + + /** + * Set the active Statistics object to the named one, creating it if necessary, and write the completion + * timestamp into the currently active one. + */ + static void setActive(std::string name); +}; diff --git a/projects/clr/hipamd/hipify-clang/src/Types.h b/projects/clr/hipamd/hipify-clang/src/Types.h deleted file mode 100644 index f30e4895ca..0000000000 --- a/projects/clr/hipamd/hipify-clang/src/Types.h +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -enum ConvTypes { - CONV_VERSION = 0, - CONV_INIT, - CONV_DEVICE, - CONV_MEM, - CONV_KERN, - CONV_COORD_FUNC, - CONV_MATH_FUNC, - CONV_SPECIAL_FUNC, - CONV_STREAM, - CONV_EVENT, - CONV_OCCUPANCY, - CONV_CONTEXT, - CONV_PEER, - CONV_MODULE, - CONV_CACHE, - CONV_EXEC, - CONV_ERROR, - CONV_DEF, - CONV_TEX, - CONV_GL, - CONV_GRAPHICS, - CONV_SURFACE, - CONV_JIT, - CONV_D3D9, - CONV_D3D10, - CONV_D3D11, - CONV_VDPAU, - CONV_EGL, - CONV_THREAD, - CONV_OTHER, - CONV_INCLUDE, - CONV_INCLUDE_CUDA_MAIN_H, - CONV_TYPE, - CONV_LITERAL, - CONV_NUMERIC_LITERAL, - CONV_LAST -}; - -enum ApiTypes { - API_DRIVER = 0, - API_RUNTIME, - API_BLAS, - API_LAST -};