Decouple the statistics system from the code translation

The original implementation had the statistics system woken very
tightly into things like PPCallbacks, with counters duplicated
in two places, and all the output code duplicated. This made it
very difficult to alter the structure of the program without
breaking the statistics system.

Since the planned approach for solving the remaining preprocessor
bugs needs the introduction of a custom FrontendAction, and such
a restructure was incompatible with the way the statistics system
was set up, this rewrite was required.

'tis rather simpler now, mind you :D

This commit also fixes an issue where some stats were counted
twice, and allows `-print-stats` to operate independently of
`-stat-output`, allowing you to print stats to a file without
printing them to a terminal (or vice-versa).


[ROCm/clr commit: dd5a60054a]

projects/clr/hipamd/hipify-clang/src/CUDA2HipMap.h

@@ -4,15 +4,7 @@
 #include <set>
 #include <map>
 
-#include "Types.h"
-
-// TODO: This shouldn't really be here. More restructuring needed...
-struct hipCounter {
-    llvm::StringRef hipName;
-    ConvTypes countType;
-    ApiTypes countApiType;
-    bool unsupported;
-};
+#include "Statistics.h"
 
 #define HIP_UNSUPPORTED true
 

projects/clr/hipamd/hipify-clang/src/Cuda2Hip.cpp

@@ -53,7 +53,6 @@ THE SOFTWARE.
 #include <sstream>
 
 #include "CUDA2HipMap.h"
-#include "Types.h"
 #include "LLVMCompat.h"
 #include "StringUtils.h"
 
@@ -64,16 +63,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "cuda2hip"
 
-const char *counterNames[CONV_LAST] = {
-    "version",      "init",     "device",  "mem",       "kern",        "coord_func", "math_func",
-    "special_func", "stream",   "event",   "occupancy", "ctx",         "peer",       "module",
-    "cache",        "exec",     "err",     "def",       "tex",         "gl",         "graphics",
-    "surface",      "jit",      "d3d9",    "d3d10",     "d3d11",       "vdpau",      "egl",
-    "thread",       "other",    "include", "include_cuda_main_header", "type",       "literal",
-    "numeric_literal"};
-
-const char *apiNames[API_LAST] = {
-    "CUDA Driver API", "CUDA RT API", "CUBLAS API"};
 
 // Set up the command line options
 static cl::OptionCategory ToolTemplateCategory("CUDA to HIP source translator options");
@@ -115,24 +104,10 @@ static cl::opt<bool> Examine("examine",
 
 static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
 
-uint64_t countRepsTotal[CONV_LAST] = { 0 };
-uint64_t countApiRepsTotal[API_LAST] = { 0 };
-uint64_t countRepsTotalUnsupported[CONV_LAST] = { 0 };
-uint64_t countApiRepsTotalUnsupported[API_LAST] = { 0 };
-std::map<std::string, uint64_t> cuda2hipConvertedTotal;
-std::map<std::string, uint64_t> cuda2hipUnconvertedTotal;
-
 class Cuda2Hip {
 public:
   Cuda2Hip(Replacements& R, const std::string &srcFileName) :
     Replace(R), mainFileName(srcFileName) {}
-  uint64_t countReps[CONV_LAST] = { 0 };
-  uint64_t countApiReps[API_LAST] = { 0 };
-  uint64_t countRepsUnsupported[CONV_LAST] = { 0 };
-  uint64_t countApiRepsUnsupported[API_LAST] = { 0 };
-  std::map<std::string, uint64_t> cuda2hipConverted;
-  std::map<std::string, uint64_t> cuda2hipUnconverted;
-  std::set<unsigned> LOCs;
 
   enum msgTypes {
     HIPIFY_ERROR = 0,
@@ -154,14 +129,15 @@ protected:
   virtual void insertReplacement(const Replacement &rep, const FullSourceLoc &fullSL) {
     llcompat::insertReplacement(Replace, rep);
     if (PrintStats) {
-      LOCs.insert(fullSL.getExpansionLineNumber());
+      rep.getLength();
+      Statistics::current().lineTouched(fullSL.getExpansionLineNumber());
+      Statistics::current().bytesChanged(rep.getLength());
     }
   }
   void insertHipHeaders(Cuda2Hip *owner, const SourceManager &SM) {
-    if (owner->countReps[CONV_INCLUDE_CUDA_MAIN_H] == 0 && countReps[CONV_INCLUDE_CUDA_MAIN_H] == 0 && Replace.size() > 0) {
+    if (Replace.size() > 0) {
       std::string repName = "#include <hip/hip_runtime.h>";
-      hipCounter counter = { repName, CONV_INCLUDE_CUDA_MAIN_H, API_RUNTIME };
-      updateCounters(counter, repName);
+      Statistics::current().incrementCounter({repName, ConvTypes::CONV_INCLUDE_CUDA_MAIN_H, ApiTypes::API_RUNTIME}, "#include <cuda>");
       SourceLocation sl = SM.getLocForStartOfFile(SM.getMainFileID());
       FullSourceLoc fullSL(sl, SM);
       Replacement Rep(SM, sl, 0, repName + "\n");
@@ -174,31 +150,6 @@ protected:
     llvm::errs() << "[HIPIFY] " << getMsgType(msgType) << ": " << mainFileName << ":" << fullSL.getExpansionLineNumber() << ":" << fullSL.getExpansionColumnNumber() << ": " << message << "\n";
   }
 
-  virtual void updateCounters(const hipCounter &counter, const std::string &cudaName) {
-    if (!PrintStats) {
-      return;
-    }
-
-    std::map<std::string, uint64_t>& map = cuda2hipConverted;
-    std::map<std::string, uint64_t>& mapTotal = cuda2hipConvertedTotal;
-    if (counter.unsupported) {
-      map = cuda2hipUnconverted;
-      mapTotal = cuda2hipUnconvertedTotal;
-      countRepsUnsupported[counter.countType]++;
-      countRepsTotalUnsupported[counter.countType]++;
-      countApiRepsUnsupported[counter.countApiType]++;
-      countApiRepsTotalUnsupported[counter.countApiType]++;
-    } else {
-      countReps[counter.countType]++;
-      countRepsTotal[counter.countType]++;
-      countApiReps[counter.countApiType]++;
-      countApiRepsTotal[counter.countApiType]++;
-    }
-
-    map[cudaName]++;
-    mapTotal[cudaName]++;
-  }
-
   void processString(StringRef s, SourceManager &SM, SourceLocation start) {
     size_t begin = 0;
     while ((begin = s.find("cu", begin)) != StringRef::npos) {
@@ -207,8 +158,8 @@ protected:
       const auto found = CUDA_RENAMES_MAP().find(name);
       if (found != CUDA_RENAMES_MAP().end()) {
         StringRef repName = found->second.hipName;
-        hipCounter counter = {"", CONV_LITERAL, API_RUNTIME, found->second.unsupported};
-        updateCounters(counter, name.str());
+        hipCounter counter = {"[string literal]", ConvTypes::CONV_LITERAL, ApiTypes::API_RUNTIME, found->second.unsupported};
+        Statistics::current().incrementCounter(counter, name.str());
         if (!counter.unsupported) {
           SourceLocation sl = start.getLocWithOffset(begin + 1);
           Replacement Rep(SM, sl, name.size(), repName);
@@ -266,7 +217,7 @@ public:
       return;
     }
 
-    updateCounters(found->second, file_name.str());
+    Statistics::current().incrementCounter(found->second, file_name.str());
     if (found->second.unsupported) {
       // An unsupported CUDA header? Oh dear. Print a warning.
       printHipifyMessage(*_sm, hash_loc, "Unsupported CUDA header used: " + file_name.str());
@@ -312,7 +263,7 @@ public:
       // So it's an identifier, but not CUDA? Boring.
       return;
     }
-    updateCounters(found->second, name.str());
+    Statistics::current().incrementCounter(found->second, name.str());
 
     SourceLocation sl = t.getLocation();
     if (found->second.unsupported) {
@@ -408,7 +359,7 @@ private:
     }
 
     const hipCounter& hipCtr = found->second;
-    updateCounters(found->second, name);
+    Statistics::current().incrementCounter(hipCtr, name);
 
     if (hipCtr.unsupported) {
       return true; // Silently fail when you find an unsupported member.
@@ -416,7 +367,6 @@ private:
     }
 
     size_t length = name.size();
-    updateCounters(found->second, name);
     Replacement Rep(*SM, sl, length, hipCtr.hipName);
     FullSourceLoc fullSL(sl, *SM);
     insertReplacement(Rep, fullSL);
@@ -517,8 +467,8 @@ private:
       Replacement Rep(*SM, launchStart, length, OS.str());
       FullSourceLoc fullSL(launchStart, *SM);
       insertReplacement(Rep, fullSL);
-      hipCounter counter = {"hipLaunchKernelGGL", CONV_KERN, API_RUNTIME};
-      updateCounters(counter, refName.str());
+      hipCounter counter = {"hipLaunchKernelGGL", ConvTypes::CONV_KERN, ApiTypes::API_RUNTIME};
+      Statistics::current().incrementCounter(counter, refName.str());
       return true;
     }
     return false;
@@ -542,7 +492,7 @@ private:
           // TODO: Make a lookup table just for builtins to improve performance.
           const auto found = CUDA_IDENTIFIER_MAP.find(name);
           if (found != CUDA_IDENTIFIER_MAP.end()) {
-            updateCounters(found->second, name.str());
+            Statistics::current().incrementCounter(found->second, name.str());
             if (!found->second.unsupported) {
               StringRef repName = found->second.hipName;
               Replacement Rep(*SM, sl, name.size(), repName);
@@ -569,7 +519,7 @@ private:
       // TODO: Make a lookup table just for enum values to improve performance.
       const auto found = CUDA_IDENTIFIER_MAP.find(name);
       if (found != CUDA_IDENTIFIER_MAP.end()) {
-        updateCounters(found->second, name.str());
+        Statistics::current().incrementCounter(found->second, name.str());
         if (!found->second.unsupported) {
           StringRef repName = found->second.hipName;
           Replacement Rep(*SM, sl, name.size(), repName);
@@ -665,8 +615,8 @@ private:
           Replacement Rep(*SM, slStart, repLength, repName);
           FullSourceLoc fullSL(slStart, *SM);
           insertReplacement(Rep, fullSL);
-          hipCounter counter = { "HIP_DYNAMIC_SHARED", CONV_MEM, API_RUNTIME };
-          updateCounters(counter, refName.str());
+          hipCounter counter = { "HIP_DYNAMIC_SHARED", ConvTypes::CONV_MEM, ApiTypes::API_RUNTIME };
+          Statistics::current().incrementCounter(counter, refName.str());
         }
       }
       return true;
@@ -787,223 +737,6 @@ void addAllMatchers(ast_matchers::MatchFinder &Finder, Cuda2HipCallback *Callbac
   );
 }
 
-/**
- * Print a named stat value to both the terminal and the CSV file.
- */
-template<typename T>
-void printStat(std::ofstream &csv, const std::string& name, T value) {
-  llvm::outs() << "  " << name << ": " << value << "\n";
-  csv << name << ";" << value << "\n";
-}
-
-int64_t printStats(const std::string &csvFile, const std::string &srcFile,
-                   HipifyPPCallbacks &PPCallbacks, Cuda2HipCallback &Callback,
-                   uint64_t replacedBytes, uint64_t totalBytes, unsigned totalLines,
-                   const std::chrono::steady_clock::time_point &start) {
-  std::ofstream csv(csvFile, std::ios::app);
-  int64_t sum = 0, sum_interm = 0;
-  std::string str;
-  const std::string hipify_info = "[HIPIFY] info: ", separator = ";";
-  for (int i = 0; i < CONV_LAST; i++) {
-    sum += Callback.countReps[i] + PPCallbacks.countReps[i];
-  }
-  int64_t sum_unsupported = 0;
-  for (int i = 0; i < CONV_LAST; i++) {
-    sum_unsupported += Callback.countRepsUnsupported[i] + PPCallbacks.countRepsUnsupported[i];
-  }
-
-  if (sum > 0 || sum_unsupported > 0) {
-    str = "file \'" + srcFile + "\' statistics:\n";
-    llvm::outs() << "\n" << hipify_info << str;
-    csv << "\n" << str;
-
-    size_t changedLines = Callback.LOCs.size() + PPCallbacks.LOCs.size();
-
-    printStat(csv, "CONVERTED refs count", sum);
-    printStat(csv, "UNCONVERTED refs count", sum_unsupported);
-    printStat(csv, "CONVERSION %", 100 - std::lround(double(sum_unsupported * 100) / double(sum + sum_unsupported)));
-    printStat(csv, "REPLACED bytes", replacedBytes);
-    printStat(csv, "TOTAL bytes", totalBytes);
-    printStat(csv, "CHANGED lines of code", changedLines);
-    printStat(csv, "TOTAL lines of code", totalLines);
-
-    if (totalBytes > 0) {
-      printStat(csv, "CODE CHANGED (in bytes) %", std::lround(double(replacedBytes * 100) / double(totalBytes)));
-    }
-
-    if (totalLines > 0) {
-      printStat(csv, "CODE CHANGED (in lines) %", std::lround(double(changedLines * 100) / double(totalLines)));
-    }
-
-    typedef std::chrono::duration<double, std::milli> duration;
-    duration elapsed = std::chrono::steady_clock::now() - start;
-    std::stringstream stream;
-    stream << std::fixed << std::setprecision(2) << elapsed.count() / 1000;
-    printStat(csv, "TIME ELAPSED s", stream.str());
-  }
-
-  if (sum > 0) {
-    llvm::outs() << hipify_info << "CONVERTED refs by type:\n";
-    csv << "\nCUDA ref type" << separator << "Count\n";
-    for (int i = 0; i < CONV_LAST; i++) {
-      sum_interm = Callback.countReps[i] + PPCallbacks.countReps[i];
-      if (0 == sum_interm) {
-        continue;
-      }
-      printStat(csv, counterNames[i], sum_interm);
-    }
-    llvm::outs() << hipify_info << "CONVERTED refs by API:\n";
-    csv << "\nCUDA API" << separator << "Count\n";
-    for (int i = 0; i < API_LAST; i++) {
-      printStat(csv, apiNames[i], Callback.countApiReps[i] + PPCallbacks.countApiReps[i]);
-    }
-    for (const auto & it : PPCallbacks.cuda2hipConverted) {
-      const auto found = Callback.cuda2hipConverted.find(it.first);
-      if (found == Callback.cuda2hipConverted.end()) {
-        Callback.cuda2hipConverted.insert(std::pair<std::string, uint64_t>(it.first, 1));
-      } else {
-        found->second += it.second;
-      }
-    }
-    llvm::outs() << hipify_info << "CONVERTED refs by names:\n";
-    csv << "\nCUDA ref name" << separator << "Count\n";
-    for (const auto & it : Callback.cuda2hipConverted) {
-      printStat(csv, it.first, it.second);
-    }
-  }
-
-  if (sum_unsupported > 0) {
-    str = "UNCONVERTED refs by type:";
-    llvm::outs() << hipify_info << str << "\n";
-    csv << "\nUNCONVERTED CUDA ref type" << separator << "Count\n";
-    for (int i = 0; i < CONV_LAST; i++) {
-      sum_interm = Callback.countRepsUnsupported[i] + PPCallbacks.countRepsUnsupported[i];
-      if (0 == sum_interm) {
-        continue;
-      }
-      printStat(csv, counterNames[i], sum_interm);
-    }
-    llvm::outs() << hipify_info << "UNCONVERTED refs by API:\n";
-    csv << "\nUNCONVERTED CUDA API" << separator << "Count\n";
-    for (int i = 0; i < API_LAST; i++) {
-      printStat(csv, apiNames[i], Callback.countApiRepsUnsupported[i] + PPCallbacks.countApiRepsUnsupported[i]);
-    }
-    for (const auto & it : PPCallbacks.cuda2hipUnconverted) {
-      const auto found = Callback.cuda2hipUnconverted.find(it.first);
-      if (found == Callback.cuda2hipUnconverted.end()) {
-        Callback.cuda2hipUnconverted.insert(std::pair<std::string, uint64_t>(it.first, 1));
-      } else {
-        found->second += it.second;
-      }
-    }
-
-    llvm::outs() << hipify_info << "UNCONVERTED refs by names:\n";
-    csv << "\nUNCONVERTED CUDA ref name" << separator << "Count\n";
-    for (const auto & it : Callback.cuda2hipUnconverted) {
-      printStat(csv, it.first, it.second);
-    }
-  }
-
-  csv.close();
-  return sum;
-}
-
-void printAllStats(const std::string &csvFile, int64_t totalFiles, int64_t convertedFiles,
-                   uint64_t replacedBytes, uint64_t totalBytes, unsigned changedLines, unsigned totalLines,
-                   const std::chrono::steady_clock::time_point &start) {
-  std::ofstream csv(csvFile, std::ios::app);
-  int64_t sum = 0, sum_interm = 0;
-  std::string str;
-  const std::string hipify_info = "[HIPIFY] info: ", separator = ";";
-  for (int i = 0; i < CONV_LAST; i++) {
-    sum += countRepsTotal[i];
-  }
-  int64_t sum_unsupported = 0;
-  for (int i = 0; i < CONV_LAST; i++) {
-    sum_unsupported += countRepsTotalUnsupported[i];
-  }
-
-  if (sum > 0 || sum_unsupported > 0) {
-    str = "TOTAL statistics:\n";
-    llvm::outs() << "\n" << hipify_info << str;
-    csv << "\n" << str;
-    printStat(csv, "CONVERTED files", convertedFiles);
-    printStat(csv, "PROCESSED files", totalFiles);
-    printStat(csv, "CONVERTED refs count", sum);
-    printStat(csv, "UNCONVERTED refs count", sum_unsupported);
-    printStat(csv, "CONVERSION %", 100 - std::lround(double(sum_unsupported * 100) / double(sum + sum_unsupported)));
-    printStat(csv, "REPLACED bytes", replacedBytes);
-    printStat(csv, "TOTAL bytes", totalBytes);
-    printStat(csv, "CHANGED lines of code", changedLines);
-    printStat(csv, "TOTAL lines of code", totalLines);
-    if (totalBytes > 0) {
-      printStat(csv, "CODE CHANGED (in bytes) %", std::lround(double(replacedBytes * 100) / double(totalBytes)));
-    }
-    if (totalLines > 0) {
-      printStat(csv, "CODE CHANGED (in lines) %", std::lround(double(changedLines * 100) / double(totalLines)));
-    }
-
-    typedef std::chrono::duration<double, std::milli> duration;
-    duration elapsed = std::chrono::steady_clock::now() - start;
-    std::stringstream stream;
-    stream << std::fixed << std::setprecision(2) << elapsed.count() / 1000;
-    printStat(csv, "TIME ELAPSED s", stream.str());
-  }
-
-  if (sum > 0) {
-    llvm::outs() << hipify_info << "CONVERTED refs by type:\n";
-    csv << "\nCUDA ref type" << separator << "Count\n";
-    for (int i = 0; i < CONV_LAST; i++) {
-      sum_interm = countRepsTotal[i];
-      if (0 == sum_interm) {
-        continue;
-      }
-
-      printStat(csv, counterNames[i], sum_interm);
-    }
-
-    llvm::outs() << hipify_info << "CONVERTED refs by API:\n";
-    csv << "\nCUDA API" << separator << "Count\n";
-    for (int i = 0; i < API_LAST; i++) {
-      printStat(csv, apiNames[i], countApiRepsTotal[i]);
-    }
-
-    llvm::outs() << hipify_info << "CONVERTED refs by names:\n";
-    csv << "\nCUDA ref name" << separator << "Count\n";
-    for (const auto & it : cuda2hipConvertedTotal) {
-      printStat(csv, it.first, it.second);
-    }
-  }
-
-  if (sum_unsupported > 0) {
-    str = "UNCONVERTED refs by type:";
-    llvm::outs() << hipify_info << str << "\n";
-    csv << "\nUNCONVERTED CUDA ref type" << separator << "Count\n";
-    for (int i = 0; i < CONV_LAST; i++) {
-      sum_interm = countRepsTotalUnsupported[i];
-      if (0 == sum_interm) {
-        continue;
-      }
-
-      printStat(csv, counterNames[i], sum_interm);
-    }
-
-    llvm::outs() << hipify_info << "UNCONVERTED refs by API:\n";
-    csv << "\nUNCONVERTED CUDA API" << separator << "Count\n";
-    for (int i = 0; i < API_LAST; i++) {
-      printStat(csv, apiNames[i], countApiRepsTotalUnsupported[i]);
-    }
-
-    llvm::outs() << hipify_info << "UNCONVERTED refs by names:\n";
-    csv << "\nUNCONVERTED CUDA ref name" << separator << "Count\n";
-    for (const auto & it : cuda2hipUnconvertedTotal) {
-      printStat(csv, it.first, it.second);
-    }
-  }
-
-  csv.close();
-}
-
 void copyFile(const std::string& src, const std::string& dst) {
   std::ifstream source(src, std::ios::binary);
   std::ofstream dest(dst, std::ios::binary);
@@ -1011,9 +744,6 @@ void copyFile(const std::string& src, const std::string& dst) {
 }
 
 int main(int argc, const char **argv) {
-  auto start = std::chrono::steady_clock::now();
-  auto begin = start;
-
   llcompat::PrintStackTraceOnErrorSignal();
 
   CommonOptionsParser OptionsParser(argc, argv, ToolTemplateCategory, llvm::cl::OneOrMore);
@@ -1023,6 +753,7 @@ int main(int argc, const char **argv) {
     llvm::errs() << "[HIPIFY] conflict: -o and multiple source files are specified.\n";
     return 1;
   }
+
   if (NoOutput) {
     if (Inplace) {
       llvm::errs() << "[HIPIFY] conflict: both -no-output and -inplace options are specified.\n";
@@ -1033,24 +764,23 @@ int main(int argc, const char **argv) {
       return 1;
     }
   }
+
   if (Examine) {
     NoOutput = PrintStats = true;
   }
+
   int Result = 0;
-  std::string csv;
+
+  // Arguments for the Statistics print routines.
+  std::unique_ptr<std::ostream> csv = nullptr;
+  llvm::raw_ostream* statPrint = nullptr;
   if (!OutputStatsFilename.empty()) {
-    csv = OutputStatsFilename;
-  } else {
-    csv = "hipify_stats.csv";
+    csv = std::unique_ptr<std::ostream>(new std::ofstream(OutputStatsFilename, std::ios_base::trunc));
   }
-  size_t filesTranslated = fileSources.size();
-  uint64_t repBytesTotal = 0;
-  uint64_t bytesTotal = 0;
-  unsigned changedLinesTotal = 0;
-  unsigned linesTotal = 0;
-  if (PrintStats && filesTranslated > 1) {
-    std::remove(csv.c_str());
+  if (PrintStats) {
+    statPrint = &llvm::errs();
   }
+
   for (const auto & src : fileSources) {
     if (dst.empty()) {
       if (Inplace) {
@@ -1070,6 +800,9 @@ int main(int argc, const char **argv) {
     // Should we fail for some reason, we'll just leak this file and not corrupt the input.
     copyFile(src, tmpFile);
 
+    // Initialise the statistics counters for this file.
+    Statistics::setActive(src);
+
     // RefactoringTool operates on the file in-place. Giving it the output path is no good,
     // because that'll break relative includes, and we don't want to overwrite the input file.
     // So what we do is operate on a copy, which we then move to the output.
@@ -1101,24 +834,8 @@ int main(int argc, const char **argv) {
     TextDiagnosticPrinter DiagnosticPrinter(llvm::errs(), &*DiagOpts);
     DiagnosticsEngine Diagnostics(IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts, &DiagnosticPrinter, false);
 
-    uint64_t repBytes = 0;
-    uint64_t bytes = 0;
-    unsigned lines = 0;
     SourceManager SM(Diagnostics, Tool.getFiles());
-    if (PrintStats) {
-      DEBUG(dbgs() << "Replacements collected by the tool:\n");
 
-      Replacements& replacements = llcompat::getReplacements(Tool, tmpFile);
-      for (const auto &replacement : replacements) {
-        DEBUG(dbgs() << replacement.toString() << "\n");
-        repBytes += replacement.getLength();
-      }
-      std::ifstream src_file(dst, std::ios::binary | std::ios::ate);
-      src_file.clear();
-      src_file.seekg(0);
-      lines = std::count(std::istreambuf_iterator<char>(src_file), std::istreambuf_iterator<char>(), '\n');
-      bytes = src_file.tellg();
-    }
     Rewriter Rewrite(SM, DefaultLangOptions);
     if (!Tool.applyAllReplacements(Rewrite)) {
       DEBUG(dbgs() << "Skipped some replacements.\n");
@@ -1131,26 +848,16 @@ int main(int argc, const char **argv) {
     } else {
       remove(tmpFile.c_str());
     }
-    if (PrintStats) {
-      if (fileSources.size() == 1) {
-        if (OutputStatsFilename.empty()) {
-          csv = dst + ".csv";
-        }
-        std::remove(csv.c_str());
-      }
-      if (0 == printStats(csv, src, *PPCallbacks, Callback, repBytes, bytes, lines, start)) {
-        filesTranslated--;
-      }
-      start = std::chrono::steady_clock::now();
-      repBytesTotal += repBytes;
-      bytesTotal += bytes;
-      changedLinesTotal += PPCallbacks->LOCs.size() + Callback.LOCs.size();
-      linesTotal += lines;
-    }
+
+    Statistics::current().markCompletion();
+    Statistics::current().print(csv.get(), statPrint);
+
     dst.clear();
   }
-  if (PrintStats && fileSources.size() > 1) {
-    printAllStats(csv, fileSources.size(), filesTranslated, repBytesTotal, bytesTotal, changedLinesTotal, linesTotal, begin);
+
+  if (fileSources.size() > 1) {
+    Statistics::printAggregate(csv.get(), statPrint);
   }
+
   return Result;
 }

projects/clr/hipamd/hipify-clang/src/Statistics.cpp

@@ -0,0 +1,227 @@
+#include "Statistics.h"
+#include <assert.h>
+#include <sstream>
+#include <iomanip>
+
+
+const char *counterNames[NUM_CONV_TYPES] = {
+    "version", "init", "device", "mem", "kern", "coord_func", "math_func",
+    "special_func", "stream", "event", "occupancy", "ctx", "peer", "module",
+    "cache", "exec", "err", "def", "tex", "gl", "graphics",
+    "surface", "jit", "d3d9", "d3d10", "d3d11", "vdpau", "egl",
+    "thread", "other", "include", "include_cuda_main_header", "type", "literal",
+    "numeric_literal"
+};
+
+const char *apiNames[NUM_API_TYPES] = {
+    "CUDA Driver API", "CUDA RT API", "CUBLAS API"
+};
+
+namespace {
+
+template<typename ST, typename ST2>
+void conditionalPrint(ST *stream1,
+                      ST2* stream2,
+                      const std::string& s1,
+                      const std::string& s2) {
+    if (stream1) {
+        *stream1 << s1;
+    }
+
+    if (stream2) {
+        *stream2 << s2;
+    }
+}
+
+
+/**
+ * Print a named stat value to both the terminal and the CSV file.
+ */
+template<typename T>
+void printStat(std::ostream *csv, llvm::raw_ostream* printOut, const std::string &name, T value) {
+    if (printOut) {
+        *printOut << "  " << name << ": " << value << "\n";
+    }
+
+    if (csv) {
+        *csv << name << ";" << value << "\n";
+    }
+}
+
+
+} // Anonymous namespace
+
+void StatCounter::incrementCounter(const hipCounter& counter, std::string name) {
+    counters[name]++;
+    apiCounters[(int) counter.countApiType]++;
+    convTypeCounters[(int) counter.countType]++;
+}
+
+void StatCounter::add(const StatCounter& other) {
+    for (const auto& p : other.counters) {
+        counters[p.first] += p.second;
+    }
+
+    for (int i = 0; i < NUM_API_TYPES; i++) {
+        apiCounters[i] += other.apiCounters[i];
+    }
+
+    for (int i = 0; i < NUM_CONV_TYPES; i++) {
+        convTypeCounters[i] += other.convTypeCounters[i];
+    }
+}
+
+int StatCounter::getConvSum() {
+    int acc = 0;
+    for (const int& i : convTypeCounters) {
+        acc += i;
+    }
+
+    return acc;
+}
+
+void StatCounter::print(std::ostream* csv, llvm::raw_ostream* printOut, std::string prefix) {
+    conditionalPrint(csv, printOut, "\nCUDA ref type;Count\n", "[HIPIFY] info: " + prefix + " refs by type:\n");
+    for (int i = 0; i < NUM_CONV_TYPES; i++) {
+        if (convTypeCounters[i] > 0) {
+            printStat(csv, printOut, counterNames[i], convTypeCounters[i]);
+        }
+    }
+
+    conditionalPrint(csv, printOut, "\nCUDA API;Count\n", "[HIPIFY] info: " + prefix + " refs by API:\n");
+    for (int i = 0; i < NUM_API_TYPES; i++) {
+        printStat(csv, printOut, apiNames[i], apiCounters[i]);
+    }
+
+    conditionalPrint(csv, printOut, "\nCUDA ref name;Count\n", "[HIPIFY] info: " + prefix + " refs by names:\n");
+    for (const auto &it : counters) {
+        printStat(csv, printOut, it.first, it.second);
+    }
+}
+
+
+Statistics::Statistics(std::string name): fileName(name) {
+    // Compute the total bytes/lines in the input file.
+    std::ifstream src_file(name, std::ios::binary | std::ios::ate);
+    src_file.clear();
+    src_file.seekg(0);
+    totalLines = (int) std::count(std::istreambuf_iterator<char>(src_file), std::istreambuf_iterator<char>(), '\n');
+    totalBytes = (int) src_file.tellg();
+
+    // Mark the start time...
+    startTime = chr::steady_clock::now();
+};
+
+
+///////// Counter update routines //////////
+
+void Statistics::incrementCounter(const hipCounter &counter, std::string name) {
+    if (counter.unsupported) {
+        unsupported.incrementCounter(counter, name);
+    } else {
+        supported.incrementCounter(counter, name);
+    }
+}
+
+void Statistics::add(const Statistics &other) {
+    supported.add(other.supported);
+    unsupported.add(other.unsupported);
+    totalBytes += other.totalBytes;
+    totalLines += other.totalLines;
+    touchedBytes += other.touchedBytes;
+}
+
+void Statistics::lineTouched(int lineNumber) {
+    touchedLines.insert(lineNumber);
+}
+void Statistics::bytesChanged(int bytes) {
+    touchedBytes += bytes;
+}
+void Statistics::markCompletion() {
+    completionTime = chr::steady_clock::now();
+}
+
+
+///////// Output functions //////////
+
+void Statistics::print(std::ostream* csv, llvm::raw_ostream* printOut, bool skipHeader) {
+   if (!skipHeader) {
+       std::string str = "file \'" + fileName + "\' statistics:\n";
+       conditionalPrint(csv, printOut, "\n" + str, "\n[HIPIFY] info: " + str);
+   }
+
+   size_t changedLines = touchedLines.size();
+
+   // Total number of (un)supported refs that were converted.
+   int supportedSum = supported.getConvSum();
+   int unsupportedSum = unsupported.getConvSum();
+
+   printStat(csv, printOut, "CONVERTED refs count", supportedSum);
+   printStat(csv, printOut, "UNCONVERTED refs count", unsupportedSum);
+   printStat(csv, printOut, "CONVERSION %", 100 - std::lround(double(unsupportedSum * 100) / double(supportedSum + unsupportedSum)));
+   printStat(csv, printOut, "REPLACED bytes", touchedBytes);
+   printStat(csv, printOut, "TOTAL bytes", totalBytes);
+   printStat(csv, printOut, "CHANGED lines of code", changedLines);
+   printStat(csv, printOut, "TOTAL lines of code", totalLines);
+
+   if (totalBytes > 0) {
+       printStat(csv, printOut, "CODE CHANGED (in bytes) %", std::lround(double(touchedBytes * 100) / double(totalBytes)));
+   }
+
+   if (totalLines > 0) {
+       printStat(csv, printOut, "CODE CHANGED (in lines) %", std::lround(double(changedLines * 100) / double(totalLines)));
+   }
+
+   typedef std::chrono::duration<double, std::milli> duration;
+   duration elapsed = completionTime - startTime;
+   std::stringstream stream;
+   stream << std::fixed << std::setprecision(2) << elapsed.count() / 1000;
+   printStat(csv, printOut, "TIME ELAPSED s", stream.str());
+
+   supported.print(csv, printOut, "CONVERTED");
+   unsupported.print(csv, printOut, "UNCONVERTED");
+}
+
+void Statistics::printAggregate(std::ostream *csv, llvm::raw_ostream* printOut) {
+    Statistics globalStats = getAggregate();
+
+    conditionalPrint(csv, printOut, "\nTOTAL statistics:\n", "\n[HIPIFY] info: TOTAL statistics:\n");
+
+    // A file is considered "converted" if we made any changes to it.
+    int convertedFiles = 0;
+    for (const auto& p : stats) {
+        if (!p.second.touchedLines.empty()) {
+            convertedFiles++;
+        }
+    }
+
+    printStat(csv, printOut, "CONVERTED files", convertedFiles);
+    printStat(csv, printOut, "PROCESSED files", stats.size());
+
+    globalStats.print(csv, printOut);
+}
+
+//// Static state management ////
+
+Statistics Statistics::getAggregate() {
+    Statistics globalStats("global");
+
+    for (const auto& p : stats) {
+        globalStats.add(p.second);
+    }
+
+    return globalStats;
+}
+
+Statistics& Statistics::current() {
+    assert(Statistics::currentStatistics);
+    return *Statistics::currentStatistics;
+}
+
+void Statistics::setActive(std::string name) {
+    stats.emplace(std::make_pair(name, Statistics{name}));
+    Statistics::currentStatistics = &stats.at(name);
+}
+
+std::map<std::string, Statistics> Statistics::stats = {};
+Statistics* Statistics::currentStatistics = nullptr;

projects/clr/hipamd/hipify-clang/src/Statistics.h

@@ -0,0 +1,174 @@
+#pragma once
+
+#include <chrono>
+#include <string>
+#include <fstream>
+#include <llvm/ADT/StringRef.h>
+#include <map>
+#include <set>
+#include <llvm/Support/raw_ostream.h>
+
+namespace chr = std::chrono;
+
+enum ConvTypes {
+    CONV_VERSION = 0,
+    CONV_INIT,
+    CONV_DEVICE,
+    CONV_MEM,
+    CONV_KERN,
+    CONV_COORD_FUNC,
+    CONV_MATH_FUNC,
+    CONV_SPECIAL_FUNC,
+    CONV_STREAM,
+    CONV_EVENT,
+    CONV_OCCUPANCY,
+    CONV_CONTEXT,
+    CONV_PEER,
+    CONV_MODULE,
+    CONV_CACHE,
+    CONV_EXEC,
+    CONV_ERROR,
+    CONV_DEF,
+    CONV_TEX,
+    CONV_GL,
+    CONV_GRAPHICS,
+    CONV_SURFACE,
+    CONV_JIT,
+    CONV_D3D9,
+    CONV_D3D10,
+    CONV_D3D11,
+    CONV_VDPAU,
+    CONV_EGL,
+    CONV_THREAD,
+    CONV_OTHER,
+    CONV_INCLUDE,
+    CONV_INCLUDE_CUDA_MAIN_H,
+    CONV_TYPE,
+    CONV_LITERAL,
+    CONV_NUMERIC_LITERAL,
+    CONV_LAST
+};
+constexpr int NUM_CONV_TYPES = (int) ConvTypes::CONV_LAST;
+
+enum ApiTypes {
+    API_DRIVER = 0,
+    API_RUNTIME,
+    API_BLAS,
+    API_LAST
+};
+constexpr int NUM_API_TYPES = (int) ApiTypes::API_LAST;
+
+// The names of various fields in in the statistics reports.
+extern const char *counterNames[NUM_CONV_TYPES];
+extern const char *apiNames[NUM_API_TYPES];
+
+
+struct hipCounter {
+    llvm::StringRef hipName;
+    ConvTypes countType;
+    ApiTypes countApiType;
+    bool unsupported;
+};
+
+
+/**
+ * Tracks a set of named counters, as well as counters for each of the type enums defined above.
+ */
+class StatCounter {
+private:
+    // Each thing we track is either "supported" or "unsupported"...
+    std::map<std::string, int> counters;
+
+    int apiCounters[NUM_API_TYPES] = {};
+    int convTypeCounters[NUM_CONV_TYPES] = {};
+
+public:
+    void incrementCounter(const hipCounter& counter, std::string name);
+
+    /**
+     * Add the counters from `other` onto the counters of this object.
+     */
+    void add(const StatCounter& other);
+
+    int getConvSum();
+
+    void print(std::ostream* csv, llvm::raw_ostream* printOut, std::string prefix);
+};
+
+/**
+ * Tracks the statistics for a single input file.
+ */
+class Statistics {
+    StatCounter supported;
+    StatCounter unsupported;
+
+    std::string fileName;
+
+    std::set<int> touchedLines = {};
+    int touchedBytes = 0;
+
+    int totalLines = 0;
+    int totalBytes = 0;
+
+    chr::steady_clock::time_point startTime;
+    chr::steady_clock::time_point completionTime;
+
+public:
+    Statistics(std::string name);
+
+    void incrementCounter(const hipCounter &counter, std::string name);
+
+    /**
+     * Add the counters from `other` onto the counters of this object.
+     */
+    void add(const Statistics &other);
+
+    void lineTouched(int lineNumber);
+    void bytesChanged(int bytes);
+
+    /**
+     * Set the completion timestamp to now.
+     */
+    void markCompletion();
+
+    /////// Output functions ///////
+
+public:
+   /**
+     * Pretty-print the statistics stored in this object.
+     *
+     * @param csv Pointer to an output stream for the CSV to write. If null, no CSV is written
+     * @param printOut Pointer to an output stream to print human-readable textual stats to. If null, no
+     *                 such stats are produced.
+     */
+    void print(std::ostream* csv, llvm::raw_ostream* printOut, bool skipHeader = false);
+
+    /// Print aggregated statistics for all registered counters.
+    static void printAggregate(std::ostream *csv, llvm::raw_ostream* printOut);
+
+    /////// Static nonsense ///////
+
+    // The Statistics for each input file.
+    static std::map<std::string, Statistics> stats;
+
+    // The Statistics objects for the currently-being-processed input file.
+    static Statistics* currentStatistics;
+
+    /**
+     * Aggregate statistics over all entries in `stats` and return the resulting Statistics object.
+     */
+    static Statistics getAggregate();
+
+    /**
+     * Convenient global entry point for updating the "active" Statistics. Since we operate single-threadedly
+     * processing one file at a time, this allows us to simply expose the stats for the current file globally,
+     * simplifying things.
+     */
+    static Statistics& current();
+
+    /**
+     * Set the active Statistics object to the named one, creating it if necessary, and write the completion
+     * timestamp into the currently active one.
+     */
+    static void setActive(std::string name);
+};

projects/clr/hipamd/hipify-clang/src/Types.h

@@ -1,47 +0,0 @@
-#pragma once
-
-enum ConvTypes {
-    CONV_VERSION = 0,
-    CONV_INIT,
-    CONV_DEVICE,
-    CONV_MEM,
-    CONV_KERN,
-    CONV_COORD_FUNC,
-    CONV_MATH_FUNC,
-    CONV_SPECIAL_FUNC,
-    CONV_STREAM,
-    CONV_EVENT,
-    CONV_OCCUPANCY,
-    CONV_CONTEXT,
-    CONV_PEER,
-    CONV_MODULE,
-    CONV_CACHE,
-    CONV_EXEC,
-    CONV_ERROR,
-    CONV_DEF,
-    CONV_TEX,
-    CONV_GL,
-    CONV_GRAPHICS,
-    CONV_SURFACE,
-    CONV_JIT,
-    CONV_D3D9,
-    CONV_D3D10,
-    CONV_D3D11,
-    CONV_VDPAU,
-    CONV_EGL,
-    CONV_THREAD,
-    CONV_OTHER,
-    CONV_INCLUDE,
-    CONV_INCLUDE_CUDA_MAIN_H,
-    CONV_TYPE,
-    CONV_LITERAL,
-    CONV_NUMERIC_LITERAL,
-    CONV_LAST
-};
-
-enum ApiTypes {
-    API_DRIVER = 0,
-    API_RUNTIME,
-    API_BLAS,
-    API_LAST
-};