Merge pull request #1090 from emankov/master

[HIPIFY][python] Initial support of hipify-python generation from hipify-clang

[ROCm/hip commit: bfb0524e13]

projects/hip/hipify-clang/src/ArgParse.cpp

@@ -39,6 +39,11 @@ cl::opt <bool> GeneratePerl("perl",
   cl::value_desc("perl"),
   cl::cat(ToolTemplateCategory));
 
+cl::opt <bool> GeneratePython("python",
+  cl::desc("Generate hipify-python"),
+  cl::value_desc("python"),
+  cl::cat(ToolTemplateCategory));
+
 cl::opt<std::string> OutputPerlMapFilename("o-perl-map",
   cl::desc("Output filename for Perl map"),
   cl::value_desc("filename"),
@@ -49,6 +54,11 @@ cl::opt<std::string> OutputPerlMapDir("o-perl-map-dir",
   cl::value_desc("directory"),
   cl::cat(ToolTemplateCategory));
 
+cl::opt<std::string> OutputPythonMapDir("o-python-map-dir",
+  cl::desc("Output direcory for Python map"),
+  cl::value_desc("directory"),
+  cl::cat(ToolTemplateCategory));
+
 cl::opt<std::string> TemporaryDir("temp-dir",
   cl::desc("Temporary direcory"),
   cl::value_desc("directory"),

projects/hip/hipify-clang/src/ArgParse.h

@@ -32,6 +32,7 @@ extern cl::OptionCategory ToolTemplateCategory;
 extern cl::opt<std::string> OutputFilename;
 extern cl::opt<std::string> OutputPerlMapFilename;
 extern cl::opt<std::string> OutputPerlMapDir;
+extern cl::opt<std::string> OutputPythonMapDir;
 extern cl::opt<std::string> OutputDir;
 extern cl::opt<std::string> TemporaryDir;
 extern cl::opt<std::string> CudaPath;
@@ -40,6 +41,7 @@ extern cl::list<std::string> MacroNames;
 extern cl::opt<bool> Inplace;
 extern cl::opt<bool> SaveTemps;
 extern cl::opt<bool> GeneratePerl;
+extern cl::opt<bool> GeneratePython;
 extern cl::opt<bool> Verbose;
 extern cl::opt<bool> NoBackup;
 extern cl::opt<bool> NoOutput;

projects/hip/hipify-clang/src/Statistics.cpp

@@ -65,6 +65,45 @@ const char *counterNames[NUM_CONV_TYPES] = {
   "define" // CONV_DEFINE
 };
 
+const char *counterTypes[NUM_CONV_TYPES] = {
+  "CONV_ERROR",
+  "CONV_INIT",
+  "CONV_VERSION",
+  "CONV_DEVICE",
+  "CONV_CONTEXT",
+  "CONV_MODULE",
+  "CONV_MEMORY",
+  "CONV_ADDRESSING",
+  "CONV_STREAM",
+  "CONV_EVENT",
+  "CONV_EXT_RES",
+  "CONV_STREAM_MEMORY",
+  "CONV_EXECUTION",
+  "CONV_GRAPH",
+  "CONV_OCCUPANCY",
+  "CONV_TEXTURE",
+  "CONV_SURFACE",
+  "CONV_PEER",
+  "CONV_GRAPHICS",
+  "CONV_PROFILER",
+  "CONV_OPENGL",
+  "CONV_D3D9",
+  "CONV_D3D10",
+  "CONV_D3D11",
+  "CONV_VDPAU",
+  "CONV_EGL",
+  "CONV_THREAD",
+  "CONV_COMPLEX",
+  "CONV_LIB_FUNC",
+  "CONV_LIB_DEVICE_FUNC",
+  "CONV_INCLUDE",
+  "CONV_INCLUDE_CUDA_MAIN_H",
+  "CONV_TYPE",
+  "CONV_LITERAL",
+  "CONV_NUMERIC_LITERAL",
+  "CONV_DEFINE"
+};
+
 const char *apiNames[NUM_API_TYPES] = {
   "CUDA Driver API",
   "CUDA RT API",
@@ -77,6 +116,18 @@ const char *apiNames[NUM_API_TYPES] = {
   "CAFFE2 API"
 };
 
+const char *apiTypes[NUM_API_TYPES] = {
+  "API_DRIVER",
+  "API_RUNTIME",
+  "API_COMPLEX",
+  "API_BLAS",
+  "API_RAND",
+  "API_DNN",
+  "API_FFT",
+  "API_SPARSE",
+  "API_CAFFE2"
+};
+
 namespace {
 
 template<typename ST, typename ST2>

projects/hip/hipify-clang/src/Statistics.h

@@ -145,7 +145,9 @@ enum SupportDegree {
 
 // The names of various fields in in the statistics reports.
 extern const char *counterNames[NUM_CONV_TYPES];
+extern const char *counterTypes[NUM_CONV_TYPES];
 extern const char *apiNames[NUM_API_TYPES];
+extern const char *apiTypes[NUM_API_TYPES];
 
 struct hipCounter {
   llvm::StringRef hipName;

projects/hip/hipify-clang/src/main.cpp

@@ -132,7 +132,7 @@ bool generatePerl(bool Generate = true) {
           continue;
         }
         if (i == ma.second.type) {
-          *perlStreamPtr.get() << "$ft{'" + std::string(counterNames[ma.second.type]) + "'} += s/\\b" + std::string(ma.first) + "\\b/" + std::string(ma.second.hipName) + "/g;\n";
+          *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;\n";
         }
       }
     } else {
@@ -141,7 +141,7 @@ bool generatePerl(bool Generate = true) {
           continue;
         }
         if (i == ma.second.type) {
-          *perlStreamPtr.get() << "$ft{'" + std::string(counterNames[ma.second.type]) + "'} += s/\\b" + std::string(ma.first) + "\\b/" + std::string(ma.second.hipName) + "/g;\n";
+          *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;\n";
         }
       }
     }
@@ -159,6 +159,72 @@ bool generatePerl(bool Generate = true) {
   return ret;
 }
 
+bool generatePython(bool Generate = true) {
+  if (!Generate) {
+    return true;
+  }
+  std::string dstPythonMap = "cuda_to_hip_mappings.py", dstPythonMapDir = OutputPythonMapDir;
+  std::error_code EC;
+  if (!dstPythonMapDir.empty()) {
+    std::string sOutputPythonMapDirAbsPath = getAbsoluteDirectoryPath(OutputPythonMapDir, EC, "output hipify-python map");
+    if (EC) {
+      return false;
+    }
+    dstPythonMap = sOutputPythonMapDirAbsPath + "/" + dstPythonMap;
+  }
+  SmallString<128> tmpFile;
+  StringRef ext = "hipify-tmp";
+  EC = sys::fs::createTemporaryFile(dstPythonMap, ext, tmpFile);
+  if (EC) {
+    llvm::errs() << "\n" << sHipify << sError << EC.message() << ": " << tmpFile << "\n";
+    return false;
+  }
+  std::unique_ptr<std::ostream> pythonStreamPtr = std::unique_ptr<std::ostream>(new std::ofstream(tmpFile.c_str(), std::ios_base::trunc));
+  *pythonStreamPtr.get() << "import collections\n\n";
+  *pythonStreamPtr.get() << "from pyHIPIFY.constants import *\n\n";
+  *pythonStreamPtr.get() << "CUDA_RENAMES_MAP = collections.OrderedDict([\n";
+  const std::string sHIP_UNS = ", HIP_UNSUPPORTED";
+  for (int i = 0; i < NUM_CONV_TYPES; i++) {
+    if (i == CONV_INCLUDE_CUDA_MAIN_H || i == CONV_INCLUDE) {
+      for (auto& ma : CUDA_INCLUDE_MAP) {
+        if (i == ma.second.type) {
+          std::string sUnsupported;
+          if (Statistics::isUnsupported(ma.second)) {
+            sUnsupported = sHIP_UNS;
+          }
+          StringRef repName = Statistics::isToRoc(ma.second) ? ma.second.rocName : ma.second.hipName;
+          *pythonStreamPtr.get() << "    (\"" << ma.first.str() << "\", (\"" << repName.str() << "\", " << counterTypes[i] << ", " << apiTypes[ma.second.apiType] << sUnsupported << ")),\n";
+        }
+      }
+    }
+    else {
+      for (auto& ma : CUDA_RENAMES_MAP()) {
+        if (i == ma.second.type) {
+          std::string sUnsupported;
+          if (Statistics::isUnsupported(ma.second)) {
+            sUnsupported = sHIP_UNS;
+          }
+          StringRef repName = Statistics::isToRoc(ma.second) ? ma.second.rocName : ma.second.hipName;
+          *pythonStreamPtr.get() << "    (\"" << ma.first.str() << "\", (\"" << repName.str() << "\", " << counterTypes[i] << ", " << apiTypes[ma.second.apiType] << sUnsupported << ")),\n";
+        }
+      }
+    }
+  }
+  *pythonStreamPtr.get() << "])\n\n";
+  *pythonStreamPtr.get() << "CUDA_TO_HIP_MAPPINGS = [CUDA_RENAMES_MAP, C10_MAPPINGS, PYTORCH_SPECIFIC_MAPPINGS]\n";
+  pythonStreamPtr.get()->flush();
+  bool ret = true;
+  EC = sys::fs::copy_file(tmpFile, dstPythonMap);
+  if (EC) {
+    llvm::errs() << "\n" << sHipify << sError << EC.message() << ": while copying " << tmpFile << " to " << dstPythonMap << "\n";
+    ret = false;
+  }
+  if (!SaveTemps) {
+    sys::fs::remove(tmpFile);
+  }
+  return true;
+}
+
 int main(int argc, const char **argv) {
   std::vector<const char*> new_argv(argv, argv + argc);
   if (std::find(new_argv.begin(), new_argv.end(), std::string("--")) == new_argv.end()) {
@@ -170,7 +236,7 @@ int main(int argc, const char **argv) {
   llcompat::PrintStackTraceOnErrorSignal();
   ct::CommonOptionsParser OptionsParser(argc, argv, ToolTemplateCategory, llvm::cl::Optional);
   std::vector<std::string> fileSources = OptionsParser.getSourcePathList();
-  if (fileSources.empty() && !GeneratePerl) {
+  if (fileSources.empty() && !GeneratePerl && !GeneratePython) {
     llvm::errs() << "\n" << sHipify << sError << "Must specify at least 1 positional argument for source file." << "\n";
     return 1;
   }
@@ -178,6 +244,14 @@ int main(int argc, const char **argv) {
     llvm::errs() << "\n" << sHipify << sError << "hipify-perl generating failed." << "\n";
     return 1;
   }
+  bool bToRoc = TranslateToRoc;
+  TranslateToRoc = true;
+  bool bToPython = generatePython(GeneratePython);
+  TranslateToRoc = bToRoc;
+  if (!bToPython) {
+    llvm::errs() << "\n" << sHipify << sError << "hipify-python generating failed." << "\n";
+    return 1;
+  }
   if (fileSources.empty()) {
     return 0;
   }