Merge 'master' into 'amd-master'

Change-Id: I390e88de292d69072474b3ed1e5537d2477e9a0f
2019-05-10 11:30:25 -04:00
@@ -296,13 +296,13 @@ def docker_upload_dockerhub( String local_org, String image_name, String remote_
 String build_config = 'Release'
 String job_name = env.JOB_NAME.toLowerCase( )

-// The following launches 3 builds in parallel: rocm-head, rocm-2.3.x and cuda-10.x
-parallel rocm_2_3:
+// The following launches 3 builds in parallel: rocm-head, rocm-2.4.x and cuda-10.x
+parallel rocm_2_4:
 {
  node('hip-rocm')
  {
-    String hcc_ver = 'rocm-2.3.x'
-    String from_image = 'ci_test_nodes/rocm-2.3.x/ubuntu-16.04:latest'
+    String hcc_ver = 'rocm-2.4.x'
+    String from_image = 'ci_test_nodes/rocm-2.4.x/ubuntu-16.04:latest'
    String inside_args = '--device=/dev/kfd --device=/dev/dri --group-add=video'

    // Checkout source code, dependencies and version files
@@ -39,6 +39,11 @@ cl::opt <bool> GeneratePerl("perl",
  cl::value_desc("perl"),
  cl::cat(ToolTemplateCategory));

+cl::opt <bool> GeneratePython("python",
+  cl::desc("Generate hipify-python"),
+  cl::value_desc("python"),
+  cl::cat(ToolTemplateCategory));
+
 cl::opt<std::string> OutputPerlMapFilename("o-perl-map",
  cl::desc("Output filename for Perl map"),
  cl::value_desc("filename"),
@@ -49,6 +54,11 @@ cl::opt<std::string> OutputPerlMapDir("o-perl-map-dir",
  cl::value_desc("directory"),
  cl::cat(ToolTemplateCategory));

+cl::opt<std::string> OutputPythonMapDir("o-python-map-dir",
+  cl::desc("Output direcory for Python map"),
+  cl::value_desc("directory"),
+  cl::cat(ToolTemplateCategory));
+
 cl::opt<std::string> TemporaryDir("temp-dir",
  cl::desc("Temporary direcory"),
  cl::value_desc("directory"),
@@ -32,6 +32,7 @@ extern cl::OptionCategory ToolTemplateCategory;
 extern cl::opt<std::string> OutputFilename;
 extern cl::opt<std::string> OutputPerlMapFilename;
 extern cl::opt<std::string> OutputPerlMapDir;
+extern cl::opt<std::string> OutputPythonMapDir;
 extern cl::opt<std::string> OutputDir;
 extern cl::opt<std::string> TemporaryDir;
 extern cl::opt<std::string> CudaPath;
@@ -40,6 +41,7 @@ extern cl::list<std::string> MacroNames;
 extern cl::opt<bool> Inplace;
 extern cl::opt<bool> SaveTemps;
 extern cl::opt<bool> GeneratePerl;
+extern cl::opt<bool> GeneratePython;
 extern cl::opt<bool> Verbose;
 extern cl::opt<bool> NoBackup;
 extern cl::opt<bool> NoOutput;
@@ -65,6 +65,45 @@ const char *counterNames[NUM_CONV_TYPES] = {
  "define" // CONV_DEFINE
 };

+const char *counterTypes[NUM_CONV_TYPES] = {
+  "CONV_ERROR",
+  "CONV_INIT",
+  "CONV_VERSION",
+  "CONV_DEVICE",
+  "CONV_CONTEXT",
+  "CONV_MODULE",
+  "CONV_MEMORY",
+  "CONV_ADDRESSING",
+  "CONV_STREAM",
+  "CONV_EVENT",
+  "CONV_EXT_RES",
+  "CONV_STREAM_MEMORY",
+  "CONV_EXECUTION",
+  "CONV_GRAPH",
+  "CONV_OCCUPANCY",
+  "CONV_TEXTURE",
+  "CONV_SURFACE",
+  "CONV_PEER",
+  "CONV_GRAPHICS",
+  "CONV_PROFILER",
+  "CONV_OPENGL",
+  "CONV_D3D9",
+  "CONV_D3D10",
+  "CONV_D3D11",
+  "CONV_VDPAU",
+  "CONV_EGL",
+  "CONV_THREAD",
+  "CONV_COMPLEX",
+  "CONV_LIB_FUNC",
+  "CONV_LIB_DEVICE_FUNC",
+  "CONV_INCLUDE",
+  "CONV_INCLUDE_CUDA_MAIN_H",
+  "CONV_TYPE",
+  "CONV_LITERAL",
+  "CONV_NUMERIC_LITERAL",
+  "CONV_DEFINE"
+};
+
 const char *apiNames[NUM_API_TYPES] = {
  "CUDA Driver API",
  "CUDA RT API",
@@ -77,6 +116,18 @@ const char *apiNames[NUM_API_TYPES] = {
  "CAFFE2 API"
 };

+const char *apiTypes[NUM_API_TYPES] = {
+  "API_DRIVER",
+  "API_RUNTIME",
+  "API_COMPLEX",
+  "API_BLAS",
+  "API_RAND",
+  "API_DNN",
+  "API_FFT",
+  "API_SPARSE",
+  "API_CAFFE2"
+};
+
 namespace {

 template<typename ST, typename ST2>
@@ -145,7 +145,9 @@ enum SupportDegree {

 // The names of various fields in in the statistics reports.
 extern const char *counterNames[NUM_CONV_TYPES];
+extern const char *counterTypes[NUM_CONV_TYPES];
 extern const char *apiNames[NUM_API_TYPES];
+extern const char *apiTypes[NUM_API_TYPES];

 struct hipCounter {
  llvm::StringRef hipName;
@@ -125,11 +125,26 @@ bool generatePerl(bool Generate = true) {
  }
  *perlStreamPtr.get() << "\"" << counterNames[NUM_CONV_TYPES - 1] << "\");\n\n";
  *perlStreamPtr.get() << sConv << "$ft{'" << counterNames[NUM_CONV_TYPES - 1] << "'};\n\n";
-  for (auto& ma : CUDA_RENAMES_MAP()) {
-    if (Statistics::isUnsupported(ma.second)) {
-      continue;
+  for (int i = 0; i < NUM_CONV_TYPES; i++) {
+    if (i == CONV_INCLUDE_CUDA_MAIN_H || i == CONV_INCLUDE) {
+      for (auto& ma : CUDA_INCLUDE_MAP) {
+        if (Statistics::isUnsupported(ma.second)) {
+          continue;
+        }
+        if (i == ma.second.type) {
+          *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;\n";
+        }
+      }
+    } else {
+      for (auto& ma : CUDA_RENAMES_MAP()) {
+        if (Statistics::isUnsupported(ma.second)) {
+          continue;
+        }
+        if (i == ma.second.type) {
+          *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;\n";
+        }
+      }
    }
-    *perlStreamPtr.get() << "$ft{'" + std::string(counterNames[ma.second.type]) + "'} += s/\\b" + std::string(ma.first) + "\\b/" + std::string(ma.second.hipName) + "/g;\n";
  }
  perlStreamPtr.get()->flush();
  bool ret = true;
@@ -144,6 +159,72 @@ bool generatePerl(bool Generate = true) {
  return ret;
 }

+bool generatePython(bool Generate = true) {
+  if (!Generate) {
+    return true;
+  }
+  std::string dstPythonMap = "cuda_to_hip_mappings.py", dstPythonMapDir = OutputPythonMapDir;
+  std::error_code EC;
+  if (!dstPythonMapDir.empty()) {
+    std::string sOutputPythonMapDirAbsPath = getAbsoluteDirectoryPath(OutputPythonMapDir, EC, "output hipify-python map");
+    if (EC) {
+      return false;
+    }
+    dstPythonMap = sOutputPythonMapDirAbsPath + "/" + dstPythonMap;
+  }
+  SmallString<128> tmpFile;
+  StringRef ext = "hipify-tmp";
+  EC = sys::fs::createTemporaryFile(dstPythonMap, ext, tmpFile);
+  if (EC) {
+    llvm::errs() << "\n" << sHipify << sError << EC.message() << ": " << tmpFile << "\n";
+    return false;
+  }
+  std::unique_ptr<std::ostream> pythonStreamPtr = std::unique_ptr<std::ostream>(new std::ofstream(tmpFile.c_str(), std::ios_base::trunc));
+  *pythonStreamPtr.get() << "import collections\n\n";
+  *pythonStreamPtr.get() << "from pyHIPIFY.constants import *\n\n";
+  *pythonStreamPtr.get() << "CUDA_RENAMES_MAP = collections.OrderedDict([\n";
+  const std::string sHIP_UNS = ", HIP_UNSUPPORTED";
+  for (int i = 0; i < NUM_CONV_TYPES; i++) {
+    if (i == CONV_INCLUDE_CUDA_MAIN_H || i == CONV_INCLUDE) {
+      for (auto& ma : CUDA_INCLUDE_MAP) {
+        if (i == ma.second.type) {
+          std::string sUnsupported;
+          if (Statistics::isUnsupported(ma.second)) {
+            sUnsupported = sHIP_UNS;
+          }
+          StringRef repName = Statistics::isToRoc(ma.second) ? ma.second.rocName : ma.second.hipName;
+          *pythonStreamPtr.get() << "    (\"" << ma.first.str() << "\", (\"" << repName.str() << "\", " << counterTypes[i] << ", " << apiTypes[ma.second.apiType] << sUnsupported << ")),\n";
+        }
+      }
+    }
+    else {
+      for (auto& ma : CUDA_RENAMES_MAP()) {
+        if (i == ma.second.type) {
+          std::string sUnsupported;
+          if (Statistics::isUnsupported(ma.second)) {
+            sUnsupported = sHIP_UNS;
+          }
+          StringRef repName = Statistics::isToRoc(ma.second) ? ma.second.rocName : ma.second.hipName;
+          *pythonStreamPtr.get() << "    (\"" << ma.first.str() << "\", (\"" << repName.str() << "\", " << counterTypes[i] << ", " << apiTypes[ma.second.apiType] << sUnsupported << ")),\n";
+        }
+      }
+    }
+  }
+  *pythonStreamPtr.get() << "])\n\n";
+  *pythonStreamPtr.get() << "CUDA_TO_HIP_MAPPINGS = [CUDA_RENAMES_MAP, C10_MAPPINGS, PYTORCH_SPECIFIC_MAPPINGS]\n";
+  pythonStreamPtr.get()->flush();
+  bool ret = true;
+  EC = sys::fs::copy_file(tmpFile, dstPythonMap);
+  if (EC) {
+    llvm::errs() << "\n" << sHipify << sError << EC.message() << ": while copying " << tmpFile << " to " << dstPythonMap << "\n";
+    ret = false;
+  }
+  if (!SaveTemps) {
+    sys::fs::remove(tmpFile);
+  }
+  return true;
+}
+
 int main(int argc, const char **argv) {
  std::vector<const char*> new_argv(argv, argv + argc);
  if (std::find(new_argv.begin(), new_argv.end(), std::string("--")) == new_argv.end()) {
@@ -155,7 +236,7 @@ int main(int argc, const char **argv) {
  llcompat::PrintStackTraceOnErrorSignal();
  ct::CommonOptionsParser OptionsParser(argc, argv, ToolTemplateCategory, llvm::cl::Optional);
  std::vector<std::string> fileSources = OptionsParser.getSourcePathList();
-  if (fileSources.empty() && !GeneratePerl) {
+  if (fileSources.empty() && !GeneratePerl && !GeneratePython) {
    llvm::errs() << "\n" << sHipify << sError << "Must specify at least 1 positional argument for source file." << "\n";
    return 1;
  }
@@ -163,6 +244,14 @@ int main(int argc, const char **argv) {
    llvm::errs() << "\n" << sHipify << sError << "hipify-perl generating failed." << "\n";
    return 1;
  }
+  bool bToRoc = TranslateToRoc;
+  TranslateToRoc = true;
+  bool bToPython = generatePython(GeneratePython);
+  TranslateToRoc = bToRoc;
+  if (!bToPython) {
+    llvm::errs() << "\n" << sHipify << sError << "hipify-python generating failed." << "\n";
+    return 1;
+  }
  if (fileSources.empty()) {
    return 0;
  }
@@ -459,6 +459,14 @@ hc_get_workitem_absolute_id(int dim)
 #undef __CUDA__
 #pragma pop_macro("__CUDA__")

+hipError_t hipExtModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
+                                    uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
+                                    uint32_t localWorkSizeX, uint32_t localWorkSizeY,
+                                    uint32_t localWorkSizeZ, size_t sharedMemBytes,
+                                    hipStream_t hStream, void** kernelParams, void** extra,
+                                    hipEvent_t startEvent = nullptr,
+                                    hipEvent_t stopEvent = nullptr,
+                                    uint32_t flags = 0);

 hipError_t hipHccModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
                                    uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
@@ -466,7 +474,8 @@ hipError_t hipHccModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
                                    uint32_t localWorkSizeZ, size_t sharedMemBytes,
                                    hipStream_t hStream, void** kernelParams, void** extra,
                                    hipEvent_t startEvent = nullptr,
-                                    hipEvent_t stopEvent = nullptr);
+                                    hipEvent_t stopEvent = nullptr)
+                                    __attribute__((deprecated("use hipExtModuleLaunchKernel instead")));

 #endif // defined(__clang__) && defined(__HIP__)

@@ -105,7 +105,7 @@ hipError_t hipHccModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
                                    hipStream_t hStream, void** kernelParams, void** extra,
                                    hipEvent_t startEvent = nullptr,
                                    hipEvent_t stopEvent = nullptr)
-                                    __attribute__((deprecated("use hipExtModuleLaunchKernel instead")));;
+                                    __attribute__((deprecated("use hipExtModuleLaunchKernel instead")));

 // doxygen end HCC-specific features
 /**