From cabe0bb799e48eda2ea8772dd32cf1f9a2d3280f Mon Sep 17 00:00:00 2001 From: Evgeny Mankov Date: Wed, 8 May 2019 19:08:55 +0300 Subject: [PATCH] [HIPIFY][python] Initial support of hipify-python generation from hipify-clang + Only a generation of transformation map of CUDA entities is implemented. + 2 hipify-clang options are added: -python, -o-python-map-dir. + Explicitly set -roc option for cuda_to_hip_mappings.py generation. + Generated file already might be used by pytorch team. [ROCm/hip commit: 6b370e7743b1e7f7de92c148f86c88e82e5d685d] --- projects/hip/hipify-clang/src/ArgParse.cpp | 10 +++ projects/hip/hipify-clang/src/ArgParse.h | 2 + projects/hip/hipify-clang/src/Statistics.cpp | 51 +++++++++++++ projects/hip/hipify-clang/src/Statistics.h | 2 + projects/hip/hipify-clang/src/main.cpp | 80 +++++++++++++++++++- 5 files changed, 142 insertions(+), 3 deletions(-) diff --git a/projects/hip/hipify-clang/src/ArgParse.cpp b/projects/hip/hipify-clang/src/ArgParse.cpp index 346cdfd1b1..28859c278b 100644 --- a/projects/hip/hipify-clang/src/ArgParse.cpp +++ b/projects/hip/hipify-clang/src/ArgParse.cpp @@ -39,6 +39,11 @@ cl::opt GeneratePerl("perl", cl::value_desc("perl"), cl::cat(ToolTemplateCategory)); +cl::opt GeneratePython("python", + cl::desc("Generate hipify-python"), + cl::value_desc("python"), + cl::cat(ToolTemplateCategory)); + cl::opt OutputPerlMapFilename("o-perl-map", cl::desc("Output filename for Perl map"), cl::value_desc("filename"), @@ -49,6 +54,11 @@ cl::opt OutputPerlMapDir("o-perl-map-dir", cl::value_desc("directory"), cl::cat(ToolTemplateCategory)); +cl::opt OutputPythonMapDir("o-python-map-dir", + cl::desc("Output direcory for Python map"), + cl::value_desc("directory"), + cl::cat(ToolTemplateCategory)); + cl::opt TemporaryDir("temp-dir", cl::desc("Temporary direcory"), cl::value_desc("directory"), diff --git a/projects/hip/hipify-clang/src/ArgParse.h b/projects/hip/hipify-clang/src/ArgParse.h index d4ea64fd10..04ba48f004 100644 --- a/projects/hip/hipify-clang/src/ArgParse.h +++ b/projects/hip/hipify-clang/src/ArgParse.h @@ -32,6 +32,7 @@ extern cl::OptionCategory ToolTemplateCategory; extern cl::opt OutputFilename; extern cl::opt OutputPerlMapFilename; extern cl::opt OutputPerlMapDir; +extern cl::opt OutputPythonMapDir; extern cl::opt OutputDir; extern cl::opt TemporaryDir; extern cl::opt CudaPath; @@ -40,6 +41,7 @@ extern cl::list MacroNames; extern cl::opt Inplace; extern cl::opt SaveTemps; extern cl::opt GeneratePerl; +extern cl::opt GeneratePython; extern cl::opt Verbose; extern cl::opt NoBackup; extern cl::opt NoOutput; diff --git a/projects/hip/hipify-clang/src/Statistics.cpp b/projects/hip/hipify-clang/src/Statistics.cpp index e673957458..4940d13da5 100644 --- a/projects/hip/hipify-clang/src/Statistics.cpp +++ b/projects/hip/hipify-clang/src/Statistics.cpp @@ -65,6 +65,45 @@ const char *counterNames[NUM_CONV_TYPES] = { "define" // CONV_DEFINE }; +const char *counterTypes[NUM_CONV_TYPES] = { + "CONV_ERROR", + "CONV_INIT", + "CONV_VERSION", + "CONV_DEVICE", + "CONV_CONTEXT", + "CONV_MODULE", + "CONV_MEMORY", + "CONV_ADDRESSING", + "CONV_STREAM", + "CONV_EVENT", + "CONV_EXT_RES", + "CONV_STREAM_MEMORY", + "CONV_EXECUTION", + "CONV_GRAPH", + "CONV_OCCUPANCY", + "CONV_TEXTURE", + "CONV_SURFACE", + "CONV_PEER", + "CONV_GRAPHICS", + "CONV_PROFILER", + "CONV_OPENGL", + "CONV_D3D9", + "CONV_D3D10", + "CONV_D3D11", + "CONV_VDPAU", + "CONV_EGL", + "CONV_THREAD", + "CONV_COMPLEX", + "CONV_LIB_FUNC", + "CONV_LIB_DEVICE_FUNC", + "CONV_INCLUDE", + "CONV_INCLUDE_CUDA_MAIN_H", + "CONV_TYPE", + "CONV_LITERAL", + "CONV_NUMERIC_LITERAL", + "CONV_DEFINE" +}; + const char *apiNames[NUM_API_TYPES] = { "CUDA Driver API", "CUDA RT API", @@ -77,6 +116,18 @@ const char *apiNames[NUM_API_TYPES] = { "CAFFE2 API" }; +const char *apiTypes[NUM_API_TYPES] = { + "API_DRIVER", + "API_RUNTIME", + "API_COMPLEX", + "API_BLAS", + "API_RAND", + "API_DNN", + "API_FFT", + "API_SPARSE", + "API_CAFFE2" +}; + namespace { template diff --git a/projects/hip/hipify-clang/src/Statistics.h b/projects/hip/hipify-clang/src/Statistics.h index 8305f0de93..974db25dd3 100644 --- a/projects/hip/hipify-clang/src/Statistics.h +++ b/projects/hip/hipify-clang/src/Statistics.h @@ -145,7 +145,9 @@ enum SupportDegree { // The names of various fields in in the statistics reports. extern const char *counterNames[NUM_CONV_TYPES]; +extern const char *counterTypes[NUM_CONV_TYPES]; extern const char *apiNames[NUM_API_TYPES]; +extern const char *apiTypes[NUM_API_TYPES]; struct hipCounter { llvm::StringRef hipName; diff --git a/projects/hip/hipify-clang/src/main.cpp b/projects/hip/hipify-clang/src/main.cpp index c92fbe3cad..25b21bf56f 100644 --- a/projects/hip/hipify-clang/src/main.cpp +++ b/projects/hip/hipify-clang/src/main.cpp @@ -132,7 +132,7 @@ bool generatePerl(bool Generate = true) { continue; } if (i == ma.second.type) { - *perlStreamPtr.get() << "$ft{'" + std::string(counterNames[ma.second.type]) + "'} += s/\\b" + std::string(ma.first) + "\\b/" + std::string(ma.second.hipName) + "/g;\n"; + *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;\n"; } } } else { @@ -141,7 +141,7 @@ bool generatePerl(bool Generate = true) { continue; } if (i == ma.second.type) { - *perlStreamPtr.get() << "$ft{'" + std::string(counterNames[ma.second.type]) + "'} += s/\\b" + std::string(ma.first) + "\\b/" + std::string(ma.second.hipName) + "/g;\n"; + *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;\n"; } } } @@ -159,6 +159,72 @@ bool generatePerl(bool Generate = true) { return ret; } +bool generatePython(bool Generate = true) { + if (!Generate) { + return true; + } + std::string dstPythonMap = "cuda_to_hip_mappings.py", dstPythonMapDir = OutputPythonMapDir; + std::error_code EC; + if (!dstPythonMapDir.empty()) { + std::string sOutputPythonMapDirAbsPath = getAbsoluteDirectoryPath(OutputPythonMapDir, EC, "output hipify-python map"); + if (EC) { + return false; + } + dstPythonMap = sOutputPythonMapDirAbsPath + "/" + dstPythonMap; + } + SmallString<128> tmpFile; + StringRef ext = "hipify-tmp"; + EC = sys::fs::createTemporaryFile(dstPythonMap, ext, tmpFile); + if (EC) { + llvm::errs() << "\n" << sHipify << sError << EC.message() << ": " << tmpFile << "\n"; + return false; + } + std::unique_ptr pythonStreamPtr = std::unique_ptr(new std::ofstream(tmpFile.c_str(), std::ios_base::trunc)); + *pythonStreamPtr.get() << "import collections\n\n"; + *pythonStreamPtr.get() << "from pyHIPIFY.constants import *\n\n"; + *pythonStreamPtr.get() << "CUDA_RENAMES_MAP = collections.OrderedDict([\n"; + const std::string sHIP_UNS = ", HIP_UNSUPPORTED"; + for (int i = 0; i < NUM_CONV_TYPES; i++) { + if (i == CONV_INCLUDE_CUDA_MAIN_H || i == CONV_INCLUDE) { + for (auto& ma : CUDA_INCLUDE_MAP) { + if (i == ma.second.type) { + std::string sUnsupported; + if (Statistics::isUnsupported(ma.second)) { + sUnsupported = sHIP_UNS; + } + StringRef repName = Statistics::isToRoc(ma.second) ? ma.second.rocName : ma.second.hipName; + *pythonStreamPtr.get() << " (\"" << ma.first.str() << "\", (\"" << repName.str() << "\", " << counterTypes[i] << ", " << apiTypes[ma.second.apiType] << sUnsupported << ")),\n"; + } + } + } + else { + for (auto& ma : CUDA_RENAMES_MAP()) { + if (i == ma.second.type) { + std::string sUnsupported; + if (Statistics::isUnsupported(ma.second)) { + sUnsupported = sHIP_UNS; + } + StringRef repName = Statistics::isToRoc(ma.second) ? ma.second.rocName : ma.second.hipName; + *pythonStreamPtr.get() << " (\"" << ma.first.str() << "\", (\"" << repName.str() << "\", " << counterTypes[i] << ", " << apiTypes[ma.second.apiType] << sUnsupported << ")),\n"; + } + } + } + } + *pythonStreamPtr.get() << "])\n\n"; + *pythonStreamPtr.get() << "CUDA_TO_HIP_MAPPINGS = [CUDA_RENAMES_MAP, C10_MAPPINGS, PYTORCH_SPECIFIC_MAPPINGS]\n"; + pythonStreamPtr.get()->flush(); + bool ret = true; + EC = sys::fs::copy_file(tmpFile, dstPythonMap); + if (EC) { + llvm::errs() << "\n" << sHipify << sError << EC.message() << ": while copying " << tmpFile << " to " << dstPythonMap << "\n"; + ret = false; + } + if (!SaveTemps) { + sys::fs::remove(tmpFile); + } + return true; +} + int main(int argc, const char **argv) { std::vector new_argv(argv, argv + argc); if (std::find(new_argv.begin(), new_argv.end(), std::string("--")) == new_argv.end()) { @@ -170,7 +236,7 @@ int main(int argc, const char **argv) { llcompat::PrintStackTraceOnErrorSignal(); ct::CommonOptionsParser OptionsParser(argc, argv, ToolTemplateCategory, llvm::cl::Optional); std::vector fileSources = OptionsParser.getSourcePathList(); - if (fileSources.empty() && !GeneratePerl) { + if (fileSources.empty() && !GeneratePerl && !GeneratePython) { llvm::errs() << "\n" << sHipify << sError << "Must specify at least 1 positional argument for source file." << "\n"; return 1; } @@ -178,6 +244,14 @@ int main(int argc, const char **argv) { llvm::errs() << "\n" << sHipify << sError << "hipify-perl generating failed." << "\n"; return 1; } + bool bToRoc = TranslateToRoc; + TranslateToRoc = true; + bool bToPython = generatePython(GeneratePython); + TranslateToRoc = bToRoc; + if (!bToPython) { + llvm::errs() << "\n" << sHipify << sError << "hipify-python generating failed." << "\n"; + return 1; + } if (fileSources.empty()) { return 0; }