From a48cfd56e9ca7676991c4b05b221f370a18119dc Mon Sep 17 00:00:00 2001 From: Evgeny Mankov Date: Fri, 27 Sep 2019 19:10:54 +0300 Subject: [PATCH] [HIPIFY] Generate more hipify-perl (continuation) + Minor hipify-perl clean-up + Minor CUDA2HIP_Perl refactoring [ROCm/clr commit: 0d36e13ce9ba9b245c42b9b8b8070c2796bdf99e] --- projects/clr/hipamd/bin/hipify-perl | 106 +++++------ .../hipamd/hipify-clang/src/CUDA2HIP_Perl.cpp | 177 +++++++++++++----- 2 files changed, 179 insertions(+), 104 deletions(-) diff --git a/projects/clr/hipamd/bin/hipify-perl b/projects/clr/hipamd/bin/hipify-perl index 2887402b4e..c3dead1506 100755 --- a/projects/clr/hipamd/bin/hipify-perl +++ b/projects/clr/hipamd/bin/hipify-perl @@ -1,4 +1,5 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl -w + ## # Copyright (c) 2015-present Advanced Micro Devices, Inc. All rights reserved. # @@ -20,60 +21,55 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. ## -#usage hipify-perl [OPTIONS] INPUT_FILE -use Getopt::Long; -my $warn_whitelist =""; +#usage hipify-perl [OPTIONS] INPUT_FILE + +use Getopt::Long; +my $whitelist = ""; + GetOptions( - "print-stats" => \$print_stats # print the command-line, like a header. - , "count-conversions" => \$count_conversions # count conversions. - , "quiet-warnings" => \$quiet_warnings # don't print warnings on unknown CUDA functions. - , "warn-whitelist=s"=> \$warn_whitelist - , "no-output" => \$no_output # don't write any translated output to stdout. - , "inplace" => \$inplace # modify input file inplace, replacing input with hipified output, save backup in ".prehip" file. - # If .prehip file exists, use that as input to hip. - , "n" => \$n # combination of print_stats + no-output. + "examine" => \$examine # Combines -no-output and -print-stats options. + , "inplace" => \$inplace # Modify input file inplace, replacing input with hipified output, save backup in .prehip file. + , "no-output" => \$no_output # Don't write any translated output to stdout. + , "print-stats" => \$print_stats # Print translation statistics. + , "quiet-warnings" => \$quiet_warnings # Don't print warnings on unknown CUDA functions. + , "whitelist=s" => \$whitelist # TODO: test it beforehand ); -$print_stats = 1 if $n; -$no_output = 1 if $n; +$print_stats = 1 if $examine; +$no_output = 1 if $examine; -# These uses of cuda[A-Z] are commonly used in CUDA code but don't actually map to any CUDA API: -# TODO - use a hash lookup for these. -@warn_whitelist = ( - "cudaDevice" - ,"cudaDevice_t" - ,"cudaIDs" - ,"cudaGridDim" - ,"cudaDimGrid" - ,"cudaDimBlock" - ,"cudaDeviceId" - ,"cudaDevices", - ,"cudaGradOutput", - ,"cudaInput", - ,"cudaOutput", - ,"cudaGradInput", - ,"cudaIndices", - ,"cudaGaugeField" - ,"cudaMom" - ,"cudaGauge" - ,"cudaInGauge" - ,"cudaColorSpinorField" - ,"cudaSiteLink" - ,"cudaFatLink" - ,"cudaStaple" - ,"cudaCloverField" - ,"cudaParam" - ); -#print "WW=@warn_whitelist\n"; +# Whitelist of cuda[A-Z] identifiers, which are commonly used in CUDA sources but don't map to any CUDA API: +@whitelist = ( + "cudaCloverField" + , "cudaColorSpinorField" + , "cudaDevice" + , "cudaDeviceId" + , "cudaDevice_t" + , "cudaDevices" + , "cudaDimBlock" + , "cudaDimGrid" + , "cudaFatLink" + , "cudaGauge" + , "cudaGaugeField" + , "cudaGradInput" + , "cudaGradOutput" + , "cudaGridDim" + , "cudaIDs" + , "cudaInGauge" + , "cudaIndices" + , "cudaInput" + , "cudaMom" + , "cudaOutput" + , "cudaParam" + , "cudaSiteLink" + , "cudaStaple" +); -# Allow users to add their own functions. -push (@warn_whitelist, split(',',$warn_whitelist)); +push(@whitelist, split(',', $whitelist)); -#Stats tracking code: @statNames = ("error", "init", "version", "device", "context", "module", "memory", "addressing", "stream", "event", "external_resource_interop", "stream_memory", "execution", "graph", "occupancy", "texture", "surface", "peer", "graphics", "profiler", "openGL", "D3D9", "D3D10", "D3D11", "VDPAU", "EGL", "thread", "complex", "library", "device_library", "include", "include_cuda_main_header", "type", "literal", "numeric_literal", "define", "kernel_func", "extern_shared", "kern_launch"); -#Compute total of all individual counts: sub totalStats { my %count = %{ shift() }; my $total = 0; @@ -90,11 +86,11 @@ sub printStats { my $warnings = shift(); my $loc = shift(); my $total = totalStats(\%counts); - printf STDERR "%s %d CUDA->HIP refs( ", $label, $total; + printf STDERR "%s %d CUDA->HIP refs ( ", $label, $total; foreach $stat (@statNames) { printf STDERR "%s:%d ", $stat, $counts{$stat}; } - printf STDERR ") warn:%d LOC:%d", $warnings, $loc; + printf STDERR ")\n warn:%d LOC:%d", $warnings, $loc; } # Add adder stats to dest. Used to add stats for current file to a running total for all files: @@ -160,8 +156,6 @@ while (@ARGV) { undef $/; # Read whole file at once, so we can match newlines. while () { - # chomp; - # next if /^(\s*(#.*)?)?$/; $ft{'error'} += s/\bcudaGetErrorName\b/hipGetErrorName/g; $ft{'error'} += s/\bcudaGetErrorString\b/hipGetErrorString/g; $ft{'error'} += s/\bcudaGetLastError\b/hipGetLastError/g; @@ -1712,7 +1706,7 @@ while (@ARGV) { $Tkernels{$1} ++; } } - if ($count_conversions) { + if ($print_stats) { while (/(\bhip[A-Z]\w+\b)/g) { $convertedTags{$1}++; #print STDERR "HIP: $1 : ", $translateTags{$1}, "\n"; @@ -1724,14 +1718,14 @@ while (@ARGV) { unless ($quiet_warnings) { #print STDERR "Check WARNINGs\n"; # copy into array of lines, process line-by-line to show warnings: - if ($hasDeviceCode or (/\bcuda/) or (/<<<.*>>>/) or (/(\bcublas[A-Z]\w+)/) or (/(\bcurand[A-Z]\w+)/) ) { + if ($hasDeviceCode or (/\bcuda/) or (/<<<.*>>>/)) { my @lines = split /\n/, $_; my $tmp = $_; # copies the whole file, could be a little smarter here... my $line_num = 0; foreach (@lines) { $line_num ++; # remove any whitelisted words: - foreach $w (@warn_whitelist) { + foreach $w (@whitelist) { s/\b$w\b/ZAP/ } my $tag; @@ -1780,7 +1774,7 @@ while (@ARGV) { my $totalConverted = totalStats(\%ft); #printf "TOTAL-CONV=%d\n", $totalConverted; if (($totalConverted+$warnings) and $print_stats) { - printStats("info: converted", \@statNames, \%ft, $warnings, $lineCount); + printStats(" info: converted", \@statNames, \%ft, $warnings, $lineCount); print STDERR " in '$fileName'\n"; } # Update totals for all files: @@ -1794,7 +1788,7 @@ while (@ARGV) { #-- Print total stats for all files processed: if ($print_stats and ($fileCount > 1)) { print STDERR "\n"; - printStats("info: TOTAL-converted", \@statNames, \%tt, $Twarnings, $TlineCount); + printStats(" info: TOTAL-converted", \@statNames, \%tt, $Twarnings, $TlineCount); print STDERR "\n"; foreach my $key (sort { $TwarningTags{$b} <=> $TwarningTags{$a} } keys %TwarningTags) { printf STDERR " warning: unconverted %s : %d\n", $key, $TwarningTags{$key}; @@ -1807,7 +1801,7 @@ if ($print_stats and ($fileCount > 1)) { print STDERR "\n"; print STDERR "\n"; } -if ($count_conversions) { +if ($print_stats) { foreach my $key (sort { $convertedTags{$b} <=> $convertedTags{$a} } keys %convertedTags) { printf STDERR " %s %d\n", $key, $convertedTags{$key}; } diff --git a/projects/clr/hipamd/hipify-clang/src/CUDA2HIP_Perl.cpp b/projects/clr/hipamd/hipify-clang/src/CUDA2HIP_Perl.cpp index 5407cae35c..d1f4423887 100644 --- a/projects/clr/hipamd/hipify-clang/src/CUDA2HIP_Perl.cpp +++ b/projects/clr/hipamd/hipify-clang/src/CUDA2HIP_Perl.cpp @@ -36,6 +36,29 @@ using namespace llvm; namespace perl { + const std::string sCopyright = + "##\n" + "# Copyright (c) 2015-present Advanced Micro Devices, Inc. All rights reserved.\n" + "#\n" + "# Permission is hereby granted, free of charge, to any person obtaining a copy\n" + "# of this software and associated documentation files (the \"Software\"), to deal\n" + "# in the Software without restriction, including without limitation the rights\n" + "# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n" + "# copies of the Software, and to permit persons to whom the Software is\n" + "# furnished to do so, subject to the following conditions:\n" + "#\n" + "# The above copyright notice and this permission notice shall be included in\n" + "# all copies or substantial portions of the Software.\n" + "#\n" + "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n" + "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n" + "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n" + "# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n" + "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n" + "# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n" + "# THE SOFTWARE.\n" + "##\n"; + const std::string tab = " "; const std::string double_tab = tab + tab; const std::string triple_tab = double_tab + tab; @@ -44,46 +67,102 @@ namespace perl { const std::string sReturn_m = "return $m;\n"; const std::string sForeach = "foreach $func (\n"; const std::string sMy = "my $m = 0;\n"; + const std::string sCudaDevice = "cudaDevice"; + const std::string sCudaDeviceId = "cudaDeviceId"; + const std::string sCudaDevices = "cudaDevices"; + const std::string sCudaDevice_t = "cudaDevice_t"; + const std::string sCudaIDs = "cudaIDs"; + const std::string sCudaGridDim = "cudaGridDim"; + const std::string sCudaDimGrid = "cudaDimGrid"; + const std::string sCudaDimBlock = "cudaDimBlock"; + const std::string sCudaGradInput = "cudaGradInput"; + const std::string sCudaGradOutput = "cudaGradOutput"; + const std::string sCudaInput = "cudaInput"; + const std::string sCudaOutput = "cudaOutput"; + const std::string sCudaIndices = "cudaIndices"; + const std::string sCudaGaugeField = "cudaGaugeField"; + const std::string sCudaMom = "cudaMom"; + const std::string sCudaGauge = "cudaGauge"; + const std::string sCudaInGauge = "cudaInGauge"; + const std::string sCudaColorSpinorField = "cudaColorSpinorField"; + const std::string sCudaSiteLink = "cudaSiteLink"; + const std::string sCudaFatLink = "cudaFatLink"; + const std::string sCudaStaple = "cudaStaple"; + const std::string sCudaCloverField = "cudaCloverField"; + const std::string sCudaParam = "cudaParam"; - void generateHostFunctions(std::unique_ptr& perlStreamPtr) { - *perlStreamPtr.get() << "\n" << sSub << " transformHostFunctions\n" << "{\n" << tab << sMy; + const std::set Whitelist{ + {sCudaDevice}, {sCudaDevice_t}, {sCudaIDs}, {sCudaGridDim}, {sCudaDimGrid}, {sCudaDimBlock}, {sCudaDeviceId}, {sCudaDevices}, + {sCudaGradInput}, {sCudaGradOutput}, {sCudaInput}, {sCudaOutput}, {sCudaIndices}, {sCudaGaugeField}, {sCudaMom}, {sCudaGauge}, + {sCudaInGauge}, {sCudaColorSpinorField}, {sCudaSiteLink}, {sCudaFatLink}, {sCudaStaple}, {sCudaCloverField}, {sCudaParam} + }; + + void generateHeader(std::unique_ptr& streamPtr) { + *streamPtr.get() << "#!/usr/bin/perl -w" << std::endl << std::endl; + *streamPtr.get() << sCopyright << std::endl; + *streamPtr.get() << "#usage hipify-perl [OPTIONS] INPUT_FILE" << std::endl << std::endl; + *streamPtr.get() << "use Getopt::Long;" << std::endl; + *streamPtr.get() << "my $whitelist = \"\";" << std::endl << std::endl; + *streamPtr.get() << "GetOptions(" << std::endl; + *streamPtr.get() << tab << " \"examine\" => \\$examine # Combines -no-output and -print-stats options." << std::endl; + *streamPtr.get() << tab << ", \"inplace\" => \\$inplace # Modify input file inplace, replacing input with hipified output, save backup in .prehip file." << std::endl; + *streamPtr.get() << tab << ", \"no-output\" => \\$no_output # Don't write any translated output to stdout." << std::endl; + *streamPtr.get() << tab << ", \"print-stats\" => \\$print_stats # Print translation statistics." << std::endl; + *streamPtr.get() << tab << ", \"quiet-warnings\" => \\$quiet_warnings # Don't print warnings on unknown CUDA functions." << std::endl; + *streamPtr.get() << tab << ", \"whitelist=s\" => \\$whitelist # TODO: test it beforehand" << std::endl; + *streamPtr.get() << ");" << std::endl << std::endl; + *streamPtr.get() << "$print_stats = 1 if $examine;" << std::endl; + *streamPtr.get() << "$no_output = 1 if $examine;" << std::endl << std::endl; + *streamPtr.get() << "# Whitelist of cuda[A-Z] identifiers, which are commonly used in CUDA sources but don't map to any CUDA API:" << std::endl; + *streamPtr.get() << "@whitelist = ("; + unsigned int num = 0; + for (const std::string &m : Whitelist) { + *streamPtr.get() << std::endl << tab << (num ? ", " : " ") << "\"" << m << "\""; + ++num; + } + *streamPtr.get() << std::endl << ");" << std::endl << std::endl; + *streamPtr.get() << "push(@whitelist, split(',', $whitelist));" << std::endl << std::endl; + } + + void generateHostFunctions(std::unique_ptr& streamPtr) { + *streamPtr.get() << std::endl << sSub << " transformHostFunctions" << std::endl << "{" << std::endl << tab << sMy; std::set &funcSet = DeviceSymbolFunctions0; const std::string s0 = "$m += s/(?second.hipName.str() << "\""; + *streamPtr.get() << (count ? ",\n" : "") << double_tab << "\"" << found->second.hipName.str() << "\""; count++; } } - *perlStreamPtr.get() << "\n" << tab << ")\n" << tab << "{\n" << double_tab; + *streamPtr.get() << std::endl << tab << ")" << std::endl << tab << "{" << std::endl << double_tab; switch (i) { - case 0: - default: - *perlStreamPtr.get() << s0 << sHIP_SYMBOL << "\\($2\\),/g\n"; break; - case 1: - *perlStreamPtr.get() << s1 << sHIP_SYMBOL << "\\($3\\)$4/g;\n"; break; - case 2: - *perlStreamPtr.get() << s0 << s_reinterpret_cast << "\\($2\\),/g\n"; break; - case 3: - *perlStreamPtr.get() << s1 << s_reinterpret_cast << "\\($3\\)$4/g;\n"; break; + case 0: + default: + *streamPtr.get() << s0 << sHIP_SYMBOL << "\\($2\\),/g" << std::endl; break; + case 1: + *streamPtr.get() << s1 << sHIP_SYMBOL << "\\($3\\)$4/g;" << std::endl; break; + case 2: + *streamPtr.get() << s0 << s_reinterpret_cast << "\\($2\\),/g" << std::endl; break; + case 3: + *streamPtr.get() << s1 << s_reinterpret_cast << "\\($3\\)$4/g;" << std::endl; break; } - *perlStreamPtr.get() << tab << "}\n"; + *streamPtr.get() << tab << "}" << std::endl; } - *perlStreamPtr.get() << tab << sReturn_m << "}\n"; + *streamPtr.get() << tab << sReturn_m << "}" << std::endl; } - void generateDeviceFunctions(std::unique_ptr& perlStreamPtr) { + void generateDeviceFunctions(std::unique_ptr& streamPtr) { unsigned int countUnsupported = 0; unsigned int countSupported = 0; std::stringstream sSupported; @@ -98,39 +177,39 @@ namespace perl { std::stringstream subWarnUnsupported; std::stringstream subCommon; std::string sCommon = tab + sMy + tab + sForeach; - subCountSupported << "\n" << sSub << " countSupportedDeviceFunctions\n" << "{\n" << (countSupported ? sCommon : tab + sReturn_0); - subWarnUnsupported << "\n" << sSub << " warnUnsupportedDeviceFunctions\n" << "{\n" << (countUnsupported ? tab + "my $line_num = shift;\n" + sCommon : tab + sReturn_0); + subCountSupported << std::endl << sSub << " countSupportedDeviceFunctions" << std::endl << "{" << std::endl << (countSupported ? sCommon : tab + sReturn_0); + subWarnUnsupported << std::endl << sSub << " warnUnsupportedDeviceFunctions" << std::endl << "{" << std::endl << (countUnsupported ? tab + "my $line_num = shift;\n" + sCommon : tab + sReturn_0); if (countSupported) { - subCountSupported << sSupported.str() << "\n" << tab << ")\n"; + subCountSupported << sSupported.str() << std::endl << tab << ")" << std::endl; } if (countUnsupported) { - subWarnUnsupported << sUnsupported.str() << "\n" << tab << ")\n"; + subWarnUnsupported << sUnsupported.str() << std::endl << tab << ")" << std::endl; } if (countSupported || countUnsupported) { - subCommon << tab << "{\n"; - subCommon << double_tab << "# match device function from the list, except those, which have a namespace prefix (aka somenamespace::umin(...));\n"; - subCommon << double_tab << "# function with only global namespace qualifier '::' (aka ::umin(...)) should be treated as a device function (and warned as well as without such qualifier);\n"; - subCommon << double_tab << "my $mt_namespace = m/(\\w+)::($func)\\s*\\(\\s*.*\\s*\\)/g;\n"; - subCommon << double_tab << "my $mt = m/($func)\\s*\\(\\s*.*\\s*\\)/g;\n"; - subCommon << double_tab << "if ($mt && !$mt_namespace) {\n"; - subCommon << triple_tab << "$m += $mt;\n"; + subCommon << tab << "{" << std::endl; + subCommon << double_tab << "# match device function from the list, except those, which have a namespace prefix (aka somenamespace::umin(...));" << std::endl; + subCommon << double_tab << "# function with only global namespace qualifier '::' (aka ::umin(...)) should be treated as a device function (and warned as well as without such qualifier);" << std::endl; + subCommon << double_tab << "my $mt_namespace = m/(\\w+)::($func)\\s*\\(\\s*.*\\s*\\)/g;" << std::endl; + subCommon << double_tab << "my $mt = m/($func)\\s*\\(\\s*.*\\s*\\)/g;" << std::endl; + subCommon << double_tab << "if ($mt && !$mt_namespace) {" << std::endl; + subCommon << triple_tab << "$m += $mt;" << std::endl; } if (countSupported) { subCountSupported << subCommon.str(); } if (countUnsupported) { subWarnUnsupported << subCommon.str(); - subWarnUnsupported << triple_tab << "print STDERR \" warning: $fileName:$line_num: unsupported device function \\\"$func\\\": $_\\n\";\n"; + subWarnUnsupported << triple_tab << "print STDERR \" warning: $fileName:$line_num: unsupported device function \\\"$func\\\": $_\\n\";" << std::endl; } if (countSupported || countUnsupported) { sCommon = double_tab + "}\n" + tab + "}\n" + tab + sReturn_m; } if (countSupported) subCountSupported << sCommon; if (countUnsupported) subWarnUnsupported << sCommon; - subCountSupported << "}\n"; - subWarnUnsupported << "}\n"; - *perlStreamPtr.get() << subCountSupported.str(); - *perlStreamPtr.get() << subWarnUnsupported.str(); + subCountSupported << "}" << std::endl; + subWarnUnsupported << "}" << std::endl; + *streamPtr.get() << subCountSupported.str(); + *streamPtr.get() << subWarnUnsupported.str(); } bool generate(bool Generate) { @@ -150,15 +229,16 @@ namespace perl { llvm::errs() << "\n" << sHipify << sError << EC.message() << ": " << tmpFile << "\n"; return false; } - std::unique_ptr perlStreamPtr = std::unique_ptr(new std::ofstream(tmpFile.c_str(), std::ios_base::trunc)); + std::unique_ptr streamPtr = std::unique_ptr(new std::ofstream(tmpFile.c_str(), std::ios_base::trunc)); + generateHeader(streamPtr); std::string sConv = "my $conversions = "; - *perlStreamPtr.get() << "@statNames = ("; + *streamPtr.get() << "@statNames = ("; for (int i = 0; i < NUM_CONV_TYPES - 1; ++i) { - *perlStreamPtr.get() << "\"" << counterNames[i] << "\", "; + *streamPtr.get() << "\"" << counterNames[i] << "\", "; sConv += "$ft{'" + std::string(counterNames[i]) + "'} + "; } - *perlStreamPtr.get() << "\"" << counterNames[NUM_CONV_TYPES - 1] << "\");\n\n"; - *perlStreamPtr.get() << sConv << "$ft{'" << counterNames[NUM_CONV_TYPES - 1] << "'};\n\n"; + *streamPtr.get() << "\"" << counterNames[NUM_CONV_TYPES - 1] << "\");" << std::endl << std::endl; + *streamPtr.get() << sConv << "$ft{'" << counterNames[NUM_CONV_TYPES - 1] << "'};" << std::endl << std::endl; for (int i = 0; i < NUM_CONV_TYPES; ++i) { if (i == CONV_INCLUDE_CUDA_MAIN_H || i == CONV_INCLUDE) { for (auto& ma : CUDA_INCLUDE_MAP) { @@ -168,21 +248,22 @@ namespace perl { std::string sHIP = ma.second.hipName.str(); sCUDA = std::regex_replace(sCUDA, std::regex("/"), "\\/"); sHIP = std::regex_replace(sHIP, std::regex("/"), "\\/"); - *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << sCUDA << "\\b/" << sHIP << "/g;\n"; + *streamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << sCUDA << "\\b/" << sHIP << "/g;" << std::endl; } } - } else { + } + else { for (auto& ma : CUDA_RENAMES_MAP()) { if (Statistics::isUnsupported(ma.second)) continue; if (i == ma.second.type) { - *perlStreamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;\n"; + *streamPtr.get() << "$ft{'" << counterNames[ma.second.type] << "'} += s/\\b" << ma.first.str() << "\\b/" << ma.second.hipName.str() << "/g;" << std::endl; } } } } - generateHostFunctions(perlStreamPtr); - generateDeviceFunctions(perlStreamPtr); - perlStreamPtr.get()->flush(); + generateHostFunctions(streamPtr); + generateDeviceFunctions(streamPtr); + streamPtr.get()->flush(); bool ret = true; EC = sys::fs::copy_file(tmpFile, dstPerlMap); if (EC) {