rocm-systems/bin/hipcc

#!/usr/bin/perl -w


# Need perl > 5.10 to use logic-defined or
use 5.006; use v5.10.1;
use File::Basename;
use File::Temp qw/ :mktemp  /;
use Cwd;
use Cwd 'abs_path';

# HIP compiler driver
# Will call NVCC or HCC (depending on target) and pass the appropriate include and library options for
# the target compiler and HIP infrastructure.

# Will pass-through options to the target compiler.  The tools calling HIPCC must ensure the compiler
# options are appropriate for the target compiler.

# Environment variable HIP_PLATFORM control compilation path:
# HIP_PLATFORM='nvcc' or HIP_PLATFORM='hcc'.
# If HIP_PLATFORM is not set hipcc will attempt auto-detect based on if nvcc is found.
#
# Other environment variable controls:
# HIP_PATH  : Path to HIP directory, default is one dir level above location of this script
# CUDA_PATH : Path to CUDA SDK (default /usr/local/cuda). Used on NVIDIA platforms only.
# HCC_HOME  : Path to HCC SDK (default /opt/rocm/hcc).  Used on AMD platforms only.
# HSA_PATH  : Path to HSA dir (default /opt/rocm/hsa).  Used on AMD platforms only.
# HIP_VDI_HOME : Path to HIP/VDI directory. Used on AMD platforms only.

if(scalar @ARGV == 0){
    print "No Arguments passed, exiting ...\n";
    exit(-1);
}

#---
# Function to parse config file
sub parse_config_file {
    my ($file, $config) = @_;
    if (open (CONFIG, "$file")) {
        while (<CONFIG>) {
            my $config_line=$_;
            chop ($config_line);
            $config_line =~ s/^\s*//;
            $config_line =~ s/\s*$//;
            if (($config_line !~ /^#/) && ($config_line ne "")) {
                my ($name, $value) = split (/=/, $config_line);
                $$config{$name} = $value;
            }
        }
        close(CONFIG);
    }
}

$verbose = $ENV{'HIPCC_VERBOSE'} // 0;
# Verbose: 0x1=commands, 0x2=paths, 0x4=hipcc args

$isWindows = $^O eq 'MSWin32';

$HIPCC_COMPILE_FLAGS_APPEND=$ENV{'HIPCC_COMPILE_FLAGS_APPEND'};
$HIPCC_LINK_FLAGS_APPEND=$ENV{'HIPCC_LINK_FLAGS_APPEND'};

$HIP_PATH=$ENV{'HIP_PATH'} // dirname (dirname $0); # use parent directory of hipcc
$HIP_VDI_HOME=$ENV{'HIP_VDI_HOME'};
$HIP_LIB_PATH=$ENV{'HIP_LIB_PATH'};
$HIP_CLANG_PATH=$ENV{'HIP_CLANG_PATH'};
$DEVICE_LIB_PATH=$ENV{'DEVICE_LIB_PATH'};
$HIP_CLANG_HCC_COMPAT_MODE=$ENV{'HIP_CLANG_HCC_COMPAT_MODE'}; # HCC compatibility mode

if (defined $HIP_VDI_HOME) {
    $HIP_INFO_PATH= "$HIP_VDI_HOME/lib/.hipInfo";
} else {
    $HIP_INFO_PATH= "$HIP_PATH/lib/.hipInfo";
}

#---
# Read .hipInfo
my %hipConfig = ();
parse_config_file("$HIP_INFO_PATH", \%hipConfig);

#---
# Temporary directories
my @tmpDirs = ();

#---
# Create a new temporary directory and return it
sub get_temp_dir {
    my $tmpdir = mkdtemp("/tmp/hipccXXXXXXXX");
    push (@tmpDirs, $tmpdir);
    return $tmpdir;
}

#---
# Delete all created temporary directories
sub delete_temp_dirs {
    if (@tmpDirs) {
        system ('rm -rf ' . join (' ', @tmpDirs));
    }
    return 0;
}

#---
#HIP_PLATFORM controls whether to use NVCC or HCC for compilation:
$HIP_PLATFORM= `$HIP_PATH/bin/hipconfig --platform` // "hcc";
$HIP_VERSION= `$HIP_PATH/bin/hipconfig --version`;
($HIP_VERSION_MAJOR, $HIP_VERSION_MINOR, $HIP_VERSION_PATCH) = split(/\./, $HIP_VERSION);
$HIP_COMPILER= $hipConfig{'HIP_COMPILER'};
$HIP_RUNTIME= $hipConfig{'HIP_RUNTIME'};

# If using VDI runtime, need to find HIP_VDI_HOME
if ($HIP_RUNTIME eq "VDI" and !defined $HIP_VDI_HOME) {
    my $hipcc_dir = dirname($0);
    if (-e "$hipcc_dir/../lib/bitcode") {
        $HIP_VDI_HOME = abs_path($hipcc_dir . "/..");
    } else {
        $HIP_VDI_HOME = "/opt/rocm/hip";
    }
}

if (defined $HIP_VDI_HOME) {
    if (!defined $HIP_CLANG_PATH and (-e "$HIP_VDI_HOME/bin/clang" or -e "$HIP_VDI_HOME/bin/clang.exe")) {
        $HIP_CLANG_PATH = "$HIP_VDI_HOME/bin";
        $HIP_CLANG_INCLUDE_PATH = "$HIP_VDI_HOME/include/clang";
    }
    if (!defined $DEVICE_LIB_PATH and -e "$HIP_VDI_HOME/lib/bitcode") {
        $DEVICE_LIB_PATH = "$HIP_VDI_HOME/lib/bitcode";
    }
    $HIP_INCLUDE_PATH = "$HIP_VDI_HOME/include";
    if (!defined $HIP_LIB_PATH) {
        $HIP_LIB_PATH = "$HIP_VDI_HOME/lib";
    }
}

if ($HIP_COMPILER eq "clang") {
  $HIP_PLATFORM = "clang";
  if (!defined $HIP_CLANG_PATH) {
    $HIP_CLANG_PATH = "/opt/rocm/llvm/bin";
  }
  if (!defined $DEVICE_LIB_PATH) {
    $DEVICE_LIB_PATH = "/opt/rocm/lib";
  }
}

if ($verbose & 0x2) {
    print ("HIP_PATH=$HIP_PATH\n");
    print ("HIP_PLATFORM=$HIP_PLATFORM\n");
}

# set if user explicitly requests -stdlib=libc++. (else we default to libstdc++ for better interop with g++):
$setStdLib = 0;  # TODO - set to 0

$target_gfx701 = 0;
$target_gfx801 = 0;
$target_gfx802 = 0;
$target_gfx803 = 0;
$target_gfx900 = 0;
$target_gfx906 = 0;
$default_amdgpu_target = 1;

if ($HIP_PLATFORM eq "clang") {
    $ROCM_PATH=$ENV{'ROCM_PATH'} // "/opt/rocm";
    $HIPCC="$HIP_CLANG_PATH/clang++";

    # If $HIPCC clang++ is not compiled, use clang instead
    if ( ! -e $HIPCC ) {
        $HIPCC="$HIP_CLANG_PATH/clang";
        $HIPLDFLAGS = "--driver-mode=g++";
    }

    $HIP_CLANG_VERSION = `$HIPCC --version`;
    $HIP_CLANG_VERSION=~/.*clang version ([^ ]+).*/;
    $HIP_CLANG_VERSION=$1;

    if (! defined $HIP_CLANG_INCLUDE_PATH) {
        $HIP_CLANG_INCLUDE_PATH = abs_path("$HIP_CLANG_PATH/../lib/clang/$HIP_CLANG_VERSION/include");
    }
    if (! defined $HIP_INCLUDE_PATH) {
        $HIP_INCLUDE_PATH = "$HIP_PATH/include";
    }
    if (! defined $HIP_LIB_PATH) {
        $HIP_LIB_PATH = "$HIP_PATH/lib";
    }
    if ($verbose & 0x2) {
        if (defined $HIP_VDI_HOME) {
            print ("HIP_VDI_HOME=$HIP_VDI_HOME\n");
        }
        print ("HIP_CLANG_PATH=$HIP_CLANG_PATH\n");
        print ("HIP_CLANG_INCLUDE_PATH=$HIP_CLANG_INCLUDE_PATH\n");
        print ("HIP_INCLUDE_PATH=$HIP_INCLUDE_PATH\n");
        print ("HIP_LIB_PATH=$HIP_LIB_PATH\n");
        print ("DEVICE_LIB_PATH=$DEVICE_LIB_PATH\n");
    }

    $HIPCXXFLAGS .= " -std=c++11 -isystem $HIP_CLANG_INCLUDE_PATH";
    $HIPLDFLAGS .= " -L$HIP_LIB_PATH";
    if (not $isWindows) {
      $HIPLDFLAGS .= " -Wl,--rpath-link=$HIP_LIB_PATH";
      $HIPLDFLAGS .= " -lhip_hcc";
    } else {
      $HIPLDFLAGS .= " -lamdhip64";
    }
    if ($HIP_CLANG_HCC_COMPAT_MODE) {
        ## Allow __fp16 as function parameter and return type.
        $HIPCXXFLAGS .= " -Xclang -fallow-half-arguments-and-returns -D__HIP_HCC_COMPAT_MODE__=1";
    }

    if ($HIP_RUNTIME eq "HCC" ) {
      $HSA_PATH=$ENV{'HSA_PATH'} // "/opt/rocm/hsa";
      $HIPCXXFLAGS .= " -isystem $HSA_PATH/include";
    }

} elsif ($HIP_PLATFORM eq "hcc") {
    $HIP_INCLUDE_PATH = "$HIP_PATH/include";
    if (! defined $HIP_LIB_PATH) {
        $HIP_LIB_PATH = "$HIP_PATH/lib";
    }
    $HSA_PATH=$ENV{'HSA_PATH'} // "/opt/rocm/hsa";

    $HCC_HOME=$ENV{'HCC_HOME'} // $hipConfig{'HCC_HOME'} // "/opt/rocm/hcc";

    $HCC_VERSION=`${HCC_HOME}/bin/hcc --version`;
    $HCC_VERSION=~/.*based on HCC ([^ ]+).*/;
    $HCC_VERSION=$1;
    $HCC_VERSION_MAJOR=$HCC_VERSION;
    $HCC_VERSION_MAJOR=~s/\..*//;

    $ROCM_PATH=$ENV{'ROCM_PATH'} // "/opt/rocm";

    $HIP_ATP_MARKER=$ENV{'HIP_ATP_MARKER'} // 1;
    $marker_path = "$ROCM_PATH/profiler/CXLActivityLogger";

    # HCC* may be used to compile src/hip_hcc.o (and also feed the HIPCXXFLAGS below)
    $HCC = "$HCC_HOME/bin/hcc";
    $HCCFLAGS = "-hc  -D__HIPCC__ -isystem $HCC_HOME/include ";

    $HIPCC=$HCC;
    $HIPCXXFLAGS = $HCCFLAGS;

    $HIPLDFLAGS = `${HCC_HOME}/bin/hcc-config --ldflags`;

    #### GCC system includes workaround ####
    $HCC_WA_FLAGS = " ";
    $HOST_OSNAME= `cat /etc/os-release | grep "^ID\=" | cut -d= -f2 | tr -d '\n'`;
    if ($HCC_VERSION_MAJOR eq 1) {
        my $GCC_CUR_VER = `gcc -dumpversion`;
        my $GPP_CUR_VER = `g++ -dumpversion`;
        $GCC_CUR_VER =~ s/\R//g;
        $GPP_CUR_VER =~ s/\R//g;

        my @GPP_VER_FIELDS = split('\.', $GPP_CUR_VER);

        # Only include the libstdc++ headers and libraries flags explicitly if the g++ is older than version 5.
        # That's because HCC already uses libstdc++ by default if a newer g++/libstdc++ is available
        # Cent OS 7 and RHEL 7.4 cannot use libstdc++ for compilation, default to libc++
        if (${GCC_CUR_VER} eq ${GPP_CUR_VER} and $GPP_VER_FIELDS[0] < 5 and ($HOST_OSNAME ne "\"centos\"") and ($HOST_OSNAME ne "\"rhel\"")) {
            $HCC_WA_FLAGS .= " -stdlib=libstdc++  -isystem /usr/include/x86_64-linux-gnu -isystem /usr/include/x86_64-linux-gnu/c++/${GCC_CUR_VER} -isystem /usr/include/c++/${GCC_CUR_VER} ";
            # Add C++ libs for GCC.
            $HIPLDFLAGS .= " -lstdc++";
        }
    }

    # Force -stdlib=libc++ on UB14.04
    $HOST_OSVER= `cat /etc/os-release | grep "^VERSION_ID\=" | cut -d= -f2 | tr -d '\n'`;
    if ($HOST_OSNAME eq "ubuntu" and $HOST_OSVER eq "\"14.04\"") {
        $HIPCXXFLAGS .= " -stdlib=libc++";
        $setStdLib = 1;
    }

    $HIPCXXFLAGS .= " -isystem $HIP_PATH/include/hip/hcc_detail/cuda";
    $HIPCXXFLAGS .= " -isystem $HSA_PATH/include";
    $HIPCXXFLAGS .= " -Wno-deprecated-register";

    $HIPLDFLAGS .= " -L$HSA_PATH/lib -L$ROCM_PATH/lib -lhsa-runtime64 -lhc_am ";
#    $HIPLDFLAGS .= " -L$HCC_HOME/compiler/lib -lLLVMAMDGPUDesc -lLLVMAMDGPUUtils -lLLVMMC -lLLVMCore -lLLVMSupport ";

    # Add trace marker library:
    # TODO - once we cleanly separate the HIP API headers from HIP library headers this logic should move to CMakebuild option - apps do not need to see the marker library.
    if ($HIP_ATP_MARKER) {
        $marker_inc_path = "$marker_path/include";
        if (-e $marker_inc_path) {
            $HIPCXXFLAGS .= " -isystem $marker_inc_path";
        }
    }

    $marker_lib_path = "$marker_path/bin/x86_64";
    if (-e $marker_lib_path) {
        $HIPLDFLAGS .= " -L$marker_lib_path -lCXLActivityLogger -Wl,--rpath=$marker_lib_path";
    }

    if (not $isWindows) {
        $HIPLDFLAGS .= " -lm";
    }

    if ($verbose & 0x2) {
        print ("HSA_PATH=$HSA_PATH\n");
        print ("HCC_HOME=$HCC_HOME\n");
    }

} elsif ($HIP_PLATFORM eq "nvcc") {
    $CUDA_PATH=$ENV{'CUDA_PATH'} // '/usr/local/cuda';
    $HIP_INCLUDE_PATH = "$HIP_PATH/include";
    if ($verbose & 0x2) {
        print ("CUDA_PATH=$CUDA_PATH\n");
    }

    $HIPCC="$CUDA_PATH/bin/nvcc";
    $HIPCXXFLAGS .= " -Wno-deprecated-gpu-targets ";
    $HIPCXXFLAGS .= " -isystem $CUDA_PATH/include";

    $HIPLDFLAGS = " -Wno-deprecated-gpu-targets -lcuda -lcudart -L$CUDA_PATH/lib64";
} else {
    printf ("error: unknown HIP_PLATFORM = '$HIP_PLATFORM'");
    exit (-1);
}

# Add paths to common HIP includes:
$HIPCXXFLAGS .= " -isystem $HIP_INCLUDE_PATH -DHIP_VERSION_MAJOR=$HIP_VERSION_MAJOR -DHIP_VERSION_MINOR=$HIP_VERSION_MINOR -DHIP_VERSION_PATCH=$HIP_VERSION_PATCH" ;

my $compileOnly = 0;
my $needCXXFLAGS = 0;  # need to add CXX flags to compile step
my $needLDFLAGS = 1;   # need to add LDFLAGS to compile step.
my $hasC = 0;          # options contain a c-style file (NVCC must force recognition as GPU file)
my $hasCU = 0;         # options contain a cu-style file (HCC must force recognition as GPU file)
my $needHipHcc = ($HIP_PLATFORM eq 'hcc');      # set if we need to link hip_hcc.o from src tree. (some builds, ie cmake, provide their own)
my $printHipVersion = 0;    # print HIP version
my $runCmd = 1;
my $buildDeps = 0;
my $linkType = 1;
my $setLinkType = 0;

my @options = ();
my @inputs  = ();

if ($verbose & 0x4) {
    print "hipcc-args: ", join (" ", @ARGV), "\n";
}

# Handle code object generation
my $ISACMD="";
if($HIP_PLATFORM eq "hcc"){
    $ISACMD .= "$HIP_PATH/bin/lpl ";
    if($ARGV[0] eq "--genco"){
        foreach $isaarg (@ARGV[1..$#ARGV]){
            $ISACMD .= " ";
            $ISACMD .= $isaarg;
        }
        if ($verbose & 0x1) {
            print "hipcc-cmd: ", $ISACMD, "\n";
        }
        system($ISACMD) and die();
        exit(0);
    }
}

if(($HIP_PLATFORM eq "hcc")){
    $ENV{HCC_EXTRA_LIBRARIES}="\n";
}

if($HIP_PLATFORM eq "nvcc"){
    $ISACMD .= "$HIP_PATH/bin/hipcc -ptx ";
    if($ARGV[0] eq "--genco"){
        foreach $isaarg (@ARGV[1..$#ARGV]){
            $ISACMD .= " ";
            $ISACMD .= $isaarg;
        }
        if ($verbose & 0x1) {
            print "hipcc-cmd: ", $ISACMD, "\n";
        }
        system($ISACMD) and die();
        exit(0);
    }
}

my $toolArgs = "";  # arguments to pass to the hcc or nvcc tool
my $optArg = ""; # -O args
my $gArg = ""; # -g args

foreach $arg (@ARGV)
{
    $trimarg = $arg;
    $trimarg =~ s/^\s+|\s+$//g;  # Remive whitespace
    my $swallowArg = 0;
    if ($arg eq '-c' or $arg eq '--genco') {
        $compileOnly = 1;
        $needCXXFLAGS = 1;
        $needLDFLAGS  = 0;
    }
    if ($arg eq '-o') {
        $needLDFLAGS = 1;
    }

    if(($trimarg eq '-stdlib=libc++') and ($setStdLib eq 0))
    {
        $HIPCXXFLAGS .= " -stdlib=libc++";
        $setStdLib = 1;
    }

    # TODO: Add support for comma separated list like HCC_AMDGPU_TARGET
    if($arg eq '--amdgpu-target=gfx701')
    {
        $target_gfx701 = 1;
        $default_amdgpu_target = 0;
    }
    if($arg eq '--amdgpu-target=gfx801')
    {
        $target_gfx801 = 1;
        $default_amdgpu_target = 0;
    }
    if($arg eq '--amdgpu-target=gfx802')
    {
        $target_gfx802 = 1;
        $default_amdgpu_target = 0;
    }
    if($arg eq '--amdgpu-target=gfx803')
    {
        $target_gfx803 = 1;
        $default_amdgpu_target = 0;
    }
    if($arg eq '--amdgpu-target=gfx900')
    {
        $target_gfx900 = 1;
        $default_amdgpu_target = 0;
    }
    if($arg eq '--amdgpu-target=gfx906')
    {
        $target_gfx906 = 1;
        $default_amdgpu_target = 0;
    }

    # hip-clang does not accept --amdgpu-target= options.
    if (($arg =~ /--amdgpu-target=/) and $HIP_PLATFORM eq 'clang' ) {
        $swallowArg = 1;
    }

    if (($arg =~ /--genco/) and $HIP_PLATFORM eq 'clang' ) {
        $arg = "--cuda-device-only";
    }

    if(($trimarg eq '-stdlib=libstdc++') and ($setStdLib eq 0))
    {
        $HIPCXXFLAGS .= $HCC_WA_FLAGS;
        $setStdLib = 1;
    }
    if($trimarg eq '--version') {
        $printHipVersion = 1;
    }
    if($trimarg eq '--short-version') {
        $printHipVersion = 1;
        $runCmd = 0;
    }
    if($trimarg eq '-M') {
        $compileOnly = 1;
        $buildDeps = 1;
    }
    if($trimarg eq '-use_fast_math') {
        $HIPCXXFLAGS .= " -DHIP_FAST_MATH ";
    }
    if(($trimarg eq '-use-staticlib') and ($setLinkType eq 0))
    {
        $linkType = 0;
        $setLinkType = 1;
    }
    if(($trimarg eq '-use-sharedlib') and ($setLinkType eq 0))
    {
        $linkType = 1;
        $setLinkType = 1;
    }
    if($arg =~ m/^-O/)
    {
        $optArg = $arg;
    }
    if($arg =~ m/^-g/)
    {
        $gArg = $arg;
    }

    ## process linker response file for hip-clang
    ## extract object files from static library and pass them directly to
    ## hip-clang in command line.
    ## ToDo: Remove this after hip-clang switch to lto and lld is able to
    ## handle clang-offload-bundler bundles.
    if ($arg =~ m/^-Wl,@/ and $HIP_PLATFORM eq 'clang') {
        my $file = substr $arg, 5;
        open my $in, "<:encoding(utf8)", $file or die "$file: $!";
        my $new_arg = "";
        my $tmpdir = get_temp_dir ();
        my $new_file = "$tmpdir/response_file";
        open my $out, ">", $new_file or die "$new_file: $!";
        while (my $line = <$in>) {
            chomp $line;
            if ($line =~ m/\.a$/ || $line =~ m/\.lo$/) {
                my $libFile = $line;
                my $path = abs_path($line);
                my @objs = split ('\n', `cd $tmpdir; ar xv $path`);
                ## Check if all files in .a are object files.
                my $allIsObj = 1;
                my $realObjs = "";
                foreach my $obj (@objs) {
                    chomp $obj;
                    $obj =~ s/^x - //;
                    $obj = "$tmpdir/$obj";
                    my $fileType = `file $obj`;
                    my $isObj = ($fileType =~ m/ELF/ or $fileType =~ m/COFF/);
                    $allIsObj = ($allIsObj and $isObj);
                    if ($isObj) {
                        $realObjs = ($realObjs . " " . $obj);
                    } else {
                        push (@inputs, $obj);
                        $new_arg = "$new_arg $obj";
                    }
                }
                chomp $realObjs;
                if ($allIsObj) {
                    print $out "$line\n";
                } elsif ($realObjs) {
                    my($libBaseName, $libDir, $libExt) = fileparse($libFile);
                    $libBaseName = mktemp($libBaseName . "XXXX") . $libExt;
                    system("cd $tmpdir; ar c $libBaseName $realObjs");
                    print $out "$tmpdir/$libBaseName\n";
                }
            } elsif ($line =~ m/\.o$/) {
                my $fileType = `file $line`;
                my $isObj = ($fileType =~ m/ELF/ or $fileType =~ m/COFF/);
                if ($isObj) {
                    print $out "$line\n";
                } else {
                    push (@inputs, $line);
                    $new_arg = "$new_arg $line";
                }
            } else {
                print $out "$line\n";
            }
        }
        close $in;
        close $out;
        $arg = "$new_arg -Wl,\@$new_file";
    } elsif (($arg =~ m/\.a$/ || $arg =~ m/\.lo$/) &&
             $HIP_PLATFORM eq 'clang') {
        ## process static library for hip-clang
        ## extract object files from static library and pass them directly to
        ## hip-clang.
        ## ToDo: Remove this after hip-clang switch to lto and lld is able to
        ## handle clang-offload-bundler bundles.
        
        my $new_arg = "";
        my $tmpdir = get_temp_dir ();
        my $libFile = $arg;
        my $path = abs_path($arg);
        my @objs = split ('\n', `cd $tmpdir; ar xv $path`);
        ## Check if all files in .a are object files.
        my $allIsObj = 1;
        my $realObjs = "";
        foreach my $obj (@objs) {
            chomp $obj;
            $obj =~ s/^x - //;
            $obj = "$tmpdir/$obj";
            my $fileType = `file $obj`;
            my $isObj = ($fileType =~ m/ELF/ or $fileType =~ m/COFF/);
            if ($fileType =~ m/ELF/) {
                my $sections = `readelf -e -W $obj`;
                $isObj = !($sections =~ m/__CLANG_OFFLOAD_BUNDLE__/);
            }
            $allIsObj = ($allIsObj and $isObj);
            if ($isObj) {
                $realObjs = ($realObjs . " " . $obj);
            } else {
                push (@inputs, $obj);
                if ($new_arg ne "") {
                    $new_arg .= " ";
                }
                $new_arg .= "$obj";
            }
        }
        chomp $realObjs;
        if ($allIsObj) {
            $new_arg = $arg;
        } elsif ($realObjs) {
            my($libBaseName, $libDir, $libExt) = fileparse($libFile);
            $libBaseName = mktemp($libBaseName . "XXXX") . $libExt;
            system("cd $tmpdir; ar c $libBaseName $realObjs");
            $new_arg .= " $tmpdir/$libBaseName";
        }
        $arg = "$new_arg";
        if ($toolArgs =~ m/-Xlinker$/) {
            $toolArgs = substr $toolArgs, 0, -8;
            chomp $toolArgs;
        }
    } elsif ($arg =~ m/^-/) {
        # options start with -

        # Process HIPCC options here:
        if ($arg =~ m/^--hipcc/) {
            $swallowArg = 1;
            #if $arg eq "--hipcc_profile") {  # Example argument here, hipcc
            #
            #}
        } else {
            push (@options, $arg);
        }
        #print "O: <$arg>\n";
    } else {
        # input files and libraries
        if (($arg =~ /\.cpp$/) or ($arg =~ /\.c$/) or ($arg =~ /\.cc$/) ) {
            $hasC = 1;
            $needCXXFLAGS = 1;
            if ($HIP_PLATFORM eq 'clang') {
                $toolArgs .= " -x hip"
            }
        }
        if (($arg =~ /\.cu$/) or ($arg =~ /\.cuh$/) or ($arg =~ /\.hip$/)) {
            $hasCU = 1;
            $needCXXFLAGS = 1;
            if ($HIP_PLATFORM eq 'clang') {
                $toolArgs .= " -x hip"
            }
        }

        push (@inputs, $arg);
        #print "I: <$arg>\n";
    }
    $toolArgs .= " $arg" unless $swallowArg;
}

if($HIP_PLATFORM eq "hcc" or $HIP_PLATFORM eq "clang"){
    # No AMDGPU target specified at commandline. So look for HCC_AMDGPU_TARGET
    if($default_amdgpu_target eq 1 and defined $ENV{HCC_AMDGPU_TARGET})
    {
        foreach my $target (split(/,/, $ENV{HCC_AMDGPU_TARGET}))
        {
            if($target eq 'gfx701')
            {
                $target_gfx701 = 1;
                $default_amdgpu_target = 0;
            }
            if($target eq 'gfx801')
            {
                $target_gfx801 = 1;
                $default_amdgpu_target = 0;
            }
            if($target eq 'gfx802')
            {
                $target_gfx802 = 1;
                $default_amdgpu_target = 0;
            }
            if($target eq 'gfx803')
            {
                $target_gfx803 = 1;
                $default_amdgpu_target = 0;
            }
            if($target eq 'gfx900')
            {
                $target_gfx900 = 1;
                $default_amdgpu_target = 0;
            }
            if($target eq 'gfx906')
            {
                $target_gfx906 = 1;
                $default_amdgpu_target = 0;
            }
        }
    }
    # Else try using rocm_agent_enumerator
    if($default_amdgpu_target eq 1)
    {
        $ROCM_AGENT_ENUM = "${ROCM_PATH}/bin/rocm_agent_enumerator";

        my $myAgents = `${ROCM_AGENT_ENUM} -t GPU`;
        my @agentsLine = split('\n', $myAgents);

        foreach my $val (@agentsLine) {
            if($val eq "gfx701") {
                $target_gfx701 = 1;
                $default_amdgpu_target = 0;
            }
            if($val eq "gfx801") {
                $target_gfx801 = 1;
                $default_amdgpu_target = 0;
            }
            if($val eq "gfx802") {
                $target_gfx802 = 1;
                $default_amdgpu_target = 0;
            }
            if($val eq "gfx803") {
                $target_gfx803 = 1;
                $default_amdgpu_target = 0;
            }
            if($val eq "gfx900") {
                $target_gfx900 = 1;
                $default_amdgpu_target = 0;
            }
            if($val eq "gfx906") {
                $target_gfx906 = 1;
                $default_amdgpu_target = 0;
            }
        }
    }
    # rocm_agent_enumerator failed! Throw an error and die if linking is required
    if ($default_amdgpu_target eq 1 and $compileOnly eq 0)
    {
        print "No valid AMD GPU target was either specified or found. Please specify a valid target using --amdgpu-target=" and die();
    }

    $ENV{HCC_EXTRA_LIBRARIES}="\n";

    if($HIP_PLATFORM eq "hcc") {
        $GPU_ARCH_OPT = " --amdgpu-target=";
    } else {
        $GPU_ARCH_OPT = " --cuda-gpu-arch=";
    }
    # Handle ROCm target platform
    if ($target_gfx701 eq 1) {
        $GPU_ARCH_ARG = $GPU_ARCH_OPT . "gfx701";
        $HIPLDFLAGS .= $GPU_ARCH_ARG;
        if ($HIP_PLATFORM eq 'clang') {
            $HIPCXXFLAGS .= $GPU_ARCH_ARG;;
        }
        $HIPCXXFLAGS .= " -D__HIP_ARCH_GFX701__=1 ";
    }
    if ($target_gfx801 eq 1) {
        $GPU_ARCH_ARG = $GPU_ARCH_OPT . "gfx801";
        $HIPLDFLAGS .= $GPU_ARCH_ARG;
        if ($HIP_PLATFORM eq 'clang') {
            $HIPCXXFLAGS .= $GPU_ARCH_ARG;;
        }
        $HIPCXXFLAGS .= " -D__HIP_ARCH_GFX801__=1 ";
    }
    if ($target_gfx802 eq 1) {
        $GPU_ARCH_ARG = $GPU_ARCH_OPT . "gfx802";
        $HIPLDFLAGS .= $GPU_ARCH_ARG;
        if ($HIP_PLATFORM eq 'clang') {
            $HIPCXXFLAGS .= $GPU_ARCH_ARG;;
        }
        $HIPCXXFLAGS .= " -D__HIP_ARCH_GFX802__=1 ";
    }
    if ($target_gfx803 eq 1) {
        $GPU_ARCH_ARG = $GPU_ARCH_OPT . "gfx803";
        $HIPLDFLAGS .= $GPU_ARCH_ARG;
        if ($HIP_PLATFORM eq 'clang') {
            $HIPCXXFLAGS .= $GPU_ARCH_ARG;;
        }
        $HIPCXXFLAGS .= " -D__HIP_ARCH_GFX803__=1 ";
    }
    if ($target_gfx900 eq 1) {
        $GPU_ARCH_ARG = $GPU_ARCH_OPT . "gfx900";
        $HIPLDFLAGS .= $GPU_ARCH_ARG;
        if ($HIP_PLATFORM eq 'clang') {
            $HIPCXXFLAGS .= $GPU_ARCH_ARG;;
        }
        $HIPCXXFLAGS .= " -D__HIP_ARCH_GFX900__=1 ";
    }
    if ($target_gfx906 eq 1) {
        $GPU_ARCH_ARG = $GPU_ARCH_OPT . "gfx906";
        $HIPLDFLAGS .= $GPU_ARCH_ARG;
        if ($HIP_PLATFORM eq 'clang') {
            $HIPCXXFLAGS .= $GPU_ARCH_ARG;;
        }
        $HIPCXXFLAGS .= " -D__HIP_ARCH_GFX906__=1 ";
    }
}

if ($hasC and $HIP_PLATFORM eq 'nvcc') {
    $HIPCXXFLAGS .= " -x cu";
}
if ($hasCU and $HIP_PLATFORM eq 'hcc') {
    $HIPCXXFLAGS .= " -x c++";
}

if ($buildDeps and $HIP_PLATFORM eq 'nvcc') {
    $HIPCXXFLAGS .= " -M -D__CUDACC__";
}

if ($buildDeps and $HIP_PLATFORM eq 'clang') {
    $HIPCXXFLAGS .= " --cuda-host-only";
}

# Add --hip-link only if there are no source files.
if (!$needCXXFLAGS and $HIP_PLATFORM eq 'clang') {
    $HIPLDFLAGS .= " --hip-link";
}

if ($setStdLib eq 0 and $HIP_PLATFORM eq 'hcc')
{
    $HIPCXXFLAGS .= $HCC_WA_FLAGS;
}

if ($needHipHcc) {
    if ($linkType eq 0) {
        substr($HIPLDFLAGS,0,0) = "  $HIP_LIB_PATH/libhip_hcc_static.a " ;
    } else {
        substr($HIPLDFLAGS,0,0) = "  -Wl,--rpath=$HIP_LIB_PATH $HIP_LIB_PATH/libhip_hcc.so ";
    }
}

# hipcc currrently requires separate compilation of source files, ie it is not possible to pass
# CPP files combined with .O files
# Reason is that NVCC uses the file extension to determine whether to compile in CUDA mode or
# pass-through CPP mode.

if ($HIP_PLATFORM eq "clang") {
    # Set default optimization level to -O3 for hip-clang.
    if ($optArg eq "" and $gArg ne "-g") {
        $HIPCXXFLAGS .= " -O3";
        $HIPLDFLAGS .= " -O3";
    }
    $HIP_DEVLIB_FLAGS = " --hip-device-lib-path=$DEVICE_LIB_PATH";
    $HIPCXXFLAGS .= " $HIP_DEVLIB_FLAGS";
    if ($isWindows) {
        $HIPCXXFLAGS .= " -std=c++14 -fms-extensions -fms-compatibility";
    } else {
        $HIPLDFLAGS .= " -lgcc_s -lgcc -lpthread -lm";
    }
}


if ($HIPCC_COMPILE_FLAGS_APPEND) {
    $HIPCXXFLAGS .= " $HIPCC_COMPILE_FLAGS_APPEND";
}
if ($HIPCC_LINK_FLAGS_APPEND) {
    $HIPLDFLAGS .= " $HIPCC_LINK_FLAGS_APPEND";
}

my $CMD="$HIPCC";
if ($needCXXFLAGS) {
    $CMD .= " $HIPCXXFLAGS";
}
if ($needLDFLAGS and not $compileOnly) {
    $CMD .= " $HIPLDFLAGS";
}
$CMD .= " $toolArgs";

if ($verbose & 0x1) {
    print "hipcc-cmd: ", $CMD, "\n";
}

if ($printHipVersion) {
    if ($runCmd) {
        print "HIP version: "
    }
    print $HIP_VERSION, "\n";
}
if ($runCmd) {
    if ($HIP_PLATFORM eq "hcc" and exists($hipConfig{'HCC_VERSION'}) and $HCC_VERSION ne $hipConfig{'HCC_VERSION'}) {
        print ("HIP ($HIP_PATH) was built using hcc $hipConfig{'HCC_VERSION'}, but you are using $HCC_HOME/hcc with version $HCC_VERSION from hipcc. Please rebuild HIP including cmake or update HCC_HOME variable.\n") ;
        die unless $ENV{'HIP_IGNORE_HCC_VERSION'};
    }
    system ("$CMD");
    if ($? == -1) {
        print "failed to execute: $!\n";
        exit($?);
    }
    elsif ($? & 127) {
        printf "child died with signal %d, %s coredump\n",
        ($? & 127),  ($? & 128) ? 'with' : 'without';
        exit($?);
    }
    else {
         $CMD_EXIT_CODE = $? >> 8;
    }
    $? or delete_temp_dirs ();
    exit($CMD_EXIT_CODE);
}

# vim: ts=4:sw=4:expandtab:smartindent