From 2576f514602ac054b33aa6c5dc48bed40190a64d Mon Sep 17 00:00:00 2001 From: foreman Date: Fri, 21 Oct 2016 13:18:35 -0400 Subject: [PATCH] P4 to Git Change 1329974 by lmoriche@lmoriche_opencl_dev on 2016/10/21 13:09:44 SWDEV-105604 - OpenCL program manager for LC on PAL (Initial check-in) - Created LightningProgram and LightningKernel classes - Implemented LightningProgram::compileImpl and LightningProgram::linkImpl using LC - Implemented LightningKernel::init (to set up args using the LC metadata) Tests: Passes clinfo and HelloWorld. Pre-checkin: http://ocltc.amd.com:8111/viewModification.html?modId=77957&personal=true&buildTypeId=&tab=vcsModificationBuilds&show_all_builds=true Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/build/Makefile.api#141 edit ... //depot/stg/opencl/drivers/opencl/compiler/Makefile#67 edit ... //depot/stg/opencl/drivers/opencl/compiler/sclibdefs.opencl#11 edit ... //depot/stg/opencl/drivers/opencl/library/build/Makefile.library#57 edit ... //depot/stg/opencl/drivers/opencl/make/llvm.git/lib/Target/Makefile#2 edit ... //depot/stg/opencl/drivers/opencl/make/llvm.git/llvmdefs#3 edit ... //depot/stg/opencl/drivers/opencl/make/llvm.git/llvmlibs#9 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/build/Makefile.pal#7 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palcompiler.cpp#4 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldefs.hpp#13 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#26 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#10 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.cpp#10 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.hpp#5 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palprogram.cpp#10 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palprogram.hpp#8 edit [ROCm/clr commit: cd1780073439345fb9ac7a18615d1b2929b3a54f] --- .../rocclr/runtime/device/pal/palcompiler.cpp | 249 ++++++++- .../clr/rocclr/runtime/device/pal/paldefs.hpp | 26 +- .../rocclr/runtime/device/pal/paldevice.cpp | 24 +- .../rocclr/runtime/device/pal/paldevice.hpp | 4 +- .../rocclr/runtime/device/pal/palkernel.cpp | 529 +++++++++++++++++- .../rocclr/runtime/device/pal/palkernel.hpp | 46 +- .../rocclr/runtime/device/pal/palprogram.cpp | 486 +++++++++++++++- .../rocclr/runtime/device/pal/palprogram.hpp | 43 +- 8 files changed, 1355 insertions(+), 52 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/pal/palcompiler.cpp b/projects/clr/rocclr/runtime/device/pal/palcompiler.cpp index b463d5a29c..3ef2f9b999 100644 --- a/projects/clr/rocclr/runtime/device/pal/palcompiler.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palcompiler.cpp @@ -12,8 +12,18 @@ #include "device/pal/palprogram.hpp" #include "device/pal/palkernel.hpp" #include "utils/options.hpp" +#if defined(WITH_LIGHTNING_COMPILER) +#include "driver/AmdCompiler.h" +#include "opencl1.2-c.amdgcn.inc" +#include "opencl2.0-c.amdgcn.inc" +#endif // !defined(WITH_LIGHTNING_COMPILER) #include +#if defined(ATI_OS_LINUX) +#include +#include +#endif // defined(ATI_OS_LINUX) + //CLC_IN_PROCESS_CHANGE extern int openclFrontEnd(const char* cmdline, std::string*, std::string* typeInfo = nullptr); @@ -26,6 +36,9 @@ HSAILProgram::compileImpl( const char** headerIncludeNames, amd::option::Options* options) { +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); +#else // !defined(WITH_LIGHTNING_COMPILER) acl_error errorCode; aclTargetInfo target; @@ -135,8 +148,242 @@ HSAILProgram::compileImpl( // Save the binary in the interface class saveBinaryAndSetType(TYPE_COMPILED); - +#endif // !defined(WITH_LIGHTNING_COMPILER) return true; } +#if defined(WITH_LIGHTNING_COMPILER) +static std::string llvmBin_(amd::Os::getEnvironment("LLVM_BIN")); + +#if defined(ATI_OS_LINUX) +static pthread_once_t once = PTHREAD_ONCE_INIT; + +static void +checkLLVM_BIN() +{ + if (llvmBin_.empty()) { + Dl_info info; + if (dladdr((const void*)&amd::Device::init, &info)) { + llvmBin_ = dirname(strdup(info.dli_fname)); + size_t pos = llvmBin_.rfind("lib"); + if (pos != std::string::npos) { + llvmBin_.replace(pos, 3, "bin"); + } + } + } +#if defined(DEBUG) + std::string clangExe(llvmBin_ + "/clang"); + struct stat buf; + if (stat(clangExe.c_str(), &buf)) { + std::string msg("Could not find the Clang binary in " + llvmBin_); + LogWarning(msg.c_str()); + } +#endif // defined(DEBUG) +} +#endif // defined(ATI_OS_LINUX) + +std::auto_ptr +LightningProgram::newCompilerInstance() +{ +#if defined(ATI_OS_LINUX) + pthread_once(&once, checkLLVM_BIN); +#endif // defined(ATI_OS_LINUX) + return std::auto_ptr( + amd::opencl_driver::CompilerFactory().CreateAMDGPUCompiler(llvmBin_)); +} + +bool +LightningProgram::compileImpl( + const std::string& sourceCode, + const std::vector& headers, + const char** headerIncludeNames, + amd::option::Options* options) +{ + using namespace amd::opencl_driver; + std::auto_ptr C(newCompilerInstance()); + std::vector inputs; + + Data* input = C->NewBufferReference(DT_CL, + sourceCode.c_str(), sourceCode.length()); + if (input == NULL) { + buildLog_ += "Error while creating data from source code"; + return false; + } + + inputs.push_back(input); + + amd::opencl_driver::Buffer* output = C->NewBuffer(DT_LLVM_BC); + if (output == NULL) { + buildLog_ += "Error while creating buffer for the LLVM bitcode"; + return false; + } + + //Set the options for the compiler + std::ostringstream ostrstr; + std::copy(options->clangOptions.begin(), options->clangOptions.end(), + std::ostream_iterator(ostrstr, " ")); + + ostrstr << " -m" << sizeof(void*) * 8; + std::string driverOptions(ostrstr.str()); + + const char* xLang = options->oVariables->XLang; + if (xLang != NULL && strcmp(xLang, "cl")) { + buildLog_ += "Unsupported OpenCL language.\n"; + } + + //FIXME_Nikolay: the program manager should be setting the language + //driverOptions.append(" -x cl"); + + driverOptions.append(" -cl-std=").append(options->oVariables->CLStd); + + // Set the -O# + std::ostringstream optLevel; + optLevel << " -O" << options->oVariables->OptLevel; + driverOptions.append(optLevel.str()); + + // Set the machine target + std::ostringstream mCPU; + mCPU << " -mcpu=gfx" << dev().hwInfo()->gfxipVersion_; + driverOptions.append(mCPU.str()); + + driverOptions.append(options->llvmOptions); + driverOptions.append(hsailOptions()); + + //Find the temp folder for the OS + std::string tempFolder = amd::Os::getEnvironment("TEMP"); + if (tempFolder.empty()) { + tempFolder = amd::Os::getEnvironment("TMP"); + if (tempFolder.empty()) { + tempFolder = WINDOWS_SWITCH(".","/tmp");; + } + } + //Iterate through each source code and dump it into tmp + std::fstream f; + std::vector headerFileNames(headers.size()); + std::vector newDirs; + for (size_t i = 0; i < headers.size(); ++i) { + std::string headerPath = tempFolder; + std::string headerIncludeName(headerIncludeNames[i]); + // replace / in path with current os's file separator + if ( amd::Os::fileSeparator() != '/') { + for (std::string::iterator it = headerIncludeName.begin(), + end = headerIncludeName.end(); + it != end; + ++it) { + if (*it == '/') *it = amd::Os::fileSeparator(); + } + } + size_t pos = headerIncludeName.rfind(amd::Os::fileSeparator()); + if (pos != std::string::npos) { + headerPath += amd::Os::fileSeparator(); + headerPath += headerIncludeName.substr(0, pos); + headerIncludeName = headerIncludeName.substr(pos+1); + } + if (!amd::Os::pathExists(headerPath)) { + bool ret = amd::Os::createPath(headerPath); + assert(ret && "failed creating path!"); + newDirs.push_back(headerPath); + } + std::string headerFullName + = headerPath + amd::Os::fileSeparator() + headerIncludeName; + headerFileNames[i] = headerFullName; + f.open(headerFullName.c_str(), std::fstream::out); + //Should we allow asserts + assert(!f.fail() && "failed creating header file!"); + f.write(headers[i]->c_str(), headers[i]->length()); + f.close(); + + Data* inc = C->NewFileReference(DT_CL_HEADER, headerFileNames[i]); + if (inc == NULL) { + buildLog_ += "Error while creating data from headers"; + return false; + } + inputs.push_back(inc); + } + + //Set the include path for the temp folder that contains the includes + if(!headers.empty()) { + driverOptions.append(" -I"); + driverOptions.append(tempFolder); + } + + if (options->isDumpFlagSet(amd::option::DUMP_CL)) { + std::ofstream f(options->getDumpFileName(".cl").c_str(), std::ios::trunc); + if(f.is_open()) { + f << "/* Compiler options:\n" \ + "-c -emit-llvm -target amdgcn-amd-amdhsa-opencl -x cl " + << driverOptions << " -include opencl-c.h " + << "\n*/\n\n" << sourceCode; + } else { + buildLog_ += + "Warning: opening the file to dump the OpenCL source failed.\n"; + } + } + + //FIXME_lmoriche: has the CL option been validated? + uint clcStd = (options->oVariables->CLStd[2] - '0') * 100 + + (options->oVariables->CLStd[4] - '0') * 10; + + std::pair hdr; + switch(clcStd) { + case 100: case 110: case 120: + hdr = std::make_pair(opencl1_2_c_amdgcn, opencl1_2_c_amdgcn_size); + break; + case 200: + hdr = std::make_pair(opencl2_0_c_amdgcn, opencl2_0_c_amdgcn_size); + break; + default: + buildLog_ += "Unsupported requested OpenCL C version (-cl-std).\n"; + return false; + } + + File* pch = C->NewTempFile(DT_CL_HEADER); + if (pch == NULL || !pch->WriteData((const char*) hdr.first, hdr.second)) { + buildLog_ += "Error while opening the opencl-c header "; + return false; + } + + driverOptions.append(" -include-pch " + pch->Name()); + driverOptions.append(" -Xclang -fno-validate-pch"); + + // Tokenize the options string into a vector of strings + std::istringstream istrstr(driverOptions); + std::istream_iterator sit(istrstr), end; + std::vector params(sit, end); + + // Compile source to IR + bool ret = C->CompileToLLVMBitcode(inputs, output, params); + buildLog_ += C->Output(); + if (!ret) { + buildLog_ += "Error: Failed to compile opencl source (from CL to LLVM IR).\n"; + return false; + } + + llvmBinary_.assign(output->Buf().data(), output->Size()); + elfSectionType_ = amd::OclElf::LLVMIR; + + if (options->isDumpFlagSet(amd::option::DUMP_BC_ORIGINAL)) { + std::ofstream f(options->getDumpFileName("_original.bc").c_str(), std::ios::trunc); + if(f.is_open()) { + f.write(llvmBinary_.data(), llvmBinary_.size()); + } else { + buildLog_ += + "Warning: opening the file to dump the compiled IR failed.\n"; + } + } + + if (clBinary()->saveSOURCE()) { + clBinary()->elfOut()->addSection( + amd::OclElf::SOURCE, sourceCode.data(), sourceCode.size()); + } + if (clBinary()->saveLLVMIR()) { + clBinary()->elfOut()->addSection( + amd::OclElf::LLVMIR, llvmBinary_.data(), llvmBinary_.size(), false); + // store the original compile options + clBinary()->storeCompileOptions(compileOptions_); + } + return true; +} +#endif // defined(WITH_LIGHTNING_COMPILER) + } // namespace pal diff --git a/projects/clr/rocclr/runtime/device/pal/paldefs.hpp b/projects/clr/rocclr/runtime/device/pal/paldefs.hpp index 39f9694f22..421bf3115a 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldefs.hpp +++ b/projects/clr/rocclr/runtime/device/pal/paldefs.hpp @@ -123,26 +123,26 @@ struct AMDDeviceInfo { }; static const AMDDeviceInfo DeviceInfo[] = { -/* Unknown */ { "", "unknown", 4, 16, 1, 256, 64 * Ki, 32, 702 }, -/* Tahiti */ { "", "tahiti", 4, 16, 1, 256, 64 * Ki, 32, 702 }, -/* Pitcairn */ { "", "pitcairn", 4, 16, 1, 256, 64 * Ki, 32, 702 }, -/* Capeverde */ { "", "bonaire", 4, 16, 1, 256, 64 * Ki, 32, 702 }, -/* Oland */ { "", "oland", 4, 16, 1, 256, 64 * Ki, 32, 702 }, -/* Hainan */ { "", "hainan", 4, 16, 1, 256, 64 * Ki, 32, 702 }, +/* Unknown */ { "", "unknown", 4, 16, 1, 256, 64 * Ki, 32, 0 }, +/* Tahiti */ { "", "tahiti", 4, 16, 1, 256, 64 * Ki, 32, 600 }, +/* Pitcairn */ { "", "pitcairn", 4, 16, 1, 256, 64 * Ki, 32, 600 }, +/* Capeverde */ { "", "bonaire", 4, 16, 1, 256, 64 * Ki, 32, 700 }, +/* Oland */ { "", "oland", 4, 16, 1, 256, 64 * Ki, 32, 600 }, +/* Hainan */ { "", "hainan", 4, 16, 1, 256, 64 * Ki, 32, 600 }, -/* Bonaire */ { "Bonaire", "bonaire", 4, 16, 1, 256, 64 * Ki, 32, 702 }, -/* Hawaii */ { "Hawaii", "hawaii", 4, 16, 1, 256, 64 * Ki, 32, 702 }, +/* Bonaire */ { "Bonaire", "bonaire", 4, 16, 1, 256, 64 * Ki, 32, 700 }, +/* Hawaii */ { "Hawaii", "hawaii", 4, 16, 1, 256, 64 * Ki, 32, 701 }, /* Kalindi */ { "Kalindi", "kalindi", 4, 16, 1, 256, 64 * Ki, 32, 702 }, /* Spectre */ { "Spectre", "spectre", 4, 16, 1, 256, 64 * Ki, 32, 701 }, -/* Carrizo */ { "Carrizo" , "carrizo", 4, 16, 1, 256, 64 * Ki, 32, 800 }, -/* Stoney */ { "Stoney", "stoney", 4, 16, 1, 256, 64 * Ki, 32, 800 }, +/* Carrizo */ { "Carrizo" , "carrizo", 4, 16, 1, 256, 64 * Ki, 32, 801 }, +/* Stoney */ { "Stoney", "stoney", 4, 16, 1, 256, 64 * Ki, 32, 810 }, /* Iceland */ { "Iceland", "iceland", 4, 16, 1, 256, 64 * Ki, 32, 800 }, /* Tonga */ { "Tonga", "tonga", 4, 16, 1, 256, 64 * Ki, 32, 800 }, -/* Fiji */ { "Fiji", "fiji", 4, 16, 1, 256, 64 * Ki, 32, 800 }, -/* Ellesmere */ { "Ellesmere", "ellesmere",4, 16, 1, 256, 64 * Ki, 32, 800 }, -/* Buffin */ { "Baffin", "baffin", 4, 16, 1, 256, 64 * Ki, 32, 800 }, +/* Fiji */ { "Fiji", "fiji", 4, 16, 1, 256, 64 * Ki, 32, 804 }, +/* Ellesmere */ { "Ellesmere", "ellesmere",4, 16, 1, 256, 64 * Ki, 32, 804 }, +/* Baffin */ { "Baffin", "baffin", 4, 16, 1, 256, 64 * Ki, 32, 804 }, }; // The GfxIpDeviceInfo table must match with GfxIpLevel enum diff --git a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp index 2203b01e66..16b845e29d 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp +++ b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp @@ -60,7 +60,7 @@ PalDeviceUnload() namespace pal { -aclCompiler* NullDevice::compiler_; +NullDevice::Compiler* NullDevice::compiler_; AppProfile Device::appProfile_; NullDevice::NullDevice() @@ -195,7 +195,11 @@ device::Program* NullDevice::createProgram(amd::option::Options* options) { device::Program* program; +#if defined(WITH_LIGHTNING_COMPILER) + program = new LightningProgram(*this); +#else // !defined(WITH_LIGHTNING_COMPILER) program = new HSAILProgram(*this); +#endif // defined(WITH_LIGHTNING_COMPILER) if (program == nullptr) { LogError("Memory allocation has failed!"); @@ -378,7 +382,13 @@ void NullDevice::fillDeviceInfo( } ::strcpy(info_.vendor_, "Advanced Micro Devices, Inc."); ::snprintf(info_.driverVersion_, sizeof(info_.driverVersion_) - 1, - AMD_BUILD_STRING "%s", " (PAL)"); + AMD_BUILD_STRING " (PAL%s)", +#if defined(WITH_LIGHTNING_COMPILER) + ",LC" +#else // ! defined(WITH_LIGHTNING_COMPILER) + ",HSAIL" +#endif // ! defined(WITH_LIGHTNING_COMPILER) + ); info_.profile_ = "FULL_PROFILE"; if (settings().oclVersion_ == OpenCL20) { @@ -878,6 +888,7 @@ Device::initializeHeapResources() // Delay compilation due to brig_loader memory allocation const char* scheduler = nullptr; const char* ocl20 = nullptr; +#if !defined(WITH_LIGHTNING_COMPILER) std::string sch = SchedulerSourceCode; if (settings().oclVersion_ == OpenCL20) { size_t loc = sch.find("%s"); @@ -885,6 +896,7 @@ Device::initializeHeapResources() scheduler = sch.c_str(); ocl20 = "-cl-std=CL2.0"; } +#endif // !defined(WITH_LIGHTNING_COMPILER) blitProgram_ = new BlitProgram(context_); // Create blit programs if (blitProgram_ == nullptr || @@ -959,7 +971,11 @@ device::Program* Device::createProgram(amd::option::Options* options) { device::Program* program; +#if defined(WITH_LIGHTNING_COMPILER) + program = new LightningProgram(*this); +#else // !defined(WITH_LIGHTNING_COMPILER) program = new HSAILProgram(*this); +#endif // defined(WITH_LIGHTNING_COMPILER) if (program == nullptr) { LogError("We failed memory allocation for program!"); } @@ -1023,6 +1039,7 @@ Device::init() bool useDeviceList = false; requestedDevices_t requestedDevices; +#if !defined(WITH_LIGHTNING_COMPILER) const char* library = getenv("HSA_COMPILER_LIBRARY"); aclCompilerOptions opts = { sizeof(aclCompilerOptions_0_8), @@ -1041,6 +1058,7 @@ Device::init() LogError("Error initializing the compiler"); return false; } +#endif // !defined(WITH_LIGHTNING_COMPILER) size_t size = Pal::GetPlatformSize(); platformObj = new char[size]; @@ -1098,9 +1116,11 @@ Device::tearDown() platform->Destroy(); delete platformObj; +#if !defined(WITH_LIGHTNING_COMPILER) if (compiler_ != nullptr) { aclCompilerFini(compiler_); } +#endif // !defined(WITH_LIGHTNING_COMPILER) } Memory* diff --git a/projects/clr/rocclr/runtime/device/pal/paldevice.hpp b/projects/clr/rocclr/runtime/device/pal/paldevice.hpp index f689ea0646..96ae0bbd85 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldevice.hpp +++ b/projects/clr/rocclr/runtime/device/pal/paldevice.hpp @@ -32,9 +32,9 @@ namespace pal { class NullDevice : public amd::Device { protected: - static aclCompiler* compiler_; + static Compiler* compiler_; public: - aclCompiler* compiler() const { return compiler_; } + Compiler* compiler() const { return compiler_; } public: static bool init(void); diff --git a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp index 0f3b84014b..8848d76c69 100644 --- a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp @@ -11,6 +11,10 @@ #include "acl.h" +#if defined(WITH_LIGHTNING_COMPILER) +#include "libamdhsacode/amdgpu_metadata.hpp" +#endif // defined(WITH_LIGHTNING_COMPILER) + #include #include #include @@ -623,6 +627,9 @@ HSAILKernel::~HSAILKernel() bool HSAILKernel::init(amd::hsa::loader::Symbol *sym, bool finalize) { +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); +#else // !defined(WITH_LIGHTNING_COMPILER) if (extraArgumentsNum_ > MaxExtraArgumentsNum) { LogError("Failed to initialize kernel: extra arguments number is bigger than is supported"); return false; @@ -789,7 +796,7 @@ HSAILKernel::init(amd::hsa::loader::Symbol *sym, bool finalize) delete[] VecTypeHint; } - +#endif // !defined(WITH_LIGHTNING_COMPILER) return true; } @@ -1011,7 +1018,8 @@ HSAILKernel::loadArguments( switch (arg->type_) { case HSAIL_ARGTYPE_POINTER: // If it is a global pointer - if (arg->addrQual_ == HSAIL_ADDRESS_GLOBAL) { + if (arg->addrQual_ == HSAIL_ADDRESS_GLOBAL + || arg->addrQual_ == HSAIL_ADDRESS_CONSTANT) { Memory* gpuMem = nullptr; amd::Memory* mem = nullptr; @@ -1183,6 +1191,14 @@ HSAILKernel::loadArguments( return nullptr; } +#if defined(WITH_LIGHTNING_COMPILER) + //!!!!!FIXME_lmoriche: fix the hidden args + size_t zero; + WriteAqlArg(&aqlArgBuf, &zero, sizeof(zero)); + WriteAqlArg(&aqlArgBuf, &zero, sizeof(zero)); + WriteAqlArg(&aqlArgBuf, &zero, sizeof(zero)); +#endif // defined(WITH_LIGHTNING_COMPILER) + // HSAIL kernarg segment size is rounded up to multiple of 16. aqlArgBuf = amd::alignUp(aqlArgBuf, 16); assert((aqlArgBuf == (gpu.cb(0)->sysMemCopy() + argsBufferSize())) && @@ -1238,4 +1254,513 @@ HSAILKernel::loadArguments( return hsaDisp; } +#if defined(WITH_LIGHTNING_COMPILER) + +const LightningProgram& +LightningKernel::prog() const +{ + return reinterpret_cast(prog_); +} + +void +LightningKernel::initPrintf(const std::vector& printfInfoStrings) +{ + for (auto str : printfInfoStrings) { + std::vector tokens; + + size_t end, pos = 0; + do { + end = str.find_first_of(':', pos); + tokens.push_back(str.substr(pos, end-pos)); + pos = end + 1; + } while (end != std::string::npos); + + if (tokens.size() < 2) { + LogPrintfWarning("Invalid PrintInfo string: \"%s\"", str.c_str()); + continue; + } + + pos = 0; + size_t printfInfoID = std::stoi(tokens[pos++]); + if (printf_.size() <= printfInfoID) { + printf_.resize(printfInfoID + 1); + } + PrintfInfo& info = printf_[printfInfoID]; + + size_t numSizes = std::stoi(tokens[pos++]); + end = pos + numSizes; + + // ensure that we have the correct number of tokens + if (tokens.size() < end + 1/*last token is the fmtString*/) { + LogPrintfWarning("Invalid PrintInfo string: \"%s\"", str.c_str()); + continue; + } + + // push the argument sizes + while (pos < end) { + info.arguments_.push_back(std::stoi(tokens[pos++])); + } + + // FIXME: We should not need this! [ + std::string& fmt = tokens[pos]; + bool need_nl = true; + + for (pos = 0; pos < fmt.size(); ++pos) { + char symbol = fmt[pos]; + need_nl = true; + if (symbol == '\\') { + switch (fmt[pos+1]) { + case 'a': + pos++; + symbol = '\a'; + break; + case 'b': + pos++; + symbol = '\b'; + break; + case 'f': + pos++; + symbol = '\f'; + break; + case 'n': + pos++; + symbol = '\n'; + need_nl = false; + break; + case 'r': + pos++; + symbol = '\r'; + break; + case 'v': + pos++; + symbol = '\v'; + break; + case '7': + if (fmt[pos+2] == '2') { + pos += 2; + symbol = '\72'; + } + break; + default: + break; + } + } + info.fmtString_.push_back(symbol); + } + if (need_nl) { + info.fmtString_ += "\n"; + } + // ] + } +} + +static inline HSAIL_ARG_TYPE +GetKernelArgType(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + switch (lcArg.Kind()) { + case AMDGPU::RuntimeMD::KernelArg::GlobalBuffer: + case AMDGPU::RuntimeMD::KernelArg::DynamicSharedPointer: + return HSAIL_ARGTYPE_POINTER; + case AMDGPU::RuntimeMD::KernelArg::ByValue: + return HSAIL_ARGTYPE_VALUE; + case AMDGPU::RuntimeMD::KernelArg::Image: + return HSAIL_ARGTYPE_IMAGE; + case AMDGPU::RuntimeMD::KernelArg::Sampler: + return HSAIL_ARGTYPE_SAMPLER; + case AMDGPU::RuntimeMD::KernelArg::HiddenGlobalOffsetX: + return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X; + case AMDGPU::RuntimeMD::KernelArg::HiddenGlobalOffsetY: + return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y; + case AMDGPU::RuntimeMD::KernelArg::HiddenGlobalOffsetZ: + return HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z; + case AMDGPU::RuntimeMD::KernelArg::HiddenPrintfBuffer: + return HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER; + case AMDGPU::RuntimeMD::KernelArg::HiddenDefaultQueue: + return HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE; + case AMDGPU::RuntimeMD::KernelArg::HiddenCompletionAction: + return HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION; + case AMDGPU::RuntimeMD::KernelArg::HiddenNone: + return HSAIL_ARGTYPE_HIDDEN_NONE; + default: + return HSAIL_ARGTYPE_ERROR; + } +} + +static inline size_t +GetKernelArgAlignment(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + return lcArg.Align(); +} + +static inline size_t +GetKernelArgPointeeAlignment(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + if (lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::DynamicSharedPointer) { + uint32_t align = lcArg.PointeeAlign(); + if (align == 0) { + LogWarning("Missing DynamicSharedPointer alignment"); + align = 128; /* worst case alignment */; + } + return align; + } + return 1; +} + +static inline HSAIL_ACCESS_TYPE +GetKernelArgAccessType(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + if (lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::GlobalBuffer + || lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::Image) { + switch (lcArg.AccQual()) { + case AMDGPU::RuntimeMD::KernelArg::ReadOnly: + return HSAIL_ACCESS_TYPE_RO; + case AMDGPU::RuntimeMD::KernelArg::WriteOnly: + return HSAIL_ACCESS_TYPE_WO; + case AMDGPU::RuntimeMD::KernelArg::ReadWrite: + default: + return HSAIL_ACCESS_TYPE_RW; + } + } + return HSAIL_ACCESS_TYPE_NONE; +} + +static inline HSAIL_ADDRESS_QUALIFIER +GetKernelAddrQual(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + if (lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::DynamicSharedPointer) { + return HSAIL_ADDRESS_LOCAL; + } + else if (lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::GlobalBuffer) { + if (lcArg.AddrQual() == AMDGPU::RuntimeMD::KernelArg::Global) { + return HSAIL_ADDRESS_GLOBAL; + } + else if (lcArg.AddrQual() == AMDGPU::RuntimeMD::KernelArg::Constant) { + return HSAIL_ADDRESS_CONSTANT; + } + LogError("Unsupported address type"); + return HSAIL_ADDRESS_ERROR; + } + else if (lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::Image + || lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::Sampler) { + return HSAIL_ADDRESS_GLOBAL; + } + return HSAIL_ADDRESS_ERROR; +} + +static inline HSAIL_DATA_TYPE +GetKernelDataType(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + aclArgDataType dataType; + + if (lcArg.Kind() != AMDGPU::RuntimeMD::KernelArg::ByValue) { + return HSAIL_DATATYPE_ERROR; + } + + switch (lcArg.ValueType()) { + case AMDGPU::RuntimeMD::KernelArg::I8: + return HSAIL_DATATYPE_S8; + case AMDGPU::RuntimeMD::KernelArg::I16: + return HSAIL_DATATYPE_S16; + case AMDGPU::RuntimeMD::KernelArg::I32: + return HSAIL_DATATYPE_S32; + case AMDGPU::RuntimeMD::KernelArg::I64: + return HSAIL_DATATYPE_S64; + case AMDGPU::RuntimeMD::KernelArg::U8: + return HSAIL_DATATYPE_U8; + case AMDGPU::RuntimeMD::KernelArg::U16: + return HSAIL_DATATYPE_U16; + case AMDGPU::RuntimeMD::KernelArg::U32: + return HSAIL_DATATYPE_U32; + case AMDGPU::RuntimeMD::KernelArg::U64: + return HSAIL_DATATYPE_U64; + case AMDGPU::RuntimeMD::KernelArg::F16: + return HSAIL_DATATYPE_F16; + case AMDGPU::RuntimeMD::KernelArg::F32: + return HSAIL_DATATYPE_F32; + case AMDGPU::RuntimeMD::KernelArg::F64: + return HSAIL_DATATYPE_F64; + case AMDGPU::RuntimeMD::KernelArg::Struct: + return HSAIL_DATATYPE_STRUCT; + default: + return HSAIL_DATATYPE_ERROR; + } +} + +static inline clk_value_type_t +GetOclType(const HSAILKernel::Argument* arg) +{ + static const clk_value_type_t ClkValueMapType[6][6] = { + { T_CHAR, T_CHAR2, T_CHAR3, T_CHAR4, T_CHAR8, T_CHAR16 }, + { T_SHORT, T_SHORT2, T_SHORT3, T_SHORT4, T_SHORT8, T_SHORT16 }, + { T_INT, T_INT2, T_INT3, T_INT4, T_INT8, T_INT16 }, + { T_LONG, T_LONG2, T_LONG3, T_LONG4, T_LONG8, T_LONG16 }, + { T_FLOAT, T_FLOAT2, T_FLOAT3, T_FLOAT4, T_FLOAT8, T_FLOAT16 }, + { T_DOUBLE, T_DOUBLE2, T_DOUBLE3, T_DOUBLE4, T_DOUBLE8, T_DOUBLE16 }, + }; + + uint sizeType; + uint numElements; + if (arg->type_ == HSAIL_ARGTYPE_POINTER || arg->type_ == HSAIL_ARGTYPE_IMAGE) { + return T_POINTER; + } + else if (arg->type_ == HSAIL_ARGTYPE_VALUE + || arg->type_ == HSAIL_ARGTYPE_REFERENCE) { + switch (arg->dataType_) { + case HSAIL_DATATYPE_S8: + case HSAIL_DATATYPE_U8: + sizeType = 0; + numElements = arg->size_; + break; + case HSAIL_DATATYPE_S16: + case HSAIL_DATATYPE_U16: + sizeType = 1; + numElements = arg->size_ / 2; + break; + case HSAIL_DATATYPE_S32: + case HSAIL_DATATYPE_U32: + sizeType = 2; + numElements = arg->size_ / 4; + break; + case HSAIL_DATATYPE_S64: + case HSAIL_DATATYPE_U64: + sizeType = 3; + numElements = arg->size_ / 8; + break; + case HSAIL_DATATYPE_F16: + sizeType = 4; + numElements = arg->size_ / 2; + break; + case HSAIL_DATATYPE_F32: + sizeType = 4; + numElements = arg->size_ / 4; + break; + case HSAIL_DATATYPE_F64: + sizeType = 5; + numElements = arg->size_ / 8; + break; + default: + return T_VOID; + } + + switch (numElements) { + case 1: return ClkValueMapType[sizeType][0]; + case 2: return ClkValueMapType[sizeType][1]; + case 3: return ClkValueMapType[sizeType][2]; + case 4: return ClkValueMapType[sizeType][3]; + case 8: return ClkValueMapType[sizeType][4]; + case 16: return ClkValueMapType[sizeType][5]; + default: return T_VOID; + } + } + else if (arg->type_ == HSAIL_ARGTYPE_SAMPLER) { + return T_SAMPLER; + } + else { + return T_VOID; + } +} + +static inline cl_kernel_arg_address_qualifier +GetOclAddrQual(const HSAILKernel::Argument* arg) +{ + if (arg->type_ == HSAIL_ARGTYPE_POINTER) { + switch (arg->addrQual_) { + case HSAIL_ADDRESS_GLOBAL: + return CL_KERNEL_ARG_ADDRESS_GLOBAL; + case HSAIL_ADDRESS_CONSTANT: + return CL_KERNEL_ARG_ADDRESS_CONSTANT; + case HSAIL_ADDRESS_LOCAL: + return CL_KERNEL_ARG_ADDRESS_LOCAL; + default: + return CL_KERNEL_ARG_ADDRESS_PRIVATE; + } + } + else if (arg->type_ == HSAIL_ARGTYPE_IMAGE) { + return CL_KERNEL_ARG_ADDRESS_GLOBAL; + } + //default for all other cases + return CL_KERNEL_ARG_ADDRESS_PRIVATE; +} + +static inline cl_kernel_arg_access_qualifier +GetOclAccessQual(const HSAILKernel::Argument* arg) +{ + if (arg->type_ == HSAIL_ARGTYPE_IMAGE) { + switch (arg->access_) { + case HSAIL_ACCESS_TYPE_RO: + return CL_KERNEL_ARG_ACCESS_READ_ONLY; + case HSAIL_ACCESS_TYPE_WO: + return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; + case HSAIL_ACCESS_TYPE_RW: + return CL_KERNEL_ARG_ACCESS_READ_WRITE; + default: + return CL_KERNEL_ARG_ACCESS_NONE; + } + } + return CL_KERNEL_ARG_ACCESS_NONE; +} + +static inline cl_kernel_arg_type_qualifier +GetOclTypeQual(const amd::hsa::code::KernelArg::Metadata& lcArg) +{ + cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE; + if (lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::GlobalBuffer + || lcArg.Kind() == AMDGPU::RuntimeMD::KernelArg::DynamicSharedPointer) { + if (lcArg.IsVolatile()) { + rv |= CL_KERNEL_ARG_TYPE_VOLATILE; + } + if (lcArg.IsRestrict()) { + rv |= CL_KERNEL_ARG_TYPE_RESTRICT; + } + if (lcArg.IsConst()) { + rv |= CL_KERNEL_ARG_TYPE_CONST; + } + } + return rv; +} + +void +LightningKernel::initArgList(const amd::hsa::code::Kernel::Metadata& kernelMD) +{ + device::Kernel::parameters_t params; + + size_t offset = 0; + + for (size_t i = 0; i < kernelMD.KernelArgCount(); ++i) { + const amd::hsa::code::KernelArg::Metadata& lcArg = + kernelMD.GetKernelArgMetadata(i); + + // Initialize HSAIL kernel argument + auto arg = new HSAILKernel::Argument; + arg->name_ = lcArg.Name(); + arg->typeName_ = lcArg.TypeName(); + arg->size_ = lcArg.Size(); + arg->type_ = GetKernelArgType(lcArg); + arg->addrQual_ = GetKernelAddrQual(lcArg); + arg->dataType_ = GetKernelDataType(lcArg); + arg->alignment_ = GetKernelArgAlignment(lcArg); + arg->access_ = GetKernelArgAccessType(lcArg); + arg->numElem_ = 1; + arg->pointeeAlignment_ = GetKernelArgPointeeAlignment(lcArg); + + bool isHidden = arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X + || arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y + || arg->type_ == HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z + || arg->type_ == HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER + || arg->type_ == HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE + || arg->type_ == HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION + || arg->type_ == HSAIL_ARGTYPE_HIDDEN_NONE; + + arg->index_ = isHidden ? uint(-1) : params.size(); + arguments_.push_back(arg); + + if (isHidden) { + continue; + } + + // Initialize Device kernel parameters + amd::KernelParameterDescriptor desc; + + desc.name_ = lcArg.Name().c_str(); + desc.type_ = GetOclType(arg); + desc.addressQualifier_ = GetOclAddrQual(arg); + desc.accessQualifier_ = GetOclAccessQual(arg); + desc.typeQualifier_ = GetOclTypeQual(lcArg); + desc.typeName_ = lcArg.TypeName().c_str(); + + // Make a check if it is local or global + if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) { + desc.size_ = 0; + } + else { + desc.size_ = arg->size_; + } + + // Make offset alignment to match CPU metadata, since + // in multidevice config abstraction layer has a single signature + // and CPU sends the parameters as they are allocated in memory + size_t size = desc.size_; + if (size == 0) { + // Local memory for CPU + size = sizeof(cl_mem); + } + offset = (size_t) amd::alignUp(offset, std::min(size, size_t(16))); + desc.offset_ = offset; + offset += amd::alignUp(size, sizeof(uint32_t)); + + params.push_back(desc); + } + + createSignature(params); +} + +bool +LightningKernel::init(amd::hsa::loader::Symbol* symbol) +{ + flags_.internalKernel_ = (compileOptions_.find("-cl-internal-kernel") != + std::string::npos) ? true: false; + + aqlCreateHWInfo(symbol); + + const amd::hsa::code::Program::Metadata* runtimeMD = prog().metadata(); + if (!runtimeMD) { + return false; + } + const amd::hsa::code::Kernel::Metadata& kernelMD = + runtimeMD->GetKernelMetadata(runtimeMD->KernelIndexByName(name())); + + // Set the argList + initArgList(kernelMD); + + if (kernelMD.HasRequiredWorkgroupSize()) { + const uint32_t* requiredWorkgroupSize = kernelMD.RequiredWorkgroupSize(); + workGroupInfo_.compileSize_[0] = requiredWorkgroupSize[0]; + workGroupInfo_.compileSize_[1] = requiredWorkgroupSize[1]; + workGroupInfo_.compileSize_[2] = requiredWorkgroupSize[2]; + } + + if (kernelMD.HasWorkgroupSizeHint()) { + const uint32_t* workgroupSizeHint = kernelMD.WorkgroupSizeHint(); + workGroupInfo_.compileSizeHint_[0] = workgroupSizeHint[0]; + workGroupInfo_.compileSizeHint_[1] = workgroupSizeHint[1]; + workGroupInfo_.compileSizeHint_[2] = workgroupSizeHint[2]; + } + + if (kernelMD.HasVecTypeHint()) { + workGroupInfo_.compileVecTypeHint_ = kernelMD.VecTypeHint().c_str(); + } + + // Copy wavefront size + workGroupInfo_.wavefrontSize_ = prog().isNull() ? 64 : + dev().properties().gfxipProperties.shaderCore.wavefrontSize; + // Find total workgroup size + if (workGroupInfo_.compileSize_[0] != 0) { + workGroupInfo_.size_ = + workGroupInfo_.compileSize_[0] * + workGroupInfo_.compileSize_[1] * + workGroupInfo_.compileSize_[2]; + } + else { + workGroupInfo_.size_ = dev().info().maxWorkGroupSize_; + } + + initPrintf(runtimeMD->PrintfInfo()); + + /*FIXME_lmoriche: + size_t sizeOfWavesPerSimdHint = sizeof(workGroupInfo_.wavesPerSimdHint_); + error = aclQueryInfo(dev().compiler(), prog().binaryElf(), + RT_WAVES_PER_SIMD_HINT, openClKernelName.c_str(), + &workGroupInfo_.wavesPerSimdHint_, &sizeOfWavesPerSimdHint); + if (error != ACL_SUCCESS) { + return false; + } + + waveLimiter_.enable(); + */ + + return true; +} +#endif // defined(WITH_LIGHTNING_COMPILER) + + } // namespace pal diff --git a/projects/clr/rocclr/runtime/device/pal/palkernel.hpp b/projects/clr/rocclr/runtime/device/pal/palkernel.hpp index c56b34ea48..c4fc7c0c74 100644 --- a/projects/clr/rocclr/runtime/device/pal/palkernel.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palkernel.hpp @@ -21,6 +21,11 @@ namespace hsa { namespace loader { class Symbol; } // loader +namespace code { +namespace Kernel { +class Metadata; +} // Kernel +} // code } // hsa } // amd @@ -31,6 +36,7 @@ class VirtualGPU; class Device; class NullDevice; class HSAILProgram; +class LightningProgram; /*! \addtogroup pal PAL Device Implementation * @{ @@ -40,6 +46,7 @@ enum HSAIL_ADDRESS_QUALIFIER{ HSAIL_ADDRESS_ERROR = 0, HSAIL_ADDRESS_GLOBAL, HSAIL_ADDRESS_LOCAL, + HSAIL_ADDRESS_CONSTANT, HSAIL_MAX_ADDRESS_QUALIFIERS } ; @@ -47,9 +54,17 @@ enum HSAIL_ARG_TYPE{ HSAIL_ARGTYPE_ERROR = 0, HSAIL_ARGTYPE_POINTER, HSAIL_ARGTYPE_VALUE, + HSAIL_ARGTYPE_REFERENCE, HSAIL_ARGTYPE_IMAGE, HSAIL_ARGTYPE_SAMPLER, HSAIL_ARGTYPE_QUEUE, + HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_X, + HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Y, + HSAIL_ARGTYPE_HIDDEN_GLOBAL_OFFSET_Z, + HSAIL_ARGTYPE_HIDDEN_PRINTF_BUFFER, + HSAIL_ARGTYPE_HIDDEN_DEFAULT_QUEUE, + HSAIL_ARGTYPE_HIDDEN_COMPLETION_ACTION, + HSAIL_ARGTYPE_HIDDEN_NONE, HSAIL_ARGMAX_ARG_TYPES }; @@ -88,11 +103,13 @@ class HSAILKernel : public device::Kernel public: struct Argument { + uint index_; //!< Argument's index in the OCL signature std::string name_; //!< Argument's name std::string typeName_; //!< Argument's type name uint size_; //!< Size in bytes uint offset_; //!< Argument's offset uint alignment_; //!< Argument's alignment + uint pointeeAlignment_; //!< Alignment of the data pointed to HSAIL_ARG_TYPE type_; //!< Type of the argument HSAIL_ADDRESS_QUALIFIER addrQual_; //!< Address qualifier of the argument HSAIL_DATA_TYPE dataType_; //!< The type of data @@ -166,13 +183,13 @@ public: ) const; //! Returns AQL packet in CPU memory - //! if the kerenl arguments were successfully loaded, otherwise NULL + //! if the kernel arguments were successfully loaded, otherwise NULL hsa_kernel_dispatch_packet_t* loadArguments( VirtualGPU& gpu, //!< Running GPU context const amd::Kernel& kernel, //!< AMD kernel object const amd::NDRangeContainer& sizes, //!< NDrange container const_address parameters, //!< Application arguments for the kernel - bool nativeMem, //!< Native memory objectes are passed + bool nativeMem, //!< Native memory objects are passed uint64_t vmDefQueue, //!< GPU VM default queue pointer uint64_t* vmParentWrap, //!< GPU VM parent aql wrap object std::vector& memList //!< Memory list for GSL/VidMM handles @@ -205,6 +222,7 @@ private: //! Disable operator= HSAILKernel& operator=(const HSAILKernel&); +protected: //! Creates AQL kernel HW info bool aqlCreateHWInfo(amd::hsa::loader::Symbol *sym); @@ -252,5 +270,29 @@ private: WaveLimiterManager waveLimiter_; //!< adaptively control number of waves }; +#if defined(WITH_LIGHTNING_COMPILER) +class LightningKernel : public HSAILKernel +{ +public: + LightningKernel(const std::string& name, + HSAILProgram* prog, + const std::string& compileOptions + ): HSAILKernel(name, prog, compileOptions, 0) + {} + + //! Returns Lightning program associated with this kernel + const LightningProgram& prog() const; + + //! Initializes the metadata required for this kernel, + bool init(amd::hsa::loader::Symbol* symbol); + + //! Initializes Hsail Argument metadata and info for LC + void initArgList(const amd::hsa::code::Kernel::Metadata& kernelMD); + + //! Initializes HSAIL Printf metadata and info for LC + void initPrintf(const std::vector& printfInfoStrings); +}; +#endif // defined(WITH_LIGHTNING_COMPILER) + /*@}*/} // namespace pal diff --git a/projects/clr/rocclr/runtime/device/pal/palprogram.cpp b/projects/clr/rocclr/runtime/device/pal/palprogram.cpp index 9223e7fd1b..f43b4d96a3 100644 --- a/projects/clr/rocclr/runtime/device/pal/palprogram.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palprogram.cpp @@ -18,6 +18,12 @@ #include "hsa.h" #include "hsa_ext_image.h" #include "amd_hsa_loader.hpp" +#if defined(WITH_LIGHTNING_COMPILER) +#include "libamdhsacode/amdgpu_metadata.hpp" +#include "driver/AmdCompiler.h" +#include "libraries.amdgcn.inc" +#include "gelf.h" +#endif // !defined(WITH_LIGHTNING_COMPILER) namespace pal { @@ -67,6 +73,7 @@ HSAILProgram::~HSAILProgram() for (auto& it : staticSamplers_) { delete it; } +#if !defined(WITH_LIGHTNING_COMPILER) if (rawBinary_ != nullptr) { aclFreeMem(binaryElf_, rawBinary_); } @@ -78,6 +85,7 @@ HSAILProgram::~HSAILProgram() LogWarning( "Error while destroying the acl binary \n" ); } } +#endif // !defined(WITH_LIGHTNING_COMPILER) releaseClBinary(); if (executable_ != nullptr) { loader_->DestroyExecutable(executable_); @@ -135,6 +143,10 @@ HSAILProgram::linkImpl( amd::option::Options *options, bool createLibrary) { +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); + return false; +#else // !defined(WITH_LIGHTNING_COMPILER) std::vector::const_iterator it = inputPrograms.begin(); std::vector::const_iterator itEnd @@ -222,11 +234,16 @@ HSAILProgram::linkImpl( } // Now call linkImpl with the new options return linkImpl(options); +#endif // !defined(WITH_LIGHTNING_COMPILER) } aclType HSAILProgram::getCompilationStagesFromBinary(std::vector& completeStages, bool& needOptionsCheck) { +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); + return ACL_TYPE_DEFAULT; +#else // !defined(WITH_LIGHTNING_COMPILER) acl_error errorCode; size_t secSize = 0; completeStages.clear(); @@ -343,10 +360,15 @@ HSAILProgram::getCompilationStagesFromBinary(std::vector& completeStage break; } return from; +#endif // !defined(WITH_LIGHTNING_COMPILER) } aclType HSAILProgram::getNextCompilationStageFromBinary(amd::option::Options* options) { +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); + return ACL_TYPE_DEFAULT; +#else // !defined(WITH_LIGHTNING_COMPILER) aclType continueCompileFrom = ACL_TYPE_DEFAULT; binary_t binary = this->binary(); // If the binary already exists @@ -423,6 +445,7 @@ HSAILProgram::getNextCompilationStageFromBinary(amd::option::Options* options) { } } return continueCompileFrom; +#endif // !defined(WITH_LIGHTNING_COMPILER) } inline static std::vector @@ -438,6 +461,10 @@ splitSpaceSeparatedString(char *str) bool HSAILProgram::linkImpl(amd::option::Options* options) { +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); + return false; +#else // !defined(WITH_LIGHTNING_COMPILER) acl_error errorCode; aclType continueCompileFrom = ACL_TYPE_LLVMIR_BINARY; bool finalize = true; @@ -580,6 +607,7 @@ HSAILProgram::linkImpl(amd::option::Options* options) saveBinaryAndSetType(TYPE_EXECUTABLE); buildLog_ += aclGetCompilerLog(dev().compiler()); return true; +#endif // !defined(WITH_LIGHTNING_COMPILER) } bool @@ -621,9 +649,11 @@ HSAILProgram::hsailOptions() if (dev().settings().reportFMA_) { hsailOptions.append(" -DFP_FAST_FMA=1"); } +#if !defined(WITH_LIGHTNING_COMPILER) if (!dev().settings().singleFpDenorm_) { hsailOptions.append(" -cl-denorms-are-zero"); } +#endif // !defined(WITH_LIGHTNING_COMPILER) // Check if the host is 64 bit or 32 bit LP64_ONLY(hsailOptions.append(" -m64")); @@ -677,7 +707,11 @@ HSAILProgram::fillResListWithKernels( } const aclTargetInfo & -HSAILProgram::info(const char * str) { +HSAILProgram::info(const char * str) +{ +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); +#else // !defined(WITH_LIGHTNING_COMPILER) acl_error err; std::string arch = "hsail"; if (dev().settings().use64BitPtr_) { @@ -688,12 +722,16 @@ HSAILProgram::info(const char * str) { if (err != ACL_SUCCESS) { LogWarning("aclGetTargetInfo failed"); } +#endif // !defined(WITH_LIGHTNING_COMPILER) return info_; } bool HSAILProgram::saveBinaryAndSetType(type_t type) { +#if defined(WITH_LIGHTNING_COMPILER) + assert(!"Should not reach here"); +#else // !defined(WITH_LIGHTNING_COMPILER) //Write binary to memory if (rawBinary_ != nullptr) { //Free memory containing rawBinary @@ -708,6 +746,7 @@ HSAILProgram::saveBinaryAndSetType(type_t type) setBinary(static_cast(rawBinary_), size); //Set the type of binary setType(type); +#endif // !defined(WITH_LIGHTNING_COMPILER) return true; } @@ -729,38 +768,21 @@ bool ORCAHSALoaderContext::IsaSupportedByAgent(hsa_agent_t agent, hsa_isa_t isa) default: LogError("Unsupported gfxip version"); return false; - case gfx700: - case gfx701: - case gfx702: + case gfx700: case gfx701: case gfx702: // gfx701 only differs from gfx700 by faster fp operations and can be loaded on either device. return isa.handle == gfx700 || isa.handle == gfx701; case gfx800: - switch (program_->dev().asicRevision()) { - case Pal::AsicRevision::Iceland: - case Pal::AsicRevision::Tonga: - return isa.handle == gfx800; - case Pal::AsicRevision::Carrizo: - return isa.handle == gfx801; - case Pal::AsicRevision::Fiji: - case Pal::AsicRevision::Ellesmere: - case Pal::AsicRevision::Baffin: - // gfx800 ISA has only sgrps limited and can be loaded. - // gfx801 ISA has XNACK limitations and can be loaded. - return isa.handle == gfx800 || isa.handle == gfx801 || isa.handle == gfx804; - case Pal::AsicRevision::Stoney: + return isa.handle == gfx800; + case gfx801: + return isa.handle == gfx801; + case gfx804: + // gfx800 ISA has only sgrps limited and can be loaded. + // gfx801 ISA has XNACK limitations and can be loaded. + return isa.handle == gfx800 || isa.handle == gfx801 || isa.handle == gfx804; + case gfx810: return isa.handle == gfx810; - default: - assert(0 && "Unknown asic!"); - return false; - } - case gfx900: - switch (program_->dev().ipLevel()) { - case Pal::GfxIpLevel::GfxIp9: - return isa.handle == gfx900 || isa.handle == gfx901; - default: - assert(0 && "Unknown asic!"); - return false; - } + case gfx900: case gfx901: + return isa.handle == gfx900 || isa.handle == gfx901; } } @@ -960,4 +982,410 @@ void ORCAHSALoaderContext::GpuMemFree(void *ptr, size_t size) } } +#if defined(WITH_LIGHTNING_COMPILER) + +static hsa_status_t +GetKernelNamesCallback( + hsa_executable_t hExec, + hsa_executable_symbol_t hSymbol, + void *data) +{ + auto symbol = Symbol::Object(hSymbol); + auto symbolNameList = reinterpret_cast*>(data); + + hsa_symbol_kind_t type; + if (!symbol->GetInfo(HSA_EXECUTABLE_SYMBOL_INFO_TYPE, &type)) { + return HSA_STATUS_ERROR; + } + + if (type == HSA_SYMBOL_KIND_KERNEL) { + uint32_t length; + if (!symbol->GetInfo(HSA_EXECUTABLE_SYMBOL_INFO_NAME_LENGTH, &length)) { + return HSA_STATUS_ERROR; + } + + char* name = (char*) alloca(length+1); + if (!symbol->GetInfo(HSA_EXECUTABLE_SYMBOL_INFO_NAME, name)) { + return HSA_STATUS_ERROR; + } + name[length] = '\0'; + + symbolNameList->push_back(std::string(name)); + } + return HSA_STATUS_SUCCESS; +} + +bool +LightningProgram::linkImpl(amd::option::Options *options) +{ + using namespace amd::opencl_driver; + + internal_ = (compileOptions_.find("-cl-internal-kernel") != + std::string::npos) ? true : false; + + aclType continueCompileFrom = llvmBinary_.empty() + ? getNextCompilationStageFromBinary(options) + : ACL_TYPE_LLVMIR_BINARY; + + if (continueCompileFrom == ACL_TYPE_ISA) { + binary_t isa = binary(); + if ((isa.first != NULL) && (isa.second > 0)) { + return setKernels(options, (void*) isa.first, isa.second ); + } + else { + buildLog_ += "Error: code object is empty \n" ; + return false; + } + return true; + } + if (continueCompileFrom != ACL_TYPE_LLVMIR_BINARY) { + buildLog_ += "Error while Codegen phase: the binary is incomplete \n" ; + return false; + } + + std::auto_ptr C(newCompilerInstance()); + // call LinkLLVMBitcode + std::vector inputs; + + // open the input IR source + Data* input = C->NewBufferReference( + DT_LLVM_BC, llvmBinary_.data(), llvmBinary_.size()); + + if (!input) { + buildLog_ += "Error: Failed to open the compiled program.\n"; + return false; + } + + inputs.push_back(input); //< must be the first input + + // open the bitcode libraries + Data* opencl_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) opencl_amdgcn, opencl_amdgcn_size); + Data* ocml_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) ocml_amdgcn, ocml_amdgcn_size); + Data* ockl_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) ockl_amdgcn, ockl_amdgcn_size); + Data* irif_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) irif_amdgcn, irif_amdgcn_size); + + if (!opencl_bc || !ocml_bc || !ockl_bc || !irif_bc) { + buildLog_ += "Error: Failed to open the bitcode library.\n"; + return false; + } + + inputs.push_back(opencl_bc); // depends on oclm & ockl + inputs.push_back(ockl_bc); // depends on irif + inputs.push_back(ocml_bc); // depends on irif + inputs.push_back(irif_bc); + + // open the control functions + std::pair isa_version; + switch (dev().hwInfo()->gfxipVersion_) { + case 700: isa_version = std::make_pair( + oclc_isa_version_700_amdgcn, oclc_isa_version_700_amdgcn_size); + break; + case 701: isa_version = std::make_pair( + oclc_isa_version_701_amdgcn, oclc_isa_version_701_amdgcn_size); + break; + case 800: isa_version = std::make_pair( + oclc_isa_version_800_amdgcn, oclc_isa_version_800_amdgcn_size); + break; + case 801: isa_version = std::make_pair( + oclc_isa_version_801_amdgcn, oclc_isa_version_801_amdgcn_size); + break; + case 802: isa_version = std::make_pair( + oclc_isa_version_802_amdgcn, oclc_isa_version_802_amdgcn_size); + break; + case 803: case 804: isa_version = std::make_pair( + oclc_isa_version_803_amdgcn, oclc_isa_version_803_amdgcn_size); + break; + case 810: isa_version = std::make_pair( + oclc_isa_version_810_amdgcn, oclc_isa_version_810_amdgcn_size); + break; + default: + buildLog_ += "Error: Linking for this device is not supported\n"; + return false; + } + + Data* isa_version_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) isa_version.first, isa_version.second); + + if (!isa_version_bc) { + buildLog_ += "Error: Failed to open the control functions.\n"; + return false; + } + + inputs.push_back(isa_version_bc); + + auto correctly_rounded_sqrt = (options->oVariables->FP32RoundDivideSqrt) + ? std::make_pair( + oclc_correctly_rounded_sqrt_on_amdgcn, + oclc_correctly_rounded_sqrt_on_amdgcn_size) + : std::make_pair( + oclc_correctly_rounded_sqrt_off_amdgcn, + oclc_correctly_rounded_sqrt_off_amdgcn_size); + + auto daz_opt = (dev().hwInfo()->gfxipVersion_ < 900 + || options->oVariables->DenormsAreZero) + ? std::make_pair( + oclc_daz_opt_on_amdgcn, + oclc_daz_opt_on_amdgcn_size) + : std::make_pair( + oclc_daz_opt_off_amdgcn, + oclc_daz_opt_off_amdgcn_size); + + auto finite_only = (options->oVariables->FiniteMathOnly + || options->oVariables->FastRelaxedMath) + ? std::make_pair( + oclc_finite_only_on_amdgcn, + oclc_finite_only_on_amdgcn_size) + : std::make_pair( + oclc_finite_only_off_amdgcn, + oclc_finite_only_off_amdgcn_size); + + auto unsafe_math = (options->oVariables->UnsafeMathOpt + || options->oVariables->FastRelaxedMath) + ? std::make_pair( + oclc_unsafe_math_on_amdgcn, + oclc_unsafe_math_on_amdgcn_size) + : std::make_pair( + oclc_unsafe_math_off_amdgcn, + oclc_unsafe_math_off_amdgcn_size); + + Data* correctly_rounded_sqrt_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) correctly_rounded_sqrt.first, correctly_rounded_sqrt.second); + Data* daz_opt_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) daz_opt.first, daz_opt.second); + Data* finite_only_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) finite_only.first, finite_only.second); + Data* unsafe_math_bc = C->NewBufferReference(DT_LLVM_BC, + (const char*) unsafe_math.first, unsafe_math.second); + + if (!correctly_rounded_sqrt_bc || !daz_opt_bc || !finite_only_bc || !unsafe_math_bc) { + buildLog_ += "Error: Failed to open the control functions.\n"; + return false; + } + + inputs.push_back(correctly_rounded_sqrt_bc); + inputs.push_back(daz_opt_bc); + inputs.push_back(finite_only_bc); + inputs.push_back(unsafe_math_bc); + + // open the linked output + std::vector linkOptions; + amd::opencl_driver::Buffer* linked_bc = C->NewBuffer(DT_LLVM_BC); + + if (!linked_bc) { + buildLog_ += "Error: Failed to open the linked program.\n"; + return false; + } + + bool ret = C->LinkLLVMBitcode(inputs, linked_bc, linkOptions); + buildLog_ += C->Output(); + if (!ret) { + buildLog_ += "Error: Linking bitcode failed: linking source & IR libraries.\n"; + return false; + } + + if (options->isDumpFlagSet(amd::option::DUMP_BC_LINKED)) { + std::ofstream f(options->getDumpFileName("_linked.bc").c_str(), std::ios::trunc); + if(f.is_open()) { + f.write(linked_bc->Buf().data(), linked_bc->Size()); + } else { + buildLog_ += + "Warning: opening the file to dump the linked IR failed.\n"; + } + } + + inputs.clear(); + inputs.push_back(linked_bc); + + amd::opencl_driver::Buffer* out_exec = C->NewBuffer(DT_EXECUTABLE); + if (!out_exec) { + buildLog_ += "Error: Failed to create the linked executable.\n"; + return false; + } + + std::string codegenOptions(options->llvmOptions); + + // Set the machine target + std::ostringstream mCPU; + mCPU << " -mcpu=gfx" << dev().hwInfo()->gfxipVersion_; + codegenOptions.append(mCPU.str()); + + // Set the -O# + std::ostringstream optLevel; + optLevel << "-O" << options->oVariables->OptLevel; + codegenOptions.append(" ").append(optLevel.str()); + + // Tokenize the options string into a vector of strings + std::istringstream strstr(codegenOptions); + std::istream_iterator sit(strstr), end; + std::vector params(sit, end); + + ret = C->CompileAndLinkExecutable(inputs, out_exec, params); + buildLog_ += C->Output(); + if (!ret) { + buildLog_ += "Error: Creating the executable failed: Compiling LLVM IRs to exe.\n"; + return false; + } + + if (options->isDumpFlagSet(amd::option::DUMP_O)) { + std::ofstream f(options->getDumpFileName(".so").c_str(), std::ios::trunc); + if(f.is_open()) { + f.write(out_exec->Buf().data(), out_exec->Size()); + } else { + buildLog_ += + "Warning: opening the file to dump the code object failed.\n"; + } + } + + return setKernels(options, out_exec->Buf().data(), out_exec->Size()); +} + +bool +LightningProgram::setKernels( + amd::option::Options *options, + void* binary, + size_t size + ) +{ + hsa_agent_t agent; + agent.handle = 1; + + executable_ = loader_->CreateExecutable(HSA_PROFILE_FULL, NULL); + if (executable_ == nullptr) { + buildLog_ += "Error: Executable for AMD HSA Code Object isn't created.\n"; + return false; + } + + hsa_code_object_t code_object; + code_object.handle = reinterpret_cast(binary); + + hsa_status_t status = executable_->LoadCodeObject(agent, code_object, nullptr); + if (status != HSA_STATUS_SUCCESS) { + buildLog_ += "Error: AMD HSA Code Object loading failed.\n"; + return false; + } + + /* FIXME_lmoriche: We need to call this! + status = executable_->Freeze(nullptr); + if (status != HSA_STATUS_SUCCESS) { + buildLog_ += "Error: Freezing the executable failed: "; + return false; + }*/ + + size_t progvarsTotalSize = 0; + + // Begin the Elf image from memory + Elf* e = elf_memory((char*) binary, size, NULL); + if (elf_kind(e) != ELF_K_ELF) { + buildLog_ += "Error while reading the ELF program binary\n"; + return false; + } + size_t shstrndx; + if (elf_getshdrstrndx(e, &shstrndx) != 0) { + buildLog_ += "Error while reading the ELF program binary\n"; + return false; + } + + // Iterate over the sections + for (Elf_Scn* scn = elf_nextscn(e, 0); scn; scn = elf_nextscn(e, scn)) { + GElf_Shdr shdr; + if (gelf_getshdr(scn, &shdr) != &shdr) { + continue; + } + // Skip non-program sections + if (shdr.sh_type != SHT_PROGBITS) { + continue; + } + // Accumulate the size of A & !X sections + if ((shdr.sh_flags & SHF_ALLOC) && !(shdr.sh_flags & SHF_EXECINSTR)) { + progvarsTotalSize += shdr.sh_size; + } + // Check if this is the metadata section + const char* name = elf_strptr(e, shstrndx , shdr.sh_name); + if (name && !strcmp(name, ".AMDGPU.runtime_metadata")) { + // Assume a single Elf_Data, the parser will fail if it isn't + Elf_Data* data = elf_getdata(scn, NULL); + if (!data) { + buildLog_ += "Error while reading ELF program binary " \ + "runtime metadata section\n"; + return false; + } + + metadata_ = new amd::hsa::code::Program::Metadata(); + if (!metadata_ || !metadata_->ReadFrom(data->d_buf, data->d_size)) { + buildLog_ += "Error while parsing ELF program binary " \ + "runtime metadata section\n"; + return false; + } + } + } + + elf_end(e); + + if (!metadata_) { + buildLog_ += "Error: runtime metadata section not present in " \ + "ELF program binary\n"; + return false; + } + + setGlobalVariableTotalSize(progvarsTotalSize); + + // Get the list of kernels + std::vector kernelNameList; + status = executable_->IterateSymbols(GetKernelNamesCallback, + (void *) &kernelNameList ); + if (status != HSA_STATUS_SUCCESS) { + buildLog_ += "Error: Failed to get kernel names\n"; + return false; + } + + for (auto &kernelName : kernelNameList) { + auto kernel = new LightningKernel( + kernelName, this, options->origOptionStr + hsailOptions()); + + kernels()[kernelName] = kernel; + + auto symbol = executable_->GetSymbol("", kernelName.c_str(), agent, 0); + if (!symbol) { + buildLog_ += "Error: Getting kernel symbol '" + kernelName + + "' from AMD HSA Code Object failed. " \ + "Kernel initialization failed.\n"; + return false; + } + if (!kernel->init(symbol)) { + buildLog_ += "Error: Kernel '" + kernelName + + "' initialization failed.\n"; + return false; + } + buildLog_ += kernel->buildLog(); + + kernel->setUniformWorkGroupSize(options->oVariables->UniformWorkGroupSize); + + // Find max scratch regs used in the program. It's used for scratch buffer preallocation + // with dynamic parallelism, since runtime doesn't know which child kernel will be called + maxScratchRegs_ = std::max(static_cast(kernel->workGroupInfo()->scratchRegs_), maxScratchRegs_); + } + + // Allocate kernel table for device enqueuing + if (!isNull() && false/*dynamicParallelism*/ && !allocKernelTable()) { + return false; + } + + // Save the binary and type + clBinary()->saveBIFBinary((char*)binary, size); + setType(TYPE_EXECUTABLE); + + return true; +} + +LightningProgram::~LightningProgram() +{ + delete metadata_; +} + +#endif // defined(WITH_LIGHTNING_COMPILER) + } // namespace pal diff --git a/projects/clr/rocclr/runtime/device/pal/palprogram.hpp b/projects/clr/rocclr/runtime/device/pal/palprogram.hpp index dbd900081b..971f4f3848 100644 --- a/projects/clr/rocclr/runtime/device/pal/palprogram.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palprogram.hpp @@ -7,6 +7,10 @@ #include "device/pal/palbinary.hpp" #include "amd_hsa_loader.hpp" +#if defined(WITH_LIGHTNING_COMPILER) +#include "libamdhsacode/amdgpu_metadata.hpp" +#endif // defined(WITH_LIGHTNING_COMPILER) + namespace amd { namespace option { class Options; @@ -148,7 +152,7 @@ public: HSAILProgram(Device& device); HSAILProgram(NullDevice& device); //! Default destructor - ~HSAILProgram(); + virtual ~HSAILProgram(); //! Returns the aclBinary associated with the progrm aclBinary* binaryElf() const { @@ -253,6 +257,7 @@ private: //! Disable operator= HSAILProgram& operator=(const HSAILProgram&); +protected: //! Returns all the options to be appended while passing to the //compiler library std::string hsailOptions(); @@ -282,4 +287,40 @@ private: ORCAHSALoaderContext loaderContext_; //!< Context for HSA Loader }; +#if defined(WITH_LIGHTNING_COMPILER) +//! \class Lightning Compiler Program +class LightningProgram : public HSAILProgram +{ +public: + LightningProgram(NullDevice& device) + : HSAILProgram(device), + metadata_(nullptr) + {} + + const amd::hsa::code::Program::Metadata* metadata() const { + return metadata_; + } +private: + virtual ~LightningProgram(); + +protected: + virtual bool compileImpl( + const std::string& sourceCode, //!< the program's source code + const std::vector& headers, + const char** headerIncludeNames, + amd::option::Options* options //!< compile options's object + ) override; + + virtual bool linkImpl(amd::option::Options* options) override; + + bool setKernels(amd::option::Options *options, void* binary, size_t size); + + //! Return a new transient compiler instance. + static std::auto_ptr newCompilerInstance(); + +private: + amd::hsa::code::Program::Metadata* metadata_; //!< Runtime metadata +}; +#endif // defined(WITH_LIGHTNING_COMPILER) + /*@}*/} // namespace pal