P4 to Git Change 1299127 by lmoriche@lmoriche_opencl_dev on 2016/08/04 14:18:16

SWDEV-94610 - Remove the g_complib instance from the ROCm device when building OpenCL/LC

Affected files ...

... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/build/Makefile.api#138 edit
... //depot/stg/opencl/drivers/opencl/runtime/build/Makefile.runtime#64 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#276 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/build/Makefile.oclrocm#6 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/roccompiler.cpp#4 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/roccompilerlib.cpp#4 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/roccompilerlib.hpp#4 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#5 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#4 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rockernel.cpp#4 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocprogram.cpp#5 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocprogram.hpp#4 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#5 edit

rocclr/runtime/device/device.hpp

@@ -16,7 +16,11 @@
 #include "elf/elf.hpp"
 #include "appprofile.hpp"
 
+#if defined(WITH_LIGHTNING_COMPILER)
+#include "driver/AmdCompiler.h"
+#endif // defined(WITH_LIGHTNING_COMPILER)
 #include "acl.h"
+
 #include "hwdebug.hpp"
 
 #include <cstdio>
@@ -1535,6 +1539,13 @@ private:
  */
 class Device : public RuntimeObject
 {
+protected:
+#if defined(WITH_LIGHTNING_COMPILER)
+    typedef amd::opencl_driver::Compiler Compiler;
+#else // !defined(WITH_LIGHTNING_COMPILER)
+    typedef aclCompiler Compiler;
+#endif // !defined(WITH_LIGHTNING_COMPILER)
+
 public:
     typedef std::list<CommandQueue*> CommandQueues;
 
@@ -1553,7 +1564,7 @@ public:
             );
     };
 
-    virtual aclCompiler* compiler() const = 0;
+    virtual Compiler* compiler() const = 0;
 
     Device(Device* parent = NULL);
     virtual ~Device();

rocclr/runtime/device/rocm/roccompiler.cpp

@@ -11,7 +11,9 @@
 #include "os/os.hpp"
 #include "rocdevice.hpp"
 #include "rocprogram.hpp"
+#if !defined(WITH_LIGHTNING_COMPILER)
 #include "roccompilerlib.hpp"
+#endif // !defined(WITH_LIGHTNING_COMPILER)
 #include "utils/options.hpp"
 #include <cstdio>
 
@@ -34,6 +36,10 @@ HSAILProgram::compileImpl(const std::string& sourceCode,
 		       const char** headerIncludeNames,
 		       amd::option::Options* options)
 {
+#if defined(WITH_LIGHTNING_COMPILER)
+    assert(!"FIXME_Wilkin");
+    return false;
+#else // !defined(WITH_LIGHTNING_COMPILER)
     acl_error errorCode;
     aclTargetInfo target;
 
@@ -154,7 +160,7 @@ HSAILProgram::compileImpl(const std::string& sourceCode,
     // Save the binary in the interface class
     saveBinaryAndSetType(TYPE_COMPILED);
     return true;
-
+#endif // !defined(WITH_LIGHTNING_COMPILER)
 }
 }
 #endif // WITHOUT_GPU_BACKEND

rocclr/runtime/device/rocm/roccompilerlib.hpp

@@ -10,6 +10,10 @@
 #include "acl.h"
 #include "utils/debug.hpp"
 
+#if defined(WITH_LIGHTNING_COMPILER)
+#error Should not include this file
+#endif // defined(WITH_LIGHTNING_COMPILER)
+
 using namespace amd;
 
 namespace roc {

rocclr/runtime/device/rocm/rocdevice.cpp

@@ -18,7 +18,11 @@
 #include "device/rocm/rocblit.hpp"
 #include "device/rocm/rocvirtual.hpp"
 #include "device/rocm/rocprogram.hpp"
+#if defined(WITH_LIGHTNING_COMPILER)
+#include "driver/AmdCompiler.h"
+#else // !defined(WITH_LIGHTNING_COMPILER)
 #include "device/rocm/roccompilerlib.hpp"
+#endif // !defined(WITH_LIGHTNING_COMPILER)
 #include "device/rocm/rocmemory.hpp"
 #include "device/rocm/rocglinterop.hpp"
 #include "kv_id.h"
@@ -41,7 +45,7 @@ extern const char* BlitSourceCode;
 }
 
 namespace roc {
-aclCompiler* NullDevice::compilerHandle_;
+amd::Device::Compiler* NullDevice::compilerHandle_;
 bool roc::Device::isHsaInitialized_ = false;
 hsa_agent_t roc::Device::cpu_agent_ = { 0 };
 std::vector<hsa_agent_t> roc::Device::gpu_agents_;
@@ -226,6 +230,16 @@ Device::~Device()
     }
 }
 bool NullDevice::initCompiler(bool isOffline) {
+#if defined(WITH_LIGHTNING_COMPILER)
+    if (!compilerHandle_) {
+        const std::string llvmbin(amd::Os::getEnvironment("LLVM_BIN"));
+        compilerHandle_ = amd::opencl_driver::CompilerFactory()
+            .CreateAMDGPUCompiler(llvmbin);
+        if (!compilerHandle_) {
+            return false;
+        }
+    }
+#else // !defined(WITH_LIGHTNING_COMPILER)
      // Initializes g_complibModule and g_complibApi if they were not initialized
     if( g_complibModule == NULL ){
         if (!LoadCompLib(isOffline)) {
@@ -245,10 +259,15 @@ bool NullDevice::initCompiler(bool isOffline) {
             return false;
         }
     }
+#endif // !defined(WITH_LIGHTNING_COMPILER)
     return true;
 }
 
 bool NullDevice::destroyCompiler() {
+#if defined(WITH_LIGHTNING_COMPILER)
+    delete compilerHandle_;
+    compilerHandle_ = NULL;
+#else // !defined(WITH_LIGHTNING_COMPILER)
     if (compilerHandle_ != NULL) {
         acl_error error = g_complibApi._aclCompilerFini(compilerHandle_);
         if (error != ACL_SUCCESS) {
@@ -259,6 +278,7 @@ bool NullDevice::destroyCompiler() {
     if( g_complibModule != NULL ){
         UnloadCompLib();
     }
+#endif // !defined(WITH_LIGHTNING_COMPILER)
     return true;
 }
 

rocclr/runtime/device/rocm/rocdevice.hpp

@@ -16,7 +16,6 @@
 #include "thread/thread.hpp"
 #include "thread/monitor.hpp"
 #include "utils/versions.hpp"
-#include "aclTypes.h"
 
 #include "device/rocm/rocsettings.hpp"
 #include "device/rocm/rocvirtual.hpp"
@@ -78,7 +77,7 @@ public:
     //! Destructor for the Null device
     virtual ~NullDevice();
 
-    aclCompiler *compiler() const { return compilerHandle_; }
+    Compiler *compiler() const { return compilerHandle_; }
 
     //! Construct an HSAIL program object from the ELF assuming it is valid
     virtual device::Program *createProgram(amd::option::Options* options = NULL);
@@ -199,7 +198,7 @@ protected:
     //! destroy compiler instance and handle
     static bool destroyCompiler();
     //! Handle to the the compiler
-    static aclCompiler* compilerHandle_;
+    static Compiler* compilerHandle_;
     //! Device Id for an HsaDevice
     AMDDeviceInfo deviceInfo_;
 private:

rocclr/runtime/device/rocm/rockernel.cpp

@@ -493,6 +493,9 @@ Kernel::Kernel(std::string name, HSAILProgram* prog,
       extraArgumentsNum_(extraArgsNum) {}
 
 bool Kernel::init(){
+#if defined(WITH_LIGHTNING_COMPILER)
+    assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
     acl_error errorCode;
     //compile kernel down to ISA
     hsa_agent_t hsaDevice = program_->hsaDevice();
@@ -601,6 +604,7 @@ bool Kernel::init(){
     // Set the Printf List
     initPrintf(reinterpret_cast<aclPrintfFmt*>(aclPrintfList.get()));
   }
+#endif // !defined(WITH_LIGHTNING_COMPILER)
   return true;
 }
 

rocclr/runtime/device/rocm/rocprogram.cpp

@@ -10,12 +10,12 @@
 #include "compiler/lib/loaders/elf/elf.hpp"
 #include "compiler/lib/utils/options.hpp"
 #include "rockernel.hpp"
-#include "roccompilerlib.hpp"
-#include "utils/bif_section_labels.hpp"
-
 #if defined(WITH_LIGHTNING_COMPILER)
 #include "driver/AmdCompiler.h"
-#endif // defined(WITH_LIGHTNING_COMPILER)
+#else // !defined(WITH_LIGHTNING_COMPILER)
+#include "roccompilerlib.hpp"
+#endif // !defined(WITH_LIGHTNING_COMPILER)
+#include "utils/bif_section_labels.hpp"
 
 #include <string>
 #include <vector>
@@ -39,7 +39,11 @@ namespace roc {
         acl_error error;
         // Free the elf binary
         if (binaryElf_ != NULL) {
+#if defined(WITH_LIGHTNING_COMPILER)
+            assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
             error = g_complibApi._aclBinaryFini(binaryElf_);
+#endif // !defined(WITH_LIGHTNING_COMPILER)
             if (error != ACL_SUCCESS) {
                 LogWarning( "Error while destroying the acl binary \n" );
             }
@@ -198,6 +202,12 @@ namespace roc {
         //! @todo Should we also check for ACL_TYPE_OPENCL & ACL_TYPE_LLVMIR_TEXT?
         // Checking llvmir in .llvmir section
         bool containsLlvmirText = true;
+#if defined(WITH_LIGHTNING_COMPILER)
+        assert(!"FIXME_Wilkin");
+        bool containsOpts = false;
+        bool containsHsailText = false;
+        bool containsBrig = false;
+#else // !defined(WITH_LIGHTNING_COMPILER)
         errorCode = g_complibApi._aclQueryInfo(device().compiler(), binaryElf_, RT_CONTAINS_LLVMIR, NULL, &containsLlvmirText, &boolSize);
         if (errorCode != ACL_SUCCESS) {
             containsLlvmirText = false;
@@ -224,6 +234,7 @@ namespace roc {
         if (errorCode != ACL_SUCCESS) {
             containsBrig = false;
         }
+#endif // !defined(WITH_LIGHTNING_COMPILER)
         if (containsBrig) {
             completeStages.push_back(from);
             from = ACL_TYPE_HSAIL_BINARY;
@@ -244,7 +255,11 @@ namespace roc {
         }
         // Checking ISA in .text section
         bool containsShaderIsa = true;
+#if defined(WITH_LIGHTNING_COMPILER)
+        assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
         errorCode = g_complibApi._aclQueryInfo(device().compiler(), binaryElf_, RT_CONTAINS_ISA, NULL, &containsShaderIsa, &boolSize);
+#endif // !defined(WITH_LIGHTNING_COMPILER)
         if (errorCode != ACL_SUCCESS) {
             containsShaderIsa = false;
         }
@@ -301,7 +316,11 @@ namespace roc {
         if ((binary.first != NULL) && (binary.second > 0)) {
             void *mem = const_cast<void *>(binary.first);
             acl_error errorCode;
+#if defined(WITH_LIGHTNING_COMPILER)
+            assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
             binaryElf_ = g_complibApi._aclReadFromMem(mem, binary.second, &errorCode);
+#endif // !defined(WITH_LIGHTNING_COMPILER)
             if (errorCode != ACL_SUCCESS) {
                 buildLog_ += "Error while BRIG Codegen phase: aclReadFromMem failure \n" ;
                 LogWarning("aclReadFromMem failed");
@@ -332,8 +351,13 @@ namespace roc {
               assert(symbol && "symbol not found");
               std::string symName = std::string(symbol->str[bif::PRE]) + std::string(symbol->str[bif::POST]);
               size_t symSize = 0;
+#if defined(WITH_LIGHTNING_COMPILER)
+              assert(!"FIXME_Wilkin");
+              const void *opts = NULL;
+#else // !defined(WITH_LIGHTNING_COMPILER)
               const void *opts = g_complibApi._aclExtractSymbol(device().compiler(),
                   binaryElf_, &symSize, aclCOMMENT, symName.c_str(), &errorCode);
+#endif // !defined(WITH_LIGHTNING_COMPILER)
               if (errorCode != ACL_SUCCESS) {
                   recompile = true;
                   break;
@@ -377,11 +401,15 @@ namespace roc {
         //Write binary to memory
         void *rawBinary = NULL;
         size_t size;
+#if defined(WITH_LIGHTNING_COMPILER)
+        assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
         if (g_complibApi._aclWriteToMem(binaryElf_, &rawBinary, &size)
             != ACL_SUCCESS) {
                 buildLog_ += "Failed to write binary to memory \n";
                 return false;
         }
+#endif // !defined(WITH_LIGHTNING_COMPILER)
         clBinary()->saveBIFBinary((char*)rawBinary, size);
         //Set the type of binary
         setType(type);
@@ -393,6 +421,10 @@ namespace roc {
     bool HSAILProgram::linkImpl(const std::vector<Program *> &inputPrograms,
         amd::option::Options *options,
         bool createLibrary) {
+#if defined(WITH_LIGHTNING_COMPILER)
+            assert(!"FIXME_Wilkin");
+            return false;
+#else // !defined(WITH_LIGHTNING_COMPILER)
             std::vector<device::Program *>::const_iterator it
                 = inputPrograms.begin();
             std::vector<device::Program *>::const_iterator itEnd
@@ -479,6 +511,7 @@ namespace roc {
 
             // Now call linkImpl with the new options
             return linkImpl(options);
+#endif // !defined(WITH_LIGHTNING_COMPILER)
     }
 
     bool HSAILProgram::initBrigModule() {
@@ -486,6 +519,10 @@ namespace roc {
         BrigModuleHeader* brig; 
         acl_error error_code;
         size_t size;
+#if defined(WITH_LIGHTNING_COMPILER)
+        assert(!"FIXME_Wilkin");
+        const void* symbol_data = NULL;
+#else // !defined(WITH_LIGHTNING_COMPILER)
         const void* symbol_data = g_complibApi._aclExtractSymbol(
             device().compiler(),
             binaryElf_,
@@ -493,6 +530,7 @@ namespace roc {
             aclBRIG,
             symbol_name,
             &error_code);
+#endif // !defined(WITH_LIGHTNING_COMPILER)
         if (error_code != ACL_SUCCESS) {
            std::string error = "Could not find Brig in BIF: ";
            error += symbol_name;
@@ -563,9 +601,13 @@ namespace roc {
         // 1. if the program is created with binary and contains only hsail text
         case ACL_TYPE_HSAIL_TEXT: {
             std::string curOptions = options->origOptionStr + hsailOptions();
+#if defined(WITH_LIGHTNING_COMPILER)
+            assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
             errorCode = g_complibApi._aclCompile(device().compiler(), binaryElf_,
                 curOptions.c_str(), continueCompileFrom, ACL_TYPE_CG, logFunction);
             buildLog_ += g_complibApi._aclGetCompilerLog(device().compiler());
+#endif // !defined(WITH_LIGHTNING_COMPILER)
             if (errorCode != ACL_SUCCESS) {
                 buildLog_ += "Error while BRIG Codegen phase: compilation error \n" ;
                 return false;
@@ -722,8 +764,12 @@ namespace roc {
                 aclMetadata md;
                 md.numHiddenKernelArgs = 0;
                 size_t sizeOfnumHiddenKernelArgs = sizeof(md.numHiddenKernelArgs);
+#if defined(WITH_LIGHTNING_COMPILER)
+                assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
                 errorCode = g_complibApi._aclQueryInfo(device().compiler(), binaryElf_, RT_NUM_KERNEL_HIDDEN_ARGS,
                     openclKernelName.c_str(), &md.numHiddenKernelArgs, &sizeOfnumHiddenKernelArgs);
+#endif // !defined(WITH_LIGHTNING_COMPILER)
                 if (errorCode != ACL_SUCCESS) {
                     buildLog_ += "Error while Finalization phase: Kernel extra arguments count querying from the ELF failed\n";
                     return false;
@@ -787,7 +833,11 @@ namespace roc {
             }
         }
         saveBinaryAndSetType(TYPE_EXECUTABLE);
+#if defined(WITH_LIGHTNING_COMPILER)
+        assert(!"FIXME_Wilkin");
+#else // !defined(WITH_LIGHTNING_COMPILER)
         buildLog_ += g_complibApi._aclGetCompilerLog(device().compiler());
+#endif // !defined(WITH_LIGHTNING_COMPILER)
         return true;
     }
 

rocclr/runtime/device/rocm/rocprogram.hpp

@@ -6,7 +6,9 @@
 #ifndef WITHOUT_HSA_BACKEND
 
 #include "rocbinary.hpp"
+#if !defined(WITH_LIGHTNING_COMPILER)
 #include "roccompilerlib.hpp"
+#endif // !defined(WITH_LIGHTNING_COMPILER)
 #include "acl.h"
 #include <string>
 #include <sstream>

rocclr/runtime/device/rocm/rocvirtual.cpp

@@ -91,7 +91,7 @@ typedef hsa_status_t
                                         void *correlation_handle);
 static void SetOclCorrelationHandle(void *tools_lib, const hsa_agent_t agent, void *handle) {
   hsa_ext_tools_set_correlation_handle func = 
-    (hsa_ext_tools_set_correlation_handle)Os::getSymbol(tools_lib, "hsa_ext_tools_set_correlation_handler");
+    (hsa_ext_tools_set_correlation_handle)amd::Os::getSymbol(tools_lib, "hsa_ext_tools_set_correlation_handler");
   if (func) {
     func(agent, handle);
   }
@@ -488,14 +488,14 @@ VirtualGPU::create(bool profilingEna)
 {
     // Set the event handle to the tools lib if the env var
     // Load the library using its advertised "soname"
-    std::string lib_name = Os::getEnvironment("HSA_TOOLS_LIB");
+    std::string lib_name = amd::Os::getEnvironment("HSA_TOOLS_LIB");
     if (lib_name != "") {
 #if defined(_WIN32) || defined(__CYGWIN__)
         const char *tools_lib_name = "hsa-runtime-tools" LP64_SWITCH("", "64") ".dll";
 #else
         const char *tools_lib_name = "libhsa-runtime-tools" LP64_SWITCH("", "64") ".so.1";
 #endif
-        tools_lib_ = Os::loadLibrary(tools_lib_name);
+        tools_lib_ = amd::Os::loadLibrary(tools_lib_name);
     }
 
     uint32_t queue_max_packets = 0;
@@ -581,7 +581,7 @@ VirtualGPU::terminate()
     }
 
     if (tools_lib_) {
-        Os::unloadLibrary(tools_lib_);
+        amd::Os::unloadLibrary(tools_lib_);
         tools_lib_ = NULL;
     }
 
@@ -633,7 +633,7 @@ VirtualGPU::allocKernArg(size_t size, size_t alignment)
 {
     char* result = nullptr;
     do {
-        result = alignUp(kernarg_pool_base_ + kernarg_pool_cur_offset_, alignment);
+        result = amd::alignUp(kernarg_pool_base_ + kernarg_pool_cur_offset_, alignment);
         const size_t pool_new_usage = (result + size) - kernarg_pool_base_;
         if (pool_new_usage <= kernarg_pool_size_) {
             kernarg_pool_cur_offset_ = pool_new_usage;
@@ -981,7 +981,7 @@ void VirtualGPU::submitSvmFillMemory(amd::SvmFillMemoryCommand& cmd)
     // in-order semantics: previous commands need to be done before we start
     releaseGpuMemoryFence();
     profilingBegin(cmd);
-    SvmBuffer::memFill(cmd.dst(), cmd.pattern(), cmd.patternSize(), cmd.times());
+    amd::SvmBuffer::memFill(cmd.dst(), cmd.pattern(), cmd.patternSize(), cmd.times());
     profilingEnd(cmd);
 }