P4 to Git Change 1343852 by lmoriche@lmoriche_opencl_dev on 2016/11/21 00:55:02

SWDEV-106175 - [ROCm CQE][OCL][G] clGetKernelInfoAMD() returns 0 vgprs ("OCLOfflineCompilation" failed with compute-roc-master#1542) - Disable ROCm offline devices - Add roc::Device::loaderQueryHostAdress to retrieve the host address of a give kernel code handle. - Extract usedSGPRs and usedVGPRs for the amd kernel code header. Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_icd.cpp#26 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#27 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#9 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rockernel.cpp#17 edit [ROCm/clr commit: 9fbdf64025]
2016-11-21 00:00:56 -06:00
@@ -282,6 +282,7 @@ bool NullDevice::init() {
    if (!initCompiler(offlineDevice_)){
        return false;
    }
+#if !defined(WITH_LIGHTNING_COMPILER)
    //If there is an HSA enabled device online then skip any offline device
    std::vector<Device*> devices;
    devices = getDevices(CL_DEVICE_TYPE_GPU | CL_HSA_ENABLED_AMD, false);
@@ -308,6 +309,7 @@ bool NullDevice::init() {
        }
        nullDevice->registerDevice();
    }
+#endif // !defined(WITH_LIGHTNING_COMPILER)
    return true;
 }
 NullDevice::~NullDevice() {
@@ -343,8 +345,19 @@ hsa_status_t Device::iterateAgentCallback(hsa_agent_t agent, void *data) {
    return HSA_STATUS_SUCCESS;
 }

+hsa_ven_amd_loader_1_00_pfn_t
+Device::amd_loader_ext_table = {nullptr};

-bool Device::init() {
+hsa_status_t
+Device::loaderQueryHostAddress(const void* device, const void** host)
+{
+    return amd_loader_ext_table.hsa_ven_amd_loader_query_host_address
+        ? amd_loader_ext_table.hsa_ven_amd_loader_query_host_address(device, host)
+        : HSA_STATUS_ERROR;
+}
+
+bool Device::init()
+{
 #if defined(__linux__)
    if (amd::Os::getEnvironment("HSA_ENABLE_SDMA").empty()) {
        ::setenv("HSA_ENABLE_SDMA", "0", false);
@@ -363,6 +376,9 @@ bool Device::init() {
        return false;
    }

+    hsa_system_get_major_extension_table(HSA_EXTENSION_AMD_LOADER, 1,
+        sizeof(amd_loader_ext_table), &amd_loader_ext_table);
+
    if (HSA_STATUS_SUCCESS !=
        hsa_iterate_agents(iterateAgentCallback, NULL)) {
        return false;
@@ -387,7 +403,8 @@ bool Device::init() {

    size_t ordinal = 0;
    for (auto agent : gpu_agents_ ) {
-        Device *roc_device = new Device(agent);
+        std::unique_ptr<Device> roc_device(new Device(agent));
+
        if (!roc_device) {
            LogError("Error creating new instance of Device on then heap.");
            return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
@@ -445,33 +462,36 @@ bool Device::init() {
            pos = end + 1;
        } while (end != std::string::npos);

-        assert(tokens.size()==5 && tokens[0]=="AMD" && tokens[1]=="AMDGPU");
+        if (tokens.size() != 5 || tokens[0] != "AMD" || tokens[1] != "AMDGPU") {
+            LogError("Not an AMD:AMDGPU ISA name");
+            continue;
+        }
+
        uint major = atoi(tokens[2].c_str());
        uint minor = atoi(tokens[3].c_str());
        uint stepping = atoi(tokens[4].c_str());
-        assert(minor < 10 && stepping < 10 && "Invalid ISA string");
+        if (minor >= 10 && stepping >= 10) {
+            LogError("Invalid ISA string");
+            continue;
+        }

-        roc_device->deviceInfo_.gfxipVersion_ = major*100 + minor*10 + stepping;
+        roc_device->deviceInfo_.gfxipVersion_ =
+            major * 100 + minor * 10 + stepping;

        if (!roc_device->mapHSADeviceToOpenCLDevice(agent)) {
            LogError("Failed mapping of HsaDevice to Device.");
-            delete roc_device;
            continue;
        }

        if (!roc_device->create()) {
            LogError("Error creating new instance of Device.");
-            delete roc_device;
            continue;
        }

        if (selectedDevices[ordinal++] && (flagIsDefault(GPU_DEVICE_NAME)
            || GPU_DEVICE_NAME == 0 || GPU_DEVICE_NAME[0] == '\0'
-            || !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) {
-            roc_device->registerDevice();  // no return code for this function
-        }
-        else {
-            delete roc_device;
+                || !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) {
+            roc_device.release()->registerDevice();
        }
    }

@@ -27,6 +27,7 @@
 #include "hsa_ext_image.h"
 #include "hsa_ext_finalize.h"
 #include "hsa_ext_amd.h"
+#include "hsa_ven_amd_loader.h"

 #include <iostream>
 #include <vector>
@@ -217,7 +218,9 @@ public:
    static hsa_status_t iterateGpuMemoryPoolCallback(
        hsa_amd_memory_pool_t region, void* data);
    static hsa_status_t iterateCpuMemoryPoolCallback(
-      hsa_amd_memory_pool_t region, void* data);
+        hsa_amd_memory_pool_t region, void* data);
+    static hsa_status_t loaderQueryHostAddress(
+        const void* device, const void** host);

    static bool loadHsaModules();

@@ -344,6 +347,8 @@ public:
    bool addMapTarget(amd::Memory* memory) const;

 private:
+    static hsa_ven_amd_loader_1_00_pfn_t amd_loader_ext_table;
+
    amd::Monitor*   mapCacheOps_;   //!< Lock to serialise cache for the map resources
    std::vector<amd::Memory*>*  mapCache_;  //!< Map cache info structure

@@ -754,9 +754,20 @@ bool Kernel::init_LC()

    workGroupInfo_.preferredSizeMultiple_ = wavefront_size;

-    workGroupInfo_.usedSGPRs_ = 0;
+    const void* kernelHostPtr = nullptr;
+    if (Device::loaderQueryHostAddress(
+            reinterpret_cast<const void*>(kernelCodeHandle_), &kernelHostPtr
+            ) == HSA_STATUS_SUCCESS) {
+        auto akc = reinterpret_cast<const amd_kernel_code_t*>(kernelHostPtr);
+        workGroupInfo_.usedSGPRs_ = akc->wavefront_sgpr_count;
+        workGroupInfo_.usedVGPRs_ = akc->workitem_vgpr_count;
+    }
+    else {
+        workGroupInfo_.usedSGPRs_ = 0;
+        workGroupInfo_.usedVGPRs_ = 0;
+    }
+
    workGroupInfo_.usedStackSize_ = 0;
-    workGroupInfo_.usedVGPRs_ = 0;

    workGroupInfo_.wavefrontPerSIMD_ =
        program_->dev().info().maxWorkItemSizes_[0] / wavefront_size;