From d06bdb9114cc331036c9068c5404e4e8c2a27f75 Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 21 Nov 2016 00:00:56 -0600 Subject: [PATCH] P4 to Git Change 1343852 by lmoriche@lmoriche_opencl_dev on 2016/11/21 00:55:02 SWDEV-106175 - [ROCm CQE][OCL][G] clGetKernelInfoAMD() returns 0 vgprs ("OCLOfflineCompilation" failed with compute-roc-master#1542) - Disable ROCm offline devices - Add roc::Device::loaderQueryHostAdress to retrieve the host address of a give kernel code handle. - Extract usedSGPRs and usedVGPRs for the amd kernel code header. Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_icd.cpp#26 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#27 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#9 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rockernel.cpp#17 edit [ROCm/clr commit: 9fbdf6402552e03b1c0c75e0b79a82411b0290be] --- .../rocclr/runtime/device/rocm/rocdevice.cpp | 44 ++++++++++++++----- .../rocclr/runtime/device/rocm/rocdevice.hpp | 7 ++- .../rocclr/runtime/device/rocm/rockernel.cpp | 15 ++++++- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp b/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp index f5a8f5224f..70d65376cd 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp @@ -282,6 +282,7 @@ bool NullDevice::init() { if (!initCompiler(offlineDevice_)){ return false; } +#if !defined(WITH_LIGHTNING_COMPILER) //If there is an HSA enabled device online then skip any offline device std::vector devices; devices = getDevices(CL_DEVICE_TYPE_GPU | CL_HSA_ENABLED_AMD, false); @@ -308,6 +309,7 @@ bool NullDevice::init() { } nullDevice->registerDevice(); } +#endif // !defined(WITH_LIGHTNING_COMPILER) return true; } NullDevice::~NullDevice() { @@ -343,8 +345,19 @@ hsa_status_t Device::iterateAgentCallback(hsa_agent_t agent, void *data) { return HSA_STATUS_SUCCESS; } +hsa_ven_amd_loader_1_00_pfn_t +Device::amd_loader_ext_table = {nullptr}; -bool Device::init() { +hsa_status_t +Device::loaderQueryHostAddress(const void* device, const void** host) +{ + return amd_loader_ext_table.hsa_ven_amd_loader_query_host_address + ? amd_loader_ext_table.hsa_ven_amd_loader_query_host_address(device, host) + : HSA_STATUS_ERROR; +} + +bool Device::init() +{ #if defined(__linux__) if (amd::Os::getEnvironment("HSA_ENABLE_SDMA").empty()) { ::setenv("HSA_ENABLE_SDMA", "0", false); @@ -363,6 +376,9 @@ bool Device::init() { return false; } + hsa_system_get_major_extension_table(HSA_EXTENSION_AMD_LOADER, 1, + sizeof(amd_loader_ext_table), &amd_loader_ext_table); + if (HSA_STATUS_SUCCESS != hsa_iterate_agents(iterateAgentCallback, NULL)) { return false; @@ -387,7 +403,8 @@ bool Device::init() { size_t ordinal = 0; for (auto agent : gpu_agents_ ) { - Device *roc_device = new Device(agent); + std::unique_ptr roc_device(new Device(agent)); + if (!roc_device) { LogError("Error creating new instance of Device on then heap."); return HSA_STATUS_ERROR_OUT_OF_RESOURCES; @@ -445,33 +462,36 @@ bool Device::init() { pos = end + 1; } while (end != std::string::npos); - assert(tokens.size()==5 && tokens[0]=="AMD" && tokens[1]=="AMDGPU"); + if (tokens.size() != 5 || tokens[0] != "AMD" || tokens[1] != "AMDGPU") { + LogError("Not an AMD:AMDGPU ISA name"); + continue; + } + uint major = atoi(tokens[2].c_str()); uint minor = atoi(tokens[3].c_str()); uint stepping = atoi(tokens[4].c_str()); - assert(minor < 10 && stepping < 10 && "Invalid ISA string"); + if (minor >= 10 && stepping >= 10) { + LogError("Invalid ISA string"); + continue; + } - roc_device->deviceInfo_.gfxipVersion_ = major*100 + minor*10 + stepping; + roc_device->deviceInfo_.gfxipVersion_ = + major * 100 + minor * 10 + stepping; if (!roc_device->mapHSADeviceToOpenCLDevice(agent)) { LogError("Failed mapping of HsaDevice to Device."); - delete roc_device; continue; } if (!roc_device->create()) { LogError("Error creating new instance of Device."); - delete roc_device; continue; } if (selectedDevices[ordinal++] && (flagIsDefault(GPU_DEVICE_NAME) || GPU_DEVICE_NAME == 0 || GPU_DEVICE_NAME[0] == '\0' - || !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) { - roc_device->registerDevice(); // no return code for this function - } - else { - delete roc_device; + || !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) { + roc_device.release()->registerDevice(); } } diff --git a/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp b/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp index eb5721d478..32213f1fb5 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp @@ -27,6 +27,7 @@ #include "hsa_ext_image.h" #include "hsa_ext_finalize.h" #include "hsa_ext_amd.h" +#include "hsa_ven_amd_loader.h" #include #include @@ -217,7 +218,9 @@ public: static hsa_status_t iterateGpuMemoryPoolCallback( hsa_amd_memory_pool_t region, void* data); static hsa_status_t iterateCpuMemoryPoolCallback( - hsa_amd_memory_pool_t region, void* data); + hsa_amd_memory_pool_t region, void* data); + static hsa_status_t loaderQueryHostAddress( + const void* device, const void** host); static bool loadHsaModules(); @@ -344,6 +347,8 @@ public: bool addMapTarget(amd::Memory* memory) const; private: + static hsa_ven_amd_loader_1_00_pfn_t amd_loader_ext_table; + amd::Monitor* mapCacheOps_; //!< Lock to serialise cache for the map resources std::vector* mapCache_; //!< Map cache info structure diff --git a/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp b/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp index b598287528..d5fe95568c 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rockernel.cpp @@ -754,9 +754,20 @@ bool Kernel::init_LC() workGroupInfo_.preferredSizeMultiple_ = wavefront_size; - workGroupInfo_.usedSGPRs_ = 0; + const void* kernelHostPtr = nullptr; + if (Device::loaderQueryHostAddress( + reinterpret_cast(kernelCodeHandle_), &kernelHostPtr + ) == HSA_STATUS_SUCCESS) { + auto akc = reinterpret_cast(kernelHostPtr); + workGroupInfo_.usedSGPRs_ = akc->wavefront_sgpr_count; + workGroupInfo_.usedVGPRs_ = akc->workitem_vgpr_count; + } + else { + workGroupInfo_.usedSGPRs_ = 0; + workGroupInfo_.usedVGPRs_ = 0; + } + workGroupInfo_.usedStackSize_ = 0; - workGroupInfo_.usedVGPRs_ = 0; workGroupInfo_.wavefrontPerSIMD_ = program_->dev().info().maxWorkItemSizes_[0] / wavefront_size;