From 1d8e131fba91a2f6dcabb9e2bef00534298280a6 Mon Sep 17 00:00:00 2001 From: foreman Date: Wed, 11 Dec 2019 03:11:19 -0500 Subject: [PATCH] P4 to Git Change 2043802 by cpaquot@cpaquot-ocl-lc-lnx on 2019/12/11 03:06:02 SWDEV-213526 - [hip] OOM issue Delay any access to device layers till HIP API is called by app. This allows the app to fork the process first and then call HIP which is legal. Doing hip calls then fork isn't legal nor supported by ROCm. Affected files ... ... //depot/stg/opencl/drivers/opencl/api/hip/hip_context.cpp#25 edit ... //depot/stg/opencl/drivers/opencl/api/hip/hip_internal.hpp#49 edit ... //depot/stg/opencl/drivers/opencl/api/hip/hip_platform.cpp#55 edit [ROCm/clr commit: f5921646e4f4f4ae3a7fdee51647b8a4d1951d68] --- projects/clr/hipamd/api/hip/hip_context.cpp | 2 + projects/clr/hipamd/api/hip/hip_internal.hpp | 26 ++++++++- projects/clr/hipamd/api/hip/hip_platform.cpp | 60 ++++++++++++-------- 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/projects/clr/hipamd/api/hip/hip_context.cpp b/projects/clr/hipamd/api/hip/hip_context.cpp index 6ff36dc987..cc590678c5 100644 --- a/projects/clr/hipamd/api/hip/hip_context.cpp +++ b/projects/clr/hipamd/api/hip/hip_context.cpp @@ -65,6 +65,8 @@ void init() { if (host_context && CL_SUCCESS != host_context->create(nullptr)) { host_context->release(); } + + PlatformState::instance().init(); } amd::Context* getCurrentContext() { diff --git a/projects/clr/hipamd/api/hip/hip_internal.hpp b/projects/clr/hipamd/api/hip/hip_internal.hpp index cfe5cca7d1..a44b5a0b5a 100644 --- a/projects/clr/hipamd/api/hip/hip_internal.hpp +++ b/projects/clr/hipamd/api/hip/hip_internal.hpp @@ -58,7 +58,7 @@ typedef struct ihipIpcMemHandle_st { if (!CL_CHECK_THREAD(thread)) { \ HIP_RETURN(hipErrorOutOfMemory); \ } \ - HIP_INIT(); \ + HIP_INIT() \ HIP_CB_SPAWNER_OBJECT(cid); #define HIP_RETURN(ret) \ @@ -126,7 +126,29 @@ struct ihipExec_t { class PlatformState { amd::Monitor lock_; + std::unordered_map>> modules_; + bool initialized_; + + void digestFatBinary(const void* data, std::vector>& programs); public: + void init(); + std::vector>* addFatBinary(const void*data) + { + if (initialized_) { + digestFatBinary(data, modules_[data]); + } + return &modules_[data]; + } + void removeFatBinary(std::vector>* module) + { + for (auto& mod : modules_) { + if (&mod.second == module) { + modules_.erase(&mod); + return; + } + } + } + struct RegisteredVar { public: RegisteredVar(): size_(0), devicePtr_(nullptr), amd_mem_obj_(nullptr) {} @@ -159,7 +181,7 @@ private: static PlatformState* platform_; - PlatformState() : lock_("Guards global function map") {} + PlatformState() : lock_("Guards global function map"), initialized_(false) {} ~PlatformState() {} public: static PlatformState& instance() { diff --git a/projects/clr/hipamd/api/hip/hip_platform.cpp b/projects/clr/hipamd/api/hip/hip_platform.cpp index 0250ebfb09..6ce138e7a3 100644 --- a/projects/clr/hipamd/api/hip/hip_platform.cpp +++ b/projects/clr/hipamd/api/hip/hip_platform.cpp @@ -85,8 +85,6 @@ bool __hipExtractCodeObjectFromFatBinary(const void* data, const std::vector& devices, std::vector>& code_objs) { - HIP_INIT(); - std::string magic((const char*)data, sizeof(CLANG_OFFLOAD_BUNDLER_MAGIC_STR) - 1); if (magic.compare(CLANG_OFFLOAD_BUNDLER_MAGIC_STR)) { return false; @@ -127,42 +125,59 @@ bool __hipExtractCodeObjectFromFatBinary(const void* data, return false; } -extern "C" std::vector< std::pair >* __hipRegisterFatBinary(const void* data) +extern "C" std::vector>* __hipRegisterFatBinary(const void* data) { - HIP_INIT(); - - if(g_devices.empty()) { - return nullptr; - } const __CudaFatBinaryWrapper* fbwrapper = reinterpret_cast(data); if (fbwrapper->magic != __hipFatMAGIC2 || fbwrapper->version != 1) { return nullptr; } - std::vector devices; + return PlatformState::instance().addFatBinary(fbwrapper->binary); +} + +void PlatformState::digestFatBinary(const void* data, std::vector>& programs) +{ std::vector> code_objs; + std::vector devices; for (size_t dev = 0; dev < g_devices.size(); ++dev) { amd::Context* ctx = g_devices[dev]; devices.push_back(ctx->devices()[0]->info().name_); -} - - if (!__hipExtractCodeObjectFromFatBinary((char*)fbwrapper->binary, devices, code_objs)) { - return nullptr; } - auto programs = new std::vector< std::pair >(g_devices.size()); + if (!__hipExtractCodeObjectFromFatBinary((char*)data, devices, code_objs)) { + return; + } + + programs.resize(g_devices.size()); + for (size_t dev = 0; dev < g_devices.size(); ++dev) { amd::Context* ctx = g_devices[dev]; amd::Program* program = new amd::Program(*ctx); if (program == nullptr) { - return nullptr; + return; } if (CL_SUCCESS == program->addDeviceProgram(*ctx->devices()[0], code_objs[dev].first, code_objs[dev].second)) { - programs->at(dev) = std::make_pair(reinterpret_cast(as_cl(program)) , false); + programs.at(dev) = std::make_pair(reinterpret_cast(as_cl(program)) , false); } } +} - return programs; +void PlatformState::init() +{ + if(initialized_ || g_devices.empty()) { + return; + } + initialized_ = true; + + for (auto& it : modules_) { + digestFatBinary(it.first, it.second); + } + for (auto& it : functions_) { + it.second.functions.resize(g_devices.size()); + } + for (auto& it : vars_) { + it.second.rvars.resize(g_devices.size()); + } } std::vector< std::pair >* PlatformState::unregisterVar(hipModule_t hmod) { @@ -188,7 +203,6 @@ std::vector< std::pair >* PlatformState::unregisterVar(hipMod PlatformState::DeviceVar* PlatformState::findVar(std::string hostVar, int deviceId, hipModule_t hmod) { DeviceVar* dvar = nullptr; - if (hmod != nullptr) { // If module is provided, then get the var only from that module auto var_range = vars_.equal_range(hostVar); @@ -302,7 +316,6 @@ hipFunction_t PlatformState::getFunc(const void* hostFunction, int deviceId) { bool PlatformState::getFuncAttr(const void* hostFunction, hipFuncAttributes* func_attr) { - if (func_attr == nullptr) { return false; } @@ -413,8 +426,7 @@ extern "C" void __hipRegisterFunction( dim3* gridDim, int* wSize) { - HIP_INIT(); - PlatformState::DeviceFunction func{ std::string{deviceName}, modules, std::vector{ g_devices.size() }}; + PlatformState::DeviceFunction func{ std::string{deviceName}, modules, std::vector{g_devices.size()}}; PlatformState::instance().registerFunction(hostFunction, func); // for (size_t i = 0; i < g_devices.size(); ++i) { // PlatformState::instance().getFunc(hostFunction, i); @@ -436,10 +448,8 @@ extern "C" void __hipRegisterVar( int constant, // Whether this variable is constant int global) // Unknown, always 0 { - HIP_INIT(); - PlatformState::DeviceVar dvar{var, std::string{ hostVar }, static_cast(size), modules, - std::vector{ g_devices.size() }, false }; + std::vector{g_devices.size()}, false }; PlatformState::instance().registerVar(hostVar, dvar); } @@ -454,7 +464,7 @@ extern "C" void __hipUnregisterFatBinary(std::vector< std::pair