From 865fe01b3710541b91943198d9705a6c90501f76 Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Fri, 18 May 2018 10:14:46 -0500 Subject: [PATCH 1/3] HIP program state re-initialization logic This commit is to support kernels dynamically loaded thru means such as dlopen() after HIP runtime initializes. [ROCm/clr commit: 04640992dcc3a5317047e2f627e92a3fb435590d] --- .../include/hip/hcc_detail/program_state.hpp | 9 +- .../clr/hipamd/src/functional_grid_launch.inl | 16 ++- projects/clr/hipamd/src/program_state.cpp | 131 +++++++++++++++--- 3 files changed, 124 insertions(+), 32 deletions(-) diff --git a/projects/clr/hipamd/include/hip/hcc_detail/program_state.hpp b/projects/clr/hipamd/include/hip/hcc_detail/program_state.hpp index ac689fdb89..bdb87b3509 100644 --- a/projects/clr/hipamd/include/hip/hcc_detail/program_state.hpp +++ b/projects/clr/hipamd/include/hip/hcc_detail/program_state.hpp @@ -93,11 +93,12 @@ public: } }; -const std::unordered_map>& executables(); +const std::unordered_map>& executables( + bool rebuild = false); const std::unordered_map>>& -functions(); -const std::unordered_map& function_names(); -std::unordered_map& globals(); +functions(bool rebuild = false); +const std::unordered_map& function_names(bool rebuild = false); +std::unordered_map& globals(bool rebuild = false); hsa_executable_t load_executable(const std::string& file, hsa_executable_t executable, hsa_agent_t agent); diff --git a/projects/clr/hipamd/src/functional_grid_launch.inl b/projects/clr/hipamd/src/functional_grid_launch.inl index 9ecad51476..336bb5d121 100644 --- a/projects/clr/hipamd/src/functional_grid_launch.inl +++ b/projects/clr/hipamd/src/functional_grid_launch.inl @@ -92,13 +92,19 @@ namespace hip_impl hipStream_t stream, void** kernarg) { - const auto it0 = functions().find(function_address); + auto it0 = functions().find(function_address); if (it0 == functions().cend()) { - throw runtime_error{ - "No device code available for function: " + - name(function_address) - }; + // Re-init device code maps once again to help locate kernels + // loaded after HIP runtime initialization via means such as + // dlopen(). + it0 = functions(true).find(function_address); + if (it0 == functions().cend()) { + throw runtime_error{ + "No device code available for function: " + + name(function_address) + }; + } } auto agent = target_agent(stream); diff --git a/projects/clr/hipamd/src/program_state.cpp b/projects/clr/hipamd/src/program_state.cpp index c4478bec2f..3ad9285b8c 100644 --- a/projects/clr/hipamd/src/program_state.cpp +++ b/projects/clr/hipamd/src/program_state.cpp @@ -74,11 +74,15 @@ vector copy_names_of_undefined_symbols(const symbol_section_accessor& se } const std::unordered_map>& -symbol_addresses() { +symbol_addresses(bool rebuild = false) { static unordered_map> r; static once_flag f; - call_once(f, []() { + auto cons = [rebuild]() { + if (rebuild) { + r.clear(); + } + dl_iterate_phdr( [](dl_phdr_info* info, size_t, void*) { static constexpr const char self[] = "/proc/self/exe"; @@ -108,7 +112,12 @@ symbol_addresses() { return 0; }, nullptr); - }); + }; + + call_once(f, cons); + if (rebuild) { + cons(); + } return r; } @@ -166,13 +175,18 @@ vector code_object_blob_for_process() { return r; } -const unordered_map>>& code_object_blobs() { +const unordered_map>>& code_object_blobs(bool rebuild = false) { static unordered_map>> r; static once_flag f; - call_once(f, []() { + auto cons = [rebuild]() { static vector> blobs{code_object_blob_for_process()}; + if (rebuild) { + blobs.clear(); + blobs.push_back(code_object_blob_for_process()); + } + dl_iterate_phdr( [](dl_phdr_info* info, std::size_t, void*) { elfio tmp; @@ -194,7 +208,13 @@ const unordered_map>>& code_object_blobs() { } } } - }); + }; + + + call_once(f, cons); + if (rebuild) { + cons(); + } return r; } @@ -216,13 +236,13 @@ vector> function_names_for(const elfio& reader, section* return r; } -const vector>& function_names_for_process() { +const vector>& function_names_for_process(bool rebuild = false) { static constexpr const char self[] = "/proc/self/exe"; static vector> r; static once_flag f; - call_once(f, []() { + auto cons = [rebuild]() { elfio reader; if (!reader.load(self)) { @@ -233,16 +253,26 @@ const vector>& function_names_for_process() { find_section_if(reader, [](const section* x) { return x->get_type() == SHT_SYMTAB; }); if (symtab) r = function_names_for(reader, symtab); - }); + }; + + call_once(f, cons); + if (rebuild) { + cons(); + } return r; } -const unordered_map>& kernels() { +const unordered_map>& kernels(bool rebuild = false) { static unordered_map> r; static once_flag f; - call_once(f, []() { + auto cons = [rebuild]() { + if (rebuild) { + r.clear(); + executables(rebuild); + } + static const auto copy_kernels = [](hsa_executable_t, hsa_agent_t, hsa_executable_symbol_t s, void*) { if (type(s) == HSA_SYMBOL_KIND_KERNEL) r[name(s)].push_back(s); @@ -256,7 +286,12 @@ const unordered_map>& kernels() { copy_kernels, nullptr); } } - }); + }; + + call_once(f, cons); + if (rebuild) { + cons(); + } return r; } @@ -295,13 +330,18 @@ void load_code_object_and_freeze_executable( namespace hip_impl { const unordered_map>& -executables() { // TODO: This leaks the hsa_executable_ts, it should use RAII. +executables(bool rebuild) { // TODO: This leaks the hsa_executable_ts, it should use RAII. static unordered_map> r; static once_flag f; - call_once(f, []() { + auto cons = [rebuild]() { static const auto accelerators = hc::accelerator::get_all(); + if (rebuild) { + r.clear(); + code_object_blobs(rebuild); + } + for (auto&& acc : accelerators) { auto agent = static_cast(acc.get_hsa_agent()); @@ -335,17 +375,29 @@ executables() { // TODO: This leaks the hsa_executable_ts, it should use RAII. }, agent); } - }); + }; + + call_once(f, cons); + if (rebuild) { + cons(); + } return r; } -const unordered_map& function_names() { +const unordered_map& function_names(bool rebuild) { static unordered_map r{function_names_for_process().cbegin(), function_names_for_process().cend()}; static once_flag f; - call_once(f, []() { + auto cons = [rebuild]() { + if (rebuild) { + r.clear(); + function_names_for_process(rebuild); + r.insert(function_names_for_process().cbegin(), + function_names_for_process().cend()); + } + dl_iterate_phdr( [](dl_phdr_info* info, size_t, void*) { elfio tmp; @@ -365,16 +417,30 @@ const unordered_map& function_names() { return 0; }, nullptr); - }); + }; + + call_once(f, cons); + if (rebuild) { + static mutex mtx; + lock_guard lck{mtx}; + cons(); + } return r; } -const unordered_map>>& functions() { +const unordered_map>>& functions(bool rebuild) { static unordered_map>> r; static once_flag f; - call_once(f, []() { + auto cons = [rebuild]() { + if (rebuild) { + r.clear(); + function_names(rebuild); + kernels(rebuild); + globals(rebuild); + } + for (auto&& function : function_names()) { const auto it = kernels().find(function.second); @@ -386,15 +452,34 @@ const unordered_map>>& fu } } } - }); + }; + + call_once(f, cons); + if (rebuild) { + static mutex mtx; + lock_guard lck{mtx}; + cons(); + } return r; } -unordered_map& globals() { +unordered_map& globals(bool rebuild) { static unordered_map r; static once_flag f; - call_once(f, []() { r.reserve(symbol_addresses().size()); }); + auto cons =[rebuild]() { + if (rebuild) { + r.clear(); + symbol_addresses(rebuild); + } + + r.reserve(symbol_addresses().size()); + }; + + call_once(f, cons); + if (rebuild) { + cons(); + } return r; } From 1d1b402e5b713064f763b3b6b8bb738139d8a2c7 Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Fri, 15 Jun 2018 16:45:03 -0500 Subject: [PATCH 2/3] Improve performance of re-initialization logic Keep track of shared libaries already discovered. Do not build HSA executables for them. [ROCm/clr commit: b883ea759d08e4bfae509f27f9fc0cfa2b0622e7] --- projects/clr/hipamd/src/program_state.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/projects/clr/hipamd/src/program_state.cpp b/projects/clr/hipamd/src/program_state.cpp index 3ad9285b8c..3eba84f12e 100644 --- a/projects/clr/hipamd/src/program_state.cpp +++ b/projects/clr/hipamd/src/program_state.cpp @@ -180,21 +180,29 @@ const unordered_map>>& code_object_blobs(bool reb static once_flag f; auto cons = [rebuild]() { + // names of shared libraries who .kernel sections already loaded + static unordered_set lib_names; static vector> blobs{code_object_blob_for_process()}; if (rebuild) { + r.clear(); blobs.clear(); - blobs.push_back(code_object_blob_for_process()); } dl_iterate_phdr( [](dl_phdr_info* info, std::size_t, void*) { elfio tmp; - if (tmp.load(info->dlpi_name)) { + if ((lib_names.find(info->dlpi_name) == lib_names.end()) && + (tmp.load(info->dlpi_name))) { const auto it = find_section_if( tmp, [](const section* x) { return x->get_name() == ".kernel"; }); - if (it) blobs.emplace_back(it->get_data(), it->get_data() + it->get_size()); + if (it) { + blobs.emplace_back( + it->get_data(), it->get_data() + it->get_size()); + // register the shared library as already loaded + lib_names.emplace(info->dlpi_name); + } } return 0; }, @@ -338,7 +346,8 @@ executables(bool rebuild) { // TODO: This leaks the hsa_executable_ts, it shoul static const auto accelerators = hc::accelerator::get_all(); if (rebuild) { - r.clear(); + // do NOT clear r so we reuse instances of hsa_executable_t + // created previously code_object_blobs(rebuild); } From ca62af218ca7b05f5a14c6f1265731a09caef8ec Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Mon, 18 Jun 2018 16:54:18 -0500 Subject: [PATCH 3/3] Keep the map which tracks GPU kernel symbols to grow monotonically [ROCm/clr commit: 8f521edff17afc3daec7ae3d5235096eebde9c04] --- projects/clr/hipamd/src/program_state.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/clr/hipamd/src/program_state.cpp b/projects/clr/hipamd/src/program_state.cpp index 3eba84f12e..fab71b1def 100644 --- a/projects/clr/hipamd/src/program_state.cpp +++ b/projects/clr/hipamd/src/program_state.cpp @@ -444,7 +444,9 @@ const unordered_map>>& fu auto cons = [rebuild]() { if (rebuild) { - r.clear(); + // do NOT clear r so we reuse instances of pair + // created previously + function_names(rebuild); kernels(rebuild); globals(rebuild);