Merge 'master' into 'amd-master'

Change-Id: Ia897c694ffbbce2ab01493d3ea60e42701ce021e

hipamd/include/hip/hcc_detail/program_state.hpp

@@ -93,11 +93,12 @@ public:
     }
 };
 
-const std::unordered_map<hsa_agent_t, std::vector<hsa_executable_t>>& executables();
+const std::unordered_map<hsa_agent_t, std::vector<hsa_executable_t>>& executables(
+    bool rebuild = false);
 const std::unordered_map<std::uintptr_t, std::vector<std::pair<hsa_agent_t, Kernel_descriptor>>>&
-functions();
+functions(bool rebuild = false);
-const std::unordered_map<std::uintptr_t, std::string>& function_names();
+const std::unordered_map<std::uintptr_t, std::string>& function_names(bool rebuild = false);
-std::unordered_map<std::string, void*>& globals();
+std::unordered_map<std::string, void*>& globals(bool rebuild = false);
 
 hsa_executable_t load_executable(const std::string& file, hsa_executable_t executable,
                                  hsa_agent_t agent);

hipamd/src/functional_grid_launch.inl

@@ -92,13 +92,19 @@ namespace hip_impl
         hipStream_t stream,
         void** kernarg)
     {
-        const auto it0 = functions().find(function_address);
+        auto it0 = functions().find(function_address);
 
         if (it0 == functions().cend()) {
-            throw runtime_error{
+            // Re-init device code maps once again to help locate kernels
-                "No device code available for function: " +
+            // loaded after HIP runtime initialization via means such as
-                name(function_address)
+            // dlopen().
-            };
+            it0 = functions(true).find(function_address);
+            if (it0 == functions().cend()) {
+                throw runtime_error{
+                    "No device code available for function: " +
+                    name(function_address)
+                };
+            }
         }
 
         auto agent = target_agent(stream);

hipamd/src/program_state.cpp

@@ -74,11 +74,15 @@ vector<string> copy_names_of_undefined_symbols(const symbol_section_accessor& se
 }
 
 const std::unordered_map<std::string, std::pair<ELFIO::Elf64_Addr, ELFIO::Elf_Xword>>&
-symbol_addresses() {
+symbol_addresses(bool rebuild = false) {
     static unordered_map<string, pair<Elf64_Addr, Elf_Xword>> r;
     static once_flag f;
 
-    call_once(f, []() {
+    auto cons = [rebuild]() {
+        if (rebuild) {
+            r.clear();
+        }
+
         dl_iterate_phdr(
             [](dl_phdr_info* info, size_t, void*) {
                 static constexpr const char self[] = "/proc/self/exe";
@@ -108,7 +112,12 @@ symbol_addresses() {
                 return 0;
             },
             nullptr);
-    });
+    };
+
+    call_once(f, cons);
+    if (rebuild) {
+        cons();
+    }
 
     return r;
 }
@@ -166,21 +175,34 @@ vector<char> code_object_blob_for_process() {
     return r;
 }
 
-const unordered_map<hsa_isa_t, vector<vector<char>>>& code_object_blobs() {
+const unordered_map<hsa_isa_t, vector<vector<char>>>& code_object_blobs(bool rebuild = false) {
     static unordered_map<hsa_isa_t, vector<vector<char>>> r;
     static once_flag f;
 
-    call_once(f, []() {
+    auto cons = [rebuild]() {
+        // names of shared libraries who .kernel sections already loaded
+        static unordered_set<string> lib_names;
         static vector<vector<char>> blobs{code_object_blob_for_process()};
 
+        if (rebuild) {
+            r.clear();
+            blobs.clear();
+        }
+
         dl_iterate_phdr(
             [](dl_phdr_info* info, std::size_t, void*) {
                 elfio tmp;
-                if (tmp.load(info->dlpi_name)) {
+                if ((lib_names.find(info->dlpi_name) == lib_names.end()) &&
+                    (tmp.load(info->dlpi_name))) {
                     const auto it = find_section_if(
                         tmp, [](const section* x) { return x->get_name() == ".kernel"; });
 
-                    if (it) blobs.emplace_back(it->get_data(), it->get_data() + it->get_size());
+                    if (it) {
+                        blobs.emplace_back(
+                            it->get_data(), it->get_data() + it->get_size());
+                        // register the shared library as already loaded
+                        lib_names.emplace(info->dlpi_name);
+                    }
                 }
                 return 0;
             },
@@ -194,7 +216,13 @@ const unordered_map<hsa_isa_t, vector<vector<char>>>& code_object_blobs() {
                 }
             }
         }
-    });
+    };
+
+
+    call_once(f, cons);
+    if (rebuild) {
+        cons();
+    }
 
     return r;
 }
@@ -216,13 +244,13 @@ vector<pair<uintptr_t, string>> function_names_for(const elfio& reader, section*
     return r;
 }
 
-const vector<pair<uintptr_t, string>>& function_names_for_process() {
+const vector<pair<uintptr_t, string>>& function_names_for_process(bool rebuild = false) {
     static constexpr const char self[] = "/proc/self/exe";
 
     static vector<pair<uintptr_t, string>> r;
     static once_flag f;
 
-    call_once(f, []() {
+    auto cons = [rebuild]() {
         elfio reader;
 
         if (!reader.load(self)) {
@@ -233,16 +261,26 @@ const vector<pair<uintptr_t, string>>& function_names_for_process() {
             find_section_if(reader, [](const section* x) { return x->get_type() == SHT_SYMTAB; });
 
         if (symtab) r = function_names_for(reader, symtab);
-    });
+    };
+
+    call_once(f, cons);
+    if (rebuild) {
+        cons();
+    }
 
     return r;
 }
 
-const unordered_map<string, vector<hsa_executable_symbol_t>>& kernels() {
+const unordered_map<string, vector<hsa_executable_symbol_t>>& kernels(bool rebuild = false) {
     static unordered_map<string, vector<hsa_executable_symbol_t>> r;
     static once_flag f;
 
-    call_once(f, []() {
+    auto cons = [rebuild]() {
+        if (rebuild) {
+            r.clear();
+            executables(rebuild);
+        }
+
         static const auto copy_kernels = [](hsa_executable_t, hsa_agent_t,
                                             hsa_executable_symbol_t s, void*) {
             if (type(s) == HSA_SYMBOL_KIND_KERNEL) r[name(s)].push_back(s);
@@ -256,7 +294,12 @@ const unordered_map<string, vector<hsa_executable_symbol_t>>& kernels() {
                                                      copy_kernels, nullptr);
             }
         }
-    });
+    };
+
+    call_once(f, cons);
+    if (rebuild) {
+        cons();
+    }
 
     return r;
 }
@@ -295,13 +338,19 @@ void load_code_object_and_freeze_executable(
 
 namespace hip_impl {
 const unordered_map<hsa_agent_t, vector<hsa_executable_t>>&
-executables() {  // TODO: This leaks the hsa_executable_ts, it should use RAII.
+executables(bool rebuild) {  // TODO: This leaks the hsa_executable_ts, it should use RAII.
     static unordered_map<hsa_agent_t, vector<hsa_executable_t>> r;
     static once_flag f;
 
-    call_once(f, []() {
+    auto cons = [rebuild]() {
         static const auto accelerators = hc::accelerator::get_all();
 
+        if (rebuild) {
+            // do NOT clear r so we reuse instances of hsa_executable_t
+            // created previously
+            code_object_blobs(rebuild);
+        }
+
         for (auto&& acc : accelerators) {
             auto agent = static_cast<hsa_agent_t*>(acc.get_hsa_agent());
 
@@ -335,17 +384,29 @@ executables() {  // TODO: This leaks the hsa_executable_ts, it should use RAII.
                                    },
                                    agent);
         }
-    });
+    };
+
+    call_once(f, cons);
+    if (rebuild) {
+        cons();
+    }
 
     return r;
 }
 
-const unordered_map<uintptr_t, string>& function_names() {
+const unordered_map<uintptr_t, string>& function_names(bool rebuild) {
     static unordered_map<uintptr_t, string> r{function_names_for_process().cbegin(),
                                               function_names_for_process().cend()};
     static once_flag f;
 
-    call_once(f, []() {
+    auto cons = [rebuild]() {
+        if (rebuild) {
+            r.clear();
+            function_names_for_process(rebuild);
+            r.insert(function_names_for_process().cbegin(),
+                     function_names_for_process().cend());
+        }
+
         dl_iterate_phdr(
             [](dl_phdr_info* info, size_t, void*) {
                 elfio tmp;
@@ -365,16 +426,32 @@ const unordered_map<uintptr_t, string>& function_names() {
                 return 0;
             },
             nullptr);
-    });
+    };
+
+    call_once(f, cons);
+    if (rebuild) {
+        static mutex mtx;
+        lock_guard<mutex> lck{mtx};
+        cons();
+    }
 
     return r;
 }
 
-const unordered_map<uintptr_t, vector<pair<hsa_agent_t, Kernel_descriptor>>>& functions() {
+const unordered_map<uintptr_t, vector<pair<hsa_agent_t, Kernel_descriptor>>>& functions(bool rebuild) {
     static unordered_map<uintptr_t, vector<pair<hsa_agent_t, Kernel_descriptor>>> r;
     static once_flag f;
 
-    call_once(f, []() {
+    auto cons = [rebuild]() {
+        if (rebuild) {
+            // do NOT clear r so we reuse instances of pair<hsa_agent_t, Kernel_descriptor>
+            // created previously
+
+            function_names(rebuild);
+            kernels(rebuild);
+            globals(rebuild);
+        }
+
         for (auto&& function : function_names()) {
             const auto it = kernels().find(function.second);
 
@@ -386,15 +463,34 @@ const unordered_map<uintptr_t, vector<pair<hsa_agent_t, Kernel_descriptor>>>& fu
                 }
             }
         }
-    });
+    };
+
+    call_once(f, cons);
+    if (rebuild) {
+        static mutex mtx;
+        lock_guard<mutex> lck{mtx};
+        cons();
+    }
 
     return r;
 }
 
-unordered_map<string, void*>& globals() {
+unordered_map<string, void*>& globals(bool rebuild) {
     static unordered_map<string, void*> r;
     static once_flag f;
-    call_once(f, []() { r.reserve(symbol_addresses().size()); });
+    auto cons =[rebuild]() {
+        if (rebuild) {
+            r.clear();
+            symbol_addresses(rebuild);
+        }
+
+        r.reserve(symbol_addresses().size());
+    };
+
+    call_once(f, cons);
+    if (rebuild) {
+        cons();
+    }
 
     return r;
 }