From 74fc268a324193446a63087c08f8945ec60ce628 Mon Sep 17 00:00:00 2001 From: habajpai-amd Date: Wed, 1 Oct 2025 19:21:26 +0530 Subject: [PATCH] Add libomptarget discovery to prevent OpenMP/HIP segfaults (#1043) This PR fixes a segmentation fault seen when running rocprof-sys-sample with multi-process OpenMP/HIP applications. The crash was caused by missing libomptarget.so on the runtime loader path or incorrect LD_PRELOAD settings. Fixes SWDEV-552804 --------- Co-authored-by: David Galiffi --- .../examples/openmp/target/CMakeLists.txt | 8 +- .../source/bin/rocprof-sys-causal/impl.cpp | 12 +- .../source/bin/rocprof-sys-run/impl.cpp | 152 +++++++++++++++--- .../source/bin/rocprof-sys-sample/impl.cpp | 23 ++- .../rocprof-sys-sample/rocprof-sys-sample.hpp | 8 +- .../source/lib/common/environment.hpp | 52 ++++++ .../source/lib/core/argparse.cpp | 24 +-- .../tests/rocprof-sys-openmp-tests.cmake | 8 +- 8 files changed, 239 insertions(+), 48 deletions(-) diff --git a/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt b/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt index 138f565d77..af4a70ca4b 100644 --- a/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt +++ b/projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt @@ -51,6 +51,8 @@ message(STATUS "ROCm root inferred from compiler: ${ROCM_ROOT_DIR}") # Candidate lib directories for libomptarget across ROCm layouts set(_LLVM_LIB_HINTS + "${ROCmVersion_DIR}/llvm/lib" + "${ROCmVersion_DIR}/lib" "${ROCM_ROOT_DIR}/lib" "${ROCM_ROOT_DIR}/llvm/lib" "$ENV{ROCM_PATH}/llvm/lib" @@ -69,7 +71,10 @@ endif() # Use the directory that actually contains the library we found get_filename_component(_rocm_llvm_lib "${LIBOMPTARGET_SO}" DIRECTORY) set(_rocm_clang_lib "${ROCM_ROOT_DIR}/lib") -set(_COMMON_RPATH "${_rocm_llvm_lib};${_rocm_clang_lib}") +set(_COMMON_RPATH "${_rocm_llvm_lib};${_rocm_clang_lib};$ORIGIN") +if(ROCmVersion_DIR) + list(APPEND _COMMON_RPATH "${ROCmVersion_DIR}/llvm/lib") +endif() list(REMOVE_DUPLICATES _COMMON_RPATH) message(STATUS "libomptarget found at: ${LIBOMPTARGET_SO}") @@ -106,6 +111,7 @@ foreach(tgt openmp-target-lib openmp-target) target_link_options( ${tgt} PUBLIC + "-Wl,--enable-new-dtags" "-L${_rocm_llvm_lib}" "-Wl,-rpath,${_rocm_llvm_lib}" "-Wl,-rpath,${_rocm_clang_lib}" diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp index 25e62004e7..bad3848df8 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp @@ -204,6 +204,14 @@ get_initial_environment() get_env("ROCPROFSYS_THREAD_POOL_SIZE", 0)); update_env(_env, "ROCPROFSYS_LAUNCHER", "rocprof-sys-causal"); + // Ensure libomptarget.so can be found by the target (OpenMP/HIP apps) + if(auto llvm_dir = + rocprofsys::common::discover_llvm_libdir_for_ompt(get_verbose() > 0); + !llvm_dir.empty()) + { + update_env(_env, "LD_LIBRARY_PATH", llvm_dir, /*append=*/true); + } + return _env; } @@ -351,13 +359,13 @@ update_env(std::vector& _environ, std::string_view _env_var, Tp&& _env_va if(_env_var == "LD_PRELOAD") { itr = - strdup(join('=', _env_var, join(_join_delim, _val, _env_val)) + strdup(join('=', _env_var, join(_join_delim, _env_val, _val)) .c_str()); } else { itr = - strdup(join('=', _env_var, join(_join_delim, _env_val, _val)) + strdup(join('=', _env_var, join(_join_delim, _val, _env_val)) .c_str()); } } diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-run/impl.cpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-run/impl.cpp index 877d4030d7..fafa638717 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-run/impl.cpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-run/impl.cpp @@ -82,31 +82,112 @@ to_string(bool _v) namespace { +auto original_envs = std::set{}; +enum update_mode : int +{ + UPD_REPLACE = 0, // no PREPEND/APPEND bits set + UPD_PREPEND = 1 << 0, // 0x01 + UPD_APPEND = 1 << 1, // 0x02 + UPD_WEAK = 1 << 2, // 0x04 +}; + +std::string +get_rocprofsys_root(void) +{ + char* _tmp = realpath("/proc/self/exe", nullptr); + std::string _exe = (_tmp) ? std::string{ _tmp } : std::string{}; + + if(_tmp) free(_tmp); + + auto _pos = _exe.find_last_of('/'); + auto _dir = std::string{ "./" }; + + if(_pos != std::string::npos) _dir = _exe.substr(0, _pos); + + return rocprofsys::common::join("/", _dir, ".."); +} + std::string get_internal_libpath(const std::string& _lib) { - auto _exe = std::string_view{ realpath("/proc/self/exe", nullptr) }; - auto _pos = _exe.find_last_of('/'); - auto _dir = std::string{ "./" }; - if(_pos != std::string_view::npos) _dir = _exe.substr(0, _pos); - return rocprofsys::common::join("/", _dir, "..", "lib", _lib); + auto _root = get_rocprofsys_root(); + return rocprofsys::common::join("/", _root, "lib", _lib); } -parser_data_t& -get_initial_environment(parser_data_t& _data) +std::string +get_internal_script_path(void) { - if(environ != nullptr) + auto _root = get_rocprofsys_root(); + return rocprofsys::common::join("/", _root, "libexec", "rocprofiler-systems"); +} + +std::string +get_realpath(const std::string& _v) +{ + if(auto* _tmp = realpath(_v.c_str(), nullptr)) { - int idx = 0; - while(environ[idx] != nullptr) - { - auto* _v = environ[idx++]; - _data.initial.emplace(_v); - _data.current.emplace_back(strdup(_v)); - } + std::string _ret{ _tmp }; + free(_tmp); + return _ret; + } + return {}; +} + +template +void +update_env(std::vector& _environ, std::string_view _env_var, Tp&& _env_val, + update_mode&& _mode, std::string_view _join_delim = ":") +{ + auto _prepend = (_mode & UPD_PREPEND) != 0; + auto _append = (_mode & UPD_APPEND) != 0; + auto _weak_upd = (_mode & UPD_WEAK) != 0; + + // if both flags are set, prefer append + if(_prepend && _append) + { + _prepend = false; } - return _data; + auto _key = join("", _env_var, "="); + for(auto& itr : _environ) + { + if(!itr) continue; + if(std::string_view{ itr }.find(_key) == 0) + { + if(_weak_upd) + { + // if the value has changed, do not update but allow overridding the value + // inherited from the initial env + if(original_envs.find(std::string{ itr }) == original_envs.end()) return; + } + + if(_prepend || _append) + { + if(std::string_view{ itr }.find(join("", _env_val)) == + std::string_view::npos) + { + auto _val = std::string{ itr }.substr(_key.length()); + free(itr); + if(_prepend) + itr = + strdup(join('=', _env_var, join(_join_delim, _env_val, _val)) + .c_str()); + else + itr = + strdup(join('=', _env_var, join(_join_delim, _val, _env_val)) + .c_str()); + } + } + else + { + free(itr); + itr = strdup(rocprofsys::common::join('=', _env_var, _env_val).c_str()); + } + return; + } + } + _environ.emplace_back( + strdup(rocprofsys::common::join('=', _env_var, _env_val).c_str())); } int @@ -121,13 +202,40 @@ get_verbose(parser_data_t& _data) return verbose; } -std::string -get_realpath(const std::string& _v) +parser_data_t& +get_initial_environment(parser_data_t& _data) { - auto* _tmp = realpath(_v.c_str(), nullptr); - auto _ret = std::string{ _tmp }; - free(_tmp); - return _ret; + if(environ != nullptr) + { + int idx = 0; + while(environ[idx] != nullptr) + { + auto* _v = environ[idx++]; + _data.initial.emplace(_v); + _data.current.emplace_back(strdup(_v)); + original_envs.emplace(_v); + } + } + + auto _libexecpath = get_realpath(get_internal_script_path()); + if(!_libexecpath.empty()) + { + update_env(_data.current, "ROCPROFSYS_SCRIPT_PATH", _libexecpath, UPD_REPLACE); + _data.updated.emplace("ROCPROFSYS_SCRIPT_PATH"); + } + + const bool verbose = (get_verbose(_data) > 0); + if(auto llvm_dir = rocprofsys::common::discover_llvm_libdir_for_ompt(verbose); + !llvm_dir.empty()) + { + update_env(_data.current, "LD_LIBRARY_PATH", llvm_dir, UPD_APPEND); + _data.updated.emplace("LD_LIBRARY_PATH"); + auto current_ld = getenv("LD_LIBRARY_PATH"); + std::string new_ld = current_ld ? (llvm_dir + ":" + current_ld) : llvm_dir; + setenv("LD_LIBRARY_PATH", new_ld.c_str(), 1); + } + + return _data; } auto diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp index 8fb47207ae..c2c8c1e1d6 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp @@ -132,6 +132,13 @@ get_initial_environment() update_env(_env, "LD_LIBRARY_PATH", tim::filepath::dirname(_dl_libpath), UPD_APPEND); update_env(_env, "ROCPROFSYS_SCRIPT_PATH", _libexecpath, UPD_REPLACE); + // Discover LLVM libdir containing libomptarget.so and append to LD_LIBRARY_PATH + if(auto llvm_dir = rocprofsys::common::discover_llvm_libdir_for_ompt(verbose > 0); + !llvm_dir.empty()) + { + update_env(_env, "LD_LIBRARY_PATH", llvm_dir, UPD_APPEND); + } + auto _mode = get_env("ROCPROFSYS_MODE", "sampling", false); update_env(_env, "ROCPROFSYS_USE_SAMPLING", (_mode != "causal")); @@ -223,9 +230,15 @@ update_env(std::vector& _environ, std::string_view _env_var, Tp&& _env_va { updated_envs.emplace(_env_var); - auto _prepend = (_mode & UPD_PREPEND) == UPD_PREPEND; - auto _append = (_mode & UPD_APPEND) == UPD_APPEND; - auto _weak_upd = (_mode & UPD_WEAK) == UPD_WEAK; + auto _prepend = (_mode & UPD_PREPEND) != 0; + auto _append = (_mode & UPD_APPEND) != 0; + auto _weak_upd = (_mode & UPD_WEAK) != 0; + + // if both flags are set, prefer append + if(_prepend && _append) + { + _prepend = false; + } auto _key = join("", _env_var, "="); for(auto& itr : _environ) @@ -249,11 +262,11 @@ update_env(std::vector& _environ, std::string_view _env_var, Tp&& _env_va free(itr); if(_prepend) itr = - strdup(join('=', _env_var, join(_join_delim, _val, _env_val)) + strdup(join('=', _env_var, join(_join_delim, _env_val, _val)) .c_str()); else itr = - strdup(join('=', _env_var, join(_join_delim, _env_val, _val)) + strdup(join('=', _env_var, join(_join_delim, _val, _env_val)) .c_str()); } } diff --git a/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/rocprof-sys-sample.hpp b/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/rocprof-sys-sample.hpp index 3ff6198567..8ca99967e3 100644 --- a/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/rocprof-sys-sample.hpp +++ b/projects/rocprofiler-systems/source/bin/rocprof-sys-sample/rocprof-sys-sample.hpp @@ -28,10 +28,10 @@ enum update_mode : int { - UPD_REPLACE = 0x1, - UPD_PREPEND = 0x2, - UPD_APPEND = 0x3, - UPD_WEAK = 0x4, + UPD_REPLACE = 0, // no PREPEND/APPEND bits set + UPD_PREPEND = 1 << 0, // 0x01 + UPD_APPEND = 1 << 1, // 0x02 + UPD_WEAK = 1 << 2, // 0x04 }; std::string diff --git a/projects/rocprofiler-systems/source/lib/common/environment.hpp b/projects/rocprofiler-systems/source/lib/common/environment.hpp index 2501208295..8c3d42728c 100644 --- a/projects/rocprofiler-systems/source/lib/common/environment.hpp +++ b/projects/rocprofiler-systems/source/lib/common/environment.hpp @@ -24,6 +24,7 @@ #include "common/defines.h" +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #include @@ -177,5 +179,55 @@ struct ROCPROFSYS_INTERNAL_API env_config return setenv(env_name.c_str(), env_value.c_str(), override); } }; + +inline std::string +discover_llvm_libdir_for_ompt(bool verbose = false) +{ + auto strip = [](std::string s) { + if(!s.empty() && s.back() == '/') s.pop_back(); + return s; + }; + + // Common ROCm envs + const auto rocm_dir = strip(get_env("ROCM_PATH", "/opt/rocm")); + const auto rocmv_dir = strip(get_env("ROCmVersion_DIR", "")); + + std::vector candidates; + candidates.reserve(6); + + auto push_unique = [&](const std::string& p) { + if(p.empty()) return; + if(std::find(candidates.begin(), candidates.end(), p) == candidates.end()) + candidates.emplace_back(p); + }; + + if(!rocmv_dir.empty()) + { + push_unique(rocmv_dir + "/llvm/lib"); + push_unique(rocmv_dir + "/lib"); + } + push_unique(rocm_dir + "/llvm/lib"); + push_unique(rocm_dir + "/lib/llvm/lib"); + push_unique("/opt/rocm/llvm/lib"); + push_unique("/opt/rocm/lib/llvm/lib"); + + auto has_libomptarget = [](const std::string& dir) { + const std::string so = dir + "/libomptarget.so"; + return ::tim::filepath::exists(so); + }; + + // Pick the first candidate that contains libomptarget.so + auto it = std::find_if(candidates.begin(), candidates.end(), has_libomptarget); + if(it != candidates.end()) + { + ROCPROFSYS_ENVIRON_LOG(verbose, "Using LLVM libdir: %s\n", it->c_str()); + return *it; + } + + ROCPROFSYS_ENVIRON_LOG(verbose, + "libomptarget.so not found in candidate LLVM libdirs\n"); + return {}; +} + } // namespace common } // namespace rocprofsys diff --git a/projects/rocprofiler-systems/source/lib/core/argparse.cpp b/projects/rocprofiler-systems/source/lib/core/argparse.cpp index 5a457269d2..de3af4ea5c 100644 --- a/projects/rocprofiler-systems/source/lib/core/argparse.cpp +++ b/projects/rocprofiler-systems/source/lib/core/argparse.cpp @@ -87,10 +87,10 @@ get_realpath(const std::string& _path) enum update_mode : int { - UPD_REPLACE = 0x1, - UPD_PREPEND = 0x2, - UPD_APPEND = 0x3, - UPD_WEAK = 0x4, + UPD_REPLACE = 0, // no PREPEND/APPEND bits set + UPD_PREPEND = 1 << 0, // 0x01 + UPD_APPEND = 1 << 1, // 0x02 + UPD_WEAK = 1 << 2, // 0x04 }; template @@ -100,9 +100,15 @@ update_env(parser_data& _data, std::string_view _env_var, Tp&& _env_val, { _data.updated.emplace(_env_var); - auto _prepend = (_mode & UPD_PREPEND) == UPD_PREPEND; - auto _append = (_mode & UPD_APPEND) == UPD_APPEND; - auto _weak_upd = (_mode & UPD_WEAK) == UPD_WEAK; + auto _prepend = (_mode & UPD_PREPEND) != 0; + auto _append = (_mode & UPD_APPEND) != 0; + auto _weak_upd = (_mode & UPD_WEAK) != 0; + + // if both flags are set, prefer append + if(_prepend && _append) + { + _prepend = false; + } auto _key = join("", _env_var, "="); for(auto& itr : _data.current) @@ -126,11 +132,11 @@ update_env(parser_data& _data, std::string_view _env_var, Tp&& _env_val, free(itr); if(_prepend) itr = - strdup(join('=', _env_var, join(_join_delim, _val, _env_val)) + strdup(join('=', _env_var, join(_join_delim, _env_val, _val)) .c_str()); else itr = - strdup(join('=', _env_var, join(_join_delim, _env_val, _val)) + strdup(join('=', _env_var, join(_join_delim, _val, _env_val)) .c_str()); } } diff --git a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake index 8395a26a68..657f5d641d 100644 --- a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake +++ b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake @@ -15,12 +15,10 @@ else() set(_rocm_root "/opt/rocm") endif() -set(_rocm_llvm_lib "${_rocm_root}/lib/llvm/lib") +# Set path to ROCm LLVM library directory containing libomptarget.so +set(_rocm_llvm_lib "${_rocm_root}/llvm/lib") -set(_rocm_ld_env - "LD_PRELOAD=libomptarget.so" - "LD_LIBRARY_PATH=${_rocm_llvm_lib}:$ENV{LD_LIBRARY_PATH}" -) +set(_rocm_ld_env "LD_LIBRARY_PATH=${_rocm_llvm_lib}:$ENV{LD_LIBRARY_PATH}") if(NOT EXISTS "${_rocm_llvm_lib}/libomptarget.so" AND ROCPROFSYS_USE_ROCM) message(