AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add libomptarget discovery to prevent OpenMP/HIP segfaults (#1043)

Browse Source 
This PR fixes a segmentation fault seen when running rocprof-sys-sample with multi-process OpenMP/HIP applications.
The crash was caused by missing libomptarget.so on the runtime loader path or incorrect LD_PRELOAD settings.

Fixes SWDEV-552804

---------

Co-authored-by: David Galiffi <David.Galiffi@amd.com>
This commit is contained in:
habajpai-amd
2025-10-01 19:21:26 +05:30
committed by GitHub
parent 36a1fd87af
commit 74fc268a32
8 changed files with 239 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocprofiler-systems/examples/openmp/target/CMakeLists.txt
+7 -1
View File
@@ -51,6 +51,8 @@ message(STATUS "ROCm root inferred from compiler: ${ROCM_ROOT_DIR}")
# Candidate lib directories for libomptarget across ROCm layouts
set(_LLVM_LIB_HINTS
"${ROCmVersion_DIR}/llvm/lib"
"${ROCmVersion_DIR}/lib"
"${ROCM_ROOT_DIR}/lib"
"${ROCM_ROOT_DIR}/llvm/lib"
"$ENV{ROCM_PATH}/llvm/lib"
@@ -69,7 +71,10 @@ endif()
# Use the directory that actually contains the library we found
get_filename_component(_rocm_llvm_lib "${LIBOMPTARGET_SO}" DIRECTORY)
set(_rocm_clang_lib "${ROCM_ROOT_DIR}/lib")
set(_COMMON_RPATH "${_rocm_llvm_lib};${_rocm_clang_lib}")
set(_COMMON_RPATH "${_rocm_llvm_lib};${_rocm_clang_lib};$ORIGIN")
if(ROCmVersion_DIR)
list(APPEND _COMMON_RPATH "${ROCmVersion_DIR}/llvm/lib")
endif()
list(REMOVE_DUPLICATES _COMMON_RPATH)
message(STATUS "libomptarget found at: ${LIBOMPTARGET_SO}")
@@ -106,6 +111,7 @@ foreach(tgt openmp-target-lib openmp-target)
target_link_options(
${tgt}
PUBLIC
"-Wl,--enable-new-dtags"
"-L${_rocm_llvm_lib}"
"-Wl,-rpath,${_rocm_llvm_lib}"
"-Wl,-rpath,${_rocm_clang_lib}"
projects/rocprofiler-systems/source/bin/rocprof-sys-causal/impl.cpp
+10 -2
View File
@@ -204,6 +204,14 @@ get_initial_environment()
get_env<int>("ROCPROFSYS_THREAD_POOL_SIZE", 0));
update_env(_env, "ROCPROFSYS_LAUNCHER", "rocprof-sys-causal");
// Ensure libomptarget.so can be found by the target (OpenMP/HIP apps)
if(auto llvm_dir =
rocprofsys::common::discover_llvm_libdir_for_ompt(get_verbose() > 0);
!llvm_dir.empty())
{
update_env(_env, "LD_LIBRARY_PATH", llvm_dir, /*append=*/true);
}
return _env;
}
@@ -351,13 +359,13 @@ update_env(std::vector<char*>& _environ, std::string_view _env_var, Tp&& _env_va
if(_env_var == "LD_PRELOAD")
{
itr =
strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
.c_str());
}
else
{
itr =
strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
.c_str());
}
}
projects/rocprofiler-systems/source/bin/rocprof-sys-run/impl.cpp
+130 -22
View File
@@ -82,31 +82,112 @@ to_string(bool _v)
namespace
{
auto original_envs = std::set<std::string>{};
enum update_mode : int
{
UPD_REPLACE = 0, // no PREPEND/APPEND bits set
UPD_PREPEND = 1 << 0, // 0x01
UPD_APPEND = 1 << 1, // 0x02
UPD_WEAK = 1 << 2, // 0x04
};
std::string
get_rocprofsys_root(void)
{
char* _tmp = realpath("/proc/self/exe", nullptr);
std::string _exe = (_tmp) ? std::string{ _tmp } : std::string{};
if(_tmp) free(_tmp);
auto _pos = _exe.find_last_of('/');
auto _dir = std::string{ "./" };
if(_pos != std::string::npos) _dir = _exe.substr(0, _pos);
return rocprofsys::common::join("/", _dir, "..");
}
std::string
get_internal_libpath(const std::string& _lib)
{
auto _exe = std::string_view{ realpath("/proc/self/exe", nullptr) };
auto _pos = _exe.find_last_of('/');
auto _dir = std::string{ "./" };
if(_pos != std::string_view::npos) _dir = _exe.substr(0, _pos);
return rocprofsys::common::join("/", _dir, "..", "lib", _lib);
auto _root = get_rocprofsys_root();
return rocprofsys::common::join("/", _root, "lib", _lib);
}
parser_data_t&
get_initial_environment(parser_data_t& _data)
std::string
get_internal_script_path(void)
{
if(environ != nullptr)
auto _root = get_rocprofsys_root();
return rocprofsys::common::join("/", _root, "libexec", "rocprofiler-systems");
}
std::string
get_realpath(const std::string& _v)
{
if(auto* _tmp = realpath(_v.c_str(), nullptr))
{
int idx = 0;
while(environ[idx] != nullptr)
{
auto* _v = environ[idx++];
_data.initial.emplace(_v);
_data.current.emplace_back(strdup(_v));
}
std::string _ret{ _tmp };
free(_tmp);
return _ret;
}
return {};
}
template <typename Tp>
void
update_env(std::vector<char*>& _environ, std::string_view _env_var, Tp&& _env_val,
update_mode&& _mode, std::string_view _join_delim = ":")
{
auto _prepend = (_mode & UPD_PREPEND) != 0;
auto _append = (_mode & UPD_APPEND) != 0;
auto _weak_upd = (_mode & UPD_WEAK) != 0;
// if both flags are set, prefer append
if(_prepend && _append)
{
_prepend = false;
}
return _data;
auto _key = join("", _env_var, "=");
for(auto& itr : _environ)
{
if(!itr) continue;
if(std::string_view{ itr }.find(_key) == 0)
{
if(_weak_upd)
{
// if the value has changed, do not update but allow overridding the value
// inherited from the initial env
if(original_envs.find(std::string{ itr }) == original_envs.end()) return;
}
if(_prepend || _append)
{
if(std::string_view{ itr }.find(join("", _env_val)) ==
std::string_view::npos)
{
auto _val = std::string{ itr }.substr(_key.length());
free(itr);
if(_prepend)
itr =
strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
.c_str());
else
itr =
strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
.c_str());
}
}
else
{
free(itr);
itr = strdup(rocprofsys::common::join('=', _env_var, _env_val).c_str());
}
return;
}
}
_environ.emplace_back(
strdup(rocprofsys::common::join('=', _env_var, _env_val).c_str()));
}
int
@@ -121,13 +202,40 @@ get_verbose(parser_data_t& _data)
return verbose;
}
std::string
get_realpath(const std::string& _v)
parser_data_t&
get_initial_environment(parser_data_t& _data)
{
auto* _tmp = realpath(_v.c_str(), nullptr);
auto _ret = std::string{ _tmp };
free(_tmp);
return _ret;
if(environ != nullptr)
{
int idx = 0;
while(environ[idx] != nullptr)
{
auto* _v = environ[idx++];
_data.initial.emplace(_v);
_data.current.emplace_back(strdup(_v));
original_envs.emplace(_v);
}
}
auto _libexecpath = get_realpath(get_internal_script_path());
if(!_libexecpath.empty())
{
update_env(_data.current, "ROCPROFSYS_SCRIPT_PATH", _libexecpath, UPD_REPLACE);
_data.updated.emplace("ROCPROFSYS_SCRIPT_PATH");
}
const bool verbose = (get_verbose(_data) > 0);
if(auto llvm_dir = rocprofsys::common::discover_llvm_libdir_for_ompt(verbose);
!llvm_dir.empty())
{
update_env(_data.current, "LD_LIBRARY_PATH", llvm_dir, UPD_APPEND);
_data.updated.emplace("LD_LIBRARY_PATH");
auto current_ld = getenv("LD_LIBRARY_PATH");
std::string new_ld = current_ld ? (llvm_dir + ":" + current_ld) : llvm_dir;
setenv("LD_LIBRARY_PATH", new_ld.c_str(), 1);
}
return _data;
}
auto
projects/rocprofiler-systems/source/bin/rocprof-sys-sample/impl.cpp
+18 -5
View File
@@ -132,6 +132,13 @@ get_initial_environment()
update_env(_env, "LD_LIBRARY_PATH", tim::filepath::dirname(_dl_libpath), UPD_APPEND);
update_env(_env, "ROCPROFSYS_SCRIPT_PATH", _libexecpath, UPD_REPLACE);
// Discover LLVM libdir containing libomptarget.so and append to LD_LIBRARY_PATH
if(auto llvm_dir = rocprofsys::common::discover_llvm_libdir_for_ompt(verbose > 0);
!llvm_dir.empty())
{
update_env(_env, "LD_LIBRARY_PATH", llvm_dir, UPD_APPEND);
}
auto _mode = get_env<std::string>("ROCPROFSYS_MODE", "sampling", false);
update_env(_env, "ROCPROFSYS_USE_SAMPLING", (_mode != "causal"));
@@ -223,9 +230,15 @@ update_env(std::vector<char*>& _environ, std::string_view _env_var, Tp&& _env_va
{
updated_envs.emplace(_env_var);
auto _prepend = (_mode & UPD_PREPEND) == UPD_PREPEND;
auto _append = (_mode & UPD_APPEND) == UPD_APPEND;
auto _weak_upd = (_mode & UPD_WEAK) == UPD_WEAK;
auto _prepend = (_mode & UPD_PREPEND) != 0;
auto _append = (_mode & UPD_APPEND) != 0;
auto _weak_upd = (_mode & UPD_WEAK) != 0;
// if both flags are set, prefer append
if(_prepend && _append)
{
_prepend = false;
}
auto _key = join("", _env_var, "=");
for(auto& itr : _environ)
@@ -249,11 +262,11 @@ update_env(std::vector<char*>& _environ, std::string_view _env_var, Tp&& _env_va
free(itr);
if(_prepend)
itr =
strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
.c_str());
else
itr =
strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
.c_str());
}
}
projects/rocprofiler-systems/source/bin/rocprof-sys-sample/rocprof-sys-sample.hpp
+4 -4
View File
@@ -28,10 +28,10 @@
enum update_mode : int
{
UPD_REPLACE = 0x1,
UPD_PREPEND = 0x2,
UPD_APPEND = 0x3,
UPD_WEAK = 0x4,
UPD_REPLACE = 0, // no PREPEND/APPEND bits set
UPD_PREPEND = 1 << 0, // 0x01
UPD_APPEND = 1 << 1, // 0x02
UPD_WEAK = 1 << 2, // 0x04
};
std::string
projects/rocprofiler-systems/source/lib/common/environment.hpp
+52
View File
@@ -24,6 +24,7 @@
#include "common/defines.h"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
@@ -31,6 +32,7 @@
#include <stdexcept>
#include <string>
#include <string_view>
#include <timemory/utility/filepath.hpp>
#include <type_traits>
#include <unistd.h>
@@ -177,5 +179,55 @@ struct ROCPROFSYS_INTERNAL_API env_config
return setenv(env_name.c_str(), env_value.c_str(), override);
}
};
inline std::string
discover_llvm_libdir_for_ompt(bool verbose = false)
{
auto strip = [](std::string s) {
if(!s.empty() && s.back() == '/') s.pop_back();
return s;
};
// Common ROCm envs
const auto rocm_dir = strip(get_env<std::string>("ROCM_PATH", "/opt/rocm"));
const auto rocmv_dir = strip(get_env<std::string>("ROCmVersion_DIR", ""));
std::vector<std::string> candidates;
candidates.reserve(6);
auto push_unique = [&](const std::string& p) {
if(p.empty()) return;
if(std::find(candidates.begin(), candidates.end(), p) == candidates.end())
candidates.emplace_back(p);
};
if(!rocmv_dir.empty())
{
push_unique(rocmv_dir + "/llvm/lib");
push_unique(rocmv_dir + "/lib");
}
push_unique(rocm_dir + "/llvm/lib");
push_unique(rocm_dir + "/lib/llvm/lib");
push_unique("/opt/rocm/llvm/lib");
push_unique("/opt/rocm/lib/llvm/lib");
auto has_libomptarget = [](const std::string& dir) {
const std::string so = dir + "/libomptarget.so";
return ::tim::filepath::exists(so);
};
// Pick the first candidate that contains libomptarget.so
auto it = std::find_if(candidates.begin(), candidates.end(), has_libomptarget);
if(it != candidates.end())
{
ROCPROFSYS_ENVIRON_LOG(verbose, "Using LLVM libdir: %s\n", it->c_str());
return *it;
}
ROCPROFSYS_ENVIRON_LOG(verbose,
"libomptarget.so not found in candidate LLVM libdirs\n");
return {};
}
} // namespace common
} // namespace rocprofsys
projects/rocprofiler-systems/source/lib/core/argparse.cpp
+15 -9
View File
@@ -87,10 +87,10 @@ get_realpath(const std::string& _path)
enum update_mode : int
{
UPD_REPLACE = 0x1,
UPD_PREPEND = 0x2,
UPD_APPEND = 0x3,
UPD_WEAK = 0x4,
UPD_REPLACE = 0, // no PREPEND/APPEND bits set
UPD_PREPEND = 1 << 0, // 0x01
UPD_APPEND = 1 << 1, // 0x02
UPD_WEAK = 1 << 2, // 0x04
};
template <typename Tp>
@@ -100,9 +100,15 @@ update_env(parser_data& _data, std::string_view _env_var, Tp&& _env_val,
{
_data.updated.emplace(_env_var);
auto _prepend = (_mode & UPD_PREPEND) == UPD_PREPEND;
auto _append = (_mode & UPD_APPEND) == UPD_APPEND;
auto _weak_upd = (_mode & UPD_WEAK) == UPD_WEAK;
auto _prepend = (_mode & UPD_PREPEND) != 0;
auto _append = (_mode & UPD_APPEND) != 0;
auto _weak_upd = (_mode & UPD_WEAK) != 0;
// if both flags are set, prefer append
if(_prepend && _append)
{
_prepend = false;
}
auto _key = join("", _env_var, "=");
for(auto& itr : _data.current)
@@ -126,11 +132,11 @@ update_env(parser_data& _data, std::string_view _env_var, Tp&& _env_val,
free(itr);
if(_prepend)
itr =
strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
.c_str());
else
itr =
strdup(join('=', _env_var, join(_join_delim, _env_val, _val))
strdup(join('=', _env_var, join(_join_delim, _val, _env_val))
.c_str());
}
}
projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake
+3 -5
View File
@@ -15,12 +15,10 @@ else()
set(_rocm_root "/opt/rocm")
endif()
set(_rocm_llvm_lib "${_rocm_root}/lib/llvm/lib")
# Set path to ROCm LLVM library directory containing libomptarget.so
set(_rocm_llvm_lib "${_rocm_root}/llvm/lib")
set(_rocm_ld_env
"LD_PRELOAD=libomptarget.so"
"LD_LIBRARY_PATH=${_rocm_llvm_lib}:$ENV{LD_LIBRARY_PATH}"
)
set(_rocm_ld_env "LD_LIBRARY_PATH=${_rocm_llvm_lib}:$ENV{LD_LIBRARY_PATH}")
if(NOT EXISTS "${_rocm_llvm_lib}/libomptarget.so" AND ROCPROFSYS_USE_ROCM)
message(