AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix reliability when KOKKOS_PROFILE_LIBRARY is set in env (#103)

Browse Source 
* Fix reliability when KOKKOS_PROFILE_LIBRARY is set in env

- in certain situations, an exe using kokkos may be instrumented
- this will cause libomnitrace to be dlopened via libomnitrace-dl
- if KOKKOS_PROFILE_LIBRARY is set to libomnitrace and not libomnitrace-dl, you will end up with different instances of libomnitrace trying to collect data

* Set OMNITRACE_MAX_THREADS=32 in CI

[ROCm/rocprofiler-systems commit: bcdec188eb]
This commit is contained in:
Jonathan R. Madsen
2022-07-24 22:01:53 -05:00
committed by GitHub
parent 3229ba1aba
commit ee788c4178
7 changed files with 179 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocprofiler-systems/.github/workflows/opensuse.yml
Vendored
+1
View File
@@ -58,6 +58,7 @@ jobs:
-DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
-DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10"
-DOMNITRACE_CI_MPI_RUN_AS_ROOT=ON
-DOMNITRACE_MAX_THREADS=32
- name: Build
timeout-minutes: 60
projects/rocprofiler-systems/.github/workflows/ubuntu-bionic.yml
Vendored
+1
View File
@@ -88,6 +88,7 @@ jobs:
-DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
-DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10"
-DLULESH_BUILD_KOKKOS=OFF
-DOMNITRACE_MAX_THREADS=32
- name: Build
timeout-minutes: 60
projects/rocprofiler-systems/.github/workflows/ubuntu-focal.yml
Vendored
+5 -3
View File
@@ -103,6 +103,7 @@ jobs:
-DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${{ matrix.static-libstdcxx }}
-DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
-DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10"
-DOMNITRACE_MAX_THREADS=32
- name: Build
timeout-minutes: 60
@@ -212,8 +213,8 @@ jobs:
git config --global --add safe.directory ${PWD} &&
cmake --version &&
cmake -B build
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=OFF
@@ -223,7 +224,7 @@ jobs:
-DOMNITRACE_USE_MPI=OFF
-DOMNITRACE_USE_MPI_HEADERS=ON
-DOMNITRACE_USE_HIP=ON
-DOMNITRACE_MAX_THREADS=64
-DOMNITRACE_MAX_THREADS=32
-DOMNITRACE_USE_SANITIZER=OFF
-DOMNITRACE_USE_PAPI=OFF
-DOMNITRACE_INSTALL_PERFETTO_TOOLS=ON
@@ -367,6 +368,7 @@ jobs:
-DDYNINST_BUILD_SHARED_LIBS=ON
-DDYNINST_BUILD_STATIC_LIBS=OFF
-DDYNINST_ELFUTILS_DOWNLOAD_VERSION=${{ env.ELFUTILS_DOWNLOAD_VERSION }}
-DOMNITRACE_MAX_THREADS=32
- name: Build
timeout-minutes: 60
projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp
+15 -4
View File
@@ -688,11 +688,22 @@ configure_mode_settings()
if(get_use_kokkosp())
{
auto _force = 0;
auto _current_kokkosp_lib = tim::get_env<std::string>("KOKKOS_PROFILE_LIBRARY");
if(std::regex_search(_current_kokkosp_lib, std::regex{ "libtimemory\\." }))
_force = 1;
tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force);
if(_current_kokkosp_lib.find("libomnitrace-dl.so") == std::string::npos &&
_current_kokkosp_lib.find("libomnitrace.so") == std::string::npos)
{
auto _force = 0;
std::string _message = {};
if(std::regex_search(_current_kokkosp_lib, std::regex{ "libtimemory\\." }))
{
_force = 1;
_message =
JOIN("", " (forced. Previous value: '", _current_kokkosp_lib, "')");
}
OMNITRACE_VERBOSE_F(1, "Setting KOKKOS_PROFILE_LIBRARY=%s%s\n",
"libomnitrace.so", _message.c_str());
tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force);
}
}
// recycle all subsequent thread ids
projects/rocprofiler-systems/source/lib/omnitrace/library/debug.hpp
+75 -14
View File
@@ -246,7 +246,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_FAIL(COND, ...) \
#define OMNITRACE_CONDITIONAL_FAILURE(COND, METHOD, ...) \
if(COND) \
{ \
::omnitrace::debug::flush(); \
@@ -257,10 +257,10 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::print_demangled_backtrace<64>(); \
::std::exit(EXIT_FAILURE); \
METHOD; \
}
#define OMNITRACE_CONDITIONAL_BASIC_FAIL(COND, ...) \
#define OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, METHOD, ...) \
if(COND) \
{ \
::omnitrace::debug::flush(); \
@@ -270,10 +270,10 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::print_demangled_backtrace<64>(); \
::std::exit(EXIT_FAILURE); \
METHOD; \
}
#define OMNITRACE_CONDITIONAL_FAIL_F(COND, ...) \
#define OMNITRACE_CONDITIONAL_FAILURE_F(COND, METHOD, ...) \
if(COND) \
{ \
::omnitrace::debug::flush(); \
@@ -284,10 +284,10 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::print_demangled_backtrace<64>(); \
::std::exit(EXIT_FAILURE); \
METHOD; \
}
#define OMNITRACE_CONDITIONAL_BASIC_FAIL_F(COND, ...) \
#define OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, METHOD, ...) \
if(COND) \
{ \
::omnitrace::debug::flush(); \
@@ -297,24 +297,69 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::print_demangled_backtrace<64>(); \
::std::exit(EXIT_FAILURE); \
METHOD; \
}
#define OMNITRACE_CI_FAIL(COND, ...) \
OMNITRACE_CONDITIONAL_FAIL(::omnitrace::get_is_continuous_integration() && (COND), \
__VA_ARGS__)
#define OMNITRACE_CI_FAILURE(COND, METHOD, ...) \
OMNITRACE_CONDITIONAL_FAILURE( \
::omnitrace::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__)
#define OMNITRACE_CI_BASIC_FAIL(COND, ...) \
OMNITRACE_CONDITIONAL_BASIC_FAIL( \
::omnitrace::get_is_continuous_integration() && (COND), __VA_ARGS__)
#define OMNITRACE_CI_BASIC_FAILURE(COND, METHOD, ...) \
OMNITRACE_CONDITIONAL_BASIC_FAILURE( \
::omnitrace::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__)
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_FAIL(COND, ...) \
OMNITRACE_CONDITIONAL_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
__VA_ARGS__)
#define OMNITRACE_CONDITIONAL_BASIC_FAIL(COND, ...) \
OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
__VA_ARGS__)
#define OMNITRACE_CONDITIONAL_FAIL_F(COND, ...) \
OMNITRACE_CONDITIONAL_FAILURE_F(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
__VA_ARGS__)
#define OMNITRACE_CONDITIONAL_BASIC_FAIL_F(COND, ...) \
OMNITRACE_CONDITIONAL_BASIC_FAILURE_F( \
COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__)
#define OMNITRACE_CI_FAIL(COND, ...) \
OMNITRACE_CI_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__)
#define OMNITRACE_CI_BASIC_FAIL(COND, ...) \
OMNITRACE_CI_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
__VA_ARGS__)
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_ABORT(COND, ...) \
OMNITRACE_CONDITIONAL_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
#define OMNITRACE_CONDITIONAL_BASIC_ABORT(COND, ...) \
OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
#define OMNITRACE_CONDITIONAL_ABORT_F(COND, ...) \
OMNITRACE_CONDITIONAL_FAILURE_F(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
#define OMNITRACE_CONDITIONAL_BASIC_ABORT_F(COND, ...) \
OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, OMNITRACE_ESC(::std::abort()), \
__VA_ARGS__)
#define OMNITRACE_CI_ABORT(COND, ...) \
OMNITRACE_CI_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
#define OMNITRACE_CI_BASIC_ABORT(COND, ...) \
OMNITRACE_CI_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
//--------------------------------------------------------------------------------------//
//
// Debug macros
//
//--------------------------------------------------------------------------------------//
#define OMNITRACE_DEBUG(...) \
OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_debug(), __VA_ARGS__)
@@ -338,6 +383,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
// Verbose macros
//
//--------------------------------------------------------------------------------------//
#define OMNITRACE_VERBOSE(LEVEL, ...) \
OMNITRACE_CONDITIONAL_PRINT( \
::omnitrace::get_debug() || (::omnitrace::get_verbose() >= LEVEL), __VA_ARGS__)
@@ -402,6 +448,21 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
#define OMNITRACE_BASIC_FAIL_F(...) OMNITRACE_CONDITIONAL_BASIC_FAIL_F(true, __VA_ARGS__)
//--------------------------------------------------------------------------------------//
//
// Abort macros
//
//--------------------------------------------------------------------------------------//
#define OMNITRACE_ABORT(...) OMNITRACE_CONDITIONAL_ABORT(true, __VA_ARGS__)
#define OMNITRACE_ABORT_F(...) OMNITRACE_CONDITIONAL_ABORT_F(true, __VA_ARGS__)
#define OMNITRACE_BASIC_ABORT(...) OMNITRACE_CONDITIONAL_BASIC_ABORT(true, __VA_ARGS__)
#define OMNITRACE_BASIC_ABORT_F(...) \
OMNITRACE_CONDITIONAL_BASIC_ABORT_F(true, __VA_ARGS__)
#include <string>
namespace std
projects/rocprofiler-systems/source/lib/omnitrace/library/kokkosp.cpp
+37 -2
View File
@@ -20,19 +20,24 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "library/defines.hpp"
#define TIMEMORY_KOKKOSP_POSTFIX OMNITRACE_PUBLIC_API
#include "library/api.hpp"
#include "library/components/user_region.hpp"
#include "library/config.hpp"
#include "library/debug.hpp"
#include "library/defines.hpp"
#include "library/perfetto.hpp"
#include "library/runtime.hpp"
#include <timemory/api/kokkosp.hpp>
#include <timemory/backends/process.hpp>
#include <timemory/hash/types.hpp>
#include <timemory/utility/procfs/maps.hpp>
#include <cstdlib>
#include <sstream>
#include <string>
namespace kokkosp = tim::kokkosp;
@@ -138,6 +143,36 @@ extern "C"
if(_standalone_initialized || (!omnitrace::config::settings_are_configured() &&
omnitrace::get_state() < omnitrace::State::Active))
{
auto _kokkos_profile_lib =
tim::get_env<std::string>("KOKKOS_PROFILE_LIBRARY");
if(_kokkos_profile_lib.find("libomnitrace.so") != std::string::npos)
{
auto _maps = tim::procfs::read_maps(tim::process::get_id());
auto _libs = std::set<std::string>{};
for(auto& itr : _maps)
{
auto&& _path = itr.pathname;
if(!_path.empty() && _path.at(0) != '[') _libs.emplace(_path);
}
for(const auto& itr : _libs)
{
if(itr.find("libomnitrace-dl.so") != std::string::npos)
{
std::stringstream _libs_str{};
for(const auto& litr : _libs)
_libs_str << " " << litr << "\n";
OMNITRACE_ABORT(
"%s was invoked with libomnitrace.so as the "
"KOKKOS_PROFILE_LIBRARY.\n"
"However, libomnitrace-dl.so has already been loaded by the "
"process.\nTo avoid duplicate collections culminating is an "
"error, please set KOKKOS_PROFILE_LIBRARY=%s.\nLoaded "
"libraries:\n%s",
__FUNCTION__, itr.c_str(), _libs_str.str().c_str());
}
}
}
OMNITRACE_BASIC_VERBOSE_F(0,
"Initializing kokkos omnitrace connector "
"(standalone, sequence %d, version: %llu)...\n",
projects/rocprofiler-systems/tests/CMakeLists.txt
+45 -47
View File
@@ -23,17 +23,15 @@ if(OMNITRACE_BUILD_DYNINST)
)
endif()
set(_base_environment
"OMNITRACE_USE_PERFETTO=ON"
"OMNITRACE_USE_TIMEMORY=ON"
"OMNITRACE_USE_SAMPLING=ON"
"OMNITRACE_USE_PROCESS_SAMPLING=ON"
"OMNITRACE_TIME_OUTPUT=OFF"
"OMP_PROC_BIND=spread"
"OMP_PLACES=threads"
"OMP_NUM_THREADS=2"
set(_test_library_path
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
)
set(_test_openmp_env "OMP_PROC_BIND=spread" "OMP_PLACES=threads" "OMP_NUM_THREADS=2")
set(_base_environment
"OMNITRACE_USE_PERFETTO=ON" "OMNITRACE_USE_TIMEMORY=ON" "OMNITRACE_USE_SAMPLING=ON"
"OMNITRACE_USE_PROCESS_SAMPLING=ON" "OMNITRACE_TIME_OUTPUT=OFF" "${_test_openmp_env}"
"${_test_library_path}")
set(_flat_environment
"OMNITRACE_USE_PERFETTO=ON"
@@ -61,44 +59,22 @@ set(_lock_environment
"OMNITRACE_TIME_OUTPUT=OFF"
"OMNITRACE_FLAT_PROFILE=ON"
"OMNITRACE_TIMELINE_PROFILE=OFF"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
)
"${_test_library_path}")
set(_ompt_environment
"OMNITRACE_USE_PERFETTO=ON"
"OMNITRACE_USE_TIMEMORY=ON"
"OMNITRACE_TIME_OUTPUT=OFF"
"OMNITRACE_USE_OMPT=ON"
"OMNITRACE_CRITICAL_TRACE=OFF"
"OMP_PROC_BIND=spread"
"OMP_PLACES=threads"
"OMP_NUM_THREADS=2"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
)
"OMNITRACE_USE_PERFETTO=ON" "OMNITRACE_USE_TIMEMORY=ON" "OMNITRACE_TIME_OUTPUT=OFF"
"OMNITRACE_USE_OMPT=ON" "OMNITRACE_CRITICAL_TRACE=OFF" "${_test_openmp_env}"
"${_test_library_path}")
set(_perfetto_environment
"OMNITRACE_USE_PERFETTO=ON"
"OMNITRACE_USE_TIMEMORY=OFF"
"OMNITRACE_USE_SAMPLING=ON"
"OMNITRACE_USE_PROCESS_SAMPLING=ON"
"OMNITRACE_TIME_OUTPUT=OFF"
"OMP_PROC_BIND=spread"
"OMP_PLACES=threads"
"OMP_NUM_THREADS=2"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
)
"OMNITRACE_USE_PERFETTO=ON" "OMNITRACE_USE_TIMEMORY=OFF" "OMNITRACE_USE_SAMPLING=ON"
"OMNITRACE_USE_PROCESS_SAMPLING=ON" "OMNITRACE_TIME_OUTPUT=OFF" "${_test_openmp_env}"
"${_test_library_path}")
set(_timemory_environment
"OMNITRACE_USE_PERFETTO=OFF"
"OMNITRACE_USE_TIMEMORY=ON"
"OMNITRACE_USE_SAMPLING=ON"
"OMNITRACE_USE_PROCESS_SAMPLING=ON"
"OMNITRACE_TIME_OUTPUT=OFF"
"OMP_PROC_BIND=spread"
"OMP_PLACES=threads"
"OMP_NUM_THREADS=2"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
)
"OMNITRACE_USE_PERFETTO=OFF" "OMNITRACE_USE_TIMEMORY=ON" "OMNITRACE_USE_SAMPLING=ON"
"OMNITRACE_USE_PROCESS_SAMPLING=ON" "OMNITRACE_TIME_OUTPUT=OFF" "${_test_openmp_env}"
"${_test_library_path}")
set(_test_environment ${_base_environment} "OMNITRACE_CRITICAL_TRACE=OFF")
@@ -111,7 +87,7 @@ set(_python_environment
"OMNITRACE_TREE_OUTPUT=OFF"
"OMNITRACE_USE_PID=OFF"
"OMNITRACE_TIMEMORY_COMPONENTS=trip_count"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
"${_test_library_path}"
"PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python/site-packages")
set(_attach_environment
@@ -124,11 +100,8 @@ set(_attach_environment
"OMNITRACE_USE_KOKKOSP=ON"
"OMNITRACE_TIME_OUTPUT=OFF"
"OMNITRACE_USE_PID=OFF"
"OMP_PROC_BIND=spread"
"OMP_PLACES=threads"
"OMP_NUM_THREADS=2"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
)
"${_test_openmp_env}"
"${_test_library_path}")
# -------------------------------------------------------------------------------------- #
@@ -690,11 +663,36 @@ omnitrace_add_test(
args
-ME
[==[lib(gomp|m-)]==]
LABELS "kokkos;kokkos-profile-library"
RUN_ARGS -i 10 -s 20 -p
ENVIRONMENT
"${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON;KOKKOS_PROFILE_LIBRARY=libomnitrace-dl.so"
)
omnitrace_add_test(
SKIP_RUNTIME SKIP_REWRITE
NAME lulesh-baseline-kokkosp-libomnitrace
TARGET lulesh
MPI ${LULESH_USE_MPI}
NUM_PROCS 8
LABELS "kokkos;kokkos-profile-library"
RUN_ARGS -i 10 -s 20 -p
ENVIRONMENT
"${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON;KOKKOS_PROFILE_LIBRARY=libomnitrace.so"
)
omnitrace_add_test(
SKIP_RUNTIME SKIP_REWRITE
NAME lulesh-baseline-kokkosp-libomnitrace-dl
TARGET lulesh
MPI ${LULESH_USE_MPI}
NUM_PROCS 8
LABELS "kokkos;kokkos-profile-library"
RUN_ARGS -i 10 -s 20 -p
ENVIRONMENT
"${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON;KOKKOS_PROFILE_LIBRARY=libomnitrace-dl.so"
)
omnitrace_add_test(
SKIP_BASELINE
NAME lulesh-kokkosp