Fix reliability when KOKKOS_PROFILE_LIBRARY is set in env (#103)
* Fix reliability when KOKKOS_PROFILE_LIBRARY is set in env - in certain situations, an exe using kokkos may be instrumented - this will cause libomnitrace to be dlopened via libomnitrace-dl - if KOKKOS_PROFILE_LIBRARY is set to libomnitrace and not libomnitrace-dl, you will end up with different instances of libomnitrace trying to collect data * Set OMNITRACE_MAX_THREADS=32 in CI
This commit is contained in:
Jonathan R. Madsen
committed by
GitHub
GitHub
parent
b1191b2831
commit
bcdec188eb
@@ -688,11 +688,22 @@ configure_mode_settings()
|
||||
|
||||
if(get_use_kokkosp())
|
||||
{
|
||||
auto _force = 0;
|
||||
auto _current_kokkosp_lib = tim::get_env<std::string>("KOKKOS_PROFILE_LIBRARY");
|
||||
if(std::regex_search(_current_kokkosp_lib, std::regex{ "libtimemory\\." }))
|
||||
_force = 1;
|
||||
tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force);
|
||||
if(_current_kokkosp_lib.find("libomnitrace-dl.so") == std::string::npos &&
|
||||
_current_kokkosp_lib.find("libomnitrace.so") == std::string::npos)
|
||||
{
|
||||
auto _force = 0;
|
||||
std::string _message = {};
|
||||
if(std::regex_search(_current_kokkosp_lib, std::regex{ "libtimemory\\." }))
|
||||
{
|
||||
_force = 1;
|
||||
_message =
|
||||
JOIN("", " (forced. Previous value: '", _current_kokkosp_lib, "')");
|
||||
}
|
||||
OMNITRACE_VERBOSE_F(1, "Setting KOKKOS_PROFILE_LIBRARY=%s%s\n",
|
||||
"libomnitrace.so", _message.c_str());
|
||||
tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force);
|
||||
}
|
||||
}
|
||||
|
||||
// recycle all subsequent thread ids
|
||||
|
||||
@@ -246,7 +246,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_FAIL(COND, ...) \
|
||||
#define OMNITRACE_CONDITIONAL_FAILURE(COND, METHOD, ...) \
|
||||
if(COND) \
|
||||
{ \
|
||||
::omnitrace::debug::flush(); \
|
||||
@@ -257,10 +257,10 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
::omnitrace::set_state(::omnitrace::State::Finalized); \
|
||||
::tim::disable_signal_detection(); \
|
||||
::tim::print_demangled_backtrace<64>(); \
|
||||
::std::exit(EXIT_FAILURE); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_FAIL(COND, ...) \
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, METHOD, ...) \
|
||||
if(COND) \
|
||||
{ \
|
||||
::omnitrace::debug::flush(); \
|
||||
@@ -270,10 +270,10 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
::omnitrace::set_state(::omnitrace::State::Finalized); \
|
||||
::tim::disable_signal_detection(); \
|
||||
::tim::print_demangled_backtrace<64>(); \
|
||||
::std::exit(EXIT_FAILURE); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_FAIL_F(COND, ...) \
|
||||
#define OMNITRACE_CONDITIONAL_FAILURE_F(COND, METHOD, ...) \
|
||||
if(COND) \
|
||||
{ \
|
||||
::omnitrace::debug::flush(); \
|
||||
@@ -284,10 +284,10 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
::omnitrace::set_state(::omnitrace::State::Finalized); \
|
||||
::tim::disable_signal_detection(); \
|
||||
::tim::print_demangled_backtrace<64>(); \
|
||||
::std::exit(EXIT_FAILURE); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_FAIL_F(COND, ...) \
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, METHOD, ...) \
|
||||
if(COND) \
|
||||
{ \
|
||||
::omnitrace::debug::flush(); \
|
||||
@@ -297,24 +297,69 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
::omnitrace::set_state(::omnitrace::State::Finalized); \
|
||||
::tim::disable_signal_detection(); \
|
||||
::tim::print_demangled_backtrace<64>(); \
|
||||
::std::exit(EXIT_FAILURE); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define OMNITRACE_CI_FAIL(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_FAIL(::omnitrace::get_is_continuous_integration() && (COND), \
|
||||
__VA_ARGS__)
|
||||
#define OMNITRACE_CI_FAILURE(COND, METHOD, ...) \
|
||||
OMNITRACE_CONDITIONAL_FAILURE( \
|
||||
::omnitrace::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CI_BASIC_FAIL(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_BASIC_FAIL( \
|
||||
::omnitrace::get_is_continuous_integration() && (COND), __VA_ARGS__)
|
||||
#define OMNITRACE_CI_BASIC_FAILURE(COND, METHOD, ...) \
|
||||
OMNITRACE_CONDITIONAL_BASIC_FAILURE( \
|
||||
::omnitrace::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_FAIL(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_FAIL(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_FAIL_F(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_FAILURE_F(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_FAIL_F(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_BASIC_FAILURE_F( \
|
||||
COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CI_FAIL(COND, ...) \
|
||||
OMNITRACE_CI_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CI_BASIC_FAIL(COND, ...) \
|
||||
OMNITRACE_CI_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \
|
||||
__VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_ABORT(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_ABORT(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_ABORT_F(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_FAILURE_F(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CONDITIONAL_BASIC_ABORT_F(COND, ...) \
|
||||
OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, OMNITRACE_ESC(::std::abort()), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CI_ABORT(COND, ...) \
|
||||
OMNITRACE_CI_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_CI_BASIC_ABORT(COND, ...) \
|
||||
OMNITRACE_CI_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Debug macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define OMNITRACE_DEBUG(...) \
|
||||
OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_debug(), __VA_ARGS__)
|
||||
|
||||
@@ -338,6 +383,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
// Verbose macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define OMNITRACE_VERBOSE(LEVEL, ...) \
|
||||
OMNITRACE_CONDITIONAL_PRINT( \
|
||||
::omnitrace::get_debug() || (::omnitrace::get_verbose() >= LEVEL), __VA_ARGS__)
|
||||
@@ -402,6 +448,21 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
|
||||
#define OMNITRACE_BASIC_FAIL_F(...) OMNITRACE_CONDITIONAL_BASIC_FAIL_F(true, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Abort macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define OMNITRACE_ABORT(...) OMNITRACE_CONDITIONAL_ABORT(true, __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_ABORT_F(...) OMNITRACE_CONDITIONAL_ABORT_F(true, __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_BASIC_ABORT(...) OMNITRACE_CONDITIONAL_BASIC_ABORT(true, __VA_ARGS__)
|
||||
|
||||
#define OMNITRACE_BASIC_ABORT_F(...) \
|
||||
OMNITRACE_CONDITIONAL_BASIC_ABORT_F(true, __VA_ARGS__)
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace std
|
||||
|
||||
@@ -20,19 +20,24 @@
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#include "library/defines.hpp"
|
||||
|
||||
#define TIMEMORY_KOKKOSP_POSTFIX OMNITRACE_PUBLIC_API
|
||||
|
||||
#include "library/api.hpp"
|
||||
#include "library/components/user_region.hpp"
|
||||
#include "library/config.hpp"
|
||||
#include "library/debug.hpp"
|
||||
#include "library/defines.hpp"
|
||||
#include "library/perfetto.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
|
||||
#include <timemory/api/kokkosp.hpp>
|
||||
#include <timemory/backends/process.hpp>
|
||||
#include <timemory/hash/types.hpp>
|
||||
#include <timemory/utility/procfs/maps.hpp>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace kokkosp = tim::kokkosp;
|
||||
|
||||
@@ -138,6 +143,36 @@ extern "C"
|
||||
if(_standalone_initialized || (!omnitrace::config::settings_are_configured() &&
|
||||
omnitrace::get_state() < omnitrace::State::Active))
|
||||
{
|
||||
auto _kokkos_profile_lib =
|
||||
tim::get_env<std::string>("KOKKOS_PROFILE_LIBRARY");
|
||||
if(_kokkos_profile_lib.find("libomnitrace.so") != std::string::npos)
|
||||
{
|
||||
auto _maps = tim::procfs::read_maps(tim::process::get_id());
|
||||
auto _libs = std::set<std::string>{};
|
||||
for(auto& itr : _maps)
|
||||
{
|
||||
auto&& _path = itr.pathname;
|
||||
if(!_path.empty() && _path.at(0) != '[') _libs.emplace(_path);
|
||||
}
|
||||
for(const auto& itr : _libs)
|
||||
{
|
||||
if(itr.find("libomnitrace-dl.so") != std::string::npos)
|
||||
{
|
||||
std::stringstream _libs_str{};
|
||||
for(const auto& litr : _libs)
|
||||
_libs_str << " " << litr << "\n";
|
||||
OMNITRACE_ABORT(
|
||||
"%s was invoked with libomnitrace.so as the "
|
||||
"KOKKOS_PROFILE_LIBRARY.\n"
|
||||
"However, libomnitrace-dl.so has already been loaded by the "
|
||||
"process.\nTo avoid duplicate collections culminating is an "
|
||||
"error, please set KOKKOS_PROFILE_LIBRARY=%s.\nLoaded "
|
||||
"libraries:\n%s",
|
||||
__FUNCTION__, itr.c_str(), _libs_str.str().c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
OMNITRACE_BASIC_VERBOSE_F(0,
|
||||
"Initializing kokkos omnitrace connector "
|
||||
"(standalone, sequence %d, version: %llu)...\n",
|
||||
|
||||
Reference in New Issue
Block a user