AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Crusher hackathon updates (#164)

Browse Source 
- improved error handling in dyninst
- improved error handling in omnitrace exe
- new logging facility for omnitrace exe
- improved backtraces
- disable concurrent kernels in rocprofiler
- updates `setup-env.sh` and modulefile
  - set `omnitrace_ROOT`
  - set `HSA_TOOLS_LIB` if roctracer or rocprofiler enabled
  - set `ROCP_TOOL_LIB` if rocprofiler enabled
  - closes #163 
- No longer make setting `HSA_ENABLE_INTERRUPT=0` the default 
  - this has performance implications
- this was set to workaround a bug in ROCR which caused an ioctl call in
ROCm to hang when interrupted. But it was only interrupted when realtime
sampling was enabled since the CPU-clock doesn't increment when waiting
  - This bug should be fixed in ROCm 5.3
- omnitrace no longer activates a realtime sampler by default when
sampling, thus this bug is no longer encountered unless the user
explicitly triggers realtime sampling
This commit is contained in:
Jonathan R. Madsen
2022-09-21 13:58:14 -05:00
committed by GitHub
parent 472e96a084
commit 90ff7188f8
34 changed files with 932 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lib/common/setup.hpp
-8
View File
@@ -142,14 +142,6 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
_omnilib = common::path::find_path(_omnilib, _verbose, _search_paths);
_omnilib_dl = common::path::find_path(_omnilib_dl, _verbose, _search_paths);
// This environment variable forces the ROCR-Runtime to use polling to wait
// for signals rather than interrupts. We set this variable to avoid issues with
// rocm/roctracer hanging when interrupted by the sampler
//
// see:
// https://github.com/ROCm-Developer-Tools/roctracer/issues/22#issuecomment-572814465
setenv("HSA_ENABLE_INTERRUPT", "0", 0);
#if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
setenv("HSA_TOOLS_LIB", _omnilib.c_str(), 0);
#endif
source/lib/omnitrace-user/CMakeLists.txt
+5 -2
View File
@@ -14,8 +14,10 @@ add_library(omnitrace-user-library SHARED)
add_library(omnitrace::omnitrace-user-library ALIAS omnitrace-user-library)
target_sources(
omnitrace-user-library PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/user.cpp
${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/user.h)
omnitrace-user-library
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/user.cpp
${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/user.h
${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/types.h)
target_include_directories(
omnitrace-user-library PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
@@ -34,6 +36,7 @@ set_target_properties(
omnitrace_strip_target(omnitrace-user-library)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/user.h
${CMAKE_CURRENT_SOURCE_DIR}/omnitrace/types.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/omnitrace)
install(
source/lib/omnitrace-user/omnitrace/types.h
+88
View File
@@ -0,0 +1,88 @@
// MIT License
//
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#ifndef OMNITRACE_TYPES_H_
#define OMNITRACE_TYPES_H_ 1
#include <stdint.h>
#if defined(__cplusplus)
extern "C"
{
#endif
/// @enum OMNITRACE_USER_ERROR
/// @brief Identifier for errors
///
typedef enum OMNITRACE_USER_ERROR
{
OMNITRACE_USER_SUCCESS = 0, ///< No error
OMNITRACE_USER_ERROR_NO_BINDING, ///< Function pointer was not assigned
OMNITRACE_USER_ERROR_BAD_VALUE, ///< Provided value was invalid
OMNITRACE_USER_ERROR_INVALID_CATEGORY, ///< Invalid user binding category
OMNITRACE_USER_ERROR_INTERNAL, ///< Internal error occurred within libomnitrace
OMNITRACE_USER_ERROR_LAST
} omnitrace_user_error_t;
/// @enum OMNITRACE_USER_BINDINGS
/// @brief Identifier for function pointer categories
/// @code{.cpp}
/// int (*omnitrace_push_region_f)(const char*) = nullptr;
///
/// int custom_push_region(const char* name)
/// {
/// // custom push region prints message before calling internal callback
/// printf("Pushing region %s\n", name);
/// return (*omnitrace_push_region_f)(name);
/// }
///
/// int main(int argc, char** argv)
/// {
/// // get the internal callback to start a user-defined region
/// omnitrace_user_get_callbacks(OMNITRACE_USER_REGION,
/// (void**) &omnitrace_push_region_f,
/// nullptr);
/// // assign the custom callback to start a user-defined region
/// if(omnitrace_push_region_f)
/// omnitrace_user_configure(OMNITRACE_USER_REGION,
/// (void*) &custom_push_region,
/// nullptr);
/// // ...
/// }
///
/// @endcode
typedef enum OMNITRACE_USER_BINDINGS
{
OMNITRACE_USER_START_STOP =
0, ///< Function pointers which control global start/stop
OMNITRACE_USER_START_STOP_THREAD, ///< Function pointers which control per-thread
///< start/stop
OMNITRACE_USER_REGION, ///< Function pointers which generate user-defined regions
OMNITRACE_USER_SAMPLE, ///< Function pointer which generate samples
OMNITRACE_USER_BINDINGS_LAST
} omnitrace_user_bindings_t;
#if defined(__cplusplus)
}
#endif
#endif // OMNITRACE_TYPES_H_
source/lib/omnitrace-user/omnitrace/user.h
+3 -64
View File
@@ -24,83 +24,22 @@
#define OMNITRACE_USER_H_ 1
#if defined(OMNITRACE_USER_SOURCE) && (OMNITRACE_USER_SOURCE > 0)
# if !defined(OMNITRACE_ATTRIBUTE)
# define OMNITRACE_ATTRIBUTE(...) __attribute__((__VA_ARGS__))
# endif
# if !defined(OMNITRACE_VISIBILITY)
# define OMNITRACE_VISIBILITY(MODE) OMNITRACE_ATTRIBUTE(visibility(MODE))
# endif
# if !defined(OMNITRACE_PUBLIC_API)
# define OMNITRACE_PUBLIC_API OMNITRACE_VISIBILITY("default")
# endif
# if !defined(OMNITRACE_HIDDEN_API)
# define OMNITRACE_HIDDEN_API OMNITRACE_VISIBILITY("hidden")
# define OMNITRACE_PUBLIC_API __attribute__((visibility("default")))
# endif
#else
# if !defined(OMNITRACE_PUBLIC_API)
# define OMNITRACE_PUBLIC_API
# endif
# if !defined(OMNITRACE_HIDDEN_API)
# define OMNITRACE_HIDDEN_API
# endif
#endif
#include "omnitrace/types.h"
#if defined(__cplusplus)
extern "C"
{
#endif
/// @enum OMNITRACE_USER_ERROR
/// @brief Identifier for errors
///
enum OMNITRACE_USER_ERROR
{
OMNITRACE_USER_SUCCESS = 0, ///< No error
OMNITRACE_USER_ERROR_NO_BINDING, ///< Function pointer was not assigned
OMNITRACE_USER_ERROR_BAD_FUNCTION_POINTER, ///< Provided function pointer was
///< invalid
OMNITRACE_USER_ERROR_INVALID_CATEGORY, ///< Invalid user binding category
OMNITRACE_USER_ERROR_INTERNAL, ///< Internal error occurred within libomnitrace
OMNITRACE_USER_ERROR_LAST
};
/// @enum OMNITRACE_USER_BINDINGS
/// @brief Identifier for function pointer categories
/// @code{.cpp}
/// int (*omnitrace_push_region_f)(const char*) = nullptr;
///
/// int custom_push_region(const char* name)
/// {
/// // custom push region prints message before calling internal callback
/// printf("Pushing region %s\n", name);
/// return (*omnitrace_push_region_f)(name);
/// }
///
/// int main(int argc, char** argv)
/// {
/// // get the internal callback to start a user-defined region
/// omnitrace_user_get_callbacks(OMNITRACE_USER_REGION,
/// (void**) &omnitrace_push_region_f,
/// nullptr);
/// // assign the custom callback to start a user-defined region
/// if(omnitrace_push_region_f)
/// omnitrace_user_configure(OMNITRACE_USER_REGION,
/// (void*) &custom_push_region,
/// nullptr);
/// // ...
/// }
///
/// @endcode
enum OMNITRACE_USER_BINDINGS
{
OMNITRACE_USER_START_STOP =
0, ///< Function pointers which control global start/stop
OMNITRACE_USER_START_STOP_THREAD, ///< Function pointers which control per-thread
///< start/stop
OMNITRACE_USER_REGION, ///< Function pointers which generate user-defined regions
OMNITRACE_USER_BINDINGS_LAST
};
/// @fn int omnitrace_user_start_trace(void)
/// @return @ref OMNITRACE_USER_ERROR value
/// @brief Enable tracing on this thread and all subsequently created threads
source/lib/omnitrace-user/user.cpp
+2
View File
@@ -25,6 +25,7 @@
#endif
#include "omnitrace/user.h"
#include "omnitrace/types.h"
#include <cstdio>
#include <cstdlib>
@@ -149,6 +150,7 @@ extern "C"
{
case OMNITRACE_USER_SUCCESS: return "Success";
case OMNITRACE_USER_ERROR_NO_BINDING: return "Function pointer not assigned";
case OMNITRACE_USER_ERROR_BAD_VALUE: return "Invalid value was provided";
case OMNITRACE_USER_ERROR_INVALID_CATEGORY:
return "Invalid user binding category";
case OMNITRACE_USER_ERROR_INTERNAL:
source/lib/omnitrace/library.cpp
+3 -3
View File
@@ -69,9 +69,9 @@ ensure_finalization(bool _static_init = false)
{
const auto& _info = thread_info::init();
auto _tid = _info->index_data;
OMNITRACE_CI_THROW(_tid->internal_value != threading::get_id(),
OMNITRACE_CI_THROW(_tid->sequent_value != threading::get_id(),
"Error! internal tid != %li :: %li", threading::get_id(),
_tid->internal_value);
_tid->sequent_value);
OMNITRACE_CI_THROW(_tid->system_value != threading::get_sys_tid(),
"Error! system tid != %li :: %li", threading::get_sys_tid(),
_tid->system_value);
@@ -670,7 +670,7 @@ omnitrace_finalize_hidden(void)
for(size_t i = 0; i < max_supported_threads; ++i)
{
auto& itr = instrumentation_bundles::instances().at(i);
const auto& _info = thread_info::get(i, InternalTID);
const auto& _info = thread_info::get(i, SequentTID);
while(!itr.bundles.empty())
{
int _lvl = 1;
source/lib/omnitrace/library/components/backtrace.cpp
+1 -1
View File
@@ -203,7 +203,7 @@ backtrace::sample(int)
// 4a. funlockfile [common but not explicitly in call-stack]
// 4b. __resume_rt [common but not explicitly in call-stack]
// 4c. killpg [common but not explicitly in call-stack]
m_data = get_unw_backtrace_raw<stack_depth, ignore_depth, with_signal_frame>();
m_data = get_unw_stack<stack_depth, ignore_depth, with_signal_frame>();
}
} // namespace component
} // namespace omnitrace
source/lib/omnitrace/library/components/backtrace_metrics.cpp
+1 -1
View File
@@ -243,7 +243,7 @@ void
backtrace_metrics::fini_perfetto(int64_t _tid)
{
auto _hw_cnt_labels = *get_papi_labels(_tid);
const auto& _thread_info = thread_info::get(_tid, InternalTID);
const auto& _thread_info = thread_info::get(_tid, SequentTID);
OMNITRACE_CI_THROW(!_thread_info, "Error! missing thread info for tid=%li\n", _tid);
if(!_thread_info) return;
source/lib/omnitrace/library/components/backtrace_timestamp.cpp
+1 -1
View File
@@ -38,7 +38,7 @@ backtrace_timestamp::operator<(const backtrace_timestamp& rhs) const
bool
backtrace_timestamp::is_valid() const
{
const auto& _info = thread_info::get(m_tid, InternalTID);
const auto& _info = thread_info::get(m_tid, SequentTID);
return (_info) ? _info->is_valid_time(m_real) : false;
}
source/lib/omnitrace/library/components/pthread_create_gotcha.cpp
+2 -2
View File
@@ -161,7 +161,7 @@ pthread_create_gotcha::wrapper::operator()() const
auto _bundle = std::shared_ptr<bundle_t>{};
auto _signals = std::set<int>{};
auto _coverage = (get_mode() == Mode::Coverage);
// const auto& _parent_info = thread_info::get(m_parent_tid, LookupTID);
// const auto& _parent_info = thread_info::get(m_parent_tid, InternalTID);
auto _dtor = [&]() {
set_thread_state(ThreadState::Internal);
if(_is_sampling)
@@ -191,7 +191,7 @@ pthread_create_gotcha::wrapper::operator()() const
if(_active && !_coverage)
{
const auto& _tid_index = thread_info::init();
_tid = _tid_index->index_data->internal_value;
_tid = _tid_index->index_data->sequent_value;
threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid).c_str());
if(!thread_bundle_data_t::instances().at(_tid))
{
source/lib/omnitrace/library/components/pthread_mutex_gotcha.cpp
+1 -1
View File
@@ -283,7 +283,7 @@ static_data<pthread_mutex_gotcha, pthread_mutex_gotcha_t>::operator()(
using thread_data_t =
omnitrace::thread_data<pthread_mutex_gotcha, std::integral_constant<size_t, N>>;
static thread_local auto& _v =
thread_data_t::instance(omnitrace::construct_on_init{}, _data);
thread_data_t::instance(omnitrace::construct_on_thread{}, _data);
return *_v;
}
} // namespace policy
source/lib/omnitrace/library/cpu_freq.cpp
+2 -2
View File
@@ -165,7 +165,7 @@ post_process()
auto _process_frequencies = [](size_t _idx, size_t _offset) {
using freq_track = perfetto_counter_track<category::cpu_freq>;
const auto& _thread_info = thread_info::get(0, LookupTID);
const auto& _thread_info = thread_info::get(0, InternalTID);
OMNITRACE_CI_THROW(!_thread_info, "Missing thread info for thread 0");
if(!_thread_info) return;
@@ -199,7 +199,7 @@ post_process()
"Page Faults", "User Time", "Kernel Time" },
{ "MB", "MB", "MB", "", "", "sec", "sec" });
const auto& _thread_info = thread_info::get(0, LookupTID);
const auto& _thread_info = thread_info::get(0, InternalTID);
OMNITRACE_CI_THROW(!_thread_info, "Missing thread info for thread 0");
if(!_thread_info) return;
source/lib/omnitrace/library/debug.hpp
+18 -15
View File
@@ -118,20 +118,21 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
# define OMNITRACE_DEBUG_BUFFER_LEN 1024
#endif
#if !defined(OMNITRACE_PROCESS_IDENTIFIER)
#if !defined(OMNITRACE_DEBUG_PROCESS_IDENTIFIER)
# if defined(TIMEMORY_USE_MPI)
# define OMNITRACE_PROCESS_IDENTIFIER static_cast<int>(::tim::dmp::rank())
# define OMNITRACE_DEBUG_PROCESS_IDENTIFIER static_cast<int>(::tim::dmp::rank())
# elif defined(TIMEMORY_USE_MPI_HEADERS)
# define OMNITRACE_PROCESS_IDENTIFIER \
# define OMNITRACE_DEBUG_PROCESS_IDENTIFIER \
(::tim::dmp::is_initialized()) ? static_cast<int>(::tim::dmp::rank()) \
: static_cast<int>(::tim::process::get_id())
# else
# define OMNITRACE_PROCESS_IDENTIFIER static_cast<int>(::tim::process::get_id())
# define OMNITRACE_DEBUG_PROCESS_IDENTIFIER \
static_cast<int>(::tim::process::get_id())
# endif
#endif
#if !defined(OMNITRACE_THREAD_IDENTIFIER)
# define OMNITRACE_THREAD_IDENTIFIER ::tim::threading::get_id()
#if !defined(OMNITRACE_DEBUG_THREAD_IDENTIFIER)
# define OMNITRACE_DEBUG_THREAD_IDENTIFIER ::tim::threading::get_id()
#endif
#if defined(__clang__) || (__GNUC__ < 9)
@@ -174,8 +175,8 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _lk{}; \
OMNITRACE_FPRINTF_STDERR_COLOR(info); \
fprintf(stderr, "[omnitrace][%i][%li]%s", OMNITRACE_PROCESS_IDENTIFIER, \
OMNITRACE_THREAD_IDENTIFIER, \
fprintf(stderr, "[omnitrace][%i][%li]%s", OMNITRACE_DEBUG_PROCESS_IDENTIFIER, \
OMNITRACE_DEBUG_THREAD_IDENTIFIER, \
::omnitrace::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
fprintf(stderr, __VA_ARGS__); \
::omnitrace::debug::flush(); \
@@ -201,8 +202,9 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _lk{}; \
OMNITRACE_FPRINTF_STDERR_COLOR(info); \
fprintf(stderr, "[omnitrace][%i][%li][%s]%s", OMNITRACE_PROCESS_IDENTIFIER, \
OMNITRACE_THREAD_IDENTIFIER, OMNITRACE_FUNCTION, \
fprintf(stderr, "[omnitrace][%i][%li][%s]%s", \
OMNITRACE_DEBUG_PROCESS_IDENTIFIER, OMNITRACE_DEBUG_THREAD_IDENTIFIER, \
OMNITRACE_FUNCTION, \
::omnitrace::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
fprintf(stderr, __VA_ARGS__); \
::omnitrace::debug::flush(); \
@@ -228,7 +230,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
{ \
char _msg_buffer[OMNITRACE_DEBUG_BUFFER_LEN]; \
snprintf(_msg_buffer, OMNITRACE_DEBUG_BUFFER_LEN, "[omnitrace][%i][%li][%s]%s", \
OMNITRACE_PROCESS_IDENTIFIER, OMNITRACE_THREAD_IDENTIFIER, \
OMNITRACE_DEBUG_PROCESS_IDENTIFIER, OMNITRACE_DEBUG_THREAD_IDENTIFIER, \
OMNITRACE_FUNCTION, \
::omnitrace::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
auto len = strlen(_msg_buffer); \
@@ -265,8 +267,8 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
{ \
::omnitrace::debug::flush(); \
OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \
fprintf(stderr, "[omnitrace][%i][%li]%s", OMNITRACE_PROCESS_IDENTIFIER, \
OMNITRACE_THREAD_IDENTIFIER, \
fprintf(stderr, "[omnitrace][%i][%li]%s", OMNITRACE_DEBUG_PROCESS_IDENTIFIER, \
OMNITRACE_DEBUG_THREAD_IDENTIFIER, \
::omnitrace::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
fprintf(stderr, __VA_ARGS__); \
::omnitrace::debug::flush(); \
@@ -296,8 +298,9 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
{ \
::omnitrace::debug::flush(); \
OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \
fprintf(stderr, "[omnitrace][%i][%li][%s]%s", OMNITRACE_PROCESS_IDENTIFIER, \
OMNITRACE_THREAD_IDENTIFIER, OMNITRACE_FUNCTION, \
fprintf(stderr, "[omnitrace][%i][%li][%s]%s", \
OMNITRACE_DEBUG_PROCESS_IDENTIFIER, OMNITRACE_DEBUG_THREAD_IDENTIFIER, \
OMNITRACE_FUNCTION, \
::omnitrace::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
fprintf(stderr, __VA_ARGS__); \
::omnitrace::debug::flush(); \
source/lib/omnitrace/library/rocm.cpp
+1 -1
View File
@@ -142,7 +142,7 @@ extern "C"
settings->timestamp_on = 1;
settings->intercept_mode = 1;
settings->hsa_intercepting = 1;
settings->k_concurrent = 1;
settings->k_concurrent = 0;
settings->obj_dumping = 0;
// settings->code_obj_tracking = 0;
// settings->memcopy_tracking = 0;
source/lib/omnitrace/library/rocm_smi.cpp
+1 -1
View File
@@ -243,7 +243,7 @@ data::post_process(uint32_t _dev_id)
auto& _rocm_smi_v = sampler_instances::instances().at(_dev_id);
auto _rocm_smi = (_rocm_smi_v) ? *_rocm_smi_v : std::deque<rocm_smi::data>{};
const auto& _thread_info = thread_info::get(0, LookupTID);
const auto& _thread_info = thread_info::get(0, InternalTID);
OMNITRACE_CI_THROW(!_thread_info, "Missing thread info for thread 0");
if(!_thread_info) return;
source/lib/omnitrace/library/sampling.cpp
+8 -6
View File
@@ -238,7 +238,7 @@ start_duration_thread()
std::set<int>
configure(bool _setup, int64_t _tid = threading::get_id())
{
const auto& _info = thread_info::get(_tid, InternalTID);
const auto& _info = thread_info::get(_tid, SequentTID);
auto& _sampler = sampling::get_sampler(_tid);
auto& _running = get_sampler_running(_tid);
bool _is_running = (!_running) ? false : *_running;
@@ -270,7 +270,7 @@ configure(bool _setup, int64_t _tid = threading::get_id())
// thus we should not start a sampler for it
if(_tid > 0 && _info && _info->is_offset) return std::set<int>{};
// if the thread state is disabled or completed, return
if(_info && _info->index_data->internal_value == _tid &&
if(_info && _info->index_data->sequent_value == _tid &&
get_thread_state() == ThreadState::Disabled)
return std::set<int>{};
@@ -287,7 +287,8 @@ configure(bool _setup, int64_t _tid = threading::get_id())
if(get_debug_sampling()) _verbose = 2;
OMNITRACE_DEBUG("Configuring sampler for thread %lu...\n", _tid);
sampling::sampler_instances::construct("omnitrace", _tid, _verbose);
sampling::sampler_instances::construct(construct_on_thread{ _tid }, "omnitrace",
_tid, _verbose);
_sampler->set_flags(SA_RESTART);
_sampler->set_verbose(_verbose);
@@ -399,7 +400,8 @@ unique_ptr_t<std::set<int>>&
get_signal_types(int64_t _tid)
{
static auto& _v = signal_type_instances::instances();
signal_type_instances::construct(omnitrace::get_sampling_signals(_tid));
signal_type_instances::construct(construct_on_thread{ _tid },
omnitrace::get_sampling_signals(_tid));
return _v.at(_tid);
}
@@ -486,7 +488,7 @@ post_process()
continue;
}
const auto& _thread_info = thread_info::get(i, InternalTID);
const auto& _thread_info = thread_info::get(i, SequentTID);
OMNITRACE_VERBOSE(3 || get_debug_sampling(),
"Getting sampler data for thread %lu...\n", i);
@@ -578,7 +580,7 @@ post_process_perfetto(int64_t _tid, const bundle_t* _init,
OMNITRACE_VERBOSE(3 || get_debug_sampling(),
"[%li] Post-processing backtraces for perfetto...\n", _tid);
const auto& _thread_info = thread_info::get(_tid, InternalTID);
const auto& _thread_info = thread_info::get(_tid, SequentTID);
OMNITRACE_CI_THROW(!_thread_info, "No valid thread info for tid=%li\n", _tid);
if(!_thread_info) return;
source/lib/omnitrace/library/thread_data.hpp
+63 -45
View File
@@ -115,6 +115,11 @@ private:
using construct_on_init = std::true_type;
struct construct_on_thread
{
int64_t index = threading::get_id();
};
template <typename Tp, typename Tag = void, size_t MaxThreads = max_supported_threads>
struct thread_data
{
@@ -123,14 +128,26 @@ struct thread_data
using construct_on_init = std::true_type;
template <typename... Args>
static void construct(Args&&...);
static void construct(construct_on_thread&&, Args&&...);
static value_type& instance();
static instance_array_t& instances();
template <typename... Args>
static value_type& instance(construct_on_init, Args&&...);
static value_type& instance(construct_on_thread&&, Args&&...);
template <typename... Args>
static instance_array_t& instances(construct_on_init, Args&&...);
template <typename... Args>
static void construct(Args&&... args)
{
construct(construct_on_thread{}, std::forward<Args>(args)...);
}
template <typename... Args>
static value_type& instance(Args&&... args)
{
return instance(construct_on_thread{}, std::forward<Args>(args)...);
}
static constexpr size_t size() { return MaxThreads; }
decltype(auto) begin() { return instances().begin(); }
@@ -143,16 +160,12 @@ struct thread_data
template <typename Tp, typename Tag, size_t MaxThreads>
template <typename... Args>
void
thread_data<Tp, Tag, MaxThreads>::construct(Args&&... _args)
thread_data<Tp, Tag, MaxThreads>::construct(construct_on_thread&& _t, Args&&... _args)
{
// construct outside of lambda to prevent data-race
static auto& _instances = instances();
static thread_local bool _v = [&]() {
_instances.at(threading::get_id()) =
generate<value_type>{}(std::forward<Args>(_args)...);
return true;
}();
(void) _v;
static auto& _instances = instances();
if(!_instances.at(_t.index))
_instances.at(_t.index) = generate<value_type>{}(std::forward<Args>(_args)...);
}
template <typename Tp, typename Tag, size_t MaxThreads>
@@ -173,10 +186,10 @@ thread_data<Tp, Tag, MaxThreads>::instances()
template <typename Tp, typename Tag, size_t MaxThreads>
template <typename... Args>
unique_ptr_t<Tp>&
thread_data<Tp, Tag, MaxThreads>::instance(construct_on_init, Args&&... _args)
thread_data<Tp, Tag, MaxThreads>::instance(construct_on_thread&& _t, Args&&... _args)
{
construct(std::forward<Args>(_args)...);
return instances().at(threading::get_id());
construct(construct_on_thread{ _t }, std::forward<Args>(_args)...);
return instances().at(_t.index);
}
template <typename Tp, typename Tag, size_t MaxThreads>
@@ -202,16 +215,15 @@ thread_data<Tp, Tag, MaxThreads>::instances(construct_on_init, Args&&... _args)
template <typename Tp, typename Tag, size_t MaxThreads>
struct thread_data<std::optional<Tp>, Tag, MaxThreads>
{
using value_type = std::optional<Tp>;
using instance_array_t = std::array<value_type, MaxThreads>;
using construct_on_init = std::true_type;
using value_type = std::optional<Tp>;
using instance_array_t = std::array<value_type, MaxThreads>;
template <typename... Args>
static void construct(Args&&...);
static void construct(construct_on_thread&&, Args&&...);
static value_type& instance();
static instance_array_t& instances();
template <typename... Args>
static value_type& instance(construct_on_init, Args&&...);
static value_type& instance(construct_on_thread&&, Args&&...);
template <typename... Args>
static instance_array_t& instances(construct_on_init, Args&&...);
@@ -227,16 +239,13 @@ struct thread_data<std::optional<Tp>, Tag, MaxThreads>
template <typename Tp, typename Tag, size_t MaxThreads>
template <typename... Args>
void
thread_data<std::optional<Tp>, Tag, MaxThreads>::construct(Args&&... _args)
thread_data<std::optional<Tp>, Tag, MaxThreads>::construct(construct_on_thread&& _t,
Args&&... _args)
{
// construct outside of lambda to prevent data-race
static auto& _instances = instances();
static thread_local bool _v = [&]() {
_instances.at(threading::get_id()) =
generate<value_type>{}(std::forward<Args>(_args)...);
return true;
}();
(void) _v;
static auto& _instances = instances();
if(!_instances.at(_t.index))
_instances.at(_t.index) = generate<value_type>{}(std::forward<Args>(_args)...);
}
template <typename Tp, typename Tag, size_t MaxThreads>
@@ -257,11 +266,11 @@ thread_data<std::optional<Tp>, Tag, MaxThreads>::instances()
template <typename Tp, typename Tag, size_t MaxThreads>
template <typename... Args>
std::optional<Tp>&
thread_data<std::optional<Tp>, Tag, MaxThreads>::instance(construct_on_init,
thread_data<std::optional<Tp>, Tag, MaxThreads>::instance(construct_on_thread&& _t,
Args&&... _args)
{
construct(std::forward<Args>(_args)...);
return instances().at(threading::get_id());
construct(construct_on_thread{ _t }, std::forward<Args>(_args)...);
return instances().at(_t.index);
}
template <typename Tp, typename Tag, size_t MaxThreads>
@@ -291,19 +300,30 @@ using tim::identity_t;
template <typename Tp, typename Tag, size_t MaxThreads>
struct thread_data<identity<Tp>, Tag, MaxThreads>
{
using value_type = Tp;
using instance_array_t = std::array<value_type, MaxThreads>;
using construct_on_init = std::true_type;
using value_type = Tp;
using instance_array_t = std::array<value_type, MaxThreads>;
template <typename... Args>
static void construct(Args&&...);
static void construct(construct_on_thread&&, Args&&...);
static value_type& instance();
static instance_array_t& instances();
template <typename... Args>
static value_type& instance(construct_on_init, Args&&...);
static value_type& instance(construct_on_thread&&, Args&&...);
template <typename... Args>
static instance_array_t& instances(construct_on_init, Args&&...);
template <typename... Args>
static void construct(Args&&... args)
{
construct(construct_on_thread{}, std::forward<Args>(args)...);
}
template <typename... Args>
static value_type& instance(Args&&... args)
{
return instance(construct_on_thread{}, std::forward<Args>(args)...);
}
static constexpr size_t size() { return MaxThreads; }
decltype(auto) begin() { return instances().begin(); }
@@ -316,16 +336,13 @@ struct thread_data<identity<Tp>, Tag, MaxThreads>
template <typename Tp, typename Tag, size_t MaxThreads>
template <typename... Args>
void
thread_data<identity<Tp>, Tag, MaxThreads>::construct(Args&&... _args)
thread_data<identity<Tp>, Tag, MaxThreads>::construct(construct_on_thread&& _t,
Args&&... _args)
{
// construct outside of lambda to prevent data-race
static auto& _instances = instances();
static thread_local bool _v = [&]() {
_instances.at(threading::get_id()) =
generate<value_type>{}(std::forward<Args>(_args)...);
return true;
}();
(void) _v;
static auto& _instances = instances();
if(!_instances.at(_t.index))
_instances.at(_t.index) = generate<value_type>{}(std::forward<Args>(_args)...);
}
template <typename Tp, typename Tag, size_t MaxThreads>
@@ -346,10 +363,11 @@ thread_data<identity<Tp>, Tag, MaxThreads>::instances()
template <typename Tp, typename Tag, size_t MaxThreads>
template <typename... Args>
Tp&
thread_data<identity<Tp>, Tag, MaxThreads>::instance(construct_on_init, Args&&... _args)
thread_data<identity<Tp>, Tag, MaxThreads>::instance(construct_on_thread&& _t,
Args&&... _args)
{
construct(std::forward<Args>(_args)...);
return instances().at(threading::get_id());
construct(construct_on_thread{ _t }, std::forward<Args>(_args)...);
return instances().at(_t.index);
}
template <typename Tp, typename Tag, size_t MaxThreads>
source/lib/omnitrace/library/thread_info.cpp
+8 -8
View File
@@ -58,13 +58,13 @@ init_index_data(bool _offset = false)
{
OMNITRACE_BASIC_VERBOSE_F(
2, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n",
itr->system_value, process::get_id(), dmp::rank(), itr->internal_value);
itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value);
}
else
{
OMNITRACE_VERBOSE_F(
2, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n",
itr->system_value, process::get_id(), dmp::rank(), itr->internal_value);
itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value);
}
}
return itr;
@@ -103,7 +103,7 @@ thread_info::get()
const std::optional<thread_info>&
thread_info::get(int64_t _tid, ThreadIdType _type)
{
if(_type == ThreadIdType::LookupTID)
if(_type == ThreadIdType::InternalTID)
return thread_info_data_t::instances().at(_tid);
else if(_type == ThreadIdType::SystemTID)
{
@@ -113,12 +113,12 @@ thread_info::get(int64_t _tid, ThreadIdType _type)
if(itr && itr->index_data->system_value == _tid) return itr;
}
}
else if(_type == ThreadIdType::InternalTID)
else if(_type == ThreadIdType::SequentTID)
{
const auto& _v = thread_info_data_t::instances();
for(const auto& itr : _v)
{
if(itr && itr->index_data->internal_value == _tid) return itr;
if(itr && itr->index_data->sequent_value == _tid) return itr;
}
}
@@ -149,7 +149,7 @@ thread_info::set_stop(uint64_t _ts)
{
for(auto& itr : thread_info_data_t::instances())
{
if(itr && itr->index_data && itr->index_data->lookup_value > _tid)
if(itr && itr->index_data && itr->index_data->internal_value > _tid)
{
if(itr->lifetime.second > _v->lifetime.second)
itr->lifetime.second = _v->lifetime.second;
@@ -203,8 +203,8 @@ thread_info::as_string() const
std::stringstream _ss{};
_ss << std::boolalpha << "is_offset=" << is_offset;
if(index_data)
_ss << ", index_data=(" << index_data->lookup_value << ", "
<< index_data->system_value << ", " << index_data->internal_value << ")";
_ss << ", index_data=(" << index_data->internal_value << ", "
<< index_data->system_value << ", " << index_data->sequent_value << ")";
_ss << ", lifetime=(" << lifetime.first << ":" << lifetime.second << ")";
return _ss.str();
}
source/lib/omnitrace/library/thread_info.hpp
+20 -5
View File
@@ -34,11 +34,26 @@
namespace omnitrace
{
// InternalTID: zero-based, process-local thread-ID from atomic increment
// from user-created threads and omnitrace-created threads.
// This value may vary based on threads created by different
// backends, e.g., roctracer will create threads
//
// SystemTID: system thread-ID. Should be same value as what is seen
// in debugger, etc.
//
// SequentTID: zero-based, process-local thread-ID based on the sequence of
// user-created threads which are created in-between the
// initialization and finalization of omnitrace.
// In theory, omnitrace will never increment this value
// because of a thread explicitly by omnitrace or
// by other of the dependent libraries. Most commonly
// used for indexing into omnitrace's thread-local data.
enum ThreadIdType : int
{
LookupTID = 0,
SystemTID = 1,
InternalTID = 2,
InternalTID = 0,
SystemTID = 1, // system thread id
SequentTID = 2,
};
struct thread_index_data
@@ -46,9 +61,9 @@ struct thread_index_data
// the lookup value is always incremented for each thread
// the system value is the tid provided by the operating system
// the internal value is the value which the user expects
int64_t lookup_value = utility::get_thread_index();
int64_t internal_value = utility::get_thread_index();
int64_t system_value = tim::threading::get_sys_tid();
int64_t internal_value = tim::threading::get_id();
int64_t sequent_value = tim::threading::get_id();
};
struct thread_info