AI/rocm-systems
2
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

CI timeout + line-info in releases (#279)

Просмотр исходного кода 
* Update perfetto args.gn.in

- remove enable_perfetto_tools_trace_to_text (unused)

* core timeout implementation

- requires OMNITRACE_CI=ON
- requires OMNITRACE_CI_TIMEOUT=<sec>
- adds pthread_self and std::this_thread::get_id to thread info
- pthread_create_gotcha stores native handles (pthread_self)

* Testing updates

- improve detection of segfault/failures with PASS_REGEX exists
- add OMNITRACE_CI_TIMEOUT env variable to all tests

* Line-info in releases

- e.g. -g1 + more options to minimize size of debug info

* Fix typo in config exit action message

* OMNITRACE_UNLIKELY around debug/verbose messages

* format fixes

* Overflow tests + capability check

* transpose example update

- link to threads library

* roctracer/rocprofiler update

- in ROCm 5.5.0, cannot include rocprofiler.h and roctracer.h in same file due to conflicting enum defs
- Moved HSA tracing setup/shutdown to component::roctracer

* roctracer update

- fix definition of roctracer::setup when disabled

* Update fork example

- detach threads on main PID
- flush io outputs when printing info

* Update overflow tests

- pass regular expressions
- overflow on PERF_COUNT_SW_CPU_CLOCK event

* fork gotcha update

- use getpid() instead of getppid()

* update fork example

- wait on threads calling fork

* timeout update

- wait on timeout thread to launch before proceeding
Этот коммит содержится в:
Jonathan R. Madsen
2023-06-14 11:55:22 -05:00
коммит произвёл GitHub
родитель 9de3a6b0b4
Коммит 3e2fa69a14
36 изменённых файлов: 1234 добавлений и 311 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы
source/lib/core/config.cpp
+1 -1
Просмотреть файл
@@ -1215,7 +1215,7 @@ omnitrace_exit_action(int nsig)
{
tim::signals::block_signals(get_sampling_signals(),
tim::signals::sigmask_scope::process);
OMNITRACE_BASIC_PRINT("Finalizing afer signal %i :: %s\n", nsig,
OMNITRACE_BASIC_PRINT("Finalizing after signal %i :: %s\n", nsig,
signal_settings::str(static_cast<sys_signal>(nsig)).c_str());
auto _handler = get_signal_handler().load();
if(_handler) (*_handler)();
source/lib/core/debug.hpp
+28 -28
Просмотреть файл
@@ -222,8 +222,8 @@ as_hex<void*>(void*, size_t);
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_PRINT_COLOR(COLOR, COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -236,8 +236,8 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_PRINT_COLOR_F(COLOR, COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -259,8 +259,8 @@ as_hex<void*>(void*, size_t);
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_PRINT(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -273,8 +273,8 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_BASIC_PRINT(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -287,8 +287,8 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_PRINT_F(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -302,8 +302,8 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_BASIC_PRINT_F(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -318,8 +318,8 @@ as_hex<void*>(void*, size_t);
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_WARN(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -332,8 +332,8 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_BASIC_WARN(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -346,8 +346,8 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_WARN_F(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -361,8 +361,8 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_BASIC_WARN_F(COND, ...) \
if((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid()) \
if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \
::omnitrace::config::get_debug_pid())) \
{ \
::omnitrace::debug::flush(); \
::omnitrace::debug::lock _debug_lk{}; \
@@ -377,7 +377,7 @@ as_hex<void*>(void*, size_t);
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_THROW_E(COND, TYPE, ...) \
if(COND) \
if(OMNITRACE_UNLIKELY((COND))) \
{ \
char _msg_buffer[OMNITRACE_DEBUG_BUFFER_LEN]; \
snprintf(_msg_buffer, OMNITRACE_DEBUG_BUFFER_LEN, "[omnitrace][%i][%li][%s]%s", \
@@ -391,7 +391,7 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_BASIC_THROW_E(COND, TYPE, ...) \
if(COND) \
if(OMNITRACE_UNLIKELY((COND))) \
{ \
char _msg_buffer[OMNITRACE_DEBUG_BUFFER_LEN]; \
snprintf(_msg_buffer, OMNITRACE_DEBUG_BUFFER_LEN, "[omnitrace][%i][%s]%s", \
@@ -428,7 +428,7 @@ as_hex<void*>(void*, size_t);
//--------------------------------------------------------------------------------------//
#define OMNITRACE_CONDITIONAL_FAILURE(COND, METHOD, ...) \
if(COND) \
if(OMNITRACE_UNLIKELY((COND))) \
{ \
::omnitrace::debug::flush(); \
OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \
@@ -443,7 +443,7 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, METHOD, ...) \
if(COND) \
if(OMNITRACE_UNLIKELY((COND))) \
{ \
::omnitrace::debug::flush(); \
OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \
@@ -458,7 +458,7 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_FAILURE_F(COND, METHOD, ...) \
if(COND) \
if(OMNITRACE_UNLIKELY((COND))) \
{ \
::omnitrace::debug::flush(); \
OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \
@@ -474,7 +474,7 @@ as_hex<void*>(void*, size_t);
}
#define OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, METHOD, ...) \
if(COND) \
if(OMNITRACE_UNLIKELY((COND))) \
{ \
::omnitrace::debug::flush(); \
OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \
@@ -621,7 +621,7 @@ as_hex<void*>(void*, size_t);
#define OMNITRACE_WARNING_OR_CI_THROW(LEVEL, ...) \
{ \
if(::omnitrace::get_is_continuous_integration()) \
if(OMNITRACE_UNLIKELY(::omnitrace::get_is_continuous_integration())) \
{ \
OMNITRACE_CI_THROW(true, __VA_ARGS__); \
} \
@@ -635,7 +635,7 @@ as_hex<void*>(void*, size_t);
#define OMNITRACE_REQUIRE(...) TIMEMORY_REQUIRE(__VA_ARGS__)
#define OMNITRACE_PREFER(COND) \
(COND) ? ::tim::log::base() \
(OMNITRACE_LIKELY(COND)) ? ::tim::log::base() \
: (::omnitrace::get_is_continuous_integration()) ? TIMEMORY_FATAL \
: TIMEMORY_WARNING
source/lib/omnitrace/CMakeLists.txt
+1 -1
Просмотреть файл
@@ -11,7 +11,7 @@ target_sources(
omnitrace-object-library
PRIVATE ${CMAKE_CURRENT_LIST_DIR}/library.cpp ${CMAKE_CURRENT_LIST_DIR}/regions.cpp
${CMAKE_CURRENT_LIST_DIR}/progress.cpp ${CMAKE_CURRENT_LIST_DIR}/api.cpp
${CMAKE_CURRENT_LIST_DIR}/api.hpp)
${CMAKE_CURRENT_LIST_DIR}/timeout.cpp ${CMAKE_CURRENT_LIST_DIR}/api.hpp)
add_subdirectory(library)
source/lib/omnitrace/library.cpp
+19
Просмотреть файл
@@ -34,6 +34,7 @@
#include "core/debug.hpp"
#include "core/defines.hpp"
#include "core/gpu.hpp"
#include "core/locking.hpp"
#include "core/perfetto_fwd.hpp"
#include "core/timemory.hpp"
#include "core/utility.hpp"
@@ -61,22 +62,28 @@
#include "omnitrace/categories.h" // in omnitrace-user
#include <timemory/hash/types.hpp>
#include <timemory/log/logger.hpp>
#include <timemory/manager/manager.hpp>
#include <timemory/mpl/type_traits.hpp>
#include <timemory/operations/types/file_output_message.hpp>
#include <timemory/process/process.hpp>
#include <timemory/process/threading.hpp>
#include <timemory/settings/types.hpp>
#include <timemory/signals/signal_handlers.hpp>
#include <timemory/signals/signal_mask.hpp>
#include <timemory/signals/types.hpp>
#include <timemory/units.hpp>
#include <timemory/utility/backtrace.hpp>
#include <timemory/utility/join.hpp>
#include <timemory/utility/procfs/maps.hpp>
#include <atomic>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <mutex>
#include <pthread.h>
#include <stdexcept>
#include <string_view>
#include <utility>
@@ -85,6 +92,15 @@ using namespace omnitrace;
//======================================================================================//
namespace omnitrace
{
namespace timeout
{
void
setup() OMNITRACE_INTERNAL_API;
}
} // namespace omnitrace
namespace
{
auto _timemory_manager = tim::manager::instance();
@@ -152,6 +168,8 @@ ensure_finalization(bool _static_init = false)
if(common::get_env("OMNITRACE_MONOCHROME", false)) tim::log::monochrome() = true;
timeout::setup();
(void) tim::manager::instance();
(void) tim::settings::shared_instance();
@@ -1001,6 +1019,7 @@ omnitrace_finalize_hidden(void)
auto _cfg = settings::compose_filename_config{};
_cfg.use_suffix = config::get_use_pid();
_cfg.suffix = settings::default_process_suffix();
_timemory_manager->write_metadata(settings::get_global_output_prefix(),
"omnitrace", _cfg);
}
source/lib/omnitrace/library/components/fork_gotcha.cpp
+1 -1
Просмотреть файл
@@ -155,7 +155,7 @@ fork_gotcha::operator()(const gotcha_data_t&, pid_t (*_real_fork)()) const
if(_pid != 0)
{
OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getppid(),
OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getpid(),
_pid);
postfork_parent();
source/lib/omnitrace/library/components/pthread_create_gotcha.cpp
+17
Просмотреть файл
@@ -138,6 +138,8 @@ stop_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args)
tim::consume_parameters(_args...);
}
}
std::set<pthread_create_gotcha::native_handle_t> native_handles = {};
} // namespace
//--------------------------------------------------------------------------------------//
@@ -291,12 +293,20 @@ pthread_create_gotcha::wrapper::wrap(void* _arg)
{
if(_arg == nullptr) return nullptr;
auto _self = pthread_self();
// convert the argument
wrapper* _wrapper = static_cast<wrapper*>(_arg);
// store the handle
native_handles.emplace(_self);
// execute the original function
void* _ret = (*_wrapper)();
// remove the handle
if(::pthread_equal(_self, pthread_self()) == 0) native_handles.erase(_self);
// eliminate memory leak
if(_ret != _arg) delete _wrapper;
@@ -378,6 +388,13 @@ pthread_create_gotcha::set_data(wrappee_t _v)
m_wrappee = _v;
}
std::set<pthread_create_gotcha::native_handle_t>
pthread_create_gotcha::get_native_handles()
{
auto _v = native_handles;
return _v;
}
// pthread_create
int
pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
source/lib/omnitrace/library/components/pthread_create_gotcha.hpp
+10 -3
Просмотреть файл
@@ -32,15 +32,18 @@
namespace omnitrace
{
struct pthread_gotcha;
namespace component
{
struct pthread_create_gotcha : tim::component::base<pthread_create_gotcha, void>
{
static constexpr size_t gotcha_capacity = 1;
using routine_t = void* (*) (void*);
using wrappee_t = int (*)(pthread_t*, const pthread_attr_t*, routine_t, void*);
using promise_t = std::shared_ptr<std::promise<void>>;
using routine_t = void* (*) (void*);
using wrappee_t = int (*)(pthread_t*, const pthread_attr_t*, routine_t, void*);
using promise_t = std::shared_ptr<std::promise<void>>;
using native_handle_t = std::thread::native_handle_type;
struct wrapper_config
{
@@ -81,6 +84,10 @@ struct pthread_create_gotcha : tim::component::base<pthread_create_gotcha, void>
void set_data(wrappee_t);
private:
friend struct ::omnitrace::pthread_gotcha;
static std::set<native_handle_t> get_native_handles();
wrappee_t m_wrappee = &pthread_create;
};
source/lib/omnitrace/library/components/pthread_gotcha.cpp
+6
Просмотреть файл
@@ -114,4 +114,10 @@ pthread_gotcha::stop()
{
get_bundle()->stop();
}
std::set<pthread_gotcha::native_handle_t>
pthread_gotcha::get_native_handles()
{
return ::omnitrace::component::pthread_create_gotcha::get_native_handles();
}
} // namespace omnitrace
source/lib/omnitrace/library/components/pthread_gotcha.hpp
+5
Просмотреть файл
@@ -25,6 +25,7 @@
#include "core/common.hpp"
#include "core/defines.hpp"
#include "core/timemory.hpp"
#include "library/thread_info.hpp"
#include <cstdint>
#include <future>
@@ -33,6 +34,8 @@ namespace omnitrace
{
struct pthread_gotcha : tim::component::base<pthread_gotcha, void>
{
using native_handle_t = std::thread::native_handle_type;
OMNITRACE_DEFAULT_OBJECT(pthread_gotcha)
// string id for component
@@ -44,5 +47,7 @@ struct pthread_gotcha : tim::component::base<pthread_gotcha, void>
static void start();
static void stop();
static std::set<native_handle_t> get_native_handles();
};
} // namespace omnitrace
source/lib/omnitrace/library/components/roctracer.cpp
+107 -2
Просмотреть файл
@@ -31,6 +31,20 @@
#include "library/runtime.hpp"
#include "library/thread_data.hpp"
#include <roctracer.h>
#define HIP_PROF_HIP_API_STRING 1
#include <roctracer_ext.h>
#include <roctracer_hip.h>
#if OMNITRACE_HIP_VERSION < 50300
# include <roctracer_hcc.h>
#endif
#define AMD_INTERNAL_BUILD 1
#include <roctracer_hsa.h>
namespace omnitrace
{
namespace component
@@ -55,7 +69,7 @@ roctracer::preinit()
void
roctracer::start()
{
if(tracker_type::start() == 0) setup();
if(tracker_type::start() == 0) setup(nullptr);
}
void
@@ -111,7 +125,7 @@ roctracer::remove_shutdown(const std::string& _lbl)
}
void
roctracer::setup()
roctracer::setup(void* table, bool on_load_trace)
{
if(!get_use_roctracer()) return;
@@ -173,6 +187,79 @@ roctracer::setup()
roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));
}
if(table != nullptr)
{
OMNITRACE_VERBOSE(1 || on_load_trace, "[OnLoad] setting up HSA...\n");
bool trace_hsa_api = get_trace_hsa_api();
// Enable HSA API callbacks/activity
if(trace_hsa_api)
{
std::vector<std::string> hsa_api_vec =
tim::delimit(get_trace_hsa_api_types());
// initialize HSA tracing
roctracer_set_properties(
static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API), (void*) table);
if(!hsa_api_vec.empty())
{
for(const auto& itr : hsa_api_vec)
{
uint32_t cid = HSA_API_ID_NUMBER;
const char* api = itr.c_str();
OMNITRACE_ROCTRACER_CALL(roctracer_op_code(
static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API), api,
&cid, nullptr));
OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_callback(
static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API), cid,
hsa_api_callback, nullptr));
OMNITRACE_VERBOSE(1 || on_load_trace, " HSA-trace(%s)", api);
}
}
else
{
OMNITRACE_VERBOSE(1 || on_load_trace, " HSA-trace()\n");
OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback(
static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_API),
hsa_api_callback, nullptr));
}
}
bool trace_hsa_activity = get_trace_hsa_activity();
// Enable HSA GPU activity
if(trace_hsa_activity)
{
#if OMNITRACE_HIP_VERSION < 50300
using namespace roctracer;
// initialize HSA tracing
const char* output_prefix = nullptr;
hsa_ops_properties_t ops_properties{
table, reinterpret_cast<activity_async_callback_t>(hsa_activity_callback),
nullptr, output_prefix
};
#elif OMNITRACE_HIP_VERSION < 50301
hsa_ops_properties_t ops_properties;
ops_properties.table = table;
ops_properties.reserved1[0] = reinterpret_cast<void*>(&hsa_activity_callback);
ops_properties.reserved1[1] = nullptr;
ops_properties.reserved1[2] = nullptr;
#else
hsa_ops_properties_t ops_properties{
table, reinterpret_cast<void*>(&hsa_activity_callback), nullptr, nullptr
};
#endif
roctracer_set_properties(
static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_OPS), &ops_properties);
OMNITRACE_VERBOSE(1 || on_load_trace, " HSA-activity-trace()\n");
OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_activity(
static_cast<activity_domain_t>(ACTIVITY_DOMAIN_HSA_OPS), HSA_OP_ID_COPY));
}
}
// callback for HSA
for(auto& itr : roctracer_setup_routines())
itr.second();
@@ -246,6 +333,24 @@ roctracer::shutdown()
roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));
}
if(get_trace_hsa_api())
{
OMNITRACE_VERBOSE_F(
2,
"executing roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HSA_API)...\n");
OMNITRACE_ROCTRACER_CALL(
roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
}
if(get_trace_hsa_api())
{
OMNITRACE_VERBOSE_F(
2, "executing roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, "
"HSA_OP_ID_COPY)...\n");
OMNITRACE_ROCTRACER_CALL(
roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
}
if(roctracer_activity_count() == 0)
{
OMNITRACE_VERBOSE_F(2, "executing roctracer_flush_activity()...\n");
source/lib/omnitrace/library/components/roctracer.hpp
+2 -3
Просмотреть файл
@@ -54,11 +54,10 @@ struct roctracer
OMNITRACE_DEFAULT_OBJECT(roctracer)
static void preinit();
static void global_init() { setup(); }
static void global_finalize() { shutdown(); }
static bool is_setup();
static void setup();
static void setup(void* hsa_api_table, bool on_load_trace = false);
static void shutdown();
static void add_setup(const std::string&, std::function<void()>&&);
static void add_shutdown(const std::string&, std::function<void()>&&);
@@ -75,7 +74,7 @@ struct roctracer
#if !defined(OMNITRACE_USE_ROCTRACER)
inline void
roctracer::setup()
roctracer::setup(void*, bool)
{}
inline void
source/lib/omnitrace/library/rocm.cpp
+7 -119
Просмотреть файл
@@ -45,25 +45,6 @@
#include <mutex>
#include <tuple>
#define HIP_PROF_HIP_API_STRING 1
#include <roctracer_ext.h>
#include <roctracer_hip.h>
#if OMNITRACE_HIP_VERSION < 50300
# include <roctracer_hcc.h>
#endif
#define AMD_INTERNAL_BUILD 1
#include <roctracer_hsa.h>
#if __has_include(<hip/amd_detail/hip_prof_str.h>) || (defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0)
# include <hip/amd_detail/hip_prof_str.h>
# define OMNITRACE_HIP_API_ARGS 1
#else
# define OMNITRACE_HIP_API_ARGS 0
#endif
#if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
# include <rocprofiler.h>
#endif
@@ -186,113 +167,20 @@ extern "C"
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
static auto _setup = [=]() {
try
{
OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
"[OnLoad] setting up HSA...\n");
bool trace_hsa_api = get_trace_hsa_api();
// Enable HSA API callbacks/activity
if(trace_hsa_api)
{
std::vector<std::string> hsa_api_vec =
tim::delimit(get_trace_hsa_api_types());
// initialize HSA tracing
roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
if(!hsa_api_vec.empty())
{
for(const auto& itr : hsa_api_vec)
{
uint32_t cid = HSA_API_ID_NUMBER;
const char* api = itr.c_str();
OMNITRACE_ROCTRACER_CALL(roctracer_op_code(
ACTIVITY_DOMAIN_HSA_API, api, &cid, nullptr));
OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_callback(
ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));
OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
" HSA-trace(%s)", api);
}
}
else
{
OMNITRACE_VERBOSE(1 || rocm::on_load_trace, " HSA-trace()\n");
OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback(
ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
}
}
bool trace_hsa_activity = get_trace_hsa_activity();
// Enable HSA GPU activity
if(trace_hsa_activity)
{
#if OMNITRACE_HIP_VERSION < 50300
using namespace roctracer;
// initialize HSA tracing
const char* output_prefix = nullptr;
hsa_ops_properties_t ops_properties{
table,
reinterpret_cast<activity_async_callback_t>(
hsa_activity_callback),
nullptr, output_prefix
};
#elif OMNITRACE_HIP_VERSION < 50301
hsa_ops_properties_t ops_properties;
ops_properties.table = table;
ops_properties.reserved1[0] =
reinterpret_cast<void*>(&hsa_activity_callback);
ops_properties.reserved1[1] = nullptr;
ops_properties.reserved1[2] = nullptr;
#else
hsa_ops_properties_t ops_properties{
table, reinterpret_cast<void*>(&hsa_activity_callback), nullptr,
nullptr
};
#endif
roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);
OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
" HSA-activity-trace()\n");
OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_activity(
ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
}
} catch(std::exception& _e)
{
OMNITRACE_BASIC_PRINT("Exception was thrown in HSA setup: %s\n",
_e.what());
}
};
static auto _shutdown = []() {
OMNITRACE_DEBUG_F("roctracer_disable_domain_callback\n");
OMNITRACE_ROCTRACER_CALL(
roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
OMNITRACE_DEBUG_F("roctracer_disable_op_activity\n");
OMNITRACE_ROCTRACER_CALL(
roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
};
#if OMNITRACE_HIP_VERSION < 50300
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Computing the roctracer clock skew...\n");
(void) omnitrace::get_clock_skew();
#endif
comp::roctracer::add_setup("hsa", _setup);
comp::roctracer::add_shutdown("hsa", _shutdown);
if(get_use_process_sampling() && get_use_rocm_smi())
{
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Setting rocm_smi state to active...\n");
rocm_smi::set_state(State::Active);
}
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Setting rocm_smi state to active...\n");
rocm_smi::set_state(State::Active);
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Requesting roctracer to setup...\n");
comp::roctracer::setup();
comp::roctracer::setup(static_cast<void*>(table), rocm::on_load_trace);
#if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
bool _force_rocprofiler_init =
source/lib/omnitrace/library/roctracer.cpp
+3 -1
Просмотреть файл
@@ -358,8 +358,10 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
}
void
hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg)
hsa_activity_callback(uint32_t op, const void* vrecord, void* arg)
{
const auto* record = static_cast<const activity_record_t*>(vrecord);
if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
return;
source/lib/omnitrace/library/roctracer.hpp
+1 -3
Просмотреть файл
@@ -28,8 +28,6 @@
#include "library/components/roctracer.hpp"
#include "library/ptl.hpp"
#include <roctracer.h>
#include <iostream>
#include <memory>
@@ -57,7 +55,7 @@ void
hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
void
hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg);
hsa_activity_callback(uint32_t op, const void* record, void* arg);
void
hip_exec_activity_callbacks(int64_t _tid);
source/lib/omnitrace/library/thread_info.cpp
+75 -10
Просмотреть файл
@@ -75,21 +75,31 @@ init_index_data(int64_t _tid, bool _offset = false)
if(!itr)
{
threading::offset_this_id(_offset);
itr = thread_index_data{};
itr = thread_index_data{};
OMNITRACE_CONDITIONAL_THROW(itr->internal_value != _tid,
"Error! thread_info::init_index_data was called for "
"thread %zi on thread %zi\n",
_tid, itr->internal_value);
int _verb = 2;
// if thread created using finalization, bump up the minimum verbosity level
if(get_state() >= State::Finalized && _offset) _verb += 2;
if(!config::settings_are_configured())
{
OMNITRACE_BASIC_VERBOSE_F(
_verb, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n",
itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value);
OMNITRACE_BASIC_VERBOSE_F(_verb,
"Thread %li on PID %i (rank: %i) assigned "
"omnitrace TID %li (internal: %li)\n",
itr->system_value, process::get_id(), dmp::rank(),
itr->sequent_value, itr->internal_value);
}
else
{
OMNITRACE_VERBOSE_F(
_verb, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n",
itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value);
OMNITRACE_VERBOSE_F(_verb,
"Thread %li on PID %i (rank: %i) assigned omnitrace TID "
"%li (internal: %li)\n",
itr->system_value, process::get_id(), dmp::rank(),
itr->sequent_value, itr->internal_value);
}
}
return itr;
@@ -191,6 +201,46 @@ thread_info::get()
return get_info_data(utility::get_thread_index());
}
const std::optional<thread_info>&
thread_info::get(native_handle_t& _tid)
{
return get(native_handle_t{ _tid });
}
const std::optional<thread_info>&
thread_info::get(native_handle_t&& _tid)
{
const auto& _v = get_info_data();
if(_v)
{
for(const auto& itr : *_v)
{
if(itr && itr->index_data &&
pthread_equal(itr->index_data->pthread_value, _tid) == 0)
return itr;
}
}
OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
return unknown_thread;
}
const std::optional<thread_info>&
thread_info::get(std::thread::id _tid)
{
const auto& _v = get_info_data();
if(_v)
{
for(const auto& itr : *_v)
{
if(itr && itr->index_data && itr->index_data->stl_value == _tid) return itr;
}
}
OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
return unknown_thread;
}
const std::optional<thread_info>&
thread_info::get(int64_t _tid, ThreadIdType _type)
{
@@ -203,7 +253,8 @@ thread_info::get(int64_t _tid, ThreadIdType _type)
{
for(const auto& itr : *_v)
{
if(itr && itr->index_data->system_value == _tid) return itr;
if(itr && itr->index_data && itr->index_data->system_value == _tid)
return itr;
}
}
}
@@ -214,10 +265,21 @@ thread_info::get(int64_t _tid, ThreadIdType _type)
{
for(const auto& itr : *_v)
{
if(itr && itr->index_data->sequent_value == _tid) return itr;
if(itr && itr->index_data && itr->index_data->sequent_value == _tid)
return itr;
}
}
}
else if(_type == ThreadIdType::PthreadID)
{
OMNITRACE_THROW("omnitrace does not support thread_info::get(int64_t, "
"ThreadIdType) with ThreadIdType::PthreadID\n");
}
else if(_type == ThreadIdType::StlThreadID)
{
OMNITRACE_THROW("omnitrace does not support thread_info::get(int64_t, "
"ThreadIdType) with ThreadIdType::StlThreadID\n");
}
OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
return unknown_thread;
@@ -302,8 +364,11 @@ thread_info::as_string() const
std::stringstream _ss{};
_ss << std::boolalpha << "is_offset=" << is_offset;
if(index_data)
{
_ss << ", index_data=(" << index_data->internal_value << ", "
<< index_data->system_value << ", " << index_data->sequent_value << ")";
<< index_data->system_value << ", " << index_data->sequent_value << ", "
<< index_data->pthread_value << ", " << index_data->stl_value << ")";
}
if(causal_count) _ss << ", causal count=" << *causal_count;
_ss << ", lifetime=(" << lifetime.first << ":" << lifetime.second << ")";
return _ss.str();
source/lib/omnitrace/library/thread_info.hpp
+33 -16
Просмотреть файл
@@ -24,6 +24,8 @@
#include "core/utility.hpp"
#include <pthread.h>
#include <thread>
#include <timemory/backends/threading.hpp>
#include <cstdint>
@@ -34,36 +36,46 @@
namespace omnitrace
{
// InternalTID: zero-based, process-local thread-ID from atomic increment
// from user-created threads and omnitrace-created threads.
// This value may vary based on threads created by different
// backends, e.g., roctracer will create threads
// InternalTID: zero-based, process-local thread-ID from atomic increment
// from user-created threads and omnitrace-created threads.
// This value may vary based on threads created by different
// backends, e.g., roctracer will create threads
//
// SystemTID: system thread-ID. Should be same value as what is seen
// in debugger, etc.
// SystemTID: system thread-ID. Should be same value as what is seen
// in debugger, etc.
//
// SequentTID: zero-based, process-local thread-ID based on the sequence of
// user-created threads which are created in-between the
// initialization and finalization of omnitrace.
// In theory, omnitrace will never increment this value
// because of a thread explicitly by omnitrace or
// by other of the dependent libraries. Most commonly
// used for indexing into omnitrace's thread-local data.
//
// NativeHandle: value of static_cast<int64_t>(pthread_self())
//
// SequentTID: zero-based, process-local thread-ID based on the sequence of
// user-created threads which are created in-between the
// initialization and finalization of omnitrace.
// In theory, omnitrace will never increment this value
// because of a thread explicitly by omnitrace or
// by other of the dependent libraries. Most commonly
// used for indexing into omnitrace's thread-local data.
enum ThreadIdType : int
{
InternalTID = 0,
SystemTID = 1, // system thread id
SequentTID = 2,
PthreadID = 3,
StlThreadID = 4,
};
struct thread_index_data
{
using stl_tid_t = std::thread::id;
using native_tid_t = pthread_t;
// the lookup value is always incremented for each thread
// the system value is the tid provided by the operating system
// the internal value is the value which the user expects
int64_t internal_value = utility::get_thread_index();
int64_t system_value = tim::threading::get_sys_tid();
int64_t sequent_value = tim::threading::get_id();
int64_t internal_value = utility::get_thread_index();
int64_t system_value = tim::threading::get_sys_tid();
int64_t sequent_value = tim::threading::get_id();
native_tid_t pthread_value = ::pthread_self();
stl_tid_t stl_value = std::this_thread::get_id();
std::string as_string() const;
};
@@ -74,6 +86,7 @@ struct thread_info
{
using index_data_t = std::optional<thread_index_data>;
using lifetime_data_t = std::pair<uint64_t, uint64_t>;
using native_handle_t = std::thread::native_handle_type;
~thread_info() = default;
thread_info(const thread_info&) = delete;
@@ -98,7 +111,11 @@ struct thread_info
static bool exists();
static const std::optional<thread_info>& init(bool _offset = false);
static const std::optional<thread_info>& get();
static const std::optional<thread_info>& get(native_handle_t&);
static const std::optional<thread_info>& get(native_handle_t&&);
static const std::optional<thread_info>& get(std::thread::id);
static const std::optional<thread_info>& get(int64_t _tid, ThreadIdType _type);
// note: get(native_handle_t) overloaded to & and && to prevent implicit conversion
bool is_offset = false;
const int64_t* causal_count = nullptr;
source/lib/omnitrace/timeout.cpp
+231
Просмотреть файл
@@ -0,0 +1,231 @@
// MIT License
//
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "core/categories.hpp"
#include "core/config.hpp"
#include "core/debug.hpp"
#include "core/locking.hpp"
#include "core/state.hpp"
#include "library/components/pthread_gotcha.hpp"
#include "library/runtime.hpp"
#include "library/thread_info.hpp"
#include <timemory/log/color.hpp>
#include <timemory/signals/types.hpp>
#include <timemory/unwind/backtrace.hpp>
#include <chrono>
#include <sstream>
#include <thread>
namespace omnitrace
{
namespace timeout
{
void
setup() OMNITRACE_INTERNAL_API;
namespace
{
namespace unwind = ::tim::unwind;
namespace signals = ::tim::signals;
namespace log = ::tim::log;
constexpr auto timeout_signal = signals::sys_signal::Hangup;
constexpr auto timeout_signal_v = static_cast<int>(timeout_signal);
auto main_thread_native_handle = pthread_self();
bool ci_timeout_active = false;
auto ci_timeout_mutex = locking::atomic_mutex{};
uint64_t ci_timeout_backtrace_global_count = 1;
uint64_t ci_timeout_backtrace_global_done = 0;
thread_local uint64_t ci_timeout_backtrace_local_count = 0;
void
ci_timeout_backtrace(int)
{
if(ci_timeout_backtrace_local_count >= ci_timeout_backtrace_global_count) return;
++ci_timeout_backtrace_local_count;
auto _err = std::stringstream{};
auto _cfg = unwind::detailed_backtrace_config{};
_cfg.proc_pid_maps = false;
_cfg.unwind_lineinfo = false;
_cfg.force_color = !log::monochrome();
unwind::detailed_backtrace<0>(_err, _cfg);
static auto _mutex = locking::atomic_mutex{};
auto _lk = locking::atomic_lock{ _mutex };
OMNITRACE_PRINT("%s\n", _err.str().c_str());
++ci_timeout_backtrace_global_done;
}
void
ensure_ci_timeout_backtrace(double _ci_timeout_seconds,
std::promise<void> _ci_timeout_ready)
{
_ci_timeout_ready.set_value();
thread_info::init(true);
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Disabled);
auto _factor = 3.0;
while(_ci_timeout_seconds <= _factor)
_factor /= 1.25;
uint64_t _ci_timeout_nitr = 0;
int64_t _ci_timeout_nanosec = (_ci_timeout_seconds - _factor) * units::sec;
auto _ci_timeout_total_count =
get_env<uint64_t>("OMNITRACE_CI_TIMEOUT_COUNT", 1, false);
const auto root_pid =
get_env<pid_t>("OMNITRACE_ROOT_PROCESS", process::get_id(), false);
while(get_state() < State::Finalized && _ci_timeout_nitr < _ci_timeout_total_count)
{
// sleep until timeout reached
std::this_thread::sleep_for(std::chrono::nanoseconds{ _ci_timeout_nanosec });
// guard against thread in fork
if(process::get_id() != root_pid)
{
ci_timeout_active = false;
setup();
return;
}
auto _tids = pthread_gotcha::get_native_handles();
int64_t _ci_timeout_pause = (_factor * units::sec) / (3 * (_tids.size() + 1));
auto _kill_thread = [_ci_timeout_pause](auto _handle) {
// execute the pthread_kill and wait until ci_timeout_backtrace increments
// ci_timeout_backtrace_global_done (or 50 iterations pass) to avoid
// the backtraces overlapping output
auto _n = 0;
auto _done_v = ci_timeout_backtrace_global_done;
if(::pthread_kill(_handle, timeout_signal_v) != 0)
{
const auto& _info = thread_info::get(_handle);
if(_info)
{
OMNITRACE_WARNING_F(
0, "pthread_kill(%zu, %i) failed for thread %zi (info: %s)\n",
_handle, timeout_signal_v, _info->index_data->sequent_value,
_info->as_string().c_str());
}
else
{
OMNITRACE_WARNING_F(0,
"pthread_kill(%zu, %i) failed. executing generic "
"kill(%i, %i)...\n",
_handle, timeout_signal_v, process::get_id(),
timeout_signal_v);
}
::kill(process::get_id(), timeout_signal_v);
}
// wait until the signal has been delivered
while(ci_timeout_backtrace_global_done == _done_v && _n++ < 50)
std::this_thread::sleep_for(
std::chrono::nanoseconds{ _ci_timeout_pause });
};
_tids.erase(main_thread_native_handle);
OMNITRACE_WARNING_F(-127,
"timeout after %8.3f seconds... Generating backtraces for "
"%zu threads...\n",
_ci_timeout_seconds, _tids.size() + 1);
for(auto itr : _tids)
_kill_thread(itr);
_kill_thread(main_thread_native_handle);
::omnitrace::debug::flush();
::omnitrace::debug::lock _debug_lk{};
if(++_ci_timeout_nitr >= _ci_timeout_total_count)
{
// use SIGQUIT because it will generate a core dump
::kill(process::get_id(), SIGQUIT);
return;
}
else
{
++ci_timeout_backtrace_global_count;
}
}
OMNITRACE_WARNING_F(0, "timeout thread exiting...\n");
}
} // namespace
void
setup()
{
// make sure there isn't any datarace for ci_timeout_active
auto _lk = locking::atomic_lock{ ci_timeout_mutex };
if(ci_timeout_active) return;
// in CI mode, if OMNITRACE_CI_TIMEOUT or OMNITRACE_CI_TIMEOUT_OVERRIDE is
// set, start a thread that will print out the backtrace for each thread
// before the timeout is hit (i.e. killed by CTest) so we can potentially
// diagnose where the code is stuck
auto _ci = get_env("OMNITRACE_CI", false, false);
if(_ci)
{
// set by CTest
auto _ci_timeout_default = get_env("OMNITRACE_CI_TIMEOUT", -1.0, false);
// allow override by user
auto _ci_timeout_seconds =
get_env("OMNITRACE_CI_TIMEOUT_OVERRIDE", _ci_timeout_default, false);
if(_ci_timeout_seconds > 0.0)
{
// lock served its purpose after setting to true
ci_timeout_active = true;
_lk.unlock();
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
// enable the signal handler for when the timeout is reached
struct sigaction _action = {};
sigemptyset(&_action.sa_mask);
_action.sa_flags = SA_RESTART;
_action.sa_handler = ci_timeout_backtrace;
sigaction(timeout_signal_v, &_action, nullptr);
// start a background thread that handles waiting for the timeout
auto _ci_timeout_ready = std::promise<void>{};
auto _ci_timeout_wait = _ci_timeout_ready.get_future();
std::thread{ ensure_ci_timeout_backtrace, _ci_timeout_seconds,
std::move(_ci_timeout_ready) }
.detach();
_ci_timeout_wait.wait_for(std::chrono::seconds{ 1 });
}
}
}
} // namespace timeout
} // namespace omnitrace