diff --git a/.cmake-format.yaml b/.cmake-format.yaml index 9cda2c7d1a..7d99a6279e 100644 --- a/.cmake-format.yaml +++ b/.cmake-format.yaml @@ -80,7 +80,6 @@ parse: omnitrace_add_bin_test: flags: - WILL_FAIL - - ADD_INVERSE kwargs: NAME: '*' ARGS: '*' diff --git a/cmake/BuildSettings.cmake b/cmake/BuildSettings.cmake index ca5dc8c826..04d9755a87 100644 --- a/cmake/BuildSettings.cmake +++ b/cmake/BuildSettings.cmake @@ -15,6 +15,7 @@ include(MacroUtilities) omnitrace_add_option( OMNITRACE_BUILD_DEVELOPER "Extra build flags for development like -Werror" ${OMNITRACE_BUILD_CI}) +omnitrace_add_option(OMNITRACE_BUILD_RELEASE "Build with minimal debug line info" OFF) omnitrace_add_option(OMNITRACE_BUILD_EXTRA_OPTIMIZATIONS "Extra optimization flags" OFF) omnitrace_add_option(OMNITRACE_BUILD_LTO "Build with link-time optimization" OFF) omnitrace_add_option(OMNITRACE_USE_COMPILE_TIMING @@ -245,6 +246,15 @@ if(OMNITRACE_BUILD_LINKER) $<$:-fuse-ld=${OMNITRACE_BUILD_LINKER}>) endif() +# ----------------------------------------------------------------------------------------# +# release build flags +# +if(OMNITRACE_BUILD_RELEASE AND NOT OMNITRACE_BUILD_DEBUG) + add_target_flag_if_avail( + omnitrace-compile-options "-g1" "-feliminate-unused-debug-symbols" + "-gno-column-info" "-gno-variable-location-views" "-gline-tables-only") +endif() + # ----------------------------------------------------------------------------------------# # visibility build flags # diff --git a/cmake/Templates/args.gn.in b/cmake/Templates/args.gn.in index 9015bce249..33ecf87bcc 100644 --- a/cmake/Templates/args.gn.in +++ b/cmake/Templates/args.gn.in @@ -15,7 +15,6 @@ enable_perfetto_fuzzers = false # enable_perfetto_stderr_crash_dump = false enable_perfetto_heapprofd = false enable_perfetto_tools = false -enable_perfetto_tools_trace_to_text = false enable_perfetto_trace_processor = false enable_perfetto_trace_processor_httpd = false enable_perfetto_trace_processor_json = false diff --git a/examples/fork/fork.cpp b/examples/fork/fork.cpp index 6744c2d89b..13e7137be1 100644 --- a/examples/fork/fork.cpp +++ b/examples/fork/fork.cpp @@ -9,16 +9,22 @@ #include #include #include +#include void print_info(const char* _name) { + fflush(stdout); + fflush(stderr); printf("[%s] pid = %i, ppid = %i\n", _name, getpid(), getppid()); + fflush(stdout); + fflush(stderr); } int run(const char* _name, int nchildren) { + auto _threads = std::vector{}; for(int i = 0; i < nchildren; ++i) { omnitrace_user_push_region("launch_child"); @@ -35,7 +41,7 @@ run(const char* _name, int nchildren) exit(EXIT_SUCCESS); } }; - std::thread{ _run }.join(); + _threads.emplace_back(_run); omnitrace_user_pop_region("launch_child"); } @@ -70,6 +76,9 @@ run(const char* _name, int nchildren) } } + for(auto& itr : _threads) + itr.join(); + omnitrace_user_pop_region("wait_for_children"); return _status; } diff --git a/examples/transpose/CMakeLists.txt b/examples/transpose/CMakeLists.txt index 48b336f737..25792c923f 100644 --- a/examples/transpose/CMakeLists.txt +++ b/examples/transpose/CMakeLists.txt @@ -43,11 +43,13 @@ endif() option(TRANSPOSE_USE_MPI "Enable MPI support in transpose exe" ${TIMEMORY_USE_MPI}) +find_package(Threads REQUIRED) if(TRANSPOSE_USE_MPI) find_package(MPI REQUIRED) endif() add_executable(transpose transpose.cpp) +target_link_libraries(transpose PRIVATE Threads::Threads) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT CMAKE_CXX_COMPILER_IS_HIPCC diff --git a/external/timemory b/external/timemory index 58536c55d7..d1412416d0 160000 --- a/external/timemory +++ b/external/timemory @@ -1 +1 @@ -Subproject commit 58536c55d73904401059a839f9355a2150ba170a +Subproject commit d1412416d0bdebbb539bc0e0578bcde87f8c8b82 diff --git a/scripts/build-release.sh b/scripts/build-release.sh index 5f789e76b1..39375a8500 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -197,7 +197,7 @@ if [ ${NJOBS} -gt ${NPROC} ]; then NJOBS=${NPROC}; fi CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCPACK_GENERATOR=STGZ" OMNITRACE_GENERAL_ARGS="-DOMNITRACE_CPACK_SYSTEM_NAME=${DISTRO} -DOMNITRACE_ROCM_VERSION=${ROCM_VERSION} -DOMNITRACE_MAX_THREADS=${MAX_THREADS} -DOMNITRACE_STRIP_LIBRARIES=${STRIP} -DOMNITRACE_INSTALL_PERFETTO_TOOLS=${PERFETTO_TOOLS}" -OMNITRACE_BUILD_ARGS="-DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_BUILD_EXAMPLES=OFF -DOMNITRACE_BUILD_PAPI=ON -DOMNITRACE_BUILD_LTO=${LTO} -DOMNITRACE_BUILD_HIDDEN_VISIBILITY=${HIDDEN_VIZ} -DOMNITRACE_BUILD_STATIC_LIBGCC=${LIBGCC} -DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${LIBSTDCXX}" +OMNITRACE_BUILD_ARGS="-DOMNITRACE_BUILD_TESTING=OFF -DOMNITRACE_BUILD_EXAMPLES=OFF -DOMNITRACE_BUILD_PAPI=ON -DOMNITRACE_BUILD_LTO=${LTO} -DOMNITRACE_BUILD_HIDDEN_VISIBILITY=${HIDDEN_VIZ} -DOMNITRACE_BUILD_STATIC_LIBGCC=${LIBGCC} -DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${LIBSTDCXX} -DOMNITRACE_BUILD_RELEASE=ON" OMNITRACE_USE_ARGS="-DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_OMPT=ON -DOMNITRACE_USE_PAPI=ON" TIMEMORY_ARGS="-DTIMEMORY_USE_LIBUNWIND=ON -DTIMEMORY_BUILD_LIBUNWIND=ON -DTIMEMORY_BUILD_PORTABLE=ON" DYNINST_ARGS="-DOMNITRACE_BUILD_DYNINST=ON -DDYNINST_USE_OpenMP=ON $(echo -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON) -DDYNINST_BOOST_DOWNLOAD_VERSION=${BOOST_VERSION}" diff --git a/source/bin/tests/CMakeLists.txt b/source/bin/tests/CMakeLists.txt index 60a9e382fd..d390182738 100644 --- a/source/bin/tests/CMakeLists.txt +++ b/source/bin/tests/CMakeLists.txt @@ -1,8 +1,12 @@ +set(OMNITRACE_ABORT_FAIL_REGEX + "### ERROR ###|address of faulting memory reference|exiting with non-zero exit code|terminate called after throwing an instance|calling abort.. in |Exit code: [1-9]" + CACHE INTERNAL "Regex to catch abnormal exits when a PASS_REGULAR_EXPRESSION is set") + # adds a ctest for executable function(OMNITRACE_ADD_BIN_TEST) cmake_parse_arguments( TEST - "ADD_INVERSE" # options + "" # options "NAME;TARGET;TIMEOUT;WORKING_DIRECTORY" # single value args "ARGS;ENVIRONMENT;LABELS;PROPERTIES;PASS_REGEX;FAIL_REGEX;SKIP_REGEX;DEPENDS;COMMAND" # multiple # value args @@ -33,9 +37,14 @@ function(OMNITRACE_ADD_BIN_TEST) endif() # common - list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON" "OMNITRACE_CONFIG_FILE=" - "OMNITRACE_OUTPUT_PATH=${PROJECT_BINARY_DIR}/omnitrace-tests-output" - "TWD=${TEST_WORKING_DIRECTORY}") + list( + APPEND + TEST_ENVIRONMENT + "OMNITRACE_CI=ON" + "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}" + "OMNITRACE_CONFIG_FILE=" + "OMNITRACE_OUTPUT_PATH=${PROJECT_BINARY_DIR}/omnitrace-tests-output" + "TWD=${TEST_WORKING_DIRECTORY}") # copy for inverse set(TEST_ENVIRONMENT_INV "${TEST_ENVIRONMENT}") @@ -43,6 +52,22 @@ function(OMNITRACE_ADD_BIN_TEST) list(APPEND TEST_ENVIRONMENT "OMNITRACE_OUTPUT_PREFIX=${TEST_NAME}/") list(APPEND TEST_ENVIRONMENT_INV "OMNITRACE_OUTPUT_PREFIX=${TEST_NAME}-inverse/") + if(NOT "${TEST_PASS_REGEX}" STREQUAL "" + AND NOT "${TEST_FAIL_REGEX}" STREQUAL "" + AND NOT "${TEST_FAIL_REGEX}" MATCHES "\\|OMNITRACE_ABORT_FAIL_REGEX") + omnitrace_message( + FATAL_ERROR + "${TEST_NAME} has set pass and fail regexes but fail regex does not include '|OMNITRACE_ABORT_FAIL_REGEX'" + ) + endif() + + if("${TEST_FAIL_REGEX}" STREQUAL "") + set(TEST_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})") + else() + string(REPLACE "|OMNITRACE_ABORT_FAIL_REGEX" "|${OMNITRACE_ABORT_FAIL_REGEX}" + TEST_FAIL_REGEX "${TEST_FAIL_REGEX}") + endif() + if(TEST_COMMAND) add_test( NAME ${TEST_NAME} @@ -66,32 +91,6 @@ function(OMNITRACE_ADD_BIN_TEST) SKIP_REGULAR_EXPRESSION "${TEST_SKIP_REGEX}" ${TEST_PROPERTIES}) - if(TEST_ADD_INVERSE) - add_test( - NAME ${TEST_NAME}-inverse - COMMAND ${TEST_COMMAND} ${TEST_ARGS} - WORKING_DIRECTORY ${TEST_WORKING_DIRECTORY}) - - set_tests_properties( - ${TEST_NAME}-inverse - PROPERTIES ENVIRONMENT - "${TEST_ENVIRONMENT_INV}" - TIMEOUT - ${TEST_TIMEOUT} - DEPENDS - "${TEST_DEPENDS}" - LABELS - "omnitrace-bin;${TEST_LABELS}" - PASS_REGULAR_EXPRESSION - "${TEST_FAIL_REGEX}" - FAIL_REGULAR_EXPRESSION - "${TEST_PASS_REGEX}" - SKIP_REGULAR_EXPRESSION - "${TEST_SKIP_REGEX}" - WILL_FAIL - ON - ${TEST_PROPERTIES}) - endif() elseif(TARGET ${TEST_TARGET}) add_test( NAME ${TEST_NAME} @@ -115,32 +114,6 @@ function(OMNITRACE_ADD_BIN_TEST) SKIP_REGULAR_EXPRESSION "${TEST_SKIP_REGEX}" ${TEST_PROPERTIES}) - if(TEST_ADD_INVERSE) - add_test( - NAME ${TEST_NAME}-inverse - COMMAND $ ${TEST_ARGS} - WORKING_DIRECTORY ${TEST_WORKING_DIRECTORY}) - - set_tests_properties( - ${TEST_NAME}-inverse - PROPERTIES ENVIRONMENT - "${TEST_ENVIRONMENT_INV}" - TIMEOUT - ${TEST_TIMEOUT} - DEPENDS - "${TEST_DEPENDS}" - LABELS - "omnitrace-bin;${TEST_LABELS}" - PASS_REGULAR_EXPRESSION - "${TEST_FAIL_REGEX}" - FAIL_REGULAR_EXPRESSION - "${TEST_PASS_REGEX}" - SKIP_REGULAR_EXPRESSION - "${TEST_SKIP_REGEX}" - WILL_FAIL - ON - ${TEST_PROPERTIES}) - endif() elseif(OMNITRACE_BUILD_TESTING) message(FATAL_ERROR "Error! ${TEST_TARGET} does not exist") endif() @@ -189,10 +162,9 @@ omnitrace_add_bin_test( TIMEOUT 60 PASS_REGEX ".*available.json.*available.txt.*available.xml.*excluded.json.*excluded.txt.*excluded.xml.*instrumented.json.*instrumented.txt.*instrumented.xml.*overlapping.json.*overlapping.txt.*overlapping.xml.*" - FAIL_REGEX "No such file or directory|not found") + FAIL_REGEX "No such file or directory|not found|OMNITRACE_ABORT_FAIL_REGEX") omnitrace_add_bin_test( - ADD_INVERSE NAME omnitrace-instrument-simulate-lib TARGET omnitrace-instrument ARGS --print-available functions -v 2 -- $ @@ -243,7 +215,7 @@ omnitrace_add_bin_test( LABELS "log" TIMEOUT 60 PASS_REGEX "user.log" - FAIL_REGEX "No such file or directory|not found") + FAIL_REGEX "No such file or directory|not found|OMNITRACE_ABORT_FAIL_REGEX") omnitrace_add_bin_test( NAME omnitrace-avail-help @@ -268,7 +240,7 @@ omnitrace_add_bin_test( ARGS --all --expand-keys LABELS "omnitrace-avail" TIMEOUT 45 - FAIL_REGEX "%[a-zA-Z_]%") + FAIL_REGEX "%[a-zA-Z_]%|OMNITRACE_ABORT_FAIL_REGEX") omnitrace_add_bin_test( NAME omnitrace-avail-all-only-available-alphabetical @@ -309,7 +281,7 @@ omnitrace_add_bin_test( TIMEOUT 45 PASS_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE|OUTPUT_PREFIX)" FAIL_REGEX - "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)" + "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)|OMNITRACE_ABORT_FAIL_REGEX" ) omnitrace_add_bin_test( @@ -320,7 +292,7 @@ omnitrace_add_bin_test( TIMEOUT 45 PASS_REGEX "OMNITRACE_(ADD_SECONDARY|SCIENTIFIC|PRECISION|MEMORY_PRECISION|TIMING_PRECISION)" - FAIL_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE)") + FAIL_REGEX "OMNITRACE_(SETTINGS_DESC|OUTPUT_FILE)|OMNITRACE_ABORT_FAIL_REGEX") omnitrace_add_bin_test( NAME omnitrace-avail-regex-negation @@ -341,7 +313,7 @@ omnitrace_add_bin_test( TIMEOUT 45 PASS_REGEX "ENVIRONMENT VARIABLE,[ \n]+OMNITRACE_THREAD_POOL_SIZE,[ \n]+OMNITRACE_USE_PID,[ \n]+" - FAIL_REGEX "OMNITRACE_USE_PERFETTO") + FAIL_REGEX "OMNITRACE_USE_PERFETTO|OMNITRACE_ABORT_FAIL_REGEX") string(REPLACE "+" "\\\+" _AVAIL_CFG_PATH "${PROJECT_BINARY_DIR}/omnitrace-tests-output/omnitrace-avail/omnitrace-") diff --git a/source/lib/core/config.cpp b/source/lib/core/config.cpp index 5e9d20ee24..6bc637066a 100644 --- a/source/lib/core/config.cpp +++ b/source/lib/core/config.cpp @@ -1215,7 +1215,7 @@ omnitrace_exit_action(int nsig) { tim::signals::block_signals(get_sampling_signals(), tim::signals::sigmask_scope::process); - OMNITRACE_BASIC_PRINT("Finalizing afer signal %i :: %s\n", nsig, + OMNITRACE_BASIC_PRINT("Finalizing after signal %i :: %s\n", nsig, signal_settings::str(static_cast(nsig)).c_str()); auto _handler = get_signal_handler().load(); if(_handler) (*_handler)(); diff --git a/source/lib/core/debug.hpp b/source/lib/core/debug.hpp index f6f78b33df..a97b922001 100644 --- a/source/lib/core/debug.hpp +++ b/source/lib/core/debug.hpp @@ -222,8 +222,8 @@ as_hex(void*, size_t); //--------------------------------------------------------------------------------------// #define OMNITRACE_CONDITIONAL_PRINT_COLOR(COLOR, COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -236,8 +236,8 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_PRINT_COLOR_F(COLOR, COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -259,8 +259,8 @@ as_hex(void*, size_t); //--------------------------------------------------------------------------------------// #define OMNITRACE_CONDITIONAL_PRINT(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -273,8 +273,8 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_BASIC_PRINT(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -287,8 +287,8 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_PRINT_F(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -302,8 +302,8 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_BASIC_PRINT_F(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -318,8 +318,8 @@ as_hex(void*, size_t); //--------------------------------------------------------------------------------------// #define OMNITRACE_CONDITIONAL_WARN(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -332,8 +332,8 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_BASIC_WARN(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -346,8 +346,8 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_WARN_F(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -361,8 +361,8 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_BASIC_WARN_F(COND, ...) \ - if((COND) && ::omnitrace::config::get_debug_tid() && \ - ::omnitrace::config::get_debug_pid()) \ + if(OMNITRACE_UNLIKELY((COND) && ::omnitrace::config::get_debug_tid() && \ + ::omnitrace::config::get_debug_pid())) \ { \ ::omnitrace::debug::flush(); \ ::omnitrace::debug::lock _debug_lk{}; \ @@ -377,7 +377,7 @@ as_hex(void*, size_t); //--------------------------------------------------------------------------------------// #define OMNITRACE_CONDITIONAL_THROW_E(COND, TYPE, ...) \ - if(COND) \ + if(OMNITRACE_UNLIKELY((COND))) \ { \ char _msg_buffer[OMNITRACE_DEBUG_BUFFER_LEN]; \ snprintf(_msg_buffer, OMNITRACE_DEBUG_BUFFER_LEN, "[omnitrace][%i][%li][%s]%s", \ @@ -391,7 +391,7 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_BASIC_THROW_E(COND, TYPE, ...) \ - if(COND) \ + if(OMNITRACE_UNLIKELY((COND))) \ { \ char _msg_buffer[OMNITRACE_DEBUG_BUFFER_LEN]; \ snprintf(_msg_buffer, OMNITRACE_DEBUG_BUFFER_LEN, "[omnitrace][%i][%s]%s", \ @@ -428,7 +428,7 @@ as_hex(void*, size_t); //--------------------------------------------------------------------------------------// #define OMNITRACE_CONDITIONAL_FAILURE(COND, METHOD, ...) \ - if(COND) \ + if(OMNITRACE_UNLIKELY((COND))) \ { \ ::omnitrace::debug::flush(); \ OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \ @@ -443,7 +443,7 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, METHOD, ...) \ - if(COND) \ + if(OMNITRACE_UNLIKELY((COND))) \ { \ ::omnitrace::debug::flush(); \ OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \ @@ -458,7 +458,7 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_FAILURE_F(COND, METHOD, ...) \ - if(COND) \ + if(OMNITRACE_UNLIKELY((COND))) \ { \ ::omnitrace::debug::flush(); \ OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \ @@ -474,7 +474,7 @@ as_hex(void*, size_t); } #define OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, METHOD, ...) \ - if(COND) \ + if(OMNITRACE_UNLIKELY((COND))) \ { \ ::omnitrace::debug::flush(); \ OMNITRACE_FPRINTF_STDERR_COLOR(fatal); \ @@ -621,7 +621,7 @@ as_hex(void*, size_t); #define OMNITRACE_WARNING_OR_CI_THROW(LEVEL, ...) \ { \ - if(::omnitrace::get_is_continuous_integration()) \ + if(OMNITRACE_UNLIKELY(::omnitrace::get_is_continuous_integration())) \ { \ OMNITRACE_CI_THROW(true, __VA_ARGS__); \ } \ @@ -635,7 +635,7 @@ as_hex(void*, size_t); #define OMNITRACE_REQUIRE(...) TIMEMORY_REQUIRE(__VA_ARGS__) #define OMNITRACE_PREFER(COND) \ - (COND) ? ::tim::log::base() \ + (OMNITRACE_LIKELY(COND)) ? ::tim::log::base() \ : (::omnitrace::get_is_continuous_integration()) ? TIMEMORY_FATAL \ : TIMEMORY_WARNING diff --git a/source/lib/omnitrace/CMakeLists.txt b/source/lib/omnitrace/CMakeLists.txt index 5629cd40f0..2107e0a0b3 100644 --- a/source/lib/omnitrace/CMakeLists.txt +++ b/source/lib/omnitrace/CMakeLists.txt @@ -11,7 +11,7 @@ target_sources( omnitrace-object-library PRIVATE ${CMAKE_CURRENT_LIST_DIR}/library.cpp ${CMAKE_CURRENT_LIST_DIR}/regions.cpp ${CMAKE_CURRENT_LIST_DIR}/progress.cpp ${CMAKE_CURRENT_LIST_DIR}/api.cpp - ${CMAKE_CURRENT_LIST_DIR}/api.hpp) + ${CMAKE_CURRENT_LIST_DIR}/timeout.cpp ${CMAKE_CURRENT_LIST_DIR}/api.hpp) add_subdirectory(library) diff --git a/source/lib/omnitrace/library.cpp b/source/lib/omnitrace/library.cpp index 3eeb941590..fa4f8c1637 100644 --- a/source/lib/omnitrace/library.cpp +++ b/source/lib/omnitrace/library.cpp @@ -34,6 +34,7 @@ #include "core/debug.hpp" #include "core/defines.hpp" #include "core/gpu.hpp" +#include "core/locking.hpp" #include "core/perfetto_fwd.hpp" #include "core/timemory.hpp" #include "core/utility.hpp" @@ -61,22 +62,28 @@ #include "omnitrace/categories.h" // in omnitrace-user #include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -85,6 +92,15 @@ using namespace omnitrace; //======================================================================================// +namespace omnitrace +{ +namespace timeout +{ +void +setup() OMNITRACE_INTERNAL_API; +} +} // namespace omnitrace + namespace { auto _timemory_manager = tim::manager::instance(); @@ -152,6 +168,8 @@ ensure_finalization(bool _static_init = false) if(common::get_env("OMNITRACE_MONOCHROME", false)) tim::log::monochrome() = true; + timeout::setup(); + (void) tim::manager::instance(); (void) tim::settings::shared_instance(); @@ -1001,6 +1019,7 @@ omnitrace_finalize_hidden(void) auto _cfg = settings::compose_filename_config{}; _cfg.use_suffix = config::get_use_pid(); + _cfg.suffix = settings::default_process_suffix(); _timemory_manager->write_metadata(settings::get_global_output_prefix(), "omnitrace", _cfg); } diff --git a/source/lib/omnitrace/library/components/fork_gotcha.cpp b/source/lib/omnitrace/library/components/fork_gotcha.cpp index ad2926f62f..cba7b5b31e 100644 --- a/source/lib/omnitrace/library/components/fork_gotcha.cpp +++ b/source/lib/omnitrace/library/components/fork_gotcha.cpp @@ -155,7 +155,7 @@ fork_gotcha::operator()(const gotcha_data_t&, pid_t (*_real_fork)()) const if(_pid != 0) { - OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getppid(), + OMNITRACE_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getpid(), _pid); postfork_parent(); diff --git a/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp b/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp index 2cafee7ba7..4eeb70362b 100644 --- a/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp +++ b/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp @@ -138,6 +138,8 @@ stop_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args) tim::consume_parameters(_args...); } } + +std::set native_handles = {}; } // namespace //--------------------------------------------------------------------------------------// @@ -291,12 +293,20 @@ pthread_create_gotcha::wrapper::wrap(void* _arg) { if(_arg == nullptr) return nullptr; + auto _self = pthread_self(); + // convert the argument wrapper* _wrapper = static_cast(_arg); + // store the handle + native_handles.emplace(_self); + // execute the original function void* _ret = (*_wrapper)(); + // remove the handle + if(::pthread_equal(_self, pthread_self()) == 0) native_handles.erase(_self); + // eliminate memory leak if(_ret != _arg) delete _wrapper; @@ -378,6 +388,13 @@ pthread_create_gotcha::set_data(wrappee_t _v) m_wrappee = _v; } +std::set +pthread_create_gotcha::get_native_handles() +{ + auto _v = native_handles; + return _v; +} + // pthread_create int pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr, diff --git a/source/lib/omnitrace/library/components/pthread_create_gotcha.hpp b/source/lib/omnitrace/library/components/pthread_create_gotcha.hpp index c0cdf74e40..4cda99cbac 100644 --- a/source/lib/omnitrace/library/components/pthread_create_gotcha.hpp +++ b/source/lib/omnitrace/library/components/pthread_create_gotcha.hpp @@ -32,15 +32,18 @@ namespace omnitrace { +struct pthread_gotcha; + namespace component { struct pthread_create_gotcha : tim::component::base { static constexpr size_t gotcha_capacity = 1; - using routine_t = void* (*) (void*); - using wrappee_t = int (*)(pthread_t*, const pthread_attr_t*, routine_t, void*); - using promise_t = std::shared_ptr>; + using routine_t = void* (*) (void*); + using wrappee_t = int (*)(pthread_t*, const pthread_attr_t*, routine_t, void*); + using promise_t = std::shared_ptr>; + using native_handle_t = std::thread::native_handle_type; struct wrapper_config { @@ -81,6 +84,10 @@ struct pthread_create_gotcha : tim::component::base void set_data(wrappee_t); private: + friend struct ::omnitrace::pthread_gotcha; + + static std::set get_native_handles(); + wrappee_t m_wrappee = &pthread_create; }; diff --git a/source/lib/omnitrace/library/components/pthread_gotcha.cpp b/source/lib/omnitrace/library/components/pthread_gotcha.cpp index 72de8334ea..b6380bfea7 100644 --- a/source/lib/omnitrace/library/components/pthread_gotcha.cpp +++ b/source/lib/omnitrace/library/components/pthread_gotcha.cpp @@ -114,4 +114,10 @@ pthread_gotcha::stop() { get_bundle()->stop(); } + +std::set +pthread_gotcha::get_native_handles() +{ + return ::omnitrace::component::pthread_create_gotcha::get_native_handles(); +} } // namespace omnitrace diff --git a/source/lib/omnitrace/library/components/pthread_gotcha.hpp b/source/lib/omnitrace/library/components/pthread_gotcha.hpp index 96e156725a..306a3d2a97 100644 --- a/source/lib/omnitrace/library/components/pthread_gotcha.hpp +++ b/source/lib/omnitrace/library/components/pthread_gotcha.hpp @@ -25,6 +25,7 @@ #include "core/common.hpp" #include "core/defines.hpp" #include "core/timemory.hpp" +#include "library/thread_info.hpp" #include #include @@ -33,6 +34,8 @@ namespace omnitrace { struct pthread_gotcha : tim::component::base { + using native_handle_t = std::thread::native_handle_type; + OMNITRACE_DEFAULT_OBJECT(pthread_gotcha) // string id for component @@ -44,5 +47,7 @@ struct pthread_gotcha : tim::component::base static void start(); static void stop(); + + static std::set get_native_handles(); }; } // namespace omnitrace diff --git a/source/lib/omnitrace/library/components/roctracer.cpp b/source/lib/omnitrace/library/components/roctracer.cpp index d72aebd165..0dd8ff76f6 100644 --- a/source/lib/omnitrace/library/components/roctracer.cpp +++ b/source/lib/omnitrace/library/components/roctracer.cpp @@ -31,6 +31,20 @@ #include "library/runtime.hpp" #include "library/thread_data.hpp" +#include + +#define HIP_PROF_HIP_API_STRING 1 + +#include +#include + +#if OMNITRACE_HIP_VERSION < 50300 +# include +#endif + +#define AMD_INTERNAL_BUILD 1 +#include + namespace omnitrace { namespace component @@ -55,7 +69,7 @@ roctracer::preinit() void roctracer::start() { - if(tracker_type::start() == 0) setup(); + if(tracker_type::start() == 0) setup(nullptr); } void @@ -111,7 +125,7 @@ roctracer::remove_shutdown(const std::string& _lbl) } void -roctracer::setup() +roctracer::setup(void* table, bool on_load_trace) { if(!get_use_roctracer()) return; @@ -173,6 +187,79 @@ roctracer::setup() roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS)); } + if(table != nullptr) + { + OMNITRACE_VERBOSE(1 || on_load_trace, "[OnLoad] setting up HSA...\n"); + + bool trace_hsa_api = get_trace_hsa_api(); + + // Enable HSA API callbacks/activity + if(trace_hsa_api) + { + std::vector hsa_api_vec = + tim::delimit(get_trace_hsa_api_types()); + + // initialize HSA tracing + roctracer_set_properties( + static_cast(ACTIVITY_DOMAIN_HSA_API), (void*) table); + + if(!hsa_api_vec.empty()) + { + for(const auto& itr : hsa_api_vec) + { + uint32_t cid = HSA_API_ID_NUMBER; + const char* api = itr.c_str(); + OMNITRACE_ROCTRACER_CALL(roctracer_op_code( + static_cast(ACTIVITY_DOMAIN_HSA_API), api, + &cid, nullptr)); + OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_callback( + static_cast(ACTIVITY_DOMAIN_HSA_API), cid, + hsa_api_callback, nullptr)); + + OMNITRACE_VERBOSE(1 || on_load_trace, " HSA-trace(%s)", api); + } + } + else + { + OMNITRACE_VERBOSE(1 || on_load_trace, " HSA-trace()\n"); + OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback( + static_cast(ACTIVITY_DOMAIN_HSA_API), + hsa_api_callback, nullptr)); + } + } + + bool trace_hsa_activity = get_trace_hsa_activity(); + // Enable HSA GPU activity + if(trace_hsa_activity) + { +#if OMNITRACE_HIP_VERSION < 50300 + using namespace roctracer; + // initialize HSA tracing + const char* output_prefix = nullptr; + hsa_ops_properties_t ops_properties{ + table, reinterpret_cast(hsa_activity_callback), + nullptr, output_prefix + }; +#elif OMNITRACE_HIP_VERSION < 50301 + hsa_ops_properties_t ops_properties; + ops_properties.table = table; + ops_properties.reserved1[0] = reinterpret_cast(&hsa_activity_callback); + ops_properties.reserved1[1] = nullptr; + ops_properties.reserved1[2] = nullptr; +#else + hsa_ops_properties_t ops_properties{ + table, reinterpret_cast(&hsa_activity_callback), nullptr, nullptr + }; +#endif + roctracer_set_properties( + static_cast(ACTIVITY_DOMAIN_HSA_OPS), &ops_properties); + + OMNITRACE_VERBOSE(1 || on_load_trace, " HSA-activity-trace()\n"); + OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_activity( + static_cast(ACTIVITY_DOMAIN_HSA_OPS), HSA_OP_ID_COPY)); + } + } + // callback for HSA for(auto& itr : roctracer_setup_routines()) itr.second(); @@ -246,6 +333,24 @@ roctracer::shutdown() roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS)); } + if(get_trace_hsa_api()) + { + OMNITRACE_VERBOSE_F( + 2, + "executing roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HSA_API)...\n"); + OMNITRACE_ROCTRACER_CALL( + roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API)); + } + + if(get_trace_hsa_api()) + { + OMNITRACE_VERBOSE_F( + 2, "executing roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, " + "HSA_OP_ID_COPY)...\n"); + OMNITRACE_ROCTRACER_CALL( + roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY)); + } + if(roctracer_activity_count() == 0) { OMNITRACE_VERBOSE_F(2, "executing roctracer_flush_activity()...\n"); diff --git a/source/lib/omnitrace/library/components/roctracer.hpp b/source/lib/omnitrace/library/components/roctracer.hpp index 20cb93134e..d587115040 100644 --- a/source/lib/omnitrace/library/components/roctracer.hpp +++ b/source/lib/omnitrace/library/components/roctracer.hpp @@ -54,11 +54,10 @@ struct roctracer OMNITRACE_DEFAULT_OBJECT(roctracer) static void preinit(); - static void global_init() { setup(); } static void global_finalize() { shutdown(); } static bool is_setup(); - static void setup(); + static void setup(void* hsa_api_table, bool on_load_trace = false); static void shutdown(); static void add_setup(const std::string&, std::function&&); static void add_shutdown(const std::string&, std::function&&); @@ -75,7 +74,7 @@ struct roctracer #if !defined(OMNITRACE_USE_ROCTRACER) inline void -roctracer::setup() +roctracer::setup(void*, bool) {} inline void diff --git a/source/lib/omnitrace/library/rocm.cpp b/source/lib/omnitrace/library/rocm.cpp index 4d9e9a2087..0fac2fb47f 100644 --- a/source/lib/omnitrace/library/rocm.cpp +++ b/source/lib/omnitrace/library/rocm.cpp @@ -45,25 +45,6 @@ #include #include -#define HIP_PROF_HIP_API_STRING 1 - -#include -#include - -#if OMNITRACE_HIP_VERSION < 50300 -# include -#endif - -#define AMD_INTERNAL_BUILD 1 -#include - -#if __has_include() || (defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0) -# include -# define OMNITRACE_HIP_API_ARGS 1 -#else -# define OMNITRACE_HIP_API_ARGS 0 -#endif - #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0 # include #endif @@ -186,113 +167,20 @@ extern "C" OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal); - static auto _setup = [=]() { - try - { - OMNITRACE_VERBOSE(1 || rocm::on_load_trace, - "[OnLoad] setting up HSA...\n"); - - bool trace_hsa_api = get_trace_hsa_api(); - - // Enable HSA API callbacks/activity - if(trace_hsa_api) - { - std::vector hsa_api_vec = - tim::delimit(get_trace_hsa_api_types()); - - // initialize HSA tracing - roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table); - - if(!hsa_api_vec.empty()) - { - for(const auto& itr : hsa_api_vec) - { - uint32_t cid = HSA_API_ID_NUMBER; - const char* api = itr.c_str(); - OMNITRACE_ROCTRACER_CALL(roctracer_op_code( - ACTIVITY_DOMAIN_HSA_API, api, &cid, nullptr)); - OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_callback( - ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr)); - - OMNITRACE_VERBOSE(1 || rocm::on_load_trace, - " HSA-trace(%s)", api); - } - } - else - { - OMNITRACE_VERBOSE(1 || rocm::on_load_trace, " HSA-trace()\n"); - OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback( - ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr)); - } - } - - bool trace_hsa_activity = get_trace_hsa_activity(); - // Enable HSA GPU activity - if(trace_hsa_activity) - { -#if OMNITRACE_HIP_VERSION < 50300 - using namespace roctracer; - // initialize HSA tracing - const char* output_prefix = nullptr; - hsa_ops_properties_t ops_properties{ - table, - reinterpret_cast( - hsa_activity_callback), - nullptr, output_prefix - }; -#elif OMNITRACE_HIP_VERSION < 50301 - hsa_ops_properties_t ops_properties; - ops_properties.table = table; - ops_properties.reserved1[0] = - reinterpret_cast(&hsa_activity_callback); - ops_properties.reserved1[1] = nullptr; - ops_properties.reserved1[2] = nullptr; -#else - hsa_ops_properties_t ops_properties{ - table, reinterpret_cast(&hsa_activity_callback), nullptr, - nullptr - }; -#endif - roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties); - - OMNITRACE_VERBOSE(1 || rocm::on_load_trace, - " HSA-activity-trace()\n"); - OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_activity( - ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY)); - } - } catch(std::exception& _e) - { - OMNITRACE_BASIC_PRINT("Exception was thrown in HSA setup: %s\n", - _e.what()); - } - }; - - static auto _shutdown = []() { - OMNITRACE_DEBUG_F("roctracer_disable_domain_callback\n"); - OMNITRACE_ROCTRACER_CALL( - roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API)); - - OMNITRACE_DEBUG_F("roctracer_disable_op_activity\n"); - OMNITRACE_ROCTRACER_CALL( - roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY)); - }; - #if OMNITRACE_HIP_VERSION < 50300 OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace, "Computing the roctracer clock skew...\n"); (void) omnitrace::get_clock_skew(); #endif - comp::roctracer::add_setup("hsa", _setup); - comp::roctracer::add_shutdown("hsa", _shutdown); + if(get_use_process_sampling() && get_use_rocm_smi()) + { + OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace, + "Setting rocm_smi state to active...\n"); + rocm_smi::set_state(State::Active); + } - OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace, - "Setting rocm_smi state to active...\n"); - rocm_smi::set_state(State::Active); - - OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace, - "Requesting roctracer to setup...\n"); - comp::roctracer::setup(); + comp::roctracer::setup(static_cast(table), rocm::on_load_trace); #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0 bool _force_rocprofiler_init = diff --git a/source/lib/omnitrace/library/roctracer.cpp b/source/lib/omnitrace/library/roctracer.cpp index 38763095cb..eea476aab5 100644 --- a/source/lib/omnitrace/library/roctracer.cpp +++ b/source/lib/omnitrace/library/roctracer.cpp @@ -358,8 +358,10 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* } void -hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg) +hsa_activity_callback(uint32_t op, const void* vrecord, void* arg) { + const auto* record = static_cast(vrecord); + if(get_state() != State::Active || !trait::runtime_enabled::get()) return; diff --git a/source/lib/omnitrace/library/roctracer.hpp b/source/lib/omnitrace/library/roctracer.hpp index 54e56987b2..187a1776c8 100644 --- a/source/lib/omnitrace/library/roctracer.hpp +++ b/source/lib/omnitrace/library/roctracer.hpp @@ -28,8 +28,6 @@ #include "library/components/roctracer.hpp" #include "library/ptl.hpp" -#include - #include #include @@ -57,7 +55,7 @@ void hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg); void -hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg); +hsa_activity_callback(uint32_t op, const void* record, void* arg); void hip_exec_activity_callbacks(int64_t _tid); diff --git a/source/lib/omnitrace/library/thread_info.cpp b/source/lib/omnitrace/library/thread_info.cpp index 6512d94a01..9e553b9b66 100644 --- a/source/lib/omnitrace/library/thread_info.cpp +++ b/source/lib/omnitrace/library/thread_info.cpp @@ -75,21 +75,31 @@ init_index_data(int64_t _tid, bool _offset = false) if(!itr) { threading::offset_this_id(_offset); - itr = thread_index_data{}; + itr = thread_index_data{}; + + OMNITRACE_CONDITIONAL_THROW(itr->internal_value != _tid, + "Error! thread_info::init_index_data was called for " + "thread %zi on thread %zi\n", + _tid, itr->internal_value); + int _verb = 2; // if thread created using finalization, bump up the minimum verbosity level if(get_state() >= State::Finalized && _offset) _verb += 2; if(!config::settings_are_configured()) { - OMNITRACE_BASIC_VERBOSE_F( - _verb, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n", - itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value); + OMNITRACE_BASIC_VERBOSE_F(_verb, + "Thread %li on PID %i (rank: %i) assigned " + "omnitrace TID %li (internal: %li)\n", + itr->system_value, process::get_id(), dmp::rank(), + itr->sequent_value, itr->internal_value); } else { - OMNITRACE_VERBOSE_F( - _verb, "Thread %li on PID %i (rank: %i) assigned omnitrace TID %li\n", - itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value); + OMNITRACE_VERBOSE_F(_verb, + "Thread %li on PID %i (rank: %i) assigned omnitrace TID " + "%li (internal: %li)\n", + itr->system_value, process::get_id(), dmp::rank(), + itr->sequent_value, itr->internal_value); } } return itr; @@ -191,6 +201,46 @@ thread_info::get() return get_info_data(utility::get_thread_index()); } +const std::optional& +thread_info::get(native_handle_t& _tid) +{ + return get(native_handle_t{ _tid }); +} + +const std::optional& +thread_info::get(native_handle_t&& _tid) +{ + const auto& _v = get_info_data(); + if(_v) + { + for(const auto& itr : *_v) + { + if(itr && itr->index_data && + pthread_equal(itr->index_data->pthread_value, _tid) == 0) + return itr; + } + } + + OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value"); + return unknown_thread; +} + +const std::optional& +thread_info::get(std::thread::id _tid) +{ + const auto& _v = get_info_data(); + if(_v) + { + for(const auto& itr : *_v) + { + if(itr && itr->index_data && itr->index_data->stl_value == _tid) return itr; + } + } + + OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value"); + return unknown_thread; +} + const std::optional& thread_info::get(int64_t _tid, ThreadIdType _type) { @@ -203,7 +253,8 @@ thread_info::get(int64_t _tid, ThreadIdType _type) { for(const auto& itr : *_v) { - if(itr && itr->index_data->system_value == _tid) return itr; + if(itr && itr->index_data && itr->index_data->system_value == _tid) + return itr; } } } @@ -214,10 +265,21 @@ thread_info::get(int64_t _tid, ThreadIdType _type) { for(const auto& itr : *_v) { - if(itr && itr->index_data->sequent_value == _tid) return itr; + if(itr && itr->index_data && itr->index_data->sequent_value == _tid) + return itr; } } } + else if(_type == ThreadIdType::PthreadID) + { + OMNITRACE_THROW("omnitrace does not support thread_info::get(int64_t, " + "ThreadIdType) with ThreadIdType::PthreadID\n"); + } + else if(_type == ThreadIdType::StlThreadID) + { + OMNITRACE_THROW("omnitrace does not support thread_info::get(int64_t, " + "ThreadIdType) with ThreadIdType::StlThreadID\n"); + } OMNITRACE_CI_THROW(unknown_thread, "Unknown thread has been assigned a value"); return unknown_thread; @@ -302,8 +364,11 @@ thread_info::as_string() const std::stringstream _ss{}; _ss << std::boolalpha << "is_offset=" << is_offset; if(index_data) + { _ss << ", index_data=(" << index_data->internal_value << ", " - << index_data->system_value << ", " << index_data->sequent_value << ")"; + << index_data->system_value << ", " << index_data->sequent_value << ", " + << index_data->pthread_value << ", " << index_data->stl_value << ")"; + } if(causal_count) _ss << ", causal count=" << *causal_count; _ss << ", lifetime=(" << lifetime.first << ":" << lifetime.second << ")"; return _ss.str(); diff --git a/source/lib/omnitrace/library/thread_info.hpp b/source/lib/omnitrace/library/thread_info.hpp index e9c0a3f326..e16c47cfbf 100644 --- a/source/lib/omnitrace/library/thread_info.hpp +++ b/source/lib/omnitrace/library/thread_info.hpp @@ -24,6 +24,8 @@ #include "core/utility.hpp" +#include +#include #include #include @@ -34,36 +36,46 @@ namespace omnitrace { -// InternalTID: zero-based, process-local thread-ID from atomic increment -// from user-created threads and omnitrace-created threads. -// This value may vary based on threads created by different -// backends, e.g., roctracer will create threads +// InternalTID: zero-based, process-local thread-ID from atomic increment +// from user-created threads and omnitrace-created threads. +// This value may vary based on threads created by different +// backends, e.g., roctracer will create threads // -// SystemTID: system thread-ID. Should be same value as what is seen -// in debugger, etc. +// SystemTID: system thread-ID. Should be same value as what is seen +// in debugger, etc. +// +// SequentTID: zero-based, process-local thread-ID based on the sequence of +// user-created threads which are created in-between the +// initialization and finalization of omnitrace. +// In theory, omnitrace will never increment this value +// because of a thread explicitly by omnitrace or +// by other of the dependent libraries. Most commonly +// used for indexing into omnitrace's thread-local data. +// +// NativeHandle: value of static_cast(pthread_self()) // -// SequentTID: zero-based, process-local thread-ID based on the sequence of -// user-created threads which are created in-between the -// initialization and finalization of omnitrace. -// In theory, omnitrace will never increment this value -// because of a thread explicitly by omnitrace or -// by other of the dependent libraries. Most commonly -// used for indexing into omnitrace's thread-local data. enum ThreadIdType : int { InternalTID = 0, SystemTID = 1, // system thread id SequentTID = 2, + PthreadID = 3, + StlThreadID = 4, }; struct thread_index_data { + using stl_tid_t = std::thread::id; + using native_tid_t = pthread_t; + // the lookup value is always incremented for each thread // the system value is the tid provided by the operating system // the internal value is the value which the user expects - int64_t internal_value = utility::get_thread_index(); - int64_t system_value = tim::threading::get_sys_tid(); - int64_t sequent_value = tim::threading::get_id(); + int64_t internal_value = utility::get_thread_index(); + int64_t system_value = tim::threading::get_sys_tid(); + int64_t sequent_value = tim::threading::get_id(); + native_tid_t pthread_value = ::pthread_self(); + stl_tid_t stl_value = std::this_thread::get_id(); std::string as_string() const; }; @@ -74,6 +86,7 @@ struct thread_info { using index_data_t = std::optional; using lifetime_data_t = std::pair; + using native_handle_t = std::thread::native_handle_type; ~thread_info() = default; thread_info(const thread_info&) = delete; @@ -98,7 +111,11 @@ struct thread_info static bool exists(); static const std::optional& init(bool _offset = false); static const std::optional& get(); + static const std::optional& get(native_handle_t&); + static const std::optional& get(native_handle_t&&); + static const std::optional& get(std::thread::id); static const std::optional& get(int64_t _tid, ThreadIdType _type); + // note: get(native_handle_t) overloaded to & and && to prevent implicit conversion bool is_offset = false; const int64_t* causal_count = nullptr; diff --git a/source/lib/omnitrace/timeout.cpp b/source/lib/omnitrace/timeout.cpp new file mode 100644 index 0000000000..ac72f46cd1 --- /dev/null +++ b/source/lib/omnitrace/timeout.cpp @@ -0,0 +1,231 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "core/categories.hpp" +#include "core/config.hpp" +#include "core/debug.hpp" +#include "core/locking.hpp" +#include "core/state.hpp" +#include "library/components/pthread_gotcha.hpp" +#include "library/runtime.hpp" +#include "library/thread_info.hpp" + +#include +#include +#include + +#include +#include +#include + +namespace omnitrace +{ +namespace timeout +{ +void +setup() OMNITRACE_INTERNAL_API; + +namespace +{ +namespace unwind = ::tim::unwind; +namespace signals = ::tim::signals; +namespace log = ::tim::log; + +constexpr auto timeout_signal = signals::sys_signal::Hangup; +constexpr auto timeout_signal_v = static_cast(timeout_signal); + +auto main_thread_native_handle = pthread_self(); +bool ci_timeout_active = false; +auto ci_timeout_mutex = locking::atomic_mutex{}; +uint64_t ci_timeout_backtrace_global_count = 1; +uint64_t ci_timeout_backtrace_global_done = 0; +thread_local uint64_t ci_timeout_backtrace_local_count = 0; + +void +ci_timeout_backtrace(int) +{ + if(ci_timeout_backtrace_local_count >= ci_timeout_backtrace_global_count) return; + ++ci_timeout_backtrace_local_count; + + auto _err = std::stringstream{}; + auto _cfg = unwind::detailed_backtrace_config{}; + _cfg.proc_pid_maps = false; + _cfg.unwind_lineinfo = false; + _cfg.force_color = !log::monochrome(); + + unwind::detailed_backtrace<0>(_err, _cfg); + + static auto _mutex = locking::atomic_mutex{}; + auto _lk = locking::atomic_lock{ _mutex }; + OMNITRACE_PRINT("%s\n", _err.str().c_str()); + + ++ci_timeout_backtrace_global_done; +} + +void +ensure_ci_timeout_backtrace(double _ci_timeout_seconds, + std::promise _ci_timeout_ready) +{ + _ci_timeout_ready.set_value(); + + thread_info::init(true); + OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Disabled); + + auto _factor = 3.0; + while(_ci_timeout_seconds <= _factor) + _factor /= 1.25; + + uint64_t _ci_timeout_nitr = 0; + int64_t _ci_timeout_nanosec = (_ci_timeout_seconds - _factor) * units::sec; + auto _ci_timeout_total_count = + get_env("OMNITRACE_CI_TIMEOUT_COUNT", 1, false); + const auto root_pid = + get_env("OMNITRACE_ROOT_PROCESS", process::get_id(), false); + + while(get_state() < State::Finalized && _ci_timeout_nitr < _ci_timeout_total_count) + { + // sleep until timeout reached + std::this_thread::sleep_for(std::chrono::nanoseconds{ _ci_timeout_nanosec }); + + // guard against thread in fork + if(process::get_id() != root_pid) + { + ci_timeout_active = false; + setup(); + return; + } + + auto _tids = pthread_gotcha::get_native_handles(); + int64_t _ci_timeout_pause = (_factor * units::sec) / (3 * (_tids.size() + 1)); + auto _kill_thread = [_ci_timeout_pause](auto _handle) { + // execute the pthread_kill and wait until ci_timeout_backtrace increments + // ci_timeout_backtrace_global_done (or 50 iterations pass) to avoid + // the backtraces overlapping output + auto _n = 0; + auto _done_v = ci_timeout_backtrace_global_done; + if(::pthread_kill(_handle, timeout_signal_v) != 0) + { + const auto& _info = thread_info::get(_handle); + if(_info) + { + OMNITRACE_WARNING_F( + 0, "pthread_kill(%zu, %i) failed for thread %zi (info: %s)\n", + _handle, timeout_signal_v, _info->index_data->sequent_value, + _info->as_string().c_str()); + } + else + { + OMNITRACE_WARNING_F(0, + "pthread_kill(%zu, %i) failed. executing generic " + "kill(%i, %i)...\n", + _handle, timeout_signal_v, process::get_id(), + timeout_signal_v); + } + + ::kill(process::get_id(), timeout_signal_v); + } + + // wait until the signal has been delivered + while(ci_timeout_backtrace_global_done == _done_v && _n++ < 50) + std::this_thread::sleep_for( + std::chrono::nanoseconds{ _ci_timeout_pause }); + }; + + _tids.erase(main_thread_native_handle); + OMNITRACE_WARNING_F(-127, + "timeout after %8.3f seconds... Generating backtraces for " + "%zu threads...\n", + _ci_timeout_seconds, _tids.size() + 1); + + for(auto itr : _tids) + _kill_thread(itr); + + _kill_thread(main_thread_native_handle); + + ::omnitrace::debug::flush(); + ::omnitrace::debug::lock _debug_lk{}; + + if(++_ci_timeout_nitr >= _ci_timeout_total_count) + { + // use SIGQUIT because it will generate a core dump + ::kill(process::get_id(), SIGQUIT); + return; + } + else + { + ++ci_timeout_backtrace_global_count; + } + } + + OMNITRACE_WARNING_F(0, "timeout thread exiting...\n"); +} +} // namespace + +void +setup() +{ + // make sure there isn't any datarace for ci_timeout_active + auto _lk = locking::atomic_lock{ ci_timeout_mutex }; + + if(ci_timeout_active) return; + + // in CI mode, if OMNITRACE_CI_TIMEOUT or OMNITRACE_CI_TIMEOUT_OVERRIDE is + // set, start a thread that will print out the backtrace for each thread + // before the timeout is hit (i.e. killed by CTest) so we can potentially + // diagnose where the code is stuck + auto _ci = get_env("OMNITRACE_CI", false, false); + if(_ci) + { + // set by CTest + auto _ci_timeout_default = get_env("OMNITRACE_CI_TIMEOUT", -1.0, false); + // allow override by user + auto _ci_timeout_seconds = + get_env("OMNITRACE_CI_TIMEOUT_OVERRIDE", _ci_timeout_default, false); + + if(_ci_timeout_seconds > 0.0) + { + // lock served its purpose after setting to true + ci_timeout_active = true; + _lk.unlock(); + + OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal); + OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false); + + // enable the signal handler for when the timeout is reached + struct sigaction _action = {}; + sigemptyset(&_action.sa_mask); + _action.sa_flags = SA_RESTART; + _action.sa_handler = ci_timeout_backtrace; + sigaction(timeout_signal_v, &_action, nullptr); + + // start a background thread that handles waiting for the timeout + auto _ci_timeout_ready = std::promise{}; + auto _ci_timeout_wait = _ci_timeout_ready.get_future(); + std::thread{ ensure_ci_timeout_backtrace, _ci_timeout_seconds, + std::move(_ci_timeout_ready) } + .detach(); + _ci_timeout_wait.wait_for(std::chrono::seconds{ 1 }); + } + } +} +} // namespace timeout +} // namespace omnitrace diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 452a6136f9..9908415122 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,5 +20,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-time-window-tests.cmake) include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-critical-trace-tests.cmake) include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-attach-tests.cmake) include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-rccl-tests.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-overflow-tests.cmake) include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-causal-tests.cmake) include(${CMAKE_CURRENT_LIST_DIR}/omnitrace-python-tests.cmake) diff --git a/tests/omnitrace-capchk.cpp b/tests/omnitrace-capchk.cpp new file mode 100644 index 0000000000..7f2d441c0c --- /dev/null +++ b/tests/omnitrace-capchk.cpp @@ -0,0 +1,476 @@ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using cap_value_t = int; + +struct cap_info +{ + const char* name = nullptr; + cap_value_t value = -1; +}; + +struct cap_status +{ + unsigned long long inherited = 0; + unsigned long long permitted = 0; + unsigned long long effective = 0; + unsigned long long bounding = 0; + unsigned long long ambient = 0; +}; + +#define CAP_INFO_ENTRY(VAL) \ + cap_info { #VAL, VAL } + +namespace +{ +std::initializer_list known_capabilities = { +#if defined(CAP_CHOWN) + CAP_INFO_ENTRY(CAP_CHOWN), +#endif + +#if defined(CAP_DAC_OVERRIDE) + CAP_INFO_ENTRY(CAP_DAC_OVERRIDE), +#endif + +#if defined(CAP_DAC_READ_SEARCH) + CAP_INFO_ENTRY(CAP_DAC_READ_SEARCH), +#endif + +#if defined(CAP_FOWNER) + CAP_INFO_ENTRY(CAP_FOWNER), +#endif + +#if defined(CAP_FSETID) + CAP_INFO_ENTRY(CAP_FSETID), +#endif + +#if defined(CAP_KILL) + CAP_INFO_ENTRY(CAP_KILL), +#endif + +#if defined(CAP_SETGID) + CAP_INFO_ENTRY(CAP_SETGID), +#endif + +#if defined(CAP_SETUID) + CAP_INFO_ENTRY(CAP_SETUID), +#endif + +#if defined(CAP_SETPCAP) + CAP_INFO_ENTRY(CAP_SETPCAP), +#endif + +#if defined(CAP_LINUX_IMMUTABLE) + CAP_INFO_ENTRY(CAP_LINUX_IMMUTABLE), +#endif + +#if defined(CAP_NET_BIND_SERVICE) + CAP_INFO_ENTRY(CAP_NET_BIND_SERVICE), +#endif + +#if defined(CAP_NET_BROADCAST) + CAP_INFO_ENTRY(CAP_NET_BROADCAST), +#endif + +#if defined(CAP_NET_ADMIN) + CAP_INFO_ENTRY(CAP_NET_ADMIN), +#endif + +#if defined(CAP_NET_RAW) + CAP_INFO_ENTRY(CAP_NET_RAW), +#endif + +#if defined(CAP_IPC_LOCK) + CAP_INFO_ENTRY(CAP_IPC_LOCK), +#endif + +#if defined(CAP_IPC_OWNER) + CAP_INFO_ENTRY(CAP_IPC_OWNER), +#endif + +#if defined(CAP_SYS_MODULE) + CAP_INFO_ENTRY(CAP_SYS_MODULE), +#endif + +#if defined(CAP_SYS_RAWIO) + CAP_INFO_ENTRY(CAP_SYS_RAWIO), +#endif + +#if defined(CAP_SYS_CHROOT) + CAP_INFO_ENTRY(CAP_SYS_CHROOT), +#endif + +#if defined(CAP_SYS_PTRACE) + CAP_INFO_ENTRY(CAP_SYS_PTRACE), +#endif + +#if defined(CAP_SYS_PACCT) + CAP_INFO_ENTRY(CAP_SYS_PACCT), +#endif + +#if defined(CAP_SYS_ADMIN) + CAP_INFO_ENTRY(CAP_SYS_ADMIN), +#endif + +#if defined(CAP_SYS_BOOT) + CAP_INFO_ENTRY(CAP_SYS_BOOT), +#endif + +#if defined(CAP_SYS_NICE) + CAP_INFO_ENTRY(CAP_SYS_NICE), +#endif + +#if defined(CAP_SYS_RESOURCE) + CAP_INFO_ENTRY(CAP_SYS_RESOURCE), +#endif + +#if defined(CAP_SYS_TIME) + CAP_INFO_ENTRY(CAP_SYS_TIME), +#endif + +#if defined(CAP_SYS_TTY_CONFIG) + CAP_INFO_ENTRY(CAP_SYS_TTY_CONFIG), +#endif + +#if defined(CAP_MKNOD) + CAP_INFO_ENTRY(CAP_MKNOD), +#endif + +#if defined(CAP_LEASE) + CAP_INFO_ENTRY(CAP_LEASE), +#endif + +#if defined(CAP_AUDIT_WRITE) + CAP_INFO_ENTRY(CAP_AUDIT_WRITE), +#endif + +#if defined(CAP_AUDIT_CONTROL) + CAP_INFO_ENTRY(CAP_AUDIT_CONTROL), +#endif + +#if defined(CAP_SETFCAP) + CAP_INFO_ENTRY(CAP_SETFCAP), +#endif + +#if defined(CAP_MAC_OVERRIDE) + CAP_INFO_ENTRY(CAP_MAC_OVERRIDE), +#endif + +#if defined(CAP_MAC_ADMIN) + CAP_INFO_ENTRY(CAP_MAC_ADMIN), +#endif + +#if defined(CAP_SYSLOG) + CAP_INFO_ENTRY(CAP_SYSLOG), +#endif + +#if defined(CAP_WAKE_ALARM) + CAP_INFO_ENTRY(CAP_WAKE_ALARM), +#endif + +#if defined(CAP_BLOCK_SUSPEND) + CAP_INFO_ENTRY(CAP_BLOCK_SUSPEND), +#endif + +#if defined(CAP_AUDIT_READ) + CAP_INFO_ENTRY(CAP_AUDIT_READ), +#endif + +#if defined(CAP_PERFMON) + CAP_INFO_ENTRY(CAP_PERFMON), +#endif + +#if defined(CAP_BPF) + CAP_INFO_ENTRY(CAP_BPF), +#endif + +#if defined(CAP_CHECKPOINT_RESTORE) + CAP_INFO_ENTRY(CAP_CHECKPOINT_RESTORE), +#endif + +#if defined(CAP_LAST_CAP) + CAP_INFO_ENTRY(CAP_LAST_CAP), +#endif +}; + +auto cap_max_bits_v = []() { + unsigned _value = 0; + for(const auto& itr : known_capabilities) + _value = std::max(_value, itr.value + 1); + return _value; +}(); + +std::string +to_lower(std::string&& _s) +{ + for(auto& citr : _s) + citr = tolower(citr); + return _s; +} + +std::string +to_upper(std::string _s) +{ + for(auto& citr : _s) + citr = toupper(citr); + return _s; +} + +cap_status +cap_read(pid_t _pid) +{ + auto fname = std::string{ "/proc/" } + std::to_string(_pid) + "/status"; + auto ifs = std::ifstream{ fname }; + if(!ifs) return cap_status{}; + + auto _lines = std::vector{}; + + while(ifs && ifs.good()) + { + auto _line = std::string{}; + std::getline(ifs, _line); + if(ifs && ifs.good() && !_line.empty()) _lines.emplace_back(std::move(_line)); + } + + auto _data = cap_status{}; + for(const auto& itr : _lines) + { + auto iss = std::istringstream{ itr }; + auto _key = std::string{}; + auto _value = std::string{}; + iss >> _key; + + if(_key.find("Cap") == 0) iss >> _value; + + if(!_value.empty()) + { + auto _key_matches = [&_key](std::string_view _cap_id_str) { + return (_key.find(_cap_id_str) == 0); + }; + if(_key_matches("CapInh")) + _data.inherited = std::stoull(_value, nullptr, 16); + else if(_key_matches("CapPrm")) + _data.permitted = std::stoull(_value, nullptr, 16); + else if(_key_matches("CapEff")) + _data.effective = std::stoull(_value, nullptr, 16); + else if(_key_matches("CapBnd")) + _data.bounding = std::stoull(_value, nullptr, 16); + else if(_key_matches("CapAmb")) + _data.ambient = std::stoull(_value, nullptr, 16); + } + } + + return _data; +} + +std::string +cap_name(cap_value_t _v) +{ + for(const auto& itr : known_capabilities) + if(itr.value == _v) return to_lower(std::string{ itr.name }); + + return std::string{}; +} + +std::vector +decode(unsigned long long value) +{ + auto _data = std::vector{}; + for(unsigned cap = 0; (cap < 64) && ((value >> cap) != 0U); ++cap) + { + auto _mask = value & (1ULL << cap); + if(_mask != 0U) + { + if(cap >= 0 && cap < cap_max_bits_v) _data.emplace_back(cap); + } + } + + return _data; +} + +std::vector +decode(const char* arg) +{ + return decode(std::strtoull(arg, nullptr, 16)); +} +/* +template +std::string +as_hex(Tp _v, size_t _width = 16) +{ + std::stringstream _ss; + _ss.fill('0'); + _ss << "0x" << std::hex << std::setw(_width) << _v; + return _ss.str(); +} + +void run(std::string&& arg) +{ + if(arg.find("0x") == 0) + arg = arg.substr(2); + + arg.insert(0, "0x"); + // arg = std::string{ "0x" } + arg; + + std::cout << arg << "="; + auto _decoded = decode(arg.c_str()); + auto _msg = std::stringstream{}; + for(auto&& itr : _decoded) + { + _msg << "," << cap_name(itr); + } + auto _msg_v = _msg.str(); + if(!_msg_v.empty()) + { + std::cout << _msg_v.substr(1); + } + std::cout << "\n"; +} + +void +run(std::string_view _label, unsigned long long arg) +{ + std::cout << " " << std::setw(12) << _label << " "; + run(as_hex(arg)); +} +*/ +} // namespace + +int +main(int argc, char** argv) +{ + const auto* _usage = R"usage( +usage: omnitrace-capchk + + Description: + Simple tool for checking the effective capabilities of a running process + + Arguments: + capability-name (string): + case-insensitive string matching CAP_* fields defined in `man 7 capabilities` + + capability-set (string; optional): + Choices: + - effective (default) + - permitted + - inherited + - bounding + - ambient + See `man 7 capabilities` for more info + + pid (numeric process-identifier; optional): + target process identifier for capability query. If not specified, queries the + capabilities of the current process (this exe). + + Exit value: + 0 if the process has the specified capability in the specified set + 1 if the process does not have the capability + 2 if the capability name is not supported + 3 if the capability set name is not supported + + Examples: + $ omnitrace-capchk CAP_SYS_ADMIN + Check if this exe (self) has CAP_SYS_ADMIN capability in the (default) effective capability set + + $ omnitrace-capchk sys_admin bounding 423032 + Check if process 423032 has CAP_SYS_ADMIN capability in the bounding capability set + )usage"; + + std::string capability_name = {}; + std::string capability_mode = "effective"; + pid_t target_pid = getpid(); + + for(int i = 1; i < argc; ++i) + { + auto arg = std::string_view{ argv[i] }; + if(arg == "-h" || arg == "--help" || arg == "-?") + { + std::cout << _usage << "\n"; + return EXIT_SUCCESS; + } + } + + if(argc > 1) capability_name = to_lower(std::string{ argv[1] }); + if(argc > 2) capability_mode = to_lower(std::string{ argv[2] }); + if(argc > 3) + { + auto _pid_s = to_lower(std::string{ argv[3] }); + if(_pid_s != "self") target_pid = std::stoul(argv[3]); + } + + if(capability_name.find("cap_") != 0) capability_name.insert(0, "cap_"); + + capability_name = to_upper(capability_name); + + const cap_info* _info = nullptr; + for(const auto& itr : known_capabilities) + { + if(capability_name == std::string_view{ itr.name }) + { + _info = &itr; + break; + } + } + + if(!_info) + { + fprintf(stderr, "Error! invalid capability: %s\n", capability_name.c_str()); + return EXIT_FAILURE + 1; + } + + auto _status = cap_read(target_pid); + auto* _dataset = &_status.effective; + + /* + std::cout << "pid=" << target_pid << ":\n"; + run("inherited", _status.inherited); + run("permitted", _status.permitted); + run("effective", _status.effective); + run("bounding", _status.bounding); + run("ambient", _status.ambient); + */ + + if(capability_mode == "effective") + _dataset = &_status.effective; + else if(capability_mode == "permitted") + _dataset = &_status.permitted; + else if(capability_mode == "inherited") + _dataset = &_status.inherited; + else if(capability_mode == "bounding") + _dataset = &_status.bounding; + else if(capability_mode == "ambient") + _dataset = &_status.ambient; + else + { + fprintf(stderr, "Error! invalid capability set: %s\n", capability_mode.c_str()); + return EXIT_FAILURE + 2; + } + + auto _ec = EXIT_FAILURE; + for(auto&& itr : decode(*_dataset)) + { + if(itr == _info->value) + { + _ec = EXIT_SUCCESS; + break; + } + } + + std::cout << ((_ec == EXIT_SUCCESS) ? "Found" : "Missing") << " capability " + << capability_name << " in " << capability_mode + << " capability set. Exit code: " << _ec << ".\n"; + return _ec; +} diff --git a/tests/omnitrace-config-tests.cmake b/tests/omnitrace-config-tests.cmake index 72b8706f19..91fd63cb11 100644 --- a/tests/omnitrace-config-tests.cmake +++ b/tests/omnitrace-config-tests.cmake @@ -24,9 +24,15 @@ add_test( set_tests_properties( omnitrace-invalid-config - PROPERTIES ENVIRONMENT - "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/invalid.cfg" TIMEOUT - 120 LABELS "config" WILL_FAIL ON) + PROPERTIES + ENVIRONMENT + "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/invalid.cfg;OMNITRACE_CI=ON;OMNITRACE_CI_TIMEOUT=120" + TIMEOUT + 120 + LABELS + "config" + WILL_FAIL + ON) add_test( NAME omnitrace-missing-config @@ -35,6 +41,12 @@ add_test( set_tests_properties( omnitrace-missing-config - PROPERTIES ENVIRONMENT - "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/missing.cfg" TIMEOUT - 120 LABELS "config" WILL_FAIL ON) + PROPERTIES + ENVIRONMENT + "OMNITRACE_CONFIG_FILE=${CMAKE_CURRENT_BINARY_DIR}/missing.cfg;OMNITRACE_CI=ON;OMNITRACE_CI_TIMEOUT=120" + TIMEOUT + 120 + LABELS + "config" + WILL_FAIL + ON) diff --git a/tests/omnitrace-critical-trace-tests.cmake b/tests/omnitrace-critical-trace-tests.cmake index d1a9633d38..761ed83a86 100644 --- a/tests/omnitrace-critical-trace-tests.cmake +++ b/tests/omnitrace-critical-trace-tests.cmake @@ -35,7 +35,9 @@ set(_parallel_overhead_critical_trace_environ "OMNITRACE_CRITICAL_TRACE_DEBUG=ON" "OMNITRACE_VERBOSE=4" "OMNITRACE_USE_PID=OFF" - "OMNITRACE_TIME_OUTPUT=OFF") + "OMNITRACE_TIME_OUTPUT=OFF" + "OMNITRACE_CI=ON" + "OMNITRACE_CI_TIMEOUT=300") set_tests_properties( parallel-overhead-process-critical-trace diff --git a/tests/omnitrace-fork-tests.cmake b/tests/omnitrace-fork-tests.cmake index 4a074a7b91..17baec8824 100644 --- a/tests/omnitrace-fork-tests.cmake +++ b/tests/omnitrace-fork-tests.cmake @@ -14,10 +14,6 @@ omnitrace_add_test( SAMPLING_PASS_REGEX "fork.. called on PID" RUNTIME_PASS_REGEX "fork.. called on PID" REWRITE_RUN_PASS_REGEX "fork.. called on PID" - SAMPLING_FAIL_REGEX - "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])" - RUNTIME_FAIL_REGEX - "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])" - REWRITE_RUN_FAIL_REGEX - "(terminate called after throwing an instance|calling abort.. in |Exit code: [1-9])" - ) + SAMPLING_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})" + RUNTIME_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})" + REWRITE_RUN_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})") diff --git a/tests/omnitrace-mpi-tests.cmake b/tests/omnitrace-mpi-tests.cmake index 93c0f485ae..3007c62791 100644 --- a/tests/omnitrace-mpi-tests.cmake +++ b/tests/omnitrace-mpi-tests.cmake @@ -29,7 +29,7 @@ omnitrace_add_test( REWRITE_RUN_PASS_REGEX "(/[A-Za-z-]+/perfetto-trace-0.proto).*(/[A-Za-z-]+/wall_clock-0.txt')" REWRITE_RUN_FAIL_REGEX - "(perfetto-trace|trip_count|sampling_percent|sampling_cpu_clock|sampling_wall_clock|wall_clock)-[0-9][0-9]+.(json|txt|proto)" + "(perfetto-trace|trip_count|sampling_percent|sampling_cpu_clock|sampling_wall_clock|wall_clock)-[0-9][0-9]+.(json|txt|proto)|OMNITRACE_ABORT_FAIL_REGEX" ) omnitrace_add_test( diff --git a/tests/omnitrace-overflow-tests.cmake b/tests/omnitrace-overflow-tests.cmake new file mode 100644 index 0000000000..a395beb6b9 --- /dev/null +++ b/tests/omnitrace-overflow-tests.cmake @@ -0,0 +1,32 @@ +# -------------------------------------------------------------------------------------- # +# +# overflow tests +# +# -------------------------------------------------------------------------------------- # + +set(_overflow_environment + "${_base_environment}" + "OMNITRACE_VERBOSE=2" + "OMNITRACE_SAMPLING_CPUTIME=OFF" + "OMNITRACE_SAMPLING_REALTIME=OFF" + "OMNITRACE_SAMPLING_OVERFLOW=ON" + "OMNITRACE_SAMPLING_OVERFLOW_EVENT=PERF_COUNT_SW_CPU_CLOCK" + "OMNITRACE_SAMPLING_OVERFLOW_FREQ=10000" + "OMNITRACE_DEBUG_THREADING_GET_ID=ON") + +if(omnitrace_perf_event_paranoid LESS_EQUAL 3 + OR omnitrace_cap_sys_admin EQUAL 0 + OR omnitrace_cap_perfmon EQUAL 0) + omnitrace_add_test( + SKIP_BASELINE + NAME overflow + TARGET parallel-overhead + RUN_ARGS 30 2 200 + REWRITE_ARGS -e -v 2 + RUNTIME_ARGS -e -v 1 + ENVIRONMENT "${_overflow_environment}" + LABELS "perf;overflow" + SAMPLING_PASS_REGEX "sampling_wall_clock.txt" + RUNTIME_PASS_REGEX "sampling_wall_clock.txt" + REWRITE_RUN_PASS_REGEX "sampling_wall_clock.txt") +endif() diff --git a/tests/omnitrace-python-tests.cmake b/tests/omnitrace-python-tests.cmake index 0458489627..32d4c2c513 100644 --- a/tests/omnitrace-python-tests.cmake +++ b/tests/omnitrace-python-tests.cmake @@ -116,7 +116,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS}) COMMAND ${OMNITRACE_CAT_COMMAND} PYTHON_VERSION ${_VERSION} FILE omnitrace-tests-output/python-external-exclude-inefficient/${_VERSION}/trip_count.txt - FAIL_REGEX "(\\\|_inefficient).*(\\\|_sum)" + FAIL_REGEX "(\\\|_inefficient).*(\\\|_sum)|OMNITRACE_ABORT_FAIL_REGEX" DEPENDS python-external-exclude-inefficient-${_VERSION} ENVIRONMENT "${_python_environment}") @@ -135,7 +135,7 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS}) PYTHON_VERSION ${_VERSION} FILE omnitrace-tests-output/python-builtin-noprofile/${_VERSION}/trip_count.txt PASS_REGEX ".(run)..(noprofile.py)." - FAIL_REGEX ".(fib|inefficient)..(noprofile.py)." + FAIL_REGEX ".(fib|inefficient)..(noprofile.py).|OMNITRACE_ABORT_FAIL_REGEX" DEPENDS python-builtin-noprofile-${_VERSION} ENVIRONMENT "${_python_environment}") else() diff --git a/tests/omnitrace-rocm-tests.cmake b/tests/omnitrace-rocm-tests.cmake index 907d4fa66c..1e12a3e538 100644 --- a/tests/omnitrace-rocm-tests.cmake +++ b/tests/omnitrace-rocm-tests.cmake @@ -81,5 +81,5 @@ if(OMNITRACE_USE_ROCPROFILER) "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_ROCTRACER=OFF;OMNITRACE_ROCM_EVENTS=${OMNITRACE_ROCM_EVENTS_TEST}" REWRITE_RUN_PASS_REGEX "rocprof-device-0-GRBM_COUNT.txt(.*)rocprof-device-0-GPUBusy.txt(.*)rocprof-device-0-SQ_WAVES.txt(.*)rocprof-device-0-SQ_INSTS_VALU.txt(.*)rocprof-device-0-VALUInsts.txt(.*)rocprof-device-0-TCC_HIT_sum.txt(.*)rocprof-device-0-TA_TA_BUSY_0.txt(.*)rocprof-device-0-TA_TA_BUSY_11.txt" - REWRITE_RUN_FAIL_REGEX "roctracer.txt") + REWRITE_RUN_FAIL_REGEX "roctracer.txt|OMNITRACE_ABORT_FAIL_REGEX") endif() diff --git a/tests/omnitrace-testing.cmake b/tests/omnitrace-testing.cmake index 41e86e5636..621f27928c 100644 --- a/tests/omnitrace-testing.cmake +++ b/tests/omnitrace-testing.cmake @@ -213,6 +213,40 @@ execute_process( RESULT_VARIABLE _mpiexec_oversubscribe OUTPUT_QUIET ERROR_QUIET) +set(omnitrace_perf_event_paranoid "4") +set(omnitrace_cap_sys_admin "1") +set(omnitrace_cap_perfmon "1") + +if(EXISTS "/proc/sys/kernel/perf_event_paranoid") + file(STRINGS "/proc/sys/kernel/perf_event_paranoid" omnitrace_perf_event_paranoid + LIMIT_COUNT 1) +endif() + +execute_process( + COMMAND ${CMAKE_CXX_COMPILER} -O2 -g -std=c++17 + ${CMAKE_CURRENT_LIST_DIR}/omnitrace-capchk.cpp -o omnitrace-capchk + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/bin + RESULT_VARIABLE _capchk_compile + OUTPUT_QUIET ERROR_QUIET) + +if(_capchk_compile EQUAL 0) + execute_process( + COMMAND ${PROJECT_BINARY_DIR}/bin/omnitrace-capchk CAP_SYS_ADMIN effective + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + RESULT_VARIABLE omnitrace_cap_sys_admin + OUTPUT_QUIET ERROR_QUIET) + + execute_process( + COMMAND ${PROJECT_BINARY_DIR}/bin/omnitrace-capchk CAP_PERFMON effective + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + RESULT_VARIABLE omnitrace_cap_perfmon + OUTPUT_QUIET ERROR_QUIET) +endif() + +omnitrace_message(STATUS "perf_event_paranoid: ${omnitrace_perf_event_paranoid}") +omnitrace_message(STATUS "CAP_SYS_ADMIN: ${omnitrace_cap_sys_admin}") +omnitrace_message(STATUS "CAP_PERFMON: ${omnitrace_cap_perfmon}") + if(_mpiexec_oversubscribe EQUAL 0) list(APPEND MPIEXEC_EXECUTABLE_ARGS --oversubscribe) endif() @@ -255,9 +289,30 @@ endif() # -------------------------------------------------------------------------------------- # +macro(OMNITRACE_CHECK_PASS_FAIL_REGEX NAME PASS FAIL) + if(NOT "${${PASS}}" STREQUAL "" + AND NOT "${${FAIL}}" STREQUAL "" + AND NOT "${${FAIL}}" MATCHES "\\|OMNITRACE_ABORT_FAIL_REGEX" + AND NOT "${${FAIL}}" MATCHES "${OMNITRACE_ABORT_FAIL_REGEX}") + omnitrace_message( + FATAL_ERROR + "${NAME} has set pass and fail regexes but fail regex does not include '|OMNITRACE_ABORT_FAIL_REGEX'" + ) + endif() + + if("${${FAIL}}" STREQUAL "") + set(${FAIL} "(${OMNITRACE_ABORT_FAIL_REGEX})") + else() + string(REPLACE "|OMNITRACE_ABORT_FAIL_REGEX" "|${OMNITRACE_ABORT_FAIL_REGEX}" + ${FAIL} "${${FAIL}}") + endif() +endmacro() + +# -------------------------------------------------------------------------------------- # + function(OMNITRACE_WRITE_TEST_CONFIG _FILE _ENV) set(_ENV_ONLY - "OMNITRACE_(CI|MODE|USE_MPIP|DEBUG_SETTINGS|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)=" + "OMNITRACE_(CI|CI_TIMEOUT|MODE|USE_MPIP|DEBUG_[A-Z_]+|FORCE_ROCPROFILER_INIT|DEFAULT_MIN_INSTRUCTIONS|MONOCHROME|VERBOSE)=" ) set(_FILE_CONTENTS) set(_ENV_CONTENTS) @@ -357,9 +412,7 @@ function(OMNITRACE_ADD_TEST) foreach(_PREFIX SAMPLING RUNTIME REWRITE REWRITE_RUN BASELINE) if("${${_PREFIX}_FAIL_REGEX}" STREQUAL "") - set(${_PREFIX}_FAIL_REGEX - "(### ERROR ###|address of faulting memory reference|exiting with non-zero exit code)" - ) + set(${_PREFIX}_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})") endif() endforeach() @@ -552,6 +605,10 @@ function(OMNITRACE_ADD_TEST) endif() endforeach() + list(APPEND _environ "OMNITRACE_CI_TIMEOUT=${_timeout}") + + omnitrace_check_pass_fail_regex("${TEST_NAME}-${_TEST}" "${_PASS_REGEX}" + "${_FAIL_REGEX}") if(TEST ${TEST_NAME}-${_TEST}) omnitrace_write_test_config(${TEST_NAME}-${_TEST}.cfg _environ) set_tests_properties( @@ -604,9 +661,7 @@ function(OMNITRACE_ADD_CAUSAL_TEST) endif() if("${TEST_CAUSAL_FAIL_REGEX}" STREQUAL "") - set(TEST_CAUSAL_FAIL_REGEX - "(### ERROR ###|address of faulting memory reference|exiting with non-zero exit code)" - ) + set(TEST_CAUSAL_FAIL_REGEX "(${OMNITRACE_ABORT_FAIL_REGEX})") endif() if(TARGET ${TEST_TARGET}) @@ -649,10 +704,16 @@ function(OMNITRACE_ADD_CAUSAL_TEST) foreach(_TEST baseline causal validate-causal) if(NOT TEST ${_TEST}-${TEST_NAME}) - continue() + if(NOT TEST ${TEST_NAME}-${_TEST}) + continue() + else() + set(_NAME "${TEST_NAME}-${_TEST}") + endif() + else() + set(_NAME "${_TEST}-${TEST_NAME}") endif() - set(_prefix "${_TEST}-${TEST_NAME}/") + set(_prefix "${_NAME}/") set(_labels "${_TEST}" "causal-profiling") if(TEST_TARGET) @@ -701,9 +762,11 @@ function(OMNITRACE_ADD_CAUSAL_TEST) endif() endforeach() - omnitrace_write_test_config(${_TEST}-${TEST_NAME}.cfg _environ) + list(APPEND _environ "OMNITRACE_CI_TIMEOUT=${_timeout}") + omnitrace_write_test_config(${_NAME}.cfg _environ) + omnitrace_check_pass_fail_regex("${_NAME}" "${_PASS_REGEX}" "${_FAIL_REGEX}") set_tests_properties( - ${_TEST}-${TEST_NAME} + ${_NAME} PROPERTIES ENVIRONMENT "${_environ}" TIMEOUT @@ -780,16 +843,10 @@ function(OMNITRACE_ADD_PYTHON_TEST) NAME ${TEST_NAME}-${TEST_PYTHON_VERSION} COMMAND ${TEST_COMMAND} ${TEST_FILE} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - add_test( - NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse - COMMAND ${TEST_COMMAND} ${TEST_FILE} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) endif() - foreach( - _TEST - ${TEST_NAME}-${TEST_PYTHON_VERSION} ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse - ${TEST_NAME}-${TEST_PYTHON_VERSION}-annotated) + foreach(_TEST ${TEST_NAME}-${TEST_PYTHON_VERSION} + ${TEST_NAME}-${TEST_PYTHON_VERSION}-annotated) if(NOT TEST "${_TEST}") continue() @@ -797,24 +854,18 @@ function(OMNITRACE_ADD_PYTHON_TEST) string(REPLACE "${TEST_NAME}-${TEST_PYTHON_VERSION}" "${TEST_NAME}" _TEST_DIR "${_TEST}") - set(_TEST_ENV "${TEST_ENVIRONMENT}" - "OMNITRACE_OUTPUT_PREFIX=${_TEST_DIR}/${TEST_PYTHON_VERSION}/") + set(_TEST_ENV + "${TEST_ENVIRONMENT}" + "OMNITRACE_OUTPUT_PREFIX=${_TEST_DIR}/${TEST_PYTHON_VERSION}/" + "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}") set(_TEST_PROPERTIES "${TEST_PROPERTIES}") - if(NOT "${_TEST}" MATCHES "inverse") - # assign pass variable to pass regex - set(_PASS_REGEX TEST_PASS_REGEX) - # assign fail variable to fail regex - set(_FAIL_REGEX TEST_FAIL_REGEX) - else() - # assign pass variable to fail regex - set(_PASS_REGEX TEST_FAIL_REGEX) - # assign fail variable to pass regex - set(_FAIL_REGEX TEST_PASS_REGEX) - # set to will fail - list(APPEND _TEST_PROPERTIES WILL_FAIL ON) - endif() + # assign pass variable to pass regex + set(_PASS_REGEX TEST_PASS_REGEX) + # assign fail variable to fail regex + set(_FAIL_REGEX TEST_FAIL_REGEX) + omnitrace_check_pass_fail_regex("${_TEST}" "${_PASS_REGEX}" "${_FAIL_REGEX}") set_tests_properties( ${_TEST} PROPERTIES ENVIRONMENT @@ -964,16 +1015,19 @@ function(OMNITRACE_ADD_VALIDATION_TEST) WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) endif() + list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI_TIMEOUT=${TEST_TIMEOUT}") + foreach(_TEST validate-${TEST_NAME}-timemory validate-${TEST_NAME}-perfetto) if(NOT TEST "${_TEST}") continue() endif() + omnitrace_check_pass_fail_regex("${_TEST}" "TEST_PASS_REGEX" "TEST_FAIL_REGEX") set_tests_properties( ${_TEST} PROPERTIES ENVIRONMENT - "${_TEST_ENV}" + "${TEST_ENVIRONMENT}" TIMEOUT ${TEST_TIMEOUT} LABELS diff --git a/tests/omnitrace-time-window-tests.cmake b/tests/omnitrace-time-window-tests.cmake index 040474a146..fcec29a916 100644 --- a/tests/omnitrace-time-window-tests.cmake +++ b/tests/omnitrace-time-window-tests.cmake @@ -24,7 +24,7 @@ omnitrace_add_validation_test( PERFETTO_METRIC "host" PERFETTO_FILE "perfetto-trace.proto" LABELS "time-window" - FAIL_REGEX "outer_d" + FAIL_REGEX "outer_d|OMNITRACE_ABORT_FAIL_REGEX" ARGS -l trace-time-window.inst outer_a @@ -49,7 +49,7 @@ omnitrace_add_validation_test( PERFETTO_METRIC "host" PERFETTO_FILE "perfetto-trace.proto" LABELS "time-window" - FAIL_REGEX "outer_d" + FAIL_REGEX "outer_d|OMNITRACE_ABORT_FAIL_REGEX" ARGS -l trace-time-window outer_a