diff --git a/.github/workflows/ubuntu-bionic.yml b/.github/workflows/ubuntu-bionic.yml index 8c09bba4b7..8acacbdf05 100644 --- a/.github/workflows/ubuntu-bionic.yml +++ b/.github/workflows/ubuntu-bionic.yml @@ -83,7 +83,7 @@ jobs: timeout-minutes: 30 run: cd build && - ctest -V --output-log build/omnitrace-ctest-ubuntu-bionic.log + ctest -V --output-log build/omnitrace-ctest-ubuntu-bionic.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/.github/workflows/ubuntu-focal-dyninst-package.yml b/.github/workflows/ubuntu-focal-dyninst-package.yml index 15a0931c6c..28b976be04 100644 --- a/.github/workflows/ubuntu-focal-dyninst-package.yml +++ b/.github/workflows/ubuntu-focal-dyninst-package.yml @@ -88,7 +88,7 @@ jobs: run: ldd ./omnitrace && ./omnitrace --help && - ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal-dyninst-package.log + ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal-dyninst-package.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/.github/workflows/ubuntu-focal-external-rocm.yml b/.github/workflows/ubuntu-focal-external-rocm.yml index df81411c69..a178aac196 100644 --- a/.github/workflows/ubuntu-focal-external-rocm.yml +++ b/.github/workflows/ubuntu-focal-external-rocm.yml @@ -8,10 +8,11 @@ on: env: BUILD_TYPE: Release - ELFUTILS_DOWNLOAD_VERSION: 0.183 OMNITRACE_DEBUG_FINALIZE: ON OMNITRACE_VERBOSE: 1 OMNITRACE_CI: ON + OMNITRACE_OUTPUT_PATH: omnitrace-tests-output + OMNITRACE_OUTPUT_PREFIX: "%argt%/" jobs: ubuntu-focal-external-rocm: @@ -80,7 +81,8 @@ jobs: run: cd build && ldd ./omnitrace && - ./omnitrace --help + ./omnitrace --help && + ctest -V --output-log omnitrace-ctest-ubuntu-focal-external-rocm.log --stop-on-failure - name: Test Install timeout-minutes: 10 @@ -97,9 +99,22 @@ jobs: ldd $(which omnitrace) omnitrace --help omnitrace -e -v 1 -o ls.inst --simulate -- ls - for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done omnitrace -e -v 1 -o ls.inst -- ls ./ls.inst omnitrace -e -v 1 --simulate -- ls - for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done omnitrace -e -v 1 -- ls + + - name: CTest Artifacts + uses: actions/upload-artifact@v2 + with: + name: ctest-log + path: | + build/*.log + + - name: Data Artifacts + uses: actions/upload-artifact@v2 + with: + name: data-files + path: | + omnitrace-tests-output/**/*.txt + build/omnitrace-tests-output/**/*.txt diff --git a/.github/workflows/ubuntu-focal-external.yml b/.github/workflows/ubuntu-focal-external.yml index 3b95a02015..3877985ad8 100644 --- a/.github/workflows/ubuntu-focal-external.yml +++ b/.github/workflows/ubuntu-focal-external.yml @@ -71,7 +71,7 @@ jobs: cd build && ldd ./omnitrace && ./omnitrace --help && - ctest -V --output-log build/omnitrace-ctest-ubuntu-focal-external.log + ctest -V --output-log build/omnitrace-ctest-ubuntu-focal-external.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index a9cafe08b6..033c4c9614 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -71,7 +71,7 @@ jobs: timeout-minutes: 30 working-directory: ${{ github.workspace }}/build run: - ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal.log + ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal.log --stop-on-failure - name: Test Install timeout-minutes: 10 diff --git a/CMakeLists.txt b/CMakeLists.txt index e3b1f4266b..b30a59e2ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,7 +159,7 @@ if(OMNITRACE_BUILD_HIDDEN_VISIBILITY) set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) endif() -if(OMNITRACE_BUILD_TESTING) +if(OMNITRACE_BUILD_TESTING OR "$ENV{OMNITRACE_CI}" MATCHES "[1-9]+|ON|on|y|yes") include(CTest) enable_testing() endif() diff --git a/examples/user-api/CMakeLists.txt b/examples/user-api/CMakeLists.txt index 31118606da..8c22cfbdf3 100644 --- a/examples/user-api/CMakeLists.txt +++ b/examples/user-api/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.15 FATAL_ERROR) project(omnitrace-user-api LANGUAGES CXX) -set(CMAKE_BUILD_TYPE "Release") +set(CMAKE_BUILD_TYPE "Debug") find_package(Threads REQUIRED) add_executable(user-api user-api.cpp) target_link_libraries(user-api PRIVATE Threads::Threads omnitrace::omnitrace-user-library) diff --git a/examples/user-api/user-api.cpp b/examples/user-api/user-api.cpp index 15ecc318ad..33df3822cd 100644 --- a/examples/user-api/user-api.cpp +++ b/examples/user-api/user-api.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -16,33 +17,25 @@ fib(long n) __attribute__((noinline)); void run(size_t nitr, long) __attribute__((noinline)); -long -fib(long n) -{ - return (n < 2) ? n : fib(n - 1) + fib(n - 2); -} +int +custom_push_region(const char* name); -#define RUN_LABEL \ - std::string{ std::string{ __FUNCTION__ } + "(" + std::to_string(n) + ") x " + \ - std::to_string(nitr) } \ - .c_str() - -void -run(size_t nitr, long n) +namespace { - omnitrace_user_stop_thread_trace(); - omnitrace_user_push_region(RUN_LABEL); - long local = 0; - for(size_t i = 0; i < nitr; ++i) - local += fib(n); - total += local; - omnitrace_user_pop_region(RUN_LABEL); - omnitrace_user_start_thread_trace(); +int (*omnitrace_push_region_f)(const char*) = nullptr; } int main(int argc, char** argv) { + // get the internal callback to start a user-defined region + omnitrace_user_get_callbacks(OMNITRACE_USER_REGION, (void**) &omnitrace_push_region_f, + nullptr); + // assign the custom callback to start a user-defined region + if(omnitrace_push_region_f) + omnitrace_user_configure(OMNITRACE_USER_REGION, (void*) &custom_push_region, + nullptr); + omnitrace_user_push_region(argv[0]); omnitrace_user_push_region("initialization"); size_t nthread = std::min(16, std::thread::hardware_concurrency()); @@ -59,22 +52,54 @@ main(int argc, char** argv) omnitrace_user_push_region("thread_creation"); std::vector threads{}; threads.reserve(nthread); + // disable instrumentation for child threads + omnitrace_user_stop_thread_trace(); for(size_t i = 0; i < nthread; ++i) { - size_t _nitr = ((i % 2) == 1) ? (nitr - (0.1 * nitr)) : (nitr + (0.1 * nitr)); - threads.emplace_back(&run, _nitr, nfib); + threads.emplace_back(&run, nitr, nfib); } + // re-enable instrumentation + omnitrace_user_start_thread_trace(); omnitrace_user_pop_region("thread_creation"); - run(nitr - 0.25 * nitr, nfib - 0.1 * nfib); - omnitrace_user_push_region("thread_wait"); for(auto& itr : threads) itr.join(); omnitrace_user_pop_region("thread_wait"); + run(nitr, nfib); + printf("[%s] fibonacci(%li) x %lu = %li\n", argv[0], nfib, nthread, total.load()); omnitrace_user_pop_region(argv[0]); return 0; } + +long +fib(long n) +{ + return (n < 2) ? n : fib(n - 1) + fib(n - 2); +} + +#define RUN_LABEL \ + std::string{ std::string{ __FUNCTION__ } + "(" + std::to_string(n) + ") x " + \ + std::to_string(nitr) } \ + .c_str() + +void +run(size_t nitr, long n) +{ + omnitrace_user_push_region(RUN_LABEL); + long local = 0; + for(size_t i = 0; i < nitr; ++i) + local += fib(n); + total += local; + omnitrace_user_pop_region(RUN_LABEL); +} + +int +custom_push_region(const char* name) +{ + printf("Pushing custom region :: %s\n", name); + return (*omnitrace_push_region_f)(name); +} diff --git a/external/timemory b/external/timemory index 4fcf962d49..de1266606c 160000 --- a/external/timemory +++ b/external/timemory @@ -1 +1 @@ -Subproject commit 4fcf962d495d024b02ea3c5dce71157550d21dd4 +Subproject commit de1266606c403409549bfed814aa1b0412411efc diff --git a/source/bin/CMakeLists.txt b/source/bin/CMakeLists.txt index 94b02f4721..e3fdff5d0b 100644 --- a/source/bin/CMakeLists.txt +++ b/source/bin/CMakeLists.txt @@ -2,6 +2,6 @@ add_subdirectory(omnitrace-avail) add_subdirectory(omnitrace-critical-trace) add_subdirectory(omnitrace) -if(OMNITRACE_BUILD_TESTING) +if(OMNITRACE_BUILD_TESTING OR "$ENV{OMNITRACE_CI}" MATCHES "[1-9]+|ON|on|y|yes") add_subdirectory(tests) endif() diff --git a/source/lib/common/invoke.hpp b/source/lib/common/invoke.hpp index 81b8046d34..ccd4734e7b 100644 --- a/source/lib/common/invoke.hpp +++ b/source/lib/common/invoke.hpp @@ -78,17 +78,18 @@ invoke(const char* _name, int _verbose, FuncT&& _func, Args... _args) int32_t _lk = get_guard()++; if(_lk == 0) { - if(_verbose > 3) + if(_verbose >= 3) { fflush(stderr); fprintf(stderr, - "[omnitrace][" OMNITRACE_COMMON_LIBRARY_NAME "][%li] %s(%s)\n", - get_thread_index(), _name, join(", ", _args...).c_str()); + "[omnitrace][" OMNITRACE_COMMON_LIBRARY_NAME + "][%li][%i] %s(%s)\n", + get_thread_index(), _lk, _name, join(", ", _args...).c_str()); fflush(stderr); } return std::invoke(std::forward(_func), _args...); } - else if(_verbose > 2) + else if(_verbose >= 2) { fflush(stderr); fprintf(stderr, diff --git a/source/lib/omnitrace-dl/CMakeLists.txt b/source/lib/omnitrace-dl/CMakeLists.txt index f7de41496e..15c2a12294 100644 --- a/source/lib/omnitrace-dl/CMakeLists.txt +++ b/source/lib/omnitrace-dl/CMakeLists.txt @@ -14,6 +14,8 @@ add_library(omnitrace::omnitrace-dl-library ALIAS omnitrace-dl-library) target_sources(omnitrace-dl-library PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/dl.cpp) target_link_libraries(omnitrace-dl-library PRIVATE ${dl_LIBRARY} omnitrace::common-library) +target_include_directories(omnitrace-dl-library + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../omnitrace-user) check_cxx_compiler_flag("-fno-exceptions" omnitrace_dl_library_fno_exceptions) check_cxx_compiler_flag("-ftls-model=local-dynamic" diff --git a/source/lib/omnitrace-dl/dl.cpp b/source/lib/omnitrace-dl/dl.cpp index c068e5f92a..5a8739d58b 100644 --- a/source/lib/omnitrace-dl/dl.cpp +++ b/source/lib/omnitrace-dl/dl.cpp @@ -30,6 +30,8 @@ #include "common/invoke.hpp" #include "common/join.hpp" +#include "omnitrace/user.h" + #include #include #include @@ -60,6 +62,11 @@ fprintf(stderr, "[omnitrace][dl][pid=%i]> %s :: %s\n", getpid(), FUNCNAME, \ dlerror()); \ } \ + else if(_omnitrace_dl_verbose > _info_verbose) \ + { \ + fprintf(stderr, "[omnitrace][dl][pid=%i]> %s :: success\n", getpid(), \ + FUNCNAME); \ + } \ } //--------------------------------------------------------------------------------------// @@ -81,14 +88,14 @@ extern "C" void omnitrace_push_trace(const char* name) OMNITRACE_PUBLIC_API; void omnitrace_pop_trace(const char* name) OMNITRACE_PUBLIC_API; - void omnitrace_user_start_trace_dl(void) OMNITRACE_HIDDEN_API; - void omnitrace_user_stop_trace_dl(void) OMNITRACE_HIDDEN_API; + int omnitrace_user_start_trace_dl(void) OMNITRACE_HIDDEN_API; + int omnitrace_user_stop_trace_dl(void) OMNITRACE_HIDDEN_API; - void omnitrace_user_start_thread_trace_dl(void) OMNITRACE_HIDDEN_API; - void omnitrace_user_stop_thread_trace_dl(void) OMNITRACE_HIDDEN_API; + int omnitrace_user_start_thread_trace_dl(void) OMNITRACE_HIDDEN_API; + int omnitrace_user_stop_thread_trace_dl(void) OMNITRACE_HIDDEN_API; - void omnitrace_user_push_region_dl(const char*) OMNITRACE_HIDDEN_API; - void omnitrace_user_pop_region_dl(const char*) OMNITRACE_HIDDEN_API; + int omnitrace_user_push_region_dl(const char*) OMNITRACE_HIDDEN_API; + int omnitrace_user_pop_region_dl(const char*) OMNITRACE_HIDDEN_API; ompt_start_tool_result_t* ompt_start_tool(unsigned int, const char*) OMNITRACE_PUBLIC_API; @@ -168,7 +175,7 @@ struct OMNITRACE_HIDDEN_API indirect : m_omnilib{ find_path(std::move(omnilib)) } , m_userlib{ find_path(std::move(userlib)) } { - if(_omnitrace_dl_verbose > 0) + if(_omnitrace_dl_verbose >= 1) { fprintf(stderr, "[omnitrace][dl][pid=%i] libomnitrace.so resolved to '%s'\n", getpid(), m_omnilib.c_str()); @@ -190,7 +197,7 @@ struct OMNITRACE_HIDDEN_API indirect if(libhandle) { - if(_omnitrace_dl_verbose > 0) + if(_omnitrace_dl_verbose >= 1) { fprintf(stderr, "[omnitrace][dl][pid=%i] dlopen(%s, %s) :: success\n", getpid(), _lib.c_str(), _omnitrace_dl_dlopen_descr); @@ -216,6 +223,7 @@ struct OMNITRACE_HIDDEN_API indirect if(!m_omnihandle) m_omnihandle = open(m_omnilib); int _warn_verbose = 0; + int _info_verbose = 2; // Initialize all pointers OMNITRACE_DLSYM(omnitrace_init_library_f, m_omnihandle, "omnitrace_init_library"); OMNITRACE_DLSYM(omnitrace_init_f, m_omnihandle, "omnitrace_init"); @@ -239,10 +247,17 @@ struct OMNITRACE_HIDDEN_API indirect if(omnitrace_user_configure_f) { (*omnitrace_user_configure_f)( - &omnitrace_user_start_trace_dl, &omnitrace_user_stop_trace_dl, - &omnitrace_user_start_thread_trace_dl, - &omnitrace_user_stop_thread_trace_dl, &omnitrace_user_push_region_dl, - &omnitrace_user_pop_region_dl); + OMNITRACE_USER_START_STOP, + reinterpret_cast(&omnitrace_user_start_trace_dl), + reinterpret_cast(&omnitrace_user_stop_trace_dl)); + (*omnitrace_user_configure_f)( + OMNITRACE_USER_START_STOP_THREAD, + reinterpret_cast(&omnitrace_user_start_thread_trace_dl), + reinterpret_cast(&omnitrace_user_stop_thread_trace_dl)); + (*omnitrace_user_configure_f)( + OMNITRACE_USER_REGION, + reinterpret_cast(&omnitrace_user_push_region_dl), + reinterpret_cast(&omnitrace_user_pop_region_dl)); } } @@ -267,18 +282,16 @@ struct OMNITRACE_HIDDEN_API indirect } public: - void (*omnitrace_init_library_f)(void) = nullptr; - void (*omnitrace_init_f)(const char*, bool, const char*) = nullptr; - void (*omnitrace_finalize_f)(void) = nullptr; - void (*omnitrace_set_env_f)(const char*, const char*) = nullptr; - void (*omnitrace_set_mpi_f)(bool, bool) = nullptr; - void (*omnitrace_push_trace_f)(const char*) = nullptr; - void (*omnitrace_pop_trace_f)(const char*) = nullptr; - void (*omnitrace_push_region_f)(const char*) = nullptr; - void (*omnitrace_pop_region_f)(const char*) = nullptr; - void (*omnitrace_user_configure_f)(void (*)(void), void (*)(void), void (*)(void), - void (*)(void), void (*)(const char*), - void (*)(const char*)) = nullptr; + void (*omnitrace_init_library_f)(void) = nullptr; + void (*omnitrace_init_f)(const char*, bool, const char*) = nullptr; + void (*omnitrace_finalize_f)(void) = nullptr; + void (*omnitrace_set_env_f)(const char*, const char*) = nullptr; + void (*omnitrace_set_mpi_f)(bool, bool) = nullptr; + void (*omnitrace_push_trace_f)(const char*) = nullptr; + void (*omnitrace_pop_trace_f)(const char*) = nullptr; + int (*omnitrace_push_region_f)(const char*) = nullptr; + int (*omnitrace_pop_region_f)(const char*) = nullptr; + int (*omnitrace_user_configure_f)(int, void*, void*) = nullptr; ompt_start_tool_result_t* (*ompt_start_tool_f)(unsigned int, const char*); private: @@ -403,21 +416,40 @@ extern "C" OMNITRACE_DL_INVOKE(get_indirect().omnitrace_set_mpi_f, a, b); } - void omnitrace_user_start_trace_dl(void) { dl::get_enabled().store(true); } - void omnitrace_user_stop_trace_dl(void) { dl::get_enabled().store(false); } - void omnitrace_user_start_thread_trace_dl(void) { dl::get_thread_enabled() = true; } - void omnitrace_user_stop_thread_trace_dl(void) { dl::get_thread_enabled() = false; } - - void omnitrace_user_push_region_dl(const char* name) + int omnitrace_user_start_trace_dl(void) { - if(!dl::get_active()) return; - OMNITRACE_DL_INVOKE(get_indirect().omnitrace_push_region_f, name); + dl::get_enabled().store(true); + return omnitrace_user_start_thread_trace_dl(); } - void omnitrace_user_pop_region_dl(const char* name) + int omnitrace_user_stop_trace_dl(void) { - if(!dl::get_active()) return; - OMNITRACE_DL_INVOKE(get_indirect().omnitrace_pop_region_f, name); + dl::get_enabled().store(false); + return omnitrace_user_stop_thread_trace_dl(); + } + + int omnitrace_user_start_thread_trace_dl(void) + { + dl::get_thread_enabled() = true; + return 0; + } + + int omnitrace_user_stop_thread_trace_dl(void) + { + dl::get_thread_enabled() = false; + return 0; + } + + int omnitrace_user_push_region_dl(const char* name) + { + if(!dl::get_active()) return 0; + return OMNITRACE_DL_INVOKE(get_indirect().omnitrace_push_region_f, name); + } + + int omnitrace_user_pop_region_dl(const char* name) + { + if(!dl::get_active()) return 0; + return OMNITRACE_DL_INVOKE(get_indirect().omnitrace_pop_region_f, name); } ompt_start_tool_result_t* ompt_start_tool(unsigned int omp_version, diff --git a/source/lib/omnitrace-user/omnitrace/user.h b/source/lib/omnitrace-user/omnitrace/user.h index a5f554399e..12c2ef2856 100644 --- a/source/lib/omnitrace-user/omnitrace/user.h +++ b/source/lib/omnitrace-user/omnitrace/user.h @@ -32,16 +32,125 @@ extern "C" { #endif - extern void omnitrace_user_start_trace(void) OMNITRACE_PUBLIC_API; - extern void omnitrace_user_stop_trace(void) OMNITRACE_PUBLIC_API; - extern void omnitrace_user_start_thread_trace(void) OMNITRACE_PUBLIC_API; - extern void omnitrace_user_stop_thread_trace(void) OMNITRACE_PUBLIC_API; - extern void omnitrace_user_push_region(const char*) OMNITRACE_PUBLIC_API; - extern void omnitrace_user_pop_region(const char*) OMNITRACE_PUBLIC_API; - extern void omnitrace_user_configure(void (*)(void), void (*)(void), void (*)(void), - void (*)(void), void (*)(const char*), - void (*)(const char*)) OMNITRACE_PUBLIC_API; + /// @enum OMNITRACE_USER_ERROR + /// @brief Identifier for errors + /// + enum OMNITRACE_USER_ERROR + { + OMNITRACE_USER_SUCCESS = 0, ///< No error + OMNITRACE_USER_ERROR_NO_BINDING, ///< Function pointer was not assigned + OMNITRACE_USER_ERROR_BAD_FUNCTION_POINTER, ///< Provided function pointer was + ///< invalid + OMNITRACE_USER_ERROR_INVALID_CATEGORY, ///< Invalid user binding category + OMNITRACE_USER_ERROR_INTERNAL, ///< Internal error occurred within libomnitrace + OMNITRACE_USER_ERROR_LAST + }; + + /// @enum OMNITRACE_USER_BINDINGS + /// @brief Identifier for function pointer categories + /// @code{.cpp} + /// int (*omnitrace_push_region_f)(const char*) = nullptr; + /// + /// int custom_push_region(const char* name) + /// { + /// // custom push region prints message before calling internal callback + /// printf("Pushing region %s\n", name); + /// return (*omnitrace_push_region_f)(name); + /// } + /// + /// int main(int argc, char** argv) + /// { + /// // get the internal callback to start a user-defined region + /// omnitrace_user_get_callbacks(OMNITRACE_USER_REGION, + /// (void**) &omnitrace_push_region_f, + /// nullptr); + /// // assign the custom callback to start a user-defined region + /// if(omnitrace_push_region_f) + /// omnitrace_user_configure(OMNITRACE_USER_REGION, + /// (void*) &custom_push_region, + /// nullptr); + /// // ... + /// } + /// + /// @endcode + enum OMNITRACE_USER_BINDINGS + { + OMNITRACE_USER_START_STOP = + 0, ///< Function pointers which control global start/stop + OMNITRACE_USER_START_STOP_THREAD, ///< Function pointers which control per-thread + ///< start/stop + OMNITRACE_USER_REGION, ///< Function pointers which generate user-defined regions + OMNITRACE_USER_BINDINGS_LAST + }; + + /// @fn int omnitrace_user_start_trace(void) + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Enable tracing on this thread and all subsequently created threads + extern int omnitrace_user_start_trace(void) OMNITRACE_PUBLIC_API; + + /// @fn int omnitrace_user_stop_trace(void) + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Disable tracing on this thread and all subsequently created threads + extern int omnitrace_user_stop_trace(void) OMNITRACE_PUBLIC_API; + + /// @fn int omnitrace_user_start_thread_trace(void) + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Enable tracing on this specific thread. Does not apply to subsequently + /// created threads + extern int omnitrace_user_start_thread_trace(void) OMNITRACE_PUBLIC_API; + + /// @fn int omnitrace_user_stop_thread_trace(void) + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Disable tracing on this specific thread. Does not apply to subsequently + /// created threads + extern int omnitrace_user_stop_thread_trace(void) OMNITRACE_PUBLIC_API; + + /// @fn int omnitrace_user_push_region(const char* id) + /// @param id The string identifier for the region + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Start a user defined region. + extern int omnitrace_user_push_region(const char*) OMNITRACE_PUBLIC_API; + + /// @fn int omnitrace_user_pop_region(const char* id) + /// @param id The string identifier for the region + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief End a user defined region. In general, user regions should be popped in + /// the inverse order that they were pushed, i.e. first-in, last-out (FILO). The + /// timemory backend was designed to accommodate asynchronous tasking, where FILO may + /// be violated, and will thus compenstate for out-of-order popping, however, the + /// perfetto backend will not; thus, out-of-order popping will result in different + /// results in timemory vs. perfetto. + extern int omnitrace_user_pop_region(const char*) OMNITRACE_PUBLIC_API; + + /// @fn int omnitrace_user_configure(int category, void* begin_func, void* end_func) + /// @param category An @ref OMNITRACE_USER_BINDINGS value + /// @param begin_func The pointer to the function which corresponds to "starting" the + /// category, e.g. omnitrace_user_start_trace or omnitrace_user_push_region + /// @param end_func The pointer to the function which corresponds to "ending" the + /// category, e.g. omnitrace_user_stop_trace or omnitrace_user_pop_region + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Configure the function pointers for a given category. This is handled by + /// omnitrace-dl at start up but the user can specify their own if desired. + extern int omnitrace_user_configure(int, void*, void*) OMNITRACE_PUBLIC_API; + + /// @fn int omnitrace_user_get_callbacks(int category, void** begin_func, void** + /// end_func) + /// @param[in] category An @ref OMNITRACE_USER_BINDINGS value + /// @param[out] begin_func The pointer to the function which corresponds to "starting" + /// the category, e.g. omnitrace_user_start_trace or omnitrace_user_push_region + /// @param[out] end_func The pointer to the function which corresponds to "ending" the + /// category, e.g. omnitrace_user_stop_trace or omnitrace_user_pop_region + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Get the current function pointers for a given category. The initial values + /// are assigned by omnitrace-dl at start up. + extern int omnitrace_user_get_callbacks(int, void**, void**) OMNITRACE_PUBLIC_API; + + /// @fn const char* omnitrace_user_error_string(int error_category) + /// @param error_category OMNITRACE_USER_ERROR value + /// @return @ref OMNITRACE_USER_ERROR value + /// @brief Return a descriptor for the provided error code + extern const char* omnitrace_user_error_string(int) OMNITRACE_PUBLIC_API; #if defined(__cplusplus) } diff --git a/source/lib/omnitrace-user/user.cpp b/source/lib/omnitrace-user/user.cpp index 1bc2c317bf..8e7478f0a9 100644 --- a/source/lib/omnitrace-user/user.cpp +++ b/source/lib/omnitrace-user/user.cpp @@ -25,60 +25,134 @@ #include "omnitrace/user.h" #include +#include namespace { -using void_func_t = void (*)(void); -using region_func_t = void (*)(const char*); +using trace_func_t = int (*)(void); +using region_func_t = int (*)(const char*); -void_func_t _start_trace = nullptr; -void_func_t _stop_trace = nullptr; -void_func_t _start_thread_trace = nullptr; -void_func_t _stop_thread_trace = nullptr; +trace_func_t _start_trace = nullptr; +trace_func_t _stop_trace = nullptr; +trace_func_t _start_thread_trace = nullptr; +trace_func_t _stop_thread_trace = nullptr; region_func_t _push_region = nullptr; region_func_t _pop_region = nullptr; + +const char* +as_string(OMNITRACE_USER_BINDINGS _category) +{ + switch(_category) + { + case OMNITRACE_USER_START_STOP: return "OMNITRACE_USER_START_STOP"; + case OMNITRACE_USER_START_STOP_THREAD: return "OMNITRACE_USER_START_STOP_THREAD"; + case OMNITRACE_USER_REGION: return "OMNITRACE_USER_REGION"; + default: + { + fprintf(stderr, "[omnitrace][user] Unknown user binding category: %i\n", + static_cast(_category)); + fflush(stderr); + break; + } + } + return "OMNITRACE_USER_BINDINGS_unknown"; +} + +template +inline auto +invoke(int (*_func)(Args...), Args... args) +{ + if(!_func) return OMNITRACE_USER_ERROR_NO_BINDING; + if((*_func)(args...) != 0) return OMNITRACE_USER_ERROR_INTERNAL; + return OMNITRACE_USER_SUCCESS; +} } // namespace extern "C" { - void omnitrace_user_start_trace(void) + int omnitrace_user_start_trace(void) { return invoke(_start_trace); } + int omnitrace_user_stop_trace(void) { return invoke(_stop_trace); } + int omnitrace_user_start_thread_trace(void) { return invoke(_start_thread_trace); } + int omnitrace_user_stop_thread_trace(void) { return invoke(_stop_thread_trace); } + int omnitrace_user_push_region(const char* id) { return invoke(_push_region, id); } + int omnitrace_user_pop_region(const char* id) { return invoke(_pop_region, id); } + + int omnitrace_user_configure(int category, void* begin_func, void* end_func) { - if(_start_trace) (*_start_trace)(); + switch(category) + { + case OMNITRACE_USER_START_STOP: + { + if(begin_func) _start_trace = reinterpret_cast(begin_func); + if(end_func) _stop_trace = reinterpret_cast(end_func); + break; + } + case OMNITRACE_USER_START_STOP_THREAD: + { + if(begin_func) + _start_thread_trace = reinterpret_cast(begin_func); + if(end_func) + _stop_thread_trace = reinterpret_cast(end_func); + break; + } + case OMNITRACE_USER_REGION: + { + if(begin_func) _push_region = reinterpret_cast(begin_func); + if(end_func) _pop_region = reinterpret_cast(end_func); + break; + } + default: + { + return OMNITRACE_USER_ERROR_INVALID_CATEGORY; + } + } + + return OMNITRACE_USER_SUCCESS; } - void omnitrace_user_stop_trace(void) + int omnitrace_user_get_callbacks(int category, void** begin_func, void** end_func) { - if(_stop_trace) (*_stop_trace)(); + switch(category) + { + case OMNITRACE_USER_START_STOP: + { + if(begin_func) *begin_func = reinterpret_cast(_start_trace); + if(end_func) *end_func = reinterpret_cast(_stop_trace); + break; + } + case OMNITRACE_USER_START_STOP_THREAD: + { + if(begin_func) *begin_func = reinterpret_cast(_start_thread_trace); + if(end_func) *end_func = reinterpret_cast(_stop_thread_trace); + break; + } + case OMNITRACE_USER_REGION: + { + if(begin_func) *begin_func = reinterpret_cast(_push_region); + if(end_func) *end_func = reinterpret_cast(_pop_region); + break; + } + default: + { + return OMNITRACE_USER_ERROR_INVALID_CATEGORY; + } + } + + return OMNITRACE_USER_SUCCESS; } - void omnitrace_user_start_thread_trace(void) + const char* omnitrace_user_error_string(int error_category) { - if(_start_thread_trace) (*_start_thread_trace)(); - } - - void omnitrace_user_stop_thread_trace(void) - { - if(_stop_thread_trace) (*_stop_thread_trace)(); - } - - void omnitrace_user_push_region(const char* name) - { - if(_push_region) (*_push_region)(name); - } - - void omnitrace_user_pop_region(const char* name) - { - if(_pop_region) (*_pop_region)(name); - } - - void omnitrace_user_configure(void (*_a)(), void (*_b)(), void (*_c)(), void (*_d)(), - void (*_e)(const char*), void (*_f)(const char*)) - { - _start_trace = _a; - _stop_trace = _b; - _start_thread_trace = _c; - _stop_thread_trace = _d; - _push_region = _e; - _pop_region = _f; + switch(error_category) + { + case OMNITRACE_USER_SUCCESS: return "Success"; + case OMNITRACE_USER_ERROR_NO_BINDING: return "Function pointer not assigned"; + case OMNITRACE_USER_ERROR_INVALID_CATEGORY: + return "Invalid user binding category"; + case OMNITRACE_USER_ERROR_INTERNAL: + return "An unknown error occurred within omnitrace library"; + default: break; + } + return "No error"; } } diff --git a/source/lib/omnitrace/include/library/api.hpp b/source/lib/omnitrace/include/library/api.hpp index 8444d23ef4..38e05cbb39 100644 --- a/source/lib/omnitrace/include/library/api.hpp +++ b/source/lib/omnitrace/include/library/api.hpp @@ -52,10 +52,10 @@ extern "C" void omnitrace_pop_trace(const char* name) OMNITRACE_PUBLIC_API; /// starts an instrumentation region (user-defined) - void omnitrace_push_region(const char* name) OMNITRACE_PUBLIC_API; + int omnitrace_push_region(const char* name) OMNITRACE_PUBLIC_API; /// stops an instrumentation region (user-defined) - void omnitrace_pop_region(const char* name) OMNITRACE_PUBLIC_API; + int omnitrace_pop_region(const char* name) OMNITRACE_PUBLIC_API; // these are the real implementations for internal calling convention void omnitrace_init_library_hidden(void) OMNITRACE_HIDDEN_API; diff --git a/source/lib/omnitrace/include/library/debug.hpp b/source/lib/omnitrace/include/library/debug.hpp index 480120c06c..e64e8155b3 100644 --- a/source/lib/omnitrace/include/library/debug.hpp +++ b/source/lib/omnitrace/include/library/debug.hpp @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,12 @@ inline namespace config bool get_debug(); +int +get_verbose(); + +bool +get_is_continuous_integration(); + bool get_debug_tid(); @@ -56,17 +63,11 @@ namespace debug namespace { template -auto get_chars(std::index_sequence) -{ - static const char _v[sizeof...(Idx) + 1] = { T::get()[Idx]..., '\0' }; - return _v; -} - -template auto -get_chars() +get_chars(T&& _c, std::index_sequence) { - return get_chars(typename T::sequence{}); + return std::array{ std::forward(_c)[Idx]..., + '\0' }; } } // namespace } // namespace debug @@ -86,25 +87,31 @@ get_chars() # define OMNITRACE_PROCESS_IDENTIFIER static_cast(::tim::process::get_id()) #endif -#define OMNITRACE_FUNCTION \ - std::string{ __FUNCTION__ } \ - .substr(0, std::string_view{ __FUNCTION__ }.find("_hidden")) \ - .c_str() - -#define OMNITRACE_CT_FUNCTION(VAR, STR) \ - static constexpr auto OMNITRACE_VARIABLE(__LINE__) = std::string_view{ STR }; \ - VAR = OMNITRACE_CT_FUNCTION_IMPL(OMNITRACE_VARIABLE(__LINE__)); - -#define OMNITRACE_CT_FUNCTION_IMPL(STR) \ - []() { \ - struct wrapper \ - { \ - static constexpr const char* get() { return STR.data(); } \ - using sequence = \ - std::make_index_sequence; \ - }; \ - return ::omnitrace::debug::get_chars(); \ - }(); +#if defined(__clang__) || (__GNUC__ < 9) +# define OMNITRACE_FUNCTION \ + std::string{ __FUNCTION__ } \ + .substr(0, std::string_view{ __FUNCTION__ }.find("_hidden")) \ + .c_str() +# define OMNITRACE_PRETTY_FUNCTION \ + std::string{ __PRETTY_FUNCTION__ } \ + .substr(0, std::string_view{ __PRETTY_FUNCTION__ }.find("_hidden")) \ + .c_str() +#else +# define OMNITRACE_FUNCTION \ + ::omnitrace::debug::get_chars( \ + std::string_view{ __FUNCTION__ }, \ + std::make_index_sequence{}) \ + .data() +# define OMNITRACE_PRETTY_FUNCTION \ + ::omnitrace::debug::get_chars( \ + std::string_view{ __PRETTY_FUNCTION__ }, \ + std::make_index_sequence{}) \ + .data() +#endif #define OMNITRACE_CONDITIONAL_PRINT(COND, ...) \ if((COND) && ::omnitrace::config::get_debug_tid() && \ diff --git a/source/lib/omnitrace/src/library.cpp b/source/lib/omnitrace/src/library.cpp index 5eb11106a5..e8950d7d44 100644 --- a/source/lib/omnitrace/src/library.cpp +++ b/source/lib/omnitrace/src/library.cpp @@ -103,11 +103,11 @@ ensure_finalization(bool _static_init = false) (void) _main_tid; if(!_static_init) { - OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); + OMNITRACE_DEBUG_F("\n"); } else { - OMNITRACE_CONDITIONAL_PRINT(get_debug_env(), "[%s]\n", __FUNCTION__); + OMNITRACE_CONDITIONAL_PRINT_F(get_debug_env(), "\n"); // This environment variable forces the ROCR-Runtime to use polling to wait // for signals rather than interrupts. We set this variable to avoid issues with // rocm/roctracer hanging when interrupted by the sampler @@ -345,9 +345,8 @@ extern "C" void omnitrace_set_env_hidden(const char* env_name, const char* env_val) { // just search env to avoid initializing the settings - OMNITRACE_CONDITIONAL_PRINT(get_debug_init() || get_verbose_env() > 2, - "[%s] Setting env: %s=%s\n", __FUNCTION__, env_name, - env_val); + OMNITRACE_CONDITIONAL_PRINT_F(get_debug_init() || get_verbose_env() > 2, + "Setting env: %s=%s\n", env_name, env_val); tim::set_env(env_name, env_val, 0); @@ -375,9 +374,9 @@ extern "C" void omnitrace_set_mpi_hidden(bool use, bool attached) { // just search env to avoid initializing the settings - OMNITRACE_CONDITIONAL_PRINT(get_debug_init() || get_verbose_env() > 2, - "[%s] use: %s, attached: %s\n", __FUNCTION__, - (use) ? "y" : "n", (attached) ? "y" : "n"); + OMNITRACE_CONDITIONAL_PRINT_F(get_debug_init() || get_verbose_env() > 2, + "use: %s, attached: %s\n", (use) ? "y" : "n", + (attached) ? "y" : "n"); _set_mpi_called = true; config::is_attached() = attached; @@ -513,7 +512,8 @@ omnitrace_init_tooling_hidden() OMNITRACE_CONDITIONAL_THROW( get_state() == State::Init, - "%s called after omnitrace_init_library() was explicitly called", __FUNCTION__); + "%s called after omnitrace_init_library() was explicitly called", + OMNITRACE_FUNCTION); OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_verbose_env() >= 0, "Instrumentation mode: %s\n", @@ -828,9 +828,9 @@ omnitrace_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _a if(get_state() != State::Finalized) omnitrace_finalize_hidden(); }); - OMNITRACE_CONDITIONAL_BASIC_PRINT( + OMNITRACE_CONDITIONAL_BASIC_PRINT_F( get_debug_env() || get_verbose_env() > 2, - "[%s] mode: %s | is binary rewrite: %s | command: %s\n", __FUNCTION__, _mode, + "mode: %s | is binary rewrite: %s | command: %s\n", _mode, (_is_binary_rewrite) ? "y" : "n", _argv0); tim::set_env("OMNITRACE_MODE", _mode, 0); @@ -1080,29 +1080,31 @@ omnitrace_finalize_hidden(void) { if(get_verbose() >= 0) fprintf(stderr, "\n"); if(get_verbose() >= 0 || get_debug()) - fprintf(stderr, "[%s]|%i> Flushing perfetto...\n", __FUNCTION__, dmp::rank()); + fprintf(stderr, "[%s]|%i> Flushing perfetto...\n", OMNITRACE_FUNCTION, + dmp::rank()); // Make sure the last event is closed for this example. perfetto::TrackEvent::Flush(); auto& tracing_session = get_trace_session(); - OMNITRACE_DEBUG_F("Stopping the blocking perfetto trace sessions...\n"); + OMNITRACE_VERBOSE_F(3, "Stopping the blocking perfetto trace sessions...\n"); tracing_session->StopBlocking(); - OMNITRACE_DEBUG_F("Getting the trace data...\n"); + OMNITRACE_VERBOSE_F(3, "Getting the trace data...\n"); std::vector trace_data{ tracing_session->ReadTraceBlocking() }; if(trace_data.empty()) { fprintf(stderr, "[%s]> trace data is empty. File '%s' will not be written...\n", - __FUNCTION__, get_perfetto_output_filename().c_str()); + OMNITRACE_FUNCTION, get_perfetto_output_filename().c_str()); return; } // Write the trace into a file. if(get_verbose() >= 0) fprintf(stderr, "[%s]|%i> Outputting '%s' (%.2f KB / %.2f MB / %.2f GB)... ", - __FUNCTION__, dmp::rank(), get_perfetto_output_filename().c_str(), + OMNITRACE_FUNCTION, dmp::rank(), + get_perfetto_output_filename().c_str(), static_cast(trace_data.size()) / units::KB, static_cast(trace_data.size()) / units::MB, static_cast(trace_data.size()) / units::GB); @@ -1110,7 +1112,7 @@ omnitrace_finalize_hidden(void) if(!tim::filepath::open(ofs, get_perfetto_output_filename(), std::ios::out | std::ios::binary)) { - fprintf(stderr, "\n[%s]> Error opening '%s'...\n", __FUNCTION__, + fprintf(stderr, "\n[%s]> Error opening '%s'...\n", OMNITRACE_FUNCTION, get_perfetto_output_filename().c_str()); _perfetto_output_error = true; } diff --git a/source/lib/omnitrace/src/library/api.cpp b/source/lib/omnitrace/src/library/api.cpp index 7d8aa32fab..09ee76c6bb 100644 --- a/source/lib/omnitrace/src/library/api.cpp +++ b/source/lib/omnitrace/src/library/api.cpp @@ -21,6 +21,10 @@ // SOFTWARE. #include "library/api.hpp" +#include "library/debug.hpp" + +#include +#include extern "C" void omnitrace_push_trace(const char* _name) @@ -34,16 +38,32 @@ omnitrace_pop_trace(const char* _name) omnitrace_pop_trace_hidden(_name); } -extern "C" void +extern "C" int omnitrace_push_region(const char* _name) { - omnitrace_push_region_hidden(_name); + try + { + omnitrace_push_region_hidden(_name); + } catch(std::exception& _e) + { + OMNITRACE_VERBOSE_F(1, "Exception caught: %s\n", _e.what()); + return -1; + } + return 0; } -extern "C" void +extern "C" int omnitrace_pop_region(const char* _name) { - omnitrace_pop_region_hidden(_name); + try + { + omnitrace_pop_region_hidden(_name); + } catch(std::exception& _e) + { + OMNITRACE_VERBOSE_F(1, "Exception caught: %s\n", _e.what()); + return -1; + } + return 0; } extern "C" void diff --git a/source/lib/omnitrace/src/library/components/mpi_gotcha.cpp b/source/lib/omnitrace/src/library/components/mpi_gotcha.cpp index 4a1022a4cf..617eb528c4 100644 --- a/source/lib/omnitrace/src/library/components/mpi_gotcha.cpp +++ b/source/lib/omnitrace/src/library/components/mpi_gotcha.cpp @@ -51,7 +51,7 @@ omnitrace_mpi_set_attr() if(mpip_index != std::numeric_limits::max()) comp::deactivate_mpip, api::omnitrace>(mpip_index); - omnitrace_finalize(); + omnitrace_finalize_hidden(); return MPI_SUCCESS; }; using copy_func_t = int (*)(MPI_Comm, int, void*, void*, void*, int*); @@ -83,8 +83,8 @@ mpi_gotcha::configure() void mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***) { - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s(int*, char***)\n", - __FUNCTION__, _data.tool_id.c_str()); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug_env(), "%s(int*, char***)\n", + _data.tool_id.c_str()); if(get_state() < ::omnitrace::State::Init) set_state(::omnitrace::State::PreInit); @@ -98,9 +98,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***) void mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***, int, int*) { - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), - "[%s] %s(int*, char***, int, int*)\n", __FUNCTION__, - _data.tool_id.c_str()); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug_env(), "%s(int*, char***, int, int*)\n", + _data.tool_id.c_str()); if(get_state() < ::omnitrace::State::Init) set_state(::omnitrace::State::PreInit); @@ -114,8 +113,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***, in void mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming) { - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s()\n", __FUNCTION__, - _data.tool_id.c_str()); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug_env(), "%s()\n", _data.tool_id.c_str()); if(mpip_index != std::numeric_limits::max()) comp::deactivate_mpip, @@ -125,15 +123,14 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming) tim::mpi::is_initialized_callback() = []() { return false; }; tim::mpi::is_finalized() = true; #else - omnitrace_finalize(); + omnitrace_finalize_hidden(); #endif } void mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, comm_t, int* _val) { - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s()\n", __FUNCTION__, - _data.tool_id.c_str()); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug_env(), "%s()\n", _data.tool_id.c_str()); omnitrace_push_trace_hidden(_data.tool_id.c_str()); if(_data.tool_id == "MPI_Comm_rank") @@ -146,16 +143,16 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, comm_t, int* _val } else { - OMNITRACE_BASIC_PRINT("[%s] %s(, %p) :: unexpected function wrapper\n", - __FUNCTION__, _data.tool_id.c_str(), _val); + OMNITRACE_BASIC_PRINT_F("%s(, %p) :: unexpected function wrapper\n", + _data.tool_id.c_str(), _val); } } void mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) { - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s() returned %i\n", - __FUNCTION__, _data.tool_id.c_str(), (int) _retval); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug_env(), "%s() returned %i\n", + _data.tool_id.c_str(), (int) _retval); if(_retval == tim::mpi::success_v && _data.tool_id.find("MPI_Init") == 0) { @@ -167,9 +164,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) // were excluded via a regex expression) if(get_use_mpip()) { - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env() || get_verbose_env() > 0, - "[%s] Activating MPI wrappers...\n", - __FUNCTION__); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug_env() || get_verbose_env() > 0, + "Activating MPI wrappers...\n"); // use env vars OMNITRACE_MPIP_PERMIT_LIST and OMNITRACE_MPIP_REJECT_LIST // to control the gotcha bindings at runtime @@ -196,8 +192,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) } else { - OMNITRACE_BASIC_PRINT("[%s] %s() returned %i :: nullptr to rank\n", - __FUNCTION__, _data.tool_id.c_str(), (int) _retval); + OMNITRACE_BASIC_PRINT_F("%s() returned %i :: nullptr to rank\n", + _data.tool_id.c_str(), (int) _retval); } } else if(_data.tool_id == "MPI_Comm_size") @@ -212,15 +208,14 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) } else { - OMNITRACE_BASIC_PRINT("[%s] %s() returned %i :: nullptr to size\n", - __FUNCTION__, _data.tool_id.c_str(), (int) _retval); + OMNITRACE_BASIC_PRINT_F("%s() returned %i :: nullptr to size\n", + _data.tool_id.c_str(), (int) _retval); } } else { - OMNITRACE_BASIC_PRINT( - "[%s] %s() returned %i :: unexpected function wrapper\n", __FUNCTION__, - _data.tool_id.c_str(), (int) _retval); + OMNITRACE_BASIC_PRINT_F("%s() returned %i :: unexpected function wrapper\n", + _data.tool_id.c_str(), (int) _retval); } } omnitrace_pop_trace_hidden(_data.tool_id.c_str()); diff --git a/source/lib/omnitrace/src/library/components/roctracer.cpp b/source/lib/omnitrace/src/library/components/roctracer.cpp index 5f0d1d9225..4c52c95393 100644 --- a/source/lib/omnitrace/src/library/components/roctracer.cpp +++ b/source/lib/omnitrace/src/library/components/roctracer.cpp @@ -170,7 +170,7 @@ roctracer::shutdown() itr.second(); #if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR >= 4 - OMNITRACE_DEBUG("[%s] redirecting roctracer warnings\n", __FUNCTION__); + OMNITRACE_DEBUG_F("redirecting roctracer warnings\n"); // HIP 4.5.0 has an invalid warning redirect _rd{ std::cerr, "roctracer_disable_callback(), get_op_end(), invalid domain ID(4) " diff --git a/source/lib/omnitrace/src/library/components/roctracer_callbacks.cpp b/source/lib/omnitrace/src/library/components/roctracer_callbacks.cpp index b69ab100ec..60280071c4 100644 --- a/source/lib/omnitrace/src/library/components/roctracer_callbacks.cpp +++ b/source/lib/omnitrace/src/library/components/roctracer_callbacks.cpp @@ -685,17 +685,16 @@ extern "C" const char* const* failed_tool_names) { pthread_gotcha::push_enable_sampling_on_child_threads(false); - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env() || get_verbose_env() > 0, - "[%s]\n", __FUNCTION__); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug_env() || get_verbose_env() > 0, + "\n"); tim::consume_parameters(table, runtime_version, failed_tool_count, failed_tool_names); auto _setup = [=]() { try { - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1, - "[%s] setting up HSA...\n", - __FUNCTION__); + OMNITRACE_CONDITIONAL_BASIC_PRINT_F(get_debug() || get_verbose() > 1, + "setting up HSA...\n"); // const char* output_prefix = getenv("ROCP_OUTPUT_DIR"); const char* output_prefix = nullptr; @@ -763,10 +762,10 @@ extern "C" }; auto _shutdown = []() { - OMNITRACE_DEBUG("[%s] roctracer_disable_domain_callback\n", __FUNCTION__); + OMNITRACE_DEBUG_F("roctracer_disable_domain_callback\n"); ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API)); - OMNITRACE_DEBUG("[%s] roctracer_disable_op_activity\n", __FUNCTION__); + OMNITRACE_DEBUG_F("roctracer_disable_op_activity\n"); ROCTRACER_CALL( roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY)); }; @@ -784,7 +783,7 @@ extern "C" // HSA-runtime on-unload method void OnUnload() { - OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); + OMNITRACE_DEBUG_F("\n"); rocm_smi::set_state(State::Finalized); comp::roctracer::shutdown(); } diff --git a/source/lib/omnitrace/src/library/config.cpp b/source/lib/omnitrace/src/library/config.cpp index b0ece73ae6..f50cee88ec 100644 --- a/source/lib/omnitrace/src/library/config.cpp +++ b/source/lib/omnitrace/src/library/config.cpp @@ -1054,10 +1054,9 @@ setup_gotchas() if(_initialized) return; _initialized = true; - OMNITRACE_CONDITIONAL_PRINT( + OMNITRACE_CONDITIONAL_PRINT_F( get_debug_env(), - "[%s] Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n", - __FUNCTION__); + "Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n"); mpi_gotcha::configure(); fork_gotcha::configure(); diff --git a/source/lib/omnitrace/src/library/ptl.cpp b/source/lib/omnitrace/src/library/ptl.cpp index 0e5b9a414b..a2cf9d98a6 100644 --- a/source/lib/omnitrace/src/library/ptl.cpp +++ b/source/lib/omnitrace/src/library/ptl.cpp @@ -40,6 +40,7 @@ auto _thread_pool_cfg = []() { _v.init = true; _v.use_affinity = false; _v.use_tbb = false; + _v.verbose = -1; _v.initializer = []() { sampling::block_signals(); threading::set_thread_name(