diff --git a/projects/rocprofiler-systems/.github/workflows/opensuse.yml b/projects/rocprofiler-systems/.github/workflows/opensuse.yml index 858d31799e..71290a7089 100644 --- a/projects/rocprofiler-systems/.github/workflows/opensuse.yml +++ b/projects/rocprofiler-systems/.github/workflows/opensuse.yml @@ -58,6 +58,7 @@ jobs: -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10" -DOMNITRACE_CI_MPI_RUN_AS_ROOT=ON + -DOMNITRACE_MAX_THREADS=32 - name: Build timeout-minutes: 60 diff --git a/projects/rocprofiler-systems/.github/workflows/ubuntu-bionic.yml b/projects/rocprofiler-systems/.github/workflows/ubuntu-bionic.yml index 5b64c1f864..823f44e69b 100644 --- a/projects/rocprofiler-systems/.github/workflows/ubuntu-bionic.yml +++ b/projects/rocprofiler-systems/.github/workflows/ubuntu-bionic.yml @@ -88,6 +88,7 @@ jobs: -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10" -DLULESH_BUILD_KOKKOS=OFF + -DOMNITRACE_MAX_THREADS=32 - name: Build timeout-minutes: 60 diff --git a/projects/rocprofiler-systems/.github/workflows/ubuntu-focal.yml b/projects/rocprofiler-systems/.github/workflows/ubuntu-focal.yml index 47715db39e..56327d0bae 100644 --- a/projects/rocprofiler-systems/.github/workflows/ubuntu-focal.yml +++ b/projects/rocprofiler-systems/.github/workflows/ubuntu-focal.yml @@ -103,6 +103,7 @@ jobs: -DOMNITRACE_BUILD_STATIC_LIBSTDCXX=${{ matrix.static-libstdcxx }} -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10" + -DOMNITRACE_MAX_THREADS=32 - name: Build timeout-minutes: 60 @@ -212,8 +213,8 @@ jobs: git config --global --add safe.directory ${PWD} && cmake --version && cmake -B build - -DCMAKE_C_COMPILER=${CC} - -DCMAKE_CXX_COMPILER=${CXX} + -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_INSTALL_PREFIX=/opt/omnitrace -DOMNITRACE_BUILD_TESTING=OFF @@ -223,7 +224,7 @@ jobs: -DOMNITRACE_USE_MPI=OFF -DOMNITRACE_USE_MPI_HEADERS=ON -DOMNITRACE_USE_HIP=ON - -DOMNITRACE_MAX_THREADS=64 + -DOMNITRACE_MAX_THREADS=32 -DOMNITRACE_USE_SANITIZER=OFF -DOMNITRACE_USE_PAPI=OFF -DOMNITRACE_INSTALL_PERFETTO_TOOLS=ON @@ -367,6 +368,7 @@ jobs: -DDYNINST_BUILD_SHARED_LIBS=ON -DDYNINST_BUILD_STATIC_LIBS=OFF -DDYNINST_ELFUTILS_DOWNLOAD_VERSION=${{ env.ELFUTILS_DOWNLOAD_VERSION }} + -DOMNITRACE_MAX_THREADS=32 - name: Build timeout-minutes: 60 diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp index dbd62228df..9bcc3903ab 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/config.cpp @@ -688,11 +688,22 @@ configure_mode_settings() if(get_use_kokkosp()) { - auto _force = 0; auto _current_kokkosp_lib = tim::get_env("KOKKOS_PROFILE_LIBRARY"); - if(std::regex_search(_current_kokkosp_lib, std::regex{ "libtimemory\\." })) - _force = 1; - tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force); + if(_current_kokkosp_lib.find("libomnitrace-dl.so") == std::string::npos && + _current_kokkosp_lib.find("libomnitrace.so") == std::string::npos) + { + auto _force = 0; + std::string _message = {}; + if(std::regex_search(_current_kokkosp_lib, std::regex{ "libtimemory\\." })) + { + _force = 1; + _message = + JOIN("", " (forced. Previous value: '", _current_kokkosp_lib, "')"); + } + OMNITRACE_VERBOSE_F(1, "Setting KOKKOS_PROFILE_LIBRARY=%s%s\n", + "libomnitrace.so", _message.c_str()); + tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force); + } } // recycle all subsequent thread ids diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/debug.hpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/debug.hpp index f7ee27c4c1..102df8a83e 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/debug.hpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/debug.hpp @@ -246,7 +246,7 @@ get_chars(T&& _c, std::index_sequence) //--------------------------------------------------------------------------------------// -#define OMNITRACE_CONDITIONAL_FAIL(COND, ...) \ +#define OMNITRACE_CONDITIONAL_FAILURE(COND, METHOD, ...) \ if(COND) \ { \ ::omnitrace::debug::flush(); \ @@ -257,10 +257,10 @@ get_chars(T&& _c, std::index_sequence) ::omnitrace::set_state(::omnitrace::State::Finalized); \ ::tim::disable_signal_detection(); \ ::tim::print_demangled_backtrace<64>(); \ - ::std::exit(EXIT_FAILURE); \ + METHOD; \ } -#define OMNITRACE_CONDITIONAL_BASIC_FAIL(COND, ...) \ +#define OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, METHOD, ...) \ if(COND) \ { \ ::omnitrace::debug::flush(); \ @@ -270,10 +270,10 @@ get_chars(T&& _c, std::index_sequence) ::omnitrace::set_state(::omnitrace::State::Finalized); \ ::tim::disable_signal_detection(); \ ::tim::print_demangled_backtrace<64>(); \ - ::std::exit(EXIT_FAILURE); \ + METHOD; \ } -#define OMNITRACE_CONDITIONAL_FAIL_F(COND, ...) \ +#define OMNITRACE_CONDITIONAL_FAILURE_F(COND, METHOD, ...) \ if(COND) \ { \ ::omnitrace::debug::flush(); \ @@ -284,10 +284,10 @@ get_chars(T&& _c, std::index_sequence) ::omnitrace::set_state(::omnitrace::State::Finalized); \ ::tim::disable_signal_detection(); \ ::tim::print_demangled_backtrace<64>(); \ - ::std::exit(EXIT_FAILURE); \ + METHOD; \ } -#define OMNITRACE_CONDITIONAL_BASIC_FAIL_F(COND, ...) \ +#define OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, METHOD, ...) \ if(COND) \ { \ ::omnitrace::debug::flush(); \ @@ -297,24 +297,69 @@ get_chars(T&& _c, std::index_sequence) ::omnitrace::set_state(::omnitrace::State::Finalized); \ ::tim::disable_signal_detection(); \ ::tim::print_demangled_backtrace<64>(); \ - ::std::exit(EXIT_FAILURE); \ + METHOD; \ } -#define OMNITRACE_CI_FAIL(COND, ...) \ - OMNITRACE_CONDITIONAL_FAIL(::omnitrace::get_is_continuous_integration() && (COND), \ - __VA_ARGS__) +#define OMNITRACE_CI_FAILURE(COND, METHOD, ...) \ + OMNITRACE_CONDITIONAL_FAILURE( \ + ::omnitrace::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__) -#define OMNITRACE_CI_BASIC_FAIL(COND, ...) \ - OMNITRACE_CONDITIONAL_BASIC_FAIL( \ - ::omnitrace::get_is_continuous_integration() && (COND), __VA_ARGS__) +#define OMNITRACE_CI_BASIC_FAILURE(COND, METHOD, ...) \ + OMNITRACE_CONDITIONAL_BASIC_FAILURE( \ + ::omnitrace::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__) //--------------------------------------------------------------------------------------// +#define OMNITRACE_CONDITIONAL_FAIL(COND, ...) \ + OMNITRACE_CONDITIONAL_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \ + __VA_ARGS__) + +#define OMNITRACE_CONDITIONAL_BASIC_FAIL(COND, ...) \ + OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \ + __VA_ARGS__) + +#define OMNITRACE_CONDITIONAL_FAIL_F(COND, ...) \ + OMNITRACE_CONDITIONAL_FAILURE_F(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \ + __VA_ARGS__) + +#define OMNITRACE_CONDITIONAL_BASIC_FAIL_F(COND, ...) \ + OMNITRACE_CONDITIONAL_BASIC_FAILURE_F( \ + COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__) + +#define OMNITRACE_CI_FAIL(COND, ...) \ + OMNITRACE_CI_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__) + +#define OMNITRACE_CI_BASIC_FAIL(COND, ...) \ + OMNITRACE_CI_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::exit(EXIT_FAILURE)), \ + __VA_ARGS__) + +//--------------------------------------------------------------------------------------// + +#define OMNITRACE_CONDITIONAL_ABORT(COND, ...) \ + OMNITRACE_CONDITIONAL_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__) + +#define OMNITRACE_CONDITIONAL_BASIC_ABORT(COND, ...) \ + OMNITRACE_CONDITIONAL_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__) + +#define OMNITRACE_CONDITIONAL_ABORT_F(COND, ...) \ + OMNITRACE_CONDITIONAL_FAILURE_F(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__) + +#define OMNITRACE_CONDITIONAL_BASIC_ABORT_F(COND, ...) \ + OMNITRACE_CONDITIONAL_BASIC_FAILURE_F(COND, OMNITRACE_ESC(::std::abort()), \ + __VA_ARGS__) + +#define OMNITRACE_CI_ABORT(COND, ...) \ + OMNITRACE_CI_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__) + +#define OMNITRACE_CI_BASIC_ABORT(COND, ...) \ + OMNITRACE_CI_BASIC_FAILURE(COND, OMNITRACE_ESC(::std::abort()), __VA_ARGS__) + //--------------------------------------------------------------------------------------// // // Debug macros // //--------------------------------------------------------------------------------------// + #define OMNITRACE_DEBUG(...) \ OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_debug(), __VA_ARGS__) @@ -338,6 +383,7 @@ get_chars(T&& _c, std::index_sequence) // Verbose macros // //--------------------------------------------------------------------------------------// + #define OMNITRACE_VERBOSE(LEVEL, ...) \ OMNITRACE_CONDITIONAL_PRINT( \ ::omnitrace::get_debug() || (::omnitrace::get_verbose() >= LEVEL), __VA_ARGS__) @@ -402,6 +448,21 @@ get_chars(T&& _c, std::index_sequence) #define OMNITRACE_BASIC_FAIL_F(...) OMNITRACE_CONDITIONAL_BASIC_FAIL_F(true, __VA_ARGS__) +//--------------------------------------------------------------------------------------// +// +// Abort macros +// +//--------------------------------------------------------------------------------------// + +#define OMNITRACE_ABORT(...) OMNITRACE_CONDITIONAL_ABORT(true, __VA_ARGS__) + +#define OMNITRACE_ABORT_F(...) OMNITRACE_CONDITIONAL_ABORT_F(true, __VA_ARGS__) + +#define OMNITRACE_BASIC_ABORT(...) OMNITRACE_CONDITIONAL_BASIC_ABORT(true, __VA_ARGS__) + +#define OMNITRACE_BASIC_ABORT_F(...) \ + OMNITRACE_CONDITIONAL_BASIC_ABORT_F(true, __VA_ARGS__) + #include namespace std diff --git a/projects/rocprofiler-systems/source/lib/omnitrace/library/kokkosp.cpp b/projects/rocprofiler-systems/source/lib/omnitrace/library/kokkosp.cpp index 6dc0578cf5..046d53b32b 100644 --- a/projects/rocprofiler-systems/source/lib/omnitrace/library/kokkosp.cpp +++ b/projects/rocprofiler-systems/source/lib/omnitrace/library/kokkosp.cpp @@ -20,19 +20,24 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "library/defines.hpp" - #define TIMEMORY_KOKKOSP_POSTFIX OMNITRACE_PUBLIC_API #include "library/api.hpp" #include "library/components/user_region.hpp" #include "library/config.hpp" #include "library/debug.hpp" +#include "library/defines.hpp" #include "library/perfetto.hpp" #include "library/runtime.hpp" #include +#include #include +#include + +#include +#include +#include namespace kokkosp = tim::kokkosp; @@ -138,6 +143,36 @@ extern "C" if(_standalone_initialized || (!omnitrace::config::settings_are_configured() && omnitrace::get_state() < omnitrace::State::Active)) { + auto _kokkos_profile_lib = + tim::get_env("KOKKOS_PROFILE_LIBRARY"); + if(_kokkos_profile_lib.find("libomnitrace.so") != std::string::npos) + { + auto _maps = tim::procfs::read_maps(tim::process::get_id()); + auto _libs = std::set{}; + for(auto& itr : _maps) + { + auto&& _path = itr.pathname; + if(!_path.empty() && _path.at(0) != '[') _libs.emplace(_path); + } + for(const auto& itr : _libs) + { + if(itr.find("libomnitrace-dl.so") != std::string::npos) + { + std::stringstream _libs_str{}; + for(const auto& litr : _libs) + _libs_str << " " << litr << "\n"; + OMNITRACE_ABORT( + "%s was invoked with libomnitrace.so as the " + "KOKKOS_PROFILE_LIBRARY.\n" + "However, libomnitrace-dl.so has already been loaded by the " + "process.\nTo avoid duplicate collections culminating is an " + "error, please set KOKKOS_PROFILE_LIBRARY=%s.\nLoaded " + "libraries:\n%s", + __FUNCTION__, itr.c_str(), _libs_str.str().c_str()); + } + } + } + OMNITRACE_BASIC_VERBOSE_F(0, "Initializing kokkos omnitrace connector " "(standalone, sequence %d, version: %llu)...\n", diff --git a/projects/rocprofiler-systems/tests/CMakeLists.txt b/projects/rocprofiler-systems/tests/CMakeLists.txt index b6faf92586..5850c0cfbe 100644 --- a/projects/rocprofiler-systems/tests/CMakeLists.txt +++ b/projects/rocprofiler-systems/tests/CMakeLists.txt @@ -23,17 +23,15 @@ if(OMNITRACE_BUILD_DYNINST) ) endif() -set(_base_environment - "OMNITRACE_USE_PERFETTO=ON" - "OMNITRACE_USE_TIMEMORY=ON" - "OMNITRACE_USE_SAMPLING=ON" - "OMNITRACE_USE_PROCESS_SAMPLING=ON" - "OMNITRACE_TIME_OUTPUT=OFF" - "OMP_PROC_BIND=spread" - "OMP_PLACES=threads" - "OMP_NUM_THREADS=2" +set(_test_library_path "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" ) +set(_test_openmp_env "OMP_PROC_BIND=spread" "OMP_PLACES=threads" "OMP_NUM_THREADS=2") + +set(_base_environment + "OMNITRACE_USE_PERFETTO=ON" "OMNITRACE_USE_TIMEMORY=ON" "OMNITRACE_USE_SAMPLING=ON" + "OMNITRACE_USE_PROCESS_SAMPLING=ON" "OMNITRACE_TIME_OUTPUT=OFF" "${_test_openmp_env}" + "${_test_library_path}") set(_flat_environment "OMNITRACE_USE_PERFETTO=ON" @@ -61,44 +59,22 @@ set(_lock_environment "OMNITRACE_TIME_OUTPUT=OFF" "OMNITRACE_FLAT_PROFILE=ON" "OMNITRACE_TIMELINE_PROFILE=OFF" - "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" - ) + "${_test_library_path}") set(_ompt_environment - "OMNITRACE_USE_PERFETTO=ON" - "OMNITRACE_USE_TIMEMORY=ON" - "OMNITRACE_TIME_OUTPUT=OFF" - "OMNITRACE_USE_OMPT=ON" - "OMNITRACE_CRITICAL_TRACE=OFF" - "OMP_PROC_BIND=spread" - "OMP_PLACES=threads" - "OMP_NUM_THREADS=2" - "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" - ) + "OMNITRACE_USE_PERFETTO=ON" "OMNITRACE_USE_TIMEMORY=ON" "OMNITRACE_TIME_OUTPUT=OFF" + "OMNITRACE_USE_OMPT=ON" "OMNITRACE_CRITICAL_TRACE=OFF" "${_test_openmp_env}" + "${_test_library_path}") set(_perfetto_environment - "OMNITRACE_USE_PERFETTO=ON" - "OMNITRACE_USE_TIMEMORY=OFF" - "OMNITRACE_USE_SAMPLING=ON" - "OMNITRACE_USE_PROCESS_SAMPLING=ON" - "OMNITRACE_TIME_OUTPUT=OFF" - "OMP_PROC_BIND=spread" - "OMP_PLACES=threads" - "OMP_NUM_THREADS=2" - "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" - ) + "OMNITRACE_USE_PERFETTO=ON" "OMNITRACE_USE_TIMEMORY=OFF" "OMNITRACE_USE_SAMPLING=ON" + "OMNITRACE_USE_PROCESS_SAMPLING=ON" "OMNITRACE_TIME_OUTPUT=OFF" "${_test_openmp_env}" + "${_test_library_path}") set(_timemory_environment - "OMNITRACE_USE_PERFETTO=OFF" - "OMNITRACE_USE_TIMEMORY=ON" - "OMNITRACE_USE_SAMPLING=ON" - "OMNITRACE_USE_PROCESS_SAMPLING=ON" - "OMNITRACE_TIME_OUTPUT=OFF" - "OMP_PROC_BIND=spread" - "OMP_PLACES=threads" - "OMP_NUM_THREADS=2" - "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" - ) + "OMNITRACE_USE_PERFETTO=OFF" "OMNITRACE_USE_TIMEMORY=ON" "OMNITRACE_USE_SAMPLING=ON" + "OMNITRACE_USE_PROCESS_SAMPLING=ON" "OMNITRACE_TIME_OUTPUT=OFF" "${_test_openmp_env}" + "${_test_library_path}") set(_test_environment ${_base_environment} "OMNITRACE_CRITICAL_TRACE=OFF") @@ -111,7 +87,7 @@ set(_python_environment "OMNITRACE_TREE_OUTPUT=OFF" "OMNITRACE_USE_PID=OFF" "OMNITRACE_TIMEMORY_COMPONENTS=trip_count" - "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" + "${_test_library_path}" "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python/site-packages") set(_attach_environment @@ -124,11 +100,8 @@ set(_attach_environment "OMNITRACE_USE_KOKKOSP=ON" "OMNITRACE_TIME_OUTPUT=OFF" "OMNITRACE_USE_PID=OFF" - "OMP_PROC_BIND=spread" - "OMP_PLACES=threads" - "OMP_NUM_THREADS=2" - "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" - ) + "${_test_openmp_env}" + "${_test_library_path}") # -------------------------------------------------------------------------------------- # @@ -690,11 +663,36 @@ omnitrace_add_test( args -ME [==[lib(gomp|m-)]==] + LABELS "kokkos;kokkos-profile-library" + RUN_ARGS -i 10 -s 20 -p + ENVIRONMENT + "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON;KOKKOS_PROFILE_LIBRARY=libomnitrace-dl.so" + ) + +omnitrace_add_test( + SKIP_RUNTIME SKIP_REWRITE + NAME lulesh-baseline-kokkosp-libomnitrace + TARGET lulesh + MPI ${LULESH_USE_MPI} + NUM_PROCS 8 + LABELS "kokkos;kokkos-profile-library" RUN_ARGS -i 10 -s 20 -p ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON;KOKKOS_PROFILE_LIBRARY=libomnitrace.so" ) +omnitrace_add_test( + SKIP_RUNTIME SKIP_REWRITE + NAME lulesh-baseline-kokkosp-libomnitrace-dl + TARGET lulesh + MPI ${LULESH_USE_MPI} + NUM_PROCS 8 + LABELS "kokkos;kokkos-profile-library" + RUN_ARGS -i 10 -s 20 -p + ENVIRONMENT + "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON;KOKKOS_PROFILE_LIBRARY=libomnitrace-dl.so" + ) + omnitrace_add_test( SKIP_BASELINE NAME lulesh-kokkosp