diff --git a/examples/mpi/CMakeLists.txt b/examples/mpi/CMakeLists.txt index 54088a172a..281148521f 100644 --- a/examples/mpi/CMakeLists.txt +++ b/examples/mpi/CMakeLists.txt @@ -2,19 +2,21 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(omnitrace-mpi-example LANGUAGES CXX) +set(CMAKE_BUILD_TYPE "Release") + find_package(MPI) if(NOT MPI_FOUND) message(AUTHOR_WARNING "MPI could not be found. Cannot build omnitrace-mpi target") return() endif() +find_package(Threads REQUIRED) + add_executable(mpi-example mpi.cpp) -if(TARGET omnitrace::omnitrace-compile-options) - target_link_libraries(mpi-example PRIVATE omnitrace::omnitrace-compile-options) -endif() - -target_link_libraries(mpi-example PRIVATE MPI::MPI_CXX) +target_link_libraries( + mpi-example PRIVATE MPI::MPI_CXX Threads::Threads + $) if(OMNITRACE_INSTALL_EXAMPLES) install( diff --git a/examples/mpi/mpi.cpp b/examples/mpi/mpi.cpp index 1ff9463e79..7625d25b77 100644 --- a/examples/mpi/mpi.cpp +++ b/examples/mpi/mpi.cpp @@ -1,24 +1,24 @@ -/* -Copyright (c) 2015-2020 Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. #include @@ -28,6 +28,7 @@ THE SOFTWARE. #include #include #include +#include #include #include #include @@ -38,7 +39,10 @@ THE SOFTWARE. #include #include -std::string _name = {}; +namespace +{ +auto _name = std::string{}; +} // namespace template void @@ -150,12 +154,9 @@ print_info(MPI_Comm _comm, bool _verbose, std::string _msg = {}) } } -int -main(int argc, char** argv) +void +run_main(int argc, char** argv) { - int _mpi_thread_provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_SINGLE, &_mpi_thread_provided); - int rank = 0; int size = 1; int nitr = 1; @@ -253,7 +254,25 @@ main(int argc, char** argv) print_info(dup, false); } +} + +int +main(int argc, char** argv) +{ + int _mpi_thread_provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &_mpi_thread_provided); + + auto _prom = std::promise{}; + auto _fut = _prom.get_future(); + + std::thread _thr{ [&]() { + run_main(argc, argv); + _prom.set_value(); + } }; + + _fut.wait(); + _thr.join(); MPI_Finalize(); - return 0; + return EXIT_SUCCESS; } diff --git a/source/lib/omnitrace/library/components/mpi_gotcha.cpp b/source/lib/omnitrace/library/components/mpi_gotcha.cpp index 01abbfba32..e6c347c198 100644 --- a/source/lib/omnitrace/library/components/mpi_gotcha.cpp +++ b/source/lib/omnitrace/library/components/mpi_gotcha.cpp @@ -22,6 +22,7 @@ #include "library/components/mpi_gotcha.hpp" #include "library/api.hpp" +#include "library/common.hpp" #include "library/components/category_region.hpp" #include "library/config.hpp" #include "library/debug.hpp" @@ -34,10 +35,15 @@ #include #include #include +#include #include namespace omnitrace { +using mpi_tooltag_t = api::omnitrace; +using mpi_toolset_t = + tim::component_tuple>; + namespace { struct comm_rank_data @@ -104,10 +110,7 @@ omnitrace_mpi_set_attr() static auto _mpi_fini = [](MPI_Comm, int, void*, void*) { OMNITRACE_DEBUG("MPI Comm attribute finalize\n"); if(mpip_index != std::numeric_limits::max()) - comp::deactivate_mpip< - tim::component_tuple< - omnitrace::component::category_region>, - api::omnitrace>(mpip_index); + comp::deactivate_mpip(mpip_index); omnitrace_finalize_hidden(); return MPI_SUCCESS; }; @@ -224,9 +227,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming) OMNITRACE_BASIC_DEBUG_F("%s()\n", _data.tool_id.c_str()); if(mpip_index != std::numeric_limits::max()) - comp::deactivate_mpip< - tim::component_tuple>, - api::omnitrace>(mpip_index); + comp::deactivate_mpip(mpip_index); #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS) tim::mpi::is_initialized_callback() = []() { return false; }; @@ -278,14 +279,16 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) // use env vars OMNITRACE_MPIP_PERMIT_LIST and OMNITRACE_MPIP_REJECT_LIST // to control the gotcha bindings at runtime - comp::configure_mpip< - tim::component_tuple< - omnitrace::component::category_region>, - api::omnitrace>(); - mpip_index = comp::activate_mpip< - tim::component_tuple< - omnitrace::component::category_region>, - api::omnitrace>(); + auto _accept = std::set{}; + auto _reject = std::set{}; + +#if defined(OMNITRACE_USE_MPI_HEADERS) && !defined(OMNITRACE_USE_MPI) + _reject.emplace("MPI_Comm_rank"); + _reject.emplace("MPI_Comm_size"); +#endif + + comp::configure_mpip(_accept, _reject); + mpip_index = comp::activate_mpip(); } auto_lock_t _lk{ type_mutex() }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c14f3003ab..b6faf92586 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -35,6 +35,22 @@ set(_base_environment "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" ) +set(_flat_environment + "OMNITRACE_USE_PERFETTO=ON" + "OMNITRACE_USE_TIMEMORY=ON" + "OMNITRACE_TIME_OUTPUT=OFF" + "OMNITRACE_COUT_OUTPUT=ON" + "OMNITRACE_FLAT_PROFILE=ON" + "OMNITRACE_TIMELINE_PROFILE=OFF" + "OMNITRACE_COLLAPSE_PROCESSES=ON" + "OMNITRACE_COLLAPSE_THREADS=ON" + "OMNITRACE_TIMEMORY_COMPONENTS=wall_clock,trip_count" + "OMP_PROC_BIND=spread" + "OMP_PLACES=threads" + "OMP_NUM_THREADS=2" + "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" + ) + set(_lock_environment "OMNITRACE_USE_SAMPLING=OFF" "OMNITRACE_USE_PROCESS_SAMPLING=OFF" @@ -605,11 +621,57 @@ omnitrace_add_test( args --min-instructions 0 - ENVIRONMENT "${_base_environment}" + ENVIRONMENT "${_base_environment};GOTCHA_DEBUG=1" REWRITE_RUN_PASS_REGEX "(/[A-Za-z-]+/perfetto-trace-0.proto).*(/[A-Za-z-]+/wall_clock-0.txt')" REWRITE_RUN_FAIL_REGEX "-[0-9][0-9]+.(json|txt|proto)") +omnitrace_add_test( + SKIP_RUNTIME SKIP_SAMPLING + NAME "mpi-flat-mpip" + TARGET mpi-example + MPI ON + NUM_PROCS 4 + LABELS "mpip" + REWRITE_ARGS + -e + -v + 2 + --label + file + line + return + args + --min-instructions + 0 + ENVIRONMENT "${_flat_environment};OMNITRACE_USE_SAMPLING=OFF;OMNITRACE_USE_MPIP=ON" + REWRITE_RUN_PASS_REGEX + ">>> main(.*\n.*)>>> MPI_Init_thread(.*\n.*)>>> MPI_Comm_size(.*\n.*)>>> MPI_Comm_rank(.*\n.*)>>> MPI_Barrier(.*\n.*)>>> MPI_Alltoall(.*\n.*)>>> pthread_join" + ) + +omnitrace_add_test( + SKIP_RUNTIME SKIP_SAMPLING + NAME "mpi-flat" + TARGET mpi-example + MPI ON + NUM_PROCS 4 + LABELS "mpip" + REWRITE_ARGS + -e + -v + 2 + --label + file + line + return + args + --min-instructions + 0 + ENVIRONMENT "${_flat_environment};OMNITRACE_USE_SAMPLING=OFF" + REWRITE_RUN_PASS_REGEX + ">>> main(.*\n.*)>>> MPI_Init_thread(.*\n.*)>>> MPI_Comm_size(.*\n.*)>>> MPI_Comm_rank(.*\n.*)>>> MPI_Barrier(.*\n.*)>>> MPI_Alltoall(.*\n.*)>>> pthread_join" + ) + omnitrace_add_test( NAME lulesh TARGET lulesh