Added new tests validating gotcha wrappers (#105)

* Added new tests validating gotcha wrappers

* Update MPI example to use thread

* Tweaks to mpi-flat test and mpi_gotcha

- enabled MPI_Comm_size and MPI_Comm_rank in mpip so disabled them at runtime
- set test to collapse threads and processes

* Tweak to test and example

- mpi test sets GOTCHA_DEBUG=1 in env
- removed checking for MPI_{Comm_dup,Comm_group,Group_incl}
- tweaked tests so pthread_join is where it is expected

examples/mpi/CMakeLists.txt

@@ -2,19 +2,21 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
 
 project(omnitrace-mpi-example LANGUAGES CXX)
 
+set(CMAKE_BUILD_TYPE "Release")
+
 find_package(MPI)
 if(NOT MPI_FOUND)
     message(AUTHOR_WARNING "MPI could not be found. Cannot build omnitrace-mpi target")
     return()
 endif()
 
+find_package(Threads REQUIRED)
+
 add_executable(mpi-example mpi.cpp)
 
-if(TARGET omnitrace::omnitrace-compile-options)
-    target_link_libraries(mpi-example PRIVATE omnitrace::omnitrace-compile-options)
-endif()
-
-target_link_libraries(mpi-example PRIVATE MPI::MPI_CXX)
+target_link_libraries(
+    mpi-example PRIVATE MPI::MPI_CXX Threads::Threads
+                        $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>)
 
 if(OMNITRACE_INSTALL_EXAMPLES)
     install(

examples/mpi/mpi.cpp

@@ -1,24 +1,24 @@
-/*
-Copyright (c) 2015-2020 Advanced Micro Devices, Inc. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
 
 #include <mpi.h>
 
@@ -28,6 +28,7 @@ THE SOFTWARE.
 #include <cstdio>
 #include <cstdlib>
 #include <fstream>
+#include <future>
 #include <iomanip>
 #include <iostream>
 #include <mutex>
@@ -38,7 +39,10 @@ THE SOFTWARE.
 #include <unistd.h>
 #include <vector>
 
-std::string _name = {};
+namespace
+{
+auto _name = std::string{};
+}  // namespace
 
 template <typename Tp, size_t N>
 void
@@ -150,12 +154,9 @@ print_info(MPI_Comm _comm, bool _verbose, std::string _msg = {})
     }
 }
 
-int
-main(int argc, char** argv)
+void
+run_main(int argc, char** argv)
 {
-    int _mpi_thread_provided;
-    MPI_Init_thread(&argc, &argv, MPI_THREAD_SINGLE, &_mpi_thread_provided);
-
     int rank = 0;
     int size = 1;
     int nitr = 1;
@@ -253,7 +254,25 @@ main(int argc, char** argv)
 
         print_info(dup, false);
     }
+}
+
+int
+main(int argc, char** argv)
+{
+    int _mpi_thread_provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &_mpi_thread_provided);
+
+    auto _prom = std::promise<void>{};
+    auto _fut  = _prom.get_future();
+
+    std::thread _thr{ [&]() {
+        run_main(argc, argv);
+        _prom.set_value();
+    } };
+
+    _fut.wait();
+    _thr.join();
 
     MPI_Finalize();
-    return 0;
+    return EXIT_SUCCESS;
 }

source/lib/omnitrace/library/components/mpi_gotcha.cpp

@@ -22,6 +22,7 @@
 
 #include "library/components/mpi_gotcha.hpp"
 #include "library/api.hpp"
+#include "library/common.hpp"
 #include "library/components/category_region.hpp"
 #include "library/config.hpp"
 #include "library/debug.hpp"
@@ -34,10 +35,15 @@
 #include <cstdint>
 #include <limits>
 #include <thread>
+#include <type_traits>
 #include <unistd.h>
 
 namespace omnitrace
 {
+using mpi_tooltag_t = api::omnitrace;
+using mpi_toolset_t =
+    tim::component_tuple<omnitrace::component::category_region<category::mpi>>;
+
 namespace
 {
 struct comm_rank_data
@@ -104,10 +110,7 @@ omnitrace_mpi_set_attr()
     static auto _mpi_fini = [](MPI_Comm, int, void*, void*) {
         OMNITRACE_DEBUG("MPI Comm attribute finalize\n");
         if(mpip_index != std::numeric_limits<uint64_t>::max())
-            comp::deactivate_mpip<
-                tim::component_tuple<
-                    omnitrace::component::category_region<category::mpi>>,
-                api::omnitrace>(mpip_index);
+            comp::deactivate_mpip<mpi_toolset_t, mpi_tooltag_t>(mpip_index);
         omnitrace_finalize_hidden();
         return MPI_SUCCESS;
     };
@@ -224,9 +227,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
     OMNITRACE_BASIC_DEBUG_F("%s()\n", _data.tool_id.c_str());
 
     if(mpip_index != std::numeric_limits<uint64_t>::max())
-        comp::deactivate_mpip<
-            tim::component_tuple<omnitrace::component::category_region<category::mpi>>,
-            api::omnitrace>(mpip_index);
+        comp::deactivate_mpip<mpi_toolset_t, mpi_tooltag_t>(mpip_index);
 
 #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS)
     tim::mpi::is_initialized_callback() = []() { return false; };
@@ -278,14 +279,16 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
 
             // use env vars OMNITRACE_MPIP_PERMIT_LIST and OMNITRACE_MPIP_REJECT_LIST
             // to control the gotcha bindings at runtime
-            comp::configure_mpip<
-                tim::component_tuple<
-                    omnitrace::component::category_region<category::mpi>>,
-                api::omnitrace>();
-            mpip_index = comp::activate_mpip<
-                tim::component_tuple<
-                    omnitrace::component::category_region<category::mpi>>,
-                api::omnitrace>();
+            auto _accept = std::set<std::string>{};
+            auto _reject = std::set<std::string>{};
+
+#if defined(OMNITRACE_USE_MPI_HEADERS) && !defined(OMNITRACE_USE_MPI)
+            _reject.emplace("MPI_Comm_rank");
+            _reject.emplace("MPI_Comm_size");
+#endif
+
+            comp::configure_mpip<mpi_toolset_t, mpi_tooltag_t>(_accept, _reject);
+            mpip_index = comp::activate_mpip<mpi_toolset_t, mpi_tooltag_t>();
         }
 
         auto_lock_t _lk{ type_mutex<mpi_gotcha>() };

tests/CMakeLists.txt

@@ -35,6 +35,22 @@ set(_base_environment
     "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
     )
 
+set(_flat_environment
+    "OMNITRACE_USE_PERFETTO=ON"
+    "OMNITRACE_USE_TIMEMORY=ON"
+    "OMNITRACE_TIME_OUTPUT=OFF"
+    "OMNITRACE_COUT_OUTPUT=ON"
+    "OMNITRACE_FLAT_PROFILE=ON"
+    "OMNITRACE_TIMELINE_PROFILE=OFF"
+    "OMNITRACE_COLLAPSE_PROCESSES=ON"
+    "OMNITRACE_COLLAPSE_THREADS=ON"
+    "OMNITRACE_TIMEMORY_COMPONENTS=wall_clock,trip_count"
+    "OMP_PROC_BIND=spread"
+    "OMP_PLACES=threads"
+    "OMP_NUM_THREADS=2"
+    "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
+    )
+
 set(_lock_environment
     "OMNITRACE_USE_SAMPLING=OFF"
     "OMNITRACE_USE_PROCESS_SAMPLING=OFF"
@@ -605,11 +621,57 @@ omnitrace_add_test(
         args
         --min-instructions
         0
-    ENVIRONMENT "${_base_environment}"
+    ENVIRONMENT "${_base_environment};GOTCHA_DEBUG=1"
     REWRITE_RUN_PASS_REGEX
         "(/[A-Za-z-]+/perfetto-trace-0.proto).*(/[A-Za-z-]+/wall_clock-0.txt')"
     REWRITE_RUN_FAIL_REGEX "-[0-9][0-9]+.(json|txt|proto)")
 
+omnitrace_add_test(
+    SKIP_RUNTIME SKIP_SAMPLING
+    NAME "mpi-flat-mpip"
+    TARGET mpi-example
+    MPI ON
+    NUM_PROCS 4
+    LABELS "mpip"
+    REWRITE_ARGS
+        -e
+        -v
+        2
+        --label
+        file
+        line
+        return
+        args
+        --min-instructions
+        0
+    ENVIRONMENT "${_flat_environment};OMNITRACE_USE_SAMPLING=OFF;OMNITRACE_USE_MPIP=ON"
+    REWRITE_RUN_PASS_REGEX
+        ">>> main(.*\n.*)>>> MPI_Init_thread(.*\n.*)>>> MPI_Comm_size(.*\n.*)>>> MPI_Comm_rank(.*\n.*)>>> MPI_Barrier(.*\n.*)>>> MPI_Alltoall(.*\n.*)>>> pthread_join"
+    )
+
+omnitrace_add_test(
+    SKIP_RUNTIME SKIP_SAMPLING
+    NAME "mpi-flat"
+    TARGET mpi-example
+    MPI ON
+    NUM_PROCS 4
+    LABELS "mpip"
+    REWRITE_ARGS
+        -e
+        -v
+        2
+        --label
+        file
+        line
+        return
+        args
+        --min-instructions
+        0
+    ENVIRONMENT "${_flat_environment};OMNITRACE_USE_SAMPLING=OFF"
+    REWRITE_RUN_PASS_REGEX
+        ">>> main(.*\n.*)>>> MPI_Init_thread(.*\n.*)>>> MPI_Comm_size(.*\n.*)>>> MPI_Comm_rank(.*\n.*)>>> MPI_Barrier(.*\n.*)>>> MPI_Alltoall(.*\n.*)>>> pthread_join"
+    )
+
 omnitrace_add_test(
     NAME lulesh
     TARGET lulesh