Remove MPI compile-time dependency (#264)

* use dlsym for MPI functions

to allow compiling without MPI support, convert the usage of MPI functions and symbols to be based on a dlopen/dlsym based mechanism. Turns out this cannot be done entirely vendor neutral, slightly different solutions might be required for Open MPI, MPICH and the new MPI ABI.

* checkpoint

more work to be done.

* checkpoint 2

* checkpoint 3

* checkpoint 4

examples compile and link correctly

* checkpoitn 5 (I think)

* Checkpoitn 6

* dyld-mpi: adapt GDA

* dyldmpi: tests that depend on MPI need to link with it themselves

* do not ../mpi_instance.h

* dyldmpi: make the symetricHeapTestFixture compile

* dyldmpi: Change cmakery, compiles and run gda w/o external MPI

* Make it also compile in external MPI mode

* dyldmpi: ipc unit tests compile but do not link

* dyldmpi: new approach, if external mpi required, link with mpi,
otherwise use ompi5 abi

* C-style comments in cmakelist..

* dyldmpi: examples: do not fail compiling if MPI not found at build time,
instead do not compile the MPI required examples

* more updates to CMake logic

* convert RO backend

and a few other cleanups

* update some unit tests

to work with the dlopen MPI environment correctly.

---------

Co-authored-by: Aurelien Bouteiller <abouteil@amd.com>

examples/CMakeLists.txt

@@ -29,6 +29,12 @@ cmake_minimum_required(VERSION 3.16.3 FATAL_ERROR)
 include(${CMAKE_SOURCE_DIR}/cmake/setup_project.cmake)
 project(rocshmem_examples VERSION 1.0.0 LANGUAGES CXX)
 
+find_package(MPI)
+find_package(hip REQUIRED PATHS /opt/rocm)
+if (NOT TARGET roc::rocshmem)
+  find_package(rocshmem REQUIRED PATHS /opt/rocm)
+endif()
+
 ###############################################################################
 # SOURCES
 ###############################################################################
@@ -38,13 +44,11 @@ set(EXAMPLE_SOURCES
   rocshmem_broadcast_test.cc
   rocshmem_getmem_test.cc
   rocshmem_put_signal_test.cc
-  rocshmem_init_attr_test.cc
 )
 
-find_package(MPI REQUIRED)
-find_package(hip REQUIRED PATHS /opt/rocm)
-if(NOT TARGET roc::rocshmem)
-  find_package(rocshmem REQUIRED PATHS /opt/rocm)
+if (HAVE_EXTERNAL_MPI)
+  list(APPEND EXAMPLE_SOURCES
+    rocshmem_init_attr_test.cc)
 endif()
 
 foreach(SOURCE_FILE IN LISTS EXAMPLE_SOURCES)
@@ -55,7 +59,7 @@ foreach(SOURCE_FILE IN LISTS EXAMPLE_SOURCES)
   target_link_libraries(
     ${EXECUTABLE_NAME}
     PRIVATE
-      MPI::MPI_CXX
+      $<TARGET_NAME_IF_EXISTS:MPI::MPI_CXX>
       roc::rocshmem
   )
 endforeach()

examples/rocshmem_init_attr_test.cc

@@ -59,6 +59,7 @@
  */
 
 #include <rocshmem/rocshmem.hpp>
+#include <mpi.h>
 
 #include "util.h"
 

examples/util.h

@@ -34,7 +34,7 @@
         hipError_t error = condition;                                     \
         if(error != hipSuccess){                                          \
             fprintf(stderr,"HIP error: %d line: %d\n", error,  __LINE__); \
-            MPI_Abort(MPI_COMM_WORLD, error);                             \
+            exit(error);                                                  \
         }                                                                 \
     }