Remove MPI compile-time dependency (#264)

* use dlsym for MPI functions

to allow compiling without MPI support, convert the usage of MPI functions and symbols to be based on a dlopen/dlsym based mechanism. Turns out this cannot be done entirely vendor neutral, slightly different solutions might be required for Open MPI, MPICH and the new MPI ABI.

* checkpoint

more work to be done.

* checkpoint 2

* checkpoint 3

* checkpoint 4

examples compile and link correctly

* checkpoitn 5 (I think)

* Checkpoitn 6

* dyld-mpi: adapt GDA

* dyldmpi: tests that depend on MPI need to link with it themselves

* do not ../mpi_instance.h

* dyldmpi: make the symetricHeapTestFixture compile

* dyldmpi: Change cmakery, compiles and run gda w/o external MPI

* Make it also compile in external MPI mode

* dyldmpi: ipc unit tests compile but do not link

* dyldmpi: new approach, if external mpi required, link with mpi,
otherwise use ompi5 abi

* C-style comments in cmakelist..

* dyldmpi: examples: do not fail compiling if MPI not found at build time,
instead do not compile the MPI required examples

* more updates to CMake logic

* convert RO backend

and a few other cleanups

* update some unit tests

to work with the dlopen MPI environment correctly.

---------

Co-authored-by: Aurelien Bouteiller <abouteil@amd.com>

[ROCm/rocshmem commit: e4c427a736]

projects/rocshmem/src/host/host.cpp

@@ -24,8 +24,6 @@
 
 #include "host.hpp"
 
-#include <mpi.h>
-
 #include "rocshmem/rocshmem_config.h"  // NOLINT(build/include_subdir)
 #include "host_helpers.hpp"
 #include "memory/window_info.hpp"
@@ -98,9 +96,9 @@ __host__ HostInterface::HostInterface(HdpPolicy* hdp_policy,
    * Duplicate a communicator from roc_shem's comm
    * world for the host interface
    */
-  MPI_Comm_dup(rocshmem_comm, &host_comm_world_);
-  MPI_Comm_rank(host_comm_world_, &my_pe_);
-  MPI_Comm_rank(host_comm_world_, &num_pes_);
+  mpilib_ftable_.Comm_dup(rocshmem_comm, &host_comm_world_);
+  mpilib_ftable_.Comm_rank(host_comm_world_, &my_pe_);
+  mpilib_ftable_.Comm_size(host_comm_world_, &num_pes_);
 
   /*
    * Create an MPI window on the HDP so that it can be flushed
@@ -136,18 +134,18 @@ __host__ HostInterface::HostInterface(HdpPolicy* hdp_policy,
 
 #if defined USE_HDP_FLUSH
 __host__ void HostInterface::create_hdp_window() {
-  MPI_Win_create(hdp_policy_->get_hdp_flush_ptr(),
-                 sizeof(unsigned int), /* size of window */
-                 sizeof(unsigned int), /* displacement */
-                 MPI_INFO_NULL, host_comm_world_, &hdp_win);
-
+  mpilib_ftable_.Win_create(hdp_policy_->get_hdp_flush_ptr(),
+                            sizeof(unsigned int), /* size of window */
+                            sizeof(unsigned int), /* displacement */
+                            MPI_INFO_NULL, host_comm_world_, &hdp_win);
+  
   /*
    * Start a shared access epoch on windows of all ranks,
    * and let the library there is no need to check for
    * lock exclusivity during operations on this window
    * (MPI_MODE_NOCHECK).
    */
-  MPI_Win_lock_all(MPI_MODE_NOCHECK, hdp_win);
+  mpilib_ftable_.Win_lock_all(MPI_MODE_NOCHECK, hdp_win);
 }
 #endif  // USE_HDP_FLUSH
 
@@ -188,9 +186,9 @@ __host__ HostInterface::HostInterface(HdpPolicy* hdp_policy,
 
 __host__ HostInterface::~HostInterface() {
 #if defined USE_HDP_FLUSH
-  MPI_Win_unlock_all(hdp_win);
+  mpilib_ftable_.Win_unlock_all(hdp_win);
 
-  MPI_Win_free(&hdp_win);
+  mpilib_ftable_.Win_free(&hdp_win);
 #endif  // USE_HDP_FLUSH
 
   /* Detroy the pool of contexts */
@@ -203,7 +201,7 @@ __host__ HostInterface::~HostInterface() {
   }
 
   if (host_comm_world_ != MPI_COMM_NULL) {
-    MPI_Comm_free(&host_comm_world_);
+    mpilib_ftable_.Comm_free(&host_comm_world_);
   }
 }
 
@@ -236,7 +234,7 @@ __host__ void HostInterface::putmem(void* dest, const void* source,
   }
   initiate_put(dest, source, nelems, pe, window_info_mpi);
 
-  MPI_Win_flush_local(pe, window_info_mpi->get_win());
+  mpilib_ftable_.Win_flush_local(pe, window_info_mpi->get_win());
 }
 
 __host__ void HostInterface::getmem(void* dest, const void* source,
@@ -248,7 +246,7 @@ __host__ void HostInterface::getmem(void* dest, const void* source,
   }
   initiate_get(dest, source, nelems, pe, window_info_mpi);
 
-  MPI_Win_flush_local(pe, window_info_mpi->get_win());
+  mpilib_ftable_.Win_flush_local(pe, window_info_mpi->get_win());
 
   /*
    * Flush local HDP to ensure that the NIC's write
@@ -295,8 +293,8 @@ __host__ void HostInterface::quiet(WindowInfo* window_info) {
 
 __host__ void HostInterface::sync_all(WindowInfo* window_info) {
   WindowInfoMPI* window_info_mpi = dynamic_cast<WindowInfoMPI*>(window_info);
-  if (!window_info_mpi) {
-    MPI_Win_sync(window_info_mpi->get_win());
+  if (window_info_mpi) {
+    mpilib_ftable_.Win_sync(window_info_mpi->get_win());
 
     hdp_policy_->hdp_flush();
     /*
@@ -305,7 +303,7 @@ __host__ void HostInterface::sync_all(WindowInfo* window_info) {
      * participating.
      */
 
-    MPI_Barrier(host_comm_world_);
+    mpilib_ftable_.Barrier(host_comm_world_);
   } else {
     hdp_policy_->hdp_flush();
     host_bootstrap_->barrier();
@@ -325,7 +323,7 @@ __host__ void HostInterface::barrier_all(WindowInfo* window_info) {
      */
     hdp_policy_->hdp_flush();
 
-    MPI_Barrier(host_comm_world_);
+    mpilib_ftable_.Barrier(host_comm_world_);
   } else {
     // Probably not required
     hdp_policy_->hdp_flush();
@@ -337,7 +335,7 @@ __host__ void HostInterface::barrier_all(WindowInfo* window_info) {
 
 __host__ void HostInterface::barrier_for_sync() {
   if (host_comm_world_ != MPI_COMM_NULL) {
-    MPI_Barrier(host_comm_world_);
+    mpilib_ftable_.Barrier(host_comm_world_);
   } else {
     host_bootstrap_->barrier();
   }

projects/rocshmem/src/host/host.hpp

@@ -34,8 +34,6 @@
  * any backend type.
  */
 
-#include <mpi.h>
-
 #include <map>
 
 #include "rocshmem/rocshmem.hpp"
@@ -43,6 +41,7 @@
 #include "memory/symmetric_heap.hpp"
 #include "memory/window_info.hpp"
 #include "bootstrap/bootstrap.hpp"
+#include "mpi_instance.hpp"
 
 namespace rocshmem {
 
@@ -268,17 +267,17 @@ class HostInterface {
       if (i == my_pe_) {
         continue;
       }
-      MPI_Put(&flush_val, 1, MPI_UNSIGNED, i, 0, 1, MPI_UNSIGNED, hdp_win);
+      mpilib_ftable_.Put(&flush_val, 1, MPI_UNSIGNED, i, 0, 1, MPI_UNSIGNED, hdp_win);
     }
-    MPI_Win_flush_all(hdp_win);
+    mpilib_ftable_.Win_flush_all(hdp_win);
 #endif // USE_HDP_FLUSH
   }
 
   __host__ void flush_remote_hdp(int pe) {
 #if defined USE_HDP_FLUSH
     unsigned flush_val{HdpPolicy::HDP_FLUSH_VAL};
-    MPI_Put(&flush_val, 1, MPI_UNSIGNED, pe, 0, 1, MPI_UNSIGNED, hdp_win);
-    MPI_Win_flush(pe, hdp_win);
+    mpilib_ftable_.Put(&flush_val, 1, MPI_UNSIGNED, pe, 0, 1, MPI_UNSIGNED, hdp_win);
+    mpilib_ftable_.Win_flush(pe, hdp_win);
 #endif // USE_HDP_FLUSH
   }
 

projects/rocshmem/src/host/host_helpers.hpp

@@ -27,6 +27,7 @@
 
 #include "host.hpp"
 #include "memory/window_info.hpp"
+#include "mpi_instance.hpp"
 
 #include <cassert>
 
@@ -42,15 +43,15 @@ __host__ inline MPI_Aint HostInterface::compute_offset(
   MPI_Aint dest_disp{};
   MPI_Aint start_disp{};
 
-  MPI_Get_address(dest, &dest_disp);
-  MPI_Get_address(win_start, &start_disp);
+  mpilib_ftable_.Get_address(dest, &dest_disp);
+  mpilib_ftable_.Get_address(win_start, &start_disp);
 
   return MPI_Aint_diff(dest_disp, start_disp);
 }
 
 __host__ inline void HostInterface::complete_all(MPI_Win win) {
-  MPI_Win_flush_all(win); /* RMA operations */
-  MPI_Win_sync(win);      /* memory stores */
+  mpilib_ftable_.Win_flush_all(win); /* RMA operations */
+  mpilib_ftable_.Win_sync(win);      /* memory stores */
 }
 
 __host__ inline void HostInterface::initiate_put(void* dest, const void* source,
@@ -74,7 +75,7 @@ __host__ inline void HostInterface::initiate_put(void* dest, const void* source,
   hdp_policy_->hdp_flush();
 
   /* Offload remote write operation to MPI */
-  MPI_Put(source, nelems, MPI_CHAR, pe, offset, nelems, MPI_CHAR, win);
+  mpilib_ftable_.Put(source, nelems, MPI_CHAR, pe, offset, nelems, MPI_CHAR, win);
 }
 
 __host__ inline void HostInterface::initiate_get(void* dest, const void* source,
@@ -88,7 +89,7 @@ __host__ inline void HostInterface::initiate_get(void* dest, const void* source,
   MPI_Aint offset = compute_offset(source, win_start, win_end);
 
   /* Offload remote fetch operation to MPI */
-  MPI_Get(dest, nelems, MPI_CHAR, pe, offset, nelems, MPI_CHAR, win);
+  mpilib_ftable_.Get(dest, nelems, MPI_CHAR, pe, offset, nelems, MPI_CHAR, win);
 }
 
 }  // namespace rocshmem

projects/rocshmem/src/host/host_templates.hpp

@@ -74,7 +74,7 @@ __host__ T HostInterface::g(const T* source, int pe, WindowInfo* window_info) {
    */
   getmem_nbi(&ret, source, sizeof(T), pe, window_info);
 
-  MPI_Win_flush_local(pe, window_info_mpi->get_win());
+  mpilib_ftable_.Win_flush_local(pe, window_info_mpi->get_win());
 
   return ret;
 }
@@ -101,7 +101,7 @@ __host__ MPI_Comm HostInterface::get_mpi_comm(int pe_start, int log_pe_stride,
    * First, check to see if the active set is the same as COMM_WORLD
    */
   int comm_world_size{-1};
-  MPI_Comm_size(host_comm_world_, &comm_world_size);
+  mpilib_ftable_.Comm_size(host_comm_world_, &comm_world_size);
 
   if (pe_start == 0 && log_pe_stride == 0 && pe_size == comm_world_size) {
     /*
@@ -139,12 +139,12 @@ __host__ MPI_Comm HostInterface::get_mpi_comm(int pe_start, int log_pe_stride,
   MPI_Group comm_world_group{};
   MPI_Group active_set_group{};
 
-  MPI_Comm_group(host_comm_world_, &comm_world_group);
+  mpilib_ftable_.Comm_group(host_comm_world_, &comm_world_group);
 
-  MPI_Group_incl(comm_world_group, pe_size, active_set_ranks.data(),
-                 &active_set_group);
+  mpilib_ftable_.Group_incl(comm_world_group, pe_size, active_set_ranks.data(),
+                            &active_set_group);
 
-  MPI_Comm_create_group(host_comm_world_, active_set_group, 0,
+  mpilib_ftable_.Comm_create_group(host_comm_world_, active_set_group, 0,
                         &active_set_comm);
 
   /*
@@ -168,7 +168,7 @@ __host__ void HostInterface::broadcast_internal(MPI_Comm mpi_comm, T* dest,
    */
   int active_set_rank{-1};
   void* buffer{nullptr};
-  MPI_Comm_rank(mpi_comm, &active_set_rank);
+  mpilib_ftable_.Comm_rank(mpi_comm, &active_set_rank);
   if (pe_root == active_set_rank) {
     buffer = const_cast<T*>(source);
   } else {
@@ -183,7 +183,7 @@ __host__ void HostInterface::broadcast_internal(MPI_Comm mpi_comm, T* dest,
   /*
    * Offload the broadcast to MPI
    */
-  MPI_Bcast(buffer, nelems * sizeof(T), MPI_CHAR, pe_root, mpi_comm);
+  mpilib_ftable_.Bcast(buffer, nelems * sizeof(T), MPI_CHAR, pe_root, mpi_comm);
 
   return;
 }
@@ -312,9 +312,9 @@ __host__ T HostInterface::amo_fetch_add(void* dst, T value, int pe,
   T ret{};
   MPI_Win win{window_info_mpi->get_win()};
   MPI_Datatype mpi_type{get_mpi_type<T>()};
-  MPI_Fetch_and_op(&value, &ret, mpi_type, pe, offset, MPI_SUM, win);
+  mpilib_ftable_.Fetch_and_op(&value, &ret, mpi_type, pe, offset, MPI_SUM, win);
 
-  MPI_Win_flush_local(pe, win);
+  mpilib_ftable_.Win_flush_local(pe, win);
 
   return ret;
 }
@@ -341,9 +341,9 @@ __host__ T HostInterface::amo_fetch_cas(void* dst, T value, T cond, int pe,
   T ret{};
   MPI_Win win{window_info_mpi->get_win()};
   MPI_Datatype mpi_type{get_mpi_type<T>()};
-  MPI_Compare_and_swap(&value, &cond, &ret, mpi_type, pe, offset, win);
+  mpilib_ftable_.Compare_and_swap(&value, &cond, &ret, mpi_type, pe, offset, win);
 
-  MPI_Win_flush_local(pe, win);
+  mpilib_ftable_.Win_flush_local(pe, win);
 
   return ret;
 }
@@ -368,8 +368,8 @@ __host__ void HostInterface::to_all_internal(MPI_Comm mpi_comm, T* dest,
   /*
    * Offload the allreduce to MPI
    */
-  MPI_Allreduce((dest == source) ? MPI_IN_PLACE : send_buf, recv_buf, nreduce,
-                mpi_type, mpi_op, mpi_comm);
+  mpilib_ftable_.Allreduce((dest == source) ? MPI_IN_PLACE : send_buf, recv_buf, nreduce,
+                           mpi_type, mpi_op, mpi_comm);
 
   return;
 }
@@ -453,9 +453,9 @@ __host__ inline int HostInterface::test_and_compare(MPI_Aint offset,
    */
   hdp_policy_->hdp_flush();
 
-  MPI_Fetch_and_op(nullptr,  // because no operation happening here
-                   &fetched_val, mpi_type, my_pe_, offset, MPI_NO_OP, win);
-  MPI_Win_flush_local(my_pe_, win);
+  mpilib_ftable_.Fetch_and_op(nullptr,  // because no operation happening here
+                              &fetched_val, mpi_type, my_pe_, offset, MPI_NO_OP, win);
+  mpilib_ftable_.Win_flush_local(my_pe_, win);
 
   /*
    * Compare based on the operation