Remove MPI compile-time dependency (#264)

* use dlsym for MPI functions

to allow compiling without MPI support, convert the usage of MPI functions and symbols to be based on a dlopen/dlsym based mechanism. Turns out this cannot be done entirely vendor neutral, slightly different solutions might be required for Open MPI, MPICH and the new MPI ABI.

* checkpoint

more work to be done.

* checkpoint 2

* checkpoint 3

* checkpoint 4

examples compile and link correctly

* checkpoitn 5 (I think)

* Checkpoitn 6

* dyld-mpi: adapt GDA

* dyldmpi: tests that depend on MPI need to link with it themselves

* do not ../mpi_instance.h

* dyldmpi: make the symetricHeapTestFixture compile

* dyldmpi: Change cmakery, compiles and run gda w/o external MPI

* Make it also compile in external MPI mode

* dyldmpi: ipc unit tests compile but do not link

* dyldmpi: new approach, if external mpi required, link with mpi,
otherwise use ompi5 abi

* C-style comments in cmakelist..

* dyldmpi: examples: do not fail compiling if MPI not found at build time,
instead do not compile the MPI required examples

* more updates to CMake logic

* convert RO backend

and a few other cleanups

* update some unit tests

to work with the dlopen MPI environment correctly.

---------

Co-authored-by: Aurelien Bouteiller <abouteil@amd.com>

[ROCm/rocshmem commit: e4c427a736]

projects/rocshmem/src/gda/backend_gda.cpp

@@ -24,15 +24,16 @@
 
 #include <cstring>
 
-#include "backend_gda.hpp"
-#include "gda_team.hpp"
-#include "util.hpp"
-#include "topology.hpp"
-
 #include <hip/hip_runtime.h>
 #include <cstdlib>
 #include <cassert>
 
+#include "backend_gda.hpp"
+#include "mpi_instance.hpp"
+#include "gda_team.hpp"
+#include "util.hpp"
+#include "topology.hpp"
+
 namespace rocshmem {
 
 #define NET_CHECK(cmd) {                                     \
@@ -303,7 +304,7 @@ void GDABackend::create_new_team([[maybe_unused]] Team *parent_team,
    * the pool of available work arrays.
    */
   if (team_comm != MPI_COMM_NULL) {
-    NET_CHECK(MPI_Allreduce(team_pool_bitmask_, team_reduced_bitmask_, team_bitmask_size_,
+    NET_CHECK(mpilib_ftable_.Allreduce(team_pool_bitmask_, team_reduced_bitmask_, team_bitmask_size_,
                             MPI_CHAR, MPI_BAND, team_comm));
   } else {
     Allreduce_char_BAND (team_pool_bitmask_, team_reduced_bitmask_, team_bitmask_size_, parent_team);
@@ -361,7 +362,7 @@ void GDABackend::dump_backend_stats() {
 
 __host__ void GDABackend::global_exit(int status) {
   if (backend_comm != MPI_COMM_NULL)
-    MPI_Abort(backend_comm, status);
+    mpilib_ftable_.Abort(backend_comm, status);
   else
     abort();
 }
@@ -536,7 +537,7 @@ void GDABackend::setup_teams() {
 
 void GDABackend::rte_barrier() {
   if (backend_comm != MPI_COMM_NULL) {
-    NET_CHECK(MPI_Barrier(backend_comm));
+    NET_CHECK(mpilib_ftable_.Barrier(backend_comm));
   } else {
     backend_bootstr->barrier();
   }
@@ -668,7 +669,7 @@ void GDABackend::exchange_qp_dest_info() {
 
   for (int i = 0; i < maximum_num_contexts_ + 1; i++) {
     if (backend_comm != MPI_COMM_NULL) {
-      MPI_Alltoall(MPI_IN_PLACE, sizeof(dest_info_t), MPI_CHAR, dest_info.data() + i * num_pes, sizeof(dest_info_t), MPI_CHAR, backend_comm);
+      mpilib_ftable_.Alltoall(MPI_IN_PLACE, sizeof(dest_info_t), MPI_CHAR, dest_info.data() + i * num_pes, sizeof(dest_info_t), MPI_CHAR, backend_comm);
     } else {
       Alltoall_char_inplace(reinterpret_cast<char*>(dest_info.data() + i * num_pes), sizeof(dest_info_t), ROCSHMEM_TEAM_WORLD);
     }
@@ -695,7 +696,7 @@ void GDABackend::setup_heap_memory_rkey() {
   CHECK_HIP(hipStreamSynchronize(stream));
 
   if (backend_comm != MPI_COMM_NULL)
-    MPI_Allgather(MPI_IN_PLACE, sizeof(uint32_t), MPI_CHAR, host_rkey_cpy, sizeof(uint32_t), MPI_CHAR, backend_comm);
+    mpilib_ftable_.Allgather(MPI_IN_PLACE, sizeof(uint32_t), MPI_CHAR, host_rkey_cpy, sizeof(uint32_t), MPI_CHAR, backend_comm);
   else
     backend_bootstr->allGather(host_rkey_cpy, sizeof(uint32_t));
 

projects/rocshmem/src/gda/backend_gda.hpp

@@ -66,24 +66,6 @@ struct mlx5dv_funcs_t {
   int (*init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
 };
 
-/* Helper Macros for handling dynamic libraries */
-#define PPCAT_NX(prefix, func_name) prefix##func_name
-#define PPCAT(prefix, func_name) PPCAT_NX(prefix, func_name)
-
-#define STRINGIFY_NX(name) #name
-#define STRINGIFY(name) STRINGIFY_NX(name)
-
-#define DLSYM_HELPER(func_struct, prefix, handle, func_name)                                \
-do {                                                                                        \
-  *(void **) (&func_struct.func_name) = dlsym(handle, STRINGIFY(PPCAT(prefix, func_name))); \
-  if (!func_struct.func_name) {                                                             \
-    DPRINTF("Failed to find function %s \n",  STRINGIFY(PPCAT(prefix, func_name)));         \
-    dlclose(handle);                                                                        \
-    handle = nullptr;                                                                       \
-    return ROCSHMEM_ERROR;                                                                  \
-  }                                                                                         \
-} while (0)
-
 namespace rocshmem {
 
 class GDAContext;

projects/rocshmem/src/gda/context_gda_host.cpp

@@ -22,11 +22,9 @@
  * IN THE SOFTWARE.
  *****************************************************************************/
 
-#include "context_gda_host.hpp"
-
-#include <mpi.h>
 
 #include "rocshmem/rocshmem_config.h"  // NOLINT(build/include_subdir)
+#include "context_gda_host.hpp"
 #include "backend_type.hpp"
 #include "context_incl.hpp"
 #include "backend_gda.hpp"

projects/rocshmem/src/gda/gda_team.cpp

@@ -27,6 +27,7 @@
 #include "constants.hpp"
 #include "backend_type.hpp"
 #include "backend_gda.hpp"
+#include "rocshmem/rocshmem_mpi.hpp"
 
 namespace rocshmem {