AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove MPI compile-time dependency (#264)

Browse Source 
* use dlsym for MPI functions

to allow compiling without MPI support, convert the usage of MPI functions and symbols to be based on a dlopen/dlsym based mechanism. Turns out this cannot be done entirely vendor neutral, slightly different solutions might be required for Open MPI, MPICH and the new MPI ABI.

* checkpoint

more work to be done.

* checkpoint 2

* checkpoint 3

* checkpoint 4

examples compile and link correctly

* checkpoitn 5 (I think)

* Checkpoitn 6

* dyld-mpi: adapt GDA

* dyldmpi: tests that depend on MPI need to link with it themselves

* do not ../mpi_instance.h

* dyldmpi: make the symetricHeapTestFixture compile

* dyldmpi: Change cmakery, compiles and run gda w/o external MPI

* Make it also compile in external MPI mode

* dyldmpi: ipc unit tests compile but do not link

* dyldmpi: new approach, if external mpi required, link with mpi,
otherwise use ompi5 abi

* C-style comments in cmakelist..

* dyldmpi: examples: do not fail compiling if MPI not found at build time,
instead do not compile the MPI required examples

* more updates to CMake logic

* convert RO backend

and a few other cleanups

* update some unit tests

to work with the dlopen MPI environment correctly.

---------

Co-authored-by: Aurelien Bouteiller <abouteil@amd.com>

[ROCm/rocshmem commit: e4c427a736]
This commit is contained in:
Edgar Gabriel
2025-10-01 08:06:56 -05:00
committed by GitHub
parent 4f955324ac
commit 53fa35b980
50 changed files with 712 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocshmem/src/mpi_instance.cpp
+116 -10
View File
@@ -22,33 +22,139 @@
* IN THE SOFTWARE.
*****************************************************************************/
#include <dlfcn.h>
#include "rocshmem/rocshmem.hpp"
#include "mpi_instance.hpp"
#include "util.hpp"
#if !defined(HAVE_EXTERNAL_MPI)
// Open MPI specific symbols
struct ompi_internal_symbols_t ompi_symbols_;
#endif //!defined(HAVE_EXTERNAL_MPI)
namespace rocshmem {
static void* mpilib_handle_{nullptr};
struct mpilib_funcs_t mpilib_ftable_;
int MPIInstance::mpilib_dl_init() {
if (mpilib_handle_ != nullptr)
return ROCSHMEM_SUCCESS;
mpilib_handle_ = dlopen("libmpi.so", RTLD_NOW);
if (!mpilib_handle_) {
printf("Could not open libmpi.so. Returning\n");
return ROCSHMEM_ERROR;
}
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Init_thread);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Initialized);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Finalize);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Finalized);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_rank);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_size);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Abort);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Get_address);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Type_size);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Iprobe);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Testsome);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_split);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_split_type);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_group);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_create_group);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_dup);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Comm_free);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Group_free);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Group_translate_ranks);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Group_incl);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Allgather);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Alltoall);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Allreduce);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Bcast);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Barrier);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Iallreduce);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Ibarrier);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_create);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_free);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_flush);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_flush_all);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_flush_local);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_lock);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_lock_all);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_unlock);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_unlock_all);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_lock);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Win_sync);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Get);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Rget);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Put);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Rput);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Compare_and_swap);
DLSYM_HELPER(mpilib_ftable_, MPI_, mpilib_handle_, Fetch_and_op);
#if !defined(HAVE_EXTERNAL_MPI)
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_comm_world);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_comm_null);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_datatype_null);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_request_null);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_info_null);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_max);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_min);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_sum);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_prod);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_band);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_bor);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_bxor);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_replace);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_op_no_op);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_char);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_unsigned_char);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_signed_char);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_short);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_unsigned_short);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_int);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_unsigned);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_long);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_unsigned_long);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_long_long_int);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_unsigned_long_long);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_float);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_double);
DLSYM_VAR_HELPER(ompi_symbols_, mpilib_handle_, ompi_mpi_long_double);
#endif //!defined(HAVE_EXTERNAL_MPI)
return ROCSHMEM_SUCCESS;
}
void MPIInstance::mpilib_dl_close() {
if (mpilib_handle_ != nullptr) {
dlclose(mpilib_handle_);
mpilib_handle_ = nullptr;
}
}
MPIInstance::MPIInstance(MPI_Comm comm) {
int is_init{0};
MPI_Initialized(&is_init);
mpilib_ftable_.Initialized(&is_init);
if (!is_init) {
int provided;
MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);
mpilib_ftable_.Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);
init_in_this_class = 1;
}
if (comm == MPI_COMM_NULL) {
comm = MPI_COMM_WORLD;
}
MPI_Comm_size(comm, &nprocs_);
MPI_Comm_rank(comm, &my_rank_);
mpilib_ftable_.Comm_size(comm, &nprocs_);
mpilib_ftable_.Comm_rank(comm, &my_rank_);
}
MPIInstance::~MPIInstance() {
int finalized{0};
MPI_Finalized(&finalized);
mpilib_ftable_.Finalized(&finalized);
if (!finalized && init_in_this_class) {
MPI_Finalize();
mpilib_ftable_.Finalize();
}
}