AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove MPI compile-time dependency (#264)

Browse Source 
* use dlsym for MPI functions

to allow compiling without MPI support, convert the usage of MPI functions and symbols to be based on a dlopen/dlsym based mechanism. Turns out this cannot be done entirely vendor neutral, slightly different solutions might be required for Open MPI, MPICH and the new MPI ABI.

* checkpoint

more work to be done.

* checkpoint 2

* checkpoint 3

* checkpoint 4

examples compile and link correctly

* checkpoitn 5 (I think)

* Checkpoitn 6

* dyld-mpi: adapt GDA

* dyldmpi: tests that depend on MPI need to link with it themselves

* do not ../mpi_instance.h

* dyldmpi: make the symetricHeapTestFixture compile

* dyldmpi: Change cmakery, compiles and run gda w/o external MPI

* Make it also compile in external MPI mode

* dyldmpi: ipc unit tests compile but do not link

* dyldmpi: new approach, if external mpi required, link with mpi,
otherwise use ompi5 abi

* C-style comments in cmakelist..

* dyldmpi: examples: do not fail compiling if MPI not found at build time,
instead do not compile the MPI required examples

* more updates to CMake logic

* convert RO backend

and a few other cleanups

* update some unit tests

to work with the dlopen MPI environment correctly.

---------

Co-authored-by: Aurelien Bouteiller <abouteil@amd.com>

[ROCm/rocshmem commit: e4c427a736]
This commit is contained in:
Edgar Gabriel
2025-10-01 08:06:56 -05:00
committed by GitHub
parent 4f955324ac
commit 53fa35b980
50 changed files with 712 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocshmem/src/gda/backend_gda.cpp
+11 -10
View File
@@ -24,15 +24,16 @@
#include <cstring>
#include "backend_gda.hpp"
#include "gda_team.hpp"
#include "util.hpp"
#include "topology.hpp"
#include <hip/hip_runtime.h>
#include <cstdlib>
#include <cassert>
#include "backend_gda.hpp"
#include "mpi_instance.hpp"
#include "gda_team.hpp"
#include "util.hpp"
#include "topology.hpp"
namespace rocshmem {
#define NET_CHECK(cmd) { \
@@ -303,7 +304,7 @@ void GDABackend::create_new_team([[maybe_unused]] Team *parent_team,
* the pool of available work arrays.
*/
if (team_comm != MPI_COMM_NULL) {
NET_CHECK(MPI_Allreduce(team_pool_bitmask_, team_reduced_bitmask_, team_bitmask_size_,
NET_CHECK(mpilib_ftable_.Allreduce(team_pool_bitmask_, team_reduced_bitmask_, team_bitmask_size_,
MPI_CHAR, MPI_BAND, team_comm));
} else {
Allreduce_char_BAND (team_pool_bitmask_, team_reduced_bitmask_, team_bitmask_size_, parent_team);
@@ -361,7 +362,7 @@ void GDABackend::dump_backend_stats() {
__host__ void GDABackend::global_exit(int status) {
if (backend_comm != MPI_COMM_NULL)
MPI_Abort(backend_comm, status);
mpilib_ftable_.Abort(backend_comm, status);
else
abort();
}
@@ -536,7 +537,7 @@ void GDABackend::setup_teams() {
void GDABackend::rte_barrier() {
if (backend_comm != MPI_COMM_NULL) {
NET_CHECK(MPI_Barrier(backend_comm));
NET_CHECK(mpilib_ftable_.Barrier(backend_comm));
} else {
backend_bootstr->barrier();
}
@@ -668,7 +669,7 @@ void GDABackend::exchange_qp_dest_info() {
for (int i = 0; i < maximum_num_contexts_ + 1; i++) {
if (backend_comm != MPI_COMM_NULL) {
MPI_Alltoall(MPI_IN_PLACE, sizeof(dest_info_t), MPI_CHAR, dest_info.data() + i * num_pes, sizeof(dest_info_t), MPI_CHAR, backend_comm);
mpilib_ftable_.Alltoall(MPI_IN_PLACE, sizeof(dest_info_t), MPI_CHAR, dest_info.data() + i * num_pes, sizeof(dest_info_t), MPI_CHAR, backend_comm);
} else {
Alltoall_char_inplace(reinterpret_cast<char*>(dest_info.data() + i * num_pes), sizeof(dest_info_t), ROCSHMEM_TEAM_WORLD);
}
@@ -695,7 +696,7 @@ void GDABackend::setup_heap_memory_rkey() {
CHECK_HIP(hipStreamSynchronize(stream));
if (backend_comm != MPI_COMM_NULL)
MPI_Allgather(MPI_IN_PLACE, sizeof(uint32_t), MPI_CHAR, host_rkey_cpy, sizeof(uint32_t), MPI_CHAR, backend_comm);
mpilib_ftable_.Allgather(MPI_IN_PLACE, sizeof(uint32_t), MPI_CHAR, host_rkey_cpy, sizeof(uint32_t), MPI_CHAR, backend_comm);
else
backend_bootstr->allGather(host_rkey_cpy, sizeof(uint32_t));