free MPI Communicators on destruction (#134)

make sure that all communicators that have been created during the
runtime of the application are correctly freed again, to avoid memory
leaks.

[ROCm/rocshmem commit: bce329abd6]

projects/rocshmem/src/backend_bc.cpp

@@ -129,7 +129,11 @@ void Backend::destroy_remaining_ctxs() {
   }
 }
 
-Backend::~Backend() { CHECK_HIP(hipFree(print_lock)); }
+Backend::~Backend() {
+  CHECK_HIP(hipFree(print_lock));
+  if (backend_comm != MPI_COMM_NULL)
+    NET_CHECK(MPI_Comm_free(&backend_comm));
+}
 
 void Backend::dump_stats() {
   printf("PE %d\n", my_pe);

projects/rocshmem/src/backend_bc.hpp

@@ -223,7 +223,7 @@ class Backend {
    * @todo document where this is used and try to coalesce this into another
    * class
    */
-  MPI_Comm backend_comm{};
+  MPI_Comm backend_comm{MPI_COMM_NULL};
 
   /**
    * @brief Object contains the interface and internal data structures

projects/rocshmem/src/reverse_offload/mpi_transport.cpp

@@ -63,7 +63,10 @@ MPITransport::MPITransport(MPI_Comm comm, Queue* q)
   NET_CHECK(MPI_Comm_rank(ro_net_comm_world, &my_pe));
 }
 
-MPITransport::~MPITransport() {}
+MPITransport::~MPITransport() {
+  if (ro_net_comm_world != MPI_COMM_NULL)
+    NET_CHECK(MPI_Comm_free(&ro_net_comm_world));
+}
 
 void MPITransport::threadProgressEngine() {
   auto *bp{backend_proxy->get()};

projects/rocshmem/src/reverse_offload/mpi_transport.hpp

@@ -169,7 +169,7 @@ private:
 
   std::vector<int> outstanding{};
 
-  MPI_Comm ro_net_comm_world{};
+  MPI_Comm ro_net_comm_world{MPI_COMM_NULL};
 
   std::map<CommKey, MPI_Comm> comm_map{};
 

projects/rocshmem/src/rocshmem.cpp

@@ -422,7 +422,7 @@ __host__ int rocshmem_team_split_strided(
      * not */
     backend->team_tracker.track(*new_team);
   }
-
+  MPI_Comm_free (&team_comm);
   return 0;
 }
 

projects/rocshmem/src/team.cpp

@@ -78,8 +78,9 @@ __host__ Team::Team(Backend* handle, TeamInfo* team_info_wrt_parent,
       tinfo_wrt_parent(team_info_wrt_parent),
       tinfo_wrt_world(team_info_wrt_world),
       num_pes(_num_pes),
-      my_pe(_my_pe),
-      mpi_comm(_mpi_comm) {}
+      my_pe(_my_pe) {
+  MPI_Comm_dup (_mpi_comm, &mpi_comm);
+}
 
 __host__ __device__ int Team::get_pe_in_world(int pe) {
   int pe_start{tinfo_wrt_world->pe_start};
@@ -108,6 +109,9 @@ __host__ __device__ int Team::get_pe_in_my_team(int pe_in_world) {
   return pe_in_my_team;
 }
 
-__host__ Team::~Team() {}
+__host__ Team::~Team() {
+  if (mpi_comm != MPI_COMM_NULL)
+    MPI_Comm_free (&mpi_comm);
+}
 
 }  // namespace rocshmem