From 39f83d5bb7a2b9edd253832294ae575ea90ec964 Mon Sep 17 00:00:00 2001
From: Pedram Alizadeh <pmohamma@amd.com>
Date: Mon, 3 Apr 2023 11:37:13 -0400
Subject: [PATCH] fixing the error message for mpirun when number of requested
 GPUs exceeds the limits (#33)

[ROCm/rccl-tests commit: e146460810a88b1a47b22308be63702485994fce]
---
 projects/rccl-tests/src/common.cu | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/projects/rccl-tests/src/common.cu b/projects/rccl-tests/src/common.cu
index 332cc3f272..eb0743a52f 100644
--- a/projects/rccl-tests/src/common.cu
+++ b/projects/rccl-tests/src/common.cu
@@ -1127,11 +1127,13 @@ int main(int argc, char* argv[]) {
   }
 
   HIPCHECK(hipGetDeviceCount(&numDevices));
+#ifndef MPI_SUPPORT  
   if (nGpus > numDevices)
   {
       fprintf(stderr, "[ERROR] The number of requested GPUs (%d) is greater than the number of GPUs available (%d)\n", nGpus, numDevices);
       return testNcclError;
   }
+#endif
   if (minBytes > maxBytes) {
     fprintf(stderr, "invalid sizes for 'minbytes' and 'maxbytes': %llu > %llu\n",
            (unsigned long long)minBytes,
@@ -1154,7 +1156,14 @@ int main(int argc, char* argv[]) {
     return -1;
   }
 #ifdef MPI_SUPPORT
+  int nProcs = 1;
   MPI_Init(&argc, &argv);
+  MPI_Comm_size(MPI_COMM_WORLD, &nProcs);
+  if (nGpus * nProcs > numDevices)
+  {
+      fprintf(stderr, "[ERROR] The number of requested GPUs (%d) is greater than the number of GPUs available (%d)\n", nGpus*nProcs, numDevices);
+      return testNcclError;
+  }
 #endif
   TESTCHECK(run());
   return 0;