Optimize MSCCL all-gather algorithms for gfx942 (#964)

src/misc/msccl/msccl_setup.cc

@@ -3,6 +3,7 @@
  * Licensed under the MIT License.
  ************************************************************************/
 
+#include "channel.h"
 #include "checks.h"
 #include "collectives.h"
 #include "proxy.h"
@@ -89,11 +90,16 @@ ncclResult_t mscclSetupSyncFlags(hipStream_t stream) {
 ncclResult_t mscclSetupConnections(struct mscclAlgo* hostAlgo, ncclComm_t comm) {
   mscclStatus& status = mscclGetStatus();
 
-  // Check whether there is enough channels
-  if (hostAlgo->nChannels > comm->nChannels) {
-    WARN("MSCCL: number of channels available (%d) less than required (%d)", comm->nChannels, hostAlgo->nChannels);
+  // Check whether there are enough channels
+  if (hostAlgo->nChannels > MAXCHANNELS) {
+    WARN("MSCCL: max number of channels available (%d) less than required (%d)", MAXCHANNELS, hostAlgo->nChannels);
     return ncclInvalidUsage;
   }
+  if (hostAlgo->nChannels > comm->nChannels) {
+    for (int channelId = comm->nChannels; channelId < hostAlgo->nChannels; channelId++) {
+      NCCLCHECK(initChannel(comm, channelId));
+    }
+  }
 
   // Flag MSCCL connections
   for (int i = 0; i < hostAlgo->nChannels; i++) {