Move MSCCL algorithm loading to initialization to workaround HIP graph conflict (#982)

* MSCCL: pre-specify channels and pre-load algorithms

* add mutex

* fix bug

* clean include

* disable all-gathers temporarily

src/include/msccl/msccl_lifecycle.h

@@ -18,6 +18,8 @@ bool mscclIsCaller();
 
 bool mscclAvailable();
 
+ncclResult_t mscclSchedulerInit(ncclComm_t comm, int* numChannelsRequired);
+
 ncclResult_t mscclInit(ncclComm_t comm);
 
 ncclResult_t mscclGroupStart();

src/include/msccl/msccl_struct.h

@@ -91,6 +91,8 @@ struct mscclAlgoMeta {
   std::string filePath;
   // number of chunks of input/output in each MSCCL algorithm loop
   int nChunksPerLoop;
+  // number of channels needed by MSCCL algorithm
+  int nChannels;
   // number of ranks required by this algorithm
   int nRanks;
   // need to times nRanks for all-gather, reduce-scatter and all-to-all