MSCCL: Improve executor and integrate scheduler (#694)

* MSCCL: improve executor and add scheduler for testing

* Use external scheduler

* Fix cmake error

* Address comments

* Fix thread safe issue

* Make MSCCL lifecycle APIs thread safe

* Make MSCCL internal scheduler aware of topology hint

* Revise error message

src/graph/rome_models.cc

@@ -392,7 +392,7 @@ static struct rcclRomeModel rome_model_56 = {
   .gdrLevel = { },
   .pattern = "40404040",
   .ringBase = "0 1 3 2 6 7 15 14 10 11 9 8 12 13 5 4|0 1 2 3 7 6 13 12 8 9 10 11 15 14 5 4|0 2 3 7 6 14 15 11 10 8 9 13 12 4 5 1|4 5 13 12 8 9 11 10 14 15 7 6 2 3 1 0|4 5 14 15 11 10 9 8 12 13 6 7 3 2 1 0|1 5 4 12 13 9 8 10 11 15 14 6 7 3 2 0",
-  .options = "pivotA2AEnabled=1,pivotA2ANumBiRings=3,tuning=1",
+  .options = "pivotA2AEnabled=1,pivotA2ANumBiRings=3,tuning=1,mscclEnabled=1",
   .treeBase = "10 11|14 15|6 7|2 3|0 1|4 5|12 13|8 9",
 };
 
@@ -842,6 +842,8 @@ static void parseOptions(struct ncclTopoSystem* system, const char *options) {
         system->ll128Enabled = (bool)atol(tokens[i*2+1]);
       } else if (strcmp(tokens[i*2], "baseBw") == 0) {
         system->baseBw = std::stof(tokens[i*2+1]);
+      } else if (strcmp(tokens[i*2], "mscclEnabled") == 0) {
+        system->mscclEnabled = (bool)atol(tokens[i*2+1]);
       }
     }
     free(str_temp);

src/graph/topo.h

@@ -167,6 +167,7 @@ struct ncclTopoSystem {
   int pivotA2ANumBiRings;
   bool ll128Enabled;
   float baseBw;
+  bool mscclEnabled;
 };
 
 ncclResult_t ncclTopoGetNode(struct ncclTopoSystem* system, struct ncclTopoNode** node, int type, uint64_t id);