Merge remote-tracking branch 'origin/develop' into 2.8.3

[ROCm/rccl commit: ab1e7a0318]

projects/rccl/src/graph/rome_models.h

@@ -213,6 +213,28 @@ static struct rcclRomeModel rome_model_44 = {
   .ringBase = "5 4 7 6 2 1 3 0|5 6 7 4 1 0 2 3",
 };
 
+static struct rcclRomeModel rome_model_45 = {
+  .nGpus = 8, .nCpus = 7, .nNics = 0, .nLinks = 3,
+  .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, },
+  .nicIds = { },
+  .gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
+  .nicNuma = { },
+  .connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
+  .pattern = "00102010002010",
+  .ringBase = "0 1 2 3 4 5 6 7|0 2 5 7 4 6 1 3|0 3 1 6 4 7 5 2|0 7 6 5 4 3 2 1",
+};
+
+static struct rcclRomeModel rome_model_46 = {
+  .nGpus = 8, .nCpus = 7, .nNics = 1, .nLinks = 3,
+  .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, },
+  .nicIds = { 0xe1000, },
+  .gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
+  .nicNuma = { 4, },
+  .connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
+  .pattern = "00102010012010",
+  .ringBase = "6 5 7 4 1 2 3 0|7 4 6 5 1 0 3 2",
+};
+
 static struct rcclRomeModel romeTopoModels[] = {
   rome_model_22,
   rome_model_25,
@@ -230,4 +252,6 @@ static struct rcclRomeModel romeTopoModels[] = {
   rome_model_40,
   rome_model_42,
   rome_model_44,
+  rome_model_45,
+  rome_model_46,
 };

projects/rccl/src/graph/search.cc

@@ -462,7 +462,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
         for (int i = 0; i<system->nodes[GPU].count; i++)
           if (paths[i].count < paths[f].count) f = i;
         int t = 1 << 10;
-        NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, &t, NET, n, f));
+        NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, (f == 0) ? FORCED_ORDER_PCI : 0, &t, NET, n, f));
         if (t == -1) *time = -1;
       }
 
@@ -998,8 +998,10 @@ static bool permuteGpuIds(int *g, int n, int last, struct rcclRomeModel* ref, st
     if (i < ref->nGpus) return false;
     // match XGMI connection
     for (i = 0; i < ref->nGpus; i++) {
-      for (j = 0; j < ref->nGpus; j++)
+      for (j = 0; j < ref->nGpus; j++) {
         if (ref->connMatrix[i*ref->nGpus+j] != topo->connMatrix[g[i]*ref->nGpus+g[j]]) break;
+        if ((ref->gpuIds[i]-ref->gpuIds[j])*(topo->gpuIds[g[i]]-topo->gpuIds[g[j]]) < 0) break;
+      }
       if (j < ref->nGpus) break;
     }
     if (i < ref->nGpus) return false;

projects/rccl/src/graph/xml.cc

@@ -492,7 +492,10 @@ ncclResult_t ncclTopoGetXmlFromSys(struct ncclXmlNode* pciNode, struct ncclXml*
     char portSpeedStr[MAX_STR_LEN];
     float portSpeed;
     NCCLCHECK(ncclTopoGetStrFromSys(path, "../max_link_speed", portSpeedStr));
-    sscanf(portSpeedStr, "%f GT/s", &portSpeed);
+    if (portSpeedStr[0])
+      sscanf(portSpeedStr, "%f GT/s", &portSpeed);
+    else
+      portSpeed = deviceSpeed;
     NCCLCHECK(xmlSetAttr(pciNode, "link_speed", portSpeed < deviceSpeed ? portSpeedStr : deviceSpeedStr));
   }
   NCCLCHECK(xmlGetAttrIndex(pciNode, "link_width", &index));
@@ -502,7 +505,11 @@ ncclResult_t ncclTopoGetXmlFromSys(struct ncclXmlNode* pciNode, struct ncclXml*
     NCCLCHECK(ncclTopoGetStrFromSys(path, "max_link_width", strValue));
     int deviceWidth = strtol(strValue, NULL, 0);
     NCCLCHECK(ncclTopoGetStrFromSys(path, "../max_link_width", strValue));
-    int portWidth = strtol(strValue, NULL, 0);
+    int portWidth;
+    if (strValue[0])
+      portWidth = strtol(strValue, NULL, 0);
+    else
+      portWidth = deviceWidth;
     NCCLCHECK(xmlSetAttrInt(pciNode, "link_width", std::min(deviceWidth,portWidth)));
   }
   struct ncclXmlNode* parent = pciNode->parent;