diff --git a/projects/rccl/src/graph/rome_models.h b/projects/rccl/src/graph/rome_models.h index 286d860842..20d08fe5b5 100755 --- a/projects/rccl/src/graph/rome_models.h +++ b/projects/rccl/src/graph/rome_models.h @@ -213,6 +213,28 @@ static struct rcclRomeModel rome_model_44 = { .ringBase = "5 4 7 6 2 1 3 0|5 6 7 4 1 0 2 3", }; +static struct rcclRomeModel rome_model_45 = { + .nGpus = 8, .nCpus = 7, .nNics = 0, .nLinks = 3, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, }, + .nicIds = { }, + .gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, }, + .nicNuma = { }, + .connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, }, + .pattern = "00102010002010", + .ringBase = "0 1 2 3 4 5 6 7|0 2 5 7 4 6 1 3|0 3 1 6 4 7 5 2|0 7 6 5 4 3 2 1", +}; + +static struct rcclRomeModel rome_model_46 = { + .nGpus = 8, .nCpus = 7, .nNics = 1, .nLinks = 3, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, }, + .nicIds = { 0xe1000, }, + .gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, }, + .nicNuma = { 4, }, + .connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, }, + .pattern = "00102010012010", + .ringBase = "6 5 7 4 1 2 3 0|7 4 6 5 1 0 3 2", +}; + static struct rcclRomeModel romeTopoModels[] = { rome_model_22, rome_model_25, @@ -230,4 +252,6 @@ static struct rcclRomeModel romeTopoModels[] = { rome_model_40, rome_model_42, rome_model_44, + rome_model_45, + rome_model_46, }; diff --git a/projects/rccl/src/graph/search.cc b/projects/rccl/src/graph/search.cc index 0a887b6a1e..a2c0d0b710 100644 --- a/projects/rccl/src/graph/search.cc +++ b/projects/rccl/src/graph/search.cc @@ -973,8 +973,10 @@ static bool permuteGpuIds(int *g, int n, int last, struct rcclRomeModel* ref, st if (i < ref->nGpus) return false; // match XGMI connection for (i = 0; i < ref->nGpus; i++) { - for (j = 0; j < ref->nGpus; j++) + for (j = 0; j < ref->nGpus; j++) { if (ref->connMatrix[i*ref->nGpus+j] != topo->connMatrix[g[i]*ref->nGpus+g[j]]) break; + if ((ref->gpuIds[i]-ref->gpuIds[j])*(topo->gpuIds[g[i]]-topo->gpuIds[g[j]]) < 0) break; + } if (j < ref->nGpus) break; } if (i < ref->nGpus) return false; diff --git a/projects/rccl/tools/scripts/topo_val.sh b/projects/rccl/tools/scripts/topo_val.sh index fcc07c3876..9afe237aa6 100755 --- a/projects/rccl/tools/scripts/topo_val.sh +++ b/projects/rccl/tools/scripts/topo_val.sh @@ -21,7 +21,7 @@ DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -for i in {0..44} +for i in {0..46} do $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log" $DIR/../TopoVisual/topo_visual.sh -i "topo_m$i.log" diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_rome_n4_1.xml b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n4_1.xml new file mode 100644 index 0000000000..b4af22a14f --- /dev/null +++ b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n4_1.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rccl/tools/topo_expl/topo_expl.cpp b/projects/rccl/tools/topo_expl/topo_expl.cpp index ff283cc5a1..48709bb14a 100644 --- a/projects/rccl/tools/topo_expl/topo_expl.cpp +++ b/projects/rccl/tools/topo_expl/topo_expl.cpp @@ -114,6 +114,8 @@ NodeModelDesc model_descs[] = { {4, "topo_4p3l_n4.xml", "4 nodes 8 gfx908 Rome NPS=4"}, {1, "topo_4p3l_n2_1.xml", "single node 8 gfx908 Rome"}, {4, "topo_4p3l_n2_1.xml", "4 nodes 8 gfx908 Rome"}, + {1, "topo_8p_rome_n4_1.xml", "single node 8 gfx908 Rome NPS=4"}, + {4, "topo_8p_rome_n4_1.xml", "4 nodes node 8 gfx908 Rome NPS=4"}, }; int main(int argc,char* argv[])