diff --git a/src/graph/rome_models.h b/src/graph/rome_models.h index 4788fb2801..4b671b0119 100755 --- a/src/graph/rome_models.h +++ b/src/graph/rome_models.h @@ -96,6 +96,76 @@ static struct rcclRomeModel rome_model_33 = { .ringBase = "1 4 5 7 0 3 2 6|4 1 7 5 6 2 3 0", }; +static struct rcclRomeModel rome_model_30 = { + .nGpus = 8, .nCpus = 8, .nNics = 0, .nLinks = 2, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xe3000, 0xc3000, 0xc6000, 0x83000, }, + .gpuNuma = { 1, 2, 2, 3, 4, 5, 5, 7, }, + .nicNuma = { }, + .connMatrix = { 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, }, + .pattern = "0010201010200010", + .ringBase = "3 0 1 2 6 7 5 4|2 1 0 3 7 6 4 5", +}; + +static struct rcclRomeModel rome_model_32 = { + .nGpus = 8, .nCpus = 8, .nNics = 0, .nLinks = 2, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xe3000, 0xc3000, 0xc6000, 0x83000, }, + .gpuNuma = { 1, 2, 2, 3, 4, 5, 5, 7, }, + .nicNuma = { }, + .connMatrix = { 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, }, + .pattern = "0010201010200010", + .ringBase = "0 6 2 3 4 5 7 1|3 2 6 0 1 7 5 4", +}; + +static struct rcclRomeModel rome_model_24 = { + .nGpus = 8, .nCpus = 4, .nNics = 0, .nLinks = 2, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xe3000, 0xc3000, 0xc6000, 0x83000, }, + .gpuNuma = { 0, 1, 1, 1, 2, 2, 2, 3, }, + .nicNuma = { }, + .connMatrix = { 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, }, + .pattern = "10303010", + .ringBase = "0 1 2 3 5 7 6 4|1 0 3 2 7 5 4 6", +}; + +static struct rcclRomeModel rome_model_26 = { + .nGpus = 8, .nCpus = 4, .nNics = 0, .nLinks = 2, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xe3000, 0xc3000, 0xc6000, 0x83000, }, + .gpuNuma = { 0, 1, 1, 1, 2, 2, 2, 3, }, + .nicNuma = { }, + .connMatrix = { 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, }, + .pattern = "10303010", + .ringBase = "4 5 7 1 0 3 2 6|3 0 6 2 1 7 5 4", +}; + +static struct rcclRomeModel rome_model_23 = { + .nGpus = 8, .nCpus = 4, .nNics = 0, .nLinks = 2, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, }, + .gpuNuma = { 0, 1, 1, 1, 2, 2, 3, 3, }, + .nicNuma = { }, + .connMatrix = { 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, }, + .pattern = "10302020", + .ringBase = "1 7 6 4 5 2 0 3|2 5 3 0 4 6 7 1", +}; + +static struct rcclRomeModel rome_model_38 = { + .nGpus = 8, .nCpus = 7, .nNics = 0, .nLinks = 2, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, }, + .gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, }, + .nicNuma = { }, + .connMatrix = { 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, }, + .pattern = "00102010002010", + .ringBase = "6 7 1 4 3 5 2 0|0 2 5 3 4 1 7 6", +}; + +static struct rcclRomeModel rome_model_28 = { + .nGpus = 8, .nCpus = 4, .nNics = 0, .nLinks = 3, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, }, + .gpuNuma = { 0, 1, 1, 1, 2, 2, 3, 3, }, + .nicNuma = { }, + .connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, }, + .pattern = "10302020", + .ringBase = "0 3 2 1 4 5 6 7|7 6 5 4 1 2 3 0|0 2 5 7 4 6 3 1|1 3 6 4 7 5 2 0", +}; + static struct rcclRomeModel romeTopoModels[] = { rome_model_22, rome_model_25, @@ -103,4 +173,11 @@ static struct rcclRomeModel romeTopoModels[] = { rome_model_29, rome_model_31, rome_model_33, -}; \ No newline at end of file + rome_model_30, + rome_model_32, + rome_model_24, + rome_model_26, + rome_model_23, + rome_model_38, + rome_model_28, +}; diff --git a/src/graph/search.cc b/src/graph/search.cc index b3589ff0c7..e6afe49a66 100644 --- a/src/graph/search.cc +++ b/src/graph/search.cc @@ -913,7 +913,7 @@ static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRo // number of GPUs and NICs on each numa node is used as first screening pattern for (int i = 0; i < romeTopo->nCpus; i++) { int g, n; - if (!getGpuNetCount(system, i, &g, &n, romeTopo->nNics, net_map)) return ncclSuccess; + getGpuNetCount(system, i, &g, &n, romeTopo->nNics, net_map); pattern[i*2] = '0' + g; pattern[i*2+1] = '0' + n; } @@ -1036,10 +1036,15 @@ static ncclResult_t parseRome4P2H(struct ncclTopoSystem* system, struct ncclTopo //printf("No solution in %.2fms (%d iter)\n", t, time); return ncclSuccess; } - //printf("Solution in %.2fms (%d iter): ", t, time); - //for (int k = 0; k < ngpus; k++) printf("%d ", g[k]); - //printf("\n"); + char line[1024]; + sprintf(line, "Found matching Rome model index %d in %.2fms (%d iter) with GPU mapping: ", i, t, time); + int offset = strlen(line); + for (int k = 0; k < ngpus; k++) { + sprintf(line+offset, "%d ", g[k]); + offset = strlen(line); + } + INFO(NCCL_GRAPH, "%s", line); // create 4P2H based on reference and remapped ids NCCLCHECK(parseGraph(romeTopoModels[i].ringBase, system, graph, g, romeTopo.nNics, net_map)); return ncclSuccess; diff --git a/tools/scripts/topo_val.sh b/tools/scripts/topo_val.sh index d20a849dcd..556c0f60b6 100755 --- a/tools/scripts/topo_val.sh +++ b/tools/scripts/topo_val.sh @@ -21,7 +21,7 @@ DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -for i in {0..37} +for i in {0..38} do $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log" $DIR/../TopoVisual/topo_visual.sh -i "topo_m$i.log" diff --git a/tools/topo_expl/models/topo_8p_rome_n2_1.xml b/tools/topo_expl/models/topo_8p_rome_n2_1.xml index 0f965f7a3a..9edf0838ca 100644 --- a/tools/topo_expl/models/topo_8p_rome_n2_1.xml +++ b/tools/topo_expl/models/topo_8p_rome_n2_1.xml @@ -1,31 +1,72 @@ - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + - + - + @@ -33,45 +74,9 @@ - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + diff --git a/tools/topo_expl/models/topo_8p_rome_n4.xml b/tools/topo_expl/models/topo_8p_rome_n4.xml new file mode 100644 index 0000000000..05a4197b18 --- /dev/null +++ b/tools/topo_expl/models/topo_8p_rome_n4.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/topo_expl/topo_expl.cpp b/tools/topo_expl/topo_expl.cpp index 67323651bf..338e63b57e 100644 --- a/tools/topo_expl/topo_expl.cpp +++ b/tools/topo_expl/topo_expl.cpp @@ -92,7 +92,7 @@ NodeModelDesc model_descs[] = { {4, "topo_8p6l_6nic.xml", "4 nodes gfx908 8P6L 6 NICs"}, {1, "topo_8p_rome_n2.xml", "single node 8 VEGA20 Rome NPS=2"}, {4, "topo_8p_rome_n2.xml", "4 nodes 8 VEGA20 Rome NPS=2"}, - {4, "topo_8p_rome_n2_1.xml", "4 nodes 8 VEGA20 Rome NPS=2 Alt. Model"}, + {1, "topo_8p_rome_n2_1.xml", "single node 8 VEGA20 Rome NPS=2 Alt. Model"}, {1, "topo_8p_ts1.xml", "single node 8 VEGA20 TS1"}, {4, "topo_8p_ts1.xml", "4 nodes 8 VEGA20 TS1"}, {1, "topo_8p_ts1_1.xml", "single node 8 VEGA20 TS1 Alt. Model"}, @@ -107,6 +107,7 @@ NodeModelDesc model_descs[] = { {4, "topo_4p3l_ia.xml", "4 nodes 8 gfx908"}, {4, "topo_8p_rome_n2_2.xml", "4 nodes 8 VEGA20 Rome NPS=2 Alt. Model 2 NET/IF"}, {4, "topo_8p_ts1_n4_2.xml", "4 nodes 8 VEGA20 TS1 NPS=4 3 NET/IF"}, + {1, "topo_8p_rome_n4.xml", "single node 8 VEGA20 Rome NPS=4"}, }; int main(int argc,char* argv[])