diff --git a/projects/rccl/src/graph/paths.cc b/projects/rccl/src/graph/paths.cc index 5131a204c7..712ac612bb 100644 --- a/projects/rccl/src/graph/paths.cc +++ b/projects/rccl/src/graph/paths.cc @@ -509,7 +509,6 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm* } } if (ret) { - system->type |= RCCL_TOPO_GDR_ALL; bool allXgmi = true; // don't trim NICs unless all GPUs are connected by XGMI for (int i = 0; i < system->nodes[GPU].count && allXgmi; i++) { @@ -524,6 +523,7 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm* } if (!allXgmi) { remove = 0; + system->type |= RCCL_TOPO_GDR_ALL; INFO(NCCL_GRAPH, "GDR is available on all GPUs"); } } diff --git a/projects/rccl/src/graph/rome_models.cc b/projects/rccl/src/graph/rome_models.cc index c43c03e3d9..8c2fb81e65 100644 --- a/projects/rccl/src/graph/rome_models.cc +++ b/projects/rccl/src/graph/rome_models.cc @@ -434,6 +434,32 @@ static struct rcclRomeModel rome_model_65 = { .netGdrLevel = 5, }; +static struct rcclRomeModel rome_model_66 = { + .nGpus = 8, .nCpus = 2, .nNics = 0, .nLinks = 3, + .gpuIds = { 0x29000, 0x2c000, 0x2f000, 0x32000, 0xad000, 0xb0000, 0xb3000, 0xb6000, }, + .nicIds = { }, + .gpuNuma = { 1, 1, 1, 1, 3, 3, 3, 3, }, + .nicNuma = { }, + .connMatrix = { 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, }, + .gdrLevel = { }, + .pattern = "4040", + .ringBase = "0 6 7 5 4 2 3 1|1 3 2 4 5 7 6 0|0 1 7 6 2 3 5 4|4 5 3 2 6 7 1 0", + .netGdrLevel = -2, +}; + +static struct rcclRomeModel rome_model_67 = { + .nGpus = 8, .nCpus = 2, .nNics = 4, .nLinks = 3, + .gpuIds = { 0x29000, 0x2c000, 0x2f000, 0x32000, 0xad000, 0xb0000, 0xb3000, 0xb6000, }, + .nicIds = { 0x1d000, 0x1e000, 0xa1000, 0xa2000, }, + .gpuNuma = { 1, 1, 1, 1, 3, 3, 3, 3, }, + .nicNuma = { 1, 1, 3, 3, }, + .connMatrix = { 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, }, + .gdrLevel = { 4, 4, 4, 4, 6, 6, 6, 6, 4, 4, 4, 4, 6, 6, 6, 6, 6, 6, 6, 6, 4, 4, 4, 4, 6, 6, 6, 6, 4, 4, 4, 4, }, + .pattern = "4242", + .ringBase = "N3 7 6 0 1 3 2 4 5 N2|N2 5 4 2 3 1 0 6 7 N3|N1 2 3 5 4 0 1 7 6 N3|N2 4 5 3 2 6 7 1 0 N0|N1 3 2 4 5 7 6 0 1 N0|N0 1 0 6 7 5 4 2 3 N1|N0 0 1 7 6 2 3 5 4 N2|N3 6 7 1 0 4 5 3 2 N1", + .netGdrLevel = -2, +}; + static struct rcclRomeModel romeTopoModels[] = { rome_model_22, rome_model_25, @@ -465,6 +491,8 @@ static struct rcclRomeModel romeTopoModels[] = { rome_model_62, rome_model_63, rome_model_65, + rome_model_66, + rome_model_67, }; /* Parse user defined rings. Format is like : diff --git a/projects/rccl/tools/scripts/topo_val.sh b/projects/rccl/tools/scripts/topo_val.sh index c9cb158345..77e513fa8e 100755 --- a/projects/rccl/tools/scripts/topo_val.sh +++ b/projects/rccl/tools/scripts/topo_val.sh @@ -21,7 +21,7 @@ DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -for i in {0..65} +for i in {0..67} do if [[ $i -eq 50 ]] || [[ $i -eq 51 ]] then diff --git a/projects/rccl/tools/topo_expl/models/topo_8p1h.xml b/projects/rccl/tools/topo_expl/models/topo_8p1h.xml new file mode 100644 index 0000000000..8345a81900 --- /dev/null +++ b/projects/rccl/tools/topo_expl/models/topo_8p1h.xml @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rccl/tools/topo_expl/topo_expl.cpp b/projects/rccl/tools/topo_expl/topo_expl.cpp index b758322fd0..4aa8dcb477 100644 --- a/projects/rccl/tools/topo_expl/topo_expl.cpp +++ b/projects/rccl/tools/topo_expl/topo_expl.cpp @@ -135,6 +135,8 @@ NodeModelDesc model_descs[] = { {4, "topo_8p_4nics.xml", "4 nodes 8P 4 NICs"}, {1, "topo_16p1h_vm.xml", "single node 16P1H VM"}, {4, "topo_16p1h_vm.xml", "4 nodes 16P1H VM"}, + {1, "topo_8p1h.xml", "single node 8P1H"}, + {4, "topo_8p1h.xml", "4 nodes 8P1H"}, }; int main(int argc,char* argv[])