AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trim NICs when all GPUs are connected by XGMI (#430)

Browse Source 
* Trim NICs when all GPUs are connected by XGMI

* Only enable clique with maximum of 2 hops
This commit is contained in:
Wenkai Du
2021-10-05 18:27:43 -07:00
committed by GitHub
parent 51a1cf428e
commit 29c729d8b6
6 changed files with 159 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graph/paths.cc
+16 -2
View File
@@ -510,8 +510,22 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm*
}
if (ret) {
system->type |= RCCL_TOPO_GDR_ALL;
remove = 0;
INFO(NCCL_GRAPH, "GDR is available on all GPUs");
bool allXgmi = true;
// don't trim NICs unless all GPUs are connected by XGMI
for (int i = 0; i < system->nodes[GPU].count && allXgmi; i++) {
int cudaDev1 = system->nodes[GPU].nodes[i].gpu.dev;
for (int j = 0; j < system->nodes[GPU].count && allXgmi; j++) {
if (i == j) continue;
int cudaDev2 = system->nodes[GPU].nodes[j].gpu.dev;
bool isXGMI;
NCCLCHECK(ncclTopoGetLinkType(comm->topo, cudaDev1, cudaDev2, &isXGMI));
allXgmi &= isXGMI;
}
}
if (!allXgmi) {
remove = 0;
INFO(NCCL_GRAPH, "GDR is available on all GPUs");
}
}
}
comm->localRanks = system->nodes[GPU].count;