AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rail optimization for rings (#1140)

Browse Source 
- Modifies the ring creation algorithm to be friendlier to rail-optimized topologies (should not affect classic fabric topologies)

[ROCm/rccl commit: 4cb62f999a]
This commit is contained in:
gilbertlee-amd
2024-04-15 12:03:57 -06:00
committed by GitHub
parent 8ddb74e3b1
commit 422a7ffcbb
11 changed files with 296 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/graph/connect.cc
+27
View File
@@ -224,6 +224,33 @@ static ncclResult_t connectRings(struct ncclComm* comm, int* ringRecv, int* ring
TRACE(NCCL_GRAPH, "Ring %d : %d -> %d -> %d", c, channel0->ring.prev, comm->rank, channel0->ring.next);
if (channel1) TRACE(NCCL_GRAPH, "Ring %d : %d -> %d -> %d", c+nChannels, channel1->ring.prev, comm->rank, channel1->ring.next);
}
// [RCCL] Print off the recv/send local ranks per node, per channel
if (comm->rank == 0)
{
char buff[2048] = "";
int offset = 0;
int inc;
int numChannels = (nChannels > MAXCHANNELS/2) ? 2 * nChannels : nChannels;
for (int c = 0; c < numChannels; c++) {
sprintf(buff + offset, " %02d%n", c, &inc);
offset += inc;
}
INFO(NCCL_GRAPH, "[RINGS] %s", buff);
for (int n = 0; n < nNodes; n++) {
offset = 0;
for (int c = 0; c < nChannels; c++) {
int recvRank = comm->rankToLocalRank[ringRecv[c*comm->nNodes+n]];
int sendRank = comm->rankToLocalRank[ringSend[c*comm->nNodes+n]];
sprintf(buff + offset, " %02d->%02d%n", recvRank, sendRank, &inc);
offset += inc;
}
INFO(NCCL_GRAPH, "[RINGS] %s", buff);
}
}
return ncclSuccess;
}