AI/rocm-systems
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Alltoallv RCCL kernel implementation (#269)

Browse Source 
* Add alltoallv API and implementation

* Extend Rome P2P channel limit to multinode and alltoall kernels

* topo_expl: fix compilation and sync up with main

* gtest: use RCCL alltoallv API

* Code review changes

[ROCm/rccl commit: b871ea3c0c]
This commit is contained in:
Wenkai Du authored and GitHub committed 2020-09-30 16:25:36 -07:00
1 parent 256de55920
commit dbde26e681
22 files changed
+300 -60
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

projects/rccl/src/proxy.cc
+19 -1
View File
@@ -166,7 +166,9 @@ ncclResult_t ncclProxySaveP2p(struct ncclInfo* info, struct ncclChannel* channel
}
ncclResult_t ncclProxySaveA2a(struct ncclProxyArgs* args, struct ncclInfo* info) {
const int peersPerChan = (info->nChannels >= info->comm->nRanks ? 1 : DIVUP(info->comm->nRanks, info->nChannels));
const int peersPerChan = DIVUP(info->comm->nRanks, info->nChannels);
const int chunkSize = info->comm->buffSizes[info->protocol]/NCCL_STEPS*info->chunkSteps;
const int loopSize = (info->nChannels >= info->comm->nRanks ? (info->nChannels/info->comm->nRanks) : 1)*info->nchunksPerLoop*chunkSize;
for (int p=0; p<peersPerChan; p++) {
if ((peersPerChan == 1 && args->channel->id >= (info->nChannels/info->comm->nRanks)*info->comm->nRanks) ||
(peersPerChan > 1 && args->channel->id*peersPerChan+p >= info->comm->nRanks))
@@ -182,6 +184,22 @@ ncclResult_t ncclProxySaveA2a(struct ncclProxyArgs* args, struct ncclInfo* info)
if (info->coll == ncclCollAllToAll || (info->coll == ncclCollGather && info->comm->rank == info->root) ||
(info->coll == ncclCollScatter && peerRecv == info->root))
NCCLCHECK(SaveProxy<proxyRecv>(peerRecv, args));
if (info->coll == ncclCollAllToAllv) {
info->nBytes = info->sendcounts[peerSend]*info->count;
int nLoops = (int)(DIVUP(info->nBytes, loopSize));
args->nsteps = info->nstepsPerLoop*nLoops*info->chunkSteps;
TRACE(NCCL_NET,"peerSend %d opCount %lx slicesteps %d spl %d cpl %d ces %d nbytes %zi -> protocol %d nchannels %d nthreads %d, nloops %d nsteps %d comm %p",
peerSend, args->opCount, args->sliceSteps, info->nstepsPerLoop, info->nchunksPerLoop, chunkSize, info->nBytes, info->protocol, info->nChannels, info->nThreads,
nLoops, args->nsteps, info->comm);
NCCLCHECK(SaveProxy<proxySend>(peerSend, args));
info->nBytes = info->recvcounts[peerRecv]*info->count;
nLoops = (int)(DIVUP(info->nBytes, loopSize));
args->nsteps = info->nstepsPerLoop*nLoops*info->chunkSteps;
TRACE(NCCL_NET,"peerRecv %d opCount %lx slicesteps %d spl %d cpl %d ces %d nbytes %zi -> protocol %d nchannels %d nthreads %d, nloops %d nsteps %d comm %p",
peerRecv, args->opCount, args->sliceSteps, info->nstepsPerLoop, info->nchunksPerLoop, chunkSize, info->nBytes, info->protocol, info->nChannels, info->nThreads,
nLoops, args->nsteps, info->comm);
NCCLCHECK(SaveProxy<proxyRecv>(peerRecv, args));
}
}
return ncclSuccess;
}