AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable local sendrecv over network if GDR is available on all GPUs (#324)

Browse Source
This commit is contained in:
Wenkai Du
2021-03-05 19:59:41 -08:00
committed by GitHub
parent f4a9b9acba
commit c018edf0f2
14 changed files with 163 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/channel.cc
+8
View File
@@ -23,6 +23,8 @@ ncclResult_t initChannel(struct ncclComm* comm, int channelid) {
for (size_t i=0; i<comm->nRanks+1; ++i) {
channel->peers[i].send.comm = comm;
channel->peers[i].recv.comm = comm;
channel->peers[i].p2pSend.comm = comm;
channel->peers[i].p2pRecv.comm = comm;
}
// Per-channel operation list.
@@ -46,10 +48,16 @@ ncclResult_t freeChannel(struct ncclChannel* channel, int nRanks) {
for (int r=0; r<nRanks+1; r++) {
struct ncclPeer* peer = channel->peers+r;
if (peer->send.transportResources) NCCLCHECK(peer->send.transportComm->free(peer->send.transportResources));
if (peer->send.transportResources == peer->p2pSend.transportResources) peer->p2pSend.transportResources = NULL;
peer->send.transportResources = NULL;
if (peer->p2pSend.transportResources) NCCLCHECK(peer->p2pSend.transportComm->free(peer->p2pSend.transportResources));
}
for (int r=0; r<nRanks+1; r++) {
struct ncclPeer* peer = channel->peers+r;
if (peer->recv.transportResources) NCCLCHECK(peer->recv.transportComm->free(peer->recv.transportResources));
if (peer->recv.transportResources == peer->p2pRecv.transportResources) peer->p2pRecv.transportResources = NULL;
peer->recv.transportResources = NULL;
if (peer->p2pRecv.transportResources) NCCLCHECK(peer->p2pRecv.transportComm->free(peer->p2pRecv.transportResources));
}
// Free the peer structures.