AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Select sendrecv path based on collective data size (#391)

Browse Source 
* Select sendrecv path based on collective data size

* Add comments on packing and unpacking group field

* Toggling RCCL_P2P_NET_DISABLE in combined calls unit tests
This commit is contained in:
Wenkai Du
2021-06-10 17:51:04 -07:00
committed by GitHub
parent f6f5e16fe6
commit 6dcae8a459
17 changed files with 102 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/init.cc
+3 -7
View File
@@ -377,7 +377,6 @@ static ncclResult_t commFree(ncclComm_t comm) {
free(comm->intraCC);
}
NCCLCHECK(ncclCudaHostFree((void *)comm->abortFlag));
NCCLCHECK(ncclCudaHostFree((void *)comm->p2pNet));
// Poison comm to try and catch a double free
commPoison(comm);
@@ -430,9 +429,6 @@ static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) {
comm->hostDevComm.abortFlag = comm->abortFlag;
STORE(comm->abortFlag, 0);
NCCLCHECK(ncclCudaHostCalloc((uint32_t**)&comm->p2pNet, 1));
comm->hostDevComm.p2pNet = comm->p2pNet;
STORE(comm->p2pNet, 0);
comm->collOpCount = 0;
comm->p2pOpCount = 0x8000;
@@ -466,8 +462,8 @@ static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) {
static_assert(MAXCHANNELS <= sizeof(*comm->connectSend)*8, "comm->connectSend must have enough bits for all channels");
static_assert(MAXCHANNELS <= sizeof(*comm->connectRecv)*8, "comm->connectRecv must have enough bits for all channels");
NCCLCHECK(ncclCalloc(&comm->connectSend, comm->nRanks));
NCCLCHECK(ncclCalloc(&comm->connectRecv, comm->nRanks));
NCCLCHECK(ncclCalloc(&comm->connectSend, comm->nRanks*NCCL_MAX_CONNS));
NCCLCHECK(ncclCalloc(&comm->connectRecv, comm->nRanks*NCCL_MAX_CONNS));
comm->p2pSendCount = comm->p2pRecvCount = 0;
NCCLCHECK(ncclCalloc(&comm->p2pSends, comm->nRanks));
@@ -837,7 +833,7 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
if ((comm->topo->type & RCCL_TOPO_4P2H_ROME) && (comm->topo->type & RCCL_TOPO_GDR_ALL)) {
if (rcclParamP2pNetDisable() == 0) {
STORE(comm->p2pNet, 1);
comm->p2pNet = 1;
INFO(NCCL_INIT, "RCCL enabled same node P2P over network");
}
else