AI/rocm-systems
2
0
Förgrening 0
Kod Ärenden Pull-förfrågningar Actions Packages Projekt Släpp Wiki Aktiviteter

2.16.5-1

Bläddra i källkod 
Add support for 400Gbit NDR network adapters (CX7)
Handle EINTR in socket poll() function
Add NCCL_PROGRESS_APPENDOP_FREQ to control op append overhead
Resource cleanup fixes
Fix double free in case of init failure
Fix crash in ncclCommAbort
Revert AMD speed commit
This commit is contained in:
Sylvain Jeaugey
2023-02-02 12:52:47 -08:00
förälder 93840e7476
incheckning f3d5166783
12 ändrade filer med 71 tillägg och 58 borttagningar
Download Patch File Download Diff File Expand all files Collapse all files
src/init.cc
+6 -23
Visa fil
@@ -498,7 +498,7 @@ static ncclResult_t computeBuffSizes(struct ncclComm* comm) {
NCCL_PARAM(GraphDumpFileRank, "GRAPH_DUMP_FILE_RANK", 0);
NCCL_PARAM(CollNetNodeThreshold, "COLLNET_NODE_THRESHOLD", 2);
NCCL_PARAM(NvbPreconnect, "NVB_PRECONNECT", 1);
NCCL_PARAM(AllocP2pNetLLBuffers, "NCCL_ALLOC_P2P_NET_LL_BUFFERS", 0);
NCCL_PARAM(AllocP2pNetLLBuffers, "ALLOC_P2P_NET_LL_BUFFERS", 0);
static ncclResult_t collNetTrySetup(ncclComm_t comm, struct ncclTopoGraph* collNetGraph) {
ncclResult_t ret = ncclSuccess;
@@ -1102,6 +1102,10 @@ static ncclResult_t ncclCommInitRankFunc(struct ncclAsyncJob* job_) {
// update communicator state
comm->initState = ncclSuccess;
// Trace this call for replay tool
TRACE_CALL("ncclCommInitRank(%p, %d, 0x%llx, %d, %d)",
*newcomm, nranks, (unsigned long long)hashUniqueId(commId), myrank, (*newcomm)->cudaDev);
INFO(NCCL_INIT,"comm %p rank %d nranks %d cudaDev %d busId %lx commId 0x%llx - Init COMPLETE", *newcomm, myrank, nranks, (*newcomm)->cudaDev, (*newcomm)->busId, (unsigned long long)hashUniqueId(commId));
exit:
return res;
@@ -1171,7 +1175,6 @@ static ncclResult_t ncclCommInitRankDev(ncclComm_t* newcomm, int nranks, ncclUni
exit:
return ncclGroupErrCheck(res);
fail:
if (job) free(job);
if (comm) {
if (comm->abortFlag) ncclCudaHostFree((void *)comm->abortFlag);
free(comm);
@@ -1396,7 +1399,6 @@ static ncclResult_t commFinalize(ncclComm_t comm, bool userCalled) {
exit:
return ncclGroupErrCheck(ret);
fail:
if (job) free(job);
goto exit;
}
@@ -1443,26 +1445,7 @@ static ncclResult_t commReclaim(ncclComm_t comm) {
NCCLCHECKGOTO(commFinalize(comm, false), ret, fail);
}
if (comm->intraComm0 == NULL) {
/* if init errors happen and comm->intraComm0 == NULL, no proxy connection is built up, and no finalize thread
* have been launched. Main thread can reclaim everything since no NCCL kernel was issued. */
struct ncclCommFinalizeAsyncJob job;
job.comm = comm;
curRank = comm->rank;
/* comm aborts, commDestroySync should not be blocked. */
if ((ret = commDestroySync((struct ncclAsyncJob*) &job)) != ncclSuccess) {
WARN("commReclaim: comm %p (rank = %d) in abort, error %d", comm, curRank, ret);
}
if ((ret = ncclProxyDestroy(comm)) != ncclSuccess) {
WARN("commReclaim: comm %p (rank = %d) destroys proxy resource error %d", comm, curRank, ret);
}
if ((ret = commCleanup(comm)) != ncclSuccess) {
WARN("commReclaim: cleanup comm %p rank %d failed in destroy/abort, error %d", comm, curRank, ret);
}
} else {
if (comm->intraComm0 != NULL) {
int curRankCnt;
int intraRanks = comm->intraRanks;
ncclComm_t intracomm0 = comm->intraComm0;