AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'nccl/master' into develop

Browse Source
This commit is contained in:
BertanDogancay
2025-10-06 15:03:19 -05:00
committed by Corey Derochie
parent feee02ca61 593de54e52
commit 3f94267f21
51 changed files with 3682 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/init.cc
+17 -5
View File
@@ -2154,7 +2154,7 @@ static ncclResult_t envConfigOverride(ncclComm_t comm) {
int minCTAsEnv;
int maxCTAsEnv;
int splitShareEnv;
int collnetEnableEnv;
const char* collnetEnableEnv;
int ctaPolicyEnv;
int shrinkShareEnv;
int nvlsCTAsEnv;
@@ -2208,9 +2208,15 @@ static ncclResult_t envConfigOverride(ncclComm_t comm) {
comm->config.shrinkShare = shrinkShareEnv;
}
collnetEnableEnv = ncclParamCollnetEnable();
if (collnetEnableEnv != NCCL_CONFIG_UNDEF_INT) {
comm->config.collnetEnable = collnetEnableEnv;
// NCCL_COLLNET_ENABLE needs to be reloaded each time for comm init
// since users might change the env on the fly to enable/disable collnet
collnetEnableEnv = ncclGetEnv("NCCL_COLLNET_ENABLE");
if (collnetEnableEnv != NULL) {
int collnetEnableInt = (int)strtol(collnetEnableEnv, NULL, 0);
if (collnetEnableInt != NCCL_CONFIG_UNDEF_INT) {
comm->config.collnetEnable = collnetEnableInt;
INFO(NCCL_ENV, "NCCL_COLLNET_ENABLE set by environment to %d.", collnetEnableInt);
}
}
ctaPolicyEnv = ncclParamCtaPolicy();
@@ -2867,6 +2873,7 @@ ncclResult_t ncclCommDestroy_impl(ncclComm_t comm) {
NVTX3_PAYLOAD(comm->commHash, nranks, rank, cudaDev));
TRACE(NCCL_INIT, "comm %p rank %d nRanks %d cudaDev %d busId %lx", comm, rank, nranks, cudaDev, comm->busId);
NCCLCHECK(ncclGroupStartInternal());
// Try and prevent a double free of the comm struct (user error)
if (comm->rank == -1 || comm->nRanks == -1 || comm->cudaDev == -1 || comm->busId == -1) {
WARN("comm %p has already been destroyed", comm);
@@ -2881,6 +2888,8 @@ ncclResult_t ncclCommDestroy_impl(ncclComm_t comm) {
NCCLCHECKGOTO(ncclAsyncLaunch((struct ncclAsyncJob*)job, commReclaim, NULL, free, comm), res, fail);
exit:
ncclGroupErrCheck(res);
NCCLCHECK(ncclGroupEndInternal());
return res;
fail:
goto exit;
@@ -2905,6 +2914,7 @@ ncclResult_t ncclCommAbort_impl(ncclComm_t comm) {
if (comm == NULL) {
return ncclSuccess;
}
NCCLCHECK(ncclGroupStartInternal());
// Ask anything that might still be running on the device to quit
NCCLCHECK(setCommAbortFlags(comm,1));
comm->destroyFlag = 1;
@@ -2927,7 +2937,9 @@ ncclResult_t ncclCommAbort_impl(ncclComm_t comm) {
NCCLCHECKGOTO(ncclAsyncLaunch((struct ncclAsyncJob*)job, commReclaim, NULL, free, comm), res, fail);
exit:
return ncclSuccess;
ncclGroupErrCheck(res);
NCCLCHECK(ncclGroupEndInternal());
return res;
fail:
goto exit;
}