diff --git a/src/enqueue.cc b/src/enqueue.cc index ca0fcf4c7a..d5416bb17e 100644 --- a/src/enqueue.cc +++ b/src/enqueue.cc @@ -2126,6 +2126,9 @@ static ncclResult_t topoGetAlgoInfo( INFO(NCCL_INIT, "post-adjustment based on threadThreshold:%i nBytes:%lu nc:%i", threadThreshold, nBytes, nc); rcclOverrideChannels(comm, info->func, nBytes, nc); } + + rcclRestrictMaxChannels(comm, nc); + #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) #else if (info->algorithm != NCCL_ALGO_NVLS && info->algorithm != NCCL_ALGO_NVLS_TREE && diff --git a/src/include/rccl_common.h b/src/include/rccl_common.h index dd7bd1b55f..9df5a2c9e2 100644 --- a/src/include/rccl_common.h +++ b/src/include/rccl_common.h @@ -98,6 +98,7 @@ inline size_t rcclGetSizePerRank(ncclFunc_t const& func, size_t const& nBytes, i return (func == ncclFuncReduceScatter || func == ncclFuncAllGather || func == ncclFuncBroadcast || func == ncclFuncReduce) ? nBytes / nRanks : nBytes; } ncclResult_t rcclOverrideChannels(struct ncclComm* comm, ncclFunc_t coll, size_t nBytes, int& nc); +void rcclRestrictMaxChannels(struct ncclComm* comm, int& nc); ncclResult_t rcclGetAlgoProtoIndex(const char *envStr, const char* algoProtoString[], int nEntries, int& result); ncclResult_t rcclOverrideProtocol(const char* ncclProtoStr[], float table[][NCCL_NUM_PROTOCOLS], struct ncclTaskColl* info); ncclResult_t rcclOverrideAlgorithm(const char* ncclAlgoStr[], float table[][NCCL_NUM_PROTOCOLS], struct ncclTaskColl* info); diff --git a/src/rccl_wrap.cc b/src/rccl_wrap.cc index 78d7adeb19..ea0b83b0d4 100644 --- a/src/rccl_wrap.cc +++ b/src/rccl_wrap.cc @@ -43,6 +43,14 @@ RCCL_PARAM(WarpSpeedEnable, "WARP_SPEED_ENABLE", 0); #endif #define RCCL_WARP_SPEED_MIN_BYTES (1ULL << 26) // 64 MB +RCCL_PARAM(ReducedCuEnable, "REDUCED_CU_ENABLE", 0); + +void rcclRestrictMaxChannels(struct ncclComm* comm, int& nc ) { + if (comm->nNodes > 1 && IsArchMatch(comm->topo->nodes[GPU].nodes[0].gpu.gcn, "gfx950") && rcclParamReducedCuEnable() == 1) { + nc = comm->nChannels = std::min(nc, 48); + } +} + void rcclUpdateCollectiveProtocol(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info) { // Honor user input for protocol choice static int userProtocolInput = -2;