NCCL 2.27.5-1
Improvements for GB200 systems * Optimize the network performance by alternating the direction of the rings and the NIC to GPU assignment across communicators to limit unnecessary sharing. * Fix the detection of C2C links in case GPU Direct RDMA is disabled between a GPU and a NIC. * Fix PXN support on MNNVL systems, where NCCL would try (and fail) to share regular host memory across multiple nodes. * Fix P2C (PXN over C2C), which is now preferred over regular PXN. This support is currently preliminary and is disabled by default; use NCCL_PXN_C2C=1 to enable. Further reduce the overheads of CUDA graph capturing, which increased in NCCL 2.26.2 for large graphs. Optimize the network performance on DGX B200 systems by adjusting the bandwidths provided to the graph search algorithm. Enable fp8 reductions in symmetric kernels on Blackwell with CUDA 12.8. Restore the plugin name handling logic to make it possible to specify a path to the plugin (Issue #1732). Restore the ability to change NCCL_COLLNET_ENABLE during execution (Issue #1741). Add an example tuner plugin with CSV-based overrides. Remove an x86 dependency from the example profiler.
This commit is contained in:
Kamil Iskra
+21
-9
@@ -175,6 +175,13 @@ ncclResult_t ncclGetLocalCpu(struct ncclTopoSystem* system, int gpu, int* retCpu
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static int mergePathType(int type0, int type1){
|
||||
int max = std::max(type0,type1);
|
||||
int min = std::min(type0,type1);
|
||||
if(max == PATH_PHB && min == PATH_C2C) return PATH_P2C;
|
||||
else return max;
|
||||
}
|
||||
|
||||
static ncclResult_t addInterStep(struct ncclTopoSystem* system, int tx, int ix, int t1, int i1, int t2, int i2) {
|
||||
struct ncclTopoNode* cpuNode = system->nodes[tx].nodes+ix;
|
||||
struct ncclTopoNode* srcNode = system->nodes[t1].nodes+i1;
|
||||
@@ -187,7 +194,7 @@ static ncclResult_t addInterStep(struct ncclTopoSystem* system, int tx, int ix,
|
||||
|
||||
// Update path characteristics
|
||||
srcNode->paths[t2][i2].count = l;
|
||||
srcNode->paths[t2][i2].type = std::max(srcNode->paths[tx][ix].type, cpuNode->paths[t2][i2].type);
|
||||
srcNode->paths[t2][i2].type = mergePathType(srcNode->paths[tx][ix].type, cpuNode->paths[t2][i2].type);
|
||||
if (tx == GPU) srcNode->paths[t2][i2].type = PATH_PXN;
|
||||
srcNode->paths[t2][i2].bw = std::min(srcNode->paths[tx][ix].bw, cpuNode->paths[t2][i2].bw);
|
||||
return ncclSuccess;
|
||||
@@ -674,9 +681,9 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
|
||||
int c;
|
||||
NCCLCHECK(ncclGetLocalCpu(system, g, &c));
|
||||
if (c == -1) continue;
|
||||
if (gpuNode->paths[NET][n].type == PATH_PHB && gpuNode->paths[CPU][c].type == PATH_C2C) {
|
||||
gpuNode->paths[NET][n].type = PATH_P2C;
|
||||
netNode->paths[GPU][g].type = PATH_P2C;
|
||||
if (mergePathType(gpuNode->paths[CPU][c].type, netNode->paths[CPU][c].type) == PATH_P2C) {
|
||||
gpuNode->paths[NET][n].type = std::min(PATH_P2C, gpuNode->paths[NET][n].type);
|
||||
netNode->paths[GPU][g].type = std::min(PATH_P2C, netNode->paths[GPU][g].type);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -695,16 +702,15 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
|
||||
// PXN = PCI + NVLink.
|
||||
struct ncclTopoNode* peerNode = system->nodes[GPU].nodes+localGpuIndex;
|
||||
// Only use PXN for NIC n if remote GPU p ...
|
||||
if (/* (1) is either connected to the NIC with PXB*/
|
||||
(peerNode->paths[NET][n].type <= PATH_PXB ||
|
||||
/* or with P2C and PxN over C2C is enabled */
|
||||
(ncclParamPxnC2c() && peerNode->paths[NET][n].type == PATH_P2C)) &&
|
||||
int pxnType = ncclParamPxnC2c() ? PATH_P2C : PATH_PXB;
|
||||
if (/* (1) is connected to the NIC with PxN type*/
|
||||
peerNode->paths[NET][n].type <= pxnType &&
|
||||
/* and (2) is connected to us through NVLink */
|
||||
peerNode->paths[GPU][g].type <= PATH_NVL &&
|
||||
/* and (3) is on the same node as us */
|
||||
NCCL_TOPO_ID_SYSTEM_ID(peerNode->id) == NCCL_TOPO_ID_SYSTEM_ID(gpu->id) &&
|
||||
/* and (4) has either higher bw to that NIC or avoid going through the CPU*/
|
||||
(peerNode->paths[NET][n].bw > gpu->paths[NET][n].bw || gpu->paths[NET][n].type > PATH_PXB))
|
||||
(peerNode->paths[NET][n].bw > gpu->paths[NET][n].bw || gpu->paths[NET][n].type > pxnType))
|
||||
// We can use that GPU as relay to communicate with that NIC.
|
||||
// Only enabling it in the GPU->NIC direction for now to favor
|
||||
// receiving locally and sending remotely (consistent with net.cc)
|
||||
@@ -725,6 +731,12 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-compute NET local gpus to accelerate search
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
|
||||
NCCLCHECK(ncclTopoGetLocalGpu(system, net->id, &net->net.localGpu));
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
|
||||
+87
-33
@@ -437,6 +437,65 @@ ncclResult_t ncclTopoCompareGraphs(struct ncclTopoSystem* system, struct ncclTop
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Add the preferred NICs ordered by GPU first
|
||||
static ncclResult_t ncclTopoPrefNetsGpuFirst(struct ncclTopoSystem* system, int gpu, int nets[NCCL_TOPO_MAX_NODES], int* netCount) {
|
||||
const int nGpus = (gpu == -1) ? system->nodes[GPU].count : 1;
|
||||
int gpuCount = nGpus;
|
||||
int gpuIds[NCCL_TOPO_MAX_NODES] = {gpu};
|
||||
int firstNets[NCCL_TOPO_MAX_NODES];
|
||||
if (gpu == -1)
|
||||
for (int g = 0; g < nGpus; g++) gpuIds[g] = g;
|
||||
|
||||
for (int c = 0; c < MAXCHANNELS; c++) {
|
||||
for (int g = 0; g < nGpus; g++) {
|
||||
if (gpuIds[g] == -1) continue;
|
||||
int localNet;
|
||||
int64_t netId;
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes + gpuIds[g];
|
||||
NCCLCHECK(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &netId, NULL));
|
||||
NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &localNet));
|
||||
// store the first net found for each GPU in case of duplicates
|
||||
if(c == 0) firstNets[g] = localNet;
|
||||
// if the NET has already been returned for channel 0, that GPU is done
|
||||
if (c > 0 && firstNets[g] == localNet) {
|
||||
gpuIds[g] = -1;
|
||||
gpuCount--;
|
||||
continue;
|
||||
}
|
||||
// only add it to the list if it doesn't already exist
|
||||
int found = 0;
|
||||
while (found < (*netCount) && nets[found] != localNet) found++;
|
||||
if (found == (*netCount)) nets[(*netCount)++] = localNet;
|
||||
}
|
||||
if (gpuCount == 0) break;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Add the preferred NICs ordered by channels first
|
||||
static ncclResult_t ncclTopoPrefNetsChannelFirst(struct ncclTopoSystem* system, int gpu, int nets[NCCL_TOPO_MAX_NODES], int* netCount) {
|
||||
for (int g = 0; g < system->nodes[GPU].count; g++) {
|
||||
if (gpu != -1 && gpu != g) continue;
|
||||
int localNetCount = 0, localNets[MAXCHANNELS];
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes + g;
|
||||
for (int c = 0; c < MAXCHANNELS; c++) {
|
||||
int64_t netId;
|
||||
NCCLCHECK(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &netId, NULL));
|
||||
NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, localNets + localNetCount));
|
||||
if (localNetCount > 0 && localNets[localNetCount] == localNets[0]) break;
|
||||
localNetCount++;
|
||||
}
|
||||
// Append NICs to list
|
||||
for (int i = 0; i < localNetCount; i++) {
|
||||
int n = localNets[i];
|
||||
int found = 0;
|
||||
while (found < (*netCount) && nets[found] != n) found++;
|
||||
if (found == (*netCount)) nets[(*netCount)++] = n;
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Build a sorted list of the NETs to try.
|
||||
//
|
||||
// "gpu" can be set to -1 to build a list suitable for all GPUs (search start) or to a given gpu
|
||||
@@ -445,39 +504,25 @@ ncclResult_t ncclTopoCompareGraphs(struct ncclTopoSystem* system, struct ncclTop
|
||||
// The list is built the following way:
|
||||
// 1. Select NETs starting with those close to GPU(s), based on paths[n].type.
|
||||
// 2. add other NETs satisfying typeInter but not already in the list.
|
||||
|
||||
NCCL_PARAM(ScatterEnable, "MNNVL_SCATTER_NETS_ENABLE", 1);
|
||||
ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int typeInter, int gpu, int nets[NCCL_TOPO_MAX_NODES], int* netCountRet) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
int netCount = 0;
|
||||
int localNetCount;
|
||||
int localNets[MAXCHANNELS];
|
||||
|
||||
// First add the preferred NICs
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
if (gpu != -1 && gpu != g) continue;
|
||||
localNetCount = 0;
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
for (int c = 0; c<MAXCHANNELS; c++) {
|
||||
int64_t netId;
|
||||
NCCLCHECK(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &netId, NULL));
|
||||
NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, localNets+localNetCount));
|
||||
if (localNetCount > 0 && localNets[localNetCount] == localNets[0]) break;
|
||||
localNetCount++;
|
||||
}
|
||||
// Append NICs to list
|
||||
for (int i=0; i<localNetCount; i++) {
|
||||
int n = localNets[i];
|
||||
int found = 0;
|
||||
while (found<netCount && nets[found] != n) found++;
|
||||
if (found == netCount) nets[netCount++] = n;
|
||||
}
|
||||
// First add the preferred NETs.
|
||||
if (system->nHosts > 1 && ncclParamScatterEnable()) {
|
||||
// For MNNVL systems, we sort the devices by GPU first, then by channel
|
||||
NCCLCHECK(ncclTopoPrefNetsGpuFirst(system, gpu, nets, &netCount));
|
||||
} else {
|
||||
// For other systems, we sort the devices by channel first, then by GPU
|
||||
NCCLCHECK(ncclTopoPrefNetsChannelFirst(system, gpu, nets, &netCount));
|
||||
}
|
||||
|
||||
// Then add others satisfying typeInter
|
||||
for (int t=0; t <= typeInter; t++) {
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
for (int g = 0; g < system->nodes[GPU].count; g++) {
|
||||
if (gpu != -1 && gpu != g) continue;
|
||||
localNetCount = 0;
|
||||
int localNetCount = 0, localNets[MAXCHANNELS];
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
struct ncclTopoLinkList* paths = gpu->paths[NET];
|
||||
for (int n=0; n<system->nodes[NET].count && n<MAXCHANNELS; n++) {
|
||||
@@ -625,8 +670,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
|
||||
if (graph->pattern == NCCL_TOPO_PATTERN_NVLS || graph->pattern == NCCL_TOPO_PATTERN_COLLNET_DIRECT) {
|
||||
// NVLS search only tries to find NIC:GPU combinations to compute the heads.
|
||||
if (graph->nChannels < netCount) {
|
||||
int gpu;
|
||||
NCCLCHECK(ncclTopoGetLocalGpu(system, net->id, &gpu));
|
||||
int gpu = net->net.localGpu;
|
||||
if (gpu != -1) {
|
||||
int duplicate = 0;
|
||||
// check whether there is duplicate head when one GPU connects with multiple NICs
|
||||
@@ -643,13 +687,12 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (graph->nChannels > 0) {
|
||||
if (graph->nChannels > 0 && graph->sameChannels == 1) {
|
||||
// Try to replay the last channel
|
||||
int g;
|
||||
NCCLCHECK(ncclTopoReplayGetGpu(system, graph, -1, &g));
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_REPLAY, time, NET, n, g));
|
||||
}
|
||||
if (graph->nChannels == 0 || graph->sameChannels == 0) {
|
||||
} else {
|
||||
if (graph->nChannels == 0 && system->nodes[NVS].count == 0) {
|
||||
// Always try the PCI order first to set a reference, but don't count in the timeout nor let it run for long
|
||||
int t = 1 << 10;
|
||||
@@ -658,11 +701,16 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
|
||||
}
|
||||
|
||||
// Then try the most local GPUs
|
||||
int localGpu = net->net.localGpu;
|
||||
if (localGpu != -1) {
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, localGpu));
|
||||
}
|
||||
int localGpus[NCCL_TOPO_MAX_NODES], localGpuCount, pathType;
|
||||
NCCLCHECK(ncclTopoGetLocal(system, NET, n, GPU, localGpus, &localGpuCount, &pathType));
|
||||
// if no GPUs are connected, skip this net
|
||||
if (pathType == PATH_DIS) continue;
|
||||
for (int g = 0; g < localGpuCount; ++g) {
|
||||
if (localGpus[g] == localGpu) continue; // We already tried this one
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, localGpus[g]));
|
||||
}
|
||||
}
|
||||
@@ -749,8 +797,8 @@ struct kvDict kvDictLinkType[] = {
|
||||
{ "NVB", PATH_NVB },
|
||||
{ "PIX", PATH_PIX },
|
||||
{ "PXB", PATH_PXB },
|
||||
{ "PXN", PATH_PXN },
|
||||
{ "P2C", PATH_P2C },
|
||||
{ "PXN", PATH_PXN },
|
||||
{ "PHB", PATH_PHB },
|
||||
{ "SYS", PATH_SYS },
|
||||
{ NULL, 0 }
|
||||
@@ -798,8 +846,10 @@ ncclResult_t ncclTopoGetGraphFromXmlSub(struct ncclXmlNode *xmlGraph, struct ncc
|
||||
NCCLCHECK(xmlGetAttrInt(xmlGraph, "nchannels", &graph->nChannels));
|
||||
NCCLCHECK(xmlGetAttrFloat(xmlGraph, "speedintra", &graph->bwIntra));
|
||||
NCCLCHECK(xmlGetAttrFloat(xmlGraph, "speedinter", &graph->bwInter));
|
||||
if (xmlGetAttrFloat(xmlGraph, "latencyinter", &graph->latencyInter) != ncclSuccess) graph->latencyInter = 0.0;
|
||||
const char* str;
|
||||
NCCLCHECK(xmlGetAttr(xmlGraph, "latencyinter", &str));
|
||||
if (!str) INFO(NCCL_GRAPH, "latencyinter not found in graph, using 0.0");
|
||||
graph->latencyInter = str ? strtof(str, NULL) : 0.0;
|
||||
NCCLCHECK(xmlGetAttr(xmlGraph, "typeintra", &str));
|
||||
NCCLCHECK(kvConvertToInt(str, &graph->typeIntra, kvDictLinkType));
|
||||
NCCLCHECK(xmlGetAttr(xmlGraph, "typeinter", &str));
|
||||
@@ -910,7 +960,7 @@ float sm90SpeedArrayInter[] = { 48.0, 45.0, 42.0, 40.0, 30.0, 24.0, 22.0, 20.0,
|
||||
#define NSPEEDSINTER_SM90 (sizeof(sm90SpeedArrayInter)/sizeof(float))
|
||||
|
||||
float sm100SpeedArrayIntra[] = { 90.0, 80.0, 70.0, 60.0, 50.0, 40.0, 30.0, 24.0, 20.0, 19.0, 18.0 };
|
||||
float sm100SpeedArrayInter[] = { 47.9, 45.0, 42.0, 40.0, 30.0, 24.0, 22.0, 20.0, 17.5, 15.0, 12.0, 6.0, 3.0, 2.4, 1.2, 0.24, 0.12 };
|
||||
float sm100SpeedArrayInter[] = { 48.0, 45.1, 42.0, 40.0, 30.0, 24.0, 22.0, 20.0, 17.5, 15.0, 12.0, 6.0, 3.0, 2.4, 1.2, 0.24, 0.12 };
|
||||
#define NSPEEDSINTRA_SM100 (sizeof(sm100SpeedArrayIntra)/sizeof(float))
|
||||
#define NSPEEDSINTER_SM100 (sizeof(sm100SpeedArrayInter)/sizeof(float))
|
||||
|
||||
@@ -1136,8 +1186,12 @@ ncclResult_t ncclTopoPrintGraph(struct ncclTopoSystem* system, struct ncclTopoGr
|
||||
offset = strlen(line);
|
||||
}
|
||||
for (int i=0; i<ngpus; i++) {
|
||||
sprintf(line+offset, " %s/%d", topoNodeTypeStr[GPU], graph->intra[ngpus*c+i]);
|
||||
int g;
|
||||
ncclTopoRankToIndex(system, graph->intra[ngpus * c + i], &g, true);
|
||||
int64_t topoId = system->nodes[GPU].nodes[g].id;
|
||||
sprintf(line + offset, " %s/%lx-%lx", topoNodeTypeStr[GPU], NCCL_TOPO_ID_SYSTEM_ID(topoId), NCCL_TOPO_ID_LOCAL_ID(topoId));
|
||||
offset = strlen(line);
|
||||
if (graph->id == 3) break; // NVLS graphs only use the first GPU
|
||||
}
|
||||
if (system->nodes[NET].count > 0) {
|
||||
sprintf(line+offset, " %s/%lx-%lx", topoNodeTypeStr[NET], NCCL_TOPO_ID_SYSTEM_ID(graph->inter[2*c+1]), NCCL_TOPO_ID_LOCAL_ID(graph->inter[2*c+1]));
|
||||
|
||||
+11
-4
@@ -21,7 +21,7 @@
|
||||
|
||||
const char* topoNodeTypeStr[] = { "GPU", "PCI", "NVS", "CPU", "NIC", "NET" };
|
||||
const char* topoLinkTypeStr[] = { "LOC", "NVL", "", "C2C", "PCI", "", "", "", "", "SYS", "NET" };
|
||||
const char* topoPathTypeStr[] = { "LOC", "NVL", "NVB", "C2C", "PIX", "PXB", "PXN", "P2C", "PHB", "SYS", "NET", "DIS" };
|
||||
const char* topoPathTypeStr[] = { "LOC", "NVL", "NVB", "C2C", "PIX", "PXB", "P2C", "PXN", "PHB", "SYS", "NET", "DIS" };
|
||||
|
||||
/******************************************************************/
|
||||
/******************* Graph Creation Functions *********************/
|
||||
@@ -677,7 +677,14 @@ ncclResult_t ncclTopoGetSystemFromXml(struct ncclXml* xml, struct ncclTopoSystem
|
||||
struct ncclXmlNode* node = topNode->subs[s];
|
||||
if (strcmp(node->name, "cpu") == 0) NCCLCHECK(ncclTopoAddCpu(node, *topoSystem));
|
||||
}
|
||||
for (int systemId=0; systemId<system->nHosts; systemId++) if (system->hostHashes[systemId] == localHostHash) system->systemId = systemId;
|
||||
|
||||
int systemId = 0;
|
||||
while (systemId < system->nHosts && system->hostHashes[systemId] != localHostHash) systemId++;
|
||||
system->systemId = systemId;
|
||||
if(systemId == system->nHosts){
|
||||
WARN("localHostHash = 0x%lx not found in the list of system hostHashes",localHostHash);
|
||||
return ncclInvalidArgument;
|
||||
}
|
||||
|
||||
NCCLCHECK(ncclTopoAddNvLinks(topNode, *topoSystem, NULL, 0));
|
||||
NCCLCHECK(ncclTopoAddC2c(topNode, *topoSystem, NULL, 0));
|
||||
@@ -1143,8 +1150,8 @@ struct kvDict nicPathKvList[] = {
|
||||
{ "PORT", PATH_PORT },
|
||||
{ "PIX", PATH_PIX },
|
||||
{ "PXB", PATH_PXB },
|
||||
{ "PXN", PATH_PXN },
|
||||
{ "P2C", PATH_P2C },
|
||||
{ "PXN", PATH_PXN },
|
||||
{ "PHB", PATH_PHB },
|
||||
{ "SYS", PATH_SYS },
|
||||
{ NULL, 0 }
|
||||
@@ -1421,7 +1428,7 @@ ncclResult_t ncclTopoGetSystem(struct ncclComm* comm, struct ncclTopoSystem** sy
|
||||
}
|
||||
|
||||
// Only update our topo tracking structure if we aren't dumping (separate steps)
|
||||
if (dumpXmlFile == NULL) NCCLCHECKGOTO(ncclTopoGetSystemFromXml(xml, system, comm->peerInfo[comm->rank].hostHash), ret, fail);
|
||||
if (dumpXmlFile == NULL) NCCLCHECKGOTO(ncclTopoGetSystemFromXml(xml, system, getHostHash()), ret, fail);
|
||||
|
||||
exit:
|
||||
if (!comm->MNNVL && localRanks) free(localRanks);
|
||||
|
||||
+6
-5
@@ -18,7 +18,7 @@
|
||||
#define SM80_NVLINK_BW 20.0
|
||||
#define SM90_NVLINK_BW 20.6
|
||||
#define SM86_NVLINK_BW 12.0
|
||||
#define SM100_NVLINK_BW 40.0
|
||||
#define SM100_NVLINK_BW 40.1
|
||||
#define PCI_BW 12.0 // PCI Gen3 x16
|
||||
#define AMD_BW 16.0
|
||||
#define BDW_QPI_BW 6.0
|
||||
@@ -76,11 +76,11 @@ extern const char* topoLinkTypeStr[];
|
||||
// Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)
|
||||
#define PATH_PXB 5
|
||||
|
||||
// Connection between a GPU and a NIC using an intermediate GPU. Used to enable rail-local, aggregated network send/recv operations.
|
||||
#define PATH_PXN 6
|
||||
|
||||
// Connection between a GPU and a NIC using the C2C connection to the CPU and the PCIe connection to the NIC
|
||||
#define PATH_P2C 7
|
||||
#define PATH_P2C 6
|
||||
|
||||
// Connection between a GPU and a NIC using an intermediate GPU. Used to enable rail-local, aggregated network send/recv operations.
|
||||
#define PATH_PXN 7
|
||||
|
||||
// Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)
|
||||
#define PATH_PHB 8
|
||||
@@ -143,6 +143,7 @@ struct ncclTopoNode {
|
||||
int gdrSupport;
|
||||
int collSupport;
|
||||
int maxChannels;
|
||||
int localGpu;
|
||||
}net;
|
||||
struct {
|
||||
int arch;
|
||||
|
||||
+9
-2
@@ -455,9 +455,16 @@ ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCom
|
||||
for (int c=0; c<NCCL_NUM_FUNCTIONS; c++) for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
||||
int pEnable = protoEnable[c*NCCL_NUM_PROTOCOLS+p];
|
||||
if (pEnable == 2 && p == NCCL_PROTO_LL128) {
|
||||
// Enable LL128 by default only on Volta/Ampere/Hopper/Blackwell+NVLink. Other cases are not tested and may cause silent data corruption.
|
||||
pEnable = 1;
|
||||
pEnable &= (graphs[a]->typeInter <= PATH_PXB || (minCompCap >= 90 && graphs[a]->typeInter <= (ncclParamLl128C2c() ? PATH_P2C : PATH_PXN)));
|
||||
if (ncclParamLl128C2c() && minCompCap >= 90) {
|
||||
// Enable LL128 by default only on Hopper/Blackwell for all connections up to P2C and PXN.
|
||||
pEnable &= (graphs[a]->typeInter <= PATH_PXN);
|
||||
} else {
|
||||
// Enable LL128 only up to PXB. Don't enable LL128 over PxN because PxN can encapsulate PxB or P2C links.
|
||||
pEnable &= (graphs[a]->typeInter <= PATH_PXB);
|
||||
if (!ncclParamLl128C2c() && minCompCap >= 90)
|
||||
INFO(NCCL_GRAPH, "Disabling LL128 over all PxN connections (PXB and C2C). This ensures that no C2C link will be used by LL128.");
|
||||
}
|
||||
pEnable &= (graphs[a]->typeIntra <= PATH_NVB);
|
||||
pEnable &= (minCompCap == maxCompCap);
|
||||
pEnable &= !(minCompCap < 70 || (minCompCap == 90 && CUDART_VERSION == 11080 && c == ncclFuncAllReduce && a == NCCL_ALGO_RING && comm->nRanks == 2));
|
||||
|
||||
Reference in New Issue
Block a user