AI/rocm-systems
2
0
Fork 0
程式碼 問題 合併請求 Actions 套件 專案 版本發布 Wiki 動態

Merge remote-tracking branch 'nccl/master' into develop

瀏覽代碼
此提交包含在：
Wenkai Du
2022-10-20 15:40:03 +00:00
父節點 bc8ef779df 99c28f2e75
當前提交 4f0e223db4
共有 51 個檔案被更改，包括 2139 行新增 和 1081 行删除
下載 Patch 檔 下載差異檔 展開所有檔案 折疊所有檔案
src/graph/paths.cc
+77 -26
查看文件
@@ -45,7 +45,7 @@ static ncclResult_t ncclTopoSetPaths(struct ncclTopoNode* baseNode, struct ncclT
struct ncclTopoLinkList* basePath;
NCCLCHECK(getPath(system, baseNode, baseNode->type, baseNode->id, &basePath));
basePath->count = 0;
basePath->width = LOC_WIDTH;
basePath->bw = LOC_BW;
basePath->type = PATH_LOC;
while (nodeList.count) {
@@ -62,8 +62,13 @@ static ncclResult_t ncclTopoSetPaths(struct ncclTopoNode* baseNode, struct ncclT
}
struct ncclTopoLinkList* remPath;
NCCLCHECK(getPath(system, remNode, baseNode->type, baseNode->id, &remPath));
float width = std::min(path->width, link->width);
if (remPath->width < width) {
float bw = std::min(path->bw, link->bw);
// allow routing through a GPU only as 1 hop
if (node != baseNode && node->type == GPU &&
(ncclParamNvbDisable() || link->type != LINK_NVL || remNode->type != GPU || path->count > 1)) continue;
if ((remPath->bw == 0 || remPath->count > path->count) && remPath->bw < bw) {
// Find reverse link
for (int l=0; l<remNode->nlinks; l++) {
if (remNode->links[l].remNode == node) {
@@ -79,7 +84,7 @@ static ncclResult_t ncclTopoSetPaths(struct ncclTopoNode* baseNode, struct ncclT
// Copy the rest of the path
for (int i=0; i<path->count; i++) remPath->list[i+1] = path->list[i];
remPath->count = path->count + 1;
remPath->width = width;
remPath->bw = bw;
// Start with path type = link type. PATH and LINK types are supposed to match.
// Don't consider LINK_NET as we only care about the NIC->GPU path.
@@ -128,9 +133,9 @@ static void printNodePaths(struct ncclTopoSystem* system, struct ncclTopoNode* n
sprintf(line+offset, "--%s->%s/%lX", topoLinkTypeStr[link->type], topoNodeTypeStr[remNode->type], remNode->id);
offset = strlen(line);
}
INFO(NCCL_GRAPH, "%s (%f)", line, node->paths[t][n].width);
INFO(NCCL_GRAPH, "%s (%f)", line, node->paths[t][n].bw);
#else
sprintf(line+offset, "%s/%lX (%d/%f/%s) ", topoNodeTypeStr[t], system->nodes[t].nodes[n].id, node->paths[t][n].count, node->paths[t][n].width, topoPathTypeStr[node->paths[t][n].type]);
sprintf(line+offset, "%s/%lX (%d/%f/%s) ", topoNodeTypeStr[t], system->nodes[t].nodes[n].id, node->paths[t][n].count, node->paths[t][n].bw, topoPathTypeStr[node->paths[t][n].type]);
offset = strlen(line);
#endif
}
@@ -184,7 +189,7 @@ static ncclResult_t addInterStep(struct ncclTopoSystem* system, int tx, int ix,
srcNode->paths[t2][i2].count = l;
srcNode->paths[t2][i2].type = std::max(srcNode->paths[tx][ix].type, cpuNode->paths[t2][i2].type);
if (tx == GPU) srcNode->paths[t2][i2].type = PATH_PXN;
srcNode->paths[t2][i2].width = std::min(srcNode->paths[tx][ix].width, cpuNode->paths[t2][i2].width);
srcNode->paths[t2][i2].bw = std::min(srcNode->paths[tx][ix].bw, cpuNode->paths[t2][i2].bw);
return ncclSuccess;
}
@@ -419,6 +424,40 @@ ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int64_t busId, int
return ncclSuccess;
}
NCCL_PARAM(NetDisableIntra, "NET_DISABLE_INTRA", 1);
// Check whether going through the network would be faster than going through P2P/SHM.
ncclResult_t ncclTopoCheckNet(struct ncclTopoSystem* system, int64_t id1, int64_t id2, int* net) {
if (ncclParamNetDisableIntra() == 1) {
*net = 0;
return ncclSuccess;
}
*net = 1;
// First check the current GPU-to-GPU speed.
int g1, g2;
if (ncclTopoIdToIndex(system, GPU, id1, &g1) != ncclSuccess ||
ncclTopoIdToIndex(system, GPU, id2, &g2) != ncclSuccess) {
return ncclSuccess;
}
struct ncclTopoNode* gpu1 = system->nodes[GPU].nodes+g1;
struct ncclTopoNode* gpu2 = system->nodes[GPU].nodes+g2;
float speed = gpu1->paths[GPU][g2].bw;
// Now check the speed each GPU can access the network through PXB or better
float netSpeed1 = 0, netSpeed2 = 0;
for (int n=0; n<system->nodes[NET].count; n++) {
struct ncclTopoLinkList* path = gpu1->paths[NET]+n;
if (path->type <= PATH_PXB && path->bw > netSpeed1) netSpeed1 = path->bw;
path = gpu2->paths[NET]+n;
if (path->type <= PATH_PXB && path->bw > netSpeed2) netSpeed2 = path->bw;
}
if (netSpeed1 > speed && netSpeed2 > speed) return ncclSuccess;
*net = 0;
return ncclSuccess;
}
ncclResult_t ncclTopoGetIntermediateRank(struct ncclTopoSystem* system, int rank, int netDev, int* intermediateRank) {
// Get GPU and NET
int n, g;
@@ -496,17 +535,23 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
// Remove everything in case we're re-computing
for (int t=0; t<NCCL_TOPO_NODE_TYPES; t++) ncclTopoRemovePathType(system, t);
// Set direct paths from/to CPUs. We need them in many cases.
// Set direct paths to CPUs. We need them in many cases.
for (int c=0; c<system->nodes[CPU].count; c++) {
NCCLCHECK(ncclTopoSetPaths(system->nodes[CPU].nodes+c, system));
}
// Set direct paths from/to GPUs.
// Set direct paths to GPUs.
for (int g=0; g<system->nodes[GPU].count; g++) {
// Compute paths to GPU g
NCCLCHECK(ncclTopoSetPaths(system->nodes[GPU].nodes+g, system));
}
// Update path when we don't want to / can't use GPU Direct P2P
// Set direct paths to NICs.
for (int n=0; n<system->nodes[NET].count; n++) {
NCCLCHECK(ncclTopoSetPaths(system->nodes[NET].nodes+n, system));
}
// Update path for GPUs when we don't want to / can't use GPU Direct P2P
for (int g=0; g<system->nodes[GPU].count; g++) {
for (int p=0; p<system->nodes[GPU].count; p++) {
int p2p;
NCCLCHECK(ncclTopoCheckP2p(system, system->nodes[GPU].nodes[p].id, system->nodes[GPU].nodes[g].id, &p2p, NULL, NULL));
@@ -519,31 +564,32 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
}
if (comm == NULL) continue;
// Remove GPUs we can't talk to because of containers.
// Remove GPUs we can't (or don't want to) communicate with through P2P or SHM
struct ncclPeerInfo* dstInfo = comm->peerInfo+system->nodes[GPU].nodes[g].gpu.rank[0];
for (int p=0; p<system->nodes[GPU].count; p++) {
if (p == g) continue;
struct ncclPeerInfo* srcInfo = comm->peerInfo+system->nodes[GPU].nodes[p].gpu.rank[0];
int shm;
NCCLCHECK(ncclTransports[TRANSPORT_SHM]->canConnect(&shm, system, NULL, srcInfo, dstInfo));
int p2p;
NCCLCHECK(ncclTransports[TRANSPORT_P2P]->canConnect(&p2p, system, NULL, srcInfo, dstInfo));
if (shm == 0 && p2p == 0) {
// Mark this peer as inaccessible. We'll trim it later.
system->nodes[GPU].nodes[p].paths[GPU][g].count = 0;
if (p2p == 0) {
int shm;
NCCLCHECK(ncclTransports[TRANSPORT_SHM]->canConnect(&shm, system, NULL, srcInfo, dstInfo));
if (shm == 0) {
// Mark this peer as inaccessible. We'll trim it later.
system->nodes[GPU].nodes[p].paths[GPU][g].count = 0;
}
}
}
}
// Set direct paths from/to NICs.
// Update paths for NICs (no GPU Direct, PXN, ...)
for (int n=0; n<system->nodes[NET].count; n++) {
struct ncclTopoNode* netNode = system->nodes[NET].nodes+n;
NCCLCHECK(ncclTopoSetPaths(netNode, system));
for (int g=0; g<system->nodes[GPU].count; g++) {
// Check whether we can access the NIC through another NVLink-connected GPU (PXN)
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
if (ncclPxnDisable(comm) != 1 && gpu->paths[NET][n].type > PATH_PXB) {
if (ncclPxnDisable(comm) != 1) {
int pxnGpu = -1;
for (int p=0; p<system->nodes[GPU].count; p++) {
@@ -551,7 +597,12 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
// PXN = PCI + NVLink.
struct ncclTopoNode* peerNode = system->nodes[GPU].nodes+p;
if (peerNode->paths[NET][n].type > PATH_PXB || peerNode->paths[GPU][g].type > PATH_NVL) continue;
// Only use PXN for NIC n if remote GPU p ...
if (peerNode->paths[NET][n].type > PATH_PXB || // Is connected to the NIC through PCI
peerNode->paths[GPU][g].type > PATH_NVL || // Is connected to us through NVLink
(peerNode->paths[NET][n].bw <= gpu->paths[NET][n].bw && // Has either higher BW to that NIC
gpu->paths[NET][n].type <= PATH_PXB)) // or avoids going through a CPU
continue;
pxnGpu = p;
@@ -630,15 +681,15 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm*
for (int j = 0; j < system->nodes[NET].count; j ++) {
if (i == j) continue;
if (system->nodes[NET].nodes[i].net.asic == system->nodes[NET].nodes[j].net.asic) {
if (system->nodes[NET].nodes[i].net.width > system->nodes[NET].nodes[j].net.width)
system->nodes[NET].nodes[j].net.width = 0;
if (system->nodes[NET].nodes[i].net.bw > system->nodes[NET].nodes[j].net.bw)
system->nodes[NET].nodes[j].net.bw = 0;
}
}
}
do {
int n;
for (n=0; n<system->nodes[NET].count; n++) {
if (system->nodes[NET].nodes[n].net.width == 0) break;
if (system->nodes[NET].nodes[n].net.bw == 0) break;
}
if (n<system->nodes[NET].count) {
NCCLCHECK(ncclTopoRemoveNode(system, NET, n));
@@ -716,8 +767,8 @@ static ncclResult_t ncclTopoGetNchannels(struct ncclTopoSystem* system, int g /*
// Local rank
path = system->nodes[GPU].nodes[peer].paths[GPU]+g;
if (path->type == PATH_NVL) {
float nvlWidth = ncclTopoXGMISpeed(system->nodes[GPU].nodes[g].gpu.cudaCompCap);
*nChannels = 2*std::max(1, (int)(path->width / nvlWidth));
float nvlBw = ncclTopoXGMISpeed(system->nodes[GPU].nodes[g].gpu.gcn);
*nChannels = 2*std::max(1, (int)(path->bw / nvlBw));
} else {
*nChannels = 2;
}