Merge remote-tracking branch 'remotes/nccl/master' into rccl_2.5.6
Cette révision appartient à :
Wenkai Du
@@ -0,0 +1,268 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "comm.h"
|
||||
#include "graph.h"
|
||||
#include "trees.h"
|
||||
#include "rings.h"
|
||||
|
||||
/******************************************************************/
|
||||
/********************* Internode connection ***********************/
|
||||
/******************************************************************/
|
||||
|
||||
ncclResult_t ncclTopoPreset(struct ncclComm* comm,
|
||||
struct ncclTopoGraph* treeGraph, struct ncclTopoGraph* ringGraph,
|
||||
struct ncclTopoRanks* topoRanks) {
|
||||
int rank = comm->rank;
|
||||
int localRanks = comm->localRanks;
|
||||
int nChannels = comm->nChannels;
|
||||
|
||||
for (int c=0; c<nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
channel->ring.prev = channel->ring.next = -1;
|
||||
channel->treeUp.up = -1;
|
||||
for (int i=0; i<NCCL_MAX_TREE_ARITY; i++) channel->treeUp.down[i] = -1;
|
||||
channel->treeDn.up = -1;
|
||||
for (int i=0; i<NCCL_MAX_TREE_ARITY; i++) channel->treeDn.down[i] = -1;
|
||||
|
||||
int* ringIntra = ringGraph->intra+c*localRanks;
|
||||
int* treeIntra = treeGraph->intra+c*localRanks;
|
||||
|
||||
for (int i=0; i<localRanks; i++) {
|
||||
if (ringIntra[i] == rank) {
|
||||
topoRanks->ringRecv[c] = ringIntra[0];
|
||||
topoRanks->ringSend[c] = ringIntra[localRanks-1];
|
||||
channel->ring.prev = (i == 0) ? -1 : ringIntra[i-1];
|
||||
channel->ring.next = (i == localRanks-1) ? -1 : ringIntra[i+1];
|
||||
}
|
||||
if (treeIntra[i] == rank) {
|
||||
int recvIndex = 0, sendIndex = treeGraph->pattern == NCCL_TOPO_PATTERN_TREE ? 0 : 1;
|
||||
int prev = (i-1+localRanks)%localRanks, next = (i+1)%localRanks;
|
||||
|
||||
// Tree loop always flows in the same direction. Other trees are symmetric, i.e.
|
||||
// up/down go in reverse directions
|
||||
int sym = treeGraph->pattern == NCCL_TOPO_PATTERN_SPLIT_TREE_LOOP ? 0 : 1;
|
||||
|
||||
// Down tree is common
|
||||
topoRanks->treeDnRecv[c] = treeIntra[recvIndex];
|
||||
topoRanks->treeDnSend[c] = treeIntra[sendIndex];
|
||||
channel->treeDn.up = treeIntra[prev];
|
||||
channel->treeDn.down[0] = treeIntra[next];
|
||||
// Up tree depends on the pattern
|
||||
topoRanks->treeUpRecv[c] = sym ? topoRanks->treeDnSend[c] : topoRanks->treeDnRecv[c];
|
||||
topoRanks->treeUpSend[c] = sym ? topoRanks->treeDnRecv[c] : topoRanks->treeDnSend[c];
|
||||
channel->treeUp.down[0] = sym ? channel->treeDn.down[0] : channel->treeDn.up ;
|
||||
channel->treeUp.up = sym ? channel->treeDn.up : channel->treeDn.down[0];
|
||||
}
|
||||
}
|
||||
topoRanks->ringPrev[c] = channel->ring.prev;
|
||||
topoRanks->ringNext[c] = channel->ring.next;
|
||||
}
|
||||
// Duplicate channels rings/trees
|
||||
struct ncclChannel* channel0 = comm->channels;
|
||||
struct ncclChannel* channel1 = channel0+nChannels;
|
||||
memcpy(channel1, channel0, nChannels*sizeof(struct ncclChannel));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t connectRings(struct ncclComm* comm, int* ringRecv, int* ringSend, int* ringPrev, int* ringNext, int* firstRanks) {
|
||||
int nChannels = comm->nChannels;
|
||||
int nNodes = comm->nNodes;
|
||||
for (int c=0; c<nChannels; c++) {
|
||||
int* recv = ringRecv+c*comm->nRanks;
|
||||
int* send = ringSend+c*comm->nRanks;
|
||||
int* prev = ringPrev+c*comm->nRanks;
|
||||
int* next = ringNext+c*comm->nRanks;
|
||||
struct ncclChannel* channel0 = comm->channels+c;
|
||||
struct ncclChannel* channel1 = channel0+nChannels;
|
||||
for (int n=0; n<nNodes; n++) {
|
||||
int recvRank = recv[firstRanks[n]];
|
||||
int prevSendRank = send[firstRanks[(n-1+nNodes)%nNodes]];
|
||||
prev[recvRank] = prevSendRank;
|
||||
if (comm->rank == recvRank) {
|
||||
channel0->ring.prev = prevSendRank;
|
||||
channel1->ring.prev = prevSendRank;
|
||||
}
|
||||
int sendRank = send[firstRanks[n]];
|
||||
int nextRecvRank = recv[firstRanks[(n+1)%nNodes]];
|
||||
next[sendRank] = nextRecvRank;
|
||||
if (comm->rank == sendRank) {
|
||||
channel0->ring.next = nextRecvRank;
|
||||
channel1->ring.next = nextRecvRank;
|
||||
}
|
||||
}
|
||||
TRACE(NCCL_GRAPH, "Ring %d : %d -> %d -> %d", c, channel0->ring.prev, comm->rank, channel0->ring.next);
|
||||
TRACE(NCCL_GRAPH, "Ring %d : %d -> %d -> %d", c+nChannels, channel1->ring.prev, comm->rank, channel1->ring.next);
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t getIndexes(int* ranks, int* indexes, int nNodes, int* firstRanks) {
|
||||
for (int n=0; n<nNodes; n++) indexes[n] = ranks[firstRanks[n]];
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t setTreeUp(struct ncclTree* tree0, struct ncclTree* tree1, int* indexes, int u0, int u1) {
|
||||
if (u0 != -1) tree0->up = indexes[u0];
|
||||
if (u1 != -1) tree1->up = indexes[u1];
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t addRanksDown(int* down, int* indexes, int r0, int r1) {
|
||||
int x = 0;
|
||||
if (down[x] >= 0) x++;
|
||||
if (down[x] >= 0) {
|
||||
WARN("Internal error : tree already has more than one child (%d %d %d)\n", down[0], down[1], down[2]);
|
||||
return ncclInternalError;
|
||||
}
|
||||
if (r0 != -1) down[x++] = indexes[r0];
|
||||
if (r1 != -1) down[x++] = indexes[r1];
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t setTreeDown(struct ncclTree* tree0, struct ncclTree* tree1, int* indexes, int d0_0, int d0_1, int d1_0, int d1_1) {
|
||||
NCCLCHECK(addRanksDown(tree0->down, indexes, d0_0, d0_1));
|
||||
NCCLCHECK(addRanksDown(tree1->down, indexes, d1_0, d1_1));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t openRing(struct ncclTree* tree, int rank, int upRank) {
|
||||
if (tree->down[0] == upRank) tree->down[0] = -1;
|
||||
if (rank == upRank) tree->up = -1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t connectTrees(struct ncclComm* comm, int* treeUpRecv, int* treeUpSend, int* treeDnRecv, int* treeDnSend, int* firstRanks) {
|
||||
const int nChannels = comm->nChannels, nNodes = comm->nNodes, node = comm->node;
|
||||
int* indexesSend, *indexesRecv;
|
||||
NCCLCHECK(ncclCalloc(&indexesSend, nNodes));
|
||||
NCCLCHECK(ncclCalloc(&indexesRecv, nNodes));
|
||||
|
||||
// Compute tree depth. Not an exact value but a good approximation in most
|
||||
// cases
|
||||
int depth = comm->nRanks/nNodes - 1 + log2i(nNodes);
|
||||
|
||||
int u0, d0_0, d0_1, u1, d1_0, d1_1;
|
||||
NCCLCHECK(ncclGetDtree(nNodes, node, &u0, &d0_0, &d0_1, &u1, &d1_0, &d1_1));
|
||||
for (int c=0; c<nChannels; c++) {
|
||||
struct ncclChannel* channel0 = comm->channels+c;
|
||||
struct ncclChannel* channel1 = channel0+nChannels;
|
||||
NCCLCHECK(getIndexes(treeUpSend+c*comm->nRanks, indexesSend, nNodes, firstRanks));
|
||||
NCCLCHECK(getIndexes(treeUpRecv+c*comm->nRanks, indexesRecv, nNodes, firstRanks));
|
||||
NCCLCHECK(openRing(&channel0->treeUp, comm->rank, indexesSend[node]));
|
||||
NCCLCHECK(openRing(&channel1->treeUp, comm->rank, indexesSend[node]));
|
||||
int root = indexesSend[node];
|
||||
if (indexesSend[node] == comm->rank) NCCLCHECK(setTreeUp(&channel0->treeUp, &channel1->treeUp, indexesRecv, u0, u1));
|
||||
if (indexesRecv[node] == comm->rank) NCCLCHECK(setTreeDown(&channel0->treeUp, &channel1->treeUp, indexesSend, d0_0, d0_1, d1_0, d1_1));
|
||||
NCCLCHECK(getIndexes(treeDnSend+c*comm->nRanks, indexesSend, nNodes, firstRanks));
|
||||
NCCLCHECK(getIndexes(treeDnRecv+c*comm->nRanks, indexesRecv, nNodes, firstRanks));
|
||||
NCCLCHECK(openRing(&channel0->treeDn, comm->rank, u0 == -1 ? root : indexesRecv[node]));
|
||||
NCCLCHECK(openRing(&channel1->treeDn, comm->rank, u1 == -1 ? root : indexesRecv[node]));
|
||||
if (indexesSend[node] == comm->rank) NCCLCHECK(setTreeDown(&channel0->treeDn, &channel1->treeDn, indexesRecv, d0_0, d0_1, d1_0, d1_1));
|
||||
if (indexesRecv[node] == comm->rank) NCCLCHECK(setTreeUp(&channel0->treeDn, &channel1->treeDn, indexesSend, u0, u1));
|
||||
TRACE(NCCL_GRAPH, "TreeUp %d : %d -> %d/%d/%d", c, channel0->treeUp.up, channel0->treeUp.down[0], channel0->treeUp.down[1], channel0->treeUp.down[2]);
|
||||
TRACE(NCCL_GRAPH, "TreeUp %d : %d -> %d/%d/%d", c+nChannels, channel1->treeUp.up, channel1->treeUp.down[0], channel1->treeUp.down[1], channel1->treeUp.down[2]);
|
||||
TRACE(NCCL_GRAPH, "TreeDn %d : %d -> %d/%d/%d", c, channel0->treeDn.up, channel0->treeDn.down[0], channel0->treeDn.down[1], channel0->treeDn.down[2]);
|
||||
TRACE(NCCL_GRAPH, "TreeDn %d : %d -> %d/%d/%d", c+nChannels, channel1->treeDn.up, channel1->treeDn.down[0], channel1->treeDn.down[1], channel1->treeDn.down[2]);
|
||||
channel0->treeUp.depth = channel1->treeUp.depth = depth;
|
||||
}
|
||||
free(indexesSend);
|
||||
free(indexesRecv);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Legacy naming
|
||||
NCCL_PARAM(MinNrings, "MIN_NRINGS", -2);
|
||||
NCCL_PARAM(MaxNrings, "MAX_NRINGS", -2);
|
||||
// New naming
|
||||
NCCL_PARAM(MinNchannels, "MIN_NCHANNELS", -2);
|
||||
NCCL_PARAM(MaxNchannels, "MAX_NCHANNELS", -2);
|
||||
|
||||
int ncclMinNchannels() {
|
||||
int minNchannels = 0;
|
||||
if (ncclParamMinNrings() != -2) minNchannels = ncclParamMinNrings();
|
||||
if (ncclParamMinNchannels() != -2) minNchannels = ncclParamMinNchannels();
|
||||
if (minNchannels > MAXCHANNELS) {
|
||||
WARN("User asked for a minimum of %d channels, limiting to %d\n", minNchannels, MAXCHANNELS);
|
||||
minNchannels = MAXCHANNELS;
|
||||
}
|
||||
if (minNchannels < 0) minNchannels = 0;
|
||||
return minNchannels;
|
||||
}
|
||||
int ncclMaxNchannels() {
|
||||
int maxNchannels = MAXCHANNELS;
|
||||
if (ncclParamMaxNrings() != -2) maxNchannels = ncclParamMaxNrings();
|
||||
if (ncclParamMaxNchannels() != -2) maxNchannels = ncclParamMaxNchannels();
|
||||
if (maxNchannels > MAXCHANNELS) maxNchannels = MAXCHANNELS;
|
||||
if (maxNchannels < 1) {
|
||||
WARN("User asked for a maximum of %d channels, setting it to 1\n", maxNchannels);
|
||||
maxNchannels = 1;
|
||||
}
|
||||
return maxNchannels;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoPostset(struct ncclComm* comm, int* firstRanks, struct ncclTopoRanks** allTopoRanks, int* rings) {
|
||||
// Gather data from all ranks
|
||||
int *ringRecv, *ringSend, *ringPrev, *ringNext, *treeUpRecv, *treeUpSend, *treeDnRecv,*treeDnSend;
|
||||
int nranks = comm->nRanks;
|
||||
int nChannels = comm->nChannels;
|
||||
NCCLCHECK(ncclCalloc(&ringRecv, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&ringSend, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&ringPrev, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&ringNext, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&treeUpRecv, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&treeUpSend, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&treeDnRecv, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&treeDnSend, nranks*MAXCHANNELS));
|
||||
for (int i=0; i<nranks; i++) {
|
||||
for (int c=0; c<nChannels;c++) {
|
||||
ringRecv[c*nranks+i] = allTopoRanks[i]->ringRecv[c];
|
||||
ringSend[c*nranks+i] = allTopoRanks[i]->ringSend[c];
|
||||
ringPrev[c*nranks+i] = allTopoRanks[i]->ringPrev[c];
|
||||
ringNext[c*nranks+i] = allTopoRanks[i]->ringNext[c];
|
||||
treeUpRecv[c*nranks+i] = allTopoRanks[i]->treeUpRecv[c];
|
||||
treeUpSend[c*nranks+i] = allTopoRanks[i]->treeUpSend[c];
|
||||
treeDnRecv[c*nranks+i] = allTopoRanks[i]->treeDnRecv[c];
|
||||
treeDnSend[c*nranks+i] = allTopoRanks[i]->treeDnSend[c];
|
||||
}
|
||||
}
|
||||
|
||||
// Connect rings and trees. This should also duplicate the channels.
|
||||
NCCLCHECK(connectRings(comm, ringRecv, ringSend, ringPrev, ringNext, firstRanks));
|
||||
NCCLCHECK(connectTrees(comm, treeUpRecv, treeUpSend, treeDnRecv, treeDnSend, firstRanks));
|
||||
|
||||
// Duplicate ringPrev/ringNext for ncclBuildRing
|
||||
memcpy(ringPrev+nChannels*nranks, ringPrev, nChannels*nranks*sizeof(int));
|
||||
memcpy(ringNext+nChannels*nranks, ringNext, nChannels*nranks*sizeof(int));
|
||||
|
||||
// Duplication should be complete now
|
||||
nChannels = comm->nChannels = std::min(MAXCHANNELS,nChannels*2);
|
||||
|
||||
// Honor NCCL_MIN_NRINGS/NCCL_MAX_NRINGS.
|
||||
// We permit combining max, then min, to only use the first channels, then duplicate them.
|
||||
nChannels = comm->nChannels = std::min((int)ncclMaxNchannels(), nChannels);
|
||||
int c;
|
||||
for (c=nChannels; c<ncclMinNchannels(); c++) {
|
||||
memcpy(ringPrev+c*nranks, ringPrev+(c-nChannels)*nranks, nranks*sizeof(int));
|
||||
memcpy(ringNext+c*nranks, ringNext+(c-nChannels)*nranks, nranks*sizeof(int));
|
||||
memcpy(comm->channels+c, comm->channels+c-nChannels, sizeof(struct ncclChannel));
|
||||
}
|
||||
nChannels = comm->nChannels = c;
|
||||
|
||||
// Create rings array and check all is fine
|
||||
NCCLCHECK(ncclBuildRings(nChannels, rings, comm->rank, comm->nRanks, ringPrev, ringNext));
|
||||
|
||||
free(ringRecv);
|
||||
free(ringSend);
|
||||
free(ringPrev);
|
||||
free(ringNext);
|
||||
free(treeUpRecv);
|
||||
free(treeUpSend);
|
||||
free(treeDnRecv);
|
||||
free(treeDnSend);
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "graph.h"
|
||||
#include "topo.h"
|
||||
#include "comm.h"
|
||||
#include "net.h"
|
||||
|
||||
// Pre-compute GPU->NIC, GPU->GPU and NIC->GPU paths
|
||||
|
||||
struct ncclTopoNodeList {
|
||||
struct ncclTopoNode* list[NCCL_TOPO_MAX_NODES];
|
||||
int count;
|
||||
};
|
||||
|
||||
static ncclResult_t getPath(struct ncclTopoSystem* system, struct ncclTopoNode* node, int t, int64_t id, struct ncclTopoLinkList** path) {
|
||||
for (int i=0; i<system->nodes[t].count; i++) {
|
||||
if (system->nodes[t].nodes[i].id == id) {
|
||||
*path = node->paths[t]+i;
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
WARN("Could not find node of type %d id %lx\n", t, id);
|
||||
return ncclInternalError;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTopoSetPaths(struct ncclTopoNode* baseNode, struct ncclTopoSystem* system) {
|
||||
if (baseNode->paths[baseNode->type] == NULL) {
|
||||
NCCLCHECK(ncclCalloc(baseNode->paths+baseNode->type, system->nodes[baseNode->type].count));
|
||||
}
|
||||
|
||||
// breadth-first search to set all paths to that node in the system
|
||||
struct ncclTopoNodeList nodeList;
|
||||
struct ncclTopoNodeList nextNodeList;
|
||||
nodeList.count = 1; nodeList.list[0] = baseNode;
|
||||
nextNodeList.count = 0;
|
||||
struct ncclTopoLinkList* basePath;
|
||||
NCCLCHECK(getPath(system, baseNode, baseNode->type, baseNode->id, &basePath));
|
||||
basePath->count = 0;
|
||||
basePath->width = LOC_WIDTH;
|
||||
basePath->type = LINK_LOC;
|
||||
|
||||
while (nodeList.count) {
|
||||
nextNodeList.count = 0;
|
||||
for (int n=0; n<nodeList.count; n++) {
|
||||
struct ncclTopoNode* node = nodeList.list[n];
|
||||
struct ncclTopoLinkList* path;
|
||||
NCCLCHECK(getPath(system, node, baseNode->type, baseNode->id, &path));
|
||||
for (int l=0; l<node->nlinks; l++) {
|
||||
struct ncclTopoLink* link = node->links+l;
|
||||
struct ncclTopoNode* remNode = link->remNode;
|
||||
if (remNode->paths[baseNode->type] == NULL) {
|
||||
NCCLCHECK(ncclCalloc(remNode->paths+baseNode->type, system->nodes[baseNode->type].count));
|
||||
}
|
||||
struct ncclTopoLinkList* remPath;
|
||||
NCCLCHECK(getPath(system, remNode, baseNode->type, baseNode->id, &remPath));
|
||||
int width = std::min(path->width, link->width);
|
||||
if (remPath->width < width) {
|
||||
// Find reverse link
|
||||
for (int l=0; l<remNode->nlinks; l++) {
|
||||
if (remNode->links[l].remNode == node) {
|
||||
remPath->list[0] = remNode->links+l;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (remPath->list[0] == NULL) {
|
||||
WARN("Failed to find reverse path from remNode id %d type %d nlinks %d to node id %d type %d",
|
||||
remNode->id, remNode->type, remNode->nlinks, node->id, node->type);
|
||||
return ncclInternalError;
|
||||
}
|
||||
// Copy the rest of the path
|
||||
for (int i=0; i<path->count; i++) remPath->list[i+1] = path->list[i];
|
||||
remPath->count = path->count + 1;
|
||||
remPath->width = width;
|
||||
|
||||
// Consider the path is QPI when going through the CPU
|
||||
// Also don't consider LINK_NET as we only care about the NIC->GPU path.
|
||||
int type = remNode->type == CPU ? LINK_QPI : link->type == LINK_NET ? 0 : link->type;
|
||||
remPath->type = std::max(path->type, type);
|
||||
|
||||
// Add to the list for the next iteration if not already in the list
|
||||
// Disallow GPUs as intermediate steps for now
|
||||
if (remNode->type != GPU) {
|
||||
int i;
|
||||
for (i=0; i<nextNodeList.count; i++) if (nextNodeList.list[i] == remNode) break;
|
||||
if (i == nextNodeList.count) nextNodeList.list[nextNodeList.count++] = remNode;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
memcpy(&nodeList, &nextNodeList, sizeof(nodeList));
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static void printNodePaths(struct ncclTopoSystem* system, struct ncclTopoNode* node) {
|
||||
char line[1024];
|
||||
#ifdef ENABLE_TRACE
|
||||
INFO(NCCL_GRAPH, "Paths from %s/%lX :", topoNodeTypeStr[node->type], node->id);
|
||||
#else
|
||||
sprintf(line, "%s/%lX :", topoNodeTypeStr[node->type], node->id);
|
||||
int offset = strlen(line);
|
||||
#endif
|
||||
for (int t=0; t<NCCL_TOPO_NODE_TYPES; t++) {
|
||||
if (node->paths[t] == NULL) continue;
|
||||
for (int n = 0; n<system->nodes[t].count; n++) {
|
||||
#ifdef ENABLE_TRACE
|
||||
line[0] = 0;
|
||||
int offset = 0;
|
||||
for (int i=0; i<node->paths[t][n].count; i++) {
|
||||
struct ncclTopoLink* link = node->paths[t][n].list[i];
|
||||
struct ncclTopoNode* remNode = link->remNode;
|
||||
sprintf(line+offset, "--%s->%s/%lX", topoLinkTypeStr[link->type], topoNodeTypeStr[remNode->type], remNode->id);
|
||||
offset = strlen(line);
|
||||
}
|
||||
INFO(NCCL_GRAPH, "%s (%d)", line, node->paths[t][n].width);
|
||||
#else
|
||||
sprintf(line+offset, "%s/%lX (%d/%d/%d) ", topoNodeTypeStr[t], system->nodes[t].nodes[n].id, node->paths[t][n].count, node->paths[t][n].width, node->paths[t][n].type);
|
||||
offset = strlen(line);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
#ifndef ENABLE_TRACE
|
||||
INFO(NCCL_GRAPH, "%s", line);
|
||||
#endif
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoPrintPaths(struct ncclTopoSystem* system) {
|
||||
for (int i=0; i<system->nodes[GPU].count; i++) {
|
||||
printNodePaths(system, system->nodes[GPU].nodes+i);
|
||||
}
|
||||
for (int i=0; i<system->nodes[NET].count; i++) {
|
||||
printNodePaths(system, system->nodes[NET].nodes+i);
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t getLocalCpu(struct ncclTopoSystem* system, int gpu, int* retCpu) {
|
||||
// Find the closest CPU to a GPU
|
||||
int minHops = 0;
|
||||
int localCpu = -1;
|
||||
struct ncclTopoLinkList* paths = system->nodes[GPU].nodes[gpu].paths[CPU];
|
||||
for (int c=0; c<system->nodes[CPU].count; c++) {
|
||||
int hops = paths[c].count;
|
||||
if (minHops == 0 || hops < minHops) {
|
||||
localCpu = c;
|
||||
minHops = hops;
|
||||
}
|
||||
}
|
||||
if (localCpu == -1) {
|
||||
WARN("Error : could not find CPU close to GPU %d", gpu);
|
||||
return ncclInternalError;
|
||||
}
|
||||
*retCpu = localCpu;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t addCpuStep(struct ncclTopoSystem* system, int c, int t1, int i1, int t2, int i2) {
|
||||
struct ncclTopoNode* cpuNode = system->nodes[CPU].nodes+c;
|
||||
struct ncclTopoNode* srcNode = system->nodes[t1].nodes+i1;
|
||||
|
||||
int l=0;
|
||||
// Node 1 -> CPU
|
||||
for (int i=0; i<srcNode->paths[CPU][c].count; i++) srcNode->paths[t2][i2].list[l++] = srcNode->paths[CPU][c].list[i];
|
||||
// CPU -> Node 2
|
||||
for (int i=0; i<cpuNode->paths[t2][i2].count; i++) srcNode->paths[t2][i2].list[l++] = cpuNode->paths[t2][i2].list[i];
|
||||
|
||||
// Update path characteristics
|
||||
srcNode->paths[t2][i2].count = l;
|
||||
srcNode->paths[t2][i2].type = LINK_QPI;
|
||||
srcNode->paths[t2][i2].width = std::min(srcNode->paths[CPU][c].width, cpuNode->paths[t2][i2].width);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Remove/free paths for a given type
|
||||
static void ncclTopoRemovePathType(struct ncclTopoSystem* system, int nodeType) {
|
||||
for (int t=0; t<NCCL_TOPO_NODE_TYPES; t++) {
|
||||
for (int n=0; n<system->nodes[t].count; n++) {
|
||||
struct ncclTopoNode* node = system->nodes[t].nodes+n;
|
||||
free(node->paths[nodeType]);
|
||||
node->paths[nodeType] = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclPeerInfo* peerInfos) {
|
||||
// Precompute paths between GPUs/NICs.
|
||||
|
||||
// Remove everything in case we're re-computing
|
||||
for (int t=0; t<NCCL_TOPO_NODE_TYPES; t++) ncclTopoRemovePathType(system, t);
|
||||
|
||||
// Set direct paths from/to CPUs. We need them in many cases.
|
||||
for (int c=0; c<system->nodes[CPU].count; c++) {
|
||||
NCCLCHECK(ncclTopoSetPaths(system->nodes[CPU].nodes+c, system));
|
||||
}
|
||||
|
||||
// Set direct paths from/to GPUs.
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
// Compute paths to GPU g
|
||||
NCCLCHECK(ncclTopoSetPaths(system->nodes[GPU].nodes+g, system));
|
||||
|
||||
if (peerInfos == NULL) continue;
|
||||
// Update paths from GPUs p to GPU g when we can't or don't want to use P2P or even SHM
|
||||
struct ncclPeerInfo* dstInfo = peerInfos+system->nodes[GPU].nodes[g].rank;
|
||||
for (int p=0; p<system->nodes[GPU].count; p++) {
|
||||
if (p == g) continue;
|
||||
struct ncclPeerInfo* srcInfo = peerInfos+system->nodes[GPU].nodes[p].rank;
|
||||
int p2p;
|
||||
NCCLCHECK(ncclTransports[TRANSPORT_P2P].canConnect(&p2p, system, NULL, srcInfo, dstInfo));
|
||||
if (p2p == 0) {
|
||||
int shm;
|
||||
NCCLCHECK(ncclTransports[TRANSPORT_SHM].canConnect(&shm, system, NULL, srcInfo, dstInfo));
|
||||
if (shm == 1) {
|
||||
// We cannot use GPU Direct, so we need all traffic to go through a CPU
|
||||
int cpu;
|
||||
NCCLCHECK(getLocalCpu(system, g, &cpu));
|
||||
NCCLCHECK(addCpuStep(system, cpu, GPU, p, GPU, g));
|
||||
} else {
|
||||
// We cannot communicate with that peer.
|
||||
system->nodes[GPU].nodes[p].paths[GPU][g].count = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set direct paths from/to NICs.
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
struct ncclTopoNode* netNode = system->nodes[NET].nodes+n;
|
||||
NCCLCHECK(ncclTopoSetPaths(netNode, system));
|
||||
|
||||
if (peerInfos == NULL) continue;
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
if ((peerInfos[system->nodes[GPU].nodes[g].rank].gdrSupport & (1 << n)) == 0) {
|
||||
// We cannot use GPU Direct RDMA, so we need all NIC<->GPU paths
|
||||
// to go through a CPU
|
||||
int localCpu;
|
||||
NCCLCHECK(getLocalCpu(system, g, &localCpu));
|
||||
NCCLCHECK(addCpuStep(system, localCpu, NET, n, GPU, g));
|
||||
NCCLCHECK(addCpuStep(system, localCpu, GPU, g, NET, n));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm* comm) {
|
||||
int *domains;
|
||||
int64_t *ids;
|
||||
NCCLCHECK(ncclCalloc(&domains, system->nodes[GPU].count));
|
||||
NCCLCHECK(ncclCalloc(&ids, system->nodes[GPU].count));
|
||||
int myDomain = 0;
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
domains[g] = g;
|
||||
ids[g] = gpu->id;
|
||||
for (int p=0; p<g; p++) {
|
||||
if (gpu->paths[GPU][p].count > 0) {
|
||||
domains[g] = std::min(domains[g], domains[p]);
|
||||
}
|
||||
}
|
||||
if (gpu->rank == comm->rank) myDomain = domains[g];
|
||||
}
|
||||
|
||||
int ngpus = system->nodes[GPU].count;
|
||||
for (int i=0; i<ngpus; i++) {
|
||||
if (domains[i] == myDomain) continue;
|
||||
struct ncclTopoNode* gpu = NULL;
|
||||
int g;
|
||||
for (g=0; g<system->nodes[GPU].count /* This one varies over the loops */; g++) {
|
||||
gpu = system->nodes[GPU].nodes+g;
|
||||
if (gpu->id == ids[i]) break; else gpu=NULL;
|
||||
}
|
||||
if (gpu == NULL) {
|
||||
WARN("Could not find id %lx", ids[i]);
|
||||
free(domains);
|
||||
free(ids);
|
||||
return ncclInternalError;
|
||||
}
|
||||
|
||||
// Remove GPUs I can't access (even indirectly) from my view of the node
|
||||
for (int t=0; t<NCCL_TOPO_NODE_TYPES; t++) {
|
||||
for (int n=0; n<system->nodes[t].count; n++) {
|
||||
struct ncclTopoNode* node = system->nodes[t].nodes+n;
|
||||
if (node == gpu) continue;
|
||||
for (int l=0; l<node->nlinks; l++) {
|
||||
while (l<node->nlinks && node->links[l].remNode == gpu) {
|
||||
if (l<node->nlinks-1)
|
||||
memmove(node->links+l, node->links+l+1, (node->nlinks-l-1)*sizeof(struct ncclTopoLink));
|
||||
node->nlinks--;
|
||||
}
|
||||
if (l<node->nlinks && node->links[l].remNode->type == GPU && node->links[l].remNode >= gpu) {
|
||||
node->links[l].remNode--;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (g != system->nodes[GPU].count-1)
|
||||
memmove(gpu, gpu+1, (system->nodes[GPU].count-g-1)*sizeof(struct ncclTopoNode));
|
||||
system->nodes[GPU].count--;
|
||||
}
|
||||
|
||||
comm->localRanks = system->nodes[GPU].count;
|
||||
if (system->nodes[GPU].count == comm->nRanks) {
|
||||
// Trim network
|
||||
ncclTopoRemovePathType(system, NET);
|
||||
system->nodes[NET].count = 0;
|
||||
}
|
||||
free(domains);
|
||||
free(ids);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t getGpuSpeed(struct ncclTopoNode* node, int* speed) {
|
||||
int nvlSpeed = 0;
|
||||
int nvlPeers = 0;
|
||||
int pciSpeed = 0;
|
||||
for (int l=0; l<node->nlinks; l++) {
|
||||
if (node->links[l].type == LINK_NVL) nvlSpeed += node->links[l].width;
|
||||
if (node->links[l].remNode->type == GPU) nvlPeers++; else nvlPeers = 2;
|
||||
if (node->links[l].type == LINK_PCI) pciSpeed = node->links[l].width;
|
||||
}
|
||||
*speed = std::min(*speed, std::max(nvlSpeed, pciSpeed));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoGetMaxSpeed(struct ncclTopoSystem* system) {
|
||||
// Compute max speed to try to accelerate the search.
|
||||
system->maxSpeed = LOC_WIDTH;
|
||||
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
NCCLCHECK(getGpuSpeed(system->nodes[GPU].nodes+g, &system->maxSpeed));
|
||||
}
|
||||
if (system->nodes[NET].count) {
|
||||
// Try to assign one NIC per GPU
|
||||
int netMaxSpeed = 0;
|
||||
int netMaxSpeedCount = 0;
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
int maxSpeed = 0;
|
||||
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
maxSpeed = std::max(maxSpeed, net->paths[GPU][g].width);
|
||||
}
|
||||
if (maxSpeed > netMaxSpeed) {
|
||||
netMaxSpeed = maxSpeed;
|
||||
netMaxSpeedCount = 1;
|
||||
} else if (maxSpeed == netMaxSpeed) {
|
||||
netMaxSpeedCount++;
|
||||
}
|
||||
}
|
||||
system->maxSpeed = std::min(system->maxSpeed, netMaxSpeedCount*NET_WIDTH);
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
void ncclTopoFree(struct ncclTopoSystem* system) {
|
||||
for (int t=0; t<NCCL_TOPO_NODE_TYPES; t++) ncclTopoRemovePathType(system, t);
|
||||
free(system);
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
|
||||
#define MAXWIDTH 20
|
||||
#define PREFIXLEN 15
|
||||
#define STRLENGTH (PREFIXLEN+5*MAXWIDTH)
|
||||
void dumpLine(int* values, int nranks, const char* prefix) {
|
||||
int prefixlen = strlen(prefix);
|
||||
char line[STRLENGTH+1];
|
||||
line[STRLENGTH] = '\0';
|
||||
memset(line, ' ', STRLENGTH);
|
||||
strncpy(line, prefix, PREFIXLEN);
|
||||
for (int i=0; i<nranks && i<MAXWIDTH; i++) sprintf(line+prefixlen+4*i, " %3d", values[i]);
|
||||
INFO(NCCL_INIT,"%s", line);
|
||||
}
|
||||
|
||||
ncclResult_t ncclBuildRings(int nrings, int* rings, int rank, int nranks, int* prev, int* next) {
|
||||
for (int r=0; r<nrings; r++) {
|
||||
char prefix[30];
|
||||
/*sprintf(prefix, "[%d] Channel %d Prev : ", rank, r);
|
||||
dumpLine(prev+r*nranks, nranks, prefix);
|
||||
sprintf(prefix, "[%d] Channel %d Next : ", rank, r);
|
||||
dumpLine(next+r*nranks, nranks, prefix);*/
|
||||
|
||||
int current = rank;
|
||||
for (int i=0; i<nranks; i++) {
|
||||
rings[r*nranks+i] = current;
|
||||
current = next[r*nranks+current];
|
||||
}
|
||||
sprintf(prefix, "Channel %02d/%02d : ", r, nrings);
|
||||
if (rank == 0) dumpLine(rings+r*nranks, nranks, prefix);
|
||||
if (current != rank) {
|
||||
WARN("Error : ring %d does not loop back to start (%d != %d)", r, current, rank);
|
||||
return ncclInternalError;
|
||||
}
|
||||
// Check that all ranks are there
|
||||
for (int i=0; i<nranks; i++) {
|
||||
int found = 0;
|
||||
for (int j=0; j<nranks; j++) {
|
||||
if (rings[r*nranks+j] == i) {
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found == 0) {
|
||||
WARN("Error : ring %d does not contain rank %d", r, i);
|
||||
return ncclInternalError;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
ncclResult_t ncclBuildRings(int nrings, int* rings, int rank, int nranks, int* prev, int* next);
|
||||
@@ -0,0 +1,594 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "graph.h"
|
||||
#include "topo.h"
|
||||
|
||||
static ncclResult_t ncclTopoFollowPath(struct ncclTopoGraph* graph, struct ncclTopoLinkList* path, struct ncclTopoNode** node, int width, int typeSave) {
|
||||
if (path->count == 0) return ncclSuccess;
|
||||
|
||||
*node = NULL;
|
||||
if (width > 0) {
|
||||
if (path->type > graph->type) return ncclSuccess;
|
||||
graph->type = std::max(graph->type, path->type);
|
||||
graph->nHops += path->count;
|
||||
} else {
|
||||
graph->type = typeSave;
|
||||
graph->nHops -= path->count;
|
||||
}
|
||||
|
||||
for (int i=0; i<path->count; i++) {
|
||||
if (path->list[i]->width < width) {
|
||||
// Can't follow this path, rewind and exit
|
||||
for (int j=0; j<i; j++) path->list[j]->width += width;
|
||||
return ncclSuccess;
|
||||
}
|
||||
path->list[i]->width -= width;
|
||||
}
|
||||
*node = path->list[path->count-1]->remNode;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static int gpuPciWidth(struct ncclTopoNode* gpu) {
|
||||
for (int l=0; l<gpu->nlinks; l++) {
|
||||
struct ncclTopoLink* gpuLink = gpu->links+l;
|
||||
if (gpuLink->type != LINK_PCI) continue;
|
||||
struct ncclTopoNode* pci = gpuLink->remNode;
|
||||
for (int l=0; l<pci->nlinks; l++) {
|
||||
struct ncclTopoLink* pciLink = pci->links+l;
|
||||
if (pciLink->remNode != gpu) continue;
|
||||
return std::min(gpuLink->width, pciLink->width);
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Choose the order in which we try next GPUs. This is critical for the search
|
||||
to quickly converge to the best solution even if it eventually times out. */
|
||||
struct ncclGpuScore {
|
||||
int g; // Retain the index
|
||||
int startIndex; // Least important
|
||||
int intraNhops;
|
||||
int intraWidth;
|
||||
int interNhops;
|
||||
int interPciWidth;
|
||||
int interWidth; // Most important
|
||||
};
|
||||
|
||||
static int cmpScore(const void * g1, const void * g2) {
|
||||
struct ncclGpuScore *s1 = (struct ncclGpuScore*)g1;
|
||||
struct ncclGpuScore *s2 = (struct ncclGpuScore*)g2;
|
||||
int d;
|
||||
if ((d = (s2->interWidth - s1->interWidth))) return d;
|
||||
if ((d = (s2->interPciWidth - s1->interPciWidth))) return d;
|
||||
if ((d = (s1->interNhops - s2->interNhops))) return d;
|
||||
if ((d = (s2->intraWidth - s1->intraWidth))) return d;
|
||||
if ((d = (s1->intraNhops - s2->intraNhops))) return d;
|
||||
return s1->startIndex - s2->startIndex;
|
||||
}
|
||||
|
||||
static int cmpIntraScores(struct ncclGpuScore* scores, int count) {
|
||||
int intraWidth = scores[0].intraWidth;
|
||||
int intraNhops = scores[0].intraNhops;
|
||||
for (int i=1; i<count; i++) {
|
||||
if (scores[i].intraWidth != intraWidth || scores[i].intraNhops != intraNhops) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static ncclResult_t getNetPaths(struct ncclTopoSystem* system, const uint64_t flag, struct ncclTopoLinkList** netPaths) {
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
if (system->nodes[NET].nodes[n].used & flag) {
|
||||
*netPaths=system->nodes[NET].nodes[n].paths[GPU];
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
return ncclInternalError;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoSearchNextGpuSort(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoNode* gpu, int* next, int* countPtr, int sortNet) {
|
||||
const uint64_t flag = 1ULL<<(graph->nChannels);
|
||||
int ngpus = system->nodes[GPU].count;
|
||||
struct ncclTopoLinkList* paths = gpu->paths[GPU];
|
||||
struct ncclTopoLinkList* netPaths = NULL;
|
||||
if (sortNet) NCCLCHECK(getNetPaths(system, flag, &netPaths));
|
||||
|
||||
struct ncclGpuScore scores[NCCL_TOPO_MAX_NODES];
|
||||
memset(scores, 0, ngpus*sizeof(struct ncclGpuScore));
|
||||
int start = gpu-system->nodes[GPU].nodes;
|
||||
int count = 0;
|
||||
for (int i=1; i<ngpus; i++) {
|
||||
int g = (start+i)%ngpus;
|
||||
if (paths[g].count == 0) continue; // There is no path to that GPU
|
||||
if (system->nodes[GPU].nodes[g].used & flag) continue;
|
||||
scores[count].g = g;
|
||||
scores[count].startIndex = i;
|
||||
scores[count].intraNhops = paths[g].count;
|
||||
scores[count].intraWidth = paths[g].width;
|
||||
if (netPaths) {
|
||||
scores[count].interNhops = netPaths[g].count;
|
||||
scores[count].interPciWidth = gpuPciWidth(system->nodes[GPU].nodes+g);
|
||||
scores[count].interWidth = netPaths[g].width;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
|
||||
// Sort GPUs
|
||||
qsort(scores, count, sizeof(struct ncclGpuScore), cmpScore);
|
||||
|
||||
// Check if all have the same intra-node score in which case we go reverse for sortNet = -1
|
||||
if (sortNet == -1 && cmpIntraScores(scores, count) == 0) {
|
||||
for (int i=0; i<count; i++) next[i] = scores[count-1-i].g;
|
||||
} else {
|
||||
for (int i=0; i<count; i++) next[i] = scores[i].g;
|
||||
}
|
||||
*countPtr = count;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoSearchRec(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int maxSpeed, int* time);
|
||||
|
||||
#define NCCL_SEARCH_TIMEOUT (1ULL<<20) // This should get contain all search within a second or so.
|
||||
|
||||
#define FORCED_ORDER_PCI 1
|
||||
#define FORCED_ORDER_REPLAY 2
|
||||
|
||||
ncclResult_t ncclTopoReplayGetGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, int step, int* g) {
|
||||
*g = -1;
|
||||
if (graph->nChannels == 0) return ncclInternalError;
|
||||
int ngpus = system->nodes[GPU].count;
|
||||
int nextRank = graph->intra[(graph->nChannels-1)*ngpus+step+1];
|
||||
for (int i=0; i<ngpus; i++) if (system->nodes[GPU].nodes[i].rank == nextRank) {
|
||||
*g = i;
|
||||
return ncclSuccess;
|
||||
}
|
||||
if (*g == -1) return ncclInternalError;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, struct ncclTopoNode* gpu, int step, int backToNet, int backToFirstRank, int forcedOrder, int maxSpeed, int *time);
|
||||
|
||||
ncclResult_t ncclTopoSearchTryGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, struct ncclTopoLinkList* paths, int step, int backToNet, int backToFirstRank, int forcedOrder, int maxSpeed, int *time, int g, int speed) {
|
||||
int typeSave = graph->type;
|
||||
const uint64_t flag = 1ULL<<(graph->nChannels);
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
if (paths) NCCLCHECK(ncclTopoFollowPath(graph, paths+g, &gpu, speed, typeSave));
|
||||
if (gpu) {
|
||||
gpu->used ^= flag;
|
||||
NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, step, backToNet, backToFirstRank, forcedOrder, maxSpeed, time));
|
||||
gpu->used ^= flag;
|
||||
if (paths) NCCLCHECK(ncclTopoFollowPath(graph, paths+g, &gpu, -speed, typeSave));
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoCompareGraphs(struct ncclTopoGraph* graph, struct ncclTopoGraph* refGraph, int* copy) {
|
||||
// 0. When we are trying to increase speedIntra, do not copy if the solution has less channels
|
||||
// since it would likely impact the rings algorithms too.
|
||||
if (graph->speedIntra > graph->speedInter && graph->nChannels < refGraph->nChannels) return ncclSuccess;
|
||||
|
||||
// 1. Try to get better bandwidth
|
||||
if (graph->nChannels*graph->speedIntra < refGraph->nChannels*refGraph->speedIntra) return ncclSuccess;
|
||||
if (graph->nChannels*graph->speedIntra > refGraph->nChannels*refGraph->speedIntra) {
|
||||
*copy = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
// 2. Give an advantage when all channels are the same
|
||||
if (graph->nChannels > 1 && graph->sameChannels && refGraph->sameChannels == 0) {
|
||||
*copy = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
// 3. Less hops
|
||||
if (graph->nHops < refGraph->nHops) *copy = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, struct ncclTopoNode* gpu, int step, int backToNet, int backToFirstRank, int forcedOrder, int maxSpeed, int *time) {
|
||||
if ((*time) <= 0) return ncclSuccess;
|
||||
(*time)--;
|
||||
|
||||
int ngpus = system->nodes[GPU].count;
|
||||
if (step == ngpus) {
|
||||
// Determine whether we found a better solution or not
|
||||
int copy = 0;
|
||||
int sameChannels = graph->sameChannels;
|
||||
if (graph->nChannels > 0) {
|
||||
int* intra = graph->intra+graph->nChannels*ngpus;
|
||||
for (int g=0; g<ngpus; g++) if (intra[g] != intra[g-ngpus]) graph->sameChannels = 0;
|
||||
}
|
||||
graph->nChannels++;
|
||||
NCCLCHECK(ncclTopoCompareGraphs(graph, saveGraph, ©));
|
||||
if (copy) {
|
||||
memcpy(saveGraph, graph, sizeof(struct ncclTopoGraph));
|
||||
if (graph->nChannels*graph->speedIntra == maxSpeed) *time = -1;
|
||||
}
|
||||
if (graph->nChannels < MAXCHANNELS/2) {
|
||||
NCCLCHECK(ncclTopoSearchRec(system, graph, saveGraph, maxSpeed, time));
|
||||
}
|
||||
graph->nChannels--;
|
||||
graph->sameChannels = sameChannels;
|
||||
return ncclSuccess;
|
||||
}
|
||||
graph->intra[graph->nChannels*ngpus+step] = gpu->rank;
|
||||
if (step == backToNet) {
|
||||
// first get back to NIC
|
||||
if (system->nodes[NET].count) {
|
||||
int maxWidth = 0;
|
||||
struct ncclTopoLinkList* paths = gpu->paths[NET];
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
if (graph->crossNic != 1 && (system->nodes[NET].nodes[n].id != graph->inter[graph->nChannels*2])) continue;
|
||||
maxWidth = std::max(paths[n].width, maxWidth);
|
||||
}
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
if (graph->crossNic != 1 && (system->nodes[NET].nodes[n].id != graph->inter[graph->nChannels*2])) continue;
|
||||
if (paths[n].width == maxWidth) {
|
||||
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
|
||||
int typeSave = graph->type;
|
||||
NCCLCHECK(ncclTopoFollowPath(graph, paths+n, &net, graph->speedInter, typeSave));
|
||||
if (net) {
|
||||
graph->inter[graph->nChannels*2+1] = net->id;
|
||||
NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, step, -1, backToFirstRank, forcedOrder, maxSpeed, time));
|
||||
NCCLCHECK(ncclTopoFollowPath(graph, paths+n, &net, -graph->speedInter, typeSave));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (step < system->nodes[GPU].count-1) {
|
||||
// Go to next GPU
|
||||
struct ncclTopoLinkList* paths = gpu->paths[GPU];
|
||||
int next[NCCL_TOPO_MAX_NODES];
|
||||
int count;
|
||||
if (forcedOrder == FORCED_ORDER_PCI) { // Try the PCI order
|
||||
next[0] = step+1;
|
||||
count = 1;
|
||||
} else if (forcedOrder == FORCED_ORDER_REPLAY) { // Try last channel order
|
||||
NCCLCHECK(ncclTopoReplayGetGpu(system, graph, step, next));
|
||||
count = 1;
|
||||
} else { // Normal search
|
||||
NCCLCHECK(ncclTopoSearchNextGpuSort(system, graph, gpu, next, &count, backToNet == -1 ? 0 : backToNet == step+1 ? 1 : -1 ));
|
||||
}
|
||||
for (int i=0; i<count; i++) {
|
||||
int g = next[i];
|
||||
int nvlink = graph->nvlink;
|
||||
graph->nvlink &= paths[g].type <= LINK_NVL ? 1 : 0;
|
||||
int speed = graph->speedIntra;
|
||||
if (paths[g].type == LINK_QPI) speed = INTEL_P2P_OVERHEAD(speed);
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, paths, step+1, backToNet, backToFirstRank, forcedOrder, maxSpeed, time, g, speed));
|
||||
graph->nvlink = nvlink;
|
||||
}
|
||||
} else if (step == backToFirstRank) {
|
||||
// Find first GPU and loop back to it
|
||||
int g;
|
||||
int rank = graph->intra[graph->nChannels*ngpus];
|
||||
for (g=0; g<ngpus; g++) {
|
||||
if (system->nodes[GPU].nodes[g].rank == rank) break;
|
||||
}
|
||||
if (g == ngpus) {
|
||||
WARN("Could not find GPU with rank %d\n", rank);
|
||||
return ncclInternalError;
|
||||
}
|
||||
struct ncclTopoLinkList* paths = gpu->paths[GPU];
|
||||
struct ncclTopoNode* firstGpu = system->nodes[GPU].nodes+g;
|
||||
int typeSave = graph->type;
|
||||
NCCLCHECK(ncclTopoFollowPath(graph, paths+g, &firstGpu, graph->speedIntra, typeSave));
|
||||
if (firstGpu) {
|
||||
NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, firstGpu, step+1, backToNet, -1, forcedOrder, maxSpeed, time));
|
||||
NCCLCHECK(ncclTopoFollowPath(graph, paths+g, &firstGpu, -graph->speedIntra, typeSave));
|
||||
}
|
||||
} else {
|
||||
// Next path
|
||||
NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, ngpus, -1, -1, forcedOrder, maxSpeed, time));
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int backToNet, int backToFirstRank, int maxSpeed, int* time) {
|
||||
const uint64_t flag = 1ULL<<(graph->nChannels);
|
||||
const int speed = graph->speedInter;
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
|
||||
struct ncclTopoNode* gpu;
|
||||
if (net->used == 0) {
|
||||
graph->inter[graph->nChannels*2] = net->id;
|
||||
for (int i=0; i<system->nodes[NET].count; i++) {
|
||||
if (system->nodes[NET].nodes[i].rank == net->rank) system->nodes[NET].nodes[i].used ^= flag;
|
||||
}
|
||||
struct ncclTopoLinkList* paths = net->paths[GPU];
|
||||
|
||||
// First try the PCI order to set a reference
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, paths, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, maxSpeed, time, 0, speed));
|
||||
// Then try to replay the last channel
|
||||
if (graph->nChannels > 0) {
|
||||
int g;
|
||||
NCCLCHECK(ncclTopoReplayGetGpu(system, graph, -1, &g));
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, paths, 0, backToNet, backToFirstRank, FORCED_ORDER_REPLAY, maxSpeed, time, g, speed));
|
||||
}
|
||||
|
||||
// Then try the most local GPUs
|
||||
int maxWidth = 0, minHops = 0xfffffff;
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
if (paths[g].width > maxWidth) {
|
||||
maxWidth = paths[g].width;
|
||||
minHops = paths[g].count;
|
||||
} else if (paths[g].width == maxWidth && paths[g].count < minHops) {
|
||||
minHops = paths[g].count;
|
||||
}
|
||||
}
|
||||
if (maxWidth >= speed) {
|
||||
// In the first loop, avoid using GPUs in both directions between channels (one channel
|
||||
// sending from that GPU and one channel receiving to that GPU), since that usually leads
|
||||
// to lower BW.
|
||||
for (int tryGpuBidir=0; tryGpuBidir<2; tryGpuBidir++) {
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
if (paths[g].width == maxWidth && paths[g].count == minHops) {
|
||||
gpu = system->nodes[GPU].nodes+g;
|
||||
int gpuUsed = gpuPciWidth(gpu) > 0 ? 0 : 1;
|
||||
if (tryGpuBidir == gpuUsed) {
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, paths, 0, backToNet, backToFirstRank, 0, maxSpeed, time, g, speed));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i=0; i<system->nodes[NET].count; i++) {
|
||||
if (system->nodes[NET].nodes[i].rank == net->rank) system->nodes[NET].nodes[i].used ^= flag;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/* Search Patterns
|
||||
*
|
||||
* Intra-node
|
||||
* Ring : GPU a -> GPU b -> .. -> GPU x -> GPU a
|
||||
* (=Split Tree Loop)
|
||||
* Tree : GPU a -> GPU b -> .. -> GPU x
|
||||
* (=Split Tree)
|
||||
*
|
||||
* Inter-node
|
||||
* Ring : NET n -> GPU a -> GPU b -> .. -> GPU x -> NET n (or m if crossNic)
|
||||
* Tree : NET n -> GPU a -> GPU b -> .. -> GPU x
|
||||
* `--> NET n (or m if crossNic)
|
||||
* Split Tree : NET n -> GPU a -> GPU b -> .. -> GPU x
|
||||
* `--> NET n (or m if crossNic)
|
||||
* Split Tree Loop : NET n -> GPU a -> GPU b -> .. -> GPU x -> GPU a
|
||||
* `--> NET n (or m if crossNic)
|
||||
*/
|
||||
ncclResult_t ncclTopoSearchParams(struct ncclTopoSystem* system, int pattern, int* backToNet, int* backToFirstRank) {
|
||||
if (system->nodes[NET].count) {
|
||||
if (pattern == NCCL_TOPO_PATTERN_RING) *backToNet = system->nodes[GPU].count-1;
|
||||
else if (pattern == NCCL_TOPO_PATTERN_TREE) *backToNet = 0;
|
||||
else *backToNet = 1;
|
||||
if (pattern == NCCL_TOPO_PATTERN_SPLIT_TREE_LOOP) *backToFirstRank = system->nodes[GPU].count-1;
|
||||
else *backToFirstRank = -1;
|
||||
} else {
|
||||
*backToNet = -1;
|
||||
if (pattern == NCCL_TOPO_PATTERN_RING || pattern == NCCL_TOPO_PATTERN_SPLIT_TREE_LOOP) *backToFirstRank = system->nodes[GPU].count-1;
|
||||
else *backToFirstRank = -1;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoSearchRec(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int maxSpeed, int* time) {
|
||||
int backToNet, backToFirstRank;
|
||||
NCCLCHECK(ncclTopoSearchParams(system, graph->pattern, &backToNet, &backToFirstRank));
|
||||
if (system->nodes[NET].count) {
|
||||
// Start from NET
|
||||
ncclTopoSearchRecNet(system, graph, saveGraph, backToNet, backToFirstRank, maxSpeed, time);
|
||||
} else {
|
||||
// Start from GPU 0
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, NULL, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, maxSpeed, time, 0, graph->speedIntra));
|
||||
if (graph->nChannels > 0) NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, NULL, 0, backToNet, backToFirstRank, FORCED_ORDER_REPLAY, maxSpeed, time, 0, graph->speedIntra));
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, NULL, 0, backToNet, backToFirstRank, 0, maxSpeed, time, 0, graph->speedIntra));
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/* Parse user defined rings. Format is like :
|
||||
* "0 1|1 0|0 1 2 3|3 2 1 0|0 2 3 1|1 3 2 0|0 1 2 3 4 5 6 7|7 6 5 4 3 2 1 0"
|
||||
* Rings with a non-matching number of ranks are ignored so we can provide
|
||||
* rings for multiple cases.
|
||||
*/
|
||||
#define MAX_ENV_RANKS 512
|
||||
static ncclResult_t parseGraph(const char* str, int* nChannelsRet, int ngpus, int* channels) {
|
||||
int ranks[MAX_ENV_RANKS];
|
||||
int nChannels = 0;
|
||||
int rank = 0;
|
||||
int offset = 0;
|
||||
int status = 0; // 0 : between numbers, 1 : inside number
|
||||
do {
|
||||
int digit = str[offset] - '0';
|
||||
if (digit >= 0 && digit <= 9) {
|
||||
if (status == 0) {
|
||||
ranks[rank] = digit;
|
||||
status = 1;
|
||||
} else {
|
||||
ranks[rank] = ranks[rank]*10+digit;
|
||||
}
|
||||
} else {
|
||||
if (status == 1) {
|
||||
rank++;
|
||||
if (rank == MAX_ENV_RANKS) goto end;
|
||||
}
|
||||
status = 0;
|
||||
if (str[offset] == '|' || str[offset] == '\0') {
|
||||
// Ignore if ngpus doesn't match
|
||||
if (rank != ngpus) goto newchannel;
|
||||
|
||||
for (int r=0; r<ngpus; r++) {
|
||||
int rank = ranks[r];
|
||||
// Ignore if ranks are out of bounds
|
||||
if (rank < 0 || rank >= ngpus) goto newchannel;
|
||||
// Ignore if ranks are duplicate
|
||||
for (int i=0; i<r; i++)
|
||||
if (ranks[i] == rank) goto newchannel;
|
||||
|
||||
channels[nChannels*ngpus+r] = rank;
|
||||
}
|
||||
nChannels++;
|
||||
newchannel:
|
||||
rank = 0;
|
||||
}
|
||||
}
|
||||
} while (str[offset++] != 0);
|
||||
end:
|
||||
*nChannelsRet = nChannels;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoCompute(ncclTopoSystem* system, struct ncclTopoGraph* graph) {
|
||||
int ngpus = system->nodes[GPU].count;
|
||||
int crossNic = (system->nodes[NET].count > 1) && graph->crossNic ? 1 : 0;
|
||||
graph->speedIntra = graph->speedInter = 0;
|
||||
if (graph->crossNic == 2) graph->crossNic = 0;
|
||||
graph->nvlink = 0;
|
||||
graph->type = LINK_LOC;
|
||||
graph->nChannels = 0;
|
||||
graph->sameChannels = 1;
|
||||
|
||||
char* str = getenv("NCCL_GRAPH");
|
||||
if (str) {
|
||||
NCCLCHECK(parseGraph(str, &graph->nChannels, ngpus, graph->intra));
|
||||
for (int i=0; i<graph->nChannels*ngpus; i++) {
|
||||
// Translate gpu numbers into ranks
|
||||
graph->intra[i] = system->nodes[GPU].nodes[graph->intra[i]].rank;
|
||||
}
|
||||
// TODO : let user specify NICs
|
||||
graph->inter[0] = graph->inter[1] = 0;
|
||||
graph->speedIntra = graph->speedInter = PCI_WIDTH+2;
|
||||
graph->nvlink = 0;
|
||||
if (graph->pattern == NCCL_TOPO_PATTERN_RING) {
|
||||
// Reverse the loop
|
||||
for (int c=0; c<graph->nChannels; c++) {
|
||||
for (int i=0; i<=ngpus/2; i++) {
|
||||
int tmp = graph->intra[ngpus*c+i];
|
||||
graph->intra[ngpus*c+i] = graph->intra[ngpus*c+(ngpus-i)%ngpus];
|
||||
graph->intra[ngpus*c+ngpus-i] = tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (graph->nChannels) return ncclSuccess;
|
||||
}
|
||||
|
||||
if (ngpus == 1) if (graph->pattern != NCCL_TOPO_PATTERN_RING) graph->pattern = NCCL_TOPO_PATTERN_TREE;
|
||||
|
||||
struct ncclTopoGraph tmpGraph;
|
||||
memcpy(&tmpGraph, graph, sizeof(struct ncclTopoGraph));
|
||||
int bestSpeed = 0;
|
||||
|
||||
// First try crossnic, then decrease speed and finally increase speedIntra.
|
||||
tmpGraph.speedIntra = tmpGraph.speedInter = system->maxWidth;
|
||||
int maxSpeed = system->maxSpeed;
|
||||
tmpGraph.pattern = graph->pattern;
|
||||
|
||||
search:
|
||||
int time = NCCL_SEARCH_TIMEOUT;
|
||||
tmpGraph.nvlink = 1;
|
||||
tmpGraph.nChannels = 0;
|
||||
tmpGraph.sameChannels = 1;
|
||||
NCCLCHECK(ncclTopoSearchRec(system, &tmpGraph, graph, maxSpeed, &time));
|
||||
#if 0
|
||||
printf("Pattern %d, crossNic %d, Speed %d/%d, type %d -> nChannels %dx%d/%d %s\n", tmpGraph.pattern, tmpGraph.crossNic, tmpGraph.speedInter, tmpGraph.speedIntra, tmpGraph.type, graph->nChannels, graph->speedInter, graph->speedIntra, time == 0 ? "TIMEOUT" : "");
|
||||
for (int c=0; c<graph->nChannels; c++) {
|
||||
printf("%2d : ", c);
|
||||
for (int g=0; g<ngpus; g++) {
|
||||
printf("%d ", graph->intra[c*ngpus+g]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
#endif
|
||||
if (time == -1) goto done;
|
||||
// We already have a solution and we timed out so lower speed will just timeout as well
|
||||
if (time == 0 && graph->nChannels > 0) goto done;
|
||||
if ((graph->nChannels > 0) && (bestSpeed == 0)) bestSpeed = graph->speedIntra;
|
||||
|
||||
if (tmpGraph.speedIntra == tmpGraph.speedInter) {
|
||||
// First pass, we don't have a solution yet ; try to go slower.
|
||||
|
||||
// Try a simpler tree
|
||||
if (tmpGraph.pattern == NCCL_TOPO_PATTERN_SPLIT_TREE_LOOP) {
|
||||
tmpGraph.pattern = NCCL_TOPO_PATTERN_SPLIT_TREE;
|
||||
goto search;
|
||||
}
|
||||
if (tmpGraph.pattern == NCCL_TOPO_PATTERN_SPLIT_TREE) {
|
||||
tmpGraph.pattern = NCCL_TOPO_PATTERN_TREE;
|
||||
goto search;
|
||||
}
|
||||
tmpGraph.pattern = graph->pattern;
|
||||
|
||||
if (tmpGraph.type < LINK_QPI) {
|
||||
tmpGraph.type += 1;
|
||||
goto search;
|
||||
}
|
||||
tmpGraph.type = graph->type;
|
||||
|
||||
if (crossNic && tmpGraph.crossNic == 0) {
|
||||
// Try again with crossNic if permitted
|
||||
tmpGraph.crossNic = crossNic;
|
||||
goto search;
|
||||
}
|
||||
tmpGraph.crossNic = graph->crossNic;
|
||||
|
||||
// Try to reduce speed per channel
|
||||
tmpGraph.speedIntra = tmpGraph.speedInter -= 3;
|
||||
if (tmpGraph.speedIntra >= bestSpeed/2 && tmpGraph.speedIntra >= 3) goto search;
|
||||
}
|
||||
|
||||
done:
|
||||
// We have a solution now. See if we can increase speedIntra
|
||||
if (tmpGraph.speedIntra == tmpGraph.speedInter) {
|
||||
time = -1;
|
||||
memcpy(&tmpGraph, graph, sizeof(tmpGraph));
|
||||
}
|
||||
if (time != 0 && tmpGraph.pattern != NCCL_TOPO_PATTERN_RING && tmpGraph.speedIntra == graph->speedIntra) {
|
||||
// Try to increase the intra speed only but keeping nChannels the same
|
||||
tmpGraph.speedIntra += 3;
|
||||
maxSpeed = tmpGraph.speedIntra * graph->nChannels;
|
||||
if (tmpGraph.speedIntra <= tmpGraph.speedInter*2) goto search;
|
||||
}
|
||||
|
||||
if (graph->nChannels == 0) {
|
||||
WARN("Could not find a path for pattern %d, falling back to simple order\n", graph->pattern);
|
||||
for (int i=0; i<ngpus; i++) graph->intra[i] = system->nodes[GPU].nodes[i].rank;
|
||||
graph->inter[0] = graph->inter[1] = 0;
|
||||
graph->speedIntra = graph->speedInter = 3;
|
||||
graph->nvlink = 0;
|
||||
graph->nChannels = 1;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoPrintGraph(struct ncclTopoSystem* system, struct ncclTopoGraph* graph) {
|
||||
INFO(NCCL_GRAPH, "Pattern %d, crossNic %d, nChannels %d, speed %d/%d, nvlink %d, type %d, sameChannels %d", graph->pattern, graph->crossNic, graph->nChannels, graph->speedIntra, graph->speedInter, graph->nvlink, graph->type, graph->sameChannels);
|
||||
int ngpus = system->nodes[GPU].count;
|
||||
|
||||
char line[1024];
|
||||
for (int c=0; c<graph->nChannels; c++) {
|
||||
sprintf(line, "%2d :", c);
|
||||
int offset = strlen(line);
|
||||
if (system->nodes[NET].count > 0) {
|
||||
sprintf(line+offset, " %s/%d", topoNodeTypeStr[NET], graph->inter[2*c]);
|
||||
offset = strlen(line);
|
||||
}
|
||||
for (int i=0; i<ngpus; i++) {
|
||||
sprintf(line+offset, " %s/%d", topoNodeTypeStr[GPU], graph->intra[ngpus*c+i]);
|
||||
offset = strlen(line);
|
||||
}
|
||||
if (system->nodes[NET].count > 0) {
|
||||
sprintf(line+offset, " %s/%d", topoNodeTypeStr[NET], graph->inter[2*c+1]);
|
||||
offset = strlen(line);
|
||||
}
|
||||
INFO(NCCL_GRAPH, "%s", line);
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoGetNetDev(struct ncclTopoGraph* graph, int dir, int channelId, int* dev) {
|
||||
*dev = graph->inter[(channelId%graph->nChannels)*2+dir];
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,678 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "graph.h"
|
||||
#include "topo.h"
|
||||
#include "comm.h"
|
||||
#include "nvmlwrap.h"
|
||||
#include "net.h"
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
#define BUSID_SIZE (sizeof("0000:00:00.0"))
|
||||
#define BUSID_REDUCED_SIZE (sizeof("0000:00"))
|
||||
|
||||
const char* pathDists[] = { "PIX", "PXB", "PHB", "NODE", "SYS" };
|
||||
|
||||
const char* topoNodeTypeStr[] = { "GPU", "PCI", "NVS", "CPU", "NIC", "NET" };
|
||||
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
|
||||
const char* topoLinkTypeStr[] = { "LOC", "XGMI", "PCI", "QPI", "NET" };
|
||||
#else
|
||||
const char* topoLinkTypeStr[] = { "LOC", "NVL", "PCI", "QPI", "NET" };
|
||||
#endif
|
||||
|
||||
/******************************************************************/
|
||||
/******************* Graph Creation Functions *********************/
|
||||
/******************************************************************/
|
||||
static int getNumaId(char *path) {
|
||||
char npath[PATH_MAX];
|
||||
snprintf(npath, PATH_MAX, "%s/numa_node", path);
|
||||
npath[PATH_MAX-1] = '\0';
|
||||
|
||||
int numaId = -1;
|
||||
FILE *file = fopen(npath, "r");
|
||||
if (file == NULL) return -1;
|
||||
if (fscanf(file, "%d", &numaId) == EOF) { fclose(file); return -1; }
|
||||
fclose(file);
|
||||
|
||||
return numaId;
|
||||
}
|
||||
|
||||
static ncclResult_t getPciPath(char* busId, char** path) {
|
||||
for (int i=0; i<BUSID_SIZE; i++) busId[i] = tolower(busId[i]);
|
||||
char busPath[] = "/sys/class/pci_bus/0000:00/../../0000:00:00.0";
|
||||
memcpy(busPath+sizeof("/sys/class/pci_bus/")-1, busId, BUSID_REDUCED_SIZE-1);
|
||||
memcpy(busPath+sizeof("/sys/class/pci_bus/0000:00/../../")-1, busId, BUSID_SIZE-1);
|
||||
*path = realpath(busPath, NULL);
|
||||
if (*path == NULL) {
|
||||
WARN("Could not find real path of %s", busPath);
|
||||
return ncclSystemError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Get an int64 from a PCI path. For example, sys/class/pci0000:00/0000:00:02.0/0000:02:00.0/ will return 0x000002000.
|
||||
ncclResult_t pciPathToInt64(char* path, int offset, int minOffset, int64_t* id) {
|
||||
char* str = path+offset;
|
||||
// Remove trailing "/"
|
||||
if (*str == '/') str--;
|
||||
// Find next /
|
||||
while (*str != '/') str--;
|
||||
str++;
|
||||
NCCLCHECK(busIdToInt64(str, id));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t idToIndex(struct ncclTopoSystem* system, int64_t id, int* index) {
|
||||
*index = -1;
|
||||
for (int i=0; i<system->nodes[GPU].count; i++) {
|
||||
if (system->nodes[GPU].nodes[i].id == id) {
|
||||
*index = i;
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
|
||||
static ncclResult_t getPath(int64_t id, char** path) {
|
||||
char busId[] = "0000:00:00.0";
|
||||
NCCLCHECK(int64ToBusId(id, busId));
|
||||
NCCLCHECK(getPciPath(busId, path));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoCudaPath(int cudaDev, char** path) {
|
||||
char busId[BUSID_SIZE];
|
||||
CUDACHECK(hipDeviceGetPCIBusId(busId, BUSID_SIZE, cudaDev));
|
||||
NCCLCHECK(getPciPath(busId, path));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
|
||||
int interCpuWidth = 0;
|
||||
int cpuPciWidth = 0;
|
||||
|
||||
static ncclResult_t getCpuWidths() {
|
||||
// Check if already detected
|
||||
if (interCpuWidth + cpuPciWidth) return ncclSuccess;
|
||||
|
||||
// Defaults
|
||||
char cpu[256];
|
||||
sprintf(cpu, "Generic");
|
||||
cpuPciWidth = interCpuWidth = PCI_WIDTH;
|
||||
|
||||
#ifdef __PPC__
|
||||
sprintf(cpu, "ppc64");
|
||||
interCpuWidth = P9_WIDTH;
|
||||
#endif
|
||||
#ifdef __x86_64__
|
||||
sprintf(cpu, "x86_64");
|
||||
union {
|
||||
struct {
|
||||
// CPUID 0 String register order
|
||||
uint32_t ebx;
|
||||
uint32_t edx;
|
||||
uint32_t ecx;
|
||||
};
|
||||
char vendor[12];
|
||||
} cpuid0;
|
||||
|
||||
asm volatile("cpuid" : "=b" (cpuid0.ebx), "=c" (cpuid0.ecx), "=d" (cpuid0.edx) : "a" (0));
|
||||
if (strncmp(cpuid0.vendor, "GenuineIntel", 12) == 0) sprintf(cpu, "Intel");
|
||||
|
||||
if (strcmp(cpu, "Intel") == 0) {
|
||||
union {
|
||||
struct {
|
||||
int steppingId:4;
|
||||
int model:4;
|
||||
int familyId:4;
|
||||
int processorType:2;
|
||||
int resv0:2;
|
||||
int extModelId:4;
|
||||
int modelId:8;
|
||||
int resv1:4;
|
||||
};
|
||||
uint32_t val;
|
||||
} cpuid1;
|
||||
asm volatile("cpuid" : "=a" (cpuid1.val) : "a" (1));
|
||||
if (cpuid1.familyId == 6 && cpuid1.modelId >= 0x55) { // Skylake
|
||||
sprintf(cpu, "Intel/Skylake (or later)");
|
||||
interCpuWidth = SKL_QPI_WIDTH;
|
||||
} else {
|
||||
interCpuWidth = QPI_WIDTH;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
INFO(NCCL_GRAPH, "%s CPU (PCI %d, InterCpu %d)", cpu, cpuPciWidth, interCpuWidth);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTopoGetInterCpuWidth(int* width) {
|
||||
NCCLCHECK(getCpuWidths());
|
||||
*width = interCpuWidth;
|
||||
return ncclSuccess;
|
||||
}
|
||||
static ncclResult_t ncclTopoGetCpuPciP2pWidth(int* width) {
|
||||
NCCLCHECK(getCpuWidths());
|
||||
*width = cpuPciWidth;
|
||||
return ncclSuccess;
|
||||
}
|
||||
static ncclResult_t ncclTopoGetPciWidth(int* width) {
|
||||
*width = PCI_WIDTH;
|
||||
return ncclSuccess;
|
||||
}
|
||||
static ncclResult_t ncclTopoGetNetWidth(int* width) {
|
||||
*width = NET_WIDTH;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
enum ncclNvLinkDeviceType {
|
||||
ncclNvLinkDeviceUnknown,
|
||||
ncclNvLinkDeviceGpu,
|
||||
ncclNvLinkDeviceSwitch,
|
||||
ncclNvLinkDeviceBridge, // IBM/Power NVLink bridge (Device 04ea)
|
||||
};
|
||||
|
||||
static ncclResult_t ncclDeviceType(const char* busId, enum ncclNvLinkDeviceType* type) {
|
||||
char classPath[] = "/sys/bus/pci/devices/0000:00:00.0/class";
|
||||
memcpy(classPath+sizeof("/sys/bus/pci/devices/")-1, busId, sizeof("0000:00:00.0")-1);
|
||||
char* rPath = realpath(classPath, NULL);
|
||||
int fd;
|
||||
if ((fd = open(rPath, O_RDONLY)) == -1) {
|
||||
// Could not find device. It might be because we're in a VM and
|
||||
// we don't see the whole machine. This is handled silently so
|
||||
// we don't want to print an INFO error.
|
||||
TRACE(NCCL_INIT, "Open of %s failed : %s\n", rPath, strerror(errno));
|
||||
return ncclSystemError;
|
||||
}
|
||||
free(rPath);
|
||||
char pciClass[9];
|
||||
strncpy(pciClass, "0x000000", 9);
|
||||
int len;
|
||||
SYSCHECKVAL(read(fd, pciClass, 8), "read", len);
|
||||
SYSCHECK(close(fd), "close");
|
||||
if (strcmp(pciClass, "0x068000") == 0) {
|
||||
// PCI device is of type "Bridge / Other Bridge Device" (NVswitch)
|
||||
*type = ncclNvLinkDeviceSwitch;
|
||||
} else if (strcmp(pciClass, "0x068001") == 0) {
|
||||
// PCI device is of type "Bridge: IBM Device 04ea"
|
||||
*type = ncclNvLinkDeviceBridge;
|
||||
} else if (strcmp(pciClass, "0x030200") == 0 // "3D Controller" (Tesla)
|
||||
|| strcmp(pciClass, "0x030000") == 0) { // "VGA Controller" (GeForce)
|
||||
*type = ncclNvLinkDeviceGpu;
|
||||
} else {
|
||||
*type = ncclNvLinkDeviceUnknown;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoConnectCpu(struct ncclTopoSystem* system, int numaId, struct ncclTopoNode* node, int linkType, int linkWidth) {
|
||||
struct ncclTopoNode* cpuNode = NULL;
|
||||
for (int c=0; c<system->nodes[CPU].count; c++) {
|
||||
if (system->nodes[CPU].nodes[c].id == numaId) cpuNode = system->nodes[CPU].nodes+c;
|
||||
}
|
||||
if (cpuNode == NULL) { // Create CPU
|
||||
NCCLCHECK(ncclTopoCreateNode(system, &cpuNode, CPU, numaId));
|
||||
}
|
||||
NCCLCHECK(ncclTopoConnectNodes(node, cpuNode, linkType, linkWidth));
|
||||
NCCLCHECK(ncclTopoConnectNodes(cpuNode, node, linkType, linkWidth));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
|
||||
#define VEGA_XGMI_WIDTH 20
|
||||
|
||||
ncclResult_t ncclTopoConnectXGMI(int num_gpus, struct ncclTopoSystem* system) {
|
||||
struct ncclTopoNode* nvsNode = NULL;
|
||||
|
||||
int minNvlinks = 2, minWidth = VEGA_XGMI_WIDTH;
|
||||
for (int g1=0; g1<system->nodes[GPU].count; g1++) {
|
||||
for(int g2=0; g2<system->nodes[GPU].count; g2++) {
|
||||
if (g1 == g2) continue;
|
||||
struct ncclTopoNode* gpu1 = system->nodes[GPU].nodes+g1;
|
||||
struct ncclTopoNode* gpu2 = system->nodes[GPU].nodes+g2;
|
||||
uint32_t link_type, hops;
|
||||
if (hipExtGetLinkTypeAndHopCount(gpu1->rank, gpu2->rank, &link_type, &hops) == hipSuccess) {
|
||||
if (link_type == HSA_AMD_LINK_INFO_TYPE_XGMI && hops == 1) {
|
||||
NCCLCHECK(ncclTopoConnectNodes(gpu1, gpu2, LINK_NVL, minWidth));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
int pciWidth;
|
||||
NCCLCHECK(ncclTopoGetPciWidth(&pciWidth));
|
||||
system->maxSpeed = minNvlinks ? minNvlinks*minWidth : pciWidth;
|
||||
system->maxWidth = minNvlinks ? minWidth : pciWidth;
|
||||
return ncclSuccess;
|
||||
}
|
||||
#else
|
||||
ncclResult_t ncclTopoConnectNVLink(nvmlDevice_t* nvmlDevs, struct ncclTopoSystem* system) {
|
||||
struct ncclTopoNode* nvsNode = NULL;
|
||||
|
||||
int minNvlinks = 6, minWidth = VOLTA_NVLINK_WIDTH;
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
int cudaMajor, cudaMinor;
|
||||
NCCLCHECK(wrapNvmlDeviceGetCudaComputeCapability(nvmlDevs[g], &cudaMajor, &cudaMinor));
|
||||
int maxNvLinks, width;
|
||||
if (cudaMajor < 6) {
|
||||
maxNvLinks = 0;
|
||||
width = 0;
|
||||
} else if (cudaMajor == 6) {
|
||||
maxNvLinks = 4;
|
||||
width = PASCAL_NVLINK_WIDTH;
|
||||
} else {
|
||||
maxNvLinks = 6;
|
||||
width = VOLTA_NVLINK_WIDTH;
|
||||
}
|
||||
|
||||
int nvlinks = 0;
|
||||
for (int l=0; l<maxNvLinks; ++l) {
|
||||
// Check whether we can use this NVLink for P2P
|
||||
unsigned canP2P;
|
||||
if ((wrapNvmlDeviceGetNvLinkCapability(nvmlDevs[g], l, NVML_NVLINK_CAP_P2P_SUPPORTED, &canP2P) != ncclSuccess) || !canP2P) continue;
|
||||
|
||||
// Make sure the Nvlink is up. The previous call should have trained the link.
|
||||
nvmlEnableState_t isActive;
|
||||
if ((wrapNvmlDeviceGetNvLinkState(nvmlDevs[g], l, &isActive) != ncclSuccess) || (isActive != NVML_FEATURE_ENABLED)) continue;
|
||||
|
||||
// Try to figure out what's on the other side of the NVLink
|
||||
nvmlPciInfo_t remoteProc;
|
||||
if (wrapNvmlDeviceGetNvLinkRemotePciInfo(nvmlDevs[g], l, &remoteProc) != ncclSuccess) continue;
|
||||
|
||||
// Make a lower case copy of the bus ID for calling ncclDeviceType
|
||||
// PCI system path is in lower case
|
||||
char* p = remoteProc.busId;
|
||||
char lowerId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE];
|
||||
for (int c=0; c<NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE; c++) {
|
||||
lowerId[c] = tolower(p[c]);
|
||||
if (p[c] == 0) break;
|
||||
}
|
||||
|
||||
enum ncclNvLinkDeviceType type;
|
||||
NCCLCHECK(ncclDeviceType(lowerId, &type));
|
||||
if (type == ncclNvLinkDeviceGpu) {
|
||||
int64_t remoteId;
|
||||
NCCLCHECK(busIdToInt64(lowerId, &remoteId));
|
||||
int peer;
|
||||
NCCLCHECK(idToIndex(system, remoteId, &peer));
|
||||
if (peer != -1) {
|
||||
NCCLCHECK(ncclTopoConnectNodes(gpu, system->nodes[GPU].nodes+peer, LINK_NVL, width));
|
||||
nvlinks++;
|
||||
}
|
||||
} else if (type == ncclNvLinkDeviceBridge) {
|
||||
// Nvlink between GPU and CPU (PPC)
|
||||
// Since the remote bridge does not have a valid numa_node, assume we
|
||||
// are connected to the closest CPU.
|
||||
char* path;
|
||||
NCCLCHECK(getPath(gpu->id, &path));
|
||||
int numaId = getNumaId(path);
|
||||
free(path);
|
||||
NCCLCHECK(ncclTopoConnectCpu(system, numaId, gpu, LINK_NVL, width));
|
||||
nvlinks++;
|
||||
} else { // Nvswitch
|
||||
if (type == ncclNvLinkDeviceUnknown) {
|
||||
// The NVLink is up but we couldn't find the PCI device on the other
|
||||
// side. Assume it's an NVswitch outside a VM.
|
||||
if (l == 0) INFO(NCCL_INIT, "%d/%d -> %s : Assuming NVLink is connected to NVswitch", g, l, lowerId);
|
||||
}
|
||||
if (nvsNode == NULL) { // Create nvswitch
|
||||
NCCLCHECK(ncclTopoCreateNode(system, &nvsNode, NVS, 0));
|
||||
}
|
||||
NCCLCHECK(ncclTopoConnectNodes(gpu, nvsNode, LINK_NVL, VOLTA_NVLINK_WIDTH));
|
||||
NCCLCHECK(ncclTopoConnectNodes(nvsNode, gpu, LINK_NVL, VOLTA_NVLINK_WIDTH));
|
||||
nvlinks++;
|
||||
}
|
||||
}
|
||||
minNvlinks = std::min(minNvlinks, nvlinks);
|
||||
minWidth = std::min(minWidth, width);
|
||||
}
|
||||
int pciWidth;
|
||||
NCCLCHECK(ncclTopoGetPciWidth(&pciWidth));
|
||||
system->maxSpeed = minNvlinks ? minNvlinks*minWidth : pciWidth;
|
||||
system->maxWidth = minNvlinks ? minWidth : pciWidth;
|
||||
return ncclSuccess;
|
||||
}
|
||||
#endif
|
||||
|
||||
ncclResult_t ncclTopoCreatePciPath(struct ncclTopoSystem* system, struct ncclTopoNode* endNode, char* path) {
|
||||
struct ncclTopoNode* lastNode = endNode;
|
||||
int pciWidth;
|
||||
NCCLCHECK(ncclTopoGetPciWidth(&pciWidth));
|
||||
// Find intermediate PCI switches
|
||||
int slashCount = 0;
|
||||
int offsetRC = 0;
|
||||
while (offsetRC < strlen(path)) {
|
||||
if (path[offsetRC] == '/') slashCount++;
|
||||
if (slashCount == 4) break;
|
||||
offsetRC++;
|
||||
}
|
||||
int offset = strlen(path);
|
||||
slashCount = 0;
|
||||
while (--offset > offsetRC) {
|
||||
if (path[offset] == '/') {
|
||||
slashCount++;
|
||||
// Find if already existing
|
||||
if ((slashCount%2) == 0) {
|
||||
int64_t pciId;
|
||||
NCCLCHECK(pciPathToInt64(path, offset, offsetRC, &pciId));
|
||||
for (int p=0; p<system->nodes[PCI].count; p++) {
|
||||
if (system->nodes[PCI].nodes[p].id == pciId) {
|
||||
// Found our PCI switch. Attach and stop since the rest should already
|
||||
// be connected
|
||||
NCCLCHECK(ncclTopoConnectNodes(system->nodes[PCI].nodes+p, lastNode, LINK_PCI, pciWidth));
|
||||
NCCLCHECK(ncclTopoConnectNodes(lastNode, system->nodes[PCI].nodes+p, LINK_PCI, pciWidth));
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
struct ncclTopoNode* pciNode;
|
||||
NCCLCHECK(ncclTopoCreateNode(system, &pciNode, PCI, pciId));
|
||||
NCCLCHECK(ncclTopoConnectNodes(pciNode, lastNode, LINK_PCI, pciWidth));
|
||||
NCCLCHECK(ncclTopoConnectNodes(lastNode, pciNode, LINK_PCI, pciWidth));
|
||||
lastNode = pciNode;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Then attach to a CPU node
|
||||
int numaId = getNumaId(path);
|
||||
int width;
|
||||
NCCLCHECK(ncclTopoGetCpuPciP2pWidth(&width));
|
||||
NCCLCHECK(ncclTopoConnectCpu(system, numaId, lastNode, LINK_PCI, width));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Try to detect if IB cards are in fact the same physical NIC, hence sharing ports.
|
||||
#include <glob.h>
|
||||
#define IB_GUID_PATH "%s/infiniband/mlx5_*/sys_image_guid"
|
||||
uint64_t getIbGuid(char* path) {
|
||||
uint64_t guid = 0ULL;
|
||||
char guidPath[PATH_MAX];
|
||||
snprintf(guidPath, PATH_MAX, IB_GUID_PATH, path);
|
||||
// PATH has a wildcard in it so use glob()
|
||||
glob_t globbuf;
|
||||
glob(guidPath, 0, NULL, &globbuf);
|
||||
if (globbuf.gl_pathc > 0)
|
||||
strncpy(guidPath, globbuf.gl_pathv[0], PATH_MAX);
|
||||
globfree(&globbuf);
|
||||
guidPath[PATH_MAX-1] = '\0';
|
||||
FILE *file = fopen(guidPath, "r");
|
||||
if (file != NULL) {
|
||||
uint64_t a, b, c, d;
|
||||
if (fscanf(file, "%04lx:%04lx:%04lx:%04lx", &a, &b, &c, &d) != EOF) {
|
||||
guid = (a << 48) + (b << 32) + (c<<16) + d;
|
||||
TRACE(NCCL_GRAPH, "Opened %s guid %lx", guidPath, guid);
|
||||
}
|
||||
fclose(file);
|
||||
}
|
||||
return guid;
|
||||
}
|
||||
|
||||
struct netInfo {
|
||||
char* path;
|
||||
int64_t nic;
|
||||
uint64_t asic;
|
||||
int port;
|
||||
int net;
|
||||
};
|
||||
|
||||
ncclResult_t ncclTopoComputeNetInfo(struct netInfo* netInfos, int ndev) {
|
||||
for (int n=0; n<ndev; n++) {
|
||||
struct netInfo* info = netInfos+n;
|
||||
uint64_t ibGuid;
|
||||
info->nic = n;
|
||||
info->asic = n;
|
||||
info->port = 0;
|
||||
info->net = n;
|
||||
if (info->path && (ibGuid = getIbGuid(info->path)) != 0) {
|
||||
info->asic = ibGuid;
|
||||
|
||||
// Ignore PCI subdevice when computing the ID to merge multi-port cards
|
||||
// and make them use the same PCI link.
|
||||
char* path = strdup(info->path);
|
||||
path[strlen(path)-1]='0';
|
||||
NCCLCHECK(pciPathToInt64(path, strlen(path), 0, &info->nic));
|
||||
free(path);
|
||||
|
||||
// Same PCI path -> different ports of the same NIC
|
||||
for (int i=0; i<n; i++) if (netInfos[i].nic == info->nic) info->port++;
|
||||
|
||||
// Same GUID -> same network links as the other NIC
|
||||
for (int i=0; i<n; i++) if (netInfos[i].asic == info->asic && netInfos[i].port == info->port) info->net = netInfos[i].net;
|
||||
}
|
||||
INFO(NCCL_GRAPH, "%s -> %x/%lx/%d/%d", info->path, info->nic, info->asic, info->port, info->net);
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoConnectPCI(struct ncclTopoSystem* system) {
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
char* path;
|
||||
NCCLCHECK(getPath(gpu->id, &path));
|
||||
NCCLCHECK(ncclTopoCreatePciPath(system, gpu, path));
|
||||
free(path);
|
||||
}
|
||||
|
||||
// Connect the NICs
|
||||
int netDevCount;
|
||||
NCCLCHECK(ncclNetDevices(&netDevCount));
|
||||
int netWidth;
|
||||
NCCLCHECK(ncclTopoGetNetWidth(&netWidth));
|
||||
|
||||
struct netInfo* netInfos;
|
||||
NCCLCHECK(ncclCalloc(&netInfos, netDevCount));
|
||||
|
||||
for (int n=0; n<netDevCount; n++) {
|
||||
ncclResult_t res = ncclNetPciPath(n, &netInfos[n].path);
|
||||
if (res != ncclSuccess) netInfos[n].path = NULL;
|
||||
}
|
||||
|
||||
NCCLCHECK(ncclTopoComputeNetInfo(netInfos, netDevCount));
|
||||
|
||||
for (int n=0; n<netDevCount; n++) {
|
||||
struct netInfo* info = netInfos+n;
|
||||
// Create NIC and attach it to the PCI tree
|
||||
struct ncclTopoNode* nicNode = NULL;
|
||||
for (int i=0; i<system->nodes[NIC].count; i++) {
|
||||
if (system->nodes[NIC].nodes[i].id == info->nic) {
|
||||
nicNode = system->nodes[NIC].nodes+i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!nicNode) {
|
||||
NCCLCHECK(ncclTopoCreateNode(system, &nicNode, NIC, info->nic));
|
||||
if (info->path) {
|
||||
// Create the PCI path
|
||||
NCCLCHECK(ncclTopoCreatePciPath(system, nicNode, info->path));
|
||||
} else {
|
||||
// This is probably a virtual NIC. Just attach it directly to CPU 0
|
||||
int width;
|
||||
NCCLCHECK(ncclTopoGetCpuPciP2pWidth(&width));
|
||||
NCCLCHECK(ncclTopoConnectCpu(system, 0, nicNode, LINK_PCI, width));
|
||||
}
|
||||
}
|
||||
free(info->path);
|
||||
|
||||
// Create the network side
|
||||
struct ncclTopoNode* netNode;
|
||||
NCCLCHECK(ncclTopoCreateNode(system, &netNode, NET, n));
|
||||
|
||||
// Use rank to store the net information
|
||||
netNode->rank = info->net;
|
||||
|
||||
NCCLCHECK(ncclTopoConnectNodes(nicNode, netNode, LINK_NET, netWidth));
|
||||
NCCLCHECK(ncclTopoConnectNodes(netNode, nicNode, LINK_NET, netWidth));
|
||||
}
|
||||
free(netInfos);
|
||||
|
||||
// And connect all CPU nodes together
|
||||
for (int n=0; n<system->nodes[CPU].count; n++) {
|
||||
for (int p=0; p<system->nodes[CPU].count; p++) {
|
||||
if (n == p) continue;
|
||||
int width;
|
||||
NCCLCHECK(ncclTopoGetInterCpuWidth(&width));
|
||||
NCCLCHECK(ncclTopoConnectNodes(system->nodes[CPU].nodes+n, system->nodes[CPU].nodes+p, LINK_QPI, width));
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTopoPrintRec(struct ncclTopoNode* node, struct ncclTopoNode* prevNode, char* line, int offset) {
|
||||
if (node->type == GPU) {
|
||||
sprintf(line+offset, "%s/%lX (%d)", topoNodeTypeStr[node->type], node->id, node->rank);
|
||||
} else {
|
||||
sprintf(line+offset, "%s/%lX", topoNodeTypeStr[node->type], node->id);
|
||||
}
|
||||
INFO(NCCL_GRAPH, "%s", line);
|
||||
for (int i=0; i<offset; i++) line[i] = ' ';
|
||||
|
||||
for (int l=0; l<node->nlinks; l++) {
|
||||
struct ncclTopoLink* link = node->links+l;
|
||||
if (link->type == LINK_LOC) continue;
|
||||
if (link->remNode != prevNode) {
|
||||
sprintf(line+offset, "+ %s[%2d] - ", topoLinkTypeStr[link->type], link->width);
|
||||
int nextOffset = strlen(line);
|
||||
if (link->type == LINK_PCI) {
|
||||
NCCLCHECK(ncclTopoPrintRec(link->remNode, node, line, nextOffset));
|
||||
} else {
|
||||
if (link->remNode->type == NET) {
|
||||
sprintf(line+nextOffset, "%s/%lX (%d)", topoNodeTypeStr[link->remNode->type], link->remNode->id, link->remNode->rank);
|
||||
} else {
|
||||
sprintf(line+nextOffset, "%s/%lX", topoNodeTypeStr[link->remNode->type], link->remNode->id);
|
||||
}
|
||||
INFO(NCCL_GRAPH, "%s", line);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoPrint(struct ncclTopoSystem* s) {
|
||||
INFO(NCCL_GRAPH, "=== System : maxWidth %2d maxSpeed %2d ===", s->maxWidth, s->maxSpeed);
|
||||
char line[1024];
|
||||
for (int n=0; n<s->nodes[CPU].count; n++) NCCLCHECK(ncclTopoPrintRec(s->nodes[CPU].nodes+n, NULL, line, 0));
|
||||
INFO(NCCL_GRAPH, "==========================================");
|
||||
NCCLCHECK(ncclTopoPrintPaths(s));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTopoSort(struct ncclTopoNode* node, struct ncclTopoNode* upNode) {
|
||||
// Shift all links to have upLink as last link
|
||||
if (upNode) {
|
||||
int l=0;
|
||||
while (node->links[l].remNode != upNode) l++;
|
||||
struct ncclTopoLink upLink;
|
||||
memcpy(&upLink, node->links+l, sizeof(struct ncclTopoLink));
|
||||
while (node->links[l+1].remNode) {
|
||||
memcpy(node->links+l, node->links+l+1, sizeof(struct ncclTopoLink));
|
||||
l++;
|
||||
}
|
||||
memcpy(node->links+l, &upLink, sizeof(struct ncclTopoLink));
|
||||
}
|
||||
|
||||
// Recursively sort the PCI tree
|
||||
for (int l=0; l<node->nlinks; l++) {
|
||||
struct ncclTopoLink* link = node->links+l;
|
||||
if (link->type == LINK_PCI && link->remNode != upNode) NCCLCHECK(ncclTopoSort(link->remNode, node));
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// We want the graph to be organized to ease/accelerate traversal :
|
||||
// 1. NVLinks (already the case)
|
||||
// 2. PCI down
|
||||
// 3. PCI up
|
||||
// 4. QPI (already the case)
|
||||
ncclResult_t ncclTopoSortSystem(struct ncclTopoSystem* system) {
|
||||
for (int n=0; n<system->nodes[CPU].count; n++) NCCLCHECK(ncclTopoSort(system->nodes[CPU].nodes+n, NULL));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoGetSystem(struct ncclComm* comm, struct ncclTopoSystem** system) {
|
||||
struct ncclTopoSystem* s;
|
||||
NCCLCHECK(ncclCalloc(&s, 1));
|
||||
nvmlDevice_t* nvmlDevs;
|
||||
int g = 0;
|
||||
NCCLCHECK(ncclCalloc(&nvmlDevs, comm->nRanks));
|
||||
for (int r=0; r<comm->nRanks; r++) {
|
||||
if (comm->peerInfo[r].hostHash == comm->peerInfo[comm->rank].hostHash) {
|
||||
// Consider the GPU as outside of our node if we can't see it through NVML.
|
||||
char busId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE];
|
||||
NCCLCHECK(int64ToBusId(comm->peerInfo[r].busId, busId));
|
||||
if (wrapNvmlDeviceGetHandleByPciBusId(busId, nvmlDevs+g) != ncclSuccess) continue;
|
||||
g++;
|
||||
struct ncclTopoNode* gpuNode;
|
||||
NCCLCHECK(ncclTopoCreateNode(s, &gpuNode, GPU, comm->peerInfo[r].busId));
|
||||
gpuNode->rank = r;
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
|
||||
NCCLCHECK(ncclTopoConnectXGMI(g, s));
|
||||
#else
|
||||
NCCLCHECK(ncclTopoConnectNVLink(nvmlDevs, s));
|
||||
#endif
|
||||
NCCLCHECK(ncclTopoConnectPCI(s));
|
||||
|
||||
free(nvmlDevs);
|
||||
NCCLCHECK(ncclTopoSortSystem(s));
|
||||
*system = s;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoGetNvlink(struct ncclTopoSystem* system, int64_t busId1, int64_t busId2, int* nvlink) {
|
||||
int g1, g2;
|
||||
NCCLCHECK(idToIndex(system, busId1, &g1));
|
||||
NCCLCHECK(idToIndex(system, busId2, &g2));
|
||||
*nvlink = g1 != -1 && g2 != -1 && system->nodes[GPU].nodes[g1].paths[GPU][g2].type == LINK_NVL;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoHasNvlink(struct ncclTopoSystem* system, int64_t busId, int* nvlink) {
|
||||
int g;
|
||||
NCCLCHECK(idToIndex(system, busId, &g));
|
||||
for (int i=0; i<system->nodes[GPU].count; i++) {
|
||||
if (i == g) continue;
|
||||
if (system->nodes[GPU].nodes[g].paths[GPU][i].type == LINK_NVL) {
|
||||
*nvlink = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
*nvlink = 0;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static int pathDistance(struct ncclTopoLinkList* links) {
|
||||
int distance = PATH_PIX;
|
||||
if (links->count > 2) distance = PATH_PXB;
|
||||
for (int l=0; l<links->count; l++) {
|
||||
// PHB if we go through 1 CPU, SYS if we go through 2 CPUs
|
||||
if (links->list[l]->remNode->type == CPU) distance = (distance == PATH_PHB) ? PATH_SYS : PATH_PHB;
|
||||
}
|
||||
return distance;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoGpuDistance(struct ncclTopoSystem* system, int64_t busId1, int64_t busId2, int* distance) {
|
||||
int g1, g2;
|
||||
NCCLCHECK(idToIndex(system, busId1, &g1));
|
||||
NCCLCHECK(idToIndex(system, busId2, &g2));
|
||||
*distance = pathDistance(system->nodes[GPU].nodes[g1].paths[GPU]+g2);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoNetDistance(struct ncclTopoSystem* system, int64_t busId, int netDev, int* distance) {
|
||||
int g;
|
||||
NCCLCHECK(idToIndex(system, busId, &g));
|
||||
*distance = pathDistance(system->nodes[GPU].nodes[g].paths[NET]+netDev);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoCpuCount(struct ncclTopoSystem* system, int* count) {
|
||||
*count = system->nodes[CPU].count;
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_TOPO_H_
|
||||
#define NCCL_TOPO_H_
|
||||
|
||||
#include "graph.h"
|
||||
#include "core.h"
|
||||
|
||||
#define LOC_WIDTH 5000
|
||||
#define PASCAL_NVLINK_WIDTH 18
|
||||
#define VOLTA_NVLINK_WIDTH 21
|
||||
#define PCI_WIDTH 12 // PCI Gen3 x16
|
||||
#define QPI_WIDTH 8
|
||||
#define SKL_QPI_WIDTH 12
|
||||
#define P9_WIDTH 32
|
||||
#define NET_WIDTH 12 // 100Gbit
|
||||
|
||||
// Intel CPU convert GPU P2P traffic into 64B PCI TLPs, to GPU
|
||||
// to GPU traffic consumed more PCI bandwidth.
|
||||
#define INTEL_P2P(speed) (speed*9/12)
|
||||
#define INTEL_P2P_OVERHEAD(speed) (speed*12/9)
|
||||
|
||||
#define NCCL_TOPO_NODE_TYPES 6
|
||||
#define GPU 0
|
||||
#define PCI 1
|
||||
#define NVS 2
|
||||
#define CPU 3 // Actually NUMA domains
|
||||
#define NIC 4
|
||||
#define NET 5
|
||||
extern const char* topoNodeTypeStr[];
|
||||
|
||||
#define LINK_LOC 0
|
||||
#define LINK_NVL 1
|
||||
#define LINK_PCI 2
|
||||
#define LINK_QPI 3
|
||||
#define LINK_NET 4
|
||||
extern const char* topoLinkTypeStr[];
|
||||
|
||||
struct ncclTopoNode;
|
||||
struct ncclTopoLink {
|
||||
int type;
|
||||
int width;
|
||||
struct ncclTopoNode* remNode;
|
||||
};
|
||||
#define NCCL_TOPO_MAX_LINKS 32
|
||||
#define NCCL_TOPO_MAX_HOPS (NCCL_TOPO_MAX_NODES*NCCL_TOPO_NODE_TYPES)
|
||||
#define SELECT_PATH 1
|
||||
#define SELECT_LAST 2
|
||||
|
||||
#define NET_GDR_MASK 0x70000000
|
||||
|
||||
struct ncclTopoLinkList {
|
||||
struct ncclTopoLink* list[NCCL_TOPO_MAX_HOPS];
|
||||
int count;
|
||||
int width;
|
||||
int type;
|
||||
};
|
||||
|
||||
struct ncclTopoNode {
|
||||
int type;
|
||||
int64_t id;
|
||||
int rank;
|
||||
int nlinks;
|
||||
struct ncclTopoLink links[NCCL_TOPO_MAX_LINKS];
|
||||
// Pre-computed paths to GPUs and NICs
|
||||
struct ncclTopoLinkList* paths[NCCL_TOPO_NODE_TYPES];
|
||||
// Used during search
|
||||
uint64_t used;
|
||||
};
|
||||
|
||||
struct ncclTopoNodeSet {
|
||||
int count;
|
||||
struct ncclTopoNode nodes[NCCL_TOPO_MAX_NODES];
|
||||
};
|
||||
|
||||
struct ncclTopoSystem {
|
||||
struct ncclTopoNodeSet nodes[NCCL_TOPO_NODE_TYPES];
|
||||
int maxSpeed;
|
||||
int maxWidth;
|
||||
int searchInitDone;
|
||||
};
|
||||
|
||||
static ncclResult_t ncclTopoCreateNode(struct ncclTopoSystem* system, struct ncclTopoNode** node, int type, uint64_t id) {
|
||||
for (int i=0; i<system->nodes[type].count; i++) {
|
||||
if (system->nodes[type].nodes[i].id == id) {
|
||||
*node = system->nodes[type].nodes+i;
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
if (system->nodes[type].count == NCCL_TOPO_MAX_NODES) {
|
||||
WARN("Error : tried to create too many nodes of type %d\n", type);
|
||||
return ncclInternalError;
|
||||
}
|
||||
struct ncclTopoNode* n = system->nodes[type].nodes+system->nodes[type].count;
|
||||
system->nodes[type].count++;
|
||||
n->type = type;
|
||||
n->id = id;
|
||||
if (type == GPU) {
|
||||
// Create link to itself (used in some corner cases)
|
||||
n->nlinks=1;
|
||||
n->links[0].type = LINK_LOC;
|
||||
n->links[0].remNode = n;
|
||||
n->links[0].width = LOC_WIDTH;
|
||||
}
|
||||
*node = n;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTopoConnectNodes(struct ncclTopoNode* node, struct ncclTopoNode* remNode, int type, int width) {
|
||||
// Aggregate links into higher width for NVLink
|
||||
struct ncclTopoLink* link;
|
||||
for (link = node->links; link->remNode; link++) {
|
||||
if (link->remNode == remNode && link->type == type) break;
|
||||
}
|
||||
if (link->remNode == NULL) node->nlinks++;
|
||||
link->type = type;
|
||||
link->remNode = remNode;
|
||||
link->width += width;
|
||||
|
||||
// Sort links in BW descending order
|
||||
struct ncclTopoLink linkSave;
|
||||
memcpy(&linkSave, link, sizeof(struct ncclTopoLink));
|
||||
while (link != node->links) {
|
||||
if ((link-1)->width >= linkSave.width) break;
|
||||
memcpy(link, link-1, sizeof(struct ncclTopoLink));
|
||||
link--;
|
||||
}
|
||||
memcpy(link, &linkSave, sizeof(struct ncclTopoLink));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoPrintPaths(struct ncclTopoSystem* system);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,106 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "nccl.h"
|
||||
|
||||
#define RANK_TO_INDEX(r) (rank > root ? rank-1 : rank)
|
||||
|
||||
/* Btree which alternates leaves and nodes.
|
||||
* Assumes root is 0, which conveniently builds a tree on powers of two,
|
||||
* (because we have pow2-1 ranks) which lets us manipulate bits.
|
||||
* Find first non-zero bit, then :
|
||||
* Find the parent :
|
||||
* xx01[0] -> xx10[0] (1,5,9 below) or xx00[0] if xx10[0] is out of bounds (13 below)
|
||||
* xx11[0] -> xx10[0] (3,7,11 below)
|
||||
* Find the children :
|
||||
* xx10[0] -> xx01[0] (2,4,6,8,10,12) or -1 (1,3,5,7,9,11,13)
|
||||
* xx10[0] -> xx11[0] (2,4,6,8,10) or xx101[0] (12) or xx1001[0] ... or -1 (1,3,5,7,9,11,13)
|
||||
*
|
||||
* Illustration :
|
||||
* 0---------------8
|
||||
* ______/ \______
|
||||
* 4 12
|
||||
* / \ / \
|
||||
* 2 6 10 \
|
||||
* / \ / \ / \ \
|
||||
* 1 3 5 7 9 11 13
|
||||
*/
|
||||
ncclResult_t ncclGetBtree(int nranks, int rank, int* u, int* d0, int* d1) {
|
||||
int up, down0, down1;
|
||||
int bit;
|
||||
for (bit=1; bit<nranks; bit<<=1) {
|
||||
if (bit & rank) break;
|
||||
}
|
||||
|
||||
if (rank == 0) {
|
||||
*u = -1;
|
||||
*d0 = nranks > 1 ? bit >> 1 : -1;
|
||||
*d1 = -1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
up = (rank ^ bit) | (bit << 1);
|
||||
if (up >= nranks) up = (rank ^ bit);
|
||||
*u = up;
|
||||
|
||||
int lowbit = bit >> 1;
|
||||
// down0 is always within bounds
|
||||
down0 = lowbit == 0 ? -1 : rank-lowbit;
|
||||
|
||||
down1 = lowbit == 0 ? -1 : rank+lowbit;
|
||||
// Make sure down1 is within bounds
|
||||
while (down1 >= nranks) {
|
||||
down1 = lowbit == 0 ? -1 : rank+lowbit;
|
||||
lowbit >>= 1;
|
||||
}
|
||||
*d0 = down0; *d1 = down1;
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/* Build a double binary tree. Take the previous tree for the first tree.
|
||||
* For the second tree, we use a mirror tree (if nranks is odd)
|
||||
*
|
||||
* 8---------0---------5
|
||||
* ______/ \______ _____/ \______
|
||||
* 4 12 1 9
|
||||
* / \ / \ / \
|
||||
* 2 6 10 3 7 10
|
||||
* / \ / \ / \ / \ / \ / \
|
||||
* 1 3 5 7 9 11 2 4 6 8 11 12
|
||||
*
|
||||
* or shift it by one rank (if nranks is even)
|
||||
*
|
||||
* 8---------0--------------9
|
||||
* ______/ \ ______/ \
|
||||
* 4 \ 5 \
|
||||
* / \ \ / \ \
|
||||
* 2 6 10 3 7 11
|
||||
* / \ / \ / \ / \ / \ / \
|
||||
* 1 3 5 7 9 11 2 4 6 8 10 1
|
||||
*/
|
||||
ncclResult_t ncclGetDtree(int nranks, int rank, int* s0, int* d0_0, int* d0_1, int* s1, int* d1_0, int* d1_1) {
|
||||
// First tree ... use a btree
|
||||
ncclGetBtree(nranks, rank, s0, d0_0, d0_1);
|
||||
// Second tree ... mirror or shift
|
||||
if (nranks % 2 == 0) {
|
||||
// shift
|
||||
int shiftrank = (rank-1+nranks) % nranks;
|
||||
int u, d0, d1;
|
||||
ncclGetBtree(nranks, shiftrank, &u, &d0, &d1);
|
||||
*s1 = u == -1 ? -1 : (u+1) % nranks;
|
||||
*d1_0 = d0 == -1 ? -1 : (d0+1) % nranks;
|
||||
*d1_1 = d1 == -1 ? -1 : (d1+1) % nranks;
|
||||
} else {
|
||||
// mirror
|
||||
int u, d0, d1;
|
||||
ncclGetBtree(nranks, nranks-1-rank, &u, &d0, &d1);
|
||||
*s1 = u == -1 ? -1 : nranks-1-u;
|
||||
*d1_0 = d0 == -1 ? -1 : nranks-1-d0;
|
||||
*d1_1 = d1 == -1 ? -1 : nranks-1-d1;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "devcomm.h"
|
||||
#include "comm.h"
|
||||
#include "topo.h"
|
||||
|
||||
NCCL_PARAM(Nthreads, "NTHREADS", -2);
|
||||
NCCL_PARAM(Ll128Nthreads, "LL128_NTHREADS", -2);
|
||||
|
||||
static int getNthreads(const char* name, int env, int min, int max, int def) {
|
||||
int nt = env;
|
||||
if (nt > 0) {
|
||||
if (nt % WARP_SIZE != 0) {
|
||||
WARN("Invalid %s %d (must be a multiple of %d)", name, nt, WARP_SIZE);
|
||||
nt = max;
|
||||
} else if (nt > max) {
|
||||
WARN("Invalid %s %d (maximum %d).", name, nt, max);
|
||||
nt = max;
|
||||
} else if (nt < min) {
|
||||
WARN("Invalid %s %d (minimum %d).", name, nt, min);
|
||||
nt = min;
|
||||
}
|
||||
} else {
|
||||
nt = def;
|
||||
}
|
||||
return nt;
|
||||
}
|
||||
|
||||
ncclResult_t parseList(const char* str, const char* elems[], int nelems, int* list) {
|
||||
int def, set;
|
||||
if (str[0] == '^') {
|
||||
def = 1; set = 0; str++;
|
||||
} else {
|
||||
def = 0; set = 1;
|
||||
}
|
||||
for (int i=0; i<nelems; i++) list[i] = def;
|
||||
char* tokStr = strdup(str);
|
||||
char* tmpStr;
|
||||
char* token = strtok_r(tokStr, ",", &tmpStr);
|
||||
while (token) {
|
||||
for (int i=0; i<nelems; i++)
|
||||
if (strcasecmp(token, elems[i]) == 0) list[i] = set;
|
||||
token = strtok_r(NULL, ",", &tmpStr);
|
||||
}
|
||||
free(tokStr);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static const char* ncclFuncStr[] = { "Broadcast", "Reduce", "AllGather", "ReduceScatter", "AllReduce" };
|
||||
static const char* ncclAlgoStr[] = { "Tree", "Ring" };
|
||||
static const char* ncclProtoStr[] = { "LL", "LL128", "Simple" };
|
||||
|
||||
// Latencies in us, Bandwidths in GB/s
|
||||
// Tree { LL, LL128, Simple } , Ring { LL, LL128, Simple }
|
||||
static const float baseLat [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] = { { 4.4, 4.4, 0 }, { 3.6, 3.6, 8.4 } };
|
||||
|
||||
// NVLink, PCI, Network
|
||||
#define NCCL_HW_NVLINK 0
|
||||
#define NCCL_HW_PCI 1
|
||||
#define NCCL_HW_NET 2
|
||||
// Tree/Simple is the latency a 256kB chunk, which is ~ base lat + 256k/12GB/s (+ 256k/12GB/s for the network).
|
||||
static const float hwLat [3][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] =
|
||||
{ /* NVLINK */
|
||||
{ /* Tree (LL/LL128/Simple)*/ { .5, 1.9, 28 }, /* Ring (LL/LL128/Simple)*/ { .4, 2.5, 5.7 } },
|
||||
/* PCI */
|
||||
{ /* Tree (LL/LL128/Simple)*/ { 1.0, 1.9, 28 }, /* Ring (LL/LL128/Simple)*/ { 1.0, 2.5, 5.7 } },
|
||||
/* NET */
|
||||
{ /* Tree (LL/LL128/Simple)*/ { 5.0, 7.5, 50 }, /* Ring (LL/LL128/Simple)*/ { .9, 2.5, 6.6 } }
|
||||
};
|
||||
|
||||
// LL128 max BW for the different collectives
|
||||
static const double ll128MaxBw[NCCL_NUM_FUNCTIONS] = { 113.0, 72.0, 110.0, 91.0, 100.0 };
|
||||
|
||||
ncclResult_t ncclSetThresholds(struct ncclComm* comm, int minCompCap, int maxCompCap, struct ncclTopoGraph* treeGraph, struct ncclTopoGraph* ringGraph) {
|
||||
int simpleDefaultThreads = (treeGraph->speedIntra*treeGraph->nChannels <= 12) ? 256 : NCCL_MAX_NTHREADS;
|
||||
comm->maxThreads[NCCL_PROTO_SIMPLE] = getNthreads("NCCL_NTHREADS", ncclParamNthreads(), 4*WARP_SIZE, NCCL_MAX_NTHREADS, simpleDefaultThreads);
|
||||
comm->maxThreads[NCCL_PROTO_LL] = getNthreads("NCCL_NTHREADS", ncclParamNthreads(), 4*WARP_SIZE, NCCL_MAX_NTHREADS, NCCL_MAX_NTHREADS);
|
||||
comm->maxThreads[NCCL_PROTO_LL128] = getNthreads("NCCL_LL128_NTHREADS", ncclParamLl128Nthreads(), NCCL_LL128_MAX_NTHREADS/4, NCCL_LL128_MAX_NTHREADS, NCCL_LL128_MAX_NTHREADS);
|
||||
|
||||
INFO(NCCL_INIT, "Threads per block : %d/%d/%d", comm->maxThreads[NCCL_PROTO_LL], comm->maxThreads[NCCL_PROTO_LL128], comm->maxThreads[NCCL_PROTO_SIMPLE]);
|
||||
|
||||
if (comm->nRanks <= 1) return ncclSuccess;
|
||||
|
||||
struct ncclTopoGraph* graphs[2] = { treeGraph, ringGraph };
|
||||
int intraHw[2], hw[2];
|
||||
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) intraHw[a] = graphs[a]->nvlink ? NCCL_HW_NVLINK : NCCL_HW_PCI;
|
||||
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) hw[a] = comm->nNodes == 1 ? intraHw[a] : NCCL_HW_NET;
|
||||
|
||||
for (int coll=0; coll<NCCL_NUM_FUNCTIONS; coll++) {
|
||||
int nsteps = coll == ncclCollAllReduce ? 2*(comm->nRanks-1) :
|
||||
coll == ncclCollReduceScatter || coll == ncclCollAllGather ? comm->nRanks-1 :
|
||||
comm->nRanks;
|
||||
|
||||
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) {
|
||||
if (coll != ncclCollAllReduce && a == NCCL_ALGO_TREE) continue;
|
||||
|
||||
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
||||
int speed = comm->nNodes <= 2 ? graphs[a]->speedIntra : graphs[a]->speedInter;
|
||||
float busBw = graphs[a]->nChannels * speed * 1.0;
|
||||
|
||||
// Various model refinements
|
||||
if (a == NCCL_ALGO_RING && p == NCCL_PROTO_LL) busBw *= 1.0/4.0;
|
||||
if (a == NCCL_ALGO_RING && p == NCCL_PROTO_LL128) busBw = std::min(busBw*120.0/128.0, ll128MaxBw[coll]);
|
||||
if (a == NCCL_ALGO_TREE) busBw = std::min(busBw*.9, comm->nNodes > 1 ? 70.0 : 90.0);
|
||||
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL) busBw *= 1.0/3.0;
|
||||
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL128) busBw *= 7.0/9.0;
|
||||
|
||||
// Convert bus BW to algorithm BW
|
||||
float ratio = a == NCCL_ALGO_TREE ? .5 : (1.0 * comm->nRanks) / nsteps;
|
||||
comm->bandwidths[coll][a][p] = busBw * ratio;
|
||||
|
||||
comm->latencies[coll][a][p] = baseLat[a][p];
|
||||
if (a == NCCL_ALGO_RING) {
|
||||
float lat = hwLat[hw[a]][a][p];
|
||||
if ((coll == ncclCollReduce || coll == ncclCollBroadcast)) {
|
||||
if (ringGraph->sameChannels) {
|
||||
comm->latencies[coll][a][p] += lat;
|
||||
} else {
|
||||
if (p == NCCL_PROTO_SIMPLE) lat = hwLat[hw[a]][NCCL_ALGO_TREE][p]; // Add some chunk latency, waiting for proper chunk modeling
|
||||
comm->latencies[coll][a][p] += nsteps*lat;
|
||||
}
|
||||
} else {
|
||||
comm->latencies[coll][a][p] += nsteps*lat;
|
||||
}
|
||||
} else {
|
||||
float intraLat = hwLat[intraHw[a]][a][p];
|
||||
float interLat = hwLat[NCCL_HW_NET][a][p];
|
||||
comm->latencies[coll][a][p] +=
|
||||
2 * ((comm->nRanks/comm->nNodes-1) * intraLat + log2i(comm->nNodes) * interLat);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Protocols/Algorithms enable/disable, and user overrides.
|
||||
// All are enabled except ll128 which is enabled by default only in certain cases.
|
||||
int protoEnable[NCCL_NUM_PROTOCOLS] = { 1, 2, 1 };
|
||||
int algoEnable[NCCL_NUM_ALGORITHMS] = { 1, 1 };
|
||||
|
||||
const char *protoStr = getenv("NCCL_PROTO");
|
||||
if (protoStr) NCCLCHECK(parseList(protoStr, ncclProtoStr, NCCL_NUM_PROTOCOLS, protoEnable));
|
||||
const char *algoStr = getenv("NCCL_ALGO");
|
||||
if (algoStr) NCCLCHECK(parseList(algoStr, ncclAlgoStr, NCCL_NUM_ALGORITHMS, algoEnable));
|
||||
|
||||
for (int c=0; c<NCCL_NUM_FUNCTIONS; c++) for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
||||
int pEnable = protoEnable[p];
|
||||
if (pEnable == 2 && p == NCCL_PROTO_LL128) {
|
||||
// Enable LL128 by default only on Volta+NVLink. Other cases are not tested and may cause silent data corruption.
|
||||
pEnable = (graphs[a]->type <= LINK_PCI) && graphs[a]->nvlink && minCompCap == 70 && maxCompCap == 70 ? 1 : 0;
|
||||
}
|
||||
if (pEnable == 0 || algoEnable[a] == 0) comm->bandwidths[c][a][p] = 0;
|
||||
}
|
||||
|
||||
if (comm->rank == 0) {
|
||||
char line[1024];
|
||||
int offset = 0;
|
||||
sprintf(line, "Latency/AlgBw |");
|
||||
offset = strlen(line);
|
||||
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) {
|
||||
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
||||
sprintf(line+offset, " %4s/%6s |", ncclAlgoStr[a], ncclProtoStr[p]);
|
||||
offset = strlen(line);
|
||||
}
|
||||
}
|
||||
INFO(NCCL_TUNING, "%s", line);
|
||||
for (int c=0; c<NCCL_NUM_FUNCTIONS; c++) {
|
||||
sprintf(line, "%13s |", ncclFuncStr[c]);
|
||||
offset = strlen(line);
|
||||
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) {
|
||||
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
||||
sprintf(line+offset, "%7.1f/%5.1f|", comm->latencies[c][a][p], comm->bandwidths[c][a][p]);
|
||||
offset = strlen(line);
|
||||
}
|
||||
}
|
||||
INFO(NCCL_TUNING, "%s", line);
|
||||
}
|
||||
}
|
||||
|
||||
// Set per-thread amount of work before we increase nThreads and nChannels
|
||||
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) {
|
||||
comm->threadThresholds[a][NCCL_PROTO_LL] = NCCL_LL_THREAD_THRESHOLD;
|
||||
comm->threadThresholds[a][NCCL_PROTO_LL128] = NCCL_LL128_THREAD_THRESHOLD;
|
||||
comm->threadThresholds[a][NCCL_PROTO_SIMPLE] = NCCL_SIMPLE_THREAD_THRESHOLD;
|
||||
}
|
||||
comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_LL] *= comm->nRanks;
|
||||
|
||||
// Override defaults with user env
|
||||
char* str = getenv("NCCL_THREAD_THRESHOLDS");
|
||||
if (str) {
|
||||
ssize_t t[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] = { -2 };
|
||||
sscanf(str, "%ld %ld %ld %ld %ld %ld", t[0], t[0]+1, t[0]+2, t[1], t[1]+1, t[1]+2);
|
||||
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) {
|
||||
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
||||
if (t[a][p] >= 0) comm->threadThresholds[a][p] = t[a][p];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
INFO(NCCL_INIT, "threadThresholds %ld/%ld/%ld | %ld/%ld/%ld",
|
||||
comm->threadThresholds[NCCL_ALGO_TREE][NCCL_PROTO_LL],
|
||||
comm->threadThresholds[NCCL_ALGO_TREE][NCCL_PROTO_LL128],
|
||||
comm->threadThresholds[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE],
|
||||
comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_LL],
|
||||
comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_LL128],
|
||||
comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE]);
|
||||
return ncclSuccess;
|
||||
}
|
||||
Référencer dans un nouveau ticket
Bloquer un utilisateur