Merge remote-tracking branch 'nccl/master' into develop
Αυτή η υποβολή περιλαμβάνεται σε:
Wenkai Du
+79
-44
@@ -1,6 +1,6 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved.
|
||||
* Copyright (c) 2016-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
@@ -429,9 +429,67 @@ ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopo
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Select only NICs with the maximum bandwidth w.r.t. GPUs, and sort them by distance.
|
||||
ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int* nets, int* netcountRet) {
|
||||
float* maxwidths;
|
||||
int* minhops;
|
||||
int netcount = 0;
|
||||
NCCLCHECK(ncclCalloc(&minhops, system->nodes[NET].count));
|
||||
NCCLCHECK(ncclCalloc(&maxwidths, system->nodes[NET].count));
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
maxwidths[n] = 0.0;
|
||||
minhops[n] = 255;
|
||||
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
|
||||
struct ncclTopoLinkList* paths = net->paths[GPU];
|
||||
for (int g=0; g<system->nodes[GPU].count; g++) {
|
||||
if (paths[g].width > maxwidths[n] || (paths[g].width == maxwidths[n] && paths[g].count < minhops[n])) {
|
||||
maxwidths[n] = paths[g].width;
|
||||
minhops[n] = paths[g].count;
|
||||
}
|
||||
}
|
||||
if (netcount && maxwidths[nets[0]] > maxwidths[n]) continue; // Do not keep NICs with lower BW
|
||||
if (netcount && maxwidths[nets[0]] < maxwidths[n]) netcount = 0; // Remove all NICs with lower BW
|
||||
int index;
|
||||
for (index = 0; index < netcount; index++) {
|
||||
if (minhops[n] < minhops[nets[index]]) break;
|
||||
}
|
||||
// Insert net at index
|
||||
// Shift all nets with higher nhops
|
||||
for (int i = netcount; i>index; i--) nets[i] = nets[i-1];
|
||||
// Insert this net at index
|
||||
nets[index] = n;
|
||||
netcount++;
|
||||
}
|
||||
|
||||
*netcountRet = netcount;
|
||||
|
||||
// Then shuffle NICs with the same nhops based on the GPU device number, so that when we have
|
||||
// 2 NICs and 2 GPUs and create communicators with only one GPU, we will use both NICs.
|
||||
for (int start = 0; start < netcount;) {
|
||||
int end = start+1;
|
||||
while (end < netcount && minhops[nets[end]] == minhops[nets[start]]) end++;
|
||||
// Shuffle
|
||||
for (int r=0; r<system->nodes[GPU].nodes[0].gpu.dev % (end-start); r++) {
|
||||
int netStart = nets[start];
|
||||
for (int i=start; i<end-1; i++) nets[i] = nets[i+1];
|
||||
nets[end-1] = netStart;
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
|
||||
free(minhops);
|
||||
free(maxwidths);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int backToNet, int backToFirstRank, int* time) {
|
||||
const int speed = graph->speedInter;
|
||||
for (int n=0; n<system->nodes[NET].count; n++) {
|
||||
int* nets;
|
||||
NCCLCHECK(ncclCalloc(&nets, system->nodes[NET].count));
|
||||
int netcount;
|
||||
NCCLCHECK(ncclTopoSelectNets(system, nets, &netcount));
|
||||
for (int i=0; i<netcount; i++) {
|
||||
int n = nets[i];
|
||||
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
|
||||
struct ncclTopoNode* gpu;
|
||||
if (graph->collNet && net->net.collSupport == 0) continue;
|
||||
@@ -457,10 +515,19 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
|
||||
if (graph->nChannels == 0) {
|
||||
// Always try the PCI order first to set a reference, but don't count in the timeout nor let it run for long
|
||||
struct ncclTopoLinkList* paths = net->paths[GPU];
|
||||
int f = 0, f_gdr = 0;
|
||||
// find the first GPU that is closest to NIC
|
||||
int f = 0;
|
||||
for (int i = 0; i<system->nodes[GPU].count; i++)
|
||||
if (paths[i].count < paths[f].count) f = i;
|
||||
for (int i = 0; i<system->nodes[GPU].count; i++) {
|
||||
if (paths[i].count <= paths[f].count) {
|
||||
// prefer GPU direct RDMA
|
||||
int gdr;
|
||||
NCCLCHECK(ncclTopoCheckGdr(system, system->nodes[GPU].nodes[i].id, net->id, 0, &gdr));
|
||||
if (paths[i].count < paths[f].count || (paths[i].count == paths[f].count && !f_gdr && gdr)) {
|
||||
f = i;
|
||||
f_gdr = gdr;
|
||||
}
|
||||
}
|
||||
}
|
||||
int t = 1 << 10;
|
||||
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, (f == 0) ? FORCED_ORDER_PCI : 0, &t, NET, n, f));
|
||||
if (t == -1) *time = -1;
|
||||
@@ -504,6 +571,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
|
||||
}
|
||||
}
|
||||
}
|
||||
free(nets);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -700,7 +768,6 @@ RCCL_PARAM(ModelMatchingDisable, "MODEL_MATCHING_DISABLE", 0);
|
||||
|
||||
ncclResult_t ncclTopoCompute(ncclTopoSystem* system, struct ncclTopoGraph* graph) {
|
||||
int ngpus = system->nodes[GPU].count;
|
||||
int nnets = system->nodes[NET].count;
|
||||
int crossNic = (system->nodes[NET].count > 1) && graph->crossNic ? 1 : 0;
|
||||
graph->speedIntra = graph->speedInter = 0;
|
||||
if (graph->crossNic == 2) graph->crossNic = 0;
|
||||
@@ -729,41 +796,13 @@ ncclResult_t ncclTopoCompute(ncclTopoSystem* system, struct ncclTopoGraph* graph
|
||||
if (graph->nChannels) {
|
||||
system->type |= RCCL_TOPO_4P2H_ROME;
|
||||
}
|
||||
} else if (!rcclParamModelMatchingDisable()) {
|
||||
} else if (!rcclParamModelMatchingDisable() && !graph->collNet) {
|
||||
// try to match 8P6L
|
||||
NCCLCHECK(parseChordalRing(system, graph));
|
||||
if (graph->nChannels) return ncclSuccess;
|
||||
// try to match Rome 4P2H
|
||||
NCCLCHECK(parseRome4P2H(system, graph));
|
||||
}
|
||||
if (graph->collNet && graph->nChannels) {
|
||||
struct ncclTopoGraph tmpGraph;
|
||||
memcpy(&tmpGraph, graph, sizeof(struct ncclTopoGraph));
|
||||
int nets[MAXCHANNELS], n = 0;
|
||||
for (int i = 0; i < tmpGraph.nChannels; i++) {
|
||||
int j;
|
||||
for (j = 0; j < n; j++) {
|
||||
if (nets[j] == tmpGraph.inter[i*2])
|
||||
break;
|
||||
}
|
||||
if (j >= n)
|
||||
nets[n++] = tmpGraph.inter[i*2];
|
||||
}
|
||||
for (int i = 0; i < n; i++) {
|
||||
int j;
|
||||
for (j = 0; j < tmpGraph.nChannels; j++) {
|
||||
if (nets[i] == tmpGraph.inter[j*2])
|
||||
break;
|
||||
}
|
||||
if (j < tmpGraph.nChannels) {
|
||||
memcpy(graph->intra+i*ngpus, &tmpGraph.intra[j*ngpus], ngpus*sizeof(int));
|
||||
memcpy(graph->inter+i*2, &tmpGraph.inter[j*2], 2*sizeof(int));
|
||||
}
|
||||
}
|
||||
memcpy(graph->intra+n*ngpus, graph->intra, ngpus*sizeof(int)*n);
|
||||
memcpy(graph->inter+n*2, graph->inter, 2*sizeof(int)*n);
|
||||
graph->nChannels = n;
|
||||
}
|
||||
if (graph->nChannels) return ncclSuccess;
|
||||
|
||||
if ((graph->pattern == NCCL_TOPO_PATTERN_RING) && (system->type & RCCL_TOPO_4P2H_ROME) && (ngpus == system->nRanks)) {
|
||||
@@ -806,6 +845,7 @@ search:
|
||||
for (int g=0; g<ngpus; g++) {
|
||||
printf("%d ", graph->intra[c*ngpus+g]);
|
||||
}
|
||||
printf("[%d %d]", graph->inter[0], graph->inter[1]);
|
||||
printf("\n");
|
||||
}
|
||||
#endif
|
||||
@@ -897,12 +937,7 @@ done:
|
||||
graph->nChannels = 1;
|
||||
}
|
||||
|
||||
if (graph->nChannels && graph->collNet) {
|
||||
// duplicate collnet channels
|
||||
memcpy(graph->intra+graph->nChannels*ngpus, graph->intra, ngpus*sizeof(int)*graph->nChannels);
|
||||
memcpy(graph->inter+graph->nChannels*2, graph->inter, 2*sizeof(int)*graph->nChannels);
|
||||
}
|
||||
else if (graph->speedIntra >= 25.0) {
|
||||
if (graph->speedIntra >= 25.0) {
|
||||
int dupChannels = std::min(graph->nChannels*2, graph->maxChannels);
|
||||
memcpy(graph->intra+graph->nChannels*ngpus, graph->intra, (dupChannels-graph->nChannels)*ngpus*sizeof(int));
|
||||
memcpy(graph->inter+graph->nChannels*2,graph->inter, (dupChannels-graph->nChannels)*2*sizeof(int));
|
||||
@@ -951,7 +986,7 @@ ncclResult_t ncclTopoDumpGraphs(struct ncclTopoSystem* system, int ngraphs, stru
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoGetNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int* dev) {
|
||||
ncclResult_t ncclTopoGetNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int rr, int* dev) {
|
||||
if (graph) {
|
||||
// Honor the net device in the graph
|
||||
int channel = channelId%graph->nChannels;
|
||||
@@ -960,7 +995,7 @@ ncclResult_t ncclTopoGetNetDev(struct ncclTopoSystem* system, int rank, struct n
|
||||
*dev = graph->inter[channel*2+index];
|
||||
} else {
|
||||
int64_t id;
|
||||
NCCLCHECK(ncclTopoGetLocalNet(system, rank, &id, channelId));
|
||||
NCCLCHECK(ncclTopoGetLocalNet(system, rank, &id, rr));
|
||||
*dev = id;
|
||||
}
|
||||
return ncclSuccess;
|
||||
|
||||
Αναφορά σε νέο ζήτημα
Block a user