AI/rocm-systems
2
0
انشعاب 0
کد مسائل تقاضاهای واکشی Actions Packages پروژه‌ها انتشارها دانشنامه فعالیت

collnet: support multiple NICs (#335)

فهرست منبع
This commit is contained in:
Wenkai Du
2021-03-25 20:59:32 -07:00
کامیت شده توسط GitHub
والد 69c0b44553
کامیت d87dc7c2e8
10فایلهای تغییر یافته به همراه209 افزوده شده و 36 حذف شده
دانلود پرونده وصله دانلود فایل تغییرات diff Expand all files Collapse all files
src/enqueue.cc
+2 -1
مشاهده پرونده
@@ -283,10 +283,11 @@ static ncclResult_t getAlgoInfo(struct ncclInfo* info) {
info->algorithm = -1;
info->protocol = -1;
int nAlgos = NCCL_NUM_ALGORITHMS;
#define SHARP_COLL_SAT_THRESHOLD 16384
// Check collNet support
int collNetTypeSupport = 0;
if (info->comm->collNetSupport)
if (info->comm->collNetSupport && info->nBytes < SHARP_COLL_SAT_THRESHOLD*comm->collNetnChannels/2)
NCCLCHECK(collNetReduceSupport(info->datatype, info->op, &collNetTypeSupport));
if (collNetTypeSupport != 1) nAlgos--;
for (int a=0; a<nAlgos; a++) {
src/graph/connect.cc
+18 -11
مشاهده پرونده
@@ -20,19 +20,15 @@ ncclResult_t ncclTopoPreset(struct ncclComm* comm,
struct ncclTopoRanks* topoRanks) {
int rank = comm->rank;
int localRanks = comm->localRanks;
int nChannels = comm->nChannels;
for (int c=0; c<nChannels; c++) {
for (int c=0; c<comm->nChannels; c++) {
struct ncclChannel* channel = comm->channels+c;
channel->ring.prev = channel->ring.next = -1;
channel->tree.up = -1;
for (int i=0; i<NCCL_MAX_TREE_ARITY; i++) channel->tree.down[i] = -1;
channel->collTree.up = -1;
for (int i=0; i<NCCL_MAX_TREE_ARITY; i++) channel->collTree.down[i] = -1;
int* ringIntra = ringGraph->intra+c*localRanks;
int* treeIntra = treeGraph->intra+c*localRanks;
int* collNetIntra = collNetGraph->intra+c*localRanks;
for (int i=0; i<localRanks; i++) {
if (ringIntra[i] == rank) {
@@ -52,6 +48,23 @@ ncclResult_t ncclTopoPreset(struct ncclComm* comm,
channel->tree.up = i == 0 ? -1 : treeIntra[i-1];
channel->tree.down[0] = i == localRanks-1 ? -1 : treeIntra[i+1];
}
}
topoRanks->ringPrev[c] = channel->ring.prev;
topoRanks->ringNext[c] = channel->ring.next;
}
// Duplicate channels rings/trees
struct ncclChannel* channel0 = comm->channels;
struct ncclChannel* channel1 = channel0+comm->nChannels;
memcpy(channel1, channel0, comm->nChannels*sizeof(struct ncclChannel));
// Setup collnet tree
for (int c=0; c<comm->collNetnChannels; c++) {
struct ncclChannel* channel = comm->channels+c;
channel->collTree.up = -1;
for (int i=0; i<NCCL_MAX_TREE_ARITY; i++) channel->collTree.down[i] = -1;
int* collNetIntra = collNetGraph->intra+c*localRanks;
for (int i=0; i<localRanks; i++) {
if (collNetIntra[i] == rank) {
int prev = (i-1+localRanks)%localRanks, next = (i+1)%localRanks;
@@ -59,13 +72,7 @@ ncclResult_t ncclTopoPreset(struct ncclComm* comm,
channel->collTree.down[0] = collNetIntra[next];
}
}
topoRanks->ringPrev[c] = channel->ring.prev;
topoRanks->ringNext[c] = channel->ring.next;
}
// Duplicate channels rings/trees
struct ncclChannel* channel0 = comm->channels;
struct ncclChannel* channel1 = channel0+nChannels;
memcpy(channel1, channel0, nChannels*sizeof(struct ncclChannel));
return ncclSuccess;
}
src/graph/search.cc
+32 -4
مشاهده پرونده
@@ -1153,9 +1153,32 @@ ncclResult_t ncclTopoCompute(ncclTopoSystem* system, struct ncclTopoGraph* graph
NCCLCHECK(parseRome4P2H(system, graph));
}
if (graph->collNet && graph->nChannels) {
graph->nChannels = 1;
memcpy(graph->intra+graph->nChannels*ngpus, graph->intra, ngpus*sizeof(int));
memcpy(graph->inter+graph->nChannels*2, graph->inter, 2*sizeof(int));
struct ncclTopoGraph tmpGraph;
memcpy(&tmpGraph, graph, sizeof(struct ncclTopoGraph));
int nets[MAXCHANNELS], n = 0;
for (int i = 0; i < tmpGraph.nChannels; i++) {
int j;
for (j = 0; j < n; j++) {
if (nets[j] == tmpGraph.inter[i*2])
break;
}
if (j >= n)
nets[n++] = tmpGraph.inter[i*2];
}
for (int i = 0; i < n; i++) {
int j;
for (j = 0; j < tmpGraph.nChannels; j++) {
if (nets[i] == tmpGraph.inter[j*2])
break;
}
if (j < tmpGraph.nChannels) {
memcpy(graph->intra+i*ngpus, &tmpGraph.intra[j*ngpus], ngpus*sizeof(int));
memcpy(graph->inter+i*2, &tmpGraph.inter[j*2], 2*sizeof(int));
}
}
memcpy(graph->intra+n*ngpus, graph->intra, ngpus*sizeof(int)*n);
memcpy(graph->inter+n*2, graph->inter, 2*sizeof(int)*n);
graph->nChannels = n;
}
if (graph->nChannels) return ncclSuccess;
@@ -1286,7 +1309,12 @@ done:
graph->nChannels = 1;
}
if (graph->speedIntra >= 25.0) {
if (graph->nChannels && graph->collNet) {
// duplicate collnet channels
memcpy(graph->intra+graph->nChannels*ngpus, graph->intra, ngpus*sizeof(int)*graph->nChannels);
memcpy(graph->inter+graph->nChannels*2, graph->inter, 2*sizeof(int)*graph->nChannels);
}
else if (graph->speedIntra >= 25.0) {
int dupChannels = std::min(graph->nChannels*2, graph->maxChannels);
memcpy(graph->intra+graph->nChannels*ngpus, graph->intra, (dupChannels-graph->nChannels)*ngpus*sizeof(int));
memcpy(graph->inter+graph->nChannels*2,graph->inter, (dupChannels-graph->nChannels)*2*sizeof(int));
src/graph/tuning.cc
+4 -4
مشاهده پرونده
@@ -63,11 +63,11 @@ static const float baseLat [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] = { { 39.0
// Tree/Simple is the latency a 256kB chunk, which is ~ base lat + 256k/12GB/s (+ 256k/12GB/s for the network).
static const float hwLat [3][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] =
{ /* NVLINK */
{ /* Tree (LL/LL128/Simple)*/ { 2.5, 2.5, 5.5 }, /* Ring (LL/LL128/Simple)*/ { 2.5, 2.5, 5 }, /* CollNet (LL/LL128/Simple)*/ { 1.2, 1.2, 3.8 } },
{ /* Tree (LL/LL128/Simple)*/ { 2.5, 2.5, 5.5 }, /* Ring (LL/LL128/Simple)*/ { 2.5, 2.5, 5 }, /* CollNet (LL/LL128/Simple)*/ { 1.1, 1.1, 3.3 } },
/* PCI */
{ /* Tree (LL/LL128/Simple)*/ { 2.2, 2.2, 5.7 }, /* Ring (LL/LL128/Simple)*/ { 1.3, 1.3, 1.9 }, /* CollNet (LL/LL128/Simple)*/ { 2.2, 2.2, 5.7 } },
{ /* Tree (LL/LL128/Simple)*/ { 2.2, 2.2, 5.7 }, /* Ring (LL/LL128/Simple)*/ { 1.3, 1.3, 1.9 }, /* CollNet (LL/LL128/Simple)*/ { 1.1, 1.1, 1.7 } },
/* NET */
{ /* Tree (LL/LL128/Simple)*/ { 28.0, 28.0, 66.0 }, /* Ring (LL/LL128/Simple)*/ { 8.5, 8.5, 19.0 }, /* CollNet (LL/LL128/Simple)*/ { 9.8, 9.8, 19.5 } }
{ /* Tree (LL/LL128/Simple)*/ { 28.0, 28.0, 66.0 }, /* Ring (LL/LL128/Simple)*/ { 8.5, 8.5, 19.0 }, /* CollNet (LL/LL128/Simple)*/ { 6.5, 6.5, 14.5 } }
};
// LL128 max BW (per channel) for the different collectives
@@ -148,7 +148,7 @@ ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCom
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL128) busBw = std::min(busBw * (nNodes == 1 ? 7.0/9.0 : 0.915 /*120.0/128.0*/), ll128MaxBwPerCh[coll]*graphs[a]->nChannels);
#endif
if (a == NCCL_ALGO_COLLNET) busBw *= .9;
if (a == NCCL_ALGO_COLLNET && p == NCCL_PROTO_LL) busBw *= 1.0/6.0; // Take into account that GDR read is disabled on both sides
if (a == NCCL_ALGO_COLLNET && p == NCCL_PROTO_LL) busBw *= 1.0/2.0; // Take into account that GDR read is disabled on both sides
if (a == NCCL_ALGO_COLLNET && p == NCCL_PROTO_LL128) busBw = 0; // CollNet does not support LL128
// Convert bus BW to algorithm BW
src/init.cc
+6 -2
مشاهده پرونده
@@ -958,6 +958,8 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
allGather3Data[rank].collNet.typeIntra = collNetGraph.typeIntra;
allGather3Data[rank].collNet.typeInter = collNetGraph.typeInter;
// CollNet channels are already duplicated
comm->collNetnChannels = 2*collNetGraph.nChannels;
NCCLCHECK(ncclTopoPreset(comm, &treeGraph, &ringGraph, &collNetGraph, &allGather3Data[rank].topoRanks));
NCCLCHECK(bootstrapAllGather(comm->bootstrap, allGather3Data, sizeof(*allGather3Data)));
@@ -1035,9 +1037,9 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
// Force 2 channels for CollNet
comm->collNetnChannels = collNetGraph.nChannels = 2;
NCCLCHECK(ncclTopoConnectCollNet(comm, &collNetGraph, rank));
} else {
comm->collNetnChannels = 0;
}
free(allTopoRanks);
@@ -1094,6 +1096,8 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
for (int c=comm->nChannels; c<comm->collNetnChannels; c++)
NCCLCHECK(initChannel(comm, c));;
int logicChannels = comm->collNetnChannels/2;
int collNetSetupFail = 0;
const int recvIndex = 0; // recv GPU index is always 0
tools/scripts/topo_val.sh
+2 -2
مشاهده پرونده
@@ -21,9 +21,9 @@
DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
for i in {0..50}
for i in {0..51}
do
if [[ $i -eq 50 ]]
if [[ $i -eq 50 ]] || [[ $i -eq 51 ]]
then
NCCL_COLLNET_ENABLE=1 $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log"
else
tools/topo_expl/include/utils.h
+2
مشاهده پرونده
@@ -36,6 +36,8 @@ struct allGather3Data_t{
struct ncclTopoRanks topoRanks;
};
void initCollNet();
ncclResult_t ncclTopoGetSystem(const char* xmlTopoFile, struct ncclTopoSystem** system);
ncclResult_t ncclTopoGetSystemFromXml(struct ncclXml* xml, struct ncclTopoSystem** topoSystem);
tools/topo_expl/models/topo_collnet_n4.xml
+126
مشاهده پرونده
@@ -0,0 +1,126 @@
<system version="2">
<cpu numaid="0" affinity="00000000,00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:43:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:46:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:48:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="0" sm="90" gcn="908" arch="38911" rank="0" gdr="1">
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:4c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:50:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="1" sm="90" gcn="908" arch="38911" rank="1" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:45:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_1" dev="0" speed="200000" port="1" guid="0x48b9170003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="1" affinity="00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:03:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:05:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:08:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="2" sm="90" gcn="908" arch="38911" rank="2" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:0b:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0d:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0f:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="3" sm="90" gcn="908" arch="38911" rank="3" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:13:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_3" dev="1" speed="200000" port="1" guid="0x18604a0003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="2" affinity="00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:c4:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c7:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c9:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cb:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="4" sm="90" gcn="908" arch="38911" rank="4" gdr="1">
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:cd:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cf:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:d1:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="5" sm="90" gcn="908" arch="38911" rank="5" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:c6:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_5" dev="2" speed="200000" port="1" guid="0xd0b9170003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="3" affinity="ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:83:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:86:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:88:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="6" sm="90" gcn="908" arch="38911" rank="6" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:8c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:90:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="7" sm="90" gcn="908" arch="38911" rank="7" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:85:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_7" dev="3" speed="200000" port="1" guid="0xd0bd170003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
</system>
tools/topo_expl/topo_expl.cpp
+6 -8
مشاهده پرونده
@@ -120,6 +120,7 @@ NodeModelDesc model_descs[] = {
{1, "topo_8p_rome_4nics.xml", "single node 8 gfx908 Rome 4 NICs"},
{4, "topo_8p_rome_4nics.xml", "4 nodes node 8 gfx908 Rome 4 NICs"},
{4, "topo_collnet_n1.xml", "4 nodes collnet 1 NICs"},
{4, "topo_collnet_n4.xml", "4 nodes collnet 4 NICs"},
};
int main(int argc,char* argv[])
@@ -148,6 +149,8 @@ int main(int argc,char* argv[])
NetworkModel network;
NodeModel* node;
initCollNet();
NodeModelDesc *desc = &model_descs[model_id];
for (int i=0; i<desc->num_nodes; i++) {
node = new NodeModel(desc->filename);
@@ -190,14 +193,9 @@ int main(int argc,char* argv[])
}
struct ncclTopoGraph *treeGraph, *ringGraph, *collNetGraph;
treeGraph = (struct ncclTopoGraph *)malloc(sizeof(struct ncclTopoGraph)*nranks);
ringGraph = (struct ncclTopoGraph *)malloc(sizeof(struct ncclTopoGraph)*nranks);
collNetGraph = (struct ncclTopoGraph *)malloc(sizeof(struct ncclTopoGraph)*nranks);
if (!treeGraph || !ringGraph || !collNetGraph) {
printf("Failed to allocate memory for graphs\n");
return -1;
}
NCCLCHECK(ncclCalloc(&treeGraph, nranks));
NCCLCHECK(ncclCalloc(&ringGraph, nranks));
NCCLCHECK(ncclCalloc(&collNetGraph, nranks));
for (int i = 0; i < nranks; i++) {
node_model = network.GetNode(i);
assert(node_model!=0);
tools/topo_expl/utils.cpp
+11 -4
مشاهده پرونده
@@ -231,13 +231,16 @@ static ncclResult_t checkCollNetSetup(struct ncclComm* comm, int rank, int collN
return ncclSuccess;
}
void initCollNet() {
if (ncclParamCollNetEnable() == 1 && ncclCollNet == 0)
ncclCollNet = (ncclCollNet_t*)0x12345678;
}
ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t *allGather1Data, struct allGather3Data_t *allGather3Data,
struct ncclTopoGraph& treeGraph, struct ncclTopoGraph& ringGraph, struct ncclTopoGraph& collNetGraph) {
int rank = comm->rank;
int nranks = comm->nRanks;
if (ncclParamCollNetEnable() == 1 && ncclCollNet == 0)
ncclCollNet = (ncclCollNet_t*)0x12345678;
//uint64_t commHash = getHash(commId->internal, NCCL_UNIQUE_ID_BYTES);
//TRACE(NCCL_INIT, "comm %p, commHash %lx, rank %d nranks %d - BEGIN", comm, commHash, rank, nranks);
//NCCLCHECK(bootstrapInit(commId, rank, nranks, &comm->bootstrap));
@@ -401,6 +404,8 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
allGather3Data[rank].collNet.typeIntra = collNetGraph.typeIntra;
allGather3Data[rank].collNet.typeInter = collNetGraph.typeInter;
// CollNet channels are already duplicated
comm->collNetnChannels = 2*collNetGraph.nChannels;
NCCLCHECK(ncclTopoPreset(comm, &treeGraph, &ringGraph, &collNetGraph, &allGather3Data[rank].topoRanks));
return ncclSuccess;
@@ -624,9 +629,9 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
// Force 2 channels for CollNet
comm->collNetnChannels = collNetGraph.nChannels = 2;
NCCLCHECK(ncclTopoConnectCollNet(comm, &collNetGraph, rank));
} else {
comm->collNetnChannels = 0;
}
free(allTopoRanks);
@@ -682,6 +687,8 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
for (int c=comm->nChannels; c<comm->collNetnChannels; c++)
NCCLCHECK(initChannel(comm, c));;
int logicChannels = comm->collNetnChannels/2;
int collNetSetupFail = 0;
const int recvIndex = 0; // recv GPU index is always 0