AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable collnet in RCCL (#333)

Browse Source 
* Enable CollNet and use different number of channels

* topo_expl: enable collnet
This commit is contained in:
Wenkai Du
2021-03-19 12:58:13 -07:00
committed by GitHub
parent 244e25d980
commit 1d6244b18d
14 changed files with 393 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/TopoVisual/extract_topo.awk
+109 -2
View File
@@ -1,5 +1,5 @@
#!/usr/bin/gawk -f
# Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved.
# Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -26,8 +26,15 @@ BEGIN {
treedns[""]=0
max_treedn=0
conn[""]=0
has_collnet=0
max_collnet=0
max_collnet_rank=0
collnet[""]=0
collnet_conn[""]=0
collnet_conn_type[""]=0
col_start=2
col_p1=col_start+1
col_p2=col_start+2
col_p3=col_start+3
col_p4=col_start+4
col_p5=col_start+5
@@ -40,6 +47,7 @@ BEGIN {
if($3=="NCCL" && $4=="INFO" && col_start==2) {
col_start=5
col_p1=col_start+1
col_p2=col_start+2
col_p3=col_start+3
col_p4=col_start+4
col_p5=col_start+5
@@ -48,7 +56,19 @@ BEGIN {
col_p8=col_start+8
}
if($col_start=="Ring" && $col_p4="->" && $col_p6=="->") {
if($5=="NCCL" && $6=="INFO" && col_start==2) {
col_start=7
col_p1=col_start+1
col_p2=col_start+2
col_p3=col_start+3
col_p4=col_start+4
col_p5=col_start+5
col_p6=col_start+6
col_p7=col_start+7
col_p8=col_start+8
}
if($col_start=="Ring" && $col_p4=="->" && $col_p6=="->") {
chan=strtonum($col_p1)
rank=strtonum($col_p5)
next_rank=strtonum($col_p7)
@@ -91,6 +111,31 @@ BEGIN {
} while ($col_1!="")
}
if($col_start=="CollNet" && $col_p1=="Channel") {
chan=strtonum($col_p2)
rank=strtonum($col_p4)
up_rank=strtonum($col_p6)
collnet[up_rank "," rank "," chan]="1"
if(has_collnet==0)
has_collnet=1
if(chan>max_collnet)
max_collnet=chan
if(up_rank>max_collnet_rank)
max_collnet_rank=up_rank
}
if($col_start=="Coll" && $col_p2==":") {
chan=strtonum($col_p1)
rank=strtonum($col_p3)
if($col_p4=="[receive]")
collnet_conn[rank "," chan]=0
else if($col_p4=="[send]")
collnet_conn[rank "," chan]=1
else
printf "Error!\n"
collnet_conn_type[rank "," chan]=$col_p6
}
if($col_p6=="via") {
match($col_p1, /([0-9]+)/, ary)
chan=strtonum(ary[1])
@@ -180,5 +225,67 @@ END {
}
printf " }\n\n"
}
for(r=0; has_collnet && r<=max_collnet; r++) {
printf " subgraph collnet_%d {\n", r
num_top_ranks=0
for(s=0;s<max_collnet_rank;s++) {
if((max_collnet_rank "," s "," r) in collnet)
top_ranks[num_top_ranks++]=s
}
for(d=0; d<num_top_ranks; d++) {
rank=top_ranks[d]
send=collnet_conn[rank "," r]
val=collnet_conn_type[rank "," r]
style="solid"
color="red"
if (match(val,"COLLNET")) {
style="dashed"
if (match(val,"GDRDMA"))
color="green"
}
if (match(val,"P2P")) {
color="green"
}
if(send)
printf " c%d_%d -> c%d_%d [dir=back label=\"%s\",color=\"%s\",style=\"%s\",fontname=\"Helvetica\"];\n", r, max_collnet_rank, r, rank, val, color, style
else
printf " c%d_%d -> c%d_%d [label=\"%s\",color=\"%s\",style=\"%s\",fontname=\"Helvetica\"];\n", r, max_collnet_rank, r, rank, val, color, style
while(1) {
for(s=0;s<max_collnet_rank;s++) {
if((rank "," s "," r) in collnet) {
if(send)
val=conn[s "," rank "," r]
else
val=conn[rank "," s "," r]
style="solid"
color="red"
if (match(val,"NET")) {
style="dashed"
if (match(val,"GDRDMA"))
color="green"
}
if (match(val,"P2P")) {
color="green"
}
if(send)
printf " c%d_%d -> c%d_%d [dir=back label=\"%s\",color=\"%s\",style=\"%s\",fontname=\"Helvetica\"];\n", r, rank, r, s, val, color, style
else
printf " c%d_%d -> c%d_%d [label=\"%s\",color=\"%s\",style=\"%s\",fontname=\"Helvetica\"];\n", r, rank, r, s, val, color, style
rank=s
break;
}
}
if(s>=max_collnet_rank) {
break;
}
}
}
printf "\n"
for(s=0;s<=max_collnet_rank;s++) {
printf " c%d_%d [label=\"%d\",fontsize=\"28\"];\n", r, s, s
}
printf " }\n\n"
}
printf "}\n"
}
tools/scripts/topo_val.sh
+7 -2
View File
@@ -21,8 +21,13 @@
DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
for i in {0..49}
for i in {0..50}
do
$DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log"
if [[ $i -eq 50 ]]
then
NCCL_COLLNET_ENABLE=1 $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log"
else
$DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log"
fi
$DIR/../TopoVisual/topo_visual.sh -i "topo_m$i.log"
done
tools/topo_expl/model.cpp
+33 -2
View File
@@ -159,8 +159,6 @@ ncclResult_t netSendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, st
return ncclSuccess;
}
NCCL_PARAM(NetGdrLevel, "NET_GDR_LEVEL", PATH_PHB);
ncclResult_t netRecvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* recv, int channelId) {
int netDev, useGdr = 0;
@@ -179,6 +177,39 @@ struct ncclTransport netTransport = {
{ netRecvSetup, NULL, NULL, NULL }
};
/* Determine if two peers can communicate with NET */
ncclResult_t collNetCanConnect(int* ret, struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2) {
*ret = 1;
return ncclSuccess;
}
ncclResult_t collNetSendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* send, int channelId) {
int netDev, useGdr = 0;
NCCLCHECK(ncclTopoGetNetDev(comm->topo, myInfo->rank, graph, channelId, &netDev));
NCCLCHECK(ncclTopoCheckGdr(comm->topo, myInfo->busId, netDev, 1, &useGdr));
INFO(NCCL_INIT|NCCL_NET,"Coll %02d : %d [send] via COLLNET/%s/%d%s", channelId, myInfo->rank, "SHARP", netDev, useGdr ? "/GDRDMA" : "");
return ncclSuccess;
}
ncclResult_t collNetRecvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* recv, int channelId) {
int netDev, useGdr = 0;
NCCLCHECK(ncclTopoGetNetDev(comm->topo, myInfo->rank, graph, channelId, &netDev));
NCCLCHECK(ncclTopoCheckGdr(comm->topo, myInfo->busId, netDev, 0, &useGdr));
INFO(NCCL_INIT|NCCL_NET,"Coll %02d : %d [receive] via COLLNET/%s/%d%s", channelId, myInfo->rank, "SHARP", netDev, useGdr ? "/GDRDMA" : "");
return ncclSuccess;
}
struct ncclTransport collNetTransport = {
"COL",
collNetCanConnect,
{ collNetSendSetup, NULL, NULL, NULL },
{ collNetRecvSetup, NULL, NULL, NULL }
};
struct ncclTransport ncclTransports[NTRANSPORTS] = {
p2pTransport,
shmTransport,
tools/topo_expl/models/topo_collnet_n1.xml
+87
View File
@@ -0,0 +1,87 @@
<system version="2">
<cpu numaid="0" affinity="00000000,00000000,00ffffff,00000000,00000000,00ffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:61:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:63:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="0" sm="90" gcn="908" arch="38911" rank="0" gdr="1">
<xgmi target="0000:43:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:27:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:03:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:41:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:43:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="1" sm="90" gcn="908" arch="38911" rank="1" gdr="1">
<xgmi target="0000:63:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:27:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:03:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</cpu>
<cpu numaid="1" affinity="00000000,0000ffff,ff000000,00000000,0000ffff,ff000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:25:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:27:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="2" sm="90" gcn="908" arch="38911" rank="2" gdr="1">
<xgmi target="0000:63:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:43:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:03:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:01:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:03:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="3" sm="90" gcn="908" arch="38911" rank="3" gdr="1">
<xgmi target="0000:63:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:43:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:27:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</cpu>
<cpu numaid="2" affinity="000000ff,ffff0000,00000000,000000ff,ffff0000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:e1:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:e3:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="4" sm="90" gcn="908" arch="38911" rank="4" gdr="1">
<xgmi target="0000:c3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:a3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:83:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:c1:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c3:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="5" sm="90" gcn="908" arch="38911" rank="5" gdr="1">
<xgmi target="0000:e3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:a3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:83:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:c4:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_0" dev="0" speed="200000" port="1" guid="0x22fd9f00039b0398" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</cpu>
<cpu numaid="3" affinity="ffffff00,00000000,00000000,ffffff00,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:a1:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:a3:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="6" sm="90" gcn="908" arch="38911" rank="6" gdr="1">
<xgmi target="0000:e3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:c3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:83:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:81:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:83:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="7" sm="90" gcn="908" arch="38911" rank="7" gdr="1">
<xgmi target="0000:e3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:c3:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:a3:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</cpu>
</system>
tools/topo_expl/topo_expl.cpp
+1
View File
@@ -119,6 +119,7 @@ NodeModelDesc model_descs[] = {
{2, "topo_8p_rome_pcie.xml", "2 nodes node 8 VEGA20 PCIe"},
{1, "topo_8p_rome_4nics.xml", "single node 8 gfx908 Rome 4 NICs"},
{4, "topo_8p_rome_4nics.xml", "4 nodes node 8 gfx908 Rome 4 NICs"},
{4, "topo_collnet_n1.xml", "4 nodes collnet 1 NICs"},
};
int main(int argc,char* argv[])
tools/topo_expl/utils.cpp
+101 -4
View File
@@ -41,7 +41,7 @@ NCCL_PARAM(CollNetEnable, "COLLNET_ENABLE", 0);
NCCL_PARAM(GraphDumpFileRank, "GRAPH_DUMP_FILE_RANK", 0);
thread_local int ncclDebugNoWarn = 0;
ncclCollNet_t* ncclCollNet = NULL;
ncclCollNet_t* ncclCollNet = 0;
// Get current Compute Capability
int ncclCudaCompCap() {
@@ -137,10 +137,107 @@ ncclResult_t bootstrapAllGather(struct ncclComm* comm, struct allGather1Data_t *
return ncclSuccess;
}
extern struct ncclTransport collNetTransport;
// All ranks must participate in collNetSetup call
// type: 0 for send, 1 for recv
// return: 0 - unsupported, 1 - supported
// We do not NCCLCHECK this call because we would fall back to P2P network in case CollNet setup fails
static int collNetSetup(struct ncclComm* comm, struct ncclTopoGraph* collNetGraph, struct ncclChannel* channel, int rank, int nranks, int masterRank, int masterPeer, int nMasters, int type) {
int rankInCollNet = -1;
int supported = 0;
int isMaster = (rank == masterRank) ? 1 : 0;
struct {
int collNetRank;
ncclConnect connect;
} sendrecvExchange;
// check if we can connect to collnet, whose root is the nranks-th rank
struct ncclPeerInfo *myInfo = comm->peerInfo+rank, *peerInfo = comm->peerInfo+nranks;
peerInfo->rank = nranks;
int ret = 1;
if (isMaster) {
NCCLCHECK(collNetTransport.canConnect(&ret, comm->topo, collNetGraph, myInfo, peerInfo));
}
// send master receives connect info from peer recv master
if (isMaster && type == 0) {
//NCCLCHECK(bootstrapRecv(comm->bootstrap, masterPeer, &sendrecvExchange, sizeof(sendrecvExchange)));
rankInCollNet = sendrecvExchange.collNetRank;
INFO(NCCL_INIT, "CollNet [send] : rank %d collNetRank %d collNetNranks %d received connect from rank %d", rank, rankInCollNet, nMasters, masterPeer);
}
// select
struct ncclPeer* root = channel->peers+nranks;
struct ncclConnector* conn = (type == 1) ? &root->recv : &root->send;
struct ncclTransportComm* transportComm = (type == 1) ? &(collNetTransport.recv) : &(collNetTransport.send);
conn->transportComm = transportComm;
// setup
struct ncclConnect myConnect;
if (isMaster && ret > 0) {
NCCLCHECK(transportComm->setup(comm, collNetGraph, myInfo, peerInfo, &myConnect, conn, channel->id));
}
// prepare connect handles
ncclResult_t res;
struct {
int isMaster;
ncclConnect connect;
} *allConnects = NULL;
ncclConnect *masterConnects = NULL;
//NCCLCHECK(ncclCalloc(&masterConnects, nMasters));
if (type == 1) { // recv side: AllGather
// all ranks must participate
//NCCLCHECK(ncclCalloc(&allConnects, nranks));
//allConnects[rank].isMaster = isMaster;
//memcpy(&(allConnects[rank].connect), &myConnect, sizeof(struct ncclConnect));
//NCCLCHECKGOTO(bootstrapAllGather(comm->bootstrap, allConnects, sizeof(*allConnects)), res, cleanup);
// consolidate
//int c = 0;
//for (int r = 0; r < nranks; r++) {
//if (allConnects[r].isMaster) {
//memcpy(masterConnects+c, &(allConnects[r].connect), sizeof(struct ncclConnect));
//if (r == rank) rankInCollNet = c;
//c++;
//}
//}
} else { // send side : copy in connect info received from peer recv master
//if (isMaster) memcpy(masterConnects+rankInCollNet, &(sendrecvExchange.connect), sizeof(struct ncclConnect));
}
// connect
if (isMaster && ret > 0) {
//NCCLCHECKGOTO(transportComm->connect(comm, masterConnects, nMasters, rankInCollNet, conn), res, cleanup);
//struct ncclPeer* devRoot = channel->devPeers+nranks;
//struct ncclConnector* devConn = (type == 1) ? &devRoot->recv : &devRoot->send;
//CUDACHECKGOTO(hipMemcpy(devConn, conn, sizeof(struct ncclConnector), hipMemcpyHostToDevice), res, cleanup);
}
// recv side sends connect info to send side
if (isMaster && type == 1) {
//sendrecvExchange.collNetRank = rankInCollNet;
//memcpy(&sendrecvExchange.connect, masterConnects+rankInCollNet, sizeof(struct ncclConnect));
//NCCLCHECKGOTO(bootstrapSend(comm->bootstrap, masterPeer, &sendrecvExchange, sizeof(sendrecvExchange)), res, cleanup);
INFO(NCCL_INIT, "CollNet [recv] : rank %d collNetRank %d collNetNranks %d sent connect to rank %d", rank, rankInCollNet, nMasters, masterPeer);
}
if (ret > 0) {
supported = 1;
}
cleanup:
//if (allConnects != NULL) free(allConnects);
//if (masterConnects != NULL) free(masterConnects);
return supported;
}
static ncclResult_t checkCollNetSetup(struct ncclComm* comm, int rank, int collNetSetupFail) {
comm->collNetSupport = 1;
return ncclSuccess;
}
ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t *allGather1Data, struct allGather3Data_t *allGather3Data,
struct ncclTopoGraph& treeGraph, struct ncclTopoGraph& ringGraph, struct ncclTopoGraph& collNetGraph) {
int rank = comm->rank;
int nranks = comm->nRanks;
if (ncclParamCollNetEnable() == 1 && ncclCollNet == 0)
ncclCollNet = (ncclCollNet_t*)0x12345678;
//uint64_t commHash = getHash(commId->internal, NCCL_UNIQUE_ID_BYTES);
//TRACE(NCCL_INIT, "comm %p, commHash %lx, rank %d nranks %d - BEGIN", comm, commHash, rank, nranks);
//NCCLCHECK(bootstrapInit(commId, rank, nranks, &comm->bootstrap));
@@ -527,6 +624,8 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
// Force 2 channels for CollNet
comm->collNetnChannels = collNetGraph.nChannels = 2;
NCCLCHECK(ncclTopoConnectCollNet(comm, &collNetGraph, rank));
}
@@ -580,11 +679,10 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
INFO(NCCL_INIT, "Connected all trees");
// Check if we can setup CollNet
#if 0
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
int logicChannels = comm->nChannels/2;
int logicChannels = comm->collNetnChannels/2;
int collNetSetupFail = 0;
const int recvIndex = 0; // recv GPU index is always 0
const int sendIndex = collNetGraph.pattern == NCCL_TOPO_PATTERN_TREE ? 0 : 1; // send GPU index depends on topo pattern
@@ -604,7 +702,6 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
// Verify CollNet setup across ranks
NCCLCHECK(checkCollNetSetup(comm, rank, collNetSetupFail));
}
#endif
TRACE(NCCL_INIT, "rank %d nranks %d - CONNECTED %d RINGS AND TREES", rank, nranks, comm->nChannels);
free(rings);