AI/rocm-systems
2
0 Fork 0
Code Issues Pull-Requests Actions Pakete Projekte Releases Wiki Aktivität

collnet: support multiple NICs (#335)

Quellcode durchsuchen
Dieser Commit ist enthalten in:
Wenkai Du verfasst und GitHub committed 2021-03-25 20:59:32 -07:00
1 Ursprung 69c0b44553
Commit d87dc7c2e8
10 Dateien geändert
+209 -36
Patch-Datei herunterladen Vergleichs-Datei herunterladen Alle Dateien ausklappen Alle Dateien einklappen

Keine Dateien entsprechen Ihrer Suche

tools/scripts/topo_val.sh
+2 -2
Datei anzeigen
@@ -21,9 +21,9 @@
DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
for i in {0..50}
for i in {0..51}
do
if [[ $i -eq 50 ]]
if [[ $i -eq 50 ]] || [[ $i -eq 51 ]]
then
NCCL_COLLNET_ENABLE=1 $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log"
else
tools/topo_expl/include/utils.h
+2
Datei anzeigen
@@ -36,6 +36,8 @@ struct allGather3Data_t{
struct ncclTopoRanks topoRanks;
};
void initCollNet();
ncclResult_t ncclTopoGetSystem(const char* xmlTopoFile, struct ncclTopoSystem** system);
ncclResult_t ncclTopoGetSystemFromXml(struct ncclXml* xml, struct ncclTopoSystem** topoSystem);
tools/topo_expl/models/topo_collnet_n4.xml
+126
Datei anzeigen
@@ -0,0 +1,126 @@
<system version="2">
<cpu numaid="0" affinity="00000000,00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:43:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:46:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:48:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="0" sm="90" gcn="908" arch="38911" rank="0" gdr="1">
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:4c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:50:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="1" sm="90" gcn="908" arch="38911" rank="1" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:45:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_1" dev="0" speed="200000" port="1" guid="0x48b9170003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="1" affinity="00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:03:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:05:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:08:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="2" sm="90" gcn="908" arch="38911" rank="2" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:0b:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0d:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0f:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="3" sm="90" gcn="908" arch="38911" rank="3" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:13:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_3" dev="1" speed="200000" port="1" guid="0x18604a0003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="2" affinity="00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:c4:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c7:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c9:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cb:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="4" sm="90" gcn="908" arch="38911" rank="4" gdr="1">
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:cd:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cf:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:d1:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="5" sm="90" gcn="908" arch="38911" rank="5" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:c6:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_5" dev="2" speed="200000" port="1" guid="0xd0b9170003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="3" affinity="ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:83:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:86:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:88:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="6" sm="90" gcn="908" arch="38911" rank="6" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:8c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:90:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="7" sm="90" gcn="908" arch="38911" rank="7" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:85:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_7" dev="3" speed="200000" port="1" guid="0xd0bd170003a1420c" maxconn="1" gdr="1" coll="1"/>
</nic>
</pci>
</pci>
</cpu>
</system>
tools/topo_expl/topo_expl.cpp
+6 -8
Datei anzeigen
@@ -120,6 +120,7 @@ NodeModelDesc model_descs[] = {
{1, "topo_8p_rome_4nics.xml", "single node 8 gfx908 Rome 4 NICs"},
{4, "topo_8p_rome_4nics.xml", "4 nodes node 8 gfx908 Rome 4 NICs"},
{4, "topo_collnet_n1.xml", "4 nodes collnet 1 NICs"},
{4, "topo_collnet_n4.xml", "4 nodes collnet 4 NICs"},
};
int main(int argc,char* argv[])
@@ -148,6 +149,8 @@ int main(int argc,char* argv[])
NetworkModel network;
NodeModel* node;
initCollNet();
NodeModelDesc *desc = &model_descs[model_id];
for (int i=0; i<desc->num_nodes; i++) {
node = new NodeModel(desc->filename);
@@ -190,14 +193,9 @@ int main(int argc,char* argv[])
}
struct ncclTopoGraph *treeGraph, *ringGraph, *collNetGraph;
treeGraph = (struct ncclTopoGraph *)malloc(sizeof(struct ncclTopoGraph)*nranks);
ringGraph = (struct ncclTopoGraph *)malloc(sizeof(struct ncclTopoGraph)*nranks);
collNetGraph = (struct ncclTopoGraph *)malloc(sizeof(struct ncclTopoGraph)*nranks);
if (!treeGraph || !ringGraph || !collNetGraph) {
printf("Failed to allocate memory for graphs\n");
return -1;
}
NCCLCHECK(ncclCalloc(&treeGraph, nranks));
NCCLCHECK(ncclCalloc(&ringGraph, nranks));
NCCLCHECK(ncclCalloc(&collNetGraph, nranks));
for (int i = 0; i < nranks; i++) {
node_model = network.GetNode(i);
assert(node_model!=0);
tools/topo_expl/utils.cpp
+11 -4
Datei anzeigen
@@ -231,13 +231,16 @@ static ncclResult_t checkCollNetSetup(struct ncclComm* comm, int rank, int collN
return ncclSuccess;
}
void initCollNet() {
if (ncclParamCollNetEnable() == 1 && ncclCollNet == 0)
ncclCollNet = (ncclCollNet_t*)0x12345678;
}
ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t *allGather1Data, struct allGather3Data_t *allGather3Data,
struct ncclTopoGraph& treeGraph, struct ncclTopoGraph& ringGraph, struct ncclTopoGraph& collNetGraph) {
int rank = comm->rank;
int nranks = comm->nRanks;
if (ncclParamCollNetEnable() == 1 && ncclCollNet == 0)
ncclCollNet = (ncclCollNet_t*)0x12345678;
//uint64_t commHash = getHash(commId->internal, NCCL_UNIQUE_ID_BYTES);
//TRACE(NCCL_INIT, "comm %p, commHash %lx, rank %d nranks %d - BEGIN", comm, commHash, rank, nranks);
//NCCLCHECK(bootstrapInit(commId, rank, nranks, &comm->bootstrap));
@@ -401,6 +404,8 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
allGather3Data[rank].collNet.typeIntra = collNetGraph.typeIntra;
allGather3Data[rank].collNet.typeInter = collNetGraph.typeInter;
// CollNet channels are already duplicated
comm->collNetnChannels = 2*collNetGraph.nChannels;
NCCLCHECK(ncclTopoPreset(comm, &treeGraph, &ringGraph, &collNetGraph, &allGather3Data[rank].topoRanks));
return ncclSuccess;
@@ -624,9 +629,9 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
// Force 2 channels for CollNet
comm->collNetnChannels = collNetGraph.nChannels = 2;
NCCLCHECK(ncclTopoConnectCollNet(comm, &collNetGraph, rank));
} else {
comm->collNetnChannels = 0;
}
free(allTopoRanks);
@@ -682,6 +687,8 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
if (comm->nNodes > 1 &&
ncclParamCollNetEnable() == 1 &&
collNetSupport() && collNetGraph.nChannels) {
for (int c=comm->nChannels; c<comm->collNetnChannels; c++)
NCCLCHECK(initChannel(comm, c));;
int logicChannels = comm->collNetnChannels/2;
int collNetSetupFail = 0;
const int recvIndex = 0; // recv GPU index is always 0