AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more Rome models (#434)

Browse Source 
* Add more Rome models

* Update models and tuning

* Update tuning

[ROCm/rccl commit: 2249a1d9d3]
This commit is contained in:
Wenkai Du
2021-10-12 08:23:20 -07:00
committed by GitHub
parent 227848b70f
commit b587b55c2e
10 changed files with 246 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/tools/topo_expl/models/topo_16p1h.xml
+2 -7
View File
@@ -191,7 +191,7 @@
</pci>
<pci busid="0000:8c:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_7" dev="7" speed="200000" port="1" guid="0x8c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_7" dev="6" speed="200000" port="1" guid="0x8c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
@@ -217,15 +217,10 @@
</pci>
<pci busid="0000:9b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_8" dev="8" speed="200000" port="1" guid="0x6c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_8" dev="7" speed="200000" port="1" guid="0x6c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</pci>
<pci busid="0000:b1:00.0" class="0x020000" vendor="0x15b3" device="0x1015" subsystem_vendor="0x15b3" subsystem_device="0x0190" link_speed="8.0 GT/s PCIe" link_width="8">
<nic>
<net name="mlx5_6" dev="6" speed="40000" port="1" guid="0x100fe0b7312aea4" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/models/topo_3p_pcie.xml
+26
View File
@@ -0,0 +1,26 @@
<system version="2">
<cpu numaid="0" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="1">
<pci busid="0000:21:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:23:00.0" class="0x038000" vendor="0x1002" device="0x740f" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1"/>
</pci>
</pci>
<pci busid="0000:41:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_0" dev="0" speed="200000" port="1" guid="0xad9c300039f59b8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
<cpu numaid="1" affinity="ffff0000,ffff0000" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="1">
<pci busid="0000:81:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:83:00.0" class="0x038000" vendor="0x1002" device="0x740f" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1"/>
</pci>
</pci>
<pci busid="0000:e2:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:e4:00.0" class="0x038000" vendor="0x1002" device="0x740f" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1"/>
</pci>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/models/topo_3p_pcie_1.xml
+26
View File
@@ -0,0 +1,26 @@
<system version="2">
<cpu numaid="0" affinity="0000ffff,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="1">
<pci busid="0000:21:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:23:00.0" class="0x038000" vendor="0x1002" device="0x740f" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1"/>
</pci>
</pci>
</cpu>
<cpu numaid="1" affinity="ffff0000,ffff0000" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="1">
<pci busid="0000:81:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:83:00.0" class="0x038000" vendor="0x1002" device="0x740f" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1"/>
</pci>
</pci>
<pci busid="0000:e2:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:e4:00.0" class="0x038000" vendor="0x1002" device="0x740f" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1"/>
</pci>
</pci>
<pci busid="0000:a1:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_1" dev="0" speed="200000" port="1" guid="0x7657900003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/models/topo_8p_4nics.xml
+126
View File
@@ -0,0 +1,126 @@
<system version="2">
<cpu numaid="0" affinity="00000000,00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:43:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:46:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:48:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="0" sm="90" gcn="908" arch="38911" rank="0" gdr="1">
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:4c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:50:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="1" sm="90" gcn="908" arch="38911" rank="1" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:45:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_1" dev="0" speed="200000" port="1" guid="0x48b9170003a1420c" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="1" affinity="00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:03:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:05:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:08:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="2" sm="90" gcn="908" arch="38911" rank="2" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:0b:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0d:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0f:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="3" sm="90" gcn="908" arch="38911" rank="3" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:13:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_3" dev="1" speed="200000" port="1" guid="0x18604a0003a1420c" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="2" affinity="00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:c4:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c7:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c9:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cb:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="4" sm="90" gcn="908" arch="38911" rank="4" gdr="1">
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:cd:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cf:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:d1:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="5" sm="90" gcn="908" arch="38911" rank="5" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:c6:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_5" dev="2" speed="200000" port="1" guid="0xd0b9170003a1420c" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="3" affinity="ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:83:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:86:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:88:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="6" sm="90" gcn="908" arch="38911" rank="6" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:8c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
<pci busid="0000:90:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
<gpu dev="7" sm="90" gcn="908" arch="38911" rank="7" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:85:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_7" dev="3" speed="200000" port="1" guid="0xd0bd170003a1420c" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/topo_expl.cpp
+5
View File
@@ -128,6 +128,11 @@ NodeModelDesc model_descs[] = {
{1, "topo_16p1h.xml", "single node 16P1H"},
{4, "topo_8p_rome_4n_2.xml", "4 nodes 8 gfx908 Rome 4 NICs NPS=4 Alt. Model"},
{1, "topo_8p_90a_1.xml", "single node gfx90a Alt. Model"},
{4, "topo_16p1h.xml", "4 nodes 16P1H"},
{4, "topo_3p_pcie.xml", "4 nodes 3P"},
{4, "topo_3p_pcie_1.xml", "4 nodes 3P Alt. Model"},
{1, "topo_8p_4nics.xml", "single nodes 8P 4 NICs"},
{4, "topo_8p_4nics.xml", "4 nodes 8P 4 NICs"},
};
int main(int argc,char* argv[])
projects/rccl/tools/topo_expl/utils.cpp
+2
View File
@@ -672,6 +672,8 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
allGather3Data[rank].nc = 4;
if (comm->topo->nodes[GPU].count == comm->topo->nRanks && comm->topo->nodes[GPU].nodes[idx].gpu.gcn == 910)
allGather3Data[rank].nc = 4;
if (comm->topo->nodes[GPU].nodes[idx].gpu.gcn == 910)
allGather3Data[rank].nc = std::max(allGather3Data[rank].nc, 4/ringGraph.nChannels);
allGather3Data[rank].tree.pattern = treeGraph.pattern;
allGather3Data[rank].tree.nChannels = treeGraph.nChannels;
allGather3Data[rank].tree.sameChannels = treeGraph.sameChannels;