AI/rocm-systems
2
0 Förgrening 0
Kod Ärenden Pull-förfrågningar Actions Packages Projekt Släpp Wiki Aktiviteter

Update Rome model matching (#461)

Bläddra i källkod 
* Update Rome model matching

* Add another Rome model

* Automatically setup NET GDR level from model

[ROCm/rccl commit: 0331e39f81]
This commit is contained in:
Wenkai Du authored and GitHub committed 2021-11-05 08:53:47 -07:00
1 förälder 709b3dc85b
incheckning fd98ee84b4
8 files changed
+276 -52
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

projects/rccl/tools/scripts/topo_val.sh
+1 -1
Visa fil
@@ -21,7 +21,7 @@
DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
for i in {0..63}
for i in {0..65}
do
if [[ $i -eq 50 ]] || [[ $i -eq 51 ]]
then
projects/rccl/tools/topo_expl/models/topo_16p1h_vm.xml
+170
Visa fil
@@ -0,0 +1,170 @@
<system version="2">
<cpu numaid="0" affinity="00000000,00000000,00ffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="3436:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1">
<xgmi target="11d1:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="d2a8:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="2589:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="11d1:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1">
<xgmi target="3436:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="0b40:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="d2a8:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="12de:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="0b40:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1">
<xgmi target="11d1:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="d2a8:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="0bd7:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="d2a8:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="3" sm="90" gcn="910" arch="38911" rank="3" gdr="1">
<xgmi target="3436:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="11d1:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="0b40:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="258d:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="c7ca:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_1" dev="1" speed="200000" port="1" guid="0x885b20003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
<pci busid="93b7:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_6" dev="6" speed="200000" port="1" guid="0x6885b20003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
<cpu numaid="1" affinity="00000000,0000ffff,ff000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="2589:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="4" sm="90" gcn="910" arch="38911" rank="4" gdr="1">
<xgmi target="3436:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="258d:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="fa78:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="258d:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="5" sm="90" gcn="910" arch="38911" rank="5" gdr="1">
<xgmi target="d2a8:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="2589:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="2460:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="e33f:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="12de:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="6" sm="90" gcn="910" arch="38911" rank="6" gdr="1">
<xgmi target="11d1:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="0bd7:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="2460:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="e33f:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="0bd7:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="7" sm="90" gcn="910" arch="38911" rank="7" gdr="1">
<xgmi target="0b40:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="12de:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="ef87:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="f9dd:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_0" dev="0" speed="200000" port="1" guid="0x1085b20003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
<pci busid="889c:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_4" dev="4" speed="200000" port="1" guid="0xd481b20003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
<cpu numaid="2" affinity="000000ff,ffff0000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="fb49:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="8" sm="90" gcn="910" arch="38911" rank="8" gdr="1">
<xgmi target="c77c:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="820c:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="0be8:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="e33f:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="c77c:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="9" sm="90" gcn="910" arch="38911" rank="9" gdr="1">
<xgmi target="fb49:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="820c:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="0be8:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="2460:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="820c:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="10" sm="90" gcn="910" arch="38911" rank="10" gdr="1">
<xgmi target="fb49:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="c77c:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="fa78:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="0be8:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="11" sm="90" gcn="910" arch="38911" rank="11" gdr="1">
<xgmi target="fb49:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="c77c:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="ef87:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="6738:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_2" dev="2" speed="200000" port="1" guid="0x482b20003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
<pci busid="982d:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_3" dev="3" speed="200000" port="1" guid="0xacb10003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
<cpu numaid="3" affinity="ffffff00,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="2460:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="12" sm="90" gcn="910" arch="38911" rank="12" gdr="1">
<xgmi target="258d:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="12de:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="c77c:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="fa78:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="fa78:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="13" sm="90" gcn="910" arch="38911" rank="13" gdr="1">
<xgmi target="2589:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="820c:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="2460:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="e33f:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="14" sm="90" gcn="910" arch="38911" rank="14" gdr="1">
<xgmi target="258d:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="12de:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="fb49:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="ef87:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="ef87:00:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="15" sm="90" gcn="910" arch="38911" rank="15" gdr="1">
<xgmi target="0bd7:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="0be8:00:00.0" count="1" tclass="0x038000"/>
<xgmi target="e33f:00:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
<pci busid="c41e:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_5" dev="5" speed="200000" port="1" guid="0x6c85b20003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
<pci busid="3146:00:00.0" class="0x020700" vendor="0x15b3" device="0x101c" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_7" dev="7" speed="200000" port="1" guid="0x4885b20003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/topo_expl.cpp
+2
Visa fil
@@ -133,6 +133,8 @@ NodeModelDesc model_descs[] = {
{4, "topo_3p_pcie_1.xml", "4 nodes 3P Alt. Model"},
{1, "topo_8p_4nics.xml", "single nodes 8P 4 NICs"},
{4, "topo_8p_4nics.xml", "4 nodes 8P 4 NICs"},
{1, "topo_16p1h_vm.xml", "single node 16P1H VM"},
{4, "topo_16p1h_vm.xml", "4 nodes 16P1H VM"},
};
int main(int argc,char* argv[])
projects/rccl/tools/topo_expl/utils.cpp
+2
Visa fil
@@ -531,6 +531,8 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
//NCCLCHECK(ncclTopoGetSystem(comm, &comm->topo));
// save nRanks to ncclTopoSystem as indicator of multi-node
comm->topo->nRanks = comm->nRanks;
// init netGdrLevel
comm->topo->netGdrLevel = -2;
// Compute paths between GPUs and NICs
NCCLCHECK(ncclTopoComputePaths(comm->topo, comm->peerInfo));
// Remove inaccessible GPUs and unused NICs