AI/rocm-systems
2
0
フォーク 0
コード イシュー プルリクエスト Actions パッケージ プロジェクト リリース Wiki アクティビティ

Add another Rome model (#483)

ソースを参照 
[ROCm/rccl commit: f8d0775a6f]
このコミットが含まれているのは:
Wenkai Du
2022-01-05 09:26:31 -08:00
committed by GitHub
親 8d6cfcbf58
コミット 93c7526fdc
5個のファイルの変更、142行の追加、2行の削除
Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む
projects/rccl/src/graph/paths.cc
+1 -1
ファイルの表示
@@ -509,7 +509,6 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm*
}
}
if (ret) {
system->type |= RCCL_TOPO_GDR_ALL;
bool allXgmi = true;
// don't trim NICs unless all GPUs are connected by XGMI
for (int i = 0; i < system->nodes[GPU].count && allXgmi; i++) {
@@ -524,6 +523,7 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm*
}
if (!allXgmi) {
remove = 0;
system->type |= RCCL_TOPO_GDR_ALL;
INFO(NCCL_GRAPH, "GDR is available on all GPUs");
}
}
projects/rccl/src/graph/rome_models.cc
+28
ファイルの表示
@@ -434,6 +434,32 @@ static struct rcclRomeModel rome_model_65 = {
.netGdrLevel = 5,
};
static struct rcclRomeModel rome_model_66 = {
.nGpus = 8, .nCpus = 2, .nNics = 0, .nLinks = 3,
.gpuIds = { 0x29000, 0x2c000, 0x2f000, 0x32000, 0xad000, 0xb0000, 0xb3000, 0xb6000, },
.nicIds = { },
.gpuNuma = { 1, 1, 1, 1, 3, 3, 3, 3, },
.nicNuma = { },
.connMatrix = { 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, },
.gdrLevel = { },
.pattern = "4040",
.ringBase = "0 6 7 5 4 2 3 1|1 3 2 4 5 7 6 0|0 1 7 6 2 3 5 4|4 5 3 2 6 7 1 0",
.netGdrLevel = -2,
};
static struct rcclRomeModel rome_model_67 = {
.nGpus = 8, .nCpus = 2, .nNics = 4, .nLinks = 3,
.gpuIds = { 0x29000, 0x2c000, 0x2f000, 0x32000, 0xad000, 0xb0000, 0xb3000, 0xb6000, },
.nicIds = { 0x1d000, 0x1e000, 0xa1000, 0xa2000, },
.gpuNuma = { 1, 1, 1, 1, 3, 3, 3, 3, },
.nicNuma = { 1, 1, 3, 3, },
.connMatrix = { 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, },
.gdrLevel = { 4, 4, 4, 4, 6, 6, 6, 6, 4, 4, 4, 4, 6, 6, 6, 6, 6, 6, 6, 6, 4, 4, 4, 4, 6, 6, 6, 6, 4, 4, 4, 4, },
.pattern = "4242",
.ringBase = "N3 7 6 0 1 3 2 4 5 N2|N2 5 4 2 3 1 0 6 7 N3|N1 2 3 5 4 0 1 7 6 N3|N2 4 5 3 2 6 7 1 0 N0|N1 3 2 4 5 7 6 0 1 N0|N0 1 0 6 7 5 4 2 3 N1|N0 0 1 7 6 2 3 5 4 N2|N3 6 7 1 0 4 5 3 2 N1",
.netGdrLevel = -2,
};
static struct rcclRomeModel romeTopoModels[] = {
rome_model_22,
rome_model_25,
@@ -465,6 +491,8 @@ static struct rcclRomeModel romeTopoModels[] = {
rome_model_62,
rome_model_63,
rome_model_65,
rome_model_66,
rome_model_67,
};
/* Parse user defined rings. Format is like :
projects/rccl/tools/scripts/topo_val.sh
+1 -1
ファイルの表示
@@ -21,7 +21,7 @@
DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
for i in {0..65}
for i in {0..67}
do
if [[ $i -eq 50 ]] || [[ $i -eq 51 ]]
then
projects/rccl/tools/topo_expl/models/topo_8p1h.xml
+110
ファイルの表示
@@ -0,0 +1,110 @@
<system version="2">
<cpu numaid="1" affinity="00000000,0000ffff,ff000000" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="1">
<pci busid="0000:19:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:25:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:27:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:29:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1">
<xgmi target="0000:2c:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:ad:00.0" count="2" tclass="0x038000"/>
<xgmi target="0000:b3:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:2a:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:2c:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1">
<xgmi target="0000:29:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:32:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:b6:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:2d:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:2f:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1">
<xgmi target="0000:32:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:ad:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:b3:00.0" count="2" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:30:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:32:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1458" subsystem_device="0x1000" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="3" sm="90" gcn="910" arch="38911" rank="3" gdr="1">
<xgmi target="0000:2c:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:2f:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:b0:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:1b:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:1d:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_0" dev="0" speed="200000" port="1" guid="0xd4ac4d0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
<pci busid="0000:1e:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_2" dev="1" speed="200000" port="1" guid="0xd07f5600039f59b8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</pci>
</cpu>
<cpu numaid="3" affinity="ffffff00,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="1">
<pci busid="0000:9d:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:a9:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:ab:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:ad:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="4" sm="90" gcn="910" arch="38911" rank="4" gdr="1">
<xgmi target="0000:29:00.0" count="2" tclass="0x038000"/>
<xgmi target="0000:2f:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:b0:00.0" count="4" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:ae:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:b0:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="5" sm="90" gcn="910" arch="38911" rank="5" gdr="1">
<xgmi target="0000:32:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:ad:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:b6:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:b1:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:b3:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="6" sm="90" gcn="910" arch="38911" rank="6" gdr="1">
<xgmi target="0000:29:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:2f:00.0" count="2" tclass="0x038000"/>
<xgmi target="0000:b6:00.0" count="4" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:b4:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:b6:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16.0 GT/s PCIe" link_width="16">
<gpu dev="7" sm="90" gcn="910" arch="38911" rank="7" gdr="1">
<xgmi target="0000:2c:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:b0:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:b3:00.0" count="4" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:9f:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:a1:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_4" dev="2" speed="200000" port="1" guid="0x50f3b100039f59b8" maxconn="262144" gdr="1"/>
</nic>
</pci>
<pci busid="0000:a2:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_6" dev="3" speed="200000" port="1" guid="0x74ae4d0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/topo_expl.cpp
+2
ファイルの表示
@@ -135,6 +135,8 @@ NodeModelDesc model_descs[] = {
{4, "topo_8p_4nics.xml", "4 nodes 8P 4 NICs"},
{1, "topo_16p1h_vm.xml", "single node 16P1H VM"},
{4, "topo_16p1h_vm.xml", "4 nodes 16P1H VM"},
{1, "topo_8p1h.xml", "single node 8P1H"},
{4, "topo_8p1h.xml", "4 nodes 8P1H"},
};
int main(int argc,char* argv[])