From 011447e4dc0a0fde02ecebca9ce8cbfbc58a1d77 Mon Sep 17 00:00:00 2001
From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com>
Date: Wed, 13 Apr 2022 11:45:40 -0700
Subject: [PATCH] Add new Rome model (#536)

[ROCm/rccl commit: 2151c79d14c34de9048a4ee39eddfd135cce03ce]
---
 projects/rccl/src/graph/rome_models.cc        | 14 +++
 projects/rccl/tools/scripts/topo_val.sh       |  2 +-
 .../tools/topo_expl/models/topo_8p1h_3.xml    | 86 +++++++++++++++++++
 projects/rccl/tools/topo_expl/topo_expl.cpp   |  1 +
 4 files changed, 102 insertions(+), 1 deletion(-)
 create mode 100644 projects/rccl/tools/topo_expl/models/topo_8p1h_3.xml

diff --git a/projects/rccl/src/graph/rome_models.cc b/projects/rccl/src/graph/rome_models.cc
index f0afd84f56..8ac7de67ca 100644
--- a/projects/rccl/src/graph/rome_models.cc
+++ b/projects/rccl/src/graph/rome_models.cc
@@ -513,6 +513,19 @@ static struct rcclRomeModel rome_model_73 = {
   .options = "",
 };
 
+static struct rcclRomeModel rome_model_74 = {
+  .nGpus = 8, .nCpus = 4, .nNics = 4, .nLinks = 3,
+  .gpuIds = { 0xc1000, 0xc6000, 0xc9000, 0xce000, 0xd1000, 0xd6000, 0xd9000, 0xde000, },
+  .nicIds = { 0xc5000, 0xcd000, 0xd5000, 0xdd000, },
+  .gpuNuma = { 3, 3, 1, 1, 0, 0, 2, 2, },
+  .nicNuma = { 3, 1, 0, 2, },
+  .connMatrix = { 0, 4, 1, 0, 0, 0, 2, 0, 4, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 4, 2, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4, 1, 0, 0, 1, 0, 0, 4, 0, 0, 1, 2, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 1, 0, 1, 4, 0, },
+  .gdrLevel = { PATH_PHB, PATH_PHB, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_PHB, PATH_PHB, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_PHB, PATH_PHB, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_SYS, PATH_PHB, PATH_PHB, },
+  .pattern = "21212121",
+  .ringBase = "N0 0 1 5 4 6 7 3 2 N1|N1 2 3 7 6 4 5 1 0 N0|N3 7 6 0 1 3 2 4 5 N2|N2 5 4 2 3 1 0 6 7 N3|N0 0 1 5 4 6 7 3 2 N1|N1 2 3 7 6 4 5 1 0 N0|N3 7 6 0 1 3 2 4 5 N2|N2 5 4 2 3 1 0 6 7 N3",
+  .options = "tuning=3",
+};
+
 static struct rcclRomeModel romeTopoModels[] = {
   rome_model_22,
   rome_model_25,
@@ -550,6 +563,7 @@ static struct rcclRomeModel romeTopoModels[] = {
   rome_model_71,
   rome_model_72,
   rome_model_73,
+  rome_model_74,
 };
 
 /* Parse user defined rings. Format is like :
diff --git a/projects/rccl/tools/scripts/topo_val.sh b/projects/rccl/tools/scripts/topo_val.sh
index 6b48ecf548..6bcce1284c 100755
--- a/projects/rccl/tools/scripts/topo_val.sh
+++ b/projects/rccl/tools/scripts/topo_val.sh
@@ -21,7 +21,7 @@
 
 DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-for i in {0..73}
+for i in {0..74}
 do
 	if [[ $i -eq 50 ]] || [[ $i -eq 51 ]]
 	then
diff --git a/projects/rccl/tools/topo_expl/models/topo_8p1h_3.xml b/projects/rccl/tools/topo_expl/models/topo_8p1h_3.xml
new file mode 100644
index 0000000000..5dbe1d0b9d
--- /dev/null
+++ b/projects/rccl/tools/topo_expl/models/topo_8p1h_3.xml
@@ -0,0 +1,86 @@
+<system version="2">
+  <cpu numaid="3" affinity="ffff0000,00000000,ffff0000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="48">
+    <pci busid="0000:c1:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1">
+        <xgmi target="0000:c6:00.0" count="4" tclass="0x038000"/>
+        <xgmi target="0000:c9:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:d9:00.0" count="2" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:c6:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1">
+        <xgmi target="0000:c1:00.0" count="4" tclass="0x038000"/>
+        <xgmi target="0000:ce:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:d6:00.0" count="1" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:c5:00.0" class="0x020000" vendor="0x17db" device="0x0501" subsystem_vendor="0x17db" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
+      <nic>
+        <net name="net0" dev="0" speed="200000" port="1" guid="0x0" maxconn="128" gdr="1"/>
+      </nic>
+    </pci>
+  </cpu>
+  <cpu numaid="1" affinity="00000000,ffff0000,00000000,ffff0000" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="48">
+    <pci busid="0000:c9:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1">
+        <xgmi target="0000:c1:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:ce:00.0" count="4" tclass="0x038000"/>
+        <xgmi target="0000:d1:00.0" count="2" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:ce:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="3" sm="90" gcn="910" arch="38911" rank="3" gdr="1">
+        <xgmi target="0000:c6:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:c9:00.0" count="4" tclass="0x038000"/>
+        <xgmi target="0000:de:00.0" count="1" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:cd:00.0" class="0x020000" vendor="0x17db" device="0x0501" subsystem_vendor="0x17db" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
+      <nic>
+        <net name="net1" dev="1" speed="200000" port="1" guid="0x1" maxconn="128" gdr="1"/>
+      </nic>
+    </pci>
+  </cpu>
+  <cpu numaid="0" affinity="00000000,0000ffff,00000000,0000ffff" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="48">
+    <pci busid="0000:d1:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="4" sm="90" gcn="910" arch="38911" rank="4" gdr="1">
+        <xgmi target="0000:c9:00.0" count="2" tclass="0x038000"/>
+        <xgmi target="0000:d6:00.0" count="4" tclass="0x038000"/>
+        <xgmi target="0000:d9:00.0" count="1" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:d6:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="5" sm="90" gcn="910" arch="38911" rank="5" gdr="1">
+        <xgmi target="0000:c6:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:d1:00.0" count="4" tclass="0x038000"/>
+        <xgmi target="0000:de:00.0" count="1" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:d5:00.0" class="0x020000" vendor="0x17db" device="0x0501" subsystem_vendor="0x17db" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
+      <nic>
+        <net name="net2" dev="2" speed="200000" port="1" guid="0x2" maxconn="128" gdr="1"/>
+      </nic>
+    </pci>
+  </cpu>
+  <cpu numaid="2" affinity="0000ffff,00000000,0000ffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="175" modelid="48">
+    <pci busid="0000:d9:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="6" sm="90" gcn="910" arch="38911" rank="6" gdr="1">
+        <xgmi target="0000:c1:00.0" count="2" tclass="0x038000"/>
+        <xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:de:00.0" count="4" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:de:00.0" class="0x038000" vendor="0x1002" device="0x7408" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="16 GT/s" link_width="16">
+      <gpu dev="7" sm="90" gcn="910" arch="38911" rank="7" gdr="1">
+        <xgmi target="0000:ce:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:d6:00.0" count="1" tclass="0x038000"/>
+        <xgmi target="0000:d9:00.0" count="4" tclass="0x038000"/>
+      </gpu>
+    </pci>
+    <pci busid="0000:dd:00.0" class="0x020000" vendor="0x17db" device="0x0501" subsystem_vendor="0x17db" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
+      <nic>
+        <net name="net3" dev="3" speed="200000" port="1" guid="0x3" maxconn="128" gdr="1"/>
+      </nic>
+    </pci>
+  </cpu>
+</system>
diff --git a/projects/rccl/tools/topo_expl/topo_expl.cpp b/projects/rccl/tools/topo_expl/topo_expl.cpp
index ed0480a73a..da70b9c021 100644
--- a/projects/rccl/tools/topo_expl/topo_expl.cpp
+++ b/projects/rccl/tools/topo_expl/topo_expl.cpp
@@ -143,6 +143,7 @@ NodeModelDesc model_descs[] = {
   {1, "topo_8p1h_1.xml",        "single node 8P1H Alt."},
   {4, "topo_8p1h_1.xml",        "4 nodes 8P1H Alt."},
   {1, "topo_8p1h_2.xml",        "single node 8P1H Alt."},
+  {4, "topo_8p1h_3.xml",        "4 nodes 8P1H Alt."},
 };
 
 int main(int argc,char* argv[])