From fe8923ebbad2ab2bab591f7857c306fb1e46ffa0 Mon Sep 17 00:00:00 2001
From: Wenkai Du <wenkai.du@amd.com>
Date: Sat, 6 Feb 2021 00:19:47 +0000
Subject: [PATCH] Add gfx908 Rome 4 NICs model

[ROCm/rccl commit: 6dfdfef98f4f2c9790678a0bc714cee8b3c05685]
---
 projects/rccl/tools/scripts/topo_val.sh       |   2 +-
 .../topo_expl/models/topo_8p_rome_4nics.xml   | 126 ++++++++++++++++++
 projects/rccl/tools/topo_expl/topo_expl.cpp   |   2 +
 3 files changed, 129 insertions(+), 1 deletion(-)
 create mode 100644 projects/rccl/tools/topo_expl/models/topo_8p_rome_4nics.xml

diff --git a/projects/rccl/tools/scripts/topo_val.sh b/projects/rccl/tools/scripts/topo_val.sh
index fe11d8b26e..07d8d050d5 100755
--- a/projects/rccl/tools/scripts/topo_val.sh
+++ b/projects/rccl/tools/scripts/topo_val.sh
@@ -21,7 +21,7 @@
 
 DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-for i in {0..47}
+for i in {0..49}
 do
 	$DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log"
 	$DIR/../TopoVisual/topo_visual.sh -i "topo_m$i.log"
diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_rome_4nics.xml b/projects/rccl/tools/topo_expl/models/topo_8p_rome_4nics.xml
new file mode 100644
index 0000000000..8129927d71
--- /dev/null
+++ b/projects/rccl/tools/topo_expl/models/topo_8p_rome_4nics.xml
@@ -0,0 +1,126 @@
+<system version="2">
+  <cpu numaid="0" affinity="00000000,00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
+    <pci busid="0000:43:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+      <pci busid="0000:46:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:48:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:4a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="0" sm="90" gcn="908" arch="38911" rank="0" gdr="1">
+              <xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:4c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:4e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:50:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="1" sm="90" gcn="908" arch="38911" rank="1" gdr="1">
+              <xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:45:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
+        <nic>
+          <net name="mlx5_1" dev="0" speed="200000" port="1" guid="0x48b9170003a1420c" maxconn="262144" gdr="1"/>
+        </nic>
+      </pci>
+    </pci>
+  </cpu>
+  <cpu numaid="1" affinity="00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
+    <pci busid="0000:03:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+      <pci busid="0000:05:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:08:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:0a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="2" sm="90" gcn="908" arch="38911" rank="2" gdr="1">
+              <xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:0b:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:0d:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:0f:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="3" sm="90" gcn="908" arch="38911" rank="3" gdr="1">
+              <xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:13:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
+        <nic>
+          <net name="mlx5_3" dev="1" speed="200000" port="1" guid="0x18604a0003a1420c" maxconn="262144" gdr="1"/>
+        </nic>
+      </pci>
+    </pci>
+  </cpu>
+  <cpu numaid="2" affinity="00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
+    <pci busid="0000:c4:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+      <pci busid="0000:c7:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:c9:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:cb:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="4" sm="90" gcn="908" arch="38911" rank="4" gdr="1">
+              <xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:cd:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:cf:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:d1:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="5" sm="90" gcn="908" arch="38911" rank="5" gdr="1">
+              <xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:c6:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
+        <nic>
+          <net name="mlx5_5" dev="2" speed="200000" port="1" guid="0xd0b9170003a1420c" maxconn="262144" gdr="1"/>
+        </nic>
+      </pci>
+    </pci>
+  </cpu>
+  <cpu numaid="3" affinity="ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
+    <pci busid="0000:83:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+      <pci busid="0000:86:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:88:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:8a:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="6" sm="90" gcn="908" arch="38911" rank="6" gdr="1">
+              <xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:8c:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+        <pci busid="0000:8e:00.0" class="0x060400" link_speed="16 GT/s" link_width="16">
+          <pci busid="0000:90:00.0" class="0x038000" link_speed="16 GT/s" link_width="16">
+            <gpu dev="7" sm="90" gcn="908" arch="38911" rank="7" gdr="1">
+              <xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
+            </gpu>
+          </pci>
+        </pci>
+      </pci>
+      <pci busid="0000:85:00.0" class="0x020700" link_speed="16 GT/s" link_width="16">
+        <nic>
+          <net name="mlx5_7" dev="3" speed="200000" port="1" guid="0xd0bd170003a1420c" maxconn="262144" gdr="1"/>
+        </nic>
+      </pci>
+    </pci>
+  </cpu>
+</system>
diff --git a/projects/rccl/tools/topo_expl/topo_expl.cpp b/projects/rccl/tools/topo_expl/topo_expl.cpp
index db9181996c..77b0597e51 100644
--- a/projects/rccl/tools/topo_expl/topo_expl.cpp
+++ b/projects/rccl/tools/topo_expl/topo_expl.cpp
@@ -117,6 +117,8 @@ NodeModelDesc model_descs[] = {
   {1, "topo_8p_rome_n4_1.xml",  "single node 8 gfx908 Rome NPS=4"},
   {4, "topo_8p_rome_n4_1.xml",  "4 nodes node 8 gfx908 Rome NPS=4"},
   {2, "topo_8p_rome_pcie.xml",  "2 nodes node 8 VEGA20 PCIe"},
+  {1, "topo_8p_rome_4nics.xml", "single node 8 gfx908 Rome 4 NICs"},
+  {4, "topo_8p_rome_4nics.xml", "4 nodes node 8 gfx908 Rome 4 NICs"},
 };
 
 int main(int argc,char* argv[])