From 15143b1cfbc6294592bf7046985838ae40dc638e Mon Sep 17 00:00:00 2001
From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com>
Date: Tue, 26 Oct 2021 10:30:20 -0700
Subject: [PATCH] Query XGMI link count through rocm_smi_lib API (#442)

[ROCm/rccl commit: 14a184eb67ee14492a015aef90345e1393b97f92]
---
 projects/rccl/src/graph/xml.cc                |  6 +-
 projects/rccl/src/include/rocm_smi_wrap.h     |  3 +-
 projects/rccl/src/misc/rocm_smi_wrap.cc       | 18 +++-
 .../tools/topo_expl/models/topo_16p1h.xml     | 96 +++++++++----------
 4 files changed, 67 insertions(+), 56 deletions(-)

diff --git a/projects/rccl/src/graph/xml.cc b/projects/rccl/src/graph/xml.cc
index 3aca65c1c6..b9cba9953d 100644
--- a/projects/rccl/src/graph/xml.cc
+++ b/projects/rccl/src/graph/xml.cc
@@ -671,8 +671,8 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvm
     for (int i=0; i<deviceCnt; i++) {
       if (i != dev) {
         RSMI_IO_LINK_TYPE rsmi_type;
-        int hops, bw;
-        if (rocm_smi_getLinkInfo(dev, i, &rsmi_type, &hops, &bw) == ncclSuccess) {
+        int hops, count;
+        if (rocm_smi_getLinkInfo(dev, i, &rsmi_type, &hops, &count) == ncclSuccess) {
           if (rsmi_type == RSMI_IOLINK_TYPE_XGMI && hops == 1) {
             char busIdStr[] = "00000000:00:00.0";
             CUDACHECK(hipDeviceGetPCIBusId(busIdStr, sizeof(busIdStr), i));
@@ -685,7 +685,7 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvm
             if (nvlNode == NULL) {
               NCCLCHECK(xmlAddNode(xml, gpuNode, "xgmi", &nvlNode));
               NCCLCHECK(xmlSetAttr(nvlNode, "target", lowerId));
-              NCCLCHECK(xmlSetAttrInt(nvlNode, "count", 1));
+              NCCLCHECK(xmlSetAttrInt(nvlNode, "count", count));
             }
           }
         }
diff --git a/projects/rccl/src/include/rocm_smi_wrap.h b/projects/rccl/src/include/rocm_smi_wrap.h
index 56b178a14e..ea33404222 100644
--- a/projects/rccl/src/include/rocm_smi_wrap.h
+++ b/projects/rccl/src/include/rocm_smi_wrap.h
@@ -24,10 +24,11 @@ THE SOFTWARE.
 #define ROCM_SMI_WRAP_H_
 
 #include "rocm_smi/rocm_smi.h"
+#include "rocm_smi/rocm_smi64Config.h"
 #include "nccl.h"
 
 ncclResult_t rocm_smi_init();
 ncclResult_t rocm_smi_getDeviceIndexByPciBusId(const char* pciBusId, uint32_t* deviceIndex);
-ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *bw);
+ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *count);
 
 #endif
diff --git a/projects/rccl/src/misc/rocm_smi_wrap.cc b/projects/rccl/src/misc/rocm_smi_wrap.cc
index 44b5fb153c..9c1936f291 100644
--- a/projects/rccl/src/misc/rocm_smi_wrap.cc
+++ b/projects/rccl/src/misc/rocm_smi_wrap.cc
@@ -74,7 +74,7 @@ ncclResult_t rocm_smi_getDeviceIndexByPciBusId(const char* pciBusId, uint32_t* d
   }
 }
 
-ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *bw) {
+ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *count) {
   char srcStr[] = "00000000:00:00.0", dstStr[] = "00000000:00:00.0";
   uint32_t srcIndex, dstIndex;
 
@@ -86,9 +86,19 @@ ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsm
   uint64_t rsmi_hops, rsmi_weight;
   ROCMSMICHECK(rsmi_topo_get_link_type(srcIndex, dstIndex, &rsmi_hops, rsmi_type));
   ROCMSMICHECK(rsmi_topo_get_link_weight(srcIndex, dstIndex, &rsmi_weight));
-
   *hops = 2;
-  *bw = 0;
-  if (*rsmi_type == RSMI_IOLINK_TYPE_XGMI && rsmi_weight == 15) *hops = 1;
+  *count = 1;
+  if (*rsmi_type == RSMI_IOLINK_TYPE_XGMI && rsmi_weight == 15) {
+    *hops = 1;
+#if rocm_smi_VERSION_MAJOR >= 5
+    uint64_t min_bw = 0, max_bw = 0;
+    rsmi_version_t version;
+    ROCMSMICHECK(rsmi_version_get(&version));
+    if (version.major >= 5)
+      ROCMSMICHECK(rsmi_minmax_bandwidth_get(srcIndex, dstIndex, &min_bw, &max_bw));
+    if (max_bw && min_bw)
+      *count = max_bw/min_bw;
+#endif
+  }
   return ncclSuccess;
 }
diff --git a/projects/rccl/tools/topo_expl/models/topo_16p1h.xml b/projects/rccl/tools/topo_expl/models/topo_16p1h.xml
index 3389967c79..44a0451414 100644
--- a/projects/rccl/tools/topo_expl/models/topo_16p1h.xml
+++ b/projects/rccl/tools/topo_expl/models/topo_16p1h.xml
@@ -2,19 +2,19 @@
   <cpu numaid="0" affinity="00000000,00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
     <pci busid="0000:41:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
       <pci busid="0000:49:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:4c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:4c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:4e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1">
-              <xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:51:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0e:00.0" count="2" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:4f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:4f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:51:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1">
-              <xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:4e:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
@@ -23,33 +23,33 @@
         </pci>
         <pci busid="0000:4b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_0" dev="0" speed="200000" port="1" guid="0xb03fbb0003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_0" dev="0" speed="200000" port="1" guid="0xd041bb0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>
       <pci busid="0000:52:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:54:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:54:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:56:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1">
               <xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:59:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:57:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:57:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:59:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="3" sm="90" gcn="910" arch="38911" rank="3" gdr="1">
               <xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:56:00.0" count="4" tclass="0x038000"/>
+              <xgmi target="0000:19:00.0" count="2" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
         <pci busid="0000:5a:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_1" dev="1" speed="200000" port="1" guid="0xfc11b30003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_1" dev="1" speed="200000" port="1" guid="0x2085ba0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>
@@ -58,20 +58,20 @@
   <cpu numaid="1" affinity="00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
     <pci busid="0000:01:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
       <pci busid="0000:09:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:0c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:0c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:0e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="4" sm="90" gcn="910" arch="38911" rank="4" gdr="1">
-              <xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:4e:00.0" count="2" tclass="0x038000"/>
+              <xgmi target="0000:11:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:0f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:0f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:11:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="5" sm="90" gcn="910" arch="38911" rank="5" gdr="1">
               <xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:0e:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
             </gpu>
@@ -79,33 +79,33 @@
         </pci>
         <pci busid="0000:0b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_2" dev="2" speed="200000" port="1" guid="0xc83fbb0003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_2" dev="2" speed="200000" port="1" guid="0xfc84ba0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>
       <pci busid="0000:12:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:14:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:14:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:16:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="6" sm="90" gcn="910" arch="38911" rank="6" gdr="1">
               <xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:19:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:17:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:17:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:19:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="7" sm="90" gcn="910" arch="38911" rank="7" gdr="1">
-              <xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:59:00.0" count="2" tclass="0x038000"/>
+              <xgmi target="0000:16:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
         <pci busid="0000:1a:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_3" dev="3" speed="200000" port="1" guid="0x3440bb0003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_3" dev="3" speed="200000" port="1" guid="0x1c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>
@@ -114,19 +114,19 @@
   <cpu numaid="2" affinity="00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
     <pci busid="0000:c2:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
       <pci busid="0000:ca:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:cd:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:cd:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:cf:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="8" sm="90" gcn="910" arch="38911" rank="8" gdr="1">
-              <xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:d2:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:8f:00.0" count="2" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:d0:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:d0:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:d2:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="9" sm="90" gcn="910" arch="38911" rank="9" gdr="1">
-              <xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:cf:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
@@ -135,33 +135,33 @@
         </pci>
         <pci busid="0000:cc:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_4" dev="4" speed="200000" port="1" guid="0xb411b30003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_4" dev="4" speed="200000" port="1" guid="0x40bb0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>
       <pci busid="0000:d3:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:d5:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:d5:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:d7:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="10" sm="90" gcn="910" arch="38911" rank="10" gdr="1">
               <xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:da:00.0" count="4" tclass="0x038000"/>
               <xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:d8:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:d8:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:da:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="11" sm="90" gcn="910" arch="38911" rank="11" gdr="1">
               <xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:d7:00.0" count="4" tclass="0x038000"/>
+              <xgmi target="0000:9a:00.0" count="2" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
         <pci busid="0000:db:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_5" dev="5" speed="200000" port="1" guid="0x4040bb0003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_5" dev="5" speed="200000" port="1" guid="0x7c85ba0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>
@@ -170,54 +170,54 @@
   <cpu numaid="3" affinity="ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
     <pci busid="0000:82:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
       <pci busid="0000:8a:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:8d:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:8d:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:8f:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="12" sm="90" gcn="910" arch="38911" rank="12" gdr="1">
               <xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:cf:00.0" count="2" tclass="0x038000"/>
+              <xgmi target="0000:92:00.0" count="4" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:90:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:90:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:92:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="13" sm="90" gcn="910" arch="38911" rank="13" gdr="1">
               <xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:8f:00.0" count="4" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
         <pci busid="0000:8c:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_7" dev="6" speed="200000" port="1" guid="0x8c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_7" dev="6" speed="200000" port="1" guid="0x85ba0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>
       <pci busid="0000:93:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
-        <pci busid="0000:95:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:95:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:97:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="14" sm="90" gcn="910" arch="38911" rank="14" gdr="1">
               <xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
               <xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:9a:00.0" count="4" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
-        <pci busid="0000:98:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
+        <pci busid="0000:98:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
           <pci busid="0000:9a:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
             <gpu dev="15" sm="90" gcn="910" arch="38911" rank="15" gdr="1">
               <xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
-              <xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
+              <xgmi target="0000:da:00.0" count="2" tclass="0x038000"/>
+              <xgmi target="0000:97:00.0" count="4" tclass="0x038000"/>
             </gpu>
           </pci>
         </pci>
         <pci busid="0000:9b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
           <nic>
-            <net name="mlx5_8" dev="7" speed="200000" port="1" guid="0x6c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
+            <net name="mlx5_8" dev="7" speed="200000" port="1" guid="0xc042bb0003f6ceb8" maxconn="262144" gdr="1"/>
           </nic>
         </pci>
       </pci>