AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Query XGMI link count through rocm_smi_lib API (#442)

Browse Source
This commit is contained in:
Wenkai Du
2021-10-26 10:30:20 -07:00
committed by GitHub
parent d23dfc12c1
commit 14a184eb67
4 changed files with 67 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graph/xml.cc
+3 -3
View File
@@ -671,8 +671,8 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvm
for (int i=0; i<deviceCnt; i++) {
if (i != dev) {
RSMI_IO_LINK_TYPE rsmi_type;
int hops, bw;
if (rocm_smi_getLinkInfo(dev, i, &rsmi_type, &hops, &bw) == ncclSuccess) {
int hops, count;
if (rocm_smi_getLinkInfo(dev, i, &rsmi_type, &hops, &count) == ncclSuccess) {
if (rsmi_type == RSMI_IOLINK_TYPE_XGMI && hops == 1) {
char busIdStr[] = "00000000:00:00.0";
CUDACHECK(hipDeviceGetPCIBusId(busIdStr, sizeof(busIdStr), i));
@@ -685,7 +685,7 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvm
if (nvlNode == NULL) {
NCCLCHECK(xmlAddNode(xml, gpuNode, "xgmi", &nvlNode));
NCCLCHECK(xmlSetAttr(nvlNode, "target", lowerId));
NCCLCHECK(xmlSetAttrInt(nvlNode, "count", 1));
NCCLCHECK(xmlSetAttrInt(nvlNode, "count", count));
}
}
}
src/include/rocm_smi_wrap.h
+2 -1
View File
@@ -24,10 +24,11 @@ THE SOFTWARE.
#define ROCM_SMI_WRAP_H_
#include "rocm_smi/rocm_smi.h"
#include "rocm_smi/rocm_smi64Config.h"
#include "nccl.h"
ncclResult_t rocm_smi_init();
ncclResult_t rocm_smi_getDeviceIndexByPciBusId(const char* pciBusId, uint32_t* deviceIndex);
ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *bw);
ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *count);
#endif
src/misc/rocm_smi_wrap.cc
+14 -4
View File
@@ -74,7 +74,7 @@ ncclResult_t rocm_smi_getDeviceIndexByPciBusId(const char* pciBusId, uint32_t* d
}
}
ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *bw) {
ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *count) {
char srcStr[] = "00000000:00:00.0", dstStr[] = "00000000:00:00.0";
uint32_t srcIndex, dstIndex;
@@ -86,9 +86,19 @@ ncclResult_t rocm_smi_getLinkInfo(int srcDev, int dstDev, RSMI_IO_LINK_TYPE* rsm
uint64_t rsmi_hops, rsmi_weight;
ROCMSMICHECK(rsmi_topo_get_link_type(srcIndex, dstIndex, &rsmi_hops, rsmi_type));
ROCMSMICHECK(rsmi_topo_get_link_weight(srcIndex, dstIndex, &rsmi_weight));
*hops = 2;
*bw = 0;
if (*rsmi_type == RSMI_IOLINK_TYPE_XGMI && rsmi_weight == 15) *hops = 1;
*count = 1;
if (*rsmi_type == RSMI_IOLINK_TYPE_XGMI && rsmi_weight == 15) {
*hops = 1;
#if rocm_smi_VERSION_MAJOR >= 5
uint64_t min_bw = 0, max_bw = 0;
rsmi_version_t version;
ROCMSMICHECK(rsmi_version_get(&version));
if (version.major >= 5)
ROCMSMICHECK(rsmi_minmax_bandwidth_get(srcIndex, dstIndex, &min_bw, &max_bw));
if (max_bw && min_bw)
*count = max_bw/min_bw;
#endif
}
return ncclSuccess;
}
tools/topo_expl/models/topo_16p1h.xml
+48 -48
View File
@@ -2,19 +2,19 @@
<cpu numaid="0" affinity="00000000,00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:41:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:49:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:4c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:4c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:4e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1">
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:51:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0e:00.0" count="2" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:4f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:4f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:51:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1">
<xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:4e:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
@@ -23,33 +23,33 @@
</pci>
<pci busid="0000:4b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_0" dev="0" speed="200000" port="1" guid="0xb03fbb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_0" dev="0" speed="200000" port="1" guid="0xd041bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:52:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:54:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:54:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:56:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1">
<xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:59:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:57:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:57:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:59:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="3" sm="90" gcn="910" arch="38911" rank="3" gdr="1">
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:56:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:19:00.0" count="2" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:5a:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_1" dev="1" speed="200000" port="1" guid="0xfc11b30003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_1" dev="1" speed="200000" port="1" guid="0x2085ba0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
@@ -58,20 +58,20 @@
<cpu numaid="1" affinity="00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:01:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:09:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:0c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:0c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:0e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="4" sm="90" gcn="910" arch="38911" rank="4" gdr="1">
<xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:4e:00.0" count="2" tclass="0x038000"/>
<xgmi target="0000:11:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:0f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:0f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:11:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="5" sm="90" gcn="910" arch="38911" rank="5" gdr="1">
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0e:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
</gpu>
@@ -79,33 +79,33 @@
</pci>
<pci busid="0000:0b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_2" dev="2" speed="200000" port="1" guid="0xc83fbb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_2" dev="2" speed="200000" port="1" guid="0xfc84ba0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:12:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:14:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:14:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:16:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="6" sm="90" gcn="910" arch="38911" rank="6" gdr="1">
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:19:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:17:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:17:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:19:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="7" sm="90" gcn="910" arch="38911" rank="7" gdr="1">
<xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:59:00.0" count="2" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:1a:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_3" dev="3" speed="200000" port="1" guid="0x3440bb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_3" dev="3" speed="200000" port="1" guid="0x1c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
@@ -114,19 +114,19 @@
<cpu numaid="2" affinity="00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:c2:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:ca:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:cd:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:cd:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:cf:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="8" sm="90" gcn="910" arch="38911" rank="8" gdr="1">
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d2:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8f:00.0" count="2" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:d0:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:d0:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:d2:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="9" sm="90" gcn="910" arch="38911" rank="9" gdr="1">
<xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:cf:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
@@ -135,33 +135,33 @@
</pci>
<pci busid="0000:cc:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_4" dev="4" speed="200000" port="1" guid="0xb411b30003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_4" dev="4" speed="200000" port="1" guid="0x40bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:d3:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:d5:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:d5:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:d7:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="10" sm="90" gcn="910" arch="38911" rank="10" gdr="1">
<xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:d8:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:d8:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:da:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="11" sm="90" gcn="910" arch="38911" rank="11" gdr="1">
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="4" tclass="0x038000"/>
<xgmi target="0000:9a:00.0" count="2" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:db:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_5" dev="5" speed="200000" port="1" guid="0x4040bb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_5" dev="5" speed="200000" port="1" guid="0x7c85ba0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
@@ -170,54 +170,54 @@
<cpu numaid="3" affinity="ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:82:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:8a:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:8d:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:8d:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:8f:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="12" sm="90" gcn="910" arch="38911" rank="12" gdr="1">
<xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:cf:00.0" count="2" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="4" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:90:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:90:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:92:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="13" sm="90" gcn="910" arch="38911" rank="13" gdr="1">
<xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8f:00.0" count="4" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:8c:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_7" dev="6" speed="200000" port="1" guid="0x8c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_7" dev="6" speed="200000" port="1" guid="0x85ba0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:93:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:95:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:95:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:97:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="14" sm="90" gcn="910" arch="38911" rank="14" gdr="1">
<xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:9a:00.0" count="4" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:98:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:98:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x1022" subsystem_device="0x14c7" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:9a:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="15" sm="90" gcn="910" arch="38911" rank="15" gdr="1">
<xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="2" tclass="0x038000"/>
<xgmi target="0000:97:00.0" count="4" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:9b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_8" dev="7" speed="200000" port="1" guid="0x6c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
<net name="mlx5_8" dev="7" speed="200000" port="1" guid="0xc042bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>