AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trim NICs when all GPUs are connected by XGMI (#430)

Browse Source 
* Trim NICs when all GPUs are connected by XGMI

* Only enable clique with maximum of 2 hops

[ROCm/rccl commit: 29c729d8b6]
This commit is contained in:
Wenkai Du
2021-10-05 18:27:43 -07:00
committed by GitHub
parent 55fa23a613
commit 91eca0d7d2
6 changed files with 159 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/graph/paths.cc
+16 -2
View File
@@ -510,8 +510,22 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm*
}
if (ret) {
system->type |= RCCL_TOPO_GDR_ALL;
remove = 0;
INFO(NCCL_GRAPH, "GDR is available on all GPUs");
bool allXgmi = true;
// don't trim NICs unless all GPUs are connected by XGMI
for (int i = 0; i < system->nodes[GPU].count && allXgmi; i++) {
int cudaDev1 = system->nodes[GPU].nodes[i].gpu.dev;
for (int j = 0; j < system->nodes[GPU].count && allXgmi; j++) {
if (i == j) continue;
int cudaDev2 = system->nodes[GPU].nodes[j].gpu.dev;
bool isXGMI;
NCCLCHECK(ncclTopoGetLinkType(comm->topo, cudaDev1, cudaDev2, &isXGMI));
allXgmi &= isXGMI;
}
}
if (!allXgmi) {
remove = 0;
INFO(NCCL_GRAPH, "GDR is available on all GPUs");
}
}
}
comm->localRanks = system->nodes[GPU].count;
projects/rccl/src/graph/search.cc
+34 -9
View File
@@ -1046,7 +1046,8 @@ ncclResult_t ncclTopoGetIntraNetDev(struct ncclTopoSystem* system, int rank, str
return ncclSuccess;
}
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, bool direct_only) {
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, int maxInter, int nInter, int *inter) {
int interGpus[MAX_XGMI_INTER_GPUS+1];
int ngpus = system->nodes[GPU].count;
*isXGMI = false;
// check for direct XGMI connection
@@ -1065,15 +1066,39 @@ ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, in
}
}
}
if (direct_only) return ncclSuccess;
// check if there is intermediate GPU that is connected to both
if (maxInter == 0) return ncclSuccess;
// check if there are intermediate GPUs that are connected to both
bool res1, res2, res3;
int j;
for (j=0; j<nInter; j++) {
bool res1;
ncclTopoGetLinkType(system, inter[j], inter[j+1], &res1, 0);
if (!res1) break;
}
if (j<nInter) return ncclSuccess;
if (nInter > 0 && inter != nullptr) {
ncclTopoGetLinkType(system, inter[nInter], cudaDev2, &res2, 0);
if (res2) {
*isXGMI = true;
return ncclSuccess;
}
memcpy(interGpus+1, inter+1, sizeof(int)*nInter);
}
interGpus[0] = cudaDev1;
// add one more intermediate GPU recursively util reaching max depth
nInter++;
if (nInter+2 > ngpus || nInter > MAX_XGMI_INTER_GPUS || nInter > maxInter) return ncclSuccess;
for (int i=0; i<ngpus; i++) {
if (system->nodes[GPU].nodes[i].gpu.dev == cudaDev1 || system->nodes[GPU].nodes[i].gpu.dev == cudaDev2)
continue;
bool res1, res2;
ncclTopoGetLinkType(system, system->nodes[GPU].nodes[i].gpu.dev, cudaDev1, &res1, true);
ncclTopoGetLinkType(system, system->nodes[GPU].nodes[i].gpu.dev, cudaDev2, &res2, true);
if (res1 && res2) {
int dev = system->nodes[GPU].nodes[i].gpu.dev;
// skip duplicated GPU
if (dev == cudaDev2) continue;
for (j=0; j<nInter; j++)
if (dev == interGpus[j]) break;
if (j<nInter) continue;
// check connectivity with intermediate GPUs
interGpus[nInter] = dev;
ncclTopoGetLinkType(system, cudaDev1, cudaDev2, &res3, maxInter, nInter, interGpus);
if (res3) {
*isXGMI = true;
return ncclSuccess;
}
projects/rccl/src/include/graph.h
+2 -1
View File
@@ -34,8 +34,9 @@ ncclResult_t ncclTopoGetNvbGpus(struct ncclTopoSystem* system, int rank, int* nr
ncclResult_t ncclTopoGetNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int rr, int* net);
ncclResult_t ncclTopoCheckP2p(struct ncclTopoSystem* system, int64_t id1, int64_t id2, int* p2p, int *read, int* intermediateRank);
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* topo, int64_t busId, int netDev, int read, int* useGdr);
#define MAX_XGMI_INTER_GPUS 4
ncclResult_t ncclTopoGetIntraNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int type, int* dev);
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, bool direct_only=false);
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, int maxInter=MAX_XGMI_INTER_GPUS, int nInter=0, int *inter=nullptr);
// Find CPU affinity
ncclResult_t ncclTopoGetCpuAffinity(struct ncclTopoSystem* system, int rank, cpu_set_t* affinity);
projects/rccl/src/init.cc
+3 -2
View File
@@ -836,7 +836,7 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
bool allXgmi = true;
{ // [RCCL] Check if clique-based kernels can be enabled and initialize CliqueManager
CliqueManager::cliqueMode_t cliqueMode = CliqueManager::CLIQUE_DISABLED;
if (comm->localRanks == comm->nRanks)
if (comm->localRanks == comm->nRanks && comm->topo->nodes[GPU].nodes[0].gpu.gcn != 910)
{
// Check that all the GPUs have peer access to one another and are XGMI connected
bool hasPeerAccess = true;
@@ -855,7 +855,8 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
}
bool isXGMI;
NCCLCHECK(ncclTopoGetLinkType(comm->topo, i, j, &isXGMI));
// Limit to single intermediate GPU for enabling clique
NCCLCHECK(ncclTopoGetLinkType(comm->topo, i, j, &isXGMI, 1));
allXgmi &= isXGMI;
}
}
projects/rccl/tools/topo_expl/models/topo_16p1h.xml
+102 -62
View File
@@ -1,19 +1,19 @@
<system version="2">
<cpu numaid="0" affinity="00000000,00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:41:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:49:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:4e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="0">
<pci busid="0000:41:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:49:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:4c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:4e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="0" sm="90" gcn="910" arch="38911" rank="0" gdr="1">
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:4f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:51:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="0">
<pci busid="0000:4f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:51:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="1" sm="90" gcn="910" arch="38911" rank="1" gdr="1">
<xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
@@ -21,11 +21,16 @@
</gpu>
</pci>
</pci>
<pci busid="0000:4b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_0" dev="0" speed="200000" port="1" guid="0xb03fbb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:52:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:54:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:56:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="0">
<pci busid="0000:52:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:54:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:56:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="2" sm="90" gcn="910" arch="38911" rank="2" gdr="1">
<xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
@@ -33,33 +38,38 @@
</gpu>
</pci>
</pci>
<pci busid="0000:57:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:59:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="3" sm="90" gcn="910" arch="38911" rank="3" gdr="0">
<pci busid="0000:57:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:59:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="3" sm="90" gcn="910" arch="38911" rank="3" gdr="1">
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:5a:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_1" dev="1" speed="200000" port="1" guid="0xfc11b30003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</pci>
</cpu>
<cpu numaid="1" affinity="00000000,00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:01:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:09:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:0e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="4" sm="90" gcn="910" arch="38911" rank="4" gdr="0">
<pci busid="0000:01:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:09:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:0c:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:0e:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="4" sm="90" gcn="910" arch="38911" rank="4" gdr="1">
<xgmi target="0000:4e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:0f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:11:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="5" sm="90" gcn="910" arch="38911" rank="5" gdr="0">
<pci busid="0000:0f:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:11:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="5" sm="90" gcn="910" arch="38911" rank="5" gdr="1">
<xgmi target="0000:51:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
@@ -67,16 +77,16 @@
</gpu>
</pci>
</pci>
<pci busid="0000:0b:00.0" class="0x020000" vendor="0x8086" device="0x10d3" subsystem_vendor="0x8086" subsystem_device="0xa01f" link_speed="2.5 GT/s" link_width="1">
<pci busid="0000:0b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="enp11s0" dev="0" speed="1000" port="0" guid="0x0" maxconn="65536" gdr="0"/>
<net name="mlx5_2" dev="2" speed="200000" port="1" guid="0xc83fbb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:12:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:14:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:16:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="6" sm="90" gcn="910" arch="38911" rank="6" gdr="0">
<pci busid="0000:12:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:14:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:16:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="6" sm="90" gcn="910" arch="38911" rank="6" gdr="1">
<xgmi target="0000:56:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
@@ -84,33 +94,38 @@
</gpu>
</pci>
</pci>
<pci busid="0000:17:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:19:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="7" sm="90" gcn="910" arch="38911" rank="7" gdr="0">
<pci busid="0000:17:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:19:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="7" sm="90" gcn="910" arch="38911" rank="7" gdr="1">
<xgmi target="0000:59:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:1a:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_3" dev="3" speed="200000" port="1" guid="0x3440bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</pci>
</cpu>
<cpu numaid="2" affinity="00000000,ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:c2:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:ca:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cd:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:cf:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="8" sm="90" gcn="910" arch="38911" rank="8" gdr="0">
<pci busid="0000:c2:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:ca:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:cd:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:cf:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="8" sm="90" gcn="910" arch="38911" rank="8" gdr="1">
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:d0:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:d2:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="9" sm="90" gcn="910" arch="38911" rank="9" gdr="0">
<pci busid="0000:d0:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:d2:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="9" sm="90" gcn="910" arch="38911" rank="9" gdr="1">
<xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
@@ -118,11 +133,16 @@
</gpu>
</pci>
</pci>
<pci busid="0000:cc:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_4" dev="4" speed="200000" port="1" guid="0xb411b30003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:d3:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:d5:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:d7:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="10" sm="90" gcn="910" arch="38911" rank="10" gdr="0">
<pci busid="0000:d3:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:d5:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:d7:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="10" sm="90" gcn="910" arch="38911" rank="10" gdr="1">
<xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
@@ -130,33 +150,38 @@
</gpu>
</pci>
</pci>
<pci busid="0000:d8:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:da:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="11" sm="90" gcn="910" arch="38911" rank="11" gdr="0">
<pci busid="0000:d8:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:da:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="11" sm="90" gcn="910" arch="38911" rank="11" gdr="1">
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:9a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:db:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_5" dev="5" speed="200000" port="1" guid="0x4040bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</pci>
</cpu>
<cpu numaid="3" affinity="ffffffff,00000000,00000000,00000000,ffffffff,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:82:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8a:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8d:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:8f:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="12" sm="90" gcn="910" arch="38911" rank="12" gdr="0">
<pci busid="0000:82:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:8a:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:8d:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:8f:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="12" sm="90" gcn="910" arch="38911" rank="12" gdr="1">
<xgmi target="0000:0e:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:cf:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:92:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:90:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:92:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="13" sm="90" gcn="910" arch="38911" rank="13" gdr="0">
<pci busid="0000:90:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:92:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="13" sm="90" gcn="910" arch="38911" rank="13" gdr="1">
<xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d2:00.0" count="1" tclass="0x038000"/>
@@ -164,11 +189,16 @@
</gpu>
</pci>
</pci>
<pci busid="0000:8c:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_7" dev="7" speed="200000" port="1" guid="0x8c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
<pci busid="0000:93:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16 GT/s" link_width="16">
<pci busid="0000:95:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:97:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="14" sm="90" gcn="910" arch="38911" rank="14" gdr="0">
<pci busid="0000:93:00.0" class="0x060400" vendor="0x1000" device="0xc010" subsystem_vendor="0x1000" subsystem_device="0xa096" link_speed="16.0 GT/s PCIe" link_width="16">
<pci busid="0000:95:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:97:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="14" sm="90" gcn="910" arch="38911" rank="14" gdr="1">
<xgmi target="0000:11:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:16:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d7:00.0" count="1" tclass="0x038000"/>
@@ -176,16 +206,26 @@
</gpu>
</pci>
</pci>
<pci busid="0000:98:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="Unknown speed" link_width="8">
<pci busid="0000:9a:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="Unknown speed" link_width="16">
<gpu dev="15" sm="90" gcn="910" arch="38911" rank="15" gdr="0">
<pci busid="0000:98:00.0" class="0x060400" vendor="0x1022" device="0x14c7" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16.0 GT/s PCIe" link_width="8">
<pci busid="0000:9a:00.0" class="0x038000" vendor="0x1002" device="0x740c" subsystem_vendor="0x1002" subsystem_device="0x0b0c" link_speed="32.0 GT/s PCIe" link_width="16">
<gpu dev="15" sm="90" gcn="910" arch="38911" rank="15" gdr="1">
<xgmi target="0000:19:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:da:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:97:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
<pci busid="0000:9b:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0007" link_speed="16.0 GT/s PCIe" link_width="16">
<nic>
<net name="mlx5_8" dev="8" speed="200000" port="1" guid="0x6c40bb0003f6ceb8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</pci>
<pci busid="0000:b1:00.0" class="0x020000" vendor="0x15b3" device="0x1015" subsystem_vendor="0x15b3" subsystem_device="0x0190" link_speed="8.0 GT/s PCIe" link_width="8">
<nic>
<net name="mlx5_6" dev="6" speed="40000" port="1" guid="0x100fe0b7312aea4" maxconn="262144" gdr="1"/>
</nic>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/utils.cpp
+2 -2
View File
@@ -576,7 +576,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
bool allXgmi = true;
{ // [RCCL] Check if clique-based kernels can be enabled and initialize CliqueManager
//CliqueManager::cliqueMode_t cliqueMode = CliqueManager::CLIQUE_DISABLED;
if (comm->localRanks == comm->nRanks)
if (comm->localRanks == comm->nRanks && comm->topo->nodes[GPU].nodes[0].gpu.gcn != 910)
{
// Check that all the GPUs have peer access to one another and are XGMI connected
bool hasPeerAccess = true;
@@ -595,7 +595,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
//}
bool isXGMI;
NCCLCHECK(ncclTopoGetLinkType(comm->topo, i, j, &isXGMI));
NCCLCHECK(ncclTopoGetLinkType(comm->topo, i, j, &isXGMI, 1));
allXgmi &= isXGMI;
}
}