AI/rocm-systems
2
0
Forkuj 0
Kod Zgłoszenia Oczekujące zmiany Actions Packages Projekty Wydania Wiki Aktywność

Update Rome models matching (#376)

Przeglądaj źródła 
[ROCm/rccl commit: 4c83adb75c]
This commit is contained in:
Wenkai Du
2021-05-25 10:12:40 -07:00
zatwierdzone przez GitHub
rodzic ea17b05518
commit aa95cc6102
4 zmienionych plików z 168 dodań i 7 usunięć
Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files
projects/rccl/src/graph/rome_models.cc
+40 -6
Wyświetl plik
@@ -27,6 +27,7 @@ THE SOFTWARE.
#include <sys/time.h>
#include "rome_models.h"
#define MAX_ROME_CPUS 8
#define MAX_ROME_GPUS 16
#define MAX_ROME_NICS 8
@@ -172,7 +173,7 @@ static struct rcclRomeModel rome_model_38 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { },
.connMatrix = { 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, },
.pattern = "00102010002010",
.pattern = "10201000201010",
.ringBase = "6 7 1 4 3 5 2 0|0 2 5 3 4 1 7 6",
};
@@ -205,7 +206,7 @@ static struct rcclRomeModel rome_model_42 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { 4, },
.connMatrix = { 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, },
.pattern = "00102010012010",
.pattern = "10201001201010",
.ringBase = "7 4 6 1 3 0 2 5|6 4 7 1 3 2 5 0",
};
@@ -227,7 +228,7 @@ static struct rcclRomeModel rome_model_45 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { },
.connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
.pattern = "00102010002010",
.pattern = "10201000201010",
.ringBase = "0 1 2 3 4 5 6 7|0 2 5 7 4 6 1 3|0 3 1 6 4 7 5 2|0 7 6 5 4 3 2 1",
};
@@ -238,7 +239,7 @@ static struct rcclRomeModel rome_model_46 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { 4, },
.connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
.pattern = "00102010012010",
.pattern = "10201001201010",
.ringBase = "6 5 7 4 1 2 3 0|7 4 6 5 1 0 3 2",
};
@@ -275,6 +276,17 @@ static struct rcclRomeModel rome_model_52 = {
.ringBase = "0 1 3 2 4 5 7 6|6 7 5 4 2 3 1 0|0 1 5 4 6 7 3 2|2 3 7 6 4 5 1 0",
};
static struct rcclRomeModel rome_model_53 = {
.nGpus = 8, .nCpus = 4, .nNics = 4, .nLinks = 3,
.gpuIds = { 0x4a000, 0x50000, 0xa000, 0xf000, 0xcb000, 0xd1000, 0x8a000, 0x90000, },
.nicIds = { 0x45000, 0x13000, 0xc6000, 0x85000, },
.gpuNuma = { 1, 1, 3, 3, 5, 5, 7, 7, },
.nicNuma = { 1, 3, 5, 7, },
.connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
.pattern = "21212121",
.ringBase = "N0 0 1 2 3 4 5 6 7 N3|N3 7 6 5 4 3 2 1 0 N0|N1 2 3 0 1 6 7 4 5 N2|N2 5 4 7 6 1 0 3 2 N1",
};
static struct rcclRomeModel romeTopoModels[] = {
rome_model_22,
rome_model_25,
@@ -297,6 +309,7 @@ static struct rcclRomeModel romeTopoModels[] = {
rome_model_48,
rome_model_49,
rome_model_52,
rome_model_53,
};
/* Parse user defined rings. Format is like :
@@ -483,6 +496,17 @@ static int cmpIds(const void * g1, const void * g2) {
return s1->dev - s2->dev;
}
struct ncclCpuNuma {
int c;
uint64_t numa;
};
static int cmpNuma(const void * g1, const void * g2) {
struct ncclCpuNuma *s1 = (struct ncclCpuNuma*)g1;
struct ncclCpuNuma *s2 = (struct ncclCpuNuma*)g2;
return s1->numa - s2->numa;
}
static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRomeModel* romeTopo, char *pattern) {
pattern[0] = 0; // pattern will be NULL for invalid topology
romeTopo->nGpus = system->nodes[GPU].count;
@@ -496,6 +520,14 @@ static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRo
scores[i].dev = system->nodes[GPU].nodes[i].gpu.dev;
}
qsort(scores, romeTopo->nGpus, sizeof(struct ncclGpuIdHIP), cmpIds);
// sort CPU devices by NUMA id
struct ncclCpuNuma cpu_scores[MAX_ROME_CPUS];
for (int i = 0; i < romeTopo->nCpus; i ++) {
cpu_scores[i].c = i;
cpu_scores[i].numa = system->nodes[CPU].nodes[i].id;
}
qsort(cpu_scores, romeTopo->nCpus, sizeof(struct ncclCpuNuma), cmpNuma);
for (int i = 0; i < romeTopo->nGpus; i ++) {
int gpu, n, m, distance;
gpu = scores[i].g;
@@ -530,6 +562,7 @@ static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRo
for (int net = 0; net < romeTopo->nNics; net++) {
int n, m, distance;
m = 0;
romeTopo->nicIds[net] = system->nodes[NET].nodes[net].net.busId;
distance = system->nodes[NET].nodes[net].paths[CPU][m].count;
for (n = 0; n < romeTopo->nCpus; n++)
if (system->nodes[NET].nodes[net].paths[CPU][n].count < distance) {
@@ -542,11 +575,12 @@ static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRo
// number of GPUs and NICs on each numa node is used as first screening pattern
for (int i = 0; i < romeTopo->nCpus; i++) {
uint64_t id = system->nodes[CPU].nodes[cpu_scores[i].c].id;
int g = 0, n = 0;
for (int j = 0; j < romeTopo->nGpus; j++)
if (romeTopo->gpuNuma[j] == i) g++;
if (romeTopo->gpuNuma[j] == id) g++;
for (int j = 0; j < romeTopo->nNics; j++)
if (romeTopo->nicNuma[j] == i) n++;
if (romeTopo->nicNuma[j] == id) n++;
pattern[i*2] = '0' + g;
pattern[i*2+1] = '0' + n;
}
projects/rccl/tools/scripts/topo_val.sh
+1 -1
Wyświetl plik
@@ -21,7 +21,7 @@
DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
for i in {0..52}
for i in {0..53}
do
if [[ $i -eq 50 ]] || [[ $i -eq 51 ]]
then
projects/rccl/tools/topo_expl/models/topo_8p_rome_4n_1.xml
+126
Wyświetl plik
@@ -0,0 +1,126 @@
<system version="2">
<cpu numaid="1" affinity="00000000,00000000,00000000,ffff0000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:43:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:46:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:48:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4a:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="0" sm="90" gcn="908" arch="38911" rank="0" gdr="1">
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:4c:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:4e:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:50:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="1" sm="90" gcn="908" arch="38911" rank="1" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:45:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_1" dev="0" speed="200000" port="1" guid="0x30654a0003a1420c" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="3" affinity="00000000,00000000,ffff0000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:03:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:05:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:08:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0a:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="2" sm="90" gcn="908" arch="38911" rank="2" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0f:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:0b:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0d:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:0f:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="3" sm="90" gcn="908" arch="38911" rank="3" gdr="1">
<xgmi target="0000:4a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:50:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:0a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:13:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_3" dev="1" speed="200000" port="1" guid="0x906b4a0003a1420c" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="5" affinity="00000000,ffff0000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:c4:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c7:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:c9:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cb:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="4" sm="90" gcn="908" arch="38911" rank="4" gdr="1">
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:cd:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:cf:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:d1:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="5" sm="90" gcn="908" arch="38911" rank="5" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:c6:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_5" dev="2" speed="200000" port="1" guid="0xb0f3b100039f59b8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
<cpu numaid="7" affinity="ffff0000,00000000,00000000,00000000" arch="x86_64" vendor="AuthenticAMD" familyid="143" modelid="49">
<pci busid="0000:83:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:86:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:88:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8a:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="6" sm="90" gcn="908" arch="38911" rank="6" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:90:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:8c:00.0" class="0x060400" vendor="0x11f8" device="0x4000" subsystem_vendor="0x11f8" subsystem_device="0xbeef" link_speed="16 GT/s" link_width="16">
<pci busid="0000:8e:00.0" class="0x060400" vendor="0x1002" device="0x14a0" subsystem_vendor="0x0000" subsystem_device="0x0000" link_speed="16 GT/s" link_width="16">
<pci busid="0000:90:00.0" class="0x038000" vendor="0x1002" device="0x738c" subsystem_vendor="0x1002" subsystem_device="0x0c34" link_speed="16 GT/s" link_width="16">
<gpu dev="7" sm="90" gcn="908" arch="38911" rank="7" gdr="1">
<xgmi target="0000:cb:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:d1:00.0" count="1" tclass="0x038000"/>
<xgmi target="0000:8a:00.0" count="1" tclass="0x038000"/>
</gpu>
</pci>
</pci>
</pci>
<pci busid="0000:85:00.0" class="0x020700" vendor="0x15b3" device="0x101b" subsystem_vendor="0x15b3" subsystem_device="0x0009" link_speed="16 GT/s" link_width="16">
<nic>
<net name="mlx5_7" dev="3" speed="200000" port="1" guid="0x50815600039f59b8" maxconn="262144" gdr="1"/>
</nic>
</pci>
</pci>
</cpu>
</system>
projects/rccl/tools/topo_expl/topo_expl.cpp
+1
Wyświetl plik
@@ -122,6 +122,7 @@ NodeModelDesc model_descs[] = {
{4, "topo_collnet_n1.xml", "4 nodes collnet 1 NICs"},
{4, "topo_collnet_n4.xml", "4 nodes collnet 4 NICs"},
{1, "topo_8p_90a.xml", "single node gfx90a"},
{4, "topo_8p_rome_4n_1.xml", "4 nodes node 8 gfx908 Rome 4 NICs NPS=4"},
};
int main(int argc,char* argv[])