AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Rome models matching (#376)

Browse Source
This commit is contained in:
Wenkai Du
2021-05-25 10:12:40 -07:00
committed by GitHub
parent 8e817ecd6d
commit 4c83adb75c
4 changed files with 168 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graph/rome_models.cc
+40 -6
View File
@@ -27,6 +27,7 @@ THE SOFTWARE.
#include <sys/time.h>
#include "rome_models.h"
#define MAX_ROME_CPUS 8
#define MAX_ROME_GPUS 16
#define MAX_ROME_NICS 8
@@ -172,7 +173,7 @@ static struct rcclRomeModel rome_model_38 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { },
.connMatrix = { 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, },
.pattern = "00102010002010",
.pattern = "10201000201010",
.ringBase = "6 7 1 4 3 5 2 0|0 2 5 3 4 1 7 6",
};
@@ -205,7 +206,7 @@ static struct rcclRomeModel rome_model_42 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { 4, },
.connMatrix = { 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, },
.pattern = "00102010012010",
.pattern = "10201001201010",
.ringBase = "7 4 6 1 3 0 2 5|6 4 7 1 3 2 5 0",
};
@@ -227,7 +228,7 @@ static struct rcclRomeModel rome_model_45 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { },
.connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
.pattern = "00102010002010",
.pattern = "10201000201010",
.ringBase = "0 1 2 3 4 5 6 7|0 2 5 7 4 6 1 3|0 3 1 6 4 7 5 2|0 7 6 5 4 3 2 1",
};
@@ -238,7 +239,7 @@ static struct rcclRomeModel rome_model_46 = {
.gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, },
.nicNuma = { 4, },
.connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
.pattern = "00102010012010",
.pattern = "10201001201010",
.ringBase = "6 5 7 4 1 2 3 0|7 4 6 5 1 0 3 2",
};
@@ -275,6 +276,17 @@ static struct rcclRomeModel rome_model_52 = {
.ringBase = "0 1 3 2 4 5 7 6|6 7 5 4 2 3 1 0|0 1 5 4 6 7 3 2|2 3 7 6 4 5 1 0",
};
static struct rcclRomeModel rome_model_53 = {
.nGpus = 8, .nCpus = 4, .nNics = 4, .nLinks = 3,
.gpuIds = { 0x4a000, 0x50000, 0xa000, 0xf000, 0xcb000, 0xd1000, 0x8a000, 0x90000, },
.nicIds = { 0x45000, 0x13000, 0xc6000, 0x85000, },
.gpuNuma = { 1, 1, 3, 3, 5, 5, 7, 7, },
.nicNuma = { 1, 3, 5, 7, },
.connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, },
.pattern = "21212121",
.ringBase = "N0 0 1 2 3 4 5 6 7 N3|N3 7 6 5 4 3 2 1 0 N0|N1 2 3 0 1 6 7 4 5 N2|N2 5 4 7 6 1 0 3 2 N1",
};
static struct rcclRomeModel romeTopoModels[] = {
rome_model_22,
rome_model_25,
@@ -297,6 +309,7 @@ static struct rcclRomeModel romeTopoModels[] = {
rome_model_48,
rome_model_49,
rome_model_52,
rome_model_53,
};
/* Parse user defined rings. Format is like :
@@ -483,6 +496,17 @@ static int cmpIds(const void * g1, const void * g2) {
return s1->dev - s2->dev;
}
struct ncclCpuNuma {
int c;
uint64_t numa;
};
static int cmpNuma(const void * g1, const void * g2) {
struct ncclCpuNuma *s1 = (struct ncclCpuNuma*)g1;
struct ncclCpuNuma *s2 = (struct ncclCpuNuma*)g2;
return s1->numa - s2->numa;
}
static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRomeModel* romeTopo, char *pattern) {
pattern[0] = 0; // pattern will be NULL for invalid topology
romeTopo->nGpus = system->nodes[GPU].count;
@@ -496,6 +520,14 @@ static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRo
scores[i].dev = system->nodes[GPU].nodes[i].gpu.dev;
}
qsort(scores, romeTopo->nGpus, sizeof(struct ncclGpuIdHIP), cmpIds);
// sort CPU devices by NUMA id
struct ncclCpuNuma cpu_scores[MAX_ROME_CPUS];
for (int i = 0; i < romeTopo->nCpus; i ++) {
cpu_scores[i].c = i;
cpu_scores[i].numa = system->nodes[CPU].nodes[i].id;
}
qsort(cpu_scores, romeTopo->nCpus, sizeof(struct ncclCpuNuma), cmpNuma);
for (int i = 0; i < romeTopo->nGpus; i ++) {
int gpu, n, m, distance;
gpu = scores[i].g;
@@ -530,6 +562,7 @@ static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRo
for (int net = 0; net < romeTopo->nNics; net++) {
int n, m, distance;
m = 0;
romeTopo->nicIds[net] = system->nodes[NET].nodes[net].net.busId;
distance = system->nodes[NET].nodes[net].paths[CPU][m].count;
for (n = 0; n < romeTopo->nCpus; n++)
if (system->nodes[NET].nodes[net].paths[CPU][n].count < distance) {
@@ -542,11 +575,12 @@ static ncclResult_t parseRomeSystem(struct ncclTopoSystem* system, struct rcclRo
// number of GPUs and NICs on each numa node is used as first screening pattern
for (int i = 0; i < romeTopo->nCpus; i++) {
uint64_t id = system->nodes[CPU].nodes[cpu_scores[i].c].id;
int g = 0, n = 0;
for (int j = 0; j < romeTopo->nGpus; j++)
if (romeTopo->gpuNuma[j] == i) g++;
if (romeTopo->gpuNuma[j] == id) g++;
for (int j = 0; j < romeTopo->nNics; j++)
if (romeTopo->nicNuma[j] == i) n++;
if (romeTopo->nicNuma[j] == id) n++;
pattern[i*2] = '0' + g;
pattern[i*2+1] = '0' + n;
}