AI/rocm-systems
2
0
Forka 0
Codice Problemi Pull Request Actions Pacchetti Progetti Rilasci Wiki Attività

all_reduce LL/LL128 and Ring/Tree multi-node tuning for MI300 (#1627)

Sfoglia il codice sorgente 
* Enabling LL128 by default on MI300

* Add missing CUDACHECK

* Adjust BW correction factors to fix the Tree->Ring switching point

* Refactor and add ll128 AR logarithmic factor to tuning models

* Move RCCL tuning changes to a separate file 

* Use enum for tunable indexing

* Use explicit indexing in tuning models to avoid mismatch issues

* Place rcclGetSizePerRank in a function

* Remove HIP ifdef for rccl-only call

---------

Co-authored-by: Mustafa Abduljabbar <mustafa.abduljabbar@amd.com>
This commit is contained in:
Pedram Alizadeh
2025-04-10 11:43:54 -04:00
committato da GitHub
parent 5b36b68d06
commit e40ff4f84a
8 ha cambiato i file con 153 aggiunte e 69 eliminazioni
Scarica il file Patch Scarica il file Diff Expand all files Collapse all files
src/graph/rome_models.cc
+1 -1
Vedi File
@@ -814,7 +814,7 @@ static struct rcclRomeModel rome_model_81 = {
"N7 7 3 2 6 0 4 1 5 N5|"
"N1 1 0 2 4 3 5 7 6 N6|",
.options = "noCpuCheck=1,tuning=5,disableNumaMatching=1",
.options = "noCpuCheck=1,tuning=5,ll128Enabled=1,disableNumaMatching=1",
.treeRail = "N0 0 1 2 4 3 6 5 7 N1|"
"N1 1 0 4 7 3 5 2 6 N0|"
src/graph/topo.h
-6
Vedi File
@@ -123,12 +123,6 @@ struct ncclTopoLinkList {
#define RCCL_TOPO_FORCE_INTRA 16
#define RCCL_TOPO_XGMI_ALL 32
#define RCCL_LL_TUNABLE_COLLS 4 // LL/LL64/LL128 tunable Collectives
#define RCCL_RS_TUNABLE 0 // reduce_scatter index
#define RCCL_AG_TUNABLE 1 // all_gather index
#define RCCL_AR_TUNABLE 2 // all_reduce index
#define RCCL_RE_TUNABLE 3 // reduce index
#define RCCL_LL_LIMITS_UNDEFINED 0
#define GCN_ARCH_NAME_LEN 16
src/graph/tuning.cc
+9 -14
Vedi File
@@ -71,7 +71,7 @@ struct tuningModel {
float bwRatio [2][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
float treeCorrectionFactor[NCCL_NUM_PROTOCOLS][27];
float ringCorrectionFactor[NCCL_NUM_PROTOCOLS][27];
uint64_t llProtoRanges[RCCL_LL_TUNABLE_COLLS][NCCL_NUM_PROTOCOLS - 1][2];
uint64_t llProtoRanges[RCCL_TUNABLE_COLLS][NCCL_NUM_PROTOCOLS - 1][RCCL_PROTOCOL_ENTRY_SIZE];
};
static struct tuningModel tuning_model_0 {
@@ -254,19 +254,18 @@ static struct tuningModel tuning_model_5 {
.treeCorrectionFactor = {
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 1.0, 1.0, 1.0, 1.0, 0.6, 1.0, 0.9, 1.0, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 1.0, 1.0, 1.0, 1.0, 0.6, 1.0, 0.9, 1.0, 1.0, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.7, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.8, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.7, 1.0, 1.0, 1.0, 1.0, 1.0, 0.7, 0.7, 0.5, 0.6, 0.6, 0.6, },
},
.ringCorrectionFactor = {
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.2, 1.0, 0.4, 0.4, 0.1, 0.2, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.2, 1.0, 0.4, 0.4, 0.1, 0.2, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.8, 1.0, 1.0, 1.0, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.8, 1.0, 1.0, 1.0, },
},
.llProtoRanges = {
/*ReduceScatter*/ {/* LL (Min/Max) */ {0, 655360} , /* LL128 (Min/Max) */ {131072, 3211264}},
/*AllGather*/ {/* LL (Min/Max) */ {0, 98304} , /* LL128 (Min/Max) */ {98304, 5046272}},
},
.llProtoRanges[RCCL_RS_TUNABLE] = /*ReduceScatter*/ {/* LL (Min/Max) */ {0, 655360, 1} , /* LL128 (Min/Max) */ {131072, 3211264, 1}},
.llProtoRanges[RCCL_AG_TUNABLE] = /*AllGather*/ {/* LL (Min/Max) */ {0, 98304, 1} , /* LL128 (Min/Max) */ {98304, 5046272, 1}},
.llProtoRanges[RCCL_AR_TUNABLE] = /*AllReduce*/ {/* LL (Min/Max) */ {0, 1048576, 1} , /* LL128 (Min/Max) */ {1048576, 9437184, 3145728}},
};
static struct tuningModel rcclTuningModel[] = {
@@ -372,13 +371,9 @@ ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCom
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) intraHw[a] = graphs[a]->typeIntra == LINK_NVL ? NCCL_HW_NVLINK : NCCL_HW_PCI;
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) hw[a] = nNodes == 1 ? intraHw[a] : NCCL_HW_NET;
memcpy(comm->minMaxLLRange[ncclFuncReduceScatter],
rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_RS_TUNABLE],
sizeof(rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_RS_TUNABLE]));
memcpy(comm->minMaxLLRange[ncclFuncAllGather],
rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_AG_TUNABLE],
sizeof(rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_AG_TUNABLE]));
memcpy(comm->minMaxLLRange,
rcclTuningModel[comm->topo->tuning].llProtoRanges,
sizeof(rcclTuningModel[comm->topo->tuning].llProtoRanges));
for (int coll=0; coll<NCCL_NUM_FUNCTIONS; coll++) {
int nsteps = coll == ncclFuncAllReduce ? 2*(nRanks-1) :