AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add AllGather LL128 multi-node tuning and include LL cutoff points in tuning models (#1618)

Browse Source 
* Enable LL/LL128 cutoff points in tuning models

* Initializing ll/ll128 model cutoffs for MI300

* Use RCCL_LL_LIMITS_UNDEFINED

---------

Co-authored-by: PedramAlizadeh <pmohamma@amd.com>
This commit is contained in:
Mustafa Abduljabbar
2025-04-02 16:26:23 -04:00
committed by GitHub
parent aace4e27f8
commit 4be06f04d8
4 changed files with 68 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graph/topo.h
+7
View File
@@ -123,6 +123,13 @@ struct ncclTopoLinkList {
#define RCCL_TOPO_FORCE_INTRA 16
#define RCCL_TOPO_XGMI_ALL 32
#define RCCL_LL_TUNABLE_COLLS 4 // LL/LL64/LL128 tunable Collectives
#define RCCL_RS_TUNABLE 0 // reduce_scatter index
#define RCCL_AG_TUNABLE 1 // all_gather index
#define RCCL_AR_TUNABLE 2 // all_reduce index
#define RCCL_RE_TUNABLE 3 // reduce index
#define RCCL_LL_LIMITS_UNDEFINED 0
#define GCN_ARCH_NAME_LEN 16
struct ncclTopoNode {
src/graph/tuning.cc
+27 -1
View File
@@ -64,11 +64,14 @@ static const float baseLat [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] = {
#define NCCL_HW_PCI 1
#define NCCL_HW_NET 2
struct tuningModel {
float hwLat [3][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
float bwRatio [2][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
float treeCorrectionFactor[NCCL_NUM_PROTOCOLS][27];
float ringCorrectionFactor[NCCL_NUM_PROTOCOLS][27];
uint64_t llProtoRanges[RCCL_LL_TUNABLE_COLLS][NCCL_NUM_PROTOCOLS - 1][2];
};
static struct tuningModel tuning_model_0 {
@@ -99,6 +102,8 @@ static struct tuningModel tuning_model_0 {
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.3, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.8, 0.7, 0.5, 0.4, 0.4, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, },
{ 1.0, 0.8, 0.2, 1.0, 1.0, 0.3, 1.0, 0.1, 0.1, 0.2, 0.2, 0.1, 0.5, 1.0, 0.8, 0.8, 1.0, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, },
},
.llProtoRanges = {RCCL_LL_LIMITS_UNDEFINED},
};
static struct tuningModel tuning_model_1 {
@@ -129,6 +134,8 @@ static struct tuningModel tuning_model_1 {
{ 1.0, 0.5, 1.0, 1.0, 0.6, 0.7, 1.0, 1.0, 0.2, 1.0, 0.9, 0.7, 1.0, 1.0, 1.0, 0.9, 0.9, 0.8, 0.8, 0.7, 0.6, 0.5, 0.5, 0.3, 0.2, 0.1, 0.1, },
{ 0.3, 1.0, 0.3, 0.1, 0.1, 0.1, 0.3, 0.7, 1.0, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2, 0.3, 0.5, 0.9, 1.0, 1.0, 1.0, 1.0, },
},
.llProtoRanges = {RCCL_LL_LIMITS_UNDEFINED},
};
static struct tuningModel tuning_model_2 {
@@ -159,6 +166,8 @@ static struct tuningModel tuning_model_2 {
{ 0.1, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.4, 1.0, 1.0, 1.0, 1.0, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.2, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2, 0.4, 0.5, 0.6, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, },
},
.llProtoRanges = {RCCL_LL_LIMITS_UNDEFINED},
};
static struct tuningModel tuning_model_3 {
@@ -189,6 +198,8 @@ static struct tuningModel tuning_model_3 {
{ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.5, 1.0, 0.1, 0.3, 0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.4, 0.7, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, },
},
.llProtoRanges = {RCCL_LL_LIMITS_UNDEFINED},
};
static struct tuningModel tuning_model_4 {
@@ -219,6 +230,8 @@ static struct tuningModel tuning_model_4 {
{ 0.4, 0.5, 0.5, 0.4, 0.4, 0.4, 0.4, 0.2, 0.2, 0.1, 0.3, 1.0, 1.0, 0.7, 0.8, 0.5, 1.0, 1.0, 1.0, 1.0, 1.0, 0.9, 0.8, 0.5, 0.4, 0.3, 0.3, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 1.0, 0.8, 0.5, 0.1, 0.7, 0.2, 0.4, 0.4, 0.6, 0.7, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, },
},
.llProtoRanges = {RCCL_LL_LIMITS_UNDEFINED},
};
static struct tuningModel tuning_model_5 {
@@ -249,6 +262,11 @@ static struct tuningModel tuning_model_5 {
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 0.2, 1.0, 0.4, 0.4, 0.1, 0.2, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, },
{ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.8, 1.0, 1.0, 1.0, },
},
.llProtoRanges = {
/*ReduceScatter*/ {/* LL (Min/Max) */ {0, 655360} , /* LL128 (Min/Max) */ {131072, 3211264}},
/*AllGather*/ {/* LL (Min/Max) */ {0, 98304} , /* LL128 (Min/Max) */ {98304, 5046272}},
},
};
static struct tuningModel rcclTuningModel[] = {
@@ -354,6 +372,14 @@ ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCom
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) intraHw[a] = graphs[a]->typeIntra == LINK_NVL ? NCCL_HW_NVLINK : NCCL_HW_PCI;
for (int a=0; a<NCCL_NUM_ALGORITHMS; a++) hw[a] = nNodes == 1 ? intraHw[a] : NCCL_HW_NET;
memcpy(comm->minMaxLLRange[ncclFuncReduceScatter],
rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_RS_TUNABLE],
sizeof(rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_RS_TUNABLE]));
memcpy(comm->minMaxLLRange[ncclFuncAllGather],
rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_AG_TUNABLE],
sizeof(rcclTuningModel[comm->topo->tuning].llProtoRanges[RCCL_AG_TUNABLE]));
for (int coll=0; coll<NCCL_NUM_FUNCTIONS; coll++) {
int nsteps = coll == ncclFuncAllReduce ? 2*(nRanks-1) :
coll == ncclFuncReduceScatter || coll == ncclFuncAllGather ? nRanks-1 :
@@ -690,4 +716,4 @@ ncclResult_t ncclTopoGetAlgoTime(struct ncclComm* comm, int coll, int algorithm,
int latCount = algorithm == NCCL_ALGO_RING ? numPipeOps : DIVUP(numPipeOps, NCCL_MAX_DEV_WORK_BATCH_COLLS);
*time = lat * latCount + nBytes / (1000 * bw);
return ncclSuccess;
}
}