AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[TUNING] gfx950 16N tuning (#1835)

Browse Source 
* change gfx950 algo/proto selection for multinode allreduce, allgather, reduceScatter
* gfx950 tuning: enable tuning for broadcast, allreduce starts LL128 earlier and switches to ring earlier, change LL128 start for allgather and reduceScatter
* lower LL128 threshold
* update reduceScatter LL128 min to match LL max for consistency
* enable multinode PXN and increase chunksize for gfx950
* change LL128 start to 128KB, adjust ring-start according to node-count
* disable code-path for fused-AR on LL128 for gfx950
* use LL128 starting from 1KB for multinode allgather on gfx950
* start LL128 earlier for multinode reduceScatter on gfx950
* start LL128 earlier for multinode broadcast on gfx950
* set multinode allreduce to start simple on 64MB for gfx950
* start LL128 from 1KB for multinode broadcast on gfx950
* setting multinode AR to use tree instead of ring at 16MB, 64MB, 128MB
* set multinode broadcast to use LL for up to 256KB depending on node-count for gfx950
* adjust algo for 32MB  multinode allreduce on gfx950
* make 32MB tree LL128 for multinode AR on gfx950
* make sure ring is not picked on 2N allreduce on small sizes
This commit is contained in:
isaki001
2025-08-15 15:12:45 -05:00
committed by GitHub
parent 1aa2570b48
commit 44121db890
3 changed files with 35 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/include/rccl_common.h
+5 -2
View File
@@ -31,7 +31,8 @@ typedef enum RcclTunableColls {
RCCL_AG_TUNABLE = 1, // all_gather index
RCCL_AR_TUNABLE = 2, // all_reduce index
RCCL_RE_TUNABLE = 3, // reduce index
RCCL_TUNABLE_COLLS = 4 // LL/LL64/LL128 tunable collectives count
RCCL_BR_TUNABLE = 4, // broadcast index
RCCL_TUNABLE_COLLS = 5 // LL/LL64/LL128 tunable collectives count
} rcclTunableIndex_t;
#define RCCL_LL_LIMITS_UNDEFINED 0
@@ -66,6 +67,8 @@ inline rcclTunableIndex_t rcclGetTunableIndex(ncclFunc_t const& func) {
return RCCL_AR_TUNABLE;
case ncclFuncReduce:
return RCCL_RE_TUNABLE;
case ncclFuncBroadcast:
return RCCL_BR_TUNABLE;
default:
return RCCL_UNSUPPORTED_TUNABLE; // Invalid or unsupported function
}
@@ -76,7 +79,7 @@ inline size_t rcclGetSizePerRank(ncclFunc_t const& func, size_t const& nBytes, i
// For AG, this is the send size per rank
// For RS, this is the recv size per rank
// For AR, this is the send/recv size per rank
return (func == ncclFuncReduceScatter || func == ncclFuncAllGather) ? nBytes / nRanks : nBytes;
return (func == ncclFuncReduceScatter || func == ncclFuncAllGather || func == ncclFuncBroadcast) ? nBytes / nRanks : nBytes;
}
void rcclUpdateCollectiveProtocol(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info);
void rcclUpdateThreadThreshold(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info, int& threadThreshold);