AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Device] Add dynamic fetch/reduce pipelining for reduction collectives - Simple protocol (#1861)

Browse Source 
* Support pipelining codegen and template specialization

* Support ReduceCopy pipelining for AllReduce, ReduceScatter, and Reduce (currently enabled for bfloat16)

* Remove need for FUNC_INDEX_TOTAL

* Add pipeline field to device function key construction logic

* Avoid unneeded codegen for LL/LL64 kernels

* Modify conditions and add pipeline dtypes env

* Optimize selection for both gfx942 and gfx950

* Increase pipeline bitfield width

* Use __forceinline__ for all device functions

* Realign reduceCopy with original form

* Add opt-out option to enable perf debugs

* Remove force-reduce-pipelining option from README

* Update CHANGELOG.md

---------

Co-authored-by: Jeffrey Novotny <jnovotny@amd.com>

[ROCm/rccl commit: 277747c199]
This commit is contained in:
Mustafa Abduljabbar
2025-08-26 15:03:54 -04:00
committed by GitHub
parent c7fce9b0eb
commit f37f290134
18 changed files with 286 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/include/comm.h
+1 -1
View File
@@ -207,7 +207,7 @@ struct ncclTaskColl {
size_t trafficBytes;
int32_t nMaxChannels:8;
int32_t nWarps:8;
int32_t algorithm:8, protocol:8;
int32_t algorithm:8, protocol:8, pipeline:8;
uint32_t isCollnet:1, isNvls:1;
uint32_t devFuncId:30;
int regBufType;
projects/rccl/src/include/device.h
+6 -4
View File
@@ -30,7 +30,7 @@ extern const char* ncclAlgoStr[NCCL_NUM_ALGORITHMS];
extern const char* ncclProtoStr[NCCL_NUM_PROTOCOLS];
extern const char* funcNames[FUNC_INDEX_TOTAL];
extern const char* funcNames[];
#define NCCL_MAX_OPS 2048
#define NCCL_STEPS 8
@@ -134,6 +134,7 @@ static_assert(NCCL_LL_CLEAN_MASK % NCCL_STEPS == 0, "Invalid NCCL_LL_CLEAN_MASK
#define RCCL_PROTO_SHIFT 8
#define RCCL_REDOP_SHIFT 12
#define RCCL_DTYPE_SHIFT 16
#define RCCL_PIPELINE_SHIFT 20
struct ncclConnInfo {
// Regular comm mechanism
@@ -701,7 +702,7 @@ inline bool ncclNvlsSupported(int devRedOp, int type) {
extern std::unordered_map<uint64_t, int> ncclDevFuncNameToId;
// `ncclDevFuncId()` needs to be in sync with 'all_colls' in generate.py
inline int ncclDevFuncId(int coll, int devRedOp, int type, int algo, int proto) {
inline int ncclDevFuncId(int coll, int devRedOp, int type, int algo, int proto, int pipeline = 0) {
int row = -1;
uint64_t key;
// Pack 4-bit fields from right (LSB) to left in order:
@@ -717,14 +718,15 @@ inline int ncclDevFuncId(int coll, int devRedOp, int type, int algo, int proto)
((uint64_t)(algo & RCCL_FUNC_ID_MASK) << RCCL_ALGO_SHIFT ) |
((uint64_t)(proto & RCCL_FUNC_ID_MASK) << RCCL_PROTO_SHIFT) |
((uint64_t)(devRedOp & RCCL_FUNC_ID_MASK) << RCCL_REDOP_SHIFT) |
((uint64_t)(type & RCCL_FUNC_ID_MASK) << RCCL_DTYPE_SHIFT);
((uint64_t)(type & RCCL_FUNC_ID_MASK) << RCCL_DTYPE_SHIFT) |
((uint64_t)(pipeline & RCCL_FUNC_ID_MASK) << RCCL_PIPELINE_SHIFT);
}
auto it = ncclDevFuncNameToId.find(key);
if (it != ncclDevFuncNameToId.end()) {
row = it->second;
}
if(row < 0) {
WARN("Fatal error: ncclDevFuncId: %llu not found for coll: %d, algo: %d, proto: %d, devRedOp: %d, type: %d", key, coll, algo, proto, devRedOp, type);
WARN("Fatal error: ncclDevFuncId: %lu not found for coll: %d, algo: %d, proto: %d, devRedOp: %d, type: %d", key, coll, algo, proto, devRedOp, type);
return -1;
}
return row;
projects/rccl/src/include/nccl_common.h
-4
View File
@@ -39,10 +39,6 @@ typedef enum {
typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...);
#define NCCL_NUM_ONERANK 12
#define AR_WITH_BIAS_FUNC_COUNTS 324
#define FUNC_INDEX_TOTAL 821 + AR_WITH_BIAS_FUNC_COUNTS + NCCL_NUM_ONERANK
#define NCCL_NUM_FUNCTIONS 5 // Send/Recv not included for now
typedef enum {
ncclFuncBroadcast = 0,
projects/rccl/src/include/rccl_common.h
+1
View File
@@ -83,6 +83,7 @@ inline size_t rcclGetSizePerRank(ncclFunc_t const& func, size_t const& nBytes, i
}
void rcclUpdateCollectiveProtocol(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info);
void rcclUpdateThreadThreshold(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info, int& threadThreshold);
void rcclSetPipelining(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info);
ncclResult_t rcclGetAlgoInfo(struct ncclComm* comm, ncclFunc_t coll, uint64_t count, ncclDataType_t dataType,
int collNetSupport, int nvlsSupport, int numPipeOps,
int* algo, int* protocol, int* maxChannels);