Mustafa Abduljabbar
f37f290134
* Support pipelining codegen and template specialization
* Support ReduceCopy pipelining for AllReduce, ReduceScatter, and Reduce (currently enabled for bfloat16)
* Remove need for FUNC_INDEX_TOTAL
* Add pipeline field to device function key construction logic
* Avoid unneeded codegen for LL/LL64 kernels
* Modify conditions and add pipeline dtypes env
* Optimize selection for both gfx942 and gfx950
* Increase pipeline bitfield width
* Use __forceinline__ for all device functions
* Realign reduceCopy with original form
* Add opt-out option to enable perf debugs
* Remove force-reduce-pipelining option from README
* Update CHANGELOG.md
---------
Co-authored-by: Jeffrey Novotny <jnovotny@amd.com>
[ROCm/rccl commit: 277747c199]
95 řádky
4.0 KiB
C
95 řádky
4.0 KiB
C
/*
|
|
Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in
|
|
all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
THE SOFTWARE.
|
|
*/
|
|
#ifndef RCCL_COMMON_H_
|
|
#define RCCL_COMMON_H_
|
|
#include "nccl_common.h"
|
|
#include "nccl.h"
|
|
#include "param.h"
|
|
|
|
typedef enum RcclTunableColls {
|
|
RCCL_UNSUPPORTED_TUNABLE = -1,
|
|
RCCL_RS_TUNABLE = 0, // reduce_scatter index
|
|
RCCL_AG_TUNABLE = 1, // all_gather index
|
|
RCCL_AR_TUNABLE = 2, // all_reduce index
|
|
RCCL_RE_TUNABLE = 3, // reduce index
|
|
RCCL_BR_TUNABLE = 4, // broadcast index
|
|
RCCL_TUNABLE_COLLS = 5 // LL/LL64/LL128 tunable collectives count
|
|
} rcclTunableIndex_t;
|
|
|
|
#define RCCL_LL_LIMITS_UNDEFINED 0
|
|
#define RCCL_PROTOCOL_ENTRY_SIZE 4
|
|
#define RCCL_PROTOCOL_MIN_IDX 0
|
|
#define RCCL_PROTOCOL_MAX_IDX 1
|
|
#define RCCL_PROTOCOL_FACTOR_IDX 2
|
|
#define RCCL_PROTOCOL_THREAD_THRESHOLD_IDX 3
|
|
|
|
typedef enum {
|
|
RCCL_VALUE_UNSET = -2,
|
|
RCCL_VALUE_INVALID = -1
|
|
} rcclValueState_t;
|
|
|
|
#ifdef RCCL_EXPOSE_STATIC
|
|
#define RCCL_STATIC_EXPOSE_CHECK()
|
|
#else
|
|
#define RCCL_STATIC_EXPOSE_CHECK() \
|
|
do { \
|
|
WARN("Attempting to use internal logic while required static functions are not exposed. Rebuild with RCCL_EXPOSE_STATIC enabled"); \
|
|
return ncclInvalidUsage; \
|
|
} while (0)
|
|
#endif
|
|
|
|
inline rcclTunableIndex_t rcclGetTunableIndex(ncclFunc_t const& func) {
|
|
switch (func) {
|
|
case ncclFuncReduceScatter:
|
|
return RCCL_RS_TUNABLE;
|
|
case ncclFuncAllGather:
|
|
return RCCL_AG_TUNABLE;
|
|
case ncclFuncAllReduce:
|
|
return RCCL_AR_TUNABLE;
|
|
case ncclFuncReduce:
|
|
return RCCL_RE_TUNABLE;
|
|
case ncclFuncBroadcast:
|
|
return RCCL_BR_TUNABLE;
|
|
default:
|
|
return RCCL_UNSUPPORTED_TUNABLE; // Invalid or unsupported function
|
|
}
|
|
}
|
|
|
|
inline size_t rcclGetSizePerRank(ncclFunc_t const& func, size_t const& nBytes, int const& nRanks) {
|
|
// Normalize the comparison to sizePerRank as this is essentially what matters in determining protocol choice for the impacted collectives
|
|
// For AG, this is the send size per rank
|
|
// For RS, this is the recv size per rank
|
|
// For AR, this is the send/recv size per rank
|
|
return (func == ncclFuncReduceScatter || func == ncclFuncAllGather || func == ncclFuncBroadcast) ? nBytes / nRanks : nBytes;
|
|
}
|
|
void rcclUpdateCollectiveProtocol(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info);
|
|
void rcclUpdateThreadThreshold(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info, int& threadThreshold);
|
|
void rcclSetPipelining(struct ncclComm* comm, size_t const& nBytes, struct ncclTaskColl* info);
|
|
ncclResult_t rcclGetAlgoInfo(struct ncclComm* comm, ncclFunc_t coll, uint64_t count, ncclDataType_t dataType,
|
|
int collNetSupport, int nvlsSupport, int numPipeOps,
|
|
int* algo, int* protocol, int* maxChannels);
|
|
void rcclSetPxn(struct ncclComm* comm, int& rcclPxnDisable);
|
|
void rcclSetP2pNetChunkSize(struct ncclComm* comm, int& rcclP2pNetChunkSize);
|
|
ncclResult_t rcclFuncMaxSendRecvCount(ncclFunc_t func, int nRanks, size_t count, size_t& maxCount);
|
|
ncclResult_t commSetUnrollFactor(struct ncclComm* comm);
|
|
#endif
|