Re-enabled MSCCL++ (#1325)
* Added restrictions around calling MSCCL++ collectives (#1281)
* Added restriction to non-zero 32-byte multiple message sizes to MSCCL++ AllGather.
* Renamed and refactored some mscclpp types.
* Only transmit the MSCCL++ unique id for non-split comm init. For splitting comm, it has already been transmitted. Instead, save the MSCCL++ communicator in child communicators when calling `ncclCommSplit`. Only destroy MSCCL++ communicators when no RCCL communicators remain that use it. Also improved trace logging.
* Disable MSCCL++ when using managed memory buffers as it isn't supported.
* Added datatype and op constraints for MSCCL++ AllReduce.
* Added documentation on MSCCL++ restrictions to the README.
* [BUILD] Support custom CMake flags in MSCCLPP (#1275)
* [BUILD] Support custom CMAKE_PREFIX_PATH in MSCCLPP
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
* [BUILD] CMake flags to support build-id in MSCCLPP
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
* [BUILD] Fix CMake warnings in MSCCLPP build
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
* Wrapped all cmake arguments passed to mscclpp to remove empty arguments and properly format them.
---------
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
Co-authored-by: Corey Derochie <corey.derochie@amd.com>
* Link to libmscclpp_nccl statically (#1282)
* Switched mscclpp_nccl to static linking. Added a build step to rename the NCCL API functions.
* Undid separation of building libmscclpp_nccl from building librccl with MSCCL++ integration. With a static build, it's either fully enabled or fully disabled.
* `nm` isn't always available in docker containers due to being stripped down. Removed use of `nm` in `cmake` and hard-coded the output into mscclpp_nccl_syms.txt.
* Removed IBVerbs dependency for integrating with MSCCL++ (#1313)
* Renamed `RCCL_ENABLE_MSCCLPP` to `RCCL_MSCCLPP_ENABLE` to conform to MSCCL. Set `RCCL_MSCCLPP_ENABLE` to 1 by default if `ENABLE_MSCCLPP` is defined, or 0 otherwise. Added a log warning if `RCCL_MSCCLPP_ENABLE` is set to 1 but `ENABLE_MSCCLPP` is not defined. (#1294)
* Include mscclpp as a git submodule (#1314)
* Added the desired mscclpp commit as a git submodule.
* Added step to automatically checkout the mscclpp submodule if it isn't already present, in case the user forgot to clone recursively.
* Added instruction to README to clone using --recurse-submodules to get the mscclpp submodule.
* Enabled MSCCL++ feature build.
---------
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
Co-authored-by: Nilesh M Negi <Nilesh.Negi@amd.com>
[ROCm/rccl commit: 736a705875]
This commit is contained in:
corey-derochie-amd
committed by
GitHub
GitHub
vanhempi
e3b39ab309
commit
9ffd893c5a
@@ -468,6 +468,25 @@ static ncclResult_t mscclFallBackSavedParams() {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
static inline bool isMscclppAllReduceSupported(ncclDataType_t dataType, ncclRedOp_t op) {
|
||||
switch (dataType) {
|
||||
case ncclFloat16:
|
||||
case ncclInt32:
|
||||
case ncclUint32:
|
||||
case ncclFloat32:
|
||||
#ifdef RCCL_BFLOAT16
|
||||
case ncclBfloat16:
|
||||
#endif
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
return (op == ncclSum);
|
||||
}
|
||||
#endif
|
||||
|
||||
ncclResult_t mscclEnqueueCheck(
|
||||
const void* sendBuff, const size_t sendCounts[], const size_t sDisPls[],
|
||||
void* recvBuff, const size_t recvCounts[], const size_t rDisPls[],
|
||||
@@ -493,8 +512,13 @@ ncclResult_t mscclEnqueueCheck(
|
||||
}
|
||||
|
||||
/* check if one rank per GPU and graph mode is enabled */
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible) {
|
||||
if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && (nBytes & 31) == 0) {
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible && nBytes > 0 && (nBytes & 31) == 0) {
|
||||
bool isManagedBuffer = false;
|
||||
if (sendBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(sendBuff)));
|
||||
if (!isManagedBuffer && recvBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(recvBuff)));
|
||||
|
||||
if (isManagedBuffer) { /* MSCCL++ not enabled for managed memory buffers */ }
|
||||
else if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && isMscclppAllReduceSupported(dataType, op)) {
|
||||
INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p",
|
||||
"mscclpp_ncclAllReduce", comm->opCount, sendBuff, recvBuff, count, dataType, op, root, comm, comm->nRanks, stream);
|
||||
NCCLCHECK(mscclpp_ncclAllReduce(sendBuff, recvBuff, count, dataType, op, comm->mscclpp_comm, stream));
|
||||
@@ -529,8 +553,13 @@ ncclResult_t mscclEnqueueCheck(
|
||||
}
|
||||
|
||||
/* check if one rank per GPU and graph mode is enabled */
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible) {
|
||||
if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && (nBytes & 31) == 0) {
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible && nBytes > 0 && (nBytes & 31) == 0) {
|
||||
bool isManagedBuffer = false;
|
||||
if (sendBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(sendBuff)));
|
||||
if (!isManagedBuffer && recvBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(recvBuff)));
|
||||
|
||||
if (isManagedBuffer) { /* MSCCL++ not enabled for managed memory buffers */ }
|
||||
else if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && isMscclppAllReduceSupported(dataType, op)) {
|
||||
INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p",
|
||||
"mscclpp_ncclAllReduce", comm->opCount, sendBuff, recvBuff, count, dataType, op, root, comm, comm->nRanks, stream);
|
||||
NCCLCHECK(mscclpp_ncclAllReduce(sendBuff, recvBuff, count, dataType, op, comm->mscclpp_comm, stream));
|
||||
|
||||
@@ -5,42 +5,8 @@
|
||||
************************************************************************/
|
||||
|
||||
#include "mscclpp/mscclpp_nccl.h"
|
||||
#include "debug.h"
|
||||
#include <dlfcn.h>
|
||||
#include <unordered_map>
|
||||
|
||||
#define MSCCLPP_DECLARE(X) decltype(mscclpp_##X) mscclpp_##X = nullptr
|
||||
#define MSCCLPP_LOAD(HANDLE, X) do { \
|
||||
(mscclpp_##X) = (decltype(mscclpp_##X))dlsym((HANDLE), (#X)); \
|
||||
const char* error; \
|
||||
if ((error = dlerror()) != nullptr) { \
|
||||
WARN("MSCCL++: failed to load %s : %s", (#X), error); \
|
||||
return false; \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
static const char mscclpp_nccl_lib_name[] = "libmscclpp_nccl.so";
|
||||
|
||||
MSCCLPP_DECLARE(ncclGetUniqueId);
|
||||
MSCCLPP_DECLARE(ncclCommInitRank);
|
||||
MSCCLPP_DECLARE(ncclCommDestroy);
|
||||
MSCCLPP_DECLARE(ncclAllReduce);
|
||||
MSCCLPP_DECLARE(ncclAllGather);
|
||||
|
||||
bool mscclpp_init() {
|
||||
void* handle = dlopen(mscclpp_nccl_lib_name, RTLD_LAZY);
|
||||
if (!handle) {
|
||||
WARN("MSCCL++: failed to access %s : %s", mscclpp_nccl_lib_name, dlerror());
|
||||
return false;
|
||||
}
|
||||
dlerror(); // Clear any errors.
|
||||
|
||||
MSCCLPP_LOAD(handle, ncclGetUniqueId);
|
||||
MSCCLPP_LOAD(handle, ncclCommInitRank);
|
||||
MSCCLPP_LOAD(handle, ncclCommDestroy);
|
||||
MSCCLPP_LOAD(handle, ncclAllReduce);
|
||||
MSCCLPP_LOAD(handle, ncclAllGather);
|
||||
return true;
|
||||
}
|
||||
|
||||
std::unordered_map<ncclUniqueId, mscclpp_ncclUniqueId> mscclpp_uniqueIdMap;
|
||||
std::unordered_map<ncclUniqueId, mscclppUniqueId> mscclpp_uniqueIdMap;
|
||||
std::unordered_map<mscclppUniqueId, std::unordered_set<ncclUniqueId>> mscclpp_uniqueIdReverseMap;
|
||||
std::unordered_map<mscclppComm_t, mscclppUniqueId> mscclpp_commToUniqueIdMap;
|
||||
std::unordered_map<ncclComm_t, ncclUniqueId> ncclCommToUniqueIdMap;
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# > ${PROJECT_BINARY_DIR}/mscclpp_nccl_syms.txt;
|
||||
# for sym in $(nm -fjust-symbols ${MSCCLPP_ROOT}/lib/libmscclpp_nccl_static.a | grep "^nccl"); do
|
||||
# echo $sym mscclpp_$sym>> ${PROJECT_BINARY_DIR}/mscclpp_nccl_syms.txt;
|
||||
# done
|
||||
ncclAllGather mscclpp_ncclAllGather
|
||||
ncclAllReduce mscclpp_ncclAllReduce
|
||||
ncclAllToAll mscclpp_ncclAllToAll
|
||||
ncclBcast mscclpp_ncclBcast
|
||||
ncclBroadcast mscclpp_ncclBroadcast
|
||||
ncclCommAbort mscclpp_ncclCommAbort
|
||||
ncclCommCount mscclpp_ncclCommCount
|
||||
ncclCommCuDevice mscclpp_ncclCommCuDevice
|
||||
ncclCommDestroy mscclpp_ncclCommDestroy
|
||||
ncclCommFinalize mscclpp_ncclCommFinalize
|
||||
ncclCommGetAsyncError mscclpp_ncclCommGetAsyncError
|
||||
ncclCommInitAll mscclpp_ncclCommInitAll
|
||||
ncclCommInitRank mscclpp_ncclCommInitRank
|
||||
ncclCommInitRankConfig mscclpp_ncclCommInitRankConfig
|
||||
ncclCommSplit mscclpp_ncclCommSplit
|
||||
ncclCommUserRank mscclpp_ncclCommUserRank
|
||||
ncclGetErrorString mscclpp_ncclGetErrorString
|
||||
ncclGetLastError mscclpp_ncclGetLastError
|
||||
ncclGetUniqueId mscclpp_ncclGetUniqueId
|
||||
ncclGetVersion mscclpp_ncclGetVersion
|
||||
ncclGroupEnd mscclpp_ncclGroupEnd
|
||||
ncclGroupStart mscclpp_ncclGroupStart
|
||||
ncclRecv mscclpp_ncclRecv
|
||||
ncclRedOpCreatePreMulSum mscclpp_ncclRedOpCreatePreMulSum
|
||||
ncclRedOpDestroy mscclpp_ncclRedOpDestroy
|
||||
ncclReduce mscclpp_ncclReduce
|
||||
ncclReduceScatter mscclpp_ncclReduceScatter
|
||||
ncclSend mscclpp_ncclSend
|
||||
Viittaa uudesa ongelmassa
Block a user