Re-enabled MSCCL++ (#1325)
* Added restrictions around calling MSCCL++ collectives (#1281) * Added restriction to non-zero 32-byte multiple message sizes to MSCCL++ AllGather. * Renamed and refactored some mscclpp types. * Only transmit the MSCCL++ unique id for non-split comm init. For splitting comm, it has already been transmitted. Instead, save the MSCCL++ communicator in child communicators when calling `ncclCommSplit`. Only destroy MSCCL++ communicators when no RCCL communicators remain that use it. Also improved trace logging. * Disable MSCCL++ when using managed memory buffers as it isn't supported. * Added datatype and op constraints for MSCCL++ AllReduce. * Added documentation on MSCCL++ restrictions to the README. * [BUILD] Support custom CMake flags in MSCCLPP (#1275) * [BUILD] Support custom CMAKE_PREFIX_PATH in MSCCLPP Signed-off-by: nileshnegi <Nilesh.Negi@amd.com> * [BUILD] CMake flags to support build-id in MSCCLPP Signed-off-by: nileshnegi <Nilesh.Negi@amd.com> * [BUILD] Fix CMake warnings in MSCCLPP build Signed-off-by: nileshnegi <Nilesh.Negi@amd.com> * Wrapped all cmake arguments passed to mscclpp to remove empty arguments and properly format them. --------- Signed-off-by: nileshnegi <Nilesh.Negi@amd.com> Co-authored-by: Corey Derochie <corey.derochie@amd.com> * Link to libmscclpp_nccl statically (#1282) * Switched mscclpp_nccl to static linking. Added a build step to rename the NCCL API functions. * Undid separation of building libmscclpp_nccl from building librccl with MSCCL++ integration. With a static build, it's either fully enabled or fully disabled. * `nm` isn't always available in docker containers due to being stripped down. Removed use of `nm` in `cmake` and hard-coded the output into mscclpp_nccl_syms.txt. * Removed IBVerbs dependency for integrating with MSCCL++ (#1313) * Renamed `RCCL_ENABLE_MSCCLPP` to `RCCL_MSCCLPP_ENABLE` to conform to MSCCL. Set `RCCL_MSCCLPP_ENABLE` to 1 by default if `ENABLE_MSCCLPP` is defined, or 0 otherwise. Added a log warning if `RCCL_MSCCLPP_ENABLE` is set to 1 but `ENABLE_MSCCLPP` is not defined. (#1294) * Include mscclpp as a git submodule (#1314) * Added the desired mscclpp commit as a git submodule. * Added step to automatically checkout the mscclpp submodule if it isn't already present, in case the user forgot to clone recursively. * Added instruction to README to clone using --recurse-submodules to get the mscclpp submodule. * Enabled MSCCL++ feature build. --------- Signed-off-by: nileshnegi <Nilesh.Negi@amd.com> Co-authored-by: Nilesh M Negi <Nilesh.Negi@amd.com>
Bu işleme şunda yer alıyor:
corey-derochie-amd
işlemeyi yapan:
GitHub
GitHub
ebeveyn
4856309413
işleme
736a705875
@@ -396,7 +396,7 @@ struct ncclComm {
|
||||
#if defined(ENABLE_MSCCLPP)
|
||||
// Whether this comm is compatible with MSCCLPP
|
||||
bool mscclppCompatible;
|
||||
struct mscclpp_ncclComm* mscclpp_comm;
|
||||
struct mscclppComm* mscclpp_comm;
|
||||
size_t mscclpp_threshold;
|
||||
#endif
|
||||
|
||||
|
||||
@@ -9,32 +9,36 @@
|
||||
|
||||
#include "nccl.h"
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
|
||||
typedef struct mscclpp_ncclComm* mscclpp_ncclComm_t;
|
||||
typedef struct mscclppComm* mscclppComm_t;
|
||||
|
||||
typedef struct { char internal[NCCL_UNIQUE_ID_BYTES]; } mscclpp_ncclUniqueId;
|
||||
typedef ncclUniqueId mscclppUniqueId;
|
||||
|
||||
bool mscclpp_init();
|
||||
/* A ncclUniqueId and a mscclppUniqueId will always be created together and used alternatively. This maps between them. */
|
||||
extern std::unordered_map<ncclUniqueId, mscclppUniqueId> mscclpp_uniqueIdMap;
|
||||
extern std::unordered_map<mscclppUniqueId, std::unordered_set<ncclUniqueId>> mscclpp_uniqueIdReverseMap;
|
||||
extern std::unordered_map<mscclppComm_t, mscclppUniqueId> mscclpp_commToUniqueIdMap;
|
||||
extern std::unordered_map<ncclComm_t, ncclUniqueId> ncclCommToUniqueIdMap;
|
||||
|
||||
/* A ncclUniqueId and a mscclpp_ncclUniqueId will always be created together and used alternatively. This maps between them. */
|
||||
extern std::unordered_map<ncclUniqueId, mscclpp_ncclUniqueId> mscclpp_uniqueIdMap;
|
||||
extern "C" {
|
||||
/* See ncclGetUniqueId. */
|
||||
ncclResult_t mscclpp_ncclGetUniqueId(mscclppUniqueId* uniqueId);
|
||||
|
||||
/* See ncclGetUniqueId. */
|
||||
extern ncclResult_t (*mscclpp_ncclGetUniqueId)(mscclpp_ncclUniqueId* uniqueId);
|
||||
/* See ncclCommInitRank. */
|
||||
ncclResult_t mscclpp_ncclCommInitRank(mscclppComm_t* comm, int nranks, mscclppUniqueId commId, int rank);
|
||||
|
||||
/* See ncclCommInitRank. */
|
||||
extern ncclResult_t (*mscclpp_ncclCommInitRank)(mscclpp_ncclComm_t* comm, int nranks, mscclpp_ncclUniqueId commId, int rank);
|
||||
/* See ncclCommDestroy. */
|
||||
ncclResult_t mscclpp_ncclCommDestroy(mscclppComm_t comm);
|
||||
|
||||
/* See ncclCommDestroy. */
|
||||
extern ncclResult_t (*mscclpp_ncclCommDestroy)(mscclpp_ncclComm_t comm);
|
||||
/* See ncclAllReduce. */
|
||||
ncclResult_t mscclpp_ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count,
|
||||
ncclDataType_t datatype, ncclRedOp_t op, mscclppComm_t comm, hipStream_t stream);
|
||||
|
||||
/* See ncclAllReduce. */
|
||||
extern ncclResult_t (*mscclpp_ncclAllReduce)(const void* sendbuff, void* recvbuff, size_t count,
|
||||
ncclDataType_t datatype, ncclRedOp_t op, mscclpp_ncclComm_t comm, hipStream_t stream);
|
||||
|
||||
/* See ncclAllGather. */
|
||||
extern ncclResult_t (*mscclpp_ncclAllGather)(const void* sendbuff, void* recvbuff, size_t sendcount,
|
||||
ncclDataType_t datatype, mscclpp_ncclComm_t comm, hipStream_t stream);
|
||||
/* See ncclAllGather. */
|
||||
ncclResult_t mscclpp_ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount,
|
||||
ncclDataType_t datatype, mscclppComm_t comm, hipStream_t stream);
|
||||
}
|
||||
|
||||
namespace std {
|
||||
template <>
|
||||
|
||||
+56
-20
@@ -101,10 +101,14 @@ bool operator ==(const ncclUniqueId& a, const ncclUniqueId& b) {
|
||||
return memcmp(a.internal, b.internal, NCCL_UNIQUE_ID_BYTES) == 0;
|
||||
}
|
||||
|
||||
RCCL_PARAM(EnableMscclpp, "ENABLE_MSCCLPP", 0);
|
||||
RCCL_PARAM(MscclppThreshold, "MSCCLPP_THRESHOLD", (size_t)(1024*1024));
|
||||
static constexpr int64_t defaultEnableMscclpp = 1;
|
||||
#else
|
||||
static constexpr int64_t defaultEnableMscclpp = 0;
|
||||
#endif
|
||||
|
||||
RCCL_PARAM(MscclppEnabled, "MSCCLPP_ENABLE", defaultEnableMscclpp);
|
||||
|
||||
// GDRCOPY support: Off by default
|
||||
NCCL_PARAM(GdrCopyEnable, "GDRCOPY_ENABLE", 0);
|
||||
|
||||
@@ -165,11 +169,6 @@ static ncclResult_t ncclInit() {
|
||||
}
|
||||
#ifndef NVTX_NO_IMPL
|
||||
initNvtxRegisteredEnums();
|
||||
#endif
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
if (rcclParamEnableMscclpp() && !mscclpp_init()) {
|
||||
return ncclSystemError;
|
||||
}
|
||||
#endif
|
||||
__atomic_store_n(&initialized, true, __ATOMIC_RELEASE);
|
||||
}
|
||||
@@ -190,21 +189,25 @@ ncclResult_t ncclGetUniqueId_impl(ncclUniqueId* out) {
|
||||
NCCLCHECK(PtrCheck(out, "GetUniqueId", "out"));
|
||||
ncclResult_t res = bootstrapGetUniqueId((struct ncclBootstrapHandle*)out);
|
||||
TRACE_CALL("ncclGetUniqueId(0x%llx)", (unsigned long long)hashUniqueId(*out));
|
||||
if (rcclParamMscclppEnabled()) {
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
if (rcclParamEnableMscclpp()) {
|
||||
NCCLCHECK(res);
|
||||
int dev;
|
||||
CUDACHECK(cudaGetDevice(&dev));
|
||||
hipDeviceProp_t devProp;
|
||||
CUDACHECK(hipGetDeviceProperties(&devProp, dev));
|
||||
if (IsArchMatch(devProp.gcnArchName, "gfx94")) {
|
||||
res = mscclpp_ncclGetUniqueId(&(mscclpp_uniqueIdMap[*out]));
|
||||
TRACE_CALL("mscclpp_ncclGetUniqueId");
|
||||
auto& mscclppUniqueId = mscclpp_uniqueIdMap[*out];
|
||||
res = mscclpp_ncclGetUniqueId(&mscclppUniqueId);
|
||||
TRACE_CALL("mscclpp_ncclGetUniqueId(0x%llx)", (unsigned long long)hashUniqueId(mscclppUniqueId));
|
||||
mscclpp_uniqueIdReverseMap[mscclppUniqueId].insert(*out);
|
||||
} else {
|
||||
WARN("MSCCL++: Cannot enable MSCCL++ on %s architecture", devProp.gcnArchName);
|
||||
}
|
||||
}
|
||||
#else
|
||||
WARN("MSCCL++: Feature not enabled. ENABLE_MSCCLPP must be defined at compile-time to enable this feature.");
|
||||
#endif
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -1914,6 +1917,9 @@ fail:
|
||||
static ncclResult_t ncclCommInitRankFunc(struct ncclAsyncJob* job_) {
|
||||
struct ncclCommInitRankAsyncJob* job = (struct ncclCommInitRankAsyncJob*)job_;
|
||||
ncclComm_t comm = job->comm;
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
ncclUniqueId origUniqueId = job->commId;
|
||||
#endif
|
||||
ncclResult_t res = ncclSuccess;
|
||||
int archMajor, archMinor;
|
||||
size_t maxLocalSizeBytes = 0;
|
||||
@@ -1967,22 +1973,41 @@ static ncclResult_t ncclCommInitRankFunc(struct ncclAsyncJob* job_) {
|
||||
NCCLCHECKGOTO(initTransportsRank(comm, job->parent), res, fail);
|
||||
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
if (rcclParamEnableMscclpp()) {
|
||||
if (job->parent) {
|
||||
if (job->parent->mscclppCompatible) {
|
||||
INFO(NCCL_INIT, "MSCCL++: Splitting a compatible communicator; using parent mscclpp_comm");
|
||||
comm->mscclppCompatible = true;
|
||||
comm->mscclpp_threshold = job->parent->mscclpp_threshold;
|
||||
comm->mscclpp_comm = job->parent->mscclpp_comm;
|
||||
auto& mscclppUniqueId = mscclpp_uniqueIdMap[origUniqueId];
|
||||
mscclpp_uniqueIdMap[job->commId] = mscclppUniqueId;
|
||||
mscclpp_uniqueIdReverseMap[mscclppUniqueId].insert(job->commId);
|
||||
}
|
||||
}
|
||||
else
|
||||
#endif
|
||||
if (rcclParamMscclppEnabled()) {
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
hipDeviceProp_t devProp;
|
||||
CUDACHECK(hipGetDeviceProperties(&devProp, cudaDev));
|
||||
comm->mscclppCompatible = IsArchMatch(devProp.gcnArchName, "gfx94");
|
||||
if (comm->mscclppCompatible) {
|
||||
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, &(mscclpp_uniqueIdMap[job->commId]), sizeof(mscclpp_ncclUniqueId)), res, fail);
|
||||
TRACE_CALL("bootstrapIntraNodeBroadcast(rank=%d, nranks=%d, root=%d, bcastData=<mscclpp_ncclUniqueId>)", comm->localRank, comm->localRanks, 0);
|
||||
auto& mscclppUniqueId = mscclpp_uniqueIdMap[job->commId];
|
||||
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, &mscclppUniqueId, sizeof(mscclppUniqueId)), res, fail);
|
||||
unsigned long long mscclppUniqueIdHash; (void)mscclppUniqueIdHash;
|
||||
TRACE_CALL("bootstrapIntraNodeBroadcast(rank=%d, nranks=%d, root=%d, bcastData=hash:0x%llx)", comm->localRank, comm->localRanks, 0, (mscclppUniqueIdHash = (unsigned long long)hashUniqueId(mscclppUniqueId)));
|
||||
comm->mscclpp_threshold = rcclParamMscclppThreshold();
|
||||
INFO(NCCL_INIT, "MSCCL++: Enabled! Msg size threshold=%zu", comm->mscclpp_threshold);
|
||||
NCCLCHECKGOTO(mscclpp_ncclCommInitRank(&(comm->mscclpp_comm), job->nranks, mscclpp_uniqueIdMap[job->commId], job->myrank), res, fail);
|
||||
TRACE_CALL("mscclpp_ncclCommInitRank (nranks=%d, myrank=%d)", job->nranks, job->myrank);
|
||||
NCCLCHECKGOTO(mscclpp_ncclCommInitRank(&(comm->mscclpp_comm), job->nranks, mscclppUniqueId, job->myrank), res, fail);
|
||||
TRACE_CALL("mscclpp_ncclCommInitRank (*comm=%p, nranks=%d, commId=hash:0x%llx, myrank=%d)", comm->mscclpp_comm, job->nranks, mscclppUniqueIdHash, job->myrank);
|
||||
mscclpp_commToUniqueIdMap[comm->mscclpp_comm] = mscclppUniqueId;
|
||||
} else {
|
||||
WARN("MSCCL++: Cannot enable MSCCL++ on %s architecture", devProp.gcnArchName);
|
||||
}
|
||||
}
|
||||
#else
|
||||
WARN("MSCCL++: Feature not enabled. ENABLE_MSCCLPP must be defined at compile-time to enable this feature.");
|
||||
#endif
|
||||
}
|
||||
|
||||
NCCLCHECKGOTO(ncclLoadTunerPlugin(&comm->tuner), res, fail);
|
||||
if (comm->tuner) {
|
||||
@@ -2584,11 +2609,22 @@ ncclResult_t ncclCommDestroy_impl(ncclComm_t comm) {
|
||||
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
if (comm->mscclppCompatible) {
|
||||
ncclResult_t res = mscclpp_ncclCommDestroy(comm->mscclpp_comm);
|
||||
TRACE_CALL("mscclpp_ncclCommDestroy");
|
||||
if (res != ncclSuccess) {
|
||||
WARN("MSCCL++: mscclpp_ncclCommDestroy failed (%s)", ncclGetErrorString(res));
|
||||
auto& mscclppUniqueId = mscclpp_commToUniqueIdMap[comm->mscclpp_comm];
|
||||
auto& uniqueIds = mscclpp_uniqueIdReverseMap[mscclppUniqueId];
|
||||
auto& ncclUniqueId = ncclCommToUniqueIdMap[comm];
|
||||
if (uniqueIds.find(ncclUniqueId) == uniqueIds.end()) {
|
||||
WARN("MSCCL++: comm=%p not found in mscclpp_uniqueIdReverseMap for key=%p", comm, comm->mscclpp_comm);
|
||||
}
|
||||
uniqueIds.erase(ncclUniqueId);
|
||||
if (uniqueIds.size() == 0) {
|
||||
mscclpp_uniqueIdReverseMap.erase(mscclppUniqueId);
|
||||
ncclResult_t res = mscclpp_ncclCommDestroy(comm->mscclpp_comm);
|
||||
TRACE_CALL("mscclpp_ncclCommDestroy");
|
||||
if (res != ncclSuccess) {
|
||||
WARN("MSCCL++: mscclpp_ncclCommDestroy failed (%s)", ncclGetErrorString(res));
|
||||
}
|
||||
}
|
||||
|
||||
comm->mscclppCompatible = false;
|
||||
comm->mscclpp_comm = nullptr;
|
||||
}
|
||||
|
||||
@@ -468,6 +468,25 @@ static ncclResult_t mscclFallBackSavedParams() {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
static inline bool isMscclppAllReduceSupported(ncclDataType_t dataType, ncclRedOp_t op) {
|
||||
switch (dataType) {
|
||||
case ncclFloat16:
|
||||
case ncclInt32:
|
||||
case ncclUint32:
|
||||
case ncclFloat32:
|
||||
#ifdef RCCL_BFLOAT16
|
||||
case ncclBfloat16:
|
||||
#endif
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
return (op == ncclSum);
|
||||
}
|
||||
#endif
|
||||
|
||||
ncclResult_t mscclEnqueueCheck(
|
||||
const void* sendBuff, const size_t sendCounts[], const size_t sDisPls[],
|
||||
void* recvBuff, const size_t recvCounts[], const size_t rDisPls[],
|
||||
@@ -493,8 +512,13 @@ ncclResult_t mscclEnqueueCheck(
|
||||
}
|
||||
|
||||
/* check if one rank per GPU and graph mode is enabled */
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible) {
|
||||
if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && (nBytes & 31) == 0) {
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible && nBytes > 0 && (nBytes & 31) == 0) {
|
||||
bool isManagedBuffer = false;
|
||||
if (sendBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(sendBuff)));
|
||||
if (!isManagedBuffer && recvBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(recvBuff)));
|
||||
|
||||
if (isManagedBuffer) { /* MSCCL++ not enabled for managed memory buffers */ }
|
||||
else if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && isMscclppAllReduceSupported(dataType, op)) {
|
||||
INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p",
|
||||
"mscclpp_ncclAllReduce", comm->opCount, sendBuff, recvBuff, count, dataType, op, root, comm, comm->nRanks, stream);
|
||||
NCCLCHECK(mscclpp_ncclAllReduce(sendBuff, recvBuff, count, dataType, op, comm->mscclpp_comm, stream));
|
||||
@@ -529,8 +553,13 @@ ncclResult_t mscclEnqueueCheck(
|
||||
}
|
||||
|
||||
/* check if one rank per GPU and graph mode is enabled */
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible) {
|
||||
if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && (nBytes & 31) == 0) {
|
||||
if ((threadLocalStatus.captureStatus != mscclNoCapture) && comm->mscclCompatible && nBytes > 0 && (nBytes & 31) == 0) {
|
||||
bool isManagedBuffer = false;
|
||||
if (sendBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(sendBuff)));
|
||||
if (!isManagedBuffer && recvBuff) CUDACHECK(hipPointerGetAttribute(&isManagedBuffer, HIP_POINTER_ATTRIBUTE_IS_MANAGED, const_cast<void*>(recvBuff)));
|
||||
|
||||
if (isManagedBuffer) { /* MSCCL++ not enabled for managed memory buffers */ }
|
||||
else if (func == mscclFuncAllReduce && nBytes <= comm->mscclpp_threshold && isMscclppAllReduceSupported(dataType, op)) {
|
||||
INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p",
|
||||
"mscclpp_ncclAllReduce", comm->opCount, sendBuff, recvBuff, count, dataType, op, root, comm, comm->nRanks, stream);
|
||||
NCCLCHECK(mscclpp_ncclAllReduce(sendBuff, recvBuff, count, dataType, op, comm->mscclpp_comm, stream));
|
||||
|
||||
@@ -5,42 +5,8 @@
|
||||
************************************************************************/
|
||||
|
||||
#include "mscclpp/mscclpp_nccl.h"
|
||||
#include "debug.h"
|
||||
#include <dlfcn.h>
|
||||
#include <unordered_map>
|
||||
|
||||
#define MSCCLPP_DECLARE(X) decltype(mscclpp_##X) mscclpp_##X = nullptr
|
||||
#define MSCCLPP_LOAD(HANDLE, X) do { \
|
||||
(mscclpp_##X) = (decltype(mscclpp_##X))dlsym((HANDLE), (#X)); \
|
||||
const char* error; \
|
||||
if ((error = dlerror()) != nullptr) { \
|
||||
WARN("MSCCL++: failed to load %s : %s", (#X), error); \
|
||||
return false; \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
static const char mscclpp_nccl_lib_name[] = "libmscclpp_nccl.so";
|
||||
|
||||
MSCCLPP_DECLARE(ncclGetUniqueId);
|
||||
MSCCLPP_DECLARE(ncclCommInitRank);
|
||||
MSCCLPP_DECLARE(ncclCommDestroy);
|
||||
MSCCLPP_DECLARE(ncclAllReduce);
|
||||
MSCCLPP_DECLARE(ncclAllGather);
|
||||
|
||||
bool mscclpp_init() {
|
||||
void* handle = dlopen(mscclpp_nccl_lib_name, RTLD_LAZY);
|
||||
if (!handle) {
|
||||
WARN("MSCCL++: failed to access %s : %s", mscclpp_nccl_lib_name, dlerror());
|
||||
return false;
|
||||
}
|
||||
dlerror(); // Clear any errors.
|
||||
|
||||
MSCCLPP_LOAD(handle, ncclGetUniqueId);
|
||||
MSCCLPP_LOAD(handle, ncclCommInitRank);
|
||||
MSCCLPP_LOAD(handle, ncclCommDestroy);
|
||||
MSCCLPP_LOAD(handle, ncclAllReduce);
|
||||
MSCCLPP_LOAD(handle, ncclAllGather);
|
||||
return true;
|
||||
}
|
||||
|
||||
std::unordered_map<ncclUniqueId, mscclpp_ncclUniqueId> mscclpp_uniqueIdMap;
|
||||
std::unordered_map<ncclUniqueId, mscclppUniqueId> mscclpp_uniqueIdMap;
|
||||
std::unordered_map<mscclppUniqueId, std::unordered_set<ncclUniqueId>> mscclpp_uniqueIdReverseMap;
|
||||
std::unordered_map<mscclppComm_t, mscclppUniqueId> mscclpp_commToUniqueIdMap;
|
||||
std::unordered_map<ncclComm_t, ncclUniqueId> ncclCommToUniqueIdMap;
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# > ${PROJECT_BINARY_DIR}/mscclpp_nccl_syms.txt;
|
||||
# for sym in $(nm -fjust-symbols ${MSCCLPP_ROOT}/lib/libmscclpp_nccl_static.a | grep "^nccl"); do
|
||||
# echo $sym mscclpp_$sym>> ${PROJECT_BINARY_DIR}/mscclpp_nccl_syms.txt;
|
||||
# done
|
||||
ncclAllGather mscclpp_ncclAllGather
|
||||
ncclAllReduce mscclpp_ncclAllReduce
|
||||
ncclAllToAll mscclpp_ncclAllToAll
|
||||
ncclBcast mscclpp_ncclBcast
|
||||
ncclBroadcast mscclpp_ncclBroadcast
|
||||
ncclCommAbort mscclpp_ncclCommAbort
|
||||
ncclCommCount mscclpp_ncclCommCount
|
||||
ncclCommCuDevice mscclpp_ncclCommCuDevice
|
||||
ncclCommDestroy mscclpp_ncclCommDestroy
|
||||
ncclCommFinalize mscclpp_ncclCommFinalize
|
||||
ncclCommGetAsyncError mscclpp_ncclCommGetAsyncError
|
||||
ncclCommInitAll mscclpp_ncclCommInitAll
|
||||
ncclCommInitRank mscclpp_ncclCommInitRank
|
||||
ncclCommInitRankConfig mscclpp_ncclCommInitRankConfig
|
||||
ncclCommSplit mscclpp_ncclCommSplit
|
||||
ncclCommUserRank mscclpp_ncclCommUserRank
|
||||
ncclGetErrorString mscclpp_ncclGetErrorString
|
||||
ncclGetLastError mscclpp_ncclGetLastError
|
||||
ncclGetUniqueId mscclpp_ncclGetUniqueId
|
||||
ncclGetVersion mscclpp_ncclGetVersion
|
||||
ncclGroupEnd mscclpp_ncclGroupEnd
|
||||
ncclGroupStart mscclpp_ncclGroupStart
|
||||
ncclRecv mscclpp_ncclRecv
|
||||
ncclRedOpCreatePreMulSum mscclpp_ncclRedOpCreatePreMulSum
|
||||
ncclRedOpDestroy mscclpp_ncclRedOpDestroy
|
||||
ncclReduce mscclpp_ncclReduce
|
||||
ncclReduceScatter mscclpp_ncclReduceScatter
|
||||
ncclSend mscclpp_ncclSend
|
||||
Yeni konuda referans
Bir kullanıcı engelle