AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'nccl/master' into develop

Browse Source 
[ROCm/rccl commit: 36343be84f]
This commit is contained in:
BertanDogancay
2025-01-23 11:48:18 -06:00
committed by Corey Derochie
parent e3a29f5eab 5ca1b6c160
commit 1b000665df
122 changed files with 9009 additions and 4702 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/misc/argcheck.cc
-2
View File
@@ -52,8 +52,6 @@ ncclResult_t ArgsCheck(struct ncclInfo* info) {
WARN("%s : invalid type %d", info->opName, info->datatype);
return ncclInvalidArgument;
}
// Type is OK, compute nbytes. Convert Allgather/Broadcast/P2P calls to chars.
NCCLCHECK(ncclInfoSetDerived(info, info->comm->nRanks));
if (info->op < 0 || ncclMaxRedOp < info->op) {
WARN("%s : invalid reduction operation %d", info->opName, info->op);
projects/rccl/src/misc/cudawrap.cc
+8
View File
@@ -59,6 +59,10 @@ DECLARE_CUDA_PFN(cuGetErrorString);
DECLARE_CUDA_PFN(cuGetErrorName);
/* enqueue.cc */
DECLARE_CUDA_PFN(cuMemGetAddressRange);
DECLARE_CUDA_PFN(cuLaunchKernel);
#if CUDA_VERSION >= 11080
DECLARE_CUDA_PFN(cuLaunchKernelEx);
#endif
/* proxy.cc */
DECLARE_CUDA_PFN(cuCtxCreate);
DECLARE_CUDA_PFN(cuCtxDestroy);
@@ -137,6 +141,10 @@ static ncclResult_t cudaPfnFuncLoader(void) {
LOAD_SYM(cuCtxGetCurrent, 1);
LOAD_SYM(cuCtxSetCurrent, 1);
LOAD_SYM(cuCtxGetDevice, 1);
LOAD_SYM(cuLaunchKernel, 1);
#if CUDA_VERSION >= 11080
LOAD_SYM(cuLaunchKernelEx, 1);
#endif
/* cuMem API support */
LOAD_SYM(cuMemAddressReserve, 1);
LOAD_SYM(cuMemAddressFree, 1);
projects/rccl/src/misc/gdrwrap.cc
+5 -5
View File
@@ -130,7 +130,7 @@ ncclResult_t wrap_gdr_pin_buffer(gdr_t g, unsigned long addr, size_t size, uint6
int ret;
GDRLOCKCALL(gdr_internal_pin_buffer(g, addr, size, p2p_token, va_space, handle), ret);
if (ret != 0) {
WARN("gdr_pin_buffer(addr %lx, size %zi) failed: %d", addr, size, ret);
WARN("gdr_pin_buffer(addr %lx, size %zu) failed: %d", addr, size, ret);
return ncclSystemError;
}
return ncclSuccess;
@@ -172,7 +172,7 @@ ncclResult_t wrap_gdr_map(gdr_t g, gdr_mh_t handle, void **va, size_t size) {
int ret;
GDRLOCKCALL(gdr_internal_map(g, handle, va, size), ret);
if (ret != 0) {
WARN("gdr_map(handle %lx, size %zi) failed: %d", handle.h, size, ret);
WARN("gdr_map(handle %lx, size %zu) failed: %d", handle.h, size, ret);
return ncclSystemError;
}
return ncclSuccess;
@@ -186,7 +186,7 @@ ncclResult_t wrap_gdr_unmap(gdr_t g, gdr_mh_t handle, void *va, size_t size) {
int ret;
GDRLOCKCALL(gdr_internal_unmap(g, handle, va, size), ret);
if (ret != 0) {
WARN("gdr_unmap(handle %lx, va %p, size %zi) failed: %d", handle.h, va, size, ret);
WARN("gdr_unmap(handle %lx, va %p, size %zu) failed: %d", handle.h, va, size, ret);
return ncclSystemError;
}
return ncclSuccess;
@@ -218,7 +218,7 @@ ncclResult_t wrap_gdr_copy_to_mapping(gdr_mh_t handle, void *map_d_ptr, const vo
int ret;
GDRLOCKCALL(gdr_internal_copy_to_mapping(handle, map_d_ptr, h_ptr, size), ret);
if (ret != 0) {
WARN("gdr_copy_to_mapping(handle %lx, map_d_ptr %p, h_ptr %p, size %zi) failed: %d", handle.h, map_d_ptr, h_ptr, size, ret);
WARN("gdr_copy_to_mapping(handle %lx, map_d_ptr %p, h_ptr %p, size %zu) failed: %d", handle.h, map_d_ptr, h_ptr, size, ret);
return ncclSystemError;
}
return ncclSuccess;
@@ -232,7 +232,7 @@ ncclResult_t wrap_gdr_copy_from_mapping(gdr_mh_t handle, void *h_ptr, const void
int ret;
GDRLOCKCALL(gdr_internal_copy_from_mapping(handle, h_ptr, map_d_ptr, size), ret);
if (ret != 0) {
WARN("gdr_copy_from_mapping(handle %lx, h_ptr %p, map_d_ptr %p, size %zi) failed: %d", handle.h, h_ptr, map_d_ptr, size, ret);
WARN("gdr_copy_from_mapping(handle %lx, h_ptr %p, map_d_ptr %p, size %zu) failed: %d", handle.h, h_ptr, map_d_ptr, size, ret);
return ncclSystemError;
}
return ncclSuccess;
projects/rccl/src/misc/ipcsocket.cc
+2 -2
View File
@@ -132,7 +132,7 @@ ncclResult_t ncclIpcSocketRecvMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
WARN("UDS: Receiving data over socket failed : %d", errno);
return ncclSystemError;
}
if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
}
if (recvFd != NULL) {
@@ -221,7 +221,7 @@ ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
WARN("UDS: Sending data over socket %s failed : %s (%d)", temp, strerror(errno), errno);
return ncclSystemError;
}
if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
}
return ncclSuccess;
projects/rccl/src/misc/msccl/msccl_lifecycle.cc
+1 -1
View File
@@ -409,7 +409,7 @@ static ncclResult_t mscclRunSavedParams() {
for (auto& param : threadLocalStatus.savedSchedulerParams) {
INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p task %d globalrank %d",
mscclFuncNames[param.p.func], param.p.opCount, param.p.sendBuff, param.p.recvBuff, param.p.count,
param.p.dataType, param.p.op, param.p.root, param.comm, param.p.nRanks, param.stream, param.comm->tasks.nTasksP2p + param.comm->tasks.nTasksColl, param.comm->localRankToRank[param.comm->localRank]);
param.p.dataType, param.p.op, param.p.root, param.comm, param.p.nRanks, param.stream, param.comm->planner.nTasksP2p + param.comm->planner.nTasksColl, param.comm->localRankToRank[param.comm->localRank]);
NCCLCHECK(mscclRunAlgo(
param.p.sendBuff, param.p.sendCounts, param.p.sDisPls,
projects/rccl/src/misc/nvmlwrap.cc
+41
View File
@@ -41,11 +41,19 @@ namespace {
NCCL_NVML_FN(nvmlDeviceGetFieldValues, nvmlReturn_t, (nvmlDevice_t device, int valuesCount, nvmlFieldValue_t *values))
// MNNVL support
NCCL_NVML_FN(nvmlDeviceGetGpuFabricInfoV, nvmlReturn_t, (nvmlDevice_t device, nvmlGpuFabricInfoV_t *gpuFabricInfo))
// CC support
NCCL_NVML_FN(nvmlSystemGetConfComputeState, nvmlReturn_t, (nvmlConfComputeSystemState_t *state));
NCCL_NVML_FN(nvmlSystemGetConfComputeSettings, nvmlReturn_t, (nvmlSystemConfComputeSettings_t *setting));
std::mutex lock; // NVML has had some thread safety bugs
bool initialized = false;
thread_local bool threadInitialized = false;
ncclResult_t initResult;
union nvmlCCInfoInternal {
nvmlConfComputeSystemState_t settingV12020;
nvmlSystemConfComputeSettings_t settingV12040;
};
}
ncclResult_t ncclNvmlEnsureInitialized() {
@@ -87,6 +95,9 @@ ncclResult_t ncclNvmlEnsureInitialized() {
{(void**)&pfn_nvmlDeviceGetFieldValues, "nvmlDeviceGetFieldValues"},
// MNNVL support
{(void**)&pfn_nvmlDeviceGetGpuFabricInfoV, "nvmlDeviceGetGpuFabricInfoV"},
// CC support
{(void**)&pfn_nvmlSystemGetConfComputeState, "nvmlSystemGetConfComputeState"},
{(void**)&pfn_nvmlSystemGetConfComputeSettings, "nvmlSystemGetConfComputeSettings"}
};
for(Symbol sym: symbols) {
*sym.ppfn = dlsym(libhandle, sym.name);
@@ -282,3 +293,33 @@ ncclResult_t ncclNvmlDeviceGetGpuFabricInfoV(nvmlDevice_t device, nvmlGpuFabricI
NVMLTRY(nvmlDeviceGetGpuFabricInfoV, device, gpuFabricInfo);
return ncclSuccess;
}
ncclResult_t ncclNvmlGetCCStatus(struct ncclNvmlCCStatus *status) {
NCCLCHECK(ncclNvmlEnsureInitialized());
std::lock_guard<std::mutex> locked(lock);
nvmlCCInfoInternal ccInfo;
if (pfn_nvmlSystemGetConfComputeSettings != NULL) {
ccInfo.settingV12040.version = nvmlSystemConfComputeSettings_v1;
NVMLTRY(nvmlSystemGetConfComputeSettings, &ccInfo.settingV12040);
if (ccInfo.settingV12040.ccFeature == NVML_CC_SYSTEM_FEATURE_ENABLED)
status->CCEnabled = true;
else
status->CCEnabled = false;
if (ccInfo.settingV12040.multiGpuMode == NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE)
status->multiGpuCCEnabled = true;
else
status->multiGpuCCEnabled = false;
} else if (pfn_nvmlSystemGetConfComputeState != NULL) {
NVMLTRY(nvmlSystemGetConfComputeState, &ccInfo.settingV12020);
if (ccInfo.settingV12020.ccFeature == NVML_CC_SYSTEM_FEATURE_ENABLED)
status->CCEnabled = true;
else
status->CCEnabled = false;
status->multiGpuCCEnabled = false;
} else {
status->CCEnabled = false;
status->multiGpuCCEnabled = false;
}
return ncclSuccess;
}
projects/rccl/src/misc/param.cc
+1 -1
View File
@@ -84,4 +84,4 @@ const char *ncclGetEnv(const char *name) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
pthread_once(&once, initEnv);
return getenv(name);
}
}
projects/rccl/src/misc/shmutils.cc
+25 -9
View File
@@ -63,13 +63,28 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
* goes down to 0, unlink should be called in order to delete shared memory file. */
if (shmPath[0] == '\0') {
sprintf(shmPath, "/dev/shm/nccl-XXXXXX");
retry_mkstemp:
fd = mkstemp(shmPath);
if (fd < 0) {
if (errno == EINTR) {
INFO(NCCL_ALL, "mkstemp: Failed to create %s, error: %s (%d) - retrying", shmPath, strerror(errno), errno);
goto retry_mkstemp;
}
WARN("Error: failed to create shared memory file %p, error %s (%d)", shmPath, strerror(errno), errno);
ret = ncclSystemError;
goto fail;
}
} else {
SYSCHECKGOTO(fd = open(shmPath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR), ret, fail);
}
retry_fallocate:
if (fallocate(fd, 0, 0, realShmSize) != 0) {
WARN("Error: failed to extend %s to %ld bytes", shmPath, realShmSize);
if (errno == EINTR) {
INFO(NCCL_ALL, "fallocate: Failed to extend %s to %ld bytes, error: %s (%d) - retrying", shmPath, realShmSize, strerror(errno), errno);
goto retry_fallocate;
}
WARN("Error: failed to extend %s to %ld bytes, error: %s (%d)", shmPath, realShmSize, strerror(errno), errno);
ret = ncclSystemError;
goto fail;
}
@@ -80,7 +95,7 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
hptr = (char*)mmap(NULL, realShmSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (hptr == MAP_FAILED) {
WARN("Could not map %s size %zi, error: %s", shmPath, realShmSize, strerror(errno));
WARN("Error: Could not map %s size %zu, error: %s (%d)", shmPath, realShmSize, strerror(errno), errno);
ret = ncclSystemError;
hptr = NULL;
goto fail;
@@ -93,7 +108,7 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
if (remref == 0) {
/* the last peer has completed attachment, it should unlink the shm mem file. */
if (unlink(shmPath) != 0) {
WARN("unlink shared memory %s failed, error: %s", shmPath, strerror(errno));
INFO(NCCL_ALLOC, "unlink shared memory %s failed, error: %s (%d)", shmPath, strerror(errno), errno);
}
}
}
@@ -110,7 +125,8 @@ exit:
*handle = (ncclShmHandle_t)tmphandle;
return ret;
fail:
WARN("Error while %s shared memory segment %s (size %ld)", create ? "creating" : "attaching to", shmPath, shmSize);
WARN("Error while %s shared memory segment %s (size %ld), error: %s (%d)", create ? "creating" : "attaching to",
shmPath, shmSize, strerror(errno), errno);
if (tmphandle) {
shmHandleInit(fd, shmPath, shmSize, realShmSize, hptr, dptr, create, tmphandle);
ncclShmClose((ncclShmHandle_t)tmphandle);
@@ -129,7 +145,7 @@ ncclResult_t ncclShmClose(ncclShmHandle_t handle) {
close(tmphandle->fd);
if (tmphandle->shmPath != NULL && tmphandle->refcount != NULL && *tmphandle->refcount > 0) {
if (unlink(tmphandle->shmPath) != 0) {
WARN("unlink shared memory %s failed, error: %s", tmphandle->shmPath, strerror(errno));
WARN("unlink shared memory %s failed, error: %s (%d)", tmphandle->shmPath, strerror(errno), errno);
ret = ncclSystemError;
}
}
@@ -139,7 +155,7 @@ ncclResult_t ncclShmClose(ncclShmHandle_t handle) {
if (tmphandle->shmPtr) {
if (tmphandle->devShmPtr) CUDACHECK(cudaHostUnregister(tmphandle->shmPtr));
if (munmap(tmphandle->shmPtr, tmphandle->realShmSize) != 0) {
WARN("munmap of shared memory %p size %ld failed, error: %s", tmphandle->shmPtr, tmphandle->realShmSize, strerror(errno));
WARN("munmap of shared memory %p size %ld failed, error: %s (%d)", tmphandle->shmPtr, tmphandle->realShmSize, strerror(errno), errno);
ret = ncclSystemError;
}
}
@@ -152,9 +168,9 @@ ncclResult_t ncclShmUnlink(ncclShmHandle_t handle) {
ncclResult_t ret = ncclSuccess;
struct shmHandleInternal* tmphandle = (struct shmHandleInternal*)handle;
if (tmphandle) {
if (tmphandle->shmPath != NULL) {
if (tmphandle->shmPath != NULL && tmphandle->refcount != NULL && *tmphandle->refcount > 0) {
if (unlink(tmphandle->shmPath) != 0) {
WARN("unlink shared memory %s failed, error: %s", tmphandle->shmPath, strerror(errno));
WARN("unlink shared memory %s failed, error: %s (%d)", tmphandle->shmPath, strerror(errno), errno);
ret = ncclSystemError;
}
free(tmphandle->shmPath);
@@ -184,7 +200,7 @@ ncclResult_t ncclShmemAllgather(struct ncclComm *comm, struct ncclShmemCollBuff
uint64_t t0 = clockNano();
while(__atomic_load_n(shmem->cnt[curRound], __ATOMIC_ACQUIRE) != comm->localRanks + 1) {
if (clockNano() - t0 >= 5 * 1000) sched_yield();
if (__atomic_load_n(comm->abortFlag, __ATOMIC_RELAXED) == 1) {
if (__atomic_load_n(comm->abortFlag, __ATOMIC_ACQUIRE) == 1) {
ret = ncclInternalError;
goto exit;
}
projects/rccl/src/misc/socket.cc
+5 -5
View File
@@ -39,7 +39,7 @@ static ncclResult_t socketProgressOpt(int op, struct ncclSocket* sock, void* ptr
}
}
(*offset) += bytes;
if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED)) {
if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE)) {
INFO(NCCL_NET, "socketProgressOpt: abort called");
return ncclInternalError;
}
@@ -623,12 +623,12 @@ ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
do {
NCCLCHECK(socketProgressState(sock));
} while (sock->asyncFlag == 0 &&
(sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED) == 0) &&
(sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE) == 0) &&
(sock->state == ncclSocketStateConnecting ||
sock->state == ncclSocketStateConnectPolling ||
sock->state == ncclSocketStateConnected));
if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
switch (sock->state) {
case ncclSocketStateConnecting:
@@ -670,11 +670,11 @@ ncclResult_t ncclSocketAccept(struct ncclSocket* sock, struct ncclSocket* listen
do {
NCCLCHECKGOTO(socketProgressState(sock), ret, exit);
} while (sock->asyncFlag == 0 &&
(sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED) == 0) &&
(sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE) == 0) &&
(sock->state == ncclSocketStateAccepting ||
sock->state == ncclSocketStateAccepted));
if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
switch (sock->state) {
case ncclSocketStateAccepting:
projects/rccl/src/misc/tuner.cc
+115 -64
View File
@@ -9,117 +9,150 @@
#include <errno.h>
#include <stdlib.h>
#include "checks.h"
#include "debug.h"
#include "nccl_tuner.h"
#include "tuner.h"
pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
static int tunerPluginRefCount;
static void* tunerPluginLib = nullptr;
ncclTuner_t* tunerSymbol = nullptr;
static ncclTuner_v3_t* tunerSymbol = nullptr;
static ncclTuner_v2_t* ncclTuner_v2 = nullptr;
static ncclTuner_v3_t ncclTuner_v2_as_v3;
static void* tryOpenDynamicLib(const char* name) {
static int hasNvlsSupport(float** collCostTable) {
// Requirements for support of different algorithms:
//
// - NVLS intra-node: nvlsSupport
// - NVLS intra+inter-node: collNetSupport
// - NVLSTree intra-node: always disabled
// - NVLSTree inter-node: nvlsSupport
// - Collnet* inter-node: collNetSupport
//
// nvlsSupport = 1 if either NVLS or NVLS_TREE entries in the cost table are not -1
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE || table[NCCL_ALGO_NVLS_TREE][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) ? 1 : 0;
}
static int hasCollNetSupport(float** collCostTable) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] == NCCL_ALGO_PROTO_IGNORE) ? 0 : 1;
}
static ncclResult_t ncclTuner_v2_as_v3_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int* nChannels) {
int algorithm = NCCL_ALGO_UNDEF;
int protocol = NCCL_PROTO_UNDEF;
int nvlsSupport = hasNvlsSupport(collCostTable);
int collNetSupport = hasCollNetSupport(collCostTable);
NCCLCHECK(ncclTuner_v2->getCollInfo(context, collType, nBytes, collNetSupport, nvlsSupport, numPipeOps, &algorithm, &protocol, nChannels));
// set time to 0 below to make sure this algorithm/protocol is selected later on
if (algorithm >= 0 && algorithm < NCCL_NUM_ALGORITHMS && protocol >= 0 && protocol < NCCL_NUM_PROTOCOLS) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
if (table[algorithm][protocol] != NCCL_ALGO_PROTO_IGNORE) table[algorithm][protocol] = 0.0;
}
return ncclSuccess;
}
static ncclResult_t ncclTuner_v2_as_v3_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
NCCLCHECK(ncclTuner_v2->init(nRanks, nNodes, logFunction, context));
ncclTuner_v2_as_v3.name = ncclTuner_v2->name;
ncclTuner_v2_as_v3.getCollInfo = ncclTuner_v2_as_v3_getCollInfo;
ncclTuner_v2_as_v3.destroy = ncclTuner_v2->destroy;
return ncclSuccess;
}
#define MAX_STR_LEN 255
static void* tryOpenLib(const char* name, int* err, char* errStr) {
*err = 0;
if (nullptr == name || strlen(name) == 0) {
return nullptr;
}
if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
name = nullptr;
}
void *handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
if (nullptr == handle) {
if (ENOENT == errno) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: No plugin found (%s)", name);
strncpy(errStr, dlerror(), MAX_STR_LEN);
errStr[MAX_STR_LEN] = '\0';
if (strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
*err = ENOENT;
}
}
return handle;
}
static void summarizeOpenTunerPluginLibErrors(char* pluginNames) {
const char *separator = " ";
int len = strlen(pluginNames);
// remove tail separator
pluginNames[len - 1] = '\0';
// remove last plugin name
while (len > 0 && pluginNames[--len] != *separator);
if (len > 0) {
pluginNames[len] = '\0';
}
// distinguish between one load attempt and multiple attempts
if (strstr(pluginNames, separator)) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Most recent plugin load returned %d : %s. All attempts to load '%s' also failed.", errno, dlerror(), pluginNames);
} else {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin load returned %d : %s : when loading %s", errno, dlerror(), pluginNames);
static char* tryOpenLibCheck(int openErr, char* openErrStr, char* nameList, int *nameListLen, char* name) {
if (openErr == ENOENT) {
snprintf(nameList, *nameListLen, " %s", name);
nameList += strlen(name) + 1;
*nameListLen -= strlen(name) + 1;
return nameList;
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: %s", openErrStr);
return nameList;
}
static void* openTunerPluginLib(void) {
static void* openTunerPluginLib(char* couldNotFindNames, int len) {
int openErr;
void *pluginLib;
#define MAX_PLUGIN_LOAD 4
int len;
char tunerPluginLibNameTried[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
char *ptr = tunerPluginLibNameTried;
char tunerPluginLibName[PATH_MAX];
char openErrStr[MAX_STR_LEN + 1] = { 0 };
const char *envTunerPluginName = getenv("NCCL_TUNER_PLUGIN");
if (envTunerPluginName && strlen(envTunerPluginName)) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: NCCL_TUNER_PLUGIN set to %s", envTunerPluginName);
snprintf(tunerPluginLibName, PATH_MAX, "%s", envTunerPluginName);
pluginLib = tryOpenDynamicLib(tunerPluginLibName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
len = PATH_MAX - strlen(ptr);
snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner-%s.so", envTunerPluginName);
pluginLib = tryOpenDynamicLib(tunerPluginLibName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
len = PATH_MAX - strlen(ptr);
snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
} else {
snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner.so");
pluginLib = tryOpenDynamicLib(tunerPluginLibName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
len = PATH_MAX - strlen(ptr);
snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
}
const char *envNetPluginName = getenv("NCCL_NET_PLUGIN");
if (envNetPluginName && strlen(envNetPluginName)) {
// Users are allowed to pack tuner into the net plugin
snprintf(tunerPluginLibName, PATH_MAX, "%s", envNetPluginName);
pluginLib = tryOpenDynamicLib(tunerPluginLibName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
len = PATH_MAX - strlen(ptr);
snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
snprintf(tunerPluginLibName, PATH_MAX, "librccl-net-%s.so", envNetPluginName);
pluginLib = tryOpenDynamicLib(tunerPluginLibName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
len = PATH_MAX - strlen(ptr);
snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
} else {
snprintf(tunerPluginLibName, PATH_MAX, "librccl-net.so");
pluginLib = tryOpenDynamicLib(tunerPluginLibName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
len = PATH_MAX - strlen(ptr);
snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
}
summarizeOpenTunerPluginLibErrors(ptr);
tunerPluginLibName[0] = '\0';
return nullptr;
}
@@ -130,10 +163,14 @@ enum {
tunerPluginLoadSuccess = 1,
};
ncclResult_t ncclTunerPluginLoad(ncclTuner_t** tuner) {
#define MAX_PLUGIN_LOAD 4
static int status = tunerPluginLoadReady;
ncclResult_t ncclTunerPluginLoad(struct ncclComm* comm) {
// Initialize to nullptr by default if plugin tuner cannot be loaded.
*tuner = nullptr;
static int status = tunerPluginLoadReady;
char couldNotFindNames[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
comm->tuner = nullptr;
if (tunerPluginLoadFailed == status) {
return ncclSuccess;
}
@@ -144,28 +181,41 @@ ncclResult_t ncclTunerPluginLoad(ncclTuner_t** tuner) {
}
if (tunerPluginLoadSuccess == status) {
*tuner = tunerSymbol;
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
goto exit;
}
tunerPluginLib = openTunerPluginLib();
tunerPluginLib = openTunerPluginLib(couldNotFindNames, MAX_PLUGIN_LOAD * PATH_MAX);
if (nullptr == tunerPluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using internal tuner plugin.");
if (strlen(couldNotFindNames)) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Could not find:%s. Using internal tuner plugin.", couldNotFindNames);
} else {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using internal tuner plugin.");
}
goto fail;
}
tunerSymbol = (ncclTuner_t*)dlsym(tunerPluginLib, NCCL_TUNER_PLUGIN_SYMBOL);
tunerSymbol = (ncclTuner_v3_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v3");
if (tunerSymbol == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find " NCCL_TUNER_PLUGIN_SYMBOL ", using internal tuner instead.");
dlclose(tunerPluginLib);
goto fail;
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v2");
if (ncclTuner_v2 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
dlclose(tunerPluginLib);
goto fail;
} else {
ncclTuner_v2_as_v3.init = ncclTuner_v2_as_v3_init;
ncclTuner_v2_as_v3.name = ncclTuner_v2->name;
tunerSymbol = &ncclTuner_v2_as_v3;
}
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", tunerSymbol->name);
*tuner = tunerSymbol;
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
status = tunerPluginLoadSuccess;
comm->tunerPluginLoaded = 1;
exit:
pthread_mutex_unlock(&tunerPluginLock);
@@ -176,15 +226,16 @@ fail:
goto exit;
}
ncclResult_t ncclTunerPluginUnload(ncclTuner_t** tuner) {
if (*tuner == nullptr) return ncclSuccess;
ncclResult_t ncclTunerPluginUnload(struct ncclComm* comm) {
pthread_mutex_lock(&tunerPluginLock);
if (0 == (--tunerPluginRefCount)) {
if (comm->tunerPluginLoaded && 0 == (--tunerPluginRefCount)) {
INFO(NCCL_TUNING, "TUNER/Plugin: Closing tuner: '%s'", tunerSymbol->name);
dlclose(tunerPluginLib);
tunerPluginLib = nullptr;
tunerSymbol = nullptr;
*tuner = nullptr;
comm->tuner = nullptr;
status = tunerPluginLoadReady;
comm->tunerPluginLoaded = 0;
}
pthread_mutex_unlock(&tunerPluginLock);
return ncclSuccess;
projects/rccl/src/misc/utils.cc
+2 -77
View File
@@ -93,7 +93,8 @@ uint64_t getHostHash(void) {
if ((hostId = ncclGetEnv("NCCL_HOSTID")) != NULL) {
INFO(NCCL_ENV, "NCCL_HOSTID set by environment to %s", hostId);
strncpy(hostHash, hostId, sizeof(hostHash));
strncpy(hostHash, hostId, sizeof(hostHash)-1);
hostHash[sizeof(hostHash)-1] = '\0';
} else {
FILE *file = fopen(HOSTID_FILE, "r");
if (file != NULL) {
@@ -291,79 +292,3 @@ void ncclMemoryStackDestruct(struct ncclMemoryStack* me) {
h = h1;
}
}
const char* ncclOpToString(ncclRedOp_t op) {
switch (op) {
case ncclSum:
return "ncclSum";
case ncclProd:
return "ncclProd";
case ncclMax:
return "ncclMax";
case ncclMin:
return "ncclMin";
case ncclAvg:
return "ncclAvg";
default:
return "Unknown";
}
}
const char* ncclDatatypeToString(ncclDataType_t type) {
switch (type) {
case ncclInt8: // ncclChar
return "ncclInt8";
case ncclInt32: // ncclInt
return "ncclInt32";
case ncclUint32:
return "ncclUint32";
case ncclInt64:
return "ncclInt64";
case ncclUint64:
return "ncclUint64";
case ncclFloat16: // ncclHalf
return "ncclFloat16";
case ncclFloat32: // ncclFloat
return "ncclFloat32";
case ncclFloat64: // ncclDouble
return "ncclFloat64";
#if defined(__CUDA_BF16_TYPES_EXIST__)
case ncclBfloat16:
return "ncclBfloat16";
#endif
default:
return "Unknown";
}
}
const char* ncclAlgoToString(int algo) {
switch (algo) {
case NCCL_ALGO_TREE:
return "TREE";
case NCCL_ALGO_RING:
return "RING";
case NCCL_ALGO_COLLNET_DIRECT:
return "COLLNET_DIRECT";
case NCCL_ALGO_COLLNET_CHAIN:
return "COLLNET_CHAIN";
case NCCL_ALGO_NVLS:
return "NVLS";
case NCCL_ALGO_NVLS_TREE:
return "NVLS_TREE";
default:
return "Unknown";
}
}
const char* ncclProtoToString(int proto) {
switch (proto) {
case NCCL_PROTO_LL:
return "LL";
case NCCL_PROTO_LL128:
return "LL128";
case NCCL_PROTO_SIMPLE:
return "SIMPLE";
default:
return "Unknown";
}
}