AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'nccl/master' into develop

Browse Source
This commit is contained in:
BertanDogancay
2025-06-20 07:53:59 -05:00
parent 12315c259a 8171af656b
commit aaf023976a
136 changed files with 8510 additions and 5421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/misc/cudawrap.cc
+31
View File
@@ -4,6 +4,7 @@
* See LICENSE.txt for license information
************************************************************************/
#include "alloc.h"
#include "nccl.h"
#include "debug.h"
#include "param.h"
@@ -67,6 +68,36 @@ int ncclCuMemHostEnable() {
ncclCumemHostEnable = paramValue;
else
ncclCumemHostEnable = (cudaDriverVersion >= 12060) ? 1 : 0;
if (ncclCumemHostEnable) {
// Verify that host allocations actually work. Docker in particular is known to disable "get_mempolicy",
// causing such allocations to fail (this can be fixed by invoking Docker with "--cap-add SYS_NICE").
int cudaDev;
CUdevice currentDev;
int cpuNumaNodeId = -1;
CUmemAllocationProp prop = {};
size_t granularity = 0;
size_t size;
CUmemGenericAllocationHandle handle;
CUDACHECK(cudaGetDevice(&cudaDev));
CUCHECK(cuDeviceGet(&currentDev, cudaDev));
CUCHECK(cuDeviceGetAttribute(&cpuNumaNodeId, CU_DEVICE_ATTRIBUTE_HOST_NUMA_ID, currentDev));
if (cpuNumaNodeId < 0) cpuNumaNodeId = 0;
prop.location.type = CU_MEM_LOCATION_TYPE_HOST_NUMA;
prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
prop.requestedHandleTypes = ncclCuMemHandleType;
prop.location.id = cpuNumaNodeId;
CUCHECK(cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM));
size = 1;
ALIGN_SIZE(size, granularity);
if (CUPFN(cuMemCreate(&handle, size, &prop, 0)) != CUDA_SUCCESS) {
INFO(NCCL_INIT, "cuMem host allocations do not appear to be working; falling back to a /dev/shm/ based "
"implementation. This could be due to the container runtime disabling NUMA support. "
"To disable this warning, set NCCL_CUMEM_HOST_ENABLE=0");
ncclCumemHostEnable = 0;
} else {
CUCHECK(cuMemRelease(handle));
}
}
}
return ncclCumemHostEnable;
error:
src/misc/ipcsocket.cc
+2 -1
View File
@@ -169,7 +169,7 @@ ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
} control_un;
struct cmsghdr *cmptr;
char dummy_buffer[1];
char dummy_buffer[1] = {'\0'};
struct sockaddr_un cliaddr;
// Construct client address to send this shareable handle to
@@ -190,6 +190,7 @@ ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
TRACE(NCCL_INIT, "UDS: Sending hdr %p len %d fd %d to UDS socket %s", hdr, hdrLen, sendFd, temp);
if (sendFd != -1) {
memset(&control_un, '\0', sizeof(control_un));
msg.msg_control = control_un.control;
msg.msg_controllen = sizeof(control_un.control);
src/misc/param.cc
+1
View File
@@ -32,6 +32,7 @@ void setEnvFile(const char* fileName) {
size_t n = 0;
ssize_t read;
while ((read = getline(&line, &n, file)) != -1) {
if (line[0] == '#') continue;
if (line[read-1] == '\n') line[read-1] = '\0';
int s=0; // Env Var Size
while (line[s] != '\0' && line[s] != '=') s++;
src/misc/profiler.cc
-635
View File
@@ -1,635 +0,0 @@
/*************************************************************************
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "param.h"
#include "checks.h"
#include "comm.h"
#include "enqueue.h"
#include "utils.h"
#include "proxy.h"
#include "profiler.h"
#include <dlfcn.h>
static pthread_mutex_t profilerLock = PTHREAD_MUTEX_INITIALIZER;
static int profilerPluginRefCount;
static void* profilerPluginLib;
static ncclProfiler_t* ncclProfiler;
static ncclProfiler_v2_t ncclProfiler_v1_as_v2;
static ncclProfiler_v1_t* ncclProfiler_v1;
static uint8_t ncclStringToFunc(const char* func) {
if (0 == strcmp(func, "AllGather")) return ncclFuncAllGather;
if (0 == strcmp(func, "AllReduce")) return ncclFuncAllReduce;
if (0 == strcmp(func, "Broadcast")) return ncclFuncBroadcast;
if (0 == strcmp(func, "Recv")) return ncclFuncRecv;
if (0 == strcmp(func, "Reduce")) return ncclFuncReduce;
if (0 == strcmp(func, "ReduceScatter")) return ncclFuncReduceScatter;
if (0 == strcmp(func, "SendRecv")) return ncclFuncSendRecv;
return ncclFuncSend;
}
static uint8_t ncclStringToAlgo(const char* algo) {
if (0 == strcmp(algo, "TREE")) return NCCL_ALGO_TREE;
if (0 == strcmp(algo, "RING")) return NCCL_ALGO_RING;
if (0 == strcmp(algo, "COLLNET_DIRECT")) return NCCL_ALGO_COLLNET_DIRECT;
if (0 == strcmp(algo, "COLLNET_CHAIN")) return NCCL_ALGO_COLLNET_CHAIN;
if (0 == strcmp(algo, "NVLS")) return NCCL_ALGO_NVLS;
if (0 == strcmp(algo, "NVLS_TREE")) return NCCL_ALGO_NVLS_TREE;
return NCCL_ALGO_PAT;
}
static uint8_t ncclStringToProto(const char* proto) {
if (0 == strcmp(proto, "LL")) return NCCL_PROTO_LL;
if (0 == strcmp(proto, "LL128")) return NCCL_PROTO_LL128;
return NCCL_PROTO_SIMPLE;
}
static uint8_t ncclStringToDatatype(const char* dt) {
if (0 == strcmp(dt, "ncclInt8")) return ncclInt8;
if (0 == strcmp(dt, "ncclInt32")) return ncclInt32;
if (0 == strcmp(dt, "ncclUint32")) return ncclUint32;
if (0 == strcmp(dt, "ncclInt64")) return ncclInt64;
if (0 == strcmp(dt, "ncclUint64")) return ncclUint64;
if (0 == strcmp(dt, "ncclFloat16")) return ncclFloat16;
if (0 == strcmp(dt, "ncclFloat32")) return ncclFloat32;
#if defined(__CUDA_BF16_TYPES_EXIST__)
if (0 == strcmp(dt, "ncclBfloat16")) return ncclBfloat16;
#endif
return ncclFloat64;
}
static ncclResult_t ncclProfiler_v1_as_v2_startEvent(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr) {
ncclProfilerEventDescr_v1_t eDescr_v1 = { 0 };
eDescr_v1.type = eDescr->type;
eDescr_v1.parentObj = eDescr->parentObj;
eDescr_v1.rank = eDescr->rank;
switch(eDescr->type) {
case ncclProfileGroup: break;
case ncclProfileColl: {
eDescr_v1.coll.name = eDescr->coll.name;
eDescr_v1.coll.commHash = eDescr->coll.commHash;
eDescr_v1.coll.seqNumber = eDescr->coll.seqNumber;
eDescr_v1.coll.func = ncclStringToFunc(eDescr->coll.func);
eDescr_v1.coll.sendBuff = eDescr->coll.sendBuff;
eDescr_v1.coll.recvBuff = eDescr->coll.recvBuff;
eDescr_v1.coll.count = eDescr->coll.count;
eDescr_v1.coll.root = eDescr->coll.root;
eDescr_v1.coll.datatype = ncclStringToDatatype(eDescr->coll.datatype);
eDescr_v1.coll.op = 0; // removed in v2
eDescr_v1.coll.trafficBytes = eDescr->coll.trafficBytes;
eDescr_v1.coll.nMaxChannels = eDescr->coll.nMaxChannels;
eDescr_v1.coll.nWarps = eDescr->coll.nWarps;
eDescr_v1.coll.algo = ncclStringToAlgo(eDescr->coll.algo);
eDescr_v1.coll.proto = ncclStringToProto(eDescr->coll.proto);
} break;
case ncclProfileP2p: {
eDescr_v1.p2p.name = eDescr->p2p.name;
eDescr_v1.p2p.commHash = eDescr->p2p.commHash;
eDescr_v1.p2p.func = ncclStringToFunc(eDescr->p2p.func);
eDescr_v1.p2p.buff = eDescr->p2p.buff;
eDescr_v1.p2p.count = eDescr->p2p.count;
eDescr_v1.p2p.datatype = ncclStringToDatatype(eDescr->p2p.datatype);
eDescr_v1.p2p.peer = eDescr->p2p.peer;
} break;
case ncclProfileProxyOp: {
eDescr_v1.proxyOp.pid = eDescr->proxyOp.pid;
eDescr_v1.proxyOp.channelId = eDescr->proxyOp.channelId;
eDescr_v1.proxyOp.peer = eDescr->proxyOp.peer;
eDescr_v1.proxyOp.nSteps = eDescr->proxyOp.nSteps;
eDescr_v1.proxyOp.chunkSize = eDescr->proxyOp.chunkSize;
eDescr_v1.proxyOp.isSend = eDescr->proxyOp.isSend;
} break;
case ncclProfileProxyStep: {
eDescr_v1.proxyStep.step = eDescr->proxyStep.step;
} break;
case ncclProfileProxyCtrl: break;
default:;
}
return ncclProfiler_v1->startEvent(context, eHandle, &eDescr_v1);
}
static ncclResult_t ncclProfiler_v1_as_v2_init(void** context, int* eActivationMask) {
ncclProfiler_v1->init(context, eActivationMask);
ncclProfiler_v1_as_v2.startEvent = ncclProfiler_v1_as_v2_startEvent;
ncclProfiler_v1_as_v2.stopEvent = ncclProfiler_v1->stopEvent;
ncclProfiler_v1_as_v2.recordEventState = ncclProfiler_v1->recordEventState;
ncclProfiler_v1_as_v2.finalize = ncclProfiler_v1->finalize;
return ncclSuccess;
}
#define MAX_STR_LEN 256
static void* tryOpenLib(char* name, int *err, char* errStr) {
if (nullptr == name || strlen(name) == 0) {
return nullptr;
}
if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
name = nullptr;
}
void *handle = dlopen(name, RTLD_NOW | RTLD_LOCAL);
if (nullptr == handle) {
strncpy(errStr, dlerror(), MAX_STR_LEN);
errStr[MAX_STR_LEN] = 0;
if (name && strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
*err = ENOENT;
}
}
return handle;
}
static char* tryOpenLibCheck(int openErr, char* openErrStr, char* nameList, int *nameListLen, char* name) {
if (openErr == ENOENT) {
snprintf(nameList, *nameListLen, " %s", name);
nameList += strlen(name) + 1;
*nameListLen -= strlen(name) + 1;
return nameList;
}
INFO(NCCL_ENV, "PROFILER/Plugin: %s", openErrStr);
return nameList;
}
static void* openProfilerPluginLib(char* couldNotFindNames, int len) {
int openErr;
void *pluginLib;
char profilerPluginLibName[PATH_MAX];
char openErrStr[MAX_STR_LEN + 1] = { 0 };
const char *envProfilerPluginName = getenv("NCCL_PROFILER_PLUGIN");
if (envProfilerPluginName && strlen(envProfilerPluginName)) {
snprintf(profilerPluginLibName, PATH_MAX, "%s", envProfilerPluginName);
pluginLib = tryOpenLib(profilerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: Plugin name set by env to %s", profilerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, profilerPluginLibName);
pluginLib = tryOpenLib(profilerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: Plugin name set by env to %s", profilerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, profilerPluginLibName);
} else {
snprintf(profilerPluginLibName, PATH_MAX, "libnccl-profiler.so");
pluginLib = tryOpenLib(profilerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, profilerPluginLibName);
}
return nullptr;
}
enum {
profilerPluginLoadFailed = -1,
profilerPluginLoadReady = 0,
profilerPluginLoadSuccess = 1,
};
static int profilerPluginStatus = profilerPluginLoadReady;
static pid_t pid;
#define MAX_PLUGIN_LOAD 2
static ncclResult_t ncclProfilerPluginLoad(void) {
if (profilerPluginLoadFailed == profilerPluginStatus) {
return ncclSuccess;
}
char couldNotFindNames[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
pthread_mutex_lock(&profilerLock);
if (profilerPluginLoadSuccess == profilerPluginStatus) {
++profilerPluginRefCount;
goto exit;
}
profilerPluginLib = openProfilerPluginLib(couldNotFindNames, MAX_PLUGIN_LOAD * PATH_MAX);
if (profilerPluginLib == nullptr) {
if (strlen(couldNotFindNames)) {
INFO(NCCL_ENV, "PROFILER/Plugin: Could not find:%s.", couldNotFindNames);
}
goto fail;
}
ncclProfiler = (ncclProfiler_v2_t*)dlsym(profilerPluginLib, "ncclProfiler_v2");
if (ncclProfiler == nullptr) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v2.");
ncclProfiler_v1 = (ncclProfiler_v1_t*)dlsym(profilerPluginLib, "ncclProfiler_v1");
if (ncclProfiler_v1 == nullptr) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v1.");
goto fail;
} else {
ncclProfiler = &ncclProfiler_v1_as_v2;
ncclProfiler_v1_as_v2.name = ncclProfiler_v1->name;
ncclProfiler_v1_as_v2.init = ncclProfiler_v1_as_v2_init;
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded ncclProfiler_v1.");
}
} else {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded ncclProfiler_v2.");
}
++profilerPluginRefCount;
profilerPluginStatus = profilerPluginLoadSuccess;
// Store the pid of the process loading the profiler.
// This is attached to the proxyOp event descriptor
// so the plugin can figure out if the parent event
// is in the same address space or not
pid = getpid();
exit:
pthread_mutex_unlock(&profilerLock);
return ncclSuccess;
fail:
if (profilerPluginLib) dlclose(profilerPluginLib);
profilerPluginStatus = profilerPluginLoadFailed;
goto exit;
}
static ncclResult_t ncclProfilerPluginUnload(void) {
pthread_mutex_lock(&profilerLock);
if (0 == (--profilerPluginRefCount)) {
INFO(NCCL_ENV, "PROFILER/Plugin: Closing profiler plugin %s", ncclProfiler->name);
dlclose(profilerPluginLib);
profilerPluginLib = nullptr;
ncclProfiler = nullptr;
profilerPluginStatus = profilerPluginLoadReady;
}
pthread_mutex_unlock(&profilerLock);
return ncclSuccess;
}
#define ENABLE_TIMER 0
#include "timer.h"
#if ENABLE_TIMER
static int64_t elapsedCount;
static int64_t initCount, finalizeCount;
static int64_t groupStartCount, groupStopCount;
static int64_t taskStartCount, taskStopCount;
static int64_t proxyOpStartCount, proxyOpStopCount;
static int64_t proxyStepStartCount, proxyStepStopCount;
static int64_t proxyCtrlStartCount, proxyCtrlStopCount;
static int64_t proxyOpRecordCount, proxyStepRecordCount, proxyCtrlRecordCount;
static double elapsedTs[2];
static double initTs[2], finalizeTs[2];
static double groupStartTs[2], groupStopTs[2];
static double taskStartTs[2], taskStopTs[2];
static double proxyOpStartTs[2], proxyOpStopTs[2];
static double proxyStepStartTs[2], proxyStepStopTs[2];
static double proxyCtrlStartTs[2], proxyCtrlStopTs[2];
static double proxyOpRecordTs[2], proxyStepRecordTs[2], proxyCtrlRecordTs[2];
#define TIME_START_EVENT(event) do { \
(event ## Count)++; \
(event ## Ts)[0] = gettime(); \
} while(0)
#define TIME_STOP_EVENT(event) do { \
double val = gettime() - (event ## Ts)[0]; \
(event ## Ts)[1] += val; \
} while(0)
#define TIME_PRINT_EVENTS(name) do { \
printf("%s ", name); \
if (elapsedCount) printf("[elapsed] %g/%ld = %g ", elapsedTs[1], elapsedCount, elapsedTs[1]/elapsedCount); \
if (initCount) printf("[init] %g/%ld = %g ", initTs[1], initCount, initTs[1]/initCount); \
if (finalizeCount) printf("[finalize] %g/%ld = %g ", finalizeTs[1], finalizeCount, finalizeTs[1]/finalizeCount); \
if (groupStartCount) printf("[groupStart] %g/%ld = %g ", groupStartTs[1], groupStartCount, groupStartTs[1]/groupStartCount); \
if (groupStopCount) printf("[groupStop] %g/%ld = %g ", groupStopTs[1], groupStopCount, groupStopTs[1]/groupStopCount); \
if (taskStartCount) printf("[taskStart] %g/%ld = %g ", taskStartTs[1], taskStartCount, taskStartTs[1]/taskStartCount); \
if (taskStopCount) printf("[taskStop] %g/%ld = %g ", taskStopTs[1], taskStopCount, taskStopTs[1]/taskStopCount); \
if (proxyOpStartCount) printf("[proxyOpStart] %g/%ld = %g ", proxyOpStartTs[1], proxyOpStartCount, proxyOpStartTs[1]/proxyOpStartCount); \
if (proxyOpStopCount) printf("[proxyOpStop] %g/%ld = %g ", proxyOpStopTs[1], proxyOpStopCount, proxyOpStopTs[1]/proxyOpStopCount); \
if (proxyStepStartCount) printf("[proxyStepStart] %g/%ld = %g ", proxyStepStartTs[1], proxyStepStartCount, proxyStepStartTs[1]/proxyStepStartCount); \
if (proxyStepStopCount) printf("[proxyStepStop] %g/%ld = %g ", proxyStepStopTs[1], proxyStepStopCount, proxyStepStopTs[1]/proxyStepStopCount); \
if (proxyCtrlStartCount) printf("[proxyCtrlStart] %g/%ld = %g ", proxyCtrlStartTs[1], proxyCtrlStartCount, proxyCtrlStartTs[1]/proxyCtrlStartCount); \
if (proxyCtrlStopCount) printf("[proxyCtrlStop] %g/%ld = %g ", proxyCtrlStopTs[1], proxyCtrlStopCount, proxyCtrlStopTs[1]/proxyCtrlStopCount); \
if (proxyOpRecordCount) printf("[proxyOpRecord] %g/%ld = %g ", proxyOpRecordTs[1], proxyOpRecordCount, proxyOpRecordTs[1]/proxyOpRecordCount); \
if (proxyStepRecordCount) printf("[proxyStepRecord] %g/%ld = %g ", proxyStepRecordTs[1], proxyStepRecordCount, proxyStepRecordTs[1]/proxyStepRecordCount); \
if (proxyCtrlRecordCount) printf("[proxyCtrlRecord] %g/%ld = %g", proxyCtrlRecordTs[1], proxyCtrlRecordCount, proxyCtrlRecordTs[1]/proxyCtrlRecordCount); \
printf("\n"); \
} while(0)
#else
#define TIME_START_EVENT(event) do {} while(0)
#define TIME_STOP_EVENT(event) do {} while(0)
#define TIME_PRINT_EVENTS(name) do {} while(0)
#endif
static int eActivationMask; // Set by profiler
static int eActivationMaskGroup; // Cached for current group
ncclResult_t ncclProfilerPluginInit(struct ncclComm* comm) {
TIME_START_EVENT(elapsed);
TIME_START_EVENT(init);
ncclProfilerPluginLoad();
if (__builtin_expect(ncclProfiler != NULL, 0)) {
int err = ncclProfiler->init(&comm->profilerContext, &eActivationMask);
if (err) {
WARN("Profiler init failed with error (%d). Continue without profiler.", err);
ncclProfiler = NULL;
}
}
TIME_STOP_EVENT(init);
return ncclSuccess;
}
ncclResult_t ncclProfilerPluginFinalize(struct ncclComm* comm) {
TIME_START_EVENT(finalize);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
ncclProfiler->finalize(comm->profilerContext);
}
ncclProfilerPluginUnload();
TIME_STOP_EVENT(finalize);
TIME_STOP_EVENT(elapsed);
TIME_PRINT_EVENTS("Profiler");
return ncclSuccess;
}
ncclResult_t ncclProfilerStartGroupEvent(struct ncclKernelPlan* plan) {
TIME_START_EVENT(groupStart);
eActivationMaskGroup = __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (eActivationMaskGroup & (ncclProfileColl | ncclProfileP2p | ncclProfileProxyOp | ncclProfileProxyStep)) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileGroup;
ncclProfiler->startEvent(plan->comm->profilerContext, &plan->groupEventHandle, &eDescr);
}
}
TIME_STOP_EVENT(groupStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStopGroupEvent(struct ncclKernelPlan* plan) {
TIME_START_EVENT(groupStop);
if (__builtin_expect(ncclProfiler != NULL, 0) && plan->groupEventHandle) {
ncclProfiler->stopEvent(plan->groupEventHandle);
}
TIME_STOP_EVENT(groupStop);
return ncclSuccess;
}
ncclResult_t ncclProfilerStartTaskEvents(struct ncclKernelPlan* plan) {
TIME_START_EVENT(taskStart);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
int enable = eActivationMaskGroup & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileColl);
if (plan->groupEventHandle && enable) {
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
while (ct) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileColl;
eDescr.parentObj = plan->groupEventHandle;
eDescr.rank = plan->comm->rank;
eDescr.coll.name = plan->comm->commName;
eDescr.coll.commHash = plan->comm->commHash;
eDescr.coll.seqNumber = plan->comm->seqNumber[ct->func]++;
eDescr.coll.func = ncclFuncToString(ct->func);
eDescr.coll.sendBuff = ct->sendbuff;
eDescr.coll.recvBuff = ct->recvbuff;
eDescr.coll.count = ct->count;
eDescr.coll.root = ct->root;
eDescr.coll.datatype = ncclDatatypeToString(ct->datatype);
eDescr.coll.trafficBytes = ct->trafficBytes;
eDescr.coll.nMaxChannels = ct->nMaxChannels;
eDescr.coll.nWarps = ct->nWarps;
eDescr.coll.algo = ncclAlgoToString(ct->algorithm);
eDescr.coll.proto = ncclProtoToString(ct->protocol);
ncclProfiler->startEvent(plan->comm->profilerContext, &ct->eventHandle, &eDescr);
// update collective task with group event activation mask
ct->eActivationMask = eActivationMaskGroup;
ct = ct->next;
}
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
while (pt) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileP2p;
eDescr.parentObj = plan->groupEventHandle;
eDescr.rank = plan->comm->rank;
eDescr.p2p.name = plan->comm->commName;
eDescr.p2p.commHash = plan->comm->commHash;
eDescr.p2p.func = ncclFuncToString(pt->func);
eDescr.p2p.buff = pt->buff;
eDescr.p2p.count = pt->count;
eDescr.p2p.datatype = ncclDatatypeToString(pt->datatype);
eDescr.p2p.peer = pt->root;
ncclProfiler->startEvent(plan->comm->profilerContext, &pt->eventHandle, &eDescr);
// update collective task with group event activation mask
pt->eActivationMask = eActivationMaskGroup;
pt = pt->next;
}
}
}
TIME_STOP_EVENT(taskStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStopTaskEvents(struct ncclKernelPlan* plan) {
TIME_START_EVENT(taskStop);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
int enable = eActivationMaskGroup & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileColl);
if (plan->groupEventHandle && enable) {
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
while (ct) {
ncclProfiler->stopEvent(ct->eventHandle);
ct = ct->next;
}
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
while (pt) {
ncclProfiler->stopEvent(pt->eventHandle);
pt = pt->next;
}
}
}
TIME_STOP_EVENT(taskStop);
return ncclSuccess;
}
// Bellow we set the proxy descriptor step number to DIVUP(step, args->sliceSteps).
// The reason is that for some ncclOp (e.g. AllReduce) one network transfer is
// made of sliceSteps steps rather than one step. In the profiler we are still
// interested in whole network transfers though, so we account for this when
// computing the actual network step number.
ncclResult_t ncclProfilerStartSendProxyOpEvent(int s, struct ncclProxyArgs* args) {
TIME_START_EVENT(proxyOpStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileProxyOp)) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyOp;
eDescr.parentObj = sub->taskEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyOp.pid = sub->pid;
eDescr.proxyOp.channelId = sub->channelId;
eDescr.proxyOp.peer = sub->peer;
eDescr.proxyOp.nSteps = DIVUP(sub->nsteps, args->sliceSteps);
eDescr.proxyOp.chunkSize = args->chunkSize * args->sliceSteps;
eDescr.proxyOp.isSend = 1;
ncclProfiler->startEvent(sub->profilerContext, &sub->opEventHandle, &eDescr);
}
}
TIME_STOP_EVENT(proxyOpStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStartRecvProxyOpEvent(int s, struct ncclProxyArgs* args) {
TIME_START_EVENT(proxyOpStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileProxyOp)) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyOp;
eDescr.parentObj = sub->taskEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyOp.pid = sub->pid;
eDescr.proxyOp.channelId = sub->channelId;
eDescr.proxyOp.peer = sub->peer;
eDescr.proxyOp.nSteps = DIVUP(sub->nsteps, args->sliceSteps);
eDescr.proxyOp.chunkSize = args->chunkSize * args->sliceSteps;
eDescr.proxyOp.isSend = 0;
ncclProfiler->startEvent(sub->profilerContext, &sub->opEventHandle, &eDescr);
}
}
TIME_STOP_EVENT(proxyOpStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStopProxyOpEvent(int s, struct ncclProxyArgs* args) {
TIME_START_EVENT(proxyOpStop);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
ncclProfiler->stopEvent(sub->opEventHandle);
sub->opEventHandle = NULL;
}
TIME_STOP_EVENT(proxyOpStop);
return ncclSuccess;
}
ncclResult_t ncclProfilerStartSendProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
TIME_START_EVENT(proxyStepStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->opEventHandle && (sub->eActivationMask & ncclProfileProxyStep)) {
int step_ = DIVUP(stepId, args->sliceSteps);
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
eDescr.parentObj = sub->opEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyStep.step = step_;
ncclProfiler->startEvent(sub->profilerContext, &sub->stepEventHandles[step_%NCCL_STEPS], &eDescr);
}
}
TIME_STOP_EVENT(proxyStepStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStartRecvProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
TIME_START_EVENT(proxyStepStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->opEventHandle && (sub->eActivationMask & ncclProfileProxyStep)) {
int step_ = DIVUP(stepId, args->sliceSteps);
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
eDescr.parentObj = sub->opEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyStep.step = step_;
ncclProfiler->startEvent(sub->profilerContext, &sub->stepEventHandles[step_%NCCL_STEPS], &eDescr);
}
}
TIME_STOP_EVENT(proxyStepStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStopProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
TIME_START_EVENT(proxyStepStop);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
int step_ = DIVUP(stepId, args->sliceSteps);
if (sub->stepEventHandles[step_%NCCL_STEPS]) {
ncclProfiler->stopEvent(sub->stepEventHandles[step_%NCCL_STEPS]);
sub->stepEventHandles[step_%NCCL_STEPS] = NULL;
}
}
TIME_STOP_EVENT(proxyStepStop);
return ncclSuccess;
}
ncclResult_t ncclProfilerStartProxyCtrlEvent(void* profilerContext, void** eHandle) {
TIME_START_EVENT(proxyCtrlStart);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
// for proxy control events we allow profiling mode to change on a per event basis
int eActivationMaskProxy = __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED);
if (eActivationMaskProxy & ncclProfileProxyCtrl) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyCtrl;
ncclProfiler->startEvent(profilerContext, eHandle, &eDescr);
TIME_STOP_EVENT(proxyCtrlStart);
return ncclSuccess;
}
}
*eHandle = NULL;
TIME_STOP_EVENT(proxyCtrlStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStopProxyCtrlEvent(void* eHandle) {
TIME_START_EVENT(proxyCtrlStop);
if (__builtin_expect(ncclProfiler != NULL, 0) && eHandle) {
ncclProfiler->stopEvent(eHandle);
}
TIME_STOP_EVENT(proxyCtrlStop);
return ncclSuccess;
}
ncclResult_t ncclProfilerRecordProxyOpEventState(int s, struct ncclProxyArgs* args, int steps, size_t transSize, ncclProfilerEventState_t eState) {
TIME_START_EVENT(proxyOpRecord);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
ncclProfilerEventStateArgs_t a = { };
a.proxyOp.steps = DIVUP(steps, args->sliceSteps);
a.proxyOp.transSize = transSize;
ncclProfiler->recordEventState(sub->opEventHandle, eState, &a);
}
TIME_STOP_EVENT(proxyOpRecord);
return ncclSuccess;
}
ncclResult_t ncclProfilerRecordProxyStepEventState(int s, struct ncclProxyArgs* args, int stepId, ncclProfilerEventState_t eState) {
TIME_START_EVENT(proxyStepRecord);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
int step_ = DIVUP(stepId, args->sliceSteps);
if (sub->stepEventHandles[step_%NCCL_STEPS]) {
ncclProfiler->recordEventState(sub->stepEventHandles[step_%NCCL_STEPS], eState, 0);
}
}
TIME_STOP_EVENT(proxyStepRecord);
return ncclSuccess;
}
ncclResult_t ncclProfilerRecordProxyCtrlEventState(void* eHandle, int appended, ncclProfilerEventState_t eState) {
TIME_START_EVENT(proxyCtrlRecord);
if (__builtin_expect(ncclProfiler != NULL, 0) && eHandle && __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED) & ncclProfileProxyCtrl) {
ncclProfilerEventStateArgs_t args = { };
args.proxyCtrl.appendedProxyOps = appended;
ncclProfiler->recordEventState(eHandle, eState, &args);
}
TIME_STOP_EVENT(proxyCtrlRecord);
return ncclSuccess;
}
ncclResult_t ncclProfilerAddPidToProxyOp(struct ncclProxyOp* op) {
op->pid = pid;
return ncclSuccess;
}
src/misc/socket.cc
+10 -1
View File
@@ -176,6 +176,7 @@ static int findInterfaces(const char* prefixList, char* names, union ncclSocketA
strncpy(names+found*maxIfNameSize, interface->ifa_name, maxIfNameSize);
// Store the IP address
int salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
memset(addrs+found, '\0', sizeof(*addrs));
memcpy(addrs+found, interface->ifa_addr, salen);
found++;
}
@@ -919,9 +920,17 @@ ncclResult_t ncclSocketShutdown(struct ncclSocket* sock, int how) {
return ncclSuccess;
}
ncclResult_t ncclSocketClose(struct ncclSocket* sock) {
ncclResult_t ncclSocketClose(struct ncclSocket* sock, bool wait) {
if (sock != NULL) {
if (sock->state > ncclSocketStateNone && sock->state < ncclSocketStateNum && sock->fd >= 0) {
if (wait) {
char data;
int closed = 0;
do {
int offset = 0;
if (ncclSocketProgress(NCCL_SOCKET_RECV, sock, &data, sizeof(char), &offset, &closed) != ncclSuccess) break;
} while (closed == 0);
}
/* shutdown() is needed to send FIN packet to proxy thread; shutdown() is not affected
* by refcount of fd, but close() is. close() won't close a fd and send FIN packet if
* the fd is duplicated (e.g. fork()). So shutdown() guarantees the correct and graceful
src/misc/strongstream.cc
+239 -270
View File
@@ -9,28 +9,61 @@
#include "checks.h"
#include "param.h"
// Tracks the chain of graph nodes for a given graph captured identified by
// its graph id. This state has to live for as long as captured work is being
// submitted. CUDA doesn't have mechanism to inform us when the user ends capture
// so the best we can do is get notified when the graph is destroyed.
struct ncclStrongStreamGraph {
struct ncclStrongStreamGraph* next;
// Atomically exchanged to false by both the main thread or the graph destructor
// callback. The last to arrive deletes the node.
bool alive;
// Tracks the captured work a given graph captured identified by its graph id.
struct ncclStrongStreamCapture {
struct ncclStrongStreamCapture* next;
cudaGraph_t graph;
unsigned long long graphId;
// For each graph we track the "tip" of the chain of graph nodes. A linear
// chain would always have just one node at its tip, but since we have to merge
// in chains from other streams (via ncclStrongStreamWaitStream) some spots
// in the chain can be wider than a single node and thus need a list, so we
// maintain a dynamically sized array of tip nodes.
int tipCount, tipCapacity;
cudaGraphNode_t* tipNodes;
cudaStream_t captureStream;
cudaGraphNode_t lastRecord;
void* acquiredBy;
};
static void ncclStrongStreamGraphDelete(struct ncclStrongStreamGraph* g) {
free(g->tipNodes);
free(g);
////////////////////////////////////////////////////////////////////////////////
static ncclCudaContext* cxtListHead = nullptr;
static pthread_mutex_t cxtListLock = PTHREAD_MUTEX_INITIALIZER;
ncclResult_t ncclCudaContextTrack(struct ncclCudaContext** out) {
ncclResult_t result = ncclSuccess;
CUcontext hcontext;
cuCtxGetCurrent(&hcontext);
pthread_mutex_lock(&cxtListLock);
struct ncclCudaContext* p = cxtListHead;
while (1) {
if (p == nullptr) {
p = (struct ncclCudaContext*)calloc(1, sizeof(struct ncclCudaContext));
p->refCount = 1;
p->hcontext = hcontext;
p->next = cxtListHead;
cxtListHead = p;
NCCLCHECKGOTO(ncclStrongStreamConstruct(&p->launchOrder), result, leave);
break;
}
if (p->hcontext == hcontext) {
p->refCount += 1;
break;
}
p = p->next;
}
leave:
pthread_mutex_unlock(&cxtListLock);
*out = p;
return ncclSuccess;
}
void ncclCudaContextDrop(struct ncclCudaContext* cxt) {
pthread_mutex_lock(&cxtListLock);
if (0 == --cxt->refCount) {
struct ncclCudaContext** pp = &cxtListHead;
while (*pp != cxt) pp = &(*pp)->next;
*pp = cxt->next; // remove from list
// Destroy resources held in cxt
ncclStrongStreamDestruct(&cxt->launchOrder);
free(cxt);
}
pthread_mutex_unlock(&cxtListLock);
}
////////////////////////////////////////////////////////////////////////////////
@@ -40,13 +73,14 @@ ncclResult_t ncclCudaGetCapturingGraph(
) {
#if ROCM_VERSION >= 60100
hipStreamCaptureStatus status;
unsigned long long gid;
CUDACHECK(hipStreamGetCaptureInfo_v2(stream, &status, &gid, &graph->graph, nullptr, nullptr));
CUDACHECK(hipStreamGetCaptureInfo_v2(stream, &status, &graph->graphId, &graph->graph, nullptr, nullptr));
if (status != hipStreamCaptureStatusActive) {
graph->origin = nullptr;
graph->graph = nullptr;
gid = ULLONG_MAX;
graph->graphId = ULLONG_MAX;
} else {
graph->origin = stream;
}
graph->graphId = gid;
#endif
return ncclSuccess;
}
@@ -68,315 +102,250 @@ ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, cudaHostFn_t
////////////////////////////////////////////////////////////////////////////////
ncclResult_t ncclStrongStreamConstruct(struct ncclStrongStream* ss) {
CUDACHECK(cudaStreamCreateWithFlags(&ss->cudaStream, cudaStreamNonBlocking));
CUDACHECK(cudaStreamCreateWithFlags(&ss->liveStream, cudaStreamNonBlocking));
#if ROCM_VERSION >= 60100
CUDACHECK(cudaEventCreateWithFlags(&ss->serialEvent, cudaEventDisableTiming));
ss->everCaptured = false;
ss->serialEventNeedsRecord = false;
ss->graphHead = nullptr;
#else
CUDACHECK(cudaEventCreateWithFlags(&ss->scratchEvent, cudaEventDisableTiming));
ss->captureHead = nullptr;
pthread_mutex_init(&ss->lock, nullptr);
CUDACHECK(cudaEventCreateWithFlags(&ss->serialEvent, cudaEventDisableTiming));
#endif
return ncclSuccess;
}
static void graphDestructor(void* arg) {
struct ncclStrongStreamGraph* g = (struct ncclStrongStreamGraph*)arg;
if (false == __atomic_exchange_n(&g->alive, false, __ATOMIC_ACQ_REL)) {
// Last to arrive deletes list node.
ncclStrongStreamGraphDelete(g);
}
}
ncclResult_t ncclStrongStreamDestruct(struct ncclStrongStream* ss) {
CUDACHECK(cudaStreamDestroy(ss->cudaStream));
CUDACHECK(cudaStreamDestroy(ss->liveStream));
#if ROCM_VERSION >= 60100
CUDACHECK(cudaEventDestroy(ss->serialEvent));
// Delete list of per-graph chains.
struct ncclStrongStreamGraph* g = ss->graphHead;
while (g != nullptr) {
struct ncclStrongStreamGraph* next = g->next;
if (false == __atomic_exchange_n(&g->alive, false, __ATOMIC_ACQ_REL)) {
// Last to arrive deletes list node.
ncclStrongStreamGraphDelete(g);
}
g = next;
struct ncclStrongStreamCapture* cap = ss->captureHead;
while (cap) {
struct ncclStrongStreamCapture* next = cap->next;
CUDACHECK(cudaStreamDestroy(cap->captureStream));
free(cap);
cap = next;
}
#else
CUDACHECK(cudaEventDestroy(ss->scratchEvent));
CUDACHECK(cudaEventDestroy(ss->serialEvent));
pthread_mutex_destroy(&ss->lock);
#endif
return ncclSuccess;
}
NCCL_PARAM(GraphMixingSupport, "GRAPH_MIXING_SUPPORT", 0)
NCCL_PARAM(LaunchRaceFatal, "LAUNCH_RACE_FATAL", 1);
constexpr char const* launchRaceFatalMsg = "Fatal: host threads racing to launch NCCL on same device.";
static void ensureTips(struct ncclStrongStreamGraph* g, int n) {
if (g->tipCapacity < n) {
g->tipNodes = (cudaGraphNode_t*)realloc(g->tipNodes, n*sizeof(cudaGraphNode_t));
g->tipCapacity = n;
}
}
static __thread char threadIdMarker;
static void* localThreadId() { return &threadIdMarker; }
ncclResult_t ncclStrongStreamAcquire(
struct ncclCudaGraph graph, struct ncclStrongStream* ss
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent,
cudaStream_t* workStream
) {
#if ROCM_VERSION >= 60100
bool mixing = ncclParamGraphMixingSupport();
if (graph.graph == nullptr) {
if (mixing && ss->everCaptured) {
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
ss->serialEventNeedsRecord = false;
if (graph.graphId == ULLONG_MAX) {
*workStream = ss->liveStream;
ss->liveAcquiredBy = localThreadId();
if (mixing && __atomic_load_n(&ss->everCaptured, __ATOMIC_RELAXED)) {
CUDACHECK(cudaStreamWaitEvent(ss->liveStream, ss->serialEvent, 0));
}
} else {
ss->everCaptured = true;
// Find the current graph in our list of graphs if it exists.
struct ncclStrongStreamGraph** pg = &ss->graphHead;
struct ncclStrongStreamGraph* g;
while (*pg != nullptr) {
g = *pg;
if (g->graphId == graph.graphId) {
// Move to front of list so that operations after acquire don't have to search the list.
*pg = g->next;
g->next = ss->graphHead;
ss->graphHead = g;
bool firstCapture = !ss->everCaptured;
__atomic_store_n(&ss->everCaptured, true, __ATOMIC_RELAXED);
ncclResult_t ret = ncclSuccess;
if (concurrent) pthread_mutex_lock(&ss->lock);
// Look for capture in our list of active captures.
struct ncclStrongStreamCapture** pcap = &ss->captureHead;
struct ncclStrongStreamCapture* cap;
struct ncclStrongStreamCapture* spare = nullptr;
while (*pcap != nullptr) {
cap = *pcap;
if (cap->graphId == graph.graphId) { // Capture node already exists.
*workStream = cap->captureStream;
cap->acquiredBy = localThreadId();
if (concurrent) pthread_mutex_unlock(&ss->lock);
return ncclSuccess;
} else if (false == __atomic_load_n(&g->alive, __ATOMIC_ACQUIRE)) {
// Unrelated graph that has been destroyed. Remove and delete.
*pg = g->next;
ncclStrongStreamGraphDelete(g);
} else {
pg = &g->next;
cudaStreamCaptureStatus status;
CUDACHECKGOTO(cudaStreamIsCapturing(cap->captureStream, &status), ret, do_unlock);
if (status == cudaStreamCaptureStatusActive) {
pcap = &cap->next; // Active capture doesn't match, on to next.
} else { // Capture no longer active
*pcap = cap->next; // Remove from current list
if (spare == nullptr) { // Keep one spare to reuse below.
spare = cap;
} else {
cudaStreamDestroy(cap->captureStream);
free(cap);
}
}
}
}
// This is a new graph so add to the list.
g = (struct ncclStrongStreamGraph*)malloc(sizeof(struct ncclStrongStreamGraph));
g->graphId = graph.graphId;
g->tipNodes = nullptr;
g->tipCapacity = 0;
g->tipCount = 0;
g->next = ss->graphHead;
ss->graphHead = g;
g->alive = true;
NCCLCHECK(ncclCudaGraphAddDestructor(graph, graphDestructor, (void*)g));
if (mixing && ss->serialEventNeedsRecord) {
// Can only be here if previous release was for uncaptured work that
// elided updating the event because no capture had yet occurred.
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->cudaStream));
// No matching capture, need a new entry.
cap = spare;
if (cap == nullptr) {
cap = (struct ncclStrongStreamCapture*)calloc(1, sizeof(struct ncclStrongStreamCapture));
CUDACHECKGOTO(cudaStreamCreateWithFlags(&cap->captureStream, cudaStreamNonBlocking), ret, do_unlock);
}
ss->serialEventNeedsRecord = false;
cap->graphId = graph.graphId;
cap->lastRecord = nullptr;
cap->acquiredBy = localThreadId();
// Push to capturing list.
cap->next = ss->captureHead;
ss->captureHead = cap;
// First node in the chain must be a wait on the serialEvent.
do_unlock:
if (concurrent) pthread_mutex_unlock(&ss->lock);
if (ret != ncclSuccess) return ret;
*workStream = cap->captureStream;
// Bring captureStream into the graph but without any dependencies.
cudaEvent_t scratch;
CUDACHECK(cudaEventCreateWithFlags(&scratch, cudaEventDisableTiming));
CUDACHECK(cudaEventRecord(scratch, graph.origin));
CUDACHECK(cudaStreamWaitEvent(cap->captureStream, scratch, 0));
CUDACHECK(cudaEventDestroy(scratch));
CUDACHECK(cudaStreamUpdateCaptureDependencies(cap->captureStream, nullptr, 0, cudaStreamSetCaptureDependencies));
if (mixing && firstCapture) {
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->liveStream));
}
if (mixing) {
ensureTips(g, 1);
CUDACHECK(cudaGraphAddEventWaitNode(&g->tipNodes[0], graph.graph, nullptr, 0, ss->serialEvent));
g->tipCount = 1;
} else {
g->tipCount = 0;
// First dependency is to wait on serialEvent
CUDACHECK(cudaStreamWaitEvent(cap->captureStream, ss->serialEvent, 0));
}
}
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamAcquireUncaptured(struct ncclStrongStream* ss) {
ncclResult_t ncclStrongStreamAcquiredWorkStream(
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent,
cudaStream_t* workStream
) {
#if ROCM_VERSION >= 60100
bool mixing = ncclParamGraphMixingSupport();
if (mixing && ss->everCaptured) {
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
if (graph.graphId == ULLONG_MAX) {
*workStream = ss->liveStream;
} else {
if (concurrent) pthread_mutex_lock(&ss->lock);
struct ncclStrongStreamCapture* cap = ss->captureHead;
while (cap->graphId != graph.graphId) cap = cap->next;
*workStream = cap->captureStream;
if (concurrent) pthread_mutex_unlock(&ss->lock);
}
ss->serialEventNeedsRecord = true; // Assume the caller is going to add work to stream.
#else
*workStream = ss->liveStream
#endif
return ncclSuccess;
}
static ncclResult_t checkGraphId(struct ncclStrongStreamGraph* g, unsigned long long id) {
if (g == nullptr || g->graphId != id) {
WARN("Expected graph id=%llu was not at head of strong stream's internal list.", id);
return ncclInternalError;
}
return ncclSuccess;
}
ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStrongStream* ss) {
ncclResult_t ncclStrongStreamRelease(
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent
) {
#if ROCM_VERSION >= 60100
bool mixing = ncclParamGraphMixingSupport();
if (mixing && ss->serialEventNeedsRecord) {
if (graph.graph == nullptr) {
if (ss->everCaptured) {
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->cudaStream));
ss->serialEventNeedsRecord = false;
if (mixing) {
if (graph.graphId == ULLONG_MAX) {
if (__atomic_load_n(&ss->everCaptured, __ATOMIC_RELAXED)) {
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->liveStream));
}
if (ss->liveAcquiredBy != localThreadId() && ncclParamLaunchRaceFatal()) {
WARN("%s", launchRaceFatalMsg);
return ncclInvalidUsage;
}
} else {
struct ncclStrongStreamGraph* g = ss->graphHead;
NCCLCHECK(checkGraphId(g, graph.graphId));
ensureTips(g, 1);
CUDACHECK(cudaGraphAddEventRecordNode(&g->tipNodes[0], graph.graph, g->tipNodes, g->tipCount, ss->serialEvent));
g->tipCount = 1;
ss->serialEventNeedsRecord = false;
if (concurrent) pthread_mutex_lock(&ss->lock);
struct ncclStrongStreamCapture* cap = ss->captureHead;
while (cap->graphId != graph.graphId) cap = cap->next;
if (concurrent) pthread_mutex_unlock(&ss->lock);
// Add event record node with dependencies added further down.
cudaGraphNode_t recordNode;
CUDACHECK(cudaGraphAddEventRecordNode(&recordNode, graph.graph, nullptr, 0, ss->serialEvent));
// Make this record order after previous record on this stream.
if (cap->lastRecord != nullptr) {
CUDACHECK(cudaGraphAddDependencies(graph.graph, &cap->lastRecord, &recordNode, 1));
}
cap->lastRecord = recordNode;
// Get current nodes from work stream so we can add them as dependencies.
cudaStreamCaptureStatus status;
cudaGraphNode_t const* nodes;
size_t count = 0;
cudaError_t res = hipStreamGetCaptureInfo_v2(cap->captureStream, &status, nullptr, nullptr, &nodes, &count);
#if CUDART_VERSION >= 12030
if (res == cudaErrorLossyQuery) { // CUDA is telling us the dependencies have edge annotations.
cudaGraphEdgeData const* edges;
CUDACHECK(cudaStreamGetCaptureInfo_v3(cap->captureStream, &status, nullptr, nullptr, &nodes, &edges, &count));
for (int i=0; i < (int)count; i++) {
CUDACHECK(cudaGraphAddDependencies_v2(graph.graph, &nodes[i], &recordNode, &edges[i], 1));
}
}
#else
if (false) {}
#endif
else {
CUDACHECK(res /* = cudaStreamGetCaptureInfo_v2(...)*/);
for (int i=0; i < (int)count; i++) {
CUDACHECK(cudaGraphAddDependencies(graph.graph, &nodes[i], &recordNode, 1));
}
}
if (cap->acquiredBy != localThreadId() && ncclParamLaunchRaceFatal()) {
WARN("%s", launchRaceFatalMsg);
return ncclInvalidUsage;
}
}
}
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamLaunchHost(
struct ncclCudaGraph graph, struct ncclStrongStream* ss, cudaHostFn_t fn, void* arg
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
CUDACHECK(cudaLaunchHostFunc(ss->cudaStream, fn, arg));
} else {
cudaHostNodeParams p;
p.fn = fn;
p.userData = arg;
struct ncclStrongStreamGraph* g = ss->graphHead;
NCCLCHECK(checkGraphId(g, graph.graphId));
ensureTips(g, 1);
CUDACHECK(cudaGraphAddHostNode(&g->tipNodes[0], graph.graph, g->tipNodes, g->tipCount, &p));
g->tipCount = 1;
}
ss->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaLaunchHostFunc(ss->cudaStream, fn, arg));
#endif
ncclResult_t ncclStreamWaitStream(cudaStream_t a, cudaStream_t b, cudaEvent_t scratchEvent) {
CUDACHECK(cudaEventRecord(scratchEvent, b));
CUDACHECK(cudaStreamWaitEvent(a, scratchEvent, 0));
return ncclSuccess;
}
ncclResult_t ncclStrongStreamLaunchKernel(
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
void* fn, dim3 grid, dim3 block, void* args[], size_t sharedMemBytes
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
CUDACHECK(cudaLaunchKernel(fn, grid, block, args, sharedMemBytes, ss->cudaStream));
} else {
cudaKernelNodeParams p;
p.func = fn;
p.gridDim = grid;
p.blockDim = block;
p.kernelParams = args;
p.sharedMemBytes = sharedMemBytes;
p.extra = nullptr;
struct ncclStrongStreamGraph* g = ss->graphHead;
NCCLCHECK(checkGraphId(g, graph.graphId));
ensureTips(g, 1);
CUDACHECK(cudaGraphAddKernelNode(&g->tipNodes[0], graph.graph, g->tipNodes, g->tipCount, &p));
g->tipCount = 1;
}
ss->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaLaunchKernel(fn, grid, block, args, sharedMemBytes, ss->cudaStream));
#endif
return ncclSuccess;
}
ncclResult_t ncclStreamAdvanceToEvent(struct ncclCudaGraph g, cudaStream_t s, cudaEvent_t e) {
if (g.graphId == ULLONG_MAX) {
CUDACHECK(cudaStreamWaitEvent(s, e, 0));
} else {
cudaStream_t tmp;
CUDACHECK(cudaStreamCreateWithFlags(&tmp, cudaStreamNonBlocking));
CUDACHECK(cudaStreamWaitEvent(tmp, e, 0));
// Merge node list `b` into list `a` but don't add duplicates.
static void mergeTips(struct ncclStrongStreamGraph* a, cudaGraphNode_t const* bNodes, int bn) {
int an = a->tipCount;
ensureTips(a, an + bn);
for (int bi=0; bi < bn; bi++) {
for (int ai=0; ai < an; ai++) {
if (a->tipNodes[ai] == bNodes[bi]) goto next_b;
cudaStreamCaptureStatus status;
cudaGraphNode_t const* nodes;
size_t count = 0;
cudaError_t res = hipStreamGetCaptureInfo_v2(tmp, &status, nullptr, nullptr, &nodes, &count);
#if CUDART_VERSION >= 12030
if (res == cudaErrorLossyQuery) { // CUDA is telling us the dependencies have edge annotations.
cudaGraphEdgeData const* edges;
CUDACHECK(cudaStreamGetCaptureInfo_v3(tmp, &status, nullptr, nullptr, &nodes, &edges, &count));
CUDACHECK(cudaStreamUpdateCaptureDependencies_v2(s, (cudaGraphNode_t*)nodes, edges, count, cudaStreamSetCaptureDependencies));
}
a->tipNodes[a->tipCount++] = bNodes[bi];
next_b:;
#else
if (false) {}
#endif
else {
CUDACHECK(res /* = cudaStreamGetCaptureInfo_v2(...)*/);
CUDACHECK(cudaStreamUpdateCaptureDependencies(s, (cudaGraphNode_t*)nodes, count, cudaStreamSetCaptureDependencies));
}
CUDACHECK(cudaStreamDestroy(tmp));
}
}
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, struct ncclStrongStream* b,
bool b_subsumes_a
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
if (b->serialEventNeedsRecord) {
b->serialEventNeedsRecord = false;
CUDACHECK(cudaEventRecord(b->serialEvent, b->cudaStream));
}
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, b->serialEvent, 0));
} else {
struct ncclStrongStreamGraph* ag = a->graphHead;
NCCLCHECK(checkGraphId(ag, graph.graphId));
struct ncclStrongStreamGraph* bg = b->graphHead;
NCCLCHECK(checkGraphId(bg, graph.graphId));
if (b_subsumes_a) ag->tipCount = 0;
mergeTips(ag, bg->tipNodes, bg->tipCount);
}
a->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaEventRecord(b->scratchEvent, b->cudaStream));
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, b->scratchEvent, 0));
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, cudaStream_t b,
bool b_subsumes_a
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
// It is ok to use a->serialEvent to record b since we'll be setting
// a->serialEventNeedsRecord so the event won't be considered accurate
// until re-recorded.
CUDACHECK(cudaEventRecord(a->serialEvent, b));
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, a->serialEvent, 0));
} else {
cudaStreamCaptureStatus status;
unsigned long long bGraphId;
cudaGraphNode_t const* bNodes;
size_t bCount = 0;
CUDACHECK(hipStreamGetCaptureInfo_v2(b, &status, &bGraphId, nullptr, &bNodes, &bCount));
if (status != cudaStreamCaptureStatusActive || graph.graphId != bGraphId) {
WARN("Stream is not being captured by the expected graph.");
return ncclInvalidUsage;
}
struct ncclStrongStreamGraph* ag = a->graphHead;
NCCLCHECK(checkGraphId(ag, graph.graphId));
if (b_subsumes_a) ag->tipCount = 0;
mergeTips(ag, bNodes, bCount);
}
a->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaEventRecord(a->scratchEvent, b));
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, a->scratchEvent, 0));
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, cudaStream_t a, struct ncclStrongStream* b,
bool b_subsumes_a
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
if (b->serialEventNeedsRecord) {
b->serialEventNeedsRecord = false;
CUDACHECK(cudaEventRecord(b->serialEvent, b->cudaStream));
}
CUDACHECK(cudaStreamWaitEvent(a, b->serialEvent, 0));
} else {
struct ncclStrongStreamGraph* bg = b->graphHead;
NCCLCHECK(checkGraphId(bg, graph.graphId));
CUDACHECK(hipStreamUpdateCaptureDependencies(a, bg->tipNodes, bg->tipCount,
b_subsumes_a ? cudaStreamSetCaptureDependencies : cudaStreamAddCaptureDependencies
));
}
#else
CUDACHECK(cudaEventRecord(b->scratchEvent, b->cudaStream));
CUDACHECK(cudaStreamWaitEvent(a, b->scratchEvent, 0));
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamSynchronize(struct ncclStrongStream* ss) {
#if ROCM_VERSION >= 60100
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
ss->serialEventNeedsRecord = false;
CUDACHECK(cudaStreamWaitEvent(ss->liveStream, ss->serialEvent, 0));
#endif
CUDACHECK(cudaStreamSynchronize(ss->cudaStream));
CUDACHECK(cudaStreamSynchronize(ss->liveStream));
return ncclSuccess;
}
src/misc/tuner.cc
-267
View File
@@ -1,267 +0,0 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#include <dlfcn.h>
#include <errno.h>
#include <stdlib.h>
#include "checks.h"
#include "debug.h"
#include "tuner.h"
pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
static int tunerPluginRefCount;
static void* tunerPluginLib = nullptr;
static ncclTuner_v4_t* tunerSymbol = nullptr;
static ncclTuner_v3_t* ncclTuner_v3 = nullptr;
static ncclTuner_v2_t* ncclTuner_v2 = nullptr;
static ncclTuner_v4_t ncclTuner_v2_as_v4;
static ncclTuner_v4_t ncclTuner_v3_as_v4;
static int hasNvlsSupport(float** collCostTable) {
// Requirements for support of different algorithms:
//
// - NVLS intra-node: nvlsSupport
// - NVLS intra+inter-node: collNetSupport
// - NVLSTree intra-node: always disabled
// - NVLSTree inter-node: nvlsSupport
// - Collnet* inter-node: collNetSupport
//
// nvlsSupport = 1 if either NVLS or NVLS_TREE entries in the cost table are not -1
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE || table[NCCL_ALGO_NVLS_TREE][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) ? 1 : 0;
}
static int hasCollNetSupport(float** collCostTable) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] == NCCL_ALGO_PROTO_IGNORE) ? 0 : 1;
}
static ncclResult_t ncclTuner_v3_as_v4_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff __attribute__((unused)), int* nChannels) {
NCCLCHECK(ncclTuner_v3->getCollInfo(context, collType, nBytes, numPipeOps, collCostTable, numAlgo, numProto, nChannels));
return ncclSuccess;
}
static ncclResult_t ncclTuner_v3_as_v4_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
NCCLCHECK(ncclTuner_v3->init(nRanks, nNodes, logFunction, context));
ncclTuner_v3_as_v4.name = ncclTuner_v3->name;
ncclTuner_v3_as_v4.getCollInfo = ncclTuner_v3_as_v4_getCollInfo;
ncclTuner_v3_as_v4.destroy = ncclTuner_v3->destroy;
return ncclSuccess;
}
static ncclResult_t ncclTuner_v2_as_v4_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int regBuff __attribute__((unused)), int* nChannels) {
int algorithm = NCCL_ALGO_UNDEF;
int protocol = NCCL_PROTO_UNDEF;
int nvlsSupport = hasNvlsSupport(collCostTable);
int collNetSupport = hasCollNetSupport(collCostTable);
NCCLCHECK(ncclTuner_v2->getCollInfo(context, collType, nBytes, collNetSupport, nvlsSupport, numPipeOps, &algorithm, &protocol, nChannels));
// set time to 0 below to make sure this algorithm/protocol is selected later on
if (algorithm >= 0 && algorithm < NCCL_NUM_ALGORITHMS && protocol >= 0 && protocol < NCCL_NUM_PROTOCOLS) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
if (table[algorithm][protocol] != NCCL_ALGO_PROTO_IGNORE) table[algorithm][protocol] = 0.0;
}
return ncclSuccess;
}
static ncclResult_t ncclTuner_v2_as_v4_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
NCCLCHECK(ncclTuner_v2->init(nRanks, nNodes, logFunction, context));
ncclTuner_v2_as_v4.name = ncclTuner_v2->name;
ncclTuner_v2_as_v4.getCollInfo = ncclTuner_v2_as_v4_getCollInfo;
ncclTuner_v2_as_v4.destroy = ncclTuner_v2->destroy;
return ncclSuccess;
}
#define MAX_STR_LEN 255
static void* tryOpenLib(const char* name, int* err, char* errStr) {
*err = 0;
if (nullptr == name || strlen(name) == 0) {
return nullptr;
}
if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
name = nullptr;
}
void *handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
if (nullptr == handle) {
strncpy(errStr, dlerror(), MAX_STR_LEN);
errStr[MAX_STR_LEN] = '\0';
// "handle" and "name" won't be NULL at the same time.
// coverity[var_deref_model]
if (strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
*err = ENOENT;
}
}
return handle;
}
static char* tryOpenLibCheck(int openErr, char* openErrStr, char* nameList, int *nameListLen, char* name) {
if (openErr == ENOENT) {
snprintf(nameList, *nameListLen, " %s", name);
nameList += strlen(name) + 1;
*nameListLen -= strlen(name) + 1;
return nameList;
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: %s", openErrStr);
return nameList;
}
static void* openTunerPluginLib(char* couldNotFindNames, int len) {
int openErr;
void *pluginLib;
char tunerPluginLibName[PATH_MAX];
char openErrStr[MAX_STR_LEN + 1] = { 0 };
const char *envTunerPluginName = getenv("NCCL_TUNER_PLUGIN");
if (envTunerPluginName && strlen(envTunerPluginName)) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: NCCL_TUNER_PLUGIN set to %s", envTunerPluginName);
snprintf(tunerPluginLibName, PATH_MAX, "%s", envTunerPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner-%s.so", envTunerPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
} else {
snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner.so");
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
}
const char *envNetPluginName = getenv("NCCL_NET_PLUGIN");
if (envNetPluginName && strlen(envNetPluginName)) {
// Users are allowed to pack tuner into the net plugin
snprintf(tunerPluginLibName, PATH_MAX, "%s", envNetPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
snprintf(tunerPluginLibName, PATH_MAX, "librccl-net-%s.so", envNetPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
} else {
snprintf(tunerPluginLibName, PATH_MAX, "librccl-net.so");
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
}
tunerPluginLibName[0] = '\0';
return nullptr;
}
enum {
tunerPluginLoadFailed = -1,
tunerPluginLoadReady = 0,
tunerPluginLoadSuccess = 1,
};
#define MAX_PLUGIN_LOAD 4
static int status = tunerPluginLoadReady;
ncclResult_t ncclTunerPluginLoad(struct ncclComm* comm) {
// Initialize to nullptr by default if plugin tuner cannot be loaded.
char couldNotFindNames[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
comm->tuner = nullptr;
if (tunerPluginLoadFailed == status) {
return ncclSuccess;
}
pthread_mutex_lock(&tunerPluginLock);
if (tunerPluginLoadFailed == status) {
goto exit;
}
if (tunerPluginLoadSuccess == status) {
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
goto exit;
}
tunerPluginLib = openTunerPluginLib(couldNotFindNames, MAX_PLUGIN_LOAD * PATH_MAX);
if (nullptr == tunerPluginLib) {
if (strlen(couldNotFindNames)) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Could not find:%s. Using internal tuner plugin.", couldNotFindNames);
} else {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using internal tuner plugin.");
}
goto fail;
}
tunerSymbol = (ncclTuner_v4_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v4");
if (tunerSymbol == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.");
ncclTuner_v3 = (ncclTuner_v3_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v3");
if (ncclTuner_v3 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v2");
if (ncclTuner_v2 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
dlclose(tunerPluginLib);
goto fail;
} else {
ncclTuner_v2_as_v4.init = ncclTuner_v2_as_v4_init;
ncclTuner_v2_as_v4.name = ncclTuner_v2->name;
tunerSymbol = &ncclTuner_v2_as_v4;
}
} else {
ncclTuner_v3_as_v4.init = ncclTuner_v3_as_v4_init;
ncclTuner_v3_as_v4.name = ncclTuner_v3->name;
tunerSymbol = &ncclTuner_v3_as_v4;
}
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", tunerSymbol->name);
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
status = tunerPluginLoadSuccess;
comm->tunerPluginLoaded = 1;
exit:
pthread_mutex_unlock(&tunerPluginLock);
return ncclSuccess;
fail:
tunerPluginLib = nullptr;
status = tunerPluginLoadFailed;
goto exit;
}
ncclResult_t ncclTunerPluginUnload(struct ncclComm* comm) {
pthread_mutex_lock(&tunerPluginLock);
if (comm->tunerPluginLoaded && 0 == (--tunerPluginRefCount)) {
INFO(NCCL_TUNING, "TUNER/Plugin: Closing tuner: '%s'", tunerSymbol->name);
dlclose(tunerPluginLib);
tunerPluginLib = nullptr;
tunerSymbol = nullptr;
comm->tuner = nullptr;
status = tunerPluginLoadReady;
comm->tunerPluginLoaded = 0;
}
pthread_mutex_unlock(&tunerPluginLock);
return ncclSuccess;
}