Merge remote-tracking branch 'nccl/master' into develop
Этот коммит содержится в:
BertanDogancay
@@ -0,0 +1,323 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "net.h"
|
||||
#include "bootstrap.h"
|
||||
#include "checks.h"
|
||||
#include "plugin.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
//#include <sys/types.h>
|
||||
//#include <sys/stat.h>
|
||||
//#include <unistd.h>
|
||||
|
||||
extern ncclNet_t* getNcclNet_v6(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v7(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v8(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v9(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v10(void* netPluginLib);
|
||||
|
||||
extern ncclCollNet_t* getNcclCollNet_v6(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v7(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v8(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v9(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v10(void* netPluginLib);
|
||||
|
||||
static pthread_mutex_t netLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
ncclNet_t* ncclNets[NCCL_NET_MAX_PLUGINS] = { nullptr, &ncclNetIb, &ncclNetSocket };
|
||||
static int ncclNetsVer[NCCL_NET_MAX_PLUGINS] = { -1, 10, 10 };
|
||||
ncclCollNet_t* ncclCollNets[NCCL_NET_MAX_PLUGINS] = { nullptr, nullptr, nullptr };
|
||||
enum ncclNetState {
|
||||
ncclNetStateInit = 0,
|
||||
ncclNetStateEnabled = 1,
|
||||
ncclNetStateDisabled = 2
|
||||
};
|
||||
enum ncclNetState ncclNetStates[NCCL_NET_MAX_PLUGINS] = { ncclNetStateInit, ncclNetStateInit, ncclNetStateInit };
|
||||
enum ncclNetState ncclCollNetStates[NCCL_NET_MAX_PLUGINS] = { ncclNetStateInit, ncclNetStateInit, ncclNetStateInit };
|
||||
|
||||
NCCL_PARAM(NetPluginRefCount, "NET_PLUGIN_REF_COUNT", 1);
|
||||
static pthread_mutex_t netPluginLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
static void* netPluginLib;
|
||||
|
||||
static int netPluginRefCount;
|
||||
static void initNetPluginRefCountOnce(void) { netPluginRefCount = ncclParamNetPluginRefCount();}
|
||||
|
||||
enum {
|
||||
netPluginLoadFailed = -1,
|
||||
netPluginLoadReady = 0,
|
||||
netPluginLoadSuccess = 1,
|
||||
};
|
||||
|
||||
static int netPluginStatus = netPluginLoadReady;
|
||||
|
||||
ncclResult_t ncclNetPluginLoad(struct ncclComm* comm) {
|
||||
static pthread_once_t netPluginRefCountOnce = PTHREAD_ONCE_INIT;
|
||||
pthread_once(&netPluginRefCountOnce, initNetPluginRefCountOnce);
|
||||
|
||||
pthread_mutex_lock(&netPluginLock);
|
||||
if (netPluginLoadFailed == netPluginStatus) {
|
||||
goto exit;
|
||||
}
|
||||
if (netPluginLoadSuccess == netPluginStatus) {
|
||||
++netPluginRefCount;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
netPluginLib = ncclOpenNetPluginLib(ncclGetEnv("NCCL_NET_PLUGIN"));
|
||||
if (netPluginLib == nullptr) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
ncclNets[0] = getNcclNet_v10(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 10;
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v9 plugin
|
||||
ncclNets[0] = getNcclNet_v9(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 9;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v8 plugin
|
||||
ncclNets[0] = getNcclNet_v8(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 8;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v7 plugin
|
||||
ncclNets[0] = getNcclNet_v7(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 7;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v6 plugin
|
||||
ncclNets[0] = getNcclNet_v6(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 6;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
// Check for CollNet
|
||||
ncclCollNets[0] = getNcclCollNet_v10(netPluginLib);
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v9(netPluginLib);
|
||||
}
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v8(netPluginLib);
|
||||
}
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v7(netPluginLib);
|
||||
}
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v6(netPluginLib);
|
||||
}
|
||||
|
||||
++netPluginRefCount;
|
||||
netPluginStatus = netPluginLoadSuccess;
|
||||
comm->netPluginLoaded = 1;
|
||||
|
||||
exit:
|
||||
pthread_mutex_unlock(&netPluginLock);
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
if (netPluginLib) NCCLCHECK(ncclClosePluginLib(netPluginLib));
|
||||
netPluginStatus = netPluginLoadFailed;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetPluginUnload(struct ncclComm* comm) {
|
||||
pthread_mutex_lock(&netPluginLock);
|
||||
if (comm->netPluginLoaded && 0 == (--netPluginRefCount)) {
|
||||
if (ncclNets[0]) {
|
||||
INFO(NCCL_NET, "NET/Plugin: Closing net plugin '%s'", ncclNets[0]->name);
|
||||
}
|
||||
if (ncclCollNets[0]) {
|
||||
INFO(NCCL_NET, "NET/Plugin: Closing collnet plugin '%s'", ncclCollNets[0]->name);
|
||||
}
|
||||
NCCLCHECK(ncclClosePluginLib(netPluginLib));
|
||||
netPluginLib = nullptr;
|
||||
ncclNets[0] = nullptr;
|
||||
ncclCollNets[0] = nullptr;
|
||||
netPluginStatus = netPluginLoadReady;
|
||||
comm->netPluginLoaded = 0;
|
||||
for (int i = 0; i < NCCL_NET_MAX_PLUGINS; ++i)
|
||||
ncclCollNetStates[i] = ncclNetStates[i] = ncclNetStateInit;
|
||||
}
|
||||
pthread_mutex_unlock(&netPluginLock);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetCheckDeviceVersion(struct ncclComm* comm, ncclNet_t* net, int dev) {
|
||||
ncclNetProperties_t props;
|
||||
|
||||
NCCLCHECK(net->getProperties(dev, &props));
|
||||
ncclNetDeviceType type = props.netDeviceType;
|
||||
if (type) switch (type) {
|
||||
case NCCL_NET_DEVICE_UNPACK:
|
||||
if (props.netDeviceVersion == NCCL_NET_DEVICE_UNPACK_VERSION) {
|
||||
INFO(NCCL_INIT, "Using NCCL_NET_DEVICE_UNPACK net plugin version %d",
|
||||
props.netDeviceVersion);
|
||||
return ncclSuccess;
|
||||
} else {
|
||||
WARN("NCCL_DEVICE_UNPACK plugin has incompatible version %d, this NCCL build is compatible with %d, not using it",
|
||||
props.netDeviceVersion, NCCL_NET_DEVICE_UNPACK_VERSION);
|
||||
return ncclInternalError;
|
||||
}
|
||||
default:
|
||||
WARN("Unknown device code index %d \n", type);
|
||||
return ncclInternalError;
|
||||
}
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t netGetState(int i, enum ncclNetState* state) {
|
||||
pthread_mutex_lock(&netLock);
|
||||
if (ncclNetStates[i] == ncclNetStateInit) {
|
||||
int ndev;
|
||||
if (ncclNets[i]->init(ncclDebugLog, ncclProfilerCallback) != ncclSuccess) ncclNetStates[i] = ncclNetStateDisabled;
|
||||
else if (ncclNets[i]->devices(&ndev) != ncclSuccess || ndev <= 0) ncclNetStates[i] = ncclNetStateDisabled;
|
||||
else ncclNetStates[i] = ncclNetStateEnabled;
|
||||
}
|
||||
*state = ncclNetStates[i];
|
||||
pthread_mutex_unlock(&netLock);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t collNetGetState(int i, enum ncclNetState* state) {
|
||||
pthread_mutex_lock(&netLock);
|
||||
if (ncclCollNetStates[i] == ncclNetStateInit) {
|
||||
int ndev;
|
||||
if (ncclCollNets[i]->init(ncclDebugLog) != ncclSuccess) ncclCollNetStates[i] = ncclNetStateDisabled;
|
||||
else if (ncclCollNets[i]->devices(&ndev) != ncclSuccess || ndev <= 0) ncclCollNetStates[i] = ncclNetStateDisabled;
|
||||
else ncclCollNetStates[i] = ncclNetStateEnabled;
|
||||
}
|
||||
*state = ncclCollNetStates[i];
|
||||
pthread_mutex_unlock(&netLock);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetInit(struct ncclComm* comm) {
|
||||
// Initialize main communication network
|
||||
const char* netName;
|
||||
bool ok = false;
|
||||
|
||||
netName = comm->config.netName;
|
||||
for (int i=0; i<3; i++) {
|
||||
if (ncclNets[i] == nullptr) continue;
|
||||
enum ncclNetState state;
|
||||
NCCLCHECK(netGetState(i, &state));
|
||||
if (state != ncclNetStateEnabled) continue;
|
||||
if (netName && strcasecmp(netName, ncclNets[i]->name) != 0) continue;
|
||||
if (ncclSuccess != ncclNetCheckDeviceVersion(comm, ncclNets[i], 0)) {
|
||||
// Mismatched device plugin version
|
||||
continue;
|
||||
}
|
||||
|
||||
comm->ncclNet = ncclNets[i];
|
||||
comm->ncclNetVer = ncclNetsVer[i];
|
||||
ok = true;
|
||||
|
||||
if (ncclCollNets[i]) {
|
||||
NCCLCHECK(collNetGetState(i, &state));
|
||||
if (state == ncclNetStateEnabled) {
|
||||
comm->ncclCollNet = ncclCollNets[i];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
WARN("Error: network %s not found.", netName ? netName : "");
|
||||
return ncclInvalidUsage;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetFinalize(struct ncclComm* comm) {
|
||||
comm->ncclNet = nullptr;
|
||||
comm->ncclCollNet = nullptr;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclGpuGdrSupport(struct ncclComm* comm, int* gdrSupport) {
|
||||
constexpr int GPU_BUF_SIZE = 2*1024*1024;
|
||||
#if CUDART_VERSION >= 11030
|
||||
// In CUDA 11.3 and later we can now query the cudaDevAttrGPUDirectRDMASupported attribute
|
||||
int driverVersion;
|
||||
CUDACHECK(cudaDriverGetVersion(&driverVersion));
|
||||
if (driverVersion >= 11030) {
|
||||
int cudaDev, attr = 0;
|
||||
CUDACHECK(cudaGetDevice(&cudaDev));
|
||||
CUDACHECK(cudaDeviceGetAttribute(&attr, cudaDevAttrGPUDirectRDMASupported, cudaDev));
|
||||
*gdrSupport = attr;
|
||||
return ncclSuccess;
|
||||
}
|
||||
#endif
|
||||
static int gdrSupportMatrix[32] = {
|
||||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 };
|
||||
if (gdrSupportMatrix[comm->cudaDev] == -1) {
|
||||
int netDevs;
|
||||
NCCLCHECK(comm->ncclNet->devices(&netDevs));
|
||||
gdrSupportMatrix[comm->cudaDev] = 0;
|
||||
for (int dev=0; dev<netDevs; dev++) {
|
||||
// Find a net device which is GDR-capable
|
||||
ncclNetProperties_t props;
|
||||
NCCLCHECK(comm->ncclNet->getProperties(dev, &props));
|
||||
if ((props.ptrSupport & NCCL_PTR_CUDA) == 0) continue;
|
||||
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
|
||||
gdrSupportMatrix[comm->cudaDev] = 1;
|
||||
break;
|
||||
#endif
|
||||
|
||||
// Allocate memory on the GPU and try to register it on the NIC.
|
||||
void *lComm = NULL, *sComm = NULL, *rComm = NULL;
|
||||
ncclNetHandle_t handle;
|
||||
char* gpuPtr = NULL;
|
||||
void* mHandle = NULL;
|
||||
ncclResult_t ret;
|
||||
ncclDebugNoWarn = NCCL_NET;
|
||||
NCCLCHECKGOTO(comm->ncclNet->listen(dev, &handle, &lComm), ret, cleanup1);
|
||||
|
||||
bool connected;
|
||||
connected = false;
|
||||
while (!connected) {
|
||||
|
||||
// If we're aborting now, skip to cleanup
|
||||
if (__atomic_load_n(comm->abortFlag, __ATOMIC_ACQUIRE)) {
|
||||
goto cleanup2;
|
||||
}
|
||||
|
||||
if (sComm == NULL)
|
||||
NCCLCHECKGOTO(comm->ncclNet->connect(dev, NULL, &handle, &sComm, NULL), ret, cleanup2);
|
||||
|
||||
if (rComm == NULL)
|
||||
NCCLCHECKGOTO(comm->ncclNet->accept(lComm, &rComm, NULL), ret, cleanup2);
|
||||
|
||||
connected = (rComm != NULL) && (sComm != NULL);
|
||||
}
|
||||
|
||||
NCCLCHECKGOTO(ncclCudaMalloc(&gpuPtr, GPU_BUF_SIZE), ret, cleanup2);
|
||||
if (comm->ncclNet->regMr(sComm, gpuPtr, GPU_BUF_SIZE, NCCL_PTR_CUDA, &mHandle) == ncclSuccess) {
|
||||
NCCLCHECK(comm->ncclNet->deregMr(sComm, mHandle));
|
||||
NCCLCHECK(comm->ncclNet->regMr(rComm, gpuPtr, GPU_BUF_SIZE, NCCL_PTR_CUDA, &mHandle));
|
||||
NCCLCHECK(comm->ncclNet->deregMr(rComm, mHandle));
|
||||
gdrSupportMatrix[comm->cudaDev] = 1;
|
||||
}
|
||||
ncclDebugNoWarn = 0;
|
||||
NCCLCHECK(ncclCudaFree(gpuPtr));
|
||||
cleanup2:
|
||||
if (rComm != NULL)
|
||||
NCCLCHECK(comm->ncclNet->closeRecv(rComm));
|
||||
if (sComm != NULL)
|
||||
NCCLCHECK(comm->ncclNet->closeSend(sComm));
|
||||
NCCLCHECK(comm->ncclNet->closeListen(lComm));
|
||||
cleanup1:
|
||||
break;
|
||||
}
|
||||
}
|
||||
*gdrSupport = gdrSupportMatrix[comm->cudaDev];
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "nccl_net.h"
|
||||
#include "net_device.h"
|
||||
#include "proxy.h"
|
||||
|
||||
static ncclNet_v10_t* ncclNet_v10;
|
||||
static ncclCollNet_v10_t* ncclCollNet_v10;
|
||||
|
||||
ncclNet_t* getNcclNet_v10(void* lib) {
|
||||
ncclNet_v10 = (ncclNet_v10_t*)dlsym(lib, "ncclNetPlugin_v10");
|
||||
if (ncclNet_v10) {
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v10)", ncclNet_v10->name);
|
||||
return ncclNet_v10;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v10 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ncclCollNet_t* getNcclCollNet_v10(void* lib) {
|
||||
ncclCollNet_v10 = (ncclCollNet_v10_t*)dlsym(lib, "ncclCollNetPlugin_v10");
|
||||
if (ncclCollNet_v10) {
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v10)", ncclNet_v10->name);
|
||||
return ncclCollNet_v10;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v10 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "nccl_net.h"
|
||||
#include "net_device.h"
|
||||
#include "proxy.h"
|
||||
#include "checks.h"
|
||||
|
||||
static ncclNet_t ncclNet;
|
||||
static ncclCollNet_t ncclCollNet;
|
||||
static ncclNet_v6_t* ncclNet_v6;
|
||||
static ncclCollNet_v6_t* ncclCollNet_v6;
|
||||
|
||||
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
ncclNetProperties_v6_t p6;
|
||||
ncclResult_t ans = ncclNet_v6->getProperties(dev, &p6);
|
||||
if (ans != ncclSuccess) return ans;
|
||||
props->name = p6.name;
|
||||
props->pciPath = p6.pciPath;
|
||||
props->guid = p6.guid;
|
||||
props->ptrSupport = p6.ptrSupport;
|
||||
props->regIsGlobal = 0;
|
||||
props->forceFlush = 0;
|
||||
props->speed = p6.speed;
|
||||
props->port = p6.port;
|
||||
props->maxComms = p6.maxComms;
|
||||
props->maxRecvs = p6.maxRecvs;
|
||||
props->latency = p6.latency;
|
||||
props->netDeviceType = NCCL_NET_DEVICE_HOST;
|
||||
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
|
||||
props->vProps.ndevs = 1;
|
||||
props->vProps.devs[0] = dev;
|
||||
props->maxP2pBytes = MAX_NET_SIZE;
|
||||
props->maxCollBytes = MAX_COLLNET_SIZE;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
|
||||
if (size >= 1UL<<31) return ncclInternalError;
|
||||
return ncclNet_v6->regMr(comm, data, (int) size, type, mhandle);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
|
||||
return ncclNet_v6->connect(dev, handle, sendComm);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_accept(void* listenComm, void** recvComm, ncclNetDeviceHandle_t** /*recvDevComm*/) {
|
||||
return ncclNet_v6->accept(listenComm, recvComm);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
|
||||
int sizeInt;
|
||||
if (size > MAX_NET_SIZE) return ncclInternalError;
|
||||
sizeInt = (int)size;
|
||||
ncclResult_t ans = ncclNet_v6->isend(sendComm, data, sizeInt, tag, mhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
|
||||
int sizesInt[NCCL_PROXY_MAX_SUBS];
|
||||
//reset to nullptr if optional receive completion is set
|
||||
if (*request == (void *)NCCL_NET_OPTIONAL_RECV_COMPLETION) *request = nullptr;
|
||||
for (int i=0; i<n; i++) {
|
||||
if (sizes[i] > MAX_NET_SIZE) return ncclInternalError;
|
||||
sizesInt[i] = (int) sizes[i];
|
||||
}
|
||||
ncclResult_t ans = ncclNet_v6->irecv(recvComm, n, data, sizesInt, tags, mhandles, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
ncclNetProperties_v6_t p6;
|
||||
ncclResult_t ans = ncclCollNet_v6->getProperties(dev, &p6);
|
||||
if (ans != ncclSuccess) return ans;
|
||||
props->name = p6.name;
|
||||
props->pciPath = p6.pciPath;
|
||||
props->guid = p6.guid;
|
||||
props->ptrSupport = p6.ptrSupport;
|
||||
props->regIsGlobal = 0;
|
||||
props->forceFlush = 0;
|
||||
props->speed = p6.speed;
|
||||
props->port = p6.port;
|
||||
props->maxComms = p6.maxComms;
|
||||
props->maxRecvs = p6.maxRecvs;
|
||||
props->latency = p6.latency;
|
||||
props->netDeviceType = NCCL_NET_DEVICE_HOST;
|
||||
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
|
||||
props->vProps.ndevs = 1;
|
||||
props->vProps.devs[0] = dev;
|
||||
props->maxP2pBytes = MAX_NET_SIZE;
|
||||
props->maxCollBytes = MAX_COLLNET_SIZE;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
|
||||
if (size >= 1UL<<31) return ncclInternalError;
|
||||
return ncclCollNet_v6->regMr(comm, data, (int) size, type, mhandle);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_iallreduce(void* collComm, void* sendData, void* recvData, size_t count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request) {
|
||||
int countInt;
|
||||
if (count > MAX_NET_SIZE) return ncclInternalError;
|
||||
countInt = (int)count;
|
||||
ncclResult_t ans = ncclCollNet_v6->iallreduce(collComm, sendData, recvData, countInt, dataType, redOp,
|
||||
sendMhandle, recvMhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
|
||||
NCCLCHECK(ncclNet_v6->init(logfn));
|
||||
ncclNet.devices = ncclNet_v6->devices;
|
||||
ncclNet.getProperties = ncclNet_getProperties;
|
||||
ncclNet.listen = ncclNet_v6->listen;
|
||||
ncclNet.connect = ncclNet_connect;
|
||||
ncclNet.accept = ncclNet_accept;
|
||||
ncclNet.regMr = ncclNet_regMr;
|
||||
ncclNet.regMrDmaBuf = ncclNet_v6->regMrDmaBuf;
|
||||
ncclNet.deregMr = ncclNet_v6->deregMr;
|
||||
ncclNet.isend = ncclNet_isend;
|
||||
ncclNet.irecv = ncclNet_irecv;
|
||||
ncclNet.iflush = ncclNet_v6->iflush;
|
||||
ncclNet.test = ncclNet_v6->test;
|
||||
ncclNet.closeSend = ncclNet_v6->closeSend;
|
||||
ncclNet.closeRecv = ncclNet_v6->closeRecv;
|
||||
ncclNet.closeListen = ncclNet_v6->closeListen;
|
||||
ncclNet.getDeviceMr = NULL;
|
||||
ncclNet.irecvConsumed = NULL;
|
||||
ncclNet.makeVDevice = NULL;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclNet_t* getNcclNet_v6(void* lib) {
|
||||
ncclNet_v6 = (ncclNet_v6_t*)dlsym(lib, "ncclNetPlugin_v6");
|
||||
if (ncclNet_v6) {
|
||||
ncclNet.name = ncclNet_v6->name;
|
||||
ncclNet.init = ncclNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v6)", ncclNet_v6->name);
|
||||
return &ncclNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v6 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
|
||||
NCCLCHECK(ncclCollNet_v6->init(logfn));
|
||||
ncclCollNet.devices = ncclCollNet_v6->devices;
|
||||
ncclCollNet.getProperties = ncclCollNet_getProperties;
|
||||
ncclCollNet.listen = ncclCollNet_v6->listen;
|
||||
ncclCollNet.connect = ncclCollNet_v6->connect;
|
||||
ncclCollNet.reduceSupport = ncclCollNet_v6->reduceSupport;
|
||||
ncclCollNet.regMr = ncclCollNet_regMr;
|
||||
ncclCollNet.regMrDmaBuf = ncclCollNet_v6->regMrDmaBuf;
|
||||
ncclCollNet.deregMr = ncclCollNet_v6->deregMr;
|
||||
ncclCollNet.iallreduce = ncclCollNet_iallreduce;
|
||||
ncclCollNet.iallgather = nullptr;
|
||||
ncclCollNet.ireducescatter = nullptr;
|
||||
ncclCollNet.iflush = ncclCollNet_v6->iflush;
|
||||
ncclCollNet.test = ncclCollNet_v6->test;
|
||||
ncclCollNet.closeColl = ncclCollNet_v6->closeColl;
|
||||
ncclCollNet.closeListen = ncclCollNet_v6->closeListen;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclCollNet_t* getNcclCollNet_v6(void* lib) {
|
||||
ncclCollNet_v6 = (ncclCollNet_v6_t*)dlsym(lib, "ncclCollNetPlugin_v6");
|
||||
if (ncclCollNet_v6) {
|
||||
ncclCollNet.name = ncclCollNet_v6->name;
|
||||
ncclCollNet.init = ncclCollNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v6)", ncclCollNet_v6->name);
|
||||
return &ncclCollNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "nccl_net.h"
|
||||
#include "net_device.h"
|
||||
#include "proxy.h"
|
||||
#include "checks.h"
|
||||
|
||||
static ncclNet_t ncclNet;
|
||||
static ncclCollNet_t ncclCollNet;
|
||||
static ncclNet_v7_t* ncclNet_v7;
|
||||
static ncclCollNet_v7_t* ncclCollNet_v7;
|
||||
|
||||
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
ncclNetProperties_v7_t p7;
|
||||
ncclResult_t ans = ncclNet_v7->getProperties(dev, &p7);
|
||||
if (ans != ncclSuccess) return ans;
|
||||
props->name = p7.name;
|
||||
props->pciPath = p7.pciPath;
|
||||
props->guid = p7.guid;
|
||||
props->ptrSupport = p7.ptrSupport;
|
||||
props->regIsGlobal = 0;
|
||||
props->forceFlush = 0;
|
||||
props->speed = p7.speed;
|
||||
props->port = p7.port;
|
||||
props->maxComms = p7.maxComms;
|
||||
props->maxRecvs = p7.maxRecvs;
|
||||
props->latency = p7.latency;
|
||||
props->netDeviceType = p7.netDeviceType;
|
||||
props->netDeviceVersion = p7.netDeviceVersion;
|
||||
props->vProps.ndevs = 1;
|
||||
props->vProps.devs[0] = dev;
|
||||
props->maxP2pBytes = MAX_NET_SIZE;
|
||||
props->maxCollBytes = MAX_COLLNET_SIZE;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) {
|
||||
return ncclNet_v7->connect(dev, handle, sendComm, sendDevComm);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
|
||||
if (size >= 1UL<<31) return ncclInternalError;
|
||||
return ncclNet_v7->regMr(comm, data, (int) size, type, mhandle);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
|
||||
int sizeInt;
|
||||
if (size > MAX_NET_SIZE) return ncclInternalError;
|
||||
sizeInt = (int)size;
|
||||
ncclResult_t ans = ncclNet_v7->isend(sendComm, data, sizeInt, tag, mhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
|
||||
int sizesInt[NCCL_PROXY_MAX_SUBS];
|
||||
//reset to nullptr if optional receive completion is set
|
||||
if (*request == (void *)NCCL_NET_OPTIONAL_RECV_COMPLETION) *request = nullptr;
|
||||
for (int i=0; i<n; i++) {
|
||||
if (sizes[i] > MAX_NET_SIZE) return ncclInternalError;
|
||||
sizesInt[i] = (int) sizes[i];
|
||||
}
|
||||
ncclResult_t ans = ncclNet_v7->irecv(recvComm, n, data, sizesInt, tags, mhandles, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
ncclNetProperties_v7_t p7;
|
||||
ncclResult_t ans = ncclCollNet_v7->getProperties(dev, &p7);
|
||||
if (ans != ncclSuccess) return ans;
|
||||
props->name = p7.name;
|
||||
props->pciPath = p7.pciPath;
|
||||
props->guid = p7.guid;
|
||||
props->ptrSupport = p7.ptrSupport;
|
||||
props->regIsGlobal = 0;
|
||||
props->forceFlush = 0;
|
||||
props->speed = p7.speed;
|
||||
props->port = p7.port;
|
||||
props->maxComms = p7.maxComms;
|
||||
props->maxRecvs = p7.maxRecvs;
|
||||
props->latency = p7.latency;
|
||||
props->netDeviceType = NCCL_NET_DEVICE_HOST;
|
||||
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
|
||||
props->vProps.ndevs = 1;
|
||||
props->vProps.devs[0] = dev;
|
||||
props->maxP2pBytes = MAX_NET_SIZE;
|
||||
props->maxCollBytes = MAX_COLLNET_SIZE;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
|
||||
if (size >= 1UL<<31) return ncclInternalError;
|
||||
return ncclCollNet_v7->regMr(comm, data, (int) size, type, mhandle);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_iallreduce(void* collComm, void* sendData, void* recvData, size_t count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request) {
|
||||
int countInt;
|
||||
if (count > MAX_NET_SIZE) return ncclInternalError;
|
||||
countInt = (int)count;
|
||||
ncclResult_t ans = ncclCollNet_v7->iallreduce(collComm, sendData, recvData, countInt, dataType, redOp,
|
||||
sendMhandle, recvMhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
|
||||
NCCLCHECK(ncclNet_v7->init(logfn));
|
||||
ncclNet.devices = ncclNet_v7->devices;
|
||||
ncclNet.getProperties = ncclNet_getProperties; // ncclNet_v5->getProperties;
|
||||
ncclNet.listen = ncclNet_v7->listen;
|
||||
ncclNet.connect = ncclNet_connect;
|
||||
ncclNet.accept = ncclNet_v7->accept;
|
||||
ncclNet.regMr = ncclNet_regMr;
|
||||
ncclNet.regMrDmaBuf = ncclNet_v7->regMrDmaBuf;
|
||||
ncclNet.deregMr = ncclNet_v7->deregMr;
|
||||
ncclNet.isend = ncclNet_isend;
|
||||
ncclNet.irecv = ncclNet_irecv;
|
||||
ncclNet.iflush = ncclNet_v7->iflush;
|
||||
ncclNet.test = ncclNet_v7->test;
|
||||
ncclNet.closeSend = ncclNet_v7->closeSend;
|
||||
ncclNet.closeRecv = ncclNet_v7->closeRecv;
|
||||
ncclNet.closeListen = ncclNet_v7->closeListen;
|
||||
ncclNet.getDeviceMr = ncclNet_v7->getDeviceMr;
|
||||
ncclNet.irecvConsumed = ncclNet_v7->irecvConsumed;
|
||||
ncclNet.makeVDevice = NULL;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclNet_t* getNcclNet_v7(void* lib) {
|
||||
ncclNet_v7 = (ncclNet_v7_t*)dlsym(lib, "ncclNetPlugin_v7");
|
||||
if (ncclNet_v7) {
|
||||
ncclNet.name = ncclNet_v7->name;
|
||||
ncclNet.init = ncclNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v7)", ncclNet_v7->name);
|
||||
return &ncclNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v7 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
|
||||
NCCLCHECK(ncclCollNet_v7->init(logfn));
|
||||
ncclCollNet.devices = ncclCollNet_v7->devices;
|
||||
ncclCollNet.getProperties = ncclCollNet_getProperties;
|
||||
ncclCollNet.listen = ncclCollNet_v7->listen;
|
||||
ncclCollNet.connect = ncclCollNet_v7->connect;
|
||||
ncclCollNet.reduceSupport = ncclCollNet_v7->reduceSupport;
|
||||
ncclCollNet.regMr = ncclCollNet_regMr;
|
||||
ncclCollNet.regMrDmaBuf = ncclCollNet_v7->regMrDmaBuf;
|
||||
ncclCollNet.deregMr = ncclCollNet_v7->deregMr;
|
||||
ncclCollNet.iallreduce = ncclCollNet_iallreduce;
|
||||
ncclCollNet.iallgather = nullptr;
|
||||
ncclCollNet.ireducescatter = nullptr;
|
||||
ncclCollNet.iflush = ncclCollNet_v7->iflush;
|
||||
ncclCollNet.test = ncclCollNet_v7->test;
|
||||
ncclCollNet.closeColl = ncclCollNet_v7->closeColl;
|
||||
ncclCollNet.closeListen = ncclCollNet_v7->closeListen;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclCollNet_t* getNcclCollNet_v7(void* lib) {
|
||||
ncclCollNet_v7 = (ncclCollNet_v7_t*)dlsym(lib, "ncclCollNetPlugin_v7");
|
||||
if (ncclCollNet_v7) {
|
||||
ncclCollNet.name = ncclCollNet_v7->name;
|
||||
ncclCollNet.init = ncclCollNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v7)", ncclCollNet_v7->name);
|
||||
return &ncclCollNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v7 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "nccl_net.h"
|
||||
#include "net_device.h"
|
||||
#include "proxy.h"
|
||||
#include "checks.h"
|
||||
|
||||
static ncclNet_t ncclNet;
|
||||
static ncclCollNet_t ncclCollNet;
|
||||
static ncclNet_v8_t* ncclNet_v8;
|
||||
static ncclCollNet_v8_t* ncclCollNet_v8;
|
||||
|
||||
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
ncclNetProperties_v8_t p8;
|
||||
ncclResult_t ans = ncclNet_v8->getProperties(dev, &p8);
|
||||
if (ans != ncclSuccess) return ans;
|
||||
props->name = p8.name;
|
||||
props->pciPath = p8.pciPath;
|
||||
props->guid = p8.guid;
|
||||
props->ptrSupport = p8.ptrSupport;
|
||||
props->regIsGlobal = p8.regIsGlobal;
|
||||
props->forceFlush = 0;
|
||||
props->speed = p8.speed;
|
||||
props->port = p8.port;
|
||||
props->maxComms = p8.maxComms;
|
||||
props->maxRecvs = p8.maxRecvs;
|
||||
props->latency = p8.latency;
|
||||
props->netDeviceType = p8.netDeviceType;
|
||||
props->netDeviceVersion = p8.netDeviceVersion;
|
||||
props->vProps.ndevs = 1;
|
||||
props->vProps.devs[0] = dev;
|
||||
props->maxP2pBytes = MAX_NET_SIZE;
|
||||
props->maxCollBytes = MAX_COLLNET_SIZE;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) {
|
||||
return ncclNet_v8->connect(dev, handle, sendComm, sendDevComm);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
|
||||
int sizeInt;
|
||||
if (size > MAX_NET_SIZE) return ncclInternalError;
|
||||
sizeInt = (int)size;
|
||||
ncclResult_t ans = ncclNet_v8->isend(sendComm, data, sizeInt, tag, mhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
|
||||
int sizesInt[NCCL_PROXY_MAX_SUBS];
|
||||
//reset to nullptr if optional receive completion is set
|
||||
if (*request == (void *)NCCL_NET_OPTIONAL_RECV_COMPLETION) *request = nullptr;
|
||||
for (int i=0; i<n; i++) {
|
||||
if (sizes[i] > MAX_NET_SIZE) return ncclInternalError;
|
||||
sizesInt[i] = (int) sizes[i];
|
||||
}
|
||||
ncclResult_t ans = ncclNet_v8->irecv(recvComm, n, data, sizesInt, tags, mhandles, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
ncclNetProperties_v8_t p8;
|
||||
ncclResult_t ans = ncclCollNet_v8->getProperties(dev, &p8);
|
||||
if (ans != ncclSuccess) return ans;
|
||||
props->name = p8.name;
|
||||
props->pciPath = p8.pciPath;
|
||||
props->guid = p8.guid;
|
||||
props->ptrSupport = p8.ptrSupport;
|
||||
props->regIsGlobal = p8.regIsGlobal;
|
||||
props->forceFlush = 0;
|
||||
props->speed = p8.speed;
|
||||
props->port = p8.port;
|
||||
props->maxComms = p8.maxComms;
|
||||
props->maxRecvs = p8.maxRecvs;
|
||||
props->latency = p8.latency;
|
||||
props->netDeviceType = NCCL_NET_DEVICE_HOST;
|
||||
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
|
||||
props->vProps.ndevs = 1;
|
||||
props->vProps.devs[0] = dev;
|
||||
props->maxP2pBytes = MAX_NET_SIZE;
|
||||
props->maxCollBytes = MAX_COLLNET_SIZE;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_iallreduce(void* collComm, void* sendData, void* recvData, size_t count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request) {
|
||||
int countInt;
|
||||
if (count > MAX_NET_SIZE) return ncclInternalError;
|
||||
countInt = (int)count;
|
||||
ncclResult_t ans = ncclCollNet_v8->iallreduce(collComm, sendData, recvData, countInt, dataType, redOp,
|
||||
sendMhandle, recvMhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_iallgather (void* collComm, void* sendData, int nRecvParts, ncclNetSGE_t* recvParts,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
void* sendMhandle, void** request) {
|
||||
ncclNetSGE_v8_t recvPartsInt;
|
||||
if (nRecvParts > 1) return ncclInternalError;
|
||||
if (recvParts->size > MAX_COLLNET_SIZE) return ncclInternalError;
|
||||
recvPartsInt.mhandle = recvParts->mhandle;
|
||||
recvPartsInt.address = recvParts->address;
|
||||
recvPartsInt.size = (int)recvParts->size;
|
||||
ncclResult_t ans = ncclCollNet_v8->iallgather(collComm, sendData, nRecvParts, &recvPartsInt,
|
||||
bytesPerRank, windowOffset, windowBytes,
|
||||
sendMhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_ireducescatter(void* collComm, int nSendParts, ncclNetSGE_t* sendParts, void* recvData,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp,
|
||||
void* recvMhandle, void** request) {
|
||||
ncclNetSGE_v8_t sendPartsInt;
|
||||
if (nSendParts > 1) return ncclInternalError;
|
||||
if (sendParts->size > MAX_COLLNET_SIZE) return ncclInternalError;
|
||||
sendPartsInt.mhandle = sendParts->mhandle;
|
||||
sendPartsInt.address = sendParts->address;
|
||||
sendPartsInt.size = (int)sendParts->size;
|
||||
ncclResult_t ans = ncclCollNet_v8->ireducescatter(collComm, nSendParts, &sendPartsInt,
|
||||
recvData, bytesPerRank, windowOffset, windowBytes,
|
||||
dataType, redOp,
|
||||
recvMhandle, request);
|
||||
return ans;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
|
||||
NCCLCHECK(ncclNet_v8->init(logfn));
|
||||
ncclNet.devices = ncclNet_v8->devices;
|
||||
ncclNet.getProperties = ncclNet_getProperties;
|
||||
ncclNet.listen = ncclNet_v8->listen;
|
||||
ncclNet.connect = ncclNet_connect;
|
||||
ncclNet.accept = ncclNet_v8->accept;
|
||||
ncclNet.regMr = ncclNet_v8->regMr;
|
||||
ncclNet.regMrDmaBuf = ncclNet_v8->regMrDmaBuf;
|
||||
ncclNet.deregMr = ncclNet_v8->deregMr;
|
||||
ncclNet.isend = ncclNet_isend;
|
||||
ncclNet.irecv = ncclNet_irecv;
|
||||
ncclNet.iflush = ncclNet_v8->iflush;
|
||||
ncclNet.test = ncclNet_v8->test;
|
||||
ncclNet.closeSend = ncclNet_v8->closeSend;
|
||||
ncclNet.closeRecv = ncclNet_v8->closeRecv;
|
||||
ncclNet.closeListen = ncclNet_v8->closeListen;
|
||||
ncclNet.getDeviceMr = ncclNet_v8->getDeviceMr;
|
||||
ncclNet.irecvConsumed = ncclNet_v8->irecvConsumed;
|
||||
ncclNet.makeVDevice = NULL;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclNet_t* getNcclNet_v8(void* lib) {
|
||||
ncclNet_v8 = (ncclNet_v8_t*)dlsym(lib, "ncclNetPlugin_v8");
|
||||
if (ncclNet_v8) {
|
||||
ncclNet.name = ncclNet_v8->name;
|
||||
ncclNet.init = ncclNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v8)", ncclNet_v8->name);
|
||||
return &ncclNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v8 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
|
||||
NCCLCHECK(ncclCollNet_v8->init(logfn));
|
||||
ncclCollNet.devices = ncclCollNet_v8->devices;
|
||||
ncclCollNet.getProperties = ncclCollNet_getProperties;
|
||||
ncclCollNet.listen = ncclCollNet_v8->listen;
|
||||
ncclCollNet.connect = ncclCollNet_v8->connect;
|
||||
ncclCollNet.reduceSupport = ncclCollNet_v8->reduceSupport;
|
||||
ncclCollNet.regMr = ncclCollNet_v8->regMr;
|
||||
ncclCollNet.regMrDmaBuf = ncclCollNet_v8->regMrDmaBuf;
|
||||
ncclCollNet.deregMr = ncclCollNet_v8->deregMr;
|
||||
ncclCollNet.iallreduce = ncclCollNet_iallreduce;
|
||||
ncclCollNet.iallgather = ncclCollNet_iallgather;
|
||||
ncclCollNet.ireducescatter = ncclCollNet_ireducescatter;
|
||||
ncclCollNet.iflush = ncclCollNet_v8->iflush;
|
||||
ncclCollNet.test = ncclCollNet_v8->test;
|
||||
ncclCollNet.closeColl = ncclCollNet_v8->closeColl;
|
||||
ncclCollNet.closeListen = ncclCollNet_v8->closeListen;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclCollNet_t* getNcclCollNet_v8(void* lib) {
|
||||
ncclCollNet_v8 = (ncclCollNet_v8_t*)dlsym(lib, "ncclCollNetPlugin_v8");
|
||||
if (ncclCollNet_v8) {
|
||||
ncclCollNet.name = ncclCollNet_v8->name;
|
||||
ncclCollNet.init = ncclCollNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v8)", ncclCollNet_v8->name);
|
||||
return &ncclCollNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v8 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "nccl_net.h"
|
||||
#include "net_device.h"
|
||||
#include "proxy.h"
|
||||
#include "checks.h"
|
||||
|
||||
static ncclNet_t ncclNet;
|
||||
static ncclCollNet_t ncclCollNet;
|
||||
static ncclNet_v9_t* ncclNet_v9;
|
||||
static ncclCollNet_v9_t* ncclCollNet_v9;
|
||||
|
||||
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
return ncclNet_v9->getProperties(dev, (ncclNetProperties_v9_t *)props);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
|
||||
return ncclNet_v9->isend(sendComm, data, size, tag, mhandle, request);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
|
||||
return ncclNet_v9->irecv(recvComm, n, data, sizes, tags, mhandles, request);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) {
|
||||
return ncclNet_v9->connect(dev, handle, sendComm, sendDevComm);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_makeVDevice(int* d, ncclNetVDeviceProps_t* props) {
|
||||
return ncclNet_v9->makeVDevice(d, (ncclNetVDeviceProps_v9_t*)props);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
|
||||
return ncclCollNet_v9->getProperties(dev, (ncclNetProperties_v9_t *)props);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_iallgather(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_t* recvParts,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
void* sendMhandle, void** request) {
|
||||
return ncclCollNet_v9->iallgather(collComm, sendData, nRecvParts, (ncclNetSGE_v9_t*)recvParts, bytesPerRank,
|
||||
windowOffset, windowBytes, sendMhandle, request);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_ireducescatter(void* collComm, int nSendParts, ncclNetSGE_t* sendParts, void* recvData,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp,
|
||||
void* recvMhandle, void** request) {
|
||||
return ncclCollNet_v9->ireducescatter(collComm, nSendParts, (ncclNetSGE_v9_t*)sendParts, recvData, bytesPerRank,
|
||||
windowOffset, windowBytes, dataType, redOp, recvMhandle, request);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
|
||||
NCCLCHECK(ncclNet_v9->init(logfn));
|
||||
ncclNet.devices = ncclNet_v9->devices;
|
||||
ncclNet.getProperties = ncclNet_getProperties;
|
||||
ncclNet.listen = ncclNet_v9->listen;
|
||||
ncclNet.connect = ncclNet_connect;
|
||||
ncclNet.accept = ncclNet_v9->accept;
|
||||
ncclNet.regMr = ncclNet_v9->regMr;
|
||||
ncclNet.regMrDmaBuf = ncclNet_v9->regMrDmaBuf;
|
||||
ncclNet.deregMr = ncclNet_v9->deregMr;
|
||||
ncclNet.isend = ncclNet_isend;
|
||||
ncclNet.irecv = ncclNet_irecv;
|
||||
ncclNet.iflush = ncclNet_v9->iflush;
|
||||
ncclNet.test = ncclNet_v9->test;
|
||||
ncclNet.closeSend = ncclNet_v9->closeSend;
|
||||
ncclNet.closeRecv = ncclNet_v9->closeRecv;
|
||||
ncclNet.closeListen = ncclNet_v9->closeListen;
|
||||
ncclNet.getDeviceMr = ncclNet_v9->getDeviceMr;
|
||||
ncclNet.irecvConsumed = ncclNet_v9->irecvConsumed;
|
||||
ncclNet.makeVDevice = (ncclNet_v9->makeVDevice) ? ncclNet_makeVDevice : nullptr;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclNet_t* getNcclNet_v9(void* lib) {
|
||||
ncclNet_v9 = (ncclNet_v9_t*)dlsym(lib, "ncclNetPlugin_v9");
|
||||
if (ncclNet_v9) {
|
||||
ncclNet.name = ncclNet_v9->name;
|
||||
ncclNet.init = ncclNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v9)", ncclNet_v9->name);
|
||||
return &ncclNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v9 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
|
||||
NCCLCHECK(ncclCollNet_v9->init(logfn));
|
||||
ncclCollNet.devices = ncclCollNet_v9->devices;
|
||||
ncclCollNet.getProperties = ncclCollNet_getProperties;
|
||||
ncclCollNet.listen = ncclCollNet_v9->listen;
|
||||
ncclCollNet.connect = ncclCollNet_v9->connect;
|
||||
ncclCollNet.reduceSupport = ncclCollNet_v9->reduceSupport;
|
||||
ncclCollNet.regMr = ncclCollNet_v9->regMr;
|
||||
ncclCollNet.regMrDmaBuf = ncclCollNet_v9->regMrDmaBuf;
|
||||
ncclCollNet.deregMr = ncclCollNet_v9->deregMr;
|
||||
ncclCollNet.iallreduce = ncclCollNet_v9->iallreduce;
|
||||
ncclCollNet.iallgather = ncclCollNet_iallgather;
|
||||
ncclCollNet.ireducescatter = ncclCollNet_ireducescatter;
|
||||
ncclCollNet.iflush = ncclCollNet_v9->iflush;
|
||||
ncclCollNet.test = ncclCollNet_v9->test;
|
||||
ncclCollNet.closeColl = ncclCollNet_v9->closeColl;
|
||||
ncclCollNet.closeListen = ncclCollNet_v9->closeListen;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclCollNet_t* getNcclCollNet_v9(void* lib) {
|
||||
ncclCollNet_v9 = (ncclCollNet_v9_t*)dlsym(lib, "ncclCollNetPlugin_v9");
|
||||
if (ncclCollNet_v9) {
|
||||
ncclCollNet.name = ncclCollNet_v9->name;
|
||||
ncclCollNet.init = ncclCollNet_init;
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v9)", ncclCollNet_v9->name);
|
||||
return &ncclCollNet;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v9 symbol.");
|
||||
return nullptr;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <dlfcn.h>
|
||||
|
||||
#include "debug.h"
|
||||
|
||||
#define MAX_STR_LEN 255
|
||||
|
||||
enum ncclPluginType {
|
||||
ncclPluginTypeNet,
|
||||
ncclPluginTypeTuner,
|
||||
ncclPluginTypeProfiler,
|
||||
};
|
||||
|
||||
#define NUM_LIBS 3
|
||||
static void *libHandles[NUM_LIBS];
|
||||
static const char *pluginNames[NUM_LIBS] = { "NET", "TUNER", "PROFILER" };
|
||||
static const char *pluginPrefix[NUM_LIBS] = { "libnccl-net", "libnccl-tuner", "libnccl-profiler" };
|
||||
static const char *pluginFallback[NUM_LIBS] = { "Using internal net plugin.", "Using internal tuner plugin.", "" };
|
||||
static unsigned long subsys[NUM_LIBS] = { NCCL_INIT|NCCL_NET, NCCL_INIT|NCCL_TUNING, NCCL_INIT };
|
||||
|
||||
static void* tryOpenLib(char* name, int* err, char* errStr) {
|
||||
*err = 0;
|
||||
if (nullptr == name || strlen(name) == 0) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
|
||||
name = nullptr;
|
||||
}
|
||||
|
||||
void *handle = dlopen(name, RTLD_NOW | RTLD_LOCAL);
|
||||
if (nullptr == handle) {
|
||||
strncpy(errStr, dlerror(), MAX_STR_LEN);
|
||||
errStr[MAX_STR_LEN] = '\0';
|
||||
// "handle" and "name" won't be NULL at the same time.
|
||||
// coverity[var_deref_model]
|
||||
if (strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
|
||||
*err = ENOENT;
|
||||
}
|
||||
}
|
||||
return handle;
|
||||
}
|
||||
|
||||
static void appendNameToList(char* nameList, int *nameListLen, char* name) {
|
||||
snprintf(nameList, *nameListLen, " %s", name);
|
||||
nameList += strlen(name) + 1;
|
||||
*nameListLen -= strlen(name) + 1;
|
||||
}
|
||||
|
||||
static void* openPluginLib(enum ncclPluginType type, const char* libName) {
|
||||
int openErr, len = PATH_MAX;
|
||||
char libName_[MAX_STR_LEN] = { 0 };
|
||||
char openErrStr[MAX_STR_LEN + 1] = { 0 };
|
||||
char eNoEntNameList[PATH_MAX] = { 0 };
|
||||
|
||||
if (libName && strlen(libName)) {
|
||||
snprintf(libName_, MAX_STR_LEN, "%s", libName);
|
||||
libHandles[type] = tryOpenLib(libName_, &openErr, openErrStr);
|
||||
if (libHandles[type]) {
|
||||
INFO(subsys[type], "%s/Plugin: Plugin name set by env to %s", pluginNames[type], libName_);
|
||||
return libHandles[type];
|
||||
}
|
||||
if (openErr == ENOENT) {
|
||||
appendNameToList(eNoEntNameList, &len, libName_);
|
||||
} else {
|
||||
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], openErrStr);
|
||||
}
|
||||
|
||||
snprintf(libName_, MAX_STR_LEN, "%s-%s.so", pluginPrefix[type], libName);
|
||||
libHandles[type] = tryOpenLib(libName_, &openErr, openErrStr);
|
||||
if (libHandles[type]) {
|
||||
INFO(subsys[type], "%s/Plugin: Plugin name set by env to %s", pluginNames[type], libName_);
|
||||
return libHandles[type];
|
||||
}
|
||||
if (openErr == ENOENT) {
|
||||
appendNameToList(eNoEntNameList, &len, libName_);
|
||||
} else {
|
||||
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], openErrStr);
|
||||
}
|
||||
} else {
|
||||
snprintf(libName_, MAX_STR_LEN, "%s.so", pluginPrefix[type]);
|
||||
libHandles[type] = tryOpenLib(libName_, &openErr, openErrStr);
|
||||
if (libHandles[type]) {
|
||||
return libHandles[type];
|
||||
}
|
||||
if (openErr == ENOENT) {
|
||||
appendNameToList(eNoEntNameList, &len, libName_);
|
||||
} else {
|
||||
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], openErrStr);
|
||||
}
|
||||
}
|
||||
|
||||
if (strlen(eNoEntNameList)) {
|
||||
INFO(subsys[type], "%s/Plugin: Could not find:%s. %s", pluginNames[type], eNoEntNameList, pluginFallback[type]);
|
||||
} else if (strlen(pluginFallback[type])) {
|
||||
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], pluginFallback[type]);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void* ncclOpenNetPluginLib(const char* name) {
|
||||
return openPluginLib(ncclPluginTypeNet, name);
|
||||
}
|
||||
|
||||
void* ncclOpenTunerPluginLib(const char* name) {
|
||||
return openPluginLib(ncclPluginTypeTuner, name);
|
||||
}
|
||||
|
||||
void* ncclOpenProfilerPluginLib(const char* name) {
|
||||
return openPluginLib(ncclPluginTypeProfiler, name);
|
||||
}
|
||||
|
||||
void* ncclGetNetPluginLib(void) {
|
||||
return libHandles[ncclPluginTypeNet];
|
||||
}
|
||||
|
||||
ncclResult_t ncclClosePluginLib(void* handle) {
|
||||
for (int l=0; l<NUM_LIBS; l++) {
|
||||
if (libHandles[l] == handle) {
|
||||
libHandles[l] = nullptr;
|
||||
dlclose(handle);
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
return ncclInternalError;
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "param.h"
|
||||
#include "checks.h"
|
||||
#include "comm.h"
|
||||
#include "enqueue.h"
|
||||
#include "utils.h"
|
||||
#include "proxy.h"
|
||||
#include "profiler.h"
|
||||
#include "transport.h"
|
||||
#include "plugin.h"
|
||||
|
||||
#include <dlfcn.h>
|
||||
|
||||
extern ncclProfiler_t* getNcclProfiler_v1(void* lib);
|
||||
extern ncclProfiler_t* getNcclProfiler_v2(void* lib);
|
||||
extern ncclProfiler_t* getNcclProfiler_v3(void* lib);
|
||||
|
||||
static pthread_mutex_t profilerLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
static int profilerPluginRefCount;
|
||||
static void* profilerPluginLib;
|
||||
static ncclProfiler_t* ncclProfiler;
|
||||
|
||||
#define MAX_STR_LEN 256
|
||||
|
||||
enum {
|
||||
profilerPluginLoadFailed = -1,
|
||||
profilerPluginLoadReady = 0,
|
||||
profilerPluginLoadSuccess = 1,
|
||||
};
|
||||
static int profilerPluginStatus = profilerPluginLoadReady;
|
||||
static pid_t pid;
|
||||
|
||||
static ncclResult_t ncclProfilerPluginLoad(void) {
|
||||
if (profilerPluginLoadFailed == profilerPluginStatus) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&profilerLock);
|
||||
if (profilerPluginLoadSuccess == profilerPluginStatus) {
|
||||
++profilerPluginRefCount;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
profilerPluginLib = ncclOpenProfilerPluginLib(ncclGetEnv("NCCL_PROFILER_PLUGIN"));
|
||||
if (profilerPluginLib == nullptr) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
ncclProfiler = getNcclProfiler_v3(profilerPluginLib);
|
||||
if (ncclProfiler == nullptr) {
|
||||
ncclProfiler = getNcclProfiler_v2(profilerPluginLib);
|
||||
}
|
||||
if (ncclProfiler == NULL) {
|
||||
ncclProfiler = getNcclProfiler_v1(profilerPluginLib);
|
||||
}
|
||||
if (ncclProfiler == NULL) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
++profilerPluginRefCount;
|
||||
profilerPluginStatus = profilerPluginLoadSuccess;
|
||||
|
||||
// Store the pid of the process loading the profiler.
|
||||
// This is attached to the proxyOp event descriptor
|
||||
// so the plugin can figure out if the parent event
|
||||
// is in the same address space or not
|
||||
pid = getpid();
|
||||
|
||||
exit:
|
||||
pthread_mutex_unlock(&profilerLock);
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
if (profilerPluginLib) NCCLCHECK(ncclClosePluginLib(profilerPluginLib));
|
||||
profilerPluginStatus = profilerPluginLoadFailed;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclProfilerPluginUnload(void) {
|
||||
pthread_mutex_lock(&profilerLock);
|
||||
if (0 == (--profilerPluginRefCount)) {
|
||||
INFO(NCCL_ENV, "PROFILER/Plugin: Closing profiler plugin %s", ncclProfiler->name);
|
||||
NCCLCHECK(ncclClosePluginLib(profilerPluginLib));
|
||||
profilerPluginLib = nullptr;
|
||||
ncclProfiler = nullptr;
|
||||
profilerPluginStatus = profilerPluginLoadReady;
|
||||
}
|
||||
pthread_mutex_unlock(&profilerLock);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
#define ENABLE_TIMER 0
|
||||
#include "timer.h"
|
||||
|
||||
#if ENABLE_TIMER
|
||||
// These counters are used to measure profiler overheads for different part of the code
|
||||
// These counters are only useful/meaningful in controlled test environments where there
|
||||
// is only one thread updating each set of counters, i.e., every communicator has its
|
||||
// own proxy thread and the network uses only one thread to make progress (this is true
|
||||
// for net_ib plugin but might not be true for net_socket plugin).
|
||||
static int64_t elapsedCount;
|
||||
static int64_t initCount, finalizeCount;
|
||||
static int64_t groupStartCount, groupStopCount;
|
||||
static int64_t taskStartCount, taskStopCount;
|
||||
static int64_t proxyOpStartCount, proxyOpStopCount;
|
||||
static int64_t proxyStepStartCount, proxyStepStopCount;
|
||||
static int64_t proxyCtrlStartCount, proxyCtrlStopCount;
|
||||
static int64_t proxyOpRecordCount, proxyStepRecordCount, proxyCtrlRecordCount;
|
||||
|
||||
static double elapsedTs[2];
|
||||
static double initTs[2], finalizeTs[2];
|
||||
static double groupStartTs[2], groupStopTs[2];
|
||||
static double taskStartTs[2], taskStopTs[2];
|
||||
static double proxyOpStartTs[2], proxyOpStopTs[2];
|
||||
static double proxyStepStartTs[2], proxyStepStopTs[2];
|
||||
static double proxyCtrlStartTs[2], proxyCtrlStopTs[2];
|
||||
static double proxyOpRecordTs[2], proxyStepRecordTs[2], proxyCtrlRecordTs[2];
|
||||
|
||||
#define TIME_START_EVENT(event) do { \
|
||||
(event ## Count)++; \
|
||||
(event ## Ts)[0] = gettime(); \
|
||||
} while(0)
|
||||
|
||||
#define TIME_STOP_EVENT(event) do { \
|
||||
double val = gettime() - (event ## Ts)[0]; \
|
||||
(event ## Ts)[1] += val; \
|
||||
} while(0)
|
||||
|
||||
#define TIME_PRINT_EVENTS(name) do { \
|
||||
printf("%s ", name); \
|
||||
if (elapsedCount) printf("[elapsed] %g/%ld = %g ", elapsedTs[1], elapsedCount, elapsedTs[1]/elapsedCount); \
|
||||
if (initCount) printf("[init] %g/%ld = %g ", initTs[1], initCount, initTs[1]/initCount); \
|
||||
if (finalizeCount) printf("[finalize] %g/%ld = %g ", finalizeTs[1], finalizeCount, finalizeTs[1]/finalizeCount); \
|
||||
if (groupStartCount) printf("[groupStart] %g/%ld = %g ", groupStartTs[1], groupStartCount, groupStartTs[1]/groupStartCount); \
|
||||
if (groupStopCount) printf("[groupStop] %g/%ld = %g ", groupStopTs[1], groupStopCount, groupStopTs[1]/groupStopCount); \
|
||||
if (taskStartCount) printf("[taskStart] %g/%ld = %g ", taskStartTs[1], taskStartCount, taskStartTs[1]/taskStartCount); \
|
||||
if (taskStopCount) printf("[taskStop] %g/%ld = %g ", taskStopTs[1], taskStopCount, taskStopTs[1]/taskStopCount); \
|
||||
if (proxyOpStartCount) printf("[proxyOpStart] %g/%ld = %g ", proxyOpStartTs[1], proxyOpStartCount, proxyOpStartTs[1]/proxyOpStartCount); \
|
||||
if (proxyOpStopCount) printf("[proxyOpStop] %g/%ld = %g ", proxyOpStopTs[1], proxyOpStopCount, proxyOpStopTs[1]/proxyOpStopCount); \
|
||||
if (proxyStepStartCount) printf("[proxyStepStart] %g/%ld = %g ", proxyStepStartTs[1], proxyStepStartCount, proxyStepStartTs[1]/proxyStepStartCount); \
|
||||
if (proxyStepStopCount) printf("[proxyStepStop] %g/%ld = %g ", proxyStepStopTs[1], proxyStepStopCount, proxyStepStopTs[1]/proxyStepStopCount); \
|
||||
if (proxyCtrlStartCount) printf("[proxyCtrlStart] %g/%ld = %g ", proxyCtrlStartTs[1], proxyCtrlStartCount, proxyCtrlStartTs[1]/proxyCtrlStartCount); \
|
||||
if (proxyCtrlStopCount) printf("[proxyCtrlStop] %g/%ld = %g ", proxyCtrlStopTs[1], proxyCtrlStopCount, proxyCtrlStopTs[1]/proxyCtrlStopCount); \
|
||||
if (proxyOpRecordCount) printf("[proxyOpRecord] %g/%ld = %g ", proxyOpRecordTs[1], proxyOpRecordCount, proxyOpRecordTs[1]/proxyOpRecordCount); \
|
||||
if (proxyStepRecordCount) printf("[proxyStepRecord] %g/%ld = %g ", proxyStepRecordTs[1], proxyStepRecordCount, proxyStepRecordTs[1]/proxyStepRecordCount); \
|
||||
if (proxyCtrlRecordCount) printf("[proxyCtrlRecord] %g/%ld = %g", proxyCtrlRecordTs[1], proxyCtrlRecordCount, proxyCtrlRecordTs[1]/proxyCtrlRecordCount); \
|
||||
printf("\n"); \
|
||||
} while(0)
|
||||
#else
|
||||
#define TIME_START_EVENT(event) do {} while(0)
|
||||
#define TIME_STOP_EVENT(event) do {} while(0)
|
||||
#define TIME_PRINT_EVENTS(name) do {} while(0)
|
||||
#endif
|
||||
|
||||
|
||||
int ncclProfilerEventMask; // Set by profiler
|
||||
|
||||
ncclResult_t ncclProfilerPluginInit(struct ncclComm* comm) {
|
||||
TIME_START_EVENT(elapsed);
|
||||
TIME_START_EVENT(init);
|
||||
ncclProfilerPluginLoad();
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
int err = ncclProfiler->init(&comm->profilerContext, &ncclProfilerEventMask);
|
||||
if (err) {
|
||||
WARN("Profiler init failed with error (%d). Continue without profiler.", err);
|
||||
ncclProfiler = NULL;
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(init);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerPluginFinalize(struct ncclComm* comm) {
|
||||
TIME_START_EVENT(finalize);
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
ncclProfiler->finalize(comm->profilerContext);
|
||||
}
|
||||
ncclProfilerPluginUnload();
|
||||
TIME_STOP_EVENT(finalize);
|
||||
TIME_STOP_EVENT(elapsed);
|
||||
TIME_PRINT_EVENTS("Profiler");
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStartGroupEvent(struct ncclKernelPlan* plan) {
|
||||
TIME_START_EVENT(groupStart);
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
// Check if any collective in the plan has a set event activation mask
|
||||
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
|
||||
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
|
||||
int eActivationMask_ = 0;
|
||||
while (ct) {
|
||||
if (ct->eActivationMask) {
|
||||
eActivationMask_ = ct->eActivationMask;
|
||||
goto startGroup;
|
||||
}
|
||||
ct = ct->next;
|
||||
}
|
||||
// Check if any pt2pt in the plan has a set event activation mask
|
||||
while (pt) {
|
||||
if (pt->eActivationMask) {
|
||||
eActivationMask_ = pt->eActivationMask;
|
||||
goto startGroup;
|
||||
}
|
||||
pt = pt->next;
|
||||
}
|
||||
|
||||
startGroup:
|
||||
if (eActivationMask_ & (ncclProfileGroup | ncclProfileColl | ncclProfileP2p | ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileKernelCh | ncclProfileNetPlugin)) {
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileGroup;
|
||||
ncclProfiler->startEvent(plan->comm->profilerContext, &plan->groupEventHandle, &eDescr);
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(groupStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStopGroupEvent(struct ncclKernelPlan* plan) {
|
||||
TIME_START_EVENT(groupStop);
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0) && plan->groupEventHandle) {
|
||||
ncclProfiler->stopEvent(plan->groupEventHandle);
|
||||
}
|
||||
TIME_STOP_EVENT(groupStop);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStartTaskEvents(struct ncclKernelPlan* plan) {
|
||||
TIME_START_EVENT(taskStart);
|
||||
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
|
||||
while (ct) {
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
if (plan->groupEventHandle) {
|
||||
int enable = ct->eActivationMask & (ncclProfileColl | ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileKernelCh | ncclProfileNetPlugin);
|
||||
if (enable) {
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileColl;
|
||||
eDescr.parentObj = plan->groupEventHandle;
|
||||
eDescr.rank = plan->comm->rank;
|
||||
eDescr.coll.name = plan->comm->commName;
|
||||
eDescr.coll.commHash = plan->comm->commHash;
|
||||
eDescr.coll.seqNumber = plan->comm->seqNumber[ct->func];
|
||||
eDescr.coll.func = ncclFuncToString(ct->func);
|
||||
eDescr.coll.sendBuff = ct->sendbuff;
|
||||
eDescr.coll.recvBuff = ct->recvbuff;
|
||||
eDescr.coll.count = ct->count;
|
||||
eDescr.coll.root = ct->root;
|
||||
eDescr.coll.datatype = ncclDatatypeToString(ct->datatype);
|
||||
eDescr.coll.nMaxChannels = ct->nMaxChannels;
|
||||
eDescr.coll.nWarps = ct->nWarps;
|
||||
eDescr.coll.algo = ncclAlgoToString(ct->algorithm);
|
||||
eDescr.coll.proto = ncclProtoToString(ct->protocol);
|
||||
ncclProfiler->startEvent(plan->comm->profilerContext, &ct->eventHandle, &eDescr);
|
||||
}
|
||||
}
|
||||
}
|
||||
// comm->seqNumber values are updated even if the plugin is not active, since they are used by RAS as well.
|
||||
// The test for "persistent" is a workaround for graph-captured collectives. In their case this function may not be
|
||||
// consistently invoked on all the ranks, which would lead to mismatched counter values and thus false-positive
|
||||
// reports from RAS. Instead, we choose not to include graph-captured collectives in our counts. An exception is
|
||||
// made if ncclProfileKernelCh profiler events are active, as they result in proxy events always being added, which
|
||||
// gives the consistency.
|
||||
if (!plan->persistent || (__builtin_expect(ncclProfiler != NULL, 0) && plan->groupEventHandle &&
|
||||
(ct->eActivationMask & ncclProfileKernelCh)))
|
||||
plan->comm->seqNumber[ct->func]++;
|
||||
ct = ct->next;
|
||||
}
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
if (plan->groupEventHandle) {
|
||||
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
|
||||
while (pt) {
|
||||
int enable = pt->eActivationMask & (ncclProfileP2p | ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileKernelCh);
|
||||
if (enable) {
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileP2p;
|
||||
eDescr.parentObj = plan->groupEventHandle;
|
||||
eDescr.rank = plan->comm->rank;
|
||||
eDescr.p2p.name = plan->comm->commName;
|
||||
eDescr.p2p.commHash = plan->comm->commHash;
|
||||
eDescr.p2p.func = ncclFuncToString(pt->func);
|
||||
eDescr.p2p.buff = pt->buff;
|
||||
eDescr.p2p.count = pt->count;
|
||||
eDescr.p2p.datatype = ncclDatatypeToString(pt->datatype);
|
||||
eDescr.p2p.peer = pt->root;
|
||||
ncclProfiler->startEvent(plan->comm->profilerContext, &pt->eventHandle, &eDescr);
|
||||
}
|
||||
pt = pt->next;
|
||||
}
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(taskStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStopTaskEvents(struct ncclKernelPlan* plan) {
|
||||
TIME_START_EVENT(taskStop);
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
if (plan->groupEventHandle) {
|
||||
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
|
||||
while (ct) {
|
||||
if (ct->eventHandle) ncclProfiler->stopEvent(ct->eventHandle);
|
||||
ct = ct->next;
|
||||
}
|
||||
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
|
||||
while (pt) {
|
||||
if (pt->eventHandle) ncclProfiler->stopEvent(pt->eventHandle);
|
||||
pt = pt->next;
|
||||
}
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(taskStop);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Bellow we set the proxy descriptor step number to DIVUP(step, args->sliceSteps).
|
||||
// The reason is that for some ncclOp (e.g. AllReduce) one network transfer is
|
||||
// made of sliceSteps steps rather than one step. In the profiler we are still
|
||||
// interested in whole network transfers though, so we account for this when
|
||||
// computing the actual network step number.
|
||||
ncclResult_t ncclProfilerStartSendProxyOpEvent(int s, struct ncclProxyArgs* args) {
|
||||
TIME_START_EVENT(proxyOpStart);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
if (sub->eActivationMask & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileNetPlugin)) {
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileProxyOp;
|
||||
eDescr.parentObj = sub->taskEventHandle;
|
||||
eDescr.rank = sub->rank;
|
||||
eDescr.proxyOp.pid = sub->pid;
|
||||
eDescr.proxyOp.channelId = sub->channelId;
|
||||
eDescr.proxyOp.peer = sub->peer;
|
||||
eDescr.proxyOp.nSteps = DIVUP(sub->nsteps, args->sliceSteps);
|
||||
eDescr.proxyOp.chunkSize = args->chunkSize * args->sliceSteps;
|
||||
eDescr.proxyOp.isSend = 1;
|
||||
ncclProfiler->startEvent(sub->profilerContext, &sub->opEventHandle, &eDescr);
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(proxyOpStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStartRecvProxyOpEvent(int s, struct ncclProxyArgs* args) {
|
||||
TIME_START_EVENT(proxyOpStart);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
if (sub->eActivationMask & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileNetPlugin)) {
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileProxyOp;
|
||||
eDescr.parentObj = sub->taskEventHandle;
|
||||
eDescr.rank = sub->rank;
|
||||
eDescr.proxyOp.pid = sub->pid;
|
||||
eDescr.proxyOp.channelId = sub->channelId;
|
||||
eDescr.proxyOp.peer = sub->peer;
|
||||
eDescr.proxyOp.nSteps = DIVUP(sub->nsteps, args->sliceSteps);
|
||||
eDescr.proxyOp.chunkSize = args->chunkSize * args->sliceSteps;
|
||||
eDescr.proxyOp.isSend = 0;
|
||||
ncclProfiler->startEvent(sub->profilerContext, &sub->opEventHandle, &eDescr);
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(proxyOpStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStopProxyOpEvent(int s, struct ncclProxyArgs* args) {
|
||||
TIME_START_EVENT(proxyOpStop);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
|
||||
ncclProfiler->stopEvent(sub->opEventHandle);
|
||||
sub->opEventHandle = NULL;
|
||||
}
|
||||
TIME_STOP_EVENT(proxyOpStop);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStartSendProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
|
||||
TIME_START_EVENT(proxyStepStart);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
if (sub->opEventHandle && (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileNetPlugin))) {
|
||||
int step_ = DIVUP(stepId, args->sliceSteps);
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileProxyStep;
|
||||
eDescr.parentObj = sub->opEventHandle;
|
||||
eDescr.rank = sub->rank;
|
||||
eDescr.proxyStep.step = step_;
|
||||
ncclProfiler->startEvent(sub->profilerContext, &sub->stepEventHandles[step_%NCCL_STEPS], &eDescr);
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(proxyStepStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStartRecvProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
|
||||
TIME_START_EVENT(proxyStepStart);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
if (sub->opEventHandle && (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileNetPlugin))) {
|
||||
int step_ = DIVUP(stepId, args->sliceSteps);
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileProxyStep;
|
||||
eDescr.parentObj = sub->opEventHandle;
|
||||
eDescr.rank = sub->rank;
|
||||
eDescr.proxyStep.step = step_;
|
||||
ncclProfiler->startEvent(sub->profilerContext, &sub->stepEventHandles[step_%NCCL_STEPS], &eDescr);
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(proxyStepStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStopProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
|
||||
TIME_START_EVENT(proxyStepStop);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
int step_ = DIVUP(stepId, args->sliceSteps);
|
||||
if (sub->stepEventHandles[step_%NCCL_STEPS]) {
|
||||
ncclProfiler->stopEvent(sub->stepEventHandles[step_%NCCL_STEPS]);
|
||||
sub->stepEventHandles[step_%NCCL_STEPS] = NULL;
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(proxyStepStop);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStartProxyCtrlEvent(void* profilerContext, void** eHandle) {
|
||||
TIME_START_EVENT(proxyCtrlStart);
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
// for proxy control events we allow profiling mode to change on a per event basis
|
||||
int eActivationMaskProxy = __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED);
|
||||
if (eActivationMaskProxy & ncclProfileProxyCtrl) {
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileProxyCtrl;
|
||||
ncclProfiler->startEvent(profilerContext, eHandle, &eDescr);
|
||||
TIME_STOP_EVENT(proxyCtrlStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
*eHandle = NULL;
|
||||
TIME_STOP_EVENT(proxyCtrlStart);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStopProxyCtrlEvent(void* eHandle) {
|
||||
TIME_START_EVENT(proxyCtrlStop);
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0) && eHandle) {
|
||||
ncclProfiler->stopEvent(eHandle);
|
||||
}
|
||||
TIME_STOP_EVENT(proxyCtrlStop);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStartKernelChEvent(struct ncclProxyArgs* args, int s) {
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (sub->eActivationMask & ncclProfileKernelCh) {
|
||||
ncclProfilerEventDescr_t eDescr = { };
|
||||
eDescr.type = ncclProfileKernelCh;
|
||||
eDescr.parentObj = sub->taskEventHandle;
|
||||
eDescr.kernelCh.channelId = sub->channelId;
|
||||
ncclProfiler->startEvent(sub->profilerContext, &sub->kernelEventHandle, &eDescr);
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerStopKernelChEvent(struct ncclProxyArgs* args, int s) {
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (sub->kernelEventHandle) {
|
||||
ncclProfiler->stopEvent(sub->kernelEventHandle);
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerRecordProxyOpEventState(int s, struct ncclProxyArgs* args, int steps, size_t transSize, ncclProfilerEventState_t eState) {
|
||||
TIME_START_EVENT(proxyOpRecord);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
|
||||
ncclProfilerEventStateArgs_t a = { };
|
||||
a.proxyOp.steps = DIVUP(steps, args->sliceSteps);
|
||||
a.proxyOp.transSize = transSize;
|
||||
ncclProfiler->recordEventState(sub->opEventHandle, eState, &a);
|
||||
}
|
||||
TIME_STOP_EVENT(proxyOpRecord);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerRecordProxyStepEventState(int s, struct ncclProxyArgs* args, int stepId, ncclProfilerEventState_t eState) {
|
||||
TIME_START_EVENT(proxyStepRecord);
|
||||
struct ncclProxySubArgs* sub = &args->subs[s];
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
|
||||
int step_ = DIVUP(stepId, args->sliceSteps);
|
||||
if (sub->stepEventHandles[step_%NCCL_STEPS]) {
|
||||
ncclProfiler->recordEventState(sub->stepEventHandles[step_%NCCL_STEPS], eState, 0);
|
||||
}
|
||||
}
|
||||
TIME_STOP_EVENT(proxyStepRecord);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerRecordProxyCtrlEventState(void* eHandle, int appended, ncclProfilerEventState_t eState) {
|
||||
TIME_START_EVENT(proxyCtrlRecord);
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0) && eHandle && __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED) & ncclProfileProxyCtrl) {
|
||||
ncclProfilerEventStateArgs_t args = { };
|
||||
args.proxyCtrl.appendedProxyOps = appended;
|
||||
ncclProfiler->recordEventState(eHandle, eState, &args);
|
||||
}
|
||||
TIME_STOP_EVENT(proxyCtrlRecord);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerAddPidToProxyOp(struct ncclProxyOp* op) {
|
||||
op->pid = pid;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static pthread_mutex_t proxyProfilerConnectLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
static ncclResult_t proxyProfilerConnect(struct ncclComm* comm, struct ncclProxyOp* op) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
pthread_mutex_lock(&proxyProfilerConnectLock);
|
||||
if (comm->profiler.initialized) goto exit;
|
||||
for (int c = 0; c < MAXCHANNELS; c++) {
|
||||
NCCLCHECKGOTO(ncclProxyConnect(comm, TRANSPORT_PROFILER, 0, comm->rank, &comm->profiler.sendProxyConn[c]), ret, exit);
|
||||
NCCLCHECKGOTO(ncclProxyCallBlocking(comm, &comm->profiler.sendProxyConn[c], ncclProxyMsgConnect, NULL, 0, NULL, 0), ret, exit);
|
||||
NCCLCHECKGOTO(ncclProxyConnect(comm, TRANSPORT_PROFILER, 0, comm->rank, &comm->profiler.recvProxyConn[c]), ret, exit);
|
||||
NCCLCHECKGOTO(ncclProxyCallBlocking(comm, &comm->profiler.recvProxyConn[c], ncclProxyMsgConnect, NULL, 0, NULL, 0), ret, exit);
|
||||
}
|
||||
comm->profiler.initialized = true;
|
||||
exit:
|
||||
pthread_mutex_unlock(&proxyProfilerConnectLock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool ncclProfilerNeedsProxy(struct ncclComm* comm, struct ncclProxyOp* op) {
|
||||
bool enabled = ncclProfilerPluginLoaded() && (op->eActivationMask & ncclProfileKernelCh);
|
||||
if (enabled && !comm->profiler.initialized) (void)proxyProfilerConnect(comm, op);
|
||||
return enabled;
|
||||
}
|
||||
|
||||
bool ncclProfilerPluginLoaded(void) {
|
||||
return (__builtin_expect(ncclProfiler != NULL, 0));
|
||||
}
|
||||
|
||||
ncclResult_t ncclProfilerCallback(void** eHandle, int type, void* pHandle, int64_t pluginId, void* extData) {
|
||||
if (__builtin_expect(ncclProfiler != NULL, 0)) {
|
||||
struct ncclProxySubArgs* sub = (struct ncclProxySubArgs*)pHandle;
|
||||
if (type == 0) { // start
|
||||
if (sub->eActivationMask & ncclProfileNetPlugin) {
|
||||
ncclProfilerEventDescr_t eDescr = { 0 };
|
||||
eDescr.type = ncclProfileNetPlugin;
|
||||
eDescr.parentObj = sub->stepEventHandles[sub->profilerSteps%NCCL_STEPS];
|
||||
eDescr.rank = sub->rank;
|
||||
eDescr.netPlugin.id = pluginId;
|
||||
eDescr.netPlugin.data = extData;
|
||||
ncclProfiler->startEvent(sub->profilerContext, eHandle, &eDescr);
|
||||
}
|
||||
} else { // stop
|
||||
ncclProfiler->stopEvent(*eHandle);
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "comm.h"
|
||||
#include "nccl_profiler.h"
|
||||
#include "checks.h"
|
||||
|
||||
static ncclProfiler_t ncclProfiler;
|
||||
static ncclProfiler_v1_t* ncclProfiler_v1;
|
||||
|
||||
static uint8_t ncclStringToFunc(const char* func) {
|
||||
if (0 == strcmp(func, "AllGather")) return ncclFuncAllGather;
|
||||
if (0 == strcmp(func, "AllReduce")) return ncclFuncAllReduce;
|
||||
if (0 == strcmp(func, "Broadcast")) return ncclFuncBroadcast;
|
||||
if (0 == strcmp(func, "Recv")) return ncclFuncRecv;
|
||||
if (0 == strcmp(func, "Reduce")) return ncclFuncReduce;
|
||||
if (0 == strcmp(func, "ReduceScatter")) return ncclFuncReduceScatter;
|
||||
if (0 == strcmp(func, "SendRecv")) return ncclFuncSendRecv;
|
||||
return ncclFuncSend;
|
||||
}
|
||||
|
||||
static uint8_t ncclStringToAlgo(const char* algo) {
|
||||
if (0 == strcmp(algo, "TREE")) return NCCL_ALGO_TREE;
|
||||
if (0 == strcmp(algo, "RING")) return NCCL_ALGO_RING;
|
||||
if (0 == strcmp(algo, "COLLNET_DIRECT")) return NCCL_ALGO_COLLNET_DIRECT;
|
||||
if (0 == strcmp(algo, "COLLNET_CHAIN")) return NCCL_ALGO_COLLNET_CHAIN;
|
||||
if (0 == strcmp(algo, "NVLS")) return NCCL_ALGO_NVLS;
|
||||
if (0 == strcmp(algo, "NVLS_TREE")) return NCCL_ALGO_NVLS_TREE;
|
||||
return NCCL_ALGO_PAT;
|
||||
}
|
||||
|
||||
static uint8_t ncclStringToProto(const char* proto) {
|
||||
if (0 == strcmp(proto, "LL")) return NCCL_PROTO_LL;
|
||||
if (0 == strcmp(proto, "LL128")) return NCCL_PROTO_LL128;
|
||||
return NCCL_PROTO_SIMPLE;
|
||||
}
|
||||
|
||||
static uint8_t ncclStringToDatatype(const char* dt) {
|
||||
if (0 == strcmp(dt, "ncclInt8")) return ncclInt8;
|
||||
if (0 == strcmp(dt, "ncclInt32")) return ncclInt32;
|
||||
if (0 == strcmp(dt, "ncclUint32")) return ncclUint32;
|
||||
if (0 == strcmp(dt, "ncclInt64")) return ncclInt64;
|
||||
if (0 == strcmp(dt, "ncclUint64")) return ncclUint64;
|
||||
if (0 == strcmp(dt, "ncclFloat16")) return ncclFloat16;
|
||||
if (0 == strcmp(dt, "ncclFloat32")) return ncclFloat32;
|
||||
#if defined(__CUDA_BF16_TYPES_EXIST__)
|
||||
if (0 == strcmp(dt, "ncclBfloat16")) return ncclBfloat16;
|
||||
#endif
|
||||
return ncclFloat64;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclProfiler_startEvent(void* context, void** eHandle, ncclProfilerEventDescr_t* eDescr) {
|
||||
ncclProfilerEventDescr_v1_t eDescr_v1 = { 0 };
|
||||
eDescr_v1.type = eDescr->type;
|
||||
eDescr_v1.parentObj = eDescr->parentObj;
|
||||
eDescr_v1.rank = eDescr->rank;
|
||||
switch(eDescr->type) {
|
||||
case ncclProfileGroup: break;
|
||||
case ncclProfileColl: {
|
||||
eDescr_v1.coll.name = eDescr->coll.name;
|
||||
eDescr_v1.coll.commHash = eDescr->coll.commHash;
|
||||
eDescr_v1.coll.seqNumber = eDescr->coll.seqNumber;
|
||||
eDescr_v1.coll.func = ncclStringToFunc(eDescr->coll.func);
|
||||
eDescr_v1.coll.sendBuff = eDescr->coll.sendBuff;
|
||||
eDescr_v1.coll.recvBuff = eDescr->coll.recvBuff;
|
||||
eDescr_v1.coll.count = eDescr->coll.count;
|
||||
eDescr_v1.coll.root = eDescr->coll.root;
|
||||
eDescr_v1.coll.datatype = ncclStringToDatatype(eDescr->coll.datatype);
|
||||
eDescr_v1.coll.op = 0; // removed in v2
|
||||
eDescr_v1.coll.trafficBytes = 0; // removed in v3
|
||||
eDescr_v1.coll.nMaxChannels = eDescr->coll.nMaxChannels;
|
||||
eDescr_v1.coll.nWarps = eDescr->coll.nWarps;
|
||||
eDescr_v1.coll.algo = ncclStringToAlgo(eDescr->coll.algo);
|
||||
eDescr_v1.coll.proto = ncclStringToProto(eDescr->coll.proto);
|
||||
} break;
|
||||
case ncclProfileP2p: {
|
||||
eDescr_v1.p2p.name = eDescr->p2p.name;
|
||||
eDescr_v1.p2p.commHash = eDescr->p2p.commHash;
|
||||
eDescr_v1.p2p.func = ncclStringToFunc(eDescr->p2p.func);
|
||||
eDescr_v1.p2p.buff = eDescr->p2p.buff;
|
||||
eDescr_v1.p2p.count = eDescr->p2p.count;
|
||||
eDescr_v1.p2p.datatype = ncclStringToDatatype(eDescr->p2p.datatype);
|
||||
eDescr_v1.p2p.peer = eDescr->p2p.peer;
|
||||
} break;
|
||||
case ncclProfileProxyOp: {
|
||||
eDescr_v1.proxyOp.pid = eDescr->proxyOp.pid;
|
||||
eDescr_v1.proxyOp.channelId = eDescr->proxyOp.channelId;
|
||||
eDescr_v1.proxyOp.peer = eDescr->proxyOp.peer;
|
||||
eDescr_v1.proxyOp.nSteps = eDescr->proxyOp.nSteps;
|
||||
eDescr_v1.proxyOp.chunkSize = eDescr->proxyOp.chunkSize;
|
||||
eDescr_v1.proxyOp.isSend = eDescr->proxyOp.isSend;
|
||||
} break;
|
||||
case ncclProfileProxyStep: {
|
||||
eDescr_v1.proxyStep.step = eDescr->proxyStep.step;
|
||||
} break;
|
||||
case ncclProfileProxyCtrl: break;
|
||||
case ncclProfileKernelCh:
|
||||
case ncclProfileNetPlugin: {
|
||||
*eHandle = NULL;
|
||||
return ncclSuccess;
|
||||
}
|
||||
default:;
|
||||
}
|
||||
return ncclProfiler_v1->startEvent(context, eHandle, &eDescr_v1);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclProfiler_recordEventState(void* eHandle, ncclProfilerEventState_t eState, ncclProfilerEventStateArgs_t* eStateArgs) {
|
||||
return ncclProfiler_v1->recordEventState(eHandle, eState, (ncclProfilerEventStateArgs_v1_t*)eStateArgs);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclProfiler_init(void** context, int* eActivationMask) {
|
||||
NCCLCHECK(ncclProfiler_v1->init(context, eActivationMask));
|
||||
ncclProfiler.startEvent = ncclProfiler_startEvent;
|
||||
ncclProfiler.stopEvent = ncclProfiler_v1->stopEvent;
|
||||
ncclProfiler.recordEventState = ncclProfiler_recordEventState;
|
||||
ncclProfiler.finalize = ncclProfiler_v1->finalize;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclProfiler_t* getNcclProfiler_v1(void* lib) {
|
||||
ncclProfiler_v1 = (ncclProfiler_v1_t*)dlsym(lib, "ncclProfiler_v1");
|
||||
if (ncclProfiler_v1) {
|
||||
ncclProfiler.name = ncclProfiler_v1->name;
|
||||
ncclProfiler.init = ncclProfiler_init;
|
||||
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded %s", ncclProfiler_v1->name);
|
||||
return &ncclProfiler;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v1.");
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "comm.h"
|
||||
#include "nccl_profiler.h"
|
||||
#include "checks.h"
|
||||
|
||||
static ncclProfiler_t ncclProfiler;
|
||||
static ncclProfiler_v2_t* ncclProfiler_v2;
|
||||
|
||||
static ncclResult_t ncclProfiler_startEvent(void* context, void** eHandle, ncclProfilerEventDescr_t* eDescr) {
|
||||
*eHandle = nullptr;
|
||||
ncclProfilerEventDescr_v2_t eDescr_v2 = { };
|
||||
eDescr_v2.type = eDescr->type;
|
||||
eDescr_v2.parentObj = eDescr->parentObj;
|
||||
eDescr_v2.rank = eDescr->rank;
|
||||
switch(eDescr->type) {
|
||||
case ncclProfileGroup: break;
|
||||
case ncclProfileColl: {
|
||||
eDescr_v2.coll.name = eDescr->coll.name;
|
||||
eDescr_v2.coll.commHash = eDescr->coll.commHash;
|
||||
eDescr_v2.coll.seqNumber = eDescr->coll.seqNumber;
|
||||
eDescr_v2.coll.func = eDescr->coll.func;
|
||||
eDescr_v2.coll.sendBuff = eDescr->coll.sendBuff;
|
||||
eDescr_v2.coll.recvBuff = eDescr->coll.recvBuff;
|
||||
eDescr_v2.coll.count = eDescr->coll.count;
|
||||
eDescr_v2.coll.root = eDescr->coll.root;
|
||||
eDescr_v2.coll.datatype = eDescr->coll.datatype;
|
||||
eDescr_v2.coll.trafficBytes = 0; // removed in v3
|
||||
eDescr_v2.coll.nMaxChannels = eDescr->coll.nMaxChannels;
|
||||
eDescr_v2.coll.nWarps = eDescr->coll.nWarps;
|
||||
eDescr_v2.coll.algo = eDescr->coll.algo;
|
||||
eDescr_v2.coll.proto = eDescr->coll.proto;
|
||||
} break;
|
||||
case ncclProfileP2p: {
|
||||
eDescr_v2.p2p.name = eDescr->p2p.name;
|
||||
eDescr_v2.p2p.commHash = eDescr->p2p.commHash;
|
||||
eDescr_v2.p2p.func = eDescr->p2p.func;
|
||||
eDescr_v2.p2p.buff = eDescr->p2p.buff;
|
||||
eDescr_v2.p2p.count = eDescr->p2p.count;
|
||||
eDescr_v2.p2p.datatype = eDescr->p2p.datatype;
|
||||
eDescr_v2.p2p.peer = eDescr->p2p.peer;
|
||||
} break;
|
||||
case ncclProfileProxyOp: {
|
||||
eDescr_v2.proxyOp.pid = eDescr->proxyOp.pid;
|
||||
eDescr_v2.proxyOp.channelId = eDescr->proxyOp.channelId;
|
||||
eDescr_v2.proxyOp.peer = eDescr->proxyOp.peer;
|
||||
eDescr_v2.proxyOp.nSteps = eDescr->proxyOp.nSteps;
|
||||
eDescr_v2.proxyOp.chunkSize = eDescr->proxyOp.chunkSize;
|
||||
eDescr_v2.proxyOp.isSend = eDescr->proxyOp.isSend;
|
||||
} break;
|
||||
case ncclProfileProxyStep: {
|
||||
eDescr_v2.proxyStep.step = eDescr->proxyStep.step;
|
||||
} break;
|
||||
case ncclProfileProxyCtrl: break;
|
||||
default: return ncclSuccess;
|
||||
}
|
||||
return ncclProfiler_v2->startEvent(context, eHandle, &eDescr_v2);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclProfiler_recordEventState(void* eHandle, ncclProfilerEventState_t eState, ncclProfilerEventStateArgs_t* eStateArgs) {
|
||||
return ncclProfiler_v2->recordEventState(eHandle, eState, (ncclProfilerEventStateArgs_v2_t *)eStateArgs);
|
||||
}
|
||||
|
||||
static ncclResult_t ncclProfiler_init(void** context, int* eActivationMask) {
|
||||
NCCLCHECK(ncclProfiler_v2->init(context, eActivationMask));
|
||||
ncclProfiler.startEvent = ncclProfiler_startEvent;
|
||||
ncclProfiler.stopEvent = ncclProfiler_v2->stopEvent;
|
||||
ncclProfiler.recordEventState = ncclProfiler_recordEventState;
|
||||
ncclProfiler.finalize = ncclProfiler_v2->finalize;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclProfiler_t* getNcclProfiler_v2(void* lib) {
|
||||
ncclProfiler_v2 = (ncclProfiler_v2_t*)dlsym(lib, "ncclProfiler_v2");
|
||||
if (ncclProfiler_v2) {
|
||||
ncclProfiler.name = ncclProfiler_v2->name;
|
||||
ncclProfiler.init = ncclProfiler_init;
|
||||
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded %s", ncclProfiler_v2->name);
|
||||
return &ncclProfiler;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v2");
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "comm.h"
|
||||
#include "nccl_profiler.h"
|
||||
|
||||
static ncclProfiler_v3_t* ncclProfiler_v3;
|
||||
|
||||
ncclProfiler_t* getNcclProfiler_v3(void* lib) {
|
||||
ncclProfiler_v3 = (ncclProfiler_v3_t*)dlsym(lib, "ncclProfiler_v3");
|
||||
if (ncclProfiler_v3) {
|
||||
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded %s", ncclProfiler_v3->name);
|
||||
return ncclProfiler_v3;
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v3");
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "checks.h"
|
||||
#include "debug.h"
|
||||
#include "tuner.h"
|
||||
#include "plugin.h"
|
||||
|
||||
extern ncclTuner_t* getNcclTuner_v2(void* lib);
|
||||
extern ncclTuner_t* getNcclTuner_v3(void* lib);
|
||||
extern ncclTuner_t* getNcclTuner_v4(void* lib);
|
||||
|
||||
pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
static int tunerPluginRefCount;
|
||||
static void* tunerPluginLib = nullptr;
|
||||
static ncclTuner_t* tunerSymbol = nullptr;
|
||||
|
||||
enum {
|
||||
tunerPluginLoadFailed = -1,
|
||||
tunerPluginLoadReady = 0,
|
||||
tunerPluginLoadSuccess = 1,
|
||||
};
|
||||
|
||||
#define MAX_PLUGIN_LOAD 4
|
||||
|
||||
static int status = tunerPluginLoadReady;
|
||||
|
||||
ncclResult_t ncclTunerPluginLoad(struct ncclComm* comm) {
|
||||
// Initialize to nullptr by default if plugin tuner cannot be loaded.
|
||||
comm->tuner = nullptr;
|
||||
if (tunerPluginLoadFailed == status) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&tunerPluginLock);
|
||||
if (tunerPluginLoadFailed == status) {
|
||||
goto exit;
|
||||
}
|
||||
|
||||
if (tunerPluginLoadSuccess == status) {
|
||||
comm->tuner = tunerSymbol;
|
||||
++tunerPluginRefCount;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
tunerPluginLib = ncclOpenTunerPluginLib(ncclGetEnv("NCCL_TUNER_PLUGIN"));
|
||||
if (nullptr == tunerPluginLib) {
|
||||
tunerPluginLib = ncclGetNetPluginLib();
|
||||
if (nullptr == tunerPluginLib) {
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
tunerSymbol = getNcclTuner_v4(tunerPluginLib);
|
||||
if (tunerSymbol == NULL) {
|
||||
tunerSymbol = getNcclTuner_v3(tunerPluginLib);
|
||||
}
|
||||
if (tunerSymbol == NULL) {
|
||||
tunerSymbol = getNcclTuner_v2(tunerPluginLib);
|
||||
}
|
||||
if (tunerSymbol == NULL) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
comm->tuner = tunerSymbol;
|
||||
++tunerPluginRefCount;
|
||||
status = tunerPluginLoadSuccess;
|
||||
comm->tunerPluginLoaded = 1;
|
||||
|
||||
exit:
|
||||
pthread_mutex_unlock(&tunerPluginLock);
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
tunerPluginLib = nullptr;
|
||||
status = tunerPluginLoadFailed;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTunerPluginUnload(struct ncclComm* comm) {
|
||||
pthread_mutex_lock(&tunerPluginLock);
|
||||
if (comm->tunerPluginLoaded && 0 == (--tunerPluginRefCount)) {
|
||||
INFO(NCCL_TUNING, "TUNER/Plugin: Closing tuner: '%s'", tunerSymbol->name);
|
||||
NCCLCHECK(ncclClosePluginLib(tunerPluginLib));
|
||||
tunerPluginLib = nullptr;
|
||||
tunerSymbol = nullptr;
|
||||
comm->tuner = nullptr;
|
||||
status = tunerPluginLoadReady;
|
||||
comm->tunerPluginLoaded = 0;
|
||||
}
|
||||
pthread_mutex_unlock(&tunerPluginLock);
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include <dlfcn.h>
|
||||
#include "debug.h"
|
||||
#include "checks.h"
|
||||
#include "nccl_tuner.h"
|
||||
|
||||
static ncclTuner_v2_t* ncclTuner_v2;
|
||||
static ncclTuner_t ncclTuner;
|
||||
|
||||
static int hasNvlsSupport(float** collCostTable) {
|
||||
// Requirements for support of different algorithms:
|
||||
//
|
||||
// - NVLS intra-node: nvlsSupport
|
||||
// - NVLS intra+inter-node: collNetSupport
|
||||
// - NVLSTree intra-node: always disabled
|
||||
// - NVLSTree inter-node: nvlsSupport
|
||||
// - Collnet* inter-node: collNetSupport
|
||||
//
|
||||
// nvlsSupport = 1 if either NVLS or NVLS_TREE entries in the cost table are not -1
|
||||
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
|
||||
return (table[NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE || table[NCCL_ALGO_NVLS_TREE][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) ? 1 : 0;
|
||||
}
|
||||
|
||||
static int hasCollNetSupport(float** collCostTable) {
|
||||
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
|
||||
return (table[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] == NCCL_ALGO_PROTO_IGNORE) ? 0 : 1;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTuner_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int regBuff __attribute__((unused)), int* nChannels) {
|
||||
int algorithm = NCCL_ALGO_UNDEF;
|
||||
int protocol = NCCL_PROTO_UNDEF;
|
||||
int nvlsSupport = hasNvlsSupport(collCostTable);
|
||||
int collNetSupport = hasCollNetSupport(collCostTable);
|
||||
NCCLCHECK(ncclTuner_v2->getCollInfo(context, collType, nBytes, collNetSupport, nvlsSupport, numPipeOps, &algorithm, &protocol, nChannels));
|
||||
// set time to 0 below to make sure this algorithm/protocol is selected later on
|
||||
if (algorithm >= 0 && algorithm < NCCL_NUM_ALGORITHMS && protocol >= 0 && protocol < NCCL_NUM_PROTOCOLS) {
|
||||
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
|
||||
if (table[algorithm][protocol] != NCCL_ALGO_PROTO_IGNORE) table[algorithm][protocol] = 0.0;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTuner_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logfn, void** context) {
|
||||
NCCLCHECK(ncclTuner_v2->init(nRanks, nNodes, logfn, context));
|
||||
ncclTuner.getCollInfo = ncclTuner_getCollInfo;
|
||||
ncclTuner.destroy = ncclTuner_v2->destroy;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclTuner_t* getNcclTuner_v2(void* lib) {
|
||||
ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(lib, "ncclTunerPlugin_v2");
|
||||
if (ncclTuner_v2) {
|
||||
ncclTuner.name = ncclTuner_v2->name;
|
||||
ncclTuner.init = ncclTuner_init;
|
||||
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", ncclTuner_v2->name);
|
||||
return &ncclTuner;
|
||||
}
|
||||
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include <dlfcn.h>
|
||||
#include "debug.h"
|
||||
#include "checks.h"
|
||||
#include "nccl_tuner.h"
|
||||
|
||||
static ncclTuner_v3_t* ncclTuner_v3;
|
||||
static ncclTuner_t ncclTuner;
|
||||
|
||||
static ncclResult_t ncclTuner_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff __attribute__((unused)), int* nChannels) {
|
||||
NCCLCHECK(ncclTuner_v3->getCollInfo(context, collType, nBytes, numPipeOps, collCostTable, numAlgo, numProto, nChannels));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclTuner_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logfn, void** context) {
|
||||
NCCLCHECK(ncclTuner_v3->init(nRanks, nNodes, logfn, context));
|
||||
ncclTuner.getCollInfo = ncclTuner_getCollInfo;
|
||||
ncclTuner.destroy = ncclTuner_v3->destroy;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclTuner_t* getNcclTuner_v3(void* lib) {
|
||||
ncclTuner_v3 = (ncclTuner_v3_t*)dlsym(lib, "ncclTunerPlugin_v3");
|
||||
if (ncclTuner_v3) {
|
||||
ncclTuner.name = ncclTuner_v3->name;
|
||||
ncclTuner.init = ncclTuner_init;
|
||||
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", ncclTuner_v3->name);
|
||||
return &ncclTuner;
|
||||
}
|
||||
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
|
||||
return NULL;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include <dlfcn.h>
|
||||
#include "debug.h"
|
||||
#include "nccl_tuner.h"
|
||||
|
||||
static ncclTuner_v4_t* ncclTuner_v4;
|
||||
|
||||
ncclTuner_t* getNcclTuner_v4(void* lib) {
|
||||
ncclTuner_v4 = (ncclTuner_v4_t*)dlsym(lib, "ncclTunerPlugin_v4");
|
||||
if (ncclTuner_v4) {
|
||||
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", ncclTuner_v4->name);
|
||||
return ncclTuner_v4;
|
||||
}
|
||||
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.");
|
||||
return NULL;
|
||||
}
|
||||
Ссылка в новой задаче
Block a user