Merge remote-tracking branch 'nccl/master' into develop
Cette révision appartient à :
BertanDogancay
+219
-171
@@ -8,6 +8,7 @@
|
||||
#include "bootstrap.h"
|
||||
#include "checks.h"
|
||||
#include "plugin.h"
|
||||
#include "nccl_net.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
@@ -15,137 +16,100 @@
|
||||
//#include <sys/stat.h>
|
||||
//#include <unistd.h>
|
||||
|
||||
extern ncclNet_t* getNcclNet_v6(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v7(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v8(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v9(void* netPluginLib);
|
||||
extern ncclNet_t* getNcclNet_v10(void* netPluginLib);
|
||||
typedef ncclNet_t* getNcclNet_t(void* netPluginLib);
|
||||
typedef ncclCollNet_t* getNcclCollNet_t(void* netPluginLib);
|
||||
|
||||
extern ncclCollNet_t* getNcclCollNet_v6(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v7(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v8(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v9(void* netPluginLib);
|
||||
extern ncclCollNet_t* getNcclCollNet_v10(void* netPluginLib);
|
||||
|
||||
static pthread_mutex_t netLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
ncclNet_t* ncclNets[NCCL_NET_MAX_PLUGINS] = { nullptr, &ncclNetIb, &ncclNetSocket };
|
||||
static int ncclNetsVer[NCCL_NET_MAX_PLUGINS] = { -1, 10, 10 };
|
||||
ncclCollNet_t* ncclCollNets[NCCL_NET_MAX_PLUGINS] = { nullptr, nullptr, nullptr };
|
||||
enum ncclNetState {
|
||||
ncclNetStateInit = 0,
|
||||
ncclNetStateEnabled = 1,
|
||||
ncclNetStateDisabled = 2
|
||||
};
|
||||
enum ncclNetState ncclNetStates[NCCL_NET_MAX_PLUGINS] = { ncclNetStateInit, ncclNetStateInit, ncclNetStateInit };
|
||||
enum ncclNetState ncclCollNetStates[NCCL_NET_MAX_PLUGINS] = { ncclNetStateInit, ncclNetStateInit, ncclNetStateInit };
|
||||
extern getNcclNet_t getNcclNet_v6;
|
||||
extern getNcclNet_t getNcclNet_v7;
|
||||
extern getNcclNet_t getNcclNet_v8;
|
||||
extern getNcclNet_t getNcclNet_v9;
|
||||
extern getNcclNet_t getNcclNet_v10;
|
||||
extern getNcclCollNet_t getNcclCollNet_v6;
|
||||
extern getNcclCollNet_t getNcclCollNet_v7;
|
||||
extern getNcclCollNet_t getNcclCollNet_v8;
|
||||
extern getNcclCollNet_t getNcclCollNet_v9;
|
||||
extern getNcclCollNet_t getNcclCollNet_v10;
|
||||
|
||||
NCCL_PARAM(NetPluginRefCount, "NET_PLUGIN_REF_COUNT", 1);
|
||||
#define NCCL_NET_VERSION_COUNT 5
|
||||
int ncclNetVersion[NCCL_NET_VERSION_COUNT] = {10, 9, 8, 7, 6};
|
||||
getNcclNet_t* getNcclNet[NCCL_NET_VERSION_COUNT] = {getNcclNet_v10, getNcclNet_v9, getNcclNet_v8, getNcclNet_v7, getNcclNet_v6};
|
||||
getNcclCollNet_t* getNcclCollNet[NCCL_NET_VERSION_COUNT] = {getNcclCollNet_v10, getNcclCollNet_v9, getNcclCollNet_v8, getNcclCollNet_v7, getNcclCollNet_v6};
|
||||
|
||||
#define NCCL_NET_NUM_INTERNAL_PLUGINS 2
|
||||
|
||||
typedef enum ncclNetPluginState {
|
||||
ncclNetPluginStateDisabled = -2, // Plugin library failed to initialize
|
||||
ncclNetPluginStateLoadFailed = -1, // Plugin library failed to load
|
||||
ncclNetPluginStateLoadReady = 0, // Plugin library is ready to be loaded
|
||||
ncclNetPluginStateInitReady = 1, // Plugin library is loaded and ready to be initialized
|
||||
ncclNetPluginStateEnabled = 2, // Plugin library is loaded and initialized
|
||||
} ncclNetPluginState_t;
|
||||
|
||||
#define MAX_STR_LEN 255
|
||||
typedef struct netPluginLib {
|
||||
char name[MAX_STR_LEN]; // Name of the plugin library
|
||||
void* dlHandle; // Handle to the plugin library
|
||||
ncclNet_t* ncclNet; // Pointer to the ncclNet_t structure
|
||||
int ncclNetVer; // Version of the nccl net plugin
|
||||
ncclCollNet_t* ncclCollNet; // Pointer to the ncclCollNet_t structure
|
||||
ncclNetPluginState_t ncclNetPluginState; // State of the nccl net plugin
|
||||
ncclNetPluginState_t ncclCollNetPluginState; // State of the nccl coll net plugin
|
||||
int ncclNetPluginRefCount; // Reference count for the nccl net plugin
|
||||
} netPluginLib_t;
|
||||
|
||||
int pluginCount = 0;
|
||||
bool netPluginLibsInitialized = false;
|
||||
netPluginLib_t netPluginLibs[NCCL_NET_MAX_PLUGINS] = { 0 };
|
||||
static pthread_mutex_t netPluginLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
static void* netPluginLib;
|
||||
static pthread_once_t initPluginLibsOnceControl = PTHREAD_ONCE_INIT;
|
||||
|
||||
static int netPluginRefCount;
|
||||
static void initNetPluginRefCountOnce(void) { netPluginRefCount = ncclParamNetPluginRefCount();}
|
||||
|
||||
enum {
|
||||
netPluginLoadFailed = -1,
|
||||
netPluginLoadReady = 0,
|
||||
netPluginLoadSuccess = 1,
|
||||
};
|
||||
|
||||
static int netPluginStatus = netPluginLoadReady;
|
||||
|
||||
ncclResult_t ncclNetPluginLoad(struct ncclComm* comm) {
|
||||
static pthread_once_t netPluginRefCountOnce = PTHREAD_ONCE_INIT;
|
||||
pthread_once(&netPluginRefCountOnce, initNetPluginRefCountOnce);
|
||||
|
||||
pthread_mutex_lock(&netPluginLock);
|
||||
if (netPluginLoadFailed == netPluginStatus) {
|
||||
goto exit;
|
||||
static ncclResult_t ncclNetPluginUnload(netPluginLib_t* pluginLib) {
|
||||
if ((pluginLib->dlHandle) && ((pluginLib->ncclNetPluginRefCount) == 0)) {
|
||||
INFO(NCCL_INIT|NCCL_NET, "Unloading plugin %s", pluginLib->name);
|
||||
NCCLCHECK(ncclClosePluginLib(pluginLib->dlHandle));
|
||||
memset(pluginLib, 0, sizeof(netPluginLib_t));
|
||||
}
|
||||
if (netPluginLoadSuccess == netPluginStatus) {
|
||||
++netPluginRefCount;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
netPluginLib = ncclOpenNetPluginLib(ncclGetEnv("NCCL_NET_PLUGIN"));
|
||||
if (netPluginLib == nullptr) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
ncclNets[0] = getNcclNet_v10(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 10;
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v9 plugin
|
||||
ncclNets[0] = getNcclNet_v9(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 9;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v8 plugin
|
||||
ncclNets[0] = getNcclNet_v8(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 8;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v7 plugin
|
||||
ncclNets[0] = getNcclNet_v7(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 7;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
// Try v6 plugin
|
||||
ncclNets[0] = getNcclNet_v6(netPluginLib);
|
||||
if (ncclNets[0]) ncclNetsVer[0] = 6;
|
||||
}
|
||||
if (ncclNets[0] == nullptr) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
// Check for CollNet
|
||||
ncclCollNets[0] = getNcclCollNet_v10(netPluginLib);
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v9(netPluginLib);
|
||||
}
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v8(netPluginLib);
|
||||
}
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v7(netPluginLib);
|
||||
}
|
||||
if (ncclCollNets[0] == nullptr) {
|
||||
ncclCollNets[0] = getNcclCollNet_v6(netPluginLib);
|
||||
}
|
||||
|
||||
++netPluginRefCount;
|
||||
netPluginStatus = netPluginLoadSuccess;
|
||||
comm->netPluginLoaded = 1;
|
||||
|
||||
exit:
|
||||
pthread_mutex_unlock(&netPluginLock);
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
if (netPluginLib) NCCLCHECK(ncclClosePluginLib(netPluginLib));
|
||||
netPluginStatus = netPluginLoadFailed;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetPluginUnload(struct ncclComm* comm) {
|
||||
pthread_mutex_lock(&netPluginLock);
|
||||
if (comm->netPluginLoaded && 0 == (--netPluginRefCount)) {
|
||||
if (ncclNets[0]) {
|
||||
INFO(NCCL_NET, "NET/Plugin: Closing net plugin '%s'", ncclNets[0]->name);
|
||||
}
|
||||
if (ncclCollNets[0]) {
|
||||
INFO(NCCL_NET, "NET/Plugin: Closing collnet plugin '%s'", ncclCollNets[0]->name);
|
||||
}
|
||||
NCCLCHECK(ncclClosePluginLib(netPluginLib));
|
||||
netPluginLib = nullptr;
|
||||
ncclNets[0] = nullptr;
|
||||
ncclCollNets[0] = nullptr;
|
||||
netPluginStatus = netPluginLoadReady;
|
||||
comm->netPluginLoaded = 0;
|
||||
for (int i = 0; i < NCCL_NET_MAX_PLUGINS; ++i)
|
||||
ncclCollNetStates[i] = ncclNetStates[i] = ncclNetStateInit;
|
||||
static ncclResult_t ncclNetPluginLoad(netPluginLib_t* pluginLib) {
|
||||
pluginLib->dlHandle = ncclOpenNetPluginLib(pluginLib->name);
|
||||
|
||||
if (pluginLib->dlHandle == nullptr) goto fail;
|
||||
// load ncclNet
|
||||
for (int i = 0; i < NCCL_NET_VERSION_COUNT; i++) {
|
||||
pluginLib->ncclNetVer = ncclNetVersion[i];
|
||||
pluginLib->ncclNet = getNcclNet[i](pluginLib->dlHandle);
|
||||
if (pluginLib->ncclNet) break;
|
||||
}
|
||||
pthread_mutex_unlock(&netPluginLock);
|
||||
|
||||
// if we fail to find a net, exit
|
||||
if (pluginLib->ncclNet == nullptr) goto fail;
|
||||
|
||||
pluginLib->ncclNetPluginState = ncclNetPluginStateInitReady;
|
||||
|
||||
// load ncclColNet
|
||||
for (int i = 0; i < NCCL_NET_VERSION_COUNT; i++) {
|
||||
pluginLib->ncclCollNet = getNcclCollNet[i](pluginLib->dlHandle);
|
||||
if (pluginLib->ncclCollNet) break;
|
||||
}
|
||||
|
||||
if (pluginLib->ncclCollNet == nullptr)
|
||||
pluginLib->ncclCollNetPluginState = ncclNetPluginStateLoadFailed;
|
||||
else
|
||||
pluginLib->ncclCollNetPluginState = ncclNetPluginStateInitReady;
|
||||
|
||||
INFO(NCCL_INIT|NCCL_NET, "Successfully loaded external plugin %s", pluginLib->name);
|
||||
exit:
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
if (pluginLib->dlHandle) {
|
||||
NCCLCHECK(ncclClosePluginLib(pluginLib->dlHandle));
|
||||
}
|
||||
pluginLib->ncclNetPluginState = ncclNetPluginStateLoadFailed;
|
||||
pluginLib->ncclCollNetPluginState = ncclNetPluginStateLoadFailed;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetCheckDeviceVersion(struct ncclComm* comm, ncclNet_t* net, int dev) {
|
||||
@@ -172,72 +136,156 @@ ncclResult_t ncclNetCheckDeviceVersion(struct ncclComm* comm, ncclNet_t* net, in
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t netGetState(int i, enum ncclNetState* state) {
|
||||
pthread_mutex_lock(&netLock);
|
||||
if (ncclNetStates[i] == ncclNetStateInit) {
|
||||
int ndev;
|
||||
if (ncclNets[i]->init(ncclDebugLog, ncclProfilerCallback) != ncclSuccess) ncclNetStates[i] = ncclNetStateDisabled;
|
||||
else if (ncclNets[i]->devices(&ndev) != ncclSuccess || ndev <= 0) ncclNetStates[i] = ncclNetStateDisabled;
|
||||
else ncclNetStates[i] = ncclNetStateEnabled;
|
||||
static ncclResult_t ncclNetPluginInit(netPluginLib_t* pluginLib) {
|
||||
int ndev;
|
||||
if (pluginLib->ncclNetPluginState == ncclNetPluginStateInitReady && pluginLib->ncclNet) {
|
||||
if (pluginLib->ncclNet->init(ncclDebugLog, ncclProfilerCallback) != ncclSuccess) goto fail;
|
||||
if (pluginLib->ncclNet->devices(&ndev) != ncclSuccess || ndev <= 0) goto fail;
|
||||
}
|
||||
pluginLib->ncclNetPluginState = ncclNetPluginStateEnabled;
|
||||
INFO(NCCL_INIT|NCCL_NET, "Initialized NET plugin %s", pluginLib->ncclNet->name);
|
||||
|
||||
if (pluginLib->ncclCollNetPluginState == ncclNetPluginStateInitReady && pluginLib->ncclCollNet) {
|
||||
if (pluginLib->ncclCollNet->init(ncclDebugLog) != ncclSuccess) pluginLib->ncclCollNetPluginState = ncclNetPluginStateDisabled;
|
||||
else if (pluginLib->ncclCollNet->devices(&ndev) != ncclSuccess || ndev <= 0) pluginLib->ncclCollNetPluginState = ncclNetPluginStateDisabled;
|
||||
else {
|
||||
pluginLib->ncclCollNetPluginState = ncclNetPluginStateEnabled;
|
||||
}
|
||||
}
|
||||
exit:
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
pluginLib->ncclNetPluginState = ncclNetPluginStateDisabled;
|
||||
pluginLib->ncclCollNetPluginState = ncclNetPluginStateDisabled;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNetPluginAssignToComm(struct ncclComm* comm, int pluginIndex, bool* isAssigned) {
|
||||
const char* netName = comm->config.netName;
|
||||
if (netName && strcasecmp(netName, netPluginLibs[pluginIndex].ncclNet->name) != 0) goto fail;
|
||||
if (ncclSuccess != ncclNetCheckDeviceVersion(comm, netPluginLibs[pluginIndex].ncclNet, 0)) goto fail;
|
||||
|
||||
if (netPluginLibs[pluginIndex].ncclNetPluginState >= ncclNetPluginStateEnabled) {
|
||||
comm->ncclNet = netPluginLibs[pluginIndex].ncclNet;
|
||||
comm->ncclNetVer = netPluginLibs[pluginIndex].ncclNetVer;
|
||||
comm->netPluginIndex = pluginIndex;
|
||||
netPluginLibs[pluginIndex].ncclNetPluginRefCount++;
|
||||
*isAssigned = true;
|
||||
INFO(NCCL_INIT|NCCL_NET, "Assigned NET plugin %s to comm", netPluginLibs[pluginIndex].ncclNet->name);
|
||||
if (netPluginLibs[pluginIndex].ncclCollNetPluginState >= ncclNetPluginStateEnabled) {
|
||||
comm->ncclCollNet = netPluginLibs[pluginIndex].ncclCollNet;
|
||||
}
|
||||
}
|
||||
exit:
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
*isAssigned = false;
|
||||
netPluginLibs[pluginIndex].ncclNetPluginState = ncclNetPluginStateEnabled;
|
||||
netPluginLibs[pluginIndex].ncclCollNetPluginState = ncclNetPluginStateEnabled;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
static ncclResult_t ncclNetPluginDisableOtherExternal(int pluginIndex) {
|
||||
// Only if an external plugin is enabled, disable other external plugins
|
||||
if (pluginIndex >= (pluginCount - NCCL_NET_NUM_INTERNAL_PLUGINS)) return ncclSuccess;
|
||||
char names[MAX_STR_LEN*(NCCL_NET_MAX_PLUGINS - NCCL_NET_NUM_INTERNAL_PLUGINS)] = { 0 };
|
||||
for (int i = 0; i < (pluginCount - NCCL_NET_NUM_INTERNAL_PLUGINS); i++) {
|
||||
if (i != pluginIndex) {
|
||||
// Append all disabled plugin names to a string
|
||||
snprintf(names+strlen(names), sizeof(names)-strlen(names), (strlen(names) == 0) ? "%s" : ", %s", netPluginLibs[i].name);
|
||||
netPluginLibs[i].ncclNetPluginState = ncclNetPluginStateDisabled;
|
||||
}
|
||||
}
|
||||
if(strlen(names) > 0) {
|
||||
INFO(NCCL_INIT|NCCL_NET, "Disabling external plugins: %s", names);
|
||||
}
|
||||
*state = ncclNetStates[i];
|
||||
pthread_mutex_unlock(&netLock);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t collNetGetState(int i, enum ncclNetState* state) {
|
||||
pthread_mutex_lock(&netLock);
|
||||
if (ncclCollNetStates[i] == ncclNetStateInit) {
|
||||
int ndev;
|
||||
if (ncclCollNets[i]->init(ncclDebugLog) != ncclSuccess) ncclCollNetStates[i] = ncclNetStateDisabled;
|
||||
else if (ncclCollNets[i]->devices(&ndev) != ncclSuccess || ndev <= 0) ncclCollNetStates[i] = ncclNetStateDisabled;
|
||||
else ncclCollNetStates[i] = ncclNetStateEnabled;
|
||||
static void initPluginLibsOnceFunc() {
|
||||
char* netPluginName = nullptr;
|
||||
const char* defaultNetPlugin = "libnccl-net.so";
|
||||
const char* envNetPlugin = nullptr;
|
||||
char* envNetPluginList = nullptr;
|
||||
char* savePtr = nullptr;
|
||||
int pluginCounter = 0;
|
||||
|
||||
memset(netPluginLibs, 0, NCCL_NET_MAX_PLUGINS * sizeof(netPluginLib_t));
|
||||
envNetPlugin = ncclGetEnv("NCCL_NET_PLUGIN");
|
||||
if (envNetPlugin) {
|
||||
envNetPluginList = strdup(envNetPlugin);
|
||||
// Iterate over list until the list is empty
|
||||
netPluginName = strtok_r(envNetPluginList, ",", &savePtr);
|
||||
while(netPluginName) {
|
||||
// We have 2 internal plugins (ib and socket)
|
||||
// So, we can have at most( NCCL_NET_MAX_PLUGINS - (NCCL_NET_NUM_INTERNAL_PLUGINS)) in the NCCL_NET_PLUGIN list
|
||||
if (pluginCounter >= (NCCL_NET_MAX_PLUGINS - (NCCL_NET_NUM_INTERNAL_PLUGINS))) {
|
||||
INFO(NCCL_NET|NCCL_INIT,"NCCL_NET_PLUGIN list contains more than %d plugins, ignoring the rest", (NCCL_NET_MAX_PLUGINS - (NCCL_NET_NUM_INTERNAL_PLUGINS + 1)));
|
||||
break;
|
||||
}
|
||||
// need to leave space for the name + "\n"
|
||||
if((strlen(netPluginName)+1) <= MAX_STR_LEN) {
|
||||
netPluginLibs[pluginCounter].ncclNetPluginState = ncclNetPluginStateLoadReady;
|
||||
netPluginLibs[pluginCounter].ncclNetPluginRefCount = ncclParamNetPluginRefCount();
|
||||
strcpy(netPluginLibs[pluginCounter].name, netPluginName);
|
||||
pluginCounter++;
|
||||
} else {
|
||||
INFO(NCCL_NET|NCCL_INIT,"NCCL_NET_PLUGIN list contains a plugin name %s longer than %d characters, ignoring it.", netPluginName, MAX_STR_LEN);
|
||||
}
|
||||
netPluginName = strtok_r(nullptr, ",", &savePtr);
|
||||
}
|
||||
if (envNetPluginList) free(envNetPluginList);
|
||||
} else {
|
||||
// Add default net plugin
|
||||
netPluginLibs[pluginCounter].ncclNetPluginState = ncclNetPluginStateLoadReady;
|
||||
netPluginLibs[pluginCounter].ncclNetPluginRefCount = ncclParamNetPluginRefCount();
|
||||
strcpy(netPluginLibs[pluginCounter++].name, defaultNetPlugin);
|
||||
}
|
||||
*state = ncclCollNetStates[i];
|
||||
pthread_mutex_unlock(&netLock);
|
||||
return ncclSuccess;
|
||||
|
||||
// Add 2 internal ib and socket plugins
|
||||
netPluginLibs[pluginCounter].ncclNet = &ncclNetIb;
|
||||
netPluginLibs[pluginCounter++].ncclNetPluginState = ncclNetPluginStateInitReady;
|
||||
netPluginLibs[pluginCounter].ncclNet = &ncclNetSocket;
|
||||
netPluginLibs[pluginCounter++].ncclNetPluginState = ncclNetPluginStateInitReady;
|
||||
pluginCount = pluginCounter;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetInit(struct ncclComm* comm) {
|
||||
// Initialize main communication network
|
||||
const char* netName;
|
||||
bool ok = false;
|
||||
|
||||
netName = comm->config.netName;
|
||||
for (int i=0; i<3; i++) {
|
||||
if (ncclNets[i] == nullptr) continue;
|
||||
enum ncclNetState state;
|
||||
NCCLCHECK(netGetState(i, &state));
|
||||
if (state != ncclNetStateEnabled) continue;
|
||||
if (netName && strcasecmp(netName, ncclNets[i]->name) != 0) continue;
|
||||
if (ncclSuccess != ncclNetCheckDeviceVersion(comm, ncclNets[i], 0)) {
|
||||
// Mismatched device plugin version
|
||||
continue;
|
||||
bool ncclNetPluginInitialized = false;
|
||||
pthread_once(&initPluginLibsOnceControl, initPluginLibsOnceFunc);
|
||||
pthread_mutex_lock(&netPluginLock);
|
||||
for (int pluginIndex = 0; pluginIndex < pluginCount; pluginIndex++) {
|
||||
if ((pluginIndex < (pluginCount - NCCL_NET_NUM_INTERNAL_PLUGINS)) && (netPluginLibs[pluginIndex].ncclNetPluginState == ncclNetPluginStateLoadReady)) {
|
||||
NCCLCHECK(ncclNetPluginLoad(&netPluginLibs[pluginIndex]));
|
||||
}
|
||||
|
||||
comm->ncclNet = ncclNets[i];
|
||||
comm->ncclNetVer = ncclNetsVer[i];
|
||||
ok = true;
|
||||
|
||||
if (ncclCollNets[i]) {
|
||||
NCCLCHECK(collNetGetState(i, &state));
|
||||
if (state == ncclNetStateEnabled) {
|
||||
comm->ncclCollNet = ncclCollNets[i];
|
||||
if (netPluginLibs[pluginIndex].ncclNetPluginState == ncclNetPluginStateInitReady) {
|
||||
NCCLCHECK(ncclNetPluginInit(&netPluginLibs[pluginIndex]));
|
||||
}
|
||||
if (netPluginLibs[pluginIndex].ncclNetPluginState == ncclNetPluginStateEnabled) {
|
||||
bool isAssigned = false;
|
||||
NCCLCHECK(ncclNetPluginAssignToComm(comm, pluginIndex, &isAssigned));
|
||||
if (isAssigned) {
|
||||
// If one external plugin is assigned to a comm, then disable all other external plugins
|
||||
ncclNetPluginDisableOtherExternal(pluginIndex);
|
||||
ncclNetPluginInitialized = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
WARN("Error: network %s not found.", netName ? netName : "");
|
||||
return ncclInvalidUsage;
|
||||
}
|
||||
return ncclSuccess;
|
||||
pthread_mutex_unlock(&netPluginLock);
|
||||
if (ncclNetPluginInitialized) return ncclSuccess;
|
||||
WARN("Failed to initialize any NET plugin");
|
||||
return ncclInvalidUsage;
|
||||
}
|
||||
|
||||
ncclResult_t ncclNetFinalize(struct ncclComm* comm) {
|
||||
comm->ncclNet = nullptr;
|
||||
comm->ncclCollNet = nullptr;
|
||||
int pluginIndex = comm->netPluginIndex;
|
||||
pthread_mutex_lock(&netPluginLock);
|
||||
netPluginLibs[pluginIndex].ncclNetPluginRefCount--;
|
||||
for (int i = 0; i < (pluginCount - NCCL_NET_NUM_INTERNAL_PLUGINS); i++) {
|
||||
NCCLCHECK(ncclNetPluginUnload(&netPluginLibs[i]));
|
||||
}
|
||||
pthread_mutex_unlock(&netPluginLock);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
|
||||
Référencer dans un nouveau ticket
Bloquer un utilisateur