AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2.11.4-1

Browse Source 
Add new API for creating a reduction operation which multiplies the input by a rank-specific scalar before doing an inter-rank summation (see: ncclRedOpCreatePreMulSum).
Improve CollNet (SHARP) performance of ncclAllReduce when captured in a CUDA Graph via user buffer registration.
Add environment variable NCCL_NET_PLUGIN="<suffix>" to allow user to choose among multiple NCCL net plugins by substituting into "libnccl-net-<suffix>.so".
Fix memory leak of NVB connections.
Fix topology detection of IB Virtual Functions (SR-IOV).
This commit is contained in:
Ke Wen
2021-09-08 13:56:25 -07:00
parent 5f2f2f670f
commit e11238b302
41 changed files with 1532 additions and 599 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/init.cc
+69 -11
View File
@@ -67,13 +67,21 @@ ncclResult_t initCollNet(ncclCollNet_t* collnet) {
}
ncclResult_t initNetPlugin(ncclNet_t** net, ncclCollNet_t** collnet) {
void* netPluginLib = dlopen("libnccl-net.so", RTLD_NOW | RTLD_LOCAL);
char ncclNetPluginName[128];
const char* envPluginName = getenv("NCCL_NET_PLUGIN");
if (envPluginName && strlen(envPluginName)) {
snprintf(ncclNetPluginName, 128, "libnccl-net-%s.so", envPluginName);
INFO(NCCL_INIT, "Plugin name set by env to %s\n", ncclNetPluginName);
} else {
sprintf(ncclNetPluginName, "libnccl-net.so");
}
void* netPluginLib = dlopen(ncclNetPluginName, RTLD_NOW | RTLD_LOCAL);
if (netPluginLib == NULL) {
// dlopen does not guarantee to set errno, but dlerror only gives us a
// string, so checking errno doesn't hurt to try to provide a better
// error message
if (errno == ENOENT) {
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin : No plugin found (libnccl-net.so), using internal implementation");
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin : No plugin found (%s), using internal implementation", ncclNetPluginName);
} else {
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin : Plugin load returned %d : %s.", errno, dlerror());
}
@@ -185,6 +193,9 @@ void NCCL_NO_OPTIMIZE commPoison(ncclComm_t comm) {
static ncclResult_t commFree(ncclComm_t comm) {
if (comm == NULL)
return ncclSuccess;
delete[] comm->userRedOps;
free(comm->connectSend);
free(comm->connectRecv);
for (int peer=0; peer<comm->nRanks; peer++) {
@@ -216,8 +227,6 @@ static ncclResult_t commFree(ncclComm_t comm) {
CUDACHECK(cudaStreamDestroy(comm->groupStream));
}
ncclDestroyQueueInfo(comm->enqueueInfo);
// Last rank frees shared resources between threads
int isLast;
NCCLCHECK(ncclCpuBarrierIn(comm, &isLast));
@@ -238,6 +247,8 @@ static ncclResult_t commFree(ncclComm_t comm) {
}
NCCL_PARAM(AggChannelSize, "AGG_CHANNEL_SIZE", -2);
NCCL_PARAM(DisableGraphHelper, "GRAPH_HELPER_DISABLE", 0);
NCCL_PARAM(GraphRegister, "GRAPH_REGISTER", 0);
static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) {
if (ndev < 1) {
@@ -294,11 +305,20 @@ static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) {
comm->asyncAllocMode = ncclComm::ROUND_ROBIN;
}
CUDACHECK(cudaDriverGetVersion(&comm->driverVersion));
NCCLCHECK(ncclCreateQueueInfo(&comm->enqueueInfo, comm));
comm->lastSetupNode = NULL;
comm->lastCudaGraphId = -1;
CUDACHECK(cudaDriverGetVersion(&comm->driverVersion));
comm->disableGraphHelper = ncclParamDisableGraphHelper();
comm->graphRegister = ncclParamGraphRegister();
#if CUDART_VERSION >= 11030
NCCLCHECK(ncclCalloc(&comm->graphHelperResources, 1));
comm->graphHelperResources->comm = comm;
if (comm->driverVersion >= 11030)
// cudaGetDriverEntryPoint requires R465 or above (enhanced compat need)
CUDACHECK(cudaGetDriverEntryPoint("cuMemGetAddressRange", (void**)&comm->pfnCuMemGetAddressRange, cudaEnableDefault));
#endif
static_assert(MAXCHANNELS <= sizeof(*comm->connectSend)*8, "comm->connectSend must have enough bits for all channels");
static_assert(MAXCHANNELS <= sizeof(*comm->connectRecv)*8, "comm->connectRecv must have enough bits for all channels");
@@ -309,6 +329,10 @@ static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) {
NCCLCHECK(ncclCalloc(&comm->p2pSends, comm->nRanks));
NCCLCHECK(ncclCalloc(&comm->p2pRecvs, comm->nRanks));
// Create a map between global rank and intra-node rank
NCCLCHECK(ncclCalloc(&comm->rankToIntraNodeRank, comm->nRanks));
memset(comm->rankToIntraNodeRank, -1, comm->nRanks*sizeof(comm->rankToIntraNodeRank[0]));
// Mark channels as non initialized.
for (int c=0; c<MAXCHANNELS; c++) comm->channels[c].id = -1;
@@ -528,13 +552,14 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
int intraNodeRank0 = -1, intraNodeRank = -1, intraNodeRanks = 0;
int myCompCap = allGather1Data[rank].cudaCompCap;
int minCompCap = myCompCap, maxCompCap = myCompCap;
int intraNodeGlobalRanks[256];
for (int i = 0; i < nranks; i++) {
if (allGather1Data[i].peerInfo.hostHash == allGather1Data[rank].peerInfo.hostHash) {
// Rank is on same node
if (intraNodeRanks == 0) intraNodeRank0 = i;
if (i == rank) intraNodeRank = intraNodeRanks;
intraNodeGlobalRanks[intraNodeRanks++] = i;
comm->intraNodeGlobalRanks[intraNodeRanks] = i;
comm->rankToIntraNodeRank[i] = intraNodeRanks;
intraNodeRanks++;
if (allGather1Data[i].peerInfo.pidHash == allGather1Data[rank].peerInfo.pidHash) {
// Rank is in same process
if (intraProcRanks == 0) intraProcRank0 = i;
@@ -563,6 +588,7 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
}
struct ncclComm* intraProcRank0Comm = allGather1Data[intraProcRank0].comm;
uint64_t intraNodeRank0pidHash = allGather1Data[intraNodeRank0].peerInfo.pidHash;
comm->intraNodeRank = intraNodeRank;
free(allGather1Data);
@@ -792,6 +818,7 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
// Check if we can setup CollNet
if (comm->collNetSupport > 0) {
int collNetSetupFail = 0;
int highestTypes[NCCL_MAX_INTRA_RANKS] = {TRANSPORT_P2P};
// Find all head ranks
int nHeads = collNetGraph.nChannels;
int *heads;
@@ -817,16 +844,26 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm
TRACE(NCCL_INIT, "rank %d Connected inter-node CollNet", rank);
// Connect intra-node CollNet
int highestTransportType0, highestTransportType1;
for (int c=0; c<comm->nChannels; c++) {
struct ncclChannel* channelRecv = comm->channels+c;
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, channelRecv, NCCL_MAX_DIRECT_ARITY, channelRecv->collTree.up, NCCL_MAX_DIRECT_ARITY, channelRecv->collTree.down, 0), ret, collnet_cleanup);
}
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 0), ret, collnet_cleanup);
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 0, &highestTransportType0), ret, collnet_cleanup);
for (int c=0; c<comm->nChannels; c++) {
struct ncclChannel* channelSend = comm->channels+c;
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, channelSend, NCCL_MAX_DIRECT_ARITY, channelSend->collTree.down, NCCL_MAX_DIRECT_ARITY, channelSend->collTree.up, 1), ret, collnet_cleanup);
}
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 1), ret, collnet_cleanup);
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 1, &highestTransportType1), ret, collnet_cleanup);
// Exchange highest intra-node transport type among ranks
// because we need to know whether all ranks can p2p each other to determine whether we can directly read/write registered user buffer
comm->intraHighestTransportType = highestTypes[comm->intraNodeRank] = highestTransportType0 > highestTransportType1 ? highestTransportType0 : highestTransportType1;
NCCLCHECK(bootstrapIntraNodeAllGather(comm->bootstrap, comm->intraNodeGlobalRanks, comm->intraNodeRank, comm->localRanks, highestTypes, sizeof(int)));
for (int i=0; i<comm->localRanks; i++) {
if (highestTypes[i] > comm->intraHighestTransportType)
comm->intraHighestTransportType = highestTypes[i];
}
INFO(NCCL_INIT, "rank %d Connected CollNet", rank);
collnet_cleanup:
@@ -874,7 +911,7 @@ collnet_cleanup:
NCCLCHECK(ncclCommSetIntraProc(comm, intraProcRank, intraProcRanks, intraProcRank0Comm));
/* Local intra-node barrier */
NCCLCHECK(bootstrapBarrier(comm->bootstrap, intraNodeGlobalRanks, (int)intraNodeRank0pidHash, intraNodeRank, intraNodeRanks));
NCCLCHECK(bootstrapBarrier(comm->bootstrap, comm->intraNodeGlobalRanks, intraNodeRank, intraNodeRanks, (int)intraNodeRank0pidHash));
if (comm->nNodes) NCCLCHECK(ncclProxyCreate(comm));
@@ -974,6 +1011,22 @@ ncclResult_t ncclCommInitAll(ncclComm_t* comms, int ndev, const int* devlist) {
return ncclSuccess;
}
static ncclResult_t ncclGraphHelperDestroy(ncclComm* comm) {
auto res = comm->graphHelperResources;
if (comm->graphHelperThread && res) {
pthread_mutex_lock(&res->threadLock);
res->threadState = ThreadStop;
pthread_cond_signal(&res->threadCond);
pthread_mutex_unlock(&res->threadLock);
pthread_join(comm->graphHelperThread, NULL);
}
if (res) {
free(res);
res = NULL;
}
return ncclSuccess;
}
static ncclResult_t commDestroy(ncclComm_t comm) {
int savedDevice;
CUDACHECK(cudaGetDevice(&savedDevice));
@@ -987,6 +1040,11 @@ static ncclResult_t commDestroy(ncclComm_t comm) {
CUDACHECK(cudaStreamSynchronize(comm->groupStream));
NCCLCHECK(ncclProxyDestroy(comm));
ncclDestroyQueueInfo(comm->enqueueInfo);
#if CUDART_VERSION >= 11030
NCCLCHECK(ncclGraphHelperDestroy(comm));
#endif
INFO(NCCL_COLL, "Created %d queue info, destroyed %d", comm->nQueueInfoCreated, comm->nQueueInfoDestroyed);
NCCLCHECK(commFree(comm));
if (savedDevice != commDevice)