Merge remote-tracking branch 'remotes/nccl/master' into rccl_2.5.6
This commit is contained in:
Wenkai Du
@@ -1,10 +1,12 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "argcheck.h"
|
||||
#include "comm.h"
|
||||
|
||||
static ncclResult_t CudaPtrCheck(const void* pointer, struct ncclComm* comm, const char* ptrname, const char* opname) {
|
||||
hipPointerAttribute_t attr;
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2015-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "group.h"
|
||||
#include "debug.h"
|
||||
#include "enqueue.h"
|
||||
|
||||
#define MAX_ASYNC_OPS 128
|
||||
thread_local pthread_t ncclGroupThreads[MAX_ASYNC_OPS];
|
||||
thread_local int ncclGroupIndex = 0;
|
||||
thread_local int ncclGroupMode = 0;
|
||||
thread_local ncclResult_t ncclGroupError = ncclSuccess;
|
||||
|
||||
bool ncclAsyncMode() {
|
||||
return ncclGroupMode > 0;
|
||||
}
|
||||
|
||||
ncclResult_t ncclAsyncErrCheck(ncclResult_t ret) {
|
||||
if (ncclGroupError == ncclSuccess || ret != ncclSuccess) ncclGroupError = ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
struct ncclInitArgs {
|
||||
ncclInitFunc_t func;
|
||||
int cudaDev;
|
||||
ncclComm_t* newcomm;
|
||||
int ndev;
|
||||
ncclUniqueId commId;
|
||||
int myrank;
|
||||
};
|
||||
struct ncclCollArgs {
|
||||
ncclComm_t comm;
|
||||
};
|
||||
|
||||
enum ncclAsyncFuncType {
|
||||
ASYNC_FUNC_INVALID = 0,
|
||||
ASYNC_FUNC_INIT = 1,
|
||||
ASYNC_FUNC_COLL = 2,
|
||||
};
|
||||
struct ncclAsyncArgs {
|
||||
ncclResult_t ret;
|
||||
enum ncclAsyncFuncType funcType;
|
||||
union {
|
||||
ncclCollArgs coll;
|
||||
ncclInitArgs init;
|
||||
};
|
||||
};
|
||||
|
||||
thread_local struct ncclAsyncArgs ncclGroupArgs[MAX_ASYNC_OPS];
|
||||
|
||||
ncclResult_t ncclSetDevice(int cudaDev) {
|
||||
CUDACHECK(hipSetDevice(cudaDev));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
#define CHECK(a) do { \
|
||||
if ((args->ret = (a)) != ncclSuccess) { \
|
||||
INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, args->ret); \
|
||||
return args; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
void* ncclAsyncThreadMain(void* args_) {
|
||||
struct ncclAsyncArgs* args = (struct ncclAsyncArgs*)args_;
|
||||
CHECK(ncclSetDevice(args->init.cudaDev));
|
||||
CHECK(args->init.func(args->init.newcomm, args->init.ndev, args->init.commId, args->init.myrank));
|
||||
return args;
|
||||
}
|
||||
|
||||
ncclResult_t ncclAsyncInit(ncclInitFunc_t func, int cudaDev, ncclComm_t* newcomm, int ndev, ncclUniqueId commId, int myrank) {
|
||||
if (ncclGroupIndex >= MAX_ASYNC_OPS) {
|
||||
WARN("Too many async operations in progress, max is %d", MAX_ASYNC_OPS);
|
||||
return ncclAsyncErrCheck(ncclInternalError);
|
||||
}
|
||||
int index = ncclGroupIndex++;
|
||||
struct ncclAsyncArgs* args = ncclGroupArgs+index;
|
||||
args->funcType = ASYNC_FUNC_INIT;
|
||||
args->init.func = func;
|
||||
args->init.cudaDev = cudaDev;
|
||||
args->init.newcomm = newcomm;
|
||||
args->init.ndev = ndev;
|
||||
memcpy(&args->init.commId, &commId, sizeof(commId));
|
||||
args->init.myrank = myrank;
|
||||
// We need to use threads for Init
|
||||
pthread_create(ncclGroupThreads+index, NULL, ncclAsyncThreadMain, args);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclAsyncColl(ncclComm_t comm) {
|
||||
struct ncclAsyncArgs* args = ncclGroupArgs;
|
||||
for (int i=0; i<ncclGroupIndex; i++) {
|
||||
if (args->coll.comm == comm) return ncclSuccess;
|
||||
args++;
|
||||
}
|
||||
if (ncclGroupIndex >= MAX_ASYNC_OPS) {
|
||||
WARN("Too many async operations in progress, max is %d", MAX_ASYNC_OPS);
|
||||
return ncclAsyncErrCheck(ncclInternalError);
|
||||
}
|
||||
ncclGroupIndex++;
|
||||
args->funcType = ASYNC_FUNC_COLL;
|
||||
args->coll.comm = comm;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
NCCL_API(ncclResult_t, ncclGroupStart);
|
||||
ncclResult_t ncclGroupStart() {
|
||||
ncclGroupMode++;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
NCCL_API(ncclResult_t, ncclGroupEnd);
|
||||
ncclResult_t ncclGroupEnd() {
|
||||
ncclGroupMode--;
|
||||
if (ncclGroupMode > 0) return ncclSuccess;
|
||||
int savedDev;
|
||||
CUDACHECK(hipGetDevice(&savedDev));
|
||||
int done = ncclGroupIndex;
|
||||
int doneArray[MAX_ASYNC_OPS];
|
||||
for (int i=0; i<ncclGroupIndex; i++) doneArray[i] = 0;
|
||||
|
||||
ncclResult_t ret = ncclGroupError;
|
||||
if (ret != ncclSuccess) goto group_cleanup;
|
||||
|
||||
/* Collectives are done in three steps :
|
||||
* 1. Barrier Check In. Only the last call may call cudaLaunchKernel[cooperative]
|
||||
* 2. Barrier Wait. No CUDA call is permitted
|
||||
* 3. Enqueue Events. CUDA event wait/enqueue.
|
||||
* This is needed because step 2 cannot call any CUDA primitive, otherwise if
|
||||
* hipFree happens between 1 and 3, it could block that CUDA call and
|
||||
* prevent some ranks from launching their network threads, which would
|
||||
* prevent the NCCL call from completing, blocking the hipFree call.
|
||||
*/
|
||||
for (int i=0; i<ncclGroupIndex; i++) {
|
||||
struct ncclAsyncArgs* args = ncclGroupArgs+i;
|
||||
if (args->funcType == ASYNC_FUNC_COLL) {
|
||||
if (args->coll.comm->userStream == NULL)
|
||||
CUDACHECKGOTO(hipSetDevice(args->coll.comm->cudaDev), ret, end);
|
||||
NCCLCHECKGOTO(ncclBarrierEnqueue(args->coll.comm), ret, end);
|
||||
}
|
||||
}
|
||||
for (int i=0; i<ncclGroupIndex; i++) {
|
||||
struct ncclAsyncArgs* args = ncclGroupArgs+i;
|
||||
if (args->funcType == ASYNC_FUNC_COLL) {
|
||||
CUDACHECKGOTO(hipSetDevice(args->coll.comm->cudaDev), ret, end);
|
||||
NCCLCHECKGOTO(ncclBarrierEnqueueWait(args->coll.comm), ret, end);
|
||||
}
|
||||
}
|
||||
for (int i=0; i<ncclGroupIndex; i++) {
|
||||
struct ncclAsyncArgs* args = ncclGroupArgs+i;
|
||||
if (args->funcType == ASYNC_FUNC_COLL) {
|
||||
if (args->coll.comm->userStream == NULL)
|
||||
CUDACHECKGOTO(hipSetDevice(args->coll.comm->cudaDev), ret, end);
|
||||
NCCLCHECKGOTO(ncclEnqueueEvents(args->coll.comm), ret, end);
|
||||
doneArray[i] = 1;
|
||||
done--;
|
||||
}
|
||||
}
|
||||
|
||||
/* For init, since we use threads, we just wait for threads to complete */
|
||||
while (done) {
|
||||
for (int i=0; i<ncclGroupIndex; i++) {
|
||||
struct ncclAsyncArgs* args = ncclGroupArgs+i;
|
||||
if (args->funcType == ASYNC_FUNC_INIT && doneArray[i] == 0) {
|
||||
int err = pthread_tryjoin_np(ncclGroupThreads[i], NULL);
|
||||
if (err == EBUSY) continue;
|
||||
if (err != 0) { ret = ncclSystemError; goto end; }
|
||||
if (args->ret != ncclSuccess) { ret = args->ret; goto end; }
|
||||
doneArray[i] = 1;
|
||||
done--;
|
||||
}
|
||||
}
|
||||
}
|
||||
goto end;
|
||||
group_cleanup:
|
||||
// At least one call in the group failed. Since we want to make that group
|
||||
// an atomic operation, we need to cancel all operations.
|
||||
for (int i=0; i<ncclGroupIndex; i++) {
|
||||
struct ncclComm* comm = ncclGroupArgs[i].coll.comm;
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
for (int i=0; i<channel->collCount; i++) {
|
||||
STORE(&channel->collectives[(channel->collStart + i)%NCCL_MAX_OPS].active, 0);
|
||||
}
|
||||
channel->collFifoTail = channel->collStart;
|
||||
channel->collCount = 0;
|
||||
}
|
||||
comm->myParams->gridDim.x = comm->myParams->blockDim.x = 0;
|
||||
comm->userStreamSet = false;
|
||||
}
|
||||
end:
|
||||
ncclGroupError = ncclSuccess;
|
||||
ncclGroupIndex = 0;
|
||||
CUDACHECK(hipSetDevice(savedDev)); // do other clean-ups first before calling hipSetDevice, because this call can fail too
|
||||
return ret;
|
||||
}
|
||||
@@ -16,6 +16,7 @@ static nvmlReturn_t (*nvmlInternalInit)(void);
|
||||
static nvmlReturn_t (*nvmlInternalShutdown)(void);
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetHandleByPciBusId)(const char* pciBusId, nvmlDevice_t* device);
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetIndex)(nvmlDevice_t device, unsigned* index);
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetHandleByIndex)(unsigned int index, nvmlDevice_t* device);
|
||||
static const char* (*nvmlInternalErrorString)(nvmlReturn_t r);
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetNvLinkState)(nvmlDevice_t device, unsigned int link, nvmlEnableState_t *isActive);
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetPciInfo)(nvmlDevice_t device, nvmlPciInfo_t* pci);
|
||||
@@ -23,7 +24,10 @@ static nvmlReturn_t (*nvmlInternalDeviceGetNvLinkRemotePciInfo)(nvmlDevice_t dev
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetNvLinkCapability)(nvmlDevice_t device, unsigned int link,
|
||||
nvmlNvLinkCapability_t capability, unsigned int *capResult);
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetMinorNumber)(nvmlDevice_t device, unsigned int* minorNumber);
|
||||
static nvmlReturn_t (*nvmlInternalDeviceGetCudaComputeCapability)(nvmlDevice_t device, int* major, int* minor);
|
||||
|
||||
// Used to make the NVML library calls thread safe
|
||||
pthread_mutex_t nvmlLock = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
ncclResult_t wrapNvmlSymbols(void) {
|
||||
if (nvmlState == nvmlInitialized)
|
||||
@@ -70,12 +74,14 @@ ncclResult_t wrapNvmlSymbols(void) {
|
||||
LOAD_SYM(nvmlhandle, "nvmlShutdown", nvmlInternalShutdown);
|
||||
LOAD_SYM(nvmlhandle, "nvmlDeviceGetHandleByPciBusId", nvmlInternalDeviceGetHandleByPciBusId);
|
||||
LOAD_SYM(nvmlhandle, "nvmlDeviceGetIndex", nvmlInternalDeviceGetIndex);
|
||||
LOAD_SYM(nvmlhandle, "nvmlDeviceGetHandleByIndex", nvmlInternalDeviceGetHandleByIndex);
|
||||
LOAD_SYM(nvmlhandle, "nvmlErrorString", nvmlInternalErrorString);
|
||||
LOAD_SYM(nvmlhandle, "nvmlDeviceGetPciInfo", nvmlInternalDeviceGetPciInfo);
|
||||
LOAD_SYM(nvmlhandle, "nvmlDeviceGetMinorNumber", nvmlInternalDeviceGetMinorNumber);
|
||||
LOAD_SYM_OPTIONAL(nvmlhandle, "nvmlDeviceGetNvLinkState", nvmlInternalDeviceGetNvLinkState);
|
||||
LOAD_SYM_OPTIONAL(nvmlhandle, "nvmlDeviceGetNvLinkRemotePciInfo", nvmlInternalDeviceGetNvLinkRemotePciInfo);
|
||||
LOAD_SYM_OPTIONAL(nvmlhandle, "nvmlDeviceGetNvLinkCapability", nvmlInternalDeviceGetNvLinkCapability);
|
||||
LOAD_SYM(nvmlhandle, "nvmlDeviceGetCudaComputeCapability", nvmlInternalDeviceGetCudaComputeCapability);
|
||||
|
||||
nvmlState = nvmlInitialized;
|
||||
return ncclSuccess;
|
||||
@@ -85,6 +91,7 @@ teardown:
|
||||
nvmlInternalShutdown = NULL;
|
||||
nvmlInternalDeviceGetHandleByPciBusId = NULL;
|
||||
nvmlInternalDeviceGetIndex = NULL;
|
||||
nvmlInternalDeviceGetHandleByIndex = NULL;
|
||||
nvmlInternalDeviceGetPciInfo = NULL;
|
||||
nvmlInternalDeviceGetMinorNumber = NULL;
|
||||
nvmlInternalDeviceGetNvLinkState = NULL;
|
||||
@@ -130,7 +137,8 @@ ncclResult_t wrapNvmlDeviceGetHandleByPciBusId(const char* pciBusId, nvmlDevice_
|
||||
WARN("lib wrapper not initialized.");
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret = nvmlInternalDeviceGetHandleByPciBusId(pciBusId, device);
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetHandleByPciBusId(pciBusId, device), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
WARN("nvmlDeviceGetHandleByPciBusId() failed: %s ",
|
||||
nvmlInternalErrorString(ret));
|
||||
@@ -144,7 +152,8 @@ ncclResult_t wrapNvmlDeviceGetIndex(nvmlDevice_t device, unsigned* index) {
|
||||
WARN("lib wrapper not initialized.");
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret = nvmlInternalDeviceGetIndex(device, index);
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetIndex(device, index), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
WARN("nvmlDeviceGetIndex() failed: %s ",
|
||||
nvmlInternalErrorString(ret));
|
||||
@@ -153,12 +162,28 @@ ncclResult_t wrapNvmlDeviceGetIndex(nvmlDevice_t device, unsigned* index) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t wrapNvmlDeviceGetHandleByIndex(unsigned int index, nvmlDevice_t* device) {
|
||||
if (nvmlInternalDeviceGetHandleByIndex == NULL) {
|
||||
WARN("lib wrapper not initialized.");
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetHandleByIndex(index, device), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
WARN("nvmlDeviceGetHandleByIndex() failed: %s ",
|
||||
nvmlInternalErrorString(ret));
|
||||
return ncclSystemError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t wrapNvmlDeviceGetPciInfo(nvmlDevice_t device, nvmlPciInfo_t* pci) {
|
||||
if (nvmlInternalDeviceGetPciInfo == NULL) {
|
||||
WARN("lib wrapper not initialized.");
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret = nvmlInternalDeviceGetPciInfo(device, pci);
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetPciInfo(device, pci), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
WARN("nvmlDeviceGetPciInfo() failed: %s ",
|
||||
nvmlInternalErrorString(ret));
|
||||
@@ -172,7 +197,8 @@ ncclResult_t wrapNvmlDeviceGetMinorNumber(nvmlDevice_t device, unsigned int* min
|
||||
WARN("lib wrapper not initialized.");
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret = nvmlInternalDeviceGetMinorNumber(device, minorNumber);
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetMinorNumber(device, minorNumber), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
WARN("nvmlDeviceGetMinorNumber() failed: %s ",
|
||||
nvmlInternalErrorString(ret));
|
||||
@@ -186,7 +212,8 @@ ncclResult_t wrapNvmlDeviceGetNvLinkState(nvmlDevice_t device, unsigned int link
|
||||
/* Do not warn, this symbol is optional. */
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret = nvmlInternalDeviceGetNvLinkState(device, link, isActive);
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetNvLinkState(device, link, isActive), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
if (ret != NVML_ERROR_NOT_SUPPORTED)
|
||||
INFO(NCCL_INIT,"nvmlDeviceGetNvLinkState() failed: %s ",
|
||||
@@ -201,7 +228,8 @@ ncclResult_t wrapNvmlDeviceGetNvLinkRemotePciInfo(nvmlDevice_t device, unsigned
|
||||
/* Do not warn, this symbol is optional. */
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret = nvmlInternalDeviceGetNvLinkRemotePciInfo(device, link, pci);
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetNvLinkRemotePciInfo(device, link, pci), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
if (ret != NVML_ERROR_NOT_SUPPORTED)
|
||||
INFO(NCCL_INIT,"nvmlDeviceGetNvLinkRemotePciInfo() failed: %s ",
|
||||
@@ -217,7 +245,8 @@ ncclResult_t wrapNvmlDeviceGetNvLinkCapability(nvmlDevice_t device, unsigned int
|
||||
/* Do not warn, this symbol is optional. */
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret = nvmlInternalDeviceGetNvLinkCapability(device, link, capability, capResult);
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetNvLinkCapability(device, link, capability, capResult), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
if (ret != NVML_ERROR_NOT_SUPPORTED)
|
||||
INFO(NCCL_INIT,"nvmlDeviceGetNvLinkCapability() failed: %s ",
|
||||
@@ -226,4 +255,19 @@ ncclResult_t wrapNvmlDeviceGetNvLinkCapability(nvmlDevice_t device, unsigned int
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t wrapNvmlDeviceGetCudaComputeCapability(nvmlDevice_t device, int* major, int* minor) {
|
||||
if (nvmlInternalDeviceGetNvLinkCapability == NULL) {
|
||||
WARN("lib wrapper not initialized.");
|
||||
return ncclInternalError;
|
||||
}
|
||||
nvmlReturn_t ret;
|
||||
NVMLLOCKCALL(nvmlInternalDeviceGetCudaComputeCapability(device, major, minor), ret);
|
||||
if (ret != NVML_SUCCESS) {
|
||||
WARN("nvmlDeviceGetCudaComputeCapability() failed: %s ",
|
||||
nvmlInternalErrorString(ret));
|
||||
return ncclSystemError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -24,6 +24,7 @@ ncclResult_t wrapNvmlDeviceGetHandleByPciBusId(const char* pciBusId, nvmlDevice_
|
||||
}
|
||||
|
||||
ncclResult_t wrapNvmlDeviceGetIndex(nvmlDevice_t device, unsigned* index) {
|
||||
*index = 0;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -32,6 +33,7 @@ ncclResult_t wrapNvmlDeviceGetPciInfo(nvmlDevice_t device, nvmlPciInfo_t* pci) {
|
||||
}
|
||||
|
||||
ncclResult_t wrapNvmlDeviceGetMinorNumber(nvmlDevice_t device, unsigned int* minorNumber) {
|
||||
*minorNumber = 0;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -45,5 +47,11 @@ ncclResult_t wrapNvmlDeviceGetNvLinkRemotePciInfo(nvmlDevice_t device, unsigned
|
||||
|
||||
ncclResult_t wrapNvmlDeviceGetNvLinkCapability(nvmlDevice_t device, unsigned int link,
|
||||
nvmlNvLinkCapability_t capability, unsigned int *capResult) {
|
||||
*capResult = 0;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t wrapNvmlDeviceGetCudaComputeCapability(nvmlDevice_t device, int* major, int* minor) {
|
||||
*major = *minor = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -1,398 +0,0 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "param.h"
|
||||
|
||||
#define NCCL_MAX_SCORE 7
|
||||
|
||||
/* Parse user defined rings. Format is like :
|
||||
* "0 1|1 0|0 1 2 3|3 2 1 0|0 2 3 1|1 3 2 0|0 1 2 3 4 5 6 7|7 6 5 4 3 2 1 0"
|
||||
* Rings with a non-matching number of ranks are ignored so we can provide
|
||||
* rings for multiple cases.
|
||||
*/
|
||||
#define MAX_ENV_RANKS 512
|
||||
static ncclResult_t parseRings(const char* str, int* nringsRet, int nranks, int* prev, int* next) {
|
||||
int ranks[MAX_ENV_RANKS];
|
||||
int nrings = 0;
|
||||
int rank = 0;
|
||||
int offset = 0;
|
||||
int status = 0; // 0 : between numbers, 1 : inside number
|
||||
do {
|
||||
int digit = str[offset] - '0';
|
||||
if (digit >= 0 && digit <= 9) {
|
||||
if (status == 0) {
|
||||
ranks[rank] = digit;
|
||||
status = 1;
|
||||
} else {
|
||||
ranks[rank] = ranks[rank]*10+digit;
|
||||
}
|
||||
} else {
|
||||
if (status == 1) {
|
||||
rank++;
|
||||
if (rank == MAX_ENV_RANKS) goto end;
|
||||
}
|
||||
status = 0;
|
||||
if (str[offset] == '|' || str[offset] == '\0') {
|
||||
int prevRank = ranks[rank-1];
|
||||
// Ignore rings if nranks doesn't match
|
||||
if (rank != nranks) goto newring;
|
||||
|
||||
for (int r=0; r<nranks; r++) {
|
||||
int rank = ranks[r];
|
||||
// Ignore rings with ranks out of bounds
|
||||
if (rank < 0 || rank >= nranks) goto newring;
|
||||
// Ignore rings with duplicate ranks
|
||||
for (int i=0; i<r; i++)
|
||||
if (ranks[i] == rank) goto newring;
|
||||
|
||||
next[nrings*nranks+prevRank] = rank;
|
||||
prev[nrings*nranks+rank] = prevRank;
|
||||
prevRank = rank;
|
||||
}
|
||||
nrings++;
|
||||
newring:
|
||||
rank = 0;
|
||||
}
|
||||
}
|
||||
} while (str[offset++] != 0);
|
||||
end:
|
||||
*nringsRet = nrings;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/*
|
||||
* Ring creation algorithm
|
||||
*
|
||||
* First, we establish hierarchical coordinates depending on the way ranks can
|
||||
* communicate. After fillCoords, we have for each rank a unique 3-int array
|
||||
* { node, pci_domain, rank } corresponding to the three transports :
|
||||
* { 2[NET], 1[SHM], 0[P2P] }.
|
||||
* Also, we renumber ranks (to indexes) based on their growing coordinates.
|
||||
*
|
||||
* Then, we ask transports to connect groups together. We start with net, then
|
||||
* shm, then p2p. We maintain two arrays, prev and next, where values are equal
|
||||
* to -1 when ranks are not yet connected, and a rank otherwise. We never
|
||||
* connect ranks outside our group, meaning that on 4 nodes of 2 sockets of 4
|
||||
* ranks, if we are rank 13, we should see something like (provided we have a
|
||||
* single net interface, hence a single ring) :
|
||||
*
|
||||
* Connecting all nodes <13>
|
||||
* 2[NET] : prev 31 -1 -1 -1 -1 -1 -1 -1 7 -1 -1 -1 -1 -1 -1 -1 15 -1 -1 -1 -1 -1 -1 -1 23 -1 -1 -1 -1 -1 -1 -1
|
||||
* next -1 -1 -1 -1 -1 -1 -1 8 -1 -1 -1 -1 -1 -1 -1 16 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 0
|
||||
*
|
||||
* Connecting P2P domains with shared memory <13>
|
||||
* 1[SHM] : prev 31 -1 -1 -1 -1 -1 -1 -1 7 -1 -1 -1 11 -1 -1 -1 15 -1 -1 -1 -1 -1 -1 -1 23 -1 -1 -1 -1 -1 -1 -1
|
||||
* next -1 -1 -1 -1 -1 -1 -1 8 -1 -1 -1 12 -1 -1 -1 16 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 0
|
||||
*
|
||||
* Connecting ranks (only inside the P2P domain) <13>
|
||||
* 0[P2P] : prev 31 -1 -1 -1 -1 -1 -1 -1 7 -1 -1 -1 11 12 13 14 15 -1 -1 -1 -1 -1 -1 -1 23 -1 -1 -1 -1 -1 -1 -1
|
||||
* next -1 -1 -1 -1 -1 -1 -1 8 -1 -1 -1 12 13 14 15 16 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 0
|
||||
*
|
||||
* Hence, when we ask a transport to connect groups, we provide it with a subview of the ranks (except for net
|
||||
* which always sees the full world). That way, P2P can bruteforce all combinations inside the node without
|
||||
* risking to explode in terms of combinations, and we scale better.
|
||||
*
|
||||
* Finally, we loop over Network scores to try to create rings with high scores (=locality) and decrease until
|
||||
* we get at least one ring.
|
||||
*/
|
||||
|
||||
static void recIsConnected(int rank, int* connected, int nranks, int* matrix, int transport) {
|
||||
connected[rank] = 1;
|
||||
for (int r=0; r<nranks; r++) {
|
||||
if (connected[r] == 0 && matrix[rank*nranks+r] == transport) {
|
||||
recIsConnected(r, connected, nranks, matrix, transport);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void isConnected(int rank, int* connected, int nranks, int* matrix, int transport) {
|
||||
for (int r=0; r<nranks; r++) connected[r] = 0;
|
||||
recIsConnected(rank, connected, nranks, matrix, transport);
|
||||
}
|
||||
|
||||
#define NEW_IDX(rank) do { \
|
||||
rankToIdx[rank] = idx; \
|
||||
idxToRank[idx] = rank; \
|
||||
for (int t=0; t<NTRANSPORTS; t++) coords[rank*NTRANSPORTS+t] = current[t]; \
|
||||
idx++; \
|
||||
} while (0)
|
||||
|
||||
int findConnected(int rank, int* matrix, int nranks, int transport, int* coords) {
|
||||
for (int r=0; r<nranks; r++) {
|
||||
if (coords[r*NTRANSPORTS] == -1 && matrix[rank*nranks+r] == transport) return r;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static ncclResult_t fillCoords(int nranks, int* matrix, int* coords, int* rankToIdx, int* idxToRank) {
|
||||
int current[NTRANSPORTS];
|
||||
int* p2pConnected;
|
||||
NCCLCHECK(ncclCalloc(&p2pConnected, nranks));
|
||||
for (int i=0; i<NTRANSPORTS; i++) current[i] = 0;
|
||||
int curRank = 0, idx = 0;
|
||||
while (1) {
|
||||
// P2P is handled separately as there is no level below it and we need to
|
||||
// cover the case of being connected to another GPU indirectly.
|
||||
// So we detect all GPUs in the same P2P domain once and add them all at
|
||||
// once.
|
||||
isConnected(curRank, p2pConnected, nranks, matrix, 0);
|
||||
for (int r=0; r<nranks; r++) {
|
||||
if (p2pConnected[r]) {
|
||||
NEW_IDX(r);
|
||||
curRank = r;
|
||||
current[0]++;
|
||||
}
|
||||
}
|
||||
current[0] = 0;
|
||||
|
||||
if (idx == nranks) {
|
||||
free(p2pConnected);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Find next group, either connected through SHM or NET.
|
||||
int rank;
|
||||
int transport = 1;
|
||||
while ((rank = findConnected(curRank, matrix, nranks, transport, coords)) == -1) {
|
||||
current[transport] = 0;
|
||||
transport++;
|
||||
if (transport == NTRANSPORTS) {
|
||||
WARN("Error : Could not find transport to connect next group\n");
|
||||
free(p2pConnected);
|
||||
return ncclInternalError; }
|
||||
}
|
||||
curRank = rank;
|
||||
current[transport]++;
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
|
||||
#define DEFAULT_MIN_NRINGS 2
|
||||
#elif defined(__PPC__)
|
||||
// Make the default NCCL_MIN_NRINGS=4 for IBM/Power nodes
|
||||
#define DEFAULT_MIN_NRINGS 4
|
||||
#else
|
||||
#define DEFAULT_MIN_NRINGS 0
|
||||
#endif
|
||||
NCCL_PARAM(MinNrings, "MIN_NRINGS", DEFAULT_MIN_NRINGS);
|
||||
NCCL_PARAM(MaxNrings, "MAX_NRINGS", 0);
|
||||
|
||||
/* Users can force the number of threads with an environment variable */
|
||||
NCCL_PARAM(Nthreads, "NTHREADS", -2);
|
||||
ncclResult_t getEnvThreads(int* nthreads) {
|
||||
int64_t nt = ncclParamNthreads();
|
||||
if (nt != -2)
|
||||
*nthreads = nt;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static inline int copyRings(int nrings, int newNrings, int nranks, int* a, int* b, int* c, int* d) {
|
||||
if (newNrings > MAXCHANNELS) newNrings = MAXCHANNELS;
|
||||
for (int r=nrings; r<newNrings; r++) {
|
||||
for (int i=0; i<nranks; i++) {
|
||||
a[r*nranks+i] = a[(r-nrings)*nranks+i];
|
||||
b[r*nranks+i] = b[(r-nrings)*nranks+i];
|
||||
c[r*nranks+i] = c[(r-nrings)*nranks+i];
|
||||
d[r*nranks+i] = d[(r-nrings)*nranks+i];
|
||||
}
|
||||
}
|
||||
return newNrings;
|
||||
}
|
||||
/* Main ring creation function */
|
||||
ncclResult_t ncclGetRings(int* nrings, int* nthreads, int rank, int nranks, int* transports, ncclTvalue_t* values, int* prev, int* next, int* treeIn, int* treeOut) {
|
||||
*nrings = 0;
|
||||
|
||||
if (nranks == 1) return ncclSuccess;
|
||||
|
||||
char* str = getenv("NCCL_RINGS");
|
||||
if (str && strlen(str)>0) {
|
||||
int ret = parseRings(str, nrings, nranks, prev, next);
|
||||
if (ret == ncclSuccess && *nrings > 0) {
|
||||
if (rank == 0) INFO(NCCL_INIT,"%d ring(s) set by environment", *nrings);
|
||||
NCCLCHECK(getEnvThreads(nthreads));
|
||||
for (int r = 0; r<*nrings; r++) {
|
||||
for (int i = 0; i<nranks; i++) {
|
||||
if (transports[i*nranks+prev[r*nranks+i]] == 2) treeIn[r*nranks+i] = 1;
|
||||
if (transports[i*nranks+next[r*nranks+i]] == 2) treeOut[r*nranks+i] = 1;
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
if (rank == 0) INFO(NCCL_INIT,"No valid ring found in environment, ignoring");
|
||||
*nrings = 0;
|
||||
}
|
||||
|
||||
// Compute hierarchical topology groups, indexes, and rank<->index tables
|
||||
int* coords, *globalIdxToRank, *globalRankToIdx;
|
||||
NCCLCHECK(ncclCalloc(&coords, nranks*NTRANSPORTS));
|
||||
for (int i=0; i<nranks*NTRANSPORTS; i++) coords[i] = -1;
|
||||
NCCLCHECK(ncclCalloc(&globalIdxToRank, nranks));
|
||||
NCCLCHECK(ncclCalloc(&globalRankToIdx, nranks));
|
||||
|
||||
NCCLCHECK(fillCoords(nranks, transports, coords, globalRankToIdx, globalIdxToRank));
|
||||
|
||||
// Start with a high score, then decrease until we find rings
|
||||
int minScore = NCCL_MAX_SCORE;
|
||||
int nringsTmp;
|
||||
int *prevTmp, *nextTmp, *idxToRank, *rankToIdx, *groups, *subgroups;
|
||||
NCCLCHECK(ncclCalloc(&prevTmp, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&nextTmp, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclCalloc(&idxToRank, nranks));
|
||||
NCCLCHECK(ncclCalloc(&rankToIdx, nranks));
|
||||
NCCLCHECK(ncclCalloc(&groups, nranks));
|
||||
NCCLCHECK(ncclCalloc(&subgroups, nranks));
|
||||
|
||||
int nThreads;
|
||||
do {
|
||||
nThreads = *nthreads;
|
||||
for (int i=0; i<nranks*MAXCHANNELS; i++) prevTmp[i] = nextTmp[i] = -1;
|
||||
nringsTmp = MAXCHANNELS;
|
||||
// Loop over transports to connect groups
|
||||
for (int t=NTRANSPORTS-1; t>=0; t--) {
|
||||
for (int i=0; i<nranks; i++) idxToRank[i] = rankToIdx[i] = -1;
|
||||
|
||||
int nidx = 0;
|
||||
for (int i=0; i<nranks; i++) {
|
||||
// Extract only ranks in the same local area as rank
|
||||
// We need to extract them in the topological order, hence we iterate over indexes, not ranks
|
||||
int r = globalIdxToRank[i];
|
||||
int sameLocal = 1;
|
||||
for (int tr = NTRANSPORTS-1; tr > t; tr--) if (coords[r*NTRANSPORTS+tr] != coords[rank*NTRANSPORTS+tr]) sameLocal = 0;
|
||||
if (!sameLocal) continue;
|
||||
|
||||
groups[nidx] = coords[r*NTRANSPORTS+t];
|
||||
subgroups[nidx] = t ? coords[r*NTRANSPORTS+t-1] : nidx;
|
||||
rankToIdx[r] = nidx;
|
||||
idxToRank[nidx] = r;
|
||||
nidx++;
|
||||
}
|
||||
|
||||
int ngroups = groups[nidx-1] + 1; // Coords should be ordered
|
||||
|
||||
ncclTvalue_t* subvalues;
|
||||
int *subprev, *subnext;
|
||||
NCCLCHECK(ncclCalloc(&subvalues, nidx*nidx));
|
||||
NCCLCHECK(ncclCalloc(&subprev, nidx*nringsTmp));
|
||||
NCCLCHECK(ncclCalloc(&subnext, nidx*nringsTmp));
|
||||
if (ngroups > 1) {
|
||||
/* Extract subvalues */
|
||||
for (int i=0; i<nidx; i++) {
|
||||
for (int j=0; j<nidx; j++) {
|
||||
if (transports[idxToRank[i]*nranks+idxToRank[j]] == t)
|
||||
subvalues[i*nidx+j] = values[idxToRank[i]*nranks+idxToRank[j]];
|
||||
else
|
||||
subvalues[i*nidx+j] = 0;
|
||||
}
|
||||
}
|
||||
/* Extract subprev/subnext */
|
||||
for (int i=0; i<nidx*nringsTmp; i++) {
|
||||
subprev[i] = subnext[i] = -1;
|
||||
}
|
||||
for (int r=0; r<nringsTmp; r++) {
|
||||
int start = -1, end = -1;
|
||||
for (int i=0; i<nranks; i++) {
|
||||
if (rankToIdx[i] == -1) continue;
|
||||
if (prevTmp[r*nranks+i] != -1) start = i;
|
||||
if (nextTmp[r*nranks+i] != -1) end = i;
|
||||
}
|
||||
if (start != -1 && end != -1) {
|
||||
subprev[r*nidx+rankToIdx[start]] = rankToIdx[end];
|
||||
subnext[r*nidx+rankToIdx[end]] = rankToIdx[start];
|
||||
}
|
||||
}
|
||||
/* Get rings */
|
||||
NCCLCHECK(ncclTransports[t].getRings(nidx, groups, subgroups, subvalues, &nringsTmp, subprev, subnext, minScore, &nThreads));
|
||||
/* Merge subprev/subnext into prev/next */
|
||||
for (int r=0; r<nringsTmp; r++) {
|
||||
for (int i=0; i<nidx; i++) {
|
||||
if ((prevTmp[r*nranks+idxToRank[i]] == -1) && (subprev[r*nidx+i] != -1)) prevTmp[r*nranks+idxToRank[i]] = idxToRank[subprev[r*nidx+i]];
|
||||
if ((nextTmp[r*nranks+idxToRank[i]] == -1) && (subnext[r*nidx+i] != -1)) nextTmp[r*nranks+idxToRank[i]] = idxToRank[subnext[r*nidx+i]];
|
||||
if (t == NTRANSPORTS-1) {
|
||||
// Save node-level masters for trees
|
||||
treeIn[r*nranks+idxToRank[i]] = prevTmp[r*nranks+idxToRank[i]] == -1 ? 0 : 1;
|
||||
treeOut[r*nranks+idxToRank[i]] = nextTmp[r*nranks+idxToRank[i]] == -1 ? 0 : 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
//for (int r=0; r<nringsTmp; r++) {
|
||||
//printf("[%d] [%d] [%d] [%d] Prev ", rank, minScore, t, r); for (int i=0; i<nranks; i++) printf("%d ", prevTmp[r*nranks+i]); printf("\n");
|
||||
//printf("[%d] [%d] [%d] [%d] Next ", rank, minScore, t, r); for (int i=0; i<nranks; i++) printf("%d ", nextTmp[r*nranks+i]); printf("\n");
|
||||
//}
|
||||
}
|
||||
free(subvalues);
|
||||
free(subprev);
|
||||
free(subnext);
|
||||
if (nringsTmp == 0) break;
|
||||
}
|
||||
minScore--;
|
||||
if (nringsTmp > *nrings) {
|
||||
*nrings = nringsTmp;
|
||||
for (int i=0; i<nranks*(*nrings); i++) {
|
||||
prev[i] = prevTmp[i];
|
||||
next[i] = nextTmp[i];
|
||||
}
|
||||
}
|
||||
} while (nringsTmp == 0 && minScore);
|
||||
|
||||
free(coords);
|
||||
free(globalRankToIdx);
|
||||
free(globalIdxToRank);
|
||||
free(prevTmp);
|
||||
free(nextTmp);
|
||||
free(idxToRank);
|
||||
free(rankToIdx);
|
||||
free(groups);
|
||||
free(subgroups);
|
||||
|
||||
*nthreads = nThreads;
|
||||
|
||||
/* Duplicate the rings in case of multinode+NVLink */
|
||||
int nnodes = 0;
|
||||
for (int r=0; r<nranks; r++) nnodes += treeIn[r];
|
||||
int nvlink;
|
||||
NCCLCHECK(ncclNvlinkGpu(&nvlink));
|
||||
if (nnodes > 1 && nvlink) {
|
||||
*nrings = copyRings(*nrings, *nrings*2, nranks, prev, next, treeIn, treeOut);
|
||||
}
|
||||
|
||||
if (*nrings == 0) {
|
||||
WARN("Could not create rings, falling back on simple ring");
|
||||
*nrings = 1;
|
||||
prev[rank] = (rank-1+nranks) % nranks;
|
||||
next[rank] = (rank+1)%nranks;
|
||||
}
|
||||
|
||||
int maxNrings = ncclParamMaxNrings();
|
||||
int minNrings = ncclParamMinNrings();
|
||||
if (maxNrings > 0 && minNrings > maxNrings) {
|
||||
if (rank == 0) WARN("NCCL_MIN_NRINGS set to a value greater than NCCL_MAX_NRINGS, ignoring NCCL_MIN_NRINGS");
|
||||
minNrings = 0;
|
||||
}
|
||||
if (minNrings > MAXCHANNELS) {
|
||||
if (rank == 0) WARN("NCCL_MIN_NRINGS set to a value greater than the maximum number of rings supported (%d), limiting it to %d", MAXCHANNELS, MAXCHANNELS);
|
||||
minNrings = MAXCHANNELS;
|
||||
}
|
||||
if (maxNrings > 0 && maxNrings <= *nrings) {
|
||||
if (rank == 0) INFO(NCCL_INIT,"Limiting to %d rings per user request.", maxNrings);
|
||||
*nrings = maxNrings;
|
||||
} else {
|
||||
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
|
||||
int defaultMinNrings = 1;
|
||||
#else
|
||||
int defaultMinNrings = ncclCudaCompCap() == 3 ? 2 : 1;
|
||||
#endif
|
||||
if (minNrings < defaultMinNrings) minNrings = defaultMinNrings;
|
||||
if (minNrings > 0 && minNrings > *nrings) {
|
||||
if (rank == 0 && minNrings > defaultMinNrings) INFO(NCCL_INIT,"Duplicating rings to %d per user request.", minNrings);
|
||||
*nrings = copyRings(*nrings, minNrings, nranks, prev, next, treeIn, treeOut);
|
||||
}
|
||||
}
|
||||
|
||||
NCCLCHECK(getEnvThreads(nthreads));
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "topo.h"
|
||||
|
||||
#define BUSID_SIZE (sizeof("0000:00:00.0"))
|
||||
#define BUSID_REDUCED_SIZE (sizeof("0000:00"))
|
||||
|
||||
ncclResult_t getCudaPath(int cudaDev, char** path) {
|
||||
char busId[BUSID_SIZE];
|
||||
CUDACHECK(hipDeviceGetPCIBusId(busId, BUSID_SIZE, cudaDev));
|
||||
for (int i=0; i<BUSID_SIZE; i++) busId[i] = tolower(busId[i]);
|
||||
char busPath[] = "/sys/class/pci_bus/0000:00/../../0000:00:00.0";
|
||||
memcpy(busPath+sizeof("/sys/class/pci_bus/")-1, busId, BUSID_REDUCED_SIZE-1);
|
||||
memcpy(busPath+sizeof("/sys/class/pci_bus/0000:00/../../")-1, busId, BUSID_SIZE-1);
|
||||
*path = realpath(busPath, NULL);
|
||||
if (*path == NULL) {
|
||||
WARN("Could not find real path of %s", busPath);
|
||||
return ncclSystemError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
const char* pathDists[] = { "PIX", "PXB", "PHB", "NODE", "SYS" };
|
||||
|
||||
int pciDistance(char* path1, char* path2) {
|
||||
int score = 0;
|
||||
int depth = 0;
|
||||
int same = 1;
|
||||
for (int i=0; i<strlen(path1); i++) {
|
||||
if (path1[i] != path2[i]) same = 0;
|
||||
if (path1[i] == '/') {
|
||||
depth++;
|
||||
if (same == 1) score++;
|
||||
}
|
||||
}
|
||||
if (score <= 3) {
|
||||
#ifdef __PPC__
|
||||
// NUMA distance detection and PATH_SYS not supported on IBM/Power nodes
|
||||
// nodes currently
|
||||
return PATH_NODE;
|
||||
#else
|
||||
/* Split the former PATH_SOC distance into PATH_NODE and PATH_SYS based on numaId */
|
||||
int numaId1 = getNumaId(path1);
|
||||
int numaId2 = getNumaId(path2);
|
||||
TRACE(NCCL_INIT, "depth %d score %d path1 %s numaId %d path2 %s numaId %d", depth, score, path1, numaId1, path2, numaId2);
|
||||
return ((numaId1 == numaId2) ? PATH_NODE : PATH_SYS);
|
||||
#endif
|
||||
}
|
||||
if (score == 4) return PATH_PHB;
|
||||
if (score == depth-1) return PATH_PIX;
|
||||
return PATH_PXB;
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "net.h"
|
||||
#include "param.h"
|
||||
|
||||
#define RANK_TO_INDEX(r) (rank > root ? rank-1 : rank)
|
||||
|
||||
/* Btree which alternates leaves and nodes.
|
||||
* Assumes root is 0, which conveniently builds a tree on powers of two,
|
||||
* (because we have pow2-1 ranks) which lets us manipulate bits.
|
||||
* Find first non-zero bit, then :
|
||||
* Find the parent :
|
||||
* xx01[0] -> xx10[0] (1,5,9 below) or xx00[0] if xx10[0] is out of bounds (13 below)
|
||||
* xx11[0] -> xx10[0] (3,7,11 below)
|
||||
* Find the children :
|
||||
* xx10[0] -> xx01[0] (2,4,6,8,10,12) or -1 (1,3,5,7,9,11,13)
|
||||
* xx10[0] -> xx11[0] (2,4,6,8,10) or xx101[0] (12) or xx1001[0] ... or -1 (1,3,5,7,9,11,13)
|
||||
*
|
||||
* Illustration :
|
||||
* 0---------------8
|
||||
* ______/ \______
|
||||
* 4 12
|
||||
* / \ / \
|
||||
* 2 6 10 \
|
||||
* / \ / \ / \ \
|
||||
* 1 3 5 7 9 11 13
|
||||
*/
|
||||
ncclResult_t ncclGetBtree(int nranks, int rank, int* u, int* d0, int* d1) {
|
||||
int up, down0, down1;
|
||||
int bit;
|
||||
for (bit=1; bit<nranks; bit<<=1) {
|
||||
if (bit & rank) break;
|
||||
}
|
||||
|
||||
if (rank == 0) {
|
||||
*u = -1;
|
||||
*d0 = nranks > 1 ? bit >> 1 : -1;
|
||||
*d1 = -1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
up = (rank ^ bit) | (bit << 1);
|
||||
if (up >= nranks) up = (rank ^ bit);
|
||||
*u = up;
|
||||
|
||||
int lowbit = bit >> 1;
|
||||
// down0 is always within bounds
|
||||
down0 = lowbit == 0 ? -1 : rank-lowbit;
|
||||
|
||||
down1 = lowbit == 0 ? -1 : rank+lowbit;
|
||||
// Make sure down1 is within bounds
|
||||
while (down1 >= nranks) {
|
||||
down1 = lowbit == 0 ? -1 : rank+lowbit;
|
||||
lowbit >>= 1;
|
||||
}
|
||||
*d0 = down0; *d1 = down1;
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/* Build a double binary tree. Take the previous tree for the first tree.
|
||||
* For the second tree, we use a mirror tree (if nranks is odd)
|
||||
*
|
||||
* 8---------0---------5
|
||||
* ______/ \______ _____/ \______
|
||||
* 4 12 1 9
|
||||
* / \ / \ / \
|
||||
* 2 6 10 3 7 10
|
||||
* / \ / \ / \ / \ / \ / \
|
||||
* 1 3 5 7 9 11 2 4 6 8 11 12
|
||||
*
|
||||
* or shift it by one rank (if nranks is even)
|
||||
*
|
||||
* 8---------0--------------9
|
||||
* ______/ \ ______/ \
|
||||
* 4 \ 5 \
|
||||
* / \ \ / \ \
|
||||
* 2 6 10 3 7 11
|
||||
* / \ / \ / \ / \ / \ / \
|
||||
* 1 3 5 7 9 11 2 4 6 8 10 1
|
||||
*/
|
||||
ncclResult_t ncclGetDtree(int nranks, int rank, int* s0, int* d0_0, int* d0_1, int* s1, int* d1_0, int* d1_1) {
|
||||
// First tree ... use a btree
|
||||
ncclGetBtree(nranks, rank, s0, d0_0, d0_1);
|
||||
// Second tree ... mirror or shift
|
||||
if (nranks % 2 == 0) {
|
||||
// shift
|
||||
int shiftrank = (rank-1+nranks) % nranks;
|
||||
int u, d0, d1;
|
||||
ncclGetBtree(nranks, shiftrank, &u, &d0, &d1);
|
||||
*s1 = u == -1 ? -1 : (u+1) % nranks;
|
||||
*d1_0 = d0 == -1 ? -1 : (d0+1) % nranks;
|
||||
*d1_1 = d1 == -1 ? -1 : (d1+1) % nranks;
|
||||
} else {
|
||||
// mirror
|
||||
int u, d0, d1;
|
||||
ncclGetBtree(nranks, nranks-1-rank, &u, &d0, &d1);
|
||||
*s1 = u == -1 ? -1 : nranks-1-u;
|
||||
*d1_0 = d0 == -1 ? -1 : nranks-1-d0;
|
||||
*d1_1 = d1 == -1 ? -1 : nranks-1-d1;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
+75
-99
@@ -6,32 +6,54 @@
|
||||
************************************************************************/
|
||||
|
||||
#include "utils.h"
|
||||
#include "debug.h"
|
||||
#include "nccl_net.h"
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#include <stdarg.h>
|
||||
|
||||
#include "nvmlwrap.h"
|
||||
#include "core.h"
|
||||
|
||||
#include "nvmlwrap.h"
|
||||
#include <hip/hip_runtime.h>
|
||||
|
||||
// Get current Compute Capability
|
||||
int ncclCudaCompCap() {
|
||||
int cudaDev;
|
||||
if (hipGetDevice(&cudaDev) != hipSuccess) return 0;
|
||||
int ccMajor, ccMinor;
|
||||
if (hipDeviceGetAttribute(&ccMajor, hipDeviceAttributeComputeCapabilityMajor, cudaDev) != hipSuccess) return 0;
|
||||
if (hipDeviceGetAttribute(&ccMinor, hipDeviceAttributeComputeCapabilityMinor, cudaDev) != hipSuccess) return 0;
|
||||
return ccMajor*10+ccMinor;
|
||||
}
|
||||
|
||||
ncclResult_t int64ToBusId(int64_t id, char* busId) {
|
||||
sprintf(busId, "%04lx:%02lx:%02lx.%01lx", (id) >> 20, (id & 0xff000) >> 12, (id & 0xff0) >> 4, (id & 0xf));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t busIdToInt64(char* busId, int64_t* id) {
|
||||
const int size = strlen(busId);
|
||||
char* hexStr;
|
||||
NCCLCHECK(ncclCalloc(&hexStr, size));
|
||||
int hexOffset = 0;
|
||||
for (int i=0; i<size; i++) {
|
||||
char c = busId[i];
|
||||
if (c == '.' || c == ':') continue;
|
||||
if ((c >= '0' && c <= '9') ||
|
||||
(c >= 'A' && c <= 'F') ||
|
||||
(c >= 'a' && c <= 'f')) {
|
||||
hexStr[hexOffset++] = busId[i];
|
||||
} else break;
|
||||
}
|
||||
hexStr[hexOffset] = '\0';
|
||||
*id = strtol(hexStr, NULL, 16);
|
||||
free(hexStr);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Convert a logical cudaDev index to the NVML device minor number
|
||||
ncclResult_t getNvmlDevice(int cudaDev, int *nvmlDev) {
|
||||
#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__)
|
||||
// assign nmvlDev to be same as cudaDev to avoid garbage numbers
|
||||
*nvmlDev = cudaDev;
|
||||
#else
|
||||
char busId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE];
|
||||
nvmlDevice_t nvmlDevice;
|
||||
unsigned int dev;
|
||||
*nvmlDev = -1;
|
||||
CUDACHECK(hipDeviceGetPCIBusId(busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, cudaDev));
|
||||
NCCLCHECK(wrapNvmlDeviceGetHandleByPciBusId(busId, &nvmlDevice));
|
||||
NCCLCHECK(wrapNvmlDeviceGetMinorNumber(nvmlDevice, &dev));
|
||||
|
||||
*nvmlDev = dev;
|
||||
#endif
|
||||
|
||||
ncclResult_t getBusId(int cudaDev, int64_t *busId) {
|
||||
// On most systems, the PCI bus ID comes back as in the 0000:00:00.0
|
||||
// format. Still need to allocate proper space in case PCI domain goes
|
||||
// higher.
|
||||
char busIdStr[] = "00000000:00:00.0";
|
||||
CUDACHECK(hipDeviceGetPCIBusId(busIdStr, sizeof(busIdStr), cudaDev));
|
||||
NCCLCHECK(busIdToInt64(busIdStr, busId));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -46,53 +68,6 @@ ncclResult_t getHostName(char* hostname, int maxlen, const char delim) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/* Common logging function used by the INFO, WARN and TRACE macros
|
||||
* Also exported to the dynamically loadable Net transport modules so
|
||||
* they can share the debugging mechanisms and output files
|
||||
*/
|
||||
void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *filefunc, int line, const char *fmt, ...) {
|
||||
if (ncclDebugLevel <= NCCL_LOG_NONE) return;
|
||||
|
||||
char hostname[1024];
|
||||
getHostName(hostname, 1024, '.');
|
||||
int cudaDev;
|
||||
hipGetDevice(&cudaDev);
|
||||
|
||||
char buffer[1024];
|
||||
size_t len = 0;
|
||||
pthread_mutex_lock(&ncclDebugOutputLock);
|
||||
if (level == NCCL_LOG_WARN && ncclDebugLevel >= NCCL_LOG_WARN)
|
||||
len = snprintf(buffer, sizeof(buffer),
|
||||
"\n%s:%d:%d [%d] %s:%d NCCL WARN ", hostname, getpid(), gettid(), cudaDev, filefunc, line);
|
||||
else if (level == NCCL_LOG_INFO && ncclDebugLevel >= NCCL_LOG_INFO && (flags & ncclDebugMask))
|
||||
len = snprintf(buffer, sizeof(buffer),
|
||||
"%s:%d:%d [%d] NCCL INFO ", hostname, getpid(), gettid(), cudaDev);
|
||||
#ifdef ENABLE_TRACE
|
||||
else if (level == NCCL_LOG_TRACE && ncclDebugLevel >= NCCL_LOG_TRACE && (flags & ncclDebugMask)) {
|
||||
auto delta = std::chrono::high_resolution_clock::now() - ncclEpoch;
|
||||
double timestamp = std::chrono::duration_cast<std::chrono::duration<double>>(delta).count()*1000;
|
||||
len = snprintf(buffer, sizeof(buffer),
|
||||
"%s:%d:%d [%d] %f %s:%d NCCL TRACE ", hostname, getpid(), gettid(), cudaDev, timestamp, filefunc, line);
|
||||
}
|
||||
#endif
|
||||
if (len) {
|
||||
va_list vargs;
|
||||
va_start(vargs, fmt);
|
||||
(void) vsnprintf(buffer+len, sizeof(buffer)-len, fmt, vargs);
|
||||
va_end(vargs);
|
||||
fprintf(ncclDebugFile,"%s\n", buffer);
|
||||
fflush(ncclDebugFile);
|
||||
}
|
||||
pthread_mutex_unlock(&ncclDebugOutputLock);
|
||||
|
||||
// If ncclDebugLevel == NCCL_LOG_ABORT then WARN() will also call abort()
|
||||
if (level == NCCL_LOG_WARN && ncclDebugLevel == NCCL_LOG_ABORT) {
|
||||
fprintf(stderr,"\n%s:%d:%d [%d] %s:%d NCCL ABORT\n",
|
||||
hostname, getpid(), gettid(), cudaDev, filefunc, line);
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t getHash(const char* string, int n) {
|
||||
// Based on DJB2, result = result * 33 + char
|
||||
uint64_t result = 5381;
|
||||
@@ -102,40 +77,43 @@ uint64_t getHash(const char* string, int n) {
|
||||
return result;
|
||||
}
|
||||
|
||||
uint64_t getnHash(const char* string, int n) {
|
||||
// Based on DJB2, result = result * 33 + char
|
||||
uint64_t result = 9527;
|
||||
for (int c = 0; c < n; c++) {
|
||||
result = ((result << 5) + result) + string[c];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Generate a hash of the unique identifying string for this host
|
||||
* that will be unique for both bare-metal and container instances
|
||||
* Equivalent of a hash of;
|
||||
*
|
||||
* $(hostname) $(readlink /proc/self/ns/uts) $(readlink /proc/self/ns/mnt)
|
||||
* $(hostname)$(cat /proc/sys/kernel/random/boot_id)
|
||||
*
|
||||
* This string can be overridden by using the NCCL_HOSTID env var.
|
||||
*/
|
||||
#define HOSTID_FILE "/proc/sys/kernel/random/boot_id"
|
||||
uint64_t getHostHash(void) {
|
||||
char uname[1024];
|
||||
// Start off with the full hostname
|
||||
(void) getHostName(uname, sizeof(uname), '\0');
|
||||
int offset = strlen(uname);
|
||||
int len;
|
||||
// $(readlink /proc/self/ns/uts)
|
||||
len = readlink("/proc/self/ns/uts", uname+offset, sizeof(uname)-1-offset);
|
||||
if (len < 0) len = 0;
|
||||
offset += len;
|
||||
// $(readlink /proc/self/ns/mnt)
|
||||
len = readlink("/proc/self/ns/mnt", uname+offset, sizeof(uname)-1-offset);
|
||||
if (len < 0) len = 0;
|
||||
offset += len;
|
||||
// Trailing '\0'
|
||||
uname[offset]='\0';
|
||||
TRACE(NCCL_INIT,"unique hostname '%s'", uname);
|
||||
char hostHash[1024];
|
||||
char *hostId;
|
||||
|
||||
return getHash(uname, strlen(uname));
|
||||
// Fall back is the full hostname if something fails
|
||||
(void) getHostName(hostHash, sizeof(hostHash), '\0');
|
||||
int offset = strlen(hostHash);
|
||||
|
||||
if ((hostId = getenv("NCCL_HOSTID")) != NULL) {
|
||||
strncpy(hostHash, hostId, sizeof(hostHash));
|
||||
} else {
|
||||
FILE *file = fopen(HOSTID_FILE, "r");
|
||||
if (file != NULL) {
|
||||
char *p;
|
||||
if (fscanf(file, "%ms", &p) == 1) {
|
||||
strncpy(hostHash+offset, p, sizeof(hostHash)-offset-1);
|
||||
free(p);
|
||||
}
|
||||
}
|
||||
fclose(file);
|
||||
}
|
||||
|
||||
// Make sure the string is terminated
|
||||
hostHash[sizeof(hostHash)-1]='\0';
|
||||
|
||||
TRACE(NCCL_INIT,"unique hostname '%s'", hostHash);
|
||||
|
||||
return getHash(hostHash, strlen(hostHash));
|
||||
}
|
||||
|
||||
/* Generate a hash of the unique identifying string for this process
|
||||
@@ -162,8 +140,6 @@ int parseStringList(const char* string, struct netIf* ifList, int maxList) {
|
||||
if (!string) return 0;
|
||||
|
||||
const char* ptr = string;
|
||||
// Ignore "^" or "=" prefix, will be detected outside of this function
|
||||
if (ptr[0] == '^' || ptr[0] == '=') ptr++;
|
||||
|
||||
int ifNum = 0;
|
||||
int ifC = 0;
|
||||
|
||||
Reference in New Issue
Block a user