AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2.14.3-1

Browse Source 
Add support for improved fault tolerance: non-blocking mode, new
init function with config, and ncclCommFinalize function.
Reintroduce collnet+chain algorithm, alongside collnet+direct.
Add LL protocol for intra-node P2P (on by default) and network
communication (off by default).
Use network instead of shared memory when performance is better.
Fix: wait for CUDA graph destroy before destroying comm with linked
graph resources.
Remove aggressive polling during enqueue.
Fix DMABUF fallback on MOFED 5.4 and earlier.
This commit is contained in:
Sylvain Jeaugey
2022-08-18 02:53:17 -07:00
parent e1d9b273b0
commit c4e2aa6c79
42 changed files with 1787 additions and 942 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/misc/cudawrap.cc
+43 -52
View File
@@ -10,32 +10,30 @@
#include <dlfcn.h>
#define DECLARE_CUDA_PFN(symbol) PFN_##symbol pfn_##symbol = nullptr
#define DECLARE_CUDA_PFN(symbol,version) PFN_##symbol##_v##version pfn_##symbol = nullptr
#if CUDART_VERSION >= 11030
/* CUDA Driver functions loaded with cuGetProcAddress for versioning */
DECLARE_CUDA_PFN(cuDeviceGet);
DECLARE_CUDA_PFN(cuDeviceGetAttribute);
DECLARE_CUDA_PFN(cuGetErrorString);
DECLARE_CUDA_PFN(cuGetErrorName);
DECLARE_CUDA_PFN(cuDeviceGet, 2000);
DECLARE_CUDA_PFN(cuDeviceGetAttribute, 2000);
DECLARE_CUDA_PFN(cuGetErrorString, 6000);
DECLARE_CUDA_PFN(cuGetErrorName, 6000);
/* enqueue.cc */
DECLARE_CUDA_PFN(cuMemGetAddressRange);
DECLARE_CUDA_PFN(cuMemGetAddressRange, 3020);
/* proxy.cc */
DECLARE_CUDA_PFN(cuCtxCreate_v3020);
DECLARE_CUDA_PFN(cuCtxDestroy);
DECLARE_CUDA_PFN(cuCtxSetCurrent);
DECLARE_CUDA_PFN(cuCtxCreate, 3020);
DECLARE_CUDA_PFN(cuCtxDestroy, 4000);
DECLARE_CUDA_PFN(cuCtxSetCurrent, 4000);
#if CUDA_VERSION >= 11070
/* transport/collNet.cc/net.cc*/
DECLARE_CUDA_PFN(cuMemGetHandleForAddressRange); // DMA-BUF support
DECLARE_CUDA_PFN(cuMemGetHandleForAddressRange, 11070); // DMA-BUF support
#endif
#endif
/* CUDA Driver functions loaded with dlsym() */
DECLARE_CUDA_PFN(cuInit);
DECLARE_CUDA_PFN(cuDriverGetVersion);
DECLARE_CUDA_PFN(cuGetProcAddress);
static enum { cudaUninitialized, cudaInitializing, cudaInitialized, cudaError } cudaState = cudaUninitialized;
DECLARE_CUDA_PFN(cuInit, 2000);
DECLARE_CUDA_PFN(cuDriverGetVersion, 2020);
DECLARE_CUDA_PFN(cuGetProcAddress, 11030);
#define CUDA_DRIVER_MIN_VERSION 11030
@@ -46,46 +44,37 @@ static int cudaDriverVersion;
/*
Load the CUDA symbols
*/
static int cudaPfnFuncLoader(void) {
static ncclResult_t cudaPfnFuncLoader(void) {
CUresult res;
#define LOAD_SYM(symbol, ignore) do { \
res = pfn_cuGetProcAddress(#symbol, (void **) (&pfn_##symbol), cudaDriverVersion, 0); \
#define LOAD_SYM(symbol, version, ignore) do { \
res = pfn_cuGetProcAddress(#symbol, (void **) (&pfn_##symbol), version, 0); \
if (res != 0) { \
if (!ignore) { \
WARN("Retrieve %s version %d failed with %d", #symbol, cudaDriverVersion, res); \
WARN("Retrieve %s version %d failed with %d", #symbol, version, res); \
return ncclSystemError; } \
} } while(0)
LOAD_SYM(cuGetErrorString, 0);
LOAD_SYM(cuGetErrorName, 0);
LOAD_SYM(cuDeviceGet, 0);
LOAD_SYM(cuDeviceGetAttribute, 0);
LOAD_SYM(cuMemGetAddressRange, 1);
LOAD_SYM(cuCtxCreate_v3020, 1);
LOAD_SYM(cuCtxDestroy, 1);
LOAD_SYM(cuCtxSetCurrent, 1);
LOAD_SYM(cuGetErrorString, 6000, 0);
LOAD_SYM(cuGetErrorName, 6000, 0);
LOAD_SYM(cuDeviceGet, 2000, 0);
LOAD_SYM(cuDeviceGetAttribute, 2000, 0);
LOAD_SYM(cuMemGetAddressRange, 3020, 1);
LOAD_SYM(cuCtxCreate, 3020, 1);
LOAD_SYM(cuCtxDestroy, 4000, 1);
LOAD_SYM(cuCtxSetCurrent, 4000, 1);
#if CUDA_VERSION >= 11070
LOAD_SYM(cuMemGetHandleForAddressRange, 1); // DMA-BUF support
LOAD_SYM(cuMemGetHandleForAddressRange, 11070, 1); // DMA-BUF support
#endif
return ncclSuccess;
}
#endif
ncclResult_t cudaLibraryInit(void) {
static pthread_once_t initOnceControl = PTHREAD_ONCE_INIT;
static ncclResult_t initResult;
static void initOnceFunc() {
CUresult res;
if (cudaState == cudaInitialized)
return ncclSuccess;
if (cudaState == cudaError)
return ncclSystemError;
if (__sync_bool_compare_and_swap(&cudaState, cudaUninitialized, cudaInitializing) == false) {
// Another thread raced in front of us. Wait for it to be done.
while (cudaState == cudaInitializing) sched_yield();
return (cudaState == cudaInitialized) ? ncclSuccess : ncclSystemError;
}
/*
* Load CUDA driver library
*/
@@ -106,13 +95,13 @@ ncclResult_t cudaLibraryInit(void) {
* Load initial CUDA functions
*/
pfn_cuInit = (PFN_cuInit) dlsym(cudaLib, "cuInit");
pfn_cuInit = (PFN_cuInit_v2000) dlsym(cudaLib, "cuInit");
if (pfn_cuInit == NULL) {
WARN("Failed to load CUDA missing symbol cuInit");
goto error;
}
pfn_cuDriverGetVersion = (PFN_cuDriverGetVersion) dlsym(cudaLib, "cuDriverGetVersion");
pfn_cuDriverGetVersion = (PFN_cuDriverGetVersion_v2020) dlsym(cudaLib, "cuDriverGetVersion");
if (pfn_cuDriverGetVersion == NULL) {
WARN("Failed to load CUDA missing symbol cuDriverGetVersion");
goto error;
@@ -132,7 +121,7 @@ ncclResult_t cudaLibraryInit(void) {
goto error;
}
pfn_cuGetProcAddress = (PFN_cuGetProcAddress) dlsym(cudaLib, "cuGetProcAddress");
pfn_cuGetProcAddress = (PFN_cuGetProcAddress_v11030) dlsym(cudaLib, "cuGetProcAddress");
if (pfn_cuGetProcAddress == NULL) {
WARN("Failed to load CUDA missing symbol cuGetProcAddress");
goto error;
@@ -145,19 +134,21 @@ ncclResult_t cudaLibraryInit(void) {
*/
pfn_cuInit(0);
#if CUDART_VERSION >= 11030
#if CUDART_VERSION >= 11030
if (cudaPfnFuncLoader()) {
WARN("CUDA some PFN functions not found in the library");
goto error;
}
#endif
cudaState = cudaInitialized;
return ncclSuccess;
#endif
initResult = ncclSuccess;
return;
error:
cudaState = cudaError;
return ncclSystemError;
initResult = ncclSystemError;
return;
}
ncclResult_t cudaLibraryInit() {
pthread_once(&initOnceControl, initOnceFunc);
return initResult;
}
src/misc/gdrwrap.cc
+12 -17
View File
@@ -9,8 +9,6 @@
#ifndef GDR_DIRECT
#include "core.h"
static enum { gdrUninitialized, gdrInitializing, gdrInitialized, gdrError } gdrState = gdrUninitialized;
/* Function pointers assigned from dlopen() */
static gdr_t (*gdr_internal_open)(void);
static int (*gdr_internal_close)(gdr_t g);
@@ -49,18 +47,10 @@ pthread_mutex_t gdrLock = PTHREAD_MUTEX_INITIALIZER;
*cast = tmp; \
} while (0)
ncclResult_t wrap_gdr_symbols(void) {
if (gdrState == gdrInitialized)
return ncclSuccess;
if (gdrState == gdrError)
return ncclSystemError;
if (__sync_bool_compare_and_swap(&gdrState, gdrUninitialized, gdrInitializing) == false) {
// Another thread raced in front of us. Wait for it to be done.
while (gdrState == gdrInitializing) sched_yield();
return (gdrState == gdrInitialized) ? ncclSuccess : ncclSystemError;
}
static pthread_once_t initOnceControl = PTHREAD_ONCE_INIT;
static ncclResult_t initResult;
static void initOnceFunc(void) {
static void* gdrhandle = NULL;
void* tmp;
void** cast;
@@ -84,8 +74,8 @@ ncclResult_t wrap_gdr_symbols(void) {
LOAD_SYM(gdrhandle, "gdr_copy_to_mapping", gdr_internal_copy_to_mapping);
LOAD_SYM(gdrhandle, "gdr_copy_from_mapping", gdr_internal_copy_from_mapping);
gdrState = gdrInitialized;
return ncclSuccess;
initResult = ncclSuccess;
return;
teardown:
gdr_internal_open = NULL;
@@ -101,11 +91,16 @@ teardown:
gdr_internal_copy_from_mapping = NULL;
if (gdrhandle != NULL) dlclose(gdrhandle);
gdrState = gdrError;
return ncclSystemError;
initResult = ncclSystemError;
return;
}
ncclResult_t wrap_gdr_symbols(void) {
pthread_once(&initOnceControl, initOnceFunc);
return initResult;
}
gdr_t wrap_gdr_open(void) {
if (gdr_internal_open == NULL) {
WARN("GDRCOPY lib wrapper not initialized.");
src/misc/ibvwrap.cc
+16 -20
View File
@@ -11,8 +11,6 @@
#include <dlfcn.h>
#include "core.h"
static enum { ibvUninitialized, ibvInitializing, ibvInitialized, ibvError } ibvState = ibvUninitialized;
/*Function Pointers*/
int (*ibv_internal_fork_init)(void);
struct ibv_device** (*ibv_internal_get_device_list)(int *num_devices);
@@ -43,18 +41,10 @@ const char * (*ibv_internal_event_type_str)(enum ibv_event_type event);
// IBVERBS Library versioning
#define IBVERBS_VERSION "IBVERBS_1.1"
ncclResult_t wrap_ibv_symbols(void) {
if (ibvState == ibvInitialized)
return ncclSuccess;
if (ibvState == ibvError)
return ncclSystemError;
if (__sync_bool_compare_and_swap(&ibvState, ibvUninitialized, ibvInitializing) == false) {
// Another thread raced in front of us. Wait for it to be done.
while (ibvState == ibvInitializing) sched_yield();
return (ibvState == ibvInitialized) ? ncclSuccess : ncclSystemError;
}
static pthread_once_t initOnceControl = PTHREAD_ONCE_INIT;
static ncclResult_t initResult;
static void initOnceFunc(void) {
static void* ibvhandle = NULL;
void* tmp;
void** cast;
@@ -111,8 +101,8 @@ ncclResult_t wrap_ibv_symbols(void) {
LOAD_SYM(ibvhandle, "ibv_fork_init", ibv_internal_fork_init);
LOAD_SYM(ibvhandle, "ibv_event_type_str", ibv_internal_event_type_str);
ibvState = ibvInitialized;
return ncclSuccess;
initResult = ncclSuccess;
return;
teardown:
ibv_internal_get_device_list = NULL;
@@ -141,8 +131,13 @@ teardown:
ibv_internal_event_type_str = NULL;
if (ibvhandle != NULL) dlclose(ibvhandle);
ibvState = ibvError;
return ncclSystemError;
initResult = ncclSystemError;
return;
}
ncclResult_t wrap_ibv_symbols(void) {
pthread_once(&initOnceControl, initOnceFunc);
return initResult;
}
#define IBV_PTR_CHECK_ERRNO(name_internal, call, retval, error_retval, name) \
@@ -256,7 +251,7 @@ ncclResult_t wrap_ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int
}
ncclResult_t wrap_ibv_alloc_pd(struct ibv_pd **ret, struct ibv_context *context) {
IBV_PTR_CHECK(ibv_internal_alloc_pd, ibv_internal_alloc_pd(context), *ret, NULL, "ibv_alloc_pd");
IBV_PTR_CHECK_ERRNO(ibv_internal_alloc_pd, ibv_internal_alloc_pd(context), *ret, NULL, "ibv_alloc_pd");
}
ncclResult_t wrap_ibv_dealloc_pd(struct ibv_pd *pd) { /*returns 0 on success, or the value of errno on failure (which indicates the failure reason)*/
@@ -290,6 +285,7 @@ ncclResult_t wrap_ibv_reg_dmabuf_mr(struct ibv_mr **ret, struct ibv_pd *pd, uint
struct ibv_mr * wrap_direct_ibv_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access) {
if (ibv_internal_reg_dmabuf_mr == NULL) {
errno = EOPNOTSUPP; // ncclIbDmaBufSupport() requires this errno being set
return NULL;
}
return ibv_internal_reg_dmabuf_mr(pd, offset, length, iova, fd, access);
@@ -300,7 +296,7 @@ ncclResult_t wrap_ibv_dereg_mr(struct ibv_mr *mr) { /*returns 0 on success, or t
}
ncclResult_t wrap_ibv_create_cq(struct ibv_cq **ret, struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector) {
IBV_PTR_CHECK(ibv_internal_create_cq, ibv_internal_create_cq(context, cqe, cq_context, channel, comp_vector), *ret, NULL, "ibv_create_cq");
IBV_PTR_CHECK_ERRNO(ibv_internal_create_cq, ibv_internal_create_cq(context, cqe, cq_context, channel, comp_vector), *ret, NULL, "ibv_create_cq");
}
ncclResult_t wrap_ibv_destroy_cq(struct ibv_cq *cq) {
@@ -312,7 +308,7 @@ ncclResult_t wrap_ibv_destroy_qp(struct ibv_qp *qp) {
}
ncclResult_t wrap_ibv_create_qp(struct ibv_qp **ret, struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr) {
IBV_PTR_CHECK(ibv_internal_create_qp, ibv_internal_create_qp(pd, qp_init_attr), *ret, NULL, "ibv_create_qp");
IBV_PTR_CHECK_ERRNO(ibv_internal_create_qp, ibv_internal_create_qp(pd, qp_init_attr), *ret, NULL, "ibv_create_qp");
}
ncclResult_t wrap_ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) { /*returns 0 on success, or the value of errno on failure (which indicates the failure reason)*/
src/misc/shmutils.cc
+7 -4
View File
@@ -38,6 +38,7 @@ static ncclResult_t ncclShmSetup(char* shmPath, const int shmSize, int* fd, void
WARN("Error: failed to extend %s to %d bytes", shmPath, shmSize);
return ncclSystemError;
}
INFO(NCCL_ALLOC, "Allocated %d bytes of shared memory in %s\n", shmSize, shmPath);
} else {
SYSCHECKVAL(open(shmPath, O_RDWR, S_IRUSR | S_IWUSR), "open", *fd);
}
@@ -81,10 +82,12 @@ ncclResult_t ncclShmUnlink(const char* shmPath) {
}
ncclResult_t ncclShmClose(void* shmPtr, void* devShmPtr, const int shmSize) {
if (devShmPtr) CUDACHECK(cudaHostUnregister(shmPtr));
if (munmap(shmPtr, shmSize) != 0) {
WARN("munmap of shared memory failed");
return ncclSystemError;
if (shmPtr) {
if (devShmPtr) CUDACHECK(cudaHostUnregister(shmPtr));
if (munmap(shmPtr, shmSize) != 0) {
WARN("munmap of shared memory failed");
return ncclSystemError;
}
}
return ncclSuccess;
}
src/misc/socket.cc
+43 -26
View File
@@ -332,10 +332,10 @@ ncclResult_t ncclSocketListen(struct ncclSocket* sock) {
#endif
}
if (sock->asyncFlag) {
EQCHECK(flags = fcntl(fd, F_GETFL), -1);
SYSCHECK(fcntl(fd, F_SETFL, flags | O_NONBLOCK), "fcntl");
}
/* The socket is set non-blocking for OS level, but asyncFlag is used to control
* blocking and non-blocking behavior in user level. */
EQCHECK(flags = fcntl(fd, F_GETFL), -1);
SYSCHECK(fcntl(fd, F_SETFL, flags | O_NONBLOCK), "fcntl");
// addr port should be 0 (Any port)
SYSCHECK(bind(fd, &sock->addr.sa, salen), "bind");
@@ -411,11 +411,10 @@ ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
const int one = 1;
SYSCHECK(setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
/* support non-blocking socket; by default, the socket is non-blocking */
if (sock->asyncFlag) {
EQCHECK(flags = fcntl(fd, F_GETFL), -1);
SYSCHECK(fcntl(fd, F_SETFL, flags | O_NONBLOCK), "fcntl");
}
/* The socket is set non-blocking for OS level, but asyncFlag is used to control
* blocking and non-blocking behavior in user level. */
EQCHECK(flags = fcntl(fd, F_GETFL), -1);
SYSCHECK(fcntl(fd, F_SETFL, flags | O_NONBLOCK), "fcntl");
/* const int bufsize = 128*1024;
SYSCHECK(setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (char*)&bufsize, sizeof(int)), "setsockopt");
@@ -430,17 +429,26 @@ retry:
/* blocking/non-blocking connect() is determined by asyncFlag. */
ret = connect(fd, &sock->addr.sa, salen);
if (!sock->asyncFlag && (errno == EAGAIN || (errno == ECONNREFUSED && ++refused_retries < RETRY_REFUSED_TIMES) ||
(errno == ETIMEDOUT && ++timedout_retries < RETRY_TIMEDOUT_TIMES))) {
if (errno == ECONNREFUSED && refused_retries % 1000 == 0) INFO(NCCL_ALL, "Call to connect returned %s, retrying", strerror(errno));
usleep(SLEEP_INT);
goto retry;
}
if (!sock->asyncFlag) {
/* blocking socket, need retry if connect fails. */
if (errno == EINPROGRESS || errno == EAGAIN || errno == EALREADY ||
(errno == ECONNREFUSED && ++refused_retries < RETRY_REFUSED_TIMES) ||
(errno == ETIMEDOUT && ++timedout_retries < RETRY_TIMEDOUT_TIMES)) {
/* check abortFlag as long as we have chance to retry. */
if (sock->abortFlag && *sock->abortFlag != 0) return ncclInternalError;
if (errno == ECONNREFUSED && refused_retries % 1000 == 0) INFO(NCCL_ALL, "Call to connect returned %s, retrying", strerror(errno));
usleep(SLEEP_INT);
goto retry;
}
/* If connect() fails with errno == EAGAIN/EINPROGRESS/ETIMEDOUT, we may want to try connect again.
* However, it can return EISCONN instead of success which indicates connection is built up in
* background already. No need to call connect() again. */
if (ret == 0 || ((errno == EINPROGRESS || errno == ECONNREFUSED) && sock->asyncFlag) || errno == EISCONN) {
/* If connect() fails with errno == EAGAIN/EINPROGRESS/ETIMEDOUT, we may want to try connect again.
* However, it can return EISCONN instead of success which indicates connection is built up in
* background already. No need to call connect() again. */
if (ret == 0 || errno == EISCONN) {
sock->fd = fd;
return ncclSuccess;
}
} else {
sock->fd = fd;
return ncclSuccess;
}
@@ -451,17 +459,26 @@ retry:
ncclResult_t ncclSocketAccept(struct ncclSocket* sock, struct ncclSocket* listenSocket) {
socklen_t socklen = sizeof(union ncclSocketAddress);
struct pollfd pollfd;
int tmpFd = sock->fd = -1;
int pollret;
do {
if (listenSocket->abortFlag) NEQCHECK(*listenSocket->abortFlag, 0);
pollfd.fd = listenSocket->fd;
pollfd.events = POLLIN;
retry:
if ((pollret = poll(&pollfd, 1, listenSocket->asyncFlag ? 0 : 100)) < 0) {
return ncclSystemError;
} else {
tmpFd = accept(listenSocket->fd, &sock->addr.sa, &socklen);
} while ((errno == EAGAIN || errno == EWOULDBLOCK) && tmpFd == -1 && !listenSocket->asyncFlag);
}
if (!listenSocket->asyncFlag) {
/* blocking socket, if tmpFd is still -1, we need to retry */
if (tmpFd == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
if (listenSocket->abortFlag && *listenSocket->abortFlag != 0) return ncclInternalError;
goto retry;
}
EQCHECK(tmpFd, -1);
} else if (tmpFd == -1 && errno != EAGAIN && errno != EWOULDBLOCK) {
return ncclSystemError;
}
sock->fd = tmpFd;
@@ -491,7 +508,7 @@ static ncclResult_t ncclSocketProgressOpt(int op, struct ncclSocket* sock, void*
char line[SOCKET_NAME_MAXLEN+1];
do {
if (op == NCCL_SOCKET_RECV) bytes = recv(sock->fd, data+(*offset), size-(*offset), block ? 0 : MSG_DONTWAIT);
if (op == NCCL_SOCKET_SEND) bytes = send(sock->fd, data+(*offset), size-(*offset), block ? 0 : MSG_DONTWAIT);
if (op == NCCL_SOCKET_SEND) bytes = send(sock->fd, data+(*offset), size-(*offset), block ? MSG_NOSIGNAL : MSG_DONTWAIT | MSG_NOSIGNAL);
if (op == NCCL_SOCKET_RECV && bytes == 0) {
*closed = 1;
return ncclSuccess;
@@ -507,7 +524,7 @@ static ncclResult_t ncclSocketProgressOpt(int op, struct ncclSocket* sock, void*
(*offset) += bytes;
if (sock->abortFlag && *sock->abortFlag != 0) {
INFO(NCCL_NET, "Socket progress: abort called");
return ncclSystemError;
return ncclInternalError;
}
} while (bytes > 0 && (*offset) < size);
return ncclSuccess;