Merge remote-tracking branch 'nccl/master' into develop
[ROCm/rccl commit: a6bf9bfc9e]
This commit is contained in:
BertanDogancay
+178
-119
@@ -14,6 +14,18 @@
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
#include "param.h"
|
||||
#include <time.h>
|
||||
|
||||
NCCL_PARAM(RetryCnt, "SOCKET_RETRY_CNT", 34);
|
||||
NCCL_PARAM(RetryTimeOut, "SOCKET_RETRY_SLEEP_MSEC", 100);
|
||||
static void msleep(unsigned int time_msec) {
|
||||
const long c_1e6 = 1e6;
|
||||
struct timespec tv = (struct timespec){
|
||||
.tv_sec = time_msec / 1000,
|
||||
.tv_nsec = (time_msec % 1000) * c_1e6,
|
||||
};
|
||||
nanosleep(&tv, NULL);
|
||||
}
|
||||
|
||||
RCCL_PARAM(SocketReuseAddr, "SOCKET_REUSEADDR", 0);
|
||||
RCCL_PARAM(SocketLinger, "SOCKET_LINGER", -1);
|
||||
@@ -31,8 +43,13 @@ static ncclResult_t socketProgressOpt(int op, struct ncclSocket* sock, void* ptr
|
||||
return ncclSuccess;
|
||||
}
|
||||
if (bytes == -1) {
|
||||
if ((op == NCCL_SOCKET_SEND && errno == EPIPE) || (op == NCCL_SOCKET_RECV && errno == ECONNRESET)) {
|
||||
*closed = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) {
|
||||
WARN("socketProgressOpt: Call to recv from %s failed : %s", ncclSocketToString(&sock->addr, line), strerror(errno));
|
||||
WARN("socketProgressOpt: Call to %s %s failed : %s", (op == NCCL_SOCKET_RECV ? "recv from" : "send to"),
|
||||
ncclSocketToString(&sock->addr, line), strerror(errno));
|
||||
return ncclRemoteError;
|
||||
} else {
|
||||
bytes = 0;
|
||||
@@ -43,17 +60,22 @@ static ncclResult_t socketProgressOpt(int op, struct ncclSocket* sock, void* ptr
|
||||
INFO(NCCL_NET, "socketProgressOpt: abort called");
|
||||
return ncclInternalError;
|
||||
}
|
||||
} while (bytes > 0 && (*offset) < size);
|
||||
} while (sock->asyncFlag == 0 && bytes > 0 && (*offset) < size);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t socketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset) {
|
||||
static ncclResult_t socketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset, int* pclosed = NULL) {
|
||||
int closed;
|
||||
NCCLCHECK(socketProgressOpt(op, sock, ptr, size, offset, 0 /*block*/, &closed));
|
||||
if (closed) {
|
||||
char line[SOCKET_NAME_MAXLEN+1];
|
||||
WARN("socketProgress: Connection closed by remote peer %s", ncclSocketToString(&sock->addr, line, 0));
|
||||
return ncclRemoteError;
|
||||
if (pclosed) {
|
||||
*pclosed = closed;
|
||||
return ncclSuccess;
|
||||
} else {
|
||||
char line[SOCKET_NAME_MAXLEN+1];
|
||||
WARN("socketProgress: Connection closed by remote peer %s", ncclSocketToString(&sock->addr, line, 0));
|
||||
return ncclRemoteError;
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -68,9 +90,9 @@ static ncclResult_t socketWait(int op, struct ncclSocket* sock, void* ptr, int s
|
||||
*
|
||||
* Output: "IPv4/IPv6 address<port>"
|
||||
*/
|
||||
const char *ncclSocketToString(union ncclSocketAddress *addr, char *buf, const int numericHostForm /*= 1*/) {
|
||||
const char *ncclSocketToString(const union ncclSocketAddress *addr, char *buf, const int numericHostForm /*= 1*/) {
|
||||
if (buf == NULL || addr == NULL) return NULL;
|
||||
struct sockaddr *saddr = &addr->sa;
|
||||
const struct sockaddr *saddr = &addr->sa;
|
||||
if (saddr->sa_family != AF_INET && saddr->sa_family != AF_INET6) { buf[0]='\0'; return buf; }
|
||||
char host[NI_MAXHOST], service[NI_MAXSERV];
|
||||
/* NI_NUMERICHOST: If set, then the numeric form of the hostname is returned.
|
||||
@@ -375,10 +397,9 @@ ncclResult_t ncclSocketListen(struct ncclSocket* sock) {
|
||||
if (socketToPort(&sock->addr)) {
|
||||
// Port is forced by env. Make sure we get the port.
|
||||
int opt = 1;
|
||||
#if defined(SO_REUSEPORT)
|
||||
SYSCHECK(setsockopt(sock->fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, sizeof(opt)), "setsockopt");
|
||||
#else
|
||||
SYSCHECK(setsockopt(sock->fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)), "setsockopt");
|
||||
#if defined(SO_REUSEPORT)
|
||||
SYSCHECK(setsockopt(sock->fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)), "setsockopt");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -417,6 +438,15 @@ static ncclResult_t socketTryAccept(struct ncclSocket* sock) {
|
||||
sock->fd = accept(sock->acceptFd, (struct sockaddr*)&sock->addr, &socklen);
|
||||
if (sock->fd != -1) {
|
||||
sock->state = ncclSocketStateAccepted;
|
||||
} else if (errno == ENETDOWN || errno == EPROTO || errno == ENOPROTOOPT || errno == EHOSTDOWN ||
|
||||
errno == ENONET || errno == EHOSTUNREACH || errno == EOPNOTSUPP || errno == ENETUNREACH) {
|
||||
/* per accept's man page, for linux sockets, the following errors might be already pending errors
|
||||
* and should be considered as EAGAIN. To avoid infinite loop in case of errors, we use the retry count*/
|
||||
if (++sock->errorRetries == ncclParamRetryCnt()) {
|
||||
WARN("socketTryAccept: exceeded error retry count (%d), %s", sock->errorRetries, strerror(errno));
|
||||
return ncclSystemError;
|
||||
}
|
||||
INFO(NCCL_ALL, "Call to accept returned %s, retrying", strerror(errno));
|
||||
} else if (errno != EAGAIN && errno != EWOULDBLOCK) {
|
||||
WARN("socketTryAccept: Accept failed: %s", strerror(errno));
|
||||
return ncclSystemError;
|
||||
@@ -424,72 +454,118 @@ static ncclResult_t socketTryAccept(struct ncclSocket* sock) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t socketSetFlags(struct ncclSocket* sock) {
|
||||
const int one = 1;
|
||||
/* Set socket as non-blocking if async or if we need to be able to abort */
|
||||
if ((sock->asyncFlag || sock->abortFlag) && sock->fd >= 0) {
|
||||
int flags;
|
||||
SYSCHECK(flags = fcntl(sock->fd, F_GETFL), "fcntl");
|
||||
SYSCHECK(fcntl(sock->fd, F_SETFL, flags | O_NONBLOCK), "fcntl");
|
||||
}
|
||||
SYSCHECK(setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t socketFinalizeAccept(struct ncclSocket* sock) {
|
||||
uint64_t magic;
|
||||
enum ncclSocketType type;
|
||||
int received = 0;
|
||||
const int one = 1;
|
||||
SYSCHECK(setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
|
||||
int received;
|
||||
// once accepted, linux sockets do NOT inherit file status flags such as O_NONBLOCK (BSD ones do)
|
||||
NCCLCHECK(socketSetFlags(sock));
|
||||
|
||||
NCCLCHECK(ncclSocketProgress(NCCL_SOCKET_RECV, sock, &magic, sizeof(magic), &received));
|
||||
if (received == 0) return ncclSuccess;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &magic, sizeof(magic), &received));
|
||||
if (magic != sock->magic) {
|
||||
WARN("socketFinalizeAccept: wrong magic %lx != %lx", magic, sock->magic);
|
||||
close(sock->fd);
|
||||
sock->fd = -1;
|
||||
// Ignore spurious connection and accept again
|
||||
sock->state = ncclSocketStateAccepting;
|
||||
return ncclSuccess;
|
||||
} else {
|
||||
received = 0;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &type, sizeof(type), &received));
|
||||
if (type != sock->type) {
|
||||
WARN("socketFinalizeAccept: wrong type %d != %d", type, sock->type);
|
||||
sock->state = ncclSocketStateError;
|
||||
if (sock->asyncFlag == 0 || sock->finalizeCounter < sizeof(magic)) {
|
||||
if (sock->asyncFlag == 0) {
|
||||
received = 0;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &magic, sizeof(magic), &received));
|
||||
} else {
|
||||
received = sock->finalizeCounter;
|
||||
NCCLCHECK(socketProgress(NCCL_SOCKET_RECV, sock, sock->finalizeBuffer, sizeof(magic), &received));
|
||||
sock->finalizeCounter = received;
|
||||
if (received < sizeof(magic)) return ncclSuccess;
|
||||
memcpy(&magic, sock->finalizeBuffer, sizeof(magic));
|
||||
}
|
||||
if (magic != sock->magic) {
|
||||
WARN("socketFinalizeAccept: wrong magic %lx != %lx", magic, sock->magic);
|
||||
close(sock->fd);
|
||||
sock->fd = -1;
|
||||
return ncclInternalError;
|
||||
} else {
|
||||
sock->state = ncclSocketStateReady;
|
||||
// Ignore spurious connection and accept again
|
||||
sock->state = ncclSocketStateAccepting;
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
if (sock->asyncFlag == 0) {
|
||||
received = 0;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &type, sizeof(type), &received));
|
||||
} else {
|
||||
received = sock->finalizeCounter - sizeof(magic);
|
||||
NCCLCHECK(socketProgress(NCCL_SOCKET_RECV, sock, sock->finalizeBuffer, sizeof(type), &received));
|
||||
sock->finalizeCounter = received + sizeof(magic);
|
||||
if (received < sizeof(type)) return ncclSuccess;
|
||||
memcpy(&type, sock->finalizeBuffer, sizeof(type));
|
||||
}
|
||||
if (type != sock->type) {
|
||||
WARN("socketFinalizeAccept: wrong type %d != %d", type, sock->type);
|
||||
sock->state = ncclSocketStateError;
|
||||
close(sock->fd);
|
||||
sock->fd = -1;
|
||||
return ncclInternalError;
|
||||
} else {
|
||||
sock->state = ncclSocketStateReady;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t socketStartConnect(struct ncclSocket* sock) {
|
||||
/* blocking/non-blocking connect() is determined by asyncFlag. */
|
||||
int ret = connect(sock->fd, &sock->addr.sa, sock->salen);
|
||||
|
||||
if (ret == 0) {
|
||||
static ncclResult_t socketResetFd(struct ncclSocket* sock) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
int fd = -1;
|
||||
SYSCHECKGOTO(fd = socket(sock->addr.sa.sa_family, SOCK_STREAM, 0), "socket", ret, cleanup);
|
||||
// if sock->fd is valid, close it and reuse its number
|
||||
if (sock->fd != -1) {
|
||||
SYSCHECKGOTO(dup2(fd, sock->fd), "dup2", ret, cleanup);
|
||||
SYSCHECKGOTO(close(fd), "close", ret, cleanup);
|
||||
} else {
|
||||
sock->fd = fd;
|
||||
}
|
||||
NCCLCHECKGOTO(socketSetFlags(sock), ret, exit);
|
||||
exit:
|
||||
return ret;
|
||||
cleanup:
|
||||
// cleanup fd, leave sock->fd untouched
|
||||
if (fd != -1) {
|
||||
(void)close(fd);
|
||||
}
|
||||
goto exit;
|
||||
}
|
||||
static ncclResult_t socketConnectCheck(struct ncclSocket* sock, int errCode, const char funcName[]) {
|
||||
if (errCode == 0) {
|
||||
sock->state = ncclSocketStateConnected;
|
||||
return ncclSuccess;
|
||||
} else if (errno == EINPROGRESS) {
|
||||
} else if (errCode == EINPROGRESS) {
|
||||
sock->state = ncclSocketStateConnectPolling;
|
||||
return ncclSuccess;
|
||||
} else if (errno == ECONNREFUSED) {
|
||||
if (++sock->refusedRetries == RETRY_REFUSED_TIMES) {
|
||||
sock->state = ncclSocketStateError;
|
||||
WARN("socketStartConnect: exceeded retries (%d)", sock->refusedRetries);
|
||||
return ncclRemoteError;
|
||||
} else if (errCode == ETIMEDOUT || errCode == EHOSTUNREACH || errCode == ECONNREFUSED) {
|
||||
if (sock->customRetry == 0) {
|
||||
if (sock->errorRetries++ == ncclParamRetryCnt()) {
|
||||
sock->state = ncclSocketStateError;
|
||||
WARN("%s: connect returned %s, exceeded error retry count (%d)", funcName, strerror(errCode), sock->errorRetries);
|
||||
return ncclRemoteError;
|
||||
}
|
||||
unsigned int sleepTime = sock->errorRetries * ncclParamRetryTimeOut();
|
||||
INFO(NCCL_ALL, "%s: connect returned %s, retrying (%d/%ld) after sleep for %u msec", funcName, strerror(errCode), sock->errorRetries, ncclParamRetryCnt(), sleepTime);
|
||||
msleep(sleepTime);
|
||||
}
|
||||
usleep(SLEEP_INT);
|
||||
if (sock->refusedRetries % 1000 == 0) INFO(NCCL_ALL, "Call to connect returned %s, retrying", strerror(errno));
|
||||
return ncclSuccess;
|
||||
} else if (errno == ETIMEDOUT) {
|
||||
if (++sock->timedOutRetries == RETRY_TIMEDOUT_TIMES) {
|
||||
sock->state = ncclSocketStateError;
|
||||
WARN("socketStartConnect: exceeded timeouts (%d)", sock->timedOutRetries);
|
||||
return ncclRemoteError;
|
||||
}
|
||||
usleep(SLEEP_INT);
|
||||
return ncclSuccess;
|
||||
NCCLCHECK(socketResetFd(sock)); /* in case of failure in connect, socket state is unspecified */
|
||||
sock->state = ncclSocketStateConnecting;
|
||||
} else {
|
||||
char line[SOCKET_NAME_MAXLEN+1];
|
||||
sock->state = ncclSocketStateError;
|
||||
WARN("socketStartConnect: Connect to %s failed : %s", ncclSocketToString(&sock->addr, line), strerror(errno));
|
||||
WARN("%s: Connect to %s failed : %s", funcName, ncclSocketToString(&sock->addr, line), strerror(errCode));
|
||||
return ncclSystemError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
static ncclResult_t socketStartConnect(struct ncclSocket* sock) {
|
||||
/* blocking/non-blocking connect() is determined by asyncFlag. */
|
||||
int ret = connect(sock->fd, &sock->addr.sa, sock->salen);
|
||||
return socketConnectCheck(sock, (ret == -1) ? errno : 0, __func__);
|
||||
}
|
||||
|
||||
static ncclResult_t socketPollConnect(struct ncclSocket* sock) {
|
||||
@@ -514,33 +590,7 @@ static ncclResult_t socketPollConnect(struct ncclSocket* sock) {
|
||||
|
||||
/* check socket status */
|
||||
SYSCHECK(getsockopt(sock->fd, SOL_SOCKET, SO_ERROR, (void*)&ret, &rlen), "getsockopt");
|
||||
|
||||
if (ret == 0) {
|
||||
sock->state = ncclSocketStateConnected;
|
||||
} else if (ret == ECONNREFUSED) {
|
||||
if (++sock->refusedRetries == RETRY_REFUSED_TIMES) {
|
||||
sock->state = ncclSocketStateError;
|
||||
WARN("socketPollConnect: exceeded retries (%d)", sock->refusedRetries);
|
||||
return ncclRemoteError;
|
||||
}
|
||||
if (sock->refusedRetries % 1000 == 0) INFO(NCCL_ALL, "Call to connect returned %s, retrying", strerror(errno));
|
||||
usleep(SLEEP_INT);
|
||||
sock->state = ncclSocketStateConnecting;
|
||||
} else if (ret == ETIMEDOUT) {
|
||||
if (++sock->timedOutRetries == RETRY_TIMEDOUT_TIMES) {
|
||||
sock->state = ncclSocketStateError;
|
||||
WARN("socketPollConnect: exceeded timeouts (%d)", sock->timedOutRetries);
|
||||
return ncclRemoteError;
|
||||
}
|
||||
usleep(SLEEP_INT);
|
||||
sock->state = ncclSocketStateConnecting;
|
||||
} else if (ret != EINPROGRESS) {
|
||||
sock->state = ncclSocketStateError;
|
||||
char line[SOCKET_NAME_MAXLEN+1];
|
||||
WARN("socketPollConnect: Connect to %s returned %d(%s) errno %d(%s)", ncclSocketToString(&sock->addr, line), ret, strerror(ret), errno, strerror(errno));
|
||||
return ncclSystemError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
return socketConnectCheck(sock, ret, __func__);
|
||||
}
|
||||
|
||||
ncclResult_t ncclSocketPollConnect(struct ncclSocket* sock) {
|
||||
@@ -553,12 +603,24 @@ ncclResult_t ncclSocketPollConnect(struct ncclSocket* sock) {
|
||||
}
|
||||
|
||||
static ncclResult_t socketFinalizeConnect(struct ncclSocket* sock) {
|
||||
int sent = 0;
|
||||
NCCLCHECK(socketProgress(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
|
||||
if (sent == 0) return ncclSuccess;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
|
||||
sent = 0;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->type, sizeof(sock->type), &sent));
|
||||
int sent;
|
||||
if (sock->asyncFlag == 0) {
|
||||
sent = 0;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
|
||||
sent = 0;
|
||||
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->type, sizeof(sock->type), &sent));
|
||||
} else {
|
||||
if (sock->finalizeCounter < sizeof(sock->magic)) {
|
||||
sent = sock->finalizeCounter;
|
||||
NCCLCHECK(socketProgress(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
|
||||
sock->finalizeCounter = sent;
|
||||
if (sent < sizeof(sock->magic)) return ncclSuccess;
|
||||
}
|
||||
sent = sock->finalizeCounter - sizeof(sock->magic);
|
||||
NCCLCHECK(socketProgress(NCCL_SOCKET_SEND, sock, &sock->type, sizeof(sock->type), &sent));
|
||||
sock->finalizeCounter = sent + sizeof(sock->magic);
|
||||
if (sent < sizeof(sock->type)) return ncclSuccess;
|
||||
}
|
||||
sock->state = ncclSocketStateReady;
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -601,7 +663,6 @@ ncclResult_t ncclSocketReady(struct ncclSocket* sock, int *running) {
|
||||
|
||||
ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
|
||||
char line[SOCKET_NAME_MAXLEN+1];
|
||||
const int one = 1;
|
||||
|
||||
if (sock == NULL) {
|
||||
WARN("ncclSocketConnect: pass NULL socket");
|
||||
@@ -619,9 +680,8 @@ ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
|
||||
}
|
||||
TRACE(NCCL_INIT|NCCL_NET,"Connecting to socket %s", ncclSocketToString(&sock->addr, line));
|
||||
|
||||
SYSCHECK(setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
|
||||
|
||||
sock->state = ncclSocketStateConnecting;
|
||||
sock->finalizeCounter = 0;
|
||||
do {
|
||||
NCCLCHECK(socketProgressState(sock));
|
||||
} while (sock->asyncFlag == 0 &&
|
||||
@@ -667,6 +727,7 @@ ncclResult_t ncclSocketAccept(struct ncclSocket* sock, struct ncclSocket* listen
|
||||
memcpy(sock, listenSock, sizeof(struct ncclSocket));
|
||||
sock->acceptFd = listenSock->fd;
|
||||
sock->state = ncclSocketStateAccepting;
|
||||
sock->finalizeCounter = 0;
|
||||
}
|
||||
|
||||
do {
|
||||
@@ -697,12 +758,11 @@ exit:
|
||||
return ret;
|
||||
}
|
||||
|
||||
ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* addr, uint64_t magic, enum ncclSocketType type, volatile uint32_t* abortFlag, int asyncFlag) {
|
||||
ncclResult_t ncclSocketInit(struct ncclSocket* sock, const union ncclSocketAddress* addr, uint64_t magic, enum ncclSocketType type, volatile uint32_t* abortFlag, int asyncFlag, int customRetry) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
|
||||
if (sock == NULL) goto exit;
|
||||
sock->timedOutRetries = 0;
|
||||
sock->refusedRetries = 0;
|
||||
sock->errorRetries = 0;
|
||||
sock->abortFlag = abortFlag;
|
||||
sock->asyncFlag = asyncFlag;
|
||||
sock->state = ncclSocketStateInitialized;
|
||||
@@ -710,6 +770,7 @@ ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* ad
|
||||
sock->type = type;
|
||||
sock->fd = -1;
|
||||
sock->acceptFd = -1;
|
||||
sock->customRetry = customRetry;
|
||||
|
||||
if (addr) {
|
||||
/* IPv4/IPv6 support */
|
||||
@@ -721,17 +782,11 @@ ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* ad
|
||||
WARN("ncclSocketInit: connecting to address %s with family %d is neither AF_INET(%d) nor AF_INET6(%d)",
|
||||
ncclSocketToString(&sock->addr, line), family, AF_INET, AF_INET6);
|
||||
ret = ncclInternalError;
|
||||
goto fail;
|
||||
goto exit;
|
||||
}
|
||||
sock->salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
|
||||
|
||||
/* Connect to a hostname / port */
|
||||
sock->fd = socket(family, SOCK_STREAM, 0);
|
||||
if (sock->fd == -1) {
|
||||
WARN("ncclSocketInit: Socket creation failed : %s", strerror(errno));
|
||||
ret = ncclSystemError;
|
||||
goto fail;
|
||||
}
|
||||
// in case of error, we close the fd before returning as it's unclear if the caller has to use ncclSocketClose for cleanup
|
||||
NCCLCHECKGOTO(socketResetFd(sock), ret, fail);
|
||||
|
||||
// [RCCL] Runtime socket options
|
||||
if (rcclParamSocketReuseAddr()) {
|
||||
@@ -746,14 +801,6 @@ ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* ad
|
||||
} else {
|
||||
memset(&sock->addr, 0, sizeof(union ncclSocketAddress));
|
||||
}
|
||||
|
||||
/* Set socket as non-blocking if async or if we need to be able to abort */
|
||||
if ((sock->asyncFlag || sock->abortFlag) && sock->fd >= 0) {
|
||||
int flags;
|
||||
SYSCHECKGOTO(flags = fcntl(sock->fd, F_GETFL), "fcntl", ret, fail);
|
||||
SYSCHECKGOTO(fcntl(sock->fd, F_SETFL, flags | O_NONBLOCK), "fcntl", ret, fail);
|
||||
}
|
||||
|
||||
exit:
|
||||
return ret;
|
||||
fail:
|
||||
@@ -764,12 +811,12 @@ fail:
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t ncclSocketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset) {
|
||||
ncclResult_t ncclSocketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset, int* closed) {
|
||||
if (sock == NULL) {
|
||||
WARN("ncclSocketProgress: pass NULL socket");
|
||||
return ncclInvalidArgument;
|
||||
}
|
||||
NCCLCHECK(socketProgress(op, sock, ptr, size, offset));
|
||||
NCCLCHECK(socketProgress(op, sock, ptr, size, offset, closed));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -802,7 +849,7 @@ ncclResult_t ncclSocketRecv(struct ncclSocket* sock, void* ptr, int size) {
|
||||
WARN("ncclSocketRecv: pass NULL socket");
|
||||
return ncclInvalidArgument;
|
||||
}
|
||||
if (sock->state != ncclSocketStateReady) {
|
||||
if (sock->state != ncclSocketStateReady && sock->state != ncclSocketStateTerminating) {
|
||||
WARN("ncclSocketRecv: socket state (%d) is not ready", sock->state);
|
||||
return ncclInternalError;
|
||||
}
|
||||
@@ -816,7 +863,8 @@ ncclResult_t ncclSocketSendRecv(struct ncclSocket* sendSock, void* sendPtr, int
|
||||
WARN("ncclSocketSendRecv: invalid socket %p/%p", sendSock, recvSock);
|
||||
return ncclInternalError;
|
||||
}
|
||||
if (sendSock->state != ncclSocketStateReady || recvSock->state != ncclSocketStateReady) {
|
||||
if (sendSock->state != ncclSocketStateReady ||
|
||||
(recvSock->state != ncclSocketStateReady && recvSock->state != ncclSocketStateTerminating)) {
|
||||
WARN("ncclSocketSendRecv: socket state (%d/%d) is not ready", sendSock->state, recvSock->state);
|
||||
return ncclInternalError;
|
||||
}
|
||||
@@ -860,9 +908,20 @@ ncclResult_t ncclSocketTryRecv(struct ncclSocket* sock, void* ptr, int size, int
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclSocketClose(struct ncclSocket* sock) {
|
||||
// Make it possible to close just one part of a socket.
|
||||
ncclResult_t ncclSocketShutdown(struct ncclSocket* sock, int how) {
|
||||
if (sock != NULL) {
|
||||
if (sock->fd >= 0) {
|
||||
shutdown(sock->fd, how);
|
||||
}
|
||||
sock->state = ncclSocketStateTerminating;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclSocketClose(struct ncclSocket* sock) {
|
||||
if (sock != NULL) {
|
||||
if (sock->state > ncclSocketStateNone && sock->state < ncclSocketStateNum && sock->fd >= 0) {
|
||||
/* shutdown() is needed to send FIN packet to proxy thread; shutdown() is not affected
|
||||
* by refcount of fd, but close() is. close() won't close a fd and send FIN packet if
|
||||
* the fd is duplicated (e.g. fork()). So shutdown() guarantees the correct and graceful
|
||||
|
||||
Reference in New Issue
Block a user