AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert re-usage of connect and listen ports

Browse Source 
[ROCm/rccl commit: 12ac20ade5]
This commit is contained in:
BertanDogancay
2024-02-01 10:03:13 -08:00
parent ef57c82b60
commit b17f5947ba
3 changed files with 6 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/misc/socket.cc
+2 -44
View File
@@ -11,15 +11,9 @@
#include <unistd.h>
#include <ifaddrs.h>
#include <net/if.h>
#include "param.h"
#include <vector>
#include <utility>
#include <unordered_set>
#include <unistd.h>
#include <sys/syscall.h>
static std::vector<std::pair<int, std::unordered_set<std::string>>> clientPortPool;
#include "param.h"
static ncclResult_t socketProgressOpt(int op, struct ncclSocket* sock, void* ptr, int size, int* offset, int block, int* closed) {
int bytes = 0;
@@ -598,7 +592,7 @@ ncclResult_t ncclSocketReady(struct ncclSocket* sock, int *running) {
return ncclSuccess;
}
ncclResult_t ncclSocketConnect(struct ncclSocket* sock, int portReuse) {
ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
char line[SOCKET_NAME_MAXLEN+1];
const int one = 1;
@@ -620,42 +614,6 @@ ncclResult_t ncclSocketConnect(struct ncclSocket* sock, int portReuse) {
SYSCHECK(setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
if (portReuse) {
int family = sock->addr.sa.sa_family;
if (family != AF_INET && family != AF_INET6) {
WARN("Net : connecting to address %s with family %d is neither AF_INET(%d) nor AF_INET6(%d)",
ncclSocketToString(&sock->addr, line), family, AF_INET, AF_INET6);
return ncclInternalError;
}
int salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); // pre-define ports according to tid, to avoid extra lock for race condition
if (clientPortPool.size() == 0) {
for (int tid = syscall(SYS_gettid), i = 1; i < 5; i++) {
clientPortPool.push_back(std::make_pair(60000 + i * 1000 + tid % 1000, std::unordered_set<std::string>()));
}
}
// find a port without conflict (different remote peer) in best effort
int reused_port = -1;
std::string remote_peer(ncclSocketToString(&sock->addr, line));
for (auto& port : clientPortPool) {
if (port.second.find(remote_peer) == port.second.end()) {
reused_port = port.first;
port.second.insert(remote_peer);
break;
}
}
// bind the port in fd for connect system call
if (reused_port != -1) {
int opt = 1;
SYSCHECK(setsockopt(sock->fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, sizeof(opt)), "setsockopt");
struct sockaddr_in sin;
sin.sin_family = family;
sin.sin_addr.s_addr = htonl(INADDR_ANY);
sin.sin_port = htons(reused_port);
SYSCHECK(bind(sock->fd, (struct sockaddr *)&sin, salen), "bind_client_port");
}
}
sock->state = ncclSocketStateConnecting;
do {
NCCLCHECK(socketProgressState(sock));