AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'nccl/master' into HEAD

Browse Source
This commit is contained in:
Wenkai Du
2023-02-04 01:43:38 +00:00
parent fddb5e6be8 93840e7476
commit e1cb45ff22
62 changed files with 3799 additions and 1447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/transport/coll_net.cc
+3 -3
View File
@@ -1,6 +1,6 @@
/*************************************************************************
* Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -445,7 +445,7 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
map->mems[NCCL_NET_MAP_HOSTMEM].gpuPtr = map->mems[NCCL_NET_MAP_HOSTMEM].cpuPtr;
if (ncclGdrCopy && ncclParamGdrCopySyncEnable()) {
uint64_t *cpuPtr, *gpuPtr;
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 1, &resources->gdrDesc));
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 1, &resources->gdrDesc, comm->sideStream));
resources->gdcSync = cpuPtr;
struct connectMapMem* gdcMem = map->mems+NCCL_NET_MAP_GDCMEM;
@@ -513,7 +513,7 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
map->mems[NCCL_NET_MAP_HOSTMEM].gpuPtr = map->mems[NCCL_NET_MAP_HOSTMEM].cpuPtr;
if (ncclGdrCopy) {
uint64_t *cpuPtr, *gpuPtr;
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 2, &resources->gdrDesc));
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 2, &resources->gdrDesc, comm->sideStream));
if (ncclParamGdrCopySyncEnable()) {
resources->gdcSync = cpuPtr;
src/transport/net.cc
+59 -49
View File
@@ -1,6 +1,6 @@
/*************************************************************************
* Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
*
* See LICENSE.txt for license information
@@ -71,6 +71,7 @@ struct connectMapMem{
char shmPath[PATH_MAX];
cudaIpcMemHandle_t ipc;
};
ncclShmHandle_t handle;
};
struct connectMap {
@@ -247,13 +248,12 @@ static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
}
static ncclResult_t netMapShm(struct connectMapMem* mem) {
NCCLCHECK(ncclShmOpen(mem->shmPath, mem->size, (void**)&mem->cpuPtr, (void**)&mem->gpuPtr, 0));
NCCLCHECK(ncclShmUnlink(mem->shmPath));
NCCLCHECK(ncclShmOpen(mem->shmPath, mem->size, (void**)&mem->cpuPtr, (void**)&mem->gpuPtr, -1, &mem->handle));
return ncclSuccess;
}
static ncclResult_t netCreateShm(struct connectMapMem* mem) {
mem->shmPath[0] = '\0'; // Let ncclShmOpen create a tmp file
NCCLCHECK(ncclShmOpen(mem->shmPath, mem->size, (void**)&mem->cpuPtr, NULL, 1));
NCCLCHECK(ncclShmOpen(mem->shmPath, mem->size, (void**)&mem->cpuPtr, NULL, 1, &mem->handle));
return ncclSuccess;
}
@@ -362,7 +362,7 @@ static ncclResult_t sendFree(struct ncclConnector* send) {
struct connectMap* map = (struct connectMap*)(send->transportResources);
if (map) {
if (map->sameProcess == 0) {
NCCLCHECK(ncclShmClose(map->mems[NCCL_NET_MAP_HOSTMEM].cpuPtr, map->mems[NCCL_NET_MAP_HOSTMEM].gpuPtr, map->mems[NCCL_NET_MAP_HOSTMEM].size));
NCCLCHECK(ncclShmClose(map->mems[NCCL_NET_MAP_HOSTMEM].handle));
if (map->mems[NCCL_NET_MAP_DEVMEM].size) {
CUDACHECK(cudaIpcCloseMemHandle(map->mems[NCCL_NET_MAP_DEVMEM].gpuPtr));
}
@@ -395,7 +395,7 @@ static ncclResult_t sharedBuffersInit(struct ncclComm* comm, int cuda, int local
struct ncclProxySharedP2p* state = type == 0 ? &peer->send : &peer->recv;
state->refcount++;
if (state->size == 0) {
state->size = nChannels*NCCL_SHARED_STEPS*comm->p2pNetChunkSize;
state->size = nChannels*NCCL_SHARED_STEPS*comm->p2pChunkSize;
}
if (size) *size = state->size;
@@ -422,7 +422,7 @@ static ncclResult_t sharedBuffersInit(struct ncclComm* comm, int cuda, int local
static ncclResult_t sharedBuffersGet(struct ncclComm* comm, int channel, int slot, int* offset) {
// Use different pools for different channels and also separate send/recv.
int globalSlot = (channel*NCCL_SHARED_STEPS)+slot;
*offset = comm->p2pNetChunkSize * globalSlot;
*offset = comm->p2pChunkSize * globalSlot;
return ncclSuccess;
}
@@ -547,6 +547,7 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
NCCLCHECK(ncclNetConnect(comm, resources->netDev, reqBuff, &resources->netSendComm));
connection->proxyAppendPtr = &connection->proxyAppend;
}
if (resources->netSendComm == NULL) {
*done = 0;
return ncclSuccess;
@@ -605,7 +606,7 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
}
if (ncclGdrCopy && map->sameProcess && ncclParamGdrCopySyncEnable()) {
uint64_t *cpuPtr, *gpuPtr;
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 1, &resources->gdrDesc));
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 1, &resources->gdrDesc, comm->sideStream));
resources->gdcSync = cpuPtr;
struct connectMapMem* gdcMem = map->mems+NCCL_NET_MAP_GDCMEM;
@@ -693,6 +694,7 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
NCCLCHECK(ncclNetAccept(comm, resources->netListenComm, &resources->netRecvComm));
connection->proxyAppendPtr = &connection->proxyAppend;
}
if (resources->netRecvComm == NULL) {
*done = 0;
return ncclSuccess;
@@ -741,7 +743,7 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
map->mems[NCCL_NET_MAP_HOSTMEM].gpuPtr = map->mems[NCCL_NET_MAP_HOSTMEM].cpuPtr;
if (ncclGdrCopy && map->sameProcess) {
uint64_t *cpuPtr, *gpuPtr;
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 2, &resources->gdrDesc));
NCCLCHECK(ncclGdrCudaCalloc(&cpuPtr, &gpuPtr, 2, &resources->gdrDesc, comm->sideStream));
if (ncclParamGdrCopySyncEnable()) {
resources->gdcSync = cpuPtr;
@@ -794,67 +796,75 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
static ncclResult_t sendProxyFree(struct ncclProxyConnection* connection, struct ncclComm* comm) {
struct sendResources* resources = (struct sendResources*)(connection->transportResources);
if (resources == NULL) { // NVB Preconnect
if (connection->state == connSharedInitialized) { // NVB Preconnect
NCCLCHECK(sharedBuffersDestroy(comm, connection->localRank, 0));
return ncclSuccess;
}
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
if (resources->buffers[p]) {
NCCLCHECK(ncclNetDeregMr(comm, resources->netSendComm, resources->mhandles[p]));
if (connection->state == connConnected) {
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
if (resources->buffers[p]) {
NCCLCHECK(ncclNetDeregMr(comm, resources->netSendComm, resources->mhandles[p]));
}
}
}
struct connectMapMem* mems = resources->map.mems;
if (resources->map.sameProcess) {
NCCLCHECK(ncclCudaHostFree(mems[NCCL_NET_MAP_HOSTMEM].cpuPtr));
} else {
NCCLCHECK(ncclShmClose(mems[NCCL_NET_MAP_HOSTMEM].cpuPtr, NULL, mems[NCCL_NET_MAP_HOSTMEM].size));
}
CUDACHECK(cudaFree(mems[NCCL_NET_MAP_DEVMEM].cpuPtr));
if (mems[NCCL_NET_MAP_GDCMEM].cpuPtr) NCCLCHECK(ncclGdrCudaFree(resources->gdrDesc));
if (resources->shared) {
NCCLCHECK(sharedBuffersDestroy(comm, resources->localRank, 0));
if (resources->maxRecvs > 1 && ncclParamNetSharedComms()) {
struct ncclSharedNetComms* comms = comm->proxyState.progressState.netComms[resources->netDev]+resources->remoteRank;
comms->sendRefCount[resources->channelId]--;
if (comms->sendRefCount[resources->channelId] == 0) NCCLCHECK(ncclNetCloseSend(comm, comms->sendComm[resources->channelId]));
struct connectMapMem* mems = resources->map.mems;
if (resources->map.sameProcess) {
NCCLCHECK(ncclCudaHostFree(mems[NCCL_NET_MAP_HOSTMEM].cpuPtr));
} else {
NCCLCHECK(ncclShmClose(mems[NCCL_NET_MAP_HOSTMEM].handle));
}
CUDACHECK(cudaFree(mems[NCCL_NET_MAP_DEVMEM].cpuPtr));
if (mems[NCCL_NET_MAP_GDCMEM].cpuPtr) NCCLCHECK(ncclGdrCudaFree(resources->gdrDesc));
if (resources->shared) {
NCCLCHECK(sharedBuffersDestroy(comm, resources->localRank, 0));
if (resources->maxRecvs > 1 && ncclParamNetSharedComms()) {
struct ncclSharedNetComms* comms = comm->proxyState.progressState.netComms[resources->netDev]+resources->remoteRank;
comms->sendRefCount[resources->channelId]--;
if (comms->sendRefCount[resources->channelId] == 0) NCCLCHECK(ncclNetCloseSend(comm, comms->sendComm[resources->channelId]));
} else {
NCCLCHECK(ncclNetCloseSend(comm, resources->netSendComm));
}
} else {
NCCLCHECK(ncclNetCloseSend(comm, resources->netSendComm));
}
} else {
NCCLCHECK(ncclNetCloseSend(comm, resources->netSendComm));
}
free(resources);
if (connection->state == connSetupDone) free(resources);
return ncclSuccess;
}
static ncclResult_t recvProxyFree(struct ncclProxyConnection* connection, struct ncclComm* comm) {
struct recvResources* resources = (struct recvResources*)(connection->transportResources);
if (resources == NULL) { // NVB Preconnect
if (connection->state == connSharedInitialized) { // NVB Preconnect
NCCLCHECK(sharedBuffersDestroy(comm, connection->localRank, 1));
return ncclSuccess;
}
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
if (resources->buffers[p]) {
NCCLCHECK(ncclNetDeregMr(comm, resources->netRecvComm, resources->mhandles[p]));
if (connection->state == connConnected) {
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
if (resources->buffers[p]) {
NCCLCHECK(ncclNetDeregMr(comm, resources->netRecvComm, resources->mhandles[p]));
}
}
}
struct connectMapMem* mems = resources->map.mems;
NCCLCHECK(ncclCudaHostFree(mems[NCCL_NET_MAP_HOSTMEM].cpuPtr));
CUDACHECK(cudaFree(mems[NCCL_NET_MAP_DEVMEM].cpuPtr));
if (mems[NCCL_NET_MAP_GDCMEM].cpuPtr) NCCLCHECK(ncclGdrCudaFree(resources->gdrDesc));
if (resources->shared) {
NCCLCHECK(sharedBuffersDestroy(comm, resources->localRank, 1));
if (resources->maxRecvs > 1 && ncclParamNetSharedComms()) {
struct ncclSharedNetComms* comms = comm->proxyState.progressState.netComms[resources->netDev]+resources->proxyRank;
comms->recvRefCount[resources->channelId]--;
if (comms->recvRefCount[resources->channelId] == 0) NCCLCHECK(ncclNetCloseRecv(comm, comms->recvComm[resources->channelId]));
struct connectMapMem* mems = resources->map.mems;
NCCLCHECK(ncclCudaHostFree(mems[NCCL_NET_MAP_HOSTMEM].cpuPtr));
CUDACHECK(cudaFree(mems[NCCL_NET_MAP_DEVMEM].cpuPtr));
if (mems[NCCL_NET_MAP_GDCMEM].cpuPtr) NCCLCHECK(ncclGdrCudaFree(resources->gdrDesc));
if (resources->shared) {
NCCLCHECK(sharedBuffersDestroy(comm, resources->localRank, 1));
if (resources->maxRecvs > 1 && ncclParamNetSharedComms()) {
struct ncclSharedNetComms* comms = comm->proxyState.progressState.netComms[resources->netDev]+resources->proxyRank;
comms->recvRefCount[resources->channelId]--;
if (comms->recvRefCount[resources->channelId] == 0) NCCLCHECK(ncclNetCloseRecv(comm, comms->recvComm[resources->channelId]));
} else {
NCCLCHECK(ncclNetCloseRecv(comm, resources->netRecvComm));
}
} else {
NCCLCHECK(ncclNetCloseRecv(comm, resources->netRecvComm));
}
} else {
NCCLCHECK(ncclNetCloseRecv(comm, resources->netRecvComm));
}
free(resources);
if (connection->state == connSetupDone) free(resources);
return ncclSuccess;
}
src/transport/net_ib.cc
+47 -41
View File
@@ -1,6 +1,6 @@
/*************************************************************************
* Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -59,6 +59,7 @@ struct alignas(64) ncclIbDev {
int realPort;
int maxQp;
struct ncclIbMrCache mrCache;
int ar; // ADAPTIVE_ROUTING
};
#define MAX_IB_PORT 15
@@ -82,6 +83,7 @@ NCCL_PARAM(IbSl, "IB_SL", 0);
NCCL_PARAM(IbTc, "IB_TC", 0);
NCCL_PARAM(IbArThreshold, "IB_AR_THRESHOLD", 8192);
NCCL_PARAM(IbPciRelaxedOrdering, "IB_PCI_RELAXED_ORDERING", 2);
NCCL_PARAM(IbAdaptiveRouting, "IB_ADAPTIVE_ROUTING", -2);
NCCL_PARAM(IbSockClientPortReuse, "IB_SOCK_CLIENT_PORT_REUSE", 0);
NCCL_PARAM(IbSockServerPortReuse, "IB_SOCK_SERVER_PORT_REUSE", 0);
@@ -228,6 +230,11 @@ ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction) {
ncclIbDevs[ncclNIbDevs].mrCache.population = 0;
ncclIbDevs[ncclNIbDevs].mrCache.slots = NULL;
// Enable ADAPTIVE_ROUTING by default on IB networks
// But allow it to be overloaded by an env parameter
ncclIbDevs[ncclNIbDevs].ar = (portAttr.link_layer == IBV_LINK_LAYER_INFINIBAND) ? 1 : 0;
if (ncclParamIbAdaptiveRouting() != -2) ncclIbDevs[ncclNIbDevs].ar = ncclParamIbAdaptiveRouting();
pthread_create(&ncclIbAsyncThread, NULL, ncclIbAsyncThreadMain, context);
ncclSetThreadName(ncclIbAsyncThread, "NCCL IbAsync %2d", ncclNIbDevs);
pthread_detach(ncclIbAsyncThread); // will not be pthread_join()'d
@@ -320,11 +327,6 @@ failure:
return ncclSystemError;
}
static ncclResult_t GetSocketAddr(union ncclSocketAddress* addr) {
memcpy(addr, &ncclIbIfAddr, sizeof(*addr));
return ncclSuccess;
}
#define NCCL_NET_IB_MAX_RECVS 8
ncclResult_t ncclIbGetProperties(int dev, ncclNetProperties_t* props) {
@@ -386,6 +388,7 @@ struct ncclIbCommStage {
struct ncclIbHandle {
union ncclSocketAddress connectAddr; // Filled by the target
uint64_t magic; // random number to help debugging
struct ncclIbCommStage stage; // Used by the other side when connecting
};
@@ -398,7 +401,7 @@ struct ncclIbRequest {
struct ncclIbVerbs* verbs;
int type;
int events;
union ncclSocketAddress *addr;
struct ncclSocket* sock;
int nreqs;
union {
struct {
@@ -449,6 +452,7 @@ struct ncclIbSendComm {
struct ibv_qp* qps[NCCL_IB_MAX_QPS];
int nqps;
struct ibv_mr* fifoMr;
int ar;
};
// The SendFifo needs to be 32-byte aligned and each element needs
// to be a 32-byte multiple, so that an entry does not get split and
@@ -593,8 +597,8 @@ ncclResult_t ncclIbListen(int dev, void* opaqueHandle, void** listenComm) {
static_assert(sizeof(struct ncclIbHandle) < NCCL_NET_HANDLE_MAXSIZE, "ncclIbHandle size too large");
memset(handle, 0, sizeof(struct ncclIbHandle));
comm->dev = dev;
comm->sock.asyncFlag = 1; /* nonblocking socket is required by network communication. */
NCCLCHECK(GetSocketAddr(&comm->sock.addr));
handle->magic = NCCL_SOCKET_MAGIC;
NCCLCHECK(ncclSocketInit(&comm->sock, &ncclIbIfAddr, handle->magic, ncclSocketTypeNetIb, NULL, 1));
if (ncclParamIbSockServerPortReuse()) {
// reuse the socket address and fd for listen system call
if (reusedSockfd == -1) {
@@ -608,16 +612,16 @@ ncclResult_t ncclIbListen(int dev, void* opaqueHandle, void** listenComm) {
} else {
NCCLCHECK(ncclSocketListen(&comm->sock));
}
memcpy(&handle->connectAddr, &comm->sock.addr, sizeof(union ncclSocketAddress));
NCCLCHECK(ncclSocketGetAddr(&comm->sock, &handle->connectAddr));
*listenComm = comm;
return ncclSuccess;
}
ncclResult_t ncclIbConnect(int dev, void* opaqueHandle, void** sendComm) {
struct ncclIbHandle* handle = (struct ncclIbHandle*) opaqueHandle;
enum ncclSocketState conState;
struct ncclIbCommStage* stage = &handle->stage;
struct ncclIbSendComm* comm = (struct ncclIbSendComm*)stage->comm;
int ready;
*sendComm = NULL;
if (stage->state == ncclIbCommStateConnect) goto ib_connect_check;
@@ -628,20 +632,15 @@ ncclResult_t ncclIbConnect(int dev, void* opaqueHandle, void** sendComm) {
}
NCCLCHECK(ncclIbMalloc((void**)&comm, sizeof(struct ncclIbSendComm)));
NCCLCHECK(ncclSocketInit(&comm->sock, &handle->connectAddr, NULL, 1));
NCCLCHECK(ncclSocketInit(&comm->sock, &handle->connectAddr, handle->magic, ncclSocketTypeNetIb, NULL, 1));
stage->comm = comm;
stage->state = ncclIbCommStateConnect;
NCCLCHECK(ncclSocketConnect(&comm->sock, ncclParamIbSockClientPortReuse()));
ib_connect_check:
/* since ncclSocketConnect is async, we must check if connection is complete */
NCCLCHECK(ncclGetSocketState(&comm->sock, &conState));
if (conState == ncclSocketConnecting) {
/* expect user to call again */
return ncclSuccess;
} else if (conState == ncclSocketError) {
return ncclRemoteError;
}
NCCLCHECK(ncclSocketReady(&comm->sock, &ready));
if (!ready) return ncclSuccess;
// IB Setup
struct ibv_context* ctx;
@@ -653,6 +652,7 @@ ib_connect_check:
for (int q=0; q<comm->nqps; q++) {
NCCLCHECK(ncclIbCreateQp(ib_port, &comm->verbs, IBV_ACCESS_REMOTE_WRITE, comm->qps+q));
}
comm->ar = ncclIbDevs[dev].ar; // ADAPTIVE_ROUTING
// Send my QP Info to receiver through the socket. Hope this won't block.
struct ibv_port_attr portAttr;
@@ -704,9 +704,10 @@ ncclResult_t ncclIbAccept(void* listenComm, void** recvComm) {
struct ncclIbListenComm* lComm = (struct ncclIbListenComm*)listenComm;
struct ncclIbCommStage* stage = &lComm->stage;
struct ncclIbRecvComm* rComm = (struct ncclIbRecvComm*)stage->comm;
int ready;
*recvComm = NULL;
if (stage->state == ncclIbCommStateAccept) goto ib_accept;
if (stage->state == ncclIbCommStateAccept) goto ib_accept_check;
if (stage->state == ncclIbCommStateRecv) goto ib_recv;
if (stage->state == ncclIbCommStateSend) goto ib_send;
if (stage->state != ncclIbCommStateStart) {
@@ -717,12 +718,12 @@ ncclResult_t ncclIbAccept(void* listenComm, void** recvComm) {
NCCLCHECK(ncclIbMalloc((void**)&rComm, sizeof(struct ncclIbRecvComm)));
stage->comm = rComm;
stage->state = ncclIbCommStateAccept;
NCCLCHECK(ncclSocketInit(&rComm->sock, NULL, lComm->sock.abortFlag, 1));
ib_accept:
NCCLCHECK(ncclSocketInit(&rComm->sock));
NCCLCHECK(ncclSocketAccept(&rComm->sock, &lComm->sock));
if (rComm->sock.fd == -1)
return ncclSuccess;
ib_accept_check:
NCCLCHECK(ncclSocketReady(&rComm->sock, &ready));
if (!ready) return ncclSuccess;
struct ncclIbQpInfo remQpInfo;
stage->state = ncclIbCommStateRecv;
@@ -825,7 +826,7 @@ ncclResult_t ncclIbGetRequest(struct ncclIbVerbs* verbs, struct ncclIbRequest**
if (r->type == NCCL_NET_IB_REQ_UNUSED) {
r->verbs = verbs;
r->events = 1;
r->addr = NULL;
r->sock = NULL;
*req = r;
return ncclSuccess;
}
@@ -1000,8 +1001,8 @@ ncclResult_t ncclIbMultiSend(struct ncclIbSendComm* comm, int slot) {
}
struct ibv_send_wr* lastWr = comm->wrs+nreqs-1;
if (nreqs > 1 || reqs[0]->send.size > ncclParamIbArThreshold()) {
// When using adaptive routing, send the bulk of the data first as an
if (nreqs > 1 || (comm->ar && reqs[0]->send.size > ncclParamIbArThreshold())) {
// When using ADAPTIVE_ROUTING, send the bulk of the data first as an
// RDMA_WRITE, then a 0-byte RDMA_WRITE_WITH_IMM to trigger a remote
// completion.
lastWr++;
@@ -1067,28 +1068,31 @@ ncclResult_t ncclIbIsend(void* sendComm, void* data, int size, int tag, void* mh
// Sanity checks to catch user collective call count/size mismatches
if (size > slots[r].size) {
char line[SOCKET_NAME_MAXLEN+1];
char line[SOCKET_NAME_MAXLEN + 1];
union ncclSocketAddress addr;
ncclSocketGetAddr(&comm->sock, &addr);
WARN("NET/IB : req %d/%d tag %x peer %s collective mismatch error, local size %d remote size %d",
r, nreqs, tag, ncclSocketToString(&comm->sock.addr, line), size, slots[r].size);
r, nreqs, tag, ncclSocketToString(&addr, line), size, slots[r].size);
return ncclInvalidUsage;
} // plus any potential programming errors
else if (slots[r].size < 0 || slots[r].addr == 0 || slots[r].rkey == 0) {
char line[SOCKET_NAME_MAXLEN+1];
WARN("NET/IB : req %d/%d tag %x peer %s posted incorrect receive info: size %d addr %lx rkey %x",
r, nreqs, tag, ncclSocketToString(&comm->sock.addr, line), slots[r].size, slots[r].addr, slots[r].rkey);
char line[SOCKET_NAME_MAXLEN + 1];
union ncclSocketAddress addr;
ncclSocketGetAddr(&comm->sock, &addr);
WARN("NET/IB : req %d/%d tag %x peer %s posted incorrect receive info: size %d addr %lx rkey %x",
r, nreqs, tag, ncclSocketToString(&addr, line), slots[r].size, slots[r].addr, slots[r].rkey);
return ncclInternalError;
}
struct ncclIbRequest* req;
NCCLCHECK(ncclIbGetRequest(&comm->verbs, &req));
req->type = NCCL_NET_IB_REQ_SEND;
req->addr = &comm->sock.addr;
req->sock = &comm->sock;
req->verbs = &comm->verbs;
req->nreqs = nreqs;
req->send.size = size;
req->send.data = data;
req->send.lkey = mr->lkey;
req->send.offset = 0;
req->addr = &comm->sock.addr;
req->events = comm->nqps;
*request = reqs[r] = req;
@@ -1181,7 +1185,7 @@ ncclResult_t ncclIbIrecv(void* recvComm, int n, void** data, int* sizes, int* ta
struct ncclIbRequest* req;
NCCLCHECK(ncclIbGetRequest(&comm->verbs, &req));
req->type = NCCL_NET_IB_REQ_RECV;
req->addr = &comm->sock.addr;
req->sock = &comm->sock;
req->nreqs = n;
for (int i=0; i<n; i++) req->recv.sizes[i] = 0;
@@ -1220,7 +1224,7 @@ ncclResult_t ncclIbIflush(void* recvComm, int n, void** data, int* sizes, void**
struct ncclIbRequest* req;
NCCLCHECK(ncclIbGetRequest(&comm->verbs, &req));
req->type = NCCL_NET_IB_REQ_FLUSH;
req->addr = &comm->sock.addr;
req->sock = &comm->sock;
struct ibv_mr* mr = (struct ibv_mr*)mhandles[last];
struct ibv_send_wr wr;
@@ -1268,8 +1272,10 @@ ncclResult_t ncclIbTest(void* request, int* done, int* sizes) {
struct ibv_wc *wc = wcs+w;
if (wc->status != IBV_WC_SUCCESS) {
char line[SOCKET_NAME_MAXLEN+1];
union ncclSocketAddress addr;
ncclSocketGetAddr(r->sock, &addr);
WARN("NET/IB : Got completion from peer %s with error %d, opcode %d, len %d, vendor err %d",
ncclSocketToString(r->addr, line), wc->status, wc->opcode, wc->byte_len, wc->vendor_err);
ncclSocketToString(&addr, line), wc->status, wc->opcode, wc->byte_len, wc->vendor_err);
return ncclRemoteError;
}
@@ -1301,7 +1307,7 @@ ncclResult_t ncclIbTest(void* request, int* done, int* sizes) {
ncclResult_t ncclIbCloseSend(void* sendComm) {
struct ncclIbSendComm* comm = (struct ncclIbSendComm*)sendComm;
if (comm) {
close(comm->sock.fd);
NCCLCHECK(ncclSocketClose(&comm->sock));
for (int q=0; q<comm->nqps; q++)
if (comm->qps[q] != NULL) NCCLCHECK(wrap_ibv_destroy_qp(comm->qps[q]));
if (comm->fifoMr != NULL) NCCLCHECK(wrap_ibv_dereg_mr(comm->fifoMr));
@@ -1315,7 +1321,7 @@ ncclResult_t ncclIbCloseSend(void* sendComm) {
ncclResult_t ncclIbCloseRecv(void* recvComm) {
struct ncclIbRecvComm* comm = (struct ncclIbRecvComm*)recvComm;
if (comm) {
close(comm->sock.fd);
if (!ncclParamIbSockServerPortReuse() || reusedSockfd != comm->sock.fd) NCCLCHECK(ncclSocketClose(&comm->sock));
for (int q=0; q<comm->nqps; q++)
if (comm->qps[q] != NULL) NCCLCHECK(wrap_ibv_destroy_qp(comm->qps[q]));
if (comm->gpuFlush.enabled) {
@@ -1332,7 +1338,7 @@ ncclResult_t ncclIbCloseRecv(void* recvComm) {
ncclResult_t ncclIbCloseListen(void* listenComm) {
struct ncclIbListenComm* comm = (struct ncclIbListenComm*)listenComm;
if (comm) {
if (!ncclParamIbSockServerPortReuse() || reusedSockfd != comm->sock.fd) close(comm->sock.fd);
NCCLCHECK(ncclSocketClose(&comm->sock));
free(comm);
}
return ncclSuccess;
src/transport/net_socket.cc
+151 -166
View File
@@ -18,16 +18,16 @@
/* Init functions */
static int ncclNetIfs = -1;
struct ncclSocketDev {
struct ncclNetSocketDev {
union ncclSocketAddress addr;
char devName[MAX_IF_NAME_SIZE];
char* pciPath;
};
static struct ncclSocketDev ncclSocketDevs[MAX_IFS];
static struct ncclNetSocketDev ncclNetSocketDevs[MAX_IFS];
pthread_mutex_t ncclSocketLock = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t ncclNetSocketLock = PTHREAD_MUTEX_INITIALIZER;
static ncclResult_t ncclSocketGetPciPath(char* devName, char** pciPath) {
static ncclResult_t ncclNetSocketGetPciPath(char* devName, char** pciPath) {
char devicePath[PATH_MAX];
snprintf(devicePath, PATH_MAX, "/sys/class/net/%s/device", devName);
// May return NULL if the file doesn't exist.
@@ -35,9 +35,9 @@ static ncclResult_t ncclSocketGetPciPath(char* devName, char** pciPath) {
return ncclSuccess;
}
ncclResult_t ncclSocketInit(ncclDebugLogger_t logFunction) {
ncclResult_t ncclNetSocketInit(ncclDebugLogger_t logFunction) {
if (ncclNetIfs == -1) {
pthread_mutex_lock(&ncclSocketLock);
pthread_mutex_lock(&ncclNetSocketLock);
if (ncclNetIfs == -1) {
char names[MAX_IF_NAME_SIZE*MAX_IFS];
union ncclSocketAddress addrs[MAX_IFS];
@@ -52,9 +52,9 @@ ncclResult_t ncclSocketInit(ncclDebugLogger_t logFunction) {
line[0] = '\0';
addrline[SOCKET_NAME_MAXLEN] = '\0';
for (int i=0; i<ncclNetIfs; i++) {
strcpy(ncclSocketDevs[i].devName, names+i*MAX_IF_NAME_SIZE);
memcpy(&ncclSocketDevs[i].addr, addrs+i, sizeof(union ncclSocketAddress));
NCCLCHECK(ncclSocketGetPciPath(ncclSocketDevs[i].devName, &ncclSocketDevs[i].pciPath));
strcpy(ncclNetSocketDevs[i].devName, names+i*MAX_IF_NAME_SIZE);
memcpy(&ncclNetSocketDevs[i].addr, addrs+i, sizeof(union ncclSocketAddress));
NCCLCHECK(ncclNetSocketGetPciPath(ncclNetSocketDevs[i].devName, &ncclNetSocketDevs[i].pciPath));
snprintf(line+strlen(line), MAX_LINE_LEN-strlen(line), " [%d]%s:%s", i, names+i*MAX_IF_NAME_SIZE,
ncclSocketToString(&addrs[i], addrline));
}
@@ -62,17 +62,17 @@ ncclResult_t ncclSocketInit(ncclDebugLogger_t logFunction) {
INFO(NCCL_INIT|NCCL_NET,"NET/Socket : Using%s", line);
}
}
pthread_mutex_unlock(&ncclSocketLock);
pthread_mutex_unlock(&ncclNetSocketLock);
}
return ncclSuccess;
}
ncclResult_t ncclSocketDevices(int* ndev) {
ncclResult_t ncclNetSocketDevices(int* ndev) {
*ndev = ncclNetIfs;
return ncclSuccess;
}
static ncclResult_t ncclSocketGetSpeed(char* devName, int* speed) {
static ncclResult_t ncclNetSocketGetSpeed(char* devName, int* speed) {
*speed = 0;
char speedPath[PATH_MAX];
sprintf(speedPath, "/sys/class/net/%s/speed", devName);
@@ -91,12 +91,12 @@ static ncclResult_t ncclSocketGetSpeed(char* devName, int* speed) {
return ncclSuccess;
}
ncclResult_t ncclSocketGetProperties(int dev, ncclNetProperties_t* props) {
props->name = ncclSocketDevs[dev].devName;
props->pciPath = ncclSocketDevs[dev].pciPath;
ncclResult_t ncclNetSocketGetProperties(int dev, ncclNetProperties_t* props) {
props->name = ncclNetSocketDevs[dev].devName;
props->pciPath = ncclNetSocketDevs[dev].pciPath;
props->guid = dev;
props->ptrSupport = NCCL_PTR_HOST;
NCCLCHECK(ncclSocketGetSpeed(props->name, &props->speed));
NCCLCHECK(ncclNetSocketGetSpeed(props->name, &props->speed));
props->latency = 0; // Not set
props->port = 0;
props->maxComms = 65536;
@@ -104,12 +104,6 @@ ncclResult_t ncclSocketGetProperties(int dev, ncclNetProperties_t* props) {
return ncclSuccess;
}
ncclResult_t GetSocketAddr(int dev, union ncclSocketAddress* addr) {
if (dev >= ncclNetIfs) return ncclInternalError;
memcpy(addr, &ncclSocketDevs[dev].addr, sizeof(*addr));
return ncclSuccess;
}
/* Communication functions */
#define MAX_SOCKETS 64
@@ -120,29 +114,30 @@ ncclResult_t GetSocketAddr(int dev, union ncclSocketAddress* addr) {
NCCL_PARAM(SocketNsocksPerThread, "NSOCKS_PERTHREAD", -2);
NCCL_PARAM(SocketNthreads, "SOCKET_NTHREADS", -2);
enum ncclSocketCommState {
ncclSocketCommStateStart = 0,
ncclSocketCommStateConnect = 1,
ncclSocketCommStateAccept = 3,
ncclSocketCommStateSend = 4,
ncclSocketCommStateRecv = 5,
enum ncclNetSocketCommState {
ncclNetSocketCommStateStart = 0,
ncclNetSocketCommStateConnect = 1,
ncclNetSocketCommStateAccept = 3,
ncclNetSocketCommStateSend = 4,
ncclNetSocketCommStateRecv = 5,
};
struct ncclSocketCommStage {
enum ncclSocketCommState state;
struct ncclNetSocketCommStage {
enum ncclNetSocketCommState state;
uint8_t iteration;
struct ncclSocket* sock;
struct ncclSocketComm* comm;
struct ncclNetSocketComm* comm;
};
struct ncclSocketHandle {
struct ncclNetSocketHandle {
union ncclSocketAddress connectAddr;
uint64_t magic; // random number to help debugging
int nSocks;
int nThreads;
struct ncclSocketCommStage stage;
struct ncclNetSocketCommStage stage;
};
struct ncclSocketTask {
struct ncclNetSocketTask {
int op;
void* data;
int size;
@@ -152,41 +147,41 @@ struct ncclSocketTask {
ncclResult_t result;
};
struct ncclSocketRequest {
struct ncclNetSocketRequest {
int op;
void* data;
int size;
struct ncclSocket* ctrlSock;
int offset;
int used;
struct ncclSocketComm* comm;
struct ncclSocketTask* tasks[MAX_SOCKETS];
struct ncclNetSocketComm* comm;
struct ncclNetSocketTask* tasks[MAX_SOCKETS];
int nSubs;
};
struct ncclSocketTaskQueue {
struct ncclNetSocketTaskQueue {
int next;
int len;
struct ncclSocketTask* tasks;
struct ncclNetSocketTask* tasks;
};
struct ncclSocketThreadResources {
struct ncclSocketTaskQueue threadTaskQueue;
struct ncclNetSocketThreadResources {
struct ncclNetSocketTaskQueue threadTaskQueue;
int stop;
struct ncclSocketComm* comm;
struct ncclNetSocketComm* comm;
pthread_mutex_t threadLock;
pthread_cond_t threadCond;
};
struct ncclSocketListenComm {
struct ncclNetSocketListenComm {
struct ncclSocket sock;
struct ncclSocketCommStage stage;
struct ncclNetSocketCommStage stage;
int nSocks;
int nThreads;
int dev;
};
struct ncclSocketComm {
struct ncclNetSocketComm {
struct ncclSocket ctrlSock;
struct ncclSocket socks[MAX_SOCKETS];
int dev;
@@ -194,15 +189,15 @@ struct ncclSocketComm {
int nSocks;
int nThreads;
int nextSock;
struct ncclSocketRequest requests[MAX_REQUESTS];
struct ncclNetSocketRequest requests[MAX_REQUESTS];
pthread_t helperThread[MAX_THREADS];
struct ncclSocketThreadResources threadResources[MAX_THREADS];
struct ncclNetSocketThreadResources threadResources[MAX_THREADS];
};
void* persistentSocketThread(void *args_) {
struct ncclSocketThreadResources* resource = (struct ncclSocketThreadResources*)args_;
struct ncclSocketComm* comm = resource->comm;
struct ncclSocketTaskQueue* myQueue = &resource->threadTaskQueue;
struct ncclNetSocketThreadResources* resource = (struct ncclNetSocketThreadResources*)args_;
struct ncclNetSocketComm* comm = resource->comm;
struct ncclNetSocketTaskQueue* myQueue = &resource->threadTaskQueue;
int nSocksPerThread = comm->nSocks / comm->nThreads;
while (1) {
int idle = 1;
@@ -212,7 +207,7 @@ void* persistentSocketThread(void *args_) {
do {
repeat = 0;
for (int j=0; j<nSocksPerThread; j++) {
struct ncclSocketTask* r = myQueue->tasks+i+j;
struct ncclNetSocketTask* r = myQueue->tasks+i+j;
if (r != NULL && r->used == 1 && r->offset < r->size) {
r->result = ncclSocketProgress(r->op, r->sock, r->data, r->size, &r->offset);
if (r->result != ncclSuccess) {
@@ -236,7 +231,7 @@ void* persistentSocketThread(void *args_) {
}
}
ncclResult_t ncclSocketGetNsockNthread(int dev, int* ns, int* nt) {
ncclResult_t ncclNetSocketGetNsockNthread(int dev, int* ns, int* nt) {
int nSocksPerThread = ncclParamSocketNsocksPerThread();
int nThreads = ncclParamSocketNthreads();
if (nThreads > MAX_THREADS) {
@@ -247,7 +242,7 @@ ncclResult_t ncclSocketGetNsockNthread(int dev, int* ns, int* nt) {
// Auto-detection
int autoNt=0, autoNs=1; // By default, we only use the main thread and do not spawn extra threads
char vendorPath[PATH_MAX];
snprintf(vendorPath, PATH_MAX, "/sys/class/net/%s/device/vendor", ncclSocketDevs[dev].devName);
snprintf(vendorPath, PATH_MAX, "/sys/class/net/%s/device/vendor", ncclNetSocketDevs[dev].devName);
char* rPath = realpath(vendorPath, NULL);
int fd = open(rPath, O_RDONLY);
free(rPath);
@@ -285,36 +280,20 @@ end:
return ncclSuccess;
}
ncclResult_t ncclSocketNewListenComm(struct ncclSocketListenComm** comm) {
NCCLCHECK(ncclCalloc(comm, 1));
(*comm)->sock.fd = -1;
return ncclSuccess;
}
ncclResult_t ncclSocketNewComm(struct ncclSocketComm** comm) {
NCCLCHECK(ncclCalloc(comm, 1));
(*comm)->ctrlSock.fd = -1;
for (int i=0; i < MAX_SOCKETS; i++) {
(*comm)->socks[i].fd = -1;
}
(*comm)->nextSock = 0;
return ncclSuccess;
}
ncclResult_t ncclSocketListen(int dev, void* opaqueHandle, void** listenComm) {
if (dev < 0) { // data transfer socket is based on specified dev
ncclResult_t ncclNetSocketListen(int dev, void* opaqueHandle, void** listenComm) {
if (dev < 0 || dev >= ncclNetIfs) { // data transfer socket is based on specified dev
return ncclInternalError;
}
struct ncclSocketHandle* handle = (struct ncclSocketHandle*) opaqueHandle;
memset(handle, 0, sizeof(struct ncclSocketHandle));
static_assert(sizeof(struct ncclSocketHandle) <= NCCL_NET_HANDLE_MAXSIZE, "ncclSocketHandle size too large");
struct ncclSocketListenComm* comm;
NCCLCHECK(ncclSocketNewListenComm(&comm));
NCCLCHECK(GetSocketAddr(dev, &comm->sock.addr));
comm->sock.asyncFlag = 1;
struct ncclNetSocketHandle* handle = (struct ncclNetSocketHandle*) opaqueHandle;
memset(handle, 0, sizeof(struct ncclNetSocketHandle));
static_assert(sizeof(struct ncclNetSocketHandle) <= NCCL_NET_HANDLE_MAXSIZE, "ncclNetSocketHandle size too large");
struct ncclNetSocketListenComm* comm;
NCCLCHECK(ncclCalloc(&comm, 1));
handle->magic = NCCL_SOCKET_MAGIC;
NCCLCHECK(ncclSocketInit(&comm->sock, &ncclNetSocketDevs[dev].addr, handle->magic, ncclSocketTypeNetSocket, NULL, 1));
NCCLCHECK(ncclSocketListen(&comm->sock));
memcpy(&handle->connectAddr, &comm->sock.addr, sizeof(union ncclSocketAddress));
NCCLCHECK(ncclSocketGetNsockNthread(dev, &comm->nSocks, &comm->nThreads));
NCCLCHECK(ncclSocketGetAddr(&comm->sock, &handle->connectAddr));
NCCLCHECK(ncclNetSocketGetNsockNthread(dev, &comm->nSocks, &comm->nThreads));
handle->nSocks = comm->nSocks;
handle->nThreads = comm->nThreads;
comm->dev = dev;
@@ -322,46 +301,41 @@ ncclResult_t ncclSocketListen(int dev, void* opaqueHandle, void** listenComm) {
return ncclSuccess;
}
ncclResult_t ncclSocketConnect(int dev, void* opaqueHandle, void** sendComm) {
if (dev < 0) { // data transfer socket is based on specified dev
ncclResult_t ncclNetSocketConnect(int dev, void* opaqueHandle, void** sendComm) {
if (dev < 0 || dev >= ncclNetIfs) { // data transfer socket is based on specified dev
return ncclInternalError;
}
enum ncclSocketState conState;
struct ncclSocketHandle* handle = (struct ncclSocketHandle*) opaqueHandle;
struct ncclSocketCommStage* stage = &handle->stage;
struct ncclSocketComm* comm = stage->comm;
int ready;
struct ncclNetSocketHandle* handle = (struct ncclNetSocketHandle*) opaqueHandle;
struct ncclNetSocketCommStage* stage = &handle->stage;
struct ncclNetSocketComm* comm = stage->comm;
uint8_t i = stage->iteration;
struct ncclSocket* sock = stage->sock;
*sendComm = NULL;
if (stage->state == ncclSocketCommStateConnect) goto socket_connect_check;
if (stage->state == ncclSocketCommStateSend) goto socket_send;
if (stage->state == ncclNetSocketCommStateConnect) goto socket_connect_check;
if (stage->state == ncclNetSocketCommStateSend) goto socket_send;
NCCLCHECK(ncclSocketNewComm(&comm));
NCCLCHECK(ncclCalloc(&comm, 1));
stage->comm = comm;
comm->nSocks = handle->nSocks;
comm->nThreads = handle->nThreads;
comm->dev = dev;
CUDACHECK(cudaGetDevice(&comm->cudaDev));
for (; i<comm->nSocks+1; i++) {
sock = i == comm->nSocks ? &comm->ctrlSock : comm->socks+i;
NCCLCHECK(ncclSocketInit(sock, &handle->connectAddr, NULL, 1));
sock = (i == comm->nSocks) ? &comm->ctrlSock : comm->socks+i;
NCCLCHECK(ncclSocketInit(sock, &handle->connectAddr, handle->magic, ncclSocketTypeNetSocket, NULL, 1));
stage->sock = sock;
stage->state = ncclSocketCommStateConnect;
stage->state = ncclNetSocketCommStateConnect;
stage->iteration = i;
NCCLCHECK(ncclSocketConnect(sock));
socket_connect_check:
NCCLCHECK(ncclGetSocketState(sock, &conState));
if (conState == ncclSocketConnecting) {
/* expect user to call again */
return ncclSuccess;
} else if (conState == ncclSocketError) {
return ncclRemoteError;
}
stage->state = ncclSocketCommStateSend;
NCCLCHECK(ncclSocketReady(sock, &ready));
if (! ready) return ncclSuccess;
stage->state = ncclNetSocketCommStateSend;
socket_send:
int done = 0;
@@ -372,59 +346,63 @@ socket_send:
return ncclSuccess;
}
ncclResult_t ncclSocketAccept(void* listenComm, void** recvComm) {
struct ncclSocketListenComm* lComm = (struct ncclSocketListenComm*)listenComm;
struct ncclSocketCommStage* stage = &lComm->stage;
struct ncclSocketComm* rComm = stage->comm;
ncclResult_t ncclNetSocketAccept(void* listenComm, void** recvComm) {
struct ncclNetSocketListenComm* lComm = (struct ncclNetSocketListenComm*)listenComm;
struct ncclNetSocketCommStage* stage = &lComm->stage;
struct ncclNetSocketComm* rComm = stage->comm;
uint8_t i = stage->iteration;
struct ncclSocket* sock = stage->sock;
int ready;
*recvComm = NULL;
if (stage->state == ncclSocketCommStateAccept) goto socket_accept;
if (stage->state == ncclSocketCommStateRecv) goto socket_recv;
if (stage->state == ncclNetSocketCommStateAccept) goto socket_accept_check;
if (stage->state == ncclNetSocketCommStateRecv) goto socket_recv;
NCCLCHECK(ncclSocketNewComm(&rComm));
NCCLCHECK(ncclCalloc(&rComm, 1));
stage->comm = rComm;
rComm->nSocks = lComm->nSocks;
rComm->nThreads = lComm->nThreads;
rComm->dev = lComm->dev;
CUDACHECK(cudaGetDevice(&rComm->cudaDev));
lComm->sock.asyncFlag = 1;
for (; i<rComm->nSocks+1; i++) {
uint8_t sendSockIdx;
ncclCalloc(&sock, 1);
NCCLCHECK(ncclSocketInit(sock, NULL, lComm->sock.abortFlag, 1));
stage->sock = sock;
stage->state = ncclSocketCommStateAccept;
stage->iteration = i;
socket_accept:
NCCLCHECK(ncclSocketAccept(sock, &lComm->sock));
if (sock->fd == -1) return ncclSuccess;
stage->state = ncclSocketCommStateRecv;
NCCLCHECK(ncclCalloc(&sock, 1));
NCCLCHECK(ncclSocketInit(sock));
stage->sock = sock;
stage->state = ncclNetSocketCommStateAccept;
stage->iteration = i;
NCCLCHECK(ncclSocketAccept(sock, &lComm->sock));
socket_accept_check:
NCCLCHECK(ncclSocketReady(sock, &ready));
if (!ready) return ncclSuccess;
stage->state = ncclNetSocketCommStateRecv;
socket_recv:
int done = 0;
NCCLCHECK(ncclSocketProgress(NCCL_SOCKET_RECV, sock, &sendSockIdx, sizeof(uint8_t), &done));
if (done == 0) return ncclSuccess;
if (sendSockIdx == rComm->nSocks) memcpy(&rComm->ctrlSock, sock, sizeof(struct ncclSocket));
else memcpy(rComm->socks+sendSockIdx, sock, sizeof(struct ncclSocket));
if (sendSockIdx == rComm->nSocks)
memcpy(&rComm->ctrlSock, sock, sizeof(struct ncclSocket));
else
memcpy(rComm->socks+sendSockIdx, sock, sizeof(struct ncclSocket));
free(sock);
}
*recvComm = rComm;
/* reset lComm state */
stage->state = ncclSocketCommStateStart;
stage->state = ncclNetSocketCommStateStart;
stage->iteration = 0;
stage->sock = NULL;
stage->comm = NULL;
return ncclSuccess;
}
ncclResult_t ncclSocketGetRequest(struct ncclSocketComm* comm, int op, void* data, int size, struct ncclSocketRequest** req) {
ncclResult_t ncclNetSocketGetRequest(struct ncclNetSocketComm* comm, int op, void* data, int size, struct ncclNetSocketRequest** req) {
for (int i=0; i<MAX_REQUESTS; i++) {
struct ncclSocketRequest* r = comm->requests+i;
struct ncclNetSocketRequest* r = comm->requests+i;
if (r->used == 0) {
r->op = op;
r->data = data;
@@ -441,10 +419,10 @@ ncclResult_t ncclSocketGetRequest(struct ncclSocketComm* comm, int op, void* dat
return ncclInternalError;
}
ncclResult_t ncclSocketGetTask(struct ncclSocketComm* comm, int op, void* data, int size, struct ncclSocketTask** req) {
ncclResult_t ncclNetSocketGetTask(struct ncclNetSocketComm* comm, int op, void* data, int size, struct ncclNetSocketTask** req) {
int tid = comm->nextSock % comm->nThreads;
struct ncclSocketThreadResources* res = comm->threadResources+tid;
struct ncclSocketTaskQueue* queue = &res->threadTaskQueue;
struct ncclNetSocketThreadResources* res = comm->threadResources+tid;
struct ncclNetSocketTaskQueue* queue = &res->threadTaskQueue;
// create helper threads and prepare per-thread task queue
if (queue->tasks == NULL) {
// each request can be divided up to nSocks tasks, and
@@ -459,12 +437,12 @@ ncclResult_t ncclSocketGetTask(struct ncclSocketComm* comm, int op, void* data,
pthread_create(comm->helperThread+tid, NULL, persistentSocketThread, res);
ncclSetThreadName(comm->helperThread[tid], "NCCL Sock%c%1u%2u%2u", op == NCCL_SOCKET_SEND ? 'S' : 'R', comm->dev, tid, comm->cudaDev);
}
struct ncclSocketTask* r = queue->tasks+queue->next;
struct ncclNetSocketTask* r = queue->tasks+queue->next;
if (r->used == 0) {
r->op = op;
r->data = data;
r->size = size;
r->sock = comm->socks+comm->nextSock;
r->sock = comm->socks + comm->nextSock;
r->offset = 0;
r->result = ncclSuccess;
comm->nextSock = (comm->nextSock + 1) % comm->nSocks;
@@ -480,9 +458,9 @@ ncclResult_t ncclSocketGetTask(struct ncclSocketComm* comm, int op, void* data,
return ncclInternalError;
}
ncclResult_t ncclSocketTest(void* request, int* done, int* size) {
ncclResult_t ncclNetSocketTest(void* request, int* done, int* size) {
*done = 0;
struct ncclSocketRequest *r = (struct ncclSocketRequest*)request;
struct ncclNetSocketRequest *r = (struct ncclNetSocketRequest*)request;
if (r == NULL) {
WARN("NET/Socket : test called with NULL request");
return ncclInternalError;
@@ -500,9 +478,11 @@ ncclResult_t ncclSocketTest(void* request, int* done, int* size) {
// Check size is less or equal to the size provided by the user
if (r->op == NCCL_SOCKET_RECV && data > r->size) {
char line[SOCKET_NAME_MAXLEN+1];
union ncclSocketAddress addr;
ncclSocketGetAddr(r->ctrlSock, &addr);
WARN("NET/Socket : peer %s message truncated : receiving %d bytes instead of %d. If you believe your socket network is in healthy state, \
there may be a mismatch in collective sizes or environment settings (e.g. NCCL_PROTO, NCCL_ALGO) between ranks",
ncclSocketToString(&r->ctrlSock->addr, line), data, r->size);
ncclSocketToString(&addr, line), data, r->size);
return ncclInvalidUsage;
}
r->size = data;
@@ -515,7 +495,7 @@ ncclResult_t ncclSocketTest(void* request, int* done, int* size) {
int taskSize = std::max(MIN_CHUNKSIZE, DIVUP(r->size, r->comm->nSocks));
while (chunkOffset < r->size) {
int chunkSize = std::min(taskSize, r->size-chunkOffset);
NCCLCHECK(ncclSocketGetTask(r->comm, r->op, (char*)(r->data)+chunkOffset, chunkSize, r->tasks+i++));
NCCLCHECK(ncclNetSocketGetTask(r->comm, r->op, (char*)(r->data)+chunkOffset, chunkSize, r->tasks+i++));
chunkOffset += chunkSize;
}
}
@@ -525,7 +505,7 @@ ncclResult_t ncclSocketTest(void* request, int* done, int* size) {
if (r->nSubs > 0) {
int nCompleted = 0;
for (int i=0; i<r->nSubs; i++) {
struct ncclSocketTask* sub = r->tasks[i];
struct ncclNetSocketTask* sub = r->tasks[i];
if (sub->result != ncclSuccess) return sub->result;
if (sub->offset == sub->size) nCompleted++;
}
@@ -534,7 +514,7 @@ ncclResult_t ncclSocketTest(void* request, int* done, int* size) {
*done = 1;
r->used = 0;
for (int i=0; i<r->nSubs; i++) {
struct ncclSocketTask* sub = r->tasks[i];
struct ncclNetSocketTask* sub = r->tasks[i];
sub->used = 0;
}
}
@@ -552,43 +532,45 @@ ncclResult_t ncclSocketTest(void* request, int* done, int* size) {
return ncclSuccess;
}
ncclResult_t ncclSocketRegMr(void* comm, void* data, int size, int type, void** mhandle) {
ncclResult_t ncclNetSocketRegMr(void* comm, void* data, int size, int type, void** mhandle) {
return (type != NCCL_PTR_HOST) ? ncclInternalError : ncclSuccess;
}
ncclResult_t ncclSocketDeregMr(void* comm, void* mhandle) { return ncclSuccess; }
ncclResult_t ncclNetSocketDeregMr(void* comm, void* mhandle) { return ncclSuccess; }
ncclResult_t ncclSocketIsend(void* sendComm, void* data, int size, int tag, void* mhandle, void** request) {
struct ncclSocketComm* comm = (struct ncclSocketComm*)sendComm;
NCCLCHECK(ncclSocketGetRequest(comm, NCCL_SOCKET_SEND, data, size, (struct ncclSocketRequest**)request));
ncclResult_t ncclNetSocketIsend(void* sendComm, void* data, int size, int tag, void* mhandle, void** request) {
struct ncclNetSocketComm* comm = (struct ncclNetSocketComm*)sendComm;
NCCLCHECK(ncclNetSocketGetRequest(comm, NCCL_SOCKET_SEND, data, size, (struct ncclNetSocketRequest**)request));
return ncclSuccess;
}
ncclResult_t ncclSocketIrecv(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request) {
struct ncclSocketComm* comm = (struct ncclSocketComm*)recvComm;
ncclResult_t ncclNetSocketIrecv(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request) {
struct ncclNetSocketComm* comm = (struct ncclNetSocketComm*)recvComm;
if (n != 1) return ncclInternalError;
NCCLCHECK(ncclSocketGetRequest(comm, NCCL_SOCKET_RECV, data[0], sizes[0], (struct ncclSocketRequest**)request));
NCCLCHECK(ncclNetSocketGetRequest(comm, NCCL_SOCKET_RECV, data[0], sizes[0], (struct ncclNetSocketRequest**)request));
return ncclSuccess;
}
ncclResult_t ncclSocketIflush(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request) {
ncclResult_t ncclNetSocketIflush(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request) {
// We don't support CUDA pointers, so we don't need a flush operation
return ncclInternalError;
}
ncclResult_t ncclSocketCloseListen(void* opaqueComm) {
struct ncclSocketListenComm* comm = (struct ncclSocketListenComm*)opaqueComm;
ncclResult_t ncclNetSocketCloseListen(void* opaqueComm) {
struct ncclNetSocketListenComm* comm = (struct ncclNetSocketListenComm*)opaqueComm;
if (comm) {
if (comm->sock.fd != -1) close(comm->sock.fd);
int ready;
NCCLCHECK(ncclSocketReady(&comm->sock, &ready));
if (ready) NCCLCHECK(ncclSocketClose(&comm->sock));
free(comm);
}
return ncclSuccess;
}
ncclResult_t ncclSocketClose(void* opaqueComm) {
struct ncclSocketComm* comm = (struct ncclSocketComm*)opaqueComm;
ncclResult_t ncclNetSocketClose(void* opaqueComm) {
struct ncclNetSocketComm* comm = (struct ncclNetSocketComm*)opaqueComm;
if (comm) {
for (int i=0; i<comm->nThreads; i++) {
struct ncclSocketThreadResources* res = comm->threadResources+i;
struct ncclNetSocketThreadResources* res = comm->threadResources+i;
if (comm->helperThread[i]) {
pthread_mutex_lock(&res->threadLock);
res->stop = 1;
@@ -598,9 +580,12 @@ ncclResult_t ncclSocketClose(void* opaqueComm) {
}
free(res->threadTaskQueue.tasks);
}
if (comm->ctrlSock.fd != -1) close(comm->ctrlSock.fd);
int ready;
NCCLCHECK(ncclSocketReady(&comm->ctrlSock, &ready));
if (ready) NCCLCHECK(ncclSocketClose(&comm->ctrlSock));
for (int i=0; i<comm->nSocks; i++) {
if (comm->socks[i].fd != -1) close(comm->socks[i].fd);
NCCLCHECK(ncclSocketReady(&comm->socks[i], &ready));
if (ready) NCCLCHECK(ncclSocketClose(&comm->socks[i]));
}
free(comm);
}
@@ -609,20 +594,20 @@ ncclResult_t ncclSocketClose(void* opaqueComm) {
ncclNet_t ncclNetSocket = {
"Socket",
ncclSocketInit,
ncclSocketDevices,
ncclSocketGetProperties,
ncclSocketListen,
ncclSocketConnect,
ncclSocketAccept,
ncclSocketRegMr,
ncclNetSocketInit,
ncclNetSocketDevices,
ncclNetSocketGetProperties,
ncclNetSocketListen,
ncclNetSocketConnect,
ncclNetSocketAccept,
ncclNetSocketRegMr,
NULL, // No DMA-BUF support
ncclSocketDeregMr,
ncclSocketIsend,
ncclSocketIrecv,
ncclSocketIflush,
ncclSocketTest,
ncclSocketClose,
ncclSocketClose,
ncclSocketCloseListen
ncclNetSocketDeregMr,
ncclNetSocketIsend,
ncclNetSocketIrecv,
ncclNetSocketIflush,
ncclNetSocketTest,
ncclNetSocketClose,
ncclNetSocketClose,
ncclNetSocketCloseListen
};
src/transport/p2p.cc
+7 -7
View File
@@ -1,6 +1,6 @@
/*************************************************************************
* Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
* Modifications Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
@@ -37,6 +37,7 @@ struct p2pProxyInfo {
struct p2pShm* devShm;
char shmName[7];
int shmSize;
ncclShmHandle_t handle;
// Intermediate step for sender
struct ncclRecvMem* ceRecvMem;
@@ -67,6 +68,7 @@ struct p2pRecvResources {
struct p2pShm* shm;
struct p2pShm* devShm;
int shmSize;
ncclShmHandle_t handle;
};
#include <sys/types.h>
@@ -379,9 +381,7 @@ ncclResult_t p2pRecvConnect(struct ncclComm* comm, struct ncclConnect* connectIn
sprintf(shmPath, "/dev/shm/nccl-%s", info->shmName);
TRACE(NCCL_SHM,"Open shmName %s shmSize %d", shmPath, info->shmSize);
resources->shmSize = info->shmSize;
NCCLCHECK(ncclShmOpen(shmPath, info->shmSize, (void**)&resources->shm, (void**)&resources->devShm, 0));
// Remove the file to ensure proper clean-up
NCCLCHECK(ncclShmUnlink(shmPath));
NCCLCHECK(ncclShmOpen(shmPath, info->shmSize, (void**)&resources->shm, (void**)&resources->devShm, -1, &resources->handle));
recv->conn.tail = &resources->devShm->recvMem.tail;
recv->conn.head = &resources->devShm->sendMem.head;
@@ -424,7 +424,7 @@ ncclResult_t p2pRecvFree(struct ncclConnector* recv) {
if (resources->sendMemIpc) CUDACHECK(cudaIpcCloseMemHandle(resources->sendMemIpc));
if (resources->recvMemIpc) CUDACHECK(cudaIpcCloseMemHandle(resources->recvMemIpc));
if (useMemcpy) {
NCCLCHECK(ncclShmClose(resources->shm, resources->devShm, resources->shmSize));
NCCLCHECK(ncclShmClose(resources->handle));
}
free(resources);
}
@@ -442,7 +442,7 @@ static ncclResult_t p2pSendProxySetup(struct ncclProxyConnection* connection, st
char shmPath[PATH_MAX];
shmPath[0] = '\0';
proxyInfo->shmSize = sizeof(struct ncclSendMem) + sizeof(struct ncclRecvMem);
NCCLCHECK(ncclShmOpen(shmPath, proxyInfo->shmSize, (void**)&proxyInfo->shm, (void**)&proxyInfo->devShm, 1));
NCCLCHECK(ncclShmOpen(shmPath, proxyInfo->shmSize, (void**)&proxyInfo->shm, (void**)&proxyInfo->devShm, 1, &proxyInfo->handle));
TRACE(NCCL_SHM,"Opened shmName %s shmSize %d", shmPath, proxyInfo->shmSize);
memcpy(proxyInfo->shmName, shmPath+sizeof("/dev/shm/nccl-")-1, sizeof(proxyInfo->shmName));
@@ -505,7 +505,7 @@ static ncclResult_t p2pSendProxyFree(struct ncclProxyConnection* connection, str
if (useMemcpy) {
struct p2pProxyInfo* proxyInfo = (struct p2pProxyInfo*)connection->transportResources;
if (proxyInfo) {
NCCLCHECK(ncclShmClose(proxyInfo->shm, proxyInfo->devShm, proxyInfo->shmSize));
NCCLCHECK(ncclShmClose(proxyInfo->handle));
NCCLCHECK(ncclCudaHostFree(proxyInfo->ceRecvMem));
CUDACHECK(cudaFree(proxyInfo->ceDevBuff));
CUDACHECK(cudaStreamDestroy(proxyInfo->stream));
src/transport/shm.cc
+12 -11
View File
@@ -17,18 +17,22 @@ struct shmSendResources {
int remShmSize;
struct ncclRecvMem* remHostMem;
struct ncclRecvMem* devRemHostMem;
ncclShmHandle_t remHandle;
int shmSize;
struct ncclSendMem* hostMem;
struct ncclSendMem* devHostMem;
ncclShmHandle_t hostHandle;
};
struct shmRecvResources {
int remShmSize;
struct ncclSendMem* remHostMem;
struct ncclSendMem* devRemHostMem;
ncclShmHandle_t remHandle;
int shmSize;
struct ncclRecvMem* hostMem;
struct ncclRecvMem* devHostMem;
ncclShmHandle_t hostHandle;
};
#define SHM_SEND_SIDE 1
@@ -84,7 +88,7 @@ static ncclResult_t shmSendSetup(struct ncclComm* comm, struct ncclTopoGraph* gr
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) shmSize += send->comm->buffSizes[p];
}
info->shmSize = resources->shmSize = shmSize;
NCCLCHECK(ncclShmOpen(shmPath, resources->shmSize, (void**)&resources->hostMem, (void**)&resources->devHostMem, 1));
NCCLCHECK(ncclShmOpen(shmPath, resources->shmSize, (void**)&resources->hostMem, (void**)&resources->devHostMem, 1, &resources->hostHandle));
TRACE(NCCL_SHM,"Opened shmName %s shmSize %d", shmPath, info->shmSize);
memcpy(info->shmName, shmPath+sizeof("/dev/shm/nccl-")-1, sizeof(info->shmName));
@@ -107,7 +111,7 @@ static ncclResult_t shmRecvSetup(struct ncclComm* comm, struct ncclTopoGraph* gr
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) shmSize += recv->comm->buffSizes[p];
}
info->shmSize = resources->shmSize = shmSize;
NCCLCHECK(ncclShmOpen(shmPath, resources->shmSize, (void**)&resources->hostMem, (void**)&resources->devHostMem, 1));
NCCLCHECK(ncclShmOpen(shmPath, resources->shmSize, (void**)&resources->hostMem, (void**)&resources->devHostMem, 1, &resources->hostHandle));
TRACE(NCCL_SHM,"Opened shmName %s shmSize %d", shmPath, info->shmSize);
memcpy(info->shmName, shmPath+sizeof("/dev/shm/nccl-")-1, sizeof(info->shmName));
@@ -137,9 +141,7 @@ static ncclResult_t shmSendConnect(struct ncclComm* comm, struct ncclConnect* co
sprintf(shmPath, "/dev/shm/nccl-%s", info->shmName);
resources->remShmSize = info->shmSize;
TRACE(NCCL_SHM,"Open shmName %s shmSize %d", shmPath, info->shmSize);
NCCLCHECK(ncclShmOpen(shmPath, resources->remShmSize, (void**)&resources->remHostMem, (void**)&resources->devRemHostMem, 0));
// Remove the file to ensure proper clean-up
NCCLCHECK(ncclShmUnlink(shmPath));
NCCLCHECK(ncclShmOpen(shmPath, resources->remShmSize, (void**)&resources->remHostMem, (void**)&resources->devRemHostMem, -1, &resources->remHandle));
char* buff = shmLocality == SHM_SEND_SIDE ? (char*)(resources->devHostMem+1) : (char*)(resources->devRemHostMem+1);
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
@@ -172,8 +174,7 @@ static ncclResult_t shmRecvConnect(struct ncclComm* comm, struct ncclConnect* co
sprintf(shmPath, "/dev/shm/nccl-%s", info->shmName);
resources->remShmSize = info->shmSize;
TRACE(NCCL_SHM,"Open shmName %s shmSize %d", shmPath, info->shmSize);
NCCLCHECK(ncclShmOpen(shmPath, resources->remShmSize, (void**)&resources->remHostMem, (void**)&resources->devRemHostMem, 0));
NCCLCHECK(ncclShmUnlink(shmPath));
NCCLCHECK(ncclShmOpen(shmPath, resources->remShmSize, (void**)&resources->remHostMem, (void**)&resources->devRemHostMem, -1, &resources->remHandle));
char* buff = shmLocality == SHM_RECV_SIDE ? (char*)(resources->devHostMem+1) : (char*)(resources->devRemHostMem+1);
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
@@ -196,8 +197,8 @@ static ncclResult_t shmRecvConnect(struct ncclComm* comm, struct ncclConnect* co
static ncclResult_t shmSendFree(struct ncclConnector* send) {
struct shmRecvResources* resources = (struct shmRecvResources*)send->transportResources;
if (resources) {
NCCLCHECK(ncclShmClose(resources->hostMem, resources->devHostMem, resources->shmSize));
NCCLCHECK(ncclShmClose(resources->remHostMem, resources->devRemHostMem, resources->remShmSize));
NCCLCHECK(ncclShmClose(resources->hostHandle));
NCCLCHECK(ncclShmClose(resources->remHandle));
free(resources);
}
return ncclSuccess;
@@ -206,8 +207,8 @@ static ncclResult_t shmSendFree(struct ncclConnector* send) {
static ncclResult_t shmRecvFree(struct ncclConnector* recv) {
struct shmRecvResources* resources = (struct shmRecvResources*)recv->transportResources;
if (resources) {
NCCLCHECK(ncclShmClose(resources->hostMem, resources->devHostMem, resources->shmSize));
NCCLCHECK(ncclShmClose(resources->remHostMem, resources->devRemHostMem, resources->remShmSize));
NCCLCHECK(ncclShmClose(resources->hostHandle));
NCCLCHECK(ncclShmClose(resources->remHandle));
free(resources);
}
return ncclSuccess;