Sylvain Jeaugey
8dcf8e8720
Add new NVLS algorithm for allreduce using NVLink SHARP (intra-node only).
Add new config options: cgaClusterSize, minCTAs, maxCTAs, netName.
Enable LL128 when we use PXN to close rings.
NVTX3 includes update.
Fix crash when one CollNet (SHARP) rail fails to initialize.
[ROCm/rccl commit: 5d3ab08b69]
577 righe
22 KiB
C++
577 righe
22 KiB
C++
/*************************************************************************
|
|
* Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* See LICENSE.txt for license information
|
|
************************************************************************/
|
|
|
|
#include "comm.h"
|
|
#include "graph.h"
|
|
#include "utils.h"
|
|
#include "shm.h"
|
|
|
|
struct ncclP2pBuff {
|
|
void* directPtr;
|
|
cudaIpcMemHandle_t devIpc;
|
|
};
|
|
|
|
struct p2pConnectInfo {
|
|
int rank;
|
|
int read;
|
|
struct ncclP2pBuff p2pBuff;
|
|
// Use by CE memcpy
|
|
char shmName[7];
|
|
int shmSize;
|
|
};
|
|
static_assert(sizeof(struct p2pConnectInfo) <= CONNECT_SIZE, "p2pConnectInfo is too large");
|
|
|
|
struct p2pShm {
|
|
struct ncclSendMem sendMem;
|
|
struct ncclRecvMem recvMem;
|
|
};
|
|
struct p2pProxyInfo {
|
|
// Shared memory between proxy and receiving GPU
|
|
struct p2pShm* shm;
|
|
struct p2pShm* devShm;
|
|
char shmName[7];
|
|
int shmSize;
|
|
ncclShmHandle_t handle;
|
|
|
|
// Intermediate step for sender
|
|
struct ncclRecvMem* ceRecvMem;
|
|
char* ceDevBuff;
|
|
|
|
// Receiver buffer
|
|
char* recvFifo;
|
|
|
|
// Used by progress only
|
|
uint64_t step;
|
|
cudaStream_t stream;
|
|
cudaEvent_t events[NCCL_STEPS];
|
|
};
|
|
static_assert(sizeof(p2pConnectInfo) <= CONNECT_SIZE, "P2P Connect info is too large");
|
|
|
|
struct p2pSendResources {
|
|
struct ncclSendMem* devMem;
|
|
void* sendMemIpc;
|
|
void* recvMemIpc;
|
|
struct p2pProxyInfo proxyInfo;
|
|
};
|
|
|
|
struct p2pRecvResources {
|
|
struct ncclRecvMem* devMem;
|
|
void* sendMemIpc;
|
|
void* recvMemIpc;
|
|
struct p2pShm* shm;
|
|
struct p2pShm* devShm;
|
|
int shmSize;
|
|
ncclShmHandle_t handle;
|
|
};
|
|
|
|
#include <sys/types.h>
|
|
|
|
/* Convert a PCI busId string into a local cudaDev device index (cf. CUDA_VISIBLE_DEVICES) */
|
|
static int busIdToCudaDev(int64_t busId) {
|
|
int ndev;
|
|
if (cudaGetDeviceCount(&ndev) != cudaSuccess)
|
|
return -1;
|
|
for (int i = 0; i < ndev; i++) {
|
|
char devBusIdStr[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE];
|
|
if (cudaDeviceGetPCIBusId(devBusIdStr, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, i) != cudaSuccess)
|
|
return -1;
|
|
int64_t devBusId;
|
|
NCCLCHECK(busIdToInt64(devBusIdStr, &devBusId));
|
|
if (busId == devBusId) return i;
|
|
}
|
|
// BusId was not found in our locally visible CUDA devices
|
|
return -1;
|
|
}
|
|
|
|
NCCL_PARAM(P2pUseCudaMemcpy, "P2P_USE_CUDA_MEMCPY", 0);
|
|
static int useMemcpy = 0;
|
|
static void initCeOperation();
|
|
|
|
/* Determine if two peers can communicate through p2p */
|
|
ncclResult_t p2pCanConnect(int* ret, struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2) {
|
|
initCeOperation();
|
|
|
|
// Rule out different nodes / isolated containers
|
|
if (info1->hostHash != info2->hostHash || info1->shmDev != info2->shmDev) {
|
|
*ret = 0;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
// Check topology / p2p level.
|
|
int intermediateRank;
|
|
NCCLCHECK(ncclTopoCheckP2p(topo, info1->busId, info2->busId, ret, NULL, &intermediateRank));
|
|
if (*ret == 0) return ncclSuccess;
|
|
if (intermediateRank != -1) {
|
|
if (useMemcpy) *ret = 0;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
// Check if NET would work better
|
|
int useNet = 0;
|
|
NCCLCHECK(ncclTopoCheckNet(topo, info1->busId, info2->busId, &useNet));
|
|
if (useNet) {
|
|
*ret = 0;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
// Convert the peer's busId into a local cudaDev index (cf. CUDA_VISIBLE_DEVICES)
|
|
int cudaDev1 = busIdToCudaDev(info1->busId);
|
|
int cudaDev2 = busIdToCudaDev(info2->busId);
|
|
if (cudaDev1 == -1 || cudaDev2 == -1) {
|
|
#if CUDART_VERSION >= 10010
|
|
// CUDA 10.1 and later can use P2P with invisible devices.
|
|
return ncclSuccess;
|
|
#else
|
|
// Peer's CUDA device is not visible in this process : we can't communicate with it.
|
|
*ret = 0;
|
|
return ncclSuccess;
|
|
#endif
|
|
}
|
|
|
|
// Check that CUDA can do P2P
|
|
int p2p;
|
|
if (cudaDeviceCanAccessPeer(&p2p, cudaDev1, cudaDev2) != cudaSuccess) {
|
|
INFO(NCCL_INIT|NCCL_P2P,"peer query failed between dev %d(=%lx) and dev %d(=%lx)",
|
|
cudaDev1, info1->busId, cudaDev2, info2->busId);
|
|
*ret = 0;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
if (p2p != 0) {
|
|
// Cached result of the legacyIPC detection
|
|
static int legacyIPC = -1;
|
|
if (legacyIPC >= 0) {
|
|
*ret = legacyIPC;
|
|
return ncclSuccess;
|
|
}
|
|
// Check that legacy IPC support is available (WSL WAR)
|
|
char *dummy;
|
|
cudaIpcMemHandle_t ipc;
|
|
NCCLCHECK(ncclCudaCalloc(&dummy, CUDA_IPC_MIN));
|
|
if (cudaIpcGetMemHandle(&ipc, dummy) != cudaSuccess) {
|
|
INFO(NCCL_INIT|NCCL_P2P,"Legacy IPC not supported");
|
|
*ret = 0;
|
|
}
|
|
CUDACHECK(cudaFree(dummy));
|
|
legacyIPC = *ret;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
if (p2p == 0) {
|
|
INFO(NCCL_INIT|NCCL_P2P,"Could not enable P2P between dev %d(=%lx) and dev %d(=%lx)",
|
|
cudaDev1, info1->busId, cudaDev2, info2->busId);
|
|
*ret = 0;
|
|
return ncclSuccess;
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
#define TRACE_DUMP_IPC(DEVIPC) \
|
|
do { \
|
|
unsigned long *devIpc = (unsigned long *) (DEVIPC); \
|
|
TRACE(P2P,"IPC: %016lx %016lx %016lx %016lx", devIpc[0], devIpc[1], devIpc[2], devIpc[3]); \
|
|
TRACE(P2P,"IPC: %016lx %016lx %016lx %016lx", devIpc[4], devIpc[5], devIpc[6], devIpc[7]); \
|
|
} while (0)
|
|
|
|
|
|
// Setting this to non zero causes P2P to use Reads rather than Writes
|
|
NCCL_PARAM(P2pReadEnable, "P2P_READ_ENABLE", -2);
|
|
NCCL_PARAM(P2pDirectDisable, "P2P_DIRECT_DISABLE", 0);
|
|
|
|
static ncclResult_t p2pGetInfo(struct ncclTopoSystem* topo, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2, int* read, int* intermediateRank) {
|
|
int p2p;
|
|
// Queries the topology to see if the GPUs are Ampere and
|
|
// connected via NVLink, if so we enable P2P Read by default
|
|
NCCLCHECK(ncclTopoCheckP2p(topo, info1->busId, info2->busId, &p2p, read, intermediateRank));
|
|
|
|
int readEnable = ncclParamP2pReadEnable();
|
|
if (readEnable != -2) *read = readEnable;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t p2pMap(struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclP2pBuff* p2pBuff, void** devMem, void** ipcPtr) {
|
|
if (myInfo->pidHash == peerInfo->pidHash) {
|
|
if (peerInfo->cudaDev != myInfo->cudaDev) {
|
|
// Enable P2P access
|
|
cudaError_t err = cudaDeviceEnablePeerAccess(peerInfo->cudaDev, 0);
|
|
if (err == cudaErrorPeerAccessAlreadyEnabled) {
|
|
cudaGetLastError();
|
|
} else if (err != cudaSuccess) {
|
|
WARN("failed to peer with device %d(=%lx): %d %s",
|
|
peerInfo->cudaDev, peerInfo->busId, err, cudaGetErrorString(err));
|
|
return ncclInternalError;
|
|
}
|
|
}
|
|
*devMem = p2pBuff->directPtr;
|
|
*ipcPtr = NULL;
|
|
} else {
|
|
CUDACHECK(cudaIpcOpenMemHandle(devMem, p2pBuff->devIpc, cudaIpcMemLazyEnablePeerAccess));
|
|
*ipcPtr = *devMem;
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
/* Send: Create and return connect structures for this peer to connect to me */
|
|
ncclResult_t p2pSendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo,
|
|
struct ncclConnect* connectInfo, struct ncclConnector* send, int channelId, int connIndex) {
|
|
struct p2pSendResources* resources;
|
|
NCCLCHECK(ncclCalloc(&resources, 1));
|
|
send->transportResources = resources;
|
|
int useRead, intermediateRank;
|
|
NCCLCHECK(p2pGetInfo(comm->topo, myInfo, peerInfo, &useRead, &intermediateRank));
|
|
if (useMemcpy) useRead = 0;
|
|
|
|
static_assert(sizeof(struct p2pConnectInfo) <= sizeof(struct ncclConnect), "p2p Connect Info is too big");
|
|
struct p2pConnectInfo* info = (struct p2pConnectInfo*)connectInfo;
|
|
info->read = useRead;
|
|
// For CollNet, use write for scatter-reduce (conn 1), read for broadcast-gather (conn 0)
|
|
if (graph && connIndex == 1) info->read = 0;
|
|
const char* useReadStr = info->read ? "/read" : "";
|
|
|
|
int sendSize = sizeof(struct ncclSendMem);
|
|
// For P2P Read the SIMPLE buffer is tagged on the end of the ncclSendMem structure
|
|
if (info->read) sendSize += send->comm->buffSizes[NCCL_PROTO_SIMPLE];
|
|
ALIGN_SIZE(sendSize, CUDA_IPC_MIN);
|
|
|
|
if (intermediateRank == -1) {
|
|
info->rank = myInfo->rank;
|
|
if (myInfo->pidHash == peerInfo->pidHash && useMemcpy == 0) {
|
|
if (ncclParamP2pDirectDisable() == 0) send->conn.flags |= info->read ? NCCL_DIRECT_READ : NCCL_DIRECT_WRITE;
|
|
INFO(NCCL_INIT|NCCL_P2P, "Channel %02d/%01d : %d[%lx] -> %d[%lx] via P2P/direct pointer%s",
|
|
channelId, connIndex, myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId, useReadStr);
|
|
} else {
|
|
send->conn.flags |= info->read ? NCCL_IPC_READ : NCCL_IPC_WRITE;
|
|
INFO(NCCL_INIT|NCCL_P2P,"Channel %02d/%01d : %d[%lx] -> %d[%lx] via P2P/IPC%s%s",
|
|
channelId, connIndex, myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId, useReadStr, useMemcpy ? "/CE" : "");
|
|
}
|
|
} else {
|
|
info->rank = intermediateRank;
|
|
INFO(NCCL_INIT|NCCL_P2P, "Channel %02d/%01d : %d[%lx] -> %d[%lx] via P2P/indirect/%d[%lx]%s",
|
|
channelId, connIndex, myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId, intermediateRank,
|
|
comm->peerInfo[intermediateRank].busId, useReadStr);
|
|
}
|
|
|
|
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_P2P, 1, info->rank, &send->proxyConn));
|
|
if (useMemcpy) {
|
|
NCCLCHECK(ncclProxyCallBlocking(&send->proxyConn, ncclProxyMsgSetup, NULL, 0, &resources->proxyInfo, sizeof(struct p2pProxyInfo)));
|
|
info->shmSize = resources->proxyInfo.shmSize;
|
|
memcpy(info->shmName, resources->proxyInfo.shmName, sizeof(info->shmName));
|
|
} else {
|
|
NCCLCHECK(ncclProxyCallBlocking(&send->proxyConn, ncclProxyMsgSetup, &sendSize, sizeof(int), &info->p2pBuff, sizeof(struct ncclP2pBuff)));
|
|
NCCLCHECK(p2pMap(myInfo, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&resources->devMem, &resources->sendMemIpc));
|
|
}
|
|
|
|
return ncclSuccess;
|
|
}
|
|
|
|
/* Create and return connect structures for this peer to connect to me */
|
|
ncclResult_t p2pRecvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo,
|
|
struct ncclConnect* connectInfo, struct ncclConnector * recv, int channelId, int connIndex) {
|
|
struct p2pRecvResources* resources;
|
|
NCCLCHECK(ncclCalloc(&resources, 1));
|
|
recv->transportResources = resources;
|
|
int useRead, intermediateRank;
|
|
NCCLCHECK(p2pGetInfo(comm->topo, myInfo, peerInfo, &useRead, &intermediateRank));
|
|
|
|
static_assert(sizeof(struct p2pConnectInfo) <= sizeof(struct ncclConnect), "p2p Connect Info is too big");
|
|
struct p2pConnectInfo* info = (struct p2pConnectInfo*)connectInfo;
|
|
info->read = useRead;
|
|
// For CollNet, use write for scatter-reduce (conn 1), read for broadcast-gather (conn 0)
|
|
if (graph && connIndex == 1) info->read = 0;
|
|
|
|
int recvSize = sizeof(struct ncclRecvMem);
|
|
// For P2P Read the SIMPLE buffer is tagged on the end of the ncclSendMem structure
|
|
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) if (!(info->read && p == NCCL_PROTO_SIMPLE)) recvSize += recv->comm->buffSizes[p];
|
|
ALIGN_SIZE(recvSize, CUDA_IPC_MIN);
|
|
|
|
if (intermediateRank == -1) {
|
|
info->rank = myInfo->rank;
|
|
if (myInfo->pidHash == peerInfo->pidHash && useMemcpy == 0) {
|
|
if (ncclParamP2pDirectDisable() == 0) recv->conn.flags |= info->read ? NCCL_DIRECT_READ : NCCL_DIRECT_WRITE;
|
|
} else {
|
|
recv->conn.flags |= info->read ? NCCL_IPC_READ : NCCL_IPC_WRITE;
|
|
}
|
|
} else {
|
|
info->rank = intermediateRank;
|
|
}
|
|
|
|
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_P2P, 0, info->rank, &recv->proxyConn));
|
|
NCCLCHECK(ncclProxyCallBlocking(&recv->proxyConn, ncclProxyMsgSetup, &recvSize, sizeof(int), &info->p2pBuff, sizeof(struct ncclP2pBuff)));
|
|
|
|
NCCLCHECK(p2pMap(myInfo, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&resources->devMem, &resources->recvMemIpc));
|
|
return ncclSuccess;
|
|
}
|
|
|
|
/* Connect/Send to this peer */
|
|
static ncclResult_t p2pSendConnect(struct ncclComm* comm, struct ncclConnect* connectInfo, int nranks, int rank, struct ncclConnector* send) {
|
|
struct p2pSendResources* resources = (struct p2pSendResources*)send->transportResources;
|
|
struct ncclRecvMem* remDevMem;
|
|
struct p2pConnectInfo* info = (struct p2pConnectInfo*)connectInfo;
|
|
|
|
NCCLCHECK(p2pMap(comm->peerInfo+rank, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&remDevMem, &resources->recvMemIpc));
|
|
|
|
char* buff = (char*)(remDevMem+1);
|
|
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
|
if (info->read && p == NCCL_PROTO_SIMPLE) {
|
|
/* For P2P Read the SIMPLE buffer is local (ncclSendMem) */
|
|
if (resources->devMem == NULL) return ncclInternalError; // We should not use read + memcpy
|
|
send->conn.buffs[p] = (char*)(resources->devMem+1);
|
|
} else {
|
|
send->conn.buffs[p] = buff;
|
|
buff += send->comm->buffSizes[p];
|
|
}
|
|
}
|
|
|
|
if (useMemcpy) {
|
|
send->conn.tail = &resources->proxyInfo.ceRecvMem->tail;
|
|
send->conn.sizesFifo = resources->proxyInfo.ceRecvMem->sizesFifo;
|
|
send->conn.head = &resources->proxyInfo.devShm->sendMem.head;
|
|
// Send SIMPLE buff to proxy, and replace it by local buffer
|
|
NCCLCHECK(ncclProxyCallBlocking(&send->proxyConn, ncclProxyMsgConnect, &send->conn.buffs[NCCL_PROTO_SIMPLE], sizeof(void*), NULL, 0));
|
|
send->conn.buffs[NCCL_PROTO_SIMPLE] = resources->proxyInfo.ceDevBuff;
|
|
} else {
|
|
send->conn.tail = &remDevMem->tail;
|
|
send->conn.head = &resources->devMem->head;
|
|
send->conn.ptrExchange = &resources->devMem->ptrExchange;
|
|
send->conn.redOpArgExchange = resources->devMem->redOpArgExchange;
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
/* Connect/Recv from this peer */
|
|
ncclResult_t p2pRecvConnect(struct ncclComm* comm, struct ncclConnect* connectInfo, int nranks, int rank, struct ncclConnector* recv) {
|
|
struct p2pRecvResources* resources = (struct p2pRecvResources*)recv->transportResources;
|
|
struct p2pConnectInfo* info = (struct p2pConnectInfo*)connectInfo;
|
|
|
|
struct ncclSendMem* remDevMem = NULL;
|
|
|
|
if (useMemcpy) {
|
|
char shmPath[PATH_MAX];
|
|
sprintf(shmPath, "/dev/shm/nccl-%s", info->shmName);
|
|
TRACE(NCCL_SHM,"Open shmName %s shmSize %d", shmPath, info->shmSize);
|
|
resources->shmSize = info->shmSize;
|
|
NCCLCHECK(ncclShmOpen(shmPath, info->shmSize, (void**)&resources->shm, (void**)&resources->devShm, -1, &resources->handle));
|
|
|
|
recv->conn.tail = &resources->devShm->recvMem.tail;
|
|
recv->conn.head = &resources->devShm->sendMem.head;
|
|
} else {
|
|
NCCLCHECK(p2pMap(comm->peerInfo+rank, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&remDevMem, &resources->sendMemIpc));
|
|
|
|
recv->conn.tail = &resources->devMem->tail;
|
|
recv->conn.head = &remDevMem->head;
|
|
recv->conn.ptrExchange = &remDevMem->ptrExchange;
|
|
recv->conn.redOpArgExchange = remDevMem->redOpArgExchange;
|
|
}
|
|
|
|
char* buff = (char*)(resources->devMem+1);
|
|
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
|
if (info->read && p == NCCL_PROTO_SIMPLE) {
|
|
if (remDevMem == NULL) return ncclInternalError; // We should not use read + memcpy
|
|
/* For P2P Read the SIMPLE buffer is remote (ncclSendMem) */
|
|
recv->conn.buffs[p] = (char*)(remDevMem+1);
|
|
} else {
|
|
recv->conn.buffs[p] = buff;
|
|
buff += recv->comm->buffSizes[p];
|
|
}
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
ncclResult_t p2pSendFree(struct ncclConnector* send) {
|
|
struct p2pSendResources* resources = (struct p2pSendResources*)send->transportResources;
|
|
if (resources) {
|
|
if (resources->sendMemIpc) CUDACHECK(cudaIpcCloseMemHandle(resources->sendMemIpc));
|
|
if (resources->recvMemIpc) CUDACHECK(cudaIpcCloseMemHandle(resources->recvMemIpc));
|
|
free(resources);
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
ncclResult_t p2pRecvFree(struct ncclConnector* recv) {
|
|
struct p2pRecvResources* resources = (struct p2pRecvResources*)recv->transportResources;
|
|
if (resources) {
|
|
if (resources->sendMemIpc) CUDACHECK(cudaIpcCloseMemHandle(resources->sendMemIpc));
|
|
if (resources->recvMemIpc) CUDACHECK(cudaIpcCloseMemHandle(resources->recvMemIpc));
|
|
if (useMemcpy) {
|
|
NCCLCHECK(ncclShmClose(resources->handle));
|
|
}
|
|
free(resources);
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t p2pSendProxySetup(struct ncclProxyConnection* connection, struct ncclComm* comm, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
|
|
if (useMemcpy) {
|
|
struct p2pProxyInfo* proxyInfo;
|
|
NCCLCHECK(ncclCalloc(&proxyInfo, 1));
|
|
connection->transportResources = proxyInfo;
|
|
|
|
NCCLCHECK(ncclCudaCalloc(&proxyInfo->ceDevBuff, comm->buffSizes[NCCL_PROTO_SIMPLE]));
|
|
|
|
char shmPath[PATH_MAX];
|
|
shmPath[0] = '\0';
|
|
proxyInfo->shmSize = sizeof(struct ncclSendMem) + sizeof(struct ncclRecvMem);
|
|
NCCLCHECK(ncclShmOpen(shmPath, proxyInfo->shmSize, (void**)&proxyInfo->shm, (void**)&proxyInfo->devShm, 1, &proxyInfo->handle));
|
|
TRACE(NCCL_SHM,"Opened shmName %s shmSize %d", shmPath, proxyInfo->shmSize);
|
|
memcpy(proxyInfo->shmName, shmPath+sizeof("/dev/shm/nccl-")-1, sizeof(proxyInfo->shmName));
|
|
|
|
NCCLCHECK(ncclCudaHostCalloc(&proxyInfo->ceRecvMem, 1));
|
|
|
|
if (respSize != sizeof(struct p2pProxyInfo)) return ncclInternalError;
|
|
memcpy(respBuff, proxyInfo, sizeof(struct p2pProxyInfo));
|
|
} else {
|
|
if (reqSize != sizeof(int)) return ncclInternalError;
|
|
int size = *((int*)reqBuff);
|
|
if (respSize != sizeof(struct ncclP2pBuff)) return ncclInternalError;
|
|
struct ncclP2pBuff* p2pBuff = (struct ncclP2pBuff*)respBuff;
|
|
NCCLCHECK(ncclCudaCalloc((char**)&p2pBuff->directPtr, size));
|
|
connection->transportResources = p2pBuff->directPtr;
|
|
cudaError_t res = cudaIpcGetMemHandle(&p2pBuff->devIpc, p2pBuff->directPtr);
|
|
if (res != cudaSuccess) {
|
|
WARN("cudaIpcGetMemHandle failed : %s", cudaGetErrorString(res));
|
|
cudaFree(p2pBuff->directPtr);
|
|
free(p2pBuff);
|
|
CUDACHECK(res);
|
|
}
|
|
}
|
|
*done = 1;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t p2pRecvProxySetup(struct ncclProxyConnection* connection, struct ncclComm* comm, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
|
|
if (reqSize != sizeof(int)) return ncclInternalError;
|
|
int size = *((int*)reqBuff);
|
|
if (respSize != sizeof(struct ncclP2pBuff)) return ncclInternalError;
|
|
struct ncclP2pBuff* p2pBuff = (struct ncclP2pBuff*)respBuff;
|
|
NCCLCHECK(ncclCudaCalloc((char**)&p2pBuff->directPtr, size));
|
|
connection->transportResources = p2pBuff->directPtr;
|
|
cudaError_t res = cudaIpcGetMemHandle(&p2pBuff->devIpc, p2pBuff->directPtr);
|
|
if (res != cudaSuccess) {
|
|
WARN("cudaIpcGetMemHandle failed : %s", cudaGetErrorString(res));
|
|
cudaFree(p2pBuff->directPtr);
|
|
free(p2pBuff);
|
|
CUDACHECK(res);
|
|
}
|
|
*done = 1;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t p2pSendProxyConnect(struct ncclProxyConnection* connection, struct ncclComm* comm, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
|
|
struct p2pProxyInfo* proxyInfo = (struct p2pProxyInfo*)connection->transportResources;
|
|
|
|
if (reqSize != sizeof(void*)) return ncclInternalError;
|
|
proxyInfo->recvFifo = *((char**)reqBuff);
|
|
|
|
CUDACHECK(cudaStreamCreateWithFlags(&proxyInfo->stream, cudaStreamNonBlocking));
|
|
for (int i=0; i<NCCL_STEPS; i++) {
|
|
CUDACHECK(cudaEventCreate(proxyInfo->events+i));
|
|
}
|
|
connection->proxyAppendPtr = &connection->proxyAppend;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t p2pSendProxyFree(struct ncclProxyConnection* connection, struct ncclComm* comm) {
|
|
if (useMemcpy) {
|
|
struct p2pProxyInfo* proxyInfo = (struct p2pProxyInfo*)connection->transportResources;
|
|
if (proxyInfo) {
|
|
NCCLCHECK(ncclShmClose(proxyInfo->handle));
|
|
NCCLCHECK(ncclCudaHostFree(proxyInfo->ceRecvMem));
|
|
CUDACHECK(cudaFree(proxyInfo->ceDevBuff));
|
|
CUDACHECK(cudaStreamDestroy(proxyInfo->stream));
|
|
for (int i=0; i<NCCL_STEPS; i++) {
|
|
CUDACHECK(cudaEventDestroy(proxyInfo->events[i]));
|
|
}
|
|
free(proxyInfo);
|
|
}
|
|
} else {
|
|
// Do not check return code as CUDA may have already shut down
|
|
cudaFree(connection->transportResources);
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t p2pRecvProxyFree(struct ncclProxyConnection* connection, struct ncclComm* comm) {
|
|
// Do not check return code as CUDA may have already shut down
|
|
cudaFree(connection->transportResources);
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t p2pSendProxyProgress(struct ncclComm* comm, struct ncclProxyArgs* args) {
|
|
if (args->state == ncclProxyOpReady) {
|
|
for (int s=0; s<args->nsubs; s++) {
|
|
struct ncclProxySubArgs* sub = args->subs+s;
|
|
struct p2pProxyInfo* resources = (struct p2pProxyInfo*) (sub->connection->transportResources);
|
|
// Round to next multiple of sliceSteps
|
|
sub->base = ROUNDUP(resources->step, args->chunkSteps);
|
|
sub->posted = sub->transmitted = sub->done = 0;
|
|
}
|
|
args->state = ncclProxyOpProgress;
|
|
}
|
|
args->idle = 1;
|
|
if (args->state == ncclProxyOpProgress) {
|
|
int p = args->protocol;
|
|
int stepSize = comm->buffSizes[p] / NCCL_STEPS;
|
|
for (int s=0; s<args->nsubs; s++) {
|
|
struct ncclProxySubArgs* sub = args->subs+s;
|
|
struct p2pProxyInfo* resources = (struct p2pProxyInfo*) (sub->connection->transportResources);
|
|
if (p != NCCL_PROTO_SIMPLE) { // Only Simple uses cudaMemcpy
|
|
resources->step = sub->base + sub->nsteps;
|
|
args->done++;
|
|
continue;
|
|
}
|
|
if (sub->transmitted < sub->done + NCCL_STEPS && sub->transmitted < sub->nsteps) {
|
|
int buffSlot = (sub->base+sub->transmitted)%NCCL_STEPS;
|
|
volatile int* sizesFifo = resources->ceRecvMem->sizesFifo;
|
|
volatile uint64_t* recvTail = &resources->ceRecvMem->tail;
|
|
// Check GPU has sent everything
|
|
if ((*recvTail > sub->base+sub->transmitted)) {
|
|
int size = sizesFifo[buffSlot];
|
|
CUDACHECK(cudaMemcpyAsync(resources->recvFifo+buffSlot*stepSize, resources->ceDevBuff+buffSlot*stepSize, size, cudaMemcpyDeviceToDevice, resources->stream));
|
|
CUDACHECK(cudaEventRecord(resources->events[buffSlot], resources->stream));
|
|
sub->transmitted += args->sliceSteps;
|
|
}
|
|
}
|
|
if (sub->done < sub->transmitted) {
|
|
int buffSlot = (sub->base+sub->done)%NCCL_STEPS;
|
|
cudaError_t res = cudaEventQuery(resources->events[buffSlot]);
|
|
if (res != cudaErrorNotReady) CUDACHECK(res);
|
|
if (res == cudaSuccess) {
|
|
sub->done += args->sliceSteps;
|
|
// Notify SHM
|
|
resources->shm->recvMem.tail = sub->base + sub->done;
|
|
}
|
|
if (sub->done == sub->nsteps) {
|
|
resources->step = sub->base + sub->nsteps;
|
|
args->done++;
|
|
}
|
|
}
|
|
}
|
|
if (args->done == args->nsubs) {
|
|
args->state = ncclProxyOpNone;
|
|
}
|
|
}
|
|
return ncclSuccess;
|
|
}
|
|
|
|
struct ncclTransport p2pTransport = {
|
|
"P2P",
|
|
p2pCanConnect,
|
|
{ p2pSendSetup, p2pSendConnect, p2pSendFree, NULL, p2pSendProxySetup, NULL, p2pSendProxyFree, NULL },
|
|
{ p2pRecvSetup, p2pRecvConnect, p2pRecvFree, NULL, p2pRecvProxySetup, NULL, p2pRecvProxyFree, NULL }
|
|
};
|
|
|
|
static void initCeOperation() {
|
|
static int init = 0;
|
|
if (!init) {
|
|
useMemcpy = ncclParamP2pUseCudaMemcpy();
|
|
if (useMemcpy) {
|
|
p2pTransport.send.proxyConnect = p2pSendProxyConnect;
|
|
p2pTransport.send.proxyProgress = p2pSendProxyProgress;
|
|
}
|
|
init = 1;
|
|
}
|
|
}
|