AI/rocm-systems
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'nccl/v2.19' into develop

Browse Source
This commit is contained in:
BertanDogancay committed 2024-01-24 15:25:33 -08:00
2 parent 18a57bac10 8c6c595185
commit 81ddf9de89
111 file changed
+4607 -2074
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/transport/coll_net.cc
+12 -2
View File
@@ -158,7 +158,7 @@ static ncclResult_t sendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
NCCLCHECK(ncclTopoCheckGdr(comm->topo, myInfo->busId, req.netDev, 1, &req.useGdr));
send->conn.flags |= req.useGdr ? NCCL_DIRECT_NIC : 0;
NCCLCHECK(ncclTopoGetLocalRank(comm->topo, myInfo->rank, &send->proxyConn.tpLocalRank));
send->proxyConn.tpLocalRank = comm->topParentLocalRanks[comm->localRank];
tpProxyRank = comm->topParentRanks[myInfo->rank];
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_COLLNET, 1, tpProxyRank, &send->proxyConn));
ncclAtomicRefCountIncrement(&comm->collNetSharedRes->refCount);
@@ -180,7 +180,7 @@ static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
// Determine whether we need to flush the GDR buffer on recv or not
if (req.useGdr) NCCLCHECK(ncclTopoNeedFlush(comm->topo, myInfo->busId, &req.needFlush));
NCCLCHECK(ncclTopoGetLocalRank(comm->topo, myInfo->rank, &recv->proxyConn.tpLocalRank));
recv->proxyConn.tpLocalRank = comm->topParentLocalRanks[comm->localRank];
tpProxyRank = comm->topParentRanks[myInfo->rank];
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_COLLNET, 0, tpProxyRank, &recv->proxyConn));
struct collNetRecvConnectInfo* info = (struct collNetRecvConnectInfo*) connectInfo;
@@ -227,6 +227,8 @@ struct collNetConnectArgs {
struct ncclConnect* connectInfos;
};
static ncclResult_t sendProxyProgress(struct ncclProxyState* proxyState, struct ncclProxyArgs* args);
static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* connectInfos, int nranks, int rank, struct ncclConnector* send) {
// We're on the same process as the proxy. We can pass a pointer to a struct.
struct collNetConnectArgs args = { rank, nranks, connectInfos };
@@ -250,9 +252,14 @@ static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* conne
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++)
send->conn.buffs[p] = NCCL_NET_MAP_GET_POINTER(map, gpu, buffs[p]);
send->proxyConn.proxyProgress = sendProxyProgress;
return ncclSuccess;
}
static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct ncclProxyArgs* args);
static ncclResult_t recvConnect(struct ncclComm* comm, struct ncclConnect* connectInfos, int nranks, int rank, struct ncclConnector* recv) {
// We're on the same process as the proxy. We can pass a pointer to a struct.
struct collNetConnectArgs args = { rank, nranks, connectInfos };
@@ -275,6 +282,9 @@ static ncclResult_t recvConnect(struct ncclComm* comm, struct ncclConnect* conne
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
recv->conn.buffs[p] = NCCL_NET_MAP_GET_POINTER(map, gpu, buffs[p]);
}
recv->proxyConn.proxyProgress = recvProxyProgress;
return ncclSuccess;
}
src/transport/net.cc
+175 -58
View File
@@ -88,7 +88,7 @@ struct connectMap {
} offsets;
};
struct sendResources {
struct sendNetResources {
struct connectMap map;
void* netSendComm;
struct ncclSendMem* sendMem;
@@ -111,10 +111,13 @@ struct sendResources {
void* mhandles[NCCL_NUM_PROTOCOLS];
uint64_t step;
uint64_t llLastCleaning;
int netDeviceVersion;
ncclNetDeviceType netDeviceType;
ncclNetDeviceHandle_t* netDeviceHandle;
volatile uint32_t* curr_hdp_reg; // Curr GPU in ring (for rdma transport use only)
};
struct recvResources {
struct recvNetResources {
struct connectMap map;
void* netListenComm;
void* netRecvComm;
@@ -141,6 +144,9 @@ struct recvResources {
void* mhandles[NCCL_NUM_PROTOCOLS];
uint64_t step;
uint64_t llLastCleaning;
int netDeviceVersion;
ncclNetDeviceType netDeviceType;
ncclNetDeviceHandle_t* netDeviceHandle;
volatile uint32_t* curr_hdp_reg; // Curr GPU in ring (for rdma transport use only)
};
@@ -173,11 +179,14 @@ struct setupReq {
uint32_t* curr_hdp_reg;
};
// Forward declaration
static ncclResult_t sendProxyProgress(struct ncclProxyState* proxyState, struct ncclProxyArgs* args);
/* Determine if we will use this transport for this peer and return connect
* information for this peer */
static ncclResult_t sendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* send, int channelId, int connIndex) {
struct setupReq req = { 0 };
int localRank, tpProxyRank;
int tpProxyRank;
send->conn.shared = req.shared = (graph || mscclIsCaller()) ? 0 : ncclParamNetSharedBuffers() != -2 ? ncclParamNetSharedBuffers() : 1;
req.channelId = channelId;
@@ -197,8 +206,7 @@ static ncclResult_t sendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
tpProxyRank = comm->topParentRanks[proxyRank];
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_NET, 1, tpProxyRank, &send->proxyConn));
NCCLCHECK(ncclTopoGetLocalRank(comm->topo, myInfo->rank, &localRank));
req.tpLocalRank = comm->topParentLocalRanks[localRank];
req.tpLocalRank = comm->topParentLocalRanks[comm->localRank];
req.tpRank = comm->topParentRanks[myInfo->rank];
req.tpRemoteRank = comm->topParentRanks[peerInfo->rank];
NCCLCHECK(ncclProxyCallBlocking(comm, &send->proxyConn, ncclProxyMsgSetup, &req, sizeof(req), NULL, 0));
@@ -222,7 +230,6 @@ NCCL_PARAM(GdrCopyFlushEnable, "GDRCOPY_FLUSH_ENABLE", 0);
/* Setup recv connector */
static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* recv, int channelId, int connIndex) {
struct setupReq req = { 0 };
int localRank;
recv->conn.shared = req.shared = (graph || mscclIsCaller()) ? 0 : ncclParamNetSharedBuffers() != -2 ? ncclParamNetSharedBuffers() : 1;
req.channelId = channelId;
@@ -242,8 +249,7 @@ static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
tpProxyRank = comm->topParentRanks[myInfo->rank];
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_NET, 0, tpProxyRank, &recv->proxyConn));
NCCLCHECK(ncclTopoGetLocalRank(comm->topo, myInfo->rank, &localRank));
req.tpLocalRank = comm->topParentLocalRanks[localRank];
req.tpLocalRank = comm->topParentLocalRanks[comm->localRank];
req.tpRank = comm->topParentRanks[myInfo->rank];
req.tpRemoteRank = comm->topParentRanks[peerInfo->rank];
NCCLCHECK(ncclProxyCallBlocking(comm, &recv->proxyConn, ncclProxyMsgSetup, &req, sizeof(req), connectInfo, sizeof(ncclNetHandle_t)));
@@ -290,6 +296,14 @@ static ncclResult_t netDumpMap(struct connectMap* map) {
return ncclSuccess;
}
struct netSendConnectArgs {
ncclNetHandle_t handle;
};
struct netRecvConnectArgs {
int proxyRank;
};
static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* connectInfo, int nranks, int rank, struct ncclConnector* send) {
struct connectMap* map = (connectMap*) send->transportResources;
@@ -302,7 +316,9 @@ static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* conne
send->transportResources = map;
opId = send;
INFO(NCCL_PROXY, "sendConnect ncclProxyCallAsync opId=%p", opId);
NCCLCHECK(ncclProxyCallAsync(comm, &send->proxyConn, ncclProxyMsgConnect, connectInfo, sizeof(ncclNetHandle_t), sizeof(struct connectMap), opId));
netSendConnectArgs args = {0};
memcpy(&args.handle, connectInfo, sizeof(ncclNetHandle_t));
NCCLCHECK(ncclProxyCallAsync(comm, &send->proxyConn, ncclProxyMsgConnect, &args, sizeof(netSendConnectArgs), sizeof(struct connectMap), opId));
} else {
opId = send;
}
@@ -316,15 +332,13 @@ static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* conne
if (map->sameProcess && !ncclCuMemEnable()) {
if (map->cudaDev != comm->cudaDev) {
if (!ncclCuMemEnable()) {
// Enable P2P access for Legacy IPC
cudaError_t err = cudaDeviceEnablePeerAccess(map->cudaDev, 0);
if (err == cudaErrorPeerAccessAlreadyEnabled) {
cudaGetLastError();
} else if (err != cudaSuccess) {
WARN("failed to peer with device %d: %d %s", map->cudaDev, err, cudaGetErrorString(err));
return ncclInternalError;
}
// Enable P2P access for Legacy IPC
cudaError_t err = cudaDeviceEnablePeerAccess(map->cudaDev, 0);
if (err == cudaErrorPeerAccessAlreadyEnabled) {
cudaGetLastError();
} else if (err != cudaSuccess) {
WARN("failed to peer with device %d: %d %s", map->cudaDev, err, cudaGetErrorString(err));
return ncclInternalError;
}
}
} else if (!(map->sameProcess && map->cudaDev == comm->cudaDev)) {
@@ -362,9 +376,30 @@ static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* conne
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++)
send->conn.buffs[p] = NCCL_NET_MAP_GET_POINTER(map, gpu, buffs[p]);
if (send->proxyConn.sameProcess) {
if (send->proxyConn.connection->netDeviceHandle) {
send->conn.netDeviceHandle = *send->proxyConn.connection->netDeviceHandle;
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++)
send->conn.mhandles[p] = send->proxyConn.connection->mhandles[p];
}
if (send->proxyConn.connection->needsProxyProgress) {
send->proxyConn.proxyProgress = sendProxyProgress;
} else {
send->proxyConn.proxyProgress = NULL;
}
} else {
send->proxyConn.proxyProgress = sendProxyProgress;
}
return ncclSuccess;
}
// Forward declare
static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct ncclProxyArgs* args);
/* Connect to this peer */
static ncclResult_t recvConnect(struct ncclComm* comm, struct ncclConnect* connectInfo, int nranks, int rank, struct ncclConnector* recv) {
struct connectMap* map = (connectMap*) recv->transportResources;
@@ -376,7 +411,9 @@ static ncclResult_t recvConnect(struct ncclComm* comm, struct ncclConnect* conne
opId = recv;
INFO(NCCL_PROXY, "recvConnect ncclProxyCallAsync opId=%p &recv->proxyConn=%p connectInfo=%p",
opId, &recv->proxyConn, connectInfo);
NCCLCHECK(ncclProxyCallAsync(comm, &recv->proxyConn, ncclProxyMsgConnect, connectInfo, sizeof(int), sizeof(struct connectMap), opId));
netRecvConnectArgs args = {0};
args.proxyRank = *((int*)connectInfo);
NCCLCHECK(ncclProxyCallAsync(comm, &recv->proxyConn, ncclProxyMsgConnect, &args, sizeof(netRecvConnectArgs), sizeof(struct connectMap), opId));
} else {
opId = recv;
}
@@ -401,6 +438,24 @@ static ncclResult_t recvConnect(struct ncclComm* comm, struct ncclConnect* conne
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++)
recv->conn.buffs[p] = NCCL_NET_MAP_GET_POINTER(map, gpu, buffs[p]);
if (recv->proxyConn.sameProcess) {
if (recv->proxyConn.connection->netDeviceHandle) {
recv->conn.netDeviceHandle = *recv->proxyConn.connection->netDeviceHandle;
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++)
recv->conn.mhandles[p] = recv->proxyConn.connection->mhandles[p];
}
if (recv->proxyConn.connection->needsProxyProgress) {
recv->proxyConn.proxyProgress = recvProxyProgress;
} else {
recv->proxyConn.proxyProgress = NULL;
}
} else {
recv->proxyConn.proxyProgress = recvProxyProgress;
}
return ncclSuccess;
}
@@ -439,7 +494,7 @@ static ncclResult_t recvFree(struct ncclConnector* recv) {
}
#define NCCL_SHARED_STEPS 16
static ncclResult_t sharedBuffersInit(struct ncclProxyState* proxyState, int cuda, int tpLocalRank, int type, int sameProcess,
static ncclResult_t sharedNetBuffersInit(struct ncclProxyState* proxyState, int cuda, int tpLocalRank, int type, int sameProcess,
int nChannels, char** gpuPtr, char** cpuPtr, int* size, ncclIpcDesc *ipcDesc) {
if (cuda == 0 && sameProcess == 0) {
WARN("PXN should not use host buffers for data");
@@ -485,7 +540,7 @@ static ncclResult_t sharedBuffersGet(struct ncclProxyState* proxyState, int chan
return ncclSuccess;
}
static ncclResult_t sharedBuffersDestroy(struct ncclProxyState* proxyState, int tpLocalRank, int type, struct ncclProxyConnection* connection) {
static ncclResult_t sharedNetBuffersDestroy(struct ncclProxyState* proxyState, int tpLocalRank, int type, struct ncclProxyConnection* connection) {
if (proxyState->progressState.localPeers == NULL) NCCLCHECK(ncclInternalError);
struct ncclProxyPeer* peer = proxyState->progressState.localPeers[tpLocalRank];
if (peer == NULL) NCCLCHECK(ncclInternalError;)
@@ -515,7 +570,7 @@ static ncclResult_t sharedBuffersDestroy(struct ncclProxyState* proxyState, int
}
static ncclResult_t proxySharedInit(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState, int nChannels) {
NCCLCHECK(sharedBuffersInit(proxyState, 1, connection->tpLocalRank, 0, connection->sameProcess, nChannels, NULL, NULL, NULL, NULL));
NCCLCHECK(sharedNetBuffersInit(proxyState, 1, connection->tpLocalRank, 0, connection->sameProcess, nChannels, NULL, NULL, NULL, NULL));
return ncclSuccess;
}
@@ -523,7 +578,7 @@ static ncclResult_t sendProxySetup(struct ncclProxyConnection* connection, struc
struct setupReq* req = (struct setupReq*) reqBuff;
if (reqSize != sizeof(struct setupReq)) return ncclInternalError;
struct sendResources* resources;
struct sendNetResources* resources;
NCCLCHECK(ncclCalloc(&resources, 1));
connection->transportResources = resources;
@@ -541,6 +596,11 @@ static ncclResult_t sendProxySetup(struct ncclProxyConnection* connection, struc
/* DMA-BUF support */
resources->useDmaBuf = resources->useGdr && proxyState->dmaBufSupport && (props.ptrSupport & NCCL_PTR_DMABUF);
resources->maxRecvs = props.maxRecvs;
resources->netDeviceVersion = props.netDeviceVersion;
resources->netDeviceType = props.netDeviceType;
resources->netDeviceVersion = props.netDeviceVersion;
resources->netDeviceType = props.netDeviceType;
// We don't return any data
if (respSize != 0) return ncclInternalError;
@@ -552,7 +612,7 @@ static ncclResult_t recvProxySetup(struct ncclProxyConnection* connection, struc
struct setupReq* req = (struct setupReq*) reqBuff;
if (reqSize != sizeof(struct setupReq)) return ncclInternalError;
struct recvResources* resources;
struct recvNetResources* resources;
NCCLCHECK(ncclCalloc(&resources, 1));
connection->transportResources = resources;
@@ -570,6 +630,8 @@ static ncclResult_t recvProxySetup(struct ncclProxyConnection* connection, struc
/* DMA-BUF support */
resources->useDmaBuf = resources->useGdr && proxyState->dmaBufSupport && (props.ptrSupport & NCCL_PTR_DMABUF);
resources->maxRecvs = props.maxRecvs;
resources->netDeviceVersion = props.netDeviceVersion;
resources->netDeviceType = props.netDeviceType;
if (respSize != sizeof(ncclNetHandle_t)) return ncclInternalError;
NCCLCHECK(proxyState->ncclNet->listen(req->netDev, respBuff, &resources->netListenComm));
@@ -578,11 +640,34 @@ static ncclResult_t recvProxySetup(struct ncclProxyConnection* connection, struc
return ncclSuccess;
}
static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
struct sendResources* resources = (struct sendResources*)(connection->transportResources);
if (reqSize != sizeof(ncclNetHandle_t)) return ncclInternalError;
ncclResult_t ret = ncclSuccess;
// This function embeds plugin-specific rules given the current versions
static ncclResult_t ncclNetGetDeviceHandle(ncclNetDeviceType type, int version, bool isRecv, ncclNetDeviceHandle_t** handle) {
bool needsDeviceHandle = false;
if (type == NCCL_NET_DEVICE_UNPACK) {
if (version == NCCL_NET_DEVICE_UNPACK_VERSION && isRecv) {
needsDeviceHandle = true;
}
}
// Don't re-alloc netDeviceHandles
if (needsDeviceHandle && (*handle == NULL)) {
NCCLCHECK(ncclCalloc(handle, 1));
(*handle)->netDeviceType = type;
(*handle)->netDeviceVersion = version;
} else if (!needsDeviceHandle) {
*handle = NULL;
}
return ncclSuccess;
}
static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
struct sendNetResources* resources = (struct sendNetResources*)(connection->transportResources);
if (reqSize != sizeof(netSendConnectArgs)) return ncclInternalError;
ncclResult_t ret = ncclSuccess;
netSendConnectArgs* req = (netSendConnectArgs*) reqBuff;
NCCLCHECK(ncclNetGetDeviceHandle(resources->netDeviceType, resources->netDeviceVersion, false /*isRecv*/, &resources->netDeviceHandle));
if (resources->shared) {
// Shared buffers
struct ncclProxyProgressState* progressState = &proxyState->progressState;
@@ -601,15 +686,15 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
NCCLCHECK(ncclCalloc(progressState->netComms + resources->netDev, proxyState->tpnRanks));
}
struct ncclSharedNetComms* comms = progressState->netComms[resources->netDev] + resources->tpRemoteRank;
if (comms->sendComm[resources->channelId] == NULL) ret = proxyState->ncclNet->connect(resources->netDev, reqBuff, comms->sendComm + resources->channelId);
if (comms->sendComm[resources->channelId] == NULL) ret = proxyState->ncclNet->connect(resources->netDev, req->handle, comms->sendComm + resources->channelId, &resources->netDeviceHandle);
resources->netSendComm = comms->sendComm[resources->channelId];
if (comms->sendComm[resources->channelId]) comms->sendRefCount[resources->channelId]++;
} else {
ret = proxyState->ncclNet->connect(resources->netDev, reqBuff, &resources->netSendComm);
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, &resources->netSendComm, &resources->netDeviceHandle);
}
} else {
// Connect to remote peer
ret = proxyState->ncclNet->connect(resources->netDev, reqBuff, &resources->netSendComm);
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, &resources->netSendComm, &resources->netDeviceHandle);
connection->proxyAppendPtr = &connection->proxyAppend;
}
@@ -620,6 +705,13 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
}
*done = 1;
if (resources->netDeviceHandle) {
connection->netDeviceHandle = resources->netDeviceHandle;
connection->needsProxyProgress = connection->netDeviceHandle->needsProxyProgress;
} else {
connection->needsProxyProgress = 1;
}
// Create structures
struct connectMap* map = &resources->map;
map->sameProcess = connection->sameProcess;
@@ -635,7 +727,7 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
// Get shared buffers
int bank = resources->useGdr ? NCCL_NET_MAP_SHARED_DEVMEM : NCCL_NET_MAP_SHARED_HOSTMEM;
struct connectMapMem* mapMem = map->mems+bank;
NCCLCHECK(sharedBuffersInit(
NCCLCHECK(sharedNetBuffersInit(
proxyState, resources->useGdr, resources->tpLocalRank, 0, map->sameProcess, proxyState->p2pnChannels,
&mapMem->gpuPtr, &mapMem->cpuPtr, &mapMem->size, &mapMem->ipcDesc));
resources->buffSizes[NCCL_PROTO_SIMPLE] = mapMem->size;
@@ -715,6 +807,10 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
{
NCCLCHECK(proxyState->ncclNet->regMr(resources->netSendComm, resources->buffers[p], resources->buffSizes[p], NCCL_NET_MAP_DEV_MEM(map, buffs[p]) ? NCCL_PTR_CUDA : NCCL_PTR_HOST, &resources->mhandles[p]));
}
// Copy the mhandle dptr, if implemented
if (resources->netDeviceHandle && proxyState->ncclNet->getDeviceMr)
NCCLCHECK(proxyState->ncclNet->getDeviceMr(resources->netSendComm, resources->mhandles[p], &connection->mhandles[p]));
}
}
@@ -725,11 +821,13 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
}
static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
if (reqSize != sizeof(int)) return ncclInternalError;
struct recvResources* resources = (struct recvResources*)(connection->transportResources);
resources->tpRemoteProxyRank = *(int*)reqBuff;
if (reqSize != sizeof(netRecvConnectArgs)) return ncclInternalError;
struct recvNetResources* resources = (struct recvNetResources*)(connection->transportResources);
netRecvConnectArgs* req = (netRecvConnectArgs*) reqBuff;
resources->tpRemoteProxyRank = req->proxyRank;
ncclResult_t ret = ncclSuccess;
NCCLCHECK(ncclNetGetDeviceHandle(resources->netDeviceType, resources->netDeviceVersion, true /*isRecv*/, &resources->netDeviceHandle));
// Finish connection establishment from remote peer
if (resources->shared) {
// Shared buffers
@@ -749,15 +847,15 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
NCCLCHECK(ncclCalloc(progressState->netComms + resources->netDev, proxyState->tpnRanks));
}
struct ncclSharedNetComms* comms = progressState->netComms[resources->netDev] + resources->tpRemoteProxyRank;
if (comms->recvComm[resources->channelId] == NULL) ret = proxyState->ncclNet->accept(resources->netListenComm, comms->recvComm+resources->channelId);
if (comms->recvComm[resources->channelId] == NULL) ret = proxyState->ncclNet->accept(resources->netListenComm, comms->recvComm+resources->channelId, &resources->netDeviceHandle);
resources->netRecvComm = comms->recvComm[resources->channelId];
if (comms->recvComm[resources->channelId]) comms->recvRefCount[resources->channelId]++;
} else {
ret = proxyState->ncclNet->accept(resources->netListenComm, &resources->netRecvComm);
ret = proxyState->ncclNet->accept(resources->netListenComm, &resources->netRecvComm, &resources->netDeviceHandle);
}
} else {
// Connect to remote peer
ret = proxyState->ncclNet->accept(resources->netListenComm, &resources->netRecvComm);
ret = proxyState->ncclNet->accept(resources->netListenComm, &resources->netRecvComm, &resources->netDeviceHandle);
connection->proxyAppendPtr = &connection->proxyAppend;
}
@@ -768,6 +866,13 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
}
*done = 1;
if (resources->netDeviceHandle) {
connection->netDeviceHandle = resources->netDeviceHandle;
connection->needsProxyProgress = connection->netDeviceHandle->needsProxyProgress;
} else {
connection->needsProxyProgress = 1;
}
NCCLCHECK(proxyState->ncclNet->closeListen(resources->netListenComm));
// Create structures
@@ -785,7 +890,7 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
// Get shared buffers
int bank = resources->useGdr ? NCCL_NET_MAP_SHARED_DEVMEM : NCCL_NET_MAP_SHARED_HOSTMEM;
struct connectMapMem* mapMem = map->mems+bank;
NCCLCHECK(sharedBuffersInit(
NCCLCHECK(sharedNetBuffersInit(
proxyState, resources->useGdr, resources->tpLocalRank, 1, 1, proxyState->p2pnChannels,
&mapMem->gpuPtr, &mapMem->cpuPtr, &mapMem->size, NULL));
resources->buffSizes[NCCL_PROTO_SIMPLE] = mapMem->size;
@@ -857,6 +962,10 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
{
NCCLCHECK(proxyState->ncclNet->regMr(resources->netRecvComm, resources->buffers[p], resources->buffSizes[p], NCCL_NET_MAP_DEV_MEM(map, buffs[p]) ? NCCL_PTR_CUDA : NCCL_PTR_HOST, &resources->mhandles[p]));
}
// Copy the mhandle dptr
if (resources->netDeviceType != NCCL_NET_DEVICE_HOST && proxyState->ncclNet->getDeviceMr)
NCCLCHECK(proxyState->ncclNet->getDeviceMr(resources->netRecvComm, resources->mhandles[p], &connection->mhandles[p]));
}
}
@@ -867,9 +976,9 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
}
static ncclResult_t sendProxyFree(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState) {
struct sendResources* resources = (struct sendResources*)(connection->transportResources);
struct sendNetResources* resources = (struct sendNetResources*)(connection->transportResources);
if (connection->state == connSharedInitialized) { // NVB Preconnect
NCCLCHECK(sharedBuffersDestroy(proxyState, connection->tpLocalRank, 0, connection));
NCCLCHECK(sharedNetBuffersDestroy(proxyState, connection->tpLocalRank, 0, connection));
return ncclSuccess;
}
@@ -894,7 +1003,7 @@ static ncclResult_t sendProxyFree(struct ncclProxyConnection* connection, struct
}
if (mems[NCCL_NET_MAP_GDCMEM].cpuPtr) NCCLCHECK(ncclGdrCudaFree(resources->gdrDesc));
if (resources->shared) {
NCCLCHECK(sharedBuffersDestroy(proxyState, resources->tpLocalRank, 0, connection));
NCCLCHECK(sharedNetBuffersDestroy(proxyState, resources->tpLocalRank, 0, connection));
if (resources->maxRecvs > 1 && ncclParamNetSharedComms()) {
struct ncclSharedNetComms* comms = proxyState->progressState.netComms[resources->netDev]+resources->tpRemoteRank;
comms->sendRefCount[resources->channelId]--;
@@ -912,9 +1021,9 @@ static ncclResult_t sendProxyFree(struct ncclProxyConnection* connection, struct
}
static ncclResult_t recvProxyFree(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState) {
struct recvResources* resources = (struct recvResources*)(connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*)(connection->transportResources);
if (connection->state == connSharedInitialized) { // NVB Preconnect
NCCLCHECK(sharedBuffersDestroy(proxyState, connection->tpLocalRank, 1, connection));
NCCLCHECK(sharedNetBuffersDestroy(proxyState, connection->tpLocalRank, 1, connection));
return ncclSuccess;
}
@@ -935,7 +1044,7 @@ static ncclResult_t recvProxyFree(struct ncclProxyConnection* connection, struct
}
if (mems[NCCL_NET_MAP_GDCMEM].cpuPtr) NCCLCHECK(ncclGdrCudaFree(resources->gdrDesc));
if (resources->shared) {
NCCLCHECK(sharedBuffersDestroy(proxyState, resources->tpLocalRank, 1, connection));
NCCLCHECK(sharedNetBuffersDestroy(proxyState, resources->tpLocalRank, 1, connection));
if (resources->maxRecvs > 1 && ncclParamNetSharedComms()) {
struct ncclSharedNetComms* comms = proxyState->progressState.netComms[resources->netDev] + resources->tpRemoteProxyRank;
comms->recvRefCount[resources->channelId]--;
@@ -965,7 +1074,7 @@ static ncclResult_t sendProxyProgress(struct ncclProxyState* proxyState, struct
if (args->state == ncclProxyOpReady) {
for (int s=0; s<args->nsubs; s++) {
struct ncclProxySubArgs* sub = args->subs+s;
struct sendResources* resources = (struct sendResources*) (sub->connection->transportResources);
struct sendNetResources* resources = (struct sendNetResources*) (sub->connection->transportResources);
// Round to next multiple of sliceSteps
sub->base = ROUNDUP(resources->step, args->chunkSteps);
sub->posted = sub->transmitted = sub->done = 0;
@@ -981,7 +1090,7 @@ static ncclResult_t sendProxyProgress(struct ncclProxyState* proxyState, struct
for (int s=0; s<args->nsubs; s++) {
struct ncclProxySubArgs* sub = args->subs+s;
if (sub->done == sub->nsteps) continue;
struct sendResources* resources = (struct sendResources*) (sub->connection->transportResources);
struct sendNetResources* resources = (struct sendNetResources*) (sub->connection->transportResources);
void* mhandle = resources->mhandles[p];
int stepSize = resources->buffSizes[p] / NCCL_STEPS;
char* localBuff = NCCL_NET_MAP_GET_POINTER(&resources->map, cpu, buffs[p]);
@@ -1175,7 +1284,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
} else if (s>0) { // Find next sub with the same recvComm
int next;
for (next=s; next<args->nsubs; next++) {
struct recvResources* nextRes = (struct recvResources*) (args->subs[next].connection->transportResources);
struct recvNetResources* nextRes = (struct recvNetResources*) (args->subs[next].connection->transportResources);
if (nextRes->netRecvComm == recvComm) break;
}
if (next == args->nsubs) { // Not found
@@ -1188,7 +1297,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
}
}
groupSize++;
struct recvResources* resources = (struct recvResources*) (sub->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (sub->connection->transportResources);
maxRecvs = resources->maxRecvs;
recvComm = resources->netRecvComm;
// Round to next multiple of sliceSteps
@@ -1215,7 +1324,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
struct ncclProxySubArgs* sub = subGroup + i;
if (sub->posted < sub->nsteps) {
if (sub->posted >= sub->done + maxDepth) { subCount = 0; break; }
struct recvResources* resources = (struct recvResources*) (sub->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (sub->connection->transportResources);
int stepSize = resources->buffSizes[p] / NCCL_STEPS;
char* localBuff = NCCL_NET_MAP_GET_POINTER(&resources->map, cpu, buffs[p]);
int buffSlot = (sub->base+sub->posted)%NCCL_STEPS;
@@ -1238,10 +1347,12 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
}
if (subCount) {
uint64_t step = subGroup->posted;
struct recvResources* resources = (struct recvResources*) (subGroup->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (subGroup->connection->transportResources);
void** requestPtr = subGroup->requests+(step%NCCL_STEPS);
NCCLCHECK(proxyState->ncclNet->irecv(resources->netRecvComm, subCount, ptrs, sizes, tags, mhandles, requestPtr));
if (*requestPtr) {
subGroup->recvRequestsCache[step%NCCL_STEPS] = *requestPtr;
subGroup->recvRequestsSubCount = subCount;
for (int i=0; i<subGroup->groupSize; i++) {
struct ncclProxySubArgs* sub = subGroup+i;
@@ -1304,14 +1415,14 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
sub->received += args->sliceSteps;
for (uint64_t step=sub->received-args->sliceSteps; step<sub->received; step++) ncclProfilingRecord(args, s+i, step, ncclProxyProfileRecvFlushWait);
if (step < sub->nsteps) {
struct recvResources* resources = (struct recvResources*) (sub->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (sub->connection->transportResources);
if (resources->useGdr) needFlush |= resources->needFlush;
}
}
subGroup->requests[step%NCCL_STEPS] = NULL;
if (totalSize > 0 && p == NCCL_PROTO_SIMPLE && needFlush) {
// GDRCOPY support
struct recvResources* resources = (struct recvResources*) (subGroup->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (subGroup->connection->transportResources);
if (resources->gdcFlush) {
#if defined (__x86_64__)
// Force a PCI-E read from GPU memory
@@ -1325,7 +1436,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
for (int i=0; i<subGroup->groupSize; i++) {
struct ncclProxySubArgs* sub = subGroup + i;
if (step < sub->nsteps) {
struct recvResources* resources = (struct recvResources*) (sub->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (sub->connection->transportResources);
int stepSize = resources->buffSizes[p] / NCCL_STEPS;
char* localBuff = NCCL_NET_MAP_GET_POINTER(&resources->map, cpu, buffs[p]);
int buffSlot = (sub->base+sub->posted)%NCCL_STEPS;
@@ -1334,7 +1445,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
subCount++;
}
}
struct recvResources* resources = (struct recvResources*) (subGroup->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (subGroup->connection->transportResources);
NCCLCHECK(proxyState->ncclNet->iflush(resources->netRecvComm, subCount, ptrs, sizes, mhandles, subGroup->requests+(step%NCCL_STEPS)));
}
}
@@ -1358,7 +1469,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
for (uint64_t step=sub->transmitted-args->sliceSteps; step<sub->transmitted; step++) ncclProfilingRecord(args, s+i, step, ncclProxyProfileRecvGPUWait);
if (step < sub->nsteps) {
__sync_synchronize();
struct recvResources* resources = (struct recvResources*) (sub->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (sub->connection->transportResources);
volatile uint64_t* recvTail = resources->gdcSync ? resources->gdcSync : &resources->recvMem->tail;
*recvTail = sub->base + sub->transmitted;
if (resources->gdcSync) wc_store_fence(); // Flush out WC write
@@ -1376,17 +1487,23 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
struct ncclProxySubArgs* sub = subGroup + i;
if (sub->done == sub->nsteps) continue;
if (sub->transmitted > sub->done) {
struct recvResources* resources = (struct recvResources*) (sub->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (sub->connection->transportResources);
volatile uint64_t* sendHead = &resources->sendMem->head;
uint64_t done = *sendHead;
while (done > sub->base + sub->done &&
// LL and LL128 can acknowledge 0-bytes send before they even happen. Don't go past what we transmitted.
sub->transmitted > sub->done) {
if (subGroup->recvRequestsCache[sub->done%NCCL_STEPS]) {
// the multirecv requests are only cached in the first sub.
if (proxyState->ncclNet->irecvConsumed)
NCCLCHECK(proxyState->ncclNet->irecvConsumed(resources->netRecvComm, subGroup->recvRequestsSubCount, subGroup->recvRequestsCache[sub->done%NCCL_STEPS]));
subGroup->recvRequestsCache[sub->done%NCCL_STEPS] = NULL;
}
sub->done += args->sliceSteps;
for (uint64_t step=sub->done-args->sliceSteps; step<sub->done; step++) ncclProfilingRecord(args, s+i, step, ncclProxyProfileEnd);
args->idle = 0;
if (sub->done == sub->nsteps) {
struct recvResources* resources = (struct recvResources*) (sub->connection->transportResources);
struct recvNetResources* resources = (struct recvNetResources*) (sub->connection->transportResources);
resources->step = sub->base + sub->nsteps;
args->done++;
break;
src/transport/net_ib.cc
+51 -11
View File
@@ -356,6 +356,8 @@ ncclResult_t ncclIbGetProperties(int dev, ncclNetProperties_t* props) {
props->port = ncclIbDevs[dev].port + ncclIbDevs[dev].realPort;
props->maxComms = ncclIbDevs[dev].maxQp;
props->maxRecvs = NCCL_NET_IB_MAX_RECVS;
props->netDeviceType = NCCL_NET_DEVICE_HOST;
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
return ncclSuccess;
}
@@ -371,6 +373,10 @@ struct ncclIbQpInfo {
uint8_t link_layer;
uint32_t qpn[NCCL_IB_MAX_QPS];
// Fields needed for ece (enhanced connection establishment)
struct ibv_ece ece[NCCL_IB_MAX_QPS];
int ece_supported[NCCL_IB_MAX_QPS];
// For RoCE
uint64_t spn;
uint64_t iid;
@@ -643,7 +649,7 @@ ncclResult_t ncclIbListen(int dev, void* opaqueHandle, void** listenComm) {
return ncclSuccess;
}
ncclResult_t ncclIbConnect(int dev, void* opaqueHandle, void** sendComm) {
ncclResult_t ncclIbConnect(int dev, void* opaqueHandle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
struct ncclIbHandle* handle = (struct ncclIbHandle*) opaqueHandle;
struct ncclIbCommStage* stage = &handle->stage;
struct ncclIbSendComm* comm = (struct ncclIbSendComm*)stage->comm;
@@ -687,7 +693,13 @@ ib_connect_check:
NCCLCHECK(wrap_ibv_query_port(ctx, ib_port, &portAttr));
struct ncclIbQpInfo qpInfo;
qpInfo.ib_port = ib_port;
for (int q=0; q<comm->nqps; q++) qpInfo.qpn[q] = comm->qps[q]->qp_num;
for (int q=0; q<comm->nqps; q++) {
qpInfo.qpn[q] = comm->qps[q]->qp_num;
// Query ece capabilities (enhanced connection establishment)
NCCLCHECK(wrap_ibv_query_ece(comm->qps[q], &qpInfo.ece[q], &qpInfo.ece_supported[q]));
}
qpInfo.mtu = portAttr.active_mtu;
// Prepare my fifo
@@ -698,15 +710,20 @@ ib_connect_check:
// RoCE support
qpInfo.lid = portAttr.lid;
qpInfo.link_layer = comm->gidInfo.link_layer = portAttr.link_layer;
if (qpInfo.link_layer == IBV_LINK_LAYER_INFINIBAND) { // IB
for (int q=0; q<comm->nqps; q++)
INFO(NCCL_NET,"NET/IB: Dev %d Port %d qpn %d mtu %d LID %d", dev, ib_port, qpInfo.qpn[q], qpInfo.mtu, qpInfo.lid);
} else { // RoCE
NCCLCHECK(wrap_ibv_query_gid(ctx, ib_port, ncclParamIbGidIndex(), &comm->gidInfo.localGid));
if (qpInfo.link_layer == IBV_LINK_LAYER_ETHERNET) {
NCCLCHECK(wrap_ibv_query_gid(ncclIbDevs[dev].context, ncclIbDevs[dev].port, ncclParamIbGidIndex(), &comm->gidInfo.localGid));
qpInfo.spn = comm->gidInfo.localGid.global.subnet_prefix;
qpInfo.iid = comm->gidInfo.localGid.global.interface_id;
}
if (qpInfo.link_layer == IBV_LINK_LAYER_INFINIBAND) { // IB
for (int q=0; q<comm->nqps; q++)
INFO(NCCL_NET,"NET/IB: Dev %d Port %d qpn %d mtu %d GID %ld (%lX/%lX)", dev, ib_port, qpInfo.qpn[q], qpInfo.mtu, ncclParamIbGidIndex(), qpInfo.spn, qpInfo.iid);
INFO(NCCL_NET,"NET/IB: Dev %d Port %d qpn %d mtu %d LID %d", dev, ncclIbDevs[dev].port, qpInfo.qpn[q], qpInfo.mtu, qpInfo.lid);
} else { // RoCE
for (int q=0; q<comm->nqps; q++)
INFO(NCCL_NET,"NET/IB: Dev %d Port %d qpn %d mtu %d query_ece={supported=%d, vendor_id=0x%x, options=0x%x, comp_mask=0x%x} GID %ld (%lX/%lX)",
dev, ncclIbDevs[dev].port, qpInfo.qpn[q], qpInfo.mtu, qpInfo.ece_supported[q], qpInfo.ece[q].vendor_id, qpInfo.ece[q].options, qpInfo.ece[q].comp_mask, ncclParamIbGidIndex(),
qpInfo.spn, qpInfo.iid);
}
stage->state = ncclIbCommStateSend;
@@ -734,10 +751,19 @@ ib_connect:
comm->gidInfo.remoteGid.global.interface_id = remQpInfo.iid;
for (int q=0; q<comm->nqps; q++) {
struct ibv_qp* qp = comm->qps[q];
if (remQpInfo.ece_supported[q] && qpInfo.ece_supported[q])
NCCLCHECK(wrap_ibv_set_ece(qp, &remQpInfo.ece[q], &qpInfo.ece_supported[q]));
NCCLCHECK(ncclIbRtrQp(qp, remQpInfo.qpn[q], &remQpInfo));
NCCLCHECK(ncclIbRtsQp(qp));
}
if (qpInfo.link_layer == IBV_LINK_LAYER_ETHERNET ) { // RoCE
for (int q=0; q<comm->nqps; q++)
INFO(NCCL_NET,"NET/IB: Dev %d Port %d qpn %d set_ece={supported=%d, vendor_id=0x%x, options=0x%x, comp_mask=0x%x}",
dev, ncclIbDevs[dev].port, qpInfo.qpn[q], remQpInfo.ece_supported[q], remQpInfo.ece[q].vendor_id, remQpInfo.ece[q].options, remQpInfo.ece[q].comp_mask);
}
comm->ready = 1;
stage->state = ncclIbCommStateConnected;
stage->offset = 0;
@@ -755,7 +781,7 @@ ib_send_ready:
NCCL_PARAM(IbGdrFlushDisable, "GDR_FLUSH_DISABLE", 0);
ncclResult_t ncclIbAccept(void* listenComm, void** recvComm) {
ncclResult_t ncclIbAccept(void* listenComm, void** recvComm, ncclNetDeviceHandle_t** /*recvDevComm*/) {
struct ncclIbListenComm* lComm = (struct ncclIbListenComm*)listenComm;
struct ncclIbCommStage* stage = &lComm->stage;
struct ncclIbRecvComm* rComm = (struct ncclIbRecvComm*)stage->comm;
@@ -816,8 +842,21 @@ ib_recv:
remQpInfo.mtu = (enum ibv_mtu)std::min(remQpInfo.mtu, portAttr.active_mtu);
// Setup QP
struct ncclIbQpInfo qpInfo;
for (int q=0; q<rComm->nqps; q++) {
struct ibv_qp* qp = rComm->qps[q];
// Set the ece (enhanced connection establishment) on this QP before RTR
if (remQpInfo.ece_supported[q]) {
NCCLCHECK(wrap_ibv_set_ece(qp, &remQpInfo.ece[q], &qpInfo.ece_supported[q]));
// Query the reduced ece for this QP (matching enhancements between the requestor and the responder)
// Store this in our own qpInfo for returning to the requestor
if (qpInfo.ece_supported[q]) {
NCCLCHECK(wrap_ibv_query_ece(qp, &qpInfo.ece[q], &qpInfo.ece_supported[q]));
}
}
NCCLCHECK(ncclIbRtrQp(qp, remQpInfo.qpn[q], &remQpInfo));
NCCLCHECK(ncclIbRtsQp(qp));
}
@@ -850,7 +889,6 @@ ib_recv:
}
// Fill Handle
struct ncclIbQpInfo qpInfo;
qpInfo.lid=portAttr.lid;
qpInfo.link_layer= rComm->gidInfo.link_layer = portAttr.link_layer;
qpInfo.ib_port=ib_port;
@@ -1415,6 +1453,8 @@ ncclNet_t ncclNetIb = {
ncclIbTest,
ncclIbCloseSend,
ncclIbCloseRecv,
ncclIbCloseListen
ncclIbCloseListen,
NULL /* getDeviceMr */,
NULL /* irecvConsumed */
};
src/transport/net_socket.cc
+7 -3
View File
@@ -101,6 +101,8 @@ ncclResult_t ncclNetSocketGetProperties(int dev, ncclNetProperties_t* props) {
props->port = 0;
props->maxComms = 65536;
props->maxRecvs = 1;
props->netDeviceType = NCCL_NET_DEVICE_HOST;
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
return ncclSuccess;
}
@@ -301,7 +303,7 @@ ncclResult_t ncclNetSocketListen(int dev, void* opaqueHandle, void** listenComm)
return ncclSuccess;
}
ncclResult_t ncclNetSocketConnect(int dev, void* opaqueHandle, void** sendComm) {
ncclResult_t ncclNetSocketConnect(int dev, void* opaqueHandle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
if (dev < 0 || dev >= ncclNetIfs) { // data transfer socket is based on specified dev
return ncclInternalError;
}
@@ -346,7 +348,7 @@ socket_send:
return ncclSuccess;
}
ncclResult_t ncclNetSocketAccept(void* listenComm, void** recvComm) {
ncclResult_t ncclNetSocketAccept(void* listenComm, void** recvComm, ncclNetDeviceHandle_t** /*recvDevComm*/) {
struct ncclNetSocketListenComm* lComm = (struct ncclNetSocketListenComm*)listenComm;
struct ncclNetSocketCommStage* stage = &lComm->stage;
struct ncclNetSocketComm* rComm = stage->comm;
@@ -609,5 +611,7 @@ ncclNet_t ncclNetSocket = {
ncclNetSocketTest,
ncclNetSocketClose,
ncclNetSocketClose,
ncclNetSocketCloseListen
ncclNetSocketCloseListen,
NULL /* getDeviceMr */,
NULL /* irecvConsumed */
};
src/transport/nvls.cc
+446 -35
View File
@@ -10,17 +10,30 @@
#include "graph.h"
#include "utils.h"
#include "proxy.h"
#include "enqueue.h"
#if CUDART_VERSION >= 12010
// Currently we only support POSIX_FILE_DESCRIPTOR handle exchange
#define USE_POSIX_FD 1
struct graphRegData {
uintptr_t offset;
size_t size;
};
#if USE_POSIX_FD
#define NVLS_CU_MEM_HANDLE_TYPE CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR
#else
#define NVLS_CU_MEM_HANDLE_TYPE CU_MEM_HANDLE_TYPE_NONE
#endif
struct localRegData {
/* Registration record data */
uintptr_t recSendbuff, recRecvbuff;
intptr_t recSendOffset, recRecvOffset;
/* Registration request data */
uintptr_t reqSendbuff, reqRecvbuff;
size_t reqSendSize, reqRecvSize;
intptr_t reqSendOffset, reqRecvOffset;
};
struct localRequestData {
uintptr_t reqBuff;
size_t reqSize;
intptr_t reqOffset;
};
ncclResult_t nvlsCanConnect(int* ret, struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2) {
// This transport cannot be used for p2p
@@ -66,24 +79,23 @@ ncclResult_t nvlsGetProperties(struct ncclComm *comm, struct ncclNvlsSharedRes*
return ncclSuccess;
}
ncclResult_t nvlsGroupCreate(struct ncclComm *comm, struct ncclNvlsSharedRes* resources, int rank, unsigned int nranks, char* shareableHandle) {
size_t size = resources->size;
ncclResult_t nvlsGroupCreate(struct ncclComm *comm, CUmulticastObjectProp *prop, int rank, unsigned int nranks, CUmemGenericAllocationHandle *mcHandle, char *shareableHandle) {
size_t size = prop->size;
// Create a Multicast group
CUmulticastObjectProp* prop = &resources->properties;
INFO(NCCL_NVLS, "NVLS Creating Multicast group nranks %d size %zi on rank %d", nranks, size, rank);
CUCHECK(cuMulticastCreate(&resources->mcHandle, prop));
CUCHECK(cuMulticastCreate(mcHandle, prop));
if (NVLS_CU_MEM_HANDLE_TYPE != CU_MEM_HANDLE_TYPE_NONE) {
if ((NVLS_CU_MEM_HANDLE_TYPE != CU_MEM_HANDLE_TYPE_NONE) && (NVLS_CU_MEM_HANDLE_TYPE != CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR)) {
// Get a handle to pass to other ranks
CUCHECK(cuMemExportToShareableHandle(shareableHandle, resources->mcHandle, NVLS_CU_MEM_HANDLE_TYPE, 0));
CUCHECK(cuMemExportToShareableHandle(shareableHandle, *mcHandle, NVLS_CU_MEM_HANDLE_TYPE, 0));
}
else {
memcpy(shareableHandle, &resources->mcHandle, sizeof(resources->mcHandle));
memcpy(shareableHandle, mcHandle, sizeof(CUmemGenericAllocationHandle));
}
INFO(NCCL_NVLS, "NVLS Created Multicast group %llx nranks %d size %zi on rank %d", resources->mcHandle, nranks, size, rank);
INFO(NCCL_NVLS, "NVLS Created Multicast group %llx nranks %d size %zi on rank %d", *mcHandle, nranks, size, rank);
return ncclSuccess;
}
@@ -94,7 +106,7 @@ ncclResult_t nvlsGroupAddDevice(struct ncclComm *comm, struct ncclNvlsSharedRes*
return ncclSuccess;
}
ncclResult_t nvlsGroupConnect(struct ncclComm *comm, struct ncclNvlsSharedRes* resources, int rank, char* shareableHandle) {
ncclResult_t nvlsGroupConnect(struct ncclComm *comm, char *shareableHandle, int rank, CUmemGenericAllocationHandle *mcHandle) {
CUmemAllocationHandleType type = NVLS_CU_MEM_HANDLE_TYPE;
INFO(NCCL_NVLS, "NVLS importing shareableHandle %p from rank %d", shareableHandle, rank);
@@ -102,36 +114,27 @@ ncclResult_t nvlsGroupConnect(struct ncclComm *comm, struct ncclNvlsSharedRes* r
// Import and map the remote memory descriptor to the local GPU
if (type == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) {
// cuMem UDS support
int fd = *(int *)shareableHandle;
TRACE(NCCL_NVLS, "NVLS rank %d Importing shareable handle from rank %d fd %d", comm->localRank, rank, fd);
int fd = -1;
TRACE(NCCL_NVLS, "NVLS rank %d Importing shareable handle %p from rank %d", comm->localRank, shareableHandle, rank);
struct ncclProxyConnector proxyConn;
int tpProxyRank = comm->topParentRanks[rank];
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_P2P, 1, tpProxyRank, &proxyConn));
TRACE(NCCL_NVLS, "NVLS rank %d request conversion of fd %d from rank %d", comm->localRank, fd, rank);
NCCLCHECK(ncclProxyClientConvertFdBlocking(comm, &proxyConn, fd, (int *)shareableHandle));
fd = *(int *)shareableHandle;
TRACE(NCCL_NVLS, "NVLS rank %d request conversion of handle 0x%lx from rank %d", comm->localRank, *(uint64_t*)shareableHandle, rank);
NCCLCHECK(ncclProxyClientGetFdBlocking(comm, &proxyConn, shareableHandle, &fd));
TRACE(NCCL_NVLS, "NVLS rank %d received converted fd %d from rank %d", comm->localRank, fd, rank);
CUCHECK(cuMemImportFromShareableHandle(&resources->mcHandle, (void *)(uintptr_t)fd, type));
CUCHECK(cuMemImportFromShareableHandle(mcHandle, (void *)(uintptr_t)fd, type));
(void) close(fd);
} else {
if (NVLS_CU_MEM_HANDLE_TYPE != CU_MEM_HANDLE_TYPE_NONE) {
CUCHECK(cuMemImportFromShareableHandle(&resources->mcHandle, (void *)shareableHandle, type));
CUCHECK(cuMemImportFromShareableHandle(mcHandle, (void *)shareableHandle, type));
} else {
memcpy(&resources->mcHandle, shareableHandle, sizeof(resources->mcHandle));
memcpy(mcHandle, shareableHandle, sizeof(CUmemGenericAllocationHandle));
}
}
return ncclSuccess;
}
ncclResult_t nvlsGroupDisconnect(struct ncclComm *comm, struct ncclNvlsSharedRes* resources) {
CUmemAllocationHandleType type = NVLS_CU_MEM_HANDLE_TYPE;
// Import and map the remote memory descriptor to the local GPU
if (type == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) {
// cuMem UDS support
int fd = *(int *)resources->shareableHandle;
(void) close(fd);
}
return ncclSuccess;
}
@@ -147,6 +150,7 @@ ncclResult_t nvlsGroupBindMem(struct ncclComm *comm, struct ncclNvlsSharedRes* r
prop.location.id = resources->dev;
prop.requestedHandleTypes = NVLS_CU_MEM_HANDLE_TYPE;
CUCHECK(cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));
resources->ucGran = granularity;
// Map a VA for UC memory
CUCHECK(cuMemAddressReserve(&ptr, size, granularity, 0U, 0));
@@ -181,6 +185,14 @@ ncclResult_t nvlsGroupUnbind(struct ncclComm *comm, struct ncclNvlsSharedRes* re
return ncclSuccess;
}
ncclResult_t ncclNvlsDeregBuffer(CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t size) {
CUCHECK(cuMulticastUnbind(*mcHandler, dev, 0/*mcOffset*/, size));
CUCHECK(cuMemUnmap(ptr, size));
CUCHECK(cuMemAddressFree(ptr, size));
CUCHECK(cuMemRelease(*mcHandler));
return ncclSuccess;
}
ncclResult_t nvlsGroupMapMem(struct ncclComm *comm, struct ncclNvlsSharedRes* resources) {
size_t size = resources->size;
CUdeviceptr ptr = 0;
@@ -263,6 +275,9 @@ ncclResult_t ncclNvlsSetup(struct ncclComm* comm, struct ncclComm* parent) {
int nHeads = comm->channels[0].nvls.nHeads;
int headRank = comm->channels[0].nvls.headRank;
char shmPath[sizeof("/dev/shm/nccl-XXXXXX")];
uintptr_t *nvlsShmem = NULL;
size_t typeSize;
CUdevice dev;
CUCHECK(cuCtxGetDevice(&dev));
@@ -313,11 +328,11 @@ ncclResult_t ncclNvlsSetup(struct ncclComm* comm, struct ncclComm* parent) {
char* shareableHandle = resources->shareableHandle;
NCCLCHECKGOTO(nvlsGetProperties(comm, resources, dev, comm->localRanks, nvlsTotalSize), res, cleanup);
if (comm->localRank == 0) {
NCCLCHECKGOTO(nvlsGroupCreate(comm, resources, comm->localRank, comm->localRanks, shareableHandle), res, cleanup);
NCCLCHECKGOTO(nvlsGroupCreate(comm, &resources->properties, comm->localRank, comm->localRanks, &resources->mcHandle, shareableHandle), res, cleanup);
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), res, cleanup);
} else {
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), res, cleanup);
NCCLCHECKGOTO(nvlsGroupConnect(comm, resources, comm->localRankToRank[0], shareableHandle), res, cleanup);
NCCLCHECKGOTO(nvlsGroupConnect(comm, shareableHandle, comm->localRankToRank[0], &resources->mcHandle), res, cleanup);
}
NCCLCHECKGOTO(nvlsGroupAddDevice(comm, resources), res, cleanup);
@@ -374,6 +389,23 @@ ncclResult_t ncclNvlsSetup(struct ncclComm* comm, struct ncclComm* parent) {
}
}
/* create shared memory for fast NVLS buffer registration */
typeSize = sizeof(struct localRegData);
if (comm->localRank == 0) {
shmPath[0] = '\0';
NCCLCHECKGOTO(ncclShmOpen(shmPath, (sizeof(size_t) + typeSize * comm->localRanks) * 2, (void**)&nvlsShmem, NULL, comm->localRanks - 1, &comm->nvlsShmemHandle), res, cleanup);
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shmPath, sizeof(shmPath)), res, cleanup);
} else {
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shmPath, sizeof(shmPath)), res, cleanup);
NCCLCHECKGOTO(ncclShmOpen(shmPath, (sizeof(size_t) + typeSize * comm->localRanks) * 2, (void**)&nvlsShmem, NULL, -1, &comm->nvlsShmemHandle), res, cleanup);
}
/* need 2 pools and a shared counter for shmem-based collectives */
comm->nvlsShmem.cnt[0] = (size_t*)nvlsShmem;
comm->nvlsShmem.ptr[0] = (void*)((char*)comm->nvlsShmem.cnt[0] + sizeof(size_t));
comm->nvlsShmem.cnt[1] = (size_t*)((char*)comm->nvlsShmem.ptr[0] + typeSize * comm->localRanks);
comm->nvlsShmem.ptr[1] = (void*)((char*)comm->nvlsShmem.cnt[1] + sizeof(size_t));
comm->nvlsShmem.round = 0;
return res;
cleanup:
@@ -394,6 +426,371 @@ ncclResult_t ncclNvlsFree(struct ncclComm* comm) {
return ncclSuccess;
}
ncclResult_t tryRegisterBuffer(struct ncclComm *comm, struct localRequestData *reqData, uintptr_t userBuff, size_t buffSize, CUdeviceptr *regAddr, bool *regUsed) {
ncclResult_t ret = ncclSuccess;
struct ncclRegRecord *regRecord = NULL;
struct localRequestData *myReqData = &reqData[comm->localRank];
CUdeviceptr regPtr = 0;
CUmulticastObjectProp prop;
char shareableHandle[NVLS_HANDLE_SIZE];
CUmemGenericAllocationHandle mcHandle;
size_t granularity;
size_t minSize;
bool localRegBufUsed = false;
/* get minimal size of nvls buffers */
minSize = reqData[0].reqSize;
for (int i = 1; i < comm->localRanks; ++i) {
if (minSize > reqData[i].reqSize)
minSize = reqData[i].reqSize;
}
/* start registration */
memcpy(&prop, &comm->nvlsResources->properties, sizeof(CUmulticastObjectProp));
prop.size = minSize;
CUCHECKGOTO(cuMulticastGetGranularity(&granularity, &prop, CU_MULTICAST_GRANULARITY_RECOMMENDED), ret, fail);
if (comm->localRank == 0) {
NCCLCHECKGOTO(nvlsGroupCreate(comm, &prop, comm->localRank, comm->localRanks, &mcHandle, shareableHandle), ret, fail);
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail);
} else {
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail);
NCCLCHECKGOTO(nvlsGroupConnect(comm, shareableHandle, comm->localRankToRank[0], &mcHandle), ret, fail);
}
CUCHECKGOTO(cuMulticastAddDevice(mcHandle, comm->nvlsResources->dev), ret, fail);
CUCHECKGOTO(cuMulticastBindAddr(mcHandle, 0, (CUdeviceptr)myReqData->reqBuff, minSize, 0), ret, fail);
// Create a VA for the NVLS
CUCHECKGOTO(cuMemAddressReserve(&regPtr, minSize, granularity, 0U, 0), ret, fail);
// Map the VA locally
CUCHECKGOTO(cuMemMap(regPtr, minSize, 0, mcHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess(regPtr, minSize, &comm->nvlsResources->accessDesc, 1), ret, fail);
NCCLCHECKGOTO(ncclCalloc(&regRecord, 1), ret, fail);
regRecord->buff = myReqData->reqBuff;
regRecord->size = myReqData->reqSize;
regRecord->regAddr = regPtr;
regRecord->regSize = minSize;
regRecord->dev = comm->nvlsResources->dev;
regRecord->mcHandle = mcHandle;
/* get all buffer addresses */
NCCLCHECKGOTO(ncclCalloc(&regRecord->addrs, comm->localRanks), ret, fail);
regRecord->addrs[comm->localRank] = regRecord->buff;
NCCLCHECKGOTO(ncclShmemAllgather(comm, &comm->nvlsShmem, regRecord->addrs + comm->localRank, regRecord->addrs, sizeof(uintptr_t)), ret, fail);
/* enqueue record */
ncclIntruQueueEnqueue(&comm->regRecordQueue, regRecord);
localRegBufUsed = true;
exit:
if (localRegBufUsed)
*regAddr = (uintptr_t)regPtr + userBuff - myReqData->reqBuff;
*regUsed = localRegBufUsed;
return ret;
fail:
localRegBufUsed = false;
goto exit;
}
ncclResult_t ncclNvlsLocalRegisterBuffer(struct ncclComm *comm, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, bool *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv) {
ncclResult_t ret = ncclSuccess;
bool localRegBufUsed = false;
struct localRegData *regData = NULL;
struct localRequestData *reqData = NULL;
struct ncclRegRecord *regRecordHead = NULL, *sendRegRecord = NULL, *recvRegRecord = NULL;
struct ncclRegRequest *regRequestHead = NULL, *sendRegRequest = NULL, *recvRegRequest = NULL;
bool sendNeedReg = false, recvNeedReg = false;
CUdeviceptr regSendPtr = 0;
CUdeviceptr regRecvPtr = 0;
*outRegBufUsed = false;
NCCLCHECKGOTO(ncclCalloc(&regData, comm->localRanks), ret, fail);
/* first check whether the buffer has been registered and matches each other globally */
regRecordHead = ncclIntruQueueHead(&comm->regRecordQueue);
while (regRecordHead && ((sendRegRecord == NULL && sendbuff != NULL) || (recvRegRecord == NULL && recvbuff != NULL))) {
/* check send reg record */
if (sendRegRecord == NULL && regRecordHead->buff <= (uintptr_t)sendbuff &&
regRecordHead->buff + regRecordHead->size >= (uintptr_t)sendbuff + sendbuffSize) {
regData[comm->localRank].recSendbuff = regRecordHead->buff;
regData[comm->localRank].recSendOffset = (uintptr_t)sendbuff - regRecordHead->buff;
sendRegRecord = regRecordHead;
}
/* check recv reg record */
if (recvRegRecord == NULL && regRecordHead->buff <= (uintptr_t)recvbuff &&
regRecordHead->buff + regRecordHead->size >= (uintptr_t)recvbuff + recvbuffSize) {
regData[comm->localRank].recRecvbuff = regRecordHead->buff;
regData[comm->localRank].recRecvOffset = (uintptr_t)recvbuff - regRecordHead->buff;
recvRegRecord = regRecordHead;
}
regRecordHead = regRecordHead->next;
}
/* prepare registration request for later reference */
regRequestHead = ncclIntruQueueHead(&comm->regRequestQueue);
while (regRequestHead && ((sendRegRequest == NULL && sendbuff != NULL) || (recvRegRequest == NULL && recvbuff != NULL))) {
/* check send reg request */
if (regRequestHead->buff <= (uintptr_t)sendbuff &&
regRequestHead->buff + regRequestHead->size >= (uintptr_t)sendbuff + sendbuffSize) {
regData[comm->localRank].reqSendbuff = regRequestHead->buff;
regData[comm->localRank].reqSendSize = regRequestHead->size;
regData[comm->localRank].reqSendOffset = (uintptr_t)sendbuff - regRequestHead->buff;
sendRegRequest = regRequestHead;
}
/* check recv reg request */
if (regRequestHead->buff <= (uintptr_t)recvbuff &&
regRequestHead->buff + regRequestHead->size >= (uintptr_t)recvbuff + recvbuffSize) {
regData[comm->localRank].reqRecvbuff = regRequestHead->buff;
regData[comm->localRank].reqRecvSize = regRequestHead->size;
regData[comm->localRank].reqRecvOffset = (uintptr_t)recvbuff - regRequestHead->buff;
recvRegRequest = regRequestHead;
}
regRequestHead = regRequestHead->next;
}
NCCLCHECKGOTO(ncclShmemAllgather(comm, &comm->nvlsShmem, regData + comm->localRank, regData, sizeof(struct localRegData)), ret, fail);
/* first check whether all local ranks find their registered buffer */
for (int i = 0; i < comm->localRanks; ++i) {
if (regData[i].recSendbuff == 0 || sendRegRecord->addrs[i] != regData[i].recSendbuff) {
sendNeedReg = true;
}
if (regData[i].recRecvbuff == 0 || recvRegRecord->addrs[i] != regData[i].recRecvbuff) {
recvNeedReg = true;
}
}
if (sendNeedReg == false) {
for (int i = 0; i < comm->localRanks - 1; ++i) {
if (regData[i].recSendOffset != regData[i + 1].recSendOffset) {
/* offset are different, we cannot apply user buffer registration */
goto fail;
}
}
/* reuse previous registered buffer if possible */
if (!sendNeedReg)
regSendPtr = (CUdeviceptr)((uintptr_t)sendRegRecord->regAddr + regData[comm->localRank].recSendOffset);
}
if (recvNeedReg == false) {
for (int i = 0; i < comm->localRanks - 1; ++i) {
if (regData[i].recRecvOffset != regData[i + 1].recRecvOffset) {
goto fail;
}
}
if (!recvNeedReg)
regRecvPtr = (CUdeviceptr)((uintptr_t)recvRegRecord->regAddr + regData[comm->localRank].recRecvOffset);
}
if ((!sendNeedReg || sendbuff == NULL) && (!recvNeedReg || recvbuff == NULL)) {
localRegBufUsed = true;
INFO(NCCL_NVLS, "rank %d reuse local-registered sendbuff %p, recvbuff %p, sendbuff size %ld, recvbuff size %ld, reg sendbuff %p, reg recvbuff %p", comm->rank, sendbuff, recvbuff, sendbuffSize, recvbuffSize, (void*)regSendPtr, (void*)regRecvPtr);
goto exit;
}
/* Start Registration. Not found registered buffers, then check whether both send and recv buffer locate
* in register request cache. */
NCCLCHECKGOTO(ncclCalloc(&reqData, comm->localRanks), ret, fail);
if (sendNeedReg && sendbuff != NULL) {
/* copy request data got from previous shmem AG */
intptr_t offset = regData[0].reqSendOffset;
for (int i = 0; i < comm->localRanks; ++i) {
if (regData[i].reqSendbuff == 0 || offset != regData[i].reqSendOffset) goto fail;
reqData[i].reqBuff = regData[i].reqSendbuff;
reqData[i].reqSize = regData[i].reqSendSize;
reqData[i].reqOffset = regData[i].reqSendOffset;
}
tryRegisterBuffer(comm, reqData, (uintptr_t)sendbuff, sendbuffSize, &regSendPtr, &localRegBufUsed);
if (localRegBufUsed == false) goto fail;
}
if (recvNeedReg && recvbuff != NULL) {
intptr_t offset = regData[0].reqRecvOffset;
for (int i = 0; i < comm->localRanks; ++i) {
if (regData[i].reqRecvbuff == 0 || offset != regData[i].reqRecvOffset) goto fail;
reqData[i].reqBuff = regData[i].reqRecvbuff;
reqData[i].reqSize = regData[i].reqRecvSize;
reqData[i].reqOffset = regData[i].reqRecvOffset;
}
tryRegisterBuffer(comm, reqData, (uintptr_t)recvbuff, recvbuffSize, &regRecvPtr, &localRegBufUsed);
if (localRegBufUsed == false) goto fail;
}
INFO(NCCL_NVLS, "rank %d successfully local-registered sendbuff %p, recvbuff %p, sendbuff size %ld, recvbuff size %ld, reg sendbuff %p, reg recvbuff %p", comm->rank, sendbuff, recvbuff, sendbuffSize, recvbuffSize, (void*)regSendPtr, (void*)regRecvPtr);
exit:
*outRegBufSend = (void*)regSendPtr;
*outRegBufRecv = (void*)regRecvPtr;
*outRegBufUsed = localRegBufUsed;
free(regData);
free(reqData);
return ncclSuccess;
fail:
localRegBufUsed = false;
goto exit;
}
ncclResult_t ncclNvlsGraphRegisterBuffer(struct ncclComm *comm, struct ncclKernelPlan *plan, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, bool *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv) {
ncclResult_t ret = ncclSuccess;
bool localRegBufUsed = false;
struct ncclNvlsMcHandleList* sendRecord = NULL;
struct ncclNvlsMcHandleList* recvRecord = NULL;
CUdeviceptr regSendPtr = 0;
CUdeviceptr regRecvPtr = 0;
CUmulticastObjectProp prop;
char shareableHandle[NVLS_HANDLE_SIZE];
CUmemGenericAllocationHandle sendMcHandle, recvMcHandle;
size_t sendGran, recvGran;
bool *regBufFlags = NULL;
struct graphRegData *rdata = NULL;
const void *baseSend = NULL;
const void *baseRecv = NULL;
size_t baseSendSize = 1;
size_t baseRecvSize = 1;
*outRegBufUsed = false;
NCCLCHECKGOTO(ncclCalloc(&regBufFlags, comm->localRanks), ret, fail);
NCCLCHECKGOTO(ncclCalloc(&rdata, comm->localRanks), ret, fail);
if (sendbuffSize > 0 || recvbuffSize > 0) {
/* retrieve base pointer and size */
if (CUPFN(cuMemGetAddressRange) == nullptr) goto fail;
if (sendbuff != NULL)
CUCHECKGOTO(cuMemGetAddressRange((CUdeviceptr *)&baseSend, &baseSendSize, (CUdeviceptr)sendbuff), ret, fail);
if (recvbuff != NULL)
CUCHECKGOTO(cuMemGetAddressRange((CUdeviceptr *)&baseRecv, &baseRecvSize, (CUdeviceptr)recvbuff), ret, fail);
memcpy(&prop, &comm->nvlsResources->properties, sizeof(CUmulticastObjectProp));
prop.size = baseSendSize;
CUCHECKGOTO(cuMulticastGetGranularity(&sendGran, &prop, CU_MULTICAST_GRANULARITY_RECOMMENDED), ret, fail);
prop.size = baseRecvSize;
CUCHECKGOTO(cuMulticastGetGranularity(&recvGran, &prop, CU_MULTICAST_GRANULARITY_RECOMMENDED), ret, fail);
localRegBufUsed = ((uint64_t)baseSend % sendGran != 0 || (uint64_t)baseRecv % recvGran != 0) ? false : true;
regBufFlags[comm->localRank] = localRegBufUsed;
NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, regBufFlags, sizeof(bool)), ret, fail);
for (int i = 0; i < comm->localRanks; ++i)
if (regBufFlags[i] == false) goto fail;
if (sendbuff != NULL) {
/* check send buffer offset and size */
rdata[comm->localRank].offset = (uintptr_t)sendbuff - (uintptr_t)baseSend;
rdata[comm->localRank].size = baseSendSize;
NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, rdata, sizeof(struct graphRegData)), ret, fail);
baseSendSize = rdata[0].size;
for (int i = 1; i < comm->localRanks; ++i) {
if (rdata[0].offset != rdata[i].offset) goto fail;
if (baseSendSize > rdata[i].size) baseSendSize = rdata[i].size;
}
if (baseSendSize % sendGran != 0) goto fail;
prop.size = baseSendSize;
/* register sendbuff */
if (comm->localRank == 0) {
NCCLCHECKGOTO(nvlsGroupCreate(comm, &prop, comm->localRank, comm->localRanks, &sendMcHandle, shareableHandle), ret, fail);
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail);
} else {
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail);
NCCLCHECKGOTO(nvlsGroupConnect(comm, shareableHandle, comm->localRankToRank[0], &sendMcHandle), ret, fail);
}
CUCHECKGOTO(cuMulticastAddDevice(sendMcHandle, comm->nvlsResources->dev), ret, fail);
CUCHECKGOTO(cuMulticastBindAddr(sendMcHandle, 0, (CUdeviceptr)baseSend, baseSendSize, 0), ret, fail);
// Create a VA for the NVLS
CUCHECKGOTO(cuMemAddressReserve(&regSendPtr, baseSendSize, sendGran, 0U, 0), ret, fail);
// Map the VA locally
CUCHECKGOTO(cuMemMap(regSendPtr, baseSendSize, 0, sendMcHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess(regSendPtr, baseSendSize, &comm->nvlsResources->accessDesc, 1), ret, fail);
sendRecord = ncclMemoryPoolAlloc<struct ncclNvlsMcHandleList>(&comm->memPool_ncclNvlsHandleList, &comm->memPermanent);
sendRecord->mcHandle = sendMcHandle;
sendRecord->ptr = regSendPtr;
sendRecord->dev = comm->nvlsResources->dev;
sendRecord->size = baseSendSize;
}
if (recvbuff != NULL) {
rdata[comm->localRank].offset = (uintptr_t)recvbuff - (uintptr_t)baseRecv;
rdata[comm->localRank].size = baseRecvSize;
NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, rdata, sizeof(struct graphRegData)), ret, fail);
baseRecvSize = rdata[0].size;
for (int i = 1; i < comm->localRanks; ++i) {
if (rdata[0].offset != rdata[i].offset) goto fail;
if (baseRecvSize > rdata[i].size) baseRecvSize = rdata[i].size;
}
if (baseRecvSize % recvGran != 0) goto fail;
prop.size = baseRecvSize;
if (comm->localRank == 0) {
NCCLCHECKGOTO(nvlsGroupCreate(comm, &prop, comm->localRank, comm->localRanks, &recvMcHandle, shareableHandle), ret, fail);
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail);
} else {
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail);
NCCLCHECKGOTO(nvlsGroupConnect(comm, shareableHandle, comm->localRankToRank[0], &recvMcHandle), ret, fail);
}
CUCHECKGOTO(cuMulticastAddDevice(recvMcHandle, comm->nvlsResources->dev), ret, fail);
CUCHECKGOTO(cuMulticastBindAddr(recvMcHandle, 0, (CUdeviceptr)baseRecv, baseRecvSize, 0), ret, fail);
// Create a VA for the NVLS
CUCHECKGOTO(cuMemAddressReserve(&regRecvPtr, baseRecvSize, recvGran, 0U, 0), ret, fail);
// Map the VA locally
CUCHECKGOTO(cuMemMap(regRecvPtr, baseRecvSize, 0, recvMcHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess(regRecvPtr, baseRecvSize, &comm->nvlsResources->accessDesc, 1), ret, fail);
recvRecord = ncclMemoryPoolAlloc<struct ncclNvlsMcHandleList>(&comm->memPool_ncclNvlsHandleList, &comm->memPermanent);
recvRecord->mcHandle = recvMcHandle;
recvRecord->ptr = regRecvPtr;
recvRecord->dev = comm->nvlsResources->dev;
recvRecord->size = baseRecvSize;
}
localRegBufUsed = true;
}
exit:
if (localRegBufUsed == false) {
if (sendRecord) {
ncclNvlsDeregBuffer(&sendRecord->mcHandle, sendRecord->ptr, sendRecord->dev, sendRecord->size);
ncclMemoryPoolFree(&comm->memPool_ncclNvlsHandleList, sendRecord);
}
if (recvRecord) {
ncclNvlsDeregBuffer(&recvRecord->mcHandle, recvRecord->ptr, recvRecord->dev, recvRecord->size);
ncclMemoryPoolFree(&comm->memPool_ncclNvlsHandleList, recvRecord);
}
} else {
if (sendRecord) {
*outRegBufSend = (void*)((uintptr_t)regSendPtr + (uintptr_t)sendbuff - (uintptr_t)baseSend);
ncclIntruQueueEnqueue(&plan->nvlsMcHandleQueue, sendRecord);
}
if (recvRecord) {
*outRegBufRecv = (void*)((uintptr_t)regRecvPtr + (uintptr_t)recvbuff - (uintptr_t)baseRecv);
ncclIntruQueueEnqueue(&plan->nvlsMcHandleQueue, recvRecord);
}
INFO(NCCL_NVLS, "rank %d successfully graph-registered sendbuff %p, recvbuff %p, sendbuff size %ld (register size %ld, sendGran %ld), recvbuff size %ld (register size %ld, recvGran %ld), reg sendbuff %p, reg recvbuff %p", comm->rank, sendbuff, recvbuff, sendbuffSize, baseSendSize, sendGran, recvbuffSize, baseRecvSize, recvGran, (void*)regSendPtr, (void*)regRecvPtr);
}
*outRegBufUsed = localRegBufUsed;
free(regBufFlags);
free(rdata);
/* always return success. */
return ncclSuccess;
fail:
localRegBufUsed = false;
goto exit;
}
#else
/*
@@ -413,4 +810,18 @@ ncclResult_t ncclNvlsFree(struct ncclComm* comm) {
return ncclSuccess;
}
ncclResult_t ncclNvlsGraphRegisterBuffer(struct ncclComm *comm, struct ncclKernelPlan *plan, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, bool *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv) {
*outRegBufUsed = false;
return ncclSuccess;
}
ncclResult_t ncclNvlsLocalRegisterBuffer(struct ncclComm *comm, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, bool *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv) {
*outRegBufUsed = false;
return ncclSuccess;
}
ncclResult_t ncclNvlsDeregBuffer(CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t size) {
return ncclSuccess;
}
#endif /* CUDA_VERSION >= 12010 */
src/transport/p2p.cc
+36 -29
View File
@@ -206,9 +206,13 @@ ncclResult_t ncclP2pAllocateShareableBuffer(size_t size, ncclIpcDesc *ipcDesc, v
// cuMem API support
CUmemAllocationHandleType type = NCCL_P2P_HANDLE_TYPE;
CUmemGenericAllocationHandle handle;
NCCLCHECK(ncclCuMemAlloc(ptr, &handle, size));
CUCHECK(cuMemExportToShareableHandle(&ipcDesc->cuDesc, handle, type, 0));
if (type == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) {
// Return the native cuMem handle for later Export/Import via UDS
memcpy(&ipcDesc->cuDesc.data, &handle, sizeof(handle));
} else {
CUCHECK(cuMemExportToShareableHandle(&ipcDesc->cuDesc, handle, type, 0));
}
#else
return ncclInternalError;
#endif
@@ -228,17 +232,6 @@ ncclResult_t ncclP2pAllocateShareableBuffer(size_t size, ncclIpcDesc *ipcDesc, v
}
ncclResult_t ncclP2pFreeShareableBuffer(ncclIpcDesc *ipcDesc) {
if (ncclCuMemEnable()) {
// cuMem API support
CUmemAllocationHandleType type = NCCL_P2P_HANDLE_TYPE;
if (type == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) {
int fd = *(int *) &ipcDesc->cuDesc.data;
if (fd <= 0) return ncclInternalError;
(void) close(fd);
}
}
return ncclSuccess;
}
@@ -255,20 +248,20 @@ ncclResult_t ncclP2pImportShareableBuffer(struct ncclComm *comm, int tpPeer, siz
if (type == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) {
// UDS fd support
struct ncclProxyConnector proxyConn;
int fd = *(int *)(&cuDesc->data);
int newFd = -1;
int fd = -1;
// Send cuMem handle to remote for conversion to an fd
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_P2P, 1, tpPeer, &proxyConn));
NCCLCHECK(ncclProxyClientConvertFdBlocking(comm, &proxyConn, fd, &newFd));
INFO(NCCL_P2P, "UDS converted fd %d -> %d on peer %d", fd, newFd, tpPeer);
CUCHECK(cuMemImportFromShareableHandle(&handle, (void *)(uintptr_t)newFd, type));
close(newFd);
NCCLCHECK(ncclProxyClientGetFdBlocking(comm, &proxyConn, &cuDesc->data, &fd));
INFO(NCCL_P2P, "UDS converted handle 0x%lx to fd %d on remote peer %d", *(uint64_t*)&cuDesc->data, fd, tpPeer);
CUCHECK(cuMemImportFromShareableHandle(&handle, (void *)(uintptr_t)fd, type));
(void) close(fd);
} else {
CUCHECK(cuMemImportFromShareableHandle(&handle, cuDesc, type));
}
CUCHECK(cuMemAddressReserve(&dptr, size, /* alignment */ 0, /* addr */ 0, /* flags */ 0));
CUCHECK(cuMemMap(dptr, size, /* offset */ 0, handle, /* flags */ 0));
TRACE(NCCL_P2P, "Imported shareable buffer size %zi handle 0x%lx dptr %p", size, (long)handle, (void*)dptr);
TRACE(NCCL_P2P, "Imported shareable buffer size %zi handle 0x%llx dptr %p", size, handle, (void*)dptr);
// Allow access by the local GPU
CUmemAccessDesc accessDesc = {};
@@ -276,7 +269,7 @@ ncclResult_t ncclP2pImportShareableBuffer(struct ncclComm *comm, int tpPeer, siz
accessDesc.location.id = comm->cudaDev;
accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
CUCHECK(cuMemSetAccess(dptr, size, &accessDesc, 1));
TRACE(NCCL_P2P, "Set Access for %p size %zi dev %d", (void*)dptr, size, accessDesc.location.id);
TRACE(NCCL_P2P, "Set Access for %p size %zi on dev %d", (void*)dptr, size, accessDesc.location.id);
*devMemPtr = (void *)dptr;
#else
@@ -307,8 +300,8 @@ static ncclResult_t p2pGetInfo(struct ncclTopoSystem* topo, struct ncclPeerInfo*
return ncclSuccess;
}
static ncclResult_t p2pMap(struct ncclComm *comm, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclP2pBuff* p2pBuff, void** devMem, void** ipcPtr) {
if (!ncclCuMemEnable() && myInfo->pidHash == peerInfo->pidHash) {
static ncclResult_t p2pMap(struct ncclComm *comm, struct ncclProxyConnector* proxyConn, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclP2pBuff* p2pBuff, void** devMem, void** ipcPtr) {
if (myInfo->pidHash == peerInfo->pidHash) {
if (peerInfo->cudaDev != myInfo->cudaDev) {
// Same PID different GPUs, enable P2P access
// Legacy CUDA IPC
@@ -320,6 +313,18 @@ static ncclResult_t p2pMap(struct ncclComm *comm, struct ncclPeerInfo* myInfo, s
peerInfo->cudaDev, peerInfo->busId, err, cudaGetErrorString(err));
return ncclInternalError;
}
#if CUDART_VERSION >= 11030
// cuMem API support
if (ncclCuMemEnable()) {
// Allow direct access to the remote buffer from the local GPU
CUmemAccessDesc accessDesc = {};
accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
accessDesc.location.id = myInfo->cudaDev;
accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
INFO(NCCL_P2P, "Set Access for buffer %p size %zi on dev %d", p2pBuff->directPtr, p2pBuff->size, peerInfo->cudaDev);
CUCHECK(cuMemSetAccess((CUdeviceptr) p2pBuff->directPtr, p2pBuff->size, &accessDesc, 1));
}
#endif
}
*devMem = p2pBuff->directPtr;
*ipcPtr = NULL;
@@ -373,7 +378,7 @@ ncclResult_t p2pSendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, st
if (intermediateRank == -1) {
info->rank = myInfo->rank;
if (myInfo->pidHash == peerInfo->pidHash && ncclParamP2pDirectDisable() == 0 && useMemcpy == 0 && !ncclCuMemEnable()) {
if (myInfo->pidHash == peerInfo->pidHash && ncclParamP2pDirectDisable() == 0 && useMemcpy == 0) {
resources->type = P2P_DIRECT;
send->conn.flags |= info->read ? NCCL_DIRECT_READ : NCCL_DIRECT_WRITE;
INFO(NCCL_INIT|NCCL_P2P, "Channel %02d/%01d : %d[%lx] -> %d[%lx] via P2P/direct pointer%s comm %p nRanks %02d",
@@ -408,7 +413,7 @@ ncclResult_t p2pSendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, st
memcpy(info->shmName, resources->proxyInfo.shmName, sizeof(info->shmName));
} else {
NCCLCHECK(ncclProxyCallBlocking(comm, &send->proxyConn, ncclProxyMsgSetup, &sendSize, sizeof(int), &info->p2pBuff, sizeof(struct ncclP2pBuff)));
NCCLCHECK(p2pMap(comm, myInfo, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&resources->sendDevMem, &resources->sendMemIpc));
NCCLCHECK(p2pMap(comm, &send->proxyConn, myInfo, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&resources->sendDevMem, &resources->sendMemIpc));
}
return ncclSuccess;
@@ -437,7 +442,7 @@ ncclResult_t p2pRecvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, st
if (intermediateRank == -1) {
info->rank = myInfo->rank;
if (myInfo->pidHash == peerInfo->pidHash && ncclParamP2pDirectDisable() == 0 && useMemcpy == 0 && !ncclCuMemEnable()) {
if (myInfo->pidHash == peerInfo->pidHash && ncclParamP2pDirectDisable() == 0 && useMemcpy == 0) {
resources->type = P2P_DIRECT;
recv->conn.flags |= info->read ? NCCL_DIRECT_READ : NCCL_DIRECT_WRITE;
} else {
@@ -461,7 +466,7 @@ ncclResult_t p2pRecvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, st
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_P2P, 0, tpProxyRank, &recv->proxyConn));
NCCLCHECK(ncclProxyCallBlocking(comm, &recv->proxyConn, ncclProxyMsgSetup, &recvSize, sizeof(int), &info->p2pBuff, sizeof(struct ncclP2pBuff)));
NCCLCHECK(p2pMap(comm, myInfo, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&resources->recvDevMem, &resources->recvMemIpc));
NCCLCHECK(p2pMap(comm, &recv->proxyConn, myInfo, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&resources->recvDevMem, &resources->recvMemIpc));
return ncclSuccess;
}
@@ -471,7 +476,7 @@ static ncclResult_t p2pSendConnect(struct ncclComm* comm, struct ncclConnect* co
struct ncclRecvMem* remDevMem = NULL;
struct p2pConnectInfo* info = (struct p2pConnectInfo*)connectInfo;
NCCLCHECK(p2pMap(comm, comm->peerInfo+rank, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&remDevMem, &resources->recvMemIpc));
NCCLCHECK(p2pMap(comm, &send->proxyConn, comm->peerInfo+rank, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&remDevMem, &resources->recvMemIpc));
char* buff = (char*)(remDevMem+1);
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
@@ -498,6 +503,8 @@ static ncclResult_t p2pSendConnect(struct ncclComm* comm, struct ncclConnect* co
send->conn.ptrExchange = &resources->sendDevMem->ptrExchange;
send->conn.redOpArgExchange = resources->sendDevMem->redOpArgExchange;
}
// We must assign the proxyConn's proxyProgress property for proper checking at enqueue-time
send->proxyConn.proxyProgress = p2pTransport.send.proxyProgress;
return ncclSuccess;
}
@@ -519,7 +526,7 @@ ncclResult_t p2pRecvConnect(struct ncclComm* comm, struct ncclConnect* connectIn
recv->conn.tail = &resources->devShm->recvMem.tail;
recv->conn.head = &resources->devShm->sendMem.head;
} else {
NCCLCHECK(p2pMap(comm, comm->peerInfo+rank, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&remDevMem, &resources->sendMemIpc));
NCCLCHECK(p2pMap(comm, &recv->proxyConn, comm->peerInfo+rank, comm->peerInfo+info->rank, &info->p2pBuff, (void**)&remDevMem, &resources->sendMemIpc));
struct ncclRecvMem* devMem = resources->recvDevMem;
recv->conn.tail = &devMem->tail;
src/transport/shm.cc
+8
View File
@@ -164,6 +164,10 @@ static ncclResult_t shmSendConnect(struct ncclComm* comm, struct ncclConnect* co
send->conn.tail = &proxyInfo.ceRecvMem->tail;
send->conn.sizesFifo = proxyInfo.ceRecvMem->sizesFifo;
}
// We must assign the proxyConn's proxyProgress property for proper checking at enqueue-time
send->proxyConn.proxyProgress = shmTransport.send.proxyProgress;
return ncclSuccess;
}
@@ -193,6 +197,10 @@ static ncclResult_t shmRecvConnect(struct ncclComm* comm, struct ncclConnect* co
recv->conn.buffs[NCCL_PROTO_SIMPLE] = proxyInfo.devFifo;
recv->conn.tail = &proxyInfo.ceRecvMem->tail;
}
// We must assign the proxyConn's proxyProgress property for proper checking at enqueue-time
recv->proxyConn.proxyProgress = shmTransport.recv.proxyProgress;
return ncclSuccess;
}