Merge remote-tracking branch 'nccl/master' into HEAD
Этот коммит содержится в:
Wenkai Du
+267
-267
@@ -1,6 +1,6 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2015-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
|
||||
* Modifications Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
@@ -245,7 +245,7 @@ static ncclResult_t selectTransport(struct ncclComm* comm, struct ncclTopoGraph*
|
||||
ncclResult_t ncclTransportP2pConnect(struct ncclComm* comm, int channelId, int nrecv, int* peerRecv, int nsend, int* peerSend, int connIndex) {
|
||||
TRACE(NCCL_INIT, "nsend %d nrecv %d", nsend, nrecv);
|
||||
struct ncclChannel* channel = &comm->channels[channelId];
|
||||
uint32_t mask = 1 << channelId;
|
||||
uint64_t mask = 1UL << channel->id;
|
||||
for (int i=0; i<nrecv; i++) {
|
||||
int peer = peerRecv[i];
|
||||
if (peer == -1 || peer >= comm->nRanks || peer == comm->rank || channel->peers[peer].recv[connIndex].connected) continue;
|
||||
@@ -270,72 +270,87 @@ void dumpData(struct ncclConnect* data, int ndata) {
|
||||
|
||||
ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, int connIndex, int* highestTransportType/*=NULL*/) {
|
||||
// Stream used during transport setup; need for P2P pre-connect + CUDA Graph
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
int highestType = TRANSPORT_P2P; // track highest transport type
|
||||
|
||||
//hipStream_t transportSetupStream;
|
||||
//CUDACHECK(hipStreamCreateWithFlags(&transportSetupStream, hipStreamNonBlocking));
|
||||
|
||||
struct ncclConnect data[2*MAXCHANNELS];
|
||||
|
||||
//NCCLCHECKGOTO(ncclStrongStreamAcquireUncaptured(&comm->hostStream), ret, fail);
|
||||
for (int i=1; i<comm->nRanks; i++) {
|
||||
int bootstrapTag = (i<<8) + (graph ? graph->id+1 : 0);
|
||||
int recvPeer = (comm->rank - i + comm->nRanks) % comm->nRanks;
|
||||
int sendPeer = (comm->rank + i) % comm->nRanks;
|
||||
uint32_t recvMask = comm->connectRecv[recvPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)];
|
||||
uint32_t sendMask = comm->connectSend[sendPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)];
|
||||
uint64_t recvMask = comm->connectRecv[recvPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)];
|
||||
uint64_t sendMask = comm->connectSend[sendPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)];
|
||||
|
||||
struct ncclConnect* recvData = data;
|
||||
int sendChannels = 0, recvChannels = 0;
|
||||
int type;
|
||||
TIME_START(0);
|
||||
for (int c=0; c<MAXCHANNELS; c++) {
|
||||
if (recvMask & (1<<c)) {
|
||||
NCCLCHECK(selectTransport<0>(comm, graph, recvData+recvChannels++, c, recvPeer, connIndex, &type));
|
||||
if (recvMask & (1UL<<c)) {
|
||||
NCCLCHECKGOTO(selectTransport<0>(comm, graph, recvData+recvChannels++, c, recvPeer, connIndex, &type), ret, fail);
|
||||
if (type > highestType) highestType = type;
|
||||
}
|
||||
}
|
||||
TIME_STOP(0);
|
||||
TIME_START(1);
|
||||
struct ncclConnect* sendData = recvData+recvChannels;
|
||||
for (int c=0; c<MAXCHANNELS; c++) {
|
||||
if (sendMask & (1<<c)) {
|
||||
NCCLCHECK(selectTransport<1>(comm, graph, sendData+sendChannels++, c, sendPeer, connIndex, &type));
|
||||
if (sendMask & (1UL<<c)) {
|
||||
NCCLCHECKGOTO(selectTransport<1>(comm, graph, sendData+sendChannels++, c, sendPeer, connIndex, &type), ret, fail);
|
||||
if (type > highestType) highestType = type;
|
||||
}
|
||||
}
|
||||
TIME_STOP(1);
|
||||
|
||||
TIME_START(2);
|
||||
if (sendPeer == recvPeer) {
|
||||
if (recvChannels+sendChannels) {
|
||||
//NCCLCHECK(bootstrapSend(comm->bootstrap, recvPeer, bootstrapTag, data, sizeof(struct ncclConnect)*(recvChannels+sendChannels)));
|
||||
//NCCLCHECK(bootstrapRecv(comm->bootstrap, recvPeer, bootstrapTag, data, sizeof(struct ncclConnect)*(recvChannels+sendChannels)));
|
||||
//NCCLCHECKGOTO(bootstrapSend(comm->bootstrap, recvPeer, bootstrapTag, data, sizeof(struct ncclConnect)*(recvChannels+sendChannels)), ret, fail);
|
||||
//NCCLCHECKGOTO(bootstrapRecv(comm->bootstrap, recvPeer, bootstrapTag, data, sizeof(struct ncclConnect)*(recvChannels+sendChannels)), ret, fail);
|
||||
sendData = data;
|
||||
recvData = data+sendChannels;
|
||||
}
|
||||
} else {
|
||||
//if (recvChannels) NCCLCHECK(bootstrapSend(comm->bootstrap, recvPeer, bootstrapTag, recvData, sizeof(struct ncclConnect)*recvChannels));
|
||||
//if (sendChannels) NCCLCHECK(bootstrapSend(comm->bootstrap, sendPeer, bootstrapTag, sendData, sizeof(struct ncclConnect)*sendChannels));
|
||||
//if (sendChannels) NCCLCHECK(bootstrapRecv(comm->bootstrap, sendPeer, bootstrapTag, sendData, sizeof(struct ncclConnect)*sendChannels));
|
||||
//if (recvChannels) NCCLCHECK(bootstrapRecv(comm->bootstrap, recvPeer, bootstrapTag, recvData, sizeof(struct ncclConnect)*recvChannels));
|
||||
//if (recvChannels) NCCLCHECKGOTO(bootstrapSend(comm->bootstrap, recvPeer, bootstrapTag, recvData, sizeof(struct ncclConnect)*recvChannels), ret, fail);
|
||||
//if (sendChannels) NCCLCHECKGOTO(bootstrapSend(comm->bootstrap, sendPeer, bootstrapTag, sendData, sizeof(struct ncclConnect)*sendChannels), ret, fail);
|
||||
//if (sendChannels) NCCLCHECKGOTO(bootstrapRecv(comm->bootstrap, sendPeer, bootstrapTag, sendData, sizeof(struct ncclConnect)*sendChannels), ret, fail);
|
||||
//if (recvChannels) NCCLCHECKGOTO(bootstrapRecv(comm->bootstrap, recvPeer, bootstrapTag, recvData, sizeof(struct ncclConnect)*recvChannels), ret, fail);
|
||||
}
|
||||
TIME_STOP(2);
|
||||
|
||||
TIME_START(3);
|
||||
for (int c=0; c<MAXCHANNELS; c++) {
|
||||
if (sendMask & (1<<c)) {
|
||||
if (sendMask & (1UL<<c)) {
|
||||
struct ncclConnector* conn = comm->channels[c].peers[sendPeer].send + connIndex;
|
||||
//NCCLCHECK(conn->transportComm->connect(comm, sendData++, 1, comm->rank, conn));
|
||||
//NCCLCHECKGOTO(conn->transportComm->connect(comm, sendData++, 1, comm->rank, conn), ret, fail);
|
||||
conn->connected = 1;
|
||||
//CUDACHECK(hipMemcpyAsync(&comm->channels[c].devPeers[sendPeer].send[connIndex], &conn->conn, sizeof(struct ncclConnInfo), hipMemcpyHostToDevice, transportSetupStream));
|
||||
//CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeers[sendPeer].send[connIndex], &conn->conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->hostStream.cudaStream), ret, fail);
|
||||
//CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeers[sendPeer].send[connIndex], &conn->conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->hostStream.cudaStream), ret, fail);
|
||||
}
|
||||
}
|
||||
TIME_STOP(3);
|
||||
TIME_START(4);
|
||||
for (int c=0; c<MAXCHANNELS; c++) {
|
||||
if (recvMask & (1<<c)) {
|
||||
if (recvMask & (1UL<<c)) {
|
||||
struct ncclConnector* conn = comm->channels[c].peers[recvPeer].recv + connIndex;
|
||||
//NCCLCHECK(conn->transportComm->connect(comm, recvData++, 1, comm->rank, conn));
|
||||
//NCCLCHECKGOTO(conn->transportComm->connect(comm, recvData++, 1, comm->rank, conn), ret, fail);
|
||||
conn->connected = 1;
|
||||
//CUDACHECK(hipMemcpyAsync(&comm->channels[c].devPeers[recvPeer].recv[connIndex], &conn->conn, sizeof(struct ncclConnInfo), hipMemcpyHostToDevice, transportSetupStream));
|
||||
//CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeers[recvPeer].recv[connIndex], &conn->conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->hostStream.cudaStream), ret, fail);
|
||||
}
|
||||
}
|
||||
comm->connectRecv[recvPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)] = comm->connectSend[sendPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)] = 0;
|
||||
TIME_STOP(4);
|
||||
comm->connectRecv[recvPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)] = comm->connectSend[sendPeer+comm->nRanks*(connIndex == NCCL_CONN_IDX_P2P_NET ? NCCL_CONN_IDX_P2P_NET : 0)] = 0UL;
|
||||
}
|
||||
//CUDACHECK(hipStreamSynchronize(transportSetupStream));
|
||||
//CUDACHECK(hipStreamDestroy(transportSetupStream));
|
||||
|
||||
if (highestTransportType != NULL) *highestTransportType = highestType;
|
||||
return ncclSuccess;
|
||||
TIME_PRINT("P2P Setup/Connect");
|
||||
exit:
|
||||
//NCCLCHECK(ncclStrongStreamWaitStream(ncclCudaGraphNone(), &comm->deviceStream, &comm->hostStream));
|
||||
//NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->hostStream));
|
||||
return ret;
|
||||
fail:
|
||||
goto exit;
|
||||
}
|
||||
|
||||
extern struct ncclTransport collNetTransport;
|
||||
@@ -466,32 +481,120 @@ RCCL_PARAM(PivotAlltoallEnable, "PIVOT_ALLTOALL_ENABLE", 0);
|
||||
NCCL_PARAM(AllocP2pNetLLBuffers, "NCCL_ALLOC_P2P_NET_LL_BUFFERS", 0);
|
||||
RCCL_PARAM(LL128ForceEnable, "LL128_FORCE_ENABLE", 0);
|
||||
|
||||
static ncclResult_t collNetTrySetup(ncclComm_t comm, struct ncclTopoGraph* collNetGraph) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
int* heads = NULL;
|
||||
int rank = comm->rank;
|
||||
int collNetSetupFail = 0;
|
||||
int highestTypes[NCCL_MAX_LOCAL_RANKS] = { TRANSPORT_P2P };
|
||||
// Find all head ranks
|
||||
int nHeads = collNetGraph->nChannels;
|
||||
int highestTransportType0, highestTransportType1;
|
||||
char line[1024];
|
||||
|
||||
NCCLCHECKGOTO(ncclCalloc(&heads, nHeads), ret, fail);
|
||||
// Head GPU index is always 0
|
||||
for (int c = 0; c < nHeads; c++) {
|
||||
heads[c] = collNetGraph->intra[c * comm->localRanks + 0];
|
||||
}
|
||||
|
||||
for (int c = 0; c < comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels + c;
|
||||
for (int h = 0; h < nHeads; h++) {
|
||||
const int head = heads[h];
|
||||
collNetSetupFail = ncclTransportCollNetSetup(comm, collNetGraph, channel, head, head, h, collNetRecv);
|
||||
if (!collNetSetupFail) collNetSetupFail = ncclTransportCollNetSetup(comm, collNetGraph, channel, head, head, h, collNetSend);
|
||||
}
|
||||
// Verify CollNet setup across ranks after trying the first channel
|
||||
if (c == 0) {
|
||||
NCCLCHECKGOTO(ncclTransportCollNetCheck(comm, collNetSetupFail), ret, fail);
|
||||
}
|
||||
}
|
||||
// Verify CollNet setup across ranks after trying all channels
|
||||
NCCLCHECKGOTO(ncclTransportCollNetCheck(comm, collNetSetupFail), ret, fail);
|
||||
TRACE(NCCL_INIT, "rank %d Connected inter-node CollNet", rank);
|
||||
|
||||
line[0] = '\0';
|
||||
for (int c = 0; c < comm->nChannels; c++) {
|
||||
struct ncclTree* chain = &comm->channels[c].collnetChain;
|
||||
snprintf(line + strlen(line), 1023 - strlen(line), " [%d] %d->%d->%d",
|
||||
c, chain->down[0], rank, chain->up);
|
||||
}
|
||||
line[1023] = '\0';
|
||||
|
||||
INFO(NCCL_INIT, "Collnet Chains %s", line);
|
||||
// Connect Collnet + chain
|
||||
for (int c = 0; c < comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels + c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->collnetChain.up, 1, channel->collnetChain.down, 0), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, collNetGraph, 0), ret, fail);
|
||||
for (int c = 0; c < comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels + c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, channel->collnetChain.down, 1, &channel->collnetChain.up, 1), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, collNetGraph, 1), ret, fail);
|
||||
INFO(NCCL_INIT, "Connected collnet + chain");
|
||||
|
||||
// Connect intra-node CollNet + Direct
|
||||
for (int c = 0; c < comm->nChannels; c++) {
|
||||
struct ncclChannel* channelRecv = comm->channels + c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, NCCL_MAX_DIRECT_ARITY, channelRecv->collnetDirect.up, NCCL_MAX_DIRECT_ARITY, channelRecv->collnetDirect.down, 0), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, collNetGraph, 0, &highestTransportType0), ret, fail);
|
||||
|
||||
for (int c = 0; c < comm->nChannels; c++) {
|
||||
struct ncclChannel* channelSend = comm->channels + c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, NCCL_MAX_DIRECT_ARITY, channelSend->collnetDirect.down, NCCL_MAX_DIRECT_ARITY, channelSend->collnetDirect.up, 1), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, collNetGraph, 1, &highestTransportType1), ret, fail);
|
||||
|
||||
#if 0
|
||||
// Exchange highest intra-node transport type among ranks
|
||||
// because we need to know whether all ranks can p2p each other to determine whether we can directly read/write registered user buffer
|
||||
comm->intraHighestTransportType = highestTypes[comm->localRank] = highestTransportType0 > highestTransportType1 ? highestTransportType0 : highestTransportType1;
|
||||
NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, highestTypes, sizeof(int)), ret, fail);
|
||||
for (int i = 0; i < comm->localRanks; i++) {
|
||||
if (highestTypes[i] > comm->intraHighestTransportType)
|
||||
comm->intraHighestTransportType = highestTypes[i];
|
||||
}
|
||||
#endif
|
||||
INFO(NCCL_INIT, "rank %d Connected CollNet", rank);
|
||||
|
||||
exit:
|
||||
free(heads);
|
||||
return ret;
|
||||
fail:
|
||||
ncclTransportCollNetFree(comm);
|
||||
comm->collNetSupport = 0;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t *allGather3Data,
|
||||
struct ncclTopoGraph& treeGraph, struct ncclTopoGraph& ringGraph, struct ncclTopoGraph& collNetGraph) {
|
||||
// We use 2 AllGathers
|
||||
// 1. { peerInfo, comm, compCap}
|
||||
// 2. { nChannels, graphInfo, topoRanks }
|
||||
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
int rank = comm->rank;
|
||||
int nranks = comm->nRanks;
|
||||
//uint64_t commHash = getHash(commId->internal, NCCL_UNIQUE_ID_BYTES);
|
||||
cpu_set_t affinitySave;
|
||||
//TRACE(NCCL_INIT, "comm %p, commHash %lx, rank %d nranks %d - BEGIN", comm, commHash, rank, nranks);
|
||||
// [RCCL] Collect the PID of the root
|
||||
int rootPid;
|
||||
//NCCLCHECK(bootstrapInit(commId, comm));
|
||||
// [/RCCL]
|
||||
//NCCLCHECKGOTO(bootstrapInit((struct ncclBootstrapHandle*)commId, comm), ret, fail);
|
||||
|
||||
// AllGather1 - begin
|
||||
//NCCLCHECK(ncclCalloc(&comm->peerInfo, nranks+1)); // Extra rank to represent CollNet root
|
||||
//NCCLCHECK(fillInfo(comm, comm->peerInfo+rank, comm->rank));
|
||||
//NCCLCHECK(bootstrapAllGather(comm->bootstrap, comm->peerInfo, sizeof(struct ncclPeerInfo)));
|
||||
//NCCLCHECKGOTO(ncclCalloc(&comm->peerInfo, nranks+1), ret, fail); // Extra rank to represent CollNet root
|
||||
//NCCLCHECKGOTO(fillInfo(comm, comm->peerInfo+rank, commHash), ret, fail);
|
||||
//NCCLCHECKGOTO(bootstrapAllGather(comm->bootstrap, comm->peerInfo, sizeof(struct ncclPeerInfo)), ret, fail);
|
||||
|
||||
//If virtualId == -1 multiRank support has not been requested by user, using original interface
|
||||
if (comm->virtualId == -1) {
|
||||
for (int i = 0; i < nranks; i++) {
|
||||
if ((i != rank) && (comm->peerInfo[i].hostHash == comm->peerInfo[rank].hostHash) && (comm->peerInfo[i].busId == comm->peerInfo[rank].busId)) {
|
||||
WARN("Duplicate GPU detected : rank %d and rank %d both on CUDA device %lx", rank, i, comm->peerInfo[rank].busId);
|
||||
return ncclInvalidUsage;
|
||||
WARN("Duplicate GPU detected : rank %d and rank %d both on CUDA device %lx", rank, i, comm->peerInfo[rank].busId);
|
||||
ret = ncclInvalidUsage;
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -512,8 +615,43 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t
|
||||
}
|
||||
// AllGather1 - end
|
||||
|
||||
do {
|
||||
// Compute intra-process ranks
|
||||
int intraProcRank0 = -1, intraProcRank = -1, intraProcRanks = 0;
|
||||
for (int i = 0; i < nranks; i++) {
|
||||
if ((comm->peerInfo[i].hostHash == comm->peerInfo[rank].hostHash)
|
||||
&& (comm->peerInfo[i].pidHash == comm->peerInfo[rank].pidHash)) {
|
||||
// Rank is in same process
|
||||
if (intraProcRanks == 0) intraProcRank0 = i;
|
||||
if (i == rank) intraProcRank = intraProcRanks;
|
||||
intraProcRanks++;
|
||||
if (intraProcRank0 == rank && rank != i) {
|
||||
comm->peerInfo[i].comm->intraNext = comm->intraNext;
|
||||
comm->intraNext = comm->peerInfo[i].comm;
|
||||
}
|
||||
}
|
||||
}
|
||||
TRACE(NCCL_INIT,"pidHash[%d] %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, comm->peerInfo[rank].pidHash, intraProcRank, intraProcRanks, intraProcRank0);
|
||||
if (intraProcRank == -1 || intraProcRank0 == -1 || comm->peerInfo[intraProcRank0].comm == NULL) {
|
||||
WARN("Failed to determine intra proc ranks rank %d hostHash %lx pidHash %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, comm->peerInfo[rank].hostHash, comm->peerInfo[rank].pidHash,
|
||||
intraProcRank, intraProcRanks, intraProcRank0);
|
||||
ret = ncclInternalError;
|
||||
goto fail;
|
||||
}
|
||||
struct ncclComm* comm0 = comm->peerInfo[intraProcRank0].comm;
|
||||
assert(intraProcRank==0 ? comm==comm0 : true);
|
||||
comm->intraComm0 = comm0;
|
||||
comm->intraRank = intraProcRank;
|
||||
comm->intraRanks = intraProcRanks;
|
||||
comm->intraBarrierPhase = 0;
|
||||
comm->intraBarrierCounter = 0;
|
||||
comm->intraBarrierGate = 0;
|
||||
} while(0);
|
||||
|
||||
// Topo detection / System graph creation
|
||||
//NCCLCHECK(ncclTopoGetSystem(comm, &comm->topo));
|
||||
//NCCLCHECKGOTO(ncclTopoGetSystem(comm, &comm->topo), ret, fail);
|
||||
// save nRanks to ncclTopoSystem as indicator of multi-node
|
||||
comm->topo->nRanks = comm->nRanks;
|
||||
// init netGdrLevel
|
||||
@@ -524,57 +662,54 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t
|
||||
// LL128
|
||||
comm->topo->ll128Enabled = false;
|
||||
// Compute paths between GPUs and NICs
|
||||
NCCLCHECK(ncclTopoComputePaths(comm->topo, comm));
|
||||
NCCLCHECKGOTO(ncclTopoComputePaths(comm->topo, comm), ret, fail);
|
||||
// Remove inaccessible GPUs and unused NICs
|
||||
NCCLCHECK(ncclTopoTrimSystem(comm->topo, comm));
|
||||
NCCLCHECKGOTO(ncclTopoTrimSystem(comm->topo, comm), ret, fail);
|
||||
// Recompute paths after trimming
|
||||
NCCLCHECK(ncclTopoComputePaths(comm->topo, comm));
|
||||
NCCLCHECKGOTO(ncclTopoComputePaths(comm->topo, comm), ret, fail);
|
||||
// Init search
|
||||
NCCLCHECK(ncclTopoSearchInit(comm->topo));
|
||||
NCCLCHECKGOTO(ncclTopoSearchInit(comm->topo), ret, fail);
|
||||
// Print final topology
|
||||
NCCLCHECK(ncclTopoPrint(comm->topo));
|
||||
NCCLCHECKGOTO(ncclTopoPrint(comm->topo), ret, fail);
|
||||
|
||||
// Set Affinity to a CPU local the our GPU, so that all memory we allocate
|
||||
// on the host is local.
|
||||
//NCCLCHECK(ncclTopoGetCpuAffinity(comm->topo, comm->rank, &comm->cpuAffinity));
|
||||
//cpu_set_t affinitySave;
|
||||
// if (CPU_COUNT(&comm->cpuAffinity)) {
|
||||
//sched_getaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
//sched_setaffinity(0, sizeof(cpu_set_t), &comm->cpuAffinity);
|
||||
//NCCLCHECKGOTO(ncclTopoGetCpuAffinity(comm->topo, comm->rank, &comm->cpuAffinity), ret, fail);
|
||||
//if (CPU_COUNT(&comm->cpuAffinity)) {
|
||||
// sched_getaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
// sched_setaffinity(0, sizeof(cpu_set_t), &comm->cpuAffinity);
|
||||
//}
|
||||
ncclResult_t ret;
|
||||
|
||||
// Launch proxy service thread
|
||||
//NCCLCHECK(ncclProxyCreate(comm));
|
||||
//NCCLCHECKGOTO(ncclProxyCreate(comm), ret, fail);
|
||||
|
||||
// Get rings and trees
|
||||
//struct ncclTopoGraph ringGraph;
|
||||
ringGraph.id = 0;
|
||||
ringGraph.pattern = NCCL_TOPO_PATTERN_RING;
|
||||
ringGraph.collNet = 0;
|
||||
ringGraph.minChannels = 1;
|
||||
ringGraph.maxChannels = MAXCHANNELS/2;
|
||||
NCCLCHECK(ncclTopoCompute(comm->topo, &ringGraph));
|
||||
NCCLCHECK(ncclTopoPrintGraph(comm->topo, &ringGraph));
|
||||
NCCLCHECKGOTO(ncclTopoCompute(comm->topo, &ringGraph), ret, fail);
|
||||
NCCLCHECKGOTO(ncclTopoPrintGraph(comm->topo, &ringGraph), ret, fail);
|
||||
|
||||
//struct ncclTopoGraph treeGraph;
|
||||
treeGraph.id = 1;
|
||||
treeGraph.pattern = NCCL_TOPO_PATTERN_BALANCED_TREE;
|
||||
treeGraph.collNet = 0;
|
||||
treeGraph.minChannels = comm->topo->nodes[NET].count != 0 ? 1 : ringGraph.nChannels;
|
||||
treeGraph.maxChannels = ringGraph.nChannels;
|
||||
NCCLCHECK(ncclTopoCompute(comm->topo, &treeGraph));
|
||||
NCCLCHECK(ncclTopoPrintGraph(comm->topo, &treeGraph));
|
||||
NCCLCHECKGOTO(ncclTopoCompute(comm->topo, &treeGraph), ret, fail);
|
||||
NCCLCHECKGOTO(ncclTopoPrintGraph(comm->topo, &treeGraph), ret, fail);
|
||||
|
||||
//struct ncclTopoGraph collNetGraph;
|
||||
collNetGraph.id = 2;
|
||||
collNetGraph.pattern = NCCL_TOPO_PATTERN_TREE;
|
||||
collNetGraph.collNet = 1;
|
||||
collNetGraph.minChannels = collNetGraph.maxChannels = ringGraph.nChannels;
|
||||
NCCLCHECK(ncclTopoCompute(comm->topo, &collNetGraph));
|
||||
NCCLCHECK(ncclTopoPrintGraph(comm->topo, &collNetGraph));
|
||||
NCCLCHECKGOTO(ncclTopoCompute(comm->topo, &collNetGraph), ret, fail);
|
||||
NCCLCHECKGOTO(ncclTopoPrintGraph(comm->topo, &collNetGraph), ret, fail);
|
||||
|
||||
bool allXgmi = true, hasPeerAccess = true;
|
||||
bool allXgmi, hasPeerAccess;
|
||||
allXgmi = true;
|
||||
hasPeerAccess = true;
|
||||
// Check that all the GPUs have peer access to one another and are XGMI connected
|
||||
for (int i = 0; i < nranks && hasPeerAccess; i++) {
|
||||
int cudaDev1 = comm->peerInfo[i].cudaDev;
|
||||
@@ -599,7 +734,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t
|
||||
|
||||
if (comm->rank == ncclParamGraphDumpFileRank()) {
|
||||
struct ncclTopoGraph* graphs[3] = { &ringGraph, &treeGraph, &collNetGraph };
|
||||
NCCLCHECK(ncclTopoDumpGraphs(comm->topo, 3, graphs));
|
||||
NCCLCHECKGOTO(ncclTopoDumpGraphs(comm->topo, 3, graphs), ret, fail);
|
||||
}
|
||||
|
||||
// Determine local CollNet support before all-gather
|
||||
@@ -623,31 +758,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t
|
||||
INFO(NCCL_INIT, "RCCL force disabled same node P2P over network");
|
||||
}
|
||||
// AllGather3 - begin
|
||||
#if 0
|
||||
struct ncclGraphInfo {
|
||||
int pattern;
|
||||
int nChannels;
|
||||
int sameChannels;
|
||||
float bwIntra;
|
||||
float bwInter;
|
||||
int typeIntra;
|
||||
int typeInter;
|
||||
};
|
||||
|
||||
struct {
|
||||
int netDev;
|
||||
int collNetSupport;
|
||||
int nc;
|
||||
struct ncclGraphInfo tree;
|
||||
struct ncclGraphInfo ring;
|
||||
struct ncclGraphInfo collNet;
|
||||
struct ncclTopoRanks topoRanks;
|
||||
bool pivotA2AEnabled;
|
||||
bool ll128Enabled;
|
||||
} *allGather3Data;
|
||||
|
||||
NCCLCHECK(ncclCalloc(&allGather3Data, nranks));
|
||||
#endif
|
||||
//NCCLCHECKGOTO(ncclCalloc(&allGather3Data, nranks), ret, fail);
|
||||
int idx;
|
||||
NCCLCHECK(ncclTopoIdToIndex(comm->topo, GPU, comm->busId, &idx));
|
||||
allGather3Data[rank].nc = 2;
|
||||
@@ -669,7 +780,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t
|
||||
allGather3Data[rank].nc = std::max(allGather3Data[rank].nc, 4/ringGraph.nChannels);
|
||||
if (ringGraph.nChannels > MAXCHANNELS/2)
|
||||
allGather3Data[rank].nc = 1;
|
||||
NCCLCHECK(ncclTopoGetLocalNet(comm->topo, rank, &allGather3Data[rank].netDev));
|
||||
NCCLCHECKGOTO(ncclTopoGetLocalNet(comm->topo, rank, &allGather3Data[rank].netDev), ret, fail);
|
||||
allGather3Data[rank].tree.pattern = treeGraph.pattern;
|
||||
allGather3Data[rank].tree.nChannels = treeGraph.nChannels;
|
||||
allGather3Data[rank].tree.sameChannels = treeGraph.sameChannels;
|
||||
@@ -698,8 +809,9 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t
|
||||
|
||||
comm->nChannels = (comm->topo->nodes[GPU].count != comm->topo->nRanks && comm->topo->nodes[NET].count)
|
||||
? std::min(treeGraph.nChannels, ringGraph.nChannels) : ringGraph.nChannels;
|
||||
NCCLCHECK(ncclTopoPreset(comm, &treeGraph, &ringGraph, &collNetGraph, &allGather3Data[rank].topoRanks));
|
||||
return ncclSuccess;
|
||||
NCCLCHECKGOTO(ncclTopoPreset(comm, &treeGraph, &ringGraph, &collNetGraph, &allGather3Data[rank].topoRanks), ret, fail);
|
||||
fail:
|
||||
return ret;
|
||||
}
|
||||
|
||||
ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t *allGather3Data,
|
||||
@@ -707,13 +819,20 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
int rank = comm->rank;
|
||||
int nranks = comm->nRanks;
|
||||
ncclResult_t ret;
|
||||
//NCCLCHECK(bootstrapAllGather(comm->bootstrap, allGather3Data, sizeof(*allGather3Data)));
|
||||
int nChannelsOrig;
|
||||
struct ncclTopoRanks** allTopoRanks = NULL;
|
||||
int *nodesFirstRank = NULL, *nodesTreePatterns = NULL;
|
||||
int *rings = NULL;
|
||||
int* nvbPeers = NULL;
|
||||
struct ncclProxyConnector proxyConn;
|
||||
int* pxnPeers = NULL;
|
||||
|
||||
//NCCLCHECKGOTO(bootstrapAllGather(comm->bootstrap, allGather3Data, sizeof(*allGather3Data)), ret, fail);
|
||||
|
||||
// Determine nNodes, firstRanks, ...
|
||||
int *nodesFirstRank, *nodesTreePatterns;
|
||||
NCCLCHECK(ncclCalloc(&nodesFirstRank, nranks));
|
||||
NCCLCHECK(ncclCalloc(&nodesTreePatterns, nranks));
|
||||
NCCLCHECK(ncclCalloc(&comm->rankToNode, comm->nRanks));
|
||||
NCCLCHECKGOTO(ncclCalloc(&nodesFirstRank, nranks), ret, fail);
|
||||
NCCLCHECKGOTO(ncclCalloc(&nodesTreePatterns, nranks), ret, fail);
|
||||
NCCLCHECKGOTO(ncclCalloc(&comm->rankToNode, comm->nRanks), ret, fail);
|
||||
for (int r=0; r<nranks; r++) {
|
||||
int node;
|
||||
int firstRank = allGather3Data[r].topoRanks.ringRecv[0];
|
||||
@@ -727,8 +846,8 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
comm->rankToNode[r] = node;
|
||||
}
|
||||
// Now that we know nNodes, alloc nodeRanks and compute localRanks for each node
|
||||
NCCLCHECK(ncclCalloc(&comm->nodeRanks, comm->nNodes));
|
||||
NCCLCHECK(ncclCalloc(&comm->rankToLocalRank, comm->nRanks));
|
||||
NCCLCHECKGOTO(ncclCalloc(&comm->nodeRanks, comm->nNodes), ret, fail);
|
||||
NCCLCHECKGOTO(ncclCalloc(&comm->rankToLocalRank, comm->nRanks), ret, fail);
|
||||
for (int r=0; r<comm->nRanks; r++) {
|
||||
int node = comm->rankToNode[r];
|
||||
comm->rankToLocalRank[r] = comm->nodeRanks[node].localRanks;
|
||||
@@ -736,7 +855,7 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
}
|
||||
// Allocate ranks arrays for each node
|
||||
for (int n=0; n<comm->nNodes; n++) {
|
||||
NCCLCHECK(ncclCalloc(&comm->nodeRanks[n].localRankToRank, comm->nodeRanks[n].localRanks));
|
||||
NCCLCHECKGOTO(ncclCalloc(&comm->nodeRanks[n].localRankToRank, comm->nodeRanks[n].localRanks), ret, fail);
|
||||
comm->maxLocalRanks = std::max(comm->maxLocalRanks, comm->nodeRanks[n].localRanks);
|
||||
comm->nodeRanks[n].localRanks = 0;
|
||||
}
|
||||
@@ -756,13 +875,14 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
WARN("Failed to determine local ranks rank %d hostHash %lx pidHash %lx localRank %d localRanks %d localRank0 %d",
|
||||
rank, comm->peerInfo[rank].hostHash, comm->peerInfo[rank].pidHash,
|
||||
comm->localRank, comm->localRanks, comm->localRankToRank[0]);
|
||||
return ncclInternalError;
|
||||
ret = ncclInternalError;
|
||||
goto fail;
|
||||
}
|
||||
|
||||
int nChannelsOrig = comm->nChannels;
|
||||
struct ncclTopoRanks** allTopoRanks;
|
||||
NCCLCHECK(ncclCalloc(&allTopoRanks, comm->nRanks));
|
||||
int nc = allGather3Data[0].nc;
|
||||
nChannelsOrig = comm->nChannels;
|
||||
NCCLCHECKGOTO(ncclCalloc(&allTopoRanks, comm->nRanks), ret, fail);
|
||||
int nc;
|
||||
nc = allGather3Data[0].nc;
|
||||
for (int i=0; i<nranks; i++) {
|
||||
comm->peerInfo[i].netDev = allGather3Data[i].netDev;
|
||||
allTopoRanks[i] = &allGather3Data[i].topoRanks;
|
||||
@@ -816,19 +936,14 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
}
|
||||
}
|
||||
|
||||
int *rings;
|
||||
NCCLCHECK(ncclCalloc(&rings, nranks*MAXCHANNELS));
|
||||
NCCLCHECK(ncclTopoPostset(comm, nodesFirstRank, nodesTreePatterns, allTopoRanks, rings, &collNetGraph, nc));
|
||||
NCCLCHECKGOTO(ncclCalloc(&rings, nranks*MAXCHANNELS), ret, fail);
|
||||
NCCLCHECKGOTO(ncclTopoPostset(comm, nodesFirstRank, nodesTreePatterns, allTopoRanks, rings, &collNetGraph, nc), ret, fail);
|
||||
|
||||
if (comm->topo->pivotA2ANumBiRings == 3) {
|
||||
NCCLCHECK(ncclTreeBasePostset(comm, &treeGraph));
|
||||
NCCLCHECK(ncclBinaryTreePostset(comm, &treeGraph));
|
||||
}
|
||||
|
||||
free(allTopoRanks);
|
||||
free(nodesTreePatterns);
|
||||
free(nodesFirstRank);
|
||||
//free(allGather3Data);
|
||||
|
||||
// AllGather3 - end
|
||||
|
||||
@@ -854,125 +969,41 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
INFO(NCCL_INIT, "BinTrees%s", binline);
|
||||
}
|
||||
|
||||
//NCCLCHECK(computeBuffSizes(comm));
|
||||
//NCCLCHECKGOTO(computeBuffSizes(comm), ret, fail);
|
||||
|
||||
// Connect with prev/next for each ring
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
NCCLCHECKGOTO(setupChannel(comm, c, rank, nranks, rings+c*nranks), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(setupChannel(comm, c, rank, nranks, rings+c*nranks), ret, fail);
|
||||
if (comm->nRanks == 1) continue;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->ring.prev, 1, &channel->ring.next, 0), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->ring.prev, 1, &channel->ring.next, 0), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &ringGraph, 0), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &ringGraph, 0), ret, fail);
|
||||
if (ringGraph.nIntraChannels && rcclParamP2pNetDisable() == 0) {
|
||||
comm->useIntraNet = 1;
|
||||
// Connect NET for intranode use
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
if (comm->nRanks == 1) continue;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->ring.prev, 1, &channel->ring.next, NCCL_CONN_IDX_P2P_NET), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->ring.prev, 1, &channel->ring.next, NCCL_CONN_IDX_P2P_NET), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &ringGraph, NCCL_CONN_IDX_P2P_NET), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &ringGraph, NCCL_CONN_IDX_P2P_NET), ret, fail);
|
||||
}
|
||||
free(rings);
|
||||
INFO(NCCL_INIT, "Connected all rings");
|
||||
|
||||
// Connect Trees
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
if (comm->nRanks == 1) continue;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, NCCL_MAX_TREE_ARITY, channel->tree.down, 1, &channel->tree.up, 0), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->tree.up, NCCL_MAX_TREE_ARITY, channel->tree.down, 0), ret, affinity_restore);
|
||||
// RCCL: need to connect binTree as well
|
||||
if (comm->topo->pivotA2ANumBiRings == 3) {
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, NCCL_MAX_TREE_ARITY, channel->binTree.down, 1, &channel->binTree.up, 0), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->binTree.up, NCCL_MAX_TREE_ARITY, channel->binTree.down, 0), ret, affinity_restore);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, NCCL_MAX_TREE_ARITY, channel->tree.down, 1, &channel->tree.up, 0), ret, fail);
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->tree.up, NCCL_MAX_TREE_ARITY, channel->tree.down, 0), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &treeGraph, 0), ret, affinity_restore);
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &treeGraph, 0), ret, fail);
|
||||
INFO(NCCL_INIT, "Connected all trees");
|
||||
|
||||
// Check if we can setup CollNet
|
||||
if (comm->collNetSupport > 0) {
|
||||
int collNetSetupFail = 0;
|
||||
int highestTypes[NCCL_MAX_LOCAL_RANKS] = {TRANSPORT_P2P};
|
||||
// Find all head ranks
|
||||
int nHeads = collNetGraph.nChannels;
|
||||
int *heads;
|
||||
NCCLCHECK(ncclCalloc(&heads, nHeads));
|
||||
// Head GPU index is always 0
|
||||
for (int c=0; c<nHeads; c++) {
|
||||
heads[c] = collNetGraph.intra[c*comm->localRanks+0];
|
||||
}
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
for (int h=0; h<nHeads; h++) {
|
||||
const int head = heads[h];
|
||||
collNetSetupFail = ncclTransportCollNetSetup(comm, &collNetGraph, channel, head, head, h, collNetRecv);
|
||||
collNetSetupFail += ncclTransportCollNetSetup(comm, &collNetGraph, channel, head, head, h, collNetSend);
|
||||
}
|
||||
// Verify CollNet setup across ranks after trying the first channel
|
||||
if (c == 0) {
|
||||
NCCLCHECKGOTO(ncclTransportCollNetCheck(comm, collNetSetupFail), ret, collnet_cleanup);
|
||||
}
|
||||
}
|
||||
// Verify CollNet setup across ranks after trying all channels
|
||||
NCCLCHECKGOTO(ncclTransportCollNetCheck(comm, collNetSetupFail), ret, collnet_cleanup);
|
||||
TRACE(NCCL_INIT, "rank %d Connected inter-node CollNet", rank);
|
||||
if (comm->collNetSupport > 0) collNetTrySetup(comm, &collNetGraph);
|
||||
|
||||
char line[1024];
|
||||
line[0]='\0';
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclTree* chain = &comm->channels[c].collnetChain;
|
||||
snprintf(line+strlen(line), 1023-strlen(line), " [%d] %d->%d->%d",
|
||||
c, chain->down[0], rank, chain->up);
|
||||
}
|
||||
line[1023] = '\0';
|
||||
INFO(NCCL_INIT, "Collnet Chains %s", line);
|
||||
// Connect Collnet + chain
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, &channel->collnetChain.up, 1, channel->collnetChain.down, 0), ret, collnet_cleanup);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 0), ret, collnet_cleanup);
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channel = comm->channels+c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, 1, channel->collnetChain.down, 1, &channel->collnetChain.up, 1), ret, collnet_cleanup);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 1), ret, collnet_cleanup);
|
||||
INFO(NCCL_INIT, "Connected collnet + chain");
|
||||
|
||||
// Connect intra-node CollNet + Direct
|
||||
int highestTransportType0, highestTransportType1;
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channelRecv = comm->channels+c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, NCCL_MAX_DIRECT_ARITY, channelRecv->collnetDirect.up, NCCL_MAX_DIRECT_ARITY, channelRecv->collnetDirect.down, 0), ret, collnet_cleanup);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 0, &highestTransportType0), ret, collnet_cleanup);
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
struct ncclChannel* channelSend = comm->channels+c;
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, c, NCCL_MAX_DIRECT_ARITY, channelSend->collnetDirect.down, NCCL_MAX_DIRECT_ARITY, channelSend->collnetDirect.up, 1), ret, collnet_cleanup);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &collNetGraph, 1, &highestTransportType1), ret, collnet_cleanup);
|
||||
|
||||
// Exchange highest intra-node transport type among ranks
|
||||
// because we need to know whether all ranks can p2p each other to determine whether we can directly read/write registered user buffer
|
||||
comm->intraHighestTransportType = highestTypes[comm->localRank] = highestTransportType0 > highestTransportType1 ? highestTransportType0 : highestTransportType1;
|
||||
//NCCLCHECK(bootstrapIntraNodeAllGather(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, highestTypes, sizeof(int)));
|
||||
for (int i=0; i<comm->localRanks; i++) {
|
||||
if (highestTypes[i] > comm->intraHighestTransportType)
|
||||
comm->intraHighestTransportType = highestTypes[i];
|
||||
}
|
||||
INFO(NCCL_INIT, "rank %d Connected CollNet comm %p nRanks %02d", rank, comm, comm->nRanks);
|
||||
|
||||
collnet_cleanup:
|
||||
free(heads);
|
||||
if (ret != ncclSuccess) {
|
||||
NCCLCHECK(ncclTransportCollNetFree(comm));
|
||||
comm->collNetSupport = 0;
|
||||
ret = ncclSuccess;
|
||||
}
|
||||
}
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d - CONNECTED %d RINGS AND TREES", rank, nranks, comm->nChannels);
|
||||
|
||||
// Compute time models for algorithm and protocol combinations
|
||||
@@ -983,11 +1014,11 @@ collnet_cleanup:
|
||||
minCompCap = std::min(comm->peerInfo[i].cudaCompCap, minCompCap);
|
||||
maxCompCap = std::max(comm->peerInfo[i].cudaCompCap, maxCompCap);
|
||||
}
|
||||
NCCLCHECK(ncclTopoTuneModel(comm, minCompCap, maxCompCap, &treeGraph, &ringGraph, &collNetGraph));
|
||||
NCCLCHECKGOTO(ncclTopoTuneModel(comm, minCompCap, maxCompCap, &treeGraph, &ringGraph, &collNetGraph), ret, fail);
|
||||
} while(0);
|
||||
|
||||
// Compute nChannels per peer for p2p
|
||||
NCCLCHECK(ncclTopoComputeP2pChannels(comm));
|
||||
NCCLCHECKGOTO(ncclTopoComputeP2pChannels(comm), ret, fail);
|
||||
#if 0
|
||||
do { // Setup p2p structures in comm->tasks
|
||||
struct ncclTasks* tasks = &comm->tasks;
|
||||
@@ -1038,79 +1069,40 @@ collnet_cleanup:
|
||||
if (ncclParamNvbPreconnect()) {
|
||||
// Connect p2p when using NVB path
|
||||
int nvbNpeers;
|
||||
int* nvbPeers;
|
||||
NCCLCHECK(ncclTopoGetNvbGpus(comm->topo, comm->rank, &nvbNpeers, &nvbPeers));
|
||||
NCCLCHECKGOTO(ncclTopoGetNvbGpus(comm->topo, comm->rank, &nvbNpeers, &nvbPeers), ret, fail);
|
||||
for (int r=0; r<nvbNpeers; r++) {
|
||||
int peer = nvbPeers[r];
|
||||
int channelId;
|
||||
for (int c=0; c<comm->p2pnChannelsPerPeer; c++) {
|
||||
NCCLCHECK(ncclChannelCompute(comm, peer, c, ncclFuncSend, &channelId));
|
||||
NCCLCHECKGOTO(ncclChannelCompute(comm, peer, c, ncclFuncSend, &channelId), ret, fail);
|
||||
if (comm->channels[channelId].peers[peer].send[1].connected == 0) {
|
||||
comm->connectSend[peer] |= (1<<channelId);
|
||||
comm->connectSend[peer] |= (1UL<<channelId);
|
||||
}
|
||||
}
|
||||
for (int c=0; c<comm->p2pnChannelsPerPeer; c++) {
|
||||
NCCLCHECK(ncclChannelCompute(comm, peer, c, ncclFuncRecv, &channelId));
|
||||
NCCLCHECKGOTO(ncclChannelCompute(comm, peer, c, ncclFuncRecv, &channelId), ret, fail);
|
||||
if (comm->channels[channelId].peers[peer].recv[1].connected == 0) {
|
||||
comm->connectRecv[peer] |= (1<<channelId);
|
||||
comm->connectRecv[peer] |= (1UL<<channelId);
|
||||
}
|
||||
}
|
||||
}
|
||||
NCCLCHECK(ncclTransportP2pSetup(comm, NULL, 1));
|
||||
free(nvbPeers);
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, NULL, 1), ret, fail);
|
||||
}
|
||||
#endif
|
||||
// Connect to local net proxy
|
||||
struct ncclProxyConnector proxyConn;
|
||||
//NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_NET, 1, comm->rank, &proxyConn));
|
||||
//NCCLCHECK(ncclProxyCall(&proxyConn, ncclProxyMsgSharedInit, &comm->p2pnChannels, sizeof(int), NULL, 0));
|
||||
//NCCLCHECKGOTO(ncclProxyConnect(comm, TRANSPORT_NET, 1, comm->rank, &proxyConn), ret, fail);
|
||||
//NCCLCHECKGOTO(ncclProxyCall(&proxyConn, ncclProxyMsgSharedInit, &comm->p2pnChannels, sizeof(int), NULL, 0), ret, fail);
|
||||
|
||||
// Then to remote ones when using PXN
|
||||
if (ncclPxnDisable(comm) == 0) {
|
||||
int nranks;
|
||||
int* pxnPeers;
|
||||
NCCLCHECK(ncclTopoGetPxnRanks(comm, &pxnPeers, &nranks));
|
||||
NCCLCHECKGOTO(ncclTopoGetPxnRanks(comm, &pxnPeers, &nranks), ret, fail);
|
||||
for (int r=0; r<nranks; r++) {
|
||||
//NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_NET, 1, pxnPeers[r], &proxyConn));
|
||||
// NCCLCHECK(ncclProxyCall(&proxyConn, ncclProxyMsgSharedInit, &comm->p2pnChannels, sizeof(int), NULL, 0));
|
||||
//NCCLCHECKGOTO(ncclProxyConnect(comm, TRANSPORT_NET, 1, pxnPeers[r], &proxyConn), ret, fail);
|
||||
//NCCLCHECKGOTO(ncclProxyCall(&proxyConn, ncclProxyMsgSharedInit, &comm->p2pnChannels, sizeof(int), NULL, 0), ret, fail);
|
||||
}
|
||||
free(pxnPeers);
|
||||
}
|
||||
|
||||
do {
|
||||
// Compute intra-process ranks
|
||||
int intraProcRank0 = -1, intraProcRank = -1, intraProcRanks = 0;
|
||||
for (int i = 0; i < nranks; i++) {
|
||||
if ((comm->peerInfo[i].hostHash == comm->peerInfo[rank].hostHash)
|
||||
&& (comm->peerInfo[i].pidHash == comm->peerInfo[rank].pidHash)) {
|
||||
// Rank is in same process
|
||||
if (intraProcRanks == 0) intraProcRank0 = i;
|
||||
if (i == rank) intraProcRank = intraProcRanks;
|
||||
intraProcRanks++;
|
||||
if (intraProcRank0 == rank && rank != i) {
|
||||
comm->peerInfo[i].comm->intraNext = comm->intraNext;
|
||||
comm->intraNext = comm->peerInfo[i].comm;
|
||||
}
|
||||
}
|
||||
}
|
||||
TRACE(NCCL_INIT,"pidHash[%d] %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, comm->peerInfo[rank].pidHash, intraProcRank, intraProcRanks, intraProcRank0);
|
||||
if (intraProcRank == -1 || intraProcRank0 == -1 || comm->peerInfo[intraProcRank0].comm == NULL) {
|
||||
WARN("Failed to determine intra proc ranks rank %d hostHash %lx pidHash %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, comm->peerInfo[rank].hostHash, comm->peerInfo[rank].pidHash,
|
||||
intraProcRank, intraProcRanks, intraProcRank0);
|
||||
return ncclInternalError;
|
||||
}
|
||||
struct ncclComm* comm0 = comm->peerInfo[intraProcRank0].comm;
|
||||
assert(intraProcRank==0 ? comm==comm0 : true);
|
||||
comm->intraComm0 = comm0;
|
||||
comm->intraRefs = intraProcRank==0 ? intraProcRanks : 0;
|
||||
comm->intraRank = intraProcRank;
|
||||
comm->intraRanks = intraProcRanks;
|
||||
comm->intraBarrierPhase = 0;
|
||||
comm->intraBarrierCounter = 0;
|
||||
comm->intraBarrierGate = 0;
|
||||
} while(0);
|
||||
|
||||
#if 0
|
||||
if (comm->intraRank == 0) { // Load ncclParamLaunchMode
|
||||
@@ -1129,23 +1121,31 @@ collnet_cleanup:
|
||||
}
|
||||
}
|
||||
|
||||
NCCLCHECKGOTO(devCommSetup(comm), ret, affinity_restore);
|
||||
// Call devCommSetup before the last barrier, making sure we don't have a thread running in front and starting to
|
||||
// launch NCCL kernels before all cuda mem allocation is complete. That could cause a deadlock.
|
||||
NCCLCHECKGOTO(devCommSetup(comm), ret, fail);
|
||||
|
||||
/* Local intra-node barrier */
|
||||
//NCCLCHECK(bootstrapBarrier(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, comm->localRankToRank[0]));
|
||||
|
||||
// Unlink proxy shm to make sure it will be properly cleaned up.
|
||||
NCCLCHECK(ncclProxyShmUnlink(comm));
|
||||
NCCLCHECKGOTO(bootstrapBarrier(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, comm->localRankToRank[0]), ret, fail);
|
||||
#endif
|
||||
|
||||
// We should have allocated all buffers, collective fifos, ... we can
|
||||
// restore the affinity.
|
||||
affinity_restore:
|
||||
//if (CPU_COUNT(&comm->cpuAffinity)) sched_setaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
if (ret != ncclSuccess) return ret;
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d - DONE", rank, nranks);
|
||||
return ncclSuccess;
|
||||
|
||||
exit:
|
||||
//if (CPU_COUNT(&comm->cpuAffinity)) sched_setaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
// Unlink proxy shm to make sure it will be properly cleaned up.
|
||||
//ncclProxyShmUnlink(comm);
|
||||
free(allTopoRanks);
|
||||
free(nodesTreePatterns);
|
||||
free(nodesFirstRank);
|
||||
//free(allGather3Data);
|
||||
free(rings);
|
||||
free(nvbPeers);
|
||||
free(pxnPeers);
|
||||
return ret;
|
||||
fail:
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t rocm_smi_init() {
|
||||
|
||||
Ссылка в новой задаче
Block a user