Merge remote-tracking branch 'nccl/master' into develop
Этот коммит содержится в:
Wenkai Du
+264
-196
@@ -37,10 +37,8 @@ const char* ncclProtoStr[NCCL_NUM_PROTOCOLS] = { "LL", "LL128", "Simple" };
|
||||
|
||||
extern NodeModel *node_model;
|
||||
|
||||
NCCL_PARAM(CrossNic, "CROSS_NIC", 2);
|
||||
NCCL_PARAM(CollNetEnable, "COLLNET_ENABLE", 0);
|
||||
NCCL_PARAM(GraphDumpFileRank, "GRAPH_DUMP_FILE_RANK", 0);
|
||||
RCCL_PARAM(P2pNetDisable, "P2P_NET_DISABLE", 0);
|
||||
NCCL_PARAM(CollNetNodeThreshold, "COLLNET_NODE_THRESHOLD", 2);
|
||||
|
||||
thread_local int ncclDebugNoWarn = 0;
|
||||
@@ -111,11 +109,13 @@ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *file
|
||||
vsprintf(buffer+len, fmt, args);
|
||||
va_end(args);
|
||||
printf("%s\n", buffer);
|
||||
#if 0
|
||||
if (level == NCCL_LOG_WARN) {
|
||||
fprintf(stderr,"[%d:%d] %s:%d TOPO EXPL ABORT\n",
|
||||
node_model->nodeId, node_model->currRank, filefunc, line);
|
||||
abort();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoGetSystem(const char* xmlTopoFile, struct ncclTopoSystem** system) {
|
||||
@@ -128,20 +128,6 @@ ncclResult_t ncclTopoGetSystem(const char* xmlTopoFile, struct ncclTopoSystem**
|
||||
}
|
||||
|
||||
|
||||
ncclResult_t bootstrapAllGather(struct ncclComm* comm, struct allGather1Data_t * allGather1Data) {
|
||||
// AllGather1 - begin
|
||||
allGather1Data[comm->rank].comm = comm;
|
||||
allGather1Data[comm->rank].cudaCompCap = 1;
|
||||
allGather1Data[comm->rank].peerInfo.rank = comm->rank;
|
||||
allGather1Data[comm->rank].peerInfo.cudaDev = node_model->rankToCudaDev(comm->rank);
|
||||
allGather1Data[comm->rank].peerInfo.gdrSupport = 1;
|
||||
allGather1Data[comm->rank].peerInfo.hostHash = node_model->hostHash;
|
||||
allGather1Data[comm->rank].peerInfo.pidHash = node_model->pidHash;
|
||||
allGather1Data[comm->rank].peerInfo.shmDev = 0x19;
|
||||
allGather1Data[comm->rank].peerInfo.busId = node_model->getGpuBusId(comm->rank);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
void initCollNet() {
|
||||
if (ncclParamCollNetEnable() == 1 && ncclCollNet == 0)
|
||||
ncclCollNet = (ncclCollNet_t*)0x12345678;
|
||||
@@ -182,6 +168,30 @@ ncclResult_t initChannel(struct ncclComm* comm, int channelid) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t fillInfo(struct ncclComm* comm, struct ncclPeerInfo* info, uint64_t commHash) {
|
||||
info->rank = comm->rank;
|
||||
info->cudaDev = node_model->rankToCudaDev(comm->rank);
|
||||
info->hostHash = node_model->hostHash;
|
||||
info->pidHash = node_model->pidHash;
|
||||
|
||||
// Get the device MAJOR:MINOR of /dev/shm so we can use that
|
||||
// information to decide whether we can use SHM for inter-process
|
||||
// communication in a container environment
|
||||
//struct stat statbuf;
|
||||
//SYSCHECK(stat("/dev/shm", &statbuf), "stat");
|
||||
info->shmDev = 0x19;
|
||||
|
||||
info->busId = node_model->getGpuBusId(comm->rank);
|
||||
|
||||
// detect if fine grained memory is available on this GPU
|
||||
info->hasFineGrain = true;
|
||||
info->gdrSupport = 1;
|
||||
|
||||
info->comm = comm;
|
||||
info->cudaCompCap = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t setupChannel(struct ncclComm* comm, int channelId, int rank, int nranks, int* ringRanks) {
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d", rank, nranks);
|
||||
NCCLCHECK(initChannel(comm, channelId));
|
||||
@@ -230,8 +240,8 @@ static ncclResult_t selectTransport(struct ncclComm* comm, struct ncclTopoGraph*
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
WARN("No transport found !");
|
||||
return ncclInternalError;
|
||||
WARN("No transport found for rank %d[%lx] -> rank %d[%lx]", myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId);
|
||||
return ncclSystemError;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTransportP2pConnect(struct ncclComm* comm, struct ncclChannel* channel, int nrecv, int* peerRecv, int nsend, int* peerSend, int connIndex) {
|
||||
@@ -250,12 +260,19 @@ ncclResult_t ncclTransportP2pConnect(struct ncclComm* comm, struct ncclChannel*
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
void dumpData(struct ncclConnect* data, int ndata) {
|
||||
for (int n=0; n<ndata; n++) {
|
||||
printf("[%d] ", n);
|
||||
uint8_t* d = (uint8_t*)data;
|
||||
for (int i=0; i<sizeof(struct ncclConnect); i++) printf("%02x", d[i]);
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
|
||||
ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, int connIndex, int* highestTransportType/*=NULL*/) {
|
||||
#if CUDART_VERSION >= 11030
|
||||
// Stream used during transport setup; need for P2P pre-connect + CUDA Graph
|
||||
hipStream_t transportSetupStream;
|
||||
CUDACHECK(hipStreamCreateWithFlags(&transportSetupStream, hipStreamNonBlocking));
|
||||
#endif
|
||||
//hipStream_t transportSetupStream;
|
||||
//CUDACHECK(hipStreamCreateWithFlags(&transportSetupStream, hipStreamNonBlocking));
|
||||
int highestType = TRANSPORT_P2P; // track highest transport type
|
||||
|
||||
struct ncclConnect data[2*MAXCHANNELS];
|
||||
@@ -302,11 +319,7 @@ ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph*
|
||||
struct ncclConnector* conn = comm->channels[c].peers[sendPeer].send + connIndex;
|
||||
//NCCLCHECK(conn->transportComm->connect(comm, sendData++, 1, comm->rank, conn));
|
||||
conn->connected = 1;
|
||||
#if CUDART_VERSION >= 11030
|
||||
//CUDACHECK(hipMemcpyAsync(comm->channels[c].devPeers[sendPeer].send+connIndex, conn, sizeof(struct ncclConnector), hipMemcpyHostToDevice, transportSetupStream));
|
||||
#else
|
||||
//CUDACHECK(hipMemcpy(comm->channels[c].devPeers[sendPeer].send+connIndex, conn, sizeof(struct ncclConnector), hipMemcpyHostToDevice));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
for (int c=0; c<MAXCHANNELS; c++) {
|
||||
@@ -314,19 +327,13 @@ ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph*
|
||||
struct ncclConnector* conn = comm->channels[c].peers[recvPeer].recv + connIndex;
|
||||
//NCCLCHECK(conn->transportComm->connect(comm, recvData++, 1, comm->rank, conn));
|
||||
conn->connected = 1;
|
||||
#if CUDART_VERSION >= 11030
|
||||
//CUDACHECK(hipMemcpyAsync(comm->channels[c].devPeers[recvPeer].recv+connIndex, conn, sizeof(struct ncclConnector), hipMemcpyHostToDevice, transportSetupStream));
|
||||
#else
|
||||
//CUDACHECK(hipMemcpy(comm->channels[c].devPeers[recvPeer].recv+connIndex, conn, sizeof(struct ncclConnector), hipMemcpyHostToDevice));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
comm->connectRecv[recvPeer+comm->nRanks*connIndex] = comm->connectSend[sendPeer+comm->nRanks*connIndex] = 0;
|
||||
}
|
||||
#if CUDART_VERSION >= 11030
|
||||
CUDACHECK(hipStreamSynchronize(transportSetupStream));
|
||||
CUDACHECK(hipStreamDestroy(transportSetupStream));
|
||||
#endif
|
||||
//CUDACHECK(hipStreamSynchronize(transportSetupStream));
|
||||
//CUDACHECK(hipStreamDestroy(transportSetupStream));
|
||||
if (highestTransportType != NULL) *highestTransportType = highestType;
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -422,9 +429,9 @@ cleanup:
|
||||
|
||||
ncclResult_t ncclTransportCollNetCheck(struct ncclComm* comm, int collNetSetupFail) {
|
||||
// AllGather collNet setup results
|
||||
int allGatherFailures[NCCL_MAX_INTRA_RANKS] = {0};
|
||||
allGatherFailures[comm->intraNodeRank] = collNetSetupFail;
|
||||
//NCCLCHECK(bootstrapIntraNodeAllGather(comm->bootstrap, comm->intraNodeGlobalRanks, comm->intraNodeRank, comm->localRanks, allGatherFailures, sizeof(int)));
|
||||
int allGatherFailures[NCCL_MAX_LOCAL_RANKS] = {0};
|
||||
allGatherFailures[comm->localRank] = collNetSetupFail;
|
||||
//NCCLCHECK(bootstrapIntraNodeAllGather(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, allGatherFailures, sizeof(int)));
|
||||
for (int i=0; i<comm->localRanks; i++) {
|
||||
if (allGatherFailures[i] != 0) {
|
||||
collNetSetupFail = 1;
|
||||
@@ -432,7 +439,7 @@ ncclResult_t ncclTransportCollNetCheck(struct ncclComm* comm, int collNetSetupFa
|
||||
}
|
||||
}
|
||||
if (collNetSetupFail) {
|
||||
if (comm->intraNodeRank == 0) WARN("Cannot initialize CollNet, using point-to-point network instead");
|
||||
if (comm->localRank == 0) WARN("Cannot initialize CollNet, using point-to-point network instead");
|
||||
return ncclSystemError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
@@ -457,80 +464,33 @@ ncclResult_t ncclTransportCollNetFree(struct ncclComm* comm) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t *allGather1Data, struct allGather3Data_t *allGather3Data,
|
||||
ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather3Data_t *allGather3Data,
|
||||
struct ncclTopoGraph& treeGraph, struct ncclTopoGraph& ringGraph, struct ncclTopoGraph& collNetGraph) {
|
||||
// We use 2 AllGathers
|
||||
// 1. { peerInfo, comm, compCap}
|
||||
// 2. { nChannels, graphInfo, topoRanks }
|
||||
|
||||
int rank = comm->rank;
|
||||
int nranks = comm->nRanks;
|
||||
//uint64_t commHash = getHash(commId->internal, NCCL_UNIQUE_ID_BYTES);
|
||||
//TRACE(NCCL_INIT, "comm %p, commHash %lx, rank %d nranks %d - BEGIN", comm, commHash, rank, nranks);
|
||||
//NCCLCHECK(bootstrapInit(commId, rank, nranks, &comm->bootstrap));
|
||||
// [RCCL] Collect the PID of the root
|
||||
int rootPid;
|
||||
//NCCLCHECK(bootstrapInit(commId, comm));
|
||||
// [/RCCL]
|
||||
|
||||
// AllGather1 - begin
|
||||
//struct {
|
||||
// struct ncclPeerInfo peerInfo;
|
||||
// struct ncclComm* comm;
|
||||
// int cudaCompCap;
|
||||
//} *allGather1Data;
|
||||
//NCCLCHECK(ncclCalloc(&comm->peerInfo, nranks+1)); // Extra rank to represent CollNet root
|
||||
//NCCLCHECK(fillInfo(comm, comm->peerInfo+rank, comm->rank));
|
||||
//NCCLCHECK(bootstrapAllGather(comm->bootstrap, comm->peerInfo, sizeof(struct ncclPeerInfo)));
|
||||
|
||||
//NCCLCHECK(ncclCalloc(&allGather1Data, nranks));
|
||||
//allGather1Data[rank].comm = comm;
|
||||
//allGather1Data[rank].cudaCompCap = ncclCudaCompCap();
|
||||
struct ncclPeerInfo* myInfo = &allGather1Data[rank].peerInfo;
|
||||
//NCCLCHECK(fillInfo(comm, myInfo, commHash));
|
||||
//NCCLCHECK(bootstrapAllGather(comm->bootstrap, allGather1Data, sizeof(*allGather1Data)));
|
||||
|
||||
NCCLCHECK(ncclCalloc(&comm->peerInfo, nranks+1)); // Extra rank to represent CollNet root
|
||||
for (int i = 0; i < nranks; i++) {
|
||||
memcpy(comm->peerInfo+i, &allGather1Data[i].peerInfo, sizeof(struct ncclPeerInfo));
|
||||
if ((i != rank) && (comm->peerInfo[i].hostHash == myInfo->hostHash) && (comm->peerInfo[i].busId == myInfo->busId)) {
|
||||
WARN("Duplicate GPU detected : rank %d and rank %d both on CUDA device %lx", rank, i, myInfo->busId);
|
||||
if ((i != rank) && (comm->peerInfo[i].hostHash == comm->peerInfo[rank].hostHash) && (comm->peerInfo[i].busId == comm->peerInfo[rank].busId)) {
|
||||
WARN("Duplicate GPU detected : rank %d and rank %d both on CUDA device %lx", rank, i, comm->peerInfo[rank].busId);
|
||||
return ncclInvalidUsage;
|
||||
}
|
||||
}
|
||||
|
||||
// Compute intra ranks and minimum CUDA Compute capabilities of intra-node GPUs and all GPUs
|
||||
int intraProcRank0 = -1, intraProcRank = -1, intraProcRanks = 0;
|
||||
int intraNodeRank0 = -1, intraNodeRank = -1, intraNodeRanks = 0;
|
||||
int myCompCap = allGather1Data[rank].cudaCompCap;
|
||||
int minCompCap = myCompCap, maxCompCap = myCompCap;
|
||||
for (int i = 0; i < nranks; i++) {
|
||||
if (allGather1Data[i].peerInfo.hostHash == allGather1Data[rank].peerInfo.hostHash) {
|
||||
// Rank is on same node
|
||||
if (intraNodeRanks == 0) intraNodeRank0 = i;
|
||||
if (i == rank) intraNodeRank = intraNodeRanks;
|
||||
comm->intraNodeGlobalRanks[intraNodeRanks] = i;
|
||||
comm->rankToIntraNodeRank[i] = intraNodeRanks;
|
||||
intraNodeRanks++;
|
||||
if (allGather1Data[i].peerInfo.pidHash == allGather1Data[rank].peerInfo.pidHash) {
|
||||
// Rank is in same process
|
||||
if (intraProcRanks == 0) intraProcRank0 = i;
|
||||
if (i == rank) intraProcRank = intraProcRanks;
|
||||
intraProcRanks++;
|
||||
}
|
||||
}
|
||||
minCompCap = std::min(allGather1Data[i].cudaCompCap, minCompCap);
|
||||
maxCompCap = std::max(allGather1Data[i].cudaCompCap, maxCompCap);
|
||||
}
|
||||
TRACE(NCCL_INIT,"hostHash[%d] %lx intraNodeRank %d intraNodeRanks %d intraNodeRank0 %d",
|
||||
rank, allGather1Data[rank].peerInfo.hostHash, intraNodeRank, intraNodeRanks, intraNodeRank0);
|
||||
TRACE(NCCL_INIT,"pidHash[%d] %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, allGather1Data[rank].peerInfo.pidHash, intraProcRank, intraProcRanks, intraProcRank0);
|
||||
if (intraProcRank == -1 || intraProcRank0 == -1 || allGather1Data[intraProcRank0].comm == NULL) {
|
||||
WARN("Failed to determine intra proc ranks rank %d hostHash %lx pidHash %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, allGather1Data[rank].peerInfo.hostHash, allGather1Data[rank].peerInfo.pidHash,
|
||||
intraProcRank, intraProcRanks, intraProcRank0);
|
||||
return ncclInternalError;
|
||||
}
|
||||
if (intraNodeRank == -1 || intraNodeRank0 == -1 || intraNodeRanks == 0) {
|
||||
WARN("Failed to determine intra node ranks rank %d hostHash %lx pidHash %lx intraNodeRank %d intraNodeRanks %d intraNodeRank0 %d",
|
||||
rank, allGather1Data[rank].peerInfo.hostHash, allGather1Data[rank].peerInfo.pidHash,
|
||||
intraNodeRank, intraNodeRanks, intraNodeRank0);
|
||||
return ncclInternalError;
|
||||
}
|
||||
struct ncclComm* intraProcRank0Comm = allGather1Data[intraProcRank0].comm;
|
||||
uint64_t intraNodeRank0pidHash = allGather1Data[intraNodeRank0].peerInfo.pidHash;
|
||||
comm->intraNodeRank = intraNodeRank;
|
||||
|
||||
// AllGather1 - end
|
||||
|
||||
// Topo detection / System graph creation
|
||||
@@ -550,11 +510,23 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
|
||||
// Print final topology
|
||||
NCCLCHECK(ncclTopoPrint(comm->topo));
|
||||
|
||||
// Set Affinity to a CPU local the our GPU, so that all memory we allocate
|
||||
// on the host is local.
|
||||
//NCCLCHECK(ncclTopoGetCpuAffinity(comm->topo, comm->rank, &comm->cpuAffinity));
|
||||
//cpu_set_t affinitySave;
|
||||
// if (CPU_COUNT(&comm->cpuAffinity)) {
|
||||
//sched_getaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
//sched_setaffinity(0, sizeof(cpu_set_t), &comm->cpuAffinity);
|
||||
//}
|
||||
ncclResult_t ret;
|
||||
|
||||
// Launch proxy service thread
|
||||
//NCCLCHECK(ncclProxyCreate(comm));
|
||||
|
||||
// Get rings and trees
|
||||
//struct ncclTopoGraph ringGraph;
|
||||
ringGraph.id = 0;
|
||||
ringGraph.pattern = NCCL_TOPO_PATTERN_RING;
|
||||
ringGraph.crossNic = ncclParamCrossNic();
|
||||
ringGraph.collNet = 0;
|
||||
ringGraph.minChannels = 1;
|
||||
ringGraph.maxChannels = MAXCHANNELS/2;
|
||||
@@ -564,7 +536,6 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
|
||||
//struct ncclTopoGraph treeGraph;
|
||||
treeGraph.id = 1;
|
||||
treeGraph.pattern = NCCL_TOPO_PATTERN_BALANCED_TREE;
|
||||
treeGraph.crossNic = ncclParamCrossNic();
|
||||
treeGraph.collNet = 0;
|
||||
treeGraph.minChannels = comm->topo->nodes[NET].count != 0 ? 1 : ringGraph.nChannels;
|
||||
treeGraph.maxChannels = ringGraph.nChannels;
|
||||
@@ -575,56 +546,55 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
|
||||
collNetGraph.id = 2;
|
||||
collNetGraph.pattern = NCCL_TOPO_PATTERN_TREE;
|
||||
collNetGraph.collNet = 1;
|
||||
collNetGraph.crossNic = ncclParamCrossNic();
|
||||
collNetGraph.minChannels = 1;
|
||||
collNetGraph.maxChannels = ringGraph.nChannels;
|
||||
collNetGraph.minChannels = collNetGraph.maxChannels = ringGraph.nChannels;
|
||||
NCCLCHECK(ncclTopoCompute(comm->topo, &collNetGraph));
|
||||
NCCLCHECK(ncclTopoPrintGraph(comm->topo, &collNetGraph));
|
||||
|
||||
bool allXgmi = true;
|
||||
bool allXgmi = true, hasPeerAccess = true;
|
||||
// Check that all the GPUs have peer access to one another and are XGMI connected
|
||||
for (int i = 0; i < nranks && hasPeerAccess; i++) {
|
||||
int cudaDev1 = comm->peerInfo[i].cudaDev;
|
||||
for (int j = 0; j < nranks; j++) {
|
||||
if (i == j) continue;
|
||||
int cudaDev2 = comm->peerInfo[j].cudaDev;
|
||||
int p2p;
|
||||
if (hipDeviceCanAccessPeer(&p2p, cudaDev1, cudaDev2) != hipSuccess || !p2p)
|
||||
{
|
||||
hasPeerAccess = false;
|
||||
break;
|
||||
}
|
||||
|
||||
bool isXGMI;
|
||||
// Limit to single intermediate GPU for enabling clique
|
||||
NCCLCHECK(ncclTopoGetLinkType(comm->topo, i, j, &isXGMI, 1));
|
||||
allXgmi &= isXGMI;
|
||||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
{ // [RCCL] Check if clique-based kernels can be enabled and initialize CliqueManager
|
||||
//CliqueManager::cliqueMode_t cliqueMode = CliqueManager::CLIQUE_DISABLED;
|
||||
CliqueManager::cliqueMode_t cliqueMode = CliqueManager::CLIQUE_DISABLED;
|
||||
if (comm->localRanks == comm->nRanks && comm->topo->nodes[GPU].nodes[0].gpu.gcn != 910)
|
||||
{
|
||||
// Check that all the GPUs have peer access to one another and are XGMI connected
|
||||
bool hasPeerAccess = true;
|
||||
for (int i = 0; i < nranks && hasPeerAccess; i++)
|
||||
if (hasPeerAccess)
|
||||
{
|
||||
int cudaDev1 = allGather1Data[i].peerInfo.cudaDev;
|
||||
for (int j = 0; j < nranks; j++)
|
||||
{
|
||||
if (i == j) continue;
|
||||
int cudaDev2 = allGather1Data[j].peerInfo.cudaDev;
|
||||
//int p2p;
|
||||
//if (hipDeviceCanAccessPeer(&p2p, cudaDev1, cudaDev2) != hipSuccess || !p2p)
|
||||
//{
|
||||
// hasPeerAccess = false;
|
||||
// break;
|
||||
//}
|
||||
|
||||
bool isXGMI;
|
||||
NCCLCHECK(ncclTopoGetLinkType(comm->topo, i, j, &isXGMI, 1));
|
||||
allXgmi &= isXGMI;
|
||||
}
|
||||
if (intraProcRanks == nranks)
|
||||
cliqueMode = CliqueManager::CLIQUE_SINGLE_PROCESS;
|
||||
else
|
||||
cliqueMode = CliqueManager::CLIQUE_SINGLE_NODE;
|
||||
}
|
||||
//if (hasPeerAccess)
|
||||
//{
|
||||
// if (intraRanks == nranks)
|
||||
// cliqueMode = CliqueManager::CLIQUE_SINGLE_PROCESS;
|
||||
// else
|
||||
// cliqueMode = CliqueManager::CLIQUE_SINGLE_NODE;
|
||||
//}
|
||||
|
||||
// For now, only enable clique-based kernels on nodes where all GPUs are XGMI connected
|
||||
//if (!allXgmi && !rcclParamCliqueIgnoreTopo())
|
||||
//{
|
||||
// INFO(NCCL_INIT, "Disabling clique-based kernels due to topology (ignore with RCCL_CLIQUE_IGNORE_TOPO)");
|
||||
// cliqueMode = CliqueManager::CLIQUE_DISABLED;
|
||||
//}
|
||||
if (!allXgmi && !rcclParamCliqueIgnoreTopo())
|
||||
{
|
||||
INFO(NCCL_INIT, "Disabling clique-based kernels due to topology (ignore with RCCL_CLIQUE_IGNORE_TOPO)");
|
||||
cliqueMode = CliqueManager::CLIQUE_DISABLED;
|
||||
}
|
||||
}
|
||||
//comm->cliqueManager = new CliqueManager(rank, nranks, cliqueMode);
|
||||
//NCCLCHECK(comm->cliqueManager->Init(commId, rootPid));
|
||||
comm->cliqueManager = new CliqueManager(rank, nranks, cliqueMode);
|
||||
NCCLCHECK(comm->cliqueManager->Init(commId, rootPid));
|
||||
} // [/RCCL]
|
||||
#endif
|
||||
|
||||
if (comm->rank == ncclParamGraphDumpFileRank()) {
|
||||
struct ncclTopoGraph* graphs[3] = { &ringGraph, &treeGraph, &collNetGraph };
|
||||
@@ -633,19 +603,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
|
||||
|
||||
// Determine local CollNet support before all-gather
|
||||
if (ncclParamCollNetEnable() == 1 && collNetSupport() == 1 && collNetGraph.nChannels > 0) comm->collNetSupport = 1;
|
||||
if (intraNodeRanks > 8) {
|
||||
if (comm->collNetSupport == 1) WARN("CollNet currently only supports up to 8 GPUs per node");
|
||||
comm->collNetSupport = 0;
|
||||
}
|
||||
|
||||
if ((comm->topo->type & RCCL_TOPO_4P2H_ROME) && (comm->topo->type & RCCL_TOPO_GDR_ALL)) {
|
||||
if (rcclParamP2pNetDisable() == 0) {
|
||||
comm->p2pNet = 1;
|
||||
INFO(NCCL_INIT, "RCCL enabled same node P2P over network");
|
||||
}
|
||||
else
|
||||
INFO(NCCL_INIT, "RCCL force disabled same node P2P over network");
|
||||
}
|
||||
// AllGather3 - begin
|
||||
#if 0
|
||||
struct ncclGraphInfo {
|
||||
@@ -659,6 +617,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
|
||||
};
|
||||
|
||||
struct {
|
||||
int netDev;
|
||||
int collNetSupport;
|
||||
int nc;
|
||||
struct ncclGraphInfo tree;
|
||||
@@ -670,7 +629,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
|
||||
NCCLCHECK(ncclCalloc(&allGather3Data, nranks));
|
||||
#endif
|
||||
int idx;
|
||||
NCCLCHECK(ncclTopoIdToIndex(comm->topo, GPU, myInfo->busId, &idx));
|
||||
NCCLCHECK(ncclTopoIdToIndex(comm->topo, GPU, comm->busId, &idx));
|
||||
allGather3Data[rank].nc = 2;
|
||||
if (comm->topo->nodes[GPU].count == comm->topo->nRanks && comm->topo->nodes[GPU].nodes[idx].gpu.gcn == 906 && allXgmi)
|
||||
allGather3Data[rank].nc = 4;
|
||||
@@ -684,6 +643,7 @@ ncclResult_t initTransportsRank_1(struct ncclComm* comm, struct allGather1Data_t
|
||||
allGather3Data[rank].nc = std::max(allGather3Data[rank].nc, 4/ringGraph.nChannels);
|
||||
if (ringGraph.nChannels > MAXCHANNELS/2)
|
||||
allGather3Data[rank].nc = 1;
|
||||
NCCLCHECK(ncclTopoGetLocalNet(comm->topo, rank, &allGather3Data[rank].netDev));
|
||||
allGather3Data[rank].tree.pattern = treeGraph.pattern;
|
||||
allGather3Data[rank].tree.nChannels = treeGraph.nChannels;
|
||||
allGather3Data[rank].tree.sameChannels = treeGraph.sameChannels;
|
||||
@@ -717,25 +677,57 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
struct ncclTopoGraph& treeGraph, struct ncclTopoGraph& ringGraph, struct ncclTopoGraph& collNetGraph) {
|
||||
int rank = comm->rank;
|
||||
int nranks = comm->nRanks;
|
||||
ncclResult_t ret;
|
||||
//NCCLCHECK(bootstrapAllGather(comm->bootstrap, allGather3Data, sizeof(*allGather3Data)));
|
||||
|
||||
// Determine nNodes, firstRanks, ...
|
||||
int *nodesFirstRank, *nodesTreePatterns;
|
||||
NCCLCHECK(ncclCalloc(&nodesFirstRank, nranks));
|
||||
NCCLCHECK(ncclCalloc(&nodesTreePatterns, nranks));
|
||||
for (int i=0; i<nranks; i++) {
|
||||
int node = -1;
|
||||
int firstRank = allGather3Data[i].topoRanks.ringRecv[0];
|
||||
for (int n=0; n<comm->nNodes; n++) {
|
||||
if (nodesFirstRank[n] == firstRank) node = n;
|
||||
}
|
||||
if (node == -1) {
|
||||
node = comm->nNodes++;
|
||||
NCCLCHECK(ncclCalloc(&comm->rankToNode, comm->nRanks));
|
||||
for (int r=0; r<nranks; r++) {
|
||||
int node;
|
||||
int firstRank = allGather3Data[r].topoRanks.ringRecv[0];
|
||||
for (node=0; node<comm->nNodes && nodesFirstRank[node] != firstRank; node++);
|
||||
if (node == comm->nNodes) {
|
||||
comm->nNodes++;
|
||||
nodesFirstRank[node] = firstRank;
|
||||
// Record tree pattern of each node as they can be different depending on sm arch
|
||||
nodesTreePatterns[node] = allGather3Data[i].tree.pattern;
|
||||
nodesTreePatterns[node] = allGather3Data[r].tree.pattern;
|
||||
}
|
||||
if (i == comm->rank) comm->node = node;
|
||||
comm->rankToNode[r] = node;
|
||||
}
|
||||
// Now that we know nNodes, alloc nodeRanks and compute localRanks for each node
|
||||
NCCLCHECK(ncclCalloc(&comm->nodeRanks, comm->nNodes));
|
||||
NCCLCHECK(ncclCalloc(&comm->rankToLocalRank, comm->nRanks));
|
||||
for (int r=0; r<comm->nRanks; r++) {
|
||||
int node = comm->rankToNode[r];
|
||||
comm->rankToLocalRank[r] = comm->nodeRanks[node].localRanks;
|
||||
comm->nodeRanks[node].localRanks++;
|
||||
}
|
||||
// Allocate ranks arrays for each node
|
||||
for (int n=0; n<comm->nNodes; n++) {
|
||||
NCCLCHECK(ncclCalloc(&comm->nodeRanks[n].localRankToRank, comm->nodeRanks[n].localRanks));
|
||||
comm->maxLocalRanks = std::max(comm->maxLocalRanks, comm->nodeRanks[n].localRanks);
|
||||
comm->nodeRanks[n].localRanks = 0;
|
||||
}
|
||||
// And fill the ranks arrays
|
||||
for (int r=0; r<comm->nRanks; r++) {
|
||||
int node = comm->rankToNode[r];
|
||||
comm->nodeRanks[node].localRankToRank[comm->nodeRanks[node].localRanks++] = r;
|
||||
}
|
||||
comm->node = comm->rankToNode[rank];
|
||||
comm->localRankToRank = comm->nodeRanks[comm->node].localRankToRank;
|
||||
comm->localRank = comm->rankToLocalRank[rank];
|
||||
comm->localRanks = comm->nodeRanks[comm->node].localRanks;
|
||||
|
||||
TRACE(NCCL_INIT,"hostHash[%d] %lx localRank %d localRanks %d localRank0 %d",
|
||||
rank, comm->peerInfo[rank].hostHash, comm->localRank, comm->localRanks, comm->localRankToRank[0]);
|
||||
if (comm->localRank == -1 || comm->localRankToRank[0] == -1 || comm->localRanks == 0) {
|
||||
WARN("Failed to determine local ranks rank %d hostHash %lx pidHash %lx localRank %d localRanks %d localRank0 %d",
|
||||
rank, comm->peerInfo[rank].hostHash, comm->peerInfo[rank].pidHash,
|
||||
comm->localRank, comm->localRanks, comm->localRankToRank[0]);
|
||||
return ncclInternalError;
|
||||
}
|
||||
|
||||
int nChannelsOrig = comm->nChannels;
|
||||
@@ -743,6 +735,7 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
NCCLCHECK(ncclCalloc(&allTopoRanks, comm->nRanks));
|
||||
int nc = allGather3Data[0].nc;
|
||||
for (int i=0; i<nranks; i++) {
|
||||
comm->peerInfo[i].netDev = allGather3Data[i].netDev;
|
||||
allTopoRanks[i] = &allGather3Data[i].topoRanks;
|
||||
nc = std::min(allGather3Data[i].nc, nc);
|
||||
// Make sure we align all ranks so that the tuning is consistent across ranks
|
||||
@@ -750,20 +743,20 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
treeGraph.sameChannels = std::min(allGather3Data[i].tree.sameChannels, treeGraph.sameChannels);
|
||||
treeGraph.speedIntra = std::min(allGather3Data[i].tree.speedIntra, treeGraph.speedIntra);
|
||||
treeGraph.speedInter = std::min(allGather3Data[i].tree.speedInter, treeGraph.speedInter);
|
||||
treeGraph.typeIntra = std::min(allGather3Data[i].tree.typeIntra, treeGraph.typeIntra);
|
||||
treeGraph.typeInter = std::min(allGather3Data[i].tree.typeInter, treeGraph.typeInter);
|
||||
treeGraph.typeIntra = std::max(allGather3Data[i].tree.typeIntra, treeGraph.typeIntra);
|
||||
treeGraph.typeInter = std::max(allGather3Data[i].tree.typeInter, treeGraph.typeInter);
|
||||
ringGraph.nChannels = std::min(allGather3Data[i].ring.nChannels, ringGraph.nChannels);
|
||||
ringGraph.sameChannels = std::min(allGather3Data[i].ring.sameChannels, ringGraph.sameChannels);
|
||||
ringGraph.speedIntra = std::min(allGather3Data[i].ring.speedIntra, ringGraph.speedIntra);
|
||||
ringGraph.speedInter = std::min(allGather3Data[i].ring.speedInter, ringGraph.speedInter);
|
||||
ringGraph.typeIntra = std::min(allGather3Data[i].ring.typeIntra, ringGraph.typeIntra);
|
||||
ringGraph.typeInter = std::min(allGather3Data[i].ring.typeInter, ringGraph.typeInter);
|
||||
ringGraph.typeIntra = std::max(allGather3Data[i].ring.typeIntra, ringGraph.typeIntra);
|
||||
ringGraph.typeInter = std::max(allGather3Data[i].ring.typeInter, ringGraph.typeInter);
|
||||
collNetGraph.nChannels = std::min(allGather3Data[i].collNet.nChannels, collNetGraph.nChannels);
|
||||
collNetGraph.sameChannels = std::min(allGather3Data[i].collNet.sameChannels, collNetGraph.sameChannels);
|
||||
collNetGraph.speedIntra = std::min(allGather3Data[i].collNet.speedIntra, collNetGraph.speedIntra);
|
||||
collNetGraph.speedInter = std::min(allGather3Data[i].collNet.speedInter, collNetGraph.speedInter);
|
||||
collNetGraph.typeIntra = std::min(allGather3Data[i].collNet.typeIntra, collNetGraph.typeIntra);
|
||||
collNetGraph.typeInter = std::min(allGather3Data[i].collNet.typeInter, collNetGraph.typeInter);
|
||||
collNetGraph.typeIntra = std::max(allGather3Data[i].collNet.typeIntra, collNetGraph.typeIntra);
|
||||
collNetGraph.typeInter = std::max(allGather3Data[i].collNet.typeInter, collNetGraph.typeInter);
|
||||
comm->collNetSupport = std::min(allGather3Data[i].collNetSupport, comm->collNetSupport);
|
||||
}
|
||||
|
||||
@@ -776,12 +769,20 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
for (int i=0; i<comm->nChannels; i++) memcpy(comm->channels+comm->nChannels+i, comm->channels+nChannelsOrig+i, sizeof(struct ncclChannel));
|
||||
}
|
||||
|
||||
// Determine CollNet support after all-gather now that we know nNodes
|
||||
int collNetNodeThreshold = ncclParamCollNetNodeThreshold();
|
||||
if (comm->nNodes < collNetNodeThreshold) {
|
||||
if (comm->collNetSupport == 1)
|
||||
// Determine CollNet support after all-gather now that we know nNodes and each node localRanks
|
||||
if (comm->collNetSupport == 1) {
|
||||
int collNetNodeThreshold = ncclParamCollNetNodeThreshold();
|
||||
if (comm->nNodes < collNetNodeThreshold) {
|
||||
INFO(NCCL_INIT, "Communicator has %d nodes which is less than CollNet node threshold %d, disabling CollNet", comm->nNodes, collNetNodeThreshold);
|
||||
comm->collNetSupport = 0;
|
||||
comm->collNetSupport = 0;
|
||||
}
|
||||
for (int n=0; n<comm->nNodes; n++) {
|
||||
if (comm->nodeRanks[n].localRanks > NCCL_MAX_DIRECT_ARITY+1) {
|
||||
WARN("CollNet currently only supports up to %d GPUs per node, disabling CollNet", NCCL_MAX_DIRECT_ARITY+1);
|
||||
comm->collNetSupport = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int *rings;
|
||||
@@ -808,16 +809,6 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
line[1023] = '\0';
|
||||
INFO(NCCL_INIT, "Trees%s", line);
|
||||
|
||||
// Set Affinity to a CPU local the our GPU, so that all memory we allocate
|
||||
// on the host is local.
|
||||
//NCCLCHECK(ncclTopoGetCpuAffinity(comm->topo, comm->rank, &comm->cpuAffinity));
|
||||
//cpu_set_t affinitySave;
|
||||
//if (CPU_COUNT(&comm->cpuAffinity)) {
|
||||
// sched_getaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
// sched_setaffinity(0, sizeof(cpu_set_t), &comm->cpuAffinity);
|
||||
//}
|
||||
ncclResult_t ret;
|
||||
|
||||
//NCCLCHECK(computeBuffSizes(comm));
|
||||
|
||||
// Connect with prev/next for each ring
|
||||
@@ -828,7 +819,7 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
NCCLCHECKGOTO(ncclTransportP2pConnect(comm, channel, 1, &channel->ring.prev, 1, &channel->ring.next, 0), ret, affinity_restore);
|
||||
}
|
||||
NCCLCHECKGOTO(ncclTransportP2pSetup(comm, &ringGraph, 0), ret, affinity_restore);
|
||||
if (ringGraph.nIntraChannels && rcclParamP2pNetDisable() == 0) {
|
||||
if (ringGraph.nIntraChannels) {
|
||||
comm->useIntraNet = 1;
|
||||
// Connect NET for intranode use
|
||||
for (int c=0; c<comm->nChannels; c++) {
|
||||
@@ -854,7 +845,7 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
// Check if we can setup CollNet
|
||||
if (comm->collNetSupport > 0) {
|
||||
int collNetSetupFail = 0;
|
||||
int highestTypes[NCCL_MAX_INTRA_RANKS] = {TRANSPORT_P2P};
|
||||
int highestTypes[NCCL_MAX_LOCAL_RANKS] = {TRANSPORT_P2P};
|
||||
// Find all head ranks
|
||||
int nHeads = collNetGraph.nChannels;
|
||||
int *heads;
|
||||
@@ -894,13 +885,13 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGather3Data_t
|
||||
|
||||
// Exchange highest intra-node transport type among ranks
|
||||
// because we need to know whether all ranks can p2p each other to determine whether we can directly read/write registered user buffer
|
||||
comm->intraHighestTransportType = highestTypes[comm->intraNodeRank] = highestTransportType0 > highestTransportType1 ? highestTransportType0 : highestTransportType1;
|
||||
//NCCLCHECK(bootstrapIntraNodeAllGather(comm->bootstrap, comm->intraNodeGlobalRanks, comm->intraNodeRank, comm->localRanks, highestTypes, sizeof(int)));
|
||||
//for (int i=0; i<comm->localRanks; i++) {
|
||||
//if (highestTypes[i] > comm->intraHighestTransportType)
|
||||
//comm->intraHighestTransportType = highestTypes[i];
|
||||
//}
|
||||
INFO(NCCL_INIT, "rank %d Connected CollNet", rank);
|
||||
comm->intraHighestTransportType = highestTypes[comm->localRank] = highestTransportType0 > highestTransportType1 ? highestTransportType0 : highestTransportType1;
|
||||
//NCCLCHECK(bootstrapIntraNodeAllGather(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, highestTypes, sizeof(int)));
|
||||
for (int i=0; i<comm->localRanks; i++) {
|
||||
if (highestTypes[i] > comm->intraHighestTransportType)
|
||||
comm->intraHighestTransportType = highestTypes[i];
|
||||
}
|
||||
INFO(NCCL_INIT, "rank %d Connected CollNet comm %p nRanks %02d", rank, comm, comm->nRanks);
|
||||
|
||||
collnet_cleanup:
|
||||
free(heads);
|
||||
@@ -913,19 +904,96 @@ collnet_cleanup:
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d - CONNECTED %d RINGS AND TREES", rank, nranks, comm->nChannels);
|
||||
|
||||
// Compute time models for algorithm and protocol combinations
|
||||
NCCLCHECK(ncclTopoTuneModel(comm, 1, 1, &treeGraph, &ringGraph, &collNetGraph));
|
||||
do {
|
||||
int myCompCap = comm->peerInfo[rank].cudaCompCap;
|
||||
int minCompCap = myCompCap, maxCompCap = myCompCap;
|
||||
for (int i = 0; i < nranks; i++) {
|
||||
minCompCap = std::min(comm->peerInfo[i].cudaCompCap, minCompCap);
|
||||
maxCompCap = std::max(comm->peerInfo[i].cudaCompCap, maxCompCap);
|
||||
}
|
||||
NCCLCHECK(ncclTopoTuneModel(comm, minCompCap, maxCompCap, &treeGraph, &ringGraph, &collNetGraph));
|
||||
} while(0);
|
||||
|
||||
// Compute nChannels per peer for p2p
|
||||
NCCLCHECK(ncclTopoComputeP2pChannels(comm));
|
||||
#if 0
|
||||
if (ncclParamNvbPreconnect()) {
|
||||
// Connect p2p when using NVB path
|
||||
int nvbNpeers;
|
||||
int* nvbPeers;
|
||||
NCCLCHECK(ncclTopoGetNvbGpus(comm->topo, comm->rank, &nvbNpeers, &nvbPeers));
|
||||
for (int r=0; r<nvbNpeers; r++) {
|
||||
int peer = nvbPeers[r];
|
||||
int delta = (comm->nRanks + (comm->rank-peer)) % comm->nRanks;
|
||||
for (int c=0; c<comm->p2pnChannelsPerPeer; c++) {
|
||||
int channelId = (delta+comm->p2pChannels[c]) % comm->p2pnChannels;
|
||||
if (comm->channels[channelId].peers[peer].recv[1].connected == 0) { // P2P uses only 1 connector
|
||||
comm->connectRecv[peer] |= (1<<channelId);
|
||||
}
|
||||
}
|
||||
delta = (comm->nRanks - (comm->rank-peer)) % comm->nRanks;
|
||||
for (int c=0; c<comm->p2pnChannelsPerPeer; c++) {
|
||||
int channelId = (delta+comm->p2pChannels[c]) % comm->p2pnChannels;
|
||||
if (comm->channels[channelId].peers[peer].send[1].connected == 0) { // P2P uses only 1 connector
|
||||
comm->connectSend[peer] |= (1<<channelId);
|
||||
}
|
||||
}
|
||||
}
|
||||
NCCLCHECK(ncclTransportP2pSetup(comm, NULL, 1));
|
||||
free(nvbPeers);
|
||||
}
|
||||
#endif
|
||||
// Connect to local net proxy
|
||||
struct ncclProxyConnector proxyConn;
|
||||
NCCLCHECK(ncclTopoGetLocalRank(comm->topo, comm->rank, &proxyConn.localRank));
|
||||
//NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_NET, 1, comm->rank, &proxyConn));
|
||||
//NCCLCHECK(ncclProxyCall(&proxyConn, ncclProxyMsgSharedInit, &comm->p2pnChannels, sizeof(int), NULL, 0));
|
||||
|
||||
//NCCLCHECK(ncclCommSetIntra(comm, intraRank, intraRanks, intraRank0Comm));
|
||||
// Then to remote ones when using PXN
|
||||
if (ncclPxnDisable() == 0) {
|
||||
int nranks;
|
||||
int* pxnPeers;
|
||||
NCCLCHECK(ncclTopoGetPxnRanks(comm, &pxnPeers, &nranks));
|
||||
for (int r=0; r<nranks; r++) {
|
||||
//NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_NET, 1, pxnPeers[r], &proxyConn));
|
||||
//NCCLCHECK(ncclProxyCall(&proxyConn, ncclProxyMsgSharedInit, &comm->p2pnChannels, sizeof(int), NULL, 0));
|
||||
}
|
||||
free(pxnPeers);
|
||||
}
|
||||
|
||||
//if (comm->nNodes) NCCLCHECK(ncclProxyCreate(comm));
|
||||
do {
|
||||
// Compute intra-process ranks
|
||||
int intraProcRank0 = -1, intraProcRank = -1, intraProcRanks = 0;
|
||||
for (int i = 0; i < nranks; i++) {
|
||||
if ((comm->peerInfo[i].hostHash == comm->peerInfo[rank].hostHash)
|
||||
&& (comm->peerInfo[i].pidHash == comm->peerInfo[rank].pidHash)) {
|
||||
// Rank is in same process
|
||||
if (intraProcRanks == 0) intraProcRank0 = i;
|
||||
if (i == rank) intraProcRank = intraProcRanks;
|
||||
intraProcRanks++;
|
||||
}
|
||||
}
|
||||
TRACE(NCCL_INIT,"pidHash[%d] %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, comm->peerInfo[rank].pidHash, intraProcRank, intraProcRanks, intraProcRank0);
|
||||
if (intraProcRank == -1 || intraProcRank0 == -1 || comm->peerInfo[intraProcRank0].comm == NULL) {
|
||||
WARN("Failed to determine intra proc ranks rank %d hostHash %lx pidHash %lx intraProcRank %d intraProcRanks %d intraProcRank0 %d",
|
||||
rank, comm->peerInfo[rank].hostHash, comm->peerInfo[rank].pidHash,
|
||||
intraProcRank, intraProcRanks, intraProcRank0);
|
||||
return ncclInternalError;
|
||||
}
|
||||
//NCCLCHECK(ncclCommSetIntraProc(comm, intraProcRank, intraProcRanks, comm->peerInfo[intraProcRank0].comm));
|
||||
} while(0);
|
||||
|
||||
/* Local intra-node barrier */
|
||||
//NCCLCHECK(bootstrapBarrier(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, comm->localRankToRank[0]));
|
||||
|
||||
// Unlink proxy shm to make sure it will be properly cleaned up.
|
||||
//NCCLCHECK(ncclProxyShmUnlink(comm));
|
||||
|
||||
// We should have allocated all buffers, collective fifos, ... we can
|
||||
// restore the affinity.
|
||||
affinity_restore:
|
||||
//sched_setaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
//if (CPU_COUNT(&comm->cpuAffinity)) sched_setaffinity(0, sizeof(cpu_set_t), &affinitySave);
|
||||
if (ret != ncclSuccess) return ret;
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d - DONE", rank, nranks);
|
||||
|
||||
Ссылка в новой задаче
Block a user