Sylvain Jeaugey
d97a32fac8
Add support for IB SHARP to NVLS (NVLink SHARP algorithm). Add NVLS+Tree algorithm. Add support for memory management using cuMem* functions. Use all NICs for Send/Receive operations on systems with more than one NIC per GPU (#804). Add ncclCommSplit primitive, with resource sharing option in config. Fix alltoallv hang (#788) Increase number of channels on H100 when we're not limited by NVLink. Improve error reporting in case of IB failure, printing local and remote ID (#779). Add build option to allow compilation against RDMA includes instead of dynamically loading IB verbs symbols (#802). Fix context creation for progress thread (#803). NET/IB: add option to use multiple QPs in round-robin mode. Fix tree performance issue when NVB is disabled on HCM topologies.
49 行
2.1 KiB
C
49 行
2.1 KiB
C
/*************************************************************************
|
|
* Copyright (c) 2015-2019, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* See LICENSE.txt for license information
|
|
************************************************************************/
|
|
|
|
#ifndef NCCL_CHANNEL_H_
|
|
#define NCCL_CHANNEL_H_
|
|
#include "comm.h"
|
|
|
|
ncclResult_t initChannel(struct ncclComm* comm, int channelid);
|
|
ncclResult_t initNvlsChannel(struct ncclComm* comm, int channelId, struct ncclComm* parent, bool share);
|
|
ncclResult_t initCollnetChannel(struct ncclComm* comm, int channelId, struct ncclComm* parent, bool share);
|
|
ncclResult_t freeChannel(struct ncclChannel* channel, int nRanks, int collnetNRanks, int nvlsNRanks);
|
|
static ncclResult_t ncclChannelComputeBase(struct ncclComm* comm, int peer, int coll, int*channelBase) {
|
|
int p2pGroupSize = NCCL_MAX_WORK_ELEMENTS_P2P/2;
|
|
int peerNode = comm->rankToNode[peer];
|
|
int peerIndex = comm->rankToLocalRank[peer];
|
|
int nsteps = comm->maxLocalRanks;
|
|
int rankIndex = comm->rankToLocalRank[comm->rank];
|
|
int step, delta;
|
|
if (coll == ncclFuncSend) {
|
|
step = (nsteps + peerIndex - rankIndex)%nsteps;
|
|
delta = (comm->nNodes + peerNode - comm->node) % comm->nNodes;
|
|
} else if (coll == ncclFuncRecv) {
|
|
step = (nsteps + rankIndex - peerIndex)%nsteps;
|
|
delta = (comm->nNodes + comm->node - peerNode) % comm->nNodes;
|
|
} else {
|
|
return ncclInternalError;
|
|
}
|
|
*channelBase = comm->nNodes > 1 ? delta+(step/p2pGroupSize) : step;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t ncclChannelComputeFromBase(struct ncclComm* comm, int base, int channelInc, int*channelId) {
|
|
//*channelId = (base+comm->p2pChannels[channelInc]) % comm->p2pnChannels;
|
|
*channelId = (comm->p2pChannels[base%comm->p2pnChannels]+channelInc) % comm->p2pnChannels;
|
|
return ncclSuccess;
|
|
}
|
|
|
|
static ncclResult_t ncclChannelCompute(struct ncclComm* comm, int peer, int channelInc, int coll, int*channelId) {
|
|
int base;
|
|
NCCLCHECK(ncclChannelComputeBase(comm, peer, coll, &base));
|
|
NCCLCHECK(ncclChannelComputeFromBase(comm, base, channelInc, channelId));
|
|
return ncclSuccess;
|
|
}
|
|
|
|
#endif
|