Merge remote-tracking branch 'nccl/master' into develop
Этот коммит содержится в:
Wenkai Du
+84
-94
@@ -22,64 +22,18 @@
|
||||
|
||||
static void* const ncclKernelGeneric = (void*)NCCL_KERN_NAME(SendRecv, RING, SIMPLE, Sum, int8_t);
|
||||
|
||||
// Only generate inline kernels for LL
|
||||
#define NCCL_FUNC5(func, algo, devredop, dtype) \
|
||||
/*LL */(void*)NCCL_KERN_NAME(func, algo, LL, devredop, dtype), \
|
||||
/*LL128 */nullptr /*(void*)NCCL_KERN_NAME(func, algo, LL, devredop, dtype)*/, \
|
||||
/*SIMPLE*/nullptr /*(void*)NCCL_KERN_NAME(func, algo, LL, devredop, dtype)*/
|
||||
|
||||
#define NCCL_FUNC4(func, devredop, type) \
|
||||
(void*)NCCL_FUNC5(func, TREE, devredop, type), \
|
||||
(void*)NCCL_FUNC5(func, RING, devredop, type), \
|
||||
(void*)NCCL_FUNC5(func, COLLNET, devredop, type)
|
||||
|
||||
// Must be consistent with ncclDataType_t
|
||||
#define NCCL_FUNCS3A(func, devredop) \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, uint8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int32_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, uint32_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int64_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, uint64_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, half), \
|
||||
(void*)NCCL_FUNC4(func, devredop, float), \
|
||||
(void*)NCCL_FUNC4(func, devredop, double), \
|
||||
(void*)NCCL_FUNC4(func, devredop, rccl_bfloat16)
|
||||
#define NCCL_FUNCS3B(func, devredop) \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t), \
|
||||
(void*)NCCL_FUNC4(func, devredop, int8_t)
|
||||
|
||||
// Must be consistent with ncclDevRedOp_t -- but we only generate kernel for sums.
|
||||
#define NCCL_FUNCS2A(func) \
|
||||
NCCL_FUNCS3A(func, Sum), /*Sum*/ \
|
||||
NCCL_FUNCS3A(func, Sum), /*Prod*/ \
|
||||
NCCL_FUNCS3A(func, Sum), /*Max*/ \
|
||||
NCCL_FUNCS3A(func, Sum), /*Min*/ \
|
||||
NCCL_FUNCS3A(func, Sum), /*PreMulSum*/ \
|
||||
NCCL_FUNCS3A(func, Sum) /*SumPostDiv*/
|
||||
#define NCCL_FUNCS2B(func) \
|
||||
NCCL_FUNCS3B(func, Sum), /*Sum*/ \
|
||||
NCCL_FUNCS3B(func, Sum), /*Prod*/ \
|
||||
NCCL_FUNCS3B(func, Sum), /*Max*/ \
|
||||
NCCL_FUNCS3B(func, Sum), /*Min*/ \
|
||||
NCCL_FUNCS3B(func, Sum), /*PreMulSum*/ \
|
||||
NCCL_FUNCS3B(func, Sum) /*SumPostDiv*/
|
||||
struct ncclKernelMatch {
|
||||
void* kernelFn;
|
||||
bool specialized;
|
||||
};
|
||||
|
||||
typedef void(*ncclKern_t)(struct ncclDevComm* comm, uint64_t channelMask, struct ncclWork* workHead);
|
||||
// Must be consistent with the ncclFuncSet enum
|
||||
static ncclKern_t const ncclKerns[4] = {
|
||||
NCCL_KERN_NAME(SendRecv, RING, SIMPLE, Sum, int8_t),
|
||||
NCCL_KERN_NAME_DEBUG(SendRecv, RING, SIMPLE, Sum, int8_t),
|
||||
NCCL_KERN_NAME_LL128(SendRecv, RING, SIMPLE, Sum, int8_t),
|
||||
NCCL_KERN_NAME_LL128_DEBUG(SendRecv, RING, SIMPLE, Sum, int8_t),
|
||||
static ncclKernelMatch const ncclKerns[4] = {
|
||||
{(void *)NCCL_KERN_NAME(SendRecv, RING, SIMPLE, Sum, int8_t), true},
|
||||
{(void *)NCCL_KERN_NAME_DEBUG(SendRecv, RING, SIMPLE, Sum, int8_t), true},
|
||||
{(void *)NCCL_KERN_NAME_LL128(SendRecv, RING, SIMPLE, Sum, int8_t), true},
|
||||
{(void *)NCCL_KERN_NAME_LL128_DEBUG(SendRecv, RING, SIMPLE, Sum, int8_t), true},
|
||||
};
|
||||
|
||||
static ncclResult_t computeColl(struct ncclInfo* info /* input */, int* workFuncIndex, struct ncclWorkElem* work, struct ncclProxyOp* proxyOp /* output */);
|
||||
@@ -91,8 +45,8 @@ size_t ncclKernMaxLocalSize() {
|
||||
hipFuncAttributes attr = {0};
|
||||
size_t max = 0;
|
||||
for (int i = 0; i < numNcclKerns; i++) {
|
||||
if (ncclKerns[i] != nullptr) {
|
||||
CUDACHECKGOTO(hipFuncGetAttributes(&attr, reinterpret_cast<const void*>(ncclKerns[i])), res, error);
|
||||
if (ncclKerns[i].kernelFn != nullptr) {
|
||||
CUDACHECKGOTO(hipFuncGetAttributes(&attr, reinterpret_cast<const void*>(ncclKerns[i].kernelFn)), res, error);
|
||||
if (attr.localSizeBytes > max) max = attr.localSizeBytes;
|
||||
}
|
||||
}
|
||||
@@ -107,7 +61,7 @@ size_t ncclKernLocalSize(int i) {
|
||||
int numNcclKerns = sizeof(ncclKerns)/sizeof(ncclKerns[0]);
|
||||
hipFuncAttributes attr = {0};
|
||||
if (i < numNcclKerns)
|
||||
CUDACHECKGOTO(hipFuncGetAttributes(&attr, (const void*)(ncclKerns[i])), res, error);
|
||||
CUDACHECKGOTO(hipFuncGetAttributes(&attr, (const void*)(ncclKerns[i].kernelFn)), res, error);
|
||||
|
||||
error:
|
||||
return (res != ncclSuccess) ? 0 : attr.localSizeBytes;
|
||||
@@ -119,7 +73,7 @@ ncclResult_t ncclKernSetSharedMemoryCarveout(int carveOut) {
|
||||
ncclResult_t res = ncclSuccess;
|
||||
int numNcclKerns = sizeof(ncclKerns)/sizeof(ncclKerns[0]);
|
||||
for (int i = 0; i < numNcclKerns; i++) {
|
||||
CUDACHECKGOTO(hipFuncSetAttribute((const void *)ncclKerns[i], hipFuncAttributePreferredSharedMemoryCarveout, carveOut), res, error);
|
||||
CUDACHECKGOTO(hipFuncSetAttribute((const void *)ncclKerns[i].kernelFn, hipFuncAttributePreferredSharedMemoryCarveout, carveOut), res, error);
|
||||
}
|
||||
|
||||
error:
|
||||
@@ -311,14 +265,14 @@ static ncclResult_t addCollToPlan(
|
||||
workElemReg.elem = *workElem; // C++ struct assignment
|
||||
workElemReg.elem.regUsed = 1;
|
||||
for (int i=0; i < NCCL_MAX_DIRECT_ARITY; i++) {
|
||||
int peer = channel->collTree.down[i];
|
||||
int peer = channel->collnetDirect.down[i];
|
||||
if (peer == -1) break;
|
||||
int j = comm->rankToLocalRank[peer]; // Get intra-node slot
|
||||
workElemReg.dnInputs[i] = regBufSend[j]; // Input buffer of leaf peer
|
||||
workElemReg.dnOutputs[i] = regBufRecv[j]; // Output buffer of leaf peer
|
||||
}
|
||||
for (int i=0; i < NCCL_MAX_DIRECT_ARITY; i++) {
|
||||
int peer = channel->collTree.up[i];
|
||||
int peer = channel->collnetDirect.up[i];
|
||||
if (peer == -1) break;
|
||||
int j = comm->rankToLocalRank[peer];
|
||||
// Output buffer of root peer
|
||||
@@ -340,6 +294,8 @@ static ncclResult_t addCollToPlan(
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
NCCL_PARAM(P2pLLThreshold, "P2P_LL_THRESHOLD", 16384);
|
||||
|
||||
// Put p2p op in plan assuming there is space in nWorkBudget, so you must
|
||||
// ensure *nWorkBudget >= 1 upon entry.
|
||||
static ncclResult_t addP2pToPlan(
|
||||
@@ -357,11 +313,17 @@ static ncclResult_t addP2pToPlan(
|
||||
NCCLCHECK(ncclChannelCompute(comm, peer, chunk%comm->p2pnChannelsPerPeer, info.coll, &channelId));
|
||||
info.channelId = channelId;
|
||||
|
||||
// 1 is connIndex
|
||||
struct ncclConnInfo* conn = isSendNotRecv ?
|
||||
&comm->channels[channelId].peers[peer].send[1].conn : &comm->channels[channelId].peers[peer].recv[1].conn;
|
||||
info.protocol = ((conn->buffs[NCCL_PROTO_LL] != nullptr) && bytes <= ncclParamP2pLLThreshold()) ? NCCL_PROTO_LL : NCCL_PROTO_SIMPLE;
|
||||
|
||||
struct ncclProxyOp proxyOp = {};
|
||||
NCCLCHECK(ncclProxyComputeP2p(&info, &proxyOp));
|
||||
proxyOp.connIndex = connIndex;
|
||||
|
||||
struct ncclWorkElemP2p elem = {0};
|
||||
elem.proto = info.protocol;
|
||||
elem.peer = peer;
|
||||
elem.nWarps = NCCL_MAX_NTHREADS/comm->WarpSize;
|
||||
elem.p2pType = isSendNotRecv ? ncclWorkP2pTypeSend : ncclWorkP2pTypeRecv;
|
||||
@@ -404,9 +366,7 @@ static void finishPlan(struct ncclKernelPlan* plan) {
|
||||
plan->channelCount = channelCount;
|
||||
plan->channelMask = channelMask;
|
||||
plan->hasProxyOps = hasProxyOps;
|
||||
if (plan->kernelFn == nullptr)
|
||||
plan->kernelFn = ncclKernelGeneric;
|
||||
plan->threadPerBlock = std::max(plan->threadPerBlock, 3*plan->comm->WarpSize);
|
||||
plan->threadPerBlock = std::max(plan->threadPerBlock, 4*WARP_SIZE);
|
||||
}
|
||||
|
||||
static ncclResult_t registerIntraNodeBuffers(
|
||||
@@ -565,7 +525,7 @@ static ncclResult_t scheduleCollTasksToPlan(
|
||||
void* regBufSend[NCCL_MAX_LOCAL_RANKS];
|
||||
void* regBufRecv[NCCL_MAX_LOCAL_RANKS];
|
||||
if (plan->persistent && ncclParamGraphRegister() &&
|
||||
info.algorithm == NCCL_ALGO_COLLNET && // limited to CollNet for now
|
||||
info.algorithm == NCCL_ALGO_COLLNET_DIRECT && // limited to CollNetDirect for now
|
||||
comm->intraHighestTransportType == TRANSPORT_P2P && // only when all ranks can p2p each other
|
||||
comm->intraRanks < comm->localRanks) { // only with inter-process & intra-node peers
|
||||
NCCLCHECK(registerIntraNodeBuffers(comm, plan, &info, ®BufUsed, regBufSend, regBufRecv));
|
||||
@@ -579,8 +539,10 @@ static ncclResult_t scheduleCollTasksToPlan(
|
||||
head = ncclIntruQueueHead(&tasks->collQueue);
|
||||
|
||||
plan->threadPerBlock = std::max(plan->threadPerBlock, info.nThreads);
|
||||
if (ncclKerns[ncclGetKernelIndex(comm)] != nullptr)
|
||||
plan->kernelFn = (void *)ncclKerns[ncclGetKernelIndex(comm)];
|
||||
if (!plan->kernelSpecialized) {
|
||||
plan->kernelFn = ncclKerns[ncclGetKernelIndex(comm)].kernelFn;
|
||||
plan->kernelSpecialized = ncclKerns[ncclGetKernelIndex(comm)].specialized;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
@@ -608,11 +570,15 @@ static ncclResult_t scheduleP2pTasksToPlan(
|
||||
int const *recvOrder = tasks->p2pRecvOrder;
|
||||
|
||||
plan->threadPerBlock = std::max(plan->threadPerBlock, NCCL_MAX_NTHREADS);
|
||||
if (!plan->kernelSpecialized) {
|
||||
plan->kernelFn = ncclKerns[ncclGetKernelIndex(comm)].kernelFn;
|
||||
plan->kernelSpecialized = ncclKerns[ncclGetKernelIndex(comm)].specialized;
|
||||
}
|
||||
|
||||
// Compute how much to split operations
|
||||
// Natural step size matching buffer steps.
|
||||
ssize_t stepSize = comm->buffSizes[NCCL_PROTO_SIMPLE]/NCCL_STEPS;
|
||||
if (comm->nNodes > 1) stepSize /= SENDRECV_SLICEFACTOR;
|
||||
if (comm->nNodes > 1) stepSize = comm->p2pNetChunkSize;
|
||||
// Try to use all channels
|
||||
int nChannelsMax = comm->p2pnChannelsPerPeer;
|
||||
int nChannelsMin = nChannelsMax;
|
||||
@@ -714,7 +680,6 @@ static inline uint32_t rollingMin32(uint32_t a, uint32_t b) {
|
||||
// Spin until its safe to increase comm->workFifoSent to desiredSent.
|
||||
static void waitWorkFifoAvailable(struct ncclComm* comm, uint32_t desiredSent) {
|
||||
if (__builtin_expect(rollingLess32(comm->workFifoAckdMin + comm->workFifoDepth, desiredSent), false)) {
|
||||
uint64_t t0 = clockNano();
|
||||
while (1) {
|
||||
// We have to poll for notifications from device.
|
||||
uint32_t* doneLive = comm->workFifoDone;
|
||||
@@ -747,8 +712,7 @@ static void waitWorkFifoAvailable(struct ncclComm* comm, uint32_t desiredSent) {
|
||||
|
||||
// See if that was enough.
|
||||
if (!rollingLess32(comm->workFifoAckdMin + comm->workFifoDepth, desiredSent)) break;
|
||||
// Nope. Maintain vigorous spin for first 5us, then start yielding.
|
||||
if (clockNano()-t0 >= 5*1000) sched_yield();
|
||||
sched_yield();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -874,10 +838,10 @@ static ncclResult_t reclaimPlan(struct ncclComm* comm, struct ncclCommCallback*
|
||||
struct ncclKernelPlan* plan = (struct ncclKernelPlan*)me; // cast from first member `reclaim`
|
||||
if (plan->persistent) {
|
||||
comm->persistentRefs -= 1;
|
||||
if (!ncclMainExited) NCCLCHECK(ncclCudaFree(plan->workHead));
|
||||
NCCLCHECK(ncclCudaFree(plan->workHead));
|
||||
while (!ncclIntruQueueEmpty(&plan->ipcMemQueue)) {
|
||||
struct ncclPointerList* q = ncclIntruQueueDequeue(&plan->ipcMemQueue);
|
||||
if (!ncclMainExited) CUDACHECKIGNORE(hipIpcCloseMemHandle(q->ptr));
|
||||
CUDACHECKIGNORE(hipIpcCloseMemHandle(q->ptr));
|
||||
ncclMemoryPoolFree(&comm->memPool_ncclPointerList, q);
|
||||
}
|
||||
}
|
||||
@@ -904,7 +868,7 @@ ncclResult_t ncclLaunchPrepare(struct ncclComm* comm) {
|
||||
|
||||
// Poll for callbacks sent to us from other threads. Typically these free
|
||||
// resources from to our memory pools.
|
||||
NCCLCHECK(ncclCommPollCallbacks(comm));
|
||||
NCCLCHECK(ncclCommPollCallbacks(comm, /*waitSome=*/false));
|
||||
|
||||
// We already have one frame present which holds all of our tasks (which we
|
||||
// are about to schedule). Now push an additional frame for allocating
|
||||
@@ -1082,7 +1046,7 @@ static ncclResult_t getAlgoInfo(struct ncclInfo* info, int collNetTypeSupport, i
|
||||
info->protocol = -1;
|
||||
int nAlgos = NCCL_NUM_ALGORITHMS;
|
||||
for (int a=0; a<nAlgos; a++) {
|
||||
if (a == NCCL_ALGO_COLLNET && collNetTypeSupport != 1) continue;
|
||||
if ((a == NCCL_ALGO_COLLNET_DIRECT || a == NCCL_ALGO_COLLNET_CHAIN) && collNetTypeSupport != 1) continue;
|
||||
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
|
||||
float time;
|
||||
NCCLCHECK(ncclTopoGetAlgoTime(info, a, p, numPipeOps, &time));
|
||||
@@ -1104,12 +1068,12 @@ static ncclResult_t getAlgoInfo(struct ncclInfo* info, int collNetTypeSupport, i
|
||||
int nc = (info->nChannels > 0) ? info->nChannels : comm->nChannels;
|
||||
int nt = comm->maxThreads[info->algorithm][info->protocol];
|
||||
int threadThreshold = comm->threadThresholds[info->algorithm][info->protocol];
|
||||
if (info->algorithm == NCCL_ALGO_COLLNET) {
|
||||
if (info->algorithm == NCCL_ALGO_COLLNET_DIRECT) {
|
||||
// CollNet channel tuning
|
||||
int ncSwitch = 16;
|
||||
bool flag = true;
|
||||
while (ncSwitch >= 1 && flag) {
|
||||
while ((flag = info->nBytes < nc*nt*info->comm->channels[0].collTree.nHeads*threadThreshold) && nc > ncSwitch) {
|
||||
while ((flag = info->nBytes < nc*nt*info->comm->channels[0].collnetDirect.nHeads*threadThreshold) && nc > ncSwitch) {
|
||||
if (nc == ncSwitch+ncSwitch/2) threadThreshold /= 2;
|
||||
nc--;
|
||||
}
|
||||
@@ -1133,7 +1097,8 @@ static ncclResult_t getAlgoInfo(struct ncclInfo* info, int collNetTypeSupport, i
|
||||
nt += WARP_SIZE; // Extra warp for sync
|
||||
// More threads or sync warps needed due to split thread model
|
||||
if (info->algorithm == NCCL_ALGO_TREE) nt += 3*WARP_SIZE;
|
||||
if (info->algorithm == NCCL_ALGO_COLLNET) nt += 3*WARP_SIZE;
|
||||
if (info->algorithm == NCCL_ALGO_COLLNET_DIRECT) nt += 3*WARP_SIZE;
|
||||
if (info->algorithm == NCCL_ALGO_COLLNET_CHAIN) nt += 3*WARP_SIZE;
|
||||
}
|
||||
nt = nt/WARP_SIZE < 3 ? 3*WARP_SIZE : nt;
|
||||
#endif
|
||||
@@ -1180,7 +1145,11 @@ static ncclResult_t getPatternInfo(struct ncclInfo* info) {
|
||||
case ncclFuncAllToAllPivot:
|
||||
info->pattern = ncclPatternRing; break;
|
||||
case ncclFuncAllReduce:
|
||||
info->pattern = info->algorithm == NCCL_ALGO_COLLNET ? ncclPatternCollTreeUpDown : info->algorithm == NCCL_ALGO_TREE ? ncclPatternTreeUpDown : ncclPatternRingTwice; break;
|
||||
info->pattern =
|
||||
info->algorithm == NCCL_ALGO_COLLNET_DIRECT ? ncclPatternCollnetDirect :
|
||||
info->algorithm == NCCL_ALGO_COLLNET_CHAIN ? ncclPatternCollnetChain :
|
||||
info->algorithm == NCCL_ALGO_TREE ? ncclPatternTreeUpDown :
|
||||
ncclPatternRingTwice; break;
|
||||
default:
|
||||
WARN("Unknown pattern for collective %d algorithm %d", info->coll, info->algorithm);
|
||||
return ncclInternalError;
|
||||
@@ -1195,9 +1164,10 @@ static ncclResult_t getLoopInfo(struct ncclInfo* info) {
|
||||
case ncclPatternTreeUpDown:
|
||||
case ncclPatternPipelineFrom:
|
||||
case ncclPatternPipelineTo:
|
||||
case ncclPatternCollnetChain:
|
||||
info->nstepsPerLoop = info-> nchunksPerLoop = 1; break;
|
||||
case ncclPatternCollTreeUpDown:
|
||||
info->nstepsPerLoop = 1; info->nchunksPerLoop = info->comm->channels[0].collTree.nHeads; break;
|
||||
case ncclPatternCollnetDirect:
|
||||
info->nstepsPerLoop = 1; info->nchunksPerLoop = info->comm->channels[0].collnetDirect.nHeads; break;
|
||||
case ncclPatternRing:
|
||||
info->nstepsPerLoop = info->comm->nRanks-1; info->nchunksPerLoop = info->comm->nRanks; break;
|
||||
case ncclPatternRingTwice:
|
||||
@@ -1274,15 +1244,22 @@ comp_next:
|
||||
}
|
||||
// Use lastChunkSize as chunkSize
|
||||
work->lastChunkSize = chunkSize / ncclTypeSize(info->datatype);
|
||||
} else if (info->algorithm == NCCL_ALGO_COLLNET && info->protocol == NCCL_PROTO_SIMPLE) {
|
||||
} else if (info->algorithm == NCCL_ALGO_COLLNET_DIRECT) {
|
||||
// Optimize chunkSize / nSteps
|
||||
while (info->nBytes / (info->nChannels*info->comm->channels[0].collTree.nHeads*chunkSize) < info->comm->channels[0].collTree.depth*64 && chunkSize > 131072) chunkSize /= 2;
|
||||
while (info->nBytes / (info->nChannels*info->comm->channels[0].collTree.nHeads*chunkSize) < info->comm->channels[0].collTree.depth*8 && chunkSize > 65536) chunkSize /= 2;
|
||||
while (info->nBytes / (info->nChannels*info->comm->channels[0].collTree.nHeads*chunkSize) < info->comm->channels[0].collTree.depth*8 && chunkSize > 32768) chunkSize /= 2;
|
||||
while (info->nBytes / (info->nChannels*info->comm->channels[0].collnetDirect.nHeads*chunkSize) < info->comm->channels[0].collnetDirect.depth*64 && chunkSize > 131072) chunkSize /= 2;
|
||||
while (info->nBytes / (info->nChannels*info->comm->channels[0].collnetDirect.nHeads*chunkSize) < info->comm->channels[0].collnetDirect.depth*8 && chunkSize > 65536) chunkSize /= 2;
|
||||
while (info->nBytes / (info->nChannels*info->comm->channels[0].collnetDirect.nHeads*chunkSize) < info->comm->channels[0].collnetDirect.depth*8 && chunkSize > 32768) chunkSize /= 2;
|
||||
// Use lastChunkSize as chunkSize
|
||||
work->lastChunkSize = chunkSize / ncclTypeSize(info->datatype);
|
||||
// Set direct direction for broadcast-gather (read or write)
|
||||
work->direct = (info->nBytes / info->nChannels <= 1024*1024) ? NCCL_DIRECT_WRITE : NCCL_DIRECT_READ;
|
||||
} else if (info->algorithm == NCCL_ALGO_COLLNET_CHAIN) {
|
||||
stepSize = info->comm->buffSizes[NCCL_PROTO_SIMPLE]/NCCL_STEPS;
|
||||
chunkSize = std::min(256*1024, stepSize*chunkSteps);
|
||||
while (info->nBytes / (info->nChannels*chunkSize) < info->comm->channels[0].collnetChain.depth*64 && chunkSize > 131072) chunkSize /= 2;
|
||||
while (info->nBytes / (info->nChannels*chunkSize) < info->comm->channels[0].collnetChain.depth*8 && chunkSize > 65536) chunkSize /= 2;
|
||||
while (info->nBytes / (info->nChannels*chunkSize) < info->comm->channels[0].collnetChain.depth && chunkSize > 32768) chunkSize /= 2;
|
||||
work->lastChunkSize = chunkSize / ncclTypeSize(info->datatype);
|
||||
} else if (info->protocol == NCCL_PROTO_LL) {
|
||||
const ssize_t sliceSize = stepSize*sizeof(uint64_t)/sizeof(union ncclLLFifoLine);
|
||||
const ssize_t loopSize = info->nChannels*info->nchunksPerLoop*(ssize_t)sliceSize;
|
||||
@@ -1311,7 +1288,7 @@ comp_next:
|
||||
proxyOp->chunkSize = chunkSize;
|
||||
proxyOp->protocol = info->protocol;
|
||||
proxyOp->dtype = info->datatype;
|
||||
proxyOp->redOp = info->algorithm != NCCL_ALGO_COLLNET ? ncclNumOps : // Only set redOp when using CollNet
|
||||
proxyOp->redOp = (info->algorithm != NCCL_ALGO_COLLNET_DIRECT && info->algorithm != NCCL_ALGO_COLLNET_CHAIN) ? ncclNumOps : // Only set redOp when using CollNet
|
||||
info->opFull.op==ncclDevPreMulSum || info->opFull.op==ncclDevSumPostDiv ? ncclSum : // Network sees avg as sum
|
||||
info->op;
|
||||
proxyOp->pattern = info->pattern;
|
||||
@@ -1514,30 +1491,43 @@ ncclResult_t ncclEnqueueCheck(struct ncclInfo* info) {
|
||||
NCCLCHECK(ncclGroupStartInternal());
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
int devOld = -1;
|
||||
NCCLCHECKGOTO(PtrCheck(info->comm, info->opName, "comm"), ret, end0);
|
||||
|
||||
NCCLCHECKGOTO(PtrCheck(info->comm, info->opName, "comm"), ret, fail);
|
||||
// Check whether communicator is ready to communicate
|
||||
NCCLCHECKGOTO(ncclCommEnsureReady(info->comm), ret, fail);
|
||||
|
||||
if (info->comm->checkPointers) {
|
||||
CUDACHECKGOTO(hipGetDevice(&devOld), ret, end0);
|
||||
CUDACHECKGOTO(hipSetDevice(info->comm->cudaDev), ret, end0);
|
||||
CUDACHECKGOTO(hipGetDevice(&devOld), ret, fail);
|
||||
CUDACHECKGOTO(hipSetDevice(info->comm->cudaDev), ret, fail);
|
||||
}
|
||||
NCCLCHECKGOTO(ArgsCheck(info), ret, end1);
|
||||
NCCLCHECKGOTO(ArgsCheck(info), ret, fail);
|
||||
|
||||
INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p",
|
||||
info->opName, info->comm->opCount, info->sendbuff, info->recvbuff, info->count,
|
||||
info->datatype, info->op, info->root, info->comm, info->comm->nRanks, info->stream);
|
||||
TRACE_CALL("nccl%s(%" PRIx64 ",%" PRIx64 ",%zi,%d,%d,%d,%p,%p)", info->opName, reinterpret_cast<int64_t>(info->sendbuff), reinterpret_cast<int64_t>(info->recvbuff), info->count, info->datatype, info->op, info->root, info->comm, info->stream);
|
||||
|
||||
NCCLCHECKGOTO(taskAppend(info->comm, info), ret, end1);
|
||||
NCCLCHECKGOTO(taskAppend(info->comm, info), ret, fail);
|
||||
|
||||
end1:
|
||||
if (devOld != -1) CUDACHECKGOTO(hipSetDevice(devOld), ret, end0);
|
||||
end0:
|
||||
exit:
|
||||
if (devOld != -1) CUDACHECK(hipSetDevice(devOld));
|
||||
ncclGroupErrCheck(ret);
|
||||
NCCLCHECK(ncclGroupEndInternal());
|
||||
/* if depth is 1, ncclGroupEndInternal() will trigger group ops. The state can change
|
||||
* so we have to check state here. */
|
||||
if (info->comm && !info->comm->blocking) { NCCLCHECK(ncclCommGetAsyncError(info->comm, &ret)) };
|
||||
return ret;
|
||||
fail:
|
||||
if (info->comm && !info->comm->blocking) (void) ncclCommSetAsyncError(info->comm, ret);
|
||||
goto exit;
|
||||
}
|
||||
|
||||
NCCL_API(ncclResult_t, ncclRedOpCreatePreMulSum, ncclRedOp_t *op, void *scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm);
|
||||
ncclResult_t ncclRedOpCreatePreMulSum(ncclRedOp_t *op, void *scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm) {
|
||||
NCCLCHECK(PtrCheck(comm, "ncclRedOpCreatePreMulSum", "comm"));
|
||||
/* join init thread before creating PreMulSum op. */
|
||||
NCCLCHECK(ncclCommEnsureReady(comm));
|
||||
|
||||
if (comm->userRedOpFreeHead == comm->userRedOpCapacity) {
|
||||
// double capacity and resize
|
||||
int cap = 2*comm->userRedOpCapacity;
|
||||
|
||||
Ссылка в новой задаче
Block a user