AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'nccl/master' into develop

Browse Source
This commit is contained in:
BertanDogancay
2025-01-23 11:48:18 -06:00
committed by Corey Derochie
parent b6377e0b8c 178b6b7590
commit 36343be84f
122 changed files with 9009 additions and 4702 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/include/comm.h
+242 -64
View File
@@ -8,7 +8,7 @@
#ifndef NCCL_COMM_H_
#define NCCL_COMM_H_
#include "transport.h"
// #include "transport.h"
#include "p2p.h"
#include "collectives.h"
#include "nccl_tuner.h"
@@ -16,6 +16,8 @@
#include "strongstream.h"
#include "nccl_net.h"
#include "register.h"
#include "graph.h"
#include "nvmlwrap.h"
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
#define HIPRT_CB
@@ -149,7 +151,7 @@ struct ncclChannel {
struct ncclNvls nvls;
int id; // index of this channel
uint32_t workFifoSent; // last used work index+1
uint32_t workFifoProduced; // +1 successor of last used work fifo byte
/* comm split sharable resources */
struct ncclChannelPeer* collnetPeers;
@@ -158,22 +160,15 @@ struct ncclChannel {
struct ncclDevChannelPeer* nvlsDevPeers;
};
struct ncclWorkList {
struct ncclWorkBatchList {
struct ncclWorkBatchList* next;
struct ncclDevWorkBatch batch;
};
struct alignas(16) ncclWorkList {
struct ncclWorkList* next;
struct ncclWork work;
};
struct ncclPointerList {
struct ncclPointerList* next;
void *ptr;
};
struct ncclNvlsMcHandleList {
struct ncclNvlsMcHandleList *next;
CUmemGenericAllocationHandle mcHandle;
CUdeviceptr ptr;
int dev;
size_t size;
enum ncclDevWorkType workType;
int size; // Size of struct following this node
// ncclDevWorkColl, ncclDevWorkColLReg, ncclDevWorkP2p[]...
};
struct ncclCollnetHandleList {
@@ -184,10 +179,6 @@ struct ncclCollnetHandleList {
struct ncclProxyConnector* proxyconn;
};
struct channelMasks {
uint64_t masks[MAXCHANNELS/64];
};
struct ncclKernelPlan {
// A kernel plan is also a callback that reclaims itself. Hence this must
// be the first member.
@@ -197,37 +188,213 @@ struct ncclKernelPlan {
struct ncclKernelPlan* next;
bool persistent; // aka captured in a graph
enum ncclDevWorkStorageType workStorageType;
bool kernelSpecialized;
void *kernelFn;
int channelUbound; // only channels c < channelUbound are present
int channelCount; // number of channels present
struct ncclDevKernelArgs* kernelArgs;
size_t kernelArgsSize;
struct channelMasks channelMask;
bool hasProxyOps; // does any channel have a non-empty proxyOpQueue
int threadPerBlock;
// workHeap fields are null until uploadWorkFifo() or preparePersistentKernel()
struct ncclWork* workHead;
int collOpCount; // zero based for this plan
int collOpCount; // Number of collectives in this plan.
int nWorkBatches; // Number of work batches.
size_t workBytes; // Sum size of all work (in the fifo) in bytes.
struct ncclIntruQueue<struct ncclWorkList, &ncclWorkList::next> workQueue;
struct ncclIntruQueue<struct ncclCommCallback, &ncclCommCallback::next> cleanupQueue;
void* workBufPersistent;
struct ncclIntruQueue<struct ncclPointerList, &ncclPointerList::next> ipcMemQueue;
struct ncclIntruQueue<struct ncclNvlsMcHandleList, &ncclNvlsMcHandleList::next> nvlsMcHandleQueue;
struct ncclIntruQueue<struct ncclCollnetHandleList, &ncclCollnetHandleList::next> collnetHandleQueue;
struct ncclIntruQueue<struct ncclProxyOp, &ncclProxyOp::enqNext> proxyOpQueue;
};
struct Channel {
int nWork;
union {
int nWorkElem; // used for coll and reg coll
int p2pTailElem[2]; // used for p2p, indexed by ncclWorkElemP2pType-1
};
size_t collBytes;
struct ncclIntruQueue<struct ncclWorkList, &ncclWorkList::next> workQueue;
struct ncclIntruQueue<struct ncclProxyOp, &ncclProxyOp::enqNext> proxyOpQueue;
} channels[MAXCHANNELS];
size_t maxBytesPerChannel;
////////////////////////////////////////////////////////////////////////////////
struct ncclTaskColl {
struct ncclTaskColl* next;
ncclFunc_t func;
void const* sendbuff;
void* recvbuff;
size_t count;
int root;
ncclDataType_t datatype;
ncclRedOp_t opHost;
struct ncclDevRedOpFull opDev;
int chunkSteps, sliceSteps;
// Computed later:
size_t trafficBytes;
int32_t nMaxChannels:8;
int32_t nWarps:8;
int32_t algorithm:8, protocol:8;
uint32_t isCollnet:1, isNvls:1;
uint32_t devFuncId:30;
enum ncclRegBufferType regBufType;
// number of elements in planner->ipcMemQueue associated with this collective
int nCleanupQueueElts;
void* sendMhandle;
void* recvMhandle;
};
struct ncclTaskP2p {
struct ncclTaskP2p* next;
void* buff;
size_t bytes;
};
////////////////////////////////////////////////////////////////////////////////
// Roughly sorts ncclTaskColl's by their size descending. This structure is
// self-referential, meaning that pointers it contains internally may point
// into the structure itself. This means that it is NOT memcpy-moveable:
struct ncclTaskCollSorter {
static constexpr int UnitLog2 = 10; // 1K
static constexpr size_t UnitSize = 1<<UnitLog2;
static constexpr int MaxLog2 = 30; // 1GB
static constexpr size_t MaxSize = 1ull<<MaxLog2;
// Number of bins between powers of 2. For 4 bins, the worst case out-of-order
// relative magnitude is (5/4)-1 = 25%
static constexpr int BitsPerPow2 = 2;
static constexpr int BinsPerPow2 = 1<<BitsPerPow2;
static constexpr int BinCount = 1 + (MaxLog2-UnitLog2)*BinsPerPow2;
struct ncclTaskColl* head;
struct ncclTaskColl* tail;
// Least bin such that it and all above are empty.
int binEdge;
// Pointer to the pointer to this bin's head node which is either the
// previous node's `next` field or `head`.
struct ncclTaskColl** bins[BinCount];
};
inline void ncclTaskCollSorterInsert(
struct ncclTaskCollSorter* me, struct ncclTaskColl* x, size_t size
) {
constexpr int UnitLog2 = ncclTaskCollSorter::UnitLog2;
constexpr size_t MaxSize = ncclTaskCollSorter::MaxSize;
constexpr int BitsPerPow2 = ncclTaskCollSorter::BitsPerPow2;
constexpr int BinCount = ncclTaskCollSorter::BinCount;
int bin = u32fpEncode(std::min(MaxSize, size)>>UnitLog2, BitsPerPow2);
bin = BinCount-1 - bin; // descending bin
if (me->bins[bin] == nullptr) {
if (me->binEdge <= bin) {
me->binEdge = bin+1;
me->bins[bin] = me->tail ? &me->tail->next : &me->head;
me->tail = x;
} else {
// Find successor non-empty bin after this one.
int succ = bin+1;
while (me->bins[succ] == nullptr) succ++;
// What was our successor's head's previous is now our head's previous.
me->bins[bin] = me->bins[succ];
// The first node we insert is our tail, so that becomes our successor's
// head's new previous.
me->bins[succ] = &x->next;
}
}
// Push a new head for this bin.
x->next = *me->bins[bin];
*me->bins[bin] = x;
}
inline bool ncclTaskCollSorterEmpty(struct ncclTaskCollSorter* me) {
return me->head == nullptr;
}
// Reset sorter and return sorted linked list of its coll tasks.
inline struct ncclTaskColl* ncclTaskCollSorterDequeueAll(struct ncclTaskCollSorter* me) {
struct ncclTaskColl* head = me->head;
if (head != nullptr) memset(me, 0, sizeof(*me));
return head;
}
////////////////////////////////////////////////////////////////////////////////
struct ncclCudaStreamList {
struct ncclCudaStreamList *next;
cudaStream_t stream;
};
struct ncclKernelPlanner {
//////////////////////////////////////////////////////////////////////////////
// State for accumulating tasks between ncclGroupStart/End()
//////////////////////////////////////////////////////////////////////////////
struct Peer {
bool sendSeen, recvSeen;
struct ncclIntruQueue<struct ncclTaskP2p, &ncclTaskP2p::next> sendQueue;
struct ncclIntruQueue<struct ncclTaskP2p, &ncclTaskP2p::next> recvQueue;
};
struct ncclTaskCollSorter collSorter;
struct Peer* peers/*[nRanks]*/;
int nTasksColl, nTasksP2p;
bool persistent;
// The list of user streams aggregated over all tasks present.
struct ncclCudaStreamList* streams;
// Keep track of the number of user streams
int numStreams;
// The most recent user stream. Ignored if streams==nullptr
cudaStream_t streamRecent;
// The graph capturing all user streams or invalid if none. Thus we restrict the
// user that all streams must be captured in the same graph or not captured
// at all. Technically we could probably relax this, but that would mean
// collecting a different `ncclTasks` per graph and one for non-graph.
struct ncclCudaGraph capturingGraph;
//////////////////////////////////////////////////////////////////////////////
// Lists of tasks to be assembled into plans.
//////////////////////////////////////////////////////////////////////////////
struct ncclIntruQueue<struct ncclTaskColl, &ncclTaskColl::next> collTaskQueue;
struct ncclIntruQueue<struct ncclWorkList, &ncclWorkList::next> collWorkQueue;
struct ncclIntruQueue<struct ncclCommCallback, &ncclCommCallback::next> collCleanupQueue;
//////////////////////////////////////////////////////////////////////////////
// State for building current (Work-In-Progress) plan:
//////////////////////////////////////////////////////////////////////////////
struct WipPlan {
struct Channel {
struct {
int workBytes; // Sum size of work metadata referenced by this batch.
int nP2ps; // Number of p2p works in this batch
int p2pRounds[NCCL_MAX_DEV_WORK_P2P_PER_BATCH]; // which rounds are present in this batch.
} wipBatch; // work-in-progress batch which will be next tail of workBatchQueue
int nWorkBatchesP2p; // number of p2p batches for this channel.
struct ncclIntruQueue<struct ncclWorkBatchList, &ncclWorkBatchList::next> workBatchQueue;
struct ncclIntruQueue<struct ncclProxyOp, &ncclProxyOp::enqNext> proxyOpQueue;
} channels[MAXCHANNELS];
} wipPlan;
//////////////////////////////////////////////////////////////////////////////
// State for launching built plans:
//////////////////////////////////////////////////////////////////////////////
// List of kernel plans built form tasks.
struct ncclIntruQueue<struct ncclKernelPlan, &ncclKernelPlan::next> planQueue;
// First of the unlaunched kernels in `planQueue`
struct ncclKernelPlan* unlaunchedPlansHead;
};
#define NCCL_MAGIC 0x0280028002800280 // Nickel atomic number is 28.
struct ncclPeerInfo {
int rank;
int cudaDev;
int nvmlDev;
int gdrSupport;
bool hasFineGrain;
uint64_t hostHash;
uint64_t pidHash;
dev_t shmDev;
int64_t busId;
struct ncclComm* comm;
int cudaCompCap;
// MNNVL support
nvmlGpuFabricInfoV_t fabricInfo;
int cuMemSupport;
};
struct ncclComm {
uint64_t startMagic;
struct ncclMemoryStack memPermanent, memScoped;
@@ -242,12 +409,18 @@ struct ncclComm {
struct ncclPeerInfo* peerInfo;
struct ncclTopoSystem* topo;
int netPluginLoaded;
ncclNet_t* ncclNet;
ncclNetDeviceType netDeviceType;
ncclCollNet_t* ncclCollNet;
void* bootstrap;
// Bitmasks for ncclTransportP2pSetup
struct channelMasks* connectSend;
struct channelMasks* connectRecv;
struct ncclTopoGraph graphs[NCCL_NUM_ALGORITHMS];
bool initAlgoChannels[NCCL_NUM_ALGORITHMS];
bool runtimeConn; // if dynamic connection is supported
int cuMemSupport;
uint64_t magic; // Magic number for all network communication. Not a security key -- only goal is to detect mismatches.
@@ -263,6 +436,9 @@ struct ncclComm {
int WarpSize;
int cudaArch; // matches __CUDA_ARCH__ of device
int cpuArch; // architecture - As defined in src/include/graph.h, e.g. x86/arm/ppc/mixed
int cpuVendor; // vendor - As defined in src/include/graph.h
int node;
int nNodes;
int localRank;
@@ -288,10 +464,11 @@ struct ncclComm {
int nChannels; // connection nChannels
int collChannels; // enqueue nChannels
int nvlsChannels; // enqueue nChannels
// all nvls heads stored to check if we can splitShare
int nvlsHeads[MAXCHANNELS];
// Channels (per peer) for p2p
int p2pnChannels;
int p2pnChannelsPerPeer;
int p2pChannels[MAXCHANNELS];
// Should this comm allocate LL buffers for network P2P connections?
bool allocP2pNetLLBuffers;
@@ -313,9 +490,12 @@ struct ncclComm {
ncclResult_t asyncResult;
// Flag to ask NCCL kernels to abort
volatile uint32_t *abortFlag;
volatile uint32_t *childAbortFlag;
uint32_t *abortFlagRefCount;
uint32_t* abortFlag;
uint32_t* abortFlagDev;
int* abortFlagRefCount;
uint32_t* childAbortFlag;
uint32_t* childAbortFlagDev;
uint32_t destroyFlag;
// Flags for enable P2P NET
uint32_t p2pNet;
@@ -325,16 +505,18 @@ struct ncclComm {
// Device side of the communicator (for cudaFree's)
struct ncclDevComm* devComm; // actually = &ncclDevCommAndChannels::comm
// Operation pool.
int workFifoDepth; // size of workFifoHeap[], power of 2
struct ncclWork* workFifoHeap;
struct ncclWork* devWorkFifoHeap;
void* workFifoHeapGdrHandle;
uint32_t workArgsBytes; // max size of kernel args
uint32_t workFifoBytes; // size of workFifoBuf, power of 2
void* workFifoBuf;
void* workFifoBufDev;
void* workFifoBufGdrHandle;
// Work completion notificaion
uint32_t* workFifoDone/*[MAXCHANNELS]*/; // in cudaHost memory
uint32_t workFifoSent; // Monotonic (mod 1<<32) index of next unused fifo slot.
uint32_t workFifoAckdMin; // Monotonic index of least unprocessed fifo slot over all channels.
// Monotonic number of bytes (mod 1<<32) consumed per channel. In cudaHost memory.
uint32_t* workFifoConsumed/*[MAXCHANNELS]*/;
// Last observed value of: min(workFifoConsumed[c] for c < MAXCHANNELS)
uint32_t workFifoConsumedLeast;
// Monotonic number of bytes (mod 1<<32) sent to fifo.
uint32_t workFifoProduced;
// Intra-process sync
struct ncclComm* intraComm0; // leader of intra-process comms (self possible)
@@ -352,7 +534,7 @@ struct ncclComm {
// Whether this communicator uses collNet
int collNetSupport;
bool collNetRegSupport;
uint8_t collNetSupportMatrix[4/*sum,prod,min,max*/][ncclNumTypes];
uint8_t collNetSupportMatrix[4/*sum,prod,max,min*/][ncclNumTypes];
int intraHighestTransportType;
int* collNetHeads;
int collNetHeadsNum;
@@ -370,16 +552,16 @@ struct ncclComm {
// pools backed by comm->memPermanent
struct ncclMemoryPool memPool_ncclProxyOp;
struct ncclMemoryPool memPool_ncclKernelPlan;
struct ncclMemoryPool memPool_ncclPointerList;
struct ncclMemoryPool memPool_ncclNvlsHandleList;
struct ncclMemoryPool memPool_ncclCollnetHandleList;
// Next comm in this thread's active ncclGroup[Start|End](). Holds "0x1" when
// this comm is not yet in a group.
struct ncclComm* groupNext;
// Subset of those in groupNext list. Holds 0x1 if not needing preconnect.
struct ncclComm* preconnectNext;
int persistentRefs; // number of persistent plan-lists capturing this comm
struct ncclTasks tasks;
struct P2pSchedulePair { int sendRank; int recvRank; } *p2pSchedule;
struct ncclKernelPlanner planner;
hipStream_t sideStream; // [RCCL] Cached non-captured stream
@@ -390,11 +572,6 @@ struct ncclComm {
// Queue of things for the main thread to do
struct ncclIntruQueueMpsc<struct ncclCommCallback, &ncclCommCallback::next> callbackQueue;
// List of kernel plans built form tasks.
struct ncclIntruQueue<struct ncclKernelPlan, &ncclKernelPlan::next> planQueue;
// First of the unlaunched kernels in `planQueue`
struct ncclKernelPlan* unlaunchedPlansHead;
hipEvent_t doneEvent;
hipStream_t lastStream;
@@ -426,13 +603,14 @@ struct ncclComm {
struct ncclGroupJob *groupJob;
// Tuning plugin
int tunerPluginLoaded;
ncclTuner_t* tuner;
void *tunerContext;
// buffer registration cache
struct ncclRegCache regCache;
uint64_t endMagic;
// Unroll factor for plan [RCCL]
// Unroll factor for comm [RCCL]
int unroll;
};