AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2.14.3-1

Browse Source 
Add support for improved fault tolerance: non-blocking mode, new
init function with config, and ncclCommFinalize function.
Reintroduce collnet+chain algorithm, alongside collnet+direct.
Add LL protocol for intra-node P2P (on by default) and network
communication (off by default).
Use network instead of shared memory when performance is better.
Fix: wait for CUDA graph destroy before destroying comm with linked
graph resources.
Remove aggressive polling during enqueue.
Fix DMABUF fallback on MOFED 5.4 and earlier.
This commit is contained in:
Sylvain Jeaugey
2022-08-18 02:53:17 -07:00
parent e1d9b273b0
commit c4e2aa6c79
42 changed files with 1787 additions and 942 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/include/comm.h
+28 -11
View File
@@ -102,7 +102,8 @@ struct ncclChannel {
struct ncclRing ring;
int* devRingUserRanks;
struct ncclTree tree;
struct ncclDirect collTree;
struct ncclTree collnetChain;
struct ncclDirect collnetDirect;
int id; // index of this channel
uint32_t workFifoSent; // last used work index+1
uint64_t p2pOpCount;
@@ -128,6 +129,7 @@ struct ncclKernelPlan {
struct ncclKernelPlan* next;
bool persistent; // aka captured in a graph
bool kernelSpecialized;
void *kernelFn;
int channelUbound; // only channels c < channelUbound are present
int channelCount; // number of channels present
@@ -201,8 +203,12 @@ struct ncclComm {
int p2pnChannelsPerPeer;
int p2pChannels[MAXCHANNELS];
// Should this comm allocate LL buffers for network P2P connections?
bool allocP2pNetLLBuffers;
// Buffer sizes
int buffSizes[NCCL_NUM_PROTOCOLS];
int p2pNetChunkSize;
// Algorithm/Protocols thresholds
ssize_t threadThresholds[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
@@ -210,8 +216,9 @@ struct ncclComm {
float bandwidths[NCCL_NUM_FUNCTIONS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
int maxThreads[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
// Whether there has been a fatal error in this communicator.
ncclResult_t fatalError;
/* This attribute can indicate the states of communicators and return code of
* asynchronous NCCL operations. */
ncclResult_t asyncResult;
// Flag to ask NCCL kernels to abort
volatile uint32_t *abortFlag;
@@ -276,12 +283,16 @@ struct ncclComm {
struct ncclIntruQueue<struct ncclKernelPlan, &ncclKernelPlan::next> planQueue;
// First of the unlaunched kernels in `planQueue`
struct ncclKernelPlan* unlaunchedPlansHead;
};
// Set to true during an `atexit()` handler. We use this to intentionally leak
// unfreed CUDA resources when cleaning up after return of `main()` to avoid
// CUDA calls after CUDA runtime teardown.
extern bool ncclMainExited;
// communicator mode
int blocking;
// initState is to more conveniently reclaim resources when errors happen.
ncclResult_t initState;
// flag to indicate if ncclCommFinalize() is called
bool finalizeCalled;
// shared structures for finalization
int finalizeRankCnt;
};
enum ncclLaunchMode {
ncclLaunchModeInvalid=0,
@@ -295,13 +306,16 @@ void ncclCommPushCudaFree(struct ncclComm* comm, void* buf);
void ncclCommPushCudaHostFree(struct ncclComm* comm, void* buf);
void ncclCommPushCudaGdrFree(struct ncclComm* comm, void* handle);
inline ncclResult_t ncclCommPollCallbacks(struct ncclComm* comm) {
struct ncclCommCallback* cb = ncclIntruQueueMpscDequeueAll(&comm->callbackQueue, /*waitSome=*/false);
inline ncclResult_t ncclCommPollCallbacks(struct ncclComm* comm, bool waitSome) {
ncclResult_t result = ncclSuccess;
struct ncclCommCallback* cb = ncclIntruQueueMpscDequeueAll(&comm->callbackQueue, waitSome);
while (cb != nullptr) {
struct ncclCommCallback* next = cb->next;
NCCLCHECK(cb->fn(comm, cb)); // may reclaim memory of cb
ncclResult_t res1 = cb->fn(comm, cb); // may reclaim memory of cb
if (res1 != ncclSuccess) result = res1;
cb = next;
}
NCCLCHECK(result);
return ncclSuccess;
}
@@ -358,4 +372,7 @@ static inline ncclRedOp_t ncclUserRedOpMangle(ncclComm *comm, ncclRedOp_t op) {
return op1 < int(ncclNumOps) ? op : ncclRedOp_t(op1);
}
ncclResult_t ncclCommEnsureReady(ncclComm_t comm);
ncclResult_t ncclCommSetAsyncError(ncclComm_t comm, ncclResult_t nextState);
#endif