2.14.3-1

Add support for improved fault tolerance: non-blocking mode, new
init function with config, and ncclCommFinalize function.
Reintroduce collnet+chain algorithm, alongside collnet+direct.
Add LL protocol for intra-node P2P (on by default) and network
communication (off by default).
Use network instead of shared memory when performance is better.
Fix: wait for CUDA graph destroy before destroying comm with linked
graph resources.
Remove aggressive polling during enqueue.
Fix DMABUF fallback on MOFED 5.4 and earlier.

src/misc/gdrwrap.cc

@@ -9,8 +9,6 @@
 #ifndef GDR_DIRECT
 #include "core.h"
 
-static enum { gdrUninitialized, gdrInitializing, gdrInitialized, gdrError } gdrState = gdrUninitialized;
-
 /* Function pointers assigned from dlopen() */
 static gdr_t (*gdr_internal_open)(void);
 static int (*gdr_internal_close)(gdr_t g);
@@ -49,18 +47,10 @@ pthread_mutex_t gdrLock = PTHREAD_MUTEX_INITIALIZER;
     *cast = tmp;                                         \
   } while (0)
 
-ncclResult_t wrap_gdr_symbols(void) {
-  if (gdrState == gdrInitialized)
-    return ncclSuccess;
-  if (gdrState == gdrError)
-    return ncclSystemError;
-
-  if (__sync_bool_compare_and_swap(&gdrState, gdrUninitialized, gdrInitializing) == false) {
-    // Another thread raced in front of us. Wait for it to be done.
-    while (gdrState == gdrInitializing) sched_yield();
-    return (gdrState == gdrInitialized) ? ncclSuccess : ncclSystemError;
-  }
+static pthread_once_t initOnceControl = PTHREAD_ONCE_INIT;
+static ncclResult_t initResult;
 
+static void initOnceFunc(void) {
   static void* gdrhandle = NULL;
   void* tmp;
   void** cast;
@@ -84,8 +74,8 @@ ncclResult_t wrap_gdr_symbols(void) {
   LOAD_SYM(gdrhandle, "gdr_copy_to_mapping", gdr_internal_copy_to_mapping);
   LOAD_SYM(gdrhandle, "gdr_copy_from_mapping", gdr_internal_copy_from_mapping);
 
-  gdrState = gdrInitialized;
-  return ncclSuccess;
+  initResult = ncclSuccess;
+  return;
 
 teardown:
   gdr_internal_open = NULL;
@@ -101,11 +91,16 @@ teardown:
   gdr_internal_copy_from_mapping = NULL;
 
   if (gdrhandle != NULL) dlclose(gdrhandle);
-  gdrState = gdrError;
-  return ncclSystemError;
+  initResult = ncclSystemError;
+  return;
 }
 
 
+ncclResult_t wrap_gdr_symbols(void) {
+  pthread_once(&initOnceControl, initOnceFunc);
+  return initResult;
+}
+
 gdr_t wrap_gdr_open(void) {
   if (gdr_internal_open == NULL) {
     WARN("GDRCOPY lib wrapper not initialized.");