Add support for external network.

Dynamically load external network from libnccl-net.so.
Add init function in networks.
Move PCI scoring to net.cu, only ask transport to provide a path.
Simplify CUDA PCI path detection.
Add dummy external network

src/misc/enqueue.cu

@@ -175,7 +175,7 @@ ncclResult_t ncclBarrierEnqueueWait(ncclComm_t comm) {
   // We can't print the CG mode before the first barrier happened.
   if (comm->rank == 0 && *comm->intraCGMode & 0x10) {
     *comm->intraCGMode ^= 0x10;
-    INFO(INIT,"Launch mode %s%s%s",
+    INFO(NCCL_INIT,"Launch mode %s%s%s",
         comm->launchMode == ncclComm::GROUP ? "Group" : "Parallel",
         *comm->intraCGMode ? "/CGMD" : "",
         (comm->launchMode == ncclComm::GROUP && comm->groupCudaStream) ? "/Stream" : "");

src/misc/group.cu

@@ -58,7 +58,7 @@ ncclResult_t ncclSetDevice(int cudaDev) {
 
 #define CHECK(a) do { \
   if ((args->ret = (a)) != ncclSuccess) { \
-    INFO(INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, args->ret); \
+    INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, args->ret); \
     return args; \
   } \
 } while(0)

src/misc/nvmlwrap.cu

@@ -61,7 +61,7 @@ ncclResult_t wrapNvmlSymbols(void) {
     cast = (void**)&funcptr;                             \
     tmp = dlsym(handle, symbol);                         \
     if (tmp == NULL) {                                   \
-      INFO(INIT,"dlsym failed on %s, ignoring", symbol); \
+      INFO(NCCL_INIT,"dlsym failed on %s, ignoring", symbol); \
     }                                                    \
     *cast = tmp;                                         \
   } while (0)
@@ -208,7 +208,7 @@ ncclResult_t wrapNvmlDeviceGetNvLinkState(nvmlDevice_t device, unsigned int link
   }
   nvmlReturn_t ret = nvmlInternalDeviceGetNvLinkState(device, link, isActive);
   if (ret != NVML_SUCCESS) {
-    INFO(INIT,"nvmlDeviceGetNvLinkState() failed: %s ",
+    INFO(NCCL_INIT,"nvmlDeviceGetNvLinkState() failed: %s ",
         nvmlInternalErrorString(ret));
     return ncclSystemError;
   }
@@ -223,7 +223,7 @@ ncclResult_t wrapNvmlDeviceGetNvLinkRemotePciInfo(nvmlDevice_t device, unsigned
   nvmlReturn_t ret = nvmlInternalDeviceGetNvLinkRemotePciInfo(device, link, pci);
   if (ret != NVML_SUCCESS) {
     if (ret != NVML_ERROR_NOT_SUPPORTED)
-      INFO(INIT,"nvmlDeviceGetNvLinkRemotePciInfo() failed: %s ",
+      INFO(NCCL_INIT,"nvmlDeviceGetNvLinkRemotePciInfo() failed: %s ",
           nvmlInternalErrorString(ret));
     return ncclSystemError;
   }
@@ -239,7 +239,7 @@ ncclResult_t wrapNvmlDeviceGetNvLinkCapability(nvmlDevice_t device, unsigned int
   nvmlReturn_t ret = nvmlInternalDeviceGetNvLinkCapability(device, link, capability, capResult);
   if (ret != NVML_SUCCESS) {
     if (ret != NVML_ERROR_NOT_SUPPORTED)
-      INFO(INIT,"nvmlDeviceGetNvLinkCapability() failed: %s ",
+      INFO(NCCL_INIT,"nvmlDeviceGetNvLinkCapability() failed: %s ",
           nvmlInternalErrorString(ret));
     return ncclSystemError;
   }

src/misc/rings.cu

@@ -5,9 +5,10 @@
  ************************************************************************/
 
 #include "core.h"
-#include "net.h"
 #include "param.h"
 
+#define NCCL_MAX_SCORE 7
+
 /* Parse user defined rings. Format is like :
  * "0 1|1 0|0 1 2 3|3 2 1 0|0 2 3 1|1 3 2 0|0 1 2 3 4 5 6 7|7 6 5 4 3 2 1 0"
  * Rings with a non-matching number of ranks are ignored so we can provide
@@ -188,11 +189,11 @@ ncclResult_t ncclGetRings(int* nrings, int* nthreads, int rank, int nranks, int*
   if (str && strlen(str)>0) {
     int ret = parseRings(str, nrings, nranks, prev, next);
     if (ret == ncclSuccess && *nrings > 0) {
-      if (rank == 0) INFO(INIT,"%d ring(s) set by environment", *nrings);
+      if (rank == 0) INFO(NCCL_INIT,"%d ring(s) set by environment", *nrings);
       NCCLCHECK(getEnvThreads(nthreads));
       return ncclSuccess;
     }
-    if (rank == 0) INFO(INIT,"No valid ring found in environment, ignoring");
+    if (rank == 0) INFO(NCCL_INIT,"No valid ring found in environment, ignoring");
     *nrings = 0;
   }
 
@@ -333,13 +334,13 @@ ncclResult_t ncclGetRings(int* nrings, int* nthreads, int rank, int nranks, int*
     minNrings = MAXRINGS;
   }
   if (maxNrings > 0 && maxNrings <= *nrings) {
-    if (rank == 0) INFO(INIT,"Limiting to %d rings per user request.", maxNrings);
+    if (rank == 0) INFO(NCCL_INIT,"Limiting to %d rings per user request.", maxNrings);
     *nrings = maxNrings;
   } else {
     int defaultMinNrings = ncclCudaCompCap() == 3 ? 2 : 1;
     if (minNrings < defaultMinNrings) minNrings = defaultMinNrings;
     if (minNrings > 0 && minNrings > *nrings) {
-      if (rank == 0 && minNrings > defaultMinNrings) INFO(INIT,"Duplicating rings to %d per user request.", minNrings);
+      if (rank == 0 && minNrings > defaultMinNrings) INFO(NCCL_INIT,"Duplicating rings to %d per user request.", minNrings);
       for (int r=*nrings; r<MAXRINGS && r <minNrings; r++) {
         for (int i=0; i<nranks; i++) {
           prev[r*nranks+i] = prev[(r-*nrings)*nranks+i];

src/misc/utils.cu

@@ -6,8 +6,10 @@
 
 #include "utils.h"
 #include "debug.h"
+#include "nccl_net.h"
 #include <unistd.h>
 #include <string.h>
+#include <stdarg.h>
 
 ncclResult_t getHostName(char* hostname, int maxlen) {
   if (gethostname(hostname, maxlen) != 0) {
@@ -20,6 +22,53 @@ ncclResult_t getHostName(char* hostname, int maxlen) {
   return ncclSuccess;
 }
 
+/* Common logging function used by the INFO, WARN and TRACE macros
+ * Also exported to the dynamically loadable Net transport modules so
+ * they can share the debugging mechanisms and output files
+ */
+void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *filefunc, int line, const char *fmt, ...) {
+  if (ncclDebugLevel <= NCCL_LOG_NONE) return;
+
+  char hostname[1024];
+  getHostName(hostname, 1024);
+  int cudaDev;
+  cudaGetDevice(&cudaDev);
+
+  char buffer[1024];
+  size_t len = 0;
+  pthread_mutex_lock(&ncclDebugOutputLock);
+  if (level == NCCL_LOG_WARN && ncclDebugLevel >= NCCL_LOG_WARN)
+    len = snprintf(buffer, sizeof(buffer),
+                   "\n%s:%d:%d [%d] %s:%d NCCL WARN ", hostname, getpid(), gettid(), cudaDev, filefunc, line);
+  else if (level == NCCL_LOG_INFO && ncclDebugLevel >= NCCL_LOG_INFO && (flags & ncclDebugMask))
+    len = snprintf(buffer, sizeof(buffer),
+                   "%s:%d:%d [%d] NCCL INFO ", hostname, getpid(), gettid(), cudaDev);
+#ifdef ENABLE_TRACE
+  else if (level == NCCL_LOG_TRACE && ncclDebugLevel >= NCCL_LOG_TRACE && (flags & ncclDebugMask)) {
+    auto delta = std::chrono::high_resolution_clock::now() - ncclEpoch;
+    double timestamp = std::chrono::duration_cast<std::chrono::duration<double>>(delta).count()*1000;
+    len = snprintf(buffer, sizeof(buffer),
+                   "%s:%d:%d [%d] %f %s:%d NCCL TRACE ", hostname, getpid(), gettid(), cudaDev, timestamp, filefunc, line);
+  }
+#endif
+  if (len) {
+    va_list vargs;
+    va_start(vargs, fmt);
+    (void) vsnprintf(buffer+len, sizeof(buffer)-len, fmt, vargs);
+    va_end(vargs);
+    fprintf(ncclDebugFile,"%s\n", buffer);
+    fflush(ncclDebugFile);
+  }
+  pthread_mutex_unlock(&ncclDebugOutputLock);
+
+  // If ncclDebugLevel == NCCL_LOG_ABORT then WARN() will also call abort()
+  if (level == NCCL_LOG_WARN && ncclDebugLevel == NCCL_LOG_ABORT) {
+    fprintf(stderr,"\n%s:%d:%d [%d] %s:%d NCCL ABORT\n",
+            hostname, getpid(), gettid(), cudaDev, filefunc, line);
+    abort();
+  }
+}
+
 uint64_t getHash(const char* string) {
   // Based on DJB2, result = result * 33 + char
   uint64_t result = 5381;
@@ -51,7 +100,7 @@ uint64_t getHostHash(void) {
   offset += len;
   // Trailing '\0'
   uname[offset]='\0';
-  TRACE(INIT,"unique hostname '%s'", uname);
+  TRACE(NCCL_INIT,"unique hostname '%s'", uname);
 
   return getHash(uname);
 }
@@ -71,7 +120,7 @@ uint64_t getPidHash(void) {
   if (len < 0) len = 0;
 
   pname[plen+len]='\0';
-  TRACE(INIT,"unique PID '%s'", pname);
+  TRACE(NCCL_INIT,"unique PID '%s'", pname);
 
   return getHash(pname);
 }