2.22.3-1

Rework core for NVIDIA Trusted Computing
 * Compress work structs so that they are shared between channels
 * Utilize the full amount of kernel argument space permitted (4k)
   before resorting to work fifo.
 * Rework the task preprocessing phase.
 * Use a separate abortDevFlag which is kept in sync with abortFlag
   using cudaMemcpy operations.
 * Rename src/include/align.h to src/include/bitops.h

Add lazy connection establishment for collective operations
 * Move buffer allocation and connection establishment to the first
   collective operation using that algorithm.
 * Accelerate init time and reduce memory usage.
 * Avoid allocating NVLS buffers if all calls are registered.
 * Compute algo/proto in ncclLaunchCollTasksInfo early on.
 * Connect peers in ncclCollPreconnectFunc if not connected already.
 * Also move shared buffer creation to the first send/recv call.

Accelerate intra-node NVLink detection
 * Make each rank only detect NVLinks attached to its GPU.
 * Fuse XMLs to reconstruct the full NVLink topology

Add init profiling to report time spend in different init phases.
 * Report timings of bootstrap, allgather, search, connect, etc.
 * Add new "PROFILE" category for NCCL_DEBUG_SUBSYS.

Add support for PCI p2p on split PCI switches
 * Detect split PCI switches through a kernel module exposing
   switch information.
 * Update the topology XML and graph to add those inter-switch
   connections.

Add cost estimation API
 * Add a new ncclGroupEndSimulate primitive to return the estimated
   time a group would take.

Net/IB: Add separate traffic class for fifo messages
 * Add NCCL_IB_FIFO_TC to control the traffic class of fifo messages
   independently from NCCL_IB_TC.
   Merges PR #1194

Net/IB: Add support for IB router
 * Use flid instead of lid if subnets do not match
 * Warn if flid is 0

Optimizations and fixes for device network offload (unpack)
 * Double the default number of channels
 * Cache netDeviceType
 * Fix save/increment head logic to enable Tree support.

Support ncclGroupStart/End for ncclCommAbort/Destroy
 * Allow Abort/Destroy to be called within a group when managing
   multiple GPUs with a single process.

Improve Tuner API
 * Provide to the plugin the original cost table so that the plugin
   can leave unknown or disabled algo/proto combinations untouched.
 * Remove nvlsSupport and collnetSupport.

Do not print version to stdout when using a debug file
 * Also print version from all processes with INFO debug level.
   Fixes issue #1271

Fix clang warnings in NVTX headers
 * Update NVTX headers to the latest version
   Fixes issue #1270

Disable port fusion in heterogeneous systems
 * Do not fuse ports if a mix of multi-port and single port are detected.

Fix NVLS graphs search for dual NICs.
 * Fix NVLS graph search when we have more than one NIC per GPU.

Fix crash with collnetDirect
 * Add separate graph search for collnetDirect, testing alltoall paths
   and working similarly to the NVLS search.

Fix hang when nodes have different CPU types
 * Add the CPU type to the rank peer info.
 * Align all ranks on the CPU type after the first allgather.
 * Only use the aligned CPU type for all tuning operations.
   Fixes issue #1136
   Fixes issue #1184

Fix performance of registered send/recv operations
 * Allow for single full size operations
 * Add INFO to confirm the registration of send/recv buffers.

Move all sync ops to finalize stage
 * Ensure ncclCommDestroy is non-blocking if ncclCommFinalize has
   been called.

Improve error reporting during SHM segment creation

Improve support of various compilers
   Merges PR #1177
   Merges PR #1228

Allow net and tuner plugins to be statically linked
 * Search for ncclNet or ncclTuner symbols in the main binary.
   Merges PR #979

Plugin examples includes cleanup
 * Harmonize err.h and common.h usage.
 * Add mixed plugin with both net and tuner.


[ROCm/rccl commit: 178b6b7590]

projects/rccl/src/misc/argcheck.cc

@@ -52,8 +52,6 @@ ncclResult_t ArgsCheck(struct ncclInfo* info) {
     WARN("%s : invalid type %d", info->opName, info->datatype);
     return ncclInvalidArgument;
   }
-  // Type is OK, compute nbytes. Convert Allgather/Broadcast/P2P calls to chars.
-  NCCLCHECK(ncclInfoSetDerived(info, info->comm->nRanks));
 
   if (info->op < 0 || ncclMaxRedOp < info->op) {
     WARN("%s : invalid reduction operation %d", info->opName, info->op);

projects/rccl/src/misc/cudawrap.cc

@@ -59,6 +59,10 @@ DECLARE_CUDA_PFN(cuGetErrorString);
 DECLARE_CUDA_PFN(cuGetErrorName);
 /* enqueue.cc */
 DECLARE_CUDA_PFN(cuMemGetAddressRange);
+DECLARE_CUDA_PFN(cuLaunchKernel);
+#if CUDA_VERSION >= 11080
+DECLARE_CUDA_PFN(cuLaunchKernelEx);
+#endif
 /* proxy.cc */
 DECLARE_CUDA_PFN(cuCtxCreate);
 DECLARE_CUDA_PFN(cuCtxDestroy);
@@ -137,6 +141,10 @@ static ncclResult_t cudaPfnFuncLoader(void) {
   LOAD_SYM(cuCtxGetCurrent, 1);
   LOAD_SYM(cuCtxSetCurrent, 1);
   LOAD_SYM(cuCtxGetDevice, 1);
+  LOAD_SYM(cuLaunchKernel, 1);
+#if CUDA_VERSION >= 11080
+  LOAD_SYM(cuLaunchKernelEx, 1);
+#endif
 /* cuMem API support */
   LOAD_SYM(cuMemAddressReserve, 1);
   LOAD_SYM(cuMemAddressFree, 1);

projects/rccl/src/misc/gdrwrap.cc

@@ -130,7 +130,7 @@ ncclResult_t wrap_gdr_pin_buffer(gdr_t g, unsigned long addr, size_t size, uint6
   int ret;
   GDRLOCKCALL(gdr_internal_pin_buffer(g, addr, size, p2p_token, va_space, handle), ret);
   if (ret != 0) {
-    WARN("gdr_pin_buffer(addr %lx, size %zi) failed: %d", addr, size, ret);
+    WARN("gdr_pin_buffer(addr %lx, size %zu) failed: %d", addr, size, ret);
     return ncclSystemError;
   }
   return ncclSuccess;
@@ -172,7 +172,7 @@ ncclResult_t wrap_gdr_map(gdr_t g, gdr_mh_t handle, void **va, size_t size) {
   int ret;
   GDRLOCKCALL(gdr_internal_map(g, handle, va, size), ret);
   if (ret != 0) {
-    WARN("gdr_map(handle %lx, size %zi) failed: %d", handle.h, size, ret);
+    WARN("gdr_map(handle %lx, size %zu) failed: %d", handle.h, size, ret);
     return ncclSystemError;
   }
   return ncclSuccess;
@@ -186,7 +186,7 @@ ncclResult_t wrap_gdr_unmap(gdr_t g, gdr_mh_t handle, void *va, size_t size) {
   int ret;
   GDRLOCKCALL(gdr_internal_unmap(g, handle, va, size), ret);
   if (ret != 0) {
-    WARN("gdr_unmap(handle %lx, va %p, size %zi) failed: %d", handle.h, va, size, ret);
+    WARN("gdr_unmap(handle %lx, va %p, size %zu) failed: %d", handle.h, va, size, ret);
     return ncclSystemError;
   }
   return ncclSuccess;
@@ -218,7 +218,7 @@ ncclResult_t wrap_gdr_copy_to_mapping(gdr_mh_t handle, void *map_d_ptr, const vo
   int ret;
   GDRLOCKCALL(gdr_internal_copy_to_mapping(handle, map_d_ptr, h_ptr, size), ret);
   if (ret != 0) {
-    WARN("gdr_copy_to_mapping(handle %lx, map_d_ptr %p, h_ptr %p, size %zi) failed: %d", handle.h, map_d_ptr, h_ptr, size, ret);
+    WARN("gdr_copy_to_mapping(handle %lx, map_d_ptr %p, h_ptr %p, size %zu) failed: %d", handle.h, map_d_ptr, h_ptr, size, ret);
     return ncclSystemError;
   }
   return ncclSuccess;
@@ -232,7 +232,7 @@ ncclResult_t wrap_gdr_copy_from_mapping(gdr_mh_t handle, void *h_ptr, const void
   int ret;
   GDRLOCKCALL(gdr_internal_copy_from_mapping(handle, h_ptr, map_d_ptr, size), ret);
   if (ret != 0) {
-    WARN("gdr_copy_from_mapping(handle %lx, h_ptr %p, map_d_ptr %p, size %zi) failed: %d", handle.h, h_ptr, map_d_ptr, size, ret);
+    WARN("gdr_copy_from_mapping(handle %lx, h_ptr %p, map_d_ptr %p, size %zu) failed: %d", handle.h, h_ptr, map_d_ptr, size, ret);
     return ncclSystemError;
   }
   return ncclSuccess;

projects/rccl/src/misc/ipcsocket.cc

@@ -132,7 +132,7 @@ ncclResult_t ncclIpcSocketRecvMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
       WARN("UDS: Receiving data over socket failed : %d", errno);
       return ncclSystemError;
     }
-    if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
+    if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
   }
 
   if (recvFd != NULL) {
@@ -221,7 +221,7 @@ ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
       WARN("UDS: Sending data over socket %s failed : %s (%d)", temp, strerror(errno), errno);
       return ncclSystemError;
     }
-    if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
+    if (handle->abortFlag && __atomic_load_n(handle->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
   }
 
   return ncclSuccess;

projects/rccl/src/misc/nvmlwrap.cc

@@ -41,11 +41,19 @@ namespace {
   NCCL_NVML_FN(nvmlDeviceGetFieldValues, nvmlReturn_t, (nvmlDevice_t device, int valuesCount, nvmlFieldValue_t *values))
   // MNNVL support
   NCCL_NVML_FN(nvmlDeviceGetGpuFabricInfoV, nvmlReturn_t, (nvmlDevice_t device, nvmlGpuFabricInfoV_t *gpuFabricInfo))
+  // CC support
+  NCCL_NVML_FN(nvmlSystemGetConfComputeState, nvmlReturn_t, (nvmlConfComputeSystemState_t *state));
+  NCCL_NVML_FN(nvmlSystemGetConfComputeSettings, nvmlReturn_t, (nvmlSystemConfComputeSettings_t *setting));
 
   std::mutex lock; // NVML has had some thread safety bugs
   bool initialized = false;
   thread_local bool threadInitialized = false;
   ncclResult_t initResult;
+
+  union nvmlCCInfoInternal {
+    nvmlConfComputeSystemState_t settingV12020;
+    nvmlSystemConfComputeSettings_t settingV12040;
+  };
 }
 
 ncclResult_t ncclNvmlEnsureInitialized() {
@@ -87,6 +95,9 @@ ncclResult_t ncclNvmlEnsureInitialized() {
       {(void**)&pfn_nvmlDeviceGetFieldValues, "nvmlDeviceGetFieldValues"},
       // MNNVL support
       {(void**)&pfn_nvmlDeviceGetGpuFabricInfoV, "nvmlDeviceGetGpuFabricInfoV"},
+      // CC support
+      {(void**)&pfn_nvmlSystemGetConfComputeState, "nvmlSystemGetConfComputeState"},
+      {(void**)&pfn_nvmlSystemGetConfComputeSettings, "nvmlSystemGetConfComputeSettings"}
     };
     for(Symbol sym: symbols) {
       *sym.ppfn = dlsym(libhandle, sym.name);
@@ -282,3 +293,33 @@ ncclResult_t ncclNvmlDeviceGetGpuFabricInfoV(nvmlDevice_t device, nvmlGpuFabricI
   NVMLTRY(nvmlDeviceGetGpuFabricInfoV, device, gpuFabricInfo);
   return ncclSuccess;
 }
+
+ncclResult_t ncclNvmlGetCCStatus(struct ncclNvmlCCStatus *status) {
+  NCCLCHECK(ncclNvmlEnsureInitialized());
+  std::lock_guard<std::mutex> locked(lock);
+  nvmlCCInfoInternal ccInfo;
+  if (pfn_nvmlSystemGetConfComputeSettings != NULL) {
+    ccInfo.settingV12040.version = nvmlSystemConfComputeSettings_v1;
+    NVMLTRY(nvmlSystemGetConfComputeSettings, &ccInfo.settingV12040);
+    if (ccInfo.settingV12040.ccFeature == NVML_CC_SYSTEM_FEATURE_ENABLED)
+      status->CCEnabled = true;
+    else
+      status->CCEnabled = false;
+
+    if (ccInfo.settingV12040.multiGpuMode == NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE)
+      status->multiGpuCCEnabled = true;
+    else
+      status->multiGpuCCEnabled = false;
+  } else if (pfn_nvmlSystemGetConfComputeState != NULL) {
+    NVMLTRY(nvmlSystemGetConfComputeState, &ccInfo.settingV12020);
+    if (ccInfo.settingV12020.ccFeature == NVML_CC_SYSTEM_FEATURE_ENABLED)
+      status->CCEnabled = true;
+    else
+      status->CCEnabled = false;
+    status->multiGpuCCEnabled = false;
+  } else {
+    status->CCEnabled = false;
+    status->multiGpuCCEnabled = false;
+  }
+  return ncclSuccess;
+}

projects/rccl/src/misc/param.cc

@@ -84,4 +84,4 @@ const char *ncclGetEnv(const char *name) {
   static pthread_once_t once = PTHREAD_ONCE_INIT;
   pthread_once(&once, initEnv);
   return getenv(name);
-}
+}

projects/rccl/src/misc/shmutils.cc

@@ -63,13 +63,28 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
      * goes down to 0, unlink should be called in order to delete shared memory file. */
     if (shmPath[0] == '\0') {
       sprintf(shmPath, "/dev/shm/nccl-XXXXXX");
+    retry_mkstemp:
       fd = mkstemp(shmPath);
+      if (fd < 0) {
+        if (errno == EINTR) {
+          INFO(NCCL_ALL, "mkstemp: Failed to create %s, error: %s (%d) - retrying", shmPath, strerror(errno), errno);
+          goto retry_mkstemp;
+        }
+        WARN("Error: failed to create shared memory file %p, error %s (%d)", shmPath, strerror(errno), errno);
+        ret = ncclSystemError;
+        goto fail;
+      }
     } else {
       SYSCHECKGOTO(fd = open(shmPath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR), ret, fail);
     }
 
+  retry_fallocate:
     if (fallocate(fd, 0, 0, realShmSize) != 0) {
-      WARN("Error: failed to extend %s to %ld bytes", shmPath, realShmSize);
+      if (errno == EINTR) {
+        INFO(NCCL_ALL, "fallocate: Failed to extend %s to %ld bytes, error: %s (%d) - retrying", shmPath, realShmSize, strerror(errno), errno);
+        goto retry_fallocate;
+      }
+      WARN("Error: failed to extend %s to %ld bytes, error: %s (%d)", shmPath, realShmSize, strerror(errno), errno);
       ret = ncclSystemError;
       goto fail;
     }
@@ -80,7 +95,7 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
 
   hptr = (char*)mmap(NULL, realShmSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
   if (hptr == MAP_FAILED) {
-    WARN("Could not map %s size %zi, error: %s", shmPath, realShmSize, strerror(errno));
+    WARN("Error: Could not map %s size %zu, error: %s (%d)", shmPath, realShmSize, strerror(errno), errno);
     ret = ncclSystemError;
     hptr = NULL;
     goto fail;
@@ -93,7 +108,7 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
     if (remref == 0) {
       /* the last peer has completed attachment, it should unlink the shm mem file. */
       if (unlink(shmPath) != 0) {
-        WARN("unlink shared memory %s failed, error: %s", shmPath, strerror(errno));
+        INFO(NCCL_ALLOC, "unlink shared memory %s failed, error: %s (%d)", shmPath, strerror(errno), errno);
       }
     }
   }
@@ -110,7 +125,8 @@ exit:
   *handle = (ncclShmHandle_t)tmphandle;
   return ret;
 fail:
-  WARN("Error while %s shared memory segment %s (size %ld)", create ? "creating" : "attaching to", shmPath, shmSize);
+  WARN("Error while %s shared memory segment %s (size %ld), error: %s (%d)", create ? "creating" : "attaching to",
+       shmPath, shmSize, strerror(errno), errno);
   if (tmphandle) {
     shmHandleInit(fd, shmPath, shmSize, realShmSize, hptr, dptr, create, tmphandle);
     ncclShmClose((ncclShmHandle_t)tmphandle);
@@ -129,7 +145,7 @@ ncclResult_t ncclShmClose(ncclShmHandle_t handle) {
       close(tmphandle->fd);
       if (tmphandle->shmPath != NULL && tmphandle->refcount != NULL && *tmphandle->refcount > 0) {
         if (unlink(tmphandle->shmPath) != 0) {
-          WARN("unlink shared memory %s failed, error: %s", tmphandle->shmPath, strerror(errno));
+          WARN("unlink shared memory %s failed, error: %s (%d)", tmphandle->shmPath, strerror(errno), errno);
           ret = ncclSystemError;
         }
       }
@@ -139,7 +155,7 @@ ncclResult_t ncclShmClose(ncclShmHandle_t handle) {
     if (tmphandle->shmPtr) {
       if (tmphandle->devShmPtr) CUDACHECK(cudaHostUnregister(tmphandle->shmPtr));
       if (munmap(tmphandle->shmPtr, tmphandle->realShmSize) != 0) {
-        WARN("munmap of shared memory %p size %ld failed, error: %s", tmphandle->shmPtr, tmphandle->realShmSize, strerror(errno));
+        WARN("munmap of shared memory %p size %ld failed, error: %s (%d)", tmphandle->shmPtr, tmphandle->realShmSize, strerror(errno), errno);
         ret = ncclSystemError;
       }
     }
@@ -152,9 +168,9 @@ ncclResult_t ncclShmUnlink(ncclShmHandle_t handle) {
   ncclResult_t ret = ncclSuccess;
   struct shmHandleInternal* tmphandle = (struct shmHandleInternal*)handle;
   if (tmphandle) {
-    if (tmphandle->shmPath != NULL) {
+    if (tmphandle->shmPath != NULL && tmphandle->refcount != NULL && *tmphandle->refcount > 0) {
       if (unlink(tmphandle->shmPath) != 0) {
-        WARN("unlink shared memory %s failed, error: %s", tmphandle->shmPath, strerror(errno));
+        WARN("unlink shared memory %s failed, error: %s (%d)", tmphandle->shmPath, strerror(errno), errno);
         ret = ncclSystemError;
       }
       free(tmphandle->shmPath);
@@ -184,7 +200,7 @@ ncclResult_t ncclShmemAllgather(struct ncclComm *comm, struct ncclShmemCollBuff
     uint64_t t0 = clockNano();
     while(__atomic_load_n(shmem->cnt[curRound], __ATOMIC_ACQUIRE) != comm->localRanks + 1) {
       if (clockNano() - t0 >= 5 * 1000) sched_yield();
-      if (__atomic_load_n(comm->abortFlag, __ATOMIC_RELAXED) == 1) {
+      if (__atomic_load_n(comm->abortFlag, __ATOMIC_ACQUIRE) == 1) {
         ret = ncclInternalError;
         goto exit;
       }

projects/rccl/src/misc/socket.cc

@@ -34,7 +34,7 @@ static ncclResult_t socketProgressOpt(int op, struct ncclSocket* sock, void* ptr
       }
     }
     (*offset) += bytes;
-    if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED)) {
+    if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE)) {
       INFO(NCCL_NET, "socketProgressOpt: abort called");
       return ncclInternalError;
     }
@@ -620,12 +620,12 @@ ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
   do {
     NCCLCHECK(socketProgressState(sock));
   } while (sock->asyncFlag == 0 &&
-      (sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED) == 0) &&
+      (sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE) == 0) &&
       (sock->state == ncclSocketStateConnecting ||
        sock->state == ncclSocketStateConnectPolling ||
        sock->state == ncclSocketStateConnected));
 
-  if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
+  if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
 
   switch (sock->state) {
     case ncclSocketStateConnecting:
@@ -667,11 +667,11 @@ ncclResult_t ncclSocketAccept(struct ncclSocket* sock, struct ncclSocket* listen
   do {
     NCCLCHECKGOTO(socketProgressState(sock), ret, exit);
   } while (sock->asyncFlag == 0 &&
-      (sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED) == 0) &&
+      (sock->abortFlag == NULL || __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE) == 0) &&
       (sock->state == ncclSocketStateAccepting ||
        sock->state == ncclSocketStateAccepted));
 
-  if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_RELAXED)) return ncclInternalError;
+  if (sock->abortFlag && __atomic_load_n(sock->abortFlag, __ATOMIC_ACQUIRE)) return ncclInternalError;
 
   switch (sock->state) {
     case ncclSocketStateAccepting:

projects/rccl/src/misc/tuner.cc

@@ -9,117 +9,150 @@
 #include <errno.h>
 #include <stdlib.h>
 
+#include "checks.h"
 #include "debug.h"
-#include "nccl_tuner.h"
+#include "tuner.h"
 
 pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
 static int tunerPluginRefCount;
 static void* tunerPluginLib = nullptr;
-ncclTuner_t* tunerSymbol = nullptr;
+static ncclTuner_v3_t* tunerSymbol = nullptr;
+static ncclTuner_v2_t* ncclTuner_v2 = nullptr;
+static ncclTuner_v3_t ncclTuner_v2_as_v3;
 
-static void* tryOpenDynamicLib(const char* name) {
+static int hasNvlsSupport(float** collCostTable) {
+  // Requirements for support of different algorithms:
+  //
+  // - NVLS intra-node: nvlsSupport
+  // - NVLS intra+inter-node: collNetSupport
+  // - NVLSTree intra-node: always disabled
+  // - NVLSTree inter-node: nvlsSupport
+  // - Collnet* inter-node: collNetSupport
+  //
+  // nvlsSupport = 1 if either NVLS or NVLS_TREE entries in the cost table are not -1
+  float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
+  return (table[NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE || table[NCCL_ALGO_NVLS_TREE][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) ? 1 : 0;
+}
+
+static int hasCollNetSupport(float** collCostTable) {
+  float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
+  return (table[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] == NCCL_ALGO_PROTO_IGNORE) ? 0 : 1;
+}
+
+static ncclResult_t ncclTuner_v2_as_v3_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int* nChannels) {
+  int algorithm = NCCL_ALGO_UNDEF;
+  int protocol = NCCL_PROTO_UNDEF;
+  int nvlsSupport = hasNvlsSupport(collCostTable);
+  int collNetSupport = hasCollNetSupport(collCostTable);
+  NCCLCHECK(ncclTuner_v2->getCollInfo(context, collType, nBytes, collNetSupport, nvlsSupport, numPipeOps, &algorithm, &protocol, nChannels));
+  // set time to 0 below to make sure this algorithm/protocol is selected later on
+  if (algorithm >= 0 && algorithm < NCCL_NUM_ALGORITHMS && protocol >= 0 && protocol < NCCL_NUM_PROTOCOLS) {
+    float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
+    if (table[algorithm][protocol] != NCCL_ALGO_PROTO_IGNORE) table[algorithm][protocol] = 0.0;
+  }
+  return ncclSuccess;
+}
+
+static ncclResult_t ncclTuner_v2_as_v3_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
+  NCCLCHECK(ncclTuner_v2->init(nRanks, nNodes, logFunction, context));
+  ncclTuner_v2_as_v3.name = ncclTuner_v2->name;
+  ncclTuner_v2_as_v3.getCollInfo = ncclTuner_v2_as_v3_getCollInfo;
+  ncclTuner_v2_as_v3.destroy = ncclTuner_v2->destroy;
+  return ncclSuccess;
+}
+
+#define MAX_STR_LEN 255
+
+static void* tryOpenLib(const char* name, int* err, char* errStr) {
+  *err = 0;
   if (nullptr == name || strlen(name) == 0) {
     return nullptr;
   }
+
+  if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
+    name = nullptr;
+  }
+
   void *handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
   if (nullptr == handle) {
-    if (ENOENT == errno) {
-      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: No plugin found (%s)", name);
+    strncpy(errStr, dlerror(), MAX_STR_LEN);
+    errStr[MAX_STR_LEN] = '\0';
+    if (strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
+      *err = ENOENT;
     }
   }
   return handle;
 }
 
-static void summarizeOpenTunerPluginLibErrors(char* pluginNames) {
-  const char *separator = " ";
-  int len = strlen(pluginNames);
-  // remove tail separator
-  pluginNames[len - 1] = '\0';
-
-  // remove last plugin name
-  while (len > 0 && pluginNames[--len] != *separator);
-  if (len > 0) {
-    pluginNames[len] = '\0';
-  }
-
-  // distinguish between one load attempt and multiple attempts
-  if (strstr(pluginNames, separator)) {
-    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Most recent plugin load returned %d : %s. All attempts to load '%s' also failed.", errno, dlerror(), pluginNames);
-  } else {
-    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin load returned %d : %s : when loading %s", errno, dlerror(), pluginNames);
+static char* tryOpenLibCheck(int openErr, char* openErrStr, char* nameList, int *nameListLen, char* name) {
+  if (openErr == ENOENT) {
+    snprintf(nameList, *nameListLen, " %s", name);
+    nameList += strlen(name) + 1;
+    *nameListLen -= strlen(name) + 1;
+    return nameList;
   }
+  INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: %s", openErrStr);
+  return nameList;
 }
 
-static void* openTunerPluginLib(void) {
+static void* openTunerPluginLib(char* couldNotFindNames, int len) {
+  int openErr;
   void *pluginLib;
-
-#define MAX_PLUGIN_LOAD 4
-
-  int len;
-  char tunerPluginLibNameTried[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
-  char *ptr = tunerPluginLibNameTried;
   char tunerPluginLibName[PATH_MAX];
+  char openErrStr[MAX_STR_LEN + 1] = { 0 };
   const char *envTunerPluginName = getenv("NCCL_TUNER_PLUGIN");
   if (envTunerPluginName && strlen(envTunerPluginName)) {
     INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: NCCL_TUNER_PLUGIN set to %s", envTunerPluginName);
     snprintf(tunerPluginLibName, PATH_MAX, "%s", envTunerPluginName);
-    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
     if (pluginLib) {
       INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
       return pluginLib;
     }
-    len = PATH_MAX - strlen(ptr);
-    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+    couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
 
     snprintf(tunerPluginLibName, PATH_MAX, "libnccl-tuner-%s.so", envTunerPluginName);
-    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
     if (pluginLib) {
       INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
       return pluginLib;
     }
-    len = PATH_MAX - strlen(ptr);
-    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+    couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
   } else {
     snprintf(tunerPluginLibName, PATH_MAX, "libnccl-tuner.so");
-    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
     if (pluginLib) {
       return pluginLib;
     }
-    len = PATH_MAX - strlen(ptr);
-    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+    couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
   }
 
   const char *envNetPluginName = getenv("NCCL_NET_PLUGIN");
   if (envNetPluginName && strlen(envNetPluginName)) {
     // Users are allowed to pack tuner into the net plugin
     snprintf(tunerPluginLibName, PATH_MAX, "%s", envNetPluginName);
-    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
     if (pluginLib) {
       INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
       return pluginLib;
     }
-    len = PATH_MAX - strlen(ptr);
-    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+    couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
 
     snprintf(tunerPluginLibName, PATH_MAX, "libnccl-net-%s.so", envNetPluginName);
-    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
     if (pluginLib) {
       INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
       return pluginLib;
     }
-    len = PATH_MAX - strlen(ptr);
-    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+    couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
   } else {
     snprintf(tunerPluginLibName, PATH_MAX, "libnccl-net.so");
-    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
     if (pluginLib) {
       return pluginLib;
     }
-    len = PATH_MAX - strlen(ptr);
-    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+    couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
   }
-  summarizeOpenTunerPluginLibErrors(ptr);
-
   tunerPluginLibName[0] = '\0';
   return nullptr;
 }
@@ -130,10 +163,14 @@ enum {
   tunerPluginLoadSuccess =  1,
 };
 
-ncclResult_t ncclTunerPluginLoad(ncclTuner_t** tuner) {
+#define MAX_PLUGIN_LOAD 4
+
+static int status = tunerPluginLoadReady;
+
+ncclResult_t ncclTunerPluginLoad(struct ncclComm* comm) {
   // Initialize to nullptr by default if plugin tuner cannot be loaded.
-  *tuner = nullptr;
-  static int status = tunerPluginLoadReady;
+  char couldNotFindNames[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
+  comm->tuner = nullptr;
   if (tunerPluginLoadFailed == status) {
     return ncclSuccess;
   }
@@ -144,28 +181,41 @@ ncclResult_t ncclTunerPluginLoad(ncclTuner_t** tuner) {
   }
 
   if (tunerPluginLoadSuccess == status) {
-    *tuner = tunerSymbol;
+    comm->tuner = tunerSymbol;
     ++tunerPluginRefCount;
     goto exit;
   }
 
-  tunerPluginLib = openTunerPluginLib();
+  tunerPluginLib = openTunerPluginLib(couldNotFindNames, MAX_PLUGIN_LOAD * PATH_MAX);
   if (nullptr == tunerPluginLib) {
-    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using internal tuner plugin.");
+    if (strlen(couldNotFindNames)) {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Could not find:%s. Using internal tuner plugin.", couldNotFindNames);
+    } else {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using internal tuner plugin.");
+    }
     goto fail;
   }
 
-  tunerSymbol = (ncclTuner_t*)dlsym(tunerPluginLib, NCCL_TUNER_PLUGIN_SYMBOL);
+  tunerSymbol = (ncclTuner_v3_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v3");
   if (tunerSymbol == nullptr) {
-    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find " NCCL_TUNER_PLUGIN_SYMBOL ", using internal tuner instead.");
-    dlclose(tunerPluginLib);
-    goto fail;
+    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
+    ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v2");
+    if (ncclTuner_v2 == nullptr) {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
+      dlclose(tunerPluginLib);
+      goto fail;
+    } else {
+      ncclTuner_v2_as_v3.init = ncclTuner_v2_as_v3_init;
+      ncclTuner_v2_as_v3.name = ncclTuner_v2->name;
+      tunerSymbol = &ncclTuner_v2_as_v3;
+    }
   }
 
   INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", tunerSymbol->name);
-  *tuner = tunerSymbol;
+  comm->tuner = tunerSymbol;
   ++tunerPluginRefCount;
   status = tunerPluginLoadSuccess;
+  comm->tunerPluginLoaded = 1;
 
 exit:
   pthread_mutex_unlock(&tunerPluginLock);
@@ -176,15 +226,16 @@ fail:
   goto exit;
 }
 
-ncclResult_t ncclTunerPluginUnload(ncclTuner_t** tuner) {
-  if (*tuner == nullptr) return ncclSuccess;
+ncclResult_t ncclTunerPluginUnload(struct ncclComm* comm) {
   pthread_mutex_lock(&tunerPluginLock);
-  if (0 == (--tunerPluginRefCount)) {
+  if (comm->tunerPluginLoaded && 0 == (--tunerPluginRefCount)) {
     INFO(NCCL_TUNING, "TUNER/Plugin: Closing tuner: '%s'", tunerSymbol->name);
     dlclose(tunerPluginLib);
     tunerPluginLib = nullptr;
     tunerSymbol = nullptr;
-    *tuner = nullptr;
+    comm->tuner = nullptr;
+    status = tunerPluginLoadReady;
+    comm->tunerPluginLoaded = 0;
   }
   pthread_mutex_unlock(&tunerPluginLock);
   return ncclSuccess;

projects/rccl/src/misc/utils.cc

@@ -93,7 +93,8 @@ uint64_t getHostHash(void) {
 
   if ((hostId = ncclGetEnv("NCCL_HOSTID")) != NULL) {
     INFO(NCCL_ENV, "NCCL_HOSTID set by environment to %s", hostId);
-    strncpy(hostHash, hostId, sizeof(hostHash));
+    strncpy(hostHash, hostId, sizeof(hostHash)-1);
+    hostHash[sizeof(hostHash)-1] = '\0';
   } else {
     FILE *file = fopen(HOSTID_FILE, "r");
     if (file != NULL) {
@@ -291,79 +292,3 @@ void ncclMemoryStackDestruct(struct ncclMemoryStack* me) {
     h = h1;
   }
 }
-
-const char* ncclOpToString(ncclRedOp_t op) {
-  switch (op) {
-    case ncclSum:
-      return "ncclSum";
-    case ncclProd:
-      return "ncclProd";
-    case ncclMax:
-      return "ncclMax";
-    case ncclMin:
-      return "ncclMin";
-    case ncclAvg:
-      return "ncclAvg";
-    default:
-      return "Unknown";
-  }
-}
-
-const char* ncclDatatypeToString(ncclDataType_t type) {
-  switch (type) {
-    case ncclInt8: // ncclChar
-      return "ncclInt8";
-    case ncclInt32: // ncclInt
-      return "ncclInt32";
-    case ncclUint32:
-      return "ncclUint32";
-    case ncclInt64:
-      return "ncclInt64";
-    case ncclUint64:
-      return "ncclUint64";
-    case ncclFloat16: // ncclHalf
-      return "ncclFloat16";
-    case ncclFloat32: // ncclFloat
-      return "ncclFloat32";
-    case ncclFloat64: // ncclDouble
-      return "ncclFloat64";
-#if defined(__CUDA_BF16_TYPES_EXIST__)
-    case ncclBfloat16:
-      return "ncclBfloat16";
-#endif
-    default:
-      return "Unknown";
-  }
-}
-
-const char* ncclAlgoToString(int algo) {
-  switch (algo) {
-    case NCCL_ALGO_TREE:
-      return "TREE";
-    case NCCL_ALGO_RING:
-      return "RING";
-    case NCCL_ALGO_COLLNET_DIRECT:
-      return "COLLNET_DIRECT";
-    case NCCL_ALGO_COLLNET_CHAIN:
-      return "COLLNET_CHAIN";
-    case NCCL_ALGO_NVLS:
-      return "NVLS";
-    case NCCL_ALGO_NVLS_TREE:
-      return "NVLS_TREE";
-    default:
-      return "Unknown";
-  }
-}
-
-const char* ncclProtoToString(int proto) {
-  switch (proto) {
-    case NCCL_PROTO_LL:
-      return "LL";
-    case NCCL_PROTO_LL128:
-      return "LL128";
-    case NCCL_PROTO_SIMPLE:
-      return "SIMPLE";
-    default:
-      return "Unknown";
-  }
-}