Merge remote-tracking branch 'nccl/master' into develop

[ROCm/rccl commit: 84081064a0]

projects/rccl/src/misc/argcheck.cc

@@ -33,6 +33,15 @@ ncclResult_t PtrCheck(void* ptr, const char* opname, const char* ptrname) {
   return ncclSuccess;
 }
 
+ncclResult_t CommCheck(struct ncclComm* comm, const char* opname, const char* ptrname) {
+  NCCLCHECK(PtrCheck(comm, opname, ptrname));
+  if (comm->startMagic != NCCL_MAGIC || comm->endMagic != NCCL_MAGIC) {
+    WARN("Error: corrupted comm object detected");
+    return ncclInvalidArgument;
+  }
+  return ncclSuccess;
+}
+
 ncclResult_t ArgsCheck(struct ncclInfo* info) {
   // First, the easy ones
   if (info->root < 0 || info->root >= info->comm->nRanks) {

projects/rccl/src/misc/cudawrap.cc

@@ -9,8 +9,6 @@
 #include "param.h"
 #include "cudawrap.h"
 
-#include <dlfcn.h>
-
 // This env var (NCCL_CUMEM_ENABLE) toggles cuMem API usage
 NCCL_PARAM(CuMemEnable, "CUMEM_ENABLE", -2);
 
@@ -51,112 +49,119 @@ int ncclCuMemEnable() {
   return  param >= 0 ? param : (param == -2 && ncclCuMemSupported);
 }
 
-#define DECLARE_CUDA_PFN(symbol,version) PFN_##symbol##_v##version pfn_##symbol = nullptr
+#define DECLARE_CUDA_PFN(symbol) PFN_##symbol pfn_##symbol = nullptr
 
 #if CUDART_VERSION >= 11030
 /* CUDA Driver functions loaded with cuGetProcAddress for versioning */
-DECLARE_CUDA_PFN(cuDeviceGet, 2000);
-DECLARE_CUDA_PFN(cuDeviceGetAttribute, 2000);
-DECLARE_CUDA_PFN(cuGetErrorString, 6000);
-DECLARE_CUDA_PFN(cuGetErrorName, 6000);
+DECLARE_CUDA_PFN(cuDeviceGet);
+DECLARE_CUDA_PFN(cuDeviceGetAttribute);
+DECLARE_CUDA_PFN(cuGetErrorString);
+DECLARE_CUDA_PFN(cuGetErrorName);
 /* enqueue.cc */
-DECLARE_CUDA_PFN(cuMemGetAddressRange, 3020);
+DECLARE_CUDA_PFN(cuMemGetAddressRange);
 /* proxy.cc */
-DECLARE_CUDA_PFN(cuCtxCreate, 3020);
-DECLARE_CUDA_PFN(cuCtxDestroy, 4000);
-DECLARE_CUDA_PFN(cuCtxGetCurrent, 4000);
-DECLARE_CUDA_PFN(cuCtxSetCurrent, 4000);
-DECLARE_CUDA_PFN(cuCtxGetDevice, 2000);
+DECLARE_CUDA_PFN(cuCtxCreate);
+DECLARE_CUDA_PFN(cuCtxDestroy);
+DECLARE_CUDA_PFN(cuCtxGetCurrent);
+DECLARE_CUDA_PFN(cuCtxSetCurrent);
+DECLARE_CUDA_PFN(cuCtxGetDevice);
 /* cuMem API support */
-DECLARE_CUDA_PFN(cuMemAddressReserve, 10020);
-DECLARE_CUDA_PFN(cuMemAddressFree, 10020);
-DECLARE_CUDA_PFN(cuMemCreate, 10020);
-DECLARE_CUDA_PFN(cuMemGetAllocationGranularity, 10020);
-DECLARE_CUDA_PFN(cuMemExportToShareableHandle, 10020);
-DECLARE_CUDA_PFN(cuMemImportFromShareableHandle, 10020);
-DECLARE_CUDA_PFN(cuMemMap, 10020);
-DECLARE_CUDA_PFN(cuMemRelease, 10020);
-DECLARE_CUDA_PFN(cuMemRetainAllocationHandle, 11000);
-DECLARE_CUDA_PFN(cuMemSetAccess, 10020);
-DECLARE_CUDA_PFN(cuMemUnmap, 10020);
+DECLARE_CUDA_PFN(cuMemAddressReserve);
+DECLARE_CUDA_PFN(cuMemAddressFree);
+DECLARE_CUDA_PFN(cuMemCreate);
+DECLARE_CUDA_PFN(cuMemGetAllocationGranularity);
+DECLARE_CUDA_PFN(cuMemExportToShareableHandle);
+DECLARE_CUDA_PFN(cuMemImportFromShareableHandle);
+DECLARE_CUDA_PFN(cuMemMap);
+DECLARE_CUDA_PFN(cuMemRelease);
+DECLARE_CUDA_PFN(cuMemRetainAllocationHandle);
+DECLARE_CUDA_PFN(cuMemSetAccess);
+DECLARE_CUDA_PFN(cuMemUnmap);
 /* ncclMemAlloc/Free */
-DECLARE_CUDA_PFN(cuPointerGetAttribute, 4000);
+DECLARE_CUDA_PFN(cuPointerGetAttribute);
 #if CUDA_VERSION >= 11070
 /* transport/collNet.cc/net.cc*/
-DECLARE_CUDA_PFN(cuMemGetHandleForAddressRange, 11070); // DMA-BUF support
+DECLARE_CUDA_PFN(cuMemGetHandleForAddressRange); // DMA-BUF support
 #endif
 #if CUDA_VERSION >= 12010
 /* NVSwitch Multicast support */
-DECLARE_CUDA_PFN(cuMulticastAddDevice, 12010);
-DECLARE_CUDA_PFN(cuMulticastBindMem, 12010);
-DECLARE_CUDA_PFN(cuMulticastBindAddr, 12010);
-DECLARE_CUDA_PFN(cuMulticastCreate, 12010);
-DECLARE_CUDA_PFN(cuMulticastGetGranularity, 12010);
-DECLARE_CUDA_PFN(cuMulticastUnbind, 12010);
+DECLARE_CUDA_PFN(cuMulticastAddDevice);
+DECLARE_CUDA_PFN(cuMulticastBindMem);
+DECLARE_CUDA_PFN(cuMulticastBindAddr);
+DECLARE_CUDA_PFN(cuMulticastCreate);
+DECLARE_CUDA_PFN(cuMulticastGetGranularity);
+DECLARE_CUDA_PFN(cuMulticastUnbind);
 #endif
 #endif
 
-/* CUDA Driver functions loaded with dlsym() */
-DECLARE_CUDA_PFN(cuInit, 2000);
-DECLARE_CUDA_PFN(cuDriverGetVersion, 2020);
-DECLARE_CUDA_PFN(cuGetProcAddress, 11030);
-
 #define CUDA_DRIVER_MIN_VERSION 11030
 
-static void *cudaLib;
 int ncclCudaDriverVersionCache = -1;
 bool ncclCudaLaunchBlocking = false;
 
 #if CUDART_VERSION >= 11030
+
+#if CUDART_VERSION >= 12000
+#define LOAD_SYM(symbol, ignore) do {                                   \
+    cudaDriverEntryPointQueryResult driverStatus;                       \
+    res = cudaGetDriverEntryPoint(#symbol, (void **) (&pfn_##symbol), cudaEnableDefault, &driverStatus); \
+    if (res != cudaSuccess || driverStatus != cudaDriverEntryPointSuccess) { \
+      if (!ignore) {                                                    \
+        WARN("Retrieve %s failed with %d status %d", #symbol, res, driverStatus); \
+        return ncclSystemError; }                                       \
+    } } while(0)
+#else
+#define LOAD_SYM(symbol, ignore) do {                                   \
+    res = cudaGetDriverEntryPoint(#symbol, (void **) (&pfn_##symbol), cudaEnableDefault); \
+    if (res != cudaSuccess) { \
+      if (!ignore) {                                                    \
+        WARN("Retrieve %s failed with %d", #symbol, res);               \
+        return ncclSystemError; }                                       \
+    } } while(0)
+#endif
+
 /*
   Load the CUDA symbols
  */
 static ncclResult_t cudaPfnFuncLoader(void) {
-  CUresult res;
 
-#define LOAD_SYM(symbol, version, ignore) do {                           \
-    res = pfn_cuGetProcAddress(#symbol, (void **) (&pfn_##symbol), version, 0); \
-    if (res != 0) {                                                     \
-      if (!ignore) {                                                    \
-        WARN("Retrieve %s version %d failed with %d", #symbol, version, res); \
-        return ncclSystemError; }                                       \
-    } } while(0)
+  cudaError_t res;
 
-  LOAD_SYM(cuGetErrorString, 6000, 0);
-  LOAD_SYM(cuGetErrorName, 6000, 0);
-  LOAD_SYM(cuDeviceGet, 2000, 0);
-  LOAD_SYM(cuDeviceGetAttribute, 2000, 0);
-  LOAD_SYM(cuMemGetAddressRange, 3020, 1);
-  LOAD_SYM(cuCtxCreate, 3020, 1);
-  LOAD_SYM(cuCtxDestroy, 4000, 1);
-  LOAD_SYM(cuCtxGetCurrent, 4000, 1);
-  LOAD_SYM(cuCtxSetCurrent, 4000, 1);
-  LOAD_SYM(cuCtxGetDevice, 2000, 1);
+  LOAD_SYM(cuGetErrorString, 0);
+  LOAD_SYM(cuGetErrorName, 0);
+  LOAD_SYM(cuDeviceGet, 0);
+  LOAD_SYM(cuDeviceGetAttribute, 0);
+  LOAD_SYM(cuMemGetAddressRange, 1);
+  LOAD_SYM(cuCtxCreate, 1);
+  LOAD_SYM(cuCtxDestroy, 1);
+  LOAD_SYM(cuCtxGetCurrent, 1);
+  LOAD_SYM(cuCtxSetCurrent, 1);
+  LOAD_SYM(cuCtxGetDevice, 1);
 /* cuMem API support */
-  LOAD_SYM(cuMemAddressReserve, 10020, 1);
-  LOAD_SYM(cuMemAddressFree, 10020, 1);
-  LOAD_SYM(cuMemCreate, 10020, 1);
-  LOAD_SYM(cuMemGetAllocationGranularity, 10020, 1);
-  LOAD_SYM(cuMemExportToShareableHandle, 10020, 1);
-  LOAD_SYM(cuMemImportFromShareableHandle, 10020, 1);
-  LOAD_SYM(cuMemMap, 10020, 1);
-  LOAD_SYM(cuMemRelease, 10020, 1);
-  LOAD_SYM(cuMemRetainAllocationHandle, 11000, 1);
-  LOAD_SYM(cuMemSetAccess, 10020, 1);
-  LOAD_SYM(cuMemUnmap, 10020, 1);
+  LOAD_SYM(cuMemAddressReserve, 1);
+  LOAD_SYM(cuMemAddressFree, 1);
+  LOAD_SYM(cuMemCreate, 1);
+  LOAD_SYM(cuMemGetAllocationGranularity, 1);
+  LOAD_SYM(cuMemExportToShareableHandle, 1);
+  LOAD_SYM(cuMemImportFromShareableHandle, 1);
+  LOAD_SYM(cuMemMap, 1);
+  LOAD_SYM(cuMemRelease, 1);
+  LOAD_SYM(cuMemRetainAllocationHandle, 1);
+  LOAD_SYM(cuMemSetAccess, 1);
+  LOAD_SYM(cuMemUnmap, 1);
 /* ncclMemAlloc/Free */
-  LOAD_SYM(cuPointerGetAttribute, 4000, 1);
+  LOAD_SYM(cuPointerGetAttribute, 1);
 #if CUDA_VERSION >= 11070
-  LOAD_SYM(cuMemGetHandleForAddressRange, 11070, 1); // DMA-BUF support
+  LOAD_SYM(cuMemGetHandleForAddressRange, 1); // DMA-BUF support
 #endif
 #if CUDA_VERSION >= 12010
 /* NVSwitch Multicast support */
-  LOAD_SYM(cuMulticastAddDevice, 12010, 1);
-  LOAD_SYM(cuMulticastBindMem, 12010, 1);
-  LOAD_SYM(cuMulticastBindAddr, 12010, 1);
-  LOAD_SYM(cuMulticastCreate, 12010, 1);
-  LOAD_SYM(cuMulticastGetGranularity, 12010, 1);
-  LOAD_SYM(cuMulticastUnbind, 12010, 1);
+  LOAD_SYM(cuMulticastAddDevice, 1);
+  LOAD_SYM(cuMulticastBindMem, 1);
+  LOAD_SYM(cuMulticastBindAddr, 1);
+  LOAD_SYM(cuMulticastCreate, 1);
+  LOAD_SYM(cuMulticastGetGranularity, 1);
+  LOAD_SYM(cuMulticastUnbind, 1);
 #endif
   return ncclSuccess;
 }
@@ -171,47 +176,12 @@ static void initOnceFunc() {
     ncclCudaLaunchBlocking = val!=nullptr && val[0]!=0 && !(val[0]=='0' && val[1]==0);
   } while (0);
 
-  CUresult res;
-  /*
-   * Load CUDA driver library
-   */
-  char path[1024];
-  const char *ncclCudaPath = ncclGetEnv("NCCL_CUDA_PATH");
-  if (ncclCudaPath == NULL)
-    snprintf(path, 1024, "%s", "libcuda.so");
-  else
-    snprintf(path, 1024, "%s/%s", ncclCudaPath, "libcuda.so");
-
-  (void) dlerror(); // Clear any previous errors
-  cudaLib = dlopen(path, RTLD_LAZY);
-  if (cudaLib == NULL) {
-    WARN("Failed to find CUDA library %s (NCCL_CUDA_PATH='%s') : %s", path, ncclCudaPath ? ncclCudaPath : "", dlerror());
-    goto error;
-  }
-
-  /*
-   * Load initial CUDA functions
-   */
-
-  pfn_cuInit = (PFN_cuInit_v2000) dlsym(cudaLib, "cuInit");
-  if (pfn_cuInit == NULL) {
-    WARN("Failed to load CUDA missing symbol cuInit");
-    goto error;
-  }
-
-  pfn_cuDriverGetVersion = (PFN_cuDriverGetVersion_v2020) dlsym(cudaLib, "cuDriverGetVersion");
-  if (pfn_cuDriverGetVersion == NULL) {
-    WARN("Failed to load CUDA missing symbol cuDriverGetVersion");
-    goto error;
-  }
-
+  ncclResult_t ret = ncclSuccess;
+  int cudaDev;
   int driverVersion;
-  res = pfn_cuDriverGetVersion(&driverVersion);
-  if (res != 0) {
-    WARN("cuDriverGetVersion failed with %d", res);
-    goto error;
-  }
+  CUDACHECKGOTO(cudaGetDevice(&cudaDev), ret, error); // Initialize the driver
 
+  CUDACHECKGOTO(cudaDriverGetVersion(&driverVersion), ret, error);
   INFO(NCCL_INIT, "cudaDriverVersion %d", driverVersion);
 
   if (driverVersion < CUDA_DRIVER_MIN_VERSION) {
@@ -220,19 +190,6 @@ static void initOnceFunc() {
     goto error;
   }
 
-  pfn_cuGetProcAddress = (PFN_cuGetProcAddress_v11030) dlsym(cudaLib, "cuGetProcAddress");
-  if (pfn_cuGetProcAddress == NULL) {
-    WARN("Failed to load CUDA missing symbol cuGetProcAddress");
-    goto error;
-  }
-
-  /*
-   * Required to initialize the CUDA Driver.
-   * Multiple calls of cuInit() will return immediately
-   * without making any relevant change
-   */
-  pfn_cuInit(0);
-
   #if CUDART_VERSION >= 11030
   if (cudaPfnFuncLoader()) {
     WARN("CUDA some PFN functions not found in the library");
@@ -243,7 +200,7 @@ static void initOnceFunc() {
   // Determine whether we support the cuMem APIs or not
   ncclCuMemSupported = ncclIsCuMemSupported();
 
-  initResult = ncclSuccess;
+  initResult = ret;
   return;
 error:
   initResult = ncclSystemError;

projects/rccl/src/misc/socket.cc

@@ -790,6 +790,24 @@ ncclResult_t ncclSocketRecv(struct ncclSocket* sock, void* ptr, int size) {
   return ncclSuccess;
 }
 
+ncclResult_t ncclSocketSendRecv(struct ncclSocket* sendSock, void* sendPtr, int sendSize, struct ncclSocket* recvSock, void* recvPtr, int recvSize) {
+  int sendOffset = 0, recvOffset = 0;
+  if (sendSock == NULL || recvSock == NULL) {
+    WARN("ncclSocketSendRecv: invalid socket %p/%p", sendSock, recvSock);
+    return ncclInternalError;
+  }
+  if (sendSock->state != ncclSocketStateReady || recvSock->state != ncclSocketStateReady) {
+    WARN("ncclSocketSendRecv: socket state (%d/%d) is not ready", sendSock->state, recvSock->state);
+    return ncclInternalError;
+  }
+  while (sendOffset < sendSize || recvOffset < recvSize) {
+    if (sendOffset < sendSize) NCCLCHECK(socketProgress(NCCL_SOCKET_SEND, sendSock, sendPtr, sendSize, &sendOffset));
+    if (recvOffset < recvSize) NCCLCHECK(socketProgress(NCCL_SOCKET_RECV, recvSock, recvPtr, recvSize, &recvOffset));
+  }
+  return ncclSuccess;
+}
+
+
 // Receive or detect connection closed
 ncclResult_t ncclSocketTryRecv(struct ncclSocket* sock, void* ptr, int size, int* closed, bool blocking) {
   int offset = 0;

projects/rccl/src/misc/tuner.cc

@@ -13,69 +13,178 @@
 #include "nccl_tuner.h"
 
 pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
-static int tunerPluginRefCount = -1;
+static int tunerPluginRefCount;
 static void* tunerPluginLib = nullptr;
 ncclTuner_t* tunerSymbol = nullptr;
 
-ncclResult_t ncclLoadTunerPlugin(ncclTuner_t** tuner) {
-  // Initialize to nullptr by default if plugin tuner cannot be loaded.
-  *tuner = nullptr;
-  if (tunerPluginRefCount == -2) return ncclSuccess;
-
-  pthread_mutex_lock(&tunerPluginLock);
-  if (tunerPluginRefCount == -1) {
-    tunerPluginRefCount = -2; // Default: no plugin, don't try again later
-
-    const char* name = getenv("NCCL_TUNER_PLUGIN");
-    if (name) {
-      INFO(NCCL_TUNING, "NCCL_TUNER_PLUGIN set to %s", name);
-      tunerPluginLib = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
-      if (tunerPluginLib == nullptr) {
-        // dlopen does not guarantee to set errno, but dlerror only gives us a
-        // string, so checking errno doesn't hurt to try to provide a better
-        // error message
-        if (errno == ENOENT) {
-          INFO(NCCL_TUNING, "Tuner: no plugin found '%s', using default tuner instead.", name);
-        } else {
-          INFO(NCCL_TUNING, "Tuner: plugin load '%s' returned error (%d : %s), using default tuner instead.", name, errno, dlerror());
-        }
-      } else {
-        tunerSymbol = (ncclTuner_t*)dlsym(tunerPluginLib, NCCL_TUNER_PLUGIN_SYMBOL);
-        if (tunerSymbol == nullptr) {
-          INFO(NCCL_TUNING, "Tuner: failed to find " NCCL_TUNER_PLUGIN_SYMBOL " in plugin (%s), using default tuner instead.", name);
-          dlclose(tunerPluginLib);
-          tunerPluginLib = nullptr;
-        } else {
-          INFO(NCCL_TUNING, "Opened tuner: '%s'", tunerSymbol->name);
-          tunerPluginRefCount = 0;
-        }
-      }
+static void* tryOpenDynamicLib(const char* name) {
+  if (nullptr == name || strlen(name) == 0) {
+    return nullptr;
+  }
+  void *handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
+  if (nullptr == handle) {
+    if (ENOENT == errno) {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: No plugin found (%s)", name);
     }
   }
-
-  if (tunerPluginRefCount >= 0) {
-    *tuner = tunerSymbol;
-    INFO(NCCL_INIT, "Using tuner plugin: '%s'", tunerSymbol->name);
-    tunerPluginRefCount++;
-  }
-  pthread_mutex_unlock(&tunerPluginLock);
-  return ncclSuccess;
+  return handle;
 }
 
-ncclResult_t ncclCloseTunerPlugin(ncclTuner_t** tuner) {
+static void summarizeOpenTunerPluginLibErrors(char* pluginNames) {
+  const char *separator = " ";
+  int len = strlen(pluginNames);
+  // remove tail separator
+  pluginNames[len - 1] = '\0';
+
+  // remove last plugin name
+  while (len > 0 && pluginNames[--len] != *separator);
+  if (len > 0) {
+    pluginNames[len] = '\0';
+  }
+
+  // distinguish between one load attempt and multiple attempts
+  if (strstr(pluginNames, separator)) {
+    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Most recent plugin load returned %d : %s. All attempts to load '%s' also failed.", errno, dlerror(), pluginNames);
+  } else {
+    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin load returned %d : %s : when loading %s", errno, dlerror(), pluginNames);
+  }
+}
+
+static void* openTunerPluginLib(void) {
+  void *pluginLib;
+
+#define MAX_PLUGIN_LOAD 4
+
+  int len;
+  char tunerPluginLibNameTried[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
+  char *ptr = tunerPluginLibNameTried;
+  char tunerPluginLibName[PATH_MAX];
+  const char *envTunerPluginName = getenv("NCCL_TUNER_PLUGIN");
+  if (envTunerPluginName && strlen(envTunerPluginName)) {
+    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: NCCL_TUNER_PLUGIN set to %s", envTunerPluginName);
+    snprintf(tunerPluginLibName, PATH_MAX, "%s", envTunerPluginName);
+    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    if (pluginLib) {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
+      return pluginLib;
+    }
+    len = PATH_MAX - strlen(ptr);
+    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+
+    snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner-%s.so", envTunerPluginName);
+    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    if (pluginLib) {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
+      return pluginLib;
+    }
+    len = PATH_MAX - strlen(ptr);
+    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+  } else {
+    snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner.so");
+    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    if (pluginLib) {
+      return pluginLib;
+    }
+    len = PATH_MAX - strlen(ptr);
+    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+  }
+
+  const char *envNetPluginName = getenv("NCCL_NET_PLUGIN");
+  if (envNetPluginName && strlen(envNetPluginName)) {
+    // Users are allowed to pack tuner into the net plugin
+    snprintf(tunerPluginLibName, PATH_MAX, "%s", envNetPluginName);
+    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    if (pluginLib) {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
+      return pluginLib;
+    }
+    len = PATH_MAX - strlen(ptr);
+    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+
+    snprintf(tunerPluginLibName, PATH_MAX, "librccl-net-%s.so", envNetPluginName);
+    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    if (pluginLib) {
+      INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
+      return pluginLib;
+    }
+    len = PATH_MAX - strlen(ptr);
+    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+  } else {
+    snprintf(tunerPluginLibName, PATH_MAX, "librccl-net.so");
+    pluginLib = tryOpenDynamicLib(tunerPluginLibName);
+    if (pluginLib) {
+      return pluginLib;
+    }
+    len = PATH_MAX - strlen(ptr);
+    snprintf(ptr + strlen(ptr), len + 1, "%s ", tunerPluginLibName);
+  }
+  summarizeOpenTunerPluginLibErrors(ptr);
+
+  tunerPluginLibName[0] = '\0';
+  return nullptr;
+}
+
+enum {
+  tunerPluginLoadFailed  = -1,
+  tunerPluginLoadReady   =  0,
+  tunerPluginLoadSuccess =  1,
+};
+
+ncclResult_t ncclTunerPluginLoad(ncclTuner_t** tuner) {
+  // Initialize to nullptr by default if plugin tuner cannot be loaded.
+  *tuner = nullptr;
+  static int status = tunerPluginLoadReady;
+  if (tunerPluginLoadFailed == status) {
+    return ncclSuccess;
+  }
+
+  pthread_mutex_lock(&tunerPluginLock);
+  if (tunerPluginLoadFailed == status) {
+    goto exit;
+  }
+
+  if (tunerPluginLoadSuccess == status) {
+    *tuner = tunerSymbol;
+    ++tunerPluginRefCount;
+    goto exit;
+  }
+
+  tunerPluginLib = openTunerPluginLib();
+  if (nullptr == tunerPluginLib) {
+    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using internal tuner plugin.");
+    goto fail;
+  }
+
+  tunerSymbol = (ncclTuner_t*)dlsym(tunerPluginLib, NCCL_TUNER_PLUGIN_SYMBOL);
+  if (tunerSymbol == nullptr) {
+    INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find " NCCL_TUNER_PLUGIN_SYMBOL ", using internal tuner instead.");
+    dlclose(tunerPluginLib);
+    goto fail;
+  }
+
+  INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", tunerSymbol->name);
+  *tuner = tunerSymbol;
+  ++tunerPluginRefCount;
+  status = tunerPluginLoadSuccess;
+
+exit:
+  pthread_mutex_unlock(&tunerPluginLock);
+  return ncclSuccess;
+fail:
+  tunerPluginLib = nullptr;
+  status = tunerPluginLoadFailed;
+  goto exit;
+}
+
+ncclResult_t ncclTunerPluginUnload(ncclTuner_t** tuner) {
   if (*tuner == nullptr) return ncclSuccess;
   pthread_mutex_lock(&tunerPluginLock);
-  if (--tunerPluginRefCount == 0) {
-    if (tunerPluginLib == nullptr) {
-      WARN("Tuner plugin refcount is 0, yet tunerPluginLib ptr is NULL\n");
-    } else {
-      INFO(NCCL_TUNING, "Closing tuner: '%s'", tunerSymbol->name);
-      dlclose(tunerPluginLib);
-    }
+  if (0 == (--tunerPluginRefCount)) {
+    INFO(NCCL_TUNING, "TUNER/Plugin: Closing tuner: '%s'", tunerSymbol->name);
+    dlclose(tunerPluginLib);
     tunerPluginLib = nullptr;
     tunerSymbol = nullptr;
     *tuner = nullptr;
-    tunerPluginRefCount = -1;
   }
   pthread_mutex_unlock(&tunerPluginLock);
   return ncclSuccess;