2.11.4-1

Add new API for creating a reduction operation which multiplies the input by a rank-specific scalar before doing an inter-rank summation (see: ncclRedOpCreatePreMulSum).
Improve CollNet (SHARP) performance of ncclAllReduce when captured in a CUDA Graph via user buffer registration.
Add environment variable NCCL_NET_PLUGIN="<suffix>" to allow user to choose among multiple NCCL net plugins by substituting into "libnccl-net-<suffix>.so".
Fix memory leak of NVB connections.
Fix topology detection of IB Virtual Functions (SR-IOV).

src/misc/argcheck.cc

@@ -51,10 +51,16 @@ ncclResult_t ArgsCheck(struct ncclInfo* info) {
   }
   if (info->coll == ncclFuncAllGather || info->coll == ncclFuncReduceScatter) info->nBytes *= info->comm->nRanks; // count is per rank
 
-  if (info->op < 0 || info->op >= ncclNumOps) {
+  if (info->op < 0 || ncclMaxRedOp < info->op) {
     WARN("%s : invalid reduction operation %d", info->opName, info->op);
     return ncclInvalidArgument;
   }
+  int opIx = int(ncclUserRedOpMangle(info->comm, info->op)) - int(ncclNumOps);
+  if (ncclNumOps <= info->op &&
+      (info->comm->userRedOpCapacity <= opIx || info->comm->userRedOps[opIx].freeNext != -1)) {
+    WARN("%s : reduction operation %d unknown to this communicator", info->opName, info->op);
+    return ncclInvalidArgument;
+  }
 
   if (info->comm->checkPointers) {
     if (info->coll == ncclFuncSendRecv) {

src/misc/ibvwrap.cc

@@ -60,7 +60,7 @@ ncclResult_t wrap_ibv_symbols(void) {
   if (!ibvhandle) {
     ibvhandle=dlopen("libibverbs.so.1", RTLD_NOW);
     if (!ibvhandle) {
-      WARN("Failed to open libibverbs.so[.1]");
+      INFO(NCCL_INIT, "Failed to open libibverbs.so[.1]");
       goto teardown;
     }
   }