2.12.7-1

Add network communication through another GPU connected with NVLink
(PXN).
Add aggregation of messages coming from different local GPUs through
PXN and going to the same destination.
Add new v5 plugin API with grouped receives and tags.
Add compat for v4 plugins.
Add naming of NCCL threads to help debugging.
Fix NVLink detection and avoid data corruption when some NVLinks are
down.
Add support for Relaxed Ordering for IB.
Add profiling and timing infrastructure.

src/include/enqueue.h

@@ -1,5 +1,5 @@
 /*************************************************************************
- * Copyright (c) 2015-2021, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2015-2022, NVIDIA CORPORATION. All rights reserved.
  *
  * See LICENSE.txt for license information
  ************************************************************************/
@@ -31,17 +31,17 @@ ncclResult_t ncclGetCudaGraph(ncclComm_t comm, cudaGraph_t* graph);
 ncclResult_t ncclCudaGraphHostSetup(ncclComm_t comm, cudaGraph_t graph);
 
 struct ncclBuffRegInfo {
-  void* sendbuffsBase[NCCL_MAX_INTRA_RANKS];
-  void* recvbuffsBase[NCCL_MAX_INTRA_RANKS];
-  void* sendbuffs[NCCL_MAX_INTRA_RANKS];
-  void* recvbuffs[NCCL_MAX_INTRA_RANKS];
+  void* sendbuffsBase[NCCL_MAX_LOCAL_RANKS];
+  void* recvbuffsBase[NCCL_MAX_LOCAL_RANKS];
+  void* sendbuffs[NCCL_MAX_LOCAL_RANKS];
+  void* recvbuffs[NCCL_MAX_LOCAL_RANKS];
   int nBuffs;
 };
 
 // Enqueue information (for kernel and proxy) for each operation
 struct ncclQueueElem {
-  struct ncclWorkElem work;
-  struct ncclProxyArgs proxyArgs;
+  struct ncclWork work;
+  struct ncclProxyOp proxyOp;
   struct ncclBuffRegInfo buffRegInfo;
 };
 
@@ -87,7 +87,7 @@ static void ncclDestroyQueueInfo(void* ptr) {
   // but currently the destroy function of CUDA objects does not allow CUDA API calls
   while (eqElem != NULL) {
     for (int i=0; i<eqElem->buffRegInfo.nBuffs; i++) {
-      if (i == eqInfo->comm->intraNodeRank) continue;
+      if (i == eqInfo->comm->localRank) continue;
       CUDACHECKIGNORE(cudaIpcCloseMemHandle(eqElem->buffRegInfo.sendbuffsBase[i]));
       CUDACHECKIGNORE(cudaIpcCloseMemHandle(eqElem->buffRegInfo.recvbuffsBase[i]));
     }