2.13.4-1

Optimize CUDA graph launch; avoid launching a CPU callback for
intra-node operations.
Simplify kernel common code to improve the latency of send/recv
operations.
Strengthen CUDA streams semantics.
Change NET API to v6, to add dmabuf support.
Add ncclGetLastError() function.
Add ncclRemoteError code and use it for remote network errors.
Support the use of a different NCCL_NET parameter per communicator.
Add support for SHM and P2P transfers using cudaMemcpy.

src/bootstrap.cc

@@ -105,6 +105,7 @@ static void *bootstrapRoot(void* args) {
   do {
     struct ncclSocket sock;
     sock.abortFlag = NULL;
+    /* bootstrap root thread always uses blocking ncclSocketAccept. */
     NCCLCHECKGOTO(ncclSocketAccept(&sock, listenSock), res, out);
     NCCLCHECKGOTO(bootstrapNetRecv(&sock, &info, sizeof(info)), res, out);
     close(sock.fd);
@@ -139,6 +140,7 @@ static void *bootstrapRoot(void* args) {
     int next = (r+1) % nranks;
     struct ncclSocket sock;
     sock.abortFlag = NULL;
+    sock.asyncFlag = 0;
     memcpy(&sock.addr, rankAddressesRoot+r, sizeof(union ncclSocketAddress));
     NCCLCHECKGOTO(ncclSocketConnect(&sock), res, out);
     NCCLCHECKGOTO(bootstrapNetSend(&sock, rankAddresses+next, sizeof(union ncclSocketAddress)), res, out);
@@ -316,6 +318,7 @@ ncclResult_t bootstrapSend(void* commState, int peer, int tag, void* data, int s
   struct bootstrapState* state = (struct bootstrapState*)commState;
   struct ncclSocket sock;
   sock.abortFlag = state->abortFlag;
+  sock.asyncFlag = 0;
   memcpy(&sock.addr, state->peerCommAddresses+peer, sizeof(union ncclSocketAddress));
   NCCLCHECK(ncclSocketConnect(&sock));
   NCCLCHECK(bootstrapNetSend(&sock, &state->rank, sizeof(int)));