Add env NCCL_NET_DISABLE_INTRA

Disable NET transport for intra-node communication by setting the env to 1
It provides an option to error out instead of falling back to NET when superior intra-node transports (P2P and SHM) are unavailable


[ROCm/rccl commit: c88c9f873f]

projects/rccl/src/transport.cc

@@ -36,8 +36,8 @@ static ncclResult_t selectTransport(struct ncclComm* comm, struct ncclTopoGraph*
       return ncclSuccess;
     }
   }
-  WARN("No transport found !");
-  return ncclInternalError;
+  WARN("No transport found for rank %d[%lx] -> rank %d[%lx]", myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId);
+  return ncclSystemError;
 }
 
 ncclResult_t ncclTransportP2pConnect(struct ncclComm* comm, struct ncclChannel* channel, int nrecv, int* peerRecv, int nsend, int* peerSend, int connIndex) {

projects/rccl/src/transport/net.cc

@@ -56,8 +56,18 @@ struct netRecvResources {
   uint64_t llLastCleaning;
 };
 
+NCCL_PARAM(NetDisableIntra, "NET_DISABLE_INTRA", -2);
+
 /* Determine if two peers can communicate with NET */
 ncclResult_t netCanConnect(int* ret, struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2) {
+  // Same host?
+  if (info1->hostHash == info2->hostHash) {
+    // User disabled NET for intra-node?
+    if (ncclParamNetDisableIntra() == 1) {
+      *ret = 0;
+      return ncclSuccess;
+    }
+  }
   *ret = 1;
   return ncclSuccess;
 }