diff --git a/projects/rccl/src/transport.cc b/projects/rccl/src/transport.cc index d7eadcdd87..2cb5538a69 100644 --- a/projects/rccl/src/transport.cc +++ b/projects/rccl/src/transport.cc @@ -36,8 +36,8 @@ static ncclResult_t selectTransport(struct ncclComm* comm, struct ncclTopoGraph* return ncclSuccess; } } - WARN("No transport found !"); - return ncclInternalError; + WARN("No transport found for rank %d[%lx] -> rank %d[%lx]", myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId); + return ncclSystemError; } ncclResult_t ncclTransportP2pConnect(struct ncclComm* comm, struct ncclChannel* channel, int nrecv, int* peerRecv, int nsend, int* peerSend, int connIndex) { diff --git a/projects/rccl/src/transport/net.cc b/projects/rccl/src/transport/net.cc index 2b548cee39..5abc32d15b 100644 --- a/projects/rccl/src/transport/net.cc +++ b/projects/rccl/src/transport/net.cc @@ -56,8 +56,18 @@ struct netRecvResources { uint64_t llLastCleaning; }; +NCCL_PARAM(NetDisableIntra, "NET_DISABLE_INTRA", -2); + /* Determine if two peers can communicate with NET */ ncclResult_t netCanConnect(int* ret, struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2) { + // Same host? + if (info1->hostHash == info2->hostHash) { + // User disabled NET for intra-node? + if (ncclParamNetDisableIntra() == 1) { + *ret = 0; + return ncclSuccess; + } + } *ret = 1; return ncclSuccess; }