2.21.5-1
Add support for IB SHARP 1PPN operation with user buffers. Improve support for MNNVL, add NVLS support and multi-clique support. * Detect the NVLS clique through NVML * Exchange XML between peers in the same NVLS clique and fuse XMLs before creating the topology graph. * Rework bootstrap allgather algorithms to allow for large allgather operations intra-node (XML exchange). Net/IB: add support for dynamic GID detection. * Automatically select RoCEv2/IPv4 interface by default. Allow to select IPv6 or even the network/mask. Reduce NVLS memory usage. * Add stepSize as property of a connection to allow for different sizes on different peers; set it to 128K for NVLink SHARP. Improve tuner loading * Look for more paths, be more consistent with the network device plugin. * Also search for tuner support inside the net plugin. Improve tuner API * Add context to support multi-device per process. Add magic number around comm object to detect comm corruption. * Add some basic check around communicators so that we can report a problem when a communicator gets corrupted or a wrong comm pointer is passed to NCCL. Fix net/IB error path. Github PR #1164 Fix collnet rail mapping with split comm. Fix packet reordering issue causing bootstrap mismatch * Use a different tag in ncclTransportP2pSetup for the connectInfo exchange and the following barrier. Fix hang when crossNic is inconsistent between ranks. Fix minCompCap/maxCompCap computation. Github issue #1184
This commit is contained in:
Sylvain Jeaugey committed
1 parent
6dd51f15bf
commit
ab2b89c4c3
52 files changed
+2124
-944
No files matched your search
+148
-103
@@ -80,6 +80,16 @@ static ncclResult_t bootstrapNetRecv(struct ncclSocket* sock, void* data, int si
|
||||
NCCLCHECK(ncclSocketRecv(sock, data, std::min(recvSize, size)));
|
||||
return ncclSuccess;
|
||||
}
|
||||
static ncclResult_t bootstrapNetSendRecv(struct ncclSocket* sendSock, void* sendData, int sendSize, struct ncclSocket* recvSock, void* recvData, int recvSize) {
|
||||
int senderRecvSize;
|
||||
NCCLCHECK(ncclSocketSendRecv(sendSock, &sendSize, sizeof(int), recvSock, &senderRecvSize, sizeof(int)));
|
||||
if (senderRecvSize > recvSize) {
|
||||
WARN("Message truncated : received %d bytes instead of %d", senderRecvSize, recvSize);
|
||||
return ncclInternalError;
|
||||
}
|
||||
NCCLCHECK(ncclSocketSendRecv(sendSock, sendData, sendSize, recvSock, recvData, recvSize));
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
struct extInfo {
|
||||
int rank;
|
||||
@@ -390,103 +400,40 @@ fail:
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapAllGather(void* commState, void* allData, int size) {
|
||||
// Bootstrap send/receive functions
|
||||
//
|
||||
// We do not keep connections opened with all ranks at all times, and we have no guarantee
|
||||
// that connections to our unique listen socket will arrive in the same order as we need
|
||||
// them. Therefore, when establishing a connection, the sender sends a (peer, tag) tuple to
|
||||
// allow the receiver to identify the flow, and keep it in an unexpected queue if needed.
|
||||
|
||||
ncclResult_t bootstrapConnect(void* commState, int peer, int tag, struct ncclSocket* sock) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
struct bootstrapState* state = (struct bootstrapState*)commState;
|
||||
char* data = (char*)allData;
|
||||
int rank = state->rank;
|
||||
int nranks = state->nranks;
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d", rank, nranks, size);
|
||||
|
||||
/* Simple ring based AllGather
|
||||
* At each step i receive data from (rank-i-1) from left
|
||||
* and send previous step's data from (rank-i) to right
|
||||
*/
|
||||
for (int i=0; i<nranks-1; i++) {
|
||||
size_t rslice = (rank - i - 1 + nranks) % nranks;
|
||||
size_t sslice = (rank - i + nranks) % nranks;
|
||||
|
||||
// Send slice to the right
|
||||
NCCLCHECK(bootstrapNetSend(&state->ringSendSocket, data+sslice*size, size));
|
||||
// Recv slice from the left
|
||||
NCCLCHECK(bootstrapNetRecv(&state->ringRecvSocket, data+rslice*size, size));
|
||||
}
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d - DONE", rank, nranks, size);
|
||||
NCCLCHECKGOTO(ncclSocketInit(sock, state->peerCommAddresses+peer, state->magic, ncclSocketTypeBootstrap), ret, fail);
|
||||
NCCLCHECKGOTO(ncclSocketConnect(sock), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetSend(sock, &state->rank, sizeof(int)), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetSend(sock, &tag, sizeof(int)), ret, fail);
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
NCCLCHECK(ncclSocketClose(sock));
|
||||
return ret;
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapSend(void* commState, int peer, int tag, void* data, int size) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
struct bootstrapState* state = (struct bootstrapState*)commState;
|
||||
struct ncclSocket sock;
|
||||
|
||||
NCCLCHECKGOTO(ncclSocketInit(&sock, state->peerCommAddresses+peer, state->magic, ncclSocketTypeBootstrap), ret, fail);
|
||||
NCCLCHECKGOTO(ncclSocketConnect(&sock), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetSend(&sock, &state->rank, sizeof(int)), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetSend(&sock, &tag, sizeof(int)), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetSend(&sock, data, size), ret, fail);
|
||||
TRACE(NCCL_BOOTSTRAP, "Sending to peer=%d tag=%d size=%d", peer, tag, size);
|
||||
NCCLCHECK(bootstrapConnect(commState, peer, tag, &sock));
|
||||
NCCLCHECKGOTO(bootstrapNetSend(&sock, data, size), ret, exit);
|
||||
|
||||
TRACE(NCCL_BOOTSTRAP, "Sent to peer=%d tag=%d size=%d", peer, tag, size);
|
||||
|
||||
exit:
|
||||
NCCLCHECK(ncclSocketClose(&sock));
|
||||
return ret;
|
||||
fail:
|
||||
goto exit;
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapBarrier(void* commState, int *ranks, int rank, int nranks, int tag) {
|
||||
if (nranks == 1) return ncclSuccess;
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d tag %x - ENTER", rank, nranks, tag);
|
||||
|
||||
/* Simple intra process barrier
|
||||
*
|
||||
* Based on the dissemination algorithm by Debra Hensgen, Raphael Finkel, and Udi Manbet,
|
||||
* "Two Algorithms for Barrier Synchronization," International Journal of Parallel Programming, 17(1):1-17, 1988"
|
||||
*/
|
||||
int data[1];
|
||||
for (int mask=1; mask<nranks; mask<<=1) {
|
||||
int src = (rank - mask + nranks) % nranks;
|
||||
int dst = (rank + mask) % nranks;
|
||||
NCCLCHECK(bootstrapSend(commState, ranks[dst], tag, data, sizeof(data)));
|
||||
NCCLCHECK(bootstrapRecv(commState, ranks[src], tag, data, sizeof(data)));
|
||||
}
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d tag %x - DONE", rank, nranks, tag);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapIntraNodeAllGather(void* commState, int *ranks, int rank, int nranks, void* allData, int size) {
|
||||
if (nranks == 1) return ncclSuccess;
|
||||
char* data = (char*)allData;
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d - ENTER", rank, nranks, size);
|
||||
|
||||
for (int i=1; i<nranks; i++) {
|
||||
int src = (rank - i + nranks) % nranks;
|
||||
int dst = (rank + i) % nranks;
|
||||
NCCLCHECK(bootstrapSend(commState, ranks[dst], /*tag=*/i, data+rank*size, size));
|
||||
NCCLCHECK(bootstrapRecv(commState, ranks[src], /*tag=*/i, data+src*size, size));
|
||||
}
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d - DONE", rank, nranks, size);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// IntraNode in-place Broadcast
|
||||
ncclResult_t bootstrapIntraNodeBroadcast(void* commState, int *ranks, int rank, int nranks, int root, void* bcastData, int size) {
|
||||
if (nranks == 1) return ncclSuccess;
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d root %d size %d - ENTER", rank, nranks, root, size);
|
||||
|
||||
if (rank == root) {
|
||||
for (int i=0; i<nranks; i++) {
|
||||
if (i != root) NCCLCHECK(bootstrapSend(commState, ranks[i], /*tag=*/ranks[i], bcastData, size));
|
||||
}
|
||||
}
|
||||
else {
|
||||
NCCLCHECK(bootstrapRecv(commState, ranks[root], /*tag=*/ranks[rank], bcastData, size));
|
||||
}
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d root %d size %d - DONE", rank, nranks, root, size);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t unexpectedEnqueue(struct bootstrapState* state, int peer, int tag, struct ncclSocket* sock) {
|
||||
@@ -543,38 +490,136 @@ static void unexpectedFree(struct bootstrapState* state) {
|
||||
}
|
||||
|
||||
// We can't know who we'll receive from, so we need to receive everything at once
|
||||
ncclResult_t bootstrapRecv(void* commState, int peer, int tag, void* data, int size) {
|
||||
ncclResult_t bootstrapAccept(void* commState, int peer, int tag, struct ncclSocket* sock) {
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
struct bootstrapState* state = (struct bootstrapState*)commState;
|
||||
struct ncclSocket sock;
|
||||
int newPeer, newTag;
|
||||
|
||||
// Search unexpected connections first
|
||||
int found;
|
||||
NCCLCHECK(unexpectedDequeue(state, peer, tag, &sock, &found));
|
||||
if (found) {
|
||||
NCCLCHECKGOTO(bootstrapNetRecv(&sock, ((char*)data), size), ret, fail);
|
||||
goto exit;
|
||||
}
|
||||
NCCLCHECK(unexpectedDequeue(state, peer, tag, sock, &found));
|
||||
if (found) return ncclSuccess;
|
||||
|
||||
// Then look for new connections
|
||||
while (1) {
|
||||
NCCLCHECKGOTO(ncclSocketInit(&sock), ret, fail);
|
||||
NCCLCHECKGOTO(ncclSocketAccept(&sock, &state->listenSock), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetRecv(&sock, &newPeer, sizeof(int)), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetRecv(&sock, &newTag, sizeof(int)), ret, fail);
|
||||
if (newPeer == peer && newTag == tag) {
|
||||
NCCLCHECKGOTO(bootstrapNetRecv(&sock, ((char*)data), size), ret, fail);
|
||||
goto exit;
|
||||
}
|
||||
// Unexpected connection. Save for later.
|
||||
NCCLCHECKGOTO(unexpectedEnqueue(state, newPeer, newTag, &sock), ret, fail);
|
||||
NCCLCHECKGOTO(ncclSocketInit(sock), ret, fail);
|
||||
NCCLCHECKGOTO(ncclSocketAccept(sock, &state->listenSock), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetRecv(sock, &newPeer, sizeof(int)), ret, fail);
|
||||
NCCLCHECKGOTO(bootstrapNetRecv(sock, &newTag, sizeof(int)), ret, fail);
|
||||
if (newPeer == peer && newTag == tag) return ncclSuccess;
|
||||
NCCLCHECKGOTO(unexpectedEnqueue(state, newPeer, newTag, sock), ret, fail);
|
||||
}
|
||||
return ncclSuccess;
|
||||
fail:
|
||||
NCCLCHECK(ncclSocketClose(sock));
|
||||
return ret;
|
||||
}
|
||||
|
||||
// We can't know who we'll receive from, so we need to receive everything at once
|
||||
ncclResult_t bootstrapRecv(void* commState, int peer, int tag, void* data, int size) {
|
||||
ncclResult_t ret;
|
||||
struct ncclSocket sock;
|
||||
NCCLCHECK(bootstrapAccept(commState, peer, tag, &sock));
|
||||
TRACE(NCCL_BOOTSTRAP, "Receiving tag=%d peer=%d size=%d", tag, peer, size);
|
||||
NCCLCHECKGOTO(bootstrapNetRecv(&sock, ((char*)data), size), ret, exit);
|
||||
exit:
|
||||
NCCLCHECK(ncclSocketClose(&sock));
|
||||
return ret;
|
||||
fail:
|
||||
goto exit;
|
||||
}
|
||||
|
||||
// Collective algorithms, based on bootstrapSend/Recv, and sometimes bootstrapConnect/Accept
|
||||
|
||||
ncclResult_t bootstrapRingAllGather(struct ncclSocket* prevSocket, struct ncclSocket* nextSocket, int rank, int nranks, char* data, int size) {
|
||||
/* Simple ring based AllGather
|
||||
* At each step i receive data from (rank-i-1) from prev
|
||||
* and send previous step's data from (rank-i) to next
|
||||
*/
|
||||
for (int i=0; i<nranks-1; i++) {
|
||||
size_t rslice = (rank - i - 1 + nranks) % nranks;
|
||||
size_t sslice = (rank - i + nranks) % nranks;
|
||||
|
||||
// Send slice to the right, recv slice from the left
|
||||
NCCLCHECK(bootstrapNetSendRecv(nextSocket, data+sslice*size, size, prevSocket, data+rslice*size, size));
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
ncclResult_t bootstrapAllGather(void* commState, void* allData, int size) {
|
||||
struct bootstrapState* state = (struct bootstrapState*)commState;
|
||||
int rank = state->rank;
|
||||
int nranks = state->nranks;
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d", rank, nranks, size);
|
||||
|
||||
NCCLCHECK(bootstrapRingAllGather(&state->ringRecvSocket, &state->ringSendSocket, rank, nranks, (char*)allData, size));
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d - DONE", rank, nranks, size);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapIntraNodeBarrier(void* commState, int *ranks, int rank, int nranks, int tag) {
|
||||
if (nranks == 1) return ncclSuccess;
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d tag %x - ENTER", rank, nranks, tag);
|
||||
|
||||
/* Simple [intra] process barrier
|
||||
*
|
||||
* Based on the dissemination algorithm by Debra Hensgen, Raphael Finkel, and Udi Manbet,
|
||||
* "Two Algorithms for Barrier Synchronization," International Journal of Parallel Programming, 17(1):1-17, 1988"
|
||||
*/
|
||||
int data[1];
|
||||
for (int mask=1; mask<nranks; mask<<=1) {
|
||||
int src = (rank - mask + nranks) % nranks;
|
||||
int dst = (rank + mask) % nranks;
|
||||
NCCLCHECK(bootstrapSend(commState, ranks ? ranks[dst] : dst, tag, data, sizeof(data)));
|
||||
NCCLCHECK(bootstrapRecv(commState, ranks ? ranks[src] : src, tag, data, sizeof(data)));
|
||||
}
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d tag %x - DONE", rank, nranks, tag);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapBarrier(void* commState, int rank, int nranks, int tag) {
|
||||
return bootstrapIntraNodeBarrier(commState, NULL, rank, nranks, tag);
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapIntraNodeAllGather(void* commState, int *ranks, int rank, int nranks, void* allData, int size) {
|
||||
if (nranks == 1) return ncclSuccess;
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d - ENTER", rank, nranks, size);
|
||||
|
||||
int prevRank = ranks[(rank - 1 + nranks)%nranks];
|
||||
int nextRank = ranks[(rank + 1) % nranks];
|
||||
struct ncclSocket prevSocket, nextSocket;
|
||||
NCCLCHECK(bootstrapConnect(commState, nextRank, 0, &nextSocket));
|
||||
NCCLCHECK(bootstrapAccept(commState, prevRank, 0, &prevSocket));
|
||||
|
||||
NCCLCHECK(bootstrapRingAllGather(&prevSocket, &nextSocket, rank, nranks, (char*)allData, size));
|
||||
|
||||
NCCLCHECK(ncclSocketClose(&nextSocket));
|
||||
NCCLCHECK(ncclSocketClose(&prevSocket));
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d size %d - DONE", rank, nranks, size);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// [IntraNode] in-place Broadcast
|
||||
ncclResult_t bootstrapIntraNodeBroadcast(void* commState, int *ranks, int rank, int nranks, int root, void* bcastData, int size) {
|
||||
if (nranks == 1) return ncclSuccess;
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d root %d size %d - ENTER", rank, nranks, root, size);
|
||||
|
||||
if (rank == root) {
|
||||
for (int i=0; i<nranks; i++) {
|
||||
if (i != root) NCCLCHECK(bootstrapSend(commState, ranks ? ranks[i] : i, /*tag=*/ranks ? ranks[i] : i, bcastData, size));
|
||||
}
|
||||
}
|
||||
else {
|
||||
NCCLCHECK(bootstrapRecv(commState, ranks ? ranks[root] : root, /*tag=*/ranks ? ranks[rank] : rank, bcastData, size));
|
||||
}
|
||||
|
||||
TRACE(NCCL_INIT, "rank %d nranks %d root %d size %d - DONE", rank, nranks, root, size);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapBroadcast(void* commState, int rank, int nranks, int root, void* bcastData, int size) {
|
||||
return bootstrapIntraNodeBroadcast(commState, NULL, rank, nranks, root, bcastData, size);
|
||||
}
|
||||
|
||||
ncclResult_t bootstrapClose(void* commState) {
|
||||
|
||||
Reference in new issue
Block a user