Merge remote-tracking branch 'nccl/master' into develop
Tento commit je obsažen v:
BertanDogancay
@@ -8,6 +8,7 @@
|
||||
#define NCCL_BITOPS_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
#if !__NVCC__
|
||||
#ifndef __host__
|
||||
@@ -276,13 +277,53 @@ inline __host__ __device__ uint32_t u32fp8Decode(uint8_t x) {
|
||||
return u32fpDecode(x, 3);
|
||||
}
|
||||
|
||||
inline __host__ __device__ uint64_t getHash(const char* string, int n) {
|
||||
// Based on DJB2a, result = result * 33 ^ char
|
||||
uint64_t result = 5381;
|
||||
for (int c = 0; c < n; c++) {
|
||||
result = ((result << 5) + result) ^ string[c];
|
||||
// The hash isn't just a function of the bytes but also where the bytes are split
|
||||
// into different calls to eatHash().
|
||||
inline __host__ __device__ void eatHash(uint64_t acc[2], const void* bytes, size_t size) {
|
||||
char const* ptr = (char const*)bytes;
|
||||
acc[0] ^= size;
|
||||
while (size != 0) {
|
||||
// Mix the accumulator bits.
|
||||
acc[0] += acc[1];
|
||||
acc[1] ^= acc[0];
|
||||
acc[0] ^= acc[0] >> 31;
|
||||
acc[0] *= 0x9de62bbc8cef3ce3;
|
||||
acc[1] ^= acc[1] >> 32;
|
||||
acc[1] *= 0x485cd6311b599e79;
|
||||
// Read in a chunk of input.
|
||||
size_t chunkSize = size < sizeof(uint64_t) ? size : sizeof(uint64_t);
|
||||
uint64_t x = 0;
|
||||
memcpy(&x, ptr, chunkSize);
|
||||
ptr += chunkSize;
|
||||
size -= chunkSize;
|
||||
// Add to accumulator.
|
||||
acc[0] += x;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
inline __host__ __device__ void eatHash(uint64_t acc[2], const T* bytes) {
|
||||
eatHash(acc, (const void*)bytes, sizeof(T));
|
||||
}
|
||||
|
||||
inline __host__ __device__ uint64_t digestHash(uint64_t const acc[2]) {
|
||||
uint64_t h = acc[0];
|
||||
h ^= h >> 31;
|
||||
h *= 0xbac3bd562846de6b;
|
||||
h += acc[1];
|
||||
h ^= h >> 32;
|
||||
h *= 0x995a187a14e7b445;
|
||||
return h;
|
||||
}
|
||||
|
||||
inline __host__ __device__ uint64_t getHash(const void* bytes, size_t size) {
|
||||
uint64_t acc[2] = {1, 1};
|
||||
eatHash(acc, bytes, size);
|
||||
return digestHash(acc);
|
||||
}
|
||||
template<typename T>
|
||||
inline __host__ __device__ uint64_t getHash(const T* bytes) {
|
||||
return getHash((const void*)bytes, sizeof(T));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
+239
-211
@@ -12,6 +12,7 @@
|
||||
#include "nccl.h"
|
||||
#include "nccl_common.h"
|
||||
#include "device.h"
|
||||
|
||||
#define NCCL_MAX_NET_SIZE (1024*1024*1024L) // Rather than send INT_MAX which is 2G-1, send a power of two.
|
||||
|
||||
// CHUNKSIZE must be a multiple of SLICESIZE
|
||||
@@ -396,6 +397,42 @@ public:
|
||||
~RingBCAlgorithm() {}
|
||||
};
|
||||
|
||||
#if !defined (__CUDA_ARCH__) || __CUDA_ARCH__ >= 600
|
||||
// #include <cuda/atomic>
|
||||
#endif
|
||||
|
||||
// Need a power of two to ensure it divides by parallelFactor (which is also a power of two)
|
||||
#define NCCL_PAT_NWORKERS 512
|
||||
|
||||
static constexpr int PatUsed = 0x1,
|
||||
PatSkipped = 0x2;
|
||||
|
||||
struct ncclPatStep {
|
||||
int recvDim, sendDim, recvOffset, sendOffset, stepOffset, postRecv, postSend, nelem, last, flags;
|
||||
size_t inpIx, outIx;
|
||||
};
|
||||
|
||||
struct ncclPatPeer {
|
||||
uint64_t step;
|
||||
struct ncclConnInfo* conn;
|
||||
struct ncclConnFifo* connFifo;
|
||||
void* buff;
|
||||
uint64_t *headPtr;
|
||||
uint64_t *tailPtr;
|
||||
uint64_t stepCache;
|
||||
long long int accSize;
|
||||
int connStepSize;
|
||||
};
|
||||
|
||||
#define NCCL_SHMEM_PAT_STEPS 32
|
||||
struct ncclPatShmem {
|
||||
struct ncclPatStep patSteps[NCCL_SHMEM_PAT_STEPS];
|
||||
int parallelFactor;
|
||||
long long int localAccSize;
|
||||
struct ncclPatPeer sendDims[32]; // Should cover 2^32 ranks
|
||||
struct ncclPatPeer recvDims[32];
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
class PatRSAlgorithm{
|
||||
size_t offset;
|
||||
@@ -408,18 +445,17 @@ class PatRSAlgorithm{
|
||||
int nrPow2;
|
||||
int postFreq;
|
||||
int lastA;
|
||||
|
||||
int parallelFactor;
|
||||
int aggFactor;
|
||||
int as; // aggregated steps
|
||||
int a; // step inside aggregated step
|
||||
int sendSkipped; // number of skipped steps during aggregation
|
||||
int recvSkipped; // number of skipped steps during aggregation
|
||||
int phase2recv; // receive offset for phase 2
|
||||
int stepOffset;
|
||||
int aggDelta;
|
||||
int scale;
|
||||
int phase;
|
||||
|
||||
__device__ __host__ int min(int a, int b) {
|
||||
__device__ __host__ ssize_t min(ssize_t a, ssize_t b) {
|
||||
return (a<b)?a:b;
|
||||
}
|
||||
|
||||
@@ -447,16 +483,16 @@ class PatRSAlgorithm{
|
||||
|
||||
__device__ __host__ void resetA() {
|
||||
a = 0;
|
||||
sendSkipped = recvSkipped = 0;
|
||||
sendSkipped = stepOffset = 0;
|
||||
lastA = aggFactor;
|
||||
if (phase >= 2) lastA /= 2*scale;
|
||||
if (phase == 4) lastA = 1;
|
||||
}
|
||||
|
||||
__device__ __host__ void reset() {
|
||||
nelem = getNelem();
|
||||
phase = 0;
|
||||
scale = 1;
|
||||
phase2recv = 0;
|
||||
as = aggDelta - 1;
|
||||
resetA();
|
||||
}
|
||||
@@ -479,8 +515,9 @@ class PatRSAlgorithm{
|
||||
}
|
||||
|
||||
public:
|
||||
__device__ __host__ PatRSAlgorithm(int stepSize, int stepDepth, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
|
||||
__device__ __host__ PatRSAlgorithm(int stepSize, int stepDepth, int maxParallelFactor, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
|
||||
offset(offset), end(end), count(count), chunkCount(chunkCount), rank(rank), nranks(nranks) {
|
||||
parallelFactor = maxParallelFactor;
|
||||
aggDelta = nrPow2 = (1<<log2Up(nranks));
|
||||
|
||||
aggFactor = 1;
|
||||
@@ -490,6 +527,7 @@ public:
|
||||
aggDelta /= 2;
|
||||
}
|
||||
postFreq = aggFactor;
|
||||
if (postFreq < parallelFactor) parallelFactor = postFreq;
|
||||
int d = stepDepth;
|
||||
while (d > 1 && aggFactor < nranks/2) {
|
||||
d /= 2;
|
||||
@@ -500,160 +538,151 @@ public:
|
||||
reset();
|
||||
}
|
||||
|
||||
__device__ __host__ void getNextOp(int &recvDim, int &sendDim, size_t &inpIx, size_t &outIx, int &recvOffset, int &sendOffset, int &sendStepOffset, int &nelemOut, int &postRecv, int &postSend, int &last) {
|
||||
restart:
|
||||
last = 0;
|
||||
nelemOut = nelem;
|
||||
outIx = offset;
|
||||
__device__ __host__ int getParallelFactor() {
|
||||
return parallelFactor;
|
||||
}
|
||||
|
||||
__device__ __host__ void getNextOp(struct ncclPatStep* ps) {
|
||||
ps->last = 0;
|
||||
ps->nelem = nelem;
|
||||
ps->outIx = offset;
|
||||
ps->stepOffset = stepOffset;
|
||||
int skip = 0;
|
||||
//printf("Phase %d as %d/%d a %d/%d scale %d\n", phase, as, aggDelta, a, lastA, scale);
|
||||
if (phase == 0) {
|
||||
if (a >= lastA) {
|
||||
skip = 1;
|
||||
} else if (phase == 0) {
|
||||
int s = mirrorInvert(a, lastA)*aggDelta + as;
|
||||
if (s >= nranks) skip = 1;
|
||||
int sendDataRank = (rank + s) % nranks;
|
||||
inpIx = sendDataRank * count + offset;
|
||||
recvDim = -1;
|
||||
sendDim = 0;
|
||||
outIx = 0;
|
||||
recvOffset = -1;
|
||||
sendOffset = ((a - sendSkipped)%postFreq) * nelem;
|
||||
sendStepOffset = 0;
|
||||
if ((((a - sendSkipped)%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
|
||||
postSend = 1;
|
||||
ps->inpIx = sendDataRank * count + offset;
|
||||
ps->recvDim = -1;
|
||||
ps->sendDim = 0;
|
||||
ps->outIx = 0;
|
||||
ps->recvOffset = -1;
|
||||
ps->sendOffset = (a%postFreq) * nelem;
|
||||
if (((a%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
|
||||
ps->postSend = 1;
|
||||
} else {
|
||||
postSend = 0;
|
||||
ps->postSend = 0;
|
||||
}
|
||||
postRecv = 0;
|
||||
if (skip) sendSkipped++;
|
||||
if (++a == lastA) {
|
||||
phase = as == 1 ? (aggFactor > 1 ? 2 : 4) : 1; // If as == 1, switch to phase 2
|
||||
resetA();
|
||||
}
|
||||
if (skip == 0) return;
|
||||
ps->postRecv = 0;
|
||||
} else if (phase == 1) {
|
||||
int s = mirrorInvert(a, lastA)*aggDelta + as;
|
||||
if (s >= nranks) skip = 1;
|
||||
recvDim = firstBitSet(s, nrPow2);
|
||||
sendOffset = ((a - sendSkipped)%postFreq)*nelem;
|
||||
recvOffset = ((a - recvSkipped)%postFreq)*nelem;
|
||||
postSend = 0;
|
||||
if (recvDim == 0) {
|
||||
if ((((a - sendSkipped)%postFreq) + 1 >= postFreq) || (a == lastA-1)) postSend = 1;
|
||||
sendStepOffset = 0;
|
||||
ps->recvDim = firstBitSet(s, nrPow2);
|
||||
ps->sendOffset = (a%postFreq)*nelem;
|
||||
ps->recvOffset = (a%postFreq)*nelem;
|
||||
ps->postSend = 0;
|
||||
if (ps->recvDim == 0 && (((a%postFreq) + 1 >= postFreq) || (a == lastA-1))) ps->postSend = 1;
|
||||
if (((a%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
|
||||
ps->postRecv = 1;
|
||||
} else {
|
||||
sendStepOffset = (a - sendSkipped)/postFreq;
|
||||
ps->postRecv = 0;
|
||||
}
|
||||
if ((((a - recvSkipped)%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
|
||||
postRecv = 1;
|
||||
} else {
|
||||
postRecv = 0;
|
||||
}
|
||||
s -= (1<<recvDim);
|
||||
s -= (1<<ps->recvDim);
|
||||
int recvDataRank = (rank + nranks + s) % nranks;
|
||||
inpIx = recvDataRank * count + offset;
|
||||
sendDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
if (sendDim == -1) {
|
||||
sendOffset = -1;
|
||||
sendStepOffset = 0;
|
||||
} else if (as - (1<<recvDim) == 0) {
|
||||
if (newPeer(a, aggFactor)) sendSkipped = a;
|
||||
ps->inpIx = recvDataRank * count + offset;
|
||||
ps->sendDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
if (ps->sendDim == -1) {
|
||||
ps->sendOffset = -1;
|
||||
} else if (as - (1<<ps->recvDim) == 0) {
|
||||
if (newPeer(a, aggFactor)) { sendSkipped = a; ps->stepOffset = stepOffset = 0; }
|
||||
int foffset = a - sendSkipped;
|
||||
sendStepOffset = recvDim == 0 ? 0 : foffset/postFreq;
|
||||
sendOffset = (foffset%postFreq)*nelem;
|
||||
ps->sendOffset = (foffset%postFreq)*nelem;
|
||||
}
|
||||
int recvDim = ps->recvDim;
|
||||
if (s < nranks && skip) {
|
||||
recvDim = -1;
|
||||
recvOffset = -1;
|
||||
postRecv = 0;
|
||||
ps->recvDim = -1;
|
||||
ps->recvOffset = -1;
|
||||
ps->postRecv = 0;
|
||||
skip = 0;
|
||||
}
|
||||
if (skip || recvDim == -1) recvSkipped++;
|
||||
if (skip) sendSkipped++;
|
||||
if (++a == lastA) {
|
||||
as--;
|
||||
phase = as % 2 == 1 ? 0 : 1;
|
||||
resetA();
|
||||
}
|
||||
if (skip == 0) return;
|
||||
if (recvDim > 0 && (((a-sendSkipped)%postFreq) + 1 >= postFreq) && skip == 0) stepOffset++;
|
||||
} else if (phase == 2) {
|
||||
int s = (2*mirrorInvert(a, lastA)+1)*scale*aggDelta + 1;
|
||||
postRecv = 0;
|
||||
ps->postRecv = 0;
|
||||
if (s >= nranks) skip = 1;
|
||||
recvDim = 0;
|
||||
postSend = a == lastA-1 ? 1 : 0;
|
||||
ps->recvDim = 0;
|
||||
ps->postSend = a == lastA-1 ? 1 : 0;
|
||||
s -= 1;
|
||||
if (s < nranks && skip) {
|
||||
recvDim = -1;
|
||||
recvOffset = -1;
|
||||
ps->recvDim = -1;
|
||||
ps->recvOffset = -1;
|
||||
skip = 0;
|
||||
} else if (!skip) {
|
||||
int foffset = phase2recv;
|
||||
phase2recv++;
|
||||
postRecv |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
|
||||
recvOffset = (foffset%postFreq) * nelem;
|
||||
int foffset = a + aggFactor - aggFactor/scale;
|
||||
ps->postRecv |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
|
||||
ps->recvOffset = (foffset%postFreq) * nelem;
|
||||
}
|
||||
int recvDataRank = (rank + nranks + s) % nranks;
|
||||
inpIx = recvDataRank * count + offset;
|
||||
sendDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
int foffset = a - sendSkipped;
|
||||
postSend |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
|
||||
sendStepOffset = 0;
|
||||
sendOffset = (foffset%postFreq) * nelem;
|
||||
if (skip || sendDim == -1) sendSkipped++;
|
||||
if (++a == lastA) {
|
||||
phase = 3;
|
||||
resetA();
|
||||
}
|
||||
if (skip == 0) return;
|
||||
ps->inpIx = recvDataRank * count + offset;
|
||||
ps->sendDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
int foffset = a;
|
||||
ps->postSend |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
|
||||
ps->sendOffset = (foffset%postFreq) * nelem;
|
||||
} else if (phase == 3) {
|
||||
int s = (2*mirrorInvert(a, lastA)+1)*scale*aggDelta;
|
||||
postRecv = a == lastA-1 ? 1 : 0;
|
||||
ps->postRecv = a == lastA-1 ? 1 : 0;
|
||||
if (s >= nranks) skip = 1;
|
||||
recvDim = firstBitSet(s, nrPow2);
|
||||
postSend = 0;
|
||||
s -= (1<<recvDim);
|
||||
int foffset = a - recvSkipped;
|
||||
postRecv |= (foffset+1)%postFreq == 0 ? 1 : 0;
|
||||
recvOffset = (foffset%postFreq) * nelem;
|
||||
ps->recvDim = firstBitSet(s, nrPow2);
|
||||
ps->postSend = 0;
|
||||
s -= (1<<ps->recvDim);
|
||||
int foffset = a;
|
||||
ps->postRecv |= (foffset+1)%postFreq == 0 ? 1 : 0;
|
||||
ps->recvOffset = (foffset%postFreq) * nelem;
|
||||
int recvDataRank = (rank + nranks + s) % nranks;
|
||||
inpIx = recvDataRank * count + offset;
|
||||
sendDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
ps->inpIx = recvDataRank * count + offset;
|
||||
ps->sendDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
if (s < nranks && skip) {
|
||||
recvDim = -1;
|
||||
recvOffset = -1;
|
||||
postRecv = 0;
|
||||
ps->recvDim = -1;
|
||||
ps->recvOffset = -1;
|
||||
ps->postRecv = 0;
|
||||
skip = 0;
|
||||
}
|
||||
if (newPeer(a, aggFactor/(2*scale))) sendSkipped = a;
|
||||
if (newPeer(a, aggFactor/(2*scale))) { sendSkipped = a; ps->stepOffset = stepOffset = 0; }
|
||||
foffset = a - sendSkipped;
|
||||
sendStepOffset = foffset / postFreq; // Accumulate on next steps
|
||||
sendOffset = sendDim >= 0 ? (foffset%postFreq) * nelem : -1;
|
||||
if (skip || recvDim == -1) recvSkipped++;
|
||||
if (skip) sendSkipped++;
|
||||
if (++a == lastA) {
|
||||
scale *= 2;
|
||||
phase = scale < aggFactor ? 2 : 4;
|
||||
if ((foffset%postFreq) + 1 >= postFreq && skip == 0) stepOffset++;
|
||||
ps->sendOffset = ps->sendDim >= 0 ? (foffset%postFreq) * nelem : -1;
|
||||
} else if (phase == 4) {
|
||||
ps->recvDim = 0;
|
||||
ps->sendDim = -1;
|
||||
ps->inpIx = rank * count + offset;
|
||||
ps->recvOffset = ((aggFactor-1)%postFreq) * nelem;
|
||||
ps->sendOffset = -1;
|
||||
ps->postRecv = 1;
|
||||
ps->postSend = 0;
|
||||
offset += chunkCount;
|
||||
}
|
||||
a++;
|
||||
if (a >= lastA && a >= parallelFactor) {
|
||||
int p = phase;
|
||||
if (p == 1) as--;
|
||||
if (p == 3) scale *= 2;
|
||||
phase =
|
||||
p == 0 ? as == 1 ? (aggFactor > 1 ? 2 : 4) : 1 :
|
||||
p == 1 ? as % 2 == 1 ? 0 : 1 :
|
||||
p == 2 ? 3 :
|
||||
p == 3 ? scale < aggFactor ? 2 : 4 :
|
||||
5;
|
||||
if (p == 4) {
|
||||
if (offset >= end) {
|
||||
ps->last = 2;
|
||||
} else {
|
||||
reset();
|
||||
}
|
||||
} else {
|
||||
resetA();
|
||||
}
|
||||
if (skip == 0) return;
|
||||
} else if (phase == 4) {
|
||||
recvDim = 0;
|
||||
sendDim = -1;
|
||||
inpIx = rank * count + offset;
|
||||
recvOffset = (phase2recv%postFreq) * nelem;
|
||||
sendStepOffset = 0;
|
||||
sendOffset = -1;
|
||||
postRecv = 1;
|
||||
postSend = 0;
|
||||
offset += chunkCount;
|
||||
if (offset >= end) {
|
||||
last = 1;
|
||||
} else {
|
||||
reset();
|
||||
}
|
||||
return;
|
||||
} else if (phase == 4 && offset >= end) {
|
||||
ps->last = 1;
|
||||
}
|
||||
goto restart;
|
||||
int flags = PatUsed | (skip ? PatSkipped : 0);
|
||||
#if __CUDA_ARCH__ >= 600
|
||||
cuda::atomic_ref<int, cuda::thread_scope_block> a(ps->flags);
|
||||
a.store(flags, cuda::memory_order_release);
|
||||
#else
|
||||
ps->flags = flags;
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@@ -669,14 +698,12 @@ class PatAGAlgorithm{
|
||||
int nrPow2;
|
||||
int postFreq;
|
||||
int lastA;
|
||||
|
||||
int parallelFactor;
|
||||
int aggFactor;
|
||||
int as; // aggregated steps
|
||||
int a; // step inside aggregated step
|
||||
int aggDelta;
|
||||
|
||||
int scale;
|
||||
|
||||
int phase;
|
||||
|
||||
// AS computation
|
||||
@@ -685,7 +712,7 @@ class PatAGAlgorithm{
|
||||
int bitCount[32];
|
||||
int bitZeroStep[32];
|
||||
|
||||
__device__ __host__ int min(int a, int b) {
|
||||
__device__ __host__ ssize_t min(ssize_t a, ssize_t b) {
|
||||
return (a<b)?a:b;
|
||||
}
|
||||
|
||||
@@ -752,8 +779,9 @@ class PatAGAlgorithm{
|
||||
|
||||
|
||||
public:
|
||||
__device__ __host__ PatAGAlgorithm(int stepSize, int stepDepth, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
|
||||
__device__ __host__ PatAGAlgorithm(int stepSize, int stepDepth, int maxParallelFactor, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
|
||||
offset(offset), end(end), count(count), chunkCount(chunkCount), rank(rank), nranks(nranks) {
|
||||
parallelFactor = maxParallelFactor;
|
||||
aggDelta = nrPow2 = (1<<log2Up(nranks));
|
||||
|
||||
aggFactor = 1;
|
||||
@@ -763,120 +791,120 @@ public:
|
||||
aggDelta /= 2;
|
||||
}
|
||||
postFreq = aggFactor;
|
||||
if (postFreq < parallelFactor) parallelFactor = postFreq;
|
||||
int d = stepDepth;
|
||||
while (d > 1 && aggFactor < nranks/2) {
|
||||
d /= 2;
|
||||
aggFactor *= 2;
|
||||
aggDelta /= 2;
|
||||
}
|
||||
//printf("AggFactor %d PostFreq %d AggDelta %d\n", aggFactor, postFreq, aggDelta);
|
||||
|
||||
asDim = log2Up(aggDelta);
|
||||
reset();
|
||||
}
|
||||
|
||||
__device__ __host__ void getNextOp(int &recvDim, int &sendDim, size_t &inpIx, size_t &outIx, int &recvOffset, int &sendOffset, int &recvStepOffset, int &nelemOut, int &postRecv, int &postSend, int &last) {
|
||||
restart:
|
||||
//printf("Phase %d as %d/%d a %d/%d scale %d\n", phase, as, aggDelta, a, lastA, scale);
|
||||
last = 0;
|
||||
nelemOut = nelem;
|
||||
inpIx = offset;
|
||||
__device__ __host__ int getParallelFactor() {
|
||||
return parallelFactor;
|
||||
}
|
||||
|
||||
__device__ __host__ void getNextOp(struct ncclPatStep* ps) {
|
||||
ps->last = 0;
|
||||
ps->nelem = nelem;
|
||||
ps->inpIx = offset;
|
||||
int skip = 0;
|
||||
if (phase == 0) {
|
||||
if (a >= lastA) {
|
||||
skip = 1;
|
||||
} else if (phase == 0) {
|
||||
int s = a*aggDelta + as;
|
||||
if (s >= nranks) skip = 1;
|
||||
int nextSkip = (a+1)*aggDelta + as >= nranks ? 1 : 0;
|
||||
int recvDataRank = (rank + s) % nranks;
|
||||
outIx = recvDataRank * count + offset;
|
||||
sendDim = -1;
|
||||
recvDim = 0;
|
||||
inpIx = 0;
|
||||
sendOffset = -1;
|
||||
recvOffset = (a % postFreq) * nelem;
|
||||
recvStepOffset = 0;
|
||||
postRecv = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
|
||||
postSend = 0;
|
||||
a++;
|
||||
if (nextSkip) {
|
||||
as = nextAs();
|
||||
if (as == aggDelta/2) {
|
||||
offset += chunkCount;
|
||||
if (offset >= end) {
|
||||
last = 1;
|
||||
} else {
|
||||
reset();
|
||||
}
|
||||
return;
|
||||
}
|
||||
phase = 1;
|
||||
resetA();
|
||||
}
|
||||
if (skip == 0) return;
|
||||
ps->outIx = recvDataRank * count + offset;
|
||||
ps->sendDim = -1;
|
||||
ps->recvDim = 0;
|
||||
ps->inpIx = 0;
|
||||
ps->sendOffset = -1;
|
||||
ps->recvOffset = (a % postFreq) * nelem;
|
||||
ps->stepOffset = 0;
|
||||
ps->postRecv = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
|
||||
ps->postSend = 0;
|
||||
} else if (phase == 1) {
|
||||
int s = a*aggDelta + as;
|
||||
if (s >= nranks) skip = 1;
|
||||
sendDim = firstBitSet(s, nrPow2);
|
||||
s -= (1<<sendDim);
|
||||
ps->sendDim = firstBitSet(s, nrPow2);
|
||||
s -= (1<<ps->sendDim);
|
||||
int sendDataRank = (rank + nranks + s) % nranks;
|
||||
outIx = sendDataRank * count + offset;
|
||||
recvDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
sendOffset = recvOffset = (a % postFreq) * nelem;
|
||||
postSend = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
|
||||
postRecv = (sendDim == 0) && ((a % postFreq == postFreq-1) || ((a+1)*aggDelta+as-1 >= nranks)) ? 1 : 0;
|
||||
recvStepOffset = (sendDim == 0) ? 0 : a/postFreq;
|
||||
if (recvDim == -1) {
|
||||
recvOffset = -1;
|
||||
postRecv = 0;
|
||||
} else if (as - (1<<sendDim) == 0) {
|
||||
int foffset = (a*aggDelta) >> (recvDim+1);
|
||||
recvOffset = (foffset%postFreq)*nelem;
|
||||
postRecv = (sendDim == 0) && ((foffset % postFreq == postFreq-1) || ((((foffset+1)*2)+1)<<recvDim) >= nranks) ? 1 : 0;
|
||||
recvStepOffset = (sendDim == 0) ? 0 : foffset/postFreq;
|
||||
ps->outIx = sendDataRank * count + offset;
|
||||
ps->recvDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
ps->sendOffset = ps->recvOffset = (a % postFreq) * nelem;
|
||||
ps->postSend = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
|
||||
ps->postRecv = (ps->sendDim == 0) && ((a % postFreq == postFreq-1) || ((a+1)*aggDelta+as-1 >= nranks)) ? 1 : 0;
|
||||
ps->stepOffset = (ps->sendDim == 0) ? 0 : a/postFreq;
|
||||
if (ps->recvDim == -1) {
|
||||
ps->recvOffset = -1;
|
||||
ps->postRecv = 0;
|
||||
} else if (as - (1<<ps->sendDim) == 0) {
|
||||
int foffset = (a*aggDelta) >> (ps->recvDim+1);
|
||||
ps->recvOffset = (foffset%postFreq)*nelem;
|
||||
ps->postRecv = (ps->sendDim == 0) && ((foffset % postFreq == postFreq-1) || ((((foffset+1)*2)+1)<<ps->recvDim) >= nranks) ? 1 : 0;
|
||||
ps->stepOffset = (ps->sendDim == 0) ? 0 : foffset/postFreq;
|
||||
}
|
||||
if (s < nranks && sendDim == 0 && skip) {
|
||||
if (s < nranks && ps->sendDim == 0 && skip) {
|
||||
// Don't forget to receive at least once even if we don't send afterwards
|
||||
sendDim = -1;
|
||||
sendOffset = -1;
|
||||
postSend = 0;
|
||||
ps->sendDim = -1;
|
||||
ps->sendOffset = -1;
|
||||
ps->postSend = 0;
|
||||
skip = 0;
|
||||
}
|
||||
if (++a == lastA) {
|
||||
if (as % 2 == 1) {
|
||||
phase = 0;
|
||||
} else {
|
||||
as = nextAs();
|
||||
}
|
||||
resetA();
|
||||
}
|
||||
if (skip == 0) return;
|
||||
} else if (phase == 2) {
|
||||
int s = (2*a+1)*scale*aggDelta;
|
||||
postSend = (a % postFreq == postFreq-1) || ((2*(a+1)+1)*scale*aggDelta >= nranks) ? 1 : 0;
|
||||
postRecv = 0;
|
||||
ps->postSend = (a % postFreq == postFreq-1) || ((2*(a+1)+1)*scale*aggDelta >= nranks) ? 1 : 0;
|
||||
ps->postRecv = 0;
|
||||
if (s >= nranks) skip = 1;
|
||||
sendDim = firstBitSet(s, nrPow2);
|
||||
s -= (1<<sendDim);
|
||||
sendOffset = (a%postFreq) * nelem;
|
||||
recvStepOffset = a / postFreq;
|
||||
ps->sendDim = firstBitSet(s, nrPow2);
|
||||
s -= (1<<ps->sendDim);
|
||||
ps->sendOffset = (a%postFreq) * nelem;
|
||||
ps->stepOffset = a / postFreq;
|
||||
int sendDataRank = (rank + nranks + s) % nranks;
|
||||
outIx = sendDataRank * count + offset;
|
||||
recvDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
if (recvDim == -1) {
|
||||
recvOffset = -1;
|
||||
ps->outIx = sendDataRank * count + offset;
|
||||
ps->recvDim = s ? firstBitSet(s, nrPow2) : -1;
|
||||
if (ps->recvDim == -1) {
|
||||
ps->recvOffset = -1;
|
||||
} else {
|
||||
s -= (1<<recvDim);
|
||||
int foffset = (a*2*scale*aggDelta) >> (recvDim+1);
|
||||
recvOffset = (foffset%postFreq)*nelem;
|
||||
recvStepOffset = foffset / postFreq;
|
||||
s -= (1<<ps->recvDim);
|
||||
int foffset = (a*2*scale*aggDelta) >> (ps->recvDim+1);
|
||||
ps->recvOffset = (foffset%postFreq)*nelem;
|
||||
ps->stepOffset = foffset / postFreq;
|
||||
}
|
||||
if (++a == lastA) {
|
||||
scale /= 2;
|
||||
phase = scale ? 2 : 1;
|
||||
}
|
||||
a++;
|
||||
if (a >= lastA && a >= parallelFactor) {
|
||||
int p = phase;
|
||||
if (p == 2) scale /= 2;
|
||||
phase =
|
||||
p == 2 ? scale ? 2 : 1 :
|
||||
p == 1 ? as % 2 == 1 ? 0 : 1 :
|
||||
1;
|
||||
if (p == 0 || (p == 1 && as % 2 == 0)) as = nextAs();
|
||||
if (p == 0 && as == aggDelta/2) {
|
||||
offset += chunkCount;
|
||||
if (offset >= end) {
|
||||
ps->last = 2;
|
||||
} else {
|
||||
reset();
|
||||
}
|
||||
} else {
|
||||
resetA();
|
||||
}
|
||||
if (skip == 0) return;
|
||||
} else if (phase == 0 && as == 1 && offset + chunkCount >= end && a-1 >= ((lastA-1) / parallelFactor) * parallelFactor) {
|
||||
ps->last = 1;
|
||||
}
|
||||
goto restart;
|
||||
int flags = PatUsed | (skip ? PatSkipped : 0);
|
||||
#if __CUDA_ARCH__ >= 600
|
||||
cuda::atomic_ref<int, cuda::thread_scope_block> a(ps->flags);
|
||||
a.store(flags, cuda::memory_order_release);
|
||||
#else
|
||||
ps->flags = flags;
|
||||
#endif
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
@@ -139,6 +139,9 @@ struct ncclSharedResources {
|
||||
int* tpRankToLocalRank;
|
||||
// Internal streams
|
||||
struct ncclStrongStream deviceStream, hostStream;
|
||||
int noncapturedRefs; // number of non-captured hostStreamPlanCallback on the stream
|
||||
int persistentRefs;
|
||||
cudaEvent_t launchEvent, scratchEvent;
|
||||
|
||||
/* proxy related shared res */
|
||||
struct ncclProxyState* proxyState;
|
||||
@@ -437,6 +440,7 @@ struct ncclComm {
|
||||
// List of destructors to run when comm is destructed
|
||||
struct ncclDestructor* destructorHead;
|
||||
|
||||
struct ncclCudaContext* context;
|
||||
struct ncclSharedResources* sharedRes;
|
||||
/* map to top parent ranks. */
|
||||
int* topParentRanks;
|
||||
@@ -449,6 +453,7 @@ struct ncclComm {
|
||||
|
||||
int netPluginLoaded;
|
||||
ncclNet_t* ncclNet;
|
||||
int ncclNetVer;
|
||||
ncclNetDeviceType netDeviceType;
|
||||
ncclCollNet_t* ncclCollNet;
|
||||
void* bootstrap;
|
||||
@@ -456,6 +461,7 @@ struct ncclComm {
|
||||
struct channelMasks* connectSend;
|
||||
struct channelMasks* connectRecv;
|
||||
struct ncclTopoGraph graphs[NCCL_NUM_ALGORITHMS];
|
||||
int maxTreePattern;
|
||||
bool initAlgoChannels[NCCL_NUM_ALGORITHMS];
|
||||
bool runtimeConn; // if dynamic connection is supported
|
||||
bool directMode;
|
||||
@@ -603,8 +609,7 @@ struct ncclComm {
|
||||
struct ncclComm* groupNext;
|
||||
// Subset of those in groupNext list. Holds 0x1 if not needing preconnect.
|
||||
struct ncclComm* preconnectNext;
|
||||
int persistentRefs; // number of persistent plan-lists capturing this comm
|
||||
int noncapturedRefs; // number of non-captured hostStreamPlanCallback on the stream
|
||||
int localPersistentRefs; // number of persistent plan-lists capturing this comm
|
||||
struct P2pSchedulePair { int sendRank; int recvRank; } *p2pSchedule;
|
||||
|
||||
struct ncclKernelPlanner planner;
|
||||
@@ -669,6 +674,7 @@ struct ncclComm {
|
||||
// Profiler plugin
|
||||
void* profilerContext;
|
||||
uint64_t seqNumber[NCCL_NUM_FUNCTIONS];
|
||||
struct ncclProfilerProxy profiler;
|
||||
|
||||
// buffer registration cache
|
||||
struct ncclRegCache regCache;
|
||||
|
||||
@@ -163,6 +163,7 @@ struct ncclProxyConnector {
|
||||
|
||||
struct ncclConnector {
|
||||
int connected;
|
||||
int hasSeen;
|
||||
struct ncclProxyConnector proxyConn;
|
||||
struct ncclTransportComm* transportComm;
|
||||
void* transportResources;
|
||||
@@ -256,6 +257,8 @@ struct alignas(16) ncclDevWorkP2p {
|
||||
uint8_t sendNetReg:1, recvNetReg:1;
|
||||
uint8_t sendIpcReg:1, recvIpcReg:1;
|
||||
|
||||
uint8_t profilerEnabled:1;
|
||||
|
||||
uint8_t sendConnIndex:2, recvConnIndex:2;
|
||||
};
|
||||
|
||||
@@ -304,7 +307,7 @@ struct alignas(16) ncclDevWorkColl {
|
||||
uint32_t nWarps:8;
|
||||
uint32_t redOpArgIsPtr:1, regUsed:1, netRegUsed:1, oneNode:1, direct:2, isOneRPN:1, rcclUseOneSlice:1;
|
||||
uint32_t root:30, connIndex:2;
|
||||
uint16_t pivotA2ANumBiRings;
|
||||
uint16_t pivotA2ANumBiRings:15, profilerEnabled:1;
|
||||
void* recvbuff;
|
||||
void* sendbuff;
|
||||
uintptr_t sendbuffOffset;
|
||||
@@ -498,6 +501,7 @@ struct alignas(16) ncclDevChannel {
|
||||
struct ncclTree binTree;
|
||||
struct ncclNvls nvls;
|
||||
uint32_t* workFifoDone; // Location of done counter, device writes index+1 of last work processed
|
||||
uint64_t workCounter;
|
||||
};
|
||||
|
||||
struct ncclDevComm {
|
||||
@@ -523,6 +527,10 @@ struct ncclDevComm {
|
||||
|
||||
int* rankToLocalRank;
|
||||
|
||||
// Profiler counters
|
||||
uint64_t* workStarted/*[MAXCHANNELS]*/;
|
||||
uint64_t* workCompleted/*[MAXCHANNELS]*/;
|
||||
|
||||
#if defined(ENABLE_NPKIT)
|
||||
NpKitEventCollectContext* npKitEventCollectContexts;
|
||||
uint64_t* cpuTimestamp;
|
||||
@@ -621,7 +629,7 @@ __host__ __device__ constexpr int ncclCalcUnroll(int bytePerPack, int insns, int
|
||||
|
||||
__host__ __device__ constexpr int ncclCollUnroll(int cudaArch = NCCL_CUDA_ARCH) {
|
||||
// Our collective unroll should move to the same bytes&insns model as NVLS.
|
||||
return cudaArch >= 800 ? 8 : 4;
|
||||
return cudaArch >= 800 ? (cudaArch == 1200 ? 6 : 8) : 4;
|
||||
}
|
||||
|
||||
__host__ __device__ constexpr int ncclNvlsUnrollBytes(int cudaArch = NCCL_CUDA_ARCH) { return 4*16; }
|
||||
|
||||
+19
-10
@@ -37,17 +37,24 @@ ncclResult_t ncclTopoComputeCommCPU(struct ncclComm* comm);
|
||||
ncclResult_t ncclTopoGetNetDev(struct ncclComm* comm, int rank, struct ncclTopoGraph* graph, int channelId, int peerRank, int64_t* id, int* dev, int* proxyRank);
|
||||
ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* system, int rank1, int rank2, int* p2p, int *read, int* intermediateRank);
|
||||
ncclResult_t ncclTopoCheckMNNVL(struct ncclTopoSystem* system, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2, int* ret);
|
||||
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* topo, int rank, int64_t netId, int read, int* useGdr);
|
||||
#define MAX_XGMI_INTER_GPUS 4
|
||||
ncclResult_t ncclTopoGetIntraNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int type, int64_t* id, int* dev);
|
||||
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, int maxInter=MAX_XGMI_INTER_GPUS, int nInter=0, int *inter=nullptr);
|
||||
ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int netDev, int rank, int* flush);
|
||||
enum ncclTopoGdrMode {
|
||||
ncclTopoGdrModeDisable = 0,
|
||||
ncclTopoGdrModeDefault = 1,
|
||||
ncclTopoGdrModePci = 2,
|
||||
ncclTopoGdrModeNum = 3
|
||||
};
|
||||
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* topo, int rank, int64_t netId, int read, enum ncclTopoGdrMode* gdrMode);
|
||||
ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int64_t netId, int netDev, int rank, int* flush);
|
||||
ncclResult_t ncclTopoIsGdrAvail(struct ncclTopoSystem* system, int rank, bool *avail);
|
||||
ncclResult_t ncclTopoCheckNet(struct ncclTopoSystem* system, int rank1, int rank2, int* net);
|
||||
int ncclPxnDisable(struct ncclComm* comm);
|
||||
ncclResult_t ncclTopoGetPxnRanks(struct ncclComm* comm, int** intermediateRanks, int* nranks);
|
||||
ncclResult_t ncclGetLocalCpu(struct ncclTopoSystem* system, int gpu, int* retCpu);
|
||||
|
||||
#define MAX_XGMI_INTER_GPUS 4
|
||||
ncclResult_t ncclTopoGetIntraNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int type, int64_t* id, int* dev);
|
||||
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, int maxInter=MAX_XGMI_INTER_GPUS, int nInter=0, int *inter=nullptr);
|
||||
|
||||
// Find CPU affinity
|
||||
ncclResult_t ncclTopoGetCpuAffinity(struct ncclTopoSystem* system, int rank, cpu_set_t* affinity);
|
||||
|
||||
@@ -59,11 +66,13 @@ ncclResult_t ncclTopoGetCpuAffinity(struct ncclTopoSystem* system, int rank, cpu
|
||||
#define NCCL_TOPO_CPU_VENDOR_AMD 2
|
||||
#define NCCL_TOPO_CPU_VENDOR_ZHAOXIN 3
|
||||
#define NCCL_TOPO_CPU_VENDOR_MIXED 4
|
||||
#define NCCL_TOPO_CPU_TYPE_BDW 1
|
||||
#define NCCL_TOPO_CPU_TYPE_SKL 2
|
||||
#define NCCL_TOPO_CPU_TYPE_ZEN 3
|
||||
#define NCCL_TOPO_CPU_TYPE_ROME 4
|
||||
#define NCCL_TOPO_CPU_TYPE_YONGFENG 1
|
||||
#define NCCL_TOPO_CPU_MODEL_INTEL_BDW 1
|
||||
#define NCCL_TOPO_CPU_MODEL_INTEL_SKL 2
|
||||
#define NCCL_TOPO_CPU_MODEL_INTEL_SRP 3
|
||||
#define NCCL_TOPO_CPU_MODEL_INTEL_ERP 4
|
||||
#define NCCL_TOPO_CPU_MODEL_AMD_ZEN 5
|
||||
#define NCCL_TOPO_CPU_MODEL_AMD_ROME 6
|
||||
#define NCCL_TOPO_CPU_MODEL_YONGFENG 1
|
||||
ncclResult_t ncclTopoCpuType(struct ncclTopoSystem* system, int* arch, int* vendor, int* model);
|
||||
ncclResult_t ncclTopoGetGpuCount(struct ncclTopoSystem* system, int* count);
|
||||
ncclResult_t ncclTopoGetNetCount(struct ncclTopoSystem* system, int* count);
|
||||
|
||||
@@ -108,6 +108,12 @@ inline void ncclGroupCommJoin(struct ncclComm* comm) {
|
||||
struct ncclComm** pp = &ncclGroupCommHead;
|
||||
while (*pp != nullptr && comm->intraComm0 != (*pp)->intraComm0)
|
||||
pp = &(*pp)->groupNext;
|
||||
|
||||
// didn't find its clique, we need to insert it with ascending order based on commHash
|
||||
if (*pp == nullptr) {
|
||||
pp = &ncclGroupCommHead;
|
||||
while (*pp != nullptr && (*pp)->commHash < comm->commHash) pp = &(*pp)->groupNext;
|
||||
}
|
||||
comm->groupNext = *pp;
|
||||
*pp = comm;
|
||||
// Comms gets a new memory stack scope upon joining. Each task batched for
|
||||
|
||||
@@ -1,610 +0,0 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_NET_H_
|
||||
#define NCCL_NET_H_
|
||||
|
||||
#include "nccl.h"
|
||||
#include "nccl_common.h"
|
||||
#include "net_device.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#define NCCL_NET_HANDLE_MAXSIZE 128
|
||||
//Maximum value NCCL can accept for maxP2pBytes and maxCollBytes net properties
|
||||
#define NCCL_MAX_NET_SIZE_BYTES (1*1024*1024*1024*1024L)
|
||||
#define NCCL_NET_OPTIONAL_RECV_COMPLETION 0x1
|
||||
|
||||
#define NCCL_PTR_HOST 0x1
|
||||
#define NCCL_PTR_CUDA 0x2
|
||||
#define NCCL_PTR_DMABUF 0x4
|
||||
|
||||
// Maximum number of requests per comm object
|
||||
#define NCCL_NET_MAX_REQUESTS 32
|
||||
|
||||
// Max number of ncclNet objects which can live in the same process
|
||||
#define NCCL_NET_MAX_PLUGINS 3
|
||||
|
||||
#define NCCL_NET_MAX_DEVS_PER_NIC_V9 4
|
||||
#define NCCL_NET_MAX_DEVS_PER_NIC NCCL_NET_MAX_DEVS_PER_NIC_V9
|
||||
|
||||
typedef struct {
|
||||
int ndevs;
|
||||
int devs[NCCL_NET_MAX_DEVS_PER_NIC_V9];
|
||||
} ncclNetVDeviceProps_v9_t;
|
||||
typedef ncclNetVDeviceProps_v9_t ncclNetVDeviceProps_t;
|
||||
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int regIsGlobal; // regMr is not tied to a particular comm
|
||||
int forceFlush; // Force a flush on receives
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
ncclNetDeviceType netDeviceType; // Network offload type
|
||||
int netDeviceVersion; // Version number for network offload
|
||||
ncclNetVDeviceProps_v9_t vProps;
|
||||
size_t maxP2pBytes; // Max transfer size for point-to-point operations
|
||||
size_t maxCollBytes; // Max transfer size for collective operations
|
||||
} ncclNetProperties_v9_t;
|
||||
typedef ncclNetProperties_v9_t ncclNetProperties_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v8_t** sendDevComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v8_t** recvDevComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
|
||||
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
|
||||
|
||||
// Notify the plugin that a recv has completed by the device
|
||||
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
|
||||
|
||||
// Create a virtual NIC given the specified properties, which can be accessed at device index d
|
||||
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_t* props);
|
||||
} ncclNet_v9_t;
|
||||
|
||||
typedef ncclNet_v9_t ncclNet_t;
|
||||
|
||||
#define NCCL_NET_PLUGIN_SYMBOL ncclNetPlugin_v9
|
||||
|
||||
typedef struct {
|
||||
void* mhandle;
|
||||
void* address;
|
||||
size_t size;
|
||||
} ncclNetSGE_v9_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, size_t count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v9_t* recvParts,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
void* sendMhandle, void** request);
|
||||
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v9_t* sendParts, void* recvData,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp,
|
||||
void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Create a virtual NIC given the specified properties, which can be accessed at device index d
|
||||
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_t* props);
|
||||
} ncclCollNet_v9_t;
|
||||
|
||||
typedef ncclCollNet_v9_t ncclCollNet_t;
|
||||
|
||||
#define NCCL_COLLNET_PLUGIN_SYMBOL ncclCollNetPlugin_v9
|
||||
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int regIsGlobal; // regMr is not tied to a particular comm
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
ncclNetDeviceType netDeviceType; // Network offload type
|
||||
int netDeviceVersion; // Version number for network offload
|
||||
} ncclNetProperties_v8_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v8_t** sendDevComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v8_t** recvDevComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
|
||||
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
|
||||
|
||||
// Notify the plugin that a recv has completed by the device
|
||||
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
|
||||
} ncclNet_v8_t;
|
||||
|
||||
typedef struct {
|
||||
void* mhandle;
|
||||
void* address;
|
||||
uint32_t size;
|
||||
} ncclNetSGE_v8_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v8_t* recvParts,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
void* sendMhandle, void** request);
|
||||
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v8_t* sendParts, void* recvData,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp,
|
||||
void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclCollNet_v8_t;
|
||||
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
ncclNetDeviceType netDeviceType; // Network offload type
|
||||
int netDeviceVersion; // Version number for network offload
|
||||
} ncclNetProperties_v7_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v7_t** sendDevComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v7_t** recvDevComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
|
||||
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
|
||||
|
||||
// Notify the plugin that a recv has completed by the device
|
||||
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
|
||||
} ncclNet_v7_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclCollNet_v7_t;
|
||||
|
||||
#define NCCL_NET_MAX_REQUESTS_V6 8
|
||||
|
||||
// v6 struct for backwards compatibility
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
} ncclNetProperties_v6_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclNet_v6_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclCollNet_v6_t;
|
||||
|
||||
// v5 struct for backwards compatibility
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclNet_v5_t;
|
||||
|
||||
// v5 struct for backwards compatibility
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclCollNet_v5_t;
|
||||
|
||||
// context passed from RCCL lib to n/w plugin
|
||||
typedef struct {
|
||||
// channel id
|
||||
uint32_t chId;
|
||||
} ncclNet_ctxt_t;
|
||||
|
||||
#endif // end include guard
|
||||
@@ -1,235 +0,0 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_PROFILER_H_
|
||||
#define NCCL_PROFILER_H_
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
enum {
|
||||
ncclProfileGroup = (1 << 0), // group event type
|
||||
ncclProfileColl = (1 << 1), // host collective call event type
|
||||
ncclProfileP2p = (1 << 2), // host point-to-point call event type
|
||||
ncclProfileProxyOp = (1 << 3), // proxy operation event type
|
||||
ncclProfileProxyStep = (1 << 4), // proxy step event type
|
||||
ncclProfileProxyCtrl = (1 << 5), // proxy control event type
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
uint8_t type; // event type descriptor: ncclProfileColl, ...
|
||||
void* parentObj; // pointer to the profiler parent object (for coll is the group)
|
||||
int rank; // originating rank
|
||||
union {
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
uint64_t seqNumber;
|
||||
const char* func;
|
||||
void const* sendBuff;
|
||||
void* recvBuff;
|
||||
size_t count;
|
||||
int root;
|
||||
const char* datatype;
|
||||
size_t trafficBytes;
|
||||
uint8_t nMaxChannels;
|
||||
uint8_t nWarps;
|
||||
const char* algo;
|
||||
const char* proto;
|
||||
} coll;
|
||||
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
const char* func;
|
||||
void* buff;
|
||||
const char* datatype;
|
||||
size_t count;
|
||||
int peer;
|
||||
} p2p;
|
||||
|
||||
struct {
|
||||
pid_t pid; // pid of the originating process
|
||||
uint8_t channelId; // channel id for this proxy operation
|
||||
int peer; // remote rank for send/recv
|
||||
int nSteps; // number of steps for this proxy operation
|
||||
int chunkSize; // amount of data transferred by this proxy operation
|
||||
int isSend;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int step;
|
||||
} proxyStep;
|
||||
};
|
||||
} ncclProfilerEventDescr_v2_t;
|
||||
|
||||
typedef enum {
|
||||
ncclProfilerProxyOpSendPosted,
|
||||
ncclProfilerProxyOpSendRemFifoWait,
|
||||
ncclProfilerProxyOpSendTransmitted,
|
||||
ncclProfilerProxyOpSendDone,
|
||||
ncclProfilerProxyOpRecvPosted,
|
||||
ncclProfilerProxyOpRecvReceived,
|
||||
ncclProfilerProxyOpRecvTransmitted,
|
||||
ncclProfilerProxyOpRecvDone,
|
||||
|
||||
/* Legacy proxy profiler states */
|
||||
ncclProfilerProxyStepSendGPUWait,
|
||||
ncclProfilerProxyStepSendWait,
|
||||
ncclProfilerProxyStepRecvWait,
|
||||
ncclProfilerProxyStepRecvFlushWait,
|
||||
ncclProfilerProxyStepRecvGPUWait,
|
||||
|
||||
/* Legacy proxy control states */
|
||||
ncclProfilerProxyCtrlIdle,
|
||||
ncclProfilerProxyCtrlActive,
|
||||
ncclProfilerProxyCtrlSleep,
|
||||
ncclProfilerProxyCtrlWakeup,
|
||||
ncclProfilerProxyCtrlAppend,
|
||||
ncclProfilerProxyCtrlAppendEnd,
|
||||
} ncclProfilerEventState_v2_t;
|
||||
|
||||
typedef union {
|
||||
struct {
|
||||
size_t transSize;
|
||||
int steps;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int appendedProxyOps;
|
||||
} proxyCtrl;
|
||||
} ncclProfilerEventStateArgs_v2_t;
|
||||
|
||||
typedef struct {
|
||||
const char* name;
|
||||
|
||||
// init - initialize the profiler plugin
|
||||
// Input
|
||||
// - context : opaque profiler context object for separating profiler behavior across comms
|
||||
// Output
|
||||
// - eActivationMask: bitmask of active events set by the plugin
|
||||
ncclResult_t (*init)(void** context, int* eActivationMask);
|
||||
|
||||
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
// - eDescr : pointer to ncclProfilerEventDescr_t object
|
||||
// Output
|
||||
// - eHandle: return event handle for supplied event descriptor object
|
||||
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr);
|
||||
|
||||
// stopEvent - stop/finalize an event inside and event set
|
||||
// Input
|
||||
// - eHandle: handle to event object
|
||||
ncclResult_t (*stopEvent)(void* eHandle);
|
||||
|
||||
// recordEventState - record event state transitions and event attribute updates
|
||||
// Input
|
||||
// - eHandle : handle to event object created through startEvent
|
||||
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
|
||||
// - eState : event state transition
|
||||
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v2_t eState, ncclProfilerEventStateArgs_v2_t* eStateArgs);
|
||||
|
||||
// finalize - finalize the profiler plugin
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
ncclResult_t (*finalize)(void* context);
|
||||
} ncclProfiler_v2_t;
|
||||
|
||||
typedef ncclProfilerEventDescr_v2_t ncclProfilerEventDescr_t;
|
||||
typedef ncclProfilerEventState_v2_t ncclProfilerEventState_t;
|
||||
typedef ncclProfilerEventStateArgs_v2_t ncclProfilerEventStateArgs_t;
|
||||
typedef ncclProfiler_v2_t ncclProfiler_t;
|
||||
|
||||
typedef struct {
|
||||
uint8_t type; // event type descriptor: ncclProfileColl, ...
|
||||
void* parentObj; // pointer to the profiler parent object (for coll is the group)
|
||||
int rank; // originating rank
|
||||
union {
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
uint64_t seqNumber;
|
||||
uint8_t func;
|
||||
void const* sendBuff;
|
||||
void* recvBuff;
|
||||
size_t count;
|
||||
int root;
|
||||
uint8_t datatype;
|
||||
uint32_t op;
|
||||
size_t trafficBytes;
|
||||
uint8_t nMaxChannels;
|
||||
uint8_t nWarps;
|
||||
uint8_t algo;
|
||||
uint8_t proto;
|
||||
int isCollnet;
|
||||
int isNvls;
|
||||
} coll;
|
||||
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
uint8_t func;
|
||||
void* buff;
|
||||
uint8_t datatype;
|
||||
size_t count;
|
||||
int peer;
|
||||
} p2p;
|
||||
|
||||
struct {
|
||||
pid_t pid; // pid of the originating process
|
||||
uint8_t channelId; // channel id for this proxy operation
|
||||
int peer; // remote rank for send/recv
|
||||
int nSteps; // number of steps for this proxy operation
|
||||
int chunkSize; // amount of data transferred by this proxy operation
|
||||
int isSend;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int step;
|
||||
} proxyStep;
|
||||
};
|
||||
} ncclProfilerEventDescr_v1_t;
|
||||
|
||||
typedef ncclProfilerEventState_v2_t ncclProfilerEventState_v1_t;
|
||||
typedef ncclProfilerEventStateArgs_v2_t ncclProfilerEventStateArgs_v1_t;
|
||||
|
||||
typedef struct {
|
||||
const char* name;
|
||||
|
||||
// init - initialize the profiler plugin
|
||||
// Input
|
||||
// - context : opaque profiler context object for separating profiler behavior across comms
|
||||
// Output
|
||||
// - eActivationMask: bitmask of active events set by the plugin
|
||||
ncclResult_t (*init)(void** context, int* eActivationMask);
|
||||
|
||||
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
// - eDescr : pointer to ncclProfilerEventDescr_t object
|
||||
// Output
|
||||
// - eHandle: return event handle for supplied event descriptor object
|
||||
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr);
|
||||
|
||||
// stopEvent - stop/finalize an event inside and event set
|
||||
// Input
|
||||
// - eHandle: handle to event object
|
||||
ncclResult_t (*stopEvent)(void* eHandle);
|
||||
|
||||
// recordEventState - record event state transitions and event attribute updates
|
||||
// Input
|
||||
// - eHandle : handle to event object created through startEvent
|
||||
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
|
||||
// - eState : event state transition
|
||||
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs);
|
||||
|
||||
// finalize - finalize the profiler plugin
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
ncclResult_t (*finalize)(void* context);
|
||||
} ncclProfiler_v1_t;
|
||||
|
||||
#endif
|
||||
@@ -1,149 +0,0 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_TUNER_H_
|
||||
#define NCCL_TUNER_H_
|
||||
|
||||
#include "nccl.h"
|
||||
#include "nccl_common.h"
|
||||
|
||||
// API to be implemented by external tuner
|
||||
typedef struct {
|
||||
// Name of the tuner
|
||||
const char* name;
|
||||
|
||||
// Initializes tuner states.
|
||||
// Inputs:
|
||||
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
|
||||
// - nNodes: number of nodes in current communicator.
|
||||
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
|
||||
// Outputs:
|
||||
// - context: tuner context object
|
||||
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
|
||||
|
||||
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
|
||||
// Inputs:
|
||||
// - context: tuner context object
|
||||
// - collType: collective type , e.g., allreduce, allgather…
|
||||
// - nBytes: collective size in bytes
|
||||
// - numPipeOps: number of operations in the group
|
||||
// - numAlgo: number of algorithms in collCostTable
|
||||
// - numProto: number of protocols in collCostTable
|
||||
// - regBuff: can register user buffer
|
||||
//
|
||||
// Outputs:
|
||||
// - nChannels: number of channels (hence SMs) to be used.
|
||||
//
|
||||
// InOut:
|
||||
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
|
||||
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
|
||||
//
|
||||
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
|
||||
// default tuning for the given collective.
|
||||
// Also, the plugin is allowed to not set any output, or set only the
|
||||
// algorithm and protocol, but not only the algorithm or only the protocol.
|
||||
// Unset fields will be set automatically by NCCL.
|
||||
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
|
||||
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
|
||||
int regBuff, int* nChannels);
|
||||
|
||||
// Terminates the plugin and cleans up any resources that the plugin allocated.
|
||||
// context: tuner context object
|
||||
ncclResult_t (*destroy)(void* context);
|
||||
} ncclTuner_v4_t;
|
||||
|
||||
typedef ncclTuner_v4_t ncclTuner_t;
|
||||
|
||||
#define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v4"
|
||||
|
||||
// API to be implemented by external tuner
|
||||
typedef struct {
|
||||
// Name of the tuner
|
||||
const char* name;
|
||||
|
||||
// Initializes tuner states.
|
||||
// Inputs:
|
||||
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
|
||||
// - nNodes: number of nodes in current communicator.
|
||||
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
|
||||
// Outputs:
|
||||
// - context: tuner context object
|
||||
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
|
||||
|
||||
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
|
||||
// Inputs:
|
||||
// - context: tuner context object
|
||||
// - collType: collective type , e.g., allreduce, allgather…
|
||||
// - nBytes: collective size in bytes
|
||||
// - numPipeOps: number of operations in the group
|
||||
// - numAlgo: number of algorithms in collCostTable
|
||||
// - numProto: number of protocols in collCostTable
|
||||
//
|
||||
// Outputs:
|
||||
// - nChannels: number of channels (hence SMs) to be used.
|
||||
//
|
||||
// InOut:
|
||||
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
|
||||
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
|
||||
//
|
||||
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
|
||||
// default tuning for the given collective.
|
||||
// Also, the plugin is allowed to not set any output, or set only the
|
||||
// algorithm and protocol, but not only the algorithm or only the protocol.
|
||||
// Unset fields will be set automatically by NCCL.
|
||||
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
|
||||
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
|
||||
int* nChannels);
|
||||
|
||||
// Terminates the plugin and cleans up any resources that the plugin allocated.
|
||||
// context: tuner context object
|
||||
ncclResult_t (*destroy)(void* context);
|
||||
} ncclTuner_v3_t;
|
||||
|
||||
// API to be implemented by external tuner
|
||||
typedef struct {
|
||||
// Name of the tuner
|
||||
const char* name;
|
||||
|
||||
// Initializes tuner states.
|
||||
// Inputs:
|
||||
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
|
||||
// - nNodes: number of nodes in current communicator.
|
||||
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
|
||||
// Outputs:
|
||||
// - context: tuner context object
|
||||
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
|
||||
|
||||
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
|
||||
// Inputs:
|
||||
// - context: tuner context object
|
||||
// - collType: collective type , e.g., allreduce, allgather…
|
||||
// - nBytes: collective size in bytes
|
||||
// - collNetTypeSupport: whether collnet supports this type
|
||||
// - nvlsTypeSupport: whether nvlink sharp supports this time
|
||||
// - numPipeOps: number of operations in the group
|
||||
//
|
||||
// Outputs:
|
||||
// - algorithm: selected algorithm to be used for the given collective
|
||||
// - protocol: selected protocol to be used for the give collective
|
||||
// - nChannels: number of channels (hence SMs) to be used.
|
||||
//
|
||||
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
|
||||
// default tuning for the given collective.
|
||||
// Also, the plugin is allowed to not set any output, or set only the
|
||||
// algorithm and protocol, but not only the algorithm or only the protocol.
|
||||
// Unset fields will be set automatically by NCCL.
|
||||
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
|
||||
int collNetSupport, int nvlsSupport, int numPipeOps,
|
||||
int* algorithm, int* protocol, int* nChannels);
|
||||
|
||||
// Terminates the plugin and cleans up any resources that the plugin allocated.
|
||||
// context: tuner context object
|
||||
ncclResult_t (*destroy)(void* context);
|
||||
} ncclTuner_v2_t;
|
||||
|
||||
#endif
|
||||
@@ -18,7 +18,6 @@ ncclResult_t ncclNetPluginLoad(struct ncclComm* comm);
|
||||
ncclResult_t ncclNetPluginUnload(struct ncclComm* comm);
|
||||
ncclResult_t ncclNetInit(struct ncclComm* comm);
|
||||
ncclResult_t ncclNetFinalize(struct ncclComm* comm);
|
||||
int ncclNetVersion(struct ncclComm* comm);
|
||||
|
||||
// Test whether the current GPU support GPU Direct RDMA.
|
||||
ncclResult_t ncclGpuGdrSupport(struct ncclComm* comm, int* gdrSupport);
|
||||
|
||||
@@ -26,6 +26,7 @@ typedef struct {
|
||||
|
||||
typedef ncclNetDeviceHandle_v7_t ncclNetDeviceHandle_v8_t;
|
||||
typedef ncclNetDeviceHandle_v8_t ncclNetDeviceHandle_v9_t;
|
||||
typedef ncclNetDeviceHandle_v9_t ncclNetDeviceHandle_t;
|
||||
typedef ncclNetDeviceHandle_v9_t ncclNetDeviceHandle_v10_t;
|
||||
typedef ncclNetDeviceHandle_v10_t ncclNetDeviceHandle_t;
|
||||
|
||||
#endif
|
||||
|
||||
@@ -37,9 +37,10 @@
|
||||
#define NVTX_SID_CommInitRankScalable 17 // same schema as NVTX_SID_CommInitRank
|
||||
#define NVTX_SID_CommSplit 18
|
||||
#define NVTX_SID_CommFinalize 19
|
||||
// When adding new schema IDs, DO NOT re-use/overlap with the enum schema ID below!
|
||||
|
||||
// Define static schema ID for the reduction operation.
|
||||
#define NVTX_PAYLOAD_ENTRY_NCCL_REDOP 19 + NVTX_PAYLOAD_ENTRY_TYPE_SCHEMA_ID_STATIC_START
|
||||
#define NVTX_PAYLOAD_ENTRY_NCCL_REDOP 20 + NVTX_PAYLOAD_ENTRY_TYPE_SCHEMA_ID_STATIC_START
|
||||
|
||||
extern const nvtxDomainHandle_t ncclNvtxDomainHandle;
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_NET_H_
|
||||
#define NCCL_NET_H_
|
||||
|
||||
#include "nccl.h"
|
||||
#include "nccl_common.h"
|
||||
#include "net_device.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#define NCCL_NET_HANDLE_MAXSIZE 128
|
||||
//Maximum value NCCL can accept for maxP2pBytes and maxCollBytes net properties
|
||||
#define NCCL_MAX_NET_SIZE_BYTES (1*1024*1024*1024*1024L)
|
||||
#define NCCL_NET_OPTIONAL_RECV_COMPLETION 0x1
|
||||
|
||||
#define MAX_NET_SIZE (1024*1024*1024L) // Rather than send INT_MAX which is 2G-1, send a power of two.
|
||||
#define MAX_COLLNET_SIZE (512*1024*1024L) //Set for initial collent plugins when size was not dynamically queried
|
||||
|
||||
#define NCCL_PTR_HOST 0x1
|
||||
#define NCCL_PTR_CUDA 0x2
|
||||
#define NCCL_PTR_DMABUF 0x4
|
||||
|
||||
// Maximum number of requests per comm object
|
||||
#define NCCL_NET_MAX_REQUESTS 32
|
||||
|
||||
// Max number of ncclNet objects which can live in the same process
|
||||
#define NCCL_NET_MAX_PLUGINS 3
|
||||
|
||||
// NCCL core profiler callback for network defined events instrumentation
|
||||
typedef ncclResult_t (*ncclProfilerCallback_t)(void** eHandle, int type, void* pHandle, int64_t pluginId, void* extData);
|
||||
|
||||
#include "net/net_v10.h"
|
||||
#include "net/net_v9.h"
|
||||
#include "net/net_v8.h"
|
||||
#include "net/net_v7.h"
|
||||
#include "net/net_v6.h"
|
||||
|
||||
typedef ncclNet_v10_t ncclNet_t;
|
||||
typedef ncclCollNet_v10_t ncclCollNet_t;
|
||||
typedef ncclNetSGE_v10_t ncclNetSGE_t;
|
||||
typedef ncclNetProperties_v10_t ncclNetProperties_t;
|
||||
typedef ncclNetVDeviceProps_v10_t ncclNetVDeviceProps_t;
|
||||
typedef ncclNetCommConfig_v10_t ncclNetCommConfig_t;
|
||||
|
||||
#define NCCL_NET_MAX_DEVS_PER_NIC NCCL_NET_MAX_DEVS_PER_NIC_V10
|
||||
|
||||
#define NCCL_NET_PLUGIN_SYMBOL ncclNetPlugin_v10
|
||||
#define NCCL_COLLNET_PLUGIN_SYMBOL ncclCollNetPlugin_v10
|
||||
|
||||
// context passed from RCCL lib to n/w plugin
|
||||
typedef struct {
|
||||
// channel id
|
||||
uint32_t chId;
|
||||
} ncclNet_ctxt_t;
|
||||
|
||||
#endif // end include guard
|
||||
@@ -0,0 +1,69 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_PROFILER_H_
|
||||
#define NCCL_PROFILER_H_
|
||||
|
||||
enum {
|
||||
ncclProfileGroup = (1 << 0), // group event type
|
||||
ncclProfileColl = (1 << 1), // host collective call event type
|
||||
ncclProfileP2p = (1 << 2), // host point-to-point call event type
|
||||
ncclProfileProxyOp = (1 << 3), // proxy operation event type
|
||||
ncclProfileProxyStep = (1 << 4), // proxy step event type
|
||||
ncclProfileProxyCtrl = (1 << 5), // proxy control event type
|
||||
ncclProfileKernelCh = (1 << 6), // kernel channel event type
|
||||
ncclProfileNetPlugin = (1 << 7), // network plugin-defined, events
|
||||
};
|
||||
|
||||
typedef enum {
|
||||
ncclProfilerProxyOpSendPosted,
|
||||
ncclProfilerProxyOpSendRemFifoWait,
|
||||
ncclProfilerProxyOpSendTransmitted,
|
||||
ncclProfilerProxyOpSendDone,
|
||||
ncclProfilerProxyOpRecvPosted,
|
||||
ncclProfilerProxyOpRecvReceived,
|
||||
ncclProfilerProxyOpRecvTransmitted,
|
||||
ncclProfilerProxyOpRecvDone,
|
||||
|
||||
/* Legacy proxy profiler states */
|
||||
ncclProfilerProxyStepSendGPUWait,
|
||||
ncclProfilerProxyStepSendWait,
|
||||
ncclProfilerProxyStepRecvWait,
|
||||
ncclProfilerProxyStepRecvFlushWait,
|
||||
ncclProfilerProxyStepRecvGPUWait,
|
||||
|
||||
/* Legacy proxy control states */
|
||||
ncclProfilerProxyCtrlIdle,
|
||||
ncclProfilerProxyCtrlActive,
|
||||
ncclProfilerProxyCtrlSleep,
|
||||
ncclProfilerProxyCtrlWakeup,
|
||||
ncclProfilerProxyCtrlAppend,
|
||||
ncclProfilerProxyCtrlAppendEnd,
|
||||
} ncclProfilerEventState_t;
|
||||
|
||||
typedef ncclProfilerEventState_t ncclProfilerEventState_v1_t;
|
||||
typedef ncclProfilerEventState_t ncclProfilerEventState_v2_t;
|
||||
typedef ncclProfilerEventState_t ncclProfilerEventState_v3_t;
|
||||
|
||||
#include <cstdint>
|
||||
#include "profiler/profiler_v3.h"
|
||||
#include "profiler/profiler_v2.h"
|
||||
#include "profiler/profiler_v1.h"
|
||||
|
||||
typedef ncclProfiler_v3_t ncclProfiler_t;
|
||||
typedef ncclProfilerEventDescr_v3_t ncclProfilerEventDescr_t;
|
||||
typedef ncclProfilerEventStateArgs_v3_t ncclProfilerEventStateArgs_t;
|
||||
|
||||
#define NCCL_PROFILER_NET_VER_BITS (16)
|
||||
#define NCCL_PROFILER_NET_VER_MASK (~0U >> NCCL_PROFILER_NET_VER_BITS)
|
||||
#define NCCL_PROFILER_NET_TYPE_MASK (~0U << NCCL_PROFILER_NET_VER_BITS)
|
||||
|
||||
typedef enum {
|
||||
NCCL_PROFILER_NET_TYPE_IB = (1U << NCCL_PROFILER_NET_VER_BITS),
|
||||
NCCL_PROFILER_NET_TYPE_SOCK = (2U << NCCL_PROFILER_NET_VER_BITS),
|
||||
} ncclProfilerNetType;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,22 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_TUNER_H_
|
||||
#define NCCL_TUNER_H_
|
||||
|
||||
#include "nccl.h"
|
||||
#include "nccl_common.h"
|
||||
|
||||
#include "tuner/tuner_v4.h"
|
||||
#include "tuner/tuner_v3.h"
|
||||
#include "tuner/tuner_v2.h"
|
||||
|
||||
typedef ncclTuner_v4_t ncclTuner_t;
|
||||
|
||||
#define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v4"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef NET_V10_H_
|
||||
#define NET_V10_H_
|
||||
|
||||
#define NCCL_NET_MAX_DEVS_PER_NIC_V10 4
|
||||
|
||||
typedef struct {
|
||||
int ndevs;
|
||||
int devs[NCCL_NET_MAX_DEVS_PER_NIC_V10];
|
||||
} ncclNetVDeviceProps_v10_t;
|
||||
|
||||
#define NCCL_NET_TRAFFIC_CLASS_UNDEF -1
|
||||
|
||||
typedef struct {
|
||||
// Plugin-specific TC value
|
||||
int trafficClass;
|
||||
} ncclNetCommConfig_v10_t;
|
||||
|
||||
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int regIsGlobal; // regMr is not tied to a particular comm
|
||||
int forceFlush; // Force a flush on receives
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
ncclNetDeviceType netDeviceType; // Network offload type
|
||||
int netDeviceVersion; // Version number for network offload
|
||||
ncclNetVDeviceProps_v10_t vProps;
|
||||
size_t maxP2pBytes; // Max transfer size for point-to-point operations
|
||||
size_t maxCollBytes; // Max transfer size for collective operations
|
||||
} ncclNetProperties_v10_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v10_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*connect)(int dev, ncclNetCommConfig_v10_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_v10_t** sendDevComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v10_t** recvDevComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
|
||||
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
|
||||
|
||||
// Notify the plugin that a recv has completed by the device
|
||||
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
|
||||
|
||||
// Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller
|
||||
// what index this new vNIC exists at
|
||||
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v10_t* props);
|
||||
} ncclNet_v10_t;
|
||||
|
||||
typedef struct {
|
||||
void* mhandle;
|
||||
void* address;
|
||||
size_t size;
|
||||
} ncclNetSGE_v10_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v10_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, size_t count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v10_t* recvParts,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
void* sendMhandle, void** request);
|
||||
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v10_t* sendParts, void* recvData,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp,
|
||||
void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Create a virtual NIC given the specified properties, which can be accessed at device index d
|
||||
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v10_t* props);
|
||||
} ncclCollNet_v10_t;
|
||||
|
||||
#endif // end include guard
|
||||
@@ -0,0 +1,113 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_V6_H_
|
||||
#define NET_V6_H_
|
||||
|
||||
#define NCCL_NET_MAX_REQUESTS_V6 8
|
||||
|
||||
// v6 struct for backwards compatibility
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
} ncclNetProperties_v6_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclNet_v6_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclCollNet_v6_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,120 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_V7_H_
|
||||
#define NET_V7_H_
|
||||
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
ncclNetDeviceType netDeviceType; // Network offload type
|
||||
int netDeviceVersion; // Version number for network offload
|
||||
} ncclNetProperties_v7_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v7_t** sendDevComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v7_t** recvDevComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
|
||||
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
|
||||
|
||||
// Notify the plugin that a recv has completed by the device
|
||||
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
|
||||
} ncclNet_v7_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclCollNet_v7_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,134 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_V8_H_
|
||||
#define NET_V8_H_
|
||||
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int regIsGlobal; // regMr is not tied to a particular comm
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
ncclNetDeviceType netDeviceType; // Network offload type
|
||||
int netDeviceVersion; // Version number for network offload
|
||||
} ncclNetProperties_v8_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v8_t** sendDevComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v8_t** recvDevComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
|
||||
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
|
||||
|
||||
// Notify the plugin that a recv has completed by the device
|
||||
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
|
||||
} ncclNet_v8_t;
|
||||
|
||||
typedef struct {
|
||||
void* mhandle;
|
||||
void* address;
|
||||
uint32_t size;
|
||||
} ncclNetSGE_v8_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v8_t* recvParts,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
void* sendMhandle, void** request);
|
||||
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v8_t* sendParts, void* recvData,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp,
|
||||
void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
} ncclCollNet_v8_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,152 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_V9_H_
|
||||
#define NET_V9_H_
|
||||
|
||||
#define NCCL_NET_MAX_DEVS_PER_NIC_V9 4
|
||||
|
||||
typedef struct {
|
||||
int ndevs;
|
||||
int devs[NCCL_NET_MAX_DEVS_PER_NIC_V9];
|
||||
} ncclNetVDeviceProps_v9_t;
|
||||
|
||||
typedef struct {
|
||||
char* name; // Used mostly for logging.
|
||||
char* pciPath; // Path to the PCI device in /sys.
|
||||
uint64_t guid; // Unique identifier for the NIC chip. Important for
|
||||
// cards with multiple PCI functions (Physical or virtual).
|
||||
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
|
||||
int regIsGlobal; // regMr is not tied to a particular comm
|
||||
int forceFlush; // Force a flush on receives
|
||||
int speed; // Port speed in Mbps.
|
||||
int port; // Port number.
|
||||
float latency; // Network latency
|
||||
int maxComms; // Maximum number of comms we can create
|
||||
int maxRecvs; // Maximum number of grouped receives.
|
||||
ncclNetDeviceType netDeviceType; // Network offload type
|
||||
int netDeviceVersion; // Version number for network offload
|
||||
ncclNetVDeviceProps_v9_t vProps;
|
||||
size_t maxP2pBytes; // Max transfer size for point-to-point operations
|
||||
size_t maxCollBytes; // Max transfer size for collective operations
|
||||
} ncclNetProperties_v9_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create a connection.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Connect to a handle and return a sending comm object for that peer.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with sendComm == NULL with the expectation that
|
||||
// it will be called again until sendComm != NULL.
|
||||
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v9_t** sendDevComm);
|
||||
// Finalize connection establishment after remote peer has called connect.
|
||||
// This call must not block for the connection to be established, and instead
|
||||
// should return successfully with recvComm == NULL with the expectation that
|
||||
// it will be called again until recvComm != NULL.
|
||||
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
|
||||
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v9_t** recvDevComm);
|
||||
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
|
||||
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* comm, void* mhandle);
|
||||
// Asynchronous send to a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request);
|
||||
// Asynchronous recv from a peer.
|
||||
// May return request == NULL if the call cannot be performed (or would block)
|
||||
ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* sizes);
|
||||
// Close and free send/recv comm objects
|
||||
ncclResult_t (*closeSend)(void* sendComm);
|
||||
ncclResult_t (*closeRecv)(void* recvComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
|
||||
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
|
||||
|
||||
// Notify the plugin that a recv has completed by the device
|
||||
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
|
||||
|
||||
// Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller
|
||||
// what index this new vNIC exists at
|
||||
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v9_t* props);
|
||||
} ncclNet_v9_t;
|
||||
|
||||
typedef struct {
|
||||
void* mhandle;
|
||||
void* address;
|
||||
size_t size;
|
||||
} ncclNetSGE_v9_t;
|
||||
|
||||
typedef struct {
|
||||
// Name of the collective network (mainly for logs)
|
||||
const char* name;
|
||||
// Initialize the collective network.
|
||||
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
|
||||
// Return the number of adapters capable of doing collective operations.
|
||||
// If ndev returns 0, all other functions might be set to NULL.
|
||||
ncclResult_t (*devices)(int* ndev);
|
||||
// Get various device properties.
|
||||
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
|
||||
// Create a receiving object and provide a handle to connect to it. The
|
||||
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
|
||||
// between ranks to create connections.
|
||||
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
|
||||
// Create a group for collective operations. handles have been created
|
||||
// using listen() above. rank indicates caller's rank in the collective network.
|
||||
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
|
||||
// Returns whether a reduction operation on a data type is supported.
|
||||
// 1 for supported, 0 otherwise.
|
||||
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
|
||||
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
|
||||
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
|
||||
/* DMA-BUF support */
|
||||
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
|
||||
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
|
||||
// Performs an asynchronous allreduce operation on the collective group.
|
||||
// May return request == NULL if the call cannot be performed (or would block).
|
||||
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, size_t count,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
|
||||
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v9_t* recvParts,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
void* sendMhandle, void** request);
|
||||
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v9_t* sendParts, void* recvData,
|
||||
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
|
||||
ncclDataType_t dataType, ncclRedOp_t redOp,
|
||||
void* recvMhandle, void** request);
|
||||
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
|
||||
// visible to the GPU
|
||||
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
|
||||
// Test whether a request is complete. If size is not NULL, it returns the
|
||||
// number of bytes sent/received.
|
||||
ncclResult_t (*test)(void* request, int* done, int* size);
|
||||
// Close and free collective comm objects
|
||||
ncclResult_t (*closeColl)(void* collComm);
|
||||
ncclResult_t (*closeListen)(void* listenComm);
|
||||
|
||||
// Create a virtual NIC given the specified properties, which can be accessed at device index d
|
||||
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v9_t* props);
|
||||
} ncclCollNet_v9_t;
|
||||
|
||||
#endif // end include guard
|
||||
@@ -0,0 +1,18 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NCCL_PLUGIN_H_
|
||||
#define NCCL_PLUGIN_H_
|
||||
|
||||
#include "nccl.h"
|
||||
|
||||
void* ncclOpenNetPluginLib(const char* name);
|
||||
void* ncclOpenTunerPluginLib(const char* name);
|
||||
void* ncclOpenProfilerPluginLib(const char* name);
|
||||
void* ncclGetNetPluginLib(void);
|
||||
ncclResult_t ncclClosePluginLib(void* handle);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,13 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_IB_H_
|
||||
#define NET_IB_H_
|
||||
|
||||
#include "nccl_profiler.h"
|
||||
#include "net_ib_v1.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,34 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_IB_V1_H_
|
||||
#define NET_IB_V1_H_
|
||||
|
||||
#define NCCL_PROFILER_NET_IB_VER 1
|
||||
|
||||
enum {
|
||||
ncclProfileQp = (1 << 0),
|
||||
};
|
||||
|
||||
// The data structure version is encoded in the plugin identifier bitmask and
|
||||
// passed to NCCL core through the profiler callback. NCCL copies the plugin
|
||||
// identifier in the event descriptor before calling the profiler startEvent
|
||||
// function. The profiler should inspect the plugin id to find out the source
|
||||
// plugin as well as the version of the event struct
|
||||
typedef struct {
|
||||
uint8_t type; // event type (plugin defined)
|
||||
union {
|
||||
struct {
|
||||
int device; // network device id
|
||||
uint64_t wr_id; // work request id
|
||||
int opcode; // ibv opcode
|
||||
int qpNum; // QP number
|
||||
size_t length; // work request data length
|
||||
} qp;
|
||||
};
|
||||
} ncclProfilerNetIbDescr_v1_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,13 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_SOCKET_H_
|
||||
#define NET_SOCKET_H_
|
||||
|
||||
#include "nccl_profiler.h"
|
||||
#include "net_socket_v1.h"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef NET_SOCKET_V1_H_
|
||||
#define NET_SOCKET_V1_H_
|
||||
|
||||
#define NCCL_PROFILER_NET_SOCKET_VER 1
|
||||
|
||||
enum {
|
||||
ncclProfileSocket = (1 << 0),
|
||||
};
|
||||
|
||||
// The data structure version is encoded in the plugin identifier bitmask and
|
||||
// passed to NCCL core through the profiler callback. NCCL copies the plugin
|
||||
// identifier in the event descriptor before calling the profiler startEvent
|
||||
// function. The profiler should inspect the plugin id to find out the source
|
||||
// plugin as well as the version of the event struct
|
||||
typedef struct {
|
||||
uint8_t type; // event type (plugin defined)
|
||||
union {
|
||||
struct {
|
||||
int fd;
|
||||
int op;
|
||||
size_t length;
|
||||
} sock;
|
||||
};
|
||||
} ncclProfilerNetSockDescr_v1_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,107 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef PROFILER_V1_H_
|
||||
#define PROFILER_V1_H_
|
||||
|
||||
typedef struct {
|
||||
uint8_t type; // event type descriptor: ncclProfileColl, ...
|
||||
void* parentObj; // pointer to the profiler parent object (for coll is the group)
|
||||
int rank; // originating rank
|
||||
union {
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
uint64_t seqNumber;
|
||||
uint8_t func;
|
||||
void const* sendBuff;
|
||||
void* recvBuff;
|
||||
size_t count;
|
||||
int root;
|
||||
uint8_t datatype;
|
||||
uint32_t op;
|
||||
size_t trafficBytes;
|
||||
uint8_t nMaxChannels;
|
||||
uint8_t nWarps;
|
||||
uint8_t algo;
|
||||
uint8_t proto;
|
||||
int isCollnet;
|
||||
int isNvls;
|
||||
} coll;
|
||||
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
uint8_t func;
|
||||
void* buff;
|
||||
uint8_t datatype;
|
||||
size_t count;
|
||||
int peer;
|
||||
} p2p;
|
||||
|
||||
struct {
|
||||
pid_t pid; // pid of the originating process
|
||||
uint8_t channelId; // channel id for this proxy operation
|
||||
int peer; // remote rank for send/recv
|
||||
int nSteps; // number of steps for this proxy operation
|
||||
int chunkSize; // amount of data transferred by this proxy operation
|
||||
int isSend;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int step;
|
||||
} proxyStep;
|
||||
};
|
||||
} ncclProfilerEventDescr_v1_t;
|
||||
|
||||
typedef union {
|
||||
struct {
|
||||
size_t transSize;
|
||||
int steps;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int appendedProxyOps;
|
||||
} proxyCtrl;
|
||||
} ncclProfilerEventStateArgs_v1_t;
|
||||
|
||||
typedef struct {
|
||||
const char* name;
|
||||
|
||||
// init - initialize the profiler plugin
|
||||
// Input
|
||||
// - context : opaque profiler context object for separating profiler behavior across comms
|
||||
// Output
|
||||
// - eActivationMask: bitmask of active events set by the plugin
|
||||
ncclResult_t (*init)(void** context, int* eActivationMask);
|
||||
|
||||
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
// - eDescr : pointer to ncclProfilerEventDescr_t object
|
||||
// Output
|
||||
// - eHandle: return event handle for supplied event descriptor object
|
||||
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr);
|
||||
|
||||
// stopEvent - stop/finalize an event inside and event set
|
||||
// Input
|
||||
// - eHandle: handle to event object
|
||||
ncclResult_t (*stopEvent)(void* eHandle);
|
||||
|
||||
// recordEventState - record event state transitions and event attribute updates
|
||||
// Input
|
||||
// - eHandle : handle to event object created through startEvent
|
||||
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
|
||||
// - eState : event state transition
|
||||
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs);
|
||||
|
||||
// finalize - finalize the profiler plugin
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
ncclResult_t (*finalize)(void* context);
|
||||
} ncclProfiler_v1_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,104 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef PROFILER_V2_H_
|
||||
#define PROFILER_V2_H_
|
||||
|
||||
typedef struct {
|
||||
uint8_t type; // event type descriptor: ncclProfileColl, ...
|
||||
void* parentObj; // pointer to the profiler parent object (for coll is the group)
|
||||
int rank; // originating rank
|
||||
union {
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
uint64_t seqNumber;
|
||||
const char* func;
|
||||
void const* sendBuff;
|
||||
void* recvBuff;
|
||||
size_t count;
|
||||
int root;
|
||||
const char* datatype;
|
||||
size_t trafficBytes;
|
||||
uint8_t nMaxChannels;
|
||||
uint8_t nWarps;
|
||||
const char* algo;
|
||||
const char* proto;
|
||||
} coll;
|
||||
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
const char* func;
|
||||
void* buff;
|
||||
const char* datatype;
|
||||
size_t count;
|
||||
int peer;
|
||||
} p2p;
|
||||
|
||||
struct {
|
||||
pid_t pid; // pid of the originating process
|
||||
uint8_t channelId; // channel id for this proxy operation
|
||||
int peer; // remote rank for send/recv
|
||||
int nSteps; // number of steps for this proxy operation
|
||||
int chunkSize; // amount of data transferred by this proxy operation
|
||||
int isSend;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int step;
|
||||
} proxyStep;
|
||||
};
|
||||
} ncclProfilerEventDescr_v2_t;
|
||||
|
||||
typedef union {
|
||||
struct {
|
||||
size_t transSize;
|
||||
int steps;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int appendedProxyOps;
|
||||
} proxyCtrl;
|
||||
} ncclProfilerEventStateArgs_v2_t;
|
||||
|
||||
typedef struct {
|
||||
const char* name;
|
||||
|
||||
// init - initialize the profiler plugin
|
||||
// Input
|
||||
// - context : opaque profiler context object for separating profiler behavior across comms
|
||||
// Output
|
||||
// - eActivationMask: bitmask of active events set by the plugin
|
||||
ncclResult_t (*init)(void** context, int* eActivationMask);
|
||||
|
||||
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
// - eDescr : pointer to ncclProfilerEventDescr_t object
|
||||
// Output
|
||||
// - eHandle: return event handle for supplied event descriptor object
|
||||
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr);
|
||||
|
||||
// stopEvent - stop/finalize an event inside and event set
|
||||
// Input
|
||||
// - eHandle: handle to event object
|
||||
ncclResult_t (*stopEvent)(void* eHandle);
|
||||
|
||||
// recordEventState - record event state transitions and event attribute updates
|
||||
// Input
|
||||
// - eHandle : handle to event object created through startEvent
|
||||
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
|
||||
// - eState : event state transition
|
||||
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v2_t eState, ncclProfilerEventStateArgs_v2_t* eStateArgs);
|
||||
|
||||
// finalize - finalize the profiler plugin
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
ncclResult_t (*finalize)(void* context);
|
||||
} ncclProfiler_v2_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,112 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef PROFILER_V3_H_
|
||||
#define PROFILER_V3_H_
|
||||
|
||||
typedef struct {
|
||||
uint8_t type; // event type descriptor: ncclProfileColl, ...
|
||||
void* parentObj; // pointer to the profiler parent object (for coll is the group)
|
||||
int rank; // originating rank
|
||||
union {
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
uint64_t seqNumber;
|
||||
const char* func;
|
||||
void const* sendBuff;
|
||||
void* recvBuff;
|
||||
size_t count;
|
||||
int root;
|
||||
const char* datatype;
|
||||
uint8_t nMaxChannels;
|
||||
uint8_t nWarps;
|
||||
const char* algo;
|
||||
const char* proto;
|
||||
} coll;
|
||||
|
||||
struct {
|
||||
const char* name;
|
||||
uint64_t commHash;
|
||||
const char* func;
|
||||
void* buff;
|
||||
const char* datatype;
|
||||
size_t count;
|
||||
int peer;
|
||||
} p2p;
|
||||
|
||||
struct {
|
||||
pid_t pid; // pid of the originating process
|
||||
uint8_t channelId; // channel id for this proxy operation
|
||||
int peer; // remote rank for send/recv
|
||||
int nSteps; // number of steps for this proxy operation
|
||||
int chunkSize; // amount of data transferred by this proxy operation
|
||||
int isSend;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int step;
|
||||
} proxyStep;
|
||||
|
||||
struct {
|
||||
uint8_t channelId;
|
||||
} kernelCh;
|
||||
|
||||
struct {
|
||||
int64_t id;
|
||||
void* data;
|
||||
} netPlugin;
|
||||
};
|
||||
} ncclProfilerEventDescr_v3_t;
|
||||
|
||||
typedef union {
|
||||
struct {
|
||||
size_t transSize;
|
||||
int steps;
|
||||
} proxyOp;
|
||||
|
||||
struct {
|
||||
int appendedProxyOps;
|
||||
} proxyCtrl;
|
||||
} ncclProfilerEventStateArgs_v3_t;
|
||||
|
||||
typedef struct {
|
||||
const char* name;
|
||||
|
||||
// init - initialize the profiler plugin
|
||||
// Input
|
||||
// - context : opaque profiler context object for separating profiler behavior across comms
|
||||
// Output
|
||||
// - eActivationMask: bitmask of active events set by the plugin
|
||||
ncclResult_t (*init)(void** context, int* eActivationMask);
|
||||
|
||||
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
// - eDescr : pointer to ncclProfilerEventDescr_t object
|
||||
// Output
|
||||
// - eHandle: return event handle for supplied event descriptor object
|
||||
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v3_t* eDescr);
|
||||
|
||||
// stopEvent - stop/finalize an event inside and event set
|
||||
// Input
|
||||
// - eHandle: handle to event object
|
||||
ncclResult_t (*stopEvent)(void* eHandle);
|
||||
|
||||
// recordEventState - record event state transitions and event attribute updates
|
||||
// Input
|
||||
// - eHandle : handle to event object created through startEvent
|
||||
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
|
||||
// - eState : event state transition
|
||||
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v3_t eState, ncclProfilerEventStateArgs_v3_t* eStateArgs);
|
||||
|
||||
// finalize - finalize the profiler plugin
|
||||
// Input
|
||||
// - context: opaque profiler context object
|
||||
ncclResult_t (*finalize)(void* context);
|
||||
} ncclProfiler_v3_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,53 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef TUNER_V2_H_
|
||||
#define TUNER_V2_H_
|
||||
|
||||
// API to be implemented by external tuner
|
||||
typedef struct {
|
||||
// Name of the tuner
|
||||
const char* name;
|
||||
|
||||
// Initializes tuner states.
|
||||
// Inputs:
|
||||
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
|
||||
// - nNodes: number of nodes in current communicator.
|
||||
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
|
||||
// Outputs:
|
||||
// - context: tuner context object
|
||||
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
|
||||
|
||||
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
|
||||
// Inputs:
|
||||
// - context: tuner context object
|
||||
// - collType: collective type , e.g., allreduce, allgather…
|
||||
// - nBytes: collective size in bytes
|
||||
// - collNetTypeSupport: whether collnet supports this type
|
||||
// - nvlsTypeSupport: whether nvlink sharp supports this time
|
||||
// - numPipeOps: number of operations in the group
|
||||
//
|
||||
// Outputs:
|
||||
// - algorithm: selected algorithm to be used for the given collective
|
||||
// - protocol: selected protocol to be used for the give collective
|
||||
// - nChannels: number of channels (hence SMs) to be used.
|
||||
//
|
||||
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
|
||||
// default tuning for the given collective.
|
||||
// Also, the plugin is allowed to not set any output, or set only the
|
||||
// algorithm and protocol, but not only the algorithm or only the protocol.
|
||||
// Unset fields will be set automatically by NCCL.
|
||||
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
|
||||
int collNetSupport, int nvlsSupport, int numPipeOps,
|
||||
int* algorithm, int* protocol, int* nChannels);
|
||||
|
||||
// Terminates the plugin and cleans up any resources that the plugin allocated.
|
||||
// context: tuner context object
|
||||
ncclResult_t (*destroy)(void* context);
|
||||
} ncclTuner_v2_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,55 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef TUNER_V3_H_
|
||||
#define TUNER_V3_H_
|
||||
|
||||
// API to be implemented by external tuner
|
||||
typedef struct {
|
||||
// Name of the tuner
|
||||
const char* name;
|
||||
|
||||
// Initializes tuner states.
|
||||
// Inputs:
|
||||
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
|
||||
// - nNodes: number of nodes in current communicator.
|
||||
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
|
||||
// Outputs:
|
||||
// - context: tuner context object
|
||||
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
|
||||
|
||||
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
|
||||
// Inputs:
|
||||
// - context: tuner context object
|
||||
// - collType: collective type , e.g., allreduce, allgather…
|
||||
// - nBytes: collective size in bytes
|
||||
// - numPipeOps: number of operations in the group
|
||||
// - numAlgo: number of algorithms in collCostTable
|
||||
// - numProto: number of protocols in collCostTable
|
||||
//
|
||||
// Outputs:
|
||||
// - nChannels: number of channels (hence SMs) to be used.
|
||||
//
|
||||
// InOut:
|
||||
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
|
||||
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
|
||||
//
|
||||
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
|
||||
// default tuning for the given collective.
|
||||
// Also, the plugin is allowed to not set any output, or set only the
|
||||
// algorithm and protocol, but not only the algorithm or only the protocol.
|
||||
// Unset fields will be set automatically by NCCL.
|
||||
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
|
||||
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
|
||||
int* nChannels);
|
||||
|
||||
// Terminates the plugin and cleans up any resources that the plugin allocated.
|
||||
// context: tuner context object
|
||||
ncclResult_t (*destroy)(void* context);
|
||||
} ncclTuner_v3_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,56 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef TUNER_V4_H_
|
||||
#define TUNER_V4_H_
|
||||
|
||||
// API to be implemented by external tuner
|
||||
typedef struct {
|
||||
// Name of the tuner
|
||||
const char* name;
|
||||
|
||||
// Initializes tuner states.
|
||||
// Inputs:
|
||||
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
|
||||
// - nNodes: number of nodes in current communicator.
|
||||
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
|
||||
// Outputs:
|
||||
// - context: tuner context object
|
||||
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
|
||||
|
||||
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
|
||||
// Inputs:
|
||||
// - context: tuner context object
|
||||
// - collType: collective type , e.g., allreduce, allgather…
|
||||
// - nBytes: collective size in bytes
|
||||
// - numPipeOps: number of operations in the group
|
||||
// - numAlgo: number of algorithms in collCostTable
|
||||
// - numProto: number of protocols in collCostTable
|
||||
// - regBuff: can register user buffer
|
||||
//
|
||||
// Outputs:
|
||||
// - nChannels: number of channels (hence SMs) to be used.
|
||||
//
|
||||
// InOut:
|
||||
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
|
||||
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
|
||||
//
|
||||
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
|
||||
// default tuning for the given collective.
|
||||
// Also, the plugin is allowed to not set any output, or set only the
|
||||
// algorithm and protocol, but not only the algorithm or only the protocol.
|
||||
// Unset fields will be set automatically by NCCL.
|
||||
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
|
||||
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
|
||||
int regBuff, int* nChannels);
|
||||
|
||||
// Terminates the plugin and cleans up any resources that the plugin allocated.
|
||||
// context: tuner context object
|
||||
ncclResult_t (*destroy)(void* context);
|
||||
} ncclTuner_v4_t;
|
||||
|
||||
#endif
|
||||
@@ -17,6 +17,18 @@ struct ncclTaskP2p;
|
||||
struct ncclInfo;
|
||||
struct ncclComm;
|
||||
struct ncclProxyOp;
|
||||
struct ncclProxyConnector;
|
||||
|
||||
struct ncclProfilerProxy {
|
||||
bool initialized;
|
||||
uint64_t* workStarted/*[MAXCHANNELS]*/;
|
||||
uint64_t* workCompleted/*[MAXCHANNELS]*/;
|
||||
uint64_t workCounter[MAXCHANNELS]; // host work counter
|
||||
struct ncclProxyConnector sendProxyConn[MAXCHANNELS];
|
||||
struct ncclProxyConnector recvProxyConn[MAXCHANNELS];
|
||||
};
|
||||
|
||||
extern int ncclProfilerEventMask;
|
||||
|
||||
// Plugin Init/Finalize Wrappers
|
||||
ncclResult_t ncclProfilerPluginInit(struct ncclComm* comm);
|
||||
@@ -44,6 +56,10 @@ ncclResult_t ncclProfilerStopProxyStepEvent(int sub, struct ncclProxyArgs* args,
|
||||
ncclResult_t ncclProfilerStartProxyCtrlEvent(void* profilerContext, void** eHandle);
|
||||
ncclResult_t ncclProfilerStopProxyCtrlEvent(void* eHandle);
|
||||
|
||||
// Kernel Channel Start/Stop Event Wrappers
|
||||
ncclResult_t ncclProfilerStartKernelChEvent(struct ncclProxyArgs* args, int s);
|
||||
ncclResult_t ncclProfilerStopKernelChEvent(struct ncclProxyArgs* args, int s);
|
||||
|
||||
// Record Event Wrappers
|
||||
ncclResult_t ncclProfilerRecordProxyOpEventState(int sub, struct ncclProxyArgs* args, int steps, size_t transSize, ncclProfilerEventState_t eState);
|
||||
ncclResult_t ncclProfilerRecordProxyStepEventState(int sub, struct ncclProxyArgs* args, int stepId, ncclProfilerEventState_t eState);
|
||||
@@ -51,5 +67,10 @@ ncclResult_t ncclProfilerRecordProxyCtrlEventState(void*eHandle, int appended, n
|
||||
|
||||
// Profiler utility functions
|
||||
ncclResult_t ncclProfilerAddPidToProxyOp(struct ncclProxyOp* op);
|
||||
bool ncclProfilerNeedsProxy(struct ncclComm* comm, struct ncclProxyOp* op);
|
||||
bool ncclProfilerPluginLoaded(void);
|
||||
|
||||
// Profiler callback for network plugin
|
||||
ncclResult_t ncclProfilerCallback(void** eHandle, int type, void* pHandle, int64_t pluginId, void* extData);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -34,7 +34,8 @@ typedef enum : uint8_t {
|
||||
ncclPatternPatUp,
|
||||
ncclPatternPatDown,
|
||||
ncclPatternSend,
|
||||
ncclPatternRecv
|
||||
ncclPatternRecv,
|
||||
ncclPatternProfiler,
|
||||
} ncclPattern_t;
|
||||
|
||||
enum ncclProxyOpState { ncclProxyOpNone, ncclProxyOpReady, ncclProxyOpProgress };
|
||||
@@ -93,12 +94,19 @@ struct ncclProxyOp {
|
||||
struct ncclTaskP2p* p2p;
|
||||
} task;
|
||||
|
||||
// Profiler work counter increment flag. Set to 'true' if the profiler work counter for this channel needs increment.
|
||||
// Always 'true' for collective operations. Grouped p2p operations are fused into one <send, recv> pair in the GPU kernel,
|
||||
// meaning the GPU profiler code increments the work counter for the pair rather than the individual p2p. For this
|
||||
// reason, the incWorkCounter flag is used to avoid incrementing the work counter twice in the host code. This is done
|
||||
// by setting incWorkCounter to 'true' only for one of the p2ps in the pair during enqueue.
|
||||
bool incWorkCounter;
|
||||
int eActivationMask;
|
||||
void* taskEventHandle;
|
||||
int rank;
|
||||
int peer;
|
||||
pid_t pid;
|
||||
void* profilerContext;
|
||||
uint64_t workCounter;
|
||||
|
||||
struct ncclProxyOp *enqNext;
|
||||
};
|
||||
@@ -135,12 +143,15 @@ struct ncclProxySubArgs {
|
||||
// Profiler plugin
|
||||
int eActivationMask;
|
||||
int rank;
|
||||
uint64_t profilerSteps;
|
||||
pid_t pid;
|
||||
void* profilerContext;
|
||||
void* taskEventHandle;
|
||||
void* opEventHandle;
|
||||
void* kernelEventHandle;
|
||||
void* stepEventHandles[NCCL_STEPS];
|
||||
size_t transSize;
|
||||
uint64_t workCounter;
|
||||
|
||||
void* recvRequestsCache[NCCL_STEPS];
|
||||
int recvRequestsSubCount;
|
||||
|
||||
@@ -15,6 +15,8 @@ struct rasRankInit {
|
||||
pid_t pid;
|
||||
int cudaDev;
|
||||
int nvmlDev;
|
||||
uint64_t hostHash;
|
||||
uint64_t pidHash;
|
||||
};
|
||||
|
||||
ncclResult_t ncclRasCommInit(struct ncclComm* comm, struct rasRankInit* myRank);
|
||||
|
||||
@@ -42,7 +42,7 @@ struct ncclReg {
|
||||
uintptr_t baseAddr;
|
||||
size_t baseSize;
|
||||
CUdeviceptr regAddr;
|
||||
size_t regSize;
|
||||
size_t regUCSize, regMCSize;
|
||||
int dev;
|
||||
CUmemGenericAllocationHandle mcHandle;
|
||||
uintptr_t caddrs[NCCL_MAX_LOCAL_RANKS]; /* use to check if NVLS buffers match among intra-node ranks */
|
||||
|
||||
@@ -14,7 +14,6 @@ struct shmCuIpc {
|
||||
CUmemFabricHandle handle;
|
||||
CUmemGenericAllocationHandle data;
|
||||
};
|
||||
int tpProxyRank;
|
||||
void *ptr;
|
||||
size_t size;
|
||||
};
|
||||
@@ -30,8 +29,8 @@ struct shmIpcDesc {
|
||||
|
||||
typedef struct shmIpcDesc ncclShmIpcDesc_t;
|
||||
|
||||
ncclResult_t ncclShmAllocateShareableBuffer(int tpProxyRank, size_t size, bool legacy, ncclShmIpcDesc_t *descOut, void **hptr, void **dptr);
|
||||
ncclResult_t ncclShmImportShareableBuffer(struct ncclComm *comm, ncclShmIpcDesc_t *desc, void **hptr, void **dptr, ncclShmIpcDesc_t *descOut);
|
||||
ncclResult_t ncclShmAllocateShareableBuffer(size_t size, bool legacy, ncclShmIpcDesc_t *descOut, void **hptr, void **dptr);
|
||||
ncclResult_t ncclShmImportShareableBuffer(struct ncclComm *comm, int proxyRank, ncclShmIpcDesc_t *desc, void **hptr, void **dptr, ncclShmIpcDesc_t *descOut);
|
||||
ncclResult_t ncclShmIpcClose(ncclShmIpcDesc_t *desc);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -96,5 +96,5 @@ ncclResult_t ncclSocketRecv(struct ncclSocket* sock, void* ptr, int size);
|
||||
ncclResult_t ncclSocketSendRecv(struct ncclSocket* sendSock, void* sendPtr, int sendSize, struct ncclSocket* recvSock, void* recvPtr, int recvSize);
|
||||
ncclResult_t ncclSocketTryRecv(struct ncclSocket* sock, void* ptr, int size, int* closed, bool blocking);
|
||||
ncclResult_t ncclSocketShutdown(struct ncclSocket* sock, int how);
|
||||
ncclResult_t ncclSocketClose(struct ncclSocket* sock);
|
||||
ncclResult_t ncclSocketClose(struct ncclSocket* sock, bool wait = false);
|
||||
#endif
|
||||
|
||||
@@ -10,13 +10,24 @@
|
||||
#include "nccl.h"
|
||||
#include "checks.h"
|
||||
|
||||
#include <cuda.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdint.h>
|
||||
|
||||
// ncclCudaContext: wraps a CUDA context with per-context state.
|
||||
struct ncclCudaContext;
|
||||
|
||||
// Get a ncclCudaContext to track the currently active CUDA context.
|
||||
ncclResult_t ncclCudaContextTrack(struct ncclCudaContext** out);
|
||||
// Drop reference.
|
||||
void ncclCudaContextDrop(struct ncclCudaContext* cxt);
|
||||
|
||||
/* ncclCudaGraph: Wraps a cudaGraph_t so that we can support pre-graph CUDA runtimes
|
||||
* easily.
|
||||
*/
|
||||
struct ncclCudaGraph {
|
||||
#if ROCM_VERSION >= 60100
|
||||
cudaStream_t origin;
|
||||
cudaGraph_t graph;
|
||||
unsigned long long graphId;
|
||||
#endif
|
||||
@@ -25,6 +36,7 @@ struct ncclCudaGraph {
|
||||
inline struct ncclCudaGraph ncclCudaGraphNone() {
|
||||
struct ncclCudaGraph tmp;
|
||||
#if ROCM_VERSION >= 60100
|
||||
tmp.origin = nullptr;
|
||||
tmp.graph = nullptr;
|
||||
tmp.graphId = ULLONG_MAX;
|
||||
#endif
|
||||
@@ -33,7 +45,7 @@ inline struct ncclCudaGraph ncclCudaGraphNone() {
|
||||
|
||||
inline bool ncclCudaGraphValid(struct ncclCudaGraph graph) {
|
||||
#if ROCM_VERSION >= 60100
|
||||
return graph.graph != nullptr;
|
||||
return graph.graphId != ULLONG_MAX;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
@@ -57,84 +69,69 @@ ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, cudaHostFn_t
|
||||
* streams unfit for the use of serializing access to a persistent resource.
|
||||
* Strong streams have been introduced to address this need.
|
||||
*
|
||||
* - All updates to a strong stream must be enclosed by a Acquire/Release pair.
|
||||
* All updates to a strong stream must be enclosed by a Acquire/Release pair.
|
||||
*
|
||||
* - The Acquire, Release, and all updates take a ncclCudaGraph parameter
|
||||
* indicating the currently capturing graph (or none). This parameter must be
|
||||
* the same for the entire sequence of {Acquire; ...; Release}.
|
||||
* Acquire retrieves a "work" stream (cudaStream_t) which may be used to add
|
||||
* work.
|
||||
*
|
||||
* - An {Acquire; ...; Release} sequence must not be concurrent with any
|
||||
* other operations against the strong stream including graph launches which
|
||||
* reference this stream.
|
||||
* Release publishes the work streams work into the strong stream. The Release
|
||||
* must be issued by the same thread that did the Acquire.
|
||||
*/
|
||||
struct ncclStrongStream;
|
||||
|
||||
ncclResult_t ncclStrongStreamConstruct(struct ncclStrongStream* ss);
|
||||
ncclResult_t ncclStrongStreamDestruct(struct ncclStrongStream* ss);
|
||||
|
||||
// Acquire-fence the strong stream.
|
||||
// Acquire the strong stream. Upon return `*workStream` will be usable to add work.
|
||||
// `concurrent` indicates if other threads may be using the strong stream.
|
||||
ncclResult_t ncclStrongStreamAcquire(
|
||||
struct ncclCudaGraph graph, struct ncclStrongStream* ss
|
||||
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent, cudaStream_t* workStream
|
||||
);
|
||||
|
||||
// Acquire-fence the strong stream assuming no graph is capturing. This permits
|
||||
// the caller to enqueue directly to the `ss->cudaStream` member using native CUDA
|
||||
// calls. Strong stream still must be released via:
|
||||
// ncclStrongStreamRelease(ncclCudaGraphNone(), ss);
|
||||
ncclResult_t ncclStrongStreamAcquireUncaptured(struct ncclStrongStream* ss);
|
||||
|
||||
// Release-fence of the strong stream.
|
||||
ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStrongStream* ss);
|
||||
|
||||
// Add a host launch to the stream.
|
||||
ncclResult_t ncclStrongStreamLaunchHost(
|
||||
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
|
||||
cudaHostFn_t fn, void* arg
|
||||
);
|
||||
// Add a kernel launch to the stream.
|
||||
ncclResult_t ncclStrongStreamLaunchKernel(
|
||||
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
|
||||
void* fn, dim3 grid, dim3 block, void** args, size_t sharedMemBytes
|
||||
// Get the workStream for an already acquired strong stream.
|
||||
// `concurrent` indicates if other threads may be using the strong stream.
|
||||
ncclResult_t ncclStrongStreamAcquiredWorkStream(
|
||||
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent, cudaStream_t* workStream
|
||||
);
|
||||
|
||||
// Cause `a` to wait for the current state `b`. Both `a` and `b` must be acquired.
|
||||
// `b_subsumes_a` indicates that all work in `a` is already present in `b`, thus
|
||||
// we want to fast-forward `a` to be a clone of `b`. Knowing this permits the
|
||||
// implementation to induce few graph dependencies.
|
||||
ncclResult_t ncclStrongStreamWaitStream(
|
||||
struct ncclCudaGraph graph, struct ncclStrongStream* a, struct ncclStrongStream* b, bool b_subsumes_a=false
|
||||
);
|
||||
// `b` must be capturing within `graph`.
|
||||
ncclResult_t ncclStrongStreamWaitStream(
|
||||
struct ncclCudaGraph graph, struct ncclStrongStream* a, cudaStream_t b, bool b_subsumes_a=false
|
||||
);
|
||||
// `a` must be capturing within `graph`.
|
||||
ncclResult_t ncclStrongStreamWaitStream(
|
||||
struct ncclCudaGraph graph, cudaStream_t a, struct ncclStrongStream* b, bool b_subsumes_a=false
|
||||
// Release of the strong stream.
|
||||
// `concurrent` indicates if other threads may be using the strong stream.
|
||||
ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent);
|
||||
|
||||
ncclResult_t ncclStreamWaitStream(
|
||||
cudaStream_t a, cudaStream_t b, cudaEvent_t scratchEvent
|
||||
);
|
||||
|
||||
// Like cudaStreamWaitEvent except `e` must be strictly ahead of everything in `s`.
|
||||
ncclResult_t ncclStreamAdvanceToEvent(struct ncclCudaGraph g, cudaStream_t s, cudaEvent_t e);
|
||||
|
||||
// Synchrnoization does not need the strong stream to be acquired.
|
||||
ncclResult_t ncclStrongStreamSynchronize(struct ncclStrongStream* ss);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct ncclStrongStreamGraph; // internal to ncclStrongStream
|
||||
struct ncclStrongStreamCapture; // internal to ncclStrongStream
|
||||
|
||||
struct ncclStrongStream {
|
||||
// Used when not graph capturing.
|
||||
cudaStream_t cudaStream;
|
||||
// The stream to use for non-captured work.
|
||||
cudaStream_t liveStream;
|
||||
void* liveAcquiredBy;
|
||||
#if ROCM_VERSION >= 60100
|
||||
// This stream ever appeared in a graph capture.
|
||||
bool everCaptured;
|
||||
pthread_mutex_t lock;
|
||||
struct ncclStrongStreamCapture* captureHead;
|
||||
// The event used to establish order between graphs and streams. During acquire
|
||||
// this event is waited on, during release it is recorded to.
|
||||
cudaEvent_t serialEvent;
|
||||
// This stream ever appeared in a graph capture.
|
||||
bool everCaptured;
|
||||
// Tracks whether serialEvent needs to be recorded to upon Release().
|
||||
bool serialEventNeedsRecord;
|
||||
struct ncclStrongStreamGraph* graphHead;
|
||||
#else
|
||||
cudaEvent_t scratchEvent;
|
||||
#endif
|
||||
};
|
||||
|
||||
struct ncclCudaContext {
|
||||
struct ncclCudaContext* next;
|
||||
CUcontext hcontext;
|
||||
int refCount;
|
||||
struct ncclStrongStream launchOrder;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#define TRANSPORT_SHM 1
|
||||
#define TRANSPORT_NET 2
|
||||
#define TRANSPORT_COLLNET 3
|
||||
#define TRANSPORT_PROFILER 4
|
||||
|
||||
#include "proxy.h"
|
||||
#include "comm.h"
|
||||
@@ -27,6 +28,7 @@ extern struct ncclTransport p2pTransport;
|
||||
extern struct ncclTransport shmTransport;
|
||||
extern struct ncclTransport netTransport;
|
||||
extern struct ncclTransport collNetTransport;
|
||||
extern struct ncclTransport profilerTransport;
|
||||
|
||||
extern struct ncclTransport* ncclTransports[];
|
||||
// Forward declarations
|
||||
@@ -50,8 +52,10 @@ struct ncclNvlsSharedRes {
|
||||
CUmulticastObjectProp signalProp;
|
||||
CUmemAccessDesc accessDesc;
|
||||
int dev;
|
||||
size_t buffSize;
|
||||
size_t creditSize;
|
||||
size_t creditUCSize;
|
||||
size_t creditMCSize;
|
||||
size_t buffUCSize;
|
||||
size_t buffMCSize;
|
||||
CUmemGenericAllocationHandle mcBuffHandle; // Multicast handle for NVLS buffer
|
||||
CUmemGenericAllocationHandle mcCreditHandle; // Multicast handle for NVLS credit buffer
|
||||
char* mcBuff; // Multicast NVLS buffer address
|
||||
@@ -108,7 +112,7 @@ ncclResult_t ncclNvlsBufferSetup(struct ncclComm* comm);
|
||||
ncclResult_t ncclNvlsTreeConnect(struct ncclComm* comm);
|
||||
ncclResult_t ncclNvlsGraphRegisterBuffer(struct ncclComm *comm, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, int *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv, struct ncclIntruQueue<struct ncclCommCallback, &ncclCommCallback::next>* cleanupQueue, int* nCleanupQueueElts);
|
||||
ncclResult_t ncclNvlsLocalRegisterBuffer(struct ncclComm *comm, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, int *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv);
|
||||
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t size);
|
||||
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t ucsize, size_t mcsize);
|
||||
ncclResult_t ncclNvlsFree(struct ncclComm* comm);
|
||||
|
||||
enum { collNetRecv=0, collNetSend=1 };
|
||||
|
||||
Odkázat v novém úkolu
Zablokovat Uživatele