AI/rocm-systems
2
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Merge remote-tracking branch 'nccl/master' into develop

Просмотр исходного кода
Этот коммит содержится в:
BertanDogancay
2025-06-20 07:53:59 -05:00
родитель 12315c259a 8171af656b
Коммит aaf023976a
136 изменённых файлов: 8510 добавлений и 5421 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы
src/Makefile
+8 -3
Просмотреть файл
@@ -10,11 +10,15 @@ include ../makefiles/version.mk
INCEXPORTS := nccl.h
LIBSRCFILES := \
bootstrap.cc channel.cc collectives.cc debug.cc enqueue.cc group.cc \
init.cc init_nvtx.cc net.cc proxy.cc transport.cc mnnvl.cc \
init.cc init_nvtx.cc proxy.cc transport.cc mnnvl.cc \
$(wildcard graph/*.cc) \
$(wildcard misc/*.cc) \
$(wildcard transport/*.cc) \
$(wildcard register/*.cc) \
$(wildcard plugin/*.cc) \
$(wildcard plugin/net/*.cc) \
$(wildcard plugin/tuner/*.cc) \
$(wildcard plugin/profiler/*.cc) \
$(filter-out ras/client.cc,$(wildcard ras/*.cc))
BINSRCFILES := ras/client.cc
@@ -49,6 +53,7 @@ LIBOBJ := $(LIBSRCFILES:%.cc=$(OBJDIR)/%.o)
BINOBJ := $(BINSRCFILES:%.cc=$(OBJDIR)/%.o)
DEPFILES := $(LIBOBJ:%.o=%.d) $(BINOBJ:%.o=%.d)
LDFLAGS += -L${CUDA_LIB} -l$(CUDARTLIB) -lpthread -lrt -ldl
INCPLUGIN := include/plugin
DEVMANIFEST := $(BUILDDIR)/obj/device/manifest
@@ -126,8 +131,8 @@ $(PKGDIR)/%.pc : %.pc
$(OBJDIR)/%.o : %.cc $(INCTARGETS)
@printf "Compiling %-35s > %s\n" $< $@
mkdir -p `dirname $@`
$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -c $< -o $@
@$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -M $< > $(@:%.o=%.d.tmp)
$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -I$(INCPLUGIN) -c $< -o $@
@$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -I$(INCPLUGIN) -M $< > $(@:%.o=%.d.tmp)
@sed "0,/^.*:/s//$(subst /,\/,$@):/" $(@:%.o=%.d.tmp) > $(@:%.o=%.d)
@sed -e 's/.*://' -e 's/\\$$//' < $(@:%.o=%.d.tmp) | fmt -1 | \
sed -e 's/^ *//' -e 's/$$/:/' >> $(@:%.o=%.d)
src/bootstrap.cc
+9 -7
Просмотреть файл
@@ -154,7 +154,7 @@ static ncclResult_t netIsend(ncclNet_t* net, void* sendComm, void* data, int siz
int* done) {
if (*done) return ncclSuccess;
if (!*sendReq) {
NCCLCHECK(net->isend(sendComm, data, (size_t)size, tag, dataHandle, sendReq));
NCCLCHECK(net->isend(sendComm, data, (size_t)size, tag, dataHandle, NULL, sendReq));
}
if (*sendReq) {
NCCLCHECK(net->test(*sendReq, done, NULL));
@@ -168,8 +168,8 @@ static ncclResult_t netIrecv(ncclNet_t* net, void* recvComm, void* data, int siz
int* done) {
if (*done) return ncclSuccess;
if (!*recvReq) {
size_t size64 = size;
NCCLCHECK(net->irecv(recvComm, 1, &data, &size64, &tag, &dataHandle, recvReq));
size_t size64 = size;
NCCLCHECK(net->irecv(recvComm, 1, &data, &size64, &tag, &dataHandle, NULL, recvReq));
}
if (*recvReq) {
NCCLCHECK(net->test(*recvReq, done, NULL));
@@ -485,7 +485,7 @@ static ncclResult_t netGetDevice(int rank, struct ncclComm* comm, int* dev) {
if (devOOB < 0) {
pthread_mutex_lock(&bootstrapNetLock);
if (devOOB < 0) {
char* userIfEnv = getenv("NCCL_OOB_NET_IFNAME");
const char* userIfEnv = ncclGetEnv("NCCL_OOB_NET_IFNAME");
if (userIfEnv && strlen(userIfEnv) > 0) {
INFO(NCCL_BOOTSTRAP | NCCL_ENV, "NCCL_OOB_NET_IFNAME set to %s", userIfEnv);
bool searchNot = userIfEnv && userIfEnv[0] == '^';
@@ -541,7 +541,7 @@ static ncclResult_t netRingConnect(ncclNet_t* net, struct bootstrapListen_t* lis
do {
NCCLCHECK(checkAbort(abortFlag, &abortCounter));
if (!*sendComm)
NCCLCHECK(net->connect(listen->net.dev, peerHandle, sendComm, sendDevHandle));
NCCLCHECK(net->connect(listen->net.dev, NULL, peerHandle, sendComm, sendDevHandle));
if (!*recvComm)
NCCLCHECK(net->accept(listen->net.comm, recvComm, recvDevHandle));
} while (!*sendComm || !*recvComm);
@@ -741,6 +741,8 @@ ncclResult_t bootstrapInit(int nHandles, void* handles, struct ncclComm* comm) {
rasRanks[rank].pid = getpid();
rasRanks[rank].cudaDev = comm->cudaDev;
rasRanks[rank].nvmlDev = comm->nvmlDev;
rasRanks[rank].hostHash = getHostHash();
rasRanks[rank].pidHash = getPidHash();
if (ncclRasCommInit(comm, rasRanks+rank) != ncclSuccess) {
INFO(NCCL_INIT|NCCL_RAS, "Continuing in spite of a RAS initialization error");
// We should still participate in the ringAllInfo below as the peers will be waiting for us.
@@ -972,7 +974,7 @@ ncclResult_t bootstrapRecv(void* commState, int peer, int tag, void* data, int s
NCCLCHECK(socketAccept(commState, peer, tag, &sock));
TRACE(NCCL_BOOTSTRAP, "Receiving tag=%d peer=%d size=%d", tag, peer, size);
NCCLCHECKGOTO(socketRecv(&sock, ((char*)data), size), ret, fail);
NCCLCHECK(ncclSocketClose(&sock));
NCCLCHECKGOTO(ncclSocketClose(&sock, /*wait*/true), ret, fail);
return ret;
fail:
(void)ncclSocketClose(&sock);
@@ -1067,7 +1069,7 @@ static ncclResult_t bootstrapP2PBarrier(void* commState, int* ranks, int rank, i
* Based on the dissemination algorithm by Debra Hensgen, Raphael Finkel, and Udi Manbet,
* "Two Algorithms for Barrier Synchronization," International Journal of Parallel Programming, 17(1):1-17, 1988"
*/
int data[1];
int data[1] = {0};
for (int mask = 1; mask < nranks; mask <<= 1) {
int src = (rank - mask + nranks) % nranks;
int dst = (rank + mask) % nranks;
src/channel.cc
+19 -18
Просмотреть файл
@@ -20,8 +20,8 @@ ncclResult_t initChannel(struct ncclComm* comm, int channelId) {
channel->workFifoProduced = 0;
struct ncclSharedResources* sharedRes = comm->sharedRes;
NCCLCHECK(ncclStrongStreamAcquireUncaptured(&sharedRes->deviceStream));
cudaStream_t deviceStream;
NCCLCHECK(ncclStrongStreamAcquire(ncclCudaGraphNone(), &sharedRes->deviceStream, /*concurrent=*/false, &deviceStream));
if (channel->peers == NULL) {
// The extra on nRanks+1 is for collnet root (i.e. network)
@@ -39,33 +39,33 @@ ncclResult_t initChannel(struct ncclComm* comm, int channelId) {
if (channel->devPeers == NULL) {
if (sharedRes->devPeers[channelId] == NULL) {
NCCLCHECK(ncclCudaCallocAsync(sharedRes->devPeers + channelId, sharedRes->tpNRanks, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaCallocAsync(sharedRes->devPeers + channelId, sharedRes->tpNRanks, deviceStream));
}
/* channel->devPeers is not shared, so just free it when calling commFree() */
NCCLCHECK(ncclCudaCallocAsync(&channel->devPeers, nPeers, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaCallocAsync(&channel->devPeers, nPeers, deviceStream));
ncclCommPushCudaFree(comm, channel->devPeers);
NCCLCHECK(ncclCalloc(&channel->devPeersHostPtr, nPeers));
for (int r = 0; r < nRanks; r++) {
uintptr_t addr = (uintptr_t)(comm->sharedRes->devPeers[channelId] + comm->topParentRanks[r]);
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + r), (uintptr_t*)&addr, 1, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + r), (uintptr_t*)&addr, 1, deviceStream));
channel->devPeersHostPtr[r] = (struct ncclDevChannelPeer*)addr;
}
}
channel->ring.userRanks = ncclMemoryStackAlloc<int>(&comm->memPermanent, nRanks);
NCCLCHECK(ncclCudaCallocAsync(&channel->devRingUserRanks, nRanks, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaCallocAsync(&channel->devRingUserRanks, nRanks, deviceStream));
ncclCommPushCudaFree(comm, channel->devRingUserRanks);
/* guarantee addr has been copied into channel->devPeers */
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &sharedRes->deviceStream, /*concurrent=*/false));
NCCLCHECK(ncclStrongStreamSynchronize(&sharedRes->deviceStream));
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &sharedRes->deviceStream));
return ncclSuccess;
}
ncclResult_t initNvlsChannel(struct ncclComm* comm, int channelId, struct ncclComm* parent, bool share) {
struct ncclChannel* channel = &comm->channels[channelId];
struct ncclSharedResources* sharedRes = comm->sharedRes;
cudaStream_t deviceStream;
if (channel->nvlsPeers != NULL)
return ncclSuccess;
@@ -73,7 +73,7 @@ ncclResult_t initNvlsChannel(struct ncclComm* comm, int channelId, struct ncclCo
if (channel->id == -1)
NCCLCHECK(initChannel(comm, channelId));
NCCLCHECK(ncclStrongStreamAcquireUncaptured(&sharedRes->deviceStream));
NCCLCHECK(ncclStrongStreamAcquire(ncclCudaGraphNone(), &sharedRes->deviceStream, /*concurrent=*/false, &deviceStream));
int nvlsRanks = comm->localRanks;
@@ -84,24 +84,24 @@ ncclResult_t initNvlsChannel(struct ncclComm* comm, int channelId, struct ncclCo
int tr = comm->topParentLocalRanks[r];
uintptr_t addr = (uintptr_t)(parent->channels[channelId].nvlsDevPeers + tr);
channel->peers[comm->nRanks + 1 + r] = parent->channels[channelId].nvlsPeers + tr;
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks + 1 + r), (uintptr_t*)&addr, 1, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks + 1 + r), (uintptr_t*)&addr, 1, deviceStream));
channel->devPeersHostPtr[comm->nRanks + 1 + r] = (struct ncclDevChannelPeer*)addr;
ncclAtomicRefCountIncrement(&parent->channels[channelId].nvlsPeers[tr].refCount);
}
} else {
NCCLCHECK(ncclCalloc(&channel->nvlsPeers, nvlsRanks));
NCCLCHECK(ncclCudaCallocAsync(&channel->nvlsDevPeers, nvlsRanks, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaCallocAsync(&channel->nvlsDevPeers, nvlsRanks, deviceStream));
for (int r = 0; r < nvlsRanks; ++r) {
uintptr_t addr = (uintptr_t)(channel->nvlsDevPeers + r);
channel->peers[comm->nRanks + 1 + r] = channel->nvlsPeers + r;
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks + 1 + r), (uintptr_t*)&addr, 1, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks + 1 + r), (uintptr_t*)&addr, 1, deviceStream));
channel->devPeersHostPtr[comm->nRanks + 1 + r] = (struct ncclDevChannelPeer*)addr;
ncclAtomicRefCountIncrement(&channel->nvlsPeers[r].refCount);
}
}
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &sharedRes->deviceStream, /*concurrent=*/false));
NCCLCHECK(ncclStrongStreamSynchronize(&sharedRes->deviceStream));
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &sharedRes->deviceStream));
return ncclSuccess;
}
@@ -110,6 +110,7 @@ ncclResult_t initCollnetChannel(struct ncclComm* comm, int channelId, struct ncc
struct ncclChannel* channel = &comm->channels[channelId];
struct ncclSharedResources* sharedRes = comm->sharedRes;
uintptr_t addr;
cudaStream_t deviceStream;
if (channel->collnetPeers != NULL)
return ncclSuccess;
@@ -117,28 +118,28 @@ ncclResult_t initCollnetChannel(struct ncclComm* comm, int channelId, struct ncc
if (channel->id == -1)
NCCLCHECK(initChannel(comm, channelId));
NCCLCHECK(ncclStrongStreamAcquireUncaptured(&sharedRes->deviceStream));
NCCLCHECK(ncclStrongStreamAcquire(ncclCudaGraphNone(), &sharedRes->deviceStream, /*concurrent=*/false, &deviceStream));
if (share) {
channel->collnetPeers = parent->channels[channelId].collnetPeers;
channel->collnetDevPeers = parent->channels[channelId].collnetDevPeers;
addr = (uintptr_t)parent->channels[channelId].collnetDevPeers;
channel->peers[comm->nRanks] = parent->channels[channelId].collnetPeers;
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks), (uintptr_t*)&addr, 1, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks), (uintptr_t*)&addr, 1, deviceStream));
channel->devPeersHostPtr[comm->nRanks] = (struct ncclDevChannelPeer*)addr;
ncclAtomicRefCountIncrement(&parent->channels[channelId].collnetPeers->refCount);
} else {
NCCLCHECK(ncclCalloc(&channel->collnetPeers, 1));
NCCLCHECK(ncclCudaCallocAsync(&channel->collnetDevPeers, 1, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaCallocAsync(&channel->collnetDevPeers, 1, deviceStream));
addr = (uintptr_t)channel->collnetDevPeers;
channel->peers[comm->nRanks] = channel->collnetPeers;
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks), (uintptr_t*)&addr, 1, sharedRes->deviceStream.cudaStream));
NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks), (uintptr_t*)&addr, 1, deviceStream));
channel->devPeersHostPtr[comm->nRanks] = (struct ncclDevChannelPeer*)addr;
ncclAtomicRefCountIncrement(&channel->collnetPeers->refCount);
}
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &sharedRes->deviceStream, /*concurrent=*/false));
NCCLCHECK(ncclStrongStreamSynchronize(&sharedRes->deviceStream));
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &sharedRes->deviceStream));
return ncclSuccess;
}
src/debug.cc
+149 -14
Просмотреть файл
@@ -6,6 +6,7 @@
#include "core.h"
#include "nccl_net.h"
#include <ctime>
#include <stdlib.h>
#include <stdarg.h>
#include <stdio.h>
@@ -16,6 +17,11 @@
#include "param.h"
int ncclDebugLevel = -1;
static uint32_t ncclDebugTimestampLevels = 0; // bitmaps of levels that have timestamps turned on
static char ncclDebugTimestampFormat[256]; // with space for subseconds
static int ncclDebugTimestampSubsecondsStart; // index where the subseconds starts
static uint64_t ncclDebugTimestampMaxSubseconds; // Max number of subseconds plus 1, used in duration ratio
static int ncclDebugTimestampSubsecondDigits; // Number of digits to display
static int pid = -1;
static char hostname[1024];
thread_local int ncclDebugNoWarn = 0;
@@ -114,6 +120,88 @@ static void ncclDebugInit() {
ncclWarnSetDebugInfo = value;
}
// Determine which debug levels will have timestamps.
const char* timestamps = ncclGetEnv("NCCL_DEBUG_TIMESTAMP_LEVELS");
if (timestamps == nullptr) {
ncclDebugTimestampLevels = (1<<NCCL_LOG_WARN);
} else {
int invert = 0;
if (timestamps[0] == '^') { invert = 1; ++timestamps; }
ncclDebugTimestampLevels = invert ? ~0U : 0U;
char *timestampsDup = strdup(timestamps);
char *level = strtok(timestampsDup, ",");
while (level != NULL) {
uint32_t mask = 0;
if (strcasecmp(level, "ALL") == 0) {
mask = ~0U;
} else if (strcasecmp(level, "VERSION") == 0) {
mask = (1<<NCCL_LOG_VERSION);
} else if (strcasecmp(level, "WARN") == 0) {
mask = (1<<NCCL_LOG_WARN);
} else if (strcasecmp(level, "INFO") == 0) {
mask = (1<<NCCL_LOG_INFO);
} else if (strcasecmp(level, "ABORT") == 0) {
mask = (1<<NCCL_LOG_ABORT);
} else if (strcasecmp(level, "TRACE") == 0) {
mask = (1<<NCCL_LOG_TRACE);
} else {
// Silently fail.
}
if (mask) {
if (invert) ncclDebugTimestampLevels &= ~mask;
else ncclDebugTimestampLevels |= mask;
}
level = strtok(NULL, ",");
}
free(timestampsDup);
}
// Store a copy of the timestamp format with space for the subseconds, if used.
const char* tsFormat = ncclGetEnv("NCCL_DEBUG_TIMESTAMP_FORMAT");
if (tsFormat == nullptr) tsFormat = "[%F %T] ";
ncclDebugTimestampSubsecondsStart = -1;
// Find where the subseconds are in the format.
for (int i=0; tsFormat[i] != '\0'; ++i) {
if (tsFormat[i]=='%' && tsFormat[i+1]=='%') { // Next two chars are "%"
// Skip the next character, too, and restart checking after that.
++i;
continue;
}
if (tsFormat[i]=='%' && // Found a percentage
('1' <= tsFormat[i+1] && tsFormat[i+1] <= '9') && // Next char is a digit between 1 and 9 inclusive
tsFormat[i+2]=='f' // Two characters later is an "f"
) {
constexpr int replaceLen = sizeof("%Xf") - 1;
ncclDebugTimestampSubsecondDigits = tsFormat[i+1] - '0';
if (ncclDebugTimestampSubsecondDigits + strlen(tsFormat) - replaceLen > sizeof(ncclDebugTimestampFormat) - 1) {
// Won't fit; fall back on the default.
break;
}
ncclDebugTimestampSubsecondsStart = i;
ncclDebugTimestampMaxSubseconds = 1;
memcpy(ncclDebugTimestampFormat, tsFormat, i);
for (int j=0; j<ncclDebugTimestampSubsecondDigits; ++j) {
ncclDebugTimestampFormat[i+j] = ' ';
ncclDebugTimestampMaxSubseconds *= 10;
}
strcpy(ncclDebugTimestampFormat+i+ncclDebugTimestampSubsecondDigits, tsFormat+i+replaceLen);
break;
}
}
if (ncclDebugTimestampSubsecondsStart == -1) {
if (strlen(tsFormat) < sizeof(ncclDebugTimestampFormat)) {
strcpy(ncclDebugTimestampFormat, tsFormat);
} else {
strcpy(ncclDebugTimestampFormat, "[%F %T] ");
}
}
// Replace underscore with spaces... it is hard to put spaces in command line parameters.
for (int i=0; ncclDebugTimestampFormat[i] != '\0'; ++i) {
if (ncclDebugTimestampFormat[i]=='_') ncclDebugTimestampFormat[i] = ' ';
}
// Cache pid and hostname
getHostName(hostname, 1024, '.');
pid = getpid();
@@ -194,39 +282,86 @@ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *file
tid = syscall(SYS_gettid);
}
char buffer[1024];
size_t len = 0;
// WARNs come with an extra newline at the beginning.
if (level == NCCL_LOG_WARN) {
buffer[len++] = '\n';
};
// Add the timestamp to the buffer if they are turned on for this level.
if (ncclDebugTimestampLevels & (1<<level)) {
if (ncclDebugTimestampFormat[0] != '\0') {
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts); // clock_gettime failure should never happen
std::tm nowTm;
localtime_r(&ts.tv_sec, &nowTm);
// Add the subseconds portion if it is part of the format.
char localTimestampFormat[sizeof(ncclDebugTimestampFormat)];
const char* pformat = ncclDebugTimestampFormat;
if (ncclDebugTimestampSubsecondsStart != -1) {
pformat = localTimestampFormat; // Need to use the local version which has subseconds
memcpy(localTimestampFormat, ncclDebugTimestampFormat, ncclDebugTimestampSubsecondsStart);
snprintf(localTimestampFormat + ncclDebugTimestampSubsecondsStart,
ncclDebugTimestampSubsecondDigits+1,
"%0*ld", ncclDebugTimestampSubsecondDigits,
ts.tv_nsec / (1000000000UL/ncclDebugTimestampMaxSubseconds));
strcpy( localTimestampFormat+ncclDebugTimestampSubsecondsStart+ncclDebugTimestampSubsecondDigits,
ncclDebugTimestampFormat+ncclDebugTimestampSubsecondsStart+ncclDebugTimestampSubsecondDigits);
}
// Format the time. If it runs out of space, fall back on a simpler format.
int adv = std::strftime(buffer+len, sizeof(buffer)-len, pformat, &nowTm);
if (adv==0 && ncclDebugTimestampFormat[0] != '\0') {
// Ran out of space. Fall back on the default. This should never fail.
adv = std::strftime(buffer+len, sizeof(buffer)-len, "[%F %T] ", &nowTm);
}
len += adv;
}
}
len = std::min(len, sizeof(buffer)-1); // prevent overflows
// Add hostname, pid and tid portion of the log line.
if (level != NCCL_LOG_VERSION) {
len += snprintf(buffer+len, sizeof(buffer)-len, "%s:%d:%d ", hostname, pid, tid);
len = std::min(len, sizeof(buffer)-1); // prevent overflows
}
int cudaDev = 0;
if (!(level == NCCL_LOG_TRACE && flags == NCCL_CALL)) {
(void)cudaGetDevice(&cudaDev);
}
char buffer[4096];
size_t len = 0;
// Add level specific formatting.
if (level == NCCL_LOG_WARN) {
len = snprintf(buffer, sizeof(buffer), "\n%s:%d:%d [%d] %s:%d NCCL WARN ",
hostname, pid, tid, cudaDev, filefunc, line);
len += snprintf(buffer+len, sizeof(buffer)-len, "[%d] %s:%d NCCL WARN ", cudaDev, filefunc, line);
if (ncclWarnSetDebugInfo) ncclDebugLevel = NCCL_LOG_INFO;
} else if (level == NCCL_LOG_INFO) {
len = snprintf(buffer, sizeof(buffer), "%s:%d:%d [%d] NCCL INFO ", hostname, pid, tid, cudaDev);
len += snprintf(buffer+len, sizeof(buffer)-len, "[%d] NCCL INFO ", cudaDev);
} else if (level == NCCL_LOG_TRACE && flags == NCCL_CALL) {
len = snprintf(buffer, sizeof(buffer), "%s:%d:%d NCCL CALL ", hostname, pid, tid);
len += snprintf(buffer+len, sizeof(buffer)-len, "NCCL CALL ");
} else if (level == NCCL_LOG_TRACE) {
auto delta = std::chrono::steady_clock::now() - ncclEpoch;
double timestamp = std::chrono::duration_cast<std::chrono::duration<double>>(delta).count()*1000;
len = snprintf(buffer, sizeof(buffer), "%s:%d:%d [%d] %f %s:%d NCCL TRACE ",
hostname, pid, tid, cudaDev, timestamp, filefunc, line);
len += snprintf(buffer+len, sizeof(buffer)-len, "[%d] %f %s:%d NCCL TRACE ", cudaDev, timestamp, filefunc, line);
}
len = std::min(len, sizeof(buffer)-1); // prevent overflows
// Add the message as given by the call site.
va_list vargs;
va_start(vargs, fmt);
len += vsnprintf(buffer+len, sizeof(buffer)-len, fmt, vargs);
va_end(vargs);
// vsnprintf may return len >= sizeof(buffer) in the case of a truncated output.
// Rewind len so that we can replace the final \0 by \n
if (len >= sizeof(buffer)) len = sizeof(buffer)-1;
if (len) {
buffer[len++] = '\n';
fwrite(buffer, 1, len, ncclDebugFile);
}
// Rewind len so that we can replace the final \0 by "\n"
len = std::min(len, sizeof(buffer)-1); // prevent overflows
// Add a newline and write it to the debug file. No terminating null is
// necessary since we write bytes instead of the string.
buffer[len++] = '\n';
fwrite(buffer, 1, len, ncclDebugFile);
}
NCCL_API(void, ncclResetDebugInit);
src/device/all_gather.h
+50 -12
Просмотреть файл
@@ -141,7 +141,7 @@ namespace {
}
#endif
// Final wait/copy.
prims.directRecv(offset, offset, nelem);
prims.directRecv(offset, nelem);
#if defined(ENABLE_NPKIT) && defined(ENABLE_NPKIT_EVENT_ALL_GATHER_RING_DIRECT_RECV_EXIT)
if (tid == 0) {
@@ -220,25 +220,63 @@ struct RunWorkColl<ncclFuncAllGather, T, RedOp, NCCL_ALGO_RING, NCCL_PROTO_LL128
template<typename T, typename RedOp>
struct RunWorkColl<ncclFuncAllGather, T, RedOp, NCCL_ALGO_PAT, NCCL_PROTO_SIMPLE> {
__device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) {
#if __CUDA_ARCH__ >= 600
using Proto = ProtoSimple<1, 1>;
const int nranks = ncclShmem.comm.nRanks;
const int rank = ncclShmem.comm.rank;
size_t count, channelOffset, channelCount, chunkCount;
ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), &count, &channelOffset, &channelCount, &chunkCount);
T *inputBuf = (T*)work->sendbuff;
T *outputBuf = (T*)work->recvbuff;
Primitives<T, RedOp, FanSymmetric<1>, 0, Proto, 0> prims
(tid, nthreads, NULL, NULL, inputBuf, outputBuf, work->redOpArg, 0*Proto::MaxGroupWidth, 0, 0, nullptr, nullptr, 0, primsModePatAg);
static constexpr int nworkers = NCCL_PAT_NWORKERS;
struct ncclPatShmem* shmem = (struct ncclPatShmem*)ncclScratchForWarp(0);
uint64_t pollCount = 0;
__syncthreads(); // Don't start using shared mem until everyone arrives
for (int i=tid; i<NCCL_SHMEM_PAT_STEPS; i+=nthreads) shmem->patSteps[i].flags = 0;
if (tid == 0) shmem->localAccSize = 0;
if (tid == nworkers) shmem->parallelFactor = 0;
__syncthreads();
PatAGAlgorithm<T> patAlgo(chunkCount*sizeof(T), NCCL_STEPS, channelOffset, channelOffset + channelCount, count, chunkCount, rank, nranks);
int last = 0;
while (!last) {
int recvDim, sendDim, recvOffset, sendOffset, recvStepOffset, postRecv, postSend, nelem;
size_t inpIx, outIx;
patAlgo.getNextOp(recvDim, sendDim, inpIx, outIx, recvOffset, sendOffset, recvStepOffset, nelem, postRecv, postSend, last);
prims.patCopy(recvDim, sendDim, inpIx, outIx, recvOffset, sendOffset, recvStepOffset, nelem, postRecv, postSend);
if (tid == nworkers) { // Algo computation thread
PatAGAlgorithm<T> patAlgo(chunkCount*sizeof(T), NCCL_STEPS, NCCL_PAT_NWORKERS/WARP_SIZE, channelOffset, channelOffset + channelCount, count, chunkCount, rank, nranks);
int parallelFactor = shmem->parallelFactor = patAlgo.getParallelFactor();
int step = 0;
while (1) {
struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS);
cuda::atomic_ref<int, cuda::thread_scope_block> poll(ps->flags);
while (poll.load(cuda::memory_order_acquire) != 0) pollCount++; // Wait for workers to be done with step 'step-NCCL_SHMEM_PAT_STEPS'
patAlgo.getNextOp(ps);
int last = ps->last;
step++;
if (last == 2) break;
}
} else if (tid < nworkers) { // Worker threads
T *inputBuf = (T*)work->sendbuff;
T *outputBuf = (T*)work->recvbuff;
int parallelFactor = 0;
volatile int* pfPtr = &shmem->parallelFactor;
while (parallelFactor == 0) parallelFactor = *pfPtr;
int groupSize = nworkers/(WARP_SIZE*parallelFactor) * WARP_SIZE;
int group = tid / groupSize;
int nGroups = nworkers / groupSize;
int tidInGroup = tid - group*groupSize;
// We don't use recvPeers/sendPeers so let's pass shmem structs instead
Primitives<T, RedOp, FanSymmetric<1>, 0, Proto, 0> prims
(tidInGroup, groupSize, (int*)shmem->recvDims, (int*)shmem->sendDims, inputBuf, outputBuf, work->redOpArg, group, 0, 0, nullptr, nullptr, 0, primsModePatAg);
int step = group;
while(1) {
struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS);
cuda::atomic_ref<int, cuda::thread_scope_block> poll(ps->flags);
while (poll.load(cuda::memory_order_acquire) == 0) pollCount++; // Wait for compute thread
int last = ps->last;
prims.patCopy(ps, shmem);
if (tidInGroup == 0) poll.store(0, cuda::memory_order_release); // Return element to compute thread
if (last) break;
step += nGroups;
}
}
#endif
}
};
src/device/all_reduce.h
+5 -5
Просмотреть файл
@@ -190,7 +190,7 @@ namespace {
offset = gridOffset + elemOffset + chunkOffset;
nelem = (int)min(chunkCount, remCount - chunkOffset);
prims.directRecv(offset, offset, nelem);
prims.directRecv(offset, nelem);
#if defined(ENABLE_NPKIT) && defined(ENABLE_NPKIT_EVENT_ALL_REDUCE_RING_DIRECT_RECV_EXIT)
if (tid == 0) {
@@ -329,7 +329,7 @@ namespace {
for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) {
offset = gridOffset + elemOffset;
nelem = min(chunkCount, channelCount - elemOffset);
prims.directRecv(offset, offset, nelem);
prims.directRecv(offset, nelem);
}
}
else {
@@ -528,7 +528,7 @@ namespace {
for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) {
offset = gridOffset + elemOffset;
nelem = min(chunkCount, channelCount - elemOffset);
prims.directRecv(offset, offset, nelem);
prims.directRecv(offset, nelem);
}
}
else {
@@ -1055,7 +1055,7 @@ struct RunWorkColl<ncclFuncAllReduce, T, RedOp, NCCL_ALGO_COLLNET_CHAIN, NCCL_PR
for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) {
ssize_t offset = gridOffset + bid * int(chunkSize);
int nelem = min(chunkSize, size - offset);
prims.directRecv(offset, offset, nelem, /*postOp*/true);
prims.directRecv(offset, nelem, /*postOp*/true);
}
}
} else {
@@ -1082,7 +1082,7 @@ struct RunWorkColl<ncclFuncAllReduce, T, RedOp, NCCL_ALGO_COLLNET_CHAIN, NCCL_PR
for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) {
ssize_t offset = gridOffset + bid*int(chunkSize);
int nelem = min(chunkSize, size-offset);
prims.directRecv(offset, offset, nelem);
prims.directRecv(offset, nelem);
}
} else {
for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) {
src/device/broadcast.h
+1 -1
Просмотреть файл
@@ -83,7 +83,7 @@ namespace {
prims.directCopySend(offset, offset, nelem);
}
} else if (nextRank == root) {
prims.directRecv(offset, offset, nelem);
prims.directRecv(offset, nelem);
} else {
prims.directRecvCopyDirectSend(offset, offset, nelem);
}
src/device/common.h
+54 -30
Просмотреть файл
@@ -144,6 +144,8 @@ struct ncclShmemData {
int nWorks;
int workSize;
uint32_t workConsumed;
uint64_t workCounter;
bool profilerEnabled;
struct ncclShmemGroup groups[NCCL_MAX_GROUPS];
uint64_t redOpArgs[NCCL_MAX_NVLS_ARITY+1];
@@ -236,24 +238,6 @@ __device__ inline bool barrier_red_or(bool vote, int name, int nThreads) {
: "=r"(ans) : "r"((int)vote), "r"(name), "r"(nThreads) : "memory");
return bool(ans);
}
__device__ inline bool barrier_red_or_aligned(bool vote, int name) {
int ans;
asm volatile("{ .reg .pred p;"
" setp.ne.s32 p, %1, 0;"
" barrier.red.or.pred.aligned p, %2, p; "
" selp.s32 %0, 1, 0, p; }"
: "=r"(ans) : "r"((int)vote), "r"(name) : "memory");
return bool(ans);
}
__device__ inline bool barrier_red_or_aligned(bool vote, int name, int nThreads) {
int ans;
asm("{ .reg .pred p;"
" setp.ne.s32 p, %1, 0;"
" barrier.red.or.pred.aligned p, %2, %3, p; "
" selp.s32 %0, 1, 0, p; }"
: "=r"(ans) : "r"((int)vote), "r"(name), "r"(nThreads) : "memory");
return bool(ans);
}
#ifdef ENABLE_PROFILING
#define __insert_timestamp(line_num) do { \
@@ -455,6 +439,48 @@ struct RunWorkBatch {
}
};
#define START 0
#define STOP 1
#define FINI 2
__device__ __forceinline__ bool profilerEnabled(void) {
// Check if any of the workItems in the batch is profiled. If so, there is an equivalent
// profiler ProxyOp waiting for the counter update in the host thread. If this check was
// done only for the first workItem the profiler counter for other workItems in the batch
// could never be updated, leaving the host thread spinning forever for the counter update
// and causing a hang.
bool enabled = false;
for (int i = 0; i < ncclShmem.nWorks && !enabled; i++) {
if (ncclShmem.workType == ncclDevWorkTypeP2p)
enabled = ((struct ncclDevWorkP2p*)ncclShmem.workStorage)[i].profilerEnabled;
else
enabled = ((struct ncclDevWorkColl*)ncclShmem.workStorage)[i].profilerEnabled;
}
return enabled;
}
__device__ __forceinline__ void profiler(int action) {
if (action == START) {
if (threadIdx.x == 0) {
// increment workCounter regardless of the profiler being active or not
ncclShmem.channel.workCounter += ncclShmem.nWorks;
if(!profilerEnabled()) return;
ncclShmem.comm.workStarted[ncclShmem.channelId] = ncclShmem.channel.workCounter;
}
} else if (action == STOP) {
if (threadIdx.x == 0 && profilerEnabled()) {
ncclShmem.comm.workCompleted[ncclShmem.channelId] = ncclShmem.channel.workCounter;
}
} else { // FINI
if (threadIdx.x == 0) {
// store the workCounter back to vidmem regardless of the profiler being active or not
((ncclDevCommAndChannels*)ncclShmem.args.comm)->channels[ncclShmem.channelId].workCounter = ncclShmem.channel.workCounter;
if (!profilerEnabled()) return;
ncclShmem.comm.workCompleted[ncclShmem.channelId] = ncclShmem.channel.workCounter;
}
}
}
template<int SpecializedFnId, typename SpecializedRunWorkBatch, bool COLLTRACE, int COLL_UNROLL>
__device__ __forceinline__ void ncclKernelMain(struct ncclDevKernelArgs const* args) {
const int tid = threadIdx.x;
@@ -517,8 +543,13 @@ __device__ __forceinline__ void ncclKernelMain(struct ncclDevKernelArgs const* a
break;
}
__syncthreads(); // publish ncclShmem.{args, channelId}
/* set abort flag to 0 */
if (tid == 0) {
ncclShmem.aborted = 0;
ncclShmem.channel.workCounter = ((ncclDevCommAndChannels*)ncclShmem.args.comm)->channels[ncclShmem.channelId].workCounter;
}
// Use first 2 warps to load comm and channel, and reamaining load work batch.
// Use first 2 warps to load comm and channel, and remaining load work batch.
switch (tid/WARP_SIZE) {
case 0:
{ void* dst = &ncclShmem.comm;
@@ -566,9 +597,9 @@ __device__ __forceinline__ void ncclKernelMain(struct ncclDevKernelArgs const* a
ncclShmem.comm.workConsumed[ncclShmem.channelId] = ncclShmem.workConsumed;
}
while (true) {
while (ncclShmem.aborted == 0) {
if (tid == 0) __insert_timestamp(__LINE__);
profiler(START);
if (0 <= SpecializedFnId && ncclShmem.funcId == (unsigned)SpecializedFnId) {
SpecializedRunWorkBatch().run();
} else {
@@ -586,21 +617,14 @@ __device__ __forceinline__ void ncclKernelMain(struct ncclDevKernelArgs const* a
default:
break;
}
profiler(STOP);
loadWorkBatchToShmem(tid%WARP_SIZE, tn, args, batchIx);
__syncthreads();
// Check whether the last operation was aborted and make sure all threads exit
bool aborted = false;
if (tid == 0) aborted = *ncclShmem.comm.abortFlag;
aborted = __any(aborted); // publish ncclShmem.work
if (tid == 0 && ncclShmem.args.workStorageType == ncclDevWorkStorageTypeFifo) {
// ncclShmem.workConsumed written by loadWorkBatchToShmem before barrier_red_or()
// ncclShmem.workConsumed written by loadWorkBatchToShmem before __syncthreads()
ncclShmem.comm.workConsumed[ncclShmem.channelId] = ncclShmem.workConsumed;
}
if (aborted) {
if(COLLTRACE && tid%WARP_SIZE == 0) traceAbort();
break;
}
if (COLLTRACE && tid%WARP_SIZE == 0) traceKernelLaunch(ncclCollTraceCollLaunchType, batchIx);
}
if (COLLTRACE && tid%WARP_SIZE == 0) traceKernelEnd(ncclCollTraceKernelEndType);
src/device/primitives.h
+14 -2
Просмотреть файл
@@ -13,7 +13,7 @@
#include "common_kernel.h"
#include "common.h"
#define NCCL_SPINS_BEFORE_CHECK_ABORT 1000000
#define NCCL_SPINS_BEFORE_CHECK_ABORT 10000
#define barrier_by_group_common(__THREAD_FENCE) do { \
if (nthreads == NCCL_MAX_NTHREADS) { \
@@ -154,7 +154,7 @@ struct PrimitivesWithoutDirect {
__device__ void directSendFromOutput(intptr_t outIx, int eltN) {
static_cast<RealPrimitives*>(this)->sendFromOutput(outIx, eltN);
}
__device__ void directRecv(intptr_t inpIx, intptr_t outIx, int eltN) {
__device__ void directRecv(intptr_t outIx, int eltN) {
static_cast<RealPrimitives*>(this)->recv(outIx, eltN, /*postOp=*/false);
}
__device__ void directCopySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) {
@@ -178,6 +178,18 @@ struct PrimitivesWithoutDirect {
}
};
__device__ inline int checkAbort(int &abortCache, const int abortValue, int &spins) {
if (abortCache & abortValue) return 1;
if (++spins < NCCL_SPINS_BEFORE_CHECK_ABORT) return 0;
spins = 0;
int abort = __atomic_load_n((ncclShmem.comm.abortFlag), __ATOMIC_SEQ_CST);
if (abort) {
__atomic_store_n(&ncclShmem.aborted, abort, __ATOMIC_SEQ_CST);
abortCache |= abortValue;
}
return abort;
}
#include "prims_simple.h"
#include "prims_ll.h"
#include "prims_ll128.h"
src/device/prims_ll.h
+13 -10
Просмотреть файл
@@ -85,15 +85,18 @@ private:
#endif
}
uint32_t abort = 0;
int abort = 0;
inline __device__ int checkAbort(int &spins, int send) {
spins++;
if (abort == 0 && spins == NCCL_SPINS_BEFORE_CHECK_ABORT) {
abort = __atomic_load_n((ncclShmem.comm.abortFlag), __ATOMIC_SEQ_CST);
__device__ inline int checkAbort(int &abortCache, const int abortValue, int &spins) {
if (abortCache == 0 && ++spins == NCCL_SPINS_BEFORE_CHECK_ABORT) {
int abort = __atomic_load_n((ncclShmem.comm.abortFlag), __ATOMIC_SEQ_CST);
spins = 0;
if (abort) {
__atomic_store_n(&ncclShmem.aborted, abort, __ATOMIC_SEQ_CST);
abortCache |= abortValue;
}
}
return abort;
return abortCache;
}
inline __device__ void waitSend(int nbytes) {
@@ -108,7 +111,7 @@ private:
while (sendConnHeadCache + NCCL_STEPS < sendConnHead + 1) {
__builtin_amdgcn_s_sleep(1);
sendConnHeadCache = atomicAdd((unsigned long long *)sendConnHeadPtr, 0);
if (checkAbort(spins, 1)) break;
if (checkAbort(abort, 1, spins)) break;
}
if (sendConnFifo) {
int size = ((sendConnHead & NCCL_LL_CLEAN_MASK) == NCCL_LL_CLEAN_MASK) ? stepLines*sizeof(union ncclLLFifoLine) : nbytes;
@@ -168,7 +171,7 @@ private:
#if defined(ENABLE_NPKIT) && (defined(ENABLE_NPKIT_EVENT_PRIM_LL_DATA_PROCESS_ENTRY) && defined(ENABLE_NPKIT_EVENT_PRIM_LL_DATA_PROCESS_EXIT) || defined(ENABLE_NPKIT_PRIM_COLLECT_DATA_PROCESS_TIME))
npkitWaitRecvSpins++;
#endif
if (checkAbort(spins, 0)) break;
if (checkAbort(abort, 1, spins)) break;
} while ((i4.flag1 != flag) || (i4.flag2 != flag));
uint64_t val64 = (uint64_t)(i4.data1) + (((uint64_t)i4.data2) << 32);
#else
@@ -177,7 +180,7 @@ private:
#if defined(ENABLE_NPKIT) && (defined(ENABLE_NPKIT_EVENT_PRIM_LL_DATA_PROCESS_ENTRY) && defined(ENABLE_NPKIT_EVENT_PRIM_LL_DATA_PROCESS_EXIT) || defined(ENABLE_NPKIT_PRIM_COLLECT_DATA_PROCESS_TIME))
npkitWaitRecvSpins++;
#endif
if (checkAbort(spins, 0)) break;
if (checkAbort(abort, 1, spins)) break;
} while ((flag1 != flag) || (flag2 != flag));
uint64_t val64 = data1 + (((uint64_t)data2) << 32);
#endif
@@ -241,7 +244,7 @@ private:
#if defined(ENABLE_NPKIT) && (defined(ENABLE_NPKIT_EVENT_PRIM_LL_DATA_PROCESS_ENTRY) && defined(ENABLE_NPKIT_EVENT_PRIM_LL_DATA_PROCESS_EXIT) || defined(ENABLE_NPKIT_PRIM_COLLECT_DATA_PROCESS_TIME))
npkitWaitRecvSpins++;
#endif
if (checkAbort(spins, 0)) break;
if (checkAbort(abort, 1, spins)) break;
} while(line[i].flag1 != flag || line[i].flag2 != flag);
uint64_t val64 = line[i].data1 + (((uint64_t)line[i].data2) << 32);
src/device/prims_ll128.h
+12 -10
Просмотреть файл
@@ -86,16 +86,18 @@ private:
#endif
}
uint32_t abort = 0;
uint32_t* sync;
int abort = 0;
inline __device__ int checkAbort(int &spins, int i, int send) {
spins++;
if (abort == 0 && spins == NCCL_SPINS_BEFORE_CHECK_ABORT) {
abort = __atomic_load_n(ncclShmem.comm.abortFlag, __ATOMIC_SEQ_CST);
__device__ inline int checkAbort(int &abortCache, const int abortValue, int &spins) {
if (abortCache == 0 && ++spins == NCCL_SPINS_BEFORE_CHECK_ABORT) {
int abort = __atomic_load_n((ncclShmem.comm.abortFlag), __ATOMIC_SEQ_CST);
spins = 0;
if (abort) {
__atomic_store_n(&ncclShmem.aborted, abort, __ATOMIC_SEQ_CST);
abortCache |= abortValue;
}
}
return abort;
return abortCache;
}
inline __device__ void waitSend(int nbytes) {
@@ -104,7 +106,7 @@ private:
while (sendConnHeadCache + NCCL_STEPS < sendConnHead + 1) {
__builtin_amdgcn_s_sleep(1);
sendConnHeadCache = __atomic_load_n(sendConnHeadPtr, __ATOMIC_RELAXED);
if (checkAbort(spins, wid, 1)) break;
if (checkAbort(abort, 1, spins)) break;
}
if (sendConnFifo) {
sendConnFifo[sendStep[wid]%NCCL_STEPS].size = nbytes;
@@ -241,7 +243,7 @@ private:
load128(ptr+u*WARP_SIZE, vr[u], vr[u+1]);
needReload |= flagThread && (vr[u+1] != flag);
}
needReload &= (0 == checkAbort(spins, 0, 0));
needReload &= (0 == checkAbort(abort, 1, spins));
} while (__any(needReload));
#pragma unroll
for (int u=0; u<ELEMS_PER_THREAD; u+=2)
@@ -287,7 +289,7 @@ private:
load128(ptr+u*WARP_SIZE, vr[u], vr[u+1]);
needReload |= flagThread && (vr[u+1] != flag);
}
needReload &= (0 == checkAbort(spins, i, 0));
needReload &= (0 == checkAbort(abort, 1, spins));
} while (__any(needReload));
#pragma unroll
src/device/prims_simple.h
+243 -160
Просмотреть файл
@@ -59,7 +59,7 @@ class Primitives<
uint64_t connStepCache; // Cache last seen value of (*connStepPtr)
int connStepSize; // Connection step size
void* netDeviceHandle;
uint64_t accSize; // Accumulated size. Used by PAT operations
uint64_t accSize;
uint32_t* next_hdp_reg;
uint64_t* barriers;
uint64_t barrier_next = 0;
@@ -86,19 +86,21 @@ private:
#endif
}
inline __device__ void subBarrier() {
if (nworkers == WARP_SIZE) __syncwarp();
else
barrier();
}
inline __device__ void patBarrier() {
barrier();
}
inline __device__ bool checkAbort(int &spins) {
spins++;
if (!(flags & Aborted) && spins == NCCL_SPINS_BEFORE_CHECK_ABORT) {
if (__atomic_load_n(ncclShmem.comm.abortFlag, __ATOMIC_SEQ_CST)) {
flags |= Aborted;
ncclShmem.aborted = 1;
}
spins = 0;
}
return flags & Aborted;
inline __device__ void barrierAny() {
barrier();
}
inline __device__ void subBarrierAny() {
barrier();
}
inline __device__ uint64_t loadStepValue(uint64_t* ptr) {
@@ -129,7 +131,7 @@ private:
while (connStepCache + (isSendNotRecv ? NCCL_STEPS : 0) < step + StepPerSlice) {
__builtin_amdgcn_s_sleep(1);
connStepCache = loadStepValue(connStepPtr);
if (checkAbort(spins)) break;
if (checkAbort(flags, Aborted, spins)) break;
//if (spins == 0) printf("r=%d b=%d t=%d SPUN OUT got=%d want=%d\n", ncclShmem.comm.rank, blockIdx.x, threadIdx.x, int(connStepCache + (isSendNotRecv ? NCCL_STEPS : 0)), int(step+StepPerSlice));
if (spins == 0 && repeat > 0) {
repeat --;
@@ -482,13 +484,8 @@ public:
peerPtr->recv[connIndex].step += steps;
st_relaxed_sys_global(peerPtr->recv[connIndex].head, peerPtr->recv[connIndex].step);
while (ld_volatile_global(peerPtr->recv[connIndex].tail) < peerPtr->recv[connIndex].step) {
if (spins++ == NCCL_SPINS_BEFORE_CHECK_ABORT) {
if (*ncclShmem.comm.abortFlag) {
ncclShmem.aborted = 1;
break;
}
spins = 0;
}
int abort = 0;
if (checkAbort(abort, 1, spins)) break;
}
}
@@ -503,7 +500,7 @@ public:
int spins = 0;
while (connStepCache + (isSendNotRecv ? NCCL_STEPS : 0) < step + StepPerSlice) {
connStepCache = loadStepValue(connStepPtr);
if (checkAbort(spins)) break;
if (checkAbort(flags, Aborted, spins)) break;
}
void **ptrs = isSendNotRecv ? ncclShmem.groups[group].dsts
: ncclShmem.groups[group].srcs;
@@ -754,6 +751,9 @@ public:
flags = 0;
index = -1;
if (mode == primsModeDefault) { // Connect to ranks in sendPeers/recvPeers
// // For send operations, we need an extra warp to overlap the threadfence and the copy
// this->nworkers = nthreads - (MaxSend > 0 && nthreads >= NCCL_SIMPLE_EXTRA_GROUP_IF_NTHREADS_GE ? WARP_SIZE : 0);
int nrecv=0, nsend=0;
// Yes, for some template arguments this code will be unreachable. That's fine.
// coverity[dead_error_line]
@@ -783,68 +783,84 @@ public:
if (flags & (RoleWaitRecv|RolePostRecv)) peer = recvPeers[index];
if (flags & (RoleWaitSend|RolePostSend)) peer = sendPeers[index];
// Coverity thinks that index could be -1 here but that's not actually the case.
// coverity[negative_returns:FALSE]
int sendIpcReg;
int recvIpcReg;
int sendNetReg;
int recvNetReg;
if (P2p) {
sendIpcReg = p2pWork ? p2pWork->sendIpcReg : 0;
recvIpcReg = p2pWork ? p2pWork->recvIpcReg : 0;
sendNetReg = p2pWork ? p2pWork->sendNetReg : 0;
recvNetReg = p2pWork ? p2pWork->recvNetReg : 0;
} else {
recvIpcReg = sendIpcReg = collWork ? collWork->regUsed : 0;
recvNetReg = sendNetReg = collWork ? collWork->netRegUsed : 0;
}
// coverity[overrun-call] => Coverity think prims.index can be greater than 1
if (flags & (RoleWaitRecv|RolePostRecv)) loadRecvConn(ncclShmem.channel.peers[peer], connIndexRecv, collWork ? collWork->direct : 0, recvIpcReg, recvNetReg);
// coverity[overrun-call] => Coverity think prims.index can be greater than 1
if (flags & (RoleWaitSend|RolePostSend)) loadSendConn(ncclShmem.channel.peers[peer], connIndexSend, collWork ? collWork->direct : 0, sendIpcReg, sendNetReg);
// if (barrierAny(flags & NetDeviceUnpack)) {
// flags |= AnyNetDeviceUnpack;
// // RoleWaitRecv starts at tid=0, so this creates the bitmask of which recv peers
// // have NetDeviceUnpack.
// uint32_t mask = __ballot_sync(~0u, ((flags & RoleWaitRecv) && (flags & NetDeviceUnpack)) ? 1 : 0);
// if (tid == 0) {
// ncclShmem.groups[this->group].devicePlugin.unpack.unpackNetDeviceIndexMask = mask;
// }
// }
// coverity[negative_returns:FALSE] => coverity thinks that index could be -1 but that's not actually the case
// coverity[var_deref_model] => coverity thinks work can dereferenced if NULL but this is not the case
setDataPtrs(inputBuf, outputBuf, redOpArg, (struct ncclDevWorkCollReg*)collWork, sendIpcReg || recvIpcReg, peer);
// coverity[uninit_member] => coverity thinks fan.n is not initialized
} else if (mode == primsModePatRs || mode == primsModePatAg) { // Connect to all ranks +/- 2^n
flags |= PatMode;
accSize = 0;
const int roles[5] = { RoleWaitRecv, RolePostRecv, RoleWaitSend, RolePostSend, RoleInput | RoleOutput };
if (tid < 5) flags |= roles[tid];
int nranks = ncclShmem.comm.nRanks;
int rank = ncclShmem.comm.rank;
// A thread is responsible for rank +/- 2 ^ (tid%32). That should be fine as long as rank is a 32-bits integer.
index = tid % 32;
uint32_t delta = 1 << index;
const int roles[4] = { RoleWaitRecv, RoleWaitSend, RolePostSend, RolePostRecv};
int block = tid / 32;
if (block < 4 && delta < nranks) {
int role = roles[block];
if (mode == primsModePatRs) {
if (role & (RoleWaitRecv|RolePostRecv)) peer = (rank - delta + nranks) % nranks;
if (role & (RoleWaitSend|RolePostSend)) peer = (rank + delta) % nranks;
} else if (mode == primsModePatAg) {
if (role & (RoleWaitSend|RolePostSend)) peer = (rank - delta + nranks) % nranks;
if (role & (RoleWaitRecv|RolePostRecv)) peer = (rank + delta) % nranks;
}
flags |= role;
} else if (tid == 128) {
flags |= RoleInput | RoleOutput; // Only one will be used depending on the operation
if (tid < 32 && ((1UL<<tid) < nranks)) {
int rank = ncclShmem.comm.rank;
uint32_t delta = 1 << tid;
// Load recv peer
int recvPeer = mode == primsModePatRs ? (rank - delta + nranks) % nranks : (rank + delta) % nranks;
struct ncclPatPeer* peer = ((struct ncclPatPeer*)recvPeers)+tid;
struct ncclConnInfo* conn = peer->conn = ncclShmem.channel.peers[recvPeer]->recv+connIndexRecv;
peer->step = conn->step;
peer->buff = conn->buffs[NCCL_PROTO_SIMPLE];
peer->stepCache = loadStepValue(peer->tailPtr = conn->tail);
peer->headPtr = conn->head;
peer->accSize = 0;
peer->connStepSize = conn->stepSize/sizeof(T);
// Load send peer
int sendPeer = mode == primsModePatAg ? (rank - delta + nranks) % nranks : (rank + delta) % nranks;
peer = ((struct ncclPatPeer*)sendPeers)+tid;
conn = peer->conn = ncclShmem.channel.peers[sendPeer]->send+connIndexSend;
peer->step = conn->step;
peer->connFifo = conn->connFifo;
peer->buff = conn->buffs[NCCL_PROTO_SIMPLE];
peer->stepCache = loadStepValue(peer->headPtr = conn->head);
peer->tailPtr = conn->tail;
peer->accSize = 0;
peer->connStepSize = conn->stepSize/sizeof(T);
}
if (tid==0) {
ncclShmem.groups[group].userInput = (void*)inputBuf;
ncclShmem.groups[group].userOutput = (void*)outputBuf;
ncclShmem.redOpArgs[0] = redOpArg; // scaler for local input
}
patBarrier();
}
// Coverity thinks that index could be -1 here but that's not actually the case.
// coverity[negative_returns:FALSE]
int sendIpcReg;
int recvIpcReg;
int sendNetReg;
int recvNetReg;
if (P2p) {
sendIpcReg = p2pWork ? p2pWork->sendIpcReg : 0;
recvIpcReg = p2pWork ? p2pWork->recvIpcReg : 0;
sendNetReg = p2pWork ? p2pWork->sendNetReg : 0;
recvNetReg = p2pWork ? p2pWork->recvNetReg : 0;
} else {
recvIpcReg = sendIpcReg = collWork ? collWork->regUsed : 0;
recvNetReg = sendNetReg = collWork ? collWork->netRegUsed : 0;
}
// coverity[overrun-call] => Coverity think prims.index can be greater than 1
if (flags & (RoleWaitRecv|RolePostRecv)) loadRecvConn(ncclShmem.channel.peers[peer], connIndexRecv, collWork ? collWork->direct : 0, recvIpcReg, recvNetReg);
// coverity[overrun-call] => Coverity think prims.index can be greater than 1
if (flags & (RoleWaitSend|RolePostSend)) loadSendConn(ncclShmem.channel.peers[peer], connIndexSend, collWork ? collWork->direct : 0, sendIpcReg, sendNetReg);
// if (barrierAny(flags & NetDeviceUnpack)) {
// flags |= AnyNetDeviceUnpack;
// // RoleWaitRecv starts at tid=0, so this creates the bitmask of which recv peers
// // have NetDeviceUnpack.
// uint32_t mask = __ballot_sync(~0u, ((flags & RoleWaitRecv) && (flags & NetDeviceUnpack)) ? 1 : 0);
// if (tid == 0) {
// ncclShmem.groups[this->group].devicePlugin.unpack.unpackNetDeviceIndexMask = mask;
// }
// }
// coverity[negative_returns:FALSE] => coverity thinks that index could be -1 but that's not actually the case
// coverity[var_deref_model] => coverity thinks work can dereferenced if NULL but this is not the case
setDataPtrs(inputBuf, outputBuf, redOpArg, (struct ncclDevWorkCollReg*)collWork, sendIpcReg || recvIpcReg, peer);
// coverity[uninit_member] => coverity thinks fan.n is not initialized
}
__forceinline__ __device__ ~Primitives() {
if (flags&PatMode) return;
// Save steps for the next operation
if (flags & (RolePostSend|RolePostRecv)) conn->step = step;
if ((flags & NetRegMode) && (flags & RoleWaitSend)) {
@@ -854,7 +870,7 @@ public:
uint64_t prevStep = step - StepPerSlice;
volatile ssize_t* ptr = &(connFifo[prevStep%NCCL_STEPS].size);
int spins = 0;
while (*ptr != -1) if (checkAbort(spins)) break;
while (*ptr != -1) if (checkAbort(flags, Aborted, spins)) break;
}
if (flags & NetDeviceUnpack) {
@@ -872,7 +888,7 @@ public:
int spins = 0;
volatile uint64_t* tail = conn->tail;
volatile uint64_t* head = conn->head;
while (*tail > *head) if (checkAbort(spins)) break;
while (*tail > *head) if (checkAbort(flags, Aborted, spins)) break;
}
}
@@ -895,7 +911,7 @@ public:
if (slot) {
T* exchgPtr;
directBuff = (T*)outputBuf;
while ((void *)atomicAdd((unsigned long long *) slot,0) != nullptr && !checkAbort(spins));
while ((void *)atomicAdd((unsigned long long *) slot,0) != nullptr && !checkAbort(flags, Aborted, spins));
if (P2p) {
exchgPtr = (T*)outputBuf;
} else {
@@ -912,7 +928,7 @@ public:
void* ptr;
while (slot) {
ptr = (void *)atomicAdd((unsigned long long *) slot,0);
if (ptr != nullptr || checkAbort(spins)) break;
if (ptr != nullptr || checkAbort(flags, Aborted, spins)) break;
}
if (slot) {
@@ -931,7 +947,7 @@ public:
// Wait for consumer to consume previous value before trampling it.
if (slot && argSlot0 && argSlot1) {
T* exchgPtr;
while (((void *)atomicAdd((unsigned long long *) slot,0) != nullptr || *argSlot0 != 0 || *argSlot1 != 0) && !checkAbort(spins));
while (((void *)atomicAdd((unsigned long long *) slot,0) != nullptr || *argSlot0 != 0 || *argSlot1 != 0) && !checkAbort(flags, Aborted, spins));
// If there is no recv, then we are directly pulling from input buffer (e.g. directScatter)
// Otherwise, we are pulling from output buffer (e.g. recvCopyDirectSend)
directBuff = MaxRecv == 0 ? (T*)inputBuf : (T*)outputBuf;
@@ -961,7 +977,7 @@ public:
void* ptr;
while (slot) {
ptr = (void *)atomicAdd((unsigned long long *) slot,0);
if (ptr != nullptr || checkAbort(spins)) break;
if (ptr != nullptr || checkAbort(flags, Aborted, spins)) break;
}
if (slot && argSlot0 && argSlot1) {
@@ -972,7 +988,7 @@ public:
while (true) {
arg0 = *argSlot0;
arg1 = *argSlot1;
if ((arg0 != 0 && arg1 != 0) || checkAbort(spins)) break;
if ((arg0 != 0 && arg1 != 0) || checkAbort(flags, Aborted, spins)) break;
}
ncclShmem.redOpArgs[1 + index] = ((arg1 & 0xffffffff) << 32) | (arg0 & 0xffffffff);
}
@@ -1020,8 +1036,8 @@ public:
__device__ __forceinline__ void recv(intptr_t outIx, int eltN, bool postOp=false) {
genericOp<0, 0, 1, 0, -1, Output>(-1, outIx, eltN, postOp);
}
__device__ __forceinline__ void directRecv(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) {
genericOp<1, 0, 1, 0, -1, Output>(inpIx, outIx, eltN, postOp);
__device__ __forceinline__ void directRecv(intptr_t outIx, int eltN, bool postOp=false) {
genericOp<1, 0, 1, 0, -1, Output>(outIx, outIx, eltN, postOp);
}
__device__ __forceinline__ void directRecvCopy(intptr_t inpIx, intptr_t outIx, int eltN) {
genericOp<1, 0, 1, 0, -1, Output>(inpIx, outIx, eltN, /*postOp=*/false);
@@ -1099,54 +1115,65 @@ public:
ScatterGatherOp<1, 0, 1, 0>(-1, outIx, totalElem, peerElem, peerOffset, skip, shift, /*postOp=*/false);
}
__device__ __forceinline__ void patReduce(int recvPow2, int sendPow2, intptr_t inpIx, intptr_t outIx, int recvOffset, int sendOffset, int sendStepOffset, int nelem, int postRecv, int postSend) {
nelem = nelem < 0 ? 0 : nelem;
__device__ __forceinline__ void patReduce(struct ncclPatStep* ps, struct ncclPatShmem* shmem) {
if (ps->flags & PatSkipped) { patBarrier(); patBarrier(); return; } // Skipped
int nelem = ps->nelem < 0 ? 0 : ps->nelem;
T* userInput = (T*)ncclShmem.groups[group].userInput;
T* userOutput = (T*)ncclShmem.groups[group].userOutput;
if (recvPow2 >= 0 && recvPow2 == index && (flags & RoleWaitRecv)) {
ncclShmem.groups[group].srcs[0] = (T*)(connEltsFifo + (step%NCCL_STEPS)*connStepSize) + recvOffset;
int spins = 0;
while (connStepCache < step + StepPerSlice) {
connStepCache = loadStepValue(connStepPtr);
if (checkAbort(spins)) break;
}
if (postRecv) step += StepPerSlice;
bool recv = ps->recvDim >= 0 && (flags & (RolePostRecv|RoleWaitRecv));
bool send = ps->sendDim >= 0 && (flags & (RolePostSend|RoleWaitSend));
bool postRecv = ps->postRecv && recv;
bool postSend = ps->postSend && send;
struct ncclPatPeer* peer = NULL;
if (recv) {
peer = shmem->recvDims+ps->recvDim;
step = peer->step;
}
if (sendPow2 >= 0 && sendPow2 == index && (flags & RoleWaitSend)) {
int spins = 0;
while (connStepCache + NCCL_STEPS < step + sendStepOffset + StepPerSlice) {
connStepCache = loadStepValue(connStepPtr);
if (checkAbort(spins)) break;
}
ncclShmem.groups[group].dsts[0] = (T*)(connEltsFifo + ((step+sendStepOffset)%NCCL_STEPS)*connStepSize) + sendOffset;
if (accSize < sendOffset + nelem + (step+sendStepOffset)*connStepSize) {
// New data, add our own data to it.
ncclShmem.groups[group].srcs[1] = userInput + inpIx;
accSize = sendOffset + nelem + (step+sendStepOffset)*connStepSize;
if (flags & ConnFifoEnabled)
connFifo[(step+sendStepOffset)%NCCL_STEPS].size = (sendOffset + nelem)*sizeof(T);
} else {
// There is already data in there, accumulate instead of writing to it.
ncclShmem.groups[group].srcs[1] = ncclShmem.groups[group].dsts[0];
}
if (postSend) step += StepPerSlice;
if (send) {
peer = shmem->sendDims+ps->sendDim;
step = peer->step;
}
if (sendPow2 < 0 && (flags & RoleOutput)) { // Destination is our own local buffer
ncclShmem.groups[group].dsts[0] = userOutput + outIx;
if (accSize < outIx + nelem) {
if (recv && (flags & RoleWaitRecv)) {
ncclShmem.groups[group].srcs[0] = ((T*)peer->buff) + (step%NCCL_STEPS)*peer->connStepSize + ps->recvOffset;
int spins = 0;
while (peer->stepCache < step + StepPerSlice) {
peer->stepCache = loadStepValue(peer->tailPtr);
if (checkAbort(flags, Aborted, spins)) break;
}
}
if (send && (flags & RoleWaitSend)) {
int spins = 0;
while (peer->stepCache + NCCL_STEPS < step + ps->stepOffset + StepPerSlice) {
peer->stepCache = loadStepValue(peer->headPtr);
if (checkAbort(flags, Aborted, spins)) break;
}
ncclShmem.groups[group].dsts[0] = ((T*)peer->buff) + ((step+ps->stepOffset)%NCCL_STEPS)*peer->connStepSize + ps->sendOffset;
if (peer->accSize < ps->sendOffset + nelem + (step+ps->stepOffset)*peer->connStepSize) {
// New data, add our own data to it.
ncclShmem.groups[group].srcs[1] = userInput + inpIx;
accSize = outIx + nelem;
ncclShmem.groups[group].srcs[1] = userInput + ps->inpIx;
} else {
// There is already data in there, accumulate instead of writing to it.
ncclShmem.groups[group].srcs[1] = ncclShmem.groups[group].dsts[0];
}
}
barrier();
long long int localAccSize = shmem->localAccSize;
if (ps->sendDim < 0 && (flags & RoleOutput)) { // Destination is our own local buffer
ncclShmem.groups[group].dsts[0] = userOutput + ps->outIx;
if (localAccSize < ps->outIx + nelem) {
// New data, add our own data to it.
ncclShmem.groups[group].srcs[1] = userInput + ps->inpIx;
localAccSize = ps->outIx + nelem;
} else {
// There is already data in there, accumulate instead of writing to it.
ncclShmem.groups[group].srcs[1] = ncclShmem.groups[group].dsts[0];
}
}
patBarrier();
int nSrcs = 2;
void** srcs = ncclShmem.groups[group].srcs;
if (recvPow2 < 0) { srcs++; nSrcs--; } // No peer to receive from, remove one source
if (ps->recvDim < 0) { srcs++; nSrcs--; } // No peer to receive from, remove one source
int workSize = ncclShmem.aborted ? 0 : nelem;
@@ -1154,59 +1181,92 @@ public:
(tid, nthreads, ncclShmem.redOpArgs[0], nullptr, /*postOp=*/false,
nSrcs, srcs, 1, ncclShmem.groups[group].dsts, workSize);
barrier();
if (postRecv && recvPow2 >= 0 && recvPow2 == index && (flags & RolePostRecv)) postPeer<1, 0>(0 < nelem);
if (postSend && sendPow2 >= 0 && sendPow2 == index && (flags & RolePostSend)) postPeer<0, 1>(0 < nelem);
// Store conn step here inside the two barriers to make sure next reload will see the update.
if (postSend && (flags & RolePostSend)) {
if (peer->connFifo) {
peer->connFifo[step%NCCL_STEPS].size = (ps->sendOffset + nelem)*sizeof(T);
}
peer->step = step += StepPerSlice;
st_relaxed_sys_global(&peer->conn->step, step);
}
if (postRecv && (flags & RolePostRecv)) {
peer->step = step += StepPerSlice;
st_relaxed_sys_global(&peer->conn->step, step); // Also save in global mem for next op
}
// Update accSize
if (ps->sendDim < 0 && (flags & RoleOutput)) atomicMax(&shmem->localAccSize, localAccSize);
if (ps->sendDim >= 0 && (flags & RoleWaitSend)) atomicMax(&peer->accSize, ps->sendOffset + nelem + (step+ps->stepOffset)*peer->connStepSize);
patBarrier();
if (postSend && (flags & RolePostSend)) {
if (nelem > 0 || peer->connFifo) fence_acq_rel_sys();
st_relaxed_sys_global(peer->tailPtr, step);
}
if (postRecv && (flags & RolePostRecv)) {
st_relaxed_sys_global(peer->headPtr, step);
}
}
__device__ __forceinline__ void patCopy(int recvPow2, int sendPow2, intptr_t inpIx, intptr_t outIx, int recvOffset, int sendOffset, int recvStepOffset, int nelem, int postRecv, int postSend) {
nelem = nelem < 0 ? 0 : nelem;
__device__ __forceinline__ void patCopy(struct ncclPatStep* ps, struct ncclPatShmem* shmem) {
if (ps->flags & PatSkipped) { patBarrier(); patBarrier(); return; } // Skipped
int nelem = ps->nelem < 0 ? 0 : ps->nelem;
T* userInput = (T*)ncclShmem.groups[group].userInput;
T* userOutput = (T*)ncclShmem.groups[group].userOutput;
if (recvPow2 >= 0 && recvPow2 == index && (flags & RoleWaitRecv)) {
ncclShmem.groups[group].srcs[0] = (T*)(connEltsFifo + ((step+recvStepOffset)%NCCL_STEPS)*connStepSize) + recvOffset;
int spins = 0;
while (connStepCache < step + recvStepOffset + StepPerSlice) {
connStepCache = loadStepValue(connStepPtr);
if (checkAbort(spins)) break;
}
if (accSize < recvOffset + nelem + (step+recvStepOffset)*connStepSize) {
// New data, copy to our output buffer.
ncclShmem.groups[group].dsts[1] = userOutput + outIx;
accSize = recvOffset + nelem + (step+recvStepOffset)*connStepSize;
} else {
ncclShmem.groups[group].dsts[1] = ncclShmem.groups[group].srcs[0]; // Already done
}
if (postRecv) step += StepPerSlice;
bool recv = ps->recvDim >= 0 && (flags & (RolePostRecv|RoleWaitRecv));
bool send = ps->sendDim >= 0 && (flags & (RolePostSend|RoleWaitSend));
bool postRecv = ps->postRecv && recv;
bool postSend = ps->postSend && send;
struct ncclPatPeer* peer = NULL;
if (recv) {
peer = shmem->recvDims+ps->recvDim;
step = peer->step;
}
if (sendPow2 >= 0 && sendPow2 == index && (flags & RoleWaitSend)) {
int spins = 0;
while (connStepCache + NCCL_STEPS < step + StepPerSlice) {
connStepCache = loadStepValue(connStepPtr);
if (checkAbort(spins)) break;
}
ncclShmem.groups[group].dsts[0] = (T*)(connEltsFifo + (step%NCCL_STEPS)*connStepSize) + sendOffset;
if (postSend) {
if (flags & ConnFifoEnabled)
connFifo[step%NCCL_STEPS].size = (sendOffset + nelem)*sizeof(T);
step += StepPerSlice;
}
if (send) {
peer = shmem->sendDims+ps->sendDim;
step = peer->step;
}
if (recvPow2 < 0 && (flags & RoleInput)) { // Source is our own local buffer
ncclShmem.groups[group].srcs[0] = userInput + inpIx;
if (accSize < inpIx + nelem) {
if (recv && (flags & RoleWaitRecv)) {
ncclShmem.groups[group].srcs[0] = ((T*)peer->buff) + ((step+ps->stepOffset)%NCCL_STEPS)*peer->connStepSize + ps->recvOffset;
int spins = 0;
while (peer->stepCache < step + ps->stepOffset + StepPerSlice) {
peer->stepCache = loadStepValue(peer->tailPtr);
if (checkAbort(flags, Aborted, spins)) break;
}
if (peer->accSize < ps->recvOffset + nelem + (step+ps->stepOffset)*peer->connStepSize) {
// New data, copy to our output buffer.
ncclShmem.groups[group].dsts[1] = userOutput + outIx;
accSize = inpIx + nelem;
ncclShmem.groups[group].dsts[1] = userOutput + ps->outIx;
} else {
ncclShmem.groups[group].dsts[1] = ncclShmem.groups[group].srcs[0]; // Already done
}
}
barrier();
if (send && (flags & RoleWaitSend)) {
int spins = 0;
while (peer->stepCache + NCCL_STEPS < step + StepPerSlice) {
peer->stepCache = loadStepValue(peer->headPtr);
if (checkAbort(flags, Aborted, spins)) break;
}
ncclShmem.groups[group].dsts[0] = ((T*)peer->buff) + (step%NCCL_STEPS)*peer->connStepSize + ps->sendOffset;
}
long long int localAccSize = shmem->localAccSize;
if (ps->recvDim < 0 && (flags & RoleInput)) { // Source is our own local buffer
ncclShmem.groups[group].srcs[0] = userInput + ps->inpIx;
if (localAccSize < ps->inpIx + nelem) {
// New data, copy to our output buffer.
ncclShmem.groups[group].dsts[1] = userOutput + ps->outIx;
localAccSize = ps->inpIx + nelem;
} else {
// Already done
ncclShmem.groups[group].dsts[1] = ncclShmem.groups[group].srcs[0];
}
}
patBarrier();
int nDsts = 2;
void** dsts = ncclShmem.groups[group].dsts;
if (sendPow2 < 0) { dsts++; nDsts--; } // No peer to send to, remove one dest
if (ps->sendDim < 0) { dsts++; nDsts--; } // No peer to send to, remove one dest
if (ncclShmem.groups[group].srcs[0] == ncclShmem.groups[group].dsts[1]) nDsts--; // In-place or already done.
int workSize = ncclShmem.aborted ? 0 : nelem;
@@ -1215,9 +1275,32 @@ public:
(tid, nthreads, ncclShmem.redOpArgs[0], nullptr, /*postOp=*/false,
1, ncclShmem.groups[group].srcs, nDsts, dsts, workSize);
barrier();
if (postRecv && recvPow2 >= 0 && recvPow2 == index && (flags & RolePostRecv)) postPeer<1, 0>(0 < nelem);
if (postSend && sendPow2 >= 0 && sendPow2 == index && (flags & RolePostSend)) postPeer<0, 1>(0 < nelem);
// Store conn step here inside the two barriers to make sure next reload will see the update.
if (postSend && (flags & RolePostSend)) {
if (peer->connFifo) {
peer->connFifo[step%NCCL_STEPS].size = (ps->sendOffset + nelem)*sizeof(T);
}
peer->step = step += StepPerSlice;
st_relaxed_sys_global(&peer->conn->step, step);
}
if (postRecv && (flags & RolePostRecv)) {
peer->step = step += StepPerSlice;
st_relaxed_sys_global(&peer->conn->step, step); // Also save in global mem for next op
}
// Update accSize
if (ps->recvDim < 0 && (flags & RoleInput)) atomicMax(&shmem->localAccSize, localAccSize);
if (ps->recvDim >= 0 && (flags & RoleWaitRecv)) atomicMax(&peer->accSize, ps->recvOffset + nelem + (step+ps->stepOffset)*peer->connStepSize);
patBarrier();
if (postSend && (flags & RolePostSend)) {
if (nelem > 0 || peer->connFifo) fence_acq_rel_sys();
st_relaxed_sys_global(peer->tailPtr, step);
}
if (postRecv && (flags & RolePostRecv)) {
st_relaxed_sys_global(peer->headPtr, step);
}
}
// MSCCL primitives
src/device/reduce_scatter.h
+49 -12
Просмотреть файл
@@ -170,29 +170,66 @@ struct RunWorkColl<ncclFuncReduceScatter, T, RedOp, NCCL_ALGO_RING, NCCL_PROTO_L
template<typename T, typename RedOp>
struct RunWorkColl<ncclFuncReduceScatter, T, RedOp, NCCL_ALGO_PAT, NCCL_PROTO_SIMPLE> {
__device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) {
#if __CUDA_ARCH__ >= 600
using Proto = ProtoSimple<1, 1>;
const int nranks = ncclShmem.comm.nRanks;
const int rank = ncclShmem.comm.rank;
size_t count, channelOffset, channelCount, chunkCount;
ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), &count, &channelOffset, &channelCount, &chunkCount);
T *inputBuf = (T*)work->sendbuff;
T *outputBuf = (T*)work->recvbuff;
Primitives<T, RedOp, FanSymmetric<1>, 0, Proto, 0> prims
(tid, nthreads, NULL, NULL, inputBuf, outputBuf, work->redOpArg, 0*Proto::MaxGroupWidth, 0, 0, nullptr, nullptr, 0, primsModePatRs);
static constexpr int nworkers = NCCL_PAT_NWORKERS;
struct ncclPatShmem* shmem = (struct ncclPatShmem*)ncclScratchForWarp(0);
uint64_t pollCount = 0;
__syncthreads(); // Don't start using shared mem until everyone arrives
for (int i=tid; i<NCCL_SHMEM_PAT_STEPS; i+=nthreads) shmem->patSteps[i].flags = 0;
if (tid == 0) shmem->localAccSize = 0;
if (tid == nworkers) shmem->parallelFactor = 0;
__syncthreads();
PatRSAlgorithm<T> patAlgo(chunkCount*sizeof(T), NCCL_STEPS, channelOffset, channelOffset + channelCount, count, chunkCount, rank, nranks);
int last = 0;
while (!last) {
int recvDim, sendDim, recvOffset, sendOffset, sendStepOffset, postRecv, postSend, nelem;
size_t inpIx, outIx;
patAlgo.getNextOp(recvDim, sendDim, inpIx, outIx, recvOffset, sendOffset, sendStepOffset, nelem, postRecv, postSend, last);
prims.patReduce(recvDim, sendDim, inpIx, outIx, recvOffset, sendOffset, sendStepOffset, nelem, postRecv, postSend);
if (tid == nworkers) { // Algo computation thread
PatRSAlgorithm<T> patAlgo(chunkCount*sizeof(T), NCCL_STEPS, NCCL_PAT_NWORKERS/WARP_SIZE, channelOffset, channelOffset + channelCount, count, chunkCount, rank, nranks);
int parallelFactor = shmem->parallelFactor = patAlgo.getParallelFactor();
int step = 0;
while (1) {
struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS);
cuda::atomic_ref<int, cuda::thread_scope_block> poll(ps->flags);
while (poll.load(cuda::memory_order_acquire) != 0) pollCount++; // Wait for workers to be done with step 'step-NCCL_SHMEM_PAT_STEPS'
patAlgo.getNextOp(ps);
int last = ps->last;
step++;
if (last == 2) break;
}
} else if (tid < nworkers) { // Worker threads
T *inputBuf = (T*)work->sendbuff;
T *outputBuf = (T*)work->recvbuff;
int parallelFactor = 0;
volatile int* pfPtr = &shmem->parallelFactor;
while (parallelFactor == 0) parallelFactor = *pfPtr;
int groupSize = nworkers/(WARP_SIZE*parallelFactor) * WARP_SIZE;
int group = tid / groupSize;
int nGroups = nworkers / groupSize;
int tidInGroup = tid - group*groupSize;
// We don't use recvPeers/sendPeers so let's pass shmem structs instead
Primitives<T, RedOp, FanSymmetric<1>, 0, Proto, 0> prims
(tidInGroup, groupSize, (int*)shmem->recvDims, (int*)shmem->sendDims, inputBuf, outputBuf, work->redOpArg, group, 0, 0, nullptr, nullptr, 0, primsModePatRs);
int step = group;
while(1) {
struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS);
cuda::atomic_ref<int, cuda::thread_scope_block> poll(ps->flags);
while (poll.load(cuda::memory_order_acquire) == 0) pollCount++; // Wait for compute thread
int last = ps->last;
prims.patReduce(ps, shmem);
if (tidInGroup == 0) poll.store(0, cuda::memory_order_release); // Return element to compute thread
if (last) break;
step += nGroups;
}
}
#endif
}
};
template<typename T, typename RedOp>
struct RunWorkColl<ncclFuncReduceScatter, T, RedOp, NCCL_ALGO_NVLS, NCCL_PROTO_SIMPLE> {
__device__ __forceinline__ void run(int tid, int/*nthreads*/, struct ncclDevWorkColl* work) {
src/device/sendrecv.h
+1 -1
Просмотреть файл
@@ -122,7 +122,7 @@ struct RunWorkBatch<ncclFuncSendRecv, T, RedOp, NCCL_ALGO_RING, NCCL_PROTO_SIMPL
size_t cursor = 0;
do {
int n = min(size_t(chunkSize), bytes-cursor);
prims.directRecv(cursor, cursor, n);
prims.directRecv(cursor, n);
cursor += n;
} while (cursor < bytes);
src/enqueue.cc
+214 -89
Просмотреть файл
@@ -84,7 +84,6 @@ NCCL_PARAM(L1SharedMemoryCarveout, "L1_SHARED_MEMORY_CARVEOUT", 0);
ncclResult_t ncclInitKernelsForDevice(int cudaArch, int maxSharedMem, size_t* maxStackSize) {
constexpr int KernelCount = sizeof(rcclKernelTable)/sizeof(rcclKernelTable[0]);
ncclResult_t result = ncclSuccess;
int print = 0;
if (maxStackSize) *maxStackSize = 0;
int carveout = ncclParamL1SharedMemoryCarveout();
@@ -115,11 +114,9 @@ ncclResult_t ncclInitKernelsForDevice(int cudaArch, int maxSharedMem, size_t* ma
if (ncclMaxSharedMem != 0) {
int sharedMemSize = ncclMaxSharedMem;
if (sharedMemSize > (maxSharedMem-attr.sharedSizeBytes)) {
if (print++ == 0)
INFO(NCCL_INIT, "ncclMaxSharedMem %d exceeds device/fn maxSharedMem %zu",
sharedMemSize, maxSharedMem-attr.sharedSizeBytes);
// Reduce requested MaxDynamicSharedMemorySize attribute
sharedMemSize = maxSharedMem - attr.sharedSizeBytes;
WARN("cudaArch %d ncclMaxSharedMem %d exceeds device/fn maxSharedMem %zu",
cudaArch, sharedMemSize, maxSharedMem-attr.sharedSizeBytes);
return ncclSystemError;
}
CUDACHECKGOTO(cudaFuncSetAttribute(fn,
cudaFuncAttributeMaxDynamicSharedMemorySize, sharedMemSize),
@@ -366,6 +363,7 @@ ncclResult_t ncclTasksRegAndEnqueue(struct ncclComm* comm) {
devWork.rcclUseOneSlice = comm->rcclUseOneSlice;
devWork.isOneRPN = comm->isOneRPN;
devWork.netRegUsed = devWork.regUsed = 0;
devWork.profilerEnabled = ncclProfilerPluginLoaded() && (task->eActivationMask & ncclProfileKernelCh);
if (task->regBufType & NCCL_NET_REG_BUFFER)
devWork.netRegUsed = 1;
if (task->regBufType & (NCCL_IPC_REG_BUFFER | NCCL_NVLS_REG_BUFFER))
@@ -467,6 +465,7 @@ ncclResult_t ncclPrepareTasks(struct ncclComm* comm, bool* algoNeedConnect, bool
struct ncclTaskColl* next = aggBeg->next;
aggBeg->algorithm = agg.algorithm;
aggBeg->protocol = agg.protocol;
if (aggBeg->protocol == NCCL_PROTO_LL) aggBeg->trafficBytes *= 4;
aggBeg->nMaxChannels = agg.nMaxChannels;
aggBeg->nWarps = agg.nWarps;
aggBeg->devFuncId = agg.devFuncId;
@@ -526,6 +525,7 @@ ncclResult_t ncclPrepareTasks(struct ncclComm* comm, bool* algoNeedConnect, bool
devWork.redOpArgIsPtr = task->opDev.scalarArgIsPtr;
devWork.oneNode = (comm->nNodes == 1);
devWork.netRegUsed = devWork.regUsed = 0;
devWork.profilerEnabled = ncclProfilerPluginLoaded() && (task->eActivationMask & ncclProfileKernelCh);
if (task->regBufType & NCCL_NET_REG_BUFFER)
devWork.netRegUsed = 1;
if (task->regBufType & (NCCL_IPC_REG_BUFFER | NCCL_NVLS_REG_BUFFER))
@@ -559,6 +559,14 @@ ncclResult_t ncclPrepareTasks(struct ncclComm* comm, bool* algoNeedConnect, bool
return ncclSuccess;
}
static ncclResult_t addProfilerProxyOpIfNeeded(struct ncclComm* comm, struct ncclKernelPlan* plan, struct ncclProxyOp* op) {
int tmp = op->pattern;
op->pattern = ncclPatternProfiler;
ncclResult_t ret = addProxyOpIfNeeded(comm, plan, op);
op->pattern = tmp;
return ret;
}
RCCL_PARAM(IntraNetThreshold, "INTRANET_THRESHOLD", 8388608);
static ncclResult_t scheduleCollTasksToPlan(
@@ -571,7 +579,7 @@ static ncclResult_t scheduleCollTasksToPlan(
int nChannels[2*2] = {0, 0, 0, 0}; // [collnet][nvls]
int const nMaxChannels[2*2] = {comm->nChannels, comm->nvlsChannels, // [collnet][nvls]
comm->nChannels, comm->nvlsChannels};
constexpr size_t MinTrafficPerChannel = 512; // Traffic as minimal
constexpr size_t MinTrafficPerChannel = 16 << 10; // 16K traffic as minimal
do {
size_t workBytes = 0;
struct ncclTaskColl* task = ncclIntruQueueHead(&planner->collTaskQueue);
@@ -634,11 +642,16 @@ static ncclResult_t scheduleCollTasksToPlan(
proxyOp.opCount = proxyOpId;
proxyOp.task.coll = task;
proxyOp.rank = comm->rank;
proxyOp.eActivationMask = task->eActivationMask;
proxyOp.incWorkCounter = true;
addWorkBatchToPlan(comm, plan, c, workNode->workType, task->devFuncId, plan->workBytes);
// Set pattern to profiler to add a proxy profiler for kernel events
NCCLCHECK(addProxyOpIfNeeded(comm, plan, &proxyOp));
NCCLCHECK(addProfilerProxyOpIfNeeded(comm, plan, &proxyOp));
}
} else { // not task->isCollnet
int trafficPerByte = ncclFuncTrafficPerByte(task->func, comm->nRanks);
if (task->protocol == NCCL_PROTO_LL) trafficPerByte *= 4;
size_t cellSize = divUp(divUp(MinTrafficPerChannel, (size_t)trafficPerByte), 16) * 16;
int elementsPerCell = cellSize/elementSize;
size_t cells = divUp(task->count*elementSize, cellSize);
@@ -762,6 +775,8 @@ static ncclResult_t scheduleCollTasksToPlan(
}
proxyOp->ringAlgo->incRefCount();
}
proxyOp->eActivationMask = task->eActivationMask;
proxyOp->incWorkCounter = true;
proxyOp->connIndex = 0;
if (task->protocol == NCCL_PROTO_SIMPLE && task->algorithm == NCCL_ALGO_RING) {
if (comm->useIntraNet && nBytes > rcclParamIntraNetThreshold()) {
@@ -773,6 +788,7 @@ static ncclResult_t scheduleCollTasksToPlan(
// determine if that's actually true but it's also not clear if that would be an issue.
// coverity[uninit_use_in_call:FALSE]
NCCLCHECK(addProxyOpIfNeeded(comm, plan, proxyOp));
NCCLCHECK(addProfilerProxyOpIfNeeded(comm, plan, proxyOp));
}
}
@@ -915,7 +931,8 @@ static ncclResult_t addP2pToPlan(
if (protocol[dir] == NCCL_PROTO_LL) chunkSize[dir] *= 2;
if (network[dir]) {
if (bytes[dir] > 0 && proxySameProcess[dir] && protocol[dir] == NCCL_PROTO_SIMPLE && (ncclPxnDisable(comm) || !comm->isAllNvlink)) {
bool pxnUsed = !ncclPxnDisable(comm) && comm->isAllNvlink && comm->maxLocalRanks > 1;
if (bytes[dir] > 0 && proxySameProcess[dir] && protocol[dir] == NCCL_PROTO_SIMPLE && (!pxnUsed)) {
int regFlag = 0;
NCCLCHECK(ncclCalloc(&handles[dir], nChannelsMax));
for (int part = 0; part < nChannelsMax; part++) {
@@ -991,6 +1008,7 @@ static ncclResult_t addP2pToPlan(
work->recvRank = recvRank;
work->recvAddr = recvAddr;
work->recvBytes = recvBytes==-1 ? 0 : recvBytes;
work->profilerEnabled = ncclProfilerPluginLoaded() && ((p2pTasks[0] ? p2pTasks[0] : p2pTasks[1])->eActivationMask & ncclProfileKernelCh);
work->recvConnIndex = connIndex[0];
work->recvOpCount = recvOpCount;
@@ -1010,6 +1028,7 @@ static ncclResult_t addP2pToPlan(
op->coll = p2pTasks[dir] ? p2pTasks[dir]->func : 0;
op->task.p2p = p2pTasks[dir];
op->rank = comm->rank;
op->eActivationMask = p2pTasks[dir] ? p2pTasks[dir]->eActivationMask : 0;
op->connIndex = connIndex[dir];
// The following are modified per channel part in addWorkToChannels():
// op->buffer, op->nbytes, op->nsteps = ...;
@@ -1017,6 +1036,7 @@ static ncclResult_t addP2pToPlan(
nChannelsMax = std::max(nChannels[0], nChannels[1]);
for (int part=0; part < nChannelsMax; part++) {
int incWorkCounter = -1;
int channelId = ncclP2pChannelForPart(comm->p2pnChannels, base, part, comm->p2pnChannelsPerPeer, comm->nNodes);
plan->channelMask.masks[channelId/64] |= uint64_t(1)<<(channelId%64);
// Add batch first.
@@ -1058,12 +1078,19 @@ static ncclResult_t addP2pToPlan(
}
}
// Increment work counter for <send, recv> pair rather than individual p2p
if (proxyOps[dir].nsteps && incWorkCounter < 0) {
proxyOps[dir].incWorkCounter = true;
incWorkCounter = dir;
}
if (proxyOps[dir].nsteps != 0) {
// Calculate the opCount after adding batch since then the batch count will
// equal one plus the batch index this p2p settled in.
proxyOps[dir].channelId = channelId;
proxyOps[dir].opCount = uint64_t(comm->planner.wipPlan.channels[channelId].nWorkBatchesP2p)<<1 | 1;
NCCLCHECK(addProxyOpIfNeeded(comm, plan, &proxyOps[dir]));
NCCLCHECK(addProfilerProxyOpIfNeeded(comm, plan, &proxyOps[dir]));
}
}
}
@@ -1289,22 +1316,23 @@ static ncclResult_t uploadWork(struct ncclComm* comm, struct ncclKernelPlan* pla
struct uploadWork_cleanup_t* cleanup = nullptr;
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
void* fifoBufDev = nullptr;
cudaStream_t deviceStream;
CUDACHECKGOTO(cudaThreadExchangeStreamCaptureMode(&mode), result, fail);
// Acquire deviceStream to gain access to deviceStream.cudaStream. Since the
// user's graph will be launched later, and it also acquires the deviceStream,
// it will observe this upload.
NCCLCHECKGOTO(ncclStrongStreamAcquireUncaptured(&comm->sharedRes->deviceStream), result, fail);
// Acquire deviceStream. Since the user's graph will be launched later and it also
// acquires the deviceStream, it will observe this upload.
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), result, fail);
CUDACHECKGOTO(cudaMallocAsync(&fifoBufDev, workBytes, comm->memPool, comm->sharedRes->deviceStream.cudaStream), result, fail);
CUDACHECKGOTO(cudaMallocAsync(&fifoBufDev, workBytes, comm->memPool, deviceStream), result, fail);
plan->workBufPersistent = fifoBufDev;
plan->kernelArgs->workBuf = fifoBufDev;
// coverity[uninit_use_in_call:FALSE] => fifoBufHost is never NULL
CUDACHECKGOTO(cudaMemcpyAsync(fifoBufDev, fifoBufHost, workBytes, cudaMemcpyDefault, comm->sharedRes->deviceStream.cudaStream), result, fail);
CUDACHECKGOTO(cudaMemcpyAsync(fifoBufDev, fifoBufHost, workBytes, cudaMemcpyDefault, deviceStream), result, fail);
cudaEvent_t memcpyDone;
CUDACHECKGOTO(cudaEventCreateWithFlags(&memcpyDone, cudaEventDisableTiming), result, fail);
CUDACHECKGOTO(cudaEventRecord(memcpyDone, comm->sharedRes->deviceStream.cudaStream), result, fail);
CUDACHECKGOTO(cudaEventRecord(memcpyDone, deviceStream), result, fail);
NCCLCHECKGOTO(ncclCalloc(&cleanup, 1), result, fail);
cleanup->base.fn = uploadWork_cleanup_fn;
@@ -1312,7 +1340,7 @@ static ncclResult_t uploadWork(struct ncclComm* comm, struct ncclKernelPlan* pla
cleanup->hostBuf = fifoBufHost;
ncclIntruQueueEnqueue(&comm->eventCallbackQueue, (struct ncclCommEventCallback *)cleanup);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->deviceStream), result, fail);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false), result, fail);
NCCLCHECKGOTO(ncclCommPollEventCallbacks(comm), result, fail);
finish_scope:
@@ -1386,15 +1414,38 @@ static void HIPRT_CB hostStreamPlanCallback(void *plan_) {
if (result != ncclSuccess) {
WARN("hostStreamPlanCallback() failed : %s", ncclGetErrorString(result));
}
if (!plan->persistent) ncclAtomicRefCountDecrement(&plan->comm->noncapturedRefs);
if (!plan->persistent) ncclAtomicRefCountDecrement(&plan->comm->sharedRes->noncapturedRefs);
return;
}
static ncclResult_t reclaimPlan(struct ncclComm* comm, struct ncclCommCallback* me) {
struct ncclKernelPlan* plan = (struct ncclKernelPlan*)me; // cast from first member `reclaim`
if (plan->persistent) {
comm->persistentRefs -= 1;
NCCLCHECK(ncclCudaFree(plan->workBufPersistent));
comm->sharedRes->persistentRefs -= 1;
comm->localPersistentRefs -= 1;
if (plan->workStorageType == ncclDevWorkStorageTypePersistent) {
cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed;
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
CUDACHECK(cudaFree(plan->workBufPersistent));
CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode));
}
}
// Free coll tasks
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
while (ct != nullptr) {
struct ncclTaskColl* ct1 = ct->next;
free(ct->sendNetHandles);
free(ct->recvNetHandles);
free(ct->srecvNetHandles);
ncclMemoryPoolFree(&comm->memPool_ncclTaskColl, ct);
ct = ct1;
}
// Free p2p tasks
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
while (pt != nullptr) {
struct ncclTaskP2p* pt1 = pt->next;
ncclMemoryPoolFree(&comm->memPool_ncclTaskP2p, pt);
pt = pt1;
}
// Free proxy ops
struct ncclProxyOp* q = ncclIntruQueueHead(&plan->proxyOpQueue);
@@ -1427,6 +1478,32 @@ static void persistentDestructor(void* plans_) {
}
}
NCCL_PARAM(LaunchOrderImplicit, "LAUNCH_ORDER_IMPLICIT", 0);
namespace {
enum ncclImplicitOrder {
ncclImplicitOrderNone,
ncclImplicitOrderSerial,
ncclImplicitOrderLaunch
};
}
static ncclResult_t getImplicitOrder(enum ncclImplicitOrder *mode, bool capturing, int driver=-1) {
if (ncclParamLaunchOrderImplicit()) {
#if !defined(__HIP_PLATFORM_AMD__) || !defined(__HIPCC__)
// Due to an unresolved bug in CUDA ncclImplicitOrderLaunch is not supported in graphs
if (capturing) { *mode = ncclImplicitOrderSerial; return ncclSuccess; }
if (driver < 0) { NCCLCHECK(ncclCudaDriverVersion(&driver)); }
*mode = 12030 <= std::min<int>(CUDART_VERSION, driver) ? ncclImplicitOrderLaunch : ncclImplicitOrderSerial;
#else
*mode = ncclImplicitOrderNone;
#endif
return ncclSuccess;
}
*mode = ncclImplicitOrderNone;
return ncclSuccess;
}
ncclResult_t ncclLaunchPrepare(struct ncclComm* comm) {
ncclResult_t result = ncclSuccess;
struct ncclKernelPlanner* planner = &comm->planner;
@@ -1474,63 +1551,65 @@ ncclResult_t ncclLaunchPrepare(struct ncclComm* comm) {
if (nPlans == 0) return ncclSuccess;
// Semantically we want these dependencies for the kernels launched:
// 1. Launch host task on hostStream.
// 2. Launch kernel, depends on all of {deviceStream, hostStream, userStream[i]...}
// 3. {deviceStream, userStream[i]...} depend on kernel.
// We achieve this by:
// 1. userStream[0] waits on deviceStream
// 2. deviceStream waits on each of userStream[1...]
// 3. host task launch on hostStream
// 4. userStream[0] waits on hostStream
// 5. kernel launch on userStream[0]
// 6. deviceStream waits on userStream[0]
// 7. userStream[1...] each waits on deviceStream
// The two-level fan-in fan-out is because ncclStrongStreamWaitStream() requires
// at least one of the two streams to be strong-stream.
cudaStream_t launchStream = planner->streams->stream;
NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->sharedRes->deviceStream), result, failure);
cudaStream_t deviceStream, launchOrder;
NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), result, failure);
if (planner->numStreams != 1 || persistent) {
// Create dependency for device stream on user streams. First from extra user
// streams to deviceStream. Then deviceStream to first user stream.
if (persistent || planner->numStreams != 1) {
// userStream[0] waits on each userStream[i]...
for (struct ncclCudaStreamList* l=planner->streams->next; l != nullptr; l = l->next) {
NCCLCHECKGOTO(ncclStrongStreamWaitStream(planner->capturingGraph, &comm->sharedRes->deviceStream, l->stream), result, failure);
CUDACHECKGOTO(cudaEventRecord(comm->sharedRes->scratchEvent, l->stream), result, failure);
CUDACHECKGOTO(cudaStreamWaitEvent(launchStream, comm->sharedRes->scratchEvent, 0), result, failure);
}
NCCLCHECKGOTO(ncclStrongStreamWaitStream(planner->capturingGraph, launchStream, &comm->sharedRes->deviceStream), result, failure);
// userStream[0] waits on deviceStream
NCCLCHECKGOTO(ncclStreamWaitStream(launchStream, deviceStream, comm->sharedRes->scratchEvent), result, failure);
} else if (planner->streams->stream != comm->lastStream && comm->lastStream != nullptr && !persistent) {
// Stream changed from last call, create dependency against last NCCL kernel launch
CUDACHECK(hipStreamWaitEvent(planner->streams->stream, comm->doneEvent, 0));
CUDACHECKGOTO(hipStreamWaitEvent(planner->streams->stream, comm->doneEvent, 0), result, failure);
}
if (persistent || comm->persistentRefs != 0 || ncclCudaLaunchBlocking || __atomic_load_n(&comm->noncapturedRefs, __ATOMIC_ACQUIRE)) {
bool capturing = ncclCudaGraphValid(planner->capturingGraph);
enum ncclImplicitOrder implicitOrder;
NCCLCHECKGOTO(getImplicitOrder(&implicitOrder, capturing), result, failure);
if (implicitOrder != ncclImplicitOrderNone) {
// userStream[0] waits on per-device (context) launchOrder. Concurrent strong stream access is
// required if this is a graph capture, non-captured cannot be concurrent because that would violate
// deterministic program order of launches.
bool concurrent = capturing;
NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->context->launchOrder, concurrent, &launchOrder), result, failure);
NCCLCHECKGOTO(ncclStreamWaitStream(launchStream, launchOrder, comm->sharedRes->scratchEvent), result, failure);
}
if (persistent || comm->sharedRes->persistentRefs != 0 || ncclCudaLaunchBlocking || __atomic_load_n(&comm->sharedRes->noncapturedRefs, __ATOMIC_ACQUIRE)) {
// We have to launch host tasks to push proxy args. We are careful to only
// do this if necessary since host tasks impose a high performance cost in CUDA.
bool acquired = false;
cudaStream_t hostStream;
for (struct ncclKernelPlan* plan=planHead; plan != nullptr; plan = plan->next) {
if (plan->hasProxyOps) {
if (!acquired) {
acquired = true;
NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->sharedRes->hostStream), result, failure);
NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->sharedRes->hostStream, /*concurrent=*/false, &hostStream), result, failure);
}
if (!persistent) ncclAtomicRefCountIncrement(&comm->noncapturedRefs);
if (!persistent) ncclAtomicRefCountIncrement(&comm->sharedRes->noncapturedRefs);
plan->isHostCbEnq = true;
NCCLCHECKGOTO(ncclStrongStreamLaunchHost(planner->capturingGraph, &comm->sharedRes->hostStream, hostStreamPlanCallback, plan), result, failure);
CUDACHECKGOTO(cudaLaunchHostFunc(hostStream, hostStreamPlanCallback, plan), result, failure);
}
}
if (acquired) {
// Make to-be-launched kernels dependent on just-launched host stream tasks.
NCCLCHECKGOTO(ncclStrongStreamWaitStream(planner->capturingGraph, launchStream, &comm->sharedRes->hostStream), result, failure);
NCCLCHECKGOTO(ncclStrongStreamRelease(planner->capturingGraph, &comm->sharedRes->hostStream), result, failure);
NCCLCHECKGOTO(ncclStreamWaitStream(launchStream, hostStream, comm->sharedRes->scratchEvent), result, failure);
NCCLCHECKGOTO(ncclStrongStreamRelease(planner->capturingGraph, &comm->sharedRes->hostStream, /*concurrent=*/false), result, failure);
}
}
if (persistent) {
comm->persistentRefs += nPlans;
comm->sharedRes->persistentRefs += nPlans;
comm->localPersistentRefs += nPlans;
NCCLCHECKGOTO(ncclCudaGraphAddDestructor(planner->capturingGraph, persistentDestructor, (void*)planHead), result, failure);
}
}
failure:
return result;
}
@@ -1549,6 +1628,7 @@ NCCL_PARAM(MemSyncDomain, "MEM_SYNC_DOMAIN", cudaLaunchMemSyncDomainRemote);
#endif
ncclResult_t ncclLaunchKernel(struct ncclComm* comm, struct ncclKernelPlan* plan) {
ncclResult_t ret = ncclSuccess;
struct ncclKernelPlanner* planner = &comm->planner;
int nChannels = 0;
for (int i = 0; i < MAXCHANNELS/64; i++)
@@ -1561,23 +1641,28 @@ ncclResult_t ncclLaunchKernel(struct ncclComm* comm, struct ncclKernelPlan* plan
void* extra[] = {plan->kernelArgs, &plan->kernelArgsSize};
if (planner->numStreams == 1 && !plan->persistent) {
CUDACHECK(hipExtLaunchKernel(plan->kernelFn, grid, block, extra, 0, launchStream, NULL, comm->doneEvent, 0));
comm->lastStream = planner->streams->stream;
CUDACHECKGOTO(hipExtLaunchKernel(plan->kernelFn, grid, block, extra, 0, launchStream, NULL, comm->doneEvent, 0), ret, do_return);
return ncclSuccess;
}
// CUfunction fn;
// CUDACHECK(cudaGetFuncBySymbol(&fn, sym));
#if CUDART_VERSION >= 11080
#if !defined(__HIP_PLATFORM_AMD__) || !defined(__HIPCC__)
int driverVersion;
NCCLCHECK(ncclCudaDriverVersion(&driverVersion));
if (driverVersion >= 11080) {
NCCLCHECKGOTO(ncclCudaDriverVersion(&driverVersion), ret, do_return);
CUfunction fn;
CUDACHECKGOTO(cudaGetFuncBySymbol(&fn, sym), ret, do_return);
if (CUDART_VERSION >= 11080 && driverVersion >= 11080) {
#if CUDART_VERSION >= 11080
int compCap = comm->compCap;
unsigned int clusterSize = (compCap >= 90) ? comm->config.cgaClusterSize : 0;
CUlaunchConfig launchConfig = {0};
CUlaunchAttribute launchAttrs[3];
CUlaunchAttribute launchAttrs[4] = {};
int attrs = 0;
/* Cooperative Group Array (CGA)
* On sm90 and later we have an extra level of hierarchy where we
@@ -1604,6 +1689,17 @@ ncclResult_t ncclLaunchKernel(struct ncclComm* comm, struct ncclKernelPlan* plan
launchAttrs[attrs++].value.memSyncDomain = (CUlaunchMemSyncDomain) ncclParamMemSyncDomain();
}
#endif
#if CUDART_VERSION >= 12030
bool capturing = ncclCudaGraphValid(planner->capturingGraph);
enum ncclImplicitOrder implicitOrder;
NCCLCHECKGOTO(getImplicitOrder(&implicitOrder, capturing, driverVersion), ret, do_return);
if (implicitOrder == ncclImplicitOrderLaunch) {
launchAttrs[attrs].id = CU_LAUNCH_ATTRIBUTE_LAUNCH_COMPLETION_EVENT;
launchAttrs[attrs].value.launchCompletionEvent.event = comm->sharedRes->launchEvent;
launchAttrs[attrs].value.launchCompletionEvent.flags = 0;
attrs++;
}
#endif
launchConfig.gridDimX = grid.x;
launchConfig.gridDimY = grid.y;
launchConfig.gridDimZ = grid.z;
@@ -1615,15 +1711,19 @@ ncclResult_t ncclLaunchKernel(struct ncclComm* comm, struct ncclKernelPlan* plan
launchConfig.numAttrs = attrs;
launchConfig.hStream = launchStream;
//CUDACHECK(cudaLaunchKernelExC(&launchConfig, fnAddr, args));
CUCHECK(cuLaunchKernelEx(&launchConfig, fn, nullptr, extra));
return ncclSuccess;
}
CUCHECKGOTO(cuLaunchKernelEx(&launchConfig, fn, nullptr, extra), ret, do_return);
#endif
} else {
// Standard kernel launch
CUCHECKGOTO(cuLaunchKernel(fn, grid.x, grid.y, grid.z, block.x, block.y, block.z, smem, launchStream, nullptr, extra), ret, do_return);
}
#endif
// Standard kernel launch
//cuLaunchKernel(sym, grid.x, grid.y, grid.z, block.x, block.y, block.z, smem, launchStream, nullptr, extra);
CUDACHECK(cudaLaunchKernel(sym, grid, block, extra, smem, launchStream));
return ncclSuccess;
CUDACHECKGOTO(cudaLaunchKernel(sym, grid, block, extra, smem, launchStream), ret, do_return);
do_return:
return ret;
}
ncclResult_t ncclLaunchKernelAfter_NoCuda(struct ncclComm* comm, struct ncclKernelPlan* plan) {
@@ -1643,36 +1743,51 @@ ncclResult_t ncclLaunchKernelAfter_NoCuda(struct ncclComm* comm, struct ncclKern
}
ncclResult_t ncclLaunchFinish(struct ncclComm* comm) {
ncclResult_t result = ncclSuccess;
struct ncclKernelPlanner* planner = &comm->planner;
bool persistent = ncclCudaGraphValid(planner->capturingGraph);
if (!ncclIntruQueueEmpty(&planner->planQueue)) {
// Reset queue to empty without destroying plans since those will be sent
// back to us for reclaiming via callbackQueue.
ncclIntruQueueConstruct(&planner->planQueue);
bool capturing = ncclCudaGraphValid(planner->capturingGraph);
cudaStream_t launchStream = planner->streams->stream; // First user stream gets launch
// Create dependency for deviceStream on launchStream. We know that deviceStream
// hasn't been modified since launchStream waited on it (in ncclLaunchPrepare),
// so we can say that launchStream subsumes it.
if (persistent || planner->numStreams != 1) NCCLCHECKGOTO(ncclStrongStreamWaitStream(planner->capturingGraph, &comm->sharedRes->deviceStream, launchStream, /*b_subsumes_a=*/true), result, resume1);
resume1:
// Create dependency for other user streams (skip launch stream) on deviceStream.
// Again, the user streams haven't been touched since deviceStream waited on them
// so we can say they are subsumed by deviceStream.
struct ncclCudaStreamList* sl = planner->streams->next;
planner->streams = nullptr; // Reset comm->planner.streams to empty.
while (sl != nullptr && (planner->numStreams != 1 || persistent)) {
NCCLCHECKGOTO(ncclStrongStreamWaitStream(planner->capturingGraph, sl->stream, &comm->sharedRes->deviceStream, /*b_subsumes_a=*/true), result, resume2);
resume2:
sl = sl->next;
cudaStream_t deviceStream, launchOrder;
if (capturing || planner->numStreams != 1) {
// CUDACHECK(cudaEventRecord(comm->sharedRes->scratchEvent, launchStream));
// deviceStream waits on userStream[0]
NCCLCHECK(ncclStrongStreamAcquiredWorkStream(planner->capturingGraph, &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream));
// We know that deviceStream is strictly behind the launchStream because launchStream
// synced with it before kernel launch. This allows us to to see deviceStream waiting
// on launchStream as a fast-forward. When building CUDA graphs fast forwards should
// be handled specially so as not to create graphs with a blowup in the number of edges.
// So we could do this:
// CUDACHECK(cudaStreamWaitEvent(deviceStream, comm->sharedRes->scratchEvent, 0));
// But instead we do:
NCCLCHECK(ncclStreamAdvanceToEvent(planner->capturingGraph, deviceStream, comm->sharedRes->scratchEvent));
// Each userStream[i] waits on userStream[0]
for (struct ncclCudaStreamList* l=planner->streams->next; l != nullptr; l = l->next) {
CUDACHECK(cudaStreamWaitEvent(l->stream, comm->sharedRes->scratchEvent, 0));
}
}
planner->numStreams = 0;
// Release device stream as acquired in ncclLaunchPrepare()
NCCLCHECKGOTO(ncclStrongStreamRelease(planner->capturingGraph, &comm->sharedRes->deviceStream), result, resume3);
resume3:;
enum ncclImplicitOrder implicitOrder;
NCCLCHECK(getImplicitOrder(&implicitOrder, capturing));
if (implicitOrder != ncclImplicitOrderNone) {
// As in ncclLaunchPrepare, strong stream can be non-concurrent when non-captured.
bool concurrent = capturing;
// Incorporate launch event into per-device (context) launch order.
NCCLCHECK(ncclStrongStreamAcquiredWorkStream(planner->capturingGraph, &comm->context->launchOrder, concurrent, &launchOrder));
// If we don't have launch events (requires CUDA 12.3) then just use completion event (serialize execution).
CUDACHECK(cudaStreamWaitEvent(launchOrder, implicitOrder == ncclImplicitOrderLaunch ? comm->sharedRes->launchEvent : comm->sharedRes->scratchEvent));
// Release launchOrder as acquired in ncclLaunchPrepare()
NCCLCHECK(ncclStrongStreamRelease(planner->capturingGraph, &comm->context->launchOrder, concurrent));
}
// Release deviceStream as acquired in ncclLaunchPrepare()
NCCLCHECK(ncclStrongStreamRelease(planner->capturingGraph, &comm->sharedRes->deviceStream, /*concurrent=*/false));
}
return result;
return ncclSuccess;
}
/*****************************************************************************/
@@ -1780,11 +1895,11 @@ static ncclResult_t topoGetAlgoInfo(
if (info->algorithm == NCCL_ALGO_UNDEF || info->protocol == NCCL_PROTO_UNDEF) {
char ncclAlgoEnvStr[1024] = "";
char ncclProtoEnvStr[1024] = "";
char* algoEnv = getenv("NCCL_ALGO");
const char* algoEnv = ncclGetEnv("NCCL_ALGO");
if (algoEnv) {
snprintf(ncclAlgoEnvStr, 1023, " NCCL_ALGO was set to %s.", algoEnv);
}
char* protoEnv = getenv("NCCL_PROTO");
const char* protoEnv = ncclGetEnv("NCCL_PROTO");
if (protoEnv) {
snprintf(ncclProtoEnvStr, 1023, " NCCL_PROTO was set to %s.", protoEnv);
}
@@ -2265,12 +2380,13 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) {
// Must be in thread local group before tasks can be alloc'd in `comm->memScoped`.
ncclGroupCommJoin(info->comm);
struct ncclTaskP2p* p2p = ncclMemoryStackAlloc<struct ncclTaskP2p>(&comm->memScoped);
struct ncclTaskP2p* p2p = ncclMemoryPoolAlloc<struct ncclTaskP2p>(&comm->memPool_ncclTaskP2p, &comm->memPermanent);
p2p->buff = (void*)info->recvbuff;
p2p->count = info->count;
p2p->datatype = info->datatype;
p2p->root = info->root;
p2p->bytes = nBytes;
p2p->eActivationMask = __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED);
p2p->opCount = comm->opCount;
ncclIntruQueueEnqueue(
isSendNotRecv ? &planner->peers[peer].sendQueue : &planner->peers[peer].recvQueue,
@@ -2280,6 +2396,7 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) {
// Mark channels that need pre-connect
if (comm->rank != peer) {
if (!(isSendNotRecv ? planner->peers[peer].sendSeen : planner->peers[peer].recvSeen)) {
// planner->peers[peer].send/recvSeen is private to each comm, so we need to set it anyway.
(isSendNotRecv ? planner->peers[peer].sendSeen : planner->peers[peer].recvSeen) = true;
int round = 0;
while (peer != (isSendNotRecv ? comm->p2pSchedule[round].sendRank
@@ -2290,23 +2407,30 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) {
for (int c=0; c < comm->p2pnChannelsPerPeer; c++) {
int channelId = ncclP2pChannelForPart(comm->p2pnChannels, base, c, comm->p2pnChannelsPerPeer, comm->nNodes);
if (isSendNotRecv) {
if (comm->channels[channelId].peers[peer]->send[1].connected == 0) { // P2P uses only 1 connector
if (comm->channels[channelId].peers[peer]->send[1].hasSeen == 0) { // P2P uses only 1 connector
// the send/recv connector is shared among split shared comms. We need to set hasSeen to
// 1 in order to avoid duplicate connection setup if user group sendrecv ops with split
// shared comms together.
comm->channels[channelId].peers[peer]->send[1].hasSeen = 1;
//comm->connectSend[peer] |= (1UL<<channelId);
comm->connectSend[peer].masks[channelId/64] |= (1UL<<(channelId%64));
ncclGroupCommPreconnect(comm);
}
if (comm->p2pNet && comm->channels[channelId].peers[peer]->send[NCCL_CONN_IDX_P2P_NET].connected == 0) {
if (comm->p2pNet && comm->channels[channelId].peers[peer]->send[NCCL_CONN_IDX_P2P_NET].hasSeen == 0) {
comm->channels[channelId].peers[peer]->send[1].hasSeen = 1;
//comm->connectSend[peer+comm->nRanks*NCCL_CONN_IDX_P2P_NET] |= (1UL<<channelId);
comm->connectSend[peer+comm->nRanks*NCCL_CONN_IDX_P2P_NET].masks[channelId/64] |= (1UL<<(channelId%64));
ncclGroupCommPreconnect(comm);
}
} else {
if (comm->channels[channelId].peers[peer]->recv[1].connected == 0) { // P2P uses only 1 connector
if (comm->channels[channelId].peers[peer]->recv[1].hasSeen == 0) { // P2P uses only 1 connector
comm->channels[channelId].peers[peer]->recv[1].hasSeen = 1;
//comm->connectRecv[peer] |= (1UL<<channelId);
comm->connectRecv[peer].masks[channelId/64] |= (1UL<<(channelId%64));
ncclGroupCommPreconnect(comm);
}
if (comm->p2pNet && comm->channels[channelId].peers[peer]->recv[NCCL_CONN_IDX_P2P_NET].connected == 0) {
if (comm->p2pNet && comm->channels[channelId].peers[peer]->recv[NCCL_CONN_IDX_P2P_NET].hasSeen == 0) {
comm->channels[channelId].peers[peer]->recv[1].hasSeen = 1;
//comm->connectRecv[peer+comm->nRanks*NCCL_CONN_IDX_P2P_NET] |= (1UL<<channelId);
comm->connectRecv[peer+comm->nRanks*NCCL_CONN_IDX_P2P_NET].masks[channelId/64] |= (1UL<<(channelId%64));
ncclGroupCommPreconnect(comm);
@@ -2337,7 +2461,7 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) {
} else {
// Must be in thread local group before tasks can be alloc'd in `comm->memScoped`.
ncclGroupCommJoin(info->comm);
struct ncclTaskColl* t = ncclMemoryStackAlloc<struct ncclTaskColl>(&comm->memScoped);
struct ncclTaskColl* t = ncclMemoryPoolAlloc<struct ncclTaskColl>(&comm->memPool_ncclTaskColl, &comm->memPermanent);
t->func = info->coll;
t->sendbuff = info->sendbuff;
t->recvbuff = info->recvbuff;
@@ -2355,6 +2479,7 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) {
t->opDev = opDev; // C++ struct assignment
t->chunkSteps = info->chunkSteps;
t->sliceSteps = info->sliceSteps;
t->eActivationMask = __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED);
t->opCount = comm->opCount;
planner->nTasksColl += 1;
src/graph/connect.cc
+1 -1
Просмотреть файл
@@ -731,7 +731,7 @@ ncclResult_t ncclTopoPostset(struct ncclComm* comm, int* firstRanks, int* treePa
NCCLCHECKGOTO(ncclCalloc(&nvlsHeads, nNodes*MAXCHANNELS), ret, fail);
// Alternate rings to avoid crossing rails
if (graphs[NCCL_ALGO_RING]->crossNic && (nChannels % 2) == 0) {
if (graphs[NCCL_ALGO_RING]->crossNic == 2 && (nChannels % 2) == 0) {
for (int r=0; r<comm->nRanks; r++) {
if (comm->rankToNode[r] % 2 == 1) {
// Exchange rings
src/graph/paths.cc
+78 -20
Просмотреть файл
@@ -380,8 +380,8 @@ ncclResult_t ncclTopoCheckMNNVL(struct ncclTopoSystem* system, struct ncclPeerIn
if ((((long *)&fabricInfo2->clusterUuid)[0]|((long *)fabricInfo2->clusterUuid)[1]) == 0) return ncclSuccess;
if ((memcmp(fabricInfo1->clusterUuid, fabricInfo2->clusterUuid, NVML_GPU_FABRIC_UUID_LEN) == 0) &&
(fabricInfo1->cliqueId == fabricInfo2->cliqueId)) {
INFO(NCCL_NET, "MNNVL matching peer 0x%lx UUID %lx.%lx cliqueId 0x%x",
info2->busId, ((long *)fabricInfo2->clusterUuid)[0], ((long *)fabricInfo2->clusterUuid)[1], fabricInfo2->cliqueId);
TRACE(NCCL_NET, "MNNVL matching peer 0x%lx UUID %lx.%lx cliqueId 0x%x",
info2->busId, ((long *)fabricInfo2->clusterUuid)[0], ((long *)fabricInfo2->clusterUuid)[1], fabricInfo2->cliqueId);
*ret = 1;
}
return ncclSuccess;
@@ -389,9 +389,12 @@ ncclResult_t ncclTopoCheckMNNVL(struct ncclTopoSystem* system, struct ncclPeerIn
NCCL_PARAM(NetGdrRead, "NET_GDR_READ", -2);
int ncclTopoUserGdrLevel = -1;
const char* ncclTopoGdrModeStr[ncclTopoGdrModeNum] = { "Disabled", "Default", "PCI" };
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t netId, int read, int* useGdr) {
*useGdr = 0;
NCCL_PARAM(NetGdrC2c, "NET_GDR_C2C", 0);
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t netId, int read, enum ncclTopoGdrMode* gdrMode) {
*gdrMode = ncclTopoGdrModeDisable;
// Get GPU and NET
int n, g;
@@ -434,7 +437,7 @@ ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t n
else {
int arch, vendor, model;
NCCLCHECK(ncclTopoCpuType(system, &arch, &vendor, &model));
if (arch == NCCL_TOPO_CPU_ARCH_X86 && vendor == NCCL_TOPO_CPU_VENDOR_AMD && model == NCCL_TOPO_CPU_TYPE_ROME) {
if (arch == NCCL_TOPO_CPU_ARCH_X86 && vendor == NCCL_TOPO_CPU_VENDOR_AMD && model == NCCL_TOPO_CPU_MODEL_AMD_ROME) {
int i, d1 = -1, d2 = -1;
for (i = 0; i < system->nodes[CPU].count; i++)
if (system->nodes[GPU].nodes[g].paths[CPU][i].count == 2) break;
@@ -452,25 +455,37 @@ ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t n
int distance = gpu->paths[NET][n].type;
if (distance == PATH_PXN) {
// In case of PXN, use the intermediate GPU distance instead
int proxyRank, g;
int proxyRank;
NCCLCHECK(ncclTopoGetIntermediateRank(system, gpu->gpu.rank, netId, &proxyRank));
NCCLCHECK(ncclTopoRankToIndex(system, proxyRank, &g));
struct ncclTopoNode* proxyGpu = system->nodes[GPU].nodes+g;
distance = proxyGpu->paths[NET][n].type;
gpu = system->nodes[GPU].nodes+g;
distance = gpu->paths[NET][n].type;
}
int c;
NCCLCHECK(ncclGetLocalCpu(system, g, &c));
if (ncclParamNetGdrC2c() && distance == PATH_PHB && gpu->paths[CPU][c].type == PATH_C2C) {
// On C2C platforms we can still use GDRDMA on NICs connected to the CPUs
INFO(NCCL_NET, "GPU %d / HCA %lx connected to CPU %d via C2C link", rank, netId, c);
distance = PATH_C2C;
}
if (distance > netGdrLevel) {
INFO(NCCL_NET,"GPU Direct RDMA Disabled for GPU %d / HCA %lx (distance %d > %d)", rank, netId, distance, netGdrLevel);
return ncclSuccess;
}
*useGdr = 1;
INFO(NCCL_NET,"GPU Direct RDMA Enabled for GPU %d / HCA %lx (distance %d <= %d), read %d", rank, netId, distance, netGdrLevel, read);
// Force PCIe mapping if path goes through PCI on a C2C system
if (gpu->paths[CPU][c].type == PATH_C2C && distance != PATH_C2C) *gdrMode = ncclTopoGdrModePci;
else *gdrMode = ncclTopoGdrModeDefault;
INFO(NCCL_NET,"GPU Direct RDMA Enabled for GPU %d / HCA %lx (distance %d <= %d), read %d mode %s", rank, netId, distance, netGdrLevel, read, ncclTopoGdrModeStr[*gdrMode]);
return ncclSuccess;
}
ncclResult_t ncclTopoIsGdrAvail(struct ncclTopoSystem* system, int rank, bool *avail) {
int netNum = system->nodes[NET].count;
int useGdr = 0;
enum ncclTopoGdrMode useGdr = ncclTopoGdrModeDisable;
*avail = false;
for (int n = 0; n < netNum; n++) {
int64_t netId = system->nodes[NET].nodes[n].id;
@@ -492,7 +507,7 @@ ncclResult_t ncclTopoIsGdrAvail(struct ncclTopoSystem* system, int rank, bool *a
NCCL_PARAM(NetForceFlush, "NET_FORCE_FLUSH", 0);
// Determine whether we need to flush the GDR recv buffers
ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int netDev, int rank, int* flush) {
ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int64_t netId, int netDev, int rank, int* flush) {
*flush = 1;
ncclNetProperties_t props;
NCCLCHECK(comm->ncclNet->getProperties(netDev, &props));
@@ -506,6 +521,14 @@ ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int netDev, int rank, int*
#else
// Flush is required on Ampere and earlier
if (gpu->gpu.cudaCompCap >= 90) *flush = 0;
// On C2C platforms, data could go through a PCI switch while completions and
// flags would go through C2C. In that case, force a flush.
int c, n;
NCCLCHECK(ncclGetLocalCpu(system, g, &c));
NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &n));
if (gpu->paths[NET][n].type <= PATH_PXB && gpu->paths[CPU][c].type == PATH_C2C) {
*flush = 1;
}
#endif
return ncclSuccess;
}
@@ -576,7 +599,7 @@ NCCL_PARAM(PxnDisable, "PXN_DISABLE", 1);
int ncclPxnDisable(struct ncclComm* comm) {
static int pxnDisable = -1;
if (pxnDisable == -1) {
if (comm && ncclNetVersion(comm) == 4) {
if (comm && comm->ncclNetVer == 4) {
INFO(NCCL_INIT, "PXN Disabled as plugin is v4");
pxnDisable = 1;
} else {
@@ -599,9 +622,9 @@ ncclResult_t ncclTopoGetPxnRanks(struct ncclComm* comm, int** intermediateRanks,
int proxyRank;
NCCLCHECK(ncclTopoGetNetDev(comm, comm->rank, NULL, 0, rank, &netId, NULL, &proxyRank));
if (proxyRank == comm->rank) continue;
int useGdr;
enum ncclTopoGdrMode useGdr;
NCCLCHECK(ncclTopoCheckGdr(comm->topo, comm->rank, netId, 1, &useGdr));
if (useGdr == 0) continue;
if (useGdr == ncclTopoGdrModeDisable) continue;
int found = 0;
for (int r=0; r<nr; r++) {
if (ranks[r] == proxyRank) found = 1;
@@ -746,7 +769,7 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
}
if (gpu->paths[NET][n].type < PATH_PHB) {
// Update path when we dont want to / can't use GPU Direct RDMA.
int gdr;
enum ncclTopoGdrMode gdr;
NCCLCHECK(ncclTopoCheckGdr(system, system->nodes[GPU].nodes[g].gpu.rank, netNode->id, 0, &gdr));
if (gdr == 0) {
// We cannot use GPU Direct RDMA, divert all traffic through the CPU local to the GPU
@@ -770,7 +793,7 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm*
int myDomain = 0;
int ngpus = system->nodes[GPU].count;
int remove = 1;
int gdr = 1;
enum ncclTopoGdrMode useGdr = ncclTopoGdrModeDefault;
bool allXgmi = true;
NCCLCHECK(ncclCalloc(&domains, system->nodes[GPU].count));
NCCLCHECKGOTO(ncclCalloc(&ids, system->nodes[GPU].count), ret, fail);
@@ -839,10 +862,10 @@ ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm*
for (int g = 0; g < system->nodes[GPU].count; g++) {
int64_t netId;
NCCLCHECKGOTO(ncclTopoGetLocalNet(system, system->nodes[GPU].nodes[g].gpu.rank, 0, &netId, nullptr), ret, fail);
NCCLCHECKGOTO(ncclTopoCheckGdr(system, system->nodes[GPU].nodes[g].gpu.rank, netId, 1, &gdr), ret, fail);
if (!gdr) break;
NCCLCHECKGOTO(ncclTopoCheckGdr(system, system->nodes[GPU].nodes[g].gpu.rank, netId, 1, &useGdr), ret, fail);
if (!useGdr) break;
}
if (gdr && !allXgmi) {
if (useGdr && !allXgmi) {
remove = 0;
system->type |= RCCL_TOPO_GDR_ALL;
INFO(NCCL_GRAPH, "GDR is available on all GPUs");
@@ -1014,3 +1037,38 @@ ncclResult_t ncclTopoPathAllNVLink(struct ncclTopoSystem* system, int* allNvLink
*allNvLink = maxPath >= PATH_PIX ? 0 : 1;
return ncclSuccess;
}
// Check whether we are in a split NVLink situation, with two NVLink domains, not
// connected through NVLink (e.g. QPI).
ncclResult_t ncclTopoSplitNvLink(struct ncclTopoSystem* system, int* splitNvLink) {
ncclResult_t res = ncclSuccess;
int nvlDomains = 0;
int *nvlDomain = NULL, *nvlDomainCount = NULL;
// Compute NVLink domains
NCCLCHECKGOTO(ncclCalloc(&nvlDomain, system->nodes[GPU].count), res, exit);
for (int g=0; g<system->nodes[GPU].count; g++) nvlDomain[g] = g;
for (int g=0; g<system->nodes[GPU].count; g++) {
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
int domain = nvlDomain[g];
for (int p=g+1; p<system->nodes[GPU].count; p++) {
if (gpu->paths[GPU][p].type == PATH_NVL) {
nvlDomain[p] = domain;
}
}
}
// Compute number of GPUs per NVLink domain.
NCCLCHECKGOTO(ncclCalloc(&nvlDomainCount, system->nodes[GPU].count), res, exit);
for (int g=0; g<system->nodes[GPU].count; g++) {
nvlDomainCount[nvlDomain[g]]++;
}
// Count the number of NVLink domains
for (int g=0; g<system->nodes[GPU].count; g++) {
if (nvlDomainCount[g] > 1) nvlDomains++;
}
*splitNvLink = nvlDomains == 2 ? 1 : 0;
exit:
if(nvlDomain) free(nvlDomain);
if(nvlDomainCount) free(nvlDomainCount);
return res;
}
src/graph/rome_models.cc
+4 -4
Просмотреть файл
@@ -2075,7 +2075,7 @@ ncclResult_t parseRome4P2H(struct ncclTopoSystem* system, struct ncclTopoGraph*
for (i = 0; i < sizeof(romeTopoModels)/sizeof(romeTopoModels[0]); i++) {
bool ignore_cpu = checkOption(romeTopoModels[i].options, "noCpuCheck");
if (!ignore_cpu && (arch != NCCL_TOPO_CPU_ARCH_X86 || vendor != NCCL_TOPO_CPU_VENDOR_AMD || model != NCCL_TOPO_CPU_TYPE_ROME))
if (!ignore_cpu && (arch != NCCL_TOPO_CPU_ARCH_X86 || vendor != NCCL_TOPO_CPU_VENDOR_AMD || model != NCCL_TOPO_CPU_MODEL_AMD_ROME))
continue;
bool ignore_numa = checkOption(romeTopoModels[i].options, "disableNumaMatching");
if (!ignore_numa && romeTopo.nCpus != romeTopoModels[i].nCpus) continue;
@@ -2225,7 +2225,7 @@ ncclResult_t parse1H16P(struct ncclTopoSystem* system, struct ncclTopoGraph* gra
// only valid on Rome
int arch, vendor, model;
NCCLCHECK(ncclTopoCpuType(system, &arch, &vendor, &model));
if (arch != NCCL_TOPO_CPU_ARCH_X86 || vendor != NCCL_TOPO_CPU_VENDOR_AMD || model != NCCL_TOPO_CPU_TYPE_ROME)
if (arch != NCCL_TOPO_CPU_ARCH_X86 || vendor != NCCL_TOPO_CPU_VENDOR_AMD || model != NCCL_TOPO_CPU_MODEL_AMD_ROME)
return ncclSuccess;
// number of GPUs and NICs on each numa node is used as first screening pattern
@@ -2396,7 +2396,7 @@ ncclResult_t parse4H4P(struct ncclTopoSystem* system, struct ncclTopoGraph* grap
// only valid on Rome
int arch, vendor, model;
NCCLCHECK(ncclTopoCpuType(system, &arch, &vendor, &model));
if (arch != NCCL_TOPO_CPU_ARCH_X86 || vendor != NCCL_TOPO_CPU_VENDOR_AMD || model != NCCL_TOPO_CPU_TYPE_ROME)
if (arch != NCCL_TOPO_CPU_ARCH_X86 || vendor != NCCL_TOPO_CPU_VENDOR_AMD || model != NCCL_TOPO_CPU_MODEL_AMD_ROME)
return ncclSuccess;
// number of GPUs and NICs on each numa node is used as first screening pattern
@@ -2460,7 +2460,7 @@ ncclResult_t parse4H4P(struct ncclTopoSystem* system, struct ncclTopoGraph* grap
}
}
INFO(NCCL_GRAPH, "%s", line);
if (arch == NCCL_TOPO_CPU_ARCH_X86 && vendor == NCCL_TOPO_CPU_VENDOR_AMD && model == NCCL_TOPO_CPU_TYPE_ROME)
if (arch == NCCL_TOPO_CPU_ARCH_X86 && vendor == NCCL_TOPO_CPU_VENDOR_AMD && model == NCCL_TOPO_CPU_MODEL_AMD_ROME)
system->type |= RCCL_TOPO_4P2H_ROME;
parseOptions(system, rome_model_68.options);
// create 4P4H based on reference and remapped ids
src/graph/search.cc
+35 -43
Просмотреть файл
@@ -483,12 +483,11 @@ ncclResult_t ncclTopoCompareGraphs(struct ncclTopoSystem* system, struct ncclTop
// 1. Select NETs starting with those close to GPU(s), based on paths[n].type.
// 2. add other NETs satisfying typeInter but not already in the list.
ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int typeInter, int gpu, int* nets, int* netCountRet) {
ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int typeInter, int gpu, int nets[NCCL_TOPO_MAX_NODES], int* netCountRet) {
ncclResult_t ret = ncclSuccess;
int netCount = 0;
int localNetCount;
int* localNets;
NCCLCHECK(ncclCalloc(&localNets, MAXCHANNELS));
int localNets[MAXCHANNELS];
// First add the preferred NICs
for (int g=0; g<system->nodes[GPU].count; g++) {
@@ -497,8 +496,8 @@ ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int typeInter, in
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
for (int c = 0; c<MAXCHANNELS; c++) {
int64_t netId;
NCCLCHECKGOTO(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &netId, NULL), ret, fail);
NCCLCHECKGOTO(ncclTopoIdToIndex(system, NET, netId, localNets+localNetCount), ret, fail);
NCCLCHECK(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &netId, NULL));
NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, localNets+localNetCount));
if (localNetCount > 0 && localNets[localNetCount] == localNets[0]) break;
localNetCount++;
}
@@ -506,7 +505,7 @@ ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int typeInter, in
for (int i=0; i<localNetCount; i++) {
int n = localNets[i];
int found = 0;
while (nets[found] != n && found<netCount) found++;
while (found<netCount && nets[found] != n) found++;
if (found == netCount) nets[netCount++] = n;
}
}
@@ -525,22 +524,17 @@ ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int typeInter, in
for (int i=0; i<localNetCount; i++) {
int n = localNets[i];
int found = 0;
while (nets[found] != n && found<netCount) found++;
while (found<netCount && nets[found] != n) found++;
if (found == netCount) nets[netCount++] = n;
}
}
}
*netCountRet = netCount;
exit:
free(localNets);
return ret;
fail:
goto exit;
}
ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, struct ncclTopoNode* gpu, int step, int backToNet, int backToFirstRank, int forcedOrder, int *time) {
ncclResult_t ret = ncclSuccess;
if ((*time) <= 0) return ncclSuccess;
(*time)--;
@@ -562,7 +556,7 @@ ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopo
}
graph->intra[graph->nChannels*ngpus+step] = gpu->gpu.rank;
int g = gpu - system->nodes[GPU].nodes;
int* nets = NULL;
int nets[NCCL_TOPO_MAX_NODES];
if (step == backToNet) {
// first get back to NIC
if (system->nodes[NET].count) {
@@ -570,8 +564,7 @@ ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopo
NCCLCHECK(getNetIndex(system, graph->inter[graph->nChannels*2], &startNetIndex));
struct ncclTopoNode* startNet = system->nodes[NET].nodes+startNetIndex;
int netCount;
NCCLCHECK(ncclCalloc(&nets, system->nodes[NET].count));
NCCLCHECKGOTO(ncclTopoSelectNets(system, graph->typeInter, g, nets, &netCount), ret, fail);
NCCLCHECK(ncclTopoSelectNets(system, graph->typeInter, g, nets, &netCount));
for (int i=0; i<netCount; i++) {
int n = nets[i];
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
@@ -592,14 +585,14 @@ ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopo
graph->bwInter /= 2;
}
NCCLCHECKGOTO(ncclTopoFollowPath(system, graph, GPU, g, NET, n, 1, &net), ret, fail);
NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, NET, n, 1, &net));
graph->bwInter = bwInterSave;
if (net) {
graph->inter[graph->nChannels*2+1] = net->id;
NCCLCHECKGOTO(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, step, nextBackToNet, backToFirstRank, forcedOrder, time), ret, fail);
NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, step, nextBackToNet, backToFirstRank, forcedOrder, time));
if (graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE) graph->bwInter /= 2;
NCCLCHECKGOTO(ncclTopoFollowPath(system, graph, GPU, g, NET, n, -1, &net), ret, fail);
NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, NET, n, -1, &net));
graph->bwInter = bwInterSave;
}
}
@@ -638,21 +631,15 @@ ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopo
// Next path
NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, ngpus, -1, -1, forcedOrder, time));
}
exit:
if (nets) free(nets);
return ret;
fail:
goto exit;
return ncclSuccess;
}
ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int backToNet, int backToFirstRank, int* time) {
ncclResult_t ret = ncclSuccess;
const int bw = graph->bwInter;
int* nets;
NCCLCHECK(ncclCalloc(&nets, system->nodes[NET].count));
int nets[NCCL_TOPO_MAX_NODES];
int netCount;
int graphFound = 0;
NCCLCHECKGOTO(ncclTopoSelectNets(system, graph->typeInter, -1, nets, &netCount), ret, fail);
NCCLCHECK(ncclTopoSelectNets(system, graph->typeInter, -1, nets, &netCount));
for (int i=0; i<netCount; i++) {
if ((graph->pattern == NCCL_TOPO_PATTERN_NVLS || graph->pattern == NCCL_TOPO_PATTERN_COLLNET_DIRECT) && graphFound) break;
int n = nets[(graph->nChannels+i)%netCount];
@@ -676,7 +663,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
// NVLS search only tries to find NIC:GPU combinations to compute the heads.
if (graph->nChannels < netCount) {
int gpu;
NCCLCHECKGOTO(ncclTopoGetLocalGpu(system, net->id, &gpu), ret, fail);
NCCLCHECK(ncclTopoGetLocalGpu(system, net->id, &gpu));
if (gpu != -1) {
int duplicate = 0;
// check whether there is duplicate head when one GPU connects with multiple NICs
@@ -687,7 +674,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
}
}
if (!duplicate) {
NCCLCHECKGOTO(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, gpu), ret, fail);
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, gpu));
graphFound = 1;
}
}
@@ -696,8 +683,8 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
if (graph->nChannels > 0) {
// Try to replay the last channel
int g;
NCCLCHECKGOTO(ncclTopoReplayGetGpu(system, graph, -1, &g), ret, fail);
NCCLCHECKGOTO(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_REPLAY, time, NET, n, g), ret, fail);
NCCLCHECK(ncclTopoReplayGetGpu(system, graph, -1, &g));
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_REPLAY, time, NET, n, g));
}
if (graph->nChannels == 0 || graph->sameChannels == 0) {
if (graph->nChannels == 0 && system->nodes[NVS].count == 0) {
@@ -708,16 +695,16 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
for (int i = 0; i<system->nodes[GPU].count; i++) {
if (paths[i].count <= paths[f].count) {
// prefer GPU direct RDMA
int gdr;
NCCLCHECK(ncclTopoCheckGdr(system, system->nodes[GPU].nodes[i].id, net->id, 0, &gdr));
if (paths[i].count < paths[f].count || (paths[i].count == paths[f].count && !f_gdr && gdr)) {
enum ncclTopoGdrMode useGdr;
NCCLCHECK(ncclTopoCheckGdr(system, system->nodes[GPU].nodes[i].id, net->id, 0, &useGdr));
if (paths[i].count < paths[f].count || (paths[i].count == paths[f].count && !f_gdr && useGdr)) {
f = i;
f_gdr = gdr;
f_gdr = useGdr;
}
}
}
int t = 1 << 10;
NCCLCHECKGOTO(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, &t, NET, n, 0), ret, fail);
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, &t, NET, n, 0));
if (t == -1) *time = -1;
}
@@ -737,7 +724,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
for (int i=0; i<system->nodes[GPU].count; i++) {
int g = (graph->nChannels+i)%system->nodes[GPU].count;
if (paths[g].bw == maxBw && paths[g].count == minHops) {
NCCLCHECKGOTO(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, g), ret, fail);
NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, g));
}
}
}
@@ -751,11 +738,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo
}
}
}
exit:
free(nets);
return ret;
fail:
goto exit;
return ncclSuccess;
}
/* Search Patterns
@@ -1061,7 +1044,7 @@ ncclResult_t ncclTopoCompute(ncclTopoSystem* system, struct ncclTopoGraph* graph
graph->pattern == NCCL_TOPO_PATTERN_RING ? system->hostIdx % 2 : 0));
int arch, vendor, model;
NCCLCHECK(ncclTopoCpuType(system, &arch, &vendor, &model));
if (graph->nChannels && arch == NCCL_TOPO_CPU_ARCH_X86 && vendor == NCCL_TOPO_CPU_VENDOR_AMD && model == NCCL_TOPO_CPU_TYPE_ROME) {
if (graph->nChannels && arch == NCCL_TOPO_CPU_ARCH_X86 && vendor == NCCL_TOPO_CPU_VENDOR_AMD && model == NCCL_TOPO_CPU_MODEL_AMD_ROME) {
system->type |= RCCL_TOPO_4P2H_ROME;
}
}
@@ -1107,6 +1090,15 @@ ncclResult_t ncclTopoCompute(ncclTopoSystem* system, struct ncclTopoGraph* graph
graph->minChannels = graph->maxChannels;
}
int splitNvLink;
NCCLCHECK(ncclTopoSplitNvLink(system, &splitNvLink));
if (graph->pattern == NCCL_TOPO_PATTERN_RING && splitNvLink) {
// We have two sockets with NVLink and a slower link in between (typically QPI).
// Tree is likely going to work better but it needs at least 2 channels.
// Since Tree needs to have the same number of channels as Ring, also force Ring to use 2 channels.
if (graph->maxChannels >= 2 && graph->minChannels == 1) graph->minChannels = 2;
}
struct ncclTopoGraph tmpGraph;
memcpy(&tmpGraph, graph, sizeof(struct ncclTopoGraph));
src/graph/topo.cc
+84 -42
Просмотреть файл
@@ -24,11 +24,11 @@
const char* topoNodeTypeStr[] = { "GPU", "PCI", "NVS", "CPU", "NIC", "NET" };
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
const char* topoLinkTypeStr[] = { "LOC", "XGMI", "", "PCI", "", "", "", "SYS", "NET" };
const char* topoPathTypeStr[] = { "LOC", "XGMI", "NVB", "PIX", "PXB", "PXN", "PHB", "SYS", "DIS" };
const char* topoLinkTypeStr[] = { "LOC", "XGMI", "", "C2C", "PCI", "", "", "", "SYS", "NET" };
const char* topoPathTypeStr[] = { "LOC", "XGMI", "NVB", "C2C", "PIX", "PXB", "PXN", "PHB", "SYS", "DIS" };
#else
const char* topoLinkTypeStr[] = { "LOC", "NVL", "", "PCI", "", "", "", "SYS", "NET" };
const char* topoPathTypeStr[] = { "LOC", "NVL", "NVB", "PIX", "PXB", "PXN", "PHB", "SYS", "NET", "DIS" };
const char* topoLinkTypeStr[] = { "LOC", "NVL", "", "C2C", "PCI", "", "", "", "SYS", "NET" };
const char* topoPathTypeStr[] = { "LOC", "NVL", "NVB", "C2C", "PIX", "PXB", "PXN", "PHB", "SYS", "NET", "DIS" };
#endif
/******************************************************************/
@@ -51,7 +51,7 @@ ncclResult_t pciPathToInt64(char* path, int offset, int minOffset, int64_t* id)
return ncclSuccess;
}
static ncclResult_t findLocalCpu(struct ncclTopoNode* node, struct ncclTopoNode** cpu) {
static ncclResult_t findLocalCpu(struct ncclTopoNode* node, struct ncclTopoNode** cpu, struct ncclTopoNode* from) {
*cpu = NULL;
if (node->type == CPU) {
*cpu = node;
@@ -60,9 +60,10 @@ static ncclResult_t findLocalCpu(struct ncclTopoNode* node, struct ncclTopoNode*
for (int l=0; l<node->nlinks; l++) {
// Go up the PCI tree to find the CPU. Follow only PCI switches.
if (node->links[l].type == LINK_PCI
&& (node->links[l].remNode->type == PCI
|| node->links[l].remNode->type == CPU)) {
NCCLCHECK(findLocalCpu(node->links[l].remNode, cpu));
&& node->links[l].remNode != from
&& (node->links[l].remNode->type == PCI
|| node->links[l].remNode->type == CPU)) {
NCCLCHECK(findLocalCpu(node->links[l].remNode, cpu, node));
}
if (*cpu != NULL) return ncclSuccess;
}
@@ -83,13 +84,17 @@ static ncclResult_t ncclTopoGetInterCpuBw(struct ncclTopoNode* cpu, float* bw) {
return ncclSuccess;
}
if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_X86 && cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_INTEL) {
*bw = cpu->cpu.model == NCCL_TOPO_CPU_TYPE_SKL ? SKL_QPI_BW : QPI_BW;
*bw =
cpu->cpu.model == NCCL_TOPO_CPU_MODEL_INTEL_ERP ? ERP_QPI_BW :
cpu->cpu.model == NCCL_TOPO_CPU_MODEL_INTEL_SRP ? SRP_QPI_BW :
cpu->cpu.model == NCCL_TOPO_CPU_MODEL_INTEL_SKL ? SKL_QPI_BW :
BDW_QPI_BW;
}
if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_X86 && cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_AMD) {
*bw = AMD_BW;
}
if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_X86 && cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_ZHAOXIN) {
*bw = cpu->cpu.model == NCCL_TOPO_CPU_TYPE_YONGFENG ? YONGFENG_ZPI_BW : ZPI_BW;
*bw = cpu->cpu.model == NCCL_TOPO_CPU_MODEL_YONGFENG ? YONGFENG_ZPI_BW : ZPI_BW;
}
return ncclSuccess;
}
@@ -534,19 +539,23 @@ ncclResult_t ncclTopoAddCpu(struct ncclXmlNode* xmlCpu, struct ncclTopoSystem* s
int familyId, modelId;
NCCLCHECK(xmlGetAttrInt(xmlCpu, "familyid", &familyId));
NCCLCHECK(xmlGetAttrInt(xmlCpu, "modelid", &modelId));
cpu->cpu.model = (familyId == 6 && modelId >= 0x55) ? NCCL_TOPO_CPU_TYPE_SKL : NCCL_TOPO_CPU_INTEL_BDW;
cpu->cpu.model =
(familyId == 6 && modelId >= 0xCF) ? NCCL_TOPO_CPU_MODEL_INTEL_ERP :
(familyId == 6 && modelId >= 0x8F) ? NCCL_TOPO_CPU_MODEL_INTEL_SRP :
(familyId == 6 && modelId >= 0x55) ? NCCL_TOPO_CPU_MODEL_INTEL_SKL :
NCCL_TOPO_CPU_MODEL_INTEL_BDW;
} else if (cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_ZHAOXIN) {
int familyId, modelId;
NCCLCHECK(xmlGetAttrInt(xmlCpu, "familyid", &familyId));
NCCLCHECK(xmlGetAttrInt(xmlCpu, "modelid", &modelId));
if (familyId == 7 && modelId == 0x5B) cpu->cpu.model = NCCL_TOPO_CPU_TYPE_YONGFENG;
if (familyId == 7 && modelId == 0x5B) cpu->cpu.model = NCCL_TOPO_CPU_MODEL_YONGFENG;
}
if (cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_AMD) {
int familyId, modelId;
NCCLCHECK(xmlGetAttrInt(xmlCpu, "familyid", &familyId));
NCCLCHECK(xmlGetAttrInt(xmlCpu, "modelid", &modelId));
// Treat "Milan" also as "Rome"
cpu->cpu.model = ((familyId == 143 && modelId >= 49) || familyId == 175) ? NCCL_TOPO_CPU_TYPE_ROME : NCCL_TOPO_CPU_TYPE_ZEN;
cpu->cpu.model = ((familyId == 143 && modelId >= 49) || familyId == 175) ? NCCL_TOPO_CPU_MODEL_AMD_ROME : NCCL_TOPO_CPU_MODEL_AMD_ZEN;
}
}
for (int s=0; s<xmlCpu->nSubs; s++) {
@@ -595,7 +604,7 @@ ncclResult_t ncclTopoAddXGMI(struct ncclXmlNode* node, struct ncclTopoSystem* sy
NCCLCHECK(ncclTopoGetNode(system, &remote, GPU, busId));
} else if (targetType == CPU) {
// NVL connection to the local CPU
NCCLCHECK(findLocalCpu(gpu, &remote));
NCCLCHECK(findLocalCpu(gpu, &remote, NULL));
} else {
if (system->nodes[NVS].count == 0) {
NCCLCHECK(ncclTopoCreateNode(system, &remote, NVS, 0));
@@ -647,7 +656,7 @@ ncclResult_t ncclTopoAddNvLinks(struct ncclXmlNode* node, struct ncclTopoSystem*
NCCLCHECK(ncclTopoGetNode(system, &remote, GPU, NCCL_TOPO_ID(systemId, busId)));
} else if (targetType == CPU) {
// NVL connection to the local CPU
NCCLCHECK(findLocalCpu(gpu, &remote));
NCCLCHECK(findLocalCpu(gpu, &remote, NULL));
} else {
if (system->nodes[NVS].count == 0) {
NCCLCHECK(ncclTopoCreateNode(system, &remote, NVS, 0));
@@ -725,10 +734,10 @@ ncclResult_t ncclTopoAddC2c(struct ncclXmlNode* node, struct ncclTopoSystem* sys
NCCLCHECK(xmlGetAttrInt(node, "bw", &bw));
double c2cBw = (bw*count)/1000.0;
struct ncclTopoNode* cpu = NULL;
NCCLCHECK(findLocalCpu(gpu, &cpu));
NCCLCHECK(findLocalCpu(gpu, &cpu, NULL));
if (cpu == NULL) return ncclSuccess;
NCCLCHECK(ncclTopoConnectNodes(gpu, cpu, LINK_NVL, c2cBw));
NCCLCHECK(ncclTopoConnectNodes(cpu, gpu, LINK_NVL, c2cBw));
NCCLCHECK(ncclTopoConnectNodes(gpu, cpu, LINK_C2C, c2cBw));
NCCLCHECK(ncclTopoConnectNodes(cpu, gpu, LINK_C2C, c2cBw));
} else {
if (strcmp(node->name, "cpu") == 0) {
NCCLCHECK(ncclGetSystemId(system, node, &systemId));
@@ -1048,26 +1057,31 @@ struct ncclXmlNode** physNetNodes, struct ncclXmlNode** netNode, ncclResult_t (*
// Trigger the merge, then get the new device's properties
int vDevIndex = 0;
ncclResult_t ret = makeVDevice(&vDevIndex, vProps);
if (ret == ncclInvalidUsage) {
WARN("TOPO/NET : Tried merging multiple devices together and failed. Try setting NCCL_NET_MERGE_LEVEL=LOC");
NCCLCHECK(ret);
if (ret != ncclSuccess) {
INFO(NCCL_GRAPH|NCCL_INIT|NCCL_NET, "TOPO/NET : Tried merging multiple devices together and failed. vProps={ndevs=%d, devs=[%d %d %d %d]}. Set NCCL_NET_MERGE_LEVEL=LOC to disable NIC fusion.",
vProps->ndevs, vProps->devs[0], vProps->devs[1], vProps->devs[2], vProps->devs[3]);
return ret;
}
INFO(NCCL_GRAPH, "TOPO/NET : Made vNic %d", vDevIndex);
return ncclSuccess;
}
ncclResult_t ncclTopoForceMerge(ncclComm_t comm, struct ncclXml* xml, char* str, int* placedDevs, ncclNetProperties_t* propsList, struct ncclXmlNode** physNetNodes, int nPhysDevs, ncclResult_t (*makeVDevice)(int*, ncclNetVDeviceProps_t*)) {
ncclResult_t ncclTopoForceMerge(ncclComm_t comm, struct ncclXml* xml, const char* str, int* placedDevs, ncclNetProperties_t* propsList, struct ncclXmlNode** physNetNodes, int nPhysDevs, ncclResult_t (*makeVDevice)(int*, ncclNetVDeviceProps_t*)) {
ncclResult_t ret = ncclSuccess;
INFO(NCCL_ENV|NCCL_NET, "TOPO/NET : Force-fusing NICs using NCCL_NET_FORCE_MERGE=%s", str);
char* ncStr;
NCCLCHECK(ncclCalloc(&ncStr, strlen(str)+1));
strcpy(ncStr, str);
char* semi_token;
char* semi = strtok_r(str, ";", &semi_token);
char* semi = strtok_r(ncStr, ";", &semi_token);
while (semi) {
TRACE(NCCL_NET, "Fusing %s", semi);
struct netIf userIfs[NCCL_NET_MAX_DEVS_PER_NIC];
int nUserIfs = parseStringList(semi, userIfs, NCCL_NET_MAX_DEVS_PER_NIC);
if (nUserIfs == 0) {
INFO(NCCL_NET, "NET/IB : Invalid NCCL_NET_FORCE_MERGE specified %s. Couldn't parse substring %s. Please provide a semicolon-delimited list of comma-delimited NIC groups.",
str, semi);
ncStr, semi);
continue;
}
@@ -1081,26 +1095,37 @@ ncclResult_t ncclTopoForceMerge(ncclComm_t comm, struct ncclXml* xml, char* str,
if (vProps.ndevs != nUserIfs) {
WARN("TOPO/NET : Only matched %d devices, %d requested from %s",
vProps.ndevs, nUserIfs, semi);
return ncclInvalidUsage;
ret = ncclInvalidUsage;
goto fail;
}
if (vProps.ndevs > NCCL_NET_MAX_DEVS_PER_NIC) {
WARN("Specified fused NIC %s which has too many devices (%d). Max %d", semi, vProps.ndevs, NCCL_NET_MAX_DEVS_PER_NIC);
return ncclInvalidUsage;
ret = ncclInvalidUsage;
goto fail;
}
struct ncclXmlNode* netNode;
NCCLCHECK(ncclTopoMakeVnic(comm, xml, &vProps, physNetNodes, &netNode, makeVDevice));
// Only set that a device is "placed" after successfully making a vNic (it's possible to exit before this)
for (int i = 0; i < vProps.ndevs; i++) {
placedDevs[vProps.devs[i]] = 1;
ret = ncclTopoMakeVnic(comm, xml, &vProps, physNetNodes, &netNode, makeVDevice);
if (ret == ncclSuccess) {
// Only set that a device is "placed" after successfully making a vNic (it's possible to exit before this)
for (int i = 0; i < vProps.ndevs; i++) {
placedDevs[vProps.devs[i]] = 1;
}
} else {
WARN("TOPO/NET : Could not force merge NICs %s. Please specify a valid NCCL_NET_FORCE_MERGE string.", semi);
ret = ncclInvalidUsage;
goto fail;
}
semi = strtok_r(NULL, ";", &semi_token);;
}
return ncclSuccess;
exit:
free(ncStr);
return ret;
fail:
goto exit;
}
ncclResult_t ncclTopoAutoMerge(ncclComm_t comm, struct ncclXml* xml, int mergeLevel, int* placedDevs, ncclNetProperties_t* propsList, struct ncclXmlNode** physNetNodes, int nPhysDevs, ncclResult_t (*makeVDevice)(int*, ncclNetVDeviceProps_t*)) {
@@ -1148,7 +1173,24 @@ ncclResult_t ncclTopoAutoMerge(ncclComm_t comm, struct ncclXml* xml, int mergeLe
}
struct ncclXmlNode* netNode;
NCCLCHECKGOTO(ncclTopoMakeVnic(comm, xml, &vProps, physNetNodes, &netNode, makeVDevice), res, out);
ncclResult_t ret = ncclTopoMakeVnic(comm, xml, &vProps, physNetNodes, &netNode, makeVDevice);
// Merging failed.
// Mark all as unplaced and increase their distance to disconnected (PATH_DIS)
// Set i to 0 to restart the automatic merging process and ensure all are placed
if (ret != ncclSuccess) {
INFO(NCCL_GRAPH|NCCL_INIT|NCCL_NET, "Marking physical devices as unplaced, increasing distance and restarting search.");
placedDevs[i] = 0;
TRACE(NCCL_GRAPH, "Setting dev %d as unplaced, keeping distance -> self as PATH_LOC", i);
for (int k = 1; k < vProps.ndevs; k++) {
int dev = vProps.devs[k];
placedDevs[dev] = 0;
paths[i*nPhysDevs + dev] = PATH_DIS;
paths[dev*nPhysDevs + i] = PATH_DIS;
TRACE(NCCL_GRAPH, "Setting dev %d as unplaced, setting distance -> %d as PATH_DIS", dev, i);
}
i = 0;
}
}
}
@@ -1212,16 +1254,16 @@ ncclResult_t ncclTopoMakeVNics(ncclComm_t comm, struct ncclXml* xml, ncclResult_
// By default, don't merge any devices
int mergeLevel;
mergeLevel = PATH_PORT;
char* mergeLevelEnv;
mergeLevelEnv = getenv("NCCL_NET_MERGE_LEVEL");
if (mergeLevelEnv) kvConvertToInt(mergeLevelEnv, &mergeLevel, nicPathKvList);
char* forceMerge;
forceMerge = getenv("NCCL_NET_FORCE_MERGE");
NCCLCHECK(ncclCalloc(&placedDevs, physicalDevs));
memset(placedDevs, 0, sizeof(int)*physicalDevs);
{ // Avoids warnings related to jumping to "out"
const char* mergeLevelEnv = ncclGetEnv("NCCL_NET_MERGE_LEVEL");
if (mergeLevelEnv) kvConvertToInt(mergeLevelEnv, &mergeLevel, nicPathKvList);
const char* forceMerge = ncclGetEnv("NCCL_NET_FORCE_MERGE");
NCCLCHECK(ncclCalloc(&placedDevs, physicalDevs));
memset(placedDevs, 0, sizeof(int)*physicalDevs);
if (forceMerge) {
NCCLCHECKGOTO(ncclTopoForceMerge(comm, xml, forceMerge, placedDevs, props, physNetNodes, physicalDevs, makeVDevice), res, out);
if (forceMerge) {
NCCLCHECKGOTO(ncclTopoForceMerge(comm, xml, forceMerge, placedDevs, props, physNetNodes, physicalDevs, makeVDevice), res, out);
}
}
NCCLCHECKGOTO(ncclTopoAutoMerge(comm, xml, mergeLevel, placedDevs, props, physNetNodes, physicalDevs, makeVDevice), res, out);
src/graph/topo.h
+21 -17
Просмотреть файл
@@ -21,9 +21,11 @@
#define SM86_NVLINK_BW 12.0
#define SM100_NVLINK_BW 40.0
#define PCI_BW 12.0 // PCI Gen3 x16
#define QPI_BW 6.0
#define AMD_BW 16.0
#define BDW_QPI_BW 6.0
#define SKL_QPI_BW 10.0
#define SRP_QPI_BW 22.0
#define ERP_QPI_BW 40.0
#define ZPI_BW 6.0
#define YONGFENG_ZPI_BW 9.0
#define P9_BW 32.0
@@ -51,12 +53,13 @@ extern const char* topoNodeTypeStr[];
#define LINK_LOC 0
#define LINK_NVL 1
// Skipping 2 for PATH_NVB
#define LINK_PCI 3
// Skipping 4 for PATH_PXB
// Skipping 5 for PATH_PXN
// Skipping 6 for PATH_PHB
#define LINK_SYS 7
#define LINK_NET 8
#define LINK_C2C 3
#define LINK_PCI 4
// Skipping 5 for PATH_PXB
// Skipping 6 for PATH_PXN
// Skipping 7 for PATH_PHB
#define LINK_SYS 8
#define LINK_NET 9
extern const char* topoLinkTypeStr[];
// Local (myself)
@@ -68,29 +71,32 @@ extern const char* topoLinkTypeStr[];
// Connection through NVLink using an intermediate GPU
#define PATH_NVB 2
// Connection through C2C
#define PATH_C2C 3
// Connection traversing at most a single PCIe bridge
#define PATH_PIX 3
#define PATH_PIX 4
// Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)
#define PATH_PXB 4
#define PATH_PXB 5
// Connection between a GPU and a NIC using an intermediate GPU. Used to enable rail-local, aggregated network send/recv operations.
#define PATH_PXN 5
#define PATH_PXN 6
// Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)
#define PATH_PHB 6
#define PATH_PHB 7
// Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)
#define PATH_SYS 7
#define PATH_SYS 8
// Connection through the network
#define PATH_NET 8
#define PATH_NET 9
// New type of path which should precede PATH_PIX
#define PATH_PORT PATH_NVL
// Disconnected
#define PATH_DIS 9
#define PATH_DIS 10
extern const char* topoPathTypeStr[];
struct ncclTopoNode;
@@ -110,9 +116,6 @@ struct ncclTopoLinkList {
int type;
};
#define NCCL_TOPO_CPU_INTEL_BDW 1
#define NCCL_TOPO_CPU_INTEL_SKL 2
#define NCCL_TOPO_UNDEF (-1)
#define NCCL_TOPO_ID_LOCAL_ID_MASK 0x00ffffffffffffff
@@ -212,6 +215,7 @@ ncclResult_t ncclTopoLoadSystem(const char* xmlTopoFile, struct ncclTopoSystem*
ncclResult_t ncclTopoGetIntermediateRank(struct ncclTopoSystem* system, int rank, int64_t netId, int* intermediateRank);
ncclResult_t ncclTopoGetGpuMinPath(struct ncclTopoSystem* system, int type, int* min);
ncclResult_t ncclTopoGetGpuMaxPath(struct ncclTopoSystem* system, int type, int* max);
ncclResult_t ncclTopoSplitNvLink(struct ncclTopoSystem* system, int* splitNvLink);
#define NCCL_TOPO_XML_MAX_NODES 8192
#define NCCL_GRAPH_XML_MAX_NODES 8192
src/graph/tuning.cc
+2 -1
Просмотреть файл
@@ -382,6 +382,7 @@ static const double perChMaxTreeBws[][3] = {
NCCL_PARAM(PatEnable, "PAT_ENABLE", 2);
static int ncclPatEnable(struct ncclComm* comm) {
int patEnable = ncclParamPatEnable();
if (comm->minCompCap < 60) return 0; // Need SM60 or higher for CUDA atomics
if (patEnable != 2) return patEnable;
if (comm->nNodes != comm->nRanks) return 0; // PAT only supports 1 GPU per node
if (comm->netDeviceType != NCCL_NET_DEVICE_HOST) return 0; // PAT doesn't support net device offload
@@ -486,7 +487,7 @@ ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCom
if (a == NCCL_ALGO_TREE && coll == ncclFuncAllReduce) busBw = std::min(busBw*.92, graphs[a]->nChannels*perChMaxTreeBw);
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL) busBw = std::min(busBw*1.0/3.8, llMaxBw);
if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL128) busBw = std::min(busBw * (nNodes == 1 ? 7.0/9.0 : 120.0/128.0), graphs[a]->nChannels*perChMaxTreeLL128Bw);
if (a == NCCL_ALGO_TREE && graphs[a]->pattern == NCCL_TOPO_PATTERN_TREE) busBw *= .85;
if (a == NCCL_ALGO_TREE && comm->maxTreePattern == NCCL_TOPO_PATTERN_TREE) busBw *= .85;
if (a == NCCL_ALGO_PAT) busBw *= .75;
if (a == NCCL_ALGO_COLLNET_DIRECT && p != NCCL_PROTO_SIMPLE) busBw = 0; // Not used
if (a == NCCL_ALGO_COLLNET_CHAIN && p != NCCL_PROTO_SIMPLE) busBw = 0; // Not used
src/group.cc
+36 -28
Просмотреть файл
@@ -221,7 +221,6 @@ fail:
static ncclResult_t doLaunches(struct ncclComm* head) {
ncclResult_t result = ncclSuccess;
struct ncclComm* cliqueComm0 = head->intraComm0;
struct ncclComm* cliqueHead = head;
struct ncclComm* cliqueNextHead;
bool useBarrier = ncclParamLaunchMode == ncclLaunchModeGroup;
@@ -237,7 +236,7 @@ static ncclResult_t doLaunches(struct ncclComm* head) {
NCCLCHECKGOTO(ncclLaunchPrepare(comm), result, failure);
if (useBarrier) ncclCommIntraBarrierIn(comm, 1);
comm = comm->groupNext;
} while (comm != nullptr && comm->intraComm0 == cliqueComm0);
} while (comm != nullptr && comm->intraComm0 == cliqueHead->intraComm0);
cliqueNextHead = comm;
if (capturingYes && capturingNo) {
@@ -454,38 +453,47 @@ static ncclResult_t groupLaunch(struct ncclAsyncJob *job_, ncclSimInfo_t* simInf
/* Connect channels at runtime if cumem is supported */
if (groupCommHeadMain != nullptr) {
struct ncclComm* comm = groupCommHeadMain;
struct ncclComm* cliqueHead = groupCommHeadMain;
struct ncclComm* comm = NULL;
struct ncclIntruQueue<struct ncclAsyncJob, &ncclAsyncJob::next> asyncCollJobs;
ncclIntruQueueConstruct(&asyncCollJobs);
do {
bool needConnect = false;
bool algoNeedConnect[NCCL_NUM_ALGORITHMS];
memset(algoNeedConnect, 0, sizeof(bool) * NCCL_NUM_ALGORITHMS);
// We need to preconnect connections for collectives clique by clique to avoid
// race condition for split shared comms which can connect the same connections
// at the same time.
comm = cliqueHead;
do {
bool needConnect = false;
bool algoNeedConnect[NCCL_NUM_ALGORITHMS];
memset(algoNeedConnect, 0, sizeof(bool) * NCCL_NUM_ALGORITHMS);
// CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, fail);
NCCLCHECKGOTO(ncclPrepareTasks(comm, algoNeedConnect, &needConnect, simInfo), ret, fail);
CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, fail);
NCCLCHECKGOTO(ncclPrepareTasks(comm, algoNeedConnect, &needConnect, simInfo), ret, fail);
if (comm->cuMemSupport && needConnect) {
struct ncclPreconnectJob* job;
NCCLCHECKGOTO(ncclCalloc(&job, 1), ret, fail);
job->base.func = ncclCollPreconnectFunc;
job->base.undo = nullptr;
job->base.destructor = free;
job->base.state = ncclGroupJobRunning;
job->base.abortFlag = comm->abortFlag;
job->base.abortFlagDev = comm->abortFlagDev;
job->comm = comm;
NCCLCHECKGOTO(ncclCalloc(&job->algoNeedConnect, NCCL_NUM_ALGORITHMS), ret, fail);
memcpy(job->algoNeedConnect, algoNeedConnect, sizeof(bool) * NCCL_NUM_ALGORITHMS);
ncclIntruQueueEnqueue(&asyncCollJobs, &job->base);
if (comm->cuMemSupport && needConnect) {
struct ncclPreconnectJob* job;
NCCLCHECKGOTO(ncclCalloc(&job, 1), ret, fail);
job->base.func = ncclCollPreconnectFunc;
job->base.undo = nullptr;
job->base.destructor = free;
job->base.state = ncclGroupJobRunning;
job->base.abortFlag = comm->abortFlag;
job->base.abortFlagDev = comm->abortFlagDev;
job->comm = comm;
NCCLCHECKGOTO(ncclCalloc(&job->algoNeedConnect, NCCL_NUM_ALGORITHMS), ret, fail);
memcpy(job->algoNeedConnect, algoNeedConnect, sizeof(bool) * NCCL_NUM_ALGORITHMS);
ncclIntruQueueEnqueue(&asyncCollJobs, &job->base);
}
comm = comm->groupNext;
} while (comm != nullptr && comm->intraComm0 == cliqueHead->intraComm0);
// connect
NCCLCHECKGOTO(asyncJobLaunch(&asyncCollJobs, groupAbortFlag), ret, fail);
while (!ncclIntruQueueEmpty(&asyncCollJobs)) {
struct ncclAsyncJob* job = ncclIntruQueueDequeue(&asyncCollJobs);
if (job->destructor) job->destructor((void*)job);
}
comm = comm->groupNext;
} while (comm);
NCCLCHECKGOTO(asyncJobLaunch(&asyncCollJobs, groupAbortFlag), ret, fail);
while (!ncclIntruQueueEmpty(&asyncCollJobs)) {
struct ncclAsyncJob* job = ncclIntruQueueDequeue(&asyncCollJobs);
if (job->destructor) job->destructor((void*)job);
}
cliqueHead = comm;
} while (cliqueHead != nullptr);
// done with all buffer allocation, start registration and enqueue
comm = groupCommHeadMain;
src/include/bitops.h
+47 -6
Просмотреть файл
@@ -8,6 +8,7 @@
#define NCCL_BITOPS_H_
#include <stdint.h>
#include <string.h>
#if !__NVCC__
#ifndef __host__
@@ -276,13 +277,53 @@ inline __host__ __device__ uint32_t u32fp8Decode(uint8_t x) {
return u32fpDecode(x, 3);
}
inline __host__ __device__ uint64_t getHash(const char* string, int n) {
// Based on DJB2a, result = result * 33 ^ char
uint64_t result = 5381;
for (int c = 0; c < n; c++) {
result = ((result << 5) + result) ^ string[c];
// The hash isn't just a function of the bytes but also where the bytes are split
// into different calls to eatHash().
inline __host__ __device__ void eatHash(uint64_t acc[2], const void* bytes, size_t size) {
char const* ptr = (char const*)bytes;
acc[0] ^= size;
while (size != 0) {
// Mix the accumulator bits.
acc[0] += acc[1];
acc[1] ^= acc[0];
acc[0] ^= acc[0] >> 31;
acc[0] *= 0x9de62bbc8cef3ce3;
acc[1] ^= acc[1] >> 32;
acc[1] *= 0x485cd6311b599e79;
// Read in a chunk of input.
size_t chunkSize = size < sizeof(uint64_t) ? size : sizeof(uint64_t);
uint64_t x = 0;
memcpy(&x, ptr, chunkSize);
ptr += chunkSize;
size -= chunkSize;
// Add to accumulator.
acc[0] += x;
}
return result;
}
template<typename T>
inline __host__ __device__ void eatHash(uint64_t acc[2], const T* bytes) {
eatHash(acc, (const void*)bytes, sizeof(T));
}
inline __host__ __device__ uint64_t digestHash(uint64_t const acc[2]) {
uint64_t h = acc[0];
h ^= h >> 31;
h *= 0xbac3bd562846de6b;
h += acc[1];
h ^= h >> 32;
h *= 0x995a187a14e7b445;
return h;
}
inline __host__ __device__ uint64_t getHash(const void* bytes, size_t size) {
uint64_t acc[2] = {1, 1};
eatHash(acc, bytes, size);
return digestHash(acc);
}
template<typename T>
inline __host__ __device__ uint64_t getHash(const T* bytes) {
return getHash((const void*)bytes, sizeof(T));
}
#endif
src/include/collectives.h
+239 -211
Просмотреть файл
@@ -12,6 +12,7 @@
#include "nccl.h"
#include "nccl_common.h"
#include "device.h"
#define NCCL_MAX_NET_SIZE (1024*1024*1024L) // Rather than send INT_MAX which is 2G-1, send a power of two.
// CHUNKSIZE must be a multiple of SLICESIZE
@@ -396,6 +397,42 @@ public:
~RingBCAlgorithm() {}
};
#if !defined (__CUDA_ARCH__) || __CUDA_ARCH__ >= 600
// #include <cuda/atomic>
#endif
// Need a power of two to ensure it divides by parallelFactor (which is also a power of two)
#define NCCL_PAT_NWORKERS 512
static constexpr int PatUsed = 0x1,
PatSkipped = 0x2;
struct ncclPatStep {
int recvDim, sendDim, recvOffset, sendOffset, stepOffset, postRecv, postSend, nelem, last, flags;
size_t inpIx, outIx;
};
struct ncclPatPeer {
uint64_t step;
struct ncclConnInfo* conn;
struct ncclConnFifo* connFifo;
void* buff;
uint64_t *headPtr;
uint64_t *tailPtr;
uint64_t stepCache;
long long int accSize;
int connStepSize;
};
#define NCCL_SHMEM_PAT_STEPS 32
struct ncclPatShmem {
struct ncclPatStep patSteps[NCCL_SHMEM_PAT_STEPS];
int parallelFactor;
long long int localAccSize;
struct ncclPatPeer sendDims[32]; // Should cover 2^32 ranks
struct ncclPatPeer recvDims[32];
};
template<typename T>
class PatRSAlgorithm{
size_t offset;
@@ -408,18 +445,17 @@ class PatRSAlgorithm{
int nrPow2;
int postFreq;
int lastA;
int parallelFactor;
int aggFactor;
int as; // aggregated steps
int a; // step inside aggregated step
int sendSkipped; // number of skipped steps during aggregation
int recvSkipped; // number of skipped steps during aggregation
int phase2recv; // receive offset for phase 2
int stepOffset;
int aggDelta;
int scale;
int phase;
__device__ __host__ int min(int a, int b) {
__device__ __host__ ssize_t min(ssize_t a, ssize_t b) {
return (a<b)?a:b;
}
@@ -447,16 +483,16 @@ class PatRSAlgorithm{
__device__ __host__ void resetA() {
a = 0;
sendSkipped = recvSkipped = 0;
sendSkipped = stepOffset = 0;
lastA = aggFactor;
if (phase >= 2) lastA /= 2*scale;
if (phase == 4) lastA = 1;
}
__device__ __host__ void reset() {
nelem = getNelem();
phase = 0;
scale = 1;
phase2recv = 0;
as = aggDelta - 1;
resetA();
}
@@ -479,8 +515,9 @@ class PatRSAlgorithm{
}
public:
__device__ __host__ PatRSAlgorithm(int stepSize, int stepDepth, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
__device__ __host__ PatRSAlgorithm(int stepSize, int stepDepth, int maxParallelFactor, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
offset(offset), end(end), count(count), chunkCount(chunkCount), rank(rank), nranks(nranks) {
parallelFactor = maxParallelFactor;
aggDelta = nrPow2 = (1<<log2Up(nranks));
aggFactor = 1;
@@ -490,6 +527,7 @@ public:
aggDelta /= 2;
}
postFreq = aggFactor;
if (postFreq < parallelFactor) parallelFactor = postFreq;
int d = stepDepth;
while (d > 1 && aggFactor < nranks/2) {
d /= 2;
@@ -500,160 +538,151 @@ public:
reset();
}
__device__ __host__ void getNextOp(int &recvDim, int &sendDim, size_t &inpIx, size_t &outIx, int &recvOffset, int &sendOffset, int &sendStepOffset, int &nelemOut, int &postRecv, int &postSend, int &last) {
restart:
last = 0;
nelemOut = nelem;
outIx = offset;
__device__ __host__ int getParallelFactor() {
return parallelFactor;
}
__device__ __host__ void getNextOp(struct ncclPatStep* ps) {
ps->last = 0;
ps->nelem = nelem;
ps->outIx = offset;
ps->stepOffset = stepOffset;
int skip = 0;
//printf("Phase %d as %d/%d a %d/%d scale %d\n", phase, as, aggDelta, a, lastA, scale);
if (phase == 0) {
if (a >= lastA) {
skip = 1;
} else if (phase == 0) {
int s = mirrorInvert(a, lastA)*aggDelta + as;
if (s >= nranks) skip = 1;
int sendDataRank = (rank + s) % nranks;
inpIx = sendDataRank * count + offset;
recvDim = -1;
sendDim = 0;
outIx = 0;
recvOffset = -1;
sendOffset = ((a - sendSkipped)%postFreq) * nelem;
sendStepOffset = 0;
if ((((a - sendSkipped)%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
postSend = 1;
ps->inpIx = sendDataRank * count + offset;
ps->recvDim = -1;
ps->sendDim = 0;
ps->outIx = 0;
ps->recvOffset = -1;
ps->sendOffset = (a%postFreq) * nelem;
if (((a%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
ps->postSend = 1;
} else {
postSend = 0;
ps->postSend = 0;
}
postRecv = 0;
if (skip) sendSkipped++;
if (++a == lastA) {
phase = as == 1 ? (aggFactor > 1 ? 2 : 4) : 1; // If as == 1, switch to phase 2
resetA();
}
if (skip == 0) return;
ps->postRecv = 0;
} else if (phase == 1) {
int s = mirrorInvert(a, lastA)*aggDelta + as;
if (s >= nranks) skip = 1;
recvDim = firstBitSet(s, nrPow2);
sendOffset = ((a - sendSkipped)%postFreq)*nelem;
recvOffset = ((a - recvSkipped)%postFreq)*nelem;
postSend = 0;
if (recvDim == 0) {
if ((((a - sendSkipped)%postFreq) + 1 >= postFreq) || (a == lastA-1)) postSend = 1;
sendStepOffset = 0;
ps->recvDim = firstBitSet(s, nrPow2);
ps->sendOffset = (a%postFreq)*nelem;
ps->recvOffset = (a%postFreq)*nelem;
ps->postSend = 0;
if (ps->recvDim == 0 && (((a%postFreq) + 1 >= postFreq) || (a == lastA-1))) ps->postSend = 1;
if (((a%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
ps->postRecv = 1;
} else {
sendStepOffset = (a - sendSkipped)/postFreq;
ps->postRecv = 0;
}
if ((((a - recvSkipped)%postFreq) + 1 >= postFreq) || (a == lastA-1)) {
postRecv = 1;
} else {
postRecv = 0;
}
s -= (1<<recvDim);
s -= (1<<ps->recvDim);
int recvDataRank = (rank + nranks + s) % nranks;
inpIx = recvDataRank * count + offset;
sendDim = s ? firstBitSet(s, nrPow2) : -1;
if (sendDim == -1) {
sendOffset = -1;
sendStepOffset = 0;
} else if (as - (1<<recvDim) == 0) {
if (newPeer(a, aggFactor)) sendSkipped = a;
ps->inpIx = recvDataRank * count + offset;
ps->sendDim = s ? firstBitSet(s, nrPow2) : -1;
if (ps->sendDim == -1) {
ps->sendOffset = -1;
} else if (as - (1<<ps->recvDim) == 0) {
if (newPeer(a, aggFactor)) { sendSkipped = a; ps->stepOffset = stepOffset = 0; }
int foffset = a - sendSkipped;
sendStepOffset = recvDim == 0 ? 0 : foffset/postFreq;
sendOffset = (foffset%postFreq)*nelem;
ps->sendOffset = (foffset%postFreq)*nelem;
}
int recvDim = ps->recvDim;
if (s < nranks && skip) {
recvDim = -1;
recvOffset = -1;
postRecv = 0;
ps->recvDim = -1;
ps->recvOffset = -1;
ps->postRecv = 0;
skip = 0;
}
if (skip || recvDim == -1) recvSkipped++;
if (skip) sendSkipped++;
if (++a == lastA) {
as--;
phase = as % 2 == 1 ? 0 : 1;
resetA();
}
if (skip == 0) return;
if (recvDim > 0 && (((a-sendSkipped)%postFreq) + 1 >= postFreq) && skip == 0) stepOffset++;
} else if (phase == 2) {
int s = (2*mirrorInvert(a, lastA)+1)*scale*aggDelta + 1;
postRecv = 0;
ps->postRecv = 0;
if (s >= nranks) skip = 1;
recvDim = 0;
postSend = a == lastA-1 ? 1 : 0;
ps->recvDim = 0;
ps->postSend = a == lastA-1 ? 1 : 0;
s -= 1;
if (s < nranks && skip) {
recvDim = -1;
recvOffset = -1;
ps->recvDim = -1;
ps->recvOffset = -1;
skip = 0;
} else if (!skip) {
int foffset = phase2recv;
phase2recv++;
postRecv |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
recvOffset = (foffset%postFreq) * nelem;
int foffset = a + aggFactor - aggFactor/scale;
ps->postRecv |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
ps->recvOffset = (foffset%postFreq) * nelem;
}
int recvDataRank = (rank + nranks + s) % nranks;
inpIx = recvDataRank * count + offset;
sendDim = s ? firstBitSet(s, nrPow2) : -1;
int foffset = a - sendSkipped;
postSend |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
sendStepOffset = 0;
sendOffset = (foffset%postFreq) * nelem;
if (skip || sendDim == -1) sendSkipped++;
if (++a == lastA) {
phase = 3;
resetA();
}
if (skip == 0) return;
ps->inpIx = recvDataRank * count + offset;
ps->sendDim = s ? firstBitSet(s, nrPow2) : -1;
int foffset = a;
ps->postSend |= ((foffset+1)%postFreq) == 0 ? 1 : 0;
ps->sendOffset = (foffset%postFreq) * nelem;
} else if (phase == 3) {
int s = (2*mirrorInvert(a, lastA)+1)*scale*aggDelta;
postRecv = a == lastA-1 ? 1 : 0;
ps->postRecv = a == lastA-1 ? 1 : 0;
if (s >= nranks) skip = 1;
recvDim = firstBitSet(s, nrPow2);
postSend = 0;
s -= (1<<recvDim);
int foffset = a - recvSkipped;
postRecv |= (foffset+1)%postFreq == 0 ? 1 : 0;
recvOffset = (foffset%postFreq) * nelem;
ps->recvDim = firstBitSet(s, nrPow2);
ps->postSend = 0;
s -= (1<<ps->recvDim);
int foffset = a;
ps->postRecv |= (foffset+1)%postFreq == 0 ? 1 : 0;
ps->recvOffset = (foffset%postFreq) * nelem;
int recvDataRank = (rank + nranks + s) % nranks;
inpIx = recvDataRank * count + offset;
sendDim = s ? firstBitSet(s, nrPow2) : -1;
ps->inpIx = recvDataRank * count + offset;
ps->sendDim = s ? firstBitSet(s, nrPow2) : -1;
if (s < nranks && skip) {
recvDim = -1;
recvOffset = -1;
postRecv = 0;
ps->recvDim = -1;
ps->recvOffset = -1;
ps->postRecv = 0;
skip = 0;
}
if (newPeer(a, aggFactor/(2*scale))) sendSkipped = a;
if (newPeer(a, aggFactor/(2*scale))) { sendSkipped = a; ps->stepOffset = stepOffset = 0; }
foffset = a - sendSkipped;
sendStepOffset = foffset / postFreq; // Accumulate on next steps
sendOffset = sendDim >= 0 ? (foffset%postFreq) * nelem : -1;
if (skip || recvDim == -1) recvSkipped++;
if (skip) sendSkipped++;
if (++a == lastA) {
scale *= 2;
phase = scale < aggFactor ? 2 : 4;
if ((foffset%postFreq) + 1 >= postFreq && skip == 0) stepOffset++;
ps->sendOffset = ps->sendDim >= 0 ? (foffset%postFreq) * nelem : -1;
} else if (phase == 4) {
ps->recvDim = 0;
ps->sendDim = -1;
ps->inpIx = rank * count + offset;
ps->recvOffset = ((aggFactor-1)%postFreq) * nelem;
ps->sendOffset = -1;
ps->postRecv = 1;
ps->postSend = 0;
offset += chunkCount;
}
a++;
if (a >= lastA && a >= parallelFactor) {
int p = phase;
if (p == 1) as--;
if (p == 3) scale *= 2;
phase =
p == 0 ? as == 1 ? (aggFactor > 1 ? 2 : 4) : 1 :
p == 1 ? as % 2 == 1 ? 0 : 1 :
p == 2 ? 3 :
p == 3 ? scale < aggFactor ? 2 : 4 :
5;
if (p == 4) {
if (offset >= end) {
ps->last = 2;
} else {
reset();
}
} else {
resetA();
}
if (skip == 0) return;
} else if (phase == 4) {
recvDim = 0;
sendDim = -1;
inpIx = rank * count + offset;
recvOffset = (phase2recv%postFreq) * nelem;
sendStepOffset = 0;
sendOffset = -1;
postRecv = 1;
postSend = 0;
offset += chunkCount;
if (offset >= end) {
last = 1;
} else {
reset();
}
return;
} else if (phase == 4 && offset >= end) {
ps->last = 1;
}
goto restart;
int flags = PatUsed | (skip ? PatSkipped : 0);
#if __CUDA_ARCH__ >= 600
cuda::atomic_ref<int, cuda::thread_scope_block> a(ps->flags);
a.store(flags, cuda::memory_order_release);
#else
ps->flags = flags;
#endif
}
};
@@ -669,14 +698,12 @@ class PatAGAlgorithm{
int nrPow2;
int postFreq;
int lastA;
int parallelFactor;
int aggFactor;
int as; // aggregated steps
int a; // step inside aggregated step
int aggDelta;
int scale;
int phase;
// AS computation
@@ -685,7 +712,7 @@ class PatAGAlgorithm{
int bitCount[32];
int bitZeroStep[32];
__device__ __host__ int min(int a, int b) {
__device__ __host__ ssize_t min(ssize_t a, ssize_t b) {
return (a<b)?a:b;
}
@@ -752,8 +779,9 @@ class PatAGAlgorithm{
public:
__device__ __host__ PatAGAlgorithm(int stepSize, int stepDepth, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
__device__ __host__ PatAGAlgorithm(int stepSize, int stepDepth, int maxParallelFactor, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks):
offset(offset), end(end), count(count), chunkCount(chunkCount), rank(rank), nranks(nranks) {
parallelFactor = maxParallelFactor;
aggDelta = nrPow2 = (1<<log2Up(nranks));
aggFactor = 1;
@@ -763,120 +791,120 @@ public:
aggDelta /= 2;
}
postFreq = aggFactor;
if (postFreq < parallelFactor) parallelFactor = postFreq;
int d = stepDepth;
while (d > 1 && aggFactor < nranks/2) {
d /= 2;
aggFactor *= 2;
aggDelta /= 2;
}
//printf("AggFactor %d PostFreq %d AggDelta %d\n", aggFactor, postFreq, aggDelta);
asDim = log2Up(aggDelta);
reset();
}
__device__ __host__ void getNextOp(int &recvDim, int &sendDim, size_t &inpIx, size_t &outIx, int &recvOffset, int &sendOffset, int &recvStepOffset, int &nelemOut, int &postRecv, int &postSend, int &last) {
restart:
//printf("Phase %d as %d/%d a %d/%d scale %d\n", phase, as, aggDelta, a, lastA, scale);
last = 0;
nelemOut = nelem;
inpIx = offset;
__device__ __host__ int getParallelFactor() {
return parallelFactor;
}
__device__ __host__ void getNextOp(struct ncclPatStep* ps) {
ps->last = 0;
ps->nelem = nelem;
ps->inpIx = offset;
int skip = 0;
if (phase == 0) {
if (a >= lastA) {
skip = 1;
} else if (phase == 0) {
int s = a*aggDelta + as;
if (s >= nranks) skip = 1;
int nextSkip = (a+1)*aggDelta + as >= nranks ? 1 : 0;
int recvDataRank = (rank + s) % nranks;
outIx = recvDataRank * count + offset;
sendDim = -1;
recvDim = 0;
inpIx = 0;
sendOffset = -1;
recvOffset = (a % postFreq) * nelem;
recvStepOffset = 0;
postRecv = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
postSend = 0;
a++;
if (nextSkip) {
as = nextAs();
if (as == aggDelta/2) {
offset += chunkCount;
if (offset >= end) {
last = 1;
} else {
reset();
}
return;
}
phase = 1;
resetA();
}
if (skip == 0) return;
ps->outIx = recvDataRank * count + offset;
ps->sendDim = -1;
ps->recvDim = 0;
ps->inpIx = 0;
ps->sendOffset = -1;
ps->recvOffset = (a % postFreq) * nelem;
ps->stepOffset = 0;
ps->postRecv = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
ps->postSend = 0;
} else if (phase == 1) {
int s = a*aggDelta + as;
if (s >= nranks) skip = 1;
sendDim = firstBitSet(s, nrPow2);
s -= (1<<sendDim);
ps->sendDim = firstBitSet(s, nrPow2);
s -= (1<<ps->sendDim);
int sendDataRank = (rank + nranks + s) % nranks;
outIx = sendDataRank * count + offset;
recvDim = s ? firstBitSet(s, nrPow2) : -1;
sendOffset = recvOffset = (a % postFreq) * nelem;
postSend = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
postRecv = (sendDim == 0) && ((a % postFreq == postFreq-1) || ((a+1)*aggDelta+as-1 >= nranks)) ? 1 : 0;
recvStepOffset = (sendDim == 0) ? 0 : a/postFreq;
if (recvDim == -1) {
recvOffset = -1;
postRecv = 0;
} else if (as - (1<<sendDim) == 0) {
int foffset = (a*aggDelta) >> (recvDim+1);
recvOffset = (foffset%postFreq)*nelem;
postRecv = (sendDim == 0) && ((foffset % postFreq == postFreq-1) || ((((foffset+1)*2)+1)<<recvDim) >= nranks) ? 1 : 0;
recvStepOffset = (sendDim == 0) ? 0 : foffset/postFreq;
ps->outIx = sendDataRank * count + offset;
ps->recvDim = s ? firstBitSet(s, nrPow2) : -1;
ps->sendOffset = ps->recvOffset = (a % postFreq) * nelem;
ps->postSend = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0;
ps->postRecv = (ps->sendDim == 0) && ((a % postFreq == postFreq-1) || ((a+1)*aggDelta+as-1 >= nranks)) ? 1 : 0;
ps->stepOffset = (ps->sendDim == 0) ? 0 : a/postFreq;
if (ps->recvDim == -1) {
ps->recvOffset = -1;
ps->postRecv = 0;
} else if (as - (1<<ps->sendDim) == 0) {
int foffset = (a*aggDelta) >> (ps->recvDim+1);
ps->recvOffset = (foffset%postFreq)*nelem;
ps->postRecv = (ps->sendDim == 0) && ((foffset % postFreq == postFreq-1) || ((((foffset+1)*2)+1)<<ps->recvDim) >= nranks) ? 1 : 0;
ps->stepOffset = (ps->sendDim == 0) ? 0 : foffset/postFreq;
}
if (s < nranks && sendDim == 0 && skip) {
if (s < nranks && ps->sendDim == 0 && skip) {
// Don't forget to receive at least once even if we don't send afterwards
sendDim = -1;
sendOffset = -1;
postSend = 0;
ps->sendDim = -1;
ps->sendOffset = -1;
ps->postSend = 0;
skip = 0;
}
if (++a == lastA) {
if (as % 2 == 1) {
phase = 0;
} else {
as = nextAs();
}
resetA();
}
if (skip == 0) return;
} else if (phase == 2) {
int s = (2*a+1)*scale*aggDelta;
postSend = (a % postFreq == postFreq-1) || ((2*(a+1)+1)*scale*aggDelta >= nranks) ? 1 : 0;
postRecv = 0;
ps->postSend = (a % postFreq == postFreq-1) || ((2*(a+1)+1)*scale*aggDelta >= nranks) ? 1 : 0;
ps->postRecv = 0;
if (s >= nranks) skip = 1;
sendDim = firstBitSet(s, nrPow2);
s -= (1<<sendDim);
sendOffset = (a%postFreq) * nelem;
recvStepOffset = a / postFreq;
ps->sendDim = firstBitSet(s, nrPow2);
s -= (1<<ps->sendDim);
ps->sendOffset = (a%postFreq) * nelem;
ps->stepOffset = a / postFreq;
int sendDataRank = (rank + nranks + s) % nranks;
outIx = sendDataRank * count + offset;
recvDim = s ? firstBitSet(s, nrPow2) : -1;
if (recvDim == -1) {
recvOffset = -1;
ps->outIx = sendDataRank * count + offset;
ps->recvDim = s ? firstBitSet(s, nrPow2) : -1;
if (ps->recvDim == -1) {
ps->recvOffset = -1;
} else {
s -= (1<<recvDim);
int foffset = (a*2*scale*aggDelta) >> (recvDim+1);
recvOffset = (foffset%postFreq)*nelem;
recvStepOffset = foffset / postFreq;
s -= (1<<ps->recvDim);
int foffset = (a*2*scale*aggDelta) >> (ps->recvDim+1);
ps->recvOffset = (foffset%postFreq)*nelem;
ps->stepOffset = foffset / postFreq;
}
if (++a == lastA) {
scale /= 2;
phase = scale ? 2 : 1;
}
a++;
if (a >= lastA && a >= parallelFactor) {
int p = phase;
if (p == 2) scale /= 2;
phase =
p == 2 ? scale ? 2 : 1 :
p == 1 ? as % 2 == 1 ? 0 : 1 :
1;
if (p == 0 || (p == 1 && as % 2 == 0)) as = nextAs();
if (p == 0 && as == aggDelta/2) {
offset += chunkCount;
if (offset >= end) {
ps->last = 2;
} else {
reset();
}
} else {
resetA();
}
if (skip == 0) return;
} else if (phase == 0 && as == 1 && offset + chunkCount >= end && a-1 >= ((lastA-1) / parallelFactor) * parallelFactor) {
ps->last = 1;
}
goto restart;
int flags = PatUsed | (skip ? PatSkipped : 0);
#if __CUDA_ARCH__ >= 600
cuda::atomic_ref<int, cuda::thread_scope_block> a(ps->flags);
a.store(flags, cuda::memory_order_release);
#else
ps->flags = flags;
#endif
}
};
#endif
src/include/comm.h
+8 -2
Просмотреть файл
@@ -139,6 +139,9 @@ struct ncclSharedResources {
int* tpRankToLocalRank;
// Internal streams
struct ncclStrongStream deviceStream, hostStream;
int noncapturedRefs; // number of non-captured hostStreamPlanCallback on the stream
int persistentRefs;
cudaEvent_t launchEvent, scratchEvent;
/* proxy related shared res */
struct ncclProxyState* proxyState;
@@ -437,6 +440,7 @@ struct ncclComm {
// List of destructors to run when comm is destructed
struct ncclDestructor* destructorHead;
struct ncclCudaContext* context;
struct ncclSharedResources* sharedRes;
/* map to top parent ranks. */
int* topParentRanks;
@@ -449,6 +453,7 @@ struct ncclComm {
int netPluginLoaded;
ncclNet_t* ncclNet;
int ncclNetVer;
ncclNetDeviceType netDeviceType;
ncclCollNet_t* ncclCollNet;
void* bootstrap;
@@ -456,6 +461,7 @@ struct ncclComm {
struct channelMasks* connectSend;
struct channelMasks* connectRecv;
struct ncclTopoGraph graphs[NCCL_NUM_ALGORITHMS];
int maxTreePattern;
bool initAlgoChannels[NCCL_NUM_ALGORITHMS];
bool runtimeConn; // if dynamic connection is supported
bool directMode;
@@ -603,8 +609,7 @@ struct ncclComm {
struct ncclComm* groupNext;
// Subset of those in groupNext list. Holds 0x1 if not needing preconnect.
struct ncclComm* preconnectNext;
int persistentRefs; // number of persistent plan-lists capturing this comm
int noncapturedRefs; // number of non-captured hostStreamPlanCallback on the stream
int localPersistentRefs; // number of persistent plan-lists capturing this comm
struct P2pSchedulePair { int sendRank; int recvRank; } *p2pSchedule;
struct ncclKernelPlanner planner;
@@ -669,6 +674,7 @@ struct ncclComm {
// Profiler plugin
void* profilerContext;
uint64_t seqNumber[NCCL_NUM_FUNCTIONS];
struct ncclProfilerProxy profiler;
// buffer registration cache
struct ncclRegCache regCache;
src/include/device.h
+10 -2
Просмотреть файл
@@ -163,6 +163,7 @@ struct ncclProxyConnector {
struct ncclConnector {
int connected;
int hasSeen;
struct ncclProxyConnector proxyConn;
struct ncclTransportComm* transportComm;
void* transportResources;
@@ -256,6 +257,8 @@ struct alignas(16) ncclDevWorkP2p {
uint8_t sendNetReg:1, recvNetReg:1;
uint8_t sendIpcReg:1, recvIpcReg:1;
uint8_t profilerEnabled:1;
uint8_t sendConnIndex:2, recvConnIndex:2;
};
@@ -304,7 +307,7 @@ struct alignas(16) ncclDevWorkColl {
uint32_t nWarps:8;
uint32_t redOpArgIsPtr:1, regUsed:1, netRegUsed:1, oneNode:1, direct:2, isOneRPN:1, rcclUseOneSlice:1;
uint32_t root:30, connIndex:2;
uint16_t pivotA2ANumBiRings;
uint16_t pivotA2ANumBiRings:15, profilerEnabled:1;
void* recvbuff;
void* sendbuff;
uintptr_t sendbuffOffset;
@@ -498,6 +501,7 @@ struct alignas(16) ncclDevChannel {
struct ncclTree binTree;
struct ncclNvls nvls;
uint32_t* workFifoDone; // Location of done counter, device writes index+1 of last work processed
uint64_t workCounter;
};
struct ncclDevComm {
@@ -523,6 +527,10 @@ struct ncclDevComm {
int* rankToLocalRank;
// Profiler counters
uint64_t* workStarted/*[MAXCHANNELS]*/;
uint64_t* workCompleted/*[MAXCHANNELS]*/;
#if defined(ENABLE_NPKIT)
NpKitEventCollectContext* npKitEventCollectContexts;
uint64_t* cpuTimestamp;
@@ -621,7 +629,7 @@ __host__ __device__ constexpr int ncclCalcUnroll(int bytePerPack, int insns, int
__host__ __device__ constexpr int ncclCollUnroll(int cudaArch = NCCL_CUDA_ARCH) {
// Our collective unroll should move to the same bytes&insns model as NVLS.
return cudaArch >= 800 ? 8 : 4;
return cudaArch >= 800 ? (cudaArch == 1200 ? 6 : 8) : 4;
}
__host__ __device__ constexpr int ncclNvlsUnrollBytes(int cudaArch = NCCL_CUDA_ARCH) { return 4*16; }
src/include/graph.h
+19 -10
Просмотреть файл
@@ -37,17 +37,24 @@ ncclResult_t ncclTopoComputeCommCPU(struct ncclComm* comm);
ncclResult_t ncclTopoGetNetDev(struct ncclComm* comm, int rank, struct ncclTopoGraph* graph, int channelId, int peerRank, int64_t* id, int* dev, int* proxyRank);
ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* system, int rank1, int rank2, int* p2p, int *read, int* intermediateRank);
ncclResult_t ncclTopoCheckMNNVL(struct ncclTopoSystem* system, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2, int* ret);
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* topo, int rank, int64_t netId, int read, int* useGdr);
#define MAX_XGMI_INTER_GPUS 4
ncclResult_t ncclTopoGetIntraNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int type, int64_t* id, int* dev);
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, int maxInter=MAX_XGMI_INTER_GPUS, int nInter=0, int *inter=nullptr);
ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int netDev, int rank, int* flush);
enum ncclTopoGdrMode {
ncclTopoGdrModeDisable = 0,
ncclTopoGdrModeDefault = 1,
ncclTopoGdrModePci = 2,
ncclTopoGdrModeNum = 3
};
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* topo, int rank, int64_t netId, int read, enum ncclTopoGdrMode* gdrMode);
ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int64_t netId, int netDev, int rank, int* flush);
ncclResult_t ncclTopoIsGdrAvail(struct ncclTopoSystem* system, int rank, bool *avail);
ncclResult_t ncclTopoCheckNet(struct ncclTopoSystem* system, int rank1, int rank2, int* net);
int ncclPxnDisable(struct ncclComm* comm);
ncclResult_t ncclTopoGetPxnRanks(struct ncclComm* comm, int** intermediateRanks, int* nranks);
ncclResult_t ncclGetLocalCpu(struct ncclTopoSystem* system, int gpu, int* retCpu);
#define MAX_XGMI_INTER_GPUS 4
ncclResult_t ncclTopoGetIntraNetDev(struct ncclTopoSystem* system, int rank, struct ncclTopoGraph* graph, int channelId, int type, int64_t* id, int* dev);
ncclResult_t ncclTopoGetLinkType(struct ncclTopoSystem* system, int cudaDev1, int cudaDev2, bool* isXGMI, int maxInter=MAX_XGMI_INTER_GPUS, int nInter=0, int *inter=nullptr);
// Find CPU affinity
ncclResult_t ncclTopoGetCpuAffinity(struct ncclTopoSystem* system, int rank, cpu_set_t* affinity);
@@ -59,11 +66,13 @@ ncclResult_t ncclTopoGetCpuAffinity(struct ncclTopoSystem* system, int rank, cpu
#define NCCL_TOPO_CPU_VENDOR_AMD 2
#define NCCL_TOPO_CPU_VENDOR_ZHAOXIN 3
#define NCCL_TOPO_CPU_VENDOR_MIXED 4
#define NCCL_TOPO_CPU_TYPE_BDW 1
#define NCCL_TOPO_CPU_TYPE_SKL 2
#define NCCL_TOPO_CPU_TYPE_ZEN 3
#define NCCL_TOPO_CPU_TYPE_ROME 4
#define NCCL_TOPO_CPU_TYPE_YONGFENG 1
#define NCCL_TOPO_CPU_MODEL_INTEL_BDW 1
#define NCCL_TOPO_CPU_MODEL_INTEL_SKL 2
#define NCCL_TOPO_CPU_MODEL_INTEL_SRP 3
#define NCCL_TOPO_CPU_MODEL_INTEL_ERP 4
#define NCCL_TOPO_CPU_MODEL_AMD_ZEN 5
#define NCCL_TOPO_CPU_MODEL_AMD_ROME 6
#define NCCL_TOPO_CPU_MODEL_YONGFENG 1
ncclResult_t ncclTopoCpuType(struct ncclTopoSystem* system, int* arch, int* vendor, int* model);
ncclResult_t ncclTopoGetGpuCount(struct ncclTopoSystem* system, int* count);
ncclResult_t ncclTopoGetNetCount(struct ncclTopoSystem* system, int* count);
src/include/group.h
+6
Просмотреть файл
@@ -108,6 +108,12 @@ inline void ncclGroupCommJoin(struct ncclComm* comm) {
struct ncclComm** pp = &ncclGroupCommHead;
while (*pp != nullptr && comm->intraComm0 != (*pp)->intraComm0)
pp = &(*pp)->groupNext;
// didn't find its clique, we need to insert it with ascending order based on commHash
if (*pp == nullptr) {
pp = &ncclGroupCommHead;
while (*pp != nullptr && (*pp)->commHash < comm->commHash) pp = &(*pp)->groupNext;
}
comm->groupNext = *pp;
*pp = comm;
// Comms gets a new memory stack scope upon joining. Each task batched for
src/include/nccl_net.h
-610
Просмотреть файл
@@ -1,610 +0,0 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NCCL_NET_H_
#define NCCL_NET_H_
#include "nccl.h"
#include "nccl_common.h"
#include "net_device.h"
#include <stdint.h>
#define NCCL_NET_HANDLE_MAXSIZE 128
//Maximum value NCCL can accept for maxP2pBytes and maxCollBytes net properties
#define NCCL_MAX_NET_SIZE_BYTES (1*1024*1024*1024*1024L)
#define NCCL_NET_OPTIONAL_RECV_COMPLETION 0x1
#define NCCL_PTR_HOST 0x1
#define NCCL_PTR_CUDA 0x2
#define NCCL_PTR_DMABUF 0x4
// Maximum number of requests per comm object
#define NCCL_NET_MAX_REQUESTS 32
// Max number of ncclNet objects which can live in the same process
#define NCCL_NET_MAX_PLUGINS 3
#define NCCL_NET_MAX_DEVS_PER_NIC_V9 4
#define NCCL_NET_MAX_DEVS_PER_NIC NCCL_NET_MAX_DEVS_PER_NIC_V9
typedef struct {
int ndevs;
int devs[NCCL_NET_MAX_DEVS_PER_NIC_V9];
} ncclNetVDeviceProps_v9_t;
typedef ncclNetVDeviceProps_v9_t ncclNetVDeviceProps_t;
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int regIsGlobal; // regMr is not tied to a particular comm
int forceFlush; // Force a flush on receives
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
ncclNetDeviceType netDeviceType; // Network offload type
int netDeviceVersion; // Version number for network offload
ncclNetVDeviceProps_v9_t vProps;
size_t maxP2pBytes; // Max transfer size for point-to-point operations
size_t maxCollBytes; // Max transfer size for collective operations
} ncclNetProperties_v9_t;
typedef ncclNetProperties_v9_t ncclNetProperties_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v8_t** sendDevComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v8_t** recvDevComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
// Notify the plugin that a recv has completed by the device
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
// Create a virtual NIC given the specified properties, which can be accessed at device index d
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_t* props);
} ncclNet_v9_t;
typedef ncclNet_v9_t ncclNet_t;
#define NCCL_NET_PLUGIN_SYMBOL ncclNetPlugin_v9
typedef struct {
void* mhandle;
void* address;
size_t size;
} ncclNetSGE_v9_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, size_t count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v9_t* recvParts,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
void* sendMhandle, void** request);
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v9_t* sendParts, void* recvData,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
ncclDataType_t dataType, ncclRedOp_t redOp,
void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
// Create a virtual NIC given the specified properties, which can be accessed at device index d
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_t* props);
} ncclCollNet_v9_t;
typedef ncclCollNet_v9_t ncclCollNet_t;
#define NCCL_COLLNET_PLUGIN_SYMBOL ncclCollNetPlugin_v9
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int regIsGlobal; // regMr is not tied to a particular comm
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
ncclNetDeviceType netDeviceType; // Network offload type
int netDeviceVersion; // Version number for network offload
} ncclNetProperties_v8_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v8_t** sendDevComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v8_t** recvDevComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
// Notify the plugin that a recv has completed by the device
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
} ncclNet_v8_t;
typedef struct {
void* mhandle;
void* address;
uint32_t size;
} ncclNetSGE_v8_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v8_t* recvParts,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
void* sendMhandle, void** request);
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v8_t* sendParts, void* recvData,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
ncclDataType_t dataType, ncclRedOp_t redOp,
void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclCollNet_v8_t;
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
ncclNetDeviceType netDeviceType; // Network offload type
int netDeviceVersion; // Version number for network offload
} ncclNetProperties_v7_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v7_t** sendDevComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v7_t** recvDevComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
// Notify the plugin that a recv has completed by the device
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
} ncclNet_v7_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclCollNet_v7_t;
#define NCCL_NET_MAX_REQUESTS_V6 8
// v6 struct for backwards compatibility
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
} ncclNetProperties_v6_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
ncclResult_t (*connect)(int dev, void* handle, void** sendComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
ncclResult_t (*accept)(void* listenComm, void** recvComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclNet_v6_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclCollNet_v6_t;
// v5 struct for backwards compatibility
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
ncclResult_t (*connect)(int dev, void* handle, void** sendComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
ncclResult_t (*accept)(void* listenComm, void** recvComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclNet_v5_t;
// v5 struct for backwards compatibility
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclCollNet_v5_t;
// context passed from RCCL lib to n/w plugin
typedef struct {
// channel id
uint32_t chId;
} ncclNet_ctxt_t;
#endif // end include guard
src/include/nccl_profiler.h
-235
Просмотреть файл
@@ -1,235 +0,0 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NCCL_PROFILER_H_
#define NCCL_PROFILER_H_
#include <cstdint>
enum {
ncclProfileGroup = (1 << 0), // group event type
ncclProfileColl = (1 << 1), // host collective call event type
ncclProfileP2p = (1 << 2), // host point-to-point call event type
ncclProfileProxyOp = (1 << 3), // proxy operation event type
ncclProfileProxyStep = (1 << 4), // proxy step event type
ncclProfileProxyCtrl = (1 << 5), // proxy control event type
};
typedef struct {
uint8_t type; // event type descriptor: ncclProfileColl, ...
void* parentObj; // pointer to the profiler parent object (for coll is the group)
int rank; // originating rank
union {
struct {
const char* name;
uint64_t commHash;
uint64_t seqNumber;
const char* func;
void const* sendBuff;
void* recvBuff;
size_t count;
int root;
const char* datatype;
size_t trafficBytes;
uint8_t nMaxChannels;
uint8_t nWarps;
const char* algo;
const char* proto;
} coll;
struct {
const char* name;
uint64_t commHash;
const char* func;
void* buff;
const char* datatype;
size_t count;
int peer;
} p2p;
struct {
pid_t pid; // pid of the originating process
uint8_t channelId; // channel id for this proxy operation
int peer; // remote rank for send/recv
int nSteps; // number of steps for this proxy operation
int chunkSize; // amount of data transferred by this proxy operation
int isSend;
} proxyOp;
struct {
int step;
} proxyStep;
};
} ncclProfilerEventDescr_v2_t;
typedef enum {
ncclProfilerProxyOpSendPosted,
ncclProfilerProxyOpSendRemFifoWait,
ncclProfilerProxyOpSendTransmitted,
ncclProfilerProxyOpSendDone,
ncclProfilerProxyOpRecvPosted,
ncclProfilerProxyOpRecvReceived,
ncclProfilerProxyOpRecvTransmitted,
ncclProfilerProxyOpRecvDone,
/* Legacy proxy profiler states */
ncclProfilerProxyStepSendGPUWait,
ncclProfilerProxyStepSendWait,
ncclProfilerProxyStepRecvWait,
ncclProfilerProxyStepRecvFlushWait,
ncclProfilerProxyStepRecvGPUWait,
/* Legacy proxy control states */
ncclProfilerProxyCtrlIdle,
ncclProfilerProxyCtrlActive,
ncclProfilerProxyCtrlSleep,
ncclProfilerProxyCtrlWakeup,
ncclProfilerProxyCtrlAppend,
ncclProfilerProxyCtrlAppendEnd,
} ncclProfilerEventState_v2_t;
typedef union {
struct {
size_t transSize;
int steps;
} proxyOp;
struct {
int appendedProxyOps;
} proxyCtrl;
} ncclProfilerEventStateArgs_v2_t;
typedef struct {
const char* name;
// init - initialize the profiler plugin
// Input
// - context : opaque profiler context object for separating profiler behavior across comms
// Output
// - eActivationMask: bitmask of active events set by the plugin
ncclResult_t (*init)(void** context, int* eActivationMask);
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
// Input
// - context: opaque profiler context object
// - eDescr : pointer to ncclProfilerEventDescr_t object
// Output
// - eHandle: return event handle for supplied event descriptor object
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr);
// stopEvent - stop/finalize an event inside and event set
// Input
// - eHandle: handle to event object
ncclResult_t (*stopEvent)(void* eHandle);
// recordEventState - record event state transitions and event attribute updates
// Input
// - eHandle : handle to event object created through startEvent
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
// - eState : event state transition
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v2_t eState, ncclProfilerEventStateArgs_v2_t* eStateArgs);
// finalize - finalize the profiler plugin
// Input
// - context: opaque profiler context object
ncclResult_t (*finalize)(void* context);
} ncclProfiler_v2_t;
typedef ncclProfilerEventDescr_v2_t ncclProfilerEventDescr_t;
typedef ncclProfilerEventState_v2_t ncclProfilerEventState_t;
typedef ncclProfilerEventStateArgs_v2_t ncclProfilerEventStateArgs_t;
typedef ncclProfiler_v2_t ncclProfiler_t;
typedef struct {
uint8_t type; // event type descriptor: ncclProfileColl, ...
void* parentObj; // pointer to the profiler parent object (for coll is the group)
int rank; // originating rank
union {
struct {
const char* name;
uint64_t commHash;
uint64_t seqNumber;
uint8_t func;
void const* sendBuff;
void* recvBuff;
size_t count;
int root;
uint8_t datatype;
uint32_t op;
size_t trafficBytes;
uint8_t nMaxChannels;
uint8_t nWarps;
uint8_t algo;
uint8_t proto;
int isCollnet;
int isNvls;
} coll;
struct {
const char* name;
uint64_t commHash;
uint8_t func;
void* buff;
uint8_t datatype;
size_t count;
int peer;
} p2p;
struct {
pid_t pid; // pid of the originating process
uint8_t channelId; // channel id for this proxy operation
int peer; // remote rank for send/recv
int nSteps; // number of steps for this proxy operation
int chunkSize; // amount of data transferred by this proxy operation
int isSend;
} proxyOp;
struct {
int step;
} proxyStep;
};
} ncclProfilerEventDescr_v1_t;
typedef ncclProfilerEventState_v2_t ncclProfilerEventState_v1_t;
typedef ncclProfilerEventStateArgs_v2_t ncclProfilerEventStateArgs_v1_t;
typedef struct {
const char* name;
// init - initialize the profiler plugin
// Input
// - context : opaque profiler context object for separating profiler behavior across comms
// Output
// - eActivationMask: bitmask of active events set by the plugin
ncclResult_t (*init)(void** context, int* eActivationMask);
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
// Input
// - context: opaque profiler context object
// - eDescr : pointer to ncclProfilerEventDescr_t object
// Output
// - eHandle: return event handle for supplied event descriptor object
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr);
// stopEvent - stop/finalize an event inside and event set
// Input
// - eHandle: handle to event object
ncclResult_t (*stopEvent)(void* eHandle);
// recordEventState - record event state transitions and event attribute updates
// Input
// - eHandle : handle to event object created through startEvent
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
// - eState : event state transition
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs);
// finalize - finalize the profiler plugin
// Input
// - context: opaque profiler context object
ncclResult_t (*finalize)(void* context);
} ncclProfiler_v1_t;
#endif
src/include/nccl_tuner.h
-149
Просмотреть файл
@@ -1,149 +0,0 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NCCL_TUNER_H_
#define NCCL_TUNER_H_
#include "nccl.h"
#include "nccl_common.h"
// API to be implemented by external tuner
typedef struct {
// Name of the tuner
const char* name;
// Initializes tuner states.
// Inputs:
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
// - nNodes: number of nodes in current communicator.
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
// Outputs:
// - context: tuner context object
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
// Inputs:
// - context: tuner context object
// - collType: collective type , e.g., allreduce, allgather…
// - nBytes: collective size in bytes
// - numPipeOps: number of operations in the group
// - numAlgo: number of algorithms in collCostTable
// - numProto: number of protocols in collCostTable
// - regBuff: can register user buffer
//
// Outputs:
// - nChannels: number of channels (hence SMs) to be used.
//
// InOut:
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
//
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
// default tuning for the given collective.
// Also, the plugin is allowed to not set any output, or set only the
// algorithm and protocol, but not only the algorithm or only the protocol.
// Unset fields will be set automatically by NCCL.
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
int regBuff, int* nChannels);
// Terminates the plugin and cleans up any resources that the plugin allocated.
// context: tuner context object
ncclResult_t (*destroy)(void* context);
} ncclTuner_v4_t;
typedef ncclTuner_v4_t ncclTuner_t;
#define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v4"
// API to be implemented by external tuner
typedef struct {
// Name of the tuner
const char* name;
// Initializes tuner states.
// Inputs:
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
// - nNodes: number of nodes in current communicator.
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
// Outputs:
// - context: tuner context object
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
// Inputs:
// - context: tuner context object
// - collType: collective type , e.g., allreduce, allgather…
// - nBytes: collective size in bytes
// - numPipeOps: number of operations in the group
// - numAlgo: number of algorithms in collCostTable
// - numProto: number of protocols in collCostTable
//
// Outputs:
// - nChannels: number of channels (hence SMs) to be used.
//
// InOut:
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
//
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
// default tuning for the given collective.
// Also, the plugin is allowed to not set any output, or set only the
// algorithm and protocol, but not only the algorithm or only the protocol.
// Unset fields will be set automatically by NCCL.
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
int* nChannels);
// Terminates the plugin and cleans up any resources that the plugin allocated.
// context: tuner context object
ncclResult_t (*destroy)(void* context);
} ncclTuner_v3_t;
// API to be implemented by external tuner
typedef struct {
// Name of the tuner
const char* name;
// Initializes tuner states.
// Inputs:
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
// - nNodes: number of nodes in current communicator.
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
// Outputs:
// - context: tuner context object
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
// Inputs:
// - context: tuner context object
// - collType: collective type , e.g., allreduce, allgather…
// - nBytes: collective size in bytes
// - collNetTypeSupport: whether collnet supports this type
// - nvlsTypeSupport: whether nvlink sharp supports this time
// - numPipeOps: number of operations in the group
//
// Outputs:
// - algorithm: selected algorithm to be used for the given collective
// - protocol: selected protocol to be used for the give collective
// - nChannels: number of channels (hence SMs) to be used.
//
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
// default tuning for the given collective.
// Also, the plugin is allowed to not set any output, or set only the
// algorithm and protocol, but not only the algorithm or only the protocol.
// Unset fields will be set automatically by NCCL.
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
int collNetSupport, int nvlsSupport, int numPipeOps,
int* algorithm, int* protocol, int* nChannels);
// Terminates the plugin and cleans up any resources that the plugin allocated.
// context: tuner context object
ncclResult_t (*destroy)(void* context);
} ncclTuner_v2_t;
#endif
src/include/net.h
-1
Просмотреть файл
@@ -18,7 +18,6 @@ ncclResult_t ncclNetPluginLoad(struct ncclComm* comm);
ncclResult_t ncclNetPluginUnload(struct ncclComm* comm);
ncclResult_t ncclNetInit(struct ncclComm* comm);
ncclResult_t ncclNetFinalize(struct ncclComm* comm);
int ncclNetVersion(struct ncclComm* comm);
// Test whether the current GPU support GPU Direct RDMA.
ncclResult_t ncclGpuGdrSupport(struct ncclComm* comm, int* gdrSupport);
src/include/net_device.h
+2 -1
Просмотреть файл
@@ -26,6 +26,7 @@ typedef struct {
typedef ncclNetDeviceHandle_v7_t ncclNetDeviceHandle_v8_t;
typedef ncclNetDeviceHandle_v8_t ncclNetDeviceHandle_v9_t;
typedef ncclNetDeviceHandle_v9_t ncclNetDeviceHandle_t;
typedef ncclNetDeviceHandle_v9_t ncclNetDeviceHandle_v10_t;
typedef ncclNetDeviceHandle_v10_t ncclNetDeviceHandle_t;
#endif
src/include/nvtx.h
+2 -1
Просмотреть файл
@@ -37,9 +37,10 @@
#define NVTX_SID_CommInitRankScalable 17 // same schema as NVTX_SID_CommInitRank
#define NVTX_SID_CommSplit 18
#define NVTX_SID_CommFinalize 19
// When adding new schema IDs, DO NOT re-use/overlap with the enum schema ID below!
// Define static schema ID for the reduction operation.
#define NVTX_PAYLOAD_ENTRY_NCCL_REDOP 19 + NVTX_PAYLOAD_ENTRY_TYPE_SCHEMA_ID_STATIC_START
#define NVTX_PAYLOAD_ENTRY_NCCL_REDOP 20 + NVTX_PAYLOAD_ENTRY_TYPE_SCHEMA_ID_STATIC_START
extern const nvtxDomainHandle_t ncclNvtxDomainHandle;
src/include/plugin/nccl_net.h
+60
Просмотреть файл
@@ -0,0 +1,60 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NCCL_NET_H_
#define NCCL_NET_H_
#include "nccl.h"
#include "nccl_common.h"
#include "net_device.h"
#include <stdint.h>
#define NCCL_NET_HANDLE_MAXSIZE 128
//Maximum value NCCL can accept for maxP2pBytes and maxCollBytes net properties
#define NCCL_MAX_NET_SIZE_BYTES (1*1024*1024*1024*1024L)
#define NCCL_NET_OPTIONAL_RECV_COMPLETION 0x1
#define MAX_NET_SIZE (1024*1024*1024L) // Rather than send INT_MAX which is 2G-1, send a power of two.
#define MAX_COLLNET_SIZE (512*1024*1024L) //Set for initial collent plugins when size was not dynamically queried
#define NCCL_PTR_HOST 0x1
#define NCCL_PTR_CUDA 0x2
#define NCCL_PTR_DMABUF 0x4
// Maximum number of requests per comm object
#define NCCL_NET_MAX_REQUESTS 32
// Max number of ncclNet objects which can live in the same process
#define NCCL_NET_MAX_PLUGINS 3
// NCCL core profiler callback for network defined events instrumentation
typedef ncclResult_t (*ncclProfilerCallback_t)(void** eHandle, int type, void* pHandle, int64_t pluginId, void* extData);
#include "net/net_v10.h"
#include "net/net_v9.h"
#include "net/net_v8.h"
#include "net/net_v7.h"
#include "net/net_v6.h"
typedef ncclNet_v10_t ncclNet_t;
typedef ncclCollNet_v10_t ncclCollNet_t;
typedef ncclNetSGE_v10_t ncclNetSGE_t;
typedef ncclNetProperties_v10_t ncclNetProperties_t;
typedef ncclNetVDeviceProps_v10_t ncclNetVDeviceProps_t;
typedef ncclNetCommConfig_v10_t ncclNetCommConfig_t;
#define NCCL_NET_MAX_DEVS_PER_NIC NCCL_NET_MAX_DEVS_PER_NIC_V10
#define NCCL_NET_PLUGIN_SYMBOL ncclNetPlugin_v10
#define NCCL_COLLNET_PLUGIN_SYMBOL ncclCollNetPlugin_v10
// context passed from RCCL lib to n/w plugin
typedef struct {
// channel id
uint32_t chId;
} ncclNet_ctxt_t;
#endif // end include guard
src/include/plugin/nccl_profiler.h
+69
Просмотреть файл
@@ -0,0 +1,69 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NCCL_PROFILER_H_
#define NCCL_PROFILER_H_
enum {
ncclProfileGroup = (1 << 0), // group event type
ncclProfileColl = (1 << 1), // host collective call event type
ncclProfileP2p = (1 << 2), // host point-to-point call event type
ncclProfileProxyOp = (1 << 3), // proxy operation event type
ncclProfileProxyStep = (1 << 4), // proxy step event type
ncclProfileProxyCtrl = (1 << 5), // proxy control event type
ncclProfileKernelCh = (1 << 6), // kernel channel event type
ncclProfileNetPlugin = (1 << 7), // network plugin-defined, events
};
typedef enum {
ncclProfilerProxyOpSendPosted,
ncclProfilerProxyOpSendRemFifoWait,
ncclProfilerProxyOpSendTransmitted,
ncclProfilerProxyOpSendDone,
ncclProfilerProxyOpRecvPosted,
ncclProfilerProxyOpRecvReceived,
ncclProfilerProxyOpRecvTransmitted,
ncclProfilerProxyOpRecvDone,
/* Legacy proxy profiler states */
ncclProfilerProxyStepSendGPUWait,
ncclProfilerProxyStepSendWait,
ncclProfilerProxyStepRecvWait,
ncclProfilerProxyStepRecvFlushWait,
ncclProfilerProxyStepRecvGPUWait,
/* Legacy proxy control states */
ncclProfilerProxyCtrlIdle,
ncclProfilerProxyCtrlActive,
ncclProfilerProxyCtrlSleep,
ncclProfilerProxyCtrlWakeup,
ncclProfilerProxyCtrlAppend,
ncclProfilerProxyCtrlAppendEnd,
} ncclProfilerEventState_t;
typedef ncclProfilerEventState_t ncclProfilerEventState_v1_t;
typedef ncclProfilerEventState_t ncclProfilerEventState_v2_t;
typedef ncclProfilerEventState_t ncclProfilerEventState_v3_t;
#include <cstdint>
#include "profiler/profiler_v3.h"
#include "profiler/profiler_v2.h"
#include "profiler/profiler_v1.h"
typedef ncclProfiler_v3_t ncclProfiler_t;
typedef ncclProfilerEventDescr_v3_t ncclProfilerEventDescr_t;
typedef ncclProfilerEventStateArgs_v3_t ncclProfilerEventStateArgs_t;
#define NCCL_PROFILER_NET_VER_BITS (16)
#define NCCL_PROFILER_NET_VER_MASK (~0U >> NCCL_PROFILER_NET_VER_BITS)
#define NCCL_PROFILER_NET_TYPE_MASK (~0U << NCCL_PROFILER_NET_VER_BITS)
typedef enum {
NCCL_PROFILER_NET_TYPE_IB = (1U << NCCL_PROFILER_NET_VER_BITS),
NCCL_PROFILER_NET_TYPE_SOCK = (2U << NCCL_PROFILER_NET_VER_BITS),
} ncclProfilerNetType;
#endif
src/include/plugin/nccl_tuner.h
+22
Просмотреть файл
@@ -0,0 +1,22 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NCCL_TUNER_H_
#define NCCL_TUNER_H_
#include "nccl.h"
#include "nccl_common.h"
#include "tuner/tuner_v4.h"
#include "tuner/tuner_v3.h"
#include "tuner/tuner_v2.h"
typedef ncclTuner_v4_t ncclTuner_t;
#define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v4"
#endif
src/include/plugin/net/net_v10.h
+158
Просмотреть файл
@@ -0,0 +1,158 @@
/*
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*/
#ifndef NET_V10_H_
#define NET_V10_H_
#define NCCL_NET_MAX_DEVS_PER_NIC_V10 4
typedef struct {
int ndevs;
int devs[NCCL_NET_MAX_DEVS_PER_NIC_V10];
} ncclNetVDeviceProps_v10_t;
#define NCCL_NET_TRAFFIC_CLASS_UNDEF -1
typedef struct {
// Plugin-specific TC value
int trafficClass;
} ncclNetCommConfig_v10_t;
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int regIsGlobal; // regMr is not tied to a particular comm
int forceFlush; // Force a flush on receives
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
ncclNetDeviceType netDeviceType; // Network offload type
int netDeviceVersion; // Version number for network offload
ncclNetVDeviceProps_v10_t vProps;
size_t maxP2pBytes; // Max transfer size for point-to-point operations
size_t maxCollBytes; // Max transfer size for collective operations
} ncclNetProperties_v10_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v10_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*connect)(int dev, ncclNetCommConfig_v10_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_v10_t** sendDevComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v10_t** recvDevComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
// Notify the plugin that a recv has completed by the device
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
// Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller
// what index this new vNIC exists at
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v10_t* props);
} ncclNet_v10_t;
typedef struct {
void* mhandle;
void* address;
size_t size;
} ncclNetSGE_v10_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v10_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, size_t count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v10_t* recvParts,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
void* sendMhandle, void** request);
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v10_t* sendParts, void* recvData,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
ncclDataType_t dataType, ncclRedOp_t redOp,
void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
// Create a virtual NIC given the specified properties, which can be accessed at device index d
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v10_t* props);
} ncclCollNet_v10_t;
#endif // end include guard
src/include/plugin/net/net_v6.h
+113
Просмотреть файл
@@ -0,0 +1,113 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_V6_H_
#define NET_V6_H_
#define NCCL_NET_MAX_REQUESTS_V6 8
// v6 struct for backwards compatibility
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
} ncclNetProperties_v6_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
ncclResult_t (*connect)(int dev, void* handle, void** sendComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
ncclResult_t (*accept)(void* listenComm, void** recvComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclNet_v6_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclCollNet_v6_t;
#endif
src/include/plugin/net/net_v7.h
+120
Просмотреть файл
@@ -0,0 +1,120 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_V7_H_
#define NET_V7_H_
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
ncclNetDeviceType netDeviceType; // Network offload type
int netDeviceVersion; // Version number for network offload
} ncclNetProperties_v7_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v7_t** sendDevComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v7_t** recvDevComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
// Notify the plugin that a recv has completed by the device
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
} ncclNet_v7_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclCollNet_v7_t;
#endif
src/include/plugin/net/net_v8.h
+134
Просмотреть файл
@@ -0,0 +1,134 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_V8_H_
#define NET_V8_H_
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int regIsGlobal; // regMr is not tied to a particular comm
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
ncclNetDeviceType netDeviceType; // Network offload type
int netDeviceVersion; // Version number for network offload
} ncclNetProperties_v8_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v8_t** sendDevComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v8_t** recvDevComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
// Notify the plugin that a recv has completed by the device
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
} ncclNet_v8_t;
typedef struct {
void* mhandle;
void* address;
uint32_t size;
} ncclNetSGE_v8_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v8_t* recvParts,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
void* sendMhandle, void** request);
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v8_t* sendParts, void* recvData,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
ncclDataType_t dataType, ncclRedOp_t redOp,
void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclCollNet_v8_t;
#endif
src/include/plugin/net/net_v9.h
+152
Просмотреть файл
@@ -0,0 +1,152 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_V9_H_
#define NET_V9_H_
#define NCCL_NET_MAX_DEVS_PER_NIC_V9 4
typedef struct {
int ndevs;
int devs[NCCL_NET_MAX_DEVS_PER_NIC_V9];
} ncclNetVDeviceProps_v9_t;
typedef struct {
char* name; // Used mostly for logging.
char* pciPath; // Path to the PCI device in /sys.
uint64_t guid; // Unique identifier for the NIC chip. Important for
// cards with multiple PCI functions (Physical or virtual).
int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF]
int regIsGlobal; // regMr is not tied to a particular comm
int forceFlush; // Force a flush on receives
int speed; // Port speed in Mbps.
int port; // Port number.
float latency; // Network latency
int maxComms; // Maximum number of comms we can create
int maxRecvs; // Maximum number of grouped receives.
ncclNetDeviceType netDeviceType; // Network offload type
int netDeviceVersion; // Version number for network offload
ncclNetVDeviceProps_v9_t vProps;
size_t maxP2pBytes; // Max transfer size for point-to-point operations
size_t maxCollBytes; // Max transfer size for collective operations
} ncclNetProperties_v9_t;
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create a connection.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Connect to a handle and return a sending comm object for that peer.
// This call must not block for the connection to be established, and instead
// should return successfully with sendComm == NULL with the expectation that
// it will be called again until sendComm != NULL.
// If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v9_t** sendDevComm);
// Finalize connection establishment after remote peer has called connect.
// This call must not block for the connection to be established, and instead
// should return successfully with recvComm == NULL with the expectation that
// it will be called again until recvComm != NULL.
// If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection
ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v9_t** recvDevComm);
// Register/Deregister memory. Comm can be either a sendComm or a recvComm.
// Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* comm, void* mhandle);
// Asynchronous send to a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request);
// Asynchronous recv from a peer.
// May return request == NULL if the call cannot be performed (or would block)
ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* sizes);
// Close and free send/recv comm objects
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
// Copy the given mhandle to a dptr in a format usable by this plugin's device code
ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle);
// Notify the plugin that a recv has completed by the device
ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request);
// Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller
// what index this new vNIC exists at
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v9_t* props);
} ncclNet_v9_t;
typedef struct {
void* mhandle;
void* address;
size_t size;
} ncclNetSGE_v9_t;
typedef struct {
// Name of the collective network (mainly for logs)
const char* name;
// Initialize the collective network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters capable of doing collective operations.
// If ndev returns 0, all other functions might be set to NULL.
ncclResult_t (*devices)(int* ndev);
// Get various device properties.
ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props);
// Create a receiving object and provide a handle to connect to it. The
// handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged
// between ranks to create connections.
ncclResult_t (*listen)(int dev, void* handle, void** listenComm);
// Create a group for collective operations. handles have been created
// using listen() above. rank indicates caller's rank in the collective network.
ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm);
// Returns whether a reduction operation on a data type is supported.
// 1 for supported, 0 otherwise.
ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported);
// Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA.
ncclResult_t (*regMr)(void* collComm, void* data, size_t size, int type, void** mhandle);
/* DMA-BUF support */
ncclResult_t (*regMrDmaBuf)(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle);
ncclResult_t (*deregMr)(void* collComm, void* mhandle);
// Performs an asynchronous allreduce operation on the collective group.
// May return request == NULL if the call cannot be performed (or would block).
ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, size_t count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request);
ncclResult_t (*iallgather)(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_v9_t* recvParts,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
void* sendMhandle, void** request);
ncclResult_t (*ireducescatter)(void* collComm, int nSendParts, ncclNetSGE_v9_t* sendParts, void* recvData,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
ncclDataType_t dataType, ncclRedOp_t redOp,
void* recvMhandle, void** request);
// Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is
// visible to the GPU
ncclResult_t (*iflush)(void* collComm, void* data, int size, void* mhandle, void** request);
// Test whether a request is complete. If size is not NULL, it returns the
// number of bytes sent/received.
ncclResult_t (*test)(void* request, int* done, int* size);
// Close and free collective comm objects
ncclResult_t (*closeColl)(void* collComm);
ncclResult_t (*closeListen)(void* listenComm);
// Create a virtual NIC given the specified properties, which can be accessed at device index d
ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v9_t* props);
} ncclCollNet_v9_t;
#endif // end include guard
src/include/plugin/plugin.h
+18
Просмотреть файл
@@ -0,0 +1,18 @@
/*************************************************************************
* Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NCCL_PLUGIN_H_
#define NCCL_PLUGIN_H_
#include "nccl.h"
void* ncclOpenNetPluginLib(const char* name);
void* ncclOpenTunerPluginLib(const char* name);
void* ncclOpenProfilerPluginLib(const char* name);
void* ncclGetNetPluginLib(void);
ncclResult_t ncclClosePluginLib(void* handle);
#endif
src/include/plugin/profiler/net_ib.h
+13
Просмотреть файл
@@ -0,0 +1,13 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_IB_H_
#define NET_IB_H_
#include "nccl_profiler.h"
#include "net_ib_v1.h"
#endif
src/include/plugin/profiler/net_ib_v1.h
+34
Просмотреть файл
@@ -0,0 +1,34 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_IB_V1_H_
#define NET_IB_V1_H_
#define NCCL_PROFILER_NET_IB_VER 1
enum {
ncclProfileQp = (1 << 0),
};
// The data structure version is encoded in the plugin identifier bitmask and
// passed to NCCL core through the profiler callback. NCCL copies the plugin
// identifier in the event descriptor before calling the profiler startEvent
// function. The profiler should inspect the plugin id to find out the source
// plugin as well as the version of the event struct
typedef struct {
uint8_t type; // event type (plugin defined)
union {
struct {
int device; // network device id
uint64_t wr_id; // work request id
int opcode; // ibv opcode
int qpNum; // QP number
size_t length; // work request data length
} qp;
};
} ncclProfilerNetIbDescr_v1_t;
#endif
src/include/plugin/profiler/net_socket.h
+13
Просмотреть файл
@@ -0,0 +1,13 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_SOCKET_H_
#define NET_SOCKET_H_
#include "nccl_profiler.h"
#include "net_socket_v1.h"
#endif
src/include/plugin/profiler/net_socket_v1.h
+32
Просмотреть файл
@@ -0,0 +1,32 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef NET_SOCKET_V1_H_
#define NET_SOCKET_V1_H_
#define NCCL_PROFILER_NET_SOCKET_VER 1
enum {
ncclProfileSocket = (1 << 0),
};
// The data structure version is encoded in the plugin identifier bitmask and
// passed to NCCL core through the profiler callback. NCCL copies the plugin
// identifier in the event descriptor before calling the profiler startEvent
// function. The profiler should inspect the plugin id to find out the source
// plugin as well as the version of the event struct
typedef struct {
uint8_t type; // event type (plugin defined)
union {
struct {
int fd;
int op;
size_t length;
} sock;
};
} ncclProfilerNetSockDescr_v1_t;
#endif
src/include/plugin/profiler/profiler_v1.h
+107
Просмотреть файл
@@ -0,0 +1,107 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef PROFILER_V1_H_
#define PROFILER_V1_H_
typedef struct {
uint8_t type; // event type descriptor: ncclProfileColl, ...
void* parentObj; // pointer to the profiler parent object (for coll is the group)
int rank; // originating rank
union {
struct {
const char* name;
uint64_t commHash;
uint64_t seqNumber;
uint8_t func;
void const* sendBuff;
void* recvBuff;
size_t count;
int root;
uint8_t datatype;
uint32_t op;
size_t trafficBytes;
uint8_t nMaxChannels;
uint8_t nWarps;
uint8_t algo;
uint8_t proto;
int isCollnet;
int isNvls;
} coll;
struct {
const char* name;
uint64_t commHash;
uint8_t func;
void* buff;
uint8_t datatype;
size_t count;
int peer;
} p2p;
struct {
pid_t pid; // pid of the originating process
uint8_t channelId; // channel id for this proxy operation
int peer; // remote rank for send/recv
int nSteps; // number of steps for this proxy operation
int chunkSize; // amount of data transferred by this proxy operation
int isSend;
} proxyOp;
struct {
int step;
} proxyStep;
};
} ncclProfilerEventDescr_v1_t;
typedef union {
struct {
size_t transSize;
int steps;
} proxyOp;
struct {
int appendedProxyOps;
} proxyCtrl;
} ncclProfilerEventStateArgs_v1_t;
typedef struct {
const char* name;
// init - initialize the profiler plugin
// Input
// - context : opaque profiler context object for separating profiler behavior across comms
// Output
// - eActivationMask: bitmask of active events set by the plugin
ncclResult_t (*init)(void** context, int* eActivationMask);
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
// Input
// - context: opaque profiler context object
// - eDescr : pointer to ncclProfilerEventDescr_t object
// Output
// - eHandle: return event handle for supplied event descriptor object
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr);
// stopEvent - stop/finalize an event inside and event set
// Input
// - eHandle: handle to event object
ncclResult_t (*stopEvent)(void* eHandle);
// recordEventState - record event state transitions and event attribute updates
// Input
// - eHandle : handle to event object created through startEvent
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
// - eState : event state transition
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs);
// finalize - finalize the profiler plugin
// Input
// - context: opaque profiler context object
ncclResult_t (*finalize)(void* context);
} ncclProfiler_v1_t;
#endif
src/include/plugin/profiler/profiler_v2.h
+104
Просмотреть файл
@@ -0,0 +1,104 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef PROFILER_V2_H_
#define PROFILER_V2_H_
typedef struct {
uint8_t type; // event type descriptor: ncclProfileColl, ...
void* parentObj; // pointer to the profiler parent object (for coll is the group)
int rank; // originating rank
union {
struct {
const char* name;
uint64_t commHash;
uint64_t seqNumber;
const char* func;
void const* sendBuff;
void* recvBuff;
size_t count;
int root;
const char* datatype;
size_t trafficBytes;
uint8_t nMaxChannels;
uint8_t nWarps;
const char* algo;
const char* proto;
} coll;
struct {
const char* name;
uint64_t commHash;
const char* func;
void* buff;
const char* datatype;
size_t count;
int peer;
} p2p;
struct {
pid_t pid; // pid of the originating process
uint8_t channelId; // channel id for this proxy operation
int peer; // remote rank for send/recv
int nSteps; // number of steps for this proxy operation
int chunkSize; // amount of data transferred by this proxy operation
int isSend;
} proxyOp;
struct {
int step;
} proxyStep;
};
} ncclProfilerEventDescr_v2_t;
typedef union {
struct {
size_t transSize;
int steps;
} proxyOp;
struct {
int appendedProxyOps;
} proxyCtrl;
} ncclProfilerEventStateArgs_v2_t;
typedef struct {
const char* name;
// init - initialize the profiler plugin
// Input
// - context : opaque profiler context object for separating profiler behavior across comms
// Output
// - eActivationMask: bitmask of active events set by the plugin
ncclResult_t (*init)(void** context, int* eActivationMask);
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
// Input
// - context: opaque profiler context object
// - eDescr : pointer to ncclProfilerEventDescr_t object
// Output
// - eHandle: return event handle for supplied event descriptor object
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr);
// stopEvent - stop/finalize an event inside and event set
// Input
// - eHandle: handle to event object
ncclResult_t (*stopEvent)(void* eHandle);
// recordEventState - record event state transitions and event attribute updates
// Input
// - eHandle : handle to event object created through startEvent
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
// - eState : event state transition
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v2_t eState, ncclProfilerEventStateArgs_v2_t* eStateArgs);
// finalize - finalize the profiler plugin
// Input
// - context: opaque profiler context object
ncclResult_t (*finalize)(void* context);
} ncclProfiler_v2_t;
#endif
src/include/plugin/profiler/profiler_v3.h
+112
Просмотреть файл
@@ -0,0 +1,112 @@
/*************************************************************************
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef PROFILER_V3_H_
#define PROFILER_V3_H_
typedef struct {
uint8_t type; // event type descriptor: ncclProfileColl, ...
void* parentObj; // pointer to the profiler parent object (for coll is the group)
int rank; // originating rank
union {
struct {
const char* name;
uint64_t commHash;
uint64_t seqNumber;
const char* func;
void const* sendBuff;
void* recvBuff;
size_t count;
int root;
const char* datatype;
uint8_t nMaxChannels;
uint8_t nWarps;
const char* algo;
const char* proto;
} coll;
struct {
const char* name;
uint64_t commHash;
const char* func;
void* buff;
const char* datatype;
size_t count;
int peer;
} p2p;
struct {
pid_t pid; // pid of the originating process
uint8_t channelId; // channel id for this proxy operation
int peer; // remote rank for send/recv
int nSteps; // number of steps for this proxy operation
int chunkSize; // amount of data transferred by this proxy operation
int isSend;
} proxyOp;
struct {
int step;
} proxyStep;
struct {
uint8_t channelId;
} kernelCh;
struct {
int64_t id;
void* data;
} netPlugin;
};
} ncclProfilerEventDescr_v3_t;
typedef union {
struct {
size_t transSize;
int steps;
} proxyOp;
struct {
int appendedProxyOps;
} proxyCtrl;
} ncclProfilerEventStateArgs_v3_t;
typedef struct {
const char* name;
// init - initialize the profiler plugin
// Input
// - context : opaque profiler context object for separating profiler behavior across comms
// Output
// - eActivationMask: bitmask of active events set by the plugin
ncclResult_t (*init)(void** context, int* eActivationMask);
// startEvent - initialize and start a new event for the supplied event descriptor inside the eventset
// Input
// - context: opaque profiler context object
// - eDescr : pointer to ncclProfilerEventDescr_t object
// Output
// - eHandle: return event handle for supplied event descriptor object
ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v3_t* eDescr);
// stopEvent - stop/finalize an event inside and event set
// Input
// - eHandle: handle to event object
ncclResult_t (*stopEvent)(void* eHandle);
// recordEventState - record event state transitions and event attribute updates
// Input
// - eHandle : handle to event object created through startEvent
// - eStateArgs: optional argument used to capture event attribute updates associated with the state transition
// - eState : event state transition
ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v3_t eState, ncclProfilerEventStateArgs_v3_t* eStateArgs);
// finalize - finalize the profiler plugin
// Input
// - context: opaque profiler context object
ncclResult_t (*finalize)(void* context);
} ncclProfiler_v3_t;
#endif
src/include/plugin/tuner/tuner_v2.h
+53
Просмотреть файл
@@ -0,0 +1,53 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef TUNER_V2_H_
#define TUNER_V2_H_
// API to be implemented by external tuner
typedef struct {
// Name of the tuner
const char* name;
// Initializes tuner states.
// Inputs:
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
// - nNodes: number of nodes in current communicator.
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
// Outputs:
// - context: tuner context object
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
// Inputs:
// - context: tuner context object
// - collType: collective type , e.g., allreduce, allgather…
// - nBytes: collective size in bytes
// - collNetTypeSupport: whether collnet supports this type
// - nvlsTypeSupport: whether nvlink sharp supports this time
// - numPipeOps: number of operations in the group
//
// Outputs:
// - algorithm: selected algorithm to be used for the given collective
// - protocol: selected protocol to be used for the give collective
// - nChannels: number of channels (hence SMs) to be used.
//
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
// default tuning for the given collective.
// Also, the plugin is allowed to not set any output, or set only the
// algorithm and protocol, but not only the algorithm or only the protocol.
// Unset fields will be set automatically by NCCL.
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
int collNetSupport, int nvlsSupport, int numPipeOps,
int* algorithm, int* protocol, int* nChannels);
// Terminates the plugin and cleans up any resources that the plugin allocated.
// context: tuner context object
ncclResult_t (*destroy)(void* context);
} ncclTuner_v2_t;
#endif
src/include/plugin/tuner/tuner_v3.h
+55
Просмотреть файл
@@ -0,0 +1,55 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef TUNER_V3_H_
#define TUNER_V3_H_
// API to be implemented by external tuner
typedef struct {
// Name of the tuner
const char* name;
// Initializes tuner states.
// Inputs:
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
// - nNodes: number of nodes in current communicator.
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
// Outputs:
// - context: tuner context object
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
// Inputs:
// - context: tuner context object
// - collType: collective type , e.g., allreduce, allgather…
// - nBytes: collective size in bytes
// - numPipeOps: number of operations in the group
// - numAlgo: number of algorithms in collCostTable
// - numProto: number of protocols in collCostTable
//
// Outputs:
// - nChannels: number of channels (hence SMs) to be used.
//
// InOut:
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
//
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
// default tuning for the given collective.
// Also, the plugin is allowed to not set any output, or set only the
// algorithm and protocol, but not only the algorithm or only the protocol.
// Unset fields will be set automatically by NCCL.
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
int* nChannels);
// Terminates the plugin and cleans up any resources that the plugin allocated.
// context: tuner context object
ncclResult_t (*destroy)(void* context);
} ncclTuner_v3_t;
#endif
src/include/plugin/tuner/tuner_v4.h
+56
Просмотреть файл
@@ -0,0 +1,56 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef TUNER_V4_H_
#define TUNER_V4_H_
// API to be implemented by external tuner
typedef struct {
// Name of the tuner
const char* name;
// Initializes tuner states.
// Inputs:
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
// - nNodes: number of nodes in current communicator.
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
// Outputs:
// - context: tuner context object
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
// Inputs:
// - context: tuner context object
// - collType: collective type , e.g., allreduce, allgather…
// - nBytes: collective size in bytes
// - numPipeOps: number of operations in the group
// - numAlgo: number of algorithms in collCostTable
// - numProto: number of protocols in collCostTable
// - regBuff: can register user buffer
//
// Outputs:
// - nChannels: number of channels (hence SMs) to be used.
//
// InOut:
// - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType.
// NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE).
//
// If getCollInfo() does not return ncclSuccess, NCCL will fall back to the
// default tuning for the given collective.
// Also, the plugin is allowed to not set any output, or set only the
// algorithm and protocol, but not only the algorithm or only the protocol.
// Unset fields will be set automatically by NCCL.
ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes,
int numPipeOps, float** collCostTable, int numAlgo, int numProto,
int regBuff, int* nChannels);
// Terminates the plugin and cleans up any resources that the plugin allocated.
// context: tuner context object
ncclResult_t (*destroy)(void* context);
} ncclTuner_v4_t;
#endif
src/include/profiler.h
+21
Просмотреть файл
@@ -17,6 +17,18 @@ struct ncclTaskP2p;
struct ncclInfo;
struct ncclComm;
struct ncclProxyOp;
struct ncclProxyConnector;
struct ncclProfilerProxy {
bool initialized;
uint64_t* workStarted/*[MAXCHANNELS]*/;
uint64_t* workCompleted/*[MAXCHANNELS]*/;
uint64_t workCounter[MAXCHANNELS]; // host work counter
struct ncclProxyConnector sendProxyConn[MAXCHANNELS];
struct ncclProxyConnector recvProxyConn[MAXCHANNELS];
};
extern int ncclProfilerEventMask;
// Plugin Init/Finalize Wrappers
ncclResult_t ncclProfilerPluginInit(struct ncclComm* comm);
@@ -44,6 +56,10 @@ ncclResult_t ncclProfilerStopProxyStepEvent(int sub, struct ncclProxyArgs* args,
ncclResult_t ncclProfilerStartProxyCtrlEvent(void* profilerContext, void** eHandle);
ncclResult_t ncclProfilerStopProxyCtrlEvent(void* eHandle);
// Kernel Channel Start/Stop Event Wrappers
ncclResult_t ncclProfilerStartKernelChEvent(struct ncclProxyArgs* args, int s);
ncclResult_t ncclProfilerStopKernelChEvent(struct ncclProxyArgs* args, int s);
// Record Event Wrappers
ncclResult_t ncclProfilerRecordProxyOpEventState(int sub, struct ncclProxyArgs* args, int steps, size_t transSize, ncclProfilerEventState_t eState);
ncclResult_t ncclProfilerRecordProxyStepEventState(int sub, struct ncclProxyArgs* args, int stepId, ncclProfilerEventState_t eState);
@@ -51,5 +67,10 @@ ncclResult_t ncclProfilerRecordProxyCtrlEventState(void*eHandle, int appended, n
// Profiler utility functions
ncclResult_t ncclProfilerAddPidToProxyOp(struct ncclProxyOp* op);
bool ncclProfilerNeedsProxy(struct ncclComm* comm, struct ncclProxyOp* op);
bool ncclProfilerPluginLoaded(void);
// Profiler callback for network plugin
ncclResult_t ncclProfilerCallback(void** eHandle, int type, void* pHandle, int64_t pluginId, void* extData);
#endif
src/include/proxy.h
+12 -1
Просмотреть файл
@@ -34,7 +34,8 @@ typedef enum : uint8_t {
ncclPatternPatUp,
ncclPatternPatDown,
ncclPatternSend,
ncclPatternRecv
ncclPatternRecv,
ncclPatternProfiler,
} ncclPattern_t;
enum ncclProxyOpState { ncclProxyOpNone, ncclProxyOpReady, ncclProxyOpProgress };
@@ -93,12 +94,19 @@ struct ncclProxyOp {
struct ncclTaskP2p* p2p;
} task;
// Profiler work counter increment flag. Set to 'true' if the profiler work counter for this channel needs increment.
// Always 'true' for collective operations. Grouped p2p operations are fused into one <send, recv> pair in the GPU kernel,
// meaning the GPU profiler code increments the work counter for the pair rather than the individual p2p. For this
// reason, the incWorkCounter flag is used to avoid incrementing the work counter twice in the host code. This is done
// by setting incWorkCounter to 'true' only for one of the p2ps in the pair during enqueue.
bool incWorkCounter;
int eActivationMask;
void* taskEventHandle;
int rank;
int peer;
pid_t pid;
void* profilerContext;
uint64_t workCounter;
struct ncclProxyOp *enqNext;
};
@@ -135,12 +143,15 @@ struct ncclProxySubArgs {
// Profiler plugin
int eActivationMask;
int rank;
uint64_t profilerSteps;
pid_t pid;
void* profilerContext;
void* taskEventHandle;
void* opEventHandle;
void* kernelEventHandle;
void* stepEventHandles[NCCL_STEPS];
size_t transSize;
uint64_t workCounter;
void* recvRequestsCache[NCCL_STEPS];
int recvRequestsSubCount;
src/include/ras.h
+2
Просмотреть файл
@@ -15,6 +15,8 @@ struct rasRankInit {
pid_t pid;
int cudaDev;
int nvmlDev;
uint64_t hostHash;
uint64_t pidHash;
};
ncclResult_t ncclRasCommInit(struct ncclComm* comm, struct rasRankInit* myRank);
src/include/register.h
+1 -1
Просмотреть файл
@@ -42,7 +42,7 @@ struct ncclReg {
uintptr_t baseAddr;
size_t baseSize;
CUdeviceptr regAddr;
size_t regSize;
size_t regUCSize, regMCSize;
int dev;
CUmemGenericAllocationHandle mcHandle;
uintptr_t caddrs[NCCL_MAX_LOCAL_RANKS]; /* use to check if NVLS buffers match among intra-node ranks */
src/include/shm.h
+2 -3
Просмотреть файл
@@ -14,7 +14,6 @@ struct shmCuIpc {
CUmemFabricHandle handle;
CUmemGenericAllocationHandle data;
};
int tpProxyRank;
void *ptr;
size_t size;
};
@@ -30,8 +29,8 @@ struct shmIpcDesc {
typedef struct shmIpcDesc ncclShmIpcDesc_t;
ncclResult_t ncclShmAllocateShareableBuffer(int tpProxyRank, size_t size, bool legacy, ncclShmIpcDesc_t *descOut, void **hptr, void **dptr);
ncclResult_t ncclShmImportShareableBuffer(struct ncclComm *comm, ncclShmIpcDesc_t *desc, void **hptr, void **dptr, ncclShmIpcDesc_t *descOut);
ncclResult_t ncclShmAllocateShareableBuffer(size_t size, bool legacy, ncclShmIpcDesc_t *descOut, void **hptr, void **dptr);
ncclResult_t ncclShmImportShareableBuffer(struct ncclComm *comm, int proxyRank, ncclShmIpcDesc_t *desc, void **hptr, void **dptr, ncclShmIpcDesc_t *descOut);
ncclResult_t ncclShmIpcClose(ncclShmIpcDesc_t *desc);
#endif
src/include/socket.h
+1 -1
Просмотреть файл
@@ -96,5 +96,5 @@ ncclResult_t ncclSocketRecv(struct ncclSocket* sock, void* ptr, int size);
ncclResult_t ncclSocketSendRecv(struct ncclSocket* sendSock, void* sendPtr, int sendSize, struct ncclSocket* recvSock, void* recvPtr, int recvSize);
ncclResult_t ncclSocketTryRecv(struct ncclSocket* sock, void* ptr, int size, int* closed, bool blocking);
ncclResult_t ncclSocketShutdown(struct ncclSocket* sock, int how);
ncclResult_t ncclSocketClose(struct ncclSocket* sock);
ncclResult_t ncclSocketClose(struct ncclSocket* sock, bool wait = false);
#endif
src/include/strongstream.h
+49 -52
Просмотреть файл
@@ -10,13 +10,24 @@
#include "nccl.h"
#include "checks.h"
#include <cuda.h>
#include <cuda_runtime.h>
#include <stdint.h>
// ncclCudaContext: wraps a CUDA context with per-context state.
struct ncclCudaContext;
// Get a ncclCudaContext to track the currently active CUDA context.
ncclResult_t ncclCudaContextTrack(struct ncclCudaContext** out);
// Drop reference.
void ncclCudaContextDrop(struct ncclCudaContext* cxt);
/* ncclCudaGraph: Wraps a cudaGraph_t so that we can support pre-graph CUDA runtimes
* easily.
*/
struct ncclCudaGraph {
#if ROCM_VERSION >= 60100
cudaStream_t origin;
cudaGraph_t graph;
unsigned long long graphId;
#endif
@@ -25,6 +36,7 @@ struct ncclCudaGraph {
inline struct ncclCudaGraph ncclCudaGraphNone() {
struct ncclCudaGraph tmp;
#if ROCM_VERSION >= 60100
tmp.origin = nullptr;
tmp.graph = nullptr;
tmp.graphId = ULLONG_MAX;
#endif
@@ -33,7 +45,7 @@ inline struct ncclCudaGraph ncclCudaGraphNone() {
inline bool ncclCudaGraphValid(struct ncclCudaGraph graph) {
#if ROCM_VERSION >= 60100
return graph.graph != nullptr;
return graph.graphId != ULLONG_MAX;
#else
return false;
#endif
@@ -57,84 +69,69 @@ ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, cudaHostFn_t
* streams unfit for the use of serializing access to a persistent resource.
* Strong streams have been introduced to address this need.
*
* - All updates to a strong stream must be enclosed by a Acquire/Release pair.
* All updates to a strong stream must be enclosed by a Acquire/Release pair.
*
* - The Acquire, Release, and all updates take a ncclCudaGraph parameter
* indicating the currently capturing graph (or none). This parameter must be
* the same for the entire sequence of {Acquire; ...; Release}.
* Acquire retrieves a "work" stream (cudaStream_t) which may be used to add
* work.
*
* - An {Acquire; ...; Release} sequence must not be concurrent with any
* other operations against the strong stream including graph launches which
* reference this stream.
* Release publishes the work streams work into the strong stream. The Release
* must be issued by the same thread that did the Acquire.
*/
struct ncclStrongStream;
ncclResult_t ncclStrongStreamConstruct(struct ncclStrongStream* ss);
ncclResult_t ncclStrongStreamDestruct(struct ncclStrongStream* ss);
// Acquire-fence the strong stream.
// Acquire the strong stream. Upon return `*workStream` will be usable to add work.
// `concurrent` indicates if other threads may be using the strong stream.
ncclResult_t ncclStrongStreamAcquire(
struct ncclCudaGraph graph, struct ncclStrongStream* ss
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent, cudaStream_t* workStream
);
// Acquire-fence the strong stream assuming no graph is capturing. This permits
// the caller to enqueue directly to the `ss->cudaStream` member using native CUDA
// calls. Strong stream still must be released via:
// ncclStrongStreamRelease(ncclCudaGraphNone(), ss);
ncclResult_t ncclStrongStreamAcquireUncaptured(struct ncclStrongStream* ss);
// Release-fence of the strong stream.
ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStrongStream* ss);
// Add a host launch to the stream.
ncclResult_t ncclStrongStreamLaunchHost(
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
cudaHostFn_t fn, void* arg
);
// Add a kernel launch to the stream.
ncclResult_t ncclStrongStreamLaunchKernel(
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
void* fn, dim3 grid, dim3 block, void** args, size_t sharedMemBytes
// Get the workStream for an already acquired strong stream.
// `concurrent` indicates if other threads may be using the strong stream.
ncclResult_t ncclStrongStreamAcquiredWorkStream(
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent, cudaStream_t* workStream
);
// Cause `a` to wait for the current state `b`. Both `a` and `b` must be acquired.
// `b_subsumes_a` indicates that all work in `a` is already present in `b`, thus
// we want to fast-forward `a` to be a clone of `b`. Knowing this permits the
// implementation to induce few graph dependencies.
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, struct ncclStrongStream* b, bool b_subsumes_a=false
);
// `b` must be capturing within `graph`.
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, cudaStream_t b, bool b_subsumes_a=false
);
// `a` must be capturing within `graph`.
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, cudaStream_t a, struct ncclStrongStream* b, bool b_subsumes_a=false
// Release of the strong stream.
// `concurrent` indicates if other threads may be using the strong stream.
ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent);
ncclResult_t ncclStreamWaitStream(
cudaStream_t a, cudaStream_t b, cudaEvent_t scratchEvent
);
// Like cudaStreamWaitEvent except `e` must be strictly ahead of everything in `s`.
ncclResult_t ncclStreamAdvanceToEvent(struct ncclCudaGraph g, cudaStream_t s, cudaEvent_t e);
// Synchrnoization does not need the strong stream to be acquired.
ncclResult_t ncclStrongStreamSynchronize(struct ncclStrongStream* ss);
////////////////////////////////////////////////////////////////////////////////
struct ncclStrongStreamGraph; // internal to ncclStrongStream
struct ncclStrongStreamCapture; // internal to ncclStrongStream
struct ncclStrongStream {
// Used when not graph capturing.
cudaStream_t cudaStream;
// The stream to use for non-captured work.
cudaStream_t liveStream;
void* liveAcquiredBy;
#if ROCM_VERSION >= 60100
// This stream ever appeared in a graph capture.
bool everCaptured;
pthread_mutex_t lock;
struct ncclStrongStreamCapture* captureHead;
// The event used to establish order between graphs and streams. During acquire
// this event is waited on, during release it is recorded to.
cudaEvent_t serialEvent;
// This stream ever appeared in a graph capture.
bool everCaptured;
// Tracks whether serialEvent needs to be recorded to upon Release().
bool serialEventNeedsRecord;
struct ncclStrongStreamGraph* graphHead;
#else
cudaEvent_t scratchEvent;
#endif
};
struct ncclCudaContext {
struct ncclCudaContext* next;
CUcontext hcontext;
int refCount;
struct ncclStrongStream launchOrder;
};
#endif
src/include/transport.h
+7 -3
Просмотреть файл
@@ -19,6 +19,7 @@
#define TRANSPORT_SHM 1
#define TRANSPORT_NET 2
#define TRANSPORT_COLLNET 3
#define TRANSPORT_PROFILER 4
#include "proxy.h"
#include "comm.h"
@@ -27,6 +28,7 @@ extern struct ncclTransport p2pTransport;
extern struct ncclTransport shmTransport;
extern struct ncclTransport netTransport;
extern struct ncclTransport collNetTransport;
extern struct ncclTransport profilerTransport;
extern struct ncclTransport* ncclTransports[];
// Forward declarations
@@ -50,8 +52,10 @@ struct ncclNvlsSharedRes {
CUmulticastObjectProp signalProp;
CUmemAccessDesc accessDesc;
int dev;
size_t buffSize;
size_t creditSize;
size_t creditUCSize;
size_t creditMCSize;
size_t buffUCSize;
size_t buffMCSize;
CUmemGenericAllocationHandle mcBuffHandle; // Multicast handle for NVLS buffer
CUmemGenericAllocationHandle mcCreditHandle; // Multicast handle for NVLS credit buffer
char* mcBuff; // Multicast NVLS buffer address
@@ -108,7 +112,7 @@ ncclResult_t ncclNvlsBufferSetup(struct ncclComm* comm);
ncclResult_t ncclNvlsTreeConnect(struct ncclComm* comm);
ncclResult_t ncclNvlsGraphRegisterBuffer(struct ncclComm *comm, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, int *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv, struct ncclIntruQueue<struct ncclCommCallback, &ncclCommCallback::next>* cleanupQueue, int* nCleanupQueueElts);
ncclResult_t ncclNvlsLocalRegisterBuffer(struct ncclComm *comm, const void *sendbuff, void *recvbuff, size_t sendbuffSize, size_t recvbuffSize, int *outRegBufUsed, void **outRegBufSend, void **outRegBufRecv);
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t size);
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t ucsize, size_t mcsize);
ncclResult_t ncclNvlsFree(struct ncclComm* comm);
enum { collNetRecv=0, collNetSend=1 };
src/init.cc
+76 -73
Просмотреть файл
@@ -87,17 +87,6 @@ NCCL_PARAM(RuntimeConnect, "RUNTIME_CONNECT", 1);
struct allocationTracker allocTracker[MAX_ALLOC_TRACK_NGPU] = {};
static ncclResult_t commReclaim(ncclComm_t comm);
static uint64_t hashUniqueId(ncclUniqueId const &id) {
char const *bytes = (char const*)&id;
uint64_t h = 0xdeadbeef;
for(int i=0; i < (int)sizeof(ncclUniqueId); i++) {
h ^= h >> 32;
h *= 0x8db3db47fa2994ad;
h += bytes[i];
}
return h;
}
//RCCL runtime param to set Unroll Factor
RCCL_PARAM(UnrollFactor, "UNROLL_FACTOR", 0);
@@ -131,7 +120,7 @@ ncclResult_t commSetUnrollFactor(struct ncclComm* comm) {
#ifdef ENABLE_MSCCLPP
size_t std::hash<ncclUniqueId>::operator ()(const ncclUniqueId& uniqueId) const noexcept {
return (size_t)hashUniqueId(uniqueId);
return (size_t)getHash(uniqueId.internal, NCCL_UNIQUE_ID_BYTES);
}
bool operator ==(const ncclUniqueId& a, const ncclUniqueId& b) {
@@ -237,7 +226,7 @@ ncclResult_t ncclGetUniqueId_impl(ncclUniqueId* out) {
// copy to avoid alignment mismatch
memcpy(out, &handle, sizeof(handle));
Recorder::instance().record(rrGetUniqueId, -1, -1, out);
TRACE_CALL("ncclGetUniqueId(0x%llx)", (unsigned long long)hashUniqueId(*out));
TRACE_CALL("ncclGetUniqueId(0x%llx)", (unsigned long long)getHash(out->internal, NCCL_UNIQUE_ID_BYTES));
return ncclSuccess;
}
@@ -485,6 +474,8 @@ static ncclResult_t commFree(ncclComm_t comm) {
free(comm->sharedRes->tpRankToLocalRank);
NCCLCHECK(ncclStrongStreamDestruct(&comm->sharedRes->hostStream));
NCCLCHECK(ncclStrongStreamDestruct(&comm->sharedRes->deviceStream));
CUDACHECK(cudaEventDestroy(comm->sharedRes->launchEvent));
CUDACHECK(cudaEventDestroy(comm->sharedRes->scratchEvent));
NCCLCHECK(ncclProxyDestroy(comm));
free(comm->sharedRes);
}
@@ -524,6 +515,9 @@ static ncclResult_t commFree(ncclComm_t comm) {
NCCLCHECK(ncclProfilerPluginFinalize(comm));
NCCLCHECK(ncclNetFinalize(comm));
NCCLCHECK(ncclNetPluginUnload(comm));
ncclCudaContextDrop(comm->context);
free(comm);
return ncclSuccess;
@@ -570,17 +564,12 @@ ncclResult_t ncclCommEnsureReady(ncclComm_t comm) {
ncclGroupJobAbort(comm->groupJob);
} else {
NCCLCHECK(ncclCommGetAsyncError(comm, &ret));
if (ret != ncclSuccess) {
/* if ret is not ncclInProgress, we just keep it. */
if (ret == ncclInProgress) {
WARN("Attempt to use communicator before the previous operation returned ncclSuccess");
if (ret == ncclInProgress) ret = ncclInvalidArgument;
ret = ncclInvalidArgument;
goto exit;
}
/* if there is linked group job, we should complete it. */
if (comm->groupJob) {
NCCLCHECK(ncclGroupJobComplete(comm->groupJob));
comm->groupJob = NULL;
}
/* if ret is not ncclInProgress, we just keep it. */
}
exit:
@@ -625,6 +614,8 @@ static ncclResult_t commAlloc(struct ncclComm* comm, struct ncclComm* parent, in
comm->lastStream = nullptr;
CUDACHECK(cudaGetDevice(&comm->cudaDev));
NCCLCHECK(ncclCudaContextTrack(&comm->context));
NCCLCHECK(getBusId(comm->cudaDev, &comm->busId));
char busId[]="0000:00:00.0";
NCCLCHECK(int64ToBusId(comm->busId, busId));
@@ -688,6 +679,8 @@ static ncclResult_t commAlloc(struct ncclComm* comm, struct ncclComm* parent, in
NCCLCHECK(ncclCalloc(&sharedRes->tpRankToLocalRank, comm->nRanks));
NCCLCHECK(ncclStrongStreamConstruct(&sharedRes->deviceStream));
NCCLCHECK(ncclStrongStreamConstruct(&sharedRes->hostStream));
CUDACHECK(cudaEventCreateWithFlags(&sharedRes->launchEvent, cudaEventDisableTiming));
CUDACHECK(cudaEventCreateWithFlags(&sharedRes->scratchEvent, cudaEventDisableTiming));
comm->sharedRes = sharedRes;
sharedRes->refCount = 1;
} else {
@@ -730,13 +723,14 @@ static ncclResult_t devCommSetup(ncclComm_t comm) {
struct ncclDevCommAndChannels *devCommAndChans = NULL;
struct ncclNvmlCCStatus ccStatus;
bool ccEnable = false;
cudaStream_t deviceStream;
NCCLCHECKGOTO(ncclStrongStreamAcquireUncaptured(&comm->sharedRes->deviceStream), ret, fail);
NCCLCHECKGOTO(ncclCudaCallocAsync(&devCommAndChans, 1, comm->sharedRes->deviceStream.cudaStream), ret, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), ret, fail);
NCCLCHECKGOTO(ncclCudaCallocAsync(&devCommAndChans, 1, deviceStream), ret, fail);
ncclCommPushCudaFree(comm, devCommAndChans);
NCCLCHECKGOTO(ncclCudaCallocAsync(&tmpCommAndChans.comm.rankToLocalRank, comm->nRanks, comm->sharedRes->deviceStream.cudaStream), ret, fail);
NCCLCHECKGOTO(ncclCudaCallocAsync(&tmpCommAndChans.comm.rankToLocalRank, comm->nRanks, deviceStream), ret, fail);
ncclCommPushCudaFree(comm, tmpCommAndChans.comm.rankToLocalRank);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(tmpCommAndChans.comm.rankToLocalRank, comm->rankToLocalRank, comm->nRanks, comm->sharedRes->deviceStream.cudaStream), ret, fail);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(tmpCommAndChans.comm.rankToLocalRank, comm->rankToLocalRank, comm->nRanks, deviceStream), ret, fail);
comm->devComm = &devCommAndChans->comm;
tmpCommAndChans.comm.rank = comm->rank;
tmpCommAndChans.comm.nRanks = nRanks;
@@ -759,12 +753,22 @@ static ncclResult_t devCommSetup(ncclComm_t comm) {
if (ccEnable) {
comm->workFifoBytes = 0;
} else {
comm->workFifoBytes = ncclParamWorkFifoBytes();
if (0 != (comm->workFifoBytes & (comm->workFifoBytes-1))) {
WARN("NCCL_WORK_FIFO_BYTES=%d is being ignored because it is not a power of 2.", comm->workFifoBytes);
comm->workFifoBytes = NCCL_WORK_FIFO_BYTES_DEFAULT;
int64_t workFifoBytesParam = ncclParamWorkFifoBytes();
if (workFifoBytesParam == -1) {
if (comm->MNNVL && (comm->compCap >= 100)) {
// WAR: Disable work fifo for Blackwell all2all hang issue on MNNVL
INFO(NCCL_INIT, "Disabling work fifo");
comm->workFifoBytes = 0;
} else {
comm->workFifoBytes = NCCL_WORK_FIFO_BYTES_DEFAULT;
}
} else {
if (0 != (workFifoBytesParam & (workFifoBytesParam-1))) {
WARN("NCCL_WORK_FIFO_BYTES=%ld is being ignored because it is not a power of 2.", workFifoBytesParam);
comm->workFifoBytes = NCCL_WORK_FIFO_BYTES_DEFAULT;
}
comm->workFifoBytes = std::min<uint64_t>(workFifoBytesParam, 1ul<<30);
}
comm->workFifoBytes = std::min(comm->workFifoBytes, 1u<<30);
}
#else
comm->workFifoBytes = ncclParamWorkFifoBytes();
@@ -797,10 +801,18 @@ static ncclResult_t devCommSetup(ncclComm_t comm) {
comm->workFifoConsumedLeast = 0;
tmpCommAndChans.comm.workConsumed = comm->workFifoConsumed;
// Alloc profiler counters for the kernel
NCCLCHECKGOTO(ncclCudaHostCalloc(&comm->profiler.workStarted, MAXCHANNELS), ret, fail);
NCCLCHECKGOTO(ncclCudaHostCalloc(&comm->profiler.workCompleted, MAXCHANNELS), ret, fail);
tmpCommAndChans.comm.workStarted = comm->profiler.workStarted;
tmpCommAndChans.comm.workCompleted = comm->profiler.workCompleted;
ncclCommPushCudaHostFree(comm, comm->profiler.workStarted);
ncclCommPushCudaHostFree(comm, comm->profiler.workCompleted);
if (comm->collNetDenseToUserRank != nullptr) {
NCCLCHECKGOTO(ncclCudaCallocAsync(&tmpCommAndChans.comm.collNetDenseToUserRank, nRanks, comm->sharedRes->deviceStream.cudaStream), ret, fail);
NCCLCHECKGOTO(ncclCudaCallocAsync(&tmpCommAndChans.comm.collNetDenseToUserRank, nRanks, deviceStream), ret, fail);
ncclCommPushCudaFree(comm, tmpCommAndChans.comm.collNetDenseToUserRank);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(tmpCommAndChans.comm.collNetDenseToUserRank, comm->collNetDenseToUserRank, nRanks, comm->sharedRes->deviceStream.cudaStream), ret, fail);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(tmpCommAndChans.comm.collNetDenseToUserRank, comm->collNetDenseToUserRank, nRanks, deviceStream), ret, fail);
}
for (int c=0; c < MAXCHANNELS; c++) {
@@ -814,7 +826,7 @@ static ncclResult_t devCommSetup(ncclComm_t comm) {
tmpCommAndChans.channels[c].nvls = comm->channels[c].nvls;
if (comm->channels[c].ring.userRanks != nullptr) {
NCCLCHECKGOTO(ncclCudaMemcpyAsync(tmpCommAndChans.channels[c].ring.userRanks, comm->channels[c].ring.userRanks, nRanks, comm->sharedRes->deviceStream.cudaStream), ret, fail);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(tmpCommAndChans.channels[c].ring.userRanks, comm->channels[c].ring.userRanks, nRanks, deviceStream), ret, fail);
}
}
@@ -839,10 +851,10 @@ static ncclResult_t devCommSetup(ncclComm_t comm) {
tmpCommAndChans.comm.faults = comm->faults;
#endif
NCCLCHECKGOTO(ncclCudaMemcpyAsync(devCommAndChans, &tmpCommAndChans, 1, comm->sharedRes->deviceStream.cudaStream), ret, fail);
NCCLCHECKGOTO(ncclCudaMemcpyAsync(devCommAndChans, &tmpCommAndChans, 1, deviceStream), ret, fail);
exit:
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false));
NCCLCHECK(ncclStrongStreamSynchronize(&comm->sharedRes->deviceStream));
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->deviceStream));
return ret;
fail:
goto exit;
@@ -1507,6 +1519,7 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, struct ncclComm* p
graphs[a]->typeInter = std::max(allGather3Data[i].graphInfo[a].typeInter, graphs[a]->typeInter);
graphs[a]->crossNic = std::max(allGather3Data[i].graphInfo[a].crossNic, graphs[a]->crossNic);
}
comm->maxTreePattern = std::max(comm->maxTreePattern, allGather3Data[i].graphInfo[NCCL_ALGO_TREE].pattern);
}
if (graphs[NCCL_ALGO_COLLNET_CHAIN]->nChannels == 0) comm->collNetSupport = 0;
if (graphs[NCCL_ALGO_NVLS]->nChannels == 0) comm->nvlsSupport = comm->nvlsChannels = 0;
@@ -1932,12 +1945,12 @@ static ncclResult_t ncclCommInitRankFunc(struct ncclAsyncJob* job_) {
timers[TIMER_INIT_ALLOC] = clockNano();
NCCLCHECKGOTO(commAlloc(comm, job->parent, job->nranks, job->myrank), res, fail);
timers[TIMER_INIT_ALLOC] = clockNano() - timers[TIMER_INIT_ALLOC];
// obtain a unique hash for the comm, re-using part of the parent's hash, commHash is a 64bit struct (=16 hex),
// add unique split counter and the color
ncclUniqueId tmpId;
memset(&tmpId,0,sizeof(ncclUniqueId));// must set 0 here to avoid undefined bits
snprintf((char*)&tmpId, NCCL_UNIQUE_ID_BYTES, "%016lx-%d-%d", job->parent->commHash, job->splitCount, job->color);
comm->commHash = getHash(tmpId.internal, NCCL_UNIQUE_ID_BYTES);
// child hash obtained from (parent hash, split count, color)
uint64_t hacc[2] = {1, 1};
eatHash(hacc, &job->parent->commHash);
eatHash(hacc, &job->splitCount);
eatHash(hacc, &job->color);
comm->commHash = digestHash(hacc);
INFO(NCCL_INIT, "%s comm %p rank %d nranks %d cudaDev %d nvmlDev %d busId %lx parent %p splitCount %d color %d key %d- Init START", job->funcName,
comm, comm->rank, comm->nRanks, comm->cudaDev, comm->nvmlDev, comm->busId, job->parent, job->splitCount, job->color, job->key);
timers[TIMER_INIT_BOOTSTRAP] = clockNano();
@@ -1950,8 +1963,7 @@ static ncclResult_t ncclCommInitRankFunc(struct ncclAsyncJob* job_) {
NCCLCHECKGOTO(commAlloc(comm, NULL, job->nranks, job->myrank), res, fail);
timers[TIMER_INIT_ALLOC] = clockNano() - timers[TIMER_INIT_ALLOC];
// obtain a unique hash using the first commId
comm->commHash = getHash(job->commId->internal, NCCL_UNIQUE_ID_BYTES);
commIdHash = hashUniqueId(job->commId[0]);
comm->commHash = commIdHash = getHash(job->commId->internal, NCCL_UNIQUE_ID_BYTES);
INFO(NCCL_INIT, "%s comm %p rank %d nranks %d cudaDev %d nvmlDev %d busId %lx commId 0x%llx - Init START", job->funcName,
comm, comm->rank, comm->nRanks, comm->cudaDev, comm->nvmlDev, comm->busId, commIdHash);
timers[TIMER_INIT_BOOTSTRAP] = clockNano();
@@ -1991,12 +2003,12 @@ static ncclResult_t ncclCommInitRankFunc(struct ncclAsyncJob* job_) {
auto& mscclppUniqueId = mscclpp_uniqueIdMap[*job->commId];
if (comm->localRank == 0 && !mapContainsId) {
NCCLCHECKGOTO(mscclpp_ncclGetUniqueId(&mscclppUniqueId), res, fail);
TRACE_CALL("mscclpp_ncclGetUniqueId(0x%llx)", (unsigned long long)hashUniqueId(mscclppUniqueId));
TRACE_CALL("mscclpp_ncclGetUniqueId(0x%llx)", (unsigned long long)getHash(mscclppUniqueId.internal, NCCL_UNIQUE_ID_BYTES));
}
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, &mscclppUniqueId, sizeof(mscclppUniqueId)), res, fail);
unsigned long long mscclppUniqueIdHash; (void)mscclppUniqueIdHash;
TRACE_CALL("bootstrapIntraNodeBroadcast(rank=%d, nranks=%d, root=%d, bcastData=hash:0x%llx)", comm->localRank, comm->localRanks, 0, (mscclppUniqueIdHash = (unsigned long long)hashUniqueId(mscclppUniqueId)));
TRACE_CALL("bootstrapIntraNodeBroadcast(rank=%d, nranks=%d, root=%d, bcastData=hash:0x%llx)", comm->localRank, comm->localRanks, 0, (mscclppUniqueIdHash = (unsigned long long)getHash(mscclppUniqueId.internal, NCCL_UNIQUE_ID_BYTES)));
mscclpp_uniqueIdReverseMap[mscclppUniqueId].insert(*job->commId);
comm->mscclpp_threshold = rcclParamMscclppThreshold();
@@ -2228,6 +2240,7 @@ static ncclResult_t parseCommConfig(ncclComm_t comm, ncclConfig_t *config) {
NCCL_CONFIG_DEFAULT(internalConfigPtr, maxCTAs, NCCL_CONFIG_UNDEF_INT, MAXCHANNELS, "Max CTAs", "%d");
NCCL_CONFIG_DEFAULT(internalConfigPtr, netName, NCCL_CONFIG_UNDEF_PTR, NULL, "Net name", "%s");
NCCL_CONFIG_DEFAULT(internalConfigPtr, splitShare, NCCL_CONFIG_UNDEF_INT, 0, "Split share", "%d");
NCCL_CONFIG_DEFAULT(internalConfigPtr, trafficClass, NCCL_CONFIG_UNDEF_INT, NCCL_CONFIG_UNDEF_INT, "Traffic class", "%d");
/* assign config to communicator */
comm->config.blocking = internalConfigPtr->blocking;
@@ -2236,6 +2249,7 @@ static ncclResult_t parseCommConfig(ncclComm_t comm, ncclConfig_t *config) {
comm->config.maxCTAs = internalConfigPtr->maxCTAs;
comm->config.netName = internalConfigPtr->netName;
comm->config.splitShare = internalConfigPtr->splitShare;
comm->config.trafficClass = internalConfigPtr->trafficClass;
NCCLCHECKGOTO(envConfigOverride(comm), ret, fail);
@@ -2260,6 +2274,7 @@ static ncclResult_t ncclCommInitRankDev(ncclComm_t* newcomm, int nranks, int nId
const char* commIdEnv = NULL;
ncclComm_t comm = NULL;
struct ncclCommInitRankAsyncJob* job = NULL;
bool launchedJob = false;
// first call ncclInit, this will setup the environment
NCCLCHECKGOTO(ncclInit(), res, fail);
@@ -2313,6 +2328,7 @@ static ncclResult_t ncclCommInitRankDev(ncclComm_t* newcomm, int nranks, int nId
// start the bootstrap root before bootstrapping, use only the first handle
NCCLCHECKGOTO(bootstrapCreateRoot((struct ncclBootstrapHandle*)&job->commId[0], true), res, fail);
}
launchedJob = true;
NCCLCHECKGOTO(ncclAsyncLaunch((struct ncclAsyncJob*)job, ncclCommInitRankFunc, NULL, ncclCommInitJobFree, comm), res, fail);
exit:
@@ -2321,7 +2337,7 @@ exit:
NCCLCHECK(Recorder::instance().record(rrCommInitDev, nranks, myrank, commId, comm, cudaDev));
return ncclGroupErrCheck(res);
fail:
if (job) ncclCommInitJobFree(job);
if (job && !launchedJob) ncclCommInitJobFree(job);
if (comm) {
free(comm->abortFlag);
if (comm->abortFlagDev) (void)ncclCudaHostFree((void*)comm->abortFlagDev);
@@ -2520,7 +2536,7 @@ static ncclResult_t commDestroySync(struct ncclAsyncJob* job_) {
NCCLCHECKGOTO(ncclCommPollEventCallbacks(comm), ret, fail);
NCCLCHECKGOTO(ncclCommPollCallbacks(comm, false), ret, fail);
// And keep polling until all graphs referencing us die.
while (comm->persistentRefs != 0) {
while (comm->localPersistentRefs != 0) {
NCCLCHECKGOTO(ncclCommPollCallbacks(comm, /*waitSome=*/true), ret, fail);
}
while (!ncclIntruQueueEmpty(&comm->legacyRegCleanupQueue)) {
@@ -2613,7 +2629,6 @@ exit:
}
return ret;
fail:
free(job);
if (comm && !comm->config.blocking) (void) ncclCommSetAsyncError(comm, ret);
goto exit;
}
@@ -2896,6 +2911,11 @@ ncclResult_t ncclCommGetAsyncError_impl(ncclComm_t comm, ncclResult_t *asyncErro
*asyncError = __atomic_load_n(&comm->asyncResult, __ATOMIC_ACQUIRE);
if (*asyncError == ncclSuccess && comm->proxyState) *asyncError = __atomic_load_n(&comm->proxyState->asyncResult, __ATOMIC_ACQUIRE);
/* if there is linked group job, we should complete it. */
if (*asyncError == ncclSuccess && comm->groupJob) {
NCCLCHECK(ncclGroupJobComplete(comm->groupJob));
comm->groupJob = NULL;
}
return ncclSuccess;
}
@@ -2949,16 +2969,13 @@ ncclResult_t ncclMemAlloc_impl(void **ptr, size_t size) {
#if CUDART_VERSION >= 12010
size_t memGran = 0;
size_t mcGran = 0;
CUdevice currentDev;
CUmemAllocationProp memprop = {};
CUmulticastObjectProp mcprop = {};
CUmemAccessDesc accessDesc = {};
CUmemGenericAllocationHandle handle;
int cudaDev;
int flag;
int dcnt;
int mcSupport = 0;
if (ptr == NULL || size == 0) goto fallback;
@@ -2968,6 +2985,7 @@ ncclResult_t ncclMemAlloc_impl(void **ptr, size_t size) {
CUCHECK(cuDeviceGet(&currentDev, cudaDev));
if (ncclCuMemEnable()) {
size_t handleSize = size;
int requestedHandleTypes = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR;
// Query device to see if FABRIC handle support is available
flag = 0;
@@ -2983,40 +3001,25 @@ ncclResult_t ncclMemAlloc_impl(void **ptr, size_t size) {
if (flag) memprop.allocFlags.gpuDirectRDMACapable = 1;
CUCHECK(cuMemGetAllocationGranularity(&memGran, &memprop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));
CUDACHECK(cudaGetDeviceCount(&dcnt));
if (CUPFN(cuMulticastCreate) != NULL) CUCHECK(cuDeviceGetAttribute(&mcSupport, CU_DEVICE_ATTRIBUTE_MULTICAST_SUPPORTED, currentDev));
if (mcSupport) {
/* mc property */
mcprop.size = size;
/* device cnt is a dummy value right now, it might affect mc granularity in the future. */
mcprop.numDevices = dcnt;
mcprop.handleTypes = requestedHandleTypes;
mcprop.flags = 0;
CUCHECK(cuMulticastGetGranularity(&mcGran, &mcprop, CU_MULTICAST_GRANULARITY_RECOMMENDED));
/* only size needs to be aligned to mcGran */
ALIGN_SIZE(size, mcGran);
} else {
ALIGN_SIZE(size, memGran);
}
ALIGN_SIZE(handleSize, memGran);
if (requestedHandleTypes & CU_MEM_HANDLE_TYPE_FABRIC) {
/* First try cuMemCreate() with FABRIC handle support and then remove if it fails */
CUresult err = CUPFN(cuMemCreate(&handle, size, &memprop, 0));
CUresult err = CUPFN(cuMemCreate(&handle, handleSize, &memprop, 0));
if (err == CUDA_ERROR_NOT_PERMITTED || err == CUDA_ERROR_NOT_SUPPORTED) {
requestedHandleTypes &= ~CU_MEM_HANDLE_TYPE_FABRIC;
memprop.requestedHandleTypes = (CUmemAllocationHandleType) requestedHandleTypes;
/* Allocate the physical memory on the device */
CUCHECK(cuMemCreate(&handle, size, &memprop, 0));
CUCHECK(cuMemCreate(&handle, handleSize, &memprop, 0));
}
} else {
/* Allocate the physical memory on the device */
CUCHECK(cuMemCreate(&handle, size, &memprop, 0));
CUCHECK(cuMemCreate(&handle, handleSize, &memprop, 0));
}
/* Reserve a virtual address range */
CUCHECK(cuMemAddressReserve((CUdeviceptr*)ptr, size, memGran, 0, 0));
CUCHECK(cuMemAddressReserve((CUdeviceptr*)ptr, handleSize, memGran, 0, 0));
/* Map the virtual address range to the physical allocation */
CUCHECK(cuMemMap((CUdeviceptr)*ptr, size, 0, handle, 0));
CUCHECK(cuMemMap((CUdeviceptr)*ptr, handleSize, 0, handle, 0));
/* Now allow RW access to the newly mapped memory */
for (int i = 0; i < dcnt; ++i) {
int p2p = 0;
@@ -3024,7 +3027,7 @@ ncclResult_t ncclMemAlloc_impl(void **ptr, size_t size) {
accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
accessDesc.location.id = i;
accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
CUCHECK(cuMemSetAccess((CUdeviceptr)*ptr, size, &accessDesc, 1));
CUCHECK(cuMemSetAccess((CUdeviceptr)*ptr, handleSize, &accessDesc, 1));
}
if (0 == p2p && i != cudaDev) INFO(NCCL_ALLOC, "P2P not supported between GPU%d and GPU%d", cudaDev, i);
}
src/misc/cudawrap.cc
+31
Просмотреть файл
@@ -4,6 +4,7 @@
* See LICENSE.txt for license information
************************************************************************/
#include "alloc.h"
#include "nccl.h"
#include "debug.h"
#include "param.h"
@@ -67,6 +68,36 @@ int ncclCuMemHostEnable() {
ncclCumemHostEnable = paramValue;
else
ncclCumemHostEnable = (cudaDriverVersion >= 12060) ? 1 : 0;
if (ncclCumemHostEnable) {
// Verify that host allocations actually work. Docker in particular is known to disable "get_mempolicy",
// causing such allocations to fail (this can be fixed by invoking Docker with "--cap-add SYS_NICE").
int cudaDev;
CUdevice currentDev;
int cpuNumaNodeId = -1;
CUmemAllocationProp prop = {};
size_t granularity = 0;
size_t size;
CUmemGenericAllocationHandle handle;
CUDACHECK(cudaGetDevice(&cudaDev));
CUCHECK(cuDeviceGet(&currentDev, cudaDev));
CUCHECK(cuDeviceGetAttribute(&cpuNumaNodeId, CU_DEVICE_ATTRIBUTE_HOST_NUMA_ID, currentDev));
if (cpuNumaNodeId < 0) cpuNumaNodeId = 0;
prop.location.type = CU_MEM_LOCATION_TYPE_HOST_NUMA;
prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
prop.requestedHandleTypes = ncclCuMemHandleType;
prop.location.id = cpuNumaNodeId;
CUCHECK(cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM));
size = 1;
ALIGN_SIZE(size, granularity);
if (CUPFN(cuMemCreate(&handle, size, &prop, 0)) != CUDA_SUCCESS) {
INFO(NCCL_INIT, "cuMem host allocations do not appear to be working; falling back to a /dev/shm/ based "
"implementation. This could be due to the container runtime disabling NUMA support. "
"To disable this warning, set NCCL_CUMEM_HOST_ENABLE=0");
ncclCumemHostEnable = 0;
} else {
CUCHECK(cuMemRelease(handle));
}
}
}
return ncclCumemHostEnable;
error:
src/misc/ipcsocket.cc
+2 -1
Просмотреть файл
@@ -169,7 +169,7 @@ ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
} control_un;
struct cmsghdr *cmptr;
char dummy_buffer[1];
char dummy_buffer[1] = {'\0'};
struct sockaddr_un cliaddr;
// Construct client address to send this shareable handle to
@@ -190,6 +190,7 @@ ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
TRACE(NCCL_INIT, "UDS: Sending hdr %p len %d fd %d to UDS socket %s", hdr, hdrLen, sendFd, temp);
if (sendFd != -1) {
memset(&control_un, '\0', sizeof(control_un));
msg.msg_control = control_un.control;
msg.msg_controllen = sizeof(control_un.control);
src/misc/param.cc
+1
Просмотреть файл
@@ -32,6 +32,7 @@ void setEnvFile(const char* fileName) {
size_t n = 0;
ssize_t read;
while ((read = getline(&line, &n, file)) != -1) {
if (line[0] == '#') continue;
if (line[read-1] == '\n') line[read-1] = '\0';
int s=0; // Env Var Size
while (line[s] != '\0' && line[s] != '=') s++;
src/misc/socket.cc
+10 -1
Просмотреть файл
@@ -176,6 +176,7 @@ static int findInterfaces(const char* prefixList, char* names, union ncclSocketA
strncpy(names+found*maxIfNameSize, interface->ifa_name, maxIfNameSize);
// Store the IP address
int salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
memset(addrs+found, '\0', sizeof(*addrs));
memcpy(addrs+found, interface->ifa_addr, salen);
found++;
}
@@ -919,9 +920,17 @@ ncclResult_t ncclSocketShutdown(struct ncclSocket* sock, int how) {
return ncclSuccess;
}
ncclResult_t ncclSocketClose(struct ncclSocket* sock) {
ncclResult_t ncclSocketClose(struct ncclSocket* sock, bool wait) {
if (sock != NULL) {
if (sock->state > ncclSocketStateNone && sock->state < ncclSocketStateNum && sock->fd >= 0) {
if (wait) {
char data;
int closed = 0;
do {
int offset = 0;
if (ncclSocketProgress(NCCL_SOCKET_RECV, sock, &data, sizeof(char), &offset, &closed) != ncclSuccess) break;
} while (closed == 0);
}
/* shutdown() is needed to send FIN packet to proxy thread; shutdown() is not affected
* by refcount of fd, but close() is. close() won't close a fd and send FIN packet if
* the fd is duplicated (e.g. fork()). So shutdown() guarantees the correct and graceful
src/misc/strongstream.cc
+239 -270
Просмотреть файл
@@ -9,28 +9,61 @@
#include "checks.h"
#include "param.h"
// Tracks the chain of graph nodes for a given graph captured identified by
// its graph id. This state has to live for as long as captured work is being
// submitted. CUDA doesn't have mechanism to inform us when the user ends capture
// so the best we can do is get notified when the graph is destroyed.
struct ncclStrongStreamGraph {
struct ncclStrongStreamGraph* next;
// Atomically exchanged to false by both the main thread or the graph destructor
// callback. The last to arrive deletes the node.
bool alive;
// Tracks the captured work a given graph captured identified by its graph id.
struct ncclStrongStreamCapture {
struct ncclStrongStreamCapture* next;
cudaGraph_t graph;
unsigned long long graphId;
// For each graph we track the "tip" of the chain of graph nodes. A linear
// chain would always have just one node at its tip, but since we have to merge
// in chains from other streams (via ncclStrongStreamWaitStream) some spots
// in the chain can be wider than a single node and thus need a list, so we
// maintain a dynamically sized array of tip nodes.
int tipCount, tipCapacity;
cudaGraphNode_t* tipNodes;
cudaStream_t captureStream;
cudaGraphNode_t lastRecord;
void* acquiredBy;
};
static void ncclStrongStreamGraphDelete(struct ncclStrongStreamGraph* g) {
free(g->tipNodes);
free(g);
////////////////////////////////////////////////////////////////////////////////
static ncclCudaContext* cxtListHead = nullptr;
static pthread_mutex_t cxtListLock = PTHREAD_MUTEX_INITIALIZER;
ncclResult_t ncclCudaContextTrack(struct ncclCudaContext** out) {
ncclResult_t result = ncclSuccess;
CUcontext hcontext;
cuCtxGetCurrent(&hcontext);
pthread_mutex_lock(&cxtListLock);
struct ncclCudaContext* p = cxtListHead;
while (1) {
if (p == nullptr) {
p = (struct ncclCudaContext*)calloc(1, sizeof(struct ncclCudaContext));
p->refCount = 1;
p->hcontext = hcontext;
p->next = cxtListHead;
cxtListHead = p;
NCCLCHECKGOTO(ncclStrongStreamConstruct(&p->launchOrder), result, leave);
break;
}
if (p->hcontext == hcontext) {
p->refCount += 1;
break;
}
p = p->next;
}
leave:
pthread_mutex_unlock(&cxtListLock);
*out = p;
return ncclSuccess;
}
void ncclCudaContextDrop(struct ncclCudaContext* cxt) {
pthread_mutex_lock(&cxtListLock);
if (0 == --cxt->refCount) {
struct ncclCudaContext** pp = &cxtListHead;
while (*pp != cxt) pp = &(*pp)->next;
*pp = cxt->next; // remove from list
// Destroy resources held in cxt
ncclStrongStreamDestruct(&cxt->launchOrder);
free(cxt);
}
pthread_mutex_unlock(&cxtListLock);
}
////////////////////////////////////////////////////////////////////////////////
@@ -40,13 +73,14 @@ ncclResult_t ncclCudaGetCapturingGraph(
) {
#if ROCM_VERSION >= 60100
hipStreamCaptureStatus status;
unsigned long long gid;
CUDACHECK(hipStreamGetCaptureInfo_v2(stream, &status, &gid, &graph->graph, nullptr, nullptr));
CUDACHECK(hipStreamGetCaptureInfo_v2(stream, &status, &graph->graphId, &graph->graph, nullptr, nullptr));
if (status != hipStreamCaptureStatusActive) {
graph->origin = nullptr;
graph->graph = nullptr;
gid = ULLONG_MAX;
graph->graphId = ULLONG_MAX;
} else {
graph->origin = stream;
}
graph->graphId = gid;
#endif
return ncclSuccess;
}
@@ -68,315 +102,250 @@ ncclResult_t ncclCudaGraphAddDestructor(struct ncclCudaGraph graph, cudaHostFn_t
////////////////////////////////////////////////////////////////////////////////
ncclResult_t ncclStrongStreamConstruct(struct ncclStrongStream* ss) {
CUDACHECK(cudaStreamCreateWithFlags(&ss->cudaStream, cudaStreamNonBlocking));
CUDACHECK(cudaStreamCreateWithFlags(&ss->liveStream, cudaStreamNonBlocking));
#if ROCM_VERSION >= 60100
CUDACHECK(cudaEventCreateWithFlags(&ss->serialEvent, cudaEventDisableTiming));
ss->everCaptured = false;
ss->serialEventNeedsRecord = false;
ss->graphHead = nullptr;
#else
CUDACHECK(cudaEventCreateWithFlags(&ss->scratchEvent, cudaEventDisableTiming));
ss->captureHead = nullptr;
pthread_mutex_init(&ss->lock, nullptr);
CUDACHECK(cudaEventCreateWithFlags(&ss->serialEvent, cudaEventDisableTiming));
#endif
return ncclSuccess;
}
static void graphDestructor(void* arg) {
struct ncclStrongStreamGraph* g = (struct ncclStrongStreamGraph*)arg;
if (false == __atomic_exchange_n(&g->alive, false, __ATOMIC_ACQ_REL)) {
// Last to arrive deletes list node.
ncclStrongStreamGraphDelete(g);
}
}
ncclResult_t ncclStrongStreamDestruct(struct ncclStrongStream* ss) {
CUDACHECK(cudaStreamDestroy(ss->cudaStream));
CUDACHECK(cudaStreamDestroy(ss->liveStream));
#if ROCM_VERSION >= 60100
CUDACHECK(cudaEventDestroy(ss->serialEvent));
// Delete list of per-graph chains.
struct ncclStrongStreamGraph* g = ss->graphHead;
while (g != nullptr) {
struct ncclStrongStreamGraph* next = g->next;
if (false == __atomic_exchange_n(&g->alive, false, __ATOMIC_ACQ_REL)) {
// Last to arrive deletes list node.
ncclStrongStreamGraphDelete(g);
}
g = next;
struct ncclStrongStreamCapture* cap = ss->captureHead;
while (cap) {
struct ncclStrongStreamCapture* next = cap->next;
CUDACHECK(cudaStreamDestroy(cap->captureStream));
free(cap);
cap = next;
}
#else
CUDACHECK(cudaEventDestroy(ss->scratchEvent));
CUDACHECK(cudaEventDestroy(ss->serialEvent));
pthread_mutex_destroy(&ss->lock);
#endif
return ncclSuccess;
}
NCCL_PARAM(GraphMixingSupport, "GRAPH_MIXING_SUPPORT", 0)
NCCL_PARAM(LaunchRaceFatal, "LAUNCH_RACE_FATAL", 1);
constexpr char const* launchRaceFatalMsg = "Fatal: host threads racing to launch NCCL on same device.";
static void ensureTips(struct ncclStrongStreamGraph* g, int n) {
if (g->tipCapacity < n) {
g->tipNodes = (cudaGraphNode_t*)realloc(g->tipNodes, n*sizeof(cudaGraphNode_t));
g->tipCapacity = n;
}
}
static __thread char threadIdMarker;
static void* localThreadId() { return &threadIdMarker; }
ncclResult_t ncclStrongStreamAcquire(
struct ncclCudaGraph graph, struct ncclStrongStream* ss
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent,
cudaStream_t* workStream
) {
#if ROCM_VERSION >= 60100
bool mixing = ncclParamGraphMixingSupport();
if (graph.graph == nullptr) {
if (mixing && ss->everCaptured) {
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
ss->serialEventNeedsRecord = false;
if (graph.graphId == ULLONG_MAX) {
*workStream = ss->liveStream;
ss->liveAcquiredBy = localThreadId();
if (mixing && __atomic_load_n(&ss->everCaptured, __ATOMIC_RELAXED)) {
CUDACHECK(cudaStreamWaitEvent(ss->liveStream, ss->serialEvent, 0));
}
} else {
ss->everCaptured = true;
// Find the current graph in our list of graphs if it exists.
struct ncclStrongStreamGraph** pg = &ss->graphHead;
struct ncclStrongStreamGraph* g;
while (*pg != nullptr) {
g = *pg;
if (g->graphId == graph.graphId) {
// Move to front of list so that operations after acquire don't have to search the list.
*pg = g->next;
g->next = ss->graphHead;
ss->graphHead = g;
bool firstCapture = !ss->everCaptured;
__atomic_store_n(&ss->everCaptured, true, __ATOMIC_RELAXED);
ncclResult_t ret = ncclSuccess;
if (concurrent) pthread_mutex_lock(&ss->lock);
// Look for capture in our list of active captures.
struct ncclStrongStreamCapture** pcap = &ss->captureHead;
struct ncclStrongStreamCapture* cap;
struct ncclStrongStreamCapture* spare = nullptr;
while (*pcap != nullptr) {
cap = *pcap;
if (cap->graphId == graph.graphId) { // Capture node already exists.
*workStream = cap->captureStream;
cap->acquiredBy = localThreadId();
if (concurrent) pthread_mutex_unlock(&ss->lock);
return ncclSuccess;
} else if (false == __atomic_load_n(&g->alive, __ATOMIC_ACQUIRE)) {
// Unrelated graph that has been destroyed. Remove and delete.
*pg = g->next;
ncclStrongStreamGraphDelete(g);
} else {
pg = &g->next;
cudaStreamCaptureStatus status;
CUDACHECKGOTO(cudaStreamIsCapturing(cap->captureStream, &status), ret, do_unlock);
if (status == cudaStreamCaptureStatusActive) {
pcap = &cap->next; // Active capture doesn't match, on to next.
} else { // Capture no longer active
*pcap = cap->next; // Remove from current list
if (spare == nullptr) { // Keep one spare to reuse below.
spare = cap;
} else {
cudaStreamDestroy(cap->captureStream);
free(cap);
}
}
}
}
// This is a new graph so add to the list.
g = (struct ncclStrongStreamGraph*)malloc(sizeof(struct ncclStrongStreamGraph));
g->graphId = graph.graphId;
g->tipNodes = nullptr;
g->tipCapacity = 0;
g->tipCount = 0;
g->next = ss->graphHead;
ss->graphHead = g;
g->alive = true;
NCCLCHECK(ncclCudaGraphAddDestructor(graph, graphDestructor, (void*)g));
if (mixing && ss->serialEventNeedsRecord) {
// Can only be here if previous release was for uncaptured work that
// elided updating the event because no capture had yet occurred.
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->cudaStream));
// No matching capture, need a new entry.
cap = spare;
if (cap == nullptr) {
cap = (struct ncclStrongStreamCapture*)calloc(1, sizeof(struct ncclStrongStreamCapture));
CUDACHECKGOTO(cudaStreamCreateWithFlags(&cap->captureStream, cudaStreamNonBlocking), ret, do_unlock);
}
ss->serialEventNeedsRecord = false;
cap->graphId = graph.graphId;
cap->lastRecord = nullptr;
cap->acquiredBy = localThreadId();
// Push to capturing list.
cap->next = ss->captureHead;
ss->captureHead = cap;
// First node in the chain must be a wait on the serialEvent.
do_unlock:
if (concurrent) pthread_mutex_unlock(&ss->lock);
if (ret != ncclSuccess) return ret;
*workStream = cap->captureStream;
// Bring captureStream into the graph but without any dependencies.
cudaEvent_t scratch;
CUDACHECK(cudaEventCreateWithFlags(&scratch, cudaEventDisableTiming));
CUDACHECK(cudaEventRecord(scratch, graph.origin));
CUDACHECK(cudaStreamWaitEvent(cap->captureStream, scratch, 0));
CUDACHECK(cudaEventDestroy(scratch));
CUDACHECK(cudaStreamUpdateCaptureDependencies(cap->captureStream, nullptr, 0, cudaStreamSetCaptureDependencies));
if (mixing && firstCapture) {
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->liveStream));
}
if (mixing) {
ensureTips(g, 1);
CUDACHECK(cudaGraphAddEventWaitNode(&g->tipNodes[0], graph.graph, nullptr, 0, ss->serialEvent));
g->tipCount = 1;
} else {
g->tipCount = 0;
// First dependency is to wait on serialEvent
CUDACHECK(cudaStreamWaitEvent(cap->captureStream, ss->serialEvent, 0));
}
}
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamAcquireUncaptured(struct ncclStrongStream* ss) {
ncclResult_t ncclStrongStreamAcquiredWorkStream(
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent,
cudaStream_t* workStream
) {
#if ROCM_VERSION >= 60100
bool mixing = ncclParamGraphMixingSupport();
if (mixing && ss->everCaptured) {
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
if (graph.graphId == ULLONG_MAX) {
*workStream = ss->liveStream;
} else {
if (concurrent) pthread_mutex_lock(&ss->lock);
struct ncclStrongStreamCapture* cap = ss->captureHead;
while (cap->graphId != graph.graphId) cap = cap->next;
*workStream = cap->captureStream;
if (concurrent) pthread_mutex_unlock(&ss->lock);
}
ss->serialEventNeedsRecord = true; // Assume the caller is going to add work to stream.
#else
*workStream = ss->liveStream
#endif
return ncclSuccess;
}
static ncclResult_t checkGraphId(struct ncclStrongStreamGraph* g, unsigned long long id) {
if (g == nullptr || g->graphId != id) {
WARN("Expected graph id=%llu was not at head of strong stream's internal list.", id);
return ncclInternalError;
}
return ncclSuccess;
}
ncclResult_t ncclStrongStreamRelease(struct ncclCudaGraph graph, struct ncclStrongStream* ss) {
ncclResult_t ncclStrongStreamRelease(
struct ncclCudaGraph graph, struct ncclStrongStream* ss, bool concurrent
) {
#if ROCM_VERSION >= 60100
bool mixing = ncclParamGraphMixingSupport();
if (mixing && ss->serialEventNeedsRecord) {
if (graph.graph == nullptr) {
if (ss->everCaptured) {
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->cudaStream));
ss->serialEventNeedsRecord = false;
if (mixing) {
if (graph.graphId == ULLONG_MAX) {
if (__atomic_load_n(&ss->everCaptured, __ATOMIC_RELAXED)) {
CUDACHECK(cudaEventRecord(ss->serialEvent, ss->liveStream));
}
if (ss->liveAcquiredBy != localThreadId() && ncclParamLaunchRaceFatal()) {
WARN("%s", launchRaceFatalMsg);
return ncclInvalidUsage;
}
} else {
struct ncclStrongStreamGraph* g = ss->graphHead;
NCCLCHECK(checkGraphId(g, graph.graphId));
ensureTips(g, 1);
CUDACHECK(cudaGraphAddEventRecordNode(&g->tipNodes[0], graph.graph, g->tipNodes, g->tipCount, ss->serialEvent));
g->tipCount = 1;
ss->serialEventNeedsRecord = false;
if (concurrent) pthread_mutex_lock(&ss->lock);
struct ncclStrongStreamCapture* cap = ss->captureHead;
while (cap->graphId != graph.graphId) cap = cap->next;
if (concurrent) pthread_mutex_unlock(&ss->lock);
// Add event record node with dependencies added further down.
cudaGraphNode_t recordNode;
CUDACHECK(cudaGraphAddEventRecordNode(&recordNode, graph.graph, nullptr, 0, ss->serialEvent));
// Make this record order after previous record on this stream.
if (cap->lastRecord != nullptr) {
CUDACHECK(cudaGraphAddDependencies(graph.graph, &cap->lastRecord, &recordNode, 1));
}
cap->lastRecord = recordNode;
// Get current nodes from work stream so we can add them as dependencies.
cudaStreamCaptureStatus status;
cudaGraphNode_t const* nodes;
size_t count = 0;
cudaError_t res = hipStreamGetCaptureInfo_v2(cap->captureStream, &status, nullptr, nullptr, &nodes, &count);
#if CUDART_VERSION >= 12030
if (res == cudaErrorLossyQuery) { // CUDA is telling us the dependencies have edge annotations.
cudaGraphEdgeData const* edges;
CUDACHECK(cudaStreamGetCaptureInfo_v3(cap->captureStream, &status, nullptr, nullptr, &nodes, &edges, &count));
for (int i=0; i < (int)count; i++) {
CUDACHECK(cudaGraphAddDependencies_v2(graph.graph, &nodes[i], &recordNode, &edges[i], 1));
}
}
#else
if (false) {}
#endif
else {
CUDACHECK(res /* = cudaStreamGetCaptureInfo_v2(...)*/);
for (int i=0; i < (int)count; i++) {
CUDACHECK(cudaGraphAddDependencies(graph.graph, &nodes[i], &recordNode, 1));
}
}
if (cap->acquiredBy != localThreadId() && ncclParamLaunchRaceFatal()) {
WARN("%s", launchRaceFatalMsg);
return ncclInvalidUsage;
}
}
}
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamLaunchHost(
struct ncclCudaGraph graph, struct ncclStrongStream* ss, cudaHostFn_t fn, void* arg
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
CUDACHECK(cudaLaunchHostFunc(ss->cudaStream, fn, arg));
} else {
cudaHostNodeParams p;
p.fn = fn;
p.userData = arg;
struct ncclStrongStreamGraph* g = ss->graphHead;
NCCLCHECK(checkGraphId(g, graph.graphId));
ensureTips(g, 1);
CUDACHECK(cudaGraphAddHostNode(&g->tipNodes[0], graph.graph, g->tipNodes, g->tipCount, &p));
g->tipCount = 1;
}
ss->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaLaunchHostFunc(ss->cudaStream, fn, arg));
#endif
ncclResult_t ncclStreamWaitStream(cudaStream_t a, cudaStream_t b, cudaEvent_t scratchEvent) {
CUDACHECK(cudaEventRecord(scratchEvent, b));
CUDACHECK(cudaStreamWaitEvent(a, scratchEvent, 0));
return ncclSuccess;
}
ncclResult_t ncclStrongStreamLaunchKernel(
struct ncclCudaGraph graph, struct ncclStrongStream* ss,
void* fn, dim3 grid, dim3 block, void* args[], size_t sharedMemBytes
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
CUDACHECK(cudaLaunchKernel(fn, grid, block, args, sharedMemBytes, ss->cudaStream));
} else {
cudaKernelNodeParams p;
p.func = fn;
p.gridDim = grid;
p.blockDim = block;
p.kernelParams = args;
p.sharedMemBytes = sharedMemBytes;
p.extra = nullptr;
struct ncclStrongStreamGraph* g = ss->graphHead;
NCCLCHECK(checkGraphId(g, graph.graphId));
ensureTips(g, 1);
CUDACHECK(cudaGraphAddKernelNode(&g->tipNodes[0], graph.graph, g->tipNodes, g->tipCount, &p));
g->tipCount = 1;
}
ss->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaLaunchKernel(fn, grid, block, args, sharedMemBytes, ss->cudaStream));
#endif
return ncclSuccess;
}
ncclResult_t ncclStreamAdvanceToEvent(struct ncclCudaGraph g, cudaStream_t s, cudaEvent_t e) {
if (g.graphId == ULLONG_MAX) {
CUDACHECK(cudaStreamWaitEvent(s, e, 0));
} else {
cudaStream_t tmp;
CUDACHECK(cudaStreamCreateWithFlags(&tmp, cudaStreamNonBlocking));
CUDACHECK(cudaStreamWaitEvent(tmp, e, 0));
// Merge node list `b` into list `a` but don't add duplicates.
static void mergeTips(struct ncclStrongStreamGraph* a, cudaGraphNode_t const* bNodes, int bn) {
int an = a->tipCount;
ensureTips(a, an + bn);
for (int bi=0; bi < bn; bi++) {
for (int ai=0; ai < an; ai++) {
if (a->tipNodes[ai] == bNodes[bi]) goto next_b;
cudaStreamCaptureStatus status;
cudaGraphNode_t const* nodes;
size_t count = 0;
cudaError_t res = hipStreamGetCaptureInfo_v2(tmp, &status, nullptr, nullptr, &nodes, &count);
#if CUDART_VERSION >= 12030
if (res == cudaErrorLossyQuery) { // CUDA is telling us the dependencies have edge annotations.
cudaGraphEdgeData const* edges;
CUDACHECK(cudaStreamGetCaptureInfo_v3(tmp, &status, nullptr, nullptr, &nodes, &edges, &count));
CUDACHECK(cudaStreamUpdateCaptureDependencies_v2(s, (cudaGraphNode_t*)nodes, edges, count, cudaStreamSetCaptureDependencies));
}
a->tipNodes[a->tipCount++] = bNodes[bi];
next_b:;
#else
if (false) {}
#endif
else {
CUDACHECK(res /* = cudaStreamGetCaptureInfo_v2(...)*/);
CUDACHECK(cudaStreamUpdateCaptureDependencies(s, (cudaGraphNode_t*)nodes, count, cudaStreamSetCaptureDependencies));
}
CUDACHECK(cudaStreamDestroy(tmp));
}
}
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, struct ncclStrongStream* b,
bool b_subsumes_a
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
if (b->serialEventNeedsRecord) {
b->serialEventNeedsRecord = false;
CUDACHECK(cudaEventRecord(b->serialEvent, b->cudaStream));
}
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, b->serialEvent, 0));
} else {
struct ncclStrongStreamGraph* ag = a->graphHead;
NCCLCHECK(checkGraphId(ag, graph.graphId));
struct ncclStrongStreamGraph* bg = b->graphHead;
NCCLCHECK(checkGraphId(bg, graph.graphId));
if (b_subsumes_a) ag->tipCount = 0;
mergeTips(ag, bg->tipNodes, bg->tipCount);
}
a->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaEventRecord(b->scratchEvent, b->cudaStream));
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, b->scratchEvent, 0));
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, struct ncclStrongStream* a, cudaStream_t b,
bool b_subsumes_a
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
// It is ok to use a->serialEvent to record b since we'll be setting
// a->serialEventNeedsRecord so the event won't be considered accurate
// until re-recorded.
CUDACHECK(cudaEventRecord(a->serialEvent, b));
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, a->serialEvent, 0));
} else {
cudaStreamCaptureStatus status;
unsigned long long bGraphId;
cudaGraphNode_t const* bNodes;
size_t bCount = 0;
CUDACHECK(hipStreamGetCaptureInfo_v2(b, &status, &bGraphId, nullptr, &bNodes, &bCount));
if (status != cudaStreamCaptureStatusActive || graph.graphId != bGraphId) {
WARN("Stream is not being captured by the expected graph.");
return ncclInvalidUsage;
}
struct ncclStrongStreamGraph* ag = a->graphHead;
NCCLCHECK(checkGraphId(ag, graph.graphId));
if (b_subsumes_a) ag->tipCount = 0;
mergeTips(ag, bNodes, bCount);
}
a->serialEventNeedsRecord = true;
#else
CUDACHECK(cudaEventRecord(a->scratchEvent, b));
CUDACHECK(cudaStreamWaitEvent(a->cudaStream, a->scratchEvent, 0));
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamWaitStream(
struct ncclCudaGraph graph, cudaStream_t a, struct ncclStrongStream* b,
bool b_subsumes_a
) {
#if ROCM_VERSION >= 60100
if (graph.graph == nullptr) {
if (b->serialEventNeedsRecord) {
b->serialEventNeedsRecord = false;
CUDACHECK(cudaEventRecord(b->serialEvent, b->cudaStream));
}
CUDACHECK(cudaStreamWaitEvent(a, b->serialEvent, 0));
} else {
struct ncclStrongStreamGraph* bg = b->graphHead;
NCCLCHECK(checkGraphId(bg, graph.graphId));
CUDACHECK(hipStreamUpdateCaptureDependencies(a, bg->tipNodes, bg->tipCount,
b_subsumes_a ? cudaStreamSetCaptureDependencies : cudaStreamAddCaptureDependencies
));
}
#else
CUDACHECK(cudaEventRecord(b->scratchEvent, b->cudaStream));
CUDACHECK(cudaStreamWaitEvent(a, b->scratchEvent, 0));
#endif
return ncclSuccess;
}
ncclResult_t ncclStrongStreamSynchronize(struct ncclStrongStream* ss) {
#if ROCM_VERSION >= 60100
CUDACHECK(cudaStreamWaitEvent(ss->cudaStream, ss->serialEvent, 0));
ss->serialEventNeedsRecord = false;
CUDACHECK(cudaStreamWaitEvent(ss->liveStream, ss->serialEvent, 0));
#endif
CUDACHECK(cudaStreamSynchronize(ss->cudaStream));
CUDACHECK(cudaStreamSynchronize(ss->liveStream));
return ncclSuccess;
}
src/misc/tuner.cc
-267
Просмотреть файл
@@ -1,267 +0,0 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#include <dlfcn.h>
#include <errno.h>
#include <stdlib.h>
#include "checks.h"
#include "debug.h"
#include "tuner.h"
pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
static int tunerPluginRefCount;
static void* tunerPluginLib = nullptr;
static ncclTuner_v4_t* tunerSymbol = nullptr;
static ncclTuner_v3_t* ncclTuner_v3 = nullptr;
static ncclTuner_v2_t* ncclTuner_v2 = nullptr;
static ncclTuner_v4_t ncclTuner_v2_as_v4;
static ncclTuner_v4_t ncclTuner_v3_as_v4;
static int hasNvlsSupport(float** collCostTable) {
// Requirements for support of different algorithms:
//
// - NVLS intra-node: nvlsSupport
// - NVLS intra+inter-node: collNetSupport
// - NVLSTree intra-node: always disabled
// - NVLSTree inter-node: nvlsSupport
// - Collnet* inter-node: collNetSupport
//
// nvlsSupport = 1 if either NVLS or NVLS_TREE entries in the cost table are not -1
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE || table[NCCL_ALGO_NVLS_TREE][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) ? 1 : 0;
}
static int hasCollNetSupport(float** collCostTable) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] == NCCL_ALGO_PROTO_IGNORE) ? 0 : 1;
}
static ncclResult_t ncclTuner_v3_as_v4_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff __attribute__((unused)), int* nChannels) {
NCCLCHECK(ncclTuner_v3->getCollInfo(context, collType, nBytes, numPipeOps, collCostTable, numAlgo, numProto, nChannels));
return ncclSuccess;
}
static ncclResult_t ncclTuner_v3_as_v4_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
NCCLCHECK(ncclTuner_v3->init(nRanks, nNodes, logFunction, context));
ncclTuner_v3_as_v4.name = ncclTuner_v3->name;
ncclTuner_v3_as_v4.getCollInfo = ncclTuner_v3_as_v4_getCollInfo;
ncclTuner_v3_as_v4.destroy = ncclTuner_v3->destroy;
return ncclSuccess;
}
static ncclResult_t ncclTuner_v2_as_v4_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int regBuff __attribute__((unused)), int* nChannels) {
int algorithm = NCCL_ALGO_UNDEF;
int protocol = NCCL_PROTO_UNDEF;
int nvlsSupport = hasNvlsSupport(collCostTable);
int collNetSupport = hasCollNetSupport(collCostTable);
NCCLCHECK(ncclTuner_v2->getCollInfo(context, collType, nBytes, collNetSupport, nvlsSupport, numPipeOps, &algorithm, &protocol, nChannels));
// set time to 0 below to make sure this algorithm/protocol is selected later on
if (algorithm >= 0 && algorithm < NCCL_NUM_ALGORITHMS && protocol >= 0 && protocol < NCCL_NUM_PROTOCOLS) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
if (table[algorithm][protocol] != NCCL_ALGO_PROTO_IGNORE) table[algorithm][protocol] = 0.0;
}
return ncclSuccess;
}
static ncclResult_t ncclTuner_v2_as_v4_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
NCCLCHECK(ncclTuner_v2->init(nRanks, nNodes, logFunction, context));
ncclTuner_v2_as_v4.name = ncclTuner_v2->name;
ncclTuner_v2_as_v4.getCollInfo = ncclTuner_v2_as_v4_getCollInfo;
ncclTuner_v2_as_v4.destroy = ncclTuner_v2->destroy;
return ncclSuccess;
}
#define MAX_STR_LEN 255
static void* tryOpenLib(const char* name, int* err, char* errStr) {
*err = 0;
if (nullptr == name || strlen(name) == 0) {
return nullptr;
}
if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
name = nullptr;
}
void *handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
if (nullptr == handle) {
strncpy(errStr, dlerror(), MAX_STR_LEN);
errStr[MAX_STR_LEN] = '\0';
// "handle" and "name" won't be NULL at the same time.
// coverity[var_deref_model]
if (strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
*err = ENOENT;
}
}
return handle;
}
static char* tryOpenLibCheck(int openErr, char* openErrStr, char* nameList, int *nameListLen, char* name) {
if (openErr == ENOENT) {
snprintf(nameList, *nameListLen, " %s", name);
nameList += strlen(name) + 1;
*nameListLen -= strlen(name) + 1;
return nameList;
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: %s", openErrStr);
return nameList;
}
static void* openTunerPluginLib(char* couldNotFindNames, int len) {
int openErr;
void *pluginLib;
char tunerPluginLibName[PATH_MAX];
char openErrStr[MAX_STR_LEN + 1] = { 0 };
const char *envTunerPluginName = getenv("NCCL_TUNER_PLUGIN");
if (envTunerPluginName && strlen(envTunerPluginName)) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: NCCL_TUNER_PLUGIN set to %s", envTunerPluginName);
snprintf(tunerPluginLibName, PATH_MAX, "%s", envTunerPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner-%s.so", envTunerPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
} else {
snprintf(tunerPluginLibName, PATH_MAX, "librccl-tuner.so");
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
}
const char *envNetPluginName = getenv("NCCL_NET_PLUGIN");
if (envNetPluginName && strlen(envNetPluginName)) {
// Users are allowed to pack tuner into the net plugin
snprintf(tunerPluginLibName, PATH_MAX, "%s", envNetPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
snprintf(tunerPluginLibName, PATH_MAX, "librccl-net-%s.so", envNetPluginName);
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Plugin name set by env to %s", tunerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
} else {
snprintf(tunerPluginLibName, PATH_MAX, "librccl-net.so");
pluginLib = tryOpenLib(tunerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, tunerPluginLibName);
}
tunerPluginLibName[0] = '\0';
return nullptr;
}
enum {
tunerPluginLoadFailed = -1,
tunerPluginLoadReady = 0,
tunerPluginLoadSuccess = 1,
};
#define MAX_PLUGIN_LOAD 4
static int status = tunerPluginLoadReady;
ncclResult_t ncclTunerPluginLoad(struct ncclComm* comm) {
// Initialize to nullptr by default if plugin tuner cannot be loaded.
char couldNotFindNames[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
comm->tuner = nullptr;
if (tunerPluginLoadFailed == status) {
return ncclSuccess;
}
pthread_mutex_lock(&tunerPluginLock);
if (tunerPluginLoadFailed == status) {
goto exit;
}
if (tunerPluginLoadSuccess == status) {
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
goto exit;
}
tunerPluginLib = openTunerPluginLib(couldNotFindNames, MAX_PLUGIN_LOAD * PATH_MAX);
if (nullptr == tunerPluginLib) {
if (strlen(couldNotFindNames)) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Could not find:%s. Using internal tuner plugin.", couldNotFindNames);
} else {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using internal tuner plugin.");
}
goto fail;
}
tunerSymbol = (ncclTuner_v4_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v4");
if (tunerSymbol == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.");
ncclTuner_v3 = (ncclTuner_v3_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v3");
if (ncclTuner_v3 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v2");
if (ncclTuner_v2 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
dlclose(tunerPluginLib);
goto fail;
} else {
ncclTuner_v2_as_v4.init = ncclTuner_v2_as_v4_init;
ncclTuner_v2_as_v4.name = ncclTuner_v2->name;
tunerSymbol = &ncclTuner_v2_as_v4;
}
} else {
ncclTuner_v3_as_v4.init = ncclTuner_v3_as_v4_init;
ncclTuner_v3_as_v4.name = ncclTuner_v3->name;
tunerSymbol = &ncclTuner_v3_as_v4;
}
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", tunerSymbol->name);
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
status = tunerPluginLoadSuccess;
comm->tunerPluginLoaded = 1;
exit:
pthread_mutex_unlock(&tunerPluginLock);
return ncclSuccess;
fail:
tunerPluginLib = nullptr;
status = tunerPluginLoadFailed;
goto exit;
}
ncclResult_t ncclTunerPluginUnload(struct ncclComm* comm) {
pthread_mutex_lock(&tunerPluginLock);
if (comm->tunerPluginLoaded && 0 == (--tunerPluginRefCount)) {
INFO(NCCL_TUNING, "TUNER/Plugin: Closing tuner: '%s'", tunerSymbol->name);
dlclose(tunerPluginLib);
tunerPluginLib = nullptr;
tunerSymbol = nullptr;
comm->tuner = nullptr;
status = tunerPluginLoadReady;
comm->tunerPluginLoaded = 0;
}
pthread_mutex_unlock(&tunerPluginLock);
return ncclSuccess;
}
src/nccl.h.in
+3 -1
Просмотреть файл
@@ -82,6 +82,7 @@ typedef struct ncclConfig_v21700 {
int maxCTAs; /*!< Maximum number of cooperative thread arrays (blocks) */
const char *netName; /*!< Force NCCL to use a specfic network */
int splitShare; /*!< Allow communicators to share resources */
int trafficClass; /*!< Traffic class*/
} ncclConfig_t;
/* Config initializer must be assigned to initialize config structure when it is created.
@@ -95,7 +96,8 @@ typedef struct ncclConfig_v21700 {
NCCL_CONFIG_UNDEF_INT, /* minCTAs */ \
NCCL_CONFIG_UNDEF_INT, /* maxCTAs */ \
NCCL_CONFIG_UNDEF_PTR, /* netName */ \
NCCL_CONFIG_UNDEF_INT /* splitShare */ \
NCCL_CONFIG_UNDEF_INT, /* splitShare */ \
NCCL_CONFIG_UNDEF_INT, /* trafficClass */ \
}
/*! @} */
src/net.cc
-1038
Просмотреть файл
Разница между файлами не показана из-за своего большого размера Загрузить разницу
src/plugin/net.cc
+323
Просмотреть файл
@@ -0,0 +1,323 @@
/*************************************************************************
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "net.h"
#include "bootstrap.h"
#include "checks.h"
#include "plugin.h"
#include <string.h>
#include <errno.h>
//#include <sys/types.h>
//#include <sys/stat.h>
//#include <unistd.h>
extern ncclNet_t* getNcclNet_v6(void* netPluginLib);
extern ncclNet_t* getNcclNet_v7(void* netPluginLib);
extern ncclNet_t* getNcclNet_v8(void* netPluginLib);
extern ncclNet_t* getNcclNet_v9(void* netPluginLib);
extern ncclNet_t* getNcclNet_v10(void* netPluginLib);
extern ncclCollNet_t* getNcclCollNet_v6(void* netPluginLib);
extern ncclCollNet_t* getNcclCollNet_v7(void* netPluginLib);
extern ncclCollNet_t* getNcclCollNet_v8(void* netPluginLib);
extern ncclCollNet_t* getNcclCollNet_v9(void* netPluginLib);
extern ncclCollNet_t* getNcclCollNet_v10(void* netPluginLib);
static pthread_mutex_t netLock = PTHREAD_MUTEX_INITIALIZER;
ncclNet_t* ncclNets[NCCL_NET_MAX_PLUGINS] = { nullptr, &ncclNetIb, &ncclNetSocket };
static int ncclNetsVer[NCCL_NET_MAX_PLUGINS] = { -1, 10, 10 };
ncclCollNet_t* ncclCollNets[NCCL_NET_MAX_PLUGINS] = { nullptr, nullptr, nullptr };
enum ncclNetState {
ncclNetStateInit = 0,
ncclNetStateEnabled = 1,
ncclNetStateDisabled = 2
};
enum ncclNetState ncclNetStates[NCCL_NET_MAX_PLUGINS] = { ncclNetStateInit, ncclNetStateInit, ncclNetStateInit };
enum ncclNetState ncclCollNetStates[NCCL_NET_MAX_PLUGINS] = { ncclNetStateInit, ncclNetStateInit, ncclNetStateInit };
NCCL_PARAM(NetPluginRefCount, "NET_PLUGIN_REF_COUNT", 1);
static pthread_mutex_t netPluginLock = PTHREAD_MUTEX_INITIALIZER;
static void* netPluginLib;
static int netPluginRefCount;
static void initNetPluginRefCountOnce(void) { netPluginRefCount = ncclParamNetPluginRefCount();}
enum {
netPluginLoadFailed = -1,
netPluginLoadReady = 0,
netPluginLoadSuccess = 1,
};
static int netPluginStatus = netPluginLoadReady;
ncclResult_t ncclNetPluginLoad(struct ncclComm* comm) {
static pthread_once_t netPluginRefCountOnce = PTHREAD_ONCE_INIT;
pthread_once(&netPluginRefCountOnce, initNetPluginRefCountOnce);
pthread_mutex_lock(&netPluginLock);
if (netPluginLoadFailed == netPluginStatus) {
goto exit;
}
if (netPluginLoadSuccess == netPluginStatus) {
++netPluginRefCount;
goto exit;
}
netPluginLib = ncclOpenNetPluginLib(ncclGetEnv("NCCL_NET_PLUGIN"));
if (netPluginLib == nullptr) {
goto fail;
}
ncclNets[0] = getNcclNet_v10(netPluginLib);
if (ncclNets[0]) ncclNetsVer[0] = 10;
if (ncclNets[0] == nullptr) {
// Try v9 plugin
ncclNets[0] = getNcclNet_v9(netPluginLib);
if (ncclNets[0]) ncclNetsVer[0] = 9;
}
if (ncclNets[0] == nullptr) {
// Try v8 plugin
ncclNets[0] = getNcclNet_v8(netPluginLib);
if (ncclNets[0]) ncclNetsVer[0] = 8;
}
if (ncclNets[0] == nullptr) {
// Try v7 plugin
ncclNets[0] = getNcclNet_v7(netPluginLib);
if (ncclNets[0]) ncclNetsVer[0] = 7;
}
if (ncclNets[0] == nullptr) {
// Try v6 plugin
ncclNets[0] = getNcclNet_v6(netPluginLib);
if (ncclNets[0]) ncclNetsVer[0] = 6;
}
if (ncclNets[0] == nullptr) {
goto fail;
}
// Check for CollNet
ncclCollNets[0] = getNcclCollNet_v10(netPluginLib);
if (ncclCollNets[0] == nullptr) {
ncclCollNets[0] = getNcclCollNet_v9(netPluginLib);
}
if (ncclCollNets[0] == nullptr) {
ncclCollNets[0] = getNcclCollNet_v8(netPluginLib);
}
if (ncclCollNets[0] == nullptr) {
ncclCollNets[0] = getNcclCollNet_v7(netPluginLib);
}
if (ncclCollNets[0] == nullptr) {
ncclCollNets[0] = getNcclCollNet_v6(netPluginLib);
}
++netPluginRefCount;
netPluginStatus = netPluginLoadSuccess;
comm->netPluginLoaded = 1;
exit:
pthread_mutex_unlock(&netPluginLock);
return ncclSuccess;
fail:
if (netPluginLib) NCCLCHECK(ncclClosePluginLib(netPluginLib));
netPluginStatus = netPluginLoadFailed;
goto exit;
}
ncclResult_t ncclNetPluginUnload(struct ncclComm* comm) {
pthread_mutex_lock(&netPluginLock);
if (comm->netPluginLoaded && 0 == (--netPluginRefCount)) {
if (ncclNets[0]) {
INFO(NCCL_NET, "NET/Plugin: Closing net plugin '%s'", ncclNets[0]->name);
}
if (ncclCollNets[0]) {
INFO(NCCL_NET, "NET/Plugin: Closing collnet plugin '%s'", ncclCollNets[0]->name);
}
NCCLCHECK(ncclClosePluginLib(netPluginLib));
netPluginLib = nullptr;
ncclNets[0] = nullptr;
ncclCollNets[0] = nullptr;
netPluginStatus = netPluginLoadReady;
comm->netPluginLoaded = 0;
for (int i = 0; i < NCCL_NET_MAX_PLUGINS; ++i)
ncclCollNetStates[i] = ncclNetStates[i] = ncclNetStateInit;
}
pthread_mutex_unlock(&netPluginLock);
return ncclSuccess;
}
ncclResult_t ncclNetCheckDeviceVersion(struct ncclComm* comm, ncclNet_t* net, int dev) {
ncclNetProperties_t props;
NCCLCHECK(net->getProperties(dev, &props));
ncclNetDeviceType type = props.netDeviceType;
if (type) switch (type) {
case NCCL_NET_DEVICE_UNPACK:
if (props.netDeviceVersion == NCCL_NET_DEVICE_UNPACK_VERSION) {
INFO(NCCL_INIT, "Using NCCL_NET_DEVICE_UNPACK net plugin version %d",
props.netDeviceVersion);
return ncclSuccess;
} else {
WARN("NCCL_DEVICE_UNPACK plugin has incompatible version %d, this NCCL build is compatible with %d, not using it",
props.netDeviceVersion, NCCL_NET_DEVICE_UNPACK_VERSION);
return ncclInternalError;
}
default:
WARN("Unknown device code index %d \n", type);
return ncclInternalError;
}
return ncclSuccess;
}
static ncclResult_t netGetState(int i, enum ncclNetState* state) {
pthread_mutex_lock(&netLock);
if (ncclNetStates[i] == ncclNetStateInit) {
int ndev;
if (ncclNets[i]->init(ncclDebugLog, ncclProfilerCallback) != ncclSuccess) ncclNetStates[i] = ncclNetStateDisabled;
else if (ncclNets[i]->devices(&ndev) != ncclSuccess || ndev <= 0) ncclNetStates[i] = ncclNetStateDisabled;
else ncclNetStates[i] = ncclNetStateEnabled;
}
*state = ncclNetStates[i];
pthread_mutex_unlock(&netLock);
return ncclSuccess;
}
static ncclResult_t collNetGetState(int i, enum ncclNetState* state) {
pthread_mutex_lock(&netLock);
if (ncclCollNetStates[i] == ncclNetStateInit) {
int ndev;
if (ncclCollNets[i]->init(ncclDebugLog) != ncclSuccess) ncclCollNetStates[i] = ncclNetStateDisabled;
else if (ncclCollNets[i]->devices(&ndev) != ncclSuccess || ndev <= 0) ncclCollNetStates[i] = ncclNetStateDisabled;
else ncclCollNetStates[i] = ncclNetStateEnabled;
}
*state = ncclCollNetStates[i];
pthread_mutex_unlock(&netLock);
return ncclSuccess;
}
ncclResult_t ncclNetInit(struct ncclComm* comm) {
// Initialize main communication network
const char* netName;
bool ok = false;
netName = comm->config.netName;
for (int i=0; i<3; i++) {
if (ncclNets[i] == nullptr) continue;
enum ncclNetState state;
NCCLCHECK(netGetState(i, &state));
if (state != ncclNetStateEnabled) continue;
if (netName && strcasecmp(netName, ncclNets[i]->name) != 0) continue;
if (ncclSuccess != ncclNetCheckDeviceVersion(comm, ncclNets[i], 0)) {
// Mismatched device plugin version
continue;
}
comm->ncclNet = ncclNets[i];
comm->ncclNetVer = ncclNetsVer[i];
ok = true;
if (ncclCollNets[i]) {
NCCLCHECK(collNetGetState(i, &state));
if (state == ncclNetStateEnabled) {
comm->ncclCollNet = ncclCollNets[i];
}
}
break;
}
if (!ok) {
WARN("Error: network %s not found.", netName ? netName : "");
return ncclInvalidUsage;
}
return ncclSuccess;
}
ncclResult_t ncclNetFinalize(struct ncclComm* comm) {
comm->ncclNet = nullptr;
comm->ncclCollNet = nullptr;
return ncclSuccess;
}
ncclResult_t ncclGpuGdrSupport(struct ncclComm* comm, int* gdrSupport) {
constexpr int GPU_BUF_SIZE = 2*1024*1024;
#if CUDART_VERSION >= 11030
// In CUDA 11.3 and later we can now query the cudaDevAttrGPUDirectRDMASupported attribute
int driverVersion;
CUDACHECK(cudaDriverGetVersion(&driverVersion));
if (driverVersion >= 11030) {
int cudaDev, attr = 0;
CUDACHECK(cudaGetDevice(&cudaDev));
CUDACHECK(cudaDeviceGetAttribute(&attr, cudaDevAttrGPUDirectRDMASupported, cudaDev));
*gdrSupport = attr;
return ncclSuccess;
}
#endif
static int gdrSupportMatrix[32] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 };
if (gdrSupportMatrix[comm->cudaDev] == -1) {
int netDevs;
NCCLCHECK(comm->ncclNet->devices(&netDevs));
gdrSupportMatrix[comm->cudaDev] = 0;
for (int dev=0; dev<netDevs; dev++) {
// Find a net device which is GDR-capable
ncclNetProperties_t props;
NCCLCHECK(comm->ncclNet->getProperties(dev, &props));
if ((props.ptrSupport & NCCL_PTR_CUDA) == 0) continue;
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
gdrSupportMatrix[comm->cudaDev] = 1;
break;
#endif
// Allocate memory on the GPU and try to register it on the NIC.
void *lComm = NULL, *sComm = NULL, *rComm = NULL;
ncclNetHandle_t handle;
char* gpuPtr = NULL;
void* mHandle = NULL;
ncclResult_t ret;
ncclDebugNoWarn = NCCL_NET;
NCCLCHECKGOTO(comm->ncclNet->listen(dev, &handle, &lComm), ret, cleanup1);
bool connected;
connected = false;
while (!connected) {
// If we're aborting now, skip to cleanup
if (__atomic_load_n(comm->abortFlag, __ATOMIC_ACQUIRE)) {
goto cleanup2;
}
if (sComm == NULL)
NCCLCHECKGOTO(comm->ncclNet->connect(dev, NULL, &handle, &sComm, NULL), ret, cleanup2);
if (rComm == NULL)
NCCLCHECKGOTO(comm->ncclNet->accept(lComm, &rComm, NULL), ret, cleanup2);
connected = (rComm != NULL) && (sComm != NULL);
}
NCCLCHECKGOTO(ncclCudaMalloc(&gpuPtr, GPU_BUF_SIZE), ret, cleanup2);
if (comm->ncclNet->regMr(sComm, gpuPtr, GPU_BUF_SIZE, NCCL_PTR_CUDA, &mHandle) == ncclSuccess) {
NCCLCHECK(comm->ncclNet->deregMr(sComm, mHandle));
NCCLCHECK(comm->ncclNet->regMr(rComm, gpuPtr, GPU_BUF_SIZE, NCCL_PTR_CUDA, &mHandle));
NCCLCHECK(comm->ncclNet->deregMr(rComm, mHandle));
gdrSupportMatrix[comm->cudaDev] = 1;
}
ncclDebugNoWarn = 0;
NCCLCHECK(ncclCudaFree(gpuPtr));
cleanup2:
if (rComm != NULL)
NCCLCHECK(comm->ncclNet->closeRecv(rComm));
if (sComm != NULL)
NCCLCHECK(comm->ncclNet->closeSend(sComm));
NCCLCHECK(comm->ncclNet->closeListen(lComm));
cleanup1:
break;
}
}
*gdrSupport = gdrSupportMatrix[comm->cudaDev];
return ncclSuccess;
}
src/plugin/net/net_v10.cc
+32
Просмотреть файл
@@ -0,0 +1,32 @@
/*************************************************************************
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "nccl_net.h"
#include "net_device.h"
#include "proxy.h"
static ncclNet_v10_t* ncclNet_v10;
static ncclCollNet_v10_t* ncclCollNet_v10;
ncclNet_t* getNcclNet_v10(void* lib) {
ncclNet_v10 = (ncclNet_v10_t*)dlsym(lib, "ncclNetPlugin_v10");
if (ncclNet_v10) {
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v10)", ncclNet_v10->name);
return ncclNet_v10;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v10 symbol.");
return nullptr;
}
ncclCollNet_t* getNcclCollNet_v10(void* lib) {
ncclCollNet_v10 = (ncclCollNet_v10_t*)dlsym(lib, "ncclCollNetPlugin_v10");
if (ncclCollNet_v10) {
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v10)", ncclNet_v10->name);
return ncclCollNet_v10;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v10 symbol.");
return nullptr;
}
src/plugin/net/net_v6.cc
+178
Просмотреть файл
@@ -0,0 +1,178 @@
/*************************************************************************
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "nccl_net.h"
#include "net_device.h"
#include "proxy.h"
#include "checks.h"
static ncclNet_t ncclNet;
static ncclCollNet_t ncclCollNet;
static ncclNet_v6_t* ncclNet_v6;
static ncclCollNet_v6_t* ncclCollNet_v6;
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
ncclNetProperties_v6_t p6;
ncclResult_t ans = ncclNet_v6->getProperties(dev, &p6);
if (ans != ncclSuccess) return ans;
props->name = p6.name;
props->pciPath = p6.pciPath;
props->guid = p6.guid;
props->ptrSupport = p6.ptrSupport;
props->regIsGlobal = 0;
props->forceFlush = 0;
props->speed = p6.speed;
props->port = p6.port;
props->maxComms = p6.maxComms;
props->maxRecvs = p6.maxRecvs;
props->latency = p6.latency;
props->netDeviceType = NCCL_NET_DEVICE_HOST;
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
props->vProps.ndevs = 1;
props->vProps.devs[0] = dev;
props->maxP2pBytes = MAX_NET_SIZE;
props->maxCollBytes = MAX_COLLNET_SIZE;
return ncclSuccess;
}
static ncclResult_t ncclNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
if (size >= 1UL<<31) return ncclInternalError;
return ncclNet_v6->regMr(comm, data, (int) size, type, mhandle);
}
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
return ncclNet_v6->connect(dev, handle, sendComm);
}
static ncclResult_t ncclNet_accept(void* listenComm, void** recvComm, ncclNetDeviceHandle_t** /*recvDevComm*/) {
return ncclNet_v6->accept(listenComm, recvComm);
}
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
int sizeInt;
if (size > MAX_NET_SIZE) return ncclInternalError;
sizeInt = (int)size;
ncclResult_t ans = ncclNet_v6->isend(sendComm, data, sizeInt, tag, mhandle, request);
return ans;
}
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
int sizesInt[NCCL_PROXY_MAX_SUBS];
//reset to nullptr if optional receive completion is set
if (*request == (void *)NCCL_NET_OPTIONAL_RECV_COMPLETION) *request = nullptr;
for (int i=0; i<n; i++) {
if (sizes[i] > MAX_NET_SIZE) return ncclInternalError;
sizesInt[i] = (int) sizes[i];
}
ncclResult_t ans = ncclNet_v6->irecv(recvComm, n, data, sizesInt, tags, mhandles, request);
return ans;
}
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
ncclNetProperties_v6_t p6;
ncclResult_t ans = ncclCollNet_v6->getProperties(dev, &p6);
if (ans != ncclSuccess) return ans;
props->name = p6.name;
props->pciPath = p6.pciPath;
props->guid = p6.guid;
props->ptrSupport = p6.ptrSupport;
props->regIsGlobal = 0;
props->forceFlush = 0;
props->speed = p6.speed;
props->port = p6.port;
props->maxComms = p6.maxComms;
props->maxRecvs = p6.maxRecvs;
props->latency = p6.latency;
props->netDeviceType = NCCL_NET_DEVICE_HOST;
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
props->vProps.ndevs = 1;
props->vProps.devs[0] = dev;
props->maxP2pBytes = MAX_NET_SIZE;
props->maxCollBytes = MAX_COLLNET_SIZE;
return ncclSuccess;
}
static ncclResult_t ncclCollNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
if (size >= 1UL<<31) return ncclInternalError;
return ncclCollNet_v6->regMr(comm, data, (int) size, type, mhandle);
}
static ncclResult_t ncclCollNet_iallreduce(void* collComm, void* sendData, void* recvData, size_t count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request) {
int countInt;
if (count > MAX_NET_SIZE) return ncclInternalError;
countInt = (int)count;
ncclResult_t ans = ncclCollNet_v6->iallreduce(collComm, sendData, recvData, countInt, dataType, redOp,
sendMhandle, recvMhandle, request);
return ans;
}
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
NCCLCHECK(ncclNet_v6->init(logfn));
ncclNet.devices = ncclNet_v6->devices;
ncclNet.getProperties = ncclNet_getProperties;
ncclNet.listen = ncclNet_v6->listen;
ncclNet.connect = ncclNet_connect;
ncclNet.accept = ncclNet_accept;
ncclNet.regMr = ncclNet_regMr;
ncclNet.regMrDmaBuf = ncclNet_v6->regMrDmaBuf;
ncclNet.deregMr = ncclNet_v6->deregMr;
ncclNet.isend = ncclNet_isend;
ncclNet.irecv = ncclNet_irecv;
ncclNet.iflush = ncclNet_v6->iflush;
ncclNet.test = ncclNet_v6->test;
ncclNet.closeSend = ncclNet_v6->closeSend;
ncclNet.closeRecv = ncclNet_v6->closeRecv;
ncclNet.closeListen = ncclNet_v6->closeListen;
ncclNet.getDeviceMr = NULL;
ncclNet.irecvConsumed = NULL;
ncclNet.makeVDevice = NULL;
return ncclSuccess;
}
ncclNet_t* getNcclNet_v6(void* lib) {
ncclNet_v6 = (ncclNet_v6_t*)dlsym(lib, "ncclNetPlugin_v6");
if (ncclNet_v6) {
ncclNet.name = ncclNet_v6->name;
ncclNet.init = ncclNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v6)", ncclNet_v6->name);
return &ncclNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v6 symbol.");
return nullptr;
}
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
NCCLCHECK(ncclCollNet_v6->init(logfn));
ncclCollNet.devices = ncclCollNet_v6->devices;
ncclCollNet.getProperties = ncclCollNet_getProperties;
ncclCollNet.listen = ncclCollNet_v6->listen;
ncclCollNet.connect = ncclCollNet_v6->connect;
ncclCollNet.reduceSupport = ncclCollNet_v6->reduceSupport;
ncclCollNet.regMr = ncclCollNet_regMr;
ncclCollNet.regMrDmaBuf = ncclCollNet_v6->regMrDmaBuf;
ncclCollNet.deregMr = ncclCollNet_v6->deregMr;
ncclCollNet.iallreduce = ncclCollNet_iallreduce;
ncclCollNet.iallgather = nullptr;
ncclCollNet.ireducescatter = nullptr;
ncclCollNet.iflush = ncclCollNet_v6->iflush;
ncclCollNet.test = ncclCollNet_v6->test;
ncclCollNet.closeColl = ncclCollNet_v6->closeColl;
ncclCollNet.closeListen = ncclCollNet_v6->closeListen;
return ncclSuccess;
}
ncclCollNet_t* getNcclCollNet_v6(void* lib) {
ncclCollNet_v6 = (ncclCollNet_v6_t*)dlsym(lib, "ncclCollNetPlugin_v6");
if (ncclCollNet_v6) {
ncclCollNet.name = ncclCollNet_v6->name;
ncclCollNet.init = ncclCollNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v6)", ncclCollNet_v6->name);
return &ncclCollNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol.");
return nullptr;
}
src/plugin/net/net_v7.cc
+174
Просмотреть файл
@@ -0,0 +1,174 @@
/*************************************************************************
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "nccl_net.h"
#include "net_device.h"
#include "proxy.h"
#include "checks.h"
static ncclNet_t ncclNet;
static ncclCollNet_t ncclCollNet;
static ncclNet_v7_t* ncclNet_v7;
static ncclCollNet_v7_t* ncclCollNet_v7;
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
ncclNetProperties_v7_t p7;
ncclResult_t ans = ncclNet_v7->getProperties(dev, &p7);
if (ans != ncclSuccess) return ans;
props->name = p7.name;
props->pciPath = p7.pciPath;
props->guid = p7.guid;
props->ptrSupport = p7.ptrSupport;
props->regIsGlobal = 0;
props->forceFlush = 0;
props->speed = p7.speed;
props->port = p7.port;
props->maxComms = p7.maxComms;
props->maxRecvs = p7.maxRecvs;
props->latency = p7.latency;
props->netDeviceType = p7.netDeviceType;
props->netDeviceVersion = p7.netDeviceVersion;
props->vProps.ndevs = 1;
props->vProps.devs[0] = dev;
props->maxP2pBytes = MAX_NET_SIZE;
props->maxCollBytes = MAX_COLLNET_SIZE;
return ncclSuccess;
}
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) {
return ncclNet_v7->connect(dev, handle, sendComm, sendDevComm);
}
static ncclResult_t ncclNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
if (size >= 1UL<<31) return ncclInternalError;
return ncclNet_v7->regMr(comm, data, (int) size, type, mhandle);
}
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
int sizeInt;
if (size > MAX_NET_SIZE) return ncclInternalError;
sizeInt = (int)size;
ncclResult_t ans = ncclNet_v7->isend(sendComm, data, sizeInt, tag, mhandle, request);
return ans;
}
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
int sizesInt[NCCL_PROXY_MAX_SUBS];
//reset to nullptr if optional receive completion is set
if (*request == (void *)NCCL_NET_OPTIONAL_RECV_COMPLETION) *request = nullptr;
for (int i=0; i<n; i++) {
if (sizes[i] > MAX_NET_SIZE) return ncclInternalError;
sizesInt[i] = (int) sizes[i];
}
ncclResult_t ans = ncclNet_v7->irecv(recvComm, n, data, sizesInt, tags, mhandles, request);
return ans;
}
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
ncclNetProperties_v7_t p7;
ncclResult_t ans = ncclCollNet_v7->getProperties(dev, &p7);
if (ans != ncclSuccess) return ans;
props->name = p7.name;
props->pciPath = p7.pciPath;
props->guid = p7.guid;
props->ptrSupport = p7.ptrSupport;
props->regIsGlobal = 0;
props->forceFlush = 0;
props->speed = p7.speed;
props->port = p7.port;
props->maxComms = p7.maxComms;
props->maxRecvs = p7.maxRecvs;
props->latency = p7.latency;
props->netDeviceType = NCCL_NET_DEVICE_HOST;
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
props->vProps.ndevs = 1;
props->vProps.devs[0] = dev;
props->maxP2pBytes = MAX_NET_SIZE;
props->maxCollBytes = MAX_COLLNET_SIZE;
return ncclSuccess;
}
static ncclResult_t ncclCollNet_regMr(void* comm, void* data, size_t size, int type, void** mhandle) {
if (size >= 1UL<<31) return ncclInternalError;
return ncclCollNet_v7->regMr(comm, data, (int) size, type, mhandle);
}
static ncclResult_t ncclCollNet_iallreduce(void* collComm, void* sendData, void* recvData, size_t count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request) {
int countInt;
if (count > MAX_NET_SIZE) return ncclInternalError;
countInt = (int)count;
ncclResult_t ans = ncclCollNet_v7->iallreduce(collComm, sendData, recvData, countInt, dataType, redOp,
sendMhandle, recvMhandle, request);
return ans;
}
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
NCCLCHECK(ncclNet_v7->init(logfn));
ncclNet.devices = ncclNet_v7->devices;
ncclNet.getProperties = ncclNet_getProperties; // ncclNet_v5->getProperties;
ncclNet.listen = ncclNet_v7->listen;
ncclNet.connect = ncclNet_connect;
ncclNet.accept = ncclNet_v7->accept;
ncclNet.regMr = ncclNet_regMr;
ncclNet.regMrDmaBuf = ncclNet_v7->regMrDmaBuf;
ncclNet.deregMr = ncclNet_v7->deregMr;
ncclNet.isend = ncclNet_isend;
ncclNet.irecv = ncclNet_irecv;
ncclNet.iflush = ncclNet_v7->iflush;
ncclNet.test = ncclNet_v7->test;
ncclNet.closeSend = ncclNet_v7->closeSend;
ncclNet.closeRecv = ncclNet_v7->closeRecv;
ncclNet.closeListen = ncclNet_v7->closeListen;
ncclNet.getDeviceMr = ncclNet_v7->getDeviceMr;
ncclNet.irecvConsumed = ncclNet_v7->irecvConsumed;
ncclNet.makeVDevice = NULL;
return ncclSuccess;
}
ncclNet_t* getNcclNet_v7(void* lib) {
ncclNet_v7 = (ncclNet_v7_t*)dlsym(lib, "ncclNetPlugin_v7");
if (ncclNet_v7) {
ncclNet.name = ncclNet_v7->name;
ncclNet.init = ncclNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v7)", ncclNet_v7->name);
return &ncclNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v7 symbol.");
return nullptr;
}
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
NCCLCHECK(ncclCollNet_v7->init(logfn));
ncclCollNet.devices = ncclCollNet_v7->devices;
ncclCollNet.getProperties = ncclCollNet_getProperties;
ncclCollNet.listen = ncclCollNet_v7->listen;
ncclCollNet.connect = ncclCollNet_v7->connect;
ncclCollNet.reduceSupport = ncclCollNet_v7->reduceSupport;
ncclCollNet.regMr = ncclCollNet_regMr;
ncclCollNet.regMrDmaBuf = ncclCollNet_v7->regMrDmaBuf;
ncclCollNet.deregMr = ncclCollNet_v7->deregMr;
ncclCollNet.iallreduce = ncclCollNet_iallreduce;
ncclCollNet.iallgather = nullptr;
ncclCollNet.ireducescatter = nullptr;
ncclCollNet.iflush = ncclCollNet_v7->iflush;
ncclCollNet.test = ncclCollNet_v7->test;
ncclCollNet.closeColl = ncclCollNet_v7->closeColl;
ncclCollNet.closeListen = ncclCollNet_v7->closeListen;
return ncclSuccess;
}
ncclCollNet_t* getNcclCollNet_v7(void* lib) {
ncclCollNet_v7 = (ncclCollNet_v7_t*)dlsym(lib, "ncclCollNetPlugin_v7");
if (ncclCollNet_v7) {
ncclCollNet.name = ncclCollNet_v7->name;
ncclCollNet.init = ncclCollNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v7)", ncclCollNet_v7->name);
return &ncclCollNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v7 symbol.");
return nullptr;
}
src/plugin/net/net_v8.cc
+196
Просмотреть файл
@@ -0,0 +1,196 @@
/*************************************************************************
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "nccl_net.h"
#include "net_device.h"
#include "proxy.h"
#include "checks.h"
static ncclNet_t ncclNet;
static ncclCollNet_t ncclCollNet;
static ncclNet_v8_t* ncclNet_v8;
static ncclCollNet_v8_t* ncclCollNet_v8;
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
ncclNetProperties_v8_t p8;
ncclResult_t ans = ncclNet_v8->getProperties(dev, &p8);
if (ans != ncclSuccess) return ans;
props->name = p8.name;
props->pciPath = p8.pciPath;
props->guid = p8.guid;
props->ptrSupport = p8.ptrSupport;
props->regIsGlobal = p8.regIsGlobal;
props->forceFlush = 0;
props->speed = p8.speed;
props->port = p8.port;
props->maxComms = p8.maxComms;
props->maxRecvs = p8.maxRecvs;
props->latency = p8.latency;
props->netDeviceType = p8.netDeviceType;
props->netDeviceVersion = p8.netDeviceVersion;
props->vProps.ndevs = 1;
props->vProps.devs[0] = dev;
props->maxP2pBytes = MAX_NET_SIZE;
props->maxCollBytes = MAX_COLLNET_SIZE;
return ncclSuccess;
}
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) {
return ncclNet_v8->connect(dev, handle, sendComm, sendDevComm);
}
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
int sizeInt;
if (size > MAX_NET_SIZE) return ncclInternalError;
sizeInt = (int)size;
ncclResult_t ans = ncclNet_v8->isend(sendComm, data, sizeInt, tag, mhandle, request);
return ans;
}
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
int sizesInt[NCCL_PROXY_MAX_SUBS];
//reset to nullptr if optional receive completion is set
if (*request == (void *)NCCL_NET_OPTIONAL_RECV_COMPLETION) *request = nullptr;
for (int i=0; i<n; i++) {
if (sizes[i] > MAX_NET_SIZE) return ncclInternalError;
sizesInt[i] = (int) sizes[i];
}
ncclResult_t ans = ncclNet_v8->irecv(recvComm, n, data, sizesInt, tags, mhandles, request);
return ans;
}
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
ncclNetProperties_v8_t p8;
ncclResult_t ans = ncclCollNet_v8->getProperties(dev, &p8);
if (ans != ncclSuccess) return ans;
props->name = p8.name;
props->pciPath = p8.pciPath;
props->guid = p8.guid;
props->ptrSupport = p8.ptrSupport;
props->regIsGlobal = p8.regIsGlobal;
props->forceFlush = 0;
props->speed = p8.speed;
props->port = p8.port;
props->maxComms = p8.maxComms;
props->maxRecvs = p8.maxRecvs;
props->latency = p8.latency;
props->netDeviceType = NCCL_NET_DEVICE_HOST;
props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION;
props->vProps.ndevs = 1;
props->vProps.devs[0] = dev;
props->maxP2pBytes = MAX_NET_SIZE;
props->maxCollBytes = MAX_COLLNET_SIZE;
return ncclSuccess;
}
static ncclResult_t ncclCollNet_iallreduce(void* collComm, void* sendData, void* recvData, size_t count,
ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request) {
int countInt;
if (count > MAX_NET_SIZE) return ncclInternalError;
countInt = (int)count;
ncclResult_t ans = ncclCollNet_v8->iallreduce(collComm, sendData, recvData, countInt, dataType, redOp,
sendMhandle, recvMhandle, request);
return ans;
}
static ncclResult_t ncclCollNet_iallgather (void* collComm, void* sendData, int nRecvParts, ncclNetSGE_t* recvParts,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
void* sendMhandle, void** request) {
ncclNetSGE_v8_t recvPartsInt;
if (nRecvParts > 1) return ncclInternalError;
if (recvParts->size > MAX_COLLNET_SIZE) return ncclInternalError;
recvPartsInt.mhandle = recvParts->mhandle;
recvPartsInt.address = recvParts->address;
recvPartsInt.size = (int)recvParts->size;
ncclResult_t ans = ncclCollNet_v8->iallgather(collComm, sendData, nRecvParts, &recvPartsInt,
bytesPerRank, windowOffset, windowBytes,
sendMhandle, request);
return ans;
}
static ncclResult_t ncclCollNet_ireducescatter(void* collComm, int nSendParts, ncclNetSGE_t* sendParts, void* recvData,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
ncclDataType_t dataType, ncclRedOp_t redOp,
void* recvMhandle, void** request) {
ncclNetSGE_v8_t sendPartsInt;
if (nSendParts > 1) return ncclInternalError;
if (sendParts->size > MAX_COLLNET_SIZE) return ncclInternalError;
sendPartsInt.mhandle = sendParts->mhandle;
sendPartsInt.address = sendParts->address;
sendPartsInt.size = (int)sendParts->size;
ncclResult_t ans = ncclCollNet_v8->ireducescatter(collComm, nSendParts, &sendPartsInt,
recvData, bytesPerRank, windowOffset, windowBytes,
dataType, redOp,
recvMhandle, request);
return ans;
}
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
NCCLCHECK(ncclNet_v8->init(logfn));
ncclNet.devices = ncclNet_v8->devices;
ncclNet.getProperties = ncclNet_getProperties;
ncclNet.listen = ncclNet_v8->listen;
ncclNet.connect = ncclNet_connect;
ncclNet.accept = ncclNet_v8->accept;
ncclNet.regMr = ncclNet_v8->regMr;
ncclNet.regMrDmaBuf = ncclNet_v8->regMrDmaBuf;
ncclNet.deregMr = ncclNet_v8->deregMr;
ncclNet.isend = ncclNet_isend;
ncclNet.irecv = ncclNet_irecv;
ncclNet.iflush = ncclNet_v8->iflush;
ncclNet.test = ncclNet_v8->test;
ncclNet.closeSend = ncclNet_v8->closeSend;
ncclNet.closeRecv = ncclNet_v8->closeRecv;
ncclNet.closeListen = ncclNet_v8->closeListen;
ncclNet.getDeviceMr = ncclNet_v8->getDeviceMr;
ncclNet.irecvConsumed = ncclNet_v8->irecvConsumed;
ncclNet.makeVDevice = NULL;
return ncclSuccess;
}
ncclNet_t* getNcclNet_v8(void* lib) {
ncclNet_v8 = (ncclNet_v8_t*)dlsym(lib, "ncclNetPlugin_v8");
if (ncclNet_v8) {
ncclNet.name = ncclNet_v8->name;
ncclNet.init = ncclNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v8)", ncclNet_v8->name);
return &ncclNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v8 symbol.");
return nullptr;
}
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
NCCLCHECK(ncclCollNet_v8->init(logfn));
ncclCollNet.devices = ncclCollNet_v8->devices;
ncclCollNet.getProperties = ncclCollNet_getProperties;
ncclCollNet.listen = ncclCollNet_v8->listen;
ncclCollNet.connect = ncclCollNet_v8->connect;
ncclCollNet.reduceSupport = ncclCollNet_v8->reduceSupport;
ncclCollNet.regMr = ncclCollNet_v8->regMr;
ncclCollNet.regMrDmaBuf = ncclCollNet_v8->regMrDmaBuf;
ncclCollNet.deregMr = ncclCollNet_v8->deregMr;
ncclCollNet.iallreduce = ncclCollNet_iallreduce;
ncclCollNet.iallgather = ncclCollNet_iallgather;
ncclCollNet.ireducescatter = ncclCollNet_ireducescatter;
ncclCollNet.iflush = ncclCollNet_v8->iflush;
ncclCollNet.test = ncclCollNet_v8->test;
ncclCollNet.closeColl = ncclCollNet_v8->closeColl;
ncclCollNet.closeListen = ncclCollNet_v8->closeListen;
return ncclSuccess;
}
ncclCollNet_t* getNcclCollNet_v8(void* lib) {
ncclCollNet_v8 = (ncclCollNet_v8_t*)dlsym(lib, "ncclCollNetPlugin_v8");
if (ncclCollNet_v8) {
ncclCollNet.name = ncclCollNet_v8->name;
ncclCollNet.init = ncclCollNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v8)", ncclCollNet_v8->name);
return &ncclCollNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v8 symbol.");
return nullptr;
}
src/plugin/net/net_v9.cc
+121
Просмотреть файл
@@ -0,0 +1,121 @@
/*************************************************************************
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "nccl_net.h"
#include "net_device.h"
#include "proxy.h"
#include "checks.h"
static ncclNet_t ncclNet;
static ncclCollNet_t ncclCollNet;
static ncclNet_v9_t* ncclNet_v9;
static ncclCollNet_v9_t* ncclCollNet_v9;
static ncclResult_t ncclNet_getProperties(int dev, ncclNetProperties_t* props) {
return ncclNet_v9->getProperties(dev, (ncclNetProperties_v9_t *)props);
}
static ncclResult_t ncclNet_isend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request) {
return ncclNet_v9->isend(sendComm, data, size, tag, mhandle, request);
}
static ncclResult_t ncclNet_irecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request) {
return ncclNet_v9->irecv(recvComm, n, data, sizes, tags, mhandles, request);
}
static ncclResult_t ncclNet_connect(int dev, ncclNetCommConfig_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) {
return ncclNet_v9->connect(dev, handle, sendComm, sendDevComm);
}
static ncclResult_t ncclNet_makeVDevice(int* d, ncclNetVDeviceProps_t* props) {
return ncclNet_v9->makeVDevice(d, (ncclNetVDeviceProps_v9_t*)props);
}
static ncclResult_t ncclCollNet_getProperties(int dev, ncclNetProperties_t* props) {
return ncclCollNet_v9->getProperties(dev, (ncclNetProperties_v9_t *)props);
}
static ncclResult_t ncclCollNet_iallgather(void* collComm, void* sendData, int nRecvParts, ncclNetSGE_t* recvParts,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
void* sendMhandle, void** request) {
return ncclCollNet_v9->iallgather(collComm, sendData, nRecvParts, (ncclNetSGE_v9_t*)recvParts, bytesPerRank,
windowOffset, windowBytes, sendMhandle, request);
}
static ncclResult_t ncclCollNet_ireducescatter(void* collComm, int nSendParts, ncclNetSGE_t* sendParts, void* recvData,
size_t bytesPerRank, size_t windowOffset, size_t windowBytes,
ncclDataType_t dataType, ncclRedOp_t redOp,
void* recvMhandle, void** request) {
return ncclCollNet_v9->ireducescatter(collComm, nSendParts, (ncclNetSGE_v9_t*)sendParts, recvData, bytesPerRank,
windowOffset, windowBytes, dataType, redOp, recvMhandle, request);
}
static ncclResult_t ncclNet_init(ncclDebugLogger_t logfn, ncclProfilerCallback_t proffn) {
NCCLCHECK(ncclNet_v9->init(logfn));
ncclNet.devices = ncclNet_v9->devices;
ncclNet.getProperties = ncclNet_getProperties;
ncclNet.listen = ncclNet_v9->listen;
ncclNet.connect = ncclNet_connect;
ncclNet.accept = ncclNet_v9->accept;
ncclNet.regMr = ncclNet_v9->regMr;
ncclNet.regMrDmaBuf = ncclNet_v9->regMrDmaBuf;
ncclNet.deregMr = ncclNet_v9->deregMr;
ncclNet.isend = ncclNet_isend;
ncclNet.irecv = ncclNet_irecv;
ncclNet.iflush = ncclNet_v9->iflush;
ncclNet.test = ncclNet_v9->test;
ncclNet.closeSend = ncclNet_v9->closeSend;
ncclNet.closeRecv = ncclNet_v9->closeRecv;
ncclNet.closeListen = ncclNet_v9->closeListen;
ncclNet.getDeviceMr = ncclNet_v9->getDeviceMr;
ncclNet.irecvConsumed = ncclNet_v9->irecvConsumed;
ncclNet.makeVDevice = (ncclNet_v9->makeVDevice) ? ncclNet_makeVDevice : nullptr;
return ncclSuccess;
}
ncclNet_t* getNcclNet_v9(void* lib) {
ncclNet_v9 = (ncclNet_v9_t*)dlsym(lib, "ncclNetPlugin_v9");
if (ncclNet_v9) {
ncclNet.name = ncclNet_v9->name;
ncclNet.init = ncclNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded net plugin %s (v9)", ncclNet_v9->name);
return &ncclNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclNetPlugin_v9 symbol.");
return nullptr;
}
static ncclResult_t ncclCollNet_init(ncclDebugLogger_t logfn) {
NCCLCHECK(ncclCollNet_v9->init(logfn));
ncclCollNet.devices = ncclCollNet_v9->devices;
ncclCollNet.getProperties = ncclCollNet_getProperties;
ncclCollNet.listen = ncclCollNet_v9->listen;
ncclCollNet.connect = ncclCollNet_v9->connect;
ncclCollNet.reduceSupport = ncclCollNet_v9->reduceSupport;
ncclCollNet.regMr = ncclCollNet_v9->regMr;
ncclCollNet.regMrDmaBuf = ncclCollNet_v9->regMrDmaBuf;
ncclCollNet.deregMr = ncclCollNet_v9->deregMr;
ncclCollNet.iallreduce = ncclCollNet_v9->iallreduce;
ncclCollNet.iallgather = ncclCollNet_iallgather;
ncclCollNet.ireducescatter = ncclCollNet_ireducescatter;
ncclCollNet.iflush = ncclCollNet_v9->iflush;
ncclCollNet.test = ncclCollNet_v9->test;
ncclCollNet.closeColl = ncclCollNet_v9->closeColl;
ncclCollNet.closeListen = ncclCollNet_v9->closeListen;
return ncclSuccess;
}
ncclCollNet_t* getNcclCollNet_v9(void* lib) {
ncclCollNet_v9 = (ncclCollNet_v9_t*)dlsym(lib, "ncclCollNetPlugin_v9");
if (ncclCollNet_v9) {
ncclCollNet.name = ncclCollNet_v9->name;
ncclCollNet.init = ncclCollNet_init;
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Loaded collnet plugin %s (v9)", ncclCollNet_v9->name);
return &ncclCollNet;
}
INFO(NCCL_INIT|NCCL_NET, "NET/Plugin: Failed to find ncclCollNetPlugin_v9 symbol.");
return nullptr;
}
src/plugin/plugin_open.cc
+134
Просмотреть файл
@@ -0,0 +1,134 @@
/*************************************************************************
* Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <dlfcn.h>
#include "debug.h"
#define MAX_STR_LEN 255
enum ncclPluginType {
ncclPluginTypeNet,
ncclPluginTypeTuner,
ncclPluginTypeProfiler,
};
#define NUM_LIBS 3
static void *libHandles[NUM_LIBS];
static const char *pluginNames[NUM_LIBS] = { "NET", "TUNER", "PROFILER" };
static const char *pluginPrefix[NUM_LIBS] = { "libnccl-net", "libnccl-tuner", "libnccl-profiler" };
static const char *pluginFallback[NUM_LIBS] = { "Using internal net plugin.", "Using internal tuner plugin.", "" };
static unsigned long subsys[NUM_LIBS] = { NCCL_INIT|NCCL_NET, NCCL_INIT|NCCL_TUNING, NCCL_INIT };
static void* tryOpenLib(char* name, int* err, char* errStr) {
*err = 0;
if (nullptr == name || strlen(name) == 0) {
return nullptr;
}
if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
name = nullptr;
}
void *handle = dlopen(name, RTLD_NOW | RTLD_LOCAL);
if (nullptr == handle) {
strncpy(errStr, dlerror(), MAX_STR_LEN);
errStr[MAX_STR_LEN] = '\0';
// "handle" and "name" won't be NULL at the same time.
// coverity[var_deref_model]
if (strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
*err = ENOENT;
}
}
return handle;
}
static void appendNameToList(char* nameList, int *nameListLen, char* name) {
snprintf(nameList, *nameListLen, " %s", name);
nameList += strlen(name) + 1;
*nameListLen -= strlen(name) + 1;
}
static void* openPluginLib(enum ncclPluginType type, const char* libName) {
int openErr, len = PATH_MAX;
char libName_[MAX_STR_LEN] = { 0 };
char openErrStr[MAX_STR_LEN + 1] = { 0 };
char eNoEntNameList[PATH_MAX] = { 0 };
if (libName && strlen(libName)) {
snprintf(libName_, MAX_STR_LEN, "%s", libName);
libHandles[type] = tryOpenLib(libName_, &openErr, openErrStr);
if (libHandles[type]) {
INFO(subsys[type], "%s/Plugin: Plugin name set by env to %s", pluginNames[type], libName_);
return libHandles[type];
}
if (openErr == ENOENT) {
appendNameToList(eNoEntNameList, &len, libName_);
} else {
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], openErrStr);
}
snprintf(libName_, MAX_STR_LEN, "%s-%s.so", pluginPrefix[type], libName);
libHandles[type] = tryOpenLib(libName_, &openErr, openErrStr);
if (libHandles[type]) {
INFO(subsys[type], "%s/Plugin: Plugin name set by env to %s", pluginNames[type], libName_);
return libHandles[type];
}
if (openErr == ENOENT) {
appendNameToList(eNoEntNameList, &len, libName_);
} else {
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], openErrStr);
}
} else {
snprintf(libName_, MAX_STR_LEN, "%s.so", pluginPrefix[type]);
libHandles[type] = tryOpenLib(libName_, &openErr, openErrStr);
if (libHandles[type]) {
return libHandles[type];
}
if (openErr == ENOENT) {
appendNameToList(eNoEntNameList, &len, libName_);
} else {
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], openErrStr);
}
}
if (strlen(eNoEntNameList)) {
INFO(subsys[type], "%s/Plugin: Could not find:%s. %s", pluginNames[type], eNoEntNameList, pluginFallback[type]);
} else if (strlen(pluginFallback[type])) {
INFO(subsys[type], "%s/Plugin: %s", pluginNames[type], pluginFallback[type]);
}
return nullptr;
}
void* ncclOpenNetPluginLib(const char* name) {
return openPluginLib(ncclPluginTypeNet, name);
}
void* ncclOpenTunerPluginLib(const char* name) {
return openPluginLib(ncclPluginTypeTuner, name);
}
void* ncclOpenProfilerPluginLib(const char* name) {
return openPluginLib(ncclPluginTypeProfiler, name);
}
void* ncclGetNetPluginLib(void) {
return libHandles[ncclPluginTypeNet];
}
ncclResult_t ncclClosePluginLib(void* handle) {
for (int l=0; l<NUM_LIBS; l++) {
if (libHandles[l] == handle) {
libHandles[l] = nullptr;
dlclose(handle);
return ncclSuccess;
}
}
return ncclInternalError;
}
src/misc/profiler.cc → src/plugin/profiler.cc
+182 -249
Просмотреть файл
@@ -11,183 +11,22 @@
#include "utils.h"
#include "proxy.h"
#include "profiler.h"
#include "transport.h"
#include "plugin.h"
#include <dlfcn.h>
extern ncclProfiler_t* getNcclProfiler_v1(void* lib);
extern ncclProfiler_t* getNcclProfiler_v2(void* lib);
extern ncclProfiler_t* getNcclProfiler_v3(void* lib);
static pthread_mutex_t profilerLock = PTHREAD_MUTEX_INITIALIZER;
static int profilerPluginRefCount;
static void* profilerPluginLib;
static ncclProfiler_t* ncclProfiler;
static ncclProfiler_v2_t ncclProfiler_v1_as_v2;
static ncclProfiler_v1_t* ncclProfiler_v1;
static uint8_t ncclStringToFunc(const char* func) {
if (0 == strcmp(func, "AllGather")) return ncclFuncAllGather;
if (0 == strcmp(func, "AllReduce")) return ncclFuncAllReduce;
if (0 == strcmp(func, "Broadcast")) return ncclFuncBroadcast;
if (0 == strcmp(func, "Recv")) return ncclFuncRecv;
if (0 == strcmp(func, "Reduce")) return ncclFuncReduce;
if (0 == strcmp(func, "ReduceScatter")) return ncclFuncReduceScatter;
if (0 == strcmp(func, "SendRecv")) return ncclFuncSendRecv;
return ncclFuncSend;
}
static uint8_t ncclStringToAlgo(const char* algo) {
if (0 == strcmp(algo, "TREE")) return NCCL_ALGO_TREE;
if (0 == strcmp(algo, "RING")) return NCCL_ALGO_RING;
if (0 == strcmp(algo, "COLLNET_DIRECT")) return NCCL_ALGO_COLLNET_DIRECT;
if (0 == strcmp(algo, "COLLNET_CHAIN")) return NCCL_ALGO_COLLNET_CHAIN;
if (0 == strcmp(algo, "NVLS")) return NCCL_ALGO_NVLS;
if (0 == strcmp(algo, "NVLS_TREE")) return NCCL_ALGO_NVLS_TREE;
return NCCL_ALGO_PAT;
}
static uint8_t ncclStringToProto(const char* proto) {
if (0 == strcmp(proto, "LL")) return NCCL_PROTO_LL;
if (0 == strcmp(proto, "LL128")) return NCCL_PROTO_LL128;
return NCCL_PROTO_SIMPLE;
}
static uint8_t ncclStringToDatatype(const char* dt) {
if (0 == strcmp(dt, "ncclInt8")) return ncclInt8;
if (0 == strcmp(dt, "ncclInt32")) return ncclInt32;
if (0 == strcmp(dt, "ncclUint32")) return ncclUint32;
if (0 == strcmp(dt, "ncclInt64")) return ncclInt64;
if (0 == strcmp(dt, "ncclUint64")) return ncclUint64;
if (0 == strcmp(dt, "ncclFloat16")) return ncclFloat16;
if (0 == strcmp(dt, "ncclFloat32")) return ncclFloat32;
#if defined(__CUDA_BF16_TYPES_EXIST__)
if (0 == strcmp(dt, "ncclBfloat16")) return ncclBfloat16;
#endif
return ncclFloat64;
}
static ncclResult_t ncclProfiler_v1_as_v2_startEvent(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr) {
ncclProfilerEventDescr_v1_t eDescr_v1 = { 0 };
eDescr_v1.type = eDescr->type;
eDescr_v1.parentObj = eDescr->parentObj;
eDescr_v1.rank = eDescr->rank;
switch(eDescr->type) {
case ncclProfileGroup: break;
case ncclProfileColl: {
eDescr_v1.coll.name = eDescr->coll.name;
eDescr_v1.coll.commHash = eDescr->coll.commHash;
eDescr_v1.coll.seqNumber = eDescr->coll.seqNumber;
eDescr_v1.coll.func = ncclStringToFunc(eDescr->coll.func);
eDescr_v1.coll.sendBuff = eDescr->coll.sendBuff;
eDescr_v1.coll.recvBuff = eDescr->coll.recvBuff;
eDescr_v1.coll.count = eDescr->coll.count;
eDescr_v1.coll.root = eDescr->coll.root;
eDescr_v1.coll.datatype = ncclStringToDatatype(eDescr->coll.datatype);
eDescr_v1.coll.op = 0; // removed in v2
eDescr_v1.coll.trafficBytes = eDescr->coll.trafficBytes;
eDescr_v1.coll.nMaxChannels = eDescr->coll.nMaxChannels;
eDescr_v1.coll.nWarps = eDescr->coll.nWarps;
eDescr_v1.coll.algo = ncclStringToAlgo(eDescr->coll.algo);
eDescr_v1.coll.proto = ncclStringToProto(eDescr->coll.proto);
} break;
case ncclProfileP2p: {
eDescr_v1.p2p.name = eDescr->p2p.name;
eDescr_v1.p2p.commHash = eDescr->p2p.commHash;
eDescr_v1.p2p.func = ncclStringToFunc(eDescr->p2p.func);
eDescr_v1.p2p.buff = eDescr->p2p.buff;
eDescr_v1.p2p.count = eDescr->p2p.count;
eDescr_v1.p2p.datatype = ncclStringToDatatype(eDescr->p2p.datatype);
eDescr_v1.p2p.peer = eDescr->p2p.peer;
} break;
case ncclProfileProxyOp: {
eDescr_v1.proxyOp.pid = eDescr->proxyOp.pid;
eDescr_v1.proxyOp.channelId = eDescr->proxyOp.channelId;
eDescr_v1.proxyOp.peer = eDescr->proxyOp.peer;
eDescr_v1.proxyOp.nSteps = eDescr->proxyOp.nSteps;
eDescr_v1.proxyOp.chunkSize = eDescr->proxyOp.chunkSize;
eDescr_v1.proxyOp.isSend = eDescr->proxyOp.isSend;
} break;
case ncclProfileProxyStep: {
eDescr_v1.proxyStep.step = eDescr->proxyStep.step;
} break;
case ncclProfileProxyCtrl: break;
default:;
}
return ncclProfiler_v1->startEvent(context, eHandle, &eDescr_v1);
}
static ncclResult_t ncclProfiler_v1_as_v2_init(void** context, int* eActivationMask) {
ncclProfiler_v1->init(context, eActivationMask);
ncclProfiler_v1_as_v2.startEvent = ncclProfiler_v1_as_v2_startEvent;
ncclProfiler_v1_as_v2.stopEvent = ncclProfiler_v1->stopEvent;
ncclProfiler_v1_as_v2.recordEventState = ncclProfiler_v1->recordEventState;
ncclProfiler_v1_as_v2.finalize = ncclProfiler_v1->finalize;
return ncclSuccess;
}
#define MAX_STR_LEN 256
static void* tryOpenLib(char* name, int *err, char* errStr) {
if (nullptr == name || strlen(name) == 0) {
return nullptr;
}
if (strncasecmp(name, "STATIC_PLUGIN", strlen(name)) == 0) {
name = nullptr;
}
void *handle = dlopen(name, RTLD_NOW | RTLD_LOCAL);
if (nullptr == handle) {
strncpy(errStr, dlerror(), MAX_STR_LEN);
errStr[MAX_STR_LEN] = 0;
if (name && strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
*err = ENOENT;
}
}
return handle;
}
static char* tryOpenLibCheck(int openErr, char* openErrStr, char* nameList, int *nameListLen, char* name) {
if (openErr == ENOENT) {
snprintf(nameList, *nameListLen, " %s", name);
nameList += strlen(name) + 1;
*nameListLen -= strlen(name) + 1;
return nameList;
}
INFO(NCCL_ENV, "PROFILER/Plugin: %s", openErrStr);
return nameList;
}
static void* openProfilerPluginLib(char* couldNotFindNames, int len) {
int openErr;
void *pluginLib;
char profilerPluginLibName[PATH_MAX];
char openErrStr[MAX_STR_LEN + 1] = { 0 };
const char *envProfilerPluginName = getenv("NCCL_PROFILER_PLUGIN");
if (envProfilerPluginName && strlen(envProfilerPluginName)) {
snprintf(profilerPluginLibName, PATH_MAX, "%s", envProfilerPluginName);
pluginLib = tryOpenLib(profilerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: Plugin name set by env to %s", profilerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, profilerPluginLibName);
pluginLib = tryOpenLib(profilerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: Plugin name set by env to %s", profilerPluginLibName);
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, profilerPluginLibName);
} else {
snprintf(profilerPluginLibName, PATH_MAX, "libnccl-profiler.so");
pluginLib = tryOpenLib(profilerPluginLibName, &openErr, openErrStr);
if (pluginLib) {
return pluginLib;
}
couldNotFindNames = tryOpenLibCheck(openErr, openErrStr, couldNotFindNames, &len, profilerPluginLibName);
}
return nullptr;
}
enum {
profilerPluginLoadFailed = -1,
profilerPluginLoadReady = 0,
@@ -196,43 +35,31 @@ enum {
static int profilerPluginStatus = profilerPluginLoadReady;
static pid_t pid;
#define MAX_PLUGIN_LOAD 2
static ncclResult_t ncclProfilerPluginLoad(void) {
if (profilerPluginLoadFailed == profilerPluginStatus) {
return ncclSuccess;
}
char couldNotFindNames[MAX_PLUGIN_LOAD * PATH_MAX] = { 0 };
pthread_mutex_lock(&profilerLock);
if (profilerPluginLoadSuccess == profilerPluginStatus) {
++profilerPluginRefCount;
goto exit;
}
profilerPluginLib = openProfilerPluginLib(couldNotFindNames, MAX_PLUGIN_LOAD * PATH_MAX);
profilerPluginLib = ncclOpenProfilerPluginLib(ncclGetEnv("NCCL_PROFILER_PLUGIN"));
if (profilerPluginLib == nullptr) {
if (strlen(couldNotFindNames)) {
INFO(NCCL_ENV, "PROFILER/Plugin: Could not find:%s.", couldNotFindNames);
}
goto fail;
}
ncclProfiler = (ncclProfiler_v2_t*)dlsym(profilerPluginLib, "ncclProfiler_v2");
ncclProfiler = getNcclProfiler_v3(profilerPluginLib);
if (ncclProfiler == nullptr) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v2.");
ncclProfiler_v1 = (ncclProfiler_v1_t*)dlsym(profilerPluginLib, "ncclProfiler_v1");
if (ncclProfiler_v1 == nullptr) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v1.");
goto fail;
} else {
ncclProfiler = &ncclProfiler_v1_as_v2;
ncclProfiler_v1_as_v2.name = ncclProfiler_v1->name;
ncclProfiler_v1_as_v2.init = ncclProfiler_v1_as_v2_init;
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded ncclProfiler_v1.");
}
} else {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded ncclProfiler_v2.");
ncclProfiler = getNcclProfiler_v2(profilerPluginLib);
}
if (ncclProfiler == NULL) {
ncclProfiler = getNcclProfiler_v1(profilerPluginLib);
}
if (ncclProfiler == NULL) {
goto fail;
}
++profilerPluginRefCount;
@@ -248,7 +75,7 @@ exit:
pthread_mutex_unlock(&profilerLock);
return ncclSuccess;
fail:
if (profilerPluginLib) dlclose(profilerPluginLib);
if (profilerPluginLib) NCCLCHECK(ncclClosePluginLib(profilerPluginLib));
profilerPluginStatus = profilerPluginLoadFailed;
goto exit;
}
@@ -257,7 +84,7 @@ static ncclResult_t ncclProfilerPluginUnload(void) {
pthread_mutex_lock(&profilerLock);
if (0 == (--profilerPluginRefCount)) {
INFO(NCCL_ENV, "PROFILER/Plugin: Closing profiler plugin %s", ncclProfiler->name);
dlclose(profilerPluginLib);
NCCLCHECK(ncclClosePluginLib(profilerPluginLib));
profilerPluginLib = nullptr;
ncclProfiler = nullptr;
profilerPluginStatus = profilerPluginLoadReady;
@@ -270,6 +97,11 @@ static ncclResult_t ncclProfilerPluginUnload(void) {
#include "timer.h"
#if ENABLE_TIMER
// These counters are used to measure profiler overheads for different part of the code
// These counters are only useful/meaningful in controlled test environments where there
// is only one thread updating each set of counters, i.e., every communicator has its
// own proxy thread and the network uses only one thread to make progress (this is true
// for net_ib plugin but might not be true for net_socket plugin).
static int64_t elapsedCount;
static int64_t initCount, finalizeCount;
static int64_t groupStartCount, groupStopCount;
@@ -325,15 +157,14 @@ static double proxyOpRecordTs[2], proxyStepRecordTs[2], proxyCtrlRecordTs[2];
#endif
static int eActivationMask; // Set by profiler
static int eActivationMaskGroup; // Cached for current group
int ncclProfilerEventMask; // Set by profiler
ncclResult_t ncclProfilerPluginInit(struct ncclComm* comm) {
TIME_START_EVENT(elapsed);
TIME_START_EVENT(init);
ncclProfilerPluginLoad();
if (__builtin_expect(ncclProfiler != NULL, 0)) {
int err = ncclProfiler->init(&comm->profilerContext, &eActivationMask);
int err = ncclProfiler->init(&comm->profilerContext, &ncclProfilerEventMask);
if (err) {
WARN("Profiler init failed with error (%d). Continue without profiler.", err);
ncclProfiler = NULL;
@@ -357,9 +188,29 @@ ncclResult_t ncclProfilerPluginFinalize(struct ncclComm* comm) {
ncclResult_t ncclProfilerStartGroupEvent(struct ncclKernelPlan* plan) {
TIME_START_EVENT(groupStart);
eActivationMaskGroup = __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (eActivationMaskGroup & (ncclProfileColl | ncclProfileP2p | ncclProfileProxyOp | ncclProfileProxyStep)) {
// Check if any collective in the plan has a set event activation mask
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
int eActivationMask_ = 0;
while (ct) {
if (ct->eActivationMask) {
eActivationMask_ = ct->eActivationMask;
goto startGroup;
}
ct = ct->next;
}
// Check if any pt2pt in the plan has a set event activation mask
while (pt) {
if (pt->eActivationMask) {
eActivationMask_ = pt->eActivationMask;
goto startGroup;
}
pt = pt->next;
}
startGroup:
if (eActivationMask_ & (ncclProfileGroup | ncclProfileColl | ncclProfileP2p | ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileKernelCh | ncclProfileNetPlugin)) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileGroup;
ncclProfiler->startEvent(plan->comm->profilerContext, &plan->groupEventHandle, &eDescr);
@@ -380,52 +231,63 @@ ncclResult_t ncclProfilerStopGroupEvent(struct ncclKernelPlan* plan) {
ncclResult_t ncclProfilerStartTaskEvents(struct ncclKernelPlan* plan) {
TIME_START_EVENT(taskStart);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
int enable = eActivationMaskGroup & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileColl);
if (plan->groupEventHandle && enable) {
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
while (ct) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileColl;
eDescr.parentObj = plan->groupEventHandle;
eDescr.rank = plan->comm->rank;
eDescr.coll.name = plan->comm->commName;
eDescr.coll.commHash = plan->comm->commHash;
eDescr.coll.seqNumber = plan->comm->seqNumber[ct->func]++;
eDescr.coll.func = ncclFuncToString(ct->func);
eDescr.coll.sendBuff = ct->sendbuff;
eDescr.coll.recvBuff = ct->recvbuff;
eDescr.coll.count = ct->count;
eDescr.coll.root = ct->root;
eDescr.coll.datatype = ncclDatatypeToString(ct->datatype);
eDescr.coll.trafficBytes = ct->trafficBytes;
eDescr.coll.nMaxChannels = ct->nMaxChannels;
eDescr.coll.nWarps = ct->nWarps;
eDescr.coll.algo = ncclAlgoToString(ct->algorithm);
eDescr.coll.proto = ncclProtoToString(ct->protocol);
ncclProfiler->startEvent(plan->comm->profilerContext, &ct->eventHandle, &eDescr);
// update collective task with group event activation mask
ct->eActivationMask = eActivationMaskGroup;
ct = ct->next;
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
while (ct) {
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (plan->groupEventHandle) {
int enable = ct->eActivationMask & (ncclProfileColl | ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileKernelCh | ncclProfileNetPlugin);
if (enable) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileColl;
eDescr.parentObj = plan->groupEventHandle;
eDescr.rank = plan->comm->rank;
eDescr.coll.name = plan->comm->commName;
eDescr.coll.commHash = plan->comm->commHash;
eDescr.coll.seqNumber = plan->comm->seqNumber[ct->func];
eDescr.coll.func = ncclFuncToString(ct->func);
eDescr.coll.sendBuff = ct->sendbuff;
eDescr.coll.recvBuff = ct->recvbuff;
eDescr.coll.count = ct->count;
eDescr.coll.root = ct->root;
eDescr.coll.datatype = ncclDatatypeToString(ct->datatype);
eDescr.coll.nMaxChannels = ct->nMaxChannels;
eDescr.coll.nWarps = ct->nWarps;
eDescr.coll.algo = ncclAlgoToString(ct->algorithm);
eDescr.coll.proto = ncclProtoToString(ct->protocol);
ncclProfiler->startEvent(plan->comm->profilerContext, &ct->eventHandle, &eDescr);
}
}
}
// comm->seqNumber values are updated even if the plugin is not active, since they are used by RAS as well.
// The test for "persistent" is a workaround for graph-captured collectives. In their case this function may not be
// consistently invoked on all the ranks, which would lead to mismatched counter values and thus false-positive
// reports from RAS. Instead, we choose not to include graph-captured collectives in our counts. An exception is
// made if ncclProfileKernelCh profiler events are active, as they result in proxy events always being added, which
// gives the consistency.
if (!plan->persistent || (__builtin_expect(ncclProfiler != NULL, 0) && plan->groupEventHandle &&
(ct->eActivationMask & ncclProfileKernelCh)))
plan->comm->seqNumber[ct->func]++;
ct = ct->next;
}
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (plan->groupEventHandle) {
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
while (pt) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileP2p;
eDescr.parentObj = plan->groupEventHandle;
eDescr.rank = plan->comm->rank;
eDescr.p2p.name = plan->comm->commName;
eDescr.p2p.commHash = plan->comm->commHash;
eDescr.p2p.func = ncclFuncToString(pt->func);
eDescr.p2p.buff = pt->buff;
eDescr.p2p.count = pt->count;
eDescr.p2p.datatype = ncclDatatypeToString(pt->datatype);
eDescr.p2p.peer = pt->root;
ncclProfiler->startEvent(plan->comm->profilerContext, &pt->eventHandle, &eDescr);
// update collective task with group event activation mask
pt->eActivationMask = eActivationMaskGroup;
int enable = pt->eActivationMask & (ncclProfileP2p | ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileKernelCh);
if (enable) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileP2p;
eDescr.parentObj = plan->groupEventHandle;
eDescr.rank = plan->comm->rank;
eDescr.p2p.name = plan->comm->commName;
eDescr.p2p.commHash = plan->comm->commHash;
eDescr.p2p.func = ncclFuncToString(pt->func);
eDescr.p2p.buff = pt->buff;
eDescr.p2p.count = pt->count;
eDescr.p2p.datatype = ncclDatatypeToString(pt->datatype);
eDescr.p2p.peer = pt->root;
ncclProfiler->startEvent(plan->comm->profilerContext, &pt->eventHandle, &eDescr);
}
pt = pt->next;
}
}
@@ -437,16 +299,15 @@ ncclResult_t ncclProfilerStartTaskEvents(struct ncclKernelPlan* plan) {
ncclResult_t ncclProfilerStopTaskEvents(struct ncclKernelPlan* plan) {
TIME_START_EVENT(taskStop);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
int enable = eActivationMaskGroup & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileColl);
if (plan->groupEventHandle && enable) {
if (plan->groupEventHandle) {
struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue);
while (ct) {
ncclProfiler->stopEvent(ct->eventHandle);
if (ct->eventHandle) ncclProfiler->stopEvent(ct->eventHandle);
ct = ct->next;
}
struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue);
while (pt) {
ncclProfiler->stopEvent(pt->eventHandle);
if (pt->eventHandle) ncclProfiler->stopEvent(pt->eventHandle);
pt = pt->next;
}
}
@@ -464,7 +325,7 @@ ncclResult_t ncclProfilerStartSendProxyOpEvent(int s, struct ncclProxyArgs* args
TIME_START_EVENT(proxyOpStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileProxyOp)) {
if (sub->eActivationMask & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileNetPlugin)) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyOp;
eDescr.parentObj = sub->taskEventHandle;
@@ -486,7 +347,7 @@ ncclResult_t ncclProfilerStartRecvProxyOpEvent(int s, struct ncclProxyArgs* args
TIME_START_EVENT(proxyOpStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileProxyOp)) {
if (sub->eActivationMask & (ncclProfileProxyOp | ncclProfileProxyStep | ncclProfileNetPlugin)) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyOp;
eDescr.parentObj = sub->taskEventHandle;
@@ -519,7 +380,7 @@ ncclResult_t ncclProfilerStartSendProxyStepEvent(int s, struct ncclProxyArgs* ar
TIME_START_EVENT(proxyStepStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->opEventHandle && (sub->eActivationMask & ncclProfileProxyStep)) {
if (sub->opEventHandle && (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileNetPlugin))) {
int step_ = DIVUP(stepId, args->sliceSteps);
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
@@ -537,7 +398,7 @@ ncclResult_t ncclProfilerStartRecvProxyStepEvent(int s, struct ncclProxyArgs* ar
TIME_START_EVENT(proxyStepStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->opEventHandle && (sub->eActivationMask & ncclProfileProxyStep)) {
if (sub->opEventHandle && (sub->eActivationMask & (ncclProfileProxyStep | ncclProfileNetPlugin))) {
int step_ = DIVUP(stepId, args->sliceSteps);
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
@@ -569,7 +430,7 @@ ncclResult_t ncclProfilerStartProxyCtrlEvent(void* profilerContext, void** eHand
TIME_START_EVENT(proxyCtrlStart);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
// for proxy control events we allow profiling mode to change on a per event basis
int eActivationMaskProxy = __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED);
int eActivationMaskProxy = __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED);
if (eActivationMaskProxy & ncclProfileProxyCtrl) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyCtrl;
@@ -592,6 +453,30 @@ ncclResult_t ncclProfilerStopProxyCtrlEvent(void* eHandle) {
return ncclSuccess;
}
ncclResult_t ncclProfilerStartKernelChEvent(struct ncclProxyArgs* args, int s) {
if (__builtin_expect(ncclProfiler != NULL, 0)) {
struct ncclProxySubArgs* sub = &args->subs[s];
if (sub->eActivationMask & ncclProfileKernelCh) {
ncclProfilerEventDescr_t eDescr = { };
eDescr.type = ncclProfileKernelCh;
eDescr.parentObj = sub->taskEventHandle;
eDescr.kernelCh.channelId = sub->channelId;
ncclProfiler->startEvent(sub->profilerContext, &sub->kernelEventHandle, &eDescr);
}
}
return ncclSuccess;
}
ncclResult_t ncclProfilerStopKernelChEvent(struct ncclProxyArgs* args, int s) {
if (__builtin_expect(ncclProfiler != NULL, 0)) {
struct ncclProxySubArgs* sub = &args->subs[s];
if (sub->kernelEventHandle) {
ncclProfiler->stopEvent(sub->kernelEventHandle);
}
}
return ncclSuccess;
}
ncclResult_t ncclProfilerRecordProxyOpEventState(int s, struct ncclProxyArgs* args, int steps, size_t transSize, ncclProfilerEventState_t eState) {
TIME_START_EVENT(proxyOpRecord);
struct ncclProxySubArgs* sub = &args->subs[s];
@@ -620,7 +505,7 @@ ncclResult_t ncclProfilerRecordProxyStepEventState(int s, struct ncclProxyArgs*
ncclResult_t ncclProfilerRecordProxyCtrlEventState(void* eHandle, int appended, ncclProfilerEventState_t eState) {
TIME_START_EVENT(proxyCtrlRecord);
if (__builtin_expect(ncclProfiler != NULL, 0) && eHandle && __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED) & ncclProfileProxyCtrl) {
if (__builtin_expect(ncclProfiler != NULL, 0) && eHandle && __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED) & ncclProfileProxyCtrl) {
ncclProfilerEventStateArgs_t args = { };
args.proxyCtrl.appendedProxyOps = appended;
ncclProfiler->recordEventState(eHandle, eState, &args);
@@ -633,3 +518,51 @@ ncclResult_t ncclProfilerAddPidToProxyOp(struct ncclProxyOp* op) {
op->pid = pid;
return ncclSuccess;
}
static pthread_mutex_t proxyProfilerConnectLock = PTHREAD_MUTEX_INITIALIZER;
static ncclResult_t proxyProfilerConnect(struct ncclComm* comm, struct ncclProxyOp* op) {
ncclResult_t ret = ncclSuccess;
pthread_mutex_lock(&proxyProfilerConnectLock);
if (comm->profiler.initialized) goto exit;
for (int c = 0; c < MAXCHANNELS; c++) {
NCCLCHECKGOTO(ncclProxyConnect(comm, TRANSPORT_PROFILER, 0, comm->rank, &comm->profiler.sendProxyConn[c]), ret, exit);
NCCLCHECKGOTO(ncclProxyCallBlocking(comm, &comm->profiler.sendProxyConn[c], ncclProxyMsgConnect, NULL, 0, NULL, 0), ret, exit);
NCCLCHECKGOTO(ncclProxyConnect(comm, TRANSPORT_PROFILER, 0, comm->rank, &comm->profiler.recvProxyConn[c]), ret, exit);
NCCLCHECKGOTO(ncclProxyCallBlocking(comm, &comm->profiler.recvProxyConn[c], ncclProxyMsgConnect, NULL, 0, NULL, 0), ret, exit);
}
comm->profiler.initialized = true;
exit:
pthread_mutex_unlock(&proxyProfilerConnectLock);
return ret;
}
bool ncclProfilerNeedsProxy(struct ncclComm* comm, struct ncclProxyOp* op) {
bool enabled = ncclProfilerPluginLoaded() && (op->eActivationMask & ncclProfileKernelCh);
if (enabled && !comm->profiler.initialized) (void)proxyProfilerConnect(comm, op);
return enabled;
}
bool ncclProfilerPluginLoaded(void) {
return (__builtin_expect(ncclProfiler != NULL, 0));
}
ncclResult_t ncclProfilerCallback(void** eHandle, int type, void* pHandle, int64_t pluginId, void* extData) {
if (__builtin_expect(ncclProfiler != NULL, 0)) {
struct ncclProxySubArgs* sub = (struct ncclProxySubArgs*)pHandle;
if (type == 0) { // start
if (sub->eActivationMask & ncclProfileNetPlugin) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileNetPlugin;
eDescr.parentObj = sub->stepEventHandles[sub->profilerSteps%NCCL_STEPS];
eDescr.rank = sub->rank;
eDescr.netPlugin.id = pluginId;
eDescr.netPlugin.data = extData;
ncclProfiler->startEvent(sub->profilerContext, eHandle, &eDescr);
}
} else { // stop
ncclProfiler->stopEvent(*eHandle);
}
}
return ncclSuccess;
}
src/plugin/profiler/profiler_v1.cc
+133
Просмотреть файл
@@ -0,0 +1,133 @@
/*************************************************************************
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "comm.h"
#include "nccl_profiler.h"
#include "checks.h"
static ncclProfiler_t ncclProfiler;
static ncclProfiler_v1_t* ncclProfiler_v1;
static uint8_t ncclStringToFunc(const char* func) {
if (0 == strcmp(func, "AllGather")) return ncclFuncAllGather;
if (0 == strcmp(func, "AllReduce")) return ncclFuncAllReduce;
if (0 == strcmp(func, "Broadcast")) return ncclFuncBroadcast;
if (0 == strcmp(func, "Recv")) return ncclFuncRecv;
if (0 == strcmp(func, "Reduce")) return ncclFuncReduce;
if (0 == strcmp(func, "ReduceScatter")) return ncclFuncReduceScatter;
if (0 == strcmp(func, "SendRecv")) return ncclFuncSendRecv;
return ncclFuncSend;
}
static uint8_t ncclStringToAlgo(const char* algo) {
if (0 == strcmp(algo, "TREE")) return NCCL_ALGO_TREE;
if (0 == strcmp(algo, "RING")) return NCCL_ALGO_RING;
if (0 == strcmp(algo, "COLLNET_DIRECT")) return NCCL_ALGO_COLLNET_DIRECT;
if (0 == strcmp(algo, "COLLNET_CHAIN")) return NCCL_ALGO_COLLNET_CHAIN;
if (0 == strcmp(algo, "NVLS")) return NCCL_ALGO_NVLS;
if (0 == strcmp(algo, "NVLS_TREE")) return NCCL_ALGO_NVLS_TREE;
return NCCL_ALGO_PAT;
}
static uint8_t ncclStringToProto(const char* proto) {
if (0 == strcmp(proto, "LL")) return NCCL_PROTO_LL;
if (0 == strcmp(proto, "LL128")) return NCCL_PROTO_LL128;
return NCCL_PROTO_SIMPLE;
}
static uint8_t ncclStringToDatatype(const char* dt) {
if (0 == strcmp(dt, "ncclInt8")) return ncclInt8;
if (0 == strcmp(dt, "ncclInt32")) return ncclInt32;
if (0 == strcmp(dt, "ncclUint32")) return ncclUint32;
if (0 == strcmp(dt, "ncclInt64")) return ncclInt64;
if (0 == strcmp(dt, "ncclUint64")) return ncclUint64;
if (0 == strcmp(dt, "ncclFloat16")) return ncclFloat16;
if (0 == strcmp(dt, "ncclFloat32")) return ncclFloat32;
#if defined(__CUDA_BF16_TYPES_EXIST__)
if (0 == strcmp(dt, "ncclBfloat16")) return ncclBfloat16;
#endif
return ncclFloat64;
}
static ncclResult_t ncclProfiler_startEvent(void* context, void** eHandle, ncclProfilerEventDescr_t* eDescr) {
ncclProfilerEventDescr_v1_t eDescr_v1 = { 0 };
eDescr_v1.type = eDescr->type;
eDescr_v1.parentObj = eDescr->parentObj;
eDescr_v1.rank = eDescr->rank;
switch(eDescr->type) {
case ncclProfileGroup: break;
case ncclProfileColl: {
eDescr_v1.coll.name = eDescr->coll.name;
eDescr_v1.coll.commHash = eDescr->coll.commHash;
eDescr_v1.coll.seqNumber = eDescr->coll.seqNumber;
eDescr_v1.coll.func = ncclStringToFunc(eDescr->coll.func);
eDescr_v1.coll.sendBuff = eDescr->coll.sendBuff;
eDescr_v1.coll.recvBuff = eDescr->coll.recvBuff;
eDescr_v1.coll.count = eDescr->coll.count;
eDescr_v1.coll.root = eDescr->coll.root;
eDescr_v1.coll.datatype = ncclStringToDatatype(eDescr->coll.datatype);
eDescr_v1.coll.op = 0; // removed in v2
eDescr_v1.coll.trafficBytes = 0; // removed in v3
eDescr_v1.coll.nMaxChannels = eDescr->coll.nMaxChannels;
eDescr_v1.coll.nWarps = eDescr->coll.nWarps;
eDescr_v1.coll.algo = ncclStringToAlgo(eDescr->coll.algo);
eDescr_v1.coll.proto = ncclStringToProto(eDescr->coll.proto);
} break;
case ncclProfileP2p: {
eDescr_v1.p2p.name = eDescr->p2p.name;
eDescr_v1.p2p.commHash = eDescr->p2p.commHash;
eDescr_v1.p2p.func = ncclStringToFunc(eDescr->p2p.func);
eDescr_v1.p2p.buff = eDescr->p2p.buff;
eDescr_v1.p2p.count = eDescr->p2p.count;
eDescr_v1.p2p.datatype = ncclStringToDatatype(eDescr->p2p.datatype);
eDescr_v1.p2p.peer = eDescr->p2p.peer;
} break;
case ncclProfileProxyOp: {
eDescr_v1.proxyOp.pid = eDescr->proxyOp.pid;
eDescr_v1.proxyOp.channelId = eDescr->proxyOp.channelId;
eDescr_v1.proxyOp.peer = eDescr->proxyOp.peer;
eDescr_v1.proxyOp.nSteps = eDescr->proxyOp.nSteps;
eDescr_v1.proxyOp.chunkSize = eDescr->proxyOp.chunkSize;
eDescr_v1.proxyOp.isSend = eDescr->proxyOp.isSend;
} break;
case ncclProfileProxyStep: {
eDescr_v1.proxyStep.step = eDescr->proxyStep.step;
} break;
case ncclProfileProxyCtrl: break;
case ncclProfileKernelCh:
case ncclProfileNetPlugin: {
*eHandle = NULL;
return ncclSuccess;
}
default:;
}
return ncclProfiler_v1->startEvent(context, eHandle, &eDescr_v1);
}
static ncclResult_t ncclProfiler_recordEventState(void* eHandle, ncclProfilerEventState_t eState, ncclProfilerEventStateArgs_t* eStateArgs) {
return ncclProfiler_v1->recordEventState(eHandle, eState, (ncclProfilerEventStateArgs_v1_t*)eStateArgs);
}
static ncclResult_t ncclProfiler_init(void** context, int* eActivationMask) {
NCCLCHECK(ncclProfiler_v1->init(context, eActivationMask));
ncclProfiler.startEvent = ncclProfiler_startEvent;
ncclProfiler.stopEvent = ncclProfiler_v1->stopEvent;
ncclProfiler.recordEventState = ncclProfiler_recordEventState;
ncclProfiler.finalize = ncclProfiler_v1->finalize;
return ncclSuccess;
}
ncclProfiler_t* getNcclProfiler_v1(void* lib) {
ncclProfiler_v1 = (ncclProfiler_v1_t*)dlsym(lib, "ncclProfiler_v1");
if (ncclProfiler_v1) {
ncclProfiler.name = ncclProfiler_v1->name;
ncclProfiler.init = ncclProfiler_init;
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded %s", ncclProfiler_v1->name);
return &ncclProfiler;
}
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v1.");
return NULL;
}
src/plugin/profiler/profiler_v2.cc
+87
Просмотреть файл
@@ -0,0 +1,87 @@
/*************************************************************************
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "comm.h"
#include "nccl_profiler.h"
#include "checks.h"
static ncclProfiler_t ncclProfiler;
static ncclProfiler_v2_t* ncclProfiler_v2;
static ncclResult_t ncclProfiler_startEvent(void* context, void** eHandle, ncclProfilerEventDescr_t* eDescr) {
*eHandle = nullptr;
ncclProfilerEventDescr_v2_t eDescr_v2 = { };
eDescr_v2.type = eDescr->type;
eDescr_v2.parentObj = eDescr->parentObj;
eDescr_v2.rank = eDescr->rank;
switch(eDescr->type) {
case ncclProfileGroup: break;
case ncclProfileColl: {
eDescr_v2.coll.name = eDescr->coll.name;
eDescr_v2.coll.commHash = eDescr->coll.commHash;
eDescr_v2.coll.seqNumber = eDescr->coll.seqNumber;
eDescr_v2.coll.func = eDescr->coll.func;
eDescr_v2.coll.sendBuff = eDescr->coll.sendBuff;
eDescr_v2.coll.recvBuff = eDescr->coll.recvBuff;
eDescr_v2.coll.count = eDescr->coll.count;
eDescr_v2.coll.root = eDescr->coll.root;
eDescr_v2.coll.datatype = eDescr->coll.datatype;
eDescr_v2.coll.trafficBytes = 0; // removed in v3
eDescr_v2.coll.nMaxChannels = eDescr->coll.nMaxChannels;
eDescr_v2.coll.nWarps = eDescr->coll.nWarps;
eDescr_v2.coll.algo = eDescr->coll.algo;
eDescr_v2.coll.proto = eDescr->coll.proto;
} break;
case ncclProfileP2p: {
eDescr_v2.p2p.name = eDescr->p2p.name;
eDescr_v2.p2p.commHash = eDescr->p2p.commHash;
eDescr_v2.p2p.func = eDescr->p2p.func;
eDescr_v2.p2p.buff = eDescr->p2p.buff;
eDescr_v2.p2p.count = eDescr->p2p.count;
eDescr_v2.p2p.datatype = eDescr->p2p.datatype;
eDescr_v2.p2p.peer = eDescr->p2p.peer;
} break;
case ncclProfileProxyOp: {
eDescr_v2.proxyOp.pid = eDescr->proxyOp.pid;
eDescr_v2.proxyOp.channelId = eDescr->proxyOp.channelId;
eDescr_v2.proxyOp.peer = eDescr->proxyOp.peer;
eDescr_v2.proxyOp.nSteps = eDescr->proxyOp.nSteps;
eDescr_v2.proxyOp.chunkSize = eDescr->proxyOp.chunkSize;
eDescr_v2.proxyOp.isSend = eDescr->proxyOp.isSend;
} break;
case ncclProfileProxyStep: {
eDescr_v2.proxyStep.step = eDescr->proxyStep.step;
} break;
case ncclProfileProxyCtrl: break;
default: return ncclSuccess;
}
return ncclProfiler_v2->startEvent(context, eHandle, &eDescr_v2);
}
static ncclResult_t ncclProfiler_recordEventState(void* eHandle, ncclProfilerEventState_t eState, ncclProfilerEventStateArgs_t* eStateArgs) {
return ncclProfiler_v2->recordEventState(eHandle, eState, (ncclProfilerEventStateArgs_v2_t *)eStateArgs);
}
static ncclResult_t ncclProfiler_init(void** context, int* eActivationMask) {
NCCLCHECK(ncclProfiler_v2->init(context, eActivationMask));
ncclProfiler.startEvent = ncclProfiler_startEvent;
ncclProfiler.stopEvent = ncclProfiler_v2->stopEvent;
ncclProfiler.recordEventState = ncclProfiler_recordEventState;
ncclProfiler.finalize = ncclProfiler_v2->finalize;
return ncclSuccess;
}
ncclProfiler_t* getNcclProfiler_v2(void* lib) {
ncclProfiler_v2 = (ncclProfiler_v2_t*)dlsym(lib, "ncclProfiler_v2");
if (ncclProfiler_v2) {
ncclProfiler.name = ncclProfiler_v2->name;
ncclProfiler.init = ncclProfiler_init;
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded %s", ncclProfiler_v2->name);
return &ncclProfiler;
}
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v2");
return NULL;
}
src/plugin/profiler/profiler_v3.cc
+20
Просмотреть файл
@@ -0,0 +1,20 @@
/*************************************************************************
* Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "comm.h"
#include "nccl_profiler.h"
static ncclProfiler_v3_t* ncclProfiler_v3;
ncclProfiler_t* getNcclProfiler_v3(void* lib) {
ncclProfiler_v3 = (ncclProfiler_v3_t*)dlsym(lib, "ncclProfiler_v3");
if (ncclProfiler_v3) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded %s", ncclProfiler_v3->name);
return ncclProfiler_v3;
}
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v3");
return NULL;
}
src/plugin/tuner.cc
+99
Просмотреть файл
@@ -0,0 +1,99 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#include <errno.h>
#include <stdlib.h>
#include "checks.h"
#include "debug.h"
#include "tuner.h"
#include "plugin.h"
extern ncclTuner_t* getNcclTuner_v2(void* lib);
extern ncclTuner_t* getNcclTuner_v3(void* lib);
extern ncclTuner_t* getNcclTuner_v4(void* lib);
pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
static int tunerPluginRefCount;
static void* tunerPluginLib = nullptr;
static ncclTuner_t* tunerSymbol = nullptr;
enum {
tunerPluginLoadFailed = -1,
tunerPluginLoadReady = 0,
tunerPluginLoadSuccess = 1,
};
#define MAX_PLUGIN_LOAD 4
static int status = tunerPluginLoadReady;
ncclResult_t ncclTunerPluginLoad(struct ncclComm* comm) {
// Initialize to nullptr by default if plugin tuner cannot be loaded.
comm->tuner = nullptr;
if (tunerPluginLoadFailed == status) {
return ncclSuccess;
}
pthread_mutex_lock(&tunerPluginLock);
if (tunerPluginLoadFailed == status) {
goto exit;
}
if (tunerPluginLoadSuccess == status) {
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
goto exit;
}
tunerPluginLib = ncclOpenTunerPluginLib(ncclGetEnv("NCCL_TUNER_PLUGIN"));
if (nullptr == tunerPluginLib) {
tunerPluginLib = ncclGetNetPluginLib();
if (nullptr == tunerPluginLib) {
goto fail;
}
}
tunerSymbol = getNcclTuner_v4(tunerPluginLib);
if (tunerSymbol == NULL) {
tunerSymbol = getNcclTuner_v3(tunerPluginLib);
}
if (tunerSymbol == NULL) {
tunerSymbol = getNcclTuner_v2(tunerPluginLib);
}
if (tunerSymbol == NULL) {
goto fail;
}
comm->tuner = tunerSymbol;
++tunerPluginRefCount;
status = tunerPluginLoadSuccess;
comm->tunerPluginLoaded = 1;
exit:
pthread_mutex_unlock(&tunerPluginLock);
return ncclSuccess;
fail:
tunerPluginLib = nullptr;
status = tunerPluginLoadFailed;
goto exit;
}
ncclResult_t ncclTunerPluginUnload(struct ncclComm* comm) {
pthread_mutex_lock(&tunerPluginLock);
if (comm->tunerPluginLoaded && 0 == (--tunerPluginRefCount)) {
INFO(NCCL_TUNING, "TUNER/Plugin: Closing tuner: '%s'", tunerSymbol->name);
NCCLCHECK(ncclClosePluginLib(tunerPluginLib));
tunerPluginLib = nullptr;
tunerSymbol = nullptr;
comm->tuner = nullptr;
status = tunerPluginLoadReady;
comm->tunerPluginLoaded = 0;
}
pthread_mutex_unlock(&tunerPluginLock);
return ncclSuccess;
}
src/plugin/tuner/tuner_v2.cc
+66
Просмотреть файл
@@ -0,0 +1,66 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#include <dlfcn.h>
#include "debug.h"
#include "checks.h"
#include "nccl_tuner.h"
static ncclTuner_v2_t* ncclTuner_v2;
static ncclTuner_t ncclTuner;
static int hasNvlsSupport(float** collCostTable) {
// Requirements for support of different algorithms:
//
// - NVLS intra-node: nvlsSupport
// - NVLS intra+inter-node: collNetSupport
// - NVLSTree intra-node: always disabled
// - NVLSTree inter-node: nvlsSupport
// - Collnet* inter-node: collNetSupport
//
// nvlsSupport = 1 if either NVLS or NVLS_TREE entries in the cost table are not -1
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE || table[NCCL_ALGO_NVLS_TREE][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) ? 1 : 0;
}
static int hasCollNetSupport(float** collCostTable) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
return (table[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] == NCCL_ALGO_PROTO_IGNORE) ? 0 : 1;
}
static ncclResult_t ncclTuner_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int regBuff __attribute__((unused)), int* nChannels) {
int algorithm = NCCL_ALGO_UNDEF;
int protocol = NCCL_PROTO_UNDEF;
int nvlsSupport = hasNvlsSupport(collCostTable);
int collNetSupport = hasCollNetSupport(collCostTable);
NCCLCHECK(ncclTuner_v2->getCollInfo(context, collType, nBytes, collNetSupport, nvlsSupport, numPipeOps, &algorithm, &protocol, nChannels));
// set time to 0 below to make sure this algorithm/protocol is selected later on
if (algorithm >= 0 && algorithm < NCCL_NUM_ALGORITHMS && protocol >= 0 && protocol < NCCL_NUM_PROTOCOLS) {
float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable;
if (table[algorithm][protocol] != NCCL_ALGO_PROTO_IGNORE) table[algorithm][protocol] = 0.0;
}
return ncclSuccess;
}
static ncclResult_t ncclTuner_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logfn, void** context) {
NCCLCHECK(ncclTuner_v2->init(nRanks, nNodes, logfn, context));
ncclTuner.getCollInfo = ncclTuner_getCollInfo;
ncclTuner.destroy = ncclTuner_v2->destroy;
return ncclSuccess;
}
ncclTuner_t* getNcclTuner_v2(void* lib) {
ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(lib, "ncclTunerPlugin_v2");
if (ncclTuner_v2) {
ncclTuner.name = ncclTuner_v2->name;
ncclTuner.init = ncclTuner_init;
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", ncclTuner_v2->name);
return &ncclTuner;
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
return NULL;
}
src/plugin/tuner/tuner_v3.cc
+38
Просмотреть файл
@@ -0,0 +1,38 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#include <dlfcn.h>
#include "debug.h"
#include "checks.h"
#include "nccl_tuner.h"
static ncclTuner_v3_t* ncclTuner_v3;
static ncclTuner_t ncclTuner;
static ncclResult_t ncclTuner_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff __attribute__((unused)), int* nChannels) {
NCCLCHECK(ncclTuner_v3->getCollInfo(context, collType, nBytes, numPipeOps, collCostTable, numAlgo, numProto, nChannels));
return ncclSuccess;
}
static ncclResult_t ncclTuner_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logfn, void** context) {
NCCLCHECK(ncclTuner_v3->init(nRanks, nNodes, logfn, context));
ncclTuner.getCollInfo = ncclTuner_getCollInfo;
ncclTuner.destroy = ncclTuner_v3->destroy;
return ncclSuccess;
}
ncclTuner_t* getNcclTuner_v3(void* lib) {
ncclTuner_v3 = (ncclTuner_v3_t*)dlsym(lib, "ncclTunerPlugin_v3");
if (ncclTuner_v3) {
ncclTuner.name = ncclTuner_v3->name;
ncclTuner.init = ncclTuner_init;
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", ncclTuner_v3->name);
return &ncclTuner;
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
return NULL;
}
src/plugin/tuner/tuner_v4.cc
+22
Просмотреть файл
@@ -0,0 +1,22 @@
/*************************************************************************
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2023, Meta Platforms, Inc. and affiliates.
*
* See LICENSE.txt for license information
************************************************************************/
#include <dlfcn.h>
#include "debug.h"
#include "nccl_tuner.h"
static ncclTuner_v4_t* ncclTuner_v4;
ncclTuner_t* getNcclTuner_v4(void* lib) {
ncclTuner_v4 = (ncclTuner_v4_t*)dlsym(lib, "ncclTunerPlugin_v4");
if (ncclTuner_v4) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Using tuner plugin %s", ncclTuner_v4->name);
return ncclTuner_v4;
}
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.");
return NULL;
}
src/proxy.cc
+50 -25
Просмотреть файл
@@ -11,7 +11,6 @@
#include "collectives.h"
#include "socket.h"
#include "shmutils.h"
#include "profiler.h"
#define ENABLE_TIMER 0
#include "timer.h"
#include "profiler.h"
@@ -397,6 +396,7 @@ static ncclResult_t ncclProxyOpToArgs(struct ncclProxyOp* op, struct ncclProxyAr
sub->pid = op->pid;
sub->profilerContext = op->profilerContext;
sub->ringAlgo = op->ringAlgo;
sub->workCounter = op->workCounter;
args->nsubs = subIndex+1;
if (subIndex) {
if ((args->sliceSteps != op->sliceSteps) ||
@@ -550,6 +550,25 @@ static ncclResult_t ncclLocalOpAppend(struct ncclComm* comm, struct ncclProxyCon
return ncclSuccess;
}
static void incWorkCounter(struct ncclComm* comm, struct ncclProxyOp* op) {
op->workCounter = (op->incWorkCounter) ? ++comm->profiler.workCounter[op->channelId] : comm->profiler.workCounter[op->channelId];
}
static ncclResult_t SaveProxyProfiler(struct ncclComm* comm, struct ncclProxyOp* op, bool* justInquire) {
struct ncclProxyConnector* proxyConn = (op->coll == ncclFuncRecv) ? &comm->profiler.recvProxyConn[op->channelId] : &comm->profiler.sendProxyConn[op->channelId];
if (justInquire) {
*justInquire = true;
if (!comm->planner.persistent) incWorkCounter(comm, op);
} else {
op->sendbuff = (uint8_t *)comm->profiler.workStarted;
op->recvbuff = (uint8_t *)comm->profiler.workCompleted;
// Ensure that in graph capturing the proxy workCounter is incremented to keep up with kernel workCounter
if (comm->planner.persistent) incWorkCounter(comm, op);
NCCLCHECK(ncclLocalOpAppend(comm, proxyConn, op));
}
return ncclSuccess;
}
static ncclResult_t SaveProxy(struct ncclComm* comm, struct ncclChannel* channel, int type, int peer, struct ncclProxyOp* op, int connIndex, bool* justInquire) {
if (peer < 0) return ncclSuccess;
@@ -639,20 +658,19 @@ ncclResult_t ncclProxySaveOp(struct ncclComm* comm, struct ncclProxyOp* op, bool
// Run full algorithm to count the number of steps for each peer.
ncclResult_t result = ncclSuccess;
const ssize_t size = op->nbytes/comm->nRanks;
int last = 0;
int *nstepsSend = NULL, *nstepsRecv = NULL;
const int rank = comm->rank, nranks = comm->nRanks;
PatRSAlgorithm<char> algo(op->chunkSize, NCCL_STEPS, 0, size, size, op->chunkSize, rank, nranks);
int *nstepsSend = NULL, *nstepsRecv = NULL;
PatRSAlgorithm<char> algo(op->chunkSize, NCCL_STEPS, 16, 0, size, size, op->chunkSize, rank, nranks);
NCCLCHECKGOTO(ncclCalloc(&nstepsSend, log2Up(nranks)), result, exit_pat_up);
NCCLCHECKGOTO(ncclCalloc(&nstepsRecv, log2Up(nranks)), result, exit_pat_up);
while (last == 0) {
int recvDim, sendDim, recvOffset, sendOffset, sendStepOffset, postRecv, postSend, nelem;
size_t inpIx, outIx;
algo.getNextOp(recvDim, sendDim, inpIx, outIx, recvOffset, sendOffset, sendStepOffset, nelem, postRecv, postSend, last);
if (recvDim != -1 && postRecv) nstepsRecv[recvDim]++;
if (sendDim != -1 && postSend) nstepsSend[sendDim]++;
}
struct ncclPatStep ps;
do {
algo.getNextOp(&ps);
if (ps.flags & PatSkipped) continue;
if (ps.recvDim != -1 && ps.postRecv) nstepsRecv[ps.recvDim]++;
if (ps.sendDim != -1 && ps.postSend) nstepsSend[ps.sendDim]++;
} while (ps.last != 2);
for (int i=0; i<log2Up(nranks); i++) {
if (nstepsSend[i]) {
int sendPeer = (rank + (1<<i)) % nranks;
@@ -674,20 +692,19 @@ ncclResult_t ncclProxySaveOp(struct ncclComm* comm, struct ncclProxyOp* op, bool
// Run full algorithm to count the number of steps for each peer.
ncclResult_t result = ncclSuccess;
const ssize_t size = op->nbytes/comm->nRanks;
int last = 0;
int *nstepsSend = NULL, *nstepsRecv = NULL;
const int rank = comm->rank, nranks = comm->nRanks;
PatAGAlgorithm<char> algo(op->chunkSize, NCCL_STEPS, 0, size, size, op->chunkSize, rank, nranks);
int *nstepsSend = NULL, *nstepsRecv = NULL;
PatAGAlgorithm<char> algo(op->chunkSize, NCCL_STEPS, 16, 0, size, size, op->chunkSize, rank, nranks);
NCCLCHECKGOTO(ncclCalloc(&nstepsSend, log2Up(nranks)), result, exit_pat_down);
NCCLCHECKGOTO(ncclCalloc(&nstepsRecv, log2Up(nranks)), result, exit_pat_down);
while (last == 0) {
int recvDim, sendDim, recvOffset, sendOffset, recvStepOffset, postRecv, postSend, nelem;
size_t inpIx, outIx;
algo.getNextOp(recvDim, sendDim, inpIx, outIx, recvOffset, sendOffset, recvStepOffset, nelem, postRecv, postSend, last);
if (recvDim != -1 && postRecv) nstepsRecv[recvDim]++;
if (sendDim != -1 && postSend) nstepsSend[sendDim]++;
}
struct ncclPatStep ps;
do {
algo.getNextOp(&ps);
if (ps.flags & PatSkipped) continue;
if (ps.recvDim != -1 && ps.postRecv) nstepsRecv[ps.recvDim]++;
if (ps.sendDim != -1 && ps.postSend) nstepsSend[ps.sendDim]++;
} while (ps.last != 2);
for (int i=0; i<log2Up(nranks); i++) {
if (nstepsSend[i]) {
int sendPeer = (rank - (1<<i) + nranks) % nranks;
@@ -710,6 +727,10 @@ ncclResult_t ncclProxySaveOp(struct ncclComm* comm, struct ncclProxyOp* op, bool
if (op->root == comm->rank) return ncclSuccess;
NCCLCHECK(SaveProxy(comm, channel, op->pattern == ncclPatternSend ? proxySend : proxyRecv, op->root, op, op->connIndex, justInquire));
} break;
case ncclPatternProfiler: {
if (ncclProfilerNeedsProxy(comm, op)) NCCLCHECK(SaveProxyProfiler(comm, op, justInquire));
else incWorkCounter(comm, op);
} break;
}
return ncclSuccess;
}
@@ -754,10 +775,10 @@ static ncclResult_t progressOps(struct ncclProxyState* proxyState, struct ncclPr
op->retry_total++;
if (op->state == ncclProxyOpNone) return ncclInternalError;
TIME_START(0); TIME_START(1);
NCCLCHECK(op->progress(proxyState, op));
ncclResult_t ret = op->progress(proxyState, op);
if (op->idle) { TIME_STOP(1); TIME_CANCEL(0); } else { TIME_CANCEL(1); TIME_STOP(0); }
*idle &= op->idle;
if (op->state == ncclProxyOpNone) {
if (op->state == ncclProxyOpNone || ret != ncclSuccess) {
TIME_START(2);
NCCLCHECK(removeOp(state, &op, &prevOp));
TIME_STOP(2);
@@ -940,7 +961,7 @@ void* ncclProxyProgress(void *proxyState_) {
if (ret != ncclSuccess) {
__atomic_store_n(&proxyState->asyncResult, ret, __ATOMIC_RELEASE);
INFO(NCCL_ALL,"%s:%d -> %d [Progress Thread]", __FILE__, __LINE__, ret);
continue;
break;
}
void* eHandle;
ncclProfilerStartProxyCtrlEvent(proxyState->profilerContext, &eHandle);
@@ -962,7 +983,7 @@ void* ncclProxyProgress(void *proxyState_) {
}
}
lastIdle = idle;
} while (state->stop == 0 || (state->stop == 1 && state->active));
} while ((state->stop == 0 || (state->stop == 1 && state->active)) && __atomic_load_n(proxyState->abortFlag, __ATOMIC_ACQUIRE) == 0);
return NULL;
}
@@ -1170,6 +1191,7 @@ ncclResult_t ncclProxyCallBlockingUDS(struct ncclComm* comm, struct ncclProxyCon
}
ncclIpcHdr hdr;
memset(&hdr, '\0', sizeof(hdr));
hdr.type = type;
hdr.rank = rank;
hdr.reqSize = reqSize;
@@ -1353,9 +1375,12 @@ static ncclResult_t proxyProgressInit(struct ncclProxyState* proxyState) {
pthread_mutexattr_init(&mutexAttr);
pthread_mutexattr_setpshared(&mutexAttr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(&pool->mutex, &mutexAttr);
pthread_mutexattr_destroy(&mutexAttr);
pthread_condattr_t condAttr;
pthread_condattr_init(&condAttr);
pthread_condattr_setpshared(&condAttr, PTHREAD_PROCESS_SHARED);
pthread_cond_init(&pool->cond, &condAttr);
pthread_condattr_destroy(&condAttr);
state->opsPool = pool;
memcpy(state->opsPoolShmSuffix, shmPath+sizeof("/dev/shm/nccl-")-1, sizeof("XXXXXX")-1);
src/ras/client_support.cc
+507 -344
Просмотреть файл
Разница между файлами не показана из-за своего большого размера Загрузить разницу
src/ras/collectives.cc
+470 -250
Просмотреть файл
@@ -4,7 +4,7 @@
* See LICENSE.txt for license information
************************************************************************/
#define NDEBUG // Comment out duriyng development only!
#define NDEBUG // Comment out during development only!
#include <cassert>
#include <mutex>
@@ -12,6 +12,7 @@
#include "checks.h"
#include "comm.h"
#include "nccl.h"
#include "transport.h"
#include "utils.h"
#include "ras_internal.h"
@@ -32,14 +33,14 @@ static int nRasCollHistory, rasCollHistNextIdx;
// Monotonically increased to ensure that each collective originating locally has a unique Id.
static uint64_t rasCollLastId;
// Array keeping track of ongoing collective operations (apart from broadcasts, which have no response so require
// Keeping track of ongoing collective operations (apart from broadcasts, which have no response so require
// no such tracking).
struct rasCollective* rasCollectives;
static int nRasCollectives;
struct rasCollective* rasCollectivesHead;
struct rasCollective* rasCollectivesTail;
static ncclResult_t getNewCollEntry(struct rasCollective** pColl);
static ncclResult_t rasLinkSendCollReq(struct rasLink* link, struct rasCollective* coll,
const struct rasCollRequest* req, size_t reqLen, int fromConnIdx);
const struct rasCollRequest* req, size_t reqLen, struct rasConnection* fromConn);
static ncclResult_t rasConnSendCollReq(struct rasConnection* conn, const struct rasCollRequest* req, size_t reqLen);
static ncclResult_t rasCollReadyResp(struct rasCollective* coll);
static ncclResult_t rasConnSendCollResp(struct rasConnection* conn,
@@ -47,12 +48,17 @@ static ncclResult_t rasConnSendCollResp(struct rasConnection* conn,
const union ncclSocketAddress* peers, int nPeers,
const char* data, int nData, int nLegTimeouts);
static ncclResult_t rasCollConnsInit(char** pData, int* pNData);
static ncclResult_t rasCollConnsInit(struct rasCollRequest** pReq, size_t* pReqLen, char** pData, int* pNData);
static ncclResult_t rasCollConnsMerge(struct rasCollective* coll, struct rasMsg* msg);
static ncclResult_t rasCollCommsInit(char** pData, int* pNData);
static ncclResult_t rasCollCommsInit(struct rasCollRequest** pReq, size_t* pReqLen, char** pData, int* pNData);
static ncclResult_t rasCollCommsMerge(struct rasCollective* coll, struct rasMsg* msg);
static bool rasCollCommsSkipMissing(const struct rasCollRequest* req, struct ncclComm* comm);
static int ncclCommsCompare(const void* p1, const void* p2);
static int peersHashesCompare(const void* p1, const void* p2);
static int peersHashesSearch(const void* k, const void* e);
static int rasCommIdCompare(const void* p1, const void* p2);
static int rasCollCommsMissingRankSearch(const void* k, const void* e);
///////////////////////////////////////////////////////////////////////////////////////
@@ -62,22 +68,26 @@ static int ncclCommsCompare(const void* p1, const void* p2);
// Returns the index of the first available entry in the rasCollectives array, enlarging the array if necessary.
static ncclResult_t getNewCollEntry(struct rasCollective** pColl) {
struct rasCollective* coll;
int i;
for (i = 0; i < nRasCollectives; i++)
if (rasCollectives[i].type == RAS_MSG_NONE)
break;
if (i == nRasCollectives) {
NCCLCHECK(ncclRealloc(&rasCollectives, nRasCollectives, nRasCollectives+RAS_INCREMENT));
nRasCollectives += RAS_INCREMENT;
}
int nRasConns;
NCCLCHECK(ncclCalloc(&coll, 1));
coll = rasCollectives+i;
memset(coll, '\0', sizeof(*coll));
coll->startTime = clockNano();
coll->fromConnIdx = -1;
coll->fromConn = nullptr;
// We are unlikely to use the whole array, but at least we won't need to realloc.
nRasConns = 0;
for (struct rasConnection* conn = rasConnsHead; conn; conn = conn->next)
nRasConns++;
NCCLCHECK(ncclCalloc(&coll->fwdConns, nRasConns));
if (rasCollectivesHead) {
rasCollectivesTail->next = coll;
coll->prev = rasCollectivesTail;
rasCollectivesTail = coll;
} else {
rasCollectivesHead = rasCollectivesTail = coll;
}
*pColl = coll;
return ncclSuccess;
}
@@ -95,21 +105,23 @@ void rasCollReqInit(struct rasCollRequest* req) {
// in preparation for collective response messages.
// pAllDone indicates on return if the collective operation is already finished, which is unusual, but possible
// in scenarios such as a total of two peers.
// pCollIdx provides on return an index of the allocated rasCollective structure to track this collective (unless
// pColl provides on return a pointer to the allocated rasCollective structure to track this collective (unless
// it's a broadcast, which require no such tracking).
ncclResult_t rasNetSendCollReq(const struct rasCollRequest* req, size_t reqLen, bool* pAllDone, int* pCollIdx,
int fromConnIdx) {
ncclResult_t rasNetSendCollReq(const struct rasCollRequest* req, bool* pAllDone,
struct rasCollective** pColl, struct rasConnection* fromConn) {
struct rasCollective* coll = nullptr;
struct rasCollRequest* reqMod = (struct rasCollRequest*)req;
size_t reqLen = 0;
if (req->type >= RAS_COLL_CONNS) {
// Keep track of this collective operation so that we can handle the responses appropriately.
NCCLCHECK(getNewCollEntry(&coll));
if (pCollIdx)
*pCollIdx = coll-rasCollectives;
if (pColl)
*pColl = coll;
memcpy(&coll->rootAddr, &req->rootAddr, sizeof(coll->rootAddr));
coll->rootId = req->rootId;
coll->type = req->type;
coll->timeout = req->timeout;
coll->fromConnIdx = fromConnIdx;
coll->fromConn = fromConn;
if (ncclCalloc(&coll->peers, 1) == ncclSuccess) {
memcpy(coll->peers, &rasNetListeningSocket.addr, sizeof(*coll->peers));
coll->nPeers = 1;
@@ -117,9 +129,9 @@ ncclResult_t rasNetSendCollReq(const struct rasCollRequest* req, size_t reqLen,
// Collective-specific initialization of accumulated data (using local data for now).
if (req->type == RAS_COLL_CONNS)
(void)rasCollConnsInit(&coll->data, &coll->nData);
(void)rasCollConnsInit(&reqMod, &reqLen, &coll->data, &coll->nData);
else if (req->type == RAS_COLL_COMMS)
(void)rasCollCommsInit(&coll->data, &coll->nData);
(void)rasCollCommsInit(&reqMod, &reqLen, &coll->data, &coll->nData);
} else { // req->type < RAS_COLL_CONNS
// Add the info to the collective message history.
nRasCollHistory = std::min(nRasCollHistory+1, COLL_HISTORY_SIZE);
@@ -131,42 +143,42 @@ ncclResult_t rasNetSendCollReq(const struct rasCollRequest* req, size_t reqLen,
// Collective-specific message handling.
if (req->type == RAS_BC_DEADPEER) {
bool done = false;
rasMsgHandleBCDeadPeer(req, &done);
rasMsgHandleBCDeadPeer(&reqMod, &reqLen, &done);
if (done)
goto exit;
}
} // req->type < RAS_COLL_CONNS
for (int connIdx = 0; connIdx < nRasConns; connIdx++)
rasConns[connIdx].linkFlag = false;
for (struct rasConnection* conn = rasConnsHead; conn; conn = conn->next)
conn->linkFlag = false;
(void)rasLinkSendCollReq(&rasNextLink, coll, req, reqLen, fromConnIdx);
(void)rasLinkSendCollReq(&rasPrevLink, coll, req, reqLen, fromConnIdx);
(void)rasLinkSendCollReq(&rasNextLink, coll, reqMod, reqLen, fromConn);
(void)rasLinkSendCollReq(&rasPrevLink, coll, reqMod, reqLen, fromConn);
if (coll && pAllDone)
*pAllDone = (coll->nFwdSent == coll->nFwdRecv);
exit:
if (reqMod != req)
free(reqMod);
return ncclSuccess;
}
// Sends the collective message through all connections associated with this link (with the exception of the one
// the message came from, if any).
static ncclResult_t rasLinkSendCollReq(struct rasLink* link, struct rasCollective* coll,
const struct rasCollRequest* req, size_t reqLen, int fromConnIdx) {
for (int i = 0; i < link->nConns; i++) {
struct rasLinkConn* linkConn = link->conns+i;
if (linkConn->connIdx != -1 && linkConn->connIdx != fromConnIdx) {
struct rasConnection* conn = rasConns+linkConn->connIdx;
if (!conn->linkFlag) {
// We send collective messages through fully established and operational connections only.
if (conn->sockIdx != -1 && rasSockets[conn->sockIdx].status == RAS_SOCK_READY && !conn->experiencingDelays) {
if (rasConnSendCollReq(conn, req, reqLen) == ncclSuccess && coll != nullptr)
coll->fwdConns[coll->nFwdSent++] = linkConn->connIdx;
} // if (conn->sockIdx != -1 && RAS_SOCK_READY)
conn->linkFlag = true;
} // if (!conn->linkFlag)
} // if (linkConn->connIdx != -1 && linkConn->connIdx != fromConnIdx)
} // for (i)
const struct rasCollRequest* req, size_t reqLen,
struct rasConnection* fromConn) {
for (struct rasLinkConn* linkConn = link->conns; linkConn; linkConn = linkConn->next) {
if (linkConn->conn && linkConn->conn != fromConn && !linkConn->conn->linkFlag) {
// We send collective messages through fully established and operational connections only.
if (linkConn->conn->sock && linkConn->conn->sock->status == RAS_SOCK_READY &&
!linkConn->conn->experiencingDelays) {
if (rasConnSendCollReq(linkConn->conn, req, reqLen) == ncclSuccess && coll != nullptr)
coll->fwdConns[coll->nFwdSent++] = linkConn->conn;
} // linkConn->conn is fully established and operational.
linkConn->conn->linkFlag = true;
} // if (linkConn->conn && linkConn->conn != fromConn && !linkConn->con->linkFlag)
} // for (linkConn)
return ncclSuccess;
}
@@ -190,8 +202,8 @@ static ncclResult_t rasConnSendCollReq(struct rasConnection* conn, const struct
// in which case it can immediately send the response.
ncclResult_t rasMsgHandleCollReq(struct rasMsg* msg, struct rasSocket* sock) {
bool allDone = false;
int collIdx = -1;
assert(sock->connIdx != -1);
struct rasCollective* coll = nullptr;
assert(sock->conn);
// First check if we've already handled this request (through another connection).
for (int i = 0; i < nRasCollHistory; i++) {
@@ -202,7 +214,7 @@ ncclResult_t rasMsgHandleCollReq(struct rasMsg* msg, struct rasSocket* sock) {
if (msg->collReq.type >= RAS_COLL_CONNS) {
// Send an empty response so that the sender can account for it. The non-empty response has already been
// sent through the connection that we received the request through first.
NCCLCHECK(rasConnSendCollResp(rasConns+sock->connIdx, &msg->collReq.rootAddr, msg->collReq.rootId,
NCCLCHECK(rasConnSendCollResp(sock->conn, &msg->collReq.rootAddr, msg->collReq.rootId,
/*peers*/nullptr, /*nPeers*/0, /*data*/nullptr, /*nData*/0, /*nLegTimeouts*/0));
}
goto exit;
@@ -211,31 +223,29 @@ ncclResult_t rasMsgHandleCollReq(struct rasMsg* msg, struct rasSocket* sock) {
if (msg->collReq.type >= RAS_COLL_CONNS) {
// Check if we're currently handling this collective request.
for (int i = 0; i < nRasCollectives; i++) {
struct rasCollective* coll = rasCollectives+i;
if (coll->type != RAS_MSG_NONE &&
memcmp(&msg->collReq.rootAddr, &coll->rootAddr, sizeof(msg->collReq.rootAddr)) == 0 &&
for (coll = rasCollectivesHead; coll; coll = coll->next) {
if (memcmp(&msg->collReq.rootAddr, &coll->rootAddr, sizeof(msg->collReq.rootAddr)) == 0 &&
msg->collReq.rootId == coll->rootId) {
assert(msg->collReq.type == coll->type);
// Send an empty response so that the sender can account for it. The non-empty response will be
// sent through the connection that we received the request through first.
NCCLCHECK(rasConnSendCollResp(rasConns+sock->connIdx, &msg->collReq.rootAddr, msg->collReq.rootId,
NCCLCHECK(rasConnSendCollResp(sock->conn, &msg->collReq.rootAddr, msg->collReq.rootId,
/*peers*/nullptr, /*nPeers*/0, /*data*/nullptr, /*nData*/0, /*nLegTimeouts*/0));
goto exit;
} // if match
} // for (i)
} // for (coll)
} // if (msg->collReq.type >= RAS_COLL_CONNS)
// Re-broadcast the message to my peers (minus the one it came from) and handle it locally.
NCCLCHECK(rasNetSendCollReq(&msg->collReq, rasCollDataLength(msg->collReq.type), &allDone, &collIdx, sock->connIdx));
NCCLCHECK(rasNetSendCollReq(&msg->collReq, &allDone, &coll, sock->conn));
if (msg->collReq.type >= RAS_COLL_CONNS && allDone) {
assert(collIdx != -1);
assert(coll);
// We are a leaf process -- send the response right away. This can probably trigger only for the case of a total
// of two peers, and hence just one RAS connection, or during communication issues, because normally every peer
// has more than one connection so there should always be _some_ other peer to forward the request to.
NCCLCHECK(rasCollReadyResp(rasCollectives+collIdx));
NCCLCHECK(rasCollReadyResp(coll));
}
exit:
return ncclSuccess;
@@ -245,9 +255,9 @@ exit:
// Invoked when we are finished waiting for the collective responses from other peers (i.e., either there weren't
// any peers (unlikely), the peers sent their responses (likely), or we timed out.
static ncclResult_t rasCollReadyResp(struct rasCollective* coll) {
if (coll->fromConnIdx != -1) {
if (coll->fromConn) {
// For remotely-initiated collectives, send the response back.
NCCLCHECK(rasConnSendCollResp(rasConns+coll->fromConnIdx, &coll->rootAddr, coll->rootId,
NCCLCHECK(rasConnSendCollResp(coll->fromConn, &coll->rootAddr, coll->rootId,
coll->peers, coll->nPeers, coll->data, coll->nData, coll->nLegTimeouts));
// Add the identifying info to the collective message history.
@@ -302,18 +312,15 @@ static ncclResult_t rasConnSendCollResp(struct rasConnection* conn,
// the data from the response into the accumulated data. If all the responses have been accounted for, sends the
// accumulated response back.
ncclResult_t rasMsgHandleCollResp(struct rasMsg* msg, struct rasSocket* sock) {
int collIdx;
struct rasCollective* coll = nullptr;
struct rasCollective* coll;
char line[SOCKET_NAME_MAXLEN+1];
for (collIdx = 0; collIdx < nRasCollectives; collIdx++) {
coll = rasCollectives+collIdx;
if (coll->type != RAS_MSG_NONE &&
memcmp(&msg->collResp.rootAddr, &coll->rootAddr, sizeof(msg->collResp.rootAddr)) == 0 &&
for (coll = rasCollectivesHead; coll; coll = coll->next) {
if (memcmp(&msg->collResp.rootAddr, &coll->rootAddr, sizeof(msg->collResp.rootAddr)) == 0 &&
msg->collResp.rootId == coll->rootId)
break;
}
if (collIdx == nRasCollectives) {
if (coll == nullptr) {
INFO(NCCL_RAS, "RAS failed to find a matching ongoing collective for response %s:%ld from %s!",
ncclSocketToString(&msg->collResp.rootAddr, line), msg->collResp.rootId,
ncclSocketToString(&sock->sock.addr, rasLine));
@@ -321,11 +328,11 @@ ncclResult_t rasMsgHandleCollResp(struct rasMsg* msg, struct rasSocket* sock) {
}
coll->nLegTimeouts += msg->collResp.nLegTimeouts;
assert(sock->connIdx != -1);
// Account for the received response in our collective operation tracking.
assert(sock->conn);
// Account for the received response in our collective operations tracking.
for (int i = 0; i < coll->nFwdSent; i++) {
if (coll->fwdConns[i] == sock->connIdx) {
coll->fwdConns[i] = -1;
if (coll->fwdConns[i] == sock->conn) {
coll->fwdConns[i] = nullptr;
break;
}
}
@@ -353,46 +360,53 @@ exit:
// Removes a connection from all ongoing collectives. Called when a connection is experiencing a delay or is being
// terminated.
void rasCollsPurgeConn(int connIdx) {
for (int i = 0; i < nRasCollectives; i++) {
struct rasCollective* coll = rasCollectives+i;
if (coll->type != RAS_MSG_NONE) {
char line[SOCKET_NAME_MAXLEN+1];
if (coll->fromConnIdx == connIdx) {
INFO(NCCL_RAS, "RAS purging collective %s:%ld because it comes from %s",
ncclSocketToString(&coll->rootAddr, line), coll->rootId,
ncclSocketToString(&rasConns[connIdx].addr, rasLine));
rasCollFree(coll);
} else {
for (int j = 0; j < coll->nFwdSent; j++) {
if (coll->fwdConns[j] == connIdx) {
coll->fwdConns[j] = -1;
coll->nFwdRecv++;
coll->nLegTimeouts++;
INFO(NCCL_RAS, "RAS not waiting for response from %s to collective %s:%ld "
"(nFwdSent %d, nFwdRecv %d, nLegTimeouts %d)",
ncclSocketToString(&rasConns[connIdx].addr, rasLine), ncclSocketToString(&coll->rootAddr, line),
coll->rootId, coll->nFwdSent, coll->nFwdRecv, coll->nLegTimeouts);
if (coll->nFwdSent == coll->nFwdRecv)
(void)rasCollReadyResp(coll);
break;
}
} // for (j)
} // coll->fromConnIdx != connIdx
} // !RAS_MSG_NONE
} // for (i)
void rasCollsPurgeConn(struct rasConnection* conn) {
for (struct rasCollective* coll = rasCollectivesHead; coll;) {
struct rasCollective* collNext = coll->next;
char line[SOCKET_NAME_MAXLEN+1];
if (coll->fromConn == conn) {
INFO(NCCL_RAS, "RAS purging collective %s:%ld because it comes from %s",
ncclSocketToString(&coll->rootAddr, line), coll->rootId,
ncclSocketToString(&conn->addr, rasLine));
rasCollFree(coll);
} else {
for (int i = 0; i < coll->nFwdSent; i++) {
if (coll->fwdConns[i] == conn) {
coll->fwdConns[i] = nullptr;
coll->nFwdRecv++;
coll->nLegTimeouts++;
INFO(NCCL_RAS, "RAS not waiting for response from %s to collective %s:%ld "
"(nFwdSent %d, nFwdRecv %d, nLegTimeouts %d)",
ncclSocketToString(&conn->addr, rasLine), ncclSocketToString(&coll->rootAddr, line), coll->rootId,
coll->nFwdSent, coll->nFwdRecv, coll->nLegTimeouts);
if (coll->nFwdSent == coll->nFwdRecv)
(void)rasCollReadyResp(coll);
break;
}
} // for (i)
} // coll->fromConn != conn
coll = collNext;
} // for (coll)
}
// Frees a rasCollective entry and any memory associated with it.
void rasCollFree(struct rasCollective* coll) {
if (coll == nullptr)
return;
free(coll->fwdConns);
coll->fwdConns = nullptr;
free(coll->peers);
coll->peers = nullptr;
free(coll->data);
coll->data = nullptr;
coll->fromConnIdx = -1;
coll->type = RAS_MSG_NONE;
if (coll == rasCollectivesHead)
rasCollectivesHead = rasCollectivesHead->next;
if (coll == rasCollectivesTail)
rasCollectivesTail = rasCollectivesTail->prev;
if (coll->prev)
coll->prev->next = coll->next;
if (coll->next)
coll->next->prev = coll->prev;
free(coll);
}
// Invoked from the main RAS thread loop to handle timeouts of the collectives.
@@ -407,64 +421,64 @@ void rasCollFree(struct rasCollective* coll) {
// and send back whatever we have. Unfortunately, the peer that the RAS client is connected to will in all likelihood
// time out first, so at that point any delayed responses that eventually arrive are likely to be too late...
void rasCollsHandleTimeouts(int64_t now, int64_t* nextWakeup) {
for (int collIdx = 0; collIdx < nRasCollectives; collIdx++) {
struct rasCollective* coll = rasCollectives+collIdx;
if (coll->type == RAS_MSG_NONE || coll->timeout == 0)
continue;
if (now - coll->startTime > coll->timeout) {
// We've exceeded the leg timeout. For all outstanding responses, check their connections.
if (!coll->timeoutWarned) {
INFO(NCCL_RAS, "RAS collective %s:%ld timeout warning (%lds) -- %d responses missing",
ncclSocketToString(&coll->rootAddr, rasLine), coll->rootId,
(now - coll->startTime) / CLOCK_UNITS_PER_SEC, coll->nFwdSent - coll->nFwdRecv);
coll->timeoutWarned = true;
}
for (int i = 0; i < coll->nFwdSent; i++) {
if (coll->fwdConns[i] != -1) {
struct rasConnection* conn = rasConns+coll->fwdConns[i];
char line[SOCKET_NAME_MAXLEN+1];
if (!conn->experiencingDelays && conn->sockIdx != -1) {
struct rasSocket* sock = rasSockets+conn->sockIdx;
// Ensure that the connection is fully established and operational, and that the socket hasn't been
// re-created during the handling of the collective (which would suggest that the request may have been
// lost).
if (sock->status == RAS_SOCK_READY && sock->createTime < coll->startTime)
continue;
}
// In all other cases we declare a timeout so that we can (hopefully) recover.
INFO(NCCL_RAS, "RAS not waiting for response from %s to collective %s:%ld "
"(nFwdSent %d, nFwdRecv %d, nLegTimeouts %d)",
ncclSocketToString(&conn->addr, rasLine), ncclSocketToString(&coll->rootAddr, line),
coll->rootId, coll->nFwdSent, coll->nFwdRecv, coll->nLegTimeouts);
coll->fwdConns[i] = -1;
coll->nFwdRecv++;
coll->nLegTimeouts++;
} // if (coll->fwdConns[i] != -1)
} // for (i)
if (coll->nFwdSent == coll->nFwdRecv) {
(void)rasCollReadyResp(coll);
} else {
// At least some of the delays are *not* due to this process' connections experiencing delays, i.e., they
// must be due to delays at other processes. Presumably those processes will give up waiting soon and the
// (incomplete) responses will arrive shortly, so we should wait a little longer.
if (now - coll->startTime > coll->timeout + RAS_COLLECTIVE_EXTRA_TIMEOUT) {
// We've exceeded even the longer timeout, which is unexpected. Try to return whatever we have (though
// the originator of the collective, if it's not us, may have timed out already anyway).
INFO(NCCL_RAS, "RAS collective %s:%ld timeout error (%lds) -- giving up on %d missing responses",
for (struct rasCollective* coll = rasCollectivesHead; coll;) {
struct rasCollective* collNext = coll->next;
if (coll->timeout > 0) {
if (now - coll->startTime > coll->timeout) {
// We've exceeded the leg timeout. For all outstanding responses, check their connections.
if (!coll->timeoutWarned) {
INFO(NCCL_RAS, "RAS collective %s:%ld timeout warning (%lds) -- %d responses missing",
ncclSocketToString(&coll->rootAddr, rasLine), coll->rootId,
(now - coll->startTime) / CLOCK_UNITS_PER_SEC, coll->nFwdSent - coll->nFwdRecv);
coll->nLegTimeouts += coll->nFwdSent - coll->nFwdRecv;
coll->nFwdRecv = coll->nFwdSent;
coll->timeoutWarned = true;
}
for (int i = 0; i < coll->nFwdSent; i++) {
if (coll->fwdConns[i]) {
struct rasConnection* conn = coll->fwdConns[i];
char line[SOCKET_NAME_MAXLEN+1];
if (!conn->experiencingDelays && conn->sock) {
// Ensure that the connection is fully established and operational, and that the socket hasn't been
// re-created during the handling of the collective (which would suggest that the request may have been
// lost).
if (conn->sock->status == RAS_SOCK_READY && conn->sock->createTime < coll->startTime)
continue;
}
// In all other cases we declare a timeout so that we can (hopefully) recover.
INFO(NCCL_RAS, "RAS not waiting for response from %s to collective %s:%ld "
"(nFwdSent %d, nFwdRecv %d, nLegTimeouts %d)",
ncclSocketToString(&conn->addr, rasLine), ncclSocketToString(&coll->rootAddr, line),
coll->rootId, coll->nFwdSent, coll->nFwdRecv, coll->nLegTimeouts);
coll->fwdConns[i] = nullptr;
coll->nFwdRecv++;
coll->nLegTimeouts++;
} // if (coll->fwdConns[i])
} // for (i)
if (coll->nFwdSent == coll->nFwdRecv) {
(void)rasCollReadyResp(coll);
} else {
*nextWakeup = std::min(*nextWakeup, coll->startTime+coll->timeout+RAS_COLLECTIVE_EXTRA_TIMEOUT);
}
} // conn->nFwdRecv < conn->nFwdSent
} else {
*nextWakeup = std::min(*nextWakeup, coll->startTime+coll->timeout);
}
} // for (collIdx)
// At least some of the delays are *not* due to this process' connections experiencing delays, i.e., they
// must be due to delays at other processes. Presumably those processes will give up waiting soon and the
// (incomplete) responses will arrive shortly, so we should wait a little longer.
if (now - coll->startTime > coll->timeout + RAS_COLLECTIVE_EXTRA_TIMEOUT) {
// We've exceeded even the longer timeout, which is unexpected. Try to return whatever we have (though
// the originator of the collective, if it's not us, may have timed out already anyway).
INFO(NCCL_RAS, "RAS collective %s:%ld timeout error (%lds) -- giving up on %d missing responses",
ncclSocketToString(&coll->rootAddr, rasLine), coll->rootId,
(now - coll->startTime) / CLOCK_UNITS_PER_SEC, coll->nFwdSent - coll->nFwdRecv);
coll->nLegTimeouts += coll->nFwdSent - coll->nFwdRecv;
coll->nFwdRecv = coll->nFwdSent;
(void)rasCollReadyResp(coll);
} else {
*nextWakeup = std::min(*nextWakeup, coll->startTime+coll->timeout+RAS_COLLECTIVE_EXTRA_TIMEOUT);
}
} // conn->nFwdRecv < conn->nFwdSent
} else {
*nextWakeup = std::min(*nextWakeup, coll->startTime+coll->timeout);
}
} // if (coll->timeout > 0)
coll = collNext;
} // for (coll)
}
@@ -476,15 +490,16 @@ void rasCollsHandleTimeouts(int64_t now, int64_t* nextWakeup) {
// For this particular collective, we keep some reduced statistical data (min/max/avg travel time) as well
// as connection-specific info in case we observed a negative min travel time (which, ideally, shouldn't happen,
// but the system clocks may not be perfectly in sync).
static ncclResult_t rasCollConnsInit(char** pData, int* pNData) {
static ncclResult_t rasCollConnsInit(struct rasCollRequest** pReq, size_t* pReqLen, char** pData, int* pNData) {
struct rasCollConns connsData = {.travelTimeMin = INT64_MAX, .travelTimeMax = INT64_MIN};
struct rasCollConns* pConnsData;
*pReqLen = rasCollDataLength(RAS_COLL_CONNS);
// Update the statistical data first and in the process also calculate how much connection-specific space we
// will need.
for (int i = 0; i < nRasConns; i++) {
struct rasConnection* conn = rasConns+i;
if (conn->inUse && conn->travelTimeCount > 0) {
for (struct rasConnection* conn = rasConnsHead; conn; conn = conn->next) {
if (conn->travelTimeCount > 0) {
if (connsData.travelTimeMin > conn->travelTimeMin)
connsData.travelTimeMin = conn->travelTimeMin;
if (connsData.travelTimeMax < conn->travelTimeMax)
@@ -502,9 +517,9 @@ static ncclResult_t rasCollConnsInit(char** pData, int* pNData) {
pConnsData = (struct rasCollConns*)*pData;
memcpy(pConnsData, &connsData, sizeof(*pConnsData));
if (connsData.nNegativeMins > 0) {
for (int i = 0, negMinsIdx = 0; i < nRasConns; i++) {
struct rasConnection* conn = rasConns+i;
if (conn->inUse && conn->travelTimeMin < 0) {
int negMinsIdx = 0;
for (struct rasConnection* conn = rasConnsHead; conn; conn = conn->next) {
if (conn->travelTimeMin < 0) {
struct rasCollConns::negativeMin* negativeMin = pConnsData->negativeMins+negMinsIdx;
memcpy(&negativeMin->source, &rasNetListeningSocket.addr, sizeof(negativeMin->source));
memcpy(&negativeMin->dest, &conn->addr, sizeof(negativeMin->dest));
@@ -560,10 +575,26 @@ static ncclResult_t rasCollConnsMerge(struct rasCollective* coll, struct rasMsg*
// Initializes the accumulated data with just the local data for now.
// For this particular collective, we keep for every communicator information about every rank, to help identify
// the missing ones and the discrepancies between the ones that did respond.
static ncclResult_t rasCollCommsInit(char** pData, int* pNData) {
// For any new (previously unseen) communicator we also save the basic identification data about every rank that is
// "missing" (i.e., not part of this process). During merging, this should be replaced by the actual data from
// those ranks, if they are responsive. We want to provide this information to the user (so that we can say more
// than "rank xyz missing").
// Every "new" communicator is also recorded in the (updated) request, so that when that request is forwarded to our
// peers, those peers don't needlessly send us the same data.
static ncclResult_t rasCollCommsInit(struct rasCollRequest** pReq, size_t* pReqLen, char** pData, int* pNData) {
ncclResult_t ret = ncclSuccess;
struct rasCollComms* commsData;
int nComms = 0, nRanks = 0;
int nComms = 0, nRanks = 0, nMissingRanks = 0;
bool skipMissing = false;
std::lock_guard<std::mutex> lock(ncclCommsMutex);
struct rasCollComms::comm* comm;
struct rasCollRequest* req = nullptr;
struct rasPeerInfo** peersReSorted = nullptr;
int firstNewSkipMissingIdx = -1;
*pReqLen = rasCollDataLength(RAS_COLL_COMMS) +
(*pReq)->comms.nSkipMissingRanksComms * sizeof(*(*pReq)->comms.skipMissingRanksComms);
*pData = nullptr;
// Start by counting the communicators so that we know how much space to allocate.
// We also need to sort the comms array, to make the subsequent merging easier, both between the ranks (in case
@@ -572,77 +603,152 @@ static ncclResult_t rasCollCommsInit(char** pData, int* pNData) {
qsort(ncclComms, nNcclComms, sizeof(*ncclComms), &ncclCommsCompare);
ncclCommsSorted = true;
}
for (int i = 0; i < nNcclComms; i++) {
if (ncclComms[i] == nullptr) // nullptr's are always at the end after sorting.
for (int commIdx = 0; commIdx < nNcclComms; commIdx++) {
if (ncclComms[commIdx] == nullptr) // nullptr's are always at the end after sorting.
break;
if (i == 0) {
nComms = 1;
} else if (ncclComms[i]->commHash != ncclComms[i-1]->commHash) {
nComms++;
}
nRanks++;
}
// A process may manage multiple GPUs and thus have multiple communicators with the same commHash.
// Comparing just the commHash is OK though within communicators that are part of the same process.
if (commIdx == 0 || ncclComms[commIdx]->commHash != ncclComms[commIdx-1]->commHash) {
skipMissing = rasCollCommsSkipMissing(*pReq, ncclComms[commIdx]);
if (!skipMissing) {
// Add this communicator to the request so that the processes we forward the request to know not to fill in
// the missing rank info.
struct rasCommId* skipComm;
if (req == nullptr) {
// We pessimistically allocate space for all the remaining communicators so that we don't need to reallocate.
int newSize = *pReqLen + (nNcclComms-commIdx) * sizeof(*req->comms.skipMissingRanksComms);
NCCLCHECKGOTO(ncclCalloc((char**)&req, newSize), ret, fail);
memcpy(req, *pReq, *pReqLen);
*pReq = req;
firstNewSkipMissingIdx = req->comms.nSkipMissingRanksComms;
}
skipComm = req->comms.skipMissingRanksComms + req->comms.nSkipMissingRanksComms++;
skipComm->commHash = ncclComms[commIdx]->commHash;
skipComm->hostHash = ncclComms[commIdx]->peerInfo->hostHash;
skipComm->pidHash = ncclComms[commIdx]->peerInfo->pidHash;
// rasNetCollCommsData has nested variable-length arrays, which makes the size calculation and subsequent
nMissingRanks += ncclComms[commIdx]->nRanks;
} // if (!skipMissing)
nComms++;
} // if encountered a new communicator
nRanks++;
if (!skipMissing)
nMissingRanks--;
} // for (commIdx)
// rasCollComms has nested variable-length arrays, which makes the size calculation and subsequent
// pointer manipulations somewhat unwieldy...
*pNData = sizeof(*commsData) + nComms * sizeof(*commsData->comms) + nRanks * sizeof(*commsData->comms[0].ranks);
NCCLCHECK(ncclCalloc(pData, *pNData));
// This is extra complicated because of the "hidden" array of struct rasCollCommsMissingRank following the
// ranks array for each communicator.
*pNData = sizeof(*commsData) + nComms * sizeof(*commsData->comms) + nRanks * sizeof(*commsData->comms[0].ranks) +
nMissingRanks * sizeof(struct rasCollCommsMissingRank);
NCCLCHECKGOTO(ncclCalloc(pData, *pNData), ret, fail);
commsData = (struct rasCollComms*)*pData;
commsData->nComms = nComms;
// comm points at the space in the accumulated data where the info about the current communicator is to be stored.
struct rasCollComms::comm* comm = commsData->comms;
for (int i = 0; i < nNcclComms; i++) {
struct rasCollComms::comm::rank* rank;
ncclResult_t asyncError;
if (ncclComms[i] == nullptr)
break;
if (i == 0 || ncclComms[i]->commHash != ncclComms[i-1]->commHash) {
if (i > 0)
comm = (struct rasCollComms::comm*)(((char*)(comm+1)) + comm->nRanks * sizeof(*comm->ranks));
comm->commHash = ncclComms[i]->commHash;
comm->commNRanks = ncclComms[i]->nRanks;
comm->nRanks = 0;
} else if (ncclComms[i]->nRanks != ncclComms[i-1]->nRanks) {
INFO(NCCL_RAS, "RAS encountered inconsistent communicator data: size %d != %d -- "
"possible commHash collision (0x%lx)", ncclComms[i-1]->nRanks, ncclComms[i]->nRanks, comm->commHash);
continue; // Short of failing, the best we can do is skip...
} else if (ncclComms[i]->rank == ncclComms[i-1]->rank) {
INFO(NCCL_RAS, "RAS encountered duplicate data for rank %d -- possible commHash collision (0x%lx)",
ncclComms[i]->rank, comm->commHash);
continue; // Short of failing, the best we can do is skip...
}
if (comm->nRanks == comm->commNRanks) {
INFO(NCCL_RAS,
"RAS encountered more ranks than the communicator size (%d) -- possible commHash collision (0x%lx)",
comm->commNRanks, comm->commHash);
continue; // Short of failing, the best we can do is skip...
}
rank = comm->ranks+comm->nRanks;
rank->commRank = ncclComms[i]->rank;
// rasNetSendCollReq initializes coll->peers[0] to our rasNetListeningSocket.addr, so peerIdx is initially
// always 0. It will increase after we send this response back to the peer we got the request from.
rank->peerIdx = 0;
rank->collOpCount = ncclComms[i]->collOpCount;
rank->status.initState = ncclComms[i]->initState;
if (ncclCommGetAsyncError(ncclComms[i], &asyncError) == ncclSuccess)
rank->status.asyncError = asyncError;
rank->status.finalizeCalled = (ncclComms[i]->finalizeCalled != 0);
rank->status.destroyFlag = (ncclComms[i]->destroyFlag != 0);
rank->status.abortFlag = (__atomic_load_n(ncclComms[i]->abortFlag, __ATOMIC_ACQUIRE) != 0);
rank->cudaDev = ncclComms[i]->cudaDev;
rank->nvmlDev = ncclComms[i]->nvmlDev;
comm->nRanks++;
}
assert(nComms == 0 || ((char*)(comm->ranks+comm->nRanks)) - (char*)commsData <= *pNData);
comm = commsData->comms;
// collCommIdx counts rasCollComms::comm (comm); commIdx indexes ncclComms.
for (int collCommIdx = 0, commIdx = 0; collCommIdx < nComms; collCommIdx++) {
struct ncclComm* ncclComm = ncclComms[commIdx];
return ncclSuccess;
comm->commId.commHash = ncclComm->commHash;
comm->commId.hostHash = ncclComm->peerInfo->hostHash;
comm->commId.pidHash = ncclComm->peerInfo->pidHash;
comm->commNRanks = ncclComm->nRanks;
comm->nRanks = comm->nMissingRanks = 0;
// Fill in the comm->ranks array.
for (; commIdx < nNcclComms && ncclComms[commIdx] && ncclComms[commIdx]->commHash == comm->commId.commHash;
commIdx++) {
ncclComm = ncclComms[commIdx];
struct rasCollComms::comm::rank* rank = comm->ranks+comm->nRanks;
ncclResult_t asyncError;
rank->commRank = ncclComm->rank;
// rasNetSendCollReq initializes coll->peers[0] to our rasNetListeningSocket.addr, so peerIdx is initially
// always 0. It will increase after we send this response back to the peer we got the request from.
rank->peerIdx = 0;
memcpy(rank->collOpCounts, ncclComm->seqNumber, sizeof(rank->collOpCounts));
rank->status.initState = ncclComm->initState;
if (ncclCommGetAsyncError(ncclComm, &asyncError) == ncclSuccess)
rank->status.asyncError = asyncError;
rank->status.finalizeCalled = (ncclComm->finalizeCalled != 0);
rank->status.destroyFlag = (ncclComm->destroyFlag != 0);
rank->status.abortFlag = (__atomic_load_n(ncclComm->abortFlag, __ATOMIC_ACQUIRE) != 0);
rank->cudaDev = ncclComm->cudaDev;
rank->nvmlDev = ncclComm->nvmlDev;
comm->nRanks++;
} // for (commIdx)
if (firstNewSkipMissingIdx != -1 &&
memcmp(req->comms.skipMissingRanksComms+firstNewSkipMissingIdx, &comm->commId, sizeof(comm->commId)) == 0) {
// Fill in the missingRanks array that follows the comm->ranks.
struct rasCollCommsMissingRank* missingRanks = (struct rasCollCommsMissingRank*)(comm->ranks+comm->nRanks);
if (peersReSorted == nullptr) {
// Create a lookup table to rasPeers that is sorted by hostHash and pidHash, to reduce the complexity of the
// lookups in the missingRankIdx loop below.
NCCLCHECKGOTO(ncclCalloc(&peersReSorted, nRasPeers), ret, fail);
for (int peerIdx = 0; peerIdx < nRasPeers; peerIdx++)
peersReSorted[peerIdx] = rasPeers+peerIdx;
qsort(peersReSorted, nRasPeers, sizeof(*peersReSorted), peersHashesCompare);
}
comm->nMissingRanks = comm->commNRanks - comm->nRanks;
for (int missingRankIdx = 0, rankIdx = 0; missingRankIdx < comm->nMissingRanks; missingRankIdx++) {
struct rasCollCommsMissingRank* missingRank;
struct ncclPeerInfo* info;
struct rasPeerInfo** peer;
uint64_t key[2];
// Look for the next "hole" in the ranks array.
while (rankIdx < comm->nRanks && comm->ranks[rankIdx].commRank == rankIdx+missingRankIdx)
rankIdx++;
missingRank = missingRanks + missingRankIdx;
missingRank->commRank = rankIdx + missingRankIdx;
info = ncclComm->peerInfo + missingRank->commRank;
key[0] = info->hostHash - ncclComm->commHash;
key[1] = info->pidHash - ncclComm->commHash;
peer = (struct rasPeerInfo**)bsearch(key, peersReSorted, nRasPeers, sizeof(*peersReSorted), peersHashesSearch);
if (peer)
memcpy(&missingRank->addr, &(*peer)->addr, sizeof(missingRank->addr));
missingRank->cudaDev = info->cudaDev;
missingRank->nvmlDev = info->nvmlDev;
} // for (missingRankIdx)
if (++firstNewSkipMissingIdx == req->comms.nSkipMissingRanksComms)
firstNewSkipMissingIdx = -1;
} // if need to fill in the missingRanks
comm = (struct rasCollComms::comm*)(((char*)(comm+1)) + comm->nRanks * sizeof(*comm->ranks) +
comm->nMissingRanks * sizeof(struct rasCollCommsMissingRank));
} // for (collCommIdx)
assert(((char*)comm) - (char*)commsData <= *pNData);
if (req) {
// Finish updating the request.
*pReqLen = rasCollDataLength(RAS_COLL_COMMS) +
req->comms.nSkipMissingRanksComms * sizeof(*req->comms.skipMissingRanksComms);
qsort(req->comms.skipMissingRanksComms, req->comms.nSkipMissingRanksComms,
sizeof(*req->comms.skipMissingRanksComms), rasCommIdCompare);
}
ret:
free(peersReSorted);
return ret;
fail:
if (req) {
free(req);
*pReq = nullptr;
}
free(*pData);
*pData = nullptr;
goto ret;
}
// Merges incoming collective RAS_COLL_COMMS response message into the local accumulated data.
static ncclResult_t rasCollCommsMerge(struct rasCollective* coll, struct rasMsg* msg) {
struct rasCollComms* collData;
struct rasCollComms* msgData;
struct rasCollComms* collData; // Data previously stored (locally) by our process.
struct rasCollComms* msgData; // Data just received from another process.
int dataOffset = rasMsgLength(RAS_MSG_COLLRESP) + msg->collResp.nPeers*sizeof(*msg->collResp.peers);
ALIGN_SIZE(dataOffset, alignof(int64_t));
@@ -650,7 +756,7 @@ static ncclResult_t rasCollCommsMerge(struct rasCollective* coll, struct rasMsg*
collData = (struct rasCollComms*)coll->data;
if (msgData->nComms > 0) {
struct rasCollComms* newData = nullptr;
struct rasCollComms* newData = nullptr; // Destination buffer for the merged data.
// Allocate the new buffer pessimistically (sized as the sum of the two old ones).
NCCLCHECK(ncclCalloc((char**)&newData, coll->nData + msg->collResp.nData));
@@ -661,25 +767,28 @@ static ncclResult_t rasCollCommsMerge(struct rasCollective* coll, struct rasMsg*
for (int collIdx = 0, msgIdx = 0; collIdx < collData->nComms || msgIdx < msgData->nComms; newData->nComms++) {
int cmp;
if (collIdx < collData->nComms && msgIdx < msgData->nComms)
cmp = (collComm->commHash < msgComm->commHash ? -1 : (collComm->commHash > msgComm->commHash ? 1 : 0));
cmp = rasCommIdCompare(&collComm->commId, &msgComm->commId);
else
cmp = (collIdx < collData->nComms ? -1 : 1);
if (cmp == 0 && collComm->commNRanks != msgComm->commNRanks) {
INFO(NCCL_RAS, "RAS encountered inconsistent communicator data: size %d != %d -- "
"possible commHash collision (0x%lx)", collComm->commNRanks, msgComm->commNRanks, collComm->commHash);
"possible hash collision (0x%lx, 0x%lx, 0x%lx)", collComm->commNRanks, msgComm->commNRanks,
collComm->commId.commHash, collComm->commId.hostHash, collComm->commId.pidHash);
cmp = (collComm->commNRanks < msgComm->commNRanks ? -1 : 1);
// We try to preserve both separately, although the input data might already be messed up anyway...
// We try to preserve them both separately...
}
if (cmp == 0) {
// Merge the comms.
newComm->commHash = collComm->commHash;
memcpy(&newComm->commId, &collComm->commId, sizeof(newComm->commId));
newComm->commNRanks = collComm->commNRanks;
if (collComm->nRanks + msgComm->nRanks > collComm->commNRanks) {
INFO(NCCL_RAS,
"RAS encountered more ranks (%d) than the communicator size (%d) -- possible commHash collision (0x%lx)",
collComm->nRanks + msgComm->nRanks, newComm->commNRanks, newComm->commHash);
"RAS encountered more ranks (%d) than the communicator size (%d) -- possible hash collision "
"(0x%lx, 0x%lx, 0x%lx)", collComm->nRanks + msgComm->nRanks, newComm->commNRanks,
collComm->commId.commHash, collComm->commId.hostHash, collComm->commId.pidHash);
newComm->nRanks = newComm->commNRanks;
// We'll skip the extras in the loop below.
} else {
newComm->nRanks = collComm->nRanks + msgComm->nRanks;
@@ -691,16 +800,18 @@ static ncclResult_t rasCollCommsMerge(struct rasCollective* coll, struct rasMsg*
int cmpRank;
if (newRankIdx == newComm->commNRanks)
break; // Short of failing, the best we can do is skip...
if (collRankIdx < collComm->nRanks && msgRankIdx < msgComm->nRanks)
if (collRankIdx < collComm->nRanks && msgRankIdx < msgComm->nRanks) {
cmpRank = (collComm->ranks[collRankIdx].commRank < msgComm->ranks[msgRankIdx].commRank ? -1 :
(collComm->ranks[collRankIdx].commRank > msgComm->ranks[msgRankIdx].commRank ? 1 : 0));
else
} else {
cmpRank = (collRankIdx < collComm->nRanks ? -1 : 1);
}
// There shouldn't be any overlaps in ranks between different sources.
if (cmpRank == 0) {
INFO(NCCL_RAS, "RAS encountered duplicate data for rank %d -- possible commHash collision (0x%lx)",
collComm->ranks[collRankIdx].commRank, newComm->commHash);
INFO(NCCL_RAS, "RAS encountered duplicate data for rank %d -- possible hash collision "
"(0x%lx, 0x%lx, 0x%lx)", collComm->ranks[collRankIdx].commRank,
newComm->commId.commHash, newComm->commId.hostHash, newComm->commId.pidHash);
msgRankIdx++; // Short of failing, the best we can do is skip...
}
memcpy(newComm->ranks+newRankIdx, (cmpRank <= 0 ? collComm->ranks+collRankIdx++ :
@@ -708,23 +819,63 @@ static ncclResult_t rasCollCommsMerge(struct rasCollective* coll, struct rasMsg*
if (cmpRank > 0) {
// peerIdx values from msgComm need to shift after merge.
newComm->ranks[newRankIdx].peerIdx += coll->nPeers;
}
if (collComm->nMissingRanks > 0) {
// Remove the corresponding entry from missingRanks.
struct rasCollCommsMissingRank* missingRank;
missingRank = (struct rasCollCommsMissingRank*)bsearch(&newComm->ranks[newRankIdx].commRank,
collComm->ranks+collComm->nRanks,
collComm->nMissingRanks,
sizeof(struct rasCollCommsMissingRank),
rasCollCommsMissingRankSearch);
if (missingRank) {
// Mark the entry as no longer needed.
memset(&missingRank->addr, '\0', sizeof(missingRank->addr));
} else {
INFO(NCCL_RAS, "RAS failed to find missingRank data -- internal error?");
}
} // if (collComm->nMissingRanks > 0)
} // if (cmpRank > 0)
} // for (newRankIdx)
newComm = (struct rasCollComms::comm*)(((char*)(newComm+1)) + newComm->nRanks * sizeof(*newComm->ranks));
collComm = (struct rasCollComms::comm*)(((char*)(collComm+1)) + collComm->nRanks * sizeof(*collComm->ranks));
if (collComm->nMissingRanks > 0) {
// Copy the missingRanks to newComm, skipping over any no longer needed entries.
union ncclSocketAddress emptyAddr;
struct rasCollCommsMissingRank* collMissingRanks;
struct rasCollCommsMissingRank* newMissingRanks;
int newRankIdx;
memset(&emptyAddr, '\0', sizeof(emptyAddr));
collMissingRanks = (struct rasCollCommsMissingRank*)(collComm->ranks+collComm->nRanks);
newMissingRanks = (struct rasCollCommsMissingRank*)(newComm->ranks+newComm->nRanks);
newRankIdx = 0;
for (int collRankIdx = 0; collRankIdx < collComm->nMissingRanks; collRankIdx++) {
if (memcmp(&collMissingRanks[collRankIdx].addr, &emptyAddr, sizeof(emptyAddr))) {
memcpy(newMissingRanks + newRankIdx++, collMissingRanks + collRankIdx, sizeof(*newMissingRanks));
}
}
newComm->nMissingRanks = newRankIdx;
assert(newComm->nRanks + newComm->nMissingRanks == newComm->commNRanks);
}
newComm = (struct rasCollComms::comm*)(((char*)(newComm+1)) + newComm->nRanks * sizeof(*newComm->ranks) +
newComm->nMissingRanks * sizeof(struct rasCollCommsMissingRank));
collComm = (struct rasCollComms::comm*)(((char*)(collComm+1)) + collComm->nRanks * sizeof(*collComm->ranks) +
collComm->nMissingRanks * sizeof(struct rasCollCommsMissingRank));
collIdx++;
msgComm = (struct rasCollComms::comm*)(((char*)(msgComm+1)) + msgComm->nRanks * sizeof(*msgComm->ranks));
msgComm = (struct rasCollComms::comm*)(((char*)(msgComm+1)) + msgComm->nRanks * sizeof(*msgComm->ranks) +
msgComm->nMissingRanks * sizeof(struct rasCollCommsMissingRank));
msgIdx++;
} else if (cmp < 0) {
// Copy from collComm.
int commSize = sizeof(*collComm) + collComm->nRanks * sizeof(*collComm->ranks);
int commSize = sizeof(*collComm) + collComm->nRanks * sizeof(*collComm->ranks) +
collComm->nMissingRanks * sizeof(struct rasCollCommsMissingRank);
memcpy(newComm, collComm, commSize);
newComm = (struct rasCollComms::comm*)(((char*)(newComm)) + commSize);
collComm = (struct rasCollComms::comm*)(((char*)(collComm)) + commSize);
collIdx++;
} else { // cmp > 0
// Copy from msgComm.
int commSize = sizeof(*msgComm) + msgComm->nRanks * sizeof(*msgComm->ranks);
int commSize = sizeof(*msgComm) + msgComm->nRanks * sizeof(*msgComm->ranks) +
msgComm->nMissingRanks * sizeof(struct rasCollCommsMissingRank);
memcpy(newComm, msgComm, commSize);
for (int i = 0; i < newComm->nRanks; i++) {
// peerIdx values from msgComm need to shift after merge.
@@ -745,18 +896,87 @@ static ncclResult_t rasCollCommsMerge(struct rasCollective* coll, struct rasMsg*
return ncclSuccess;
}
// Checks if a given communicator is in the skipMissingRanksComms array of the request.
static bool rasCollCommsSkipMissing(const struct rasCollRequest* req, struct ncclComm* comm) {
struct rasCommId id;
id.commHash = comm->commHash;
id.hostHash = comm->peerInfo->hostHash;
id.pidHash = comm->peerInfo->pidHash;
return (bsearch(&id, req->comms.skipMissingRanksComms, req->comms.nSkipMissingRanksComms,
sizeof(*req->comms.skipMissingRanksComms), rasCommIdCompare) != nullptr);
}
// Sorting callback for the ncclComms array.
static int ncclCommsCompare(const void* p1, const void* p2) {
const ncclComm** pc1 = (const ncclComm**)p1;
const ncclComm** pc2 = (const ncclComm**)p2;
const ncclComm* comm1 = *(const ncclComm**)p1;
const ncclComm* comm2 = *(const ncclComm**)p2;
// Put nullptr's at the end.
if (*pc1 == nullptr || *pc2 == nullptr)
return (*pc1 != nullptr ? -1 : (*pc2 != nullptr ? 1 : 0));
if (comm1 == nullptr || comm2 == nullptr)
return (comm1 != nullptr ? -1 : (comm2 != nullptr ? 1 : 0));
if ((*pc1)->commHash == (*pc2)->commHash) {
return ((*pc1)->rank < (*pc2)->rank ? -1 : ((*pc1)->rank > (*pc2)->rank ? 1 : 0));
if (comm1->commHash == comm2->commHash) {
return (comm1->rank < comm2->rank ? -1 : (comm1->rank > comm2->rank ? 1 : 0));
} else {
return ((*pc1)->commHash < (*pc2)->commHash ? -1 : 1);
return (comm1->commHash < comm2->commHash ? -1 : 1);
}
}
// Sorting callback for a lookup table to rasPeers. Sorts by the hostHash (primary) and pidHash (secondary).
static int peersHashesCompare(const void* p1, const void* p2) {
const struct rasPeerInfo* pi1 = *(const struct rasPeerInfo**)p1;
const struct rasPeerInfo* pi2 = *(const struct rasPeerInfo**)p2;
if (pi1->hostHash == pi2->hostHash) {
return (pi1->pidHash < pi2->pidHash ? -1 : (pi1->pidHash > pi2->pidHash ? 1 : 0));
} else {
return (pi1->hostHash < pi2->hostHash ? -1 : 1);
}
}
// Search callback for a lookup table to rasPeers. Searches by the hostHash and pidHash. The key is an array
// containing the hostHash at index 0 and the pidHash at index 1.
static int peersHashesSearch(const void* k, const void* e) {
const uint64_t* key = (const uint64_t*)k;
const struct rasPeerInfo* elem = *(const struct rasPeerInfo**)e;
if (key[0] == elem->hostHash) {
return (key[1] < elem->pidHash ? -1 : (key[1] > elem->pidHash ? 1 : 0));
} else {
return (key[0] < elem->hostHash ? -1 : 1);
}
}
// Sorting/searching callback for struct rasCommId. Sorts by commHash, then hostHash, then pidHash.
static int rasCommIdCompare(const void* p1, const void* p2) {
const struct rasCommId* i1 = (const struct rasCommId*)p1;
const struct rasCommId* i2 = (const struct rasCommId*)p2;
if (i1->commHash == i2->commHash) {
if (i1->hostHash == i2->hostHash) {
return (i1->pidHash < i2->pidHash ? -1 : (i1->pidHash > i2->pidHash ? 1 : 0));
} else {
return (i1->hostHash < i2->hostHash ? -1 : 1);
}
} else {
return (i1->commHash < i2->commHash ? -1 : 1);
}
}
// Search callback for rasCollComms::comm rasCollCommsMissingRank array. The key is the commRank.
static int rasCollCommsMissingRankSearch(const void* k, const void* e) {
int key = *(const int*)k;
const struct rasCollCommsMissingRank* elem = (const struct rasCollCommsMissingRank*)e;
return (key < elem->commRank ? -1 : (key > elem->commRank ? 1 : 0));
}
// Invoked during RAS termination to release all the allocated resources.
void rasCollectivesTerminate() {
for (struct rasCollective* coll = rasCollectivesHead; coll;) {
struct rasCollective* collNext = coll->next;
rasCollFree(coll);
coll = collNext;
}
// rasCollectivesHead and rasCollectivesTail are taken care of by rasCollFree().
}
src/ras/peers.cc
+90 -102
Просмотреть файл
@@ -40,10 +40,11 @@ static ncclResult_t rasRanksConvertToPeers(struct rasRankInit* ranks, int nranks
static ncclResult_t rasPeersUpdate(struct rasPeerInfo* rankPeers, int* nRankPeers, int newNRasPeers = -1);
static ncclResult_t rasNetUpdatePeers(const struct rasPeerInfo* newPeers, int nNewPeers, bool updateDeadPeers,
struct rasRankInit* ranks = nullptr, int nranks = 0, int fromConnIdx = -1);
struct rasRankInit* ranks = nullptr, int nranks = 0,
struct rasConnection* fromConn = nullptr);
static ncclResult_t rasLinkPropagateUpdate(struct rasLink* link, const struct rasPeerInfo* newPeers, int nNewPeers,
bool updateDeadPeers, struct rasRankInit* ranks, int nranks,
int fromConnIdx);
struct rasConnection* fromConn);
static ncclResult_t rasConnPropagateUpdate(struct rasConnection* conn, const struct rasPeerInfo* newPeers,
int nNewPeers, bool updateDeadPeers, struct rasRankInit* ranks, int nranks);
ncclResult_t rasMsgHandlePeersUpdate(struct rasMsg* msg, struct rasSocket* sock);
@@ -146,6 +147,8 @@ static ncclResult_t rasRanksConvertToPeers(struct rasRankInit* ranks, int nranks
rankPeer->pid = rank->pid;
rankPeer->cudaDevs = (1UL << rank->cudaDev);
rankPeer->nvmlDevs = (1UL << rank->nvmlDev);
rankPeer->hostHash = rank->hostHash;
rankPeer->pidHash = rank->pidHash;
rankPeerIdx++;
// Also check if there is already an entry with that address in the global rasPeers so that the caller can know how
@@ -357,12 +360,12 @@ int rasPeerFind(const union ncclSocketAddress* addr) {
// ranks -- if provided -- lists all the peers who are already aware of this update (because they are the members
// of the new communicator being established), and who thus don't need to be notified. updatedDeadPeers can
// be used, however, to request at least the propagation of rasDeadPeers to such peers.
// fromConnIdx -- if provided -- identified the connection used to receive this update; there's no need to
// fromConn -- if provided -- identifies the connection used to receive this update; there's no need to
// propagate the update back through it.
// Reconfigures the RAS network to accommodate the newly added peers, by modifying the links and establishing new
// connections as needed.
static ncclResult_t rasNetUpdatePeers(const struct rasPeerInfo* newPeers, int nNewPeers, bool updateDeadPeers,
struct rasRankInit* ranks, int nranks, int fromConnIdx) {
struct rasRankInit* ranks, int nranks, struct rasConnection* fromConn) {
ncclResult_t ret = ncclSuccess;
// Do we actually have anything to do?
@@ -371,8 +374,8 @@ static ncclResult_t rasNetUpdatePeers(const struct rasPeerInfo* newPeers, int nN
// Start by propagating the update through the RAS network links. We consider any errors during this process
// to be non-fatal (we can re-sync later around a keep-alive exchange).
(void)rasLinkPropagateUpdate(&rasNextLink, newPeers, nNewPeers, updateDeadPeers, ranks, nranks, fromConnIdx);
(void)rasLinkPropagateUpdate(&rasPrevLink, newPeers, nNewPeers, updateDeadPeers, ranks, nranks, fromConnIdx);
(void)rasLinkPropagateUpdate(&rasNextLink, newPeers, nNewPeers, updateDeadPeers, ranks, nranks, fromConn);
(void)rasLinkPropagateUpdate(&rasPrevLink, newPeers, nNewPeers, updateDeadPeers, ranks, nranks, fromConn);
// Calculate new link peers and open new connections if needed.
NCCLCHECKGOTO(rasLinkReinitConns(&rasNextLink), ret, fail);
@@ -388,15 +391,13 @@ fail:
// for the explanation of the function arguments.
static ncclResult_t rasLinkPropagateUpdate(struct rasLink* link, const struct rasPeerInfo* newPeers, int nNewPeers,
bool updateDeadPeers, struct rasRankInit* ranks, int nranks,
int fromConnIdx) {
for (int i = 0; i < link->nConns; i++) {
struct rasLinkConn* linkConn = link->conns+i;
struct rasConnection* fromConn) {
for (struct rasLinkConn* linkConn = link->conns; linkConn; linkConn = linkConn->next) {
// Note that we don't send the update via the connection that we received this notification from in the first
// place (while it wouldn't loop indefinitely, it would add a needless extra exchange).
if (linkConn->connIdx != -1 && linkConn->connIdx != fromConnIdx) {
struct rasConnection* conn = rasConns+linkConn->connIdx;
if (linkConn->conn && linkConn->conn != fromConn) {
// Failed propagations are not considered fatal (we will retry after a keep-alive).
(void)rasConnPropagateUpdate(conn, newPeers, nNewPeers, updateDeadPeers, ranks, nranks);
(void)rasConnPropagateUpdate(linkConn->conn, newPeers, nNewPeers, updateDeadPeers, ranks, nranks);
}
}
@@ -407,7 +408,7 @@ static ncclResult_t rasLinkPropagateUpdate(struct rasLink* link, const struct ra
// arguments.
static ncclResult_t rasConnPropagateUpdate(struct rasConnection* conn, const struct rasPeerInfo* newPeers,
int nNewPeers, bool updateDeadPeers, struct rasRankInit* ranks, int nranks) {
if (conn->sockIdx != -1 && rasSockets[conn->sockIdx].status == RAS_SOCK_READY) {
if (conn->sock && conn->sock->status == RAS_SOCK_READY) {
// If we have the rank info, check if the peer on the other side of this connection has participated in the new
// communicator.
int connRank = -1;
@@ -462,7 +463,8 @@ ncclResult_t rasConnSendPeersUpdate(struct rasConnection* conn, const struct ras
msg->peersUpdate.deadPeersHash = rasDeadPeersHash;
msg->peersUpdate.nDeadPeers = nDeadPeers;
memcpy(msg->peersUpdate.peers, peers, nPeers * sizeof(msg->peersUpdate.peers[0]));
memcpy(((char*)msg)+deadPeersOffset, rasDeadPeers, nDeadPeers * sizeof(*rasDeadPeers));
if (nDeadPeers > 0)
memcpy(((char*)msg)+deadPeersOffset, rasDeadPeers, nDeadPeers * sizeof(*rasDeadPeers));
if (nPeers > 0)
conn->lastSentPeersHash = rasPeersHash;
@@ -485,8 +487,7 @@ ncclResult_t rasMsgHandlePeersUpdate(struct rasMsg* msg, struct rasSocket* sock)
ncclResult_t ret = ncclSuccess;
struct rasMsg* newMsg = nullptr;
int newMsgLen = 0;
assert(sock->connIdx != -1);
struct rasConnection* conn = rasConns+sock->connIdx;
assert(sock->conn);
int nPeers, nDeadPeers;
int deadPeersOffset = 0;
bool updatePeers, updateDeadPeers;
@@ -496,8 +497,8 @@ ncclResult_t rasMsgHandlePeersUpdate(struct rasMsg* msg, struct rasSocket* sock)
msg->peersUpdate.nPeers, msg->peersUpdate.nDeadPeers);
INFO(NCCL_RAS, "RAS my old rasPeersHash 0x%lx, rasDeadPeersHash 0x%lx, nRasPeers %d, nRasDeadPeers %d",
rasPeersHash, rasDeadPeersHash, nRasPeers, nRasDeadPeers);
conn->lastRecvPeersHash = msg->peersUpdate.peersHash;
conn->lastRecvDeadPeersHash = msg->peersUpdate.deadPeersHash;
sock->conn->lastRecvPeersHash = msg->peersUpdate.peersHash;
sock->conn->lastRecvDeadPeersHash = msg->peersUpdate.deadPeersHash;
// Prepare ours to send back. We don't enqueue it right away because we want to make sure first that we need
// to send it. We'll find out by comparing the hash values after the merge.
@@ -545,15 +546,15 @@ ncclResult_t rasMsgHandlePeersUpdate(struct rasMsg* msg, struct rasSocket* sock)
rasDeadPeersDump();
// If post-merge the hashes are still different, send our (dead) peers back.
updatePeers = (conn->lastSentPeersHash != rasPeersHash && conn->lastRecvPeersHash != rasPeersHash);
updateDeadPeers = (conn->lastSentDeadPeersHash != rasDeadPeersHash &&
conn->lastRecvDeadPeersHash != rasDeadPeersHash);
updatePeers = (sock->conn->lastSentPeersHash != rasPeersHash && sock->conn->lastRecvPeersHash != rasPeersHash);
updateDeadPeers = (sock->conn->lastSentDeadPeersHash != rasDeadPeersHash &&
sock->conn->lastRecvDeadPeersHash != rasDeadPeersHash);
if (updatePeers || updateDeadPeers) {
newMsg->peersUpdate.peersHash = rasPeersHash;
newMsg->peersUpdate.deadPeersHash = rasDeadPeersHash;
if (updatePeers) {
assert(nPeers > 0);
conn->lastSentPeersHash = rasPeersHash;
sock->conn->lastSentPeersHash = rasPeersHash;
} else {
// If hashes match, make sure that we don't send the rasPeers back.
newMsg->peersUpdate.nPeers = 0;
@@ -564,14 +565,14 @@ ncclResult_t rasMsgHandlePeersUpdate(struct rasMsg* msg, struct rasSocket* sock)
if (updateDeadPeers) {
assert(nRasDeadPeers > 0);
conn->lastSentDeadPeersHash = rasDeadPeersHash;
sock->conn->lastSentDeadPeersHash = rasDeadPeersHash;
ALIGN_SIZE(newMsgLen, alignof(union ncclSocketAddress));
deadPeersOffset = newMsgLen;
newMsgLen += nRasDeadPeers*sizeof(*rasDeadPeers);
memcpy(((char*)newMsg)+deadPeersOffset, rasDeadPeers, nDeadPeers * sizeof(*rasDeadPeers));
conn->lastSentDeadPeersHash = rasDeadPeersHash;
sock->conn->lastSentDeadPeersHash = rasDeadPeersHash;
newMsg->peersUpdate.nDeadPeers = nRasDeadPeers;
} else {
newMsg->peersUpdate.nDeadPeers = 0;
@@ -580,13 +581,13 @@ ncclResult_t rasMsgHandlePeersUpdate(struct rasMsg* msg, struct rasSocket* sock)
INFO(NCCL_RAS, "RAS sending back a peersUpdate (nPeers %d, nDeadPeers %d)",
newMsg->peersUpdate.nPeers, newMsg->peersUpdate.nDeadPeers);
rasConnEnqueueMsg(conn, newMsg, newMsgLen);
rasConnEnqueueMsg(sock->conn, newMsg, newMsgLen);
newMsg = nullptr;
} // if (updatePeers || updateDeadPeers)
// Propagate the changes through our RAS network links.
NCCLCHECKGOTO(rasNetUpdatePeers(msg->peersUpdate.peers, msg->peersUpdate.nPeers, updateDeadPeers, nullptr, 0,
sock->connIdx), ret, fail);
sock->conn), ret, fail);
}
exit:
@@ -603,7 +604,7 @@ fail:
// Reinitializes the connection(s) of a particular link, following a peers update.
// Adding new peers can affect the calculation of the link's primary connection and also the fallbacks.
// The newly added peers could also shift all the existing peerIdx values, invalidating the values in RasLinkConn
// The newly added peers could also shift all the existing peerIdx values, invalidating the values in rasLinkConn
// structures, so it's better to drop it all and recalculate from scratch.
// We recalculate the primary peer; if an active connection to it already exists, then we're done. If there
// is no connection, we create one. If a connection exists but is experiencing delays then we add a fallback and
@@ -611,77 +612,51 @@ fail:
// External conns are dropped from the links as well (they will be re-created via keepAlive messages as needed).
static ncclResult_t rasLinkReinitConns(struct rasLink* link) {
struct rasLinkConn* linkConn;
struct rasConnection* conn = nullptr;
int newPeerIdx = myPeerIdx;
if (link->connsSize == 0) {
link->connsSize = RAS_INCREMENT;
NCCLCHECK(ncclCalloc(&link->conns, link->connsSize));
if (link->conns) {
// Free the old contents but keep the first entry for convenience (though wipe it).
for (struct rasLinkConn* linkConn = link->conns->next; linkConn;) {
struct rasLinkConn* linkConnNext = linkConn->next;
free(linkConn);
linkConn = linkConnNext;
}
memset(link->conns, '\0', sizeof(*link->conns));
link->lastUpdatePeersTime = 0;
} else { // link->conns == nullptr
NCCLCHECK(ncclCalloc(&link->conns, 1));
}
link->nConns = 0;
// Establish a connection for this link. We iterate as long as the connections we find are experiencing delays.
while (newPeerIdx != -1) {
if (link->nConns == link->connsSize) {
NCCLCHECK(ncclRealloc(&link->conns, link->connsSize, link->connsSize+RAS_INCREMENT));
link->connsSize += RAS_INCREMENT;
}
// Fill in the entry for the primary connection.
linkConn = link->conns;
linkConn->peerIdx = newPeerIdx = rasLinkCalculatePeer(link, myPeerIdx, /*isFallback*/false);
linkConn->conn = (newPeerIdx != -1 ? rasConnFind(&rasPeers[newPeerIdx].addr) : nullptr);
linkConn->external = false;
newPeerIdx = rasLinkCalculatePeer(link, newPeerIdx, /*isFallback*/link->nConns > 1);
if (newPeerIdx == -1) {
INFO(NCCL_RAS, "RAS link %d: no more fallbacks to add (nConns %d)", link->direction, link->nConns);
if (link->nConns > 0)
break;
}
linkConn = link->conns+link->nConns;
linkConn->peerIdx = newPeerIdx;
linkConn->connIdx = (newPeerIdx != -1 ? rasConnFind(&rasPeers[newPeerIdx].addr) : -1);
linkConn->external = false;
// If the calculated connection does not exist, then we are at the end of the chain and this is the last iteration.
// Depending on the circumstances, we may first need to create that connection.
if (linkConn->connIdx == - 1) {
if (link->nConns == 0) {
if (linkConn->peerIdx != -1) {
INFO(NCCL_RAS, "RAS link %d: %s primary connection with %s",
link->direction, (myPeerIdx < linkConn->peerIdx ? "opening new" : "calculated deferred"),
ncclSocketToString(&rasPeers[linkConn->peerIdx].addr, rasLine));
// We try to initiate primary connections from the side with a lower address (and thus an earlier peer index)
// to avoid races and the creation of duplicate connections.
if (myPeerIdx < linkConn->peerIdx) {
NCCLCHECK(rasConnCreate(&rasPeers[linkConn->peerIdx].addr, &linkConn->connIdx));
}
else { // If we didn't initiate the connection, start the timeout.
link->lastUpdatePeersTime = clockNano();
}
} // if (linkConn->peerIdx != -1)
} else { // link->nConns > 0
INFO(NCCL_RAS, "RAS link %d: opening new fallback connection %d with %s",
link->direction, link->nConns, ncclSocketToString(&rasPeers[linkConn->peerIdx].addr, rasLine));
NCCLCHECK(rasConnCreate(&rasPeers[newPeerIdx].addr, &linkConn->connIdx));
} // link->nConns > 0
} else { // linkConn->connIdx != -1
if (link->nConns == 0) {
INFO(NCCL_RAS, "RAS link %d: calculated existing primary connection with %s",
link->direction, ncclSocketToString(&rasPeers[linkConn->peerIdx].addr, rasLine));
} else {
INFO(NCCL_RAS, "RAS link %d: calculated existing fallback connection %d with %s",
link->direction, link->nConns, ncclSocketToString(&rasPeers[linkConn->peerIdx].addr, rasLine));
if (linkConn->conn == nullptr) {
if (linkConn->peerIdx != -1) {
// We try to initiate primary connections from the side with a lower address (and thus an earlier peer index)
// to avoid races and the creation of duplicate connections.
INFO(NCCL_RAS, "RAS link %d: %s primary connection with %s",
link->direction, (myPeerIdx < linkConn->peerIdx ? "opening new" : "calculated deferred"),
ncclSocketToString(&rasPeers[linkConn->peerIdx].addr, rasLine));
if (myPeerIdx < linkConn->peerIdx) {
NCCLCHECK(rasConnCreate(&rasPeers[linkConn->peerIdx].addr, &linkConn->conn));
}
}
link->nConns++;
if (linkConn->connIdx == -1)
break;
conn = rasConns+linkConn->connIdx;
// We check if the connection already went through the fallback calculation; if so, we'll need to create a new
// fallback in the next iteration, to ensure that RAS will keep retrying.
if (!conn->experiencingDelays)
break;
else { // If we didn't initiate the connection, start the timeout.
link->lastUpdatePeersTime = clockNano();
}
} // if (linkConn->peerIdx != -1)
} else { // linkConn->conn
INFO(NCCL_RAS, "RAS link %d: calculated existing primary connection with %s",
link->direction, ncclSocketToString(&rasPeers[linkConn->peerIdx].addr, rasLine));
} // linkConn->conn
if (linkConn->conn && linkConn->conn->experiencingDelays) {
INFO(NCCL_RAS, "RAS connection experiencingDelays %d, startRetryTime %.2fs, socket status %d",
conn->experiencingDelays, (clockNano()-conn->startRetryTime)/1e9,
(conn->sockIdx == -1 ? -1 : rasSockets[conn->sockIdx].status));
linkConn->conn->experiencingDelays, (clockNano()-linkConn->conn->startRetryTime)/1e9,
(linkConn->conn->sock ? linkConn->conn->sock->status : - 1));
NCCLCHECK(rasLinkAddFallback(link, linkConn->conn));
}
return ncclSuccess;
@@ -701,39 +676,37 @@ int rasLinkCalculatePeer(const struct rasLink* link, int peerIdx, bool isFallbac
if (isFallback && !ncclSocketsSameNode(&rasPeers[peerIdx].addr, &rasNetListeningSocket.addr)) {
// peerIdx is a fallback and it is not running on the same node as us.
int tryPeerIdx = newPeerIdx;
int tryConnIdx = -1;
struct rasConnection* tryConn = nullptr;
// Try to skip the remaining peers on the same node as peerIdx. We may end up skipping over some peers that
// are alive, which is fine -- they will still have connectivity with the rest of the RAS network, just a
// little suboptimal one.
while (ncclSocketsSameNode(&rasPeers[tryPeerIdx].addr, &rasPeers[peerIdx].addr)) {
if (!rasPeerIsDead(&rasPeers[tryPeerIdx].addr)) {
tryConnIdx = rasConnFind(&rasPeers[tryPeerIdx].addr);
if (tryConnIdx != -1) {
struct rasConnection* tryConn = rasConns+tryConnIdx;
tryConn = rasConnFind(&rasPeers[tryPeerIdx].addr);
if (tryConn) {
// Check if the connection is fully established and operational, i.e., if the underlying socket
// is ready and there's been recent communication on it.
if (tryConn->sockIdx != -1 && rasSockets[tryConn->sockIdx].status == RAS_SOCK_READY &&
!tryConn->experiencingDelays) {
if (tryConn->sock && tryConn->sock->status == RAS_SOCK_READY && !tryConn->experiencingDelays) {
// We convinced ourselves that the node is not down. We don't adjust newPeerIdx in
// this case. This is the only case when tryConnIdx != -1 after this loop.
break;
}
} // if (tryConnIdx != -1)
} // if (tryConn)
} // if (!rasPeerIsDead(&rasPeers[tryPeerIdx].addr))
tryConnIdx = -1;
tryPeerIdx = (tryPeerIdx + nRasPeers + link->direction) % nRasPeers;
tryConn = nullptr;
tryPeerIdx = (tryPeerIdx + link->direction + nRasPeers) % nRasPeers;
if (tryPeerIdx == myPeerIdx)
break;
}
if (tryConnIdx == -1)
if (tryConn == nullptr)
newPeerIdx = tryPeerIdx;
if (tryPeerIdx == myPeerIdx)
break;
} // if (isFallback && !ncclSocketsSameNode(&rasPeers[peerIdx].addr, &rasNetListeningSocket.addr))
if (rasPeerIsDead(&rasPeers[newPeerIdx].addr)) {
newPeerIdx = (newPeerIdx + nRasPeers + link->direction) % nRasPeers;
}
@@ -932,7 +905,8 @@ bool ncclSocketsSameNode(const union ncclSocketAddress* a1, const union ncclSock
static void rasPeersDump() {
for (int p = 0; p < nRasPeers; p++) {
const struct rasPeerInfo* peer = rasPeers+p;
INFO(NCCL_RAS, "RAS peer %d: %s%s", p, rasPeerDump(peer, rasLine, sizeof(rasLine)), (p == myPeerIdx ? " [this process]" : ""));
INFO(NCCL_RAS, "RAS peer %d: %s%s", p, rasPeerDump(peer, rasLine, sizeof(rasLine)),
(p == myPeerIdx ? " [this process]" : ""));
}
if (nRasPeers > 0)
INFO(NCCL_RAS, "RAS peersHash 0x%lx", rasPeersHash);
@@ -958,3 +932,17 @@ static char* rasPeerDump(const struct rasPeerInfo* peer, char* result, size_t nr
rasGpuDevsToString(peer->cudaDevs, peer->nvmlDevs, line2, sizeof(line2)));
return result;
}
// Invoked during RAS termination to release all the allocated resources.
void rasPeersTerminate() {
free(rasPeers);
rasPeers = nullptr;
nRasPeers = 0;
rasPeersHash = 0;
myPeerIdx = -1;
free(rasDeadPeers);
rasDeadPeers = nullptr;
nRasDeadPeers = rasDeadPeersSize = 0;
rasDeadPeersHash = 0;
}
src/ras/ras.cc
+109 -73
Просмотреть файл
@@ -4,8 +4,10 @@
* See LICENSE.txt for license information
************************************************************************/
#define NDEBUG // Comment out during development only!
#include <cassert>
// Workaround for libstdc++ trying to force public visibility of std:: symbols. We don't want to do that in libnccl.so.
#include <bits/c++config.h>
#undef _GLIBCXX_VISIBILITY
#define _GLIBCXX_VISIBILITY(V)
#include <cstddef>
#include <mutex>
#include <poll.h>
@@ -65,8 +67,8 @@ int nNcclComms = 0;
bool ncclCommsSorted = false; // Whether the array is currently sorted. We sort by the comms' commHash and rank.
static ncclResult_t rasLocalNotify(const struct rasNotification* msg);
static ncclResult_t rasLocalHandle();
static void rasLocalHandleTerminate();
static ncclResult_t rasLocalHandle(bool* terminate);
static void rasThreadCleanup();
static ncclResult_t rasMsgHandleConnInit(const struct rasMsg* msg, struct rasSocket* sock);
static ncclResult_t rasMsgHandleConnInitAck(const struct rasMsg* msg, struct rasSocket* sock);
@@ -74,6 +76,8 @@ static ncclResult_t rasNetSendNack(struct rasSocket* sock);
static void* rasThreadMain(void*);
static void rasTerminate() __attribute__((destructor));
NCCL_PARAM(RasTimeoutFactor, "RAS_TIMEOUT_FACTOR", 1);
//////////////////////////////////////////////////
@@ -105,7 +109,6 @@ ncclResult_t ncclRasCommInit(struct ncclComm* comm, struct rasRankInit* myRank)
PTHREADCHECKGOTO(pthread_create(&rasThread, nullptr, &rasThreadMain, nullptr), "pthread_create", ret, fail);
ncclSetThreadName(rasThread, "NCCL RAS");
(void)pthread_detach(rasThread);
rasInitialized = true;
}
@@ -157,18 +160,27 @@ ncclResult_t ncclRasCommFini(const struct ncclComm* comm) {
}
}
}
if (ncclAtomicRefCountDecrement(&rasInitRefCount) == 0) {
struct rasNotification msg;
msg.type = RAS_TERMINATE;
NCCLCHECK(rasLocalNotify(&msg));
}
ncclAtomicRefCountDecrement(&rasInitRefCount);
return ncclSuccess;
}
// Global destructor. Notifies the RAS thread to release all the resources
// and terminate. Waits for the thread to terminate.
static void rasTerminate() {
struct rasNotification msg;
if (!rasInitialized)
return;
memset(&msg, '\0', sizeof(msg));
msg.type = RAS_TERMINATE;
if (rasLocalNotify(&msg) == ncclSuccess)
(void)pthread_join(rasThread, nullptr);
}
// Invoked by regular NCCL threads on every (non-split) comm initialization. Provides info on all the ranks within
// the communicator.
ncclResult_t ncclRasAddRanks(struct rasRankInit* ranks, int nranks) {
struct rasNotification msg;
memset(&msg, '\0', sizeof(msg));
msg.type = RAS_ADD_RANKS;
msg.addRanks.ranks = ranks;
msg.addRanks.nranks = nranks;
@@ -199,7 +211,7 @@ static ncclResult_t rasLocalNotify(const struct rasNotification* msg) {
/////////////////////////////////////////////////////////////////////////////////
// Handles asynchronous local notifications arriving from regular NCCL threads.
static ncclResult_t rasLocalHandle() {
static ncclResult_t rasLocalHandle(bool* terminate) {
struct rasNotification msg;
size_t done = 0;
@@ -212,9 +224,11 @@ static ncclResult_t rasLocalHandle() {
}
if (msg.type == RAS_ADD_RANKS) {
NCCLCHECK(rasLocalHandleAddRanks(msg.addRanks.ranks, msg.addRanks.nranks));
(void)rasLocalHandleAddRanks(msg.addRanks.ranks, msg.addRanks.nranks);
// Not great if the above fails, but it shouldn't be critical; better to keep going.
} else if (msg.type == RAS_TERMINATE) {
rasLocalHandleTerminate();
INFO(NCCL_RAS, "RAS handling local termination request");
*terminate = true;
} else {
WARN("RAS received unknown notification type %d", msg.type);
return ncclInternalError;
@@ -223,10 +237,35 @@ static ncclResult_t rasLocalHandle() {
return ncclSuccess;
}
// Handles local RAS_TERMINATE notification.
static void rasLocalHandleTerminate() {
INFO(NCCL_RAS, "RAS handling local termination request");
// For now we don't do anything.
// Cleans up local RAS state, normally in response to a RAS_TERMINATE notification.
static void rasThreadCleanup() {
rasClientSupportTerminate();
rasNetTerminate();
rasCollectivesTerminate();
rasPeersTerminate();
{
std::lock_guard<std::mutex> lock(rasInitMutex);
(void)close(rasNotificationPipe[1]);
(void)close(rasNotificationPipe[0]);
// rasClientListeningSocket is taken care of by rasClientSupportTerminate().
rasNotificationPipe[0] = rasNotificationPipe[1] = -1;
(void)ncclSocketClose(&rasNetListeningSocket);
rasInitRefCount = 0;
rasInitialized = false;
}
{
std::lock_guard<std::mutex> lock(ncclCommsMutex);
free(ncclComms);
ncclComms = nullptr;
nNcclComms = 0;
ncclCommsSorted = false;
}
free(rasPfds);
rasPfds = nullptr;
nRasPfds = 0;
}
@@ -270,10 +309,10 @@ void rasConnEnqueueMsg(struct rasConnection* conn, struct rasMsg* msg, size_t ms
else
ncclIntruQueueEnqueue(&conn->sendQ, meta);
if (conn->sockIdx != -1) {
struct rasSocket* sock = rasSockets+conn->sockIdx;
if (sock->status == RAS_SOCK_READY || (sock->status == RAS_SOCK_HANDSHAKE && msg->type == RAS_MSG_CONNINIT)) {
rasPfds[sock->pfd].events |= POLLOUT;
if (conn->sock) {
if (conn->sock->status == RAS_SOCK_READY ||
(conn->sock->status == RAS_SOCK_HANDSHAKE && msg->type == RAS_MSG_CONNINIT)) {
rasPfds[conn->sock->pfd].events |= POLLOUT;
ready = true;
}
}
@@ -283,31 +322,31 @@ void rasConnEnqueueMsg(struct rasConnection* conn, struct rasMsg* msg, size_t ms
"(experiencingDelays %d, startRetryTime %.2fs, socket status %d)",
msg->type, ncclSocketToString(&conn->addr, rasLine),
conn->experiencingDelays, (conn->startRetryTime ? (clockNano()-conn->startRetryTime)/1e9 : 0.0),
(conn->sockIdx == -1 ? -1 : rasSockets[conn->sockIdx].status));
(conn->sock ? conn->sock->status : -1));
}
}
// Attempts to send the queued RAS messages to another RAS thread.
ncclResult_t rasConnSendMsg(struct rasConnection* conn, int* closed, bool* allSent) {
struct ncclSocket* sock = &rasSockets[conn->sockIdx].sock;
struct rasMsgMeta* meta;
*closed = 0;
while ((meta = ncclIntruQueueHead(&conn->sendQ)) != nullptr) {
if (rasSockets[conn->sockIdx].status == RAS_SOCK_HANDSHAKE && meta->msg.type != RAS_MSG_CONNINIT) {
if (conn->sock->status == RAS_SOCK_HANDSHAKE && meta->msg.type != RAS_MSG_CONNINIT) {
// We don't send anything beyond the handshake at this point.
meta = nullptr;
break;
}
if (meta->offset < sizeof(meta->length)) {
// Send the length of the message.
NCCLCHECK(ncclSocketProgress(NCCL_SOCKET_SEND, sock, &meta->length, sizeof(meta->length), &meta->offset, closed));
NCCLCHECK(ncclSocketProgress(NCCL_SOCKET_SEND, &conn->sock->sock, &meta->length, sizeof(meta->length),
&meta->offset, closed));
if (*closed)
return ncclSuccess;
if (meta->offset < sizeof(meta->length))
break;
}
// Send the body of the message.
NCCLCHECK(ncclSocketProgress(NCCL_SOCKET_SEND, sock, ((char*)&meta->msg)-sizeof(meta->length),
NCCLCHECK(ncclSocketProgress(NCCL_SOCKET_SEND, &conn->sock->sock, ((char*)&meta->msg)-sizeof(meta->length),
meta->length+sizeof(meta->length), &meta->offset, closed));
if (*closed)
return ncclSuccess;
@@ -377,7 +416,7 @@ ncclResult_t rasMsgHandle(struct rasMsg* msg, struct rasSocket* sock) {
static ncclResult_t rasMsgHandleConnInit(const struct rasMsg* msg, struct rasSocket* sock) {
ncclResult_t ret = ncclSuccess;
struct rasConnection* conn = nullptr;
int connIdx, peerIdx;
int peerIdx;
struct rasMsg* newMsg = nullptr;
int newMsgLen;
char line[SOCKET_NAME_MAXLEN+1];
@@ -406,19 +445,16 @@ static ncclResult_t rasMsgHandleConnInit(const struct rasMsg* msg, struct rasSoc
}
// Check for any existing connection with that RAS thread (could happen due to a network issue, or possibly a race).
connIdx = rasConnFind(&msg->connInit.listeningAddr);
if (connIdx != -1) {
conn = rasConns+connIdx;
conn = rasConnFind(&msg->connInit.listeningAddr);
if (conn) {
INFO(NCCL_RAS,
"RAS found a matching existing connection (sendQ %sempty, experiencingDelays %d, startRetryTime %.2fs)",
(ncclIntruQueueEmpty(&conn->sendQ) ? "" : "not "),
conn->experiencingDelays, (conn->startRetryTime ? (clockNano()-conn->startRetryTime)/1e9 : 0.0));
if (conn->sockIdx != -1) {
struct rasSocket* connSock = rasSockets+conn->sockIdx;
if (conn->sock) {
INFO(NCCL_RAS, "RAS found an alternative existing socket (status %d, createTime %.2fs)",
connSock->status, (clockNano()-connSock->createTime)/1e9);
conn->sock->status, (clockNano()-conn->sock->createTime)/1e9);
// In general we prefer to keep the newer connection, but "newer" can be a relative term: we may have
// a race where both sides attempt to establish a connection at roughly the same time, so the other side's
// incoming connection ends up looking newer than the locally-initiated one -- for *both* of them.
@@ -433,21 +469,19 @@ static ncclResult_t rasMsgHandleConnInit(const struct rasMsg* msg, struct rasSoc
goto exit;
} else {
INFO(NCCL_RAS, "RAS keeping the new socket and terminating the existing one");
rasSocketTerminate(connSock);
rasSocketTerminate(conn->sock);
}
}
}
if (!conn) {
} else { // conn == nullptr
NCCLCHECK(getNewConnEntry(&conn));
memcpy(&conn->addr, &msg->connInit.listeningAddr, sizeof(conn->addr));
connIdx = conn - rasConns;
}
sock->status = RAS_SOCK_READY;
// rasConnResume will reset any experiencingDelays, startRetryTime, etc.
conn->sockIdx = sock-rasSockets;
sock->connIdx = connIdx;
conn->sock = sock;
sock->conn = conn;
memcpy(&sock->sock.addr, &msg->connInit.listeningAddr, sizeof(sock->sock.addr));
// Make sure that the connection is part of the right links forming the RAS network. At this point we only
@@ -456,8 +490,8 @@ static ncclResult_t rasMsgHandleConnInit(const struct rasMsg* msg, struct rasSoc
// Note: it's possible for peerIdx to be -1 at this point if, due to races, the connInit arrives before
// the peers update.
if (peerIdx != -1) {
(void)rasLinkUpdateConn(&rasNextLink, connIdx, peerIdx);
(void)rasLinkUpdateConn(&rasPrevLink, connIdx, peerIdx);
(void)rasLinkConnUpdate(&rasNextLink, conn, peerIdx);
(void)rasLinkConnUpdate(&rasPrevLink, conn, peerIdx);
}
// Send a confirmation to the server that requested the connection (so that the resilience code can mark
@@ -504,12 +538,13 @@ static ncclResult_t rasMsgHandleConnInitAck(const struct rasMsg* msg, struct ras
}
// Handles the deadPeer broadcast.
void rasMsgHandleBCDeadPeer(const struct rasCollRequest* req, bool* pDone) {
INFO(NCCL_RAS, "RAS handling deadPeer (addr %s)", ncclSocketToString(&req->deadPeer.addr, rasLine));
void rasMsgHandleBCDeadPeer(struct rasCollRequest** pReq, size_t* pReqLen, bool* pDone) {
INFO(NCCL_RAS, "RAS handling deadPeer (addr %s)", ncclSocketToString(&(*pReq)->deadPeer.addr, rasLine));
if (!rasPeerIsDead(&req->deadPeer.addr)) {
rasConnDisconnect(&req->deadPeer.addr);
(void)rasPeerDeclareDead(&req->deadPeer.addr);
*pReqLen = rasCollDataLength(RAS_BC_DEADPEER);
if (!rasPeerIsDead(&(*pReq)->deadPeer.addr)) {
rasConnDisconnect(&(*pReq)->deadPeer.addr);
(void)rasPeerDeclareDead(&(*pReq)->deadPeer.addr);
*pDone = false;
} else {
INFO(NCCL_RAS, "RAS already knew it was dead");
@@ -530,6 +565,7 @@ static ncclResult_t rasNetSendNack(struct rasSocket* sock) {
INFO(NCCL_RAS, "RAS sending NACK to %s", ncclSocketToString(&sock->sock.addr, rasLine));
memset(&msg, '\0', sizeof(msg));
msg.type = RAS_MSG_CONNINITACK;
msg.connInitAck.nack = 1;
offset = 0;
@@ -557,16 +593,16 @@ static void* rasThreadMain(void*) {
INFO(NCCL_RAS, "RAS thread started");
// Initialize the global pollfd with the file descriptors we already have (the pipe and the listening socket).
NCCLCHECKGOTO(rasGetNewPollEntry(&pfd), ret, fail);
NCCLCHECKGOTO(rasGetNewPollEntry(&pfd), ret, exit);
rasPfds[pfd].fd = rasNotificationPipe[0];
rasPfds[pfd].events = POLLIN;
NCCLCHECKGOTO(rasGetNewPollEntry(&pfd), ret, fail);
NCCLCHECKGOTO(ncclSocketGetFd(&rasNetListeningSocket, &rasNetListeningSocketFd), ret, fail);
NCCLCHECKGOTO(rasGetNewPollEntry(&pfd), ret, exit);
NCCLCHECKGOTO(ncclSocketGetFd(&rasNetListeningSocket, &rasNetListeningSocketFd), ret, exit);
rasPfds[pfd].fd = rasNetListeningSocketFd;
rasPfds[pfd].events = POLLIN;
NCCLCHECKGOTO(rasGetNewPollEntry(&pfd), ret, fail);
NCCLCHECKGOTO(rasGetNewPollEntry(&pfd), ret, exit);
rasPfds[pfd].fd = rasClientListeningSocket;
rasPfds[pfd].events = POLLIN;
@@ -595,32 +631,37 @@ static void* rasThreadMain(void*) {
if (rasPfds[pollIdx].revents) {
nEvents--;
if (rasPfds[pollIdx].fd == rasNotificationPipe[0]) {
(void)rasLocalHandle();
bool terminate = false;
NCCLCHECKGOTO(rasLocalHandle(&terminate), ret, exit);
if (terminate)
goto exit;
} else if (rasPfds[pollIdx].fd == rasNetListeningSocketFd) {
(void)rasNetAcceptNewSocket();
} else if (rasPfds[pollIdx].fd == rasClientListeningSocket) {
(void)rasClientAcceptNewSocket();
} else {
// Check if it's one of the RAS sockets.
int sockIdx;
for (sockIdx = 0; sockIdx < nRasSockets; sockIdx++) {
struct rasSocket* sock = rasSockets+sockIdx;
if (sock->status != RAS_SOCK_CLOSED && rasPfds[pollIdx].fd == sock->sock.fd) {
rasSockEventLoop(sockIdx, pollIdx);
struct rasSocket* sock;
for (sock = rasSocketsHead; sock;) {
struct rasSocket* sockNext = sock->next;
if (rasPfds[pollIdx].fd == sock->sock.fd) {
rasSockEventLoop(sock, pollIdx);
break;
}
} // for (sockIdx)
sock = sockNext;
} // for (sock)
if (sockIdx == nRasSockets) {
if (sock == nullptr) {
// Try a client socket instead.
for (int clientIdx = 0; clientIdx < nRasClients; clientIdx++) {
struct rasClient* client = rasClients+clientIdx;
if (client->status != RAS_CLIENT_CLOSED && rasPfds[pollIdx].fd == client->sock) {
rasClientEventLoop(clientIdx, pollIdx);
for (struct rasClient* client = rasClientsHead; client;) {
struct rasClient* clientNext = client->next;
if (rasPfds[pollIdx].fd == client->sock) {
rasClientEventLoop(client, pollIdx);
break;
}
} // for (clientIdx)
} // if (sockIdx == nRasSockets)
client = clientNext;
} // for (client)
} // if (sock == nullptr)
} // dynamic fds
} // if (revents)
} // for (pollIdx)
@@ -636,14 +677,9 @@ static void* rasThreadMain(void*) {
rasCollsHandleTimeouts(now, &nextWakeup);
} // for (;;)
fail:
WARN("fatal error - RAS thread terminating");
std::lock_guard<std::mutex> lock(rasInitMutex);
(void)close(rasNotificationPipe[1]);
(void)close(rasNotificationPipe[0]);
(void)close(rasClientListeningSocket);
(void)ncclSocketClose(&rasNetListeningSocket);
rasInitialized = false;
exit:
rasThreadCleanup();
INFO(NCCL_RAS, "RAS thread terminating");
return nullptr;
}
src/ras/ras_internal.h
+92 -47
Просмотреть файл
@@ -42,6 +42,14 @@ typedef enum {
RAS_COLL_COMMS = 1002, // Collect data about all communicators.
} rasCollectiveType;
// Unique communicator identifier. commHash by itself is definitely not guaranteed to be unique.
// Combined with the two other hashes, the chance is much better...
// All three fields are used for sorting.
struct rasCommId {
uint64_t commHash;
uint64_t hostHash, pidHash; // These are the hashes of the *first* rank (comm->peerInfo[0]).
};
// Payload of a collective request message (RAS_MSG_COLLREQ).
struct rasCollRequest {
union ncclSocketAddress rootAddr;
@@ -56,6 +64,10 @@ struct rasCollRequest {
struct {
} conns;
struct {
int nSkipMissingRanksComms; // Number of elements in the array below.
// Communicators for which we do *not* need the missingRanks data in the responses
// (see struct rasCollCommsMissingRank later).
struct rasCommId skipMissingRanksComms[0]; // Variable length, sorted.
} comms;
};
};
@@ -69,8 +81,8 @@ struct rasCollResponse {
int nPeers;
int nData; // Size of data in bytes.
union ncclSocketAddress peers[0]; // Variable length.
// The peersAddrs array is followed by:
//alignas(int64_t) char data[0]; // Variable length, collective-dependent.
// The peers array is followed by:
// alignas(int64_t) char data[0]; // Variable length, collective-dependent.
};
// Describes a peer NCCL process. Every RAS thread keeps an (identical) array of them, one entry for each
@@ -80,6 +92,8 @@ struct rasPeerInfo {
pid_t pid;
uint64_t cudaDevs; // Bitmask. This is for local devices so 64 bits is enough.
uint64_t nvmlDevs; // Same, but not affected by CUDA_VISIBLE_DEVICES.
uint64_t hostHash, pidHash; // Taken from ncclComm, but with the commHash subtracted to make it
// communicator-independent.
};
// Describes a RAS message. Every message is preceded by a (32-bit) message length. All data in the host
@@ -112,7 +126,7 @@ struct rasMsg {
int nPeers;
int nDeadPeers;
struct rasPeerInfo peers[0]; // Variable length.
// The peers array is followed by the following:
// The peers array is followed by:
//union ncclSocketAddress deadPeers[0]; // Variable length.
} peersUpdate;
struct {
@@ -218,6 +232,9 @@ struct rasMsgMeta {
// Describes an ongoing collective RAS operation (apart from broadcasts, which don't need a response).
// For every collective operation, each participating RAS thread will create its own.
struct rasCollective {
struct rasCollective* next;
struct rasCollective* prev;
union ncclSocketAddress rootAddr;
uint64_t rootId;
@@ -227,15 +244,16 @@ struct rasCollective {
bool timeoutWarned;
int64_t startTime; // For timeout calculations.
int fromConnIdx; // The connection we received the request from.
struct rasConnection* fromConn; // The connection we received the request from.
int* fwdConns; // Indices of the connections we forwarded the request to; replaced by -1 as the responses arrive.
struct rasConnection** fwdConns; // Connections we forwarded the request to; replaced by nullptr's as the
// responses arrive.
int nFwdSent; // Count of the above (local process only).
int nFwdRecv; // Count of the responses received or timeouts (local process only).
int nLegTimeouts; // Collective (from this process and the responses we received).
union ncclSocketAddress* peers; // Collective (from this process and the responses we received).
union ncclSocketAddress* peers; // Collective (from this process and the responses we received). Unsorted.
int nPeers;
char* data; // Collective (from this process and the responses we received).
@@ -261,13 +279,14 @@ struct rasCollConns {
struct rasCollComms {
int nComms;
struct comm {
uint64_t commHash;
int commNRanks;
int nRanks; // number of elements in the array below, *not* in the communicator.
struct rasCommId commId;
int commNRanks; // >= nRanks + nMissingRanks
int nRanks; // Number of elements in the ranks array below, *not* in the communicator.
int nMissingRanks; // Number of elements in the missingRanks array below.
struct rank {
int commRank;
int peerIdx; // Index within rasCollective->peers, *not* rasPeers.
uint64_t collOpCount;
uint64_t collOpCounts[NCCL_NUM_FUNCTIONS];
struct {
ncclResult_t initState:4;
ncclResult_t asyncError:4;
@@ -278,34 +297,47 @@ struct rasCollComms {
char cudaDev;
char nvmlDev;
} ranks[0]; // Variable length. Sorted by commRank. Optimized for 1 GPU/process.
} comms[0]; // Variable length. Sorted by commHash.
// The ranks array is followed by:
// struct rasCollCommsMissingRank missingRanks[0]; // Variable length. Sorted by commRank.
} comms[0]; // Variable length. Sorted by commId.
};
// Provides info about missing ranks. An array of these structures can be part of struct rasCollComms above.
// Because the arrays are of variable length, we can't describe them in C. To ensure that adding
// rasCollCommsMissingRank structures doesn't mess up the alignment, we explicitly request one.
struct alignas(struct rasCollComms) rasCollCommsMissingRank {
int commRank;
union ncclSocketAddress addr;
// We don't need pid here as we can look it up in rasPeers via addr.
char cudaDev;
char nvmlDev;
};
// Holds data needed to keep track of a connection belonging to a RAS network link (either the primary one
// or one of the fallbacks).
struct rasLinkConn {
struct rasLinkConn* next;
int peerIdx; // Index in the rasPeers array of the peer this entry describes. Could be -1 (an entry initiated
// by an as of yet unknown peer -- should be a temporary situation that resolves via peer updates).
int connIdx; // Index in the rasConns array of the connection to the above peer. Could be -1 (a placeholder
// for a connection to be started by the remote peer).
struct rasConnection* conn; // The connection to the above peer. Could be nullptr (a placeholder for a connection
// to be started by the remote peer).
bool external; // true if the entry exists only due to an external request (requested by a remote peer, most
// likely as part of fault recovery). Such connections are kept as fallbacks even if there's a
// valid primary connection, in order to ensure that keep-alive messages are sent.
};
// Describes a link that forms the backbone of the RAS network. Links focus on direction (previous/next in
// case of 1-D topology) rather than a particular destination. The are implemented using rasConnections, but
// case of 1-D topology) rather than a particular destination. They are implemented using rasConnections, but
// they are persistent through the life of the RAS threads, whereas rasConnections can be terminated if the RAS
// network is reconfigured or a peer dies.
struct rasLink {
int direction; // 1 for nextLink, -1 for prevLink.
// Index 0 is the primary connection; any additional ones are fallbacks (that get created if we are having
// problems with the primary connection). The elements are de-facto ordered (highest-preference ones have
// the lowest indices).
// First element is the primary connection; any additional ones are fallbacks (that get created if we are having
// problems with the primary connection). The highest-preference elements come first; the list is de-facto sorted
// by peerIdx, though peerIdx values can wrap around (given the ring/torus topology) and they can also be -1
// (the latter are stored at the end).
struct rasLinkConn* conns;
int nConns;
int connsSize; // Array size; could be larger than nConns.
// Keep track of a timeout in case we did not create a connection during the last peers update (because we expect
// the peer on the other side to do so) but that peer failed to initiate.
@@ -315,15 +347,15 @@ struct rasLink {
// Describes a connection to another peer on the RAS network. It is meant to be more persistent than a volatile
// socket (described by the rasSocket structure), which can be affected by transient network issues.
struct rasConnection {
bool inUse;
struct rasConnection* next;
struct rasConnection* prev;
union ncclSocketAddress addr;
// Index of the current rasSocket in the rasSockets array. Note that multiple rasSocket entries may point back
// Pointer to the current rasSocket. Note that multiple rasSocket entries may point back
// to a single entry here, for sockets that are in the process of being terminated and re-established.
// We use indices, not pointers, because the arrays holding these structures can be re-alloced at run time.
// -1 if there is no such socket.
int sockIdx;
// nullptr if there is no such socket.
struct rasSocket* sock;
// We keep the rasPeersHash of remote connections to minimize the number of needless exchanges.
// There is a subtle difference in the meaning of lastSentPeersHash and lastRecvPeersHash.
@@ -371,16 +403,18 @@ typedef enum {
// Describes a socket implementing communication between two peers.
struct rasSocket {
struct rasSocket* next;
struct rasSocket* prev;
struct ncclSocket sock;
rasSocketStatus status;
int pfd; // Index in the rasPfds array.
// Index of the corresponding entry in the rasConns array.
// We use indices, not pointers, because the arrays holding these structures can be re-alloced at run time.
// -1 if there is no connection (normal condition on the accept side before the connInit message).
int connIdx;
// Pointer to the corresponding entry in the rasConns array.
// nullptr if there is no connection (a normal condition on the accept side before the connInit message).
struct rasConnection* conn;
int64_t createTime;
int64_t lastSendTime;
@@ -404,7 +438,10 @@ typedef enum {
// Describes a RAS client.
struct rasClient {
int sock;
struct rasClient* next;
struct rasClient* prev;
int sock; // File descriptor
rasClientStatus status;
@@ -420,7 +457,7 @@ struct rasClient {
int64_t timeout;
// State stored during asynchronous operations such as collectives.
int collIdx; // Index to the onging rasCollective.
struct rasCollective* coll;
};
@@ -440,31 +477,33 @@ void rasConnEnqueueMsg(struct rasConnection* conn, struct rasMsg* msg, size_t ms
ncclResult_t rasConnSendMsg(struct rasConnection* conn, int* closed, bool* allSent);
ncclResult_t rasMsgRecv(struct rasSocket* sock, struct rasMsg** msg, int* closed);
ncclResult_t rasMsgHandle(struct rasMsg* msg, struct rasSocket* sock);
void rasMsgHandleBCDeadPeer(const struct rasCollRequest* req, bool* pDone);
void rasMsgHandleBCDeadPeer(struct rasCollRequest** pReq, size_t* pReqLen, bool* pDone);
ncclResult_t rasGetNewPollEntry(int* index);
// rasnet.cc
extern struct rasLink rasNextLink, rasPrevLink;
extern struct rasConnection* rasConns;
extern int nRasConns;
extern struct rasSocket *rasSockets;
extern int nRasSockets;
extern struct rasConnection* rasConnsHead;
extern struct rasConnection* rasConnsTail;
extern struct rasSocket *rasSocketsHead;
extern struct rasSocket *rasSocketsTail;
ncclResult_t getNewConnEntry(struct rasConnection** pConn);
ncclResult_t rasConnCreate(const union ncclSocketAddress* addr, int* pConnIdx);
int rasConnFind(const union ncclSocketAddress* addr);
ncclResult_t rasConnCreate(const union ncclSocketAddress* addr, struct rasConnection** pConn);
struct rasConnection* rasConnFind(const union ncclSocketAddress* addr);
void rasConnsHandleTimeouts(int64_t now, int64_t* nextWakeup);
void rasConnDisconnect(const union ncclSocketAddress* addr);
ncclResult_t rasNetAcceptNewSocket();
void rasSocksHandleTimeouts(int64_t now, int64_t* nextWakeup);
void rasSocketTerminate(struct rasSocket* sock, bool finalize = false, uint64_t startRetryOffset = 0,
bool retry = true);
void rasSockEventLoop(int sockIdx, int pollIdx);
void rasSockEventLoop(struct rasSocket* sock, int pollIdx);
void rasNetHandleTimeouts(int64_t now, int64_t* nextWakeup);
ncclResult_t rasMsgHandleKeepAlive(const struct rasMsg* msg, struct rasSocket* sock);
ncclResult_t rasLinkUpdateConn(struct rasLink* link, int connIdx, int peerIdx, bool external = false,
bool insert = false, bool pretend = false, int* pLinkIdx = nullptr);
ncclResult_t rasLinkAddFallback(struct rasLink* link, const struct rasConnection* conn);
ncclResult_t rasLinkConnUpdate(struct rasLink* link, struct rasConnection* conn, int peerIdx);
void rasNetTerminate();
// peers.cc
extern struct rasPeerInfo* rasPeers;
@@ -483,29 +522,35 @@ ncclResult_t rasPeerDeclareDead(const union ncclSocketAddress* addr);
bool rasPeerIsDead(const union ncclSocketAddress* addr);
int ncclSocketsCompare(const void* p1, const void* p2);
bool ncclSocketsSameNode(const union ncclSocketAddress* a1, const union ncclSocketAddress* a2);
void rasPeersTerminate();
// collectives.cc
extern struct rasCollective* rasCollectives;
extern struct rasCollective* rasCollectivesHead;
extern struct rasCollective* rasCollectivesTail;
void rasCollReqInit(struct rasCollRequest* req);
ncclResult_t rasNetSendCollReq(const struct rasCollRequest* req, size_t reqLen, bool* pAllDone = nullptr,
int* pCollIdx = nullptr, int fromConnIdx = -1);
ncclResult_t rasNetSendCollReq(const struct rasCollRequest* req, bool* pAllDone = nullptr,
struct rasCollective** pColl = nullptr, struct rasConnection* fromConn = nullptr);
ncclResult_t rasMsgHandleCollReq(struct rasMsg* msg, struct rasSocket* sock);
ncclResult_t rasMsgHandleCollResp(struct rasMsg* msg, struct rasSocket* sock);
void rasCollsPurgeConn(int connIdx);
void rasCollsPurgeConn(struct rasConnection* conn);
void rasCollFree(struct rasCollective* coll);
void rasCollsHandleTimeouts(int64_t now, int64_t* nextWakeup);
void rasCollectivesTerminate();
// client_support.cc
extern int rasClientListeningSocket;
extern struct rasClient* rasClients;
extern int nRasClients;
extern struct rasClient* rasClientsHead;
extern struct rasClient* rasClientsTail;
ncclResult_t rasClientInitSocket();
ncclResult_t rasClientAcceptNewSocket();
ncclResult_t rasClientResume(struct rasCollective* coll);
void rasClientEventLoop(int clientIdx, int pollIdx);
void rasClientEventLoop(struct rasClient* client, int pollIdx);
const char* rasGpuDevsToString(uint64_t cudaDevs, uint64_t nvmlDevs, char* buf, size_t size);
void rasClientSupportTerminate();
#endif // !NCCL_RAS_CLIENT
src/ras/rasnet.cc
+652 -506
Просмотреть файл
Разница между файлами не показана из-за своего большого размера Загрузить разницу
src/register/register.cc
+2 -2
Просмотреть файл
@@ -98,8 +98,8 @@ static ncclResult_t regCleanup(struct ncclComm* comm, struct ncclReg* reg) {
}
}
if (reg->state & NVLS_REG_COMPLETE) {
if (ncclNvlsDeregBuffer(comm, &reg->mcHandle, reg->regAddr, reg->dev, reg->regSize) != ncclSuccess) {
WARN("rank %d deregister NVLS buffer %p dev %d size %ld failed", comm->rank, (void*)reg->regAddr, reg->dev, reg->regSize);
if (ncclNvlsDeregBuffer(comm, &reg->mcHandle, reg->regAddr, reg->dev, reg->regUCSize, reg->regMCSize) != ncclSuccess) {
WARN("rank %d deregister NVLS buffer %p dev %d ucsize %ld mcsize %ld failed", comm->rank, (void*)reg->regAddr, reg->dev, reg->regUCSize, reg->regMCSize);
}
reg->regAddr = (CUdeviceptr)NULL;
}
src/transport.cc
+12 -8
Просмотреть файл
@@ -12,11 +12,12 @@
#include "timer.h"
#include "transport.h"
struct ncclTransport* ncclTransports[NTRANSPORTS] = {
struct ncclTransport* ncclTransports[NTRANSPORTS+1] = {
&p2pTransport,
&shmTransport,
&netTransport,
&collNetTransport
&collNetTransport,
&profilerTransport // Not really used for transport, only to create proxy ops polling on profiler counters.
};
template <int type>
@@ -129,6 +130,7 @@ ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph*
gettimeofday(&timeStart, NULL);
timeLast = timeStart; // struct copy
bool timeReported = false;
cudaStream_t hostStream, deviceStream;
int count = 0;
int num = MAXCHANNELS/64;
@@ -137,7 +139,8 @@ ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph*
NCCLCHECKGOTO(ncclCalloc(&recvData, maxPeers), ret, fail);
NCCLCHECKGOTO(ncclCalloc(&sendData, maxPeers), ret, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquireUncaptured(&comm->sharedRes->hostStream), ret, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->hostStream, /*concurrent=*/false, &hostStream), ret, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), ret, fail);
// First time initialization
for (int i=1; i<comm->nRanks; i++) {
int bootstrapTag = (i<<8) + (graph ? graph->id+1 : 0);
@@ -234,7 +237,7 @@ ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph*
if (ret == ncclSuccess) {
conn->connected = 1;
/* comm->channels[c].devPeers[sendPeer]->send[connIndex] is a device memory access. */
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[sendPeer]->send[connIndex], &conn->conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), ret, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[sendPeer]->send[connIndex], &conn->conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), ret, fail);
} else if (ret == ncclInProgress) {
allChannelsConnected = false;
}
@@ -254,7 +257,7 @@ ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph*
if (ret == ncclSuccess) {
conn->connected = 1;
/* comm->channels[c].devPeers[recvPeer]->recv[connIndex] is a device memory access. */
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[recvPeer]->recv[connIndex], &conn->conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), ret, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[recvPeer]->recv[connIndex], &conn->conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), ret, fail);
} else if (ret == ncclInProgress) {
allChannelsConnected = false;
}
@@ -282,7 +285,7 @@ ncclResult_t ncclTransportP2pSetup(struct ncclComm* comm, struct ncclTopoGraph*
}
}
CUDACHECKGOTO(cudaStreamSynchronize(comm->sharedRes->hostStream.cudaStream), ret, fail);
CUDACHECKGOTO(cudaStreamSynchronize(hostStream), ret, fail);
{
struct timeval now;
@@ -331,8 +334,9 @@ exit:
if (sendData) free(sendData);
if (recvData) free(recvData);
NCCLCHECK(ncclStrongStreamWaitStream(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, &comm->sharedRes->hostStream));
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->hostStream));
NCCLCHECK(ncclStreamWaitStream(deviceStream, hostStream, comm->sharedRes->scratchEvent));
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->hostStream, /*concurrent=*/false));
NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false));
return ret;
fail:
goto exit;
src/transport/coll_net.cc
+72 -34
Просмотреть файл
@@ -104,7 +104,7 @@ struct sendResources {
int rank;
int nranks;
int netDev;
int useGdr;
enum ncclTopoGdrMode useGdr;
int useDmaBuf;
uint64_t* gdcSync;
void* gdrDesc;
@@ -126,7 +126,7 @@ struct recvResources {
int rank;
int nranks;
int netDev;
int useGdr;
enum ncclTopoGdrMode useGdr;
int useDmaBuf;
int needFlush;
uint64_t* gdcSync;
@@ -146,9 +146,19 @@ static ncclResult_t canConnect(int* ret, struct ncclComm* comm, struct ncclTopoG
return ncclSuccess;
}
// Returns the flags to be used by a call to cuMemGetHandleForAddressRange.
static inline int getHandleForAddressRangeFlags(ncclTopoGdrMode useGdr) {
int flags = 0;
#if CUDA_VERSION >= 12080
// Force mapping on PCIe on systems with both PCI and C2C attachments.
if (useGdr == ncclTopoGdrModePci) flags = CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE;
#endif
return flags;
}
struct setupReq {
int netDev;
int useGdr;
enum ncclTopoGdrMode useGdr;
int needFlush;
struct ncclCollNetSharedRes* collNet;
};
@@ -171,8 +181,8 @@ static ncclResult_t sendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
req.collNet = comm->collNetSharedRes;
NCCLCHECK(ncclProxyCallBlocking(comm, &send->proxyConn, ncclProxyMsgSetup, &req, sizeof(req), NULL, 0));
INFO(NCCL_INIT|NCCL_NET,"CollNet %02d/%1d : %d [send] via COLLNET/%s/%d%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, collNetName(comm), req.netDev,
req.useGdr ? "/GDRDMA" : "", comm, comm->nRanks);
INFO(NCCL_INIT|NCCL_NET,"CollNet %02d/%1d : %d [send] via COLLNET/%s/%d%s%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, collNetName(comm), req.netDev,
req.useGdr ? "/GDRDMA" : "", req.useGdr==ncclTopoGdrModePci ? "(PCI)" : "", comm, comm->nRanks);
return ncclSuccess;
}
@@ -185,7 +195,7 @@ static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
NCCLCHECK(ncclTopoCheckGdr(comm->topo, myInfo->rank, netId, 0, &req.useGdr));
recv->conn.flags |= req.useGdr ? NCCL_DIRECT_NIC : 0;
// Determine whether we need to flush the GDR buffer on recv or not
if (req.useGdr) NCCLCHECK(ncclTopoNeedFlush(comm, req.netDev, myInfo->rank, &req.needFlush));
if (req.useGdr) NCCLCHECK(ncclTopoNeedFlush(comm, netId, req.netDev, myInfo->rank, &req.needFlush));
recv->proxyConn.tpLocalRank = comm->topParentLocalRanks[comm->localRank];
NCCLCHECK(ncclProxyConnect(comm, TRANSPORT_COLLNET, 0, myInfo->rank, &recv->proxyConn));
@@ -195,8 +205,8 @@ static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
req.collNet = comm->collNetSharedRes;
NCCLCHECK(ncclProxyCallBlocking(comm, &recv->proxyConn, ncclProxyMsgSetup, &req, sizeof(req), &info->collNetHandle, sizeof(collNetHandle_t)));
INFO(NCCL_INIT|NCCL_NET,"CollNet %02d/%1d : %d [receive] via COLLNET/%s/%d%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, collNetName(comm), req.netDev,
req.useGdr ? "/GDRDMA" : "", comm, comm->nRanks);
INFO(NCCL_INIT|NCCL_NET,"CollNet %02d/%1d : %d [receive] via COLLNET/%s/%d%s%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, collNetName(comm), req.netDev,
req.useGdr ? "/GDRDMA" : "", req.useGdr==ncclTopoGdrModePci ? "(PCI)" : "", comm, comm->nRanks);
return ncclSuccess;
}
@@ -459,6 +469,7 @@ static ncclResult_t recvProxySetup(struct ncclProxyConnection* connection, struc
}
static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
ncclResult_t ret = ncclSuccess;
if (reqSize != sizeof(struct collNetConnectArgs)) { WARN("sendProxyConnect: reqSize is %d != %ld", reqSize, sizeof(struct collNetConnectArgs)); return ncclInternalError; }
struct collNetConnectArgs* args = (struct collNetConnectArgs*)reqBuff;
static_assert(sizeof(collNetSendConnectInfo) <= sizeof(struct ncclConnect), "Collnet Send Connect info is too big");
@@ -510,16 +521,17 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
int bank = resources->useGdr ? NCCL_NET_MAP_SHARED_DEVMEM : NCCL_NET_MAP_SHARED_HOSTMEM;
struct connectMapMem* mapMem = map->mems+bank;
NCCLCHECK(sharedBuffersInit(connection->collNet, resources->useGdr, &mapMem->gpuPtr, &mapMem->cpuPtr, &mapMem->size));
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr ? 1 : 0, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
int dmabuf_fd = -1;
#if CUDA_VERSION >= 11070
/* DMA-BUF support */
if (resources->useGdr && resources->useDmaBuf) {
int dmabuf_fd;
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)mapMem->cpuPtr, mapMem->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0));
NCCLCHECK(proxyState->ncclCollNet->regMrDmaBuf(resources->collNetComm, mapMem->cpuPtr, mapMem->size,
NCCL_PTR_CUDA, 0ULL, dmabuf_fd,
&resources->sendMhandles[NCCL_PROTO_SIMPLE]));
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)mapMem->cpuPtr, mapMem->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)));
NCCLCHECKGOTO(proxyState->ncclCollNet->regMrDmaBuf(resources->collNetComm, mapMem->cpuPtr, mapMem->size,
NCCL_PTR_CUDA, 0ULL, dmabuf_fd,
&resources->sendMhandles[NCCL_PROTO_SIMPLE]),
ret, fail);
(void)close(dmabuf_fd);
} else // FALL-THROUGH to nv_peermem GDR path
#endif
@@ -530,10 +542,18 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
}
*((struct connectMap**)respBuff) = &resources->map;
return ncclSuccess;
exit:
return ret;
fail:
if (dmabuf_fd != -1) {
(void)close(dmabuf_fd);
}
goto exit;
}
static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
ncclResult_t ret = ncclSuccess;
if (reqSize != sizeof(struct collNetConnectArgs)) { WARN("recvProxyConnect: reqSize is %d != %ld", reqSize, sizeof(struct collNetConnectArgs)); return ncclInternalError; }
struct collNetConnectArgs* args = (struct collNetConnectArgs*)reqBuff;
@@ -579,16 +599,17 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
int bank = resources->useGdr ? NCCL_NET_MAP_SHARED_DEVMEM : NCCL_NET_MAP_SHARED_HOSTMEM;
struct connectMapMem* mapMem = map->mems+bank;
NCCLCHECK(sharedBuffersInit(connection->collNet, resources->useGdr, &mapMem->gpuPtr, &mapMem->cpuPtr, &mapMem->size));
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr ? 1 : 0, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
int dmabuf_fd = -1;
#if CUDA_VERSION >= 11070
/* DMA-BUF support */
if (resources->useGdr && resources->useDmaBuf) {
int dmabuf_fd;
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)mapMem->cpuPtr, mapMem->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0));
NCCLCHECK(proxyState->ncclCollNet->regMrDmaBuf(resources->collNetComm, mapMem->cpuPtr, mapMem->size,
NCCL_PTR_CUDA, 0ULL, dmabuf_fd,
&resources->mhandles[NCCL_PROTO_SIMPLE]));
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)mapMem->cpuPtr, mapMem->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)));
NCCLCHECKGOTO(proxyState->ncclCollNet->regMrDmaBuf(resources->collNetComm, mapMem->cpuPtr, mapMem->size,
NCCL_PTR_CUDA, 0ULL, dmabuf_fd,
&resources->mhandles[NCCL_PROTO_SIMPLE]),
ret, fail);
(void)close(dmabuf_fd);
} else // FALL-THROUGH to nv_peermem GDR path
#endif
@@ -605,7 +626,14 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
if (respSize != sizeof(struct connectMap*)) { WARN("recvProxyConnect: respSize is %d != %ld", respSize, sizeof(void*)); return ncclInternalError; }
*((struct connectMap**)respBuff) = &resources->map;
return ncclSuccess;
exit:
return ret;
fail:
if (dmabuf_fd != -1) {
(void)close(dmabuf_fd);
}
goto exit;
}
static ncclResult_t sendProxyFree(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState) {
@@ -742,7 +770,7 @@ static ncclResult_t collNetIallreduce(struct ncclProxyState* proxyState, struct
}
static ncclResult_t collNetRegIallgather(struct ncclProxyState* proxyState, struct sendResources *resources, struct ncclProxyArgs *args, struct ncclProxySubArgs *sub, ssize_t nBytesIn, ssize_t allBeg, ssize_t recvBeg, void *recvMhandle, void **request) {
ncclNetSGE_v9_t recvParts;
ncclNetSGE_t recvParts;
ssize_t sizePerRank = args->specifics.collnetDirect.sizePerRank;
char *region = NCCL_NET_MAP_GET_POINTER(&resources->map, gpu, buffs[NCCL_PROTO_SIMPLE]);
ssize_t nBytes;
@@ -784,7 +812,7 @@ static ncclResult_t collNetRegIallgather(struct ncclProxyState* proxyState, stru
}
static ncclResult_t collNetIallgather(struct ncclProxyState* proxyState, struct sendResources *resources, struct ncclProxyArgs *args, struct ncclProxySubArgs *sub, ssize_t nBytes, ssize_t allBeg, ssize_t sendBeg, ssize_t recvBeg, void *sendMhandle, void *recvMhandle, void **request) {
ncclNetSGE_v9_t recvParts;
ncclNetSGE_t recvParts;
ssize_t sizePerRank = args->specifics.collnetDirect.sizePerRank;
char *region = NCCL_NET_MAP_GET_POINTER(&resources->map, gpu, buffs[NCCL_PROTO_SIMPLE]);
recvParts.mhandle = recvMhandle;
@@ -801,7 +829,7 @@ static ncclResult_t collNetIallgather(struct ncclProxyState* proxyState, struct
}
static ncclResult_t collNetRegIreducescatter(struct ncclProxyState* proxyState, struct sendResources *resources, struct ncclProxyArgs *args, struct ncclProxySubArgs *sub, ssize_t nBytesIn, ssize_t allBeg, ssize_t sendBeg, void *sendMhandle, void **request) {
ncclNetSGE_v9_t sendParts;
ncclNetSGE_t sendParts;
ssize_t sizePerRank = args->specifics.collnetDirect.sizePerRank;
char *region = NCCL_NET_MAP_GET_POINTER(&resources->map, gpu, buffs[NCCL_PROTO_SIMPLE]);
ssize_t nBytes;
@@ -840,7 +868,7 @@ static ncclResult_t collNetRegIreducescatter(struct ncclProxyState* proxyState,
}
static ncclResult_t collNetIreducescatter(struct ncclProxyState* proxyState, struct sendResources *resources, struct ncclProxyArgs *args, struct ncclProxySubArgs *sub, ssize_t nBytes, ssize_t allBeg, ssize_t sendBeg, ssize_t recvBeg, void *sendMhandle, void *recvMhandle, void **request) {
ncclNetSGE_v9_t sendParts;
ncclNetSGE_t sendParts;
ssize_t sizePerRank = args->specifics.collnetDirect.sizePerRank;
char *region = NCCL_NET_MAP_GET_POINTER(&resources->map, gpu, buffs[NCCL_PROTO_SIMPLE]);
sendParts.mhandle = sendMhandle;
@@ -1156,6 +1184,7 @@ struct collnetRegInfo {
static ncclResult_t collnetRegisterBuffer(struct ncclComm* comm, const void* userbuff, size_t buffSize, int type, struct ncclReg* regRecord, int* outRegBufFlag, void** outHandle) {
ncclResult_t ret = ncclSuccess;
int gdrEnable = -1;
if (regRecord) {
if (regRecord->state & COLLNET_REG_COMPLETE) {
// reuse previous registration
@@ -1171,6 +1200,7 @@ static ncclResult_t collnetRegisterBuffer(struct ncclComm* comm, const void* use
if (conn->flags & NCCL_DIRECT_NIC) {
struct ncclProxyConnector* proxyconn = (type == collNetRecv) ? &comm->channels[0].peers[comm->nRanks]->recv[type].proxyConn : &comm->channels[0].peers[comm->nRanks]->send[type].proxyConn;
gdrEnable = 1;
NCCLCHECKGOTO(ncclProxyCallBlocking(comm, proxyconn, ncclProxyMsgRegister, &info, sizeof(struct collnetRegInfo), &handle, sizeof(void*)), ret, fail);
if (handle) {
regRecord->state |= COLLNET_REG_COMPLETE;
@@ -1180,7 +1210,8 @@ static ncclResult_t collnetRegisterBuffer(struct ncclComm* comm, const void* use
INFO(NCCL_REG, "rank %d - COLLNET register userbuff %p (handle %p), buffSize %ld, type %s", comm->rank, userbuff, handle, buffSize, type == collNetRecv ? "Recv" : "Send");
}
} else {
WARN("rank %d - COLLNET failed to register userbuff %p (handle %p), buffSize %ld, type %s, GDR is not enabled", comm->rank, userbuff, handle, buffSize, type == collNetRecv ? "Recv" : "Send");
gdrEnable = 0;
goto fail;
}
}
}
@@ -1189,6 +1220,7 @@ exit:
fail:
*outRegBufFlag = 0;
*outHandle = NULL;
INFO(NCCL_REG, "rank %d - COLLNET failed to register userbuff %p, buffSize %ld, type %s, GDR %d", comm->rank, userbuff, buffSize, type == collNetRecv ? "Recv" : "Send", gdrEnable);
goto exit;
}
@@ -1274,17 +1306,20 @@ static ncclResult_t sendProxyRegBuffer(struct ncclProxyConnection* connection, s
assert(reqSize == sizeof(struct collnetRegInfo));
assert(respSize == sizeof(void*));
int dmabuf_fd = -1;
#if CUDART_VERSION >= 11070
/* DMA-BUF support */
if (resources->useGdr && resources->useDmaBuf) {
int dmabuf_fd;
CUCHECKGOTO(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0), ret, peermem);
CUCHECKGOTO(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)), ret, peermem);
NCCLCHECKGOTO(proxyState->ncclCollNet->regMrDmaBuf(resources->collNetComm, (void*)info->buffer, info->size, NCCL_PTR_CUDA, 0ULL, dmabuf_fd, &handle), ret, peermem);
(void)close(dmabuf_fd);
needReg = false;
}
#endif
peermem:
if (dmabuf_fd != -1) {
(void)close(dmabuf_fd);
dmabuf_fd = -1;
}
if (needReg) {
NCCLCHECKGOTO(proxyState->ncclCollNet->regMr(resources->collNetComm, (void*)info->buffer, info->size, NCCL_PTR_CUDA, &handle), ret, fail);
}
@@ -1307,17 +1342,20 @@ static ncclResult_t recvProxyRegBuffer(struct ncclProxyConnection* connection, s
assert(reqSize == sizeof(struct collnetRegInfo));
assert(respSize == sizeof(void*));
int dmabuf_fd = -1;
#if CUDART_VERSION >= 11070
/* DMA-BUF support */
if (resources->useGdr && resources->useDmaBuf) {
int dmabuf_fd;
CUCHECKGOTO(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0), ret, peermem);
CUCHECKGOTO(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)), ret, peermem);
NCCLCHECKGOTO(proxyState->ncclCollNet->regMrDmaBuf(resources->collNetComm, (void*)info->buffer, info->size, NCCL_PTR_CUDA, 0ULL, dmabuf_fd, &handle), ret, peermem);
(void)close(dmabuf_fd);
needReg = false;
}
#endif
peermem:
if (dmabuf_fd != -1) {
(void)close(dmabuf_fd);
dmabuf_fd = -1;
}
if (needReg) {
NCCLCHECKGOTO(proxyState->ncclCollNet->regMr(resources->collNetComm, (void*)info->buffer, info->size, NCCL_PTR_CUDA, &handle), ret, fail);
}
@@ -1606,4 +1644,4 @@ struct ncclTransport collNetTransport = {
canConnect,
{ sendSetup, sendConnect, sendFree, NULL, sendProxySetup, sendProxyConnect, sendProxyFree, sendProxyProgress, sendProxyRegBuffer, sendProxyDeregBuffer },
{ recvSetup, recvConnect, recvFree, NULL, recvProxySetup, recvProxyConnect, recvProxyFree, recvProxyProgress, recvProxyRegBuffer, recvProxyDeregBuffer }
};
};
src/transport/net.cc
+54 -32
Просмотреть файл
@@ -103,7 +103,7 @@ struct sendNetResources {
int tpLocalRank;
int tpRemoteRank;
int netDev;
int useGdr;
enum ncclTopoGdrMode useGdr;
int useDmaBuf;
int maxRecvs;
uint64_t* gdcSync;
@@ -135,7 +135,7 @@ struct recvNetResources {
int tpRemoteRank;
int tpRemoteProxyRank;
int netDev;
int useGdr;
enum ncclTopoGdrMode useGdr;
int useDmaBuf;
int needFlush;
int maxRecvs;
@@ -188,7 +188,7 @@ struct setupReq {
int tpRemoteRank;
int shared;
int netDev;
int useGdr;
enum ncclTopoGdrMode useGdr;
int needFlush;
int channelId;
int connIndex;
@@ -201,6 +201,16 @@ static_assert(sizeof(ncclNetHandle_t) + sizeof(int) <= CONNECT_SIZE, "Not large
// Forward declaration
static ncclResult_t sendProxyProgress(struct ncclProxyState* proxyState, struct ncclProxyArgs* args);
// Returns the flags to be used by a call to cuMemGetHandleForAddressRange.
static inline int getHandleForAddressRangeFlags(ncclTopoGdrMode useGdr) {
int flags = 0;
#if CUDA_VERSION >= 12080
// Force mapping on PCIe on systems with both PCI and C2C attachments.
if (useGdr == ncclTopoGdrModePci) flags = CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE;
#endif
return flags;
}
/* Determine if we will use this transport for this peer and return connect
* information for this peer */
static ncclResult_t sendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* send, int channelId, int connIndex) {
@@ -232,11 +242,14 @@ static ncclResult_t sendSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
NCCLCHECK(ncclProxyCallBlocking(comm, &send->proxyConn, ncclProxyMsgSetup, &req, sizeof(req), NULL, 0));
if (proxyRank == myInfo->rank) {
INFO(NCCL_INIT|NCCL_NET,"Channel %02d/%d : %d[%lx] -> %d[%lx] [send] via NET/%s/%d%s%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId, comm->ncclNet->name, req.netDev,
req.useGdr ? "/GDRDMA" : "", req.shared ? "/Shared" : "", comm, comm->nRanks);
INFO(NCCL_INIT|NCCL_NET,"Channel %02d/%d : %d[%d] -> %d[%d] [send] via NET/%s/%d%s%s%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, myInfo->nvmlDev, peerInfo->rank, peerInfo->nvmlDev, comm->ncclNet->name, req.netDev,
req.useGdr ? "/GDRDMA" : "", req.useGdr==ncclTopoGdrModePci ? "(PCI)" : "",
req.shared ? "/Shared" : "", comm, comm->nRanks);
} else {
INFO(NCCL_INIT|NCCL_NET,"Channel %02d/%d : %d[%lx] -> %d[%lx] [send] via NET/%s/%d(%d)%s%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId, comm->ncclNet->name, req.netDev,
proxyRank, req.useGdr ? "/GDRDMA" : "", req.shared ? "/Shared" : "", comm, comm->nRanks);
INFO(NCCL_INIT|NCCL_NET,"Channel %02d/%d : %d[%d] -> %d[%d] [send] via NET/%s/%d(%d)%s%s%s comm %p nRanks %02d", channelId, connIndex, myInfo->rank, myInfo->nvmlDev, peerInfo->rank, peerInfo->nvmlDev, comm->ncclNet->name, req.netDev,
proxyRank,
req.useGdr ? "/GDRDMA" : "", req.useGdr==ncclTopoGdrModePci ? "(PCI)" : "",
req.shared ? "/Shared" : "", comm, comm->nRanks);
}
*((int*)connectInfo) = comm->topParentRanks[proxyRank];
memcpy((uint8_t*)connectInfo + sizeof(ncclNetHandle_t), &req.useGdr, sizeof(int));
@@ -268,7 +281,7 @@ static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
// Determine whether we need to flush the GDR buffer on recv or not
if (req.useGdr) {
NCCLCHECK(ncclTopoNeedFlush(comm, req.netDev, myInfo->rank, &req.needFlush));
NCCLCHECK(ncclTopoNeedFlush(comm, netId, req.netDev, myInfo->rank, &req.needFlush));
CUDACHECK(hipDeviceGetAttribute((int*)&req.curr_hdp_reg, hipDeviceAttributeHdpMemFlushCntl, myInfo->cudaDev));
recv->conn.curr_hdp_reg = req.curr_hdp_reg;
}
@@ -281,18 +294,19 @@ static ncclResult_t recvSetup(struct ncclComm* comm, struct ncclTopoGraph* graph
req.tpRemoteRank = comm->topParentRanks[peerInfo->rank];
NCCLCHECK(ncclProxyCallBlocking(comm, &recv->proxyConn, ncclProxyMsgSetup, &req, sizeof(req), connectInfo, sizeof(ncclNetHandle_t)));
memcpy((uint8_t*)connectInfo + sizeof(ncclNetHandle_t), &req.useGdr, sizeof(int));
INFO(NCCL_INIT|NCCL_NET,"Channel %02d/%d : %d[%lx] -> %d[%lx] [receive] via NET/%s/%d%s%s comm %p nRanks %02d", channelId, connIndex, peerInfo->rank, peerInfo->busId, myInfo->rank, myInfo->busId, comm->ncclNet->name, req.netDev,
req.useGdr ? "/GDRDMA" : "", req.shared ? "/Shared" : "", comm, comm->nRanks);
INFO(NCCL_INIT|NCCL_NET,"Channel %02d/%d : %d[%d] -> %d[%d] [receive] via NET/%s/%d%s%s%s comm %p nRanks %02d", channelId, connIndex, peerInfo->rank, peerInfo->nvmlDev, myInfo->rank, myInfo->nvmlDev, comm->ncclNet->name, req.netDev,
req.useGdr ? "/GDRDMA" : "", req.useGdr==ncclTopoGdrModePci ? "(PCI)" : "",
req.shared ? "/Shared" : "", comm, comm->nRanks);
return ncclSuccess;
}
static ncclResult_t netMapShm(struct ncclComm *comm, struct connectMapMem* mem) {
NCCLCHECK(ncclShmImportShareableBuffer(comm, &mem->createDesc, (void**)&mem->cpuPtr, (void**)&mem->gpuPtr, &mem->attachDesc));
static ncclResult_t netMapShm(struct ncclComm *comm, struct ncclProxyConnector* proxyConn, struct connectMapMem* mem) {
NCCLCHECK(ncclShmImportShareableBuffer(comm, proxyConn->rank, &mem->createDesc, (void**)&mem->cpuPtr, (void**)&mem->gpuPtr, &mem->attachDesc));
return ncclSuccess;
}
static ncclResult_t netCreateShm(struct ncclProxyState* proxyState, struct connectMapMem* mem) {
NCCLCHECK(ncclShmAllocateShareableBuffer(proxyState->tpRank, mem->size, false, &mem->createDesc, (void**)&mem->cpuPtr, (void**)&mem->gpuPtr));
NCCLCHECK(ncclShmAllocateShareableBuffer(mem->size, false, &mem->createDesc, (void**)&mem->cpuPtr, (void**)&mem->gpuPtr));
return ncclSuccess;
}
@@ -326,6 +340,7 @@ static ncclResult_t netDumpMap(struct connectMap* map) {
struct netSendConnectArgs {
ncclNetHandle_t handle;
int trafficClass;
};
struct netRecvConnectArgs {
@@ -349,6 +364,7 @@ static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* conne
INFO(NCCL_PROXY, "sendConnect ncclProxyCallAsync opId=%p", opId);
netSendConnectArgs args = {0};
memcpy(&args.handle, connectInfo, sizeof(ncclNetHandle_t));
args.trafficClass = comm->config.trafficClass;
NCCLCHECK(ncclProxyCallAsync(comm, &send->proxyConn, ncclProxyMsgConnect, &args, sizeof(netSendConnectArgs), sizeof(struct connectMap), opId));
} else {
opId = send;
@@ -377,7 +393,7 @@ static ncclResult_t sendConnect(struct ncclComm* comm, struct ncclConnect* conne
}
}
} else if (!(map->sameProcess && map->cudaDev == comm->cudaDev)) {
if (!map->sameProcess) NCCLCHECK(netMapShm(comm, map->mems + NCCL_NET_MAP_HOSTMEM));
if (!map->sameProcess) NCCLCHECK(netMapShm(comm, &send->proxyConn, map->mems + NCCL_NET_MAP_HOSTMEM));
if (map->mems[NCCL_NET_MAP_DEVMEM].size) {
map->mems[NCCL_NET_MAP_DEVMEM].gpuPtr = NULL;
NCCLCHECK(ncclP2pImportShareableBuffer(comm, send->proxyConn.rank,
@@ -744,9 +760,11 @@ static ncclResult_t ncclNetGetDeviceHandle(ncclNetDeviceType type, int version,
static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, struct ncclProxyState* proxyState, void* reqBuff, int reqSize, void* respBuff, int respSize, int* done) {
ncclNet_ctxt_t ncclNetCtxt = {};
struct sendNetResources* resources = (struct sendNetResources*)(connection->transportResources);
ncclNetCommConfig_t commConfig = {0};
if (reqSize != sizeof(netSendConnectArgs)) return ncclInternalError;
ncclResult_t ret = ncclSuccess;
netSendConnectArgs* req = (netSendConnectArgs*) reqBuff;
commConfig.trafficClass = req->trafficClass == NCCL_CONFIG_UNDEF_INT ? NCCL_NET_TRAFFIC_CLASS_UNDEF : req->trafficClass;
NCCLCHECK(ncclNetGetDeviceHandle(resources->netDeviceType, resources->netDeviceVersion, false /*isRecv*/, &resources->netDeviceHandle));
bool rccl_anp = !(strcmp(proxyState->ncclNet->name, RCCL_ANP_PLUGIN_STR));
if (resources->shared) {
@@ -770,9 +788,9 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
if (comms->sendComm[resources->channelId] == NULL) {
if (rccl_anp) {
ncclNetCtxt.chId = resources->channelId;
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, comms->sendComm + resources->channelId, (ncclNetDeviceHandle_t **)&ncclNetCtxt);
ret = proxyState->ncclNet->connect(resources->netDev, &commConfig, req->handle, comms->sendComm + resources->channelId, (ncclNetDeviceHandle_t **)&ncclNetCtxt);
} else {
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, comms->sendComm + resources->channelId, &resources->netDeviceHandle);
ret = proxyState->ncclNet->connect(resources->netDev, &commConfig, req->handle, comms->sendComm + resources->channelId, &resources->netDeviceHandle);
}
}
resources->netSendComm = comms->sendComm[resources->channelId];
@@ -780,18 +798,18 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
} else {
if (rccl_anp) {
ncclNetCtxt.chId = resources->channelId;
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, &resources->netSendComm, (ncclNetDeviceHandle_t **)&ncclNetCtxt);
ret = proxyState->ncclNet->connect(resources->netDev, &commConfig, req->handle, &resources->netSendComm, (ncclNetDeviceHandle_t **)&ncclNetCtxt);
} else {
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, &resources->netSendComm, &resources->netDeviceHandle);
ret = proxyState->ncclNet->connect(resources->netDev, &commConfig, req->handle, &resources->netSendComm, &resources->netDeviceHandle);
}
}
} else {
// Connect to remote peer
if (rccl_anp) {
ncclNetCtxt.chId = resources->channelId;
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, &resources->netSendComm, (ncclNetDeviceHandle_t **)&ncclNetCtxt);
ret = proxyState->ncclNet->connect(resources->netDev, &commConfig, req->handle, &resources->netSendComm, (ncclNetDeviceHandle_t **)&ncclNetCtxt);
} else {
ret = proxyState->ncclNet->connect(resources->netDev, req->handle, &resources->netSendComm, &resources->netDeviceHandle);
ret = proxyState->ncclNet->connect(resources->netDev, &commConfig, req->handle, &resources->netSendComm, &resources->netDeviceHandle);
}
connection->proxyAppendPtr = &connection->proxyAppend;
}
@@ -818,7 +836,7 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
if (resources->shared == 0) { // Only allocate dedicated buffers for ring/tree, not for p2p
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
NCCL_NET_MAP_ADD_POINTER(map, 0, p!= NCCL_PROTO_LL && resources->useGdr, proxyState->buffSizes[p], buffs[p]);
NCCL_NET_MAP_ADD_POINTER(map, 0, p!= NCCL_PROTO_LL && resources->useGdr ? 1 : 0, proxyState->buffSizes[p], buffs[p]);
resources->buffSizes[p] = proxyState->buffSizes[p];
}
} else {
@@ -835,7 +853,7 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
resources->buffSizes[NCCL_PROTO_LL] = proxyState->buffSizes[NCCL_PROTO_LL];
}
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr ? 1 : 0, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
}
NCCL_NET_MAP_ADD_POINTER(map, 0, 0, sizeof(struct ncclSendMem), sendMem);
@@ -901,7 +919,7 @@ static ncclResult_t sendProxyConnect(struct ncclProxyConnection* connection, str
int type = NCCL_NET_MAP_DEV_MEM(map, buffs[p]) ? NCCL_PTR_CUDA : NCCL_PTR_HOST;
if (type == NCCL_PTR_CUDA && resources->useDmaBuf) {
int dmabuf_fd;
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)resources->buffers[p], resources->buffSizes[p], CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0));
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)resources->buffers[p], resources->buffSizes[p], CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)));
NCCLCHECK(proxyState->ncclNet->regMrDmaBuf(resources->netSendComm, resources->buffers[p], resources->buffSizes[p], type, 0ULL, dmabuf_fd, &resources->mhandles[p]));
(void)close(dmabuf_fd);
} else // FALL-THROUGH to nv_peermem GDR path
@@ -1016,7 +1034,7 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
if (resources->shared == 0) { // Only allocate dedicated buffers for ring/tree, not for p2p
for (int p=0; p<NCCL_NUM_PROTOCOLS; p++) {
NCCL_NET_MAP_ADD_POINTER(map, 0, resources->useGdr, proxyState->buffSizes[p], buffs[p]);
NCCL_NET_MAP_ADD_POINTER(map, 0, resources->useGdr ? 1 : 0, proxyState->buffSizes[p], buffs[p]);
resources->buffSizes[p] = proxyState->buffSizes[p];
}
} else {
@@ -1027,14 +1045,14 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
proxyState, resources->useGdr, resources->tpLocalRank, 1, 1, proxyState->p2pnChannels,
&mapMem->gpuPtr, &mapMem->cpuPtr, &mapMem->size, NULL));
resources->buffSizes[NCCL_PROTO_SIMPLE] = mapMem->size;
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
NCCL_NET_MAP_ADD_POINTER(map, 1, resources->useGdr ? 1 : 0, mapMem->size, buffs[NCCL_PROTO_SIMPLE]);
}
NCCL_NET_MAP_ADD_POINTER(map, 0, 0, sizeof(struct ncclSendMem), sendMem);
NCCL_NET_MAP_ADD_POINTER(map, 0, 0, sizeof(struct ncclRecvMem), recvMem);
if (proxyState->allocP2pNetLLBuffers) {
NCCL_NET_MAP_ADD_POINTER(map, 0, 0 /*resources->useGdr*/, proxyState->buffSizes[NCCL_PROTO_LL], buffs[NCCL_PROTO_LL]);
NCCL_NET_MAP_ADD_POINTER(map, 0, 0 /*devMem*/, proxyState->buffSizes[NCCL_PROTO_LL], buffs[NCCL_PROTO_LL]);
resources->buffSizes[NCCL_PROTO_LL] = proxyState->buffSizes[NCCL_PROTO_LL];
}
@@ -1087,7 +1105,7 @@ static ncclResult_t recvProxyConnect(struct ncclProxyConnection* connection, str
int type = NCCL_NET_MAP_DEV_MEM(map, buffs[p]) ? NCCL_PTR_CUDA : NCCL_PTR_HOST;
if (type == NCCL_PTR_CUDA && resources->useDmaBuf) {
int dmabuf_fd;
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)resources->buffers[p], resources->buffSizes[p], CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0));
CUCHECK(cuMemGetHandleForAddressRange((void *)&dmabuf_fd, (CUdeviceptr)resources->buffers[p], resources->buffSizes[p], CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)));
NCCLCHECK(proxyState->ncclNet->regMrDmaBuf(resources->netRecvComm, resources->buffers[p], resources->buffSizes[p], type, 0ULL, dmabuf_fd, &resources->mhandles[p]));
(void)close(dmabuf_fd);
} else // FALL-THROUGH to nv_peermem GDR path
@@ -1326,7 +1344,7 @@ static ncclResult_t sendProxyProgress(struct ncclProxyState* proxyState, struct
// Coverity complains about the size here as pointing to an out-of-scope temporary. Which is nonsense,
// since size is a plain integer.
// coverity[use_invalid:FALSE]
NCCLCHECK(proxyState->ncclNet->isend(resources->netSendComm, buff, size, resources->tpRank, sub->sendMhandle, sub->requests+buffSlot));
NCCLCHECK(proxyState->ncclNet->isend(resources->netSendComm, buff, size, resources->tpRank, sub->sendMhandle, sub, sub->requests+buffSlot));
if (sub->requests[buffSlot] != NULL) {
#if defined(ENABLE_NPKIT) && defined(ENABLE_NPKIT_EVENT_NET_SEND_ENTRY) && defined(ENABLE_NPKIT_EVENT_NET_SEND_EXIT)
@@ -1348,6 +1366,7 @@ static ncclResult_t sendProxyProgress(struct ncclProxyState* proxyState, struct
TRACE(NCCL_NET, "sendProxy [%ld/%d/%d] Isend posted, req %p, buff %p, size %d, proto %d, myRank %d, channelId %d, mhandle %p", sub->transmitted, buffSlot, sub->nsteps, sub->requests[buffSlot], buff, size, p, proxyState->tpRank, sub->channelId, sub->sendMhandle);
sub->transSize += size;
sub->transmitted += args->sliceSteps;
sub->profilerSteps++;
ncclProfilerRecordProxyOpEventState(s, args, sub->transmitted, sub->transSize, ncclProfilerProxyOpSendTransmitted);
ncclProfilerRecordProxyStepEventState(s, args, transmittedStepId, ncclProfilerProxyStepSendWait);
args->idle = 0;
@@ -1499,6 +1518,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
size_t sizes[NCCL_PROXY_MAX_SUBS];
int tags[NCCL_PROXY_MAX_SUBS];
void* mhandles[NCCL_PROXY_MAX_SUBS];
void* phandles[NCCL_PROXY_MAX_SUBS];
for (int i=0; i<subGroup->groupSize; i++) {
struct ncclProxySubArgs* sub = subGroup + i;
int postedStepId = sub->posted;
@@ -1542,6 +1562,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
if (sub->nbytes < sizes[subCount]) sizes[subCount] = sub->nbytes;
tags[subCount] = resources->tpRemoteRank;
mhandles[subCount] = sub->recvMhandle;
phandles[subCount] = sub;
subCount++;
}
}
@@ -1551,7 +1572,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
void** requestPtr = subGroup->requests+(step%NCCL_STEPS);
bool ignoreCompletion = ncclParamNetOptionalRecvCompletion() && ((args->protocol == NCCL_PROTO_LL128) || (args->protocol == NCCL_PROTO_LL)) && (subCount == 1);
if (ignoreCompletion) *requestPtr = (void *)NCCL_NET_OPTIONAL_RECV_COMPLETION;
NCCLCHECK(proxyState->ncclNet->irecv(resources->netRecvComm, subCount, ptrs, sizes, tags, mhandles, requestPtr));
NCCLCHECK(proxyState->ncclNet->irecv(resources->netRecvComm, subCount, ptrs, sizes, tags, mhandles, phandles, requestPtr));
if (*requestPtr) {
subGroup->recvRequestsCache[step%NCCL_STEPS] = *requestPtr;
subGroup->recvRequestsSubCount = subCount;
@@ -1576,6 +1597,7 @@ static ncclResult_t recvProxyProgress(struct ncclProxyState* proxyState, struct
#endif
sub->posted += args->sliceSteps;
sub->profilerSteps++;
ncclProfilerRecordProxyOpEventState(s+i, args, sub->posted, sub->transSize, ncclProfilerProxyOpRecvPosted);
ncclProfilerRecordProxyStepEventState(s+i, args, postedStepId, ncclProfilerProxyStepRecvWait);
}
@@ -1828,7 +1850,7 @@ exit:
return ret;
fail:
*outRegBufFlag = 0;
WARN("rank %d failed to NET register userbuff %p buffSize %ld GDR flag %d", comm->rank, userbuff, buffSize, gdrFlag);
INFO(NCCL_REG, "rank %d failed to NET register userbuff %p buffSize %ld GDR flag %d", comm->rank, userbuff, buffSize, gdrFlag);
goto exit;
}
@@ -1909,7 +1931,7 @@ static ncclResult_t sendProxyRegBuffer(struct ncclProxyConnection* connection, s
/* DMA-BUF support */
if (resources->useDmaBuf) {
int dmabuf_fd;
CUCHECKGOTO(cuMemGetHandleForAddressRange((void*)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0), ret, peermem);
CUCHECKGOTO(cuMemGetHandleForAddressRange((void*)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)), ret, peermem);
NCCLCHECKGOTO(proxyState->ncclNet->regMrDmaBuf(resources->netSendComm, (void*)info->buffer, info->size, NCCL_PTR_CUDA, 0ULL, dmabuf_fd, &handle), ret, peermem);
(void)close(dmabuf_fd);
needReg = false;
@@ -1943,7 +1965,7 @@ static ncclResult_t recvProxyRegBuffer(struct ncclProxyConnection* connection, s
/* DMA-BUF support */
if (resources->useDmaBuf) {
int dmabuf_fd;
CUCHECKGOTO(cuMemGetHandleForAddressRange((void*)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0), ret, peermem);
CUCHECKGOTO(cuMemGetHandleForAddressRange((void*)&dmabuf_fd, (CUdeviceptr)info->buffer, info->size, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, getHandleForAddressRangeFlags(resources->useGdr)), ret, peermem);
NCCLCHECKGOTO(proxyState->ncclNet->regMrDmaBuf(resources->netRecvComm, (void*)info->buffer, info->size, NCCL_PTR_CUDA, 0ULL, dmabuf_fd, &handle), ret, peermem);
(void)close(dmabuf_fd);
needReg = false;
src/transport/net_ib.cc
+161 -36
Просмотреть файл
@@ -12,6 +12,7 @@
#include "graph.h"
#include "utils.h"
#include "param.h"
#include "profiler/net_ib.h"
#include <assert.h>
#include <pthread.h>
@@ -88,6 +89,11 @@ struct ncclIbDev ncclIbDevs[MAX_IB_DEVS];
pthread_mutex_t ncclIbLock = PTHREAD_MUTEX_INITIALIZER;
static int ncclIbRelaxedOrderingEnabled = 0;
#define NCCL_IB_LLSTR(ll) (((ll) == IBV_LINK_LAYER_INFINIBAND) ? "IB" : (((ll) == IBV_LINK_LAYER_ETHERNET) ? "RoCE" : "UNSPECIFIED"))
#define NCCL_IB_SL_DEFAULT 0
#define NCCL_IB_TC_DEFAULT 0
NCCL_PARAM(IbGidIndex, "IB_GID_INDEX", -1);
NCCL_PARAM(IbRoutableFlidIbGidIndex, "IB_ROUTABLE_FLID_GID_INDEX", 1);
NCCL_PARAM(IbRoceVersionNum, "IB_ROCE_VERSION_NUM", 2);
@@ -95,8 +101,8 @@ NCCL_PARAM(IbTimeout, "IB_TIMEOUT", 20);
NCCL_PARAM(IbRetryCnt, "IB_RETRY_CNT", 7);
NCCL_PARAM(IbPkey, "IB_PKEY", 0);
NCCL_PARAM(IbUseInline, "IB_USE_INLINE", 0);
NCCL_PARAM(IbSl, "IB_SL", 0);
NCCL_PARAM(IbTc, "IB_TC", 0);
NCCL_PARAM(IbSl, "IB_SL", -1);
NCCL_PARAM(IbTc, "IB_TC", -1);
NCCL_PARAM(IbArThreshold, "IB_AR_THRESHOLD", 8192);
NCCL_PARAM(IbPciRelaxedOrdering, "IB_PCI_RELAXED_ORDERING", 2);
NCCL_PARAM(IbAdaptiveRouting, "IB_ADAPTIVE_ROUTING", -2);
@@ -330,6 +336,9 @@ static ncclResult_t ncclIbRoceGetVersionNum(const char* deviceName, int portNum,
close(fd);
if (ret == -1) {
// In containerized environments, read could return EINVAL if the GID index is not mapped to the
// container sysfs. In this case return ncclSuccess and let the caller move to next GID index.
if (errno == EINVAL) return ncclSuccess;
WARN("NET/IB: read failed in ncclIbRoceGetVersionNum: %s", strerror(errno));
return ncclSystemError;
}
@@ -362,7 +371,7 @@ static ncclResult_t ncclUpdateGidIndex(struct ibv_context* context, uint8_t port
return ncclSuccess;
}
int usrRoceVer = roceVer;
int gidRoceVerNum, gidRoceVerNumCandidate;
int gidRoceVerNum, gidRoceVerNumCandidate = -1;
const char* deviceName = wrap_ibv_get_device_name(context->device);
NCCLCHECK(ncclIbRoceGetVersionNum(deviceName, portNum, *gidIndex, &gidRoceVerNum));
NCCLCHECK(ncclIbRoceGetVersionNum(deviceName, portNum, gidIndexCandidate, &gidRoceVerNumCandidate));
@@ -533,8 +542,8 @@ ncclResult_t ncclIbMakeVDeviceInternal(int* d, ncclNetVDeviceProps_t* props) {
}
ncclIbDev* dev = ncclIbDevs + props->devs[i];
if (dev->link != dev0->link) {
WARN("NET/IB : Trying to merge multiple devices together with different link_layer properties %s -> %d, %s -> %d. Try only selecting NICs with one type of link using NCCL_IB_HCA",
dev0->devName, dev0->link, dev->devName, dev->link);
WARN("NET/IB : Attempted to merge incompatible devices: [%d]%s:%d/%s and [%d]%s:%d/%s. Try selecting NICs of only one link type using NCCL_IB_HCA",
props->devs[0], dev0->devName, dev0->portNum, NCCL_IB_LLSTR(dev0->link), props->devs[i], dev->devName, dev->portNum, NCCL_IB_LLSTR(dev->link));
return ncclInvalidUsage;
}
}
@@ -551,8 +560,11 @@ ncclResult_t ncclIbMakeVDevice(int* d, ncclNetVDeviceProps_t* props) {
return res;
}
ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction) {
static ncclProfilerCallback_t ncclProfilerFunction;
ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction) {
ncclResult_t ret = ncclSuccess;
ncclProfilerFunction = profFunction;
if (ncclParamIbDisable()) return ncclInternalError;
static int shownIbHcaEnv = 0;
if(wrap_ibv_symbols() != ncclSuccess) { return ncclInternalError; }
@@ -574,7 +586,7 @@ ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction) {
}
// Check if user defined which IB device:port to use
char* userIbEnv = getenv("NCCL_IB_HCA");
const char* userIbEnv = ncclGetEnv("NCCL_IB_HCA");
if (userIbEnv != NULL && shownIbHcaEnv++ == 0) INFO(NCCL_NET|NCCL_ENV, "NCCL_IB_HCA set to %s", userIbEnv);
struct netIf userIfs[MAX_IB_DEVS];
bool searchNot = userIbEnv && userIbEnv[0] == '^';
@@ -641,7 +653,7 @@ ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction) {
if (ncclParamIbAdaptiveRouting() != -2) ncclIbDevs[ncclNIbDevs].ar = ncclParamIbAdaptiveRouting();
TRACE(NCCL_NET,"NET/IB: [%d] %s:%s:%d/%s speed=%d context=%p pciPath=%s ar=%d", d, devices[d]->name, devices[d]->dev_name, ncclIbDevs[ncclNIbDevs].portNum,
portAttr.link_layer == IBV_LINK_LAYER_INFINIBAND ? "IB" : "RoCE", ncclIbDevs[ncclNIbDevs].speed, context, ncclIbDevs[ncclNIbDevs].pciPath, ncclIbDevs[ncclNIbDevs].ar);
NCCL_IB_LLSTR(portAttr.link_layer), ncclIbDevs[ncclNIbDevs].speed, context, ncclIbDevs[ncclNIbDevs].pciPath, ncclIbDevs[ncclNIbDevs].ar);
PTHREADCHECKGOTO(pthread_create(&ncclIbAsyncThread, NULL, ncclIbAsyncThreadMain, ncclIbDevs + ncclNIbDevs), "pthread_create", ret, fail);
ncclSetThreadName(ncclIbAsyncThread, "NCCL IbAsync %2d", ncclNIbDevs);
@@ -672,7 +684,7 @@ ncclResult_t ncclIbInit(ncclDebugLogger_t logFunction) {
ncclIbRelaxedOrderingEnabled = ncclIbRelaxedOrderingCapable();
for (int d = 0; d < ncclNIbDevs; d++) {
snprintf(line+strlen(line), sizeof(line)-strlen(line), " [%d]%s:%d/%s", d, ncclIbDevs[d].devName,
ncclIbDevs[d].portNum, ncclIbDevs[d].link == IBV_LINK_LAYER_INFINIBAND ? "IB" : "RoCE");
ncclIbDevs[d].portNum, NCCL_IB_LLSTR(ncclIbDevs[d].link));
}
char addrline[SOCKET_NAME_MAXLEN+1];
INFO(NCCL_INIT|NCCL_NET, "NET/IB : Using%s %s; OOB %s:%s", line, ncclIbRelaxedOrderingEnabled ? "[RO]" : "",
@@ -913,6 +925,8 @@ struct ncclIbConnectionMetadata {
char devName[MAX_MERGED_DEV_NAME];
uint64_t fifoAddr;
int ndevs;
int tc;
int sl;
};
enum ncclIbCommState {
@@ -954,12 +968,23 @@ struct ncclIbGidInfo {
#define NCCL_NET_IB_REQ_FLUSH 3
const char* reqTypeStr[] = { "Unused", "Send", "Recv", "Flush" };
#define MAX_QPS_PER_REQ 8
struct ncclProfilerInfo {
void* qpEventHandles[MAX_QPS_PER_REQ];
int qpIndex[MAX_QPS_PER_REQ];
int nEventHandles;
ncclProfilerNetIbDescr_v1_t data;
};
struct ncclIbRequest {
struct ncclIbNetCommBase* base;
int type;
struct ncclSocket* sock;
int events[NCCL_IB_MAX_DEVS_PER_NIC];
struct ncclIbNetCommDevBase* devBases[NCCL_IB_MAX_DEVS_PER_NIC];
#ifdef NCCL_ENABLE_NET_PROFILING
struct ncclProfilerInfo pInfo[NCCL_NET_IB_MAX_RECVS];
#endif
int nreqs;
union {
struct {
@@ -1169,7 +1194,7 @@ ncclResult_t ncclIbCreateQp(uint8_t ib_port, struct ncclIbNetCommDevBase* base,
return ncclSuccess;
}
ncclResult_t ncclIbRtrQp(struct ibv_qp* qp, struct ncclIbGidInfo* sGidInfo, uint32_t dest_qp_num, struct ncclIbDevInfo* info, bool fifoTc) {
ncclResult_t ncclIbRtrQp(struct ibv_qp* qp, struct ncclIbGidInfo* sGidInfo, uint32_t dest_qp_num, struct ncclIbDevInfo* info, bool fifoTc, int tc, int sl) {
struct ibv_qp_attr qpAttr;
memset(&qpAttr, 0, sizeof(struct ibv_qp_attr));
qpAttr.qp_state = IBV_QPS_RTR;
@@ -1185,7 +1210,7 @@ ncclResult_t ncclIbRtrQp(struct ibv_qp* qp, struct ncclIbGidInfo* sGidInfo, uint
qpAttr.ah_attr.grh.flow_label = 0;
qpAttr.ah_attr.grh.sgid_index = sGidInfo->localGidIndex;
qpAttr.ah_attr.grh.hop_limit = 255;
qpAttr.ah_attr.grh.traffic_class = fifoTc && ncclParamIbFifoTc() != -1 ? ncclParamIbFifoTc() : ncclParamIbTc();
qpAttr.ah_attr.grh.traffic_class = fifoTc && ncclParamIbFifoTc() != -1 ? ncclParamIbFifoTc() : tc;
} else {
//pick lid if subnet prefixs are same, FLID if they are not
if (ncclIbExtractLocalSubnetPrefix(sGidInfo->localGid.global.subnet_prefix) ==
@@ -1207,10 +1232,10 @@ ncclResult_t ncclIbRtrQp(struct ibv_qp* qp, struct ncclIbGidInfo* sGidInfo, uint
qpAttr.ah_attr.grh.hop_limit = 255;
}
}
qpAttr.ah_attr.sl = ncclParamIbSl();
qpAttr.ah_attr.sl = sl;
qpAttr.ah_attr.src_path_bits = 0;
qpAttr.ah_attr.port_num = info->ib_port;
TRACE(NCCL_NET, "NET/IB : ncclIbRtrQp qpn=%u mtu=%d dst=%u ll=%u port=%u", qp->qp_num, info->mtu, dest_qp_num, info->link_layer, info->ib_port);
TRACE(NCCL_NET, "NET/IB : ncclIbRtrQp qpn=%u mtu=%d dst=%u ll=%u port=%u sl: %d tc: %d", qp->qp_num, info->mtu, dest_qp_num, info->link_layer, info->ib_port, qpAttr.ah_attr.sl, qpAttr.ah_attr.grh.traffic_class);
NCCLCHECK(wrap_ibv_modify_qp(qp, &qpAttr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER));
return ncclSuccess;
}
@@ -1249,12 +1274,13 @@ fail:
goto exit;
}
ncclResult_t ncclIbConnect(int dev, void* opaqueHandle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
ncclResult_t ncclIbConnect(int dev, ncclNetCommConfig_t* config, void* opaqueHandle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
ncclResult_t ret = ncclSuccess;
struct ncclIbHandle* handle = (struct ncclIbHandle*) opaqueHandle;
struct ncclIbCommStage* stage = &handle->stage;
struct ncclIbSendComm* comm = (struct ncclIbSendComm*)stage->comm;
int ready;
uint8_t link_layer = IBV_LINK_LAYER_UNSPECIFIED;
*sendComm = NULL;
if (stage->state == ncclIbCommStateConnect) goto ib_connect_check;
@@ -1284,7 +1310,7 @@ ib_connect_check:
// IB Setup
struct ncclIbMergedDev* mergedDev;
if (dev >= ncclNMergedIbDevs) {
WARN("NET/IB : Trying to use non-existant virtual device %d", dev);
WARN("NET/IB : Trying to use non-existent virtual device %d", dev);
return ncclInternalError;
}
@@ -1390,8 +1416,17 @@ ib_recv_dev_list:
devInfo->gid.global.subnet_prefix, devInfo->gid.global.interface_id, devInfo->fifoRkey, commDev->fifoMr->lkey);
}
}
if (link_layer == IBV_LINK_LAYER_UNSPECIFIED) link_layer = devInfo->link_layer;
if (link_layer != devInfo->link_layer) {
int ibDev0 = comm->devs[0].base.ibDevN;
WARN("NET/IB : Attempted to connect incompatible devices: [%d]%s:%d/%s and [%d]%s:%d/%s. Try selecting NICs of only one link type using NCCL_IB_HCA",
commDev->base.ibDevN, ibDev->devName, ibDev->portNum, NCCL_IB_LLSTR(ibDev->portAttr.link_layer), ibDev0, ncclIbDevs[ibDev0].devName, ncclIbDevs[ibDev0].portNum, NCCL_IB_LLSTR(link_layer));
return ncclInternalError;
}
}
meta.fifoAddr = (uint64_t)comm->fifo;
meta.sl = (ncclParamIbSl() != -1) ? ncclParamIbSl() : (config && config->trafficClass != NCCL_NET_TRAFFIC_CLASS_UNDEF) ? config->trafficClass : NCCL_IB_SL_DEFAULT;
meta.tc = (ncclParamIbTc() != -1) ? ncclParamIbTc() : (config && config->trafficClass != NCCL_NET_TRAFFIC_CLASS_UNDEF) ? config->trafficClass : NCCL_IB_TC_DEFAULT;
strncpy(meta.devName, mergedDev->devName, MAX_MERGED_DEV_NAME);
stage->state = ncclIbCommStateSend;
@@ -1417,13 +1452,16 @@ ib_connect:
comm->base.nRemDevs = remMeta.ndevs;
int link_layer;
link_layer = remMeta.devs[0].link_layer;
for (int i = 1; i < remMeta.ndevs; i++) {
if (remMeta.devs[i].link_layer != link_layer) {
WARN("NET/IB : Can't connect net devices with different link_layer. i=%d remMeta.ndevs=%d link_layer=%d rem_link_layer=%d",
i, remMeta.ndevs, link_layer, remMeta.devs[i].link_layer);
return ncclInternalError;
// ensure that the remote devices have the same link layer than the local devices used in the connection.
if (comm->base.vProps.ndevs > 0) {
int ibDev0 = comm->devs[0].base.ibDevN;
link_layer = ncclIbDevs[ibDev0].portAttr.link_layer;
for (int i = 0; i < remMeta.ndevs; i++) {
if (remMeta.devs[i].link_layer != link_layer) {
WARN("NET/IB : Remote %s device is incompatible with the local [%d]%s:%d/%s. Try selecting NICs of only one link type using NCCL_IB_HCA",
NCCL_IB_LLSTR(remMeta.devs[i].link_layer), ibDev0, ncclIbDevs[ibDev0].devName, ncclIbDevs[ibDev0].portNum, NCCL_IB_LLSTR(link_layer));
return ncclInternalError;
}
}
}
@@ -1457,7 +1495,7 @@ ib_connect:
ncclIbDev* ibDev = ncclIbDevs + commDev->base.ibDevN;
remDevInfo->mtu = std::min(remDevInfo->mtu, ibDev->portAttr.active_mtu);
NCCLCHECKGOTO(ncclIbRtrQp(qp, &commDev->base.gidInfo, remQpInfo->qpn, remDevInfo, false), ret, fail);
NCCLCHECKGOTO(ncclIbRtrQp(qp, &commDev->base.gidInfo, remQpInfo->qpn, remDevInfo, false, remMeta.tc, remMeta.sl), ret, fail);
NCCLCHECKGOTO(ncclIbRtsQp(qp), ret, fail);
}
@@ -1544,6 +1582,7 @@ ncclResult_t ncclIbAccept(void* listenComm, void** recvComm, ncclNetDeviceHandle
struct ncclIbCommStage* stage = &lComm->stage;
struct ncclIbRecvComm* rComm = (struct ncclIbRecvComm*)stage->comm;
int ready;
int link_layer = IBV_LINK_LAYER_UNSPECIFIED;
*recvComm = NULL;
if (stage->state == ncclIbCommStateAccept) goto ib_accept_check;
@@ -1582,7 +1621,7 @@ ib_recv_dev_list:
ncclNetVDeviceProps_t remoteVProps;
memcpy(&remoteVProps, stage->buffer, sizeof(ncclNetVDeviceProps_t));
if (lComm->dev >= ncclNMergedIbDevs) {
WARN("NET/IB : Trying to use non-existant virtual device %d", lComm->dev);
WARN("NET/IB : Trying to use non-existent virtual device %d", lComm->dev);
return ncclInternalError;
}
@@ -1641,6 +1680,13 @@ ib_recv:
ibDev = ncclIbDevs + ibDevN;
NCCLCHECKGOTO(ncclIbGetGidIndex(ibDev->context, ibDev->portNum, &ibDev->portAttr, &rCommDev->base.gidInfo.localGidIndex), ret, fail);
NCCLCHECKGOTO(wrap_ibv_query_gid(ibDev->context, ibDev->portNum, rCommDev->base.gidInfo.localGidIndex, &rCommDev->base.gidInfo.localGid), ret, fail);
if (link_layer == IBV_LINK_LAYER_UNSPECIFIED) link_layer = ibDev->portAttr.link_layer;
if (link_layer != ibDev->portAttr.link_layer) {
int ibDev0 = rComm->devs[0].base.ibDevN;
WARN("NET/IB : Attempted to connect incompatible devices: [%d]%s:%d/%s and [%d]%s:%d/%s. Try selecting NICs of only one link type using NCCL_IB_HCA",
ibDevN, ibDev->devName, ibDev->portNum, NCCL_IB_LLSTR(ibDev->portAttr.link_layer), ibDev0, ncclIbDevs[ibDev0].devName, ncclIbDevs[ibDev0].portNum, NCCL_IB_LLSTR(link_layer));
return ncclInternalError;
}
}
// Copy remDevInfo for things like remGidInfo, remFifoAddr, etc.
@@ -1648,6 +1694,12 @@ ib_recv:
rComm->base.remDevs[i] = remMeta.devs[i];
rComm->base.remDevs[i].remoteGid.global.interface_id = rComm->base.remDevs[i].gid.global.interface_id;
rComm->base.remDevs[i].remoteGid.global.subnet_prefix = rComm->base.remDevs[i].gid.global.subnet_prefix;
if (remMeta.devs[i].link_layer != link_layer) {
int ibDev0 = rComm->devs[0].base.ibDevN;
WARN("NET/IB : Remote %s device is incompatible with the local [%d]%s:%d/%s. Try selecting NICs of only one link type using NCCL_IB_HCA",
NCCL_IB_LLSTR(remMeta.devs[i].link_layer), ibDev0, ncclIbDevs[ibDev0].devName, ncclIbDevs[ibDev0].portNum, NCCL_IB_LLSTR(link_layer));
return ncclInternalError;
}
}
// Stripe QP creation across merged devs
@@ -1675,17 +1727,18 @@ ib_recv:
// However, this has been confirmed to be intentional.
// coverity[copy_paste_error]
NCCLCHECKGOTO(wrap_ibv_set_ece(qp->qp, &remMeta.qpInfo[q].ece, &meta.qpInfo[q].ece_supported), ret, fail);
// Query the reduced ece for this QP (matching enhancements between the requestor and the responder)
// Store this in our own qpInfo for returning to the requestor
if (meta.qpInfo[q].ece_supported)
NCCLCHECKGOTO(wrap_ibv_query_ece(qp->qp, &meta.qpInfo[q].ece, &meta.qpInfo[q].ece_supported), ret, fail);
} else {
meta.qpInfo[q].ece_supported = 0;
}
NCCLCHECKGOTO(ncclIbRtrQp(qp->qp, &rCommDev->base.gidInfo, remMeta.qpInfo[q].qpn, remDevInfo, true), ret, fail);
NCCLCHECKGOTO(ncclIbRtrQp(qp->qp, &rCommDev->base.gidInfo, remMeta.qpInfo[q].qpn, remDevInfo, true, remMeta.tc, remMeta.sl), ret, fail);
NCCLCHECKGOTO(ncclIbRtsQp(qp->qp), ret, fail);
// Query the reduced ece for this QP (matching enhancements between the requestor and the responder)
// Store this in our own qpInfo for returning to the requestor
if (remMeta.qpInfo[q].ece_supported && meta.qpInfo[q].ece_supported) {
NCCLCHECKGOTO(wrap_ibv_query_ece(qp->qp, &meta.qpInfo[q].ece, &meta.qpInfo[q].ece_supported), ret, fail);
}
}
useDmaBuf = (ncclIbDmaBufSupport(lComm->dev) == ncclSuccess);
@@ -1745,7 +1798,7 @@ ib_recv:
devInfo.gid.global.subnet_prefix = rCommDev->base.gidInfo.localGid.global.subnet_prefix;
devInfo.gid.global.interface_id = rCommDev->base.gidInfo.localGid.global.interface_id;
devInfo.mtu = ibDev->portAttr.active_mtu;
NCCLCHECKGOTO(ncclIbRtrQp(rCommDev->gpuFlush.qp.qp, &rCommDev->base.gidInfo, rCommDev->gpuFlush.qp.qp->qp_num, &devInfo, false), ret, fail);
NCCLCHECKGOTO(ncclIbRtrQp(rCommDev->gpuFlush.qp.qp, &rCommDev->base.gidInfo, rCommDev->gpuFlush.qp.qp->qp_num, &devInfo, false, remMeta.tc, remMeta.sl), ret, fail);
NCCLCHECKGOTO(ncclIbRtsQp(rCommDev->gpuFlush.qp.qp), ret, fail);
}
@@ -1763,6 +1816,8 @@ ib_recv:
meta.devs[i].fifoRkey = rComm->devs[i].sizesFifoMr->rkey;
}
meta.fifoAddr = (uint64_t)rComm->sizesFifo;
meta.sl = remMeta.sl;
meta.tc = remMeta.tc;
for (int q = 0; q < rComm->base.nqps; q++) {
meta.qpInfo[q].qpn = rComm->base.qps[q].qp->qp_num;
@@ -1959,7 +2014,7 @@ ncclResult_t ncclIbDeregMr(void* comm, void* mhandle) {
NCCL_PARAM(IbSplitDataOnQps, "IB_SPLIT_DATA_ON_QPS", 0);
ncclResult_t ncclIbMultiSend(struct ncclIbSendComm* comm, int slot) {
ncclResult_t ncclIbMultiSend(struct ncclIbSendComm* comm, int slot, void* pHandle) {
struct ncclIbRequest** reqs = comm->fifoReqs[slot];
volatile struct ncclIbSendFifo* slots = comm->fifo[slot];
int nreqs = slots[0].nreqs;
@@ -1977,6 +2032,9 @@ ncclResult_t ncclIbMultiSend(struct ncclIbSendComm* comm, int slot) {
wr->wr.rdma.remote_addr = slots[r].addr;
wr->next = wr + 1;
wr_id += (reqs[r] - comm->base.reqs) << (r*8);
#ifdef NCCL_ENABLE_NET_PROFILING
reqs[r]->pInfo[0].nEventHandles = 0;
#endif
}
// Write size as immediate data. In the case of multi-send, only write
@@ -2046,6 +2104,24 @@ ncclResult_t ncclIbMultiSend(struct ncclIbSendComm* comm, int slot) {
}
struct ibv_send_wr* bad_wr;
#ifdef NCCL_ENABLE_NET_PROFILING
// QP profiling loop
for (int r=0; r<nreqs && pHandle; r++) {
// Store comm qpIndex for this request
int nEventHandles = reqs[r]->pInfo[0].nEventHandles;
reqs[r]->pInfo[0].qpIndex[nEventHandles%MAX_QPS_PER_REQ] = qpIndex;
// Store info for profiler
int pluginId = NCCL_PROFILER_NET_TYPE_IB | NCCL_PROFILER_NET_IB_VER;
reqs[r]->pInfo[0].data.type = ncclProfileQp;
reqs[r]->pInfo[0].data.qp.device = devIndex;
reqs[r]->pInfo[0].data.qp.wr_id = comm->wrs[r].wr_id;
reqs[r]->pInfo[0].data.qp.opcode = comm->wrs[r].opcode;
reqs[r]->pInfo[0].data.qp.qpNum = qp->qp->qp_num;
reqs[r]->pInfo[0].data.qp.length = comm->sges[r].length;
NCCLCHECK(ncclProfilerFunction(&reqs[r]->pInfo[0].qpEventHandles[nEventHandles%MAX_QPS_PER_REQ], 0, pHandle, pluginId, &reqs[r]->pInfo[0].data));
reqs[r]->pInfo[0].nEventHandles++;
}
#endif
NCCLCHECK(wrap_ibv_post_send(qp->qp, comm->wrs, &bad_wr));
for (int r=0; r<nreqs; r++) {
@@ -2067,7 +2143,7 @@ ncclResult_t ncclIbMultiSend(struct ncclIbSendComm* comm, int slot) {
return ncclSuccess;
}
ncclResult_t ncclIbIsend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request) {
ncclResult_t ncclIbIsend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request) {
struct ncclIbSendComm* comm = (struct ncclIbSendComm*)sendComm;
if (comm->base.ready == 0) { WARN("NET/IB: ncclIbIsend() called when comm->base.ready == 0"); return ncclInternalError; }
if (comm->base.ready == 0) { *request = NULL; return ncclSuccess; }
@@ -2140,7 +2216,7 @@ ncclResult_t ncclIbIsend(void* sendComm, void* data, size_t size, int tag, void*
}
TIME_START(0);
NCCLCHECK(ncclIbMultiSend(comm, slot));
NCCLCHECK(ncclIbMultiSend(comm, slot, phandle));
// Clear slots[0]->nreqs, as well as other fields to help debugging and sanity checks
memset((void*)slots, 0, sizeof(struct ncclIbSendFifo));
@@ -2236,7 +2312,7 @@ ncclResult_t ncclIbPostFifo(struct ncclIbRecvComm* comm, int n, void** data, siz
return ncclSuccess;
}
ncclResult_t ncclIbIrecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request) {
ncclResult_t ncclIbIrecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request) {
struct ncclIbRecvComm* comm = (struct ncclIbRecvComm*)recvComm;
if (comm->base.ready == 0) { WARN("NET/IB: ncclIbIrecv() called when comm->base.ready == 0"); return ncclInternalError; }
if (comm->base.ready == 0) { *request = NULL; return ncclSuccess; }
@@ -2248,6 +2324,9 @@ ncclResult_t ncclIbIrecv(void* recvComm, int n, void** data, size_t* sizes, int*
req->type = NCCL_NET_IB_REQ_RECV;
req->sock = &comm->base.sock;
req->nreqs = n;
#ifdef NCCL_ENABLE_NET_PROFILING
for (int r = 0; r < n && phandles; r++) req->pInfo[r].nEventHandles = 0;
#endif
for (int i = 0; i < comm->base.vProps.ndevs; i++) {
req->devBases[i] = &comm->devs[i].base;
@@ -2268,6 +2347,19 @@ ncclResult_t ncclIbIrecv(void* recvComm, int n, void** data, size_t* sizes, int*
for (int i = 0; i < nqps; i++) {
struct ncclIbQp* qp = comm->base.qps + comm->base.qpIndex;
ncclIbAddEvent(req, qp->devIndex, &comm->devs[qp->devIndex].base);
#ifdef NCCL_ENABLE_NET_PROFILING
// Start a QP event for every request in the multirecv and every qp
for (int r = 0; r < n && phandles; r++) {
// Store info for profiler
int pluginId = NCCL_PROFILER_NET_TYPE_IB | NCCL_PROFILER_NET_IB_VER;
req->pInfo[r].data.type = ncclProfileQp;
req->pInfo[r].data.qp.device = qp->devIndex;
req->pInfo[r].data.qp.wr_id = wr.wr_id;
req->pInfo[r].data.qp.qpNum = qp->qp->qp_num;
NCCLCHECK(ncclProfilerFunction(&req->pInfo[r].qpEventHandles[i], 0, phandles[r], pluginId, &req->pInfo[r].data));
req->pInfo[r].nEventHandles++;
}
#endif
NCCLCHECK(wrap_ibv_post_recv(qp->qp, &wr, &bad_wr));
comm->base.qpIndex = (comm->base.qpIndex+1)%comm->base.nqps;
}
@@ -2339,6 +2431,16 @@ ncclResult_t ncclIbIflush(void* recvComm, int n, void** data, int* sizes, void**
#define HCA_NAME(req, index) ((req)->devBases[(index)]->pd->context->device->name)
#ifdef NCCL_ENABLE_NET_PROFILING
static int getReqQpIndex(struct ncclIbRequest* req, int request, int qpNumber) {
for (int i = 0; i < MAX_QPS_PER_REQ; i++) {
int qpIndex = req->pInfo[request].qpIndex[i];
if (req->base->qps[qpIndex].qp->qp_num == qpNumber) return i;
}
return 0;
}
#endif
ncclResult_t ncclIbTest(void* request, int* done, int* sizes) {
struct ncclIbRequest *r = (struct ncclIbRequest*)request;
*done = 0;
@@ -2348,11 +2450,24 @@ ncclResult_t ncclIbTest(void* request, int* done, int* sizes) {
TRACE(NCCL_NET, "r=%p done", r);
*done = 1;
if (sizes && r->type == NCCL_NET_IB_REQ_RECV) {
for (int i=0; i<r->nreqs; i++) sizes[i] = r->recv.sizes[i];
for (int i=0; i<r->nreqs; i++) {
sizes[i] = r->recv.sizes[i];
#ifdef NCCL_ENABLE_NET_PROFILING
for (int j = 0; j < r->pInfo[i].nEventHandles; j++) {
NCCLCHECK(ncclProfilerFunction(&r->pInfo[i].qpEventHandles[j], 1, NULL, 0, NULL));
}
#endif
}
}
if (sizes && r->type == NCCL_NET_IB_REQ_SEND) {
sizes[0] = r->send.size;
#ifdef NCCL_ENABLE_NET_PROFILING
for (int j = 0; j < r->pInfo[0].nEventHandles; j++) {
NCCLCHECK(ncclProfilerFunction(&r->pInfo[0].qpEventHandles[j], 1, NULL, 0, NULL));
}
#endif
}
// Stop all remaining Qp events for this event
NCCLCHECK(ncclIbFreeRequest(r));
return ncclSuccess;
}
@@ -2407,6 +2522,10 @@ ncclResult_t ncclIbTest(void* request, int* done, int* sizes) {
return ncclInternalError;
}
sendReq->events[i]--;
#ifdef NCCL_ENABLE_NET_PROFILING
// Stop Qp event for sendReq
NCCLCHECK(ncclProfilerFunction(&sendReq->pInfo[j].qpEventHandles[getReqQpIndex(sendReq, j, wc->qp_num)], 1, NULL, 0, NULL));
#endif
}
} else {
if (req && wc->opcode == IBV_WC_RECV_RDMA_WITH_IMM) {
@@ -2419,6 +2538,12 @@ ncclResult_t ncclIbTest(void* request, int* done, int* sizes) {
}
}
req->events[i]--;
#ifdef NCCL_ENABLE_NET_PROFILING
// Stop Qp event for workFifo
for (int j = 0; j < req->nreqs; j++) {
NCCLCHECK(ncclProfilerFunction(&req->pInfo[j].qpEventHandles[getReqQpIndex(req, j, wc->qp_num)], 1, NULL, 0, NULL));
}
#endif
}
}
// Once the IB fatal event is reported in the async thread, we want to propagate this error
src/transport/net_socket.cc
+67 -6
Просмотреть файл
@@ -9,6 +9,7 @@
#include "socket.h"
#include "net.h"
#include "param.h"
#include "profiler/net_socket.h"
#include <pthread.h>
#include <stdlib.h>
@@ -35,7 +36,10 @@ static ncclResult_t ncclNetSocketGetPciPath(char* devName, char** pciPath) {
return ncclSuccess;
}
ncclResult_t ncclNetSocketInit(ncclDebugLogger_t logFunction) {
static ncclProfilerCallback_t ncclProfilerFunction;
ncclResult_t ncclNetSocketInit(ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction) {
ncclProfilerFunction = profFunction;
if (ncclNetIfs == -1) {
pthread_mutex_lock(&ncclNetSocketLock);
if (ncclNetIfs == -1) {
@@ -158,6 +162,11 @@ struct ncclNetSocketTask {
ncclResult_t result;
};
struct ncclProfilerInfo {
void* eHandle;
void* pHandle;
};
struct ncclNetSocketRequest {
int op;
void* data;
@@ -168,6 +177,7 @@ struct ncclNetSocketRequest {
struct ncclNetSocketComm* comm;
struct ncclNetSocketTask* tasks[MAX_SOCKETS];
int nSubs;
struct ncclProfilerInfo pInfo;
};
struct ncclNetSocketTaskQueue {
@@ -180,6 +190,7 @@ struct ncclNetSocketThreadResources {
struct ncclNetSocketTaskQueue threadTaskQueue;
int stop;
struct ncclNetSocketComm* comm;
struct ncclProfilerInfo* pInfo;
pthread_mutex_t threadLock;
pthread_cond_t threadCond;
};
@@ -210,6 +221,9 @@ void* persistentSocketThread(void *args_) {
struct ncclNetSocketComm* comm = resource->comm;
struct ncclNetSocketTaskQueue* myQueue = &resource->threadTaskQueue;
int nSocksPerThread = comm->nSocks / comm->nThreads;
#ifdef NCCL_ENABLE_NET_PROFILING
void* eHandle[MAX_REQUESTS*MAX_SOCKETS] = { 0 };
#endif
while (1) {
int idle = 1;
int mark = myQueue->next; // mark newest task seen
@@ -220,13 +234,33 @@ void* persistentSocketThread(void *args_) {
for (int j=0; j<nSocksPerThread; j++) {
struct ncclNetSocketTask* r = myQueue->tasks+i+j;
if (r != NULL && r->used == 1 && r->offset < r->size) {
#ifdef NCCL_ENABLE_NET_PROFILING
if (!eHandle[i+j]) {
ncclProfilerNetSockDescr_v1_t data;
data.type = ncclProfileSocket;
data.sock.fd = r->sock->fd;
data.sock.op = r->op;
data.sock.length = r->size;
ncclProfilerFunction(&eHandle[i+j], 0, resource->pInfo->pHandle, NCCL_PROFILER_NET_TYPE_SOCK | 1, &data);
}
#endif
r->result = ncclSocketProgress(r->op, r->sock, r->data, r->size, &r->offset);
if (r->result != ncclSuccess) {
#ifdef NCCL_ENABLE_NET_PROFILING
ncclProfilerFunction(&eHandle[i+j], 1, NULL, 0, NULL);
eHandle[i+j] = NULL;
#endif
WARN("NET/Socket : socket progress error");
return NULL;
}
idle = 0;
if (r->offset < r->size) repeat = 1;
#ifdef NCCL_ENABLE_NET_PROFILING
if (repeat == 0) {
ncclProfilerFunction(&eHandle[i+j], 1, NULL, 0, NULL);
eHandle[i+j] = NULL;
}
#endif
}
}
} while (repeat);
@@ -326,7 +360,7 @@ fail:
goto exit;
}
ncclResult_t ncclNetSocketConnect(int dev, void* opaqueHandle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
ncclResult_t ncclNetSocketConnect(int dev, ncclNetCommConfig_t* config, void* opaqueHandle, void** sendComm, ncclNetDeviceHandle_t** /*sendDevComm*/) {
if (dev < 0 || dev >= ncclNetIfs) { // data transfer socket is based on specified dev
return ncclInternalError;
}
@@ -444,7 +478,7 @@ ncclResult_t ncclNetSocketGetRequest(struct ncclNetSocketComm* comm, int op, voi
return ncclInternalError;
}
ncclResult_t ncclNetSocketGetTask(struct ncclNetSocketComm* comm, int op, void* data, int size, struct ncclNetSocketTask** req) {
ncclResult_t ncclNetSocketGetTask(struct ncclNetSocketComm* comm, struct ncclProfilerInfo* pInfo, int op, void* data, int size, struct ncclNetSocketTask** req) {
int tid = comm->nextSock % comm->nThreads;
struct ncclNetSocketThreadResources* res = comm->threadResources+tid;
struct ncclNetSocketTaskQueue* queue = &res->threadTaskQueue;
@@ -457,6 +491,9 @@ ncclResult_t ncclNetSocketGetTask(struct ncclNetSocketComm* comm, int op, void*
NCCLCHECK(ncclCalloc(&queue->tasks, queue->len));
queue->next = 0;
res->comm = comm;
#ifdef NCCL_ENABLE_NET_PROFILING
res->pInfo = pInfo;
#endif
pthread_mutex_init(&res->threadLock, NULL);
pthread_cond_init(&res->threadCond, NULL);
PTHREADCHECK(pthread_create(comm->helperThread+tid, NULL, persistentSocketThread, res), "pthread_create");
@@ -520,7 +557,7 @@ ncclResult_t ncclNetSocketTest(void* request, int* done, int* size) {
int taskSize = std::max(MIN_CHUNKSIZE, DIVUP(r->size, r->comm->nSocks));
while (chunkOffset < r->size) {
int chunkSize = std::min(taskSize, r->size-chunkOffset);
NCCLCHECK(ncclNetSocketGetTask(r->comm, r->op, (char*)(r->data)+chunkOffset, chunkSize, r->tasks+i++));
NCCLCHECK(ncclNetSocketGetTask(r->comm, &r->pInfo, r->op, (char*)(r->data)+chunkOffset, chunkSize, r->tasks+i++));
chunkOffset += chunkSize;
}
}
@@ -544,6 +581,16 @@ ncclResult_t ncclNetSocketTest(void* request, int* done, int* size) {
}
}
} else { // progress request using main thread
#ifdef NCCL_ENABLE_NET_PROFILING
if (!r->pInfo.eHandle) {
ncclProfilerNetSockDescr_v1_t data;
data.type = ncclProfileSocket;
data.sock.fd = r->ctrlSock->fd;
data.sock.op = r->op;
data.sock.length = r->size;
ncclProfilerFunction(&r->pInfo.eHandle, 0, r->pInfo.pHandle, NCCL_PROFILER_NET_TYPE_SOCK | 1, &data);
}
#endif
if (r->offset < r->size) {
NCCLCHECK(ncclSocketProgress(r->op, r->ctrlSock, r->data, r->size, &r->offset));
}
@@ -551,6 +598,10 @@ ncclResult_t ncclNetSocketTest(void* request, int* done, int* size) {
if (size) *size = r->size;
*done = 1;
r->used = 0;
#ifdef NCCL_ENABLE_NET_PROFILING
ncclProfilerFunction(&r->pInfo.eHandle, 1, NULL, 0, NULL);
r->pInfo.eHandle = NULL;
#endif
}
}
}
@@ -562,16 +613,26 @@ ncclResult_t ncclNetSocketRegMr(void* comm, void* data, size_t size, int type, v
}
ncclResult_t ncclNetSocketDeregMr(void* comm, void* mhandle) { return ncclSuccess; }
ncclResult_t ncclNetSocketIsend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request) {
ncclResult_t ncclNetSocketIsend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request) {
struct ncclNetSocketComm* comm = (struct ncclNetSocketComm*)sendComm;
NCCLCHECK(ncclNetSocketGetRequest(comm, NCCL_SOCKET_SEND, data, (int) size, (struct ncclNetSocketRequest**)request));
#ifdef NCCL_ENABLE_NET_PROFILING
// NCCL core profiler callback
struct ncclNetSocketRequest* req = *(struct ncclNetSocketRequest **)request;
req->pInfo.pHandle = phandle;
#endif
return ncclSuccess;
}
ncclResult_t ncclNetSocketIrecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request) {
ncclResult_t ncclNetSocketIrecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request) {
struct ncclNetSocketComm* comm = (struct ncclNetSocketComm*)recvComm;
if (n != 1) return ncclInternalError;
NCCLCHECK(ncclNetSocketGetRequest(comm, NCCL_SOCKET_RECV, data[0], (int)sizes[0], (struct ncclNetSocketRequest**)request));
#ifdef NCCL_ENABLE_NET_PROFILING
// NCCL core profiler callback
struct ncclNetSocketRequest* req = *(struct ncclNetSocketRequest **)request;
if (phandles) req->pInfo.pHandle = phandles[0];
#endif
return ncclSuccess;
}
src/transport/nvls.cc
+80 -67
Просмотреть файл
@@ -108,29 +108,29 @@ ncclResult_t nvlsGroupUnbind(struct ncclComm *comm, size_t size, CUmemGenericAll
return ncclSuccess;
}
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t size) {
CUCHECK(cuMulticastUnbind(*mcHandler, dev, 0/*mcOffset*/, size));
CUCHECK(cuMemUnmap(ptr, size));
CUCHECK(cuMemAddressFree(ptr, size));
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t ucsize, size_t mcsize) {
CUCHECK(cuMulticastUnbind(*mcHandler, dev, 0/*mcOffset*/, ucsize));
CUCHECK(cuMemUnmap(ptr, mcsize));
CUCHECK(cuMemAddressFree(ptr, mcsize));
CUCHECK(cuMemRelease(*mcHandler));
INFO(NCCL_NVLS, "rank %d - NVLS deregistered buffer %p on device %d, size %ld", comm->rank, (void*)ptr, dev, size);
INFO(NCCL_NVLS, "rank %d - NVLS deregistered buffer %p on device %d ucsize %ld mcsize %ld", comm->rank, (void*)ptr, dev, ucsize, mcsize);
return ncclSuccess;
}
ncclResult_t nvlsGroupUnmapMem(struct ncclComm *comm, size_t size, void* ucptr, CUmemGenericAllocationHandle* ucHandle, void* mcptr, CUmemGenericAllocationHandle* mcHandle) {
INFO(NCCL_NVLS, "NVLS Unmap mem UC handle 0x%llx(%p) MC handle 0x%llx(%p)", *ucHandle, ucptr, *mcHandle, mcptr);
ncclResult_t nvlsGroupUnmapMem(struct ncclComm *comm, size_t ucsize, void* ucptr, CUmemGenericAllocationHandle* ucHandle, size_t mcsize, void* mcptr, CUmemGenericAllocationHandle* mcHandle) {
INFO(NCCL_NVLS, "NVLS Unmap mem UC handle 0x%llx(%p) ucsize %zu MC handle 0x%llx(%p) mcsize %zd", *ucHandle, ucptr, ucsize, *mcHandle, mcptr, mcsize);
// Release the UC memory and mapping
if (ucptr) {
CUCHECK(cuMemUnmap((CUdeviceptr)ucptr, size));
CUCHECK(cuMemAddressFree((CUdeviceptr)ucptr, size));
CUCHECK(cuMemUnmap((CUdeviceptr)ucptr, ucsize));
CUCHECK(cuMemAddressFree((CUdeviceptr)ucptr, ucsize));
CUCHECK(cuMemRelease(*ucHandle));
}
// Release the MC memory and mapping
if (mcptr) {
CUCHECK(cuMemUnmap((CUdeviceptr)mcptr, size));
CUCHECK(cuMemAddressFree((CUdeviceptr)mcptr, size));
CUCHECK(cuMemUnmap((CUdeviceptr)mcptr, mcsize));
CUCHECK(cuMemAddressFree((CUdeviceptr)mcptr, mcsize));
CUCHECK(cuMemRelease(*mcHandle));
}
@@ -197,25 +197,27 @@ fail:
goto exit;
}
static ncclResult_t nvlsAllocateMem(struct ncclComm* comm, CUmulticastGranularity_flags mcOption, const CUmemAccessDesc* desc, size_t* sizePtr, CUmemGenericAllocationHandle* ucHandle, CUmemGenericAllocationHandle* mcHandle, void** ucptr, void** mcptr) {
static ncclResult_t nvlsAllocateMem(struct ncclComm* comm, const CUmemAccessDesc* desc, size_t size, CUmemGenericAllocationHandle* ucHandle, CUmemGenericAllocationHandle* mcHandle, void** ucptr, void** mcptr, size_t* ucsizePtr, size_t* mcsizePtr) {
char shareableHandle[NVLS_HANDLE_SIZE];
CUmulticastObjectProp mcprop;
CUmemAllocationProp ucprop;
ncclResult_t ret = ncclSuccess;
size_t size = *sizePtr;
size_t originSize = size;
size_t mcsize;
size_t ucsize;
size_t ucgran, mcgran;
int allocMcHandle = 0;
mcsize = ucsize = size;
*ucptr = *mcptr = NULL;
memset(shareableHandle, '\0', sizeof(shareableHandle));
memset(&mcprop, 0, sizeof(CUmulticastObjectProp));
mcprop.numDevices = comm->localRanks;
mcprop.handleTypes = ncclCuMemHandleType;
mcprop.flags = 0;
mcprop.size = size;
CUCHECKGOTO(cuMulticastGetGranularity(&mcgran, &mcprop, mcOption), ret, fail);
ALIGN_SIZE(size, mcgran);
*sizePtr = mcprop.size = size;
CUCHECKGOTO(cuMulticastGetGranularity(&mcgran, &mcprop, CU_MULTICAST_GRANULARITY_RECOMMENDED), ret, fail);
ALIGN_SIZE(mcsize, mcgran);
mcprop.size = mcsize;
if (comm->localRank == 0) {
NCCLCHECKGOTO(nvlsGroupCreate(comm, &mcprop, comm->localRank, comm->localRanks, mcHandle, shareableHandle), ret, fail);
@@ -235,26 +237,29 @@ static ncclResult_t nvlsAllocateMem(struct ncclComm* comm, CUmulticastGranularit
ucprop.location.id = comm->cudaDev;
ucprop.requestedHandleTypes = ncclCuMemHandleType;
CUCHECKGOTO(cuMemGetAllocationGranularity(&ucgran, &ucprop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED), ret, fail);
// Map a VA for UC memory
CUCHECKGOTO(cuMemAddressReserve((CUdeviceptr*)ucptr, size, ucgran, 0U, 0), ret, fail);
ALIGN_SIZE(ucsize, ucgran);
// Map a VA for UC memory with MC alignment and size
CUCHECKGOTO(cuMemAddressReserve((CUdeviceptr*)ucptr, ucsize, ucgran, 0U, 0), ret, fail);
// Alloc local physical mem for this NVLS group
CUCHECKGOTO(cuMemCreate(ucHandle, size, &ucprop, 0), ret, fail);
CUCHECKGOTO(cuMemMap((CUdeviceptr)*ucptr, size, 0, *ucHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess((CUdeviceptr)*ucptr, size, desc, 1), ret, fail);
CUDACHECKGOTO(cudaMemset(*ucptr, 0, size), ret, fail);
CUCHECKGOTO(cuMemCreate(ucHandle, ucsize, &ucprop, 0), ret, fail);
CUCHECKGOTO(cuMemMap((CUdeviceptr)*ucptr, ucsize, 0, *ucHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess((CUdeviceptr)*ucptr, ucsize, desc, 1), ret, fail);
CUDACHECKGOTO(cudaMemset(*ucptr, 0, ucsize), ret, fail);
// intra-node barrier to mitigate the possible hang in cuMulticastBindMem during abort
NCCLCHECKGOTO(bootstrapIntraNodeBarrier(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, comm->localRankToRank[0]), ret, fail);
// Bind physical memory to the Multicast group
// NB: It will block until all ranks have been added to the Group
CUCHECKGOTO(cuMulticastBindMem(*mcHandle, 0/*mcOffset*/, *ucHandle, 0/*memOffset*/, size, 0/*flags*/), ret, fail);
CUCHECKGOTO(cuMulticastBindMem(*mcHandle, 0/*mcOffset*/, *ucHandle, 0/*memOffset*/, ucsize, 0/*flags*/), ret, fail);
// Map mc virtual address
CUCHECKGOTO(cuMemAddressReserve((CUdeviceptr*)mcptr, size, mcgran, 0U, 0), ret, fail);
CUCHECKGOTO(cuMemMap((CUdeviceptr)*mcptr, size, 0, *mcHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess((CUdeviceptr)*mcptr, size, desc, 1), ret, fail);
INFO(NCCL_NVLS, "NVLS rank %d (dev %d) alloc done, ucptr %p ucgran %ld mcptr %p mcgran %ld size %ld (%ld)", comm->rank, comm->cudaDev, *ucptr, ucgran, *mcptr, mcgran, size, originSize);
CUCHECKGOTO(cuMemAddressReserve((CUdeviceptr*)mcptr, mcsize, mcgran, 0U, 0), ret, fail);
CUCHECKGOTO(cuMemMap((CUdeviceptr)*mcptr, mcsize, 0, *mcHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess((CUdeviceptr)*mcptr, mcsize, desc, 1), ret, fail);
*ucsizePtr = ucsize;
*mcsizePtr = mcsize;
INFO(NCCL_NVLS, "NVLS rank %d (dev %d) alloc done, ucptr %p ucgran %ld mcptr %p mcgran %ld ucsize %ld mcsize %ld (inputsize %ld)", comm->rank, comm->cudaDev, *ucptr, ucgran, *mcptr, mcgran, ucsize, mcsize, size);
exit:
return ret;
@@ -273,6 +278,7 @@ ncclResult_t ncclNvlsBufferSetup(struct ncclComm* comm) {
size_t nvlsTotalSize = 0;
struct ncclNvlsSharedRes* resources = NULL;
int nChannels = -1;
cudaStream_t deviceStream, hostStream;
if (comm->nvlsSupport == 0 || comm->nvlsResources->inited) return ncclSuccess;
// initialize after checking comm->nvlsSupport
@@ -288,10 +294,10 @@ ncclResult_t ncclNvlsBufferSetup(struct ncclComm* comm) {
INFO(NCCL_INIT | NCCL_NVLS, "NVLS comm %p headRank %d nHeads %d buffSize %zu nvlsPerRankSize %zu nvlsTotalSize %zu",
comm, headRank, nHeads, buffSize, nvlsPerRankSize, nvlsTotalSize);
NCCLCHECKGOTO(nvlsAllocateMem(comm, CU_MULTICAST_GRANULARITY_RECOMMENDED, &resources->accessDesc, &nvlsTotalSize, &resources->ucBuffHandle, &resources->mcBuffHandle, (void**)&resources->ucBuff, (void**)&resources->mcBuff), res, fail);
resources->buffSize = nvlsTotalSize;
NCCLCHECKGOTO(nvlsAllocateMem(comm, &resources->accessDesc, nvlsTotalSize, &resources->ucBuffHandle, &resources->mcBuffHandle, (void**)&resources->ucBuff, (void**)&resources->mcBuff, &resources->buffUCSize, &resources->buffMCSize), res, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquireUncaptured(&comm->sharedRes->hostStream), res, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->hostStream, /*concurrent=*/false, &hostStream), res, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), res, fail);
for (int h = 0; h < nHeads; h++) {
int nvlsPeer = comm->nRanks + 1 + h;
for (int c = 0; c < nChannels; c++) {
@@ -306,15 +312,16 @@ ncclResult_t ncclNvlsBufferSetup(struct ncclComm* comm) {
peer->recv[1].conn.buffs[NCCL_PROTO_SIMPLE] = resources->ucBuff + ((h * 2 + 1) * nChannels + c) * buffSize;
peer->send[0].conn.buffs[NCCL_PROTO_SIMPLE] = resources->mcBuff + ((h * 2 + 1) * nChannels + c) * buffSize;
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[0], &peer->send[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[0], &peer->recv[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[1], &peer->send[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[1], &peer->recv[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[0], &peer->send[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[0], &peer->recv[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[1], &peer->send[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[1], &peer->recv[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
}
}
NCCLCHECKGOTO(ncclStrongStreamWaitStream(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, &comm->sharedRes->hostStream), res, fail);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->hostStream), res, fail);
NCCLCHECKGOTO(ncclStreamWaitStream(deviceStream, hostStream, comm->sharedRes->scratchEvent), res, fail);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false), res, fail);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->hostStream, /*concurrent=*/false), res, fail);
// For now, the barrier is a must that guarantees all buffers are mc-mapped before accessing peer's buffer
NCCLCHECKGOTO(bootstrapIntraNodeBarrier(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, comm->localRankToRank[0]), res, fail);
comm->nvlsResources->inited = true;
@@ -374,6 +381,7 @@ setup:
size_t memSize = 64;
size_t creditSize = nChannels * 2 * memSize * nHeads;
int nvlsStepSize = comm->nvlsChunkSize;
cudaStream_t hostStream, deviceStream;
NCCLCHECKGOTO(ncclCalloc(&comm->nvlsResources, 1), res, fail);
comm->nvlsResources->inited = false;
@@ -398,11 +406,11 @@ setup:
resources->accessDesc.location.id = comm->cudaDev;
resources->dev = comm->cudaDev;
NCCLCHECKGOTO(nvlsAllocateMem(comm, CU_MULTICAST_GRANULARITY_MINIMUM, &resources->accessDesc, &creditSize, &resources->ucCreditHandle, &resources->mcCreditHandle, (void**)&resources->ucCredit, (void**)&resources->mcCredit), res, fail);
resources->creditSize = creditSize;
NCCLCHECKGOTO(nvlsAllocateMem(comm, &resources->accessDesc, creditSize, &resources->ucCreditHandle, &resources->mcCreditHandle, (void**)&resources->ucCredit, (void**)&resources->mcCredit, &resources->creditUCSize, &resources->creditMCSize), res, fail);
// Set up head and tail only for now
NCCLCHECKGOTO(ncclStrongStreamAcquireUncaptured(&comm->sharedRes->hostStream), res, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->hostStream, /*concurrent=*/false, &hostStream), res, fail);
NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), res, fail);
for (int h = 0; h < nHeads; h++) {
int nvlsPeer = comm->nRanks + 1 + h;
for (int c = 0; c < nChannels; c++) {
@@ -440,14 +448,15 @@ setup:
peer->send[0].conn.stepSize = nvlsStepSize;
peer->send[0].conn.flags |= NCCL_NVLS_MIN_POLL;
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[0], &peer->send[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[0], &peer->recv[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[1], &peer->send[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[1], &peer->recv[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, comm->sharedRes->hostStream.cudaStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[0], &peer->send[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[0], &peer->recv[0].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->send[1], &peer->send[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
CUDACHECKGOTO(cudaMemcpyAsync(&comm->channels[c].devPeersHostPtr[nvlsPeer]->recv[1], &peer->recv[1].conn, sizeof(struct ncclConnInfo), cudaMemcpyHostToDevice, hostStream), res, fail);
}
}
NCCLCHECKGOTO(ncclStrongStreamWaitStream(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, &comm->sharedRes->hostStream), res, fail);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->hostStream), res, fail);
NCCLCHECKGOTO(ncclStreamWaitStream(deviceStream, hostStream, comm->sharedRes->scratchEvent), res, fail);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->hostStream, /*concurrent=*/false), res, fail);
NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(), &comm->sharedRes->deviceStream, /*concurrent=*/false), res, fail);
}
// MNNVL does not support NVLS buffer registration
@@ -488,13 +497,13 @@ ncclResult_t ncclNvlsFree(struct ncclComm* comm) {
NCCLCHECK(ncclShmClose(resources->nvlsShmemHandle));
if (resources->ucCredit || resources->mcCredit) {
NCCLCHECK(nvlsGroupUnbind(comm, resources->creditSize, &resources->mcCreditHandle));
NCCLCHECK(nvlsGroupUnmapMem(comm, resources->creditSize, resources->ucCredit, &resources->ucCreditHandle, resources->mcCredit, &resources->mcCreditHandle));
NCCLCHECK(nvlsGroupUnbind(comm, resources->creditUCSize, &resources->mcCreditHandle));
NCCLCHECK(nvlsGroupUnmapMem(comm, resources->creditUCSize, resources->ucCredit, &resources->ucCreditHandle, resources->creditMCSize, resources->mcCredit, &resources->mcCreditHandle));
}
if (comm->nvlsResources->inited) {
NCCLCHECK(nvlsGroupUnbind(comm, resources->buffSize, &resources->mcBuffHandle));
NCCLCHECK(nvlsGroupUnmapMem(comm, resources->buffSize, resources->ucBuff, &resources->ucBuffHandle, resources->mcBuff, &resources->mcBuffHandle));
NCCLCHECK(nvlsGroupUnbind(comm, resources->buffUCSize, &resources->mcBuffHandle));
NCCLCHECK(nvlsGroupUnmapMem(comm, resources->buffUCSize, resources->ucBuff, &resources->ucBuffHandle, resources->buffMCSize, resources->mcBuff, &resources->mcBuffHandle));
}
free(resources);
comm->nvlsResources = NULL;
@@ -513,7 +522,7 @@ ncclResult_t tryRegisterBuffer(struct ncclComm *comm, uintptr_t userBuff, size_t
size_t minSize = SIZE_MAX;
struct localRegData* regData = NULL;
cudaPointerAttributes attr;
size_t ucgran, mcgran;
size_t ucgran, mcgran, ucsize, mcsize;
NCCLCHECKGOTO(ncclCalloc(&regData, comm->localRanks), ret, fail);
@@ -538,13 +547,12 @@ ncclResult_t tryRegisterBuffer(struct ncclComm *comm, uintptr_t userBuff, size_t
CUCHECKGOTO(cuMemGetAllocationGranularity(&ucgran, &ucprop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED), ret, fail);
CUCHECKGOTO(cuMemGetAddressRange((CUdeviceptr*)&regRecord->baseAddr, &regRecord->baseSize, (CUdeviceptr)regRecord->addr), ret, fail);
if (regSize % mcgran == 0) {
regRecord->regSize = regSize;
} else {
regRecord->regSize = regRecord->baseSize - (regRecord->addr - regRecord->baseAddr);
}
if (regRecord->addr % ucgran == 0 && regRecord->regSize % mcgran == 0) {
if (regRecord->addr % ucgran == 0) {
if (regSize % ucgran != 0) {
regRecord->regUCSize = ALIGN_SIZE(regSize, ucgran);
} else {
regRecord->regUCSize = regSize;
}
regRecord->state |= NVLS_REG_POSSIBLE;
memcpy(&regData[comm->localRank].reg, regRecord, sizeof(struct ncclReg));
regData[comm->localRank].offset = userBuff - regRecord->addr;
@@ -564,13 +572,17 @@ ncclResult_t tryRegisterBuffer(struct ncclComm *comm, uintptr_t userBuff, size_t
goto fail;
}
/* get minimal reg size of nvls buffers */
if (minSize > regData[i].reg.regSize)
minSize = regData[i].reg.regSize;
if (minSize > regData[i].reg.regUCSize)
minSize = regData[i].reg.regUCSize;
}
/* start registration */
mcsize = ucsize = minSize;
mcprop.size = minSize;
CUCHECKGOTO(cuMulticastGetGranularity(&mcgran, &mcprop, CU_MULTICAST_GRANULARITY_RECOMMENDED), ret, fail);
ALIGN_SIZE(mcsize, mcgran);
mcprop.size = mcsize;
if (comm->localRank == 0) {
NCCLCHECKGOTO(nvlsGroupCreate(comm, &mcprop, comm->localRank, comm->localRanks, &mcHandle, shareableHandle), ret, fail);
NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, comm->localRankToRank, comm->localRank, comm->localRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail);
@@ -583,16 +595,17 @@ ncclResult_t tryRegisterBuffer(struct ncclComm *comm, uintptr_t userBuff, size_t
// Coverity complains that regRecord could be NULL. That won't in practice be the case because we've already checked
// (regData[i].reg.state & NVLS_REG_POSSIBLE) of all local ranks, which would catch it and bail out.
// coverity[var_deref_op]
CUCHECKGOTO(cuMulticastBindAddr(mcHandle, 0, (CUdeviceptr)regRecord->addr, minSize, 0), ret, fail);
CUCHECKGOTO(cuMulticastBindAddr(mcHandle, 0, (CUdeviceptr)regRecord->addr, ucsize, 0), ret, fail);
// Create a VA for the NVLS
CUCHECKGOTO(cuMemAddressReserve(&regPtr, minSize, mcgran, 0U, 0), ret, fail);
CUCHECKGOTO(cuMemAddressReserve(&regPtr, mcsize, mcgran, 0U, 0), ret, fail);
// Map the VA locally
CUCHECKGOTO(cuMemMap(regPtr, minSize, 0, mcHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess(regPtr, minSize, &comm->nvlsResources->accessDesc, 1), ret, fail);
CUCHECKGOTO(cuMemMap(regPtr, mcsize, 0, mcHandle, 0), ret, fail);
CUCHECKGOTO(cuMemSetAccess(regPtr, mcsize, &comm->nvlsResources->accessDesc, 1), ret, fail);
regRecord->regAddr = regPtr;
regRecord->regSize = minSize;
regRecord->regUCSize = ucsize;
regRecord->regMCSize = mcsize;
regRecord->dev = comm->nvlsResources->dev;
regRecord->mcHandle = mcHandle;
regRecord->state |= NVLS_REG_COMPLETE;
@@ -706,7 +719,7 @@ exit:
return ncclSuccess;
fail:
regBufUsed = 0;
WARN("rank %d failed to NVLS register sendbuff %p sendbuffSize %ld recvbuff %p recvbuffSize %ld", comm->rank, sendbuff, sendbuffSize, recvbuff, recvbuffSize);
INFO(NCCL_REG, "rank %d failed to NVLS register sendbuff %p sendbuffSize %ld recvbuff %p recvbuffSize %ld", comm->rank, sendbuff, sendbuffSize, recvbuff, recvbuffSize);
goto exit;
}
@@ -843,7 +856,7 @@ ncclResult_t ncclNvlsLocalRegisterBuffer(struct ncclComm *comm, const void *send
return ncclSuccess;
}
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t size) {
ncclResult_t ncclNvlsDeregBuffer(struct ncclComm* comm, CUmemGenericAllocationHandle *mcHandler, CUdeviceptr ptr, int dev, size_t ucsize, size_t mcsize) {
return ncclSuccess;
}

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше