AI/rocm-systems
2
0
انشعاب 0
کد مسائل تقاضاهای واکشی Actions Packages پروژه‌ها انتشارها دانشنامه فعالیت

2.19.4-1

فهرست منبع 
Split transport connect phase into multiple steps to avoid port
exhaustion when connecting alltoall at large scale. Defaults to 128
peers per round.
Fix memory leaks on CUDA graph capture.
Fix alltoallv crash on self-sendrecv.
Make topology detection more deterministic when PCI speeds are not
available (fix issue #1020).
Properly close shared memory in NVLS resources.
Revert proxy detach after 5 seconds.
Add option to print progress during transport connect.
Add option to set NCCL_DEBUG to INFO on first WARN.
This commit is contained in:
Sylvain Jeaugey
2023-11-13 10:26:55 -08:00
والد 0e35f5d390
کامیت 88d44d777f
11فایلهای تغییر یافته به همراه174 افزوده شده و 183 حذف شده
دانلود پرونده وصله دانلود فایل تغییرات diff Expand all files Collapse all files
src/proxy.cc
+16 -81
مشاهده پرونده
@@ -18,14 +18,6 @@
#include <unistd.h>
#include <sys/time.h>
#define PROGRESS_RUNNING 0
#define PROGRESS_REQUEST_STOP 1
#define PROGRESS_ABORT 2
#define PROGRESS_COMPLETE 3
#define SERVICE_RUNNING 0
#define SERVICE_COMPLETE 1
enum { proxyRecv=0, proxySend=1 };
static bool NeedProxy(int type, int pattern, int root, struct ncclRing* ring, int nranks) {
@@ -720,13 +712,13 @@ static ncclResult_t ncclProxyGetPostedOps(struct ncclProxyState* proxyState, int
if (state->active == NULL) {
pthread_mutex_lock(&pool->mutex);
while (pool->nextOps == -1 && state->stop == PROGRESS_RUNNING) {
while (pool->nextOps == -1 && !state->stop) {
struct ncclProxyArgs profArgs; // Only used for profiling purposes
ncclProfilingRecord(&profArgs, 0, 0, ncclProxyProfileSleep);
pthread_cond_wait(&pool->cond, &pool->mutex);
ncclProfilingRecord(&profArgs, 0, 0, ncclProxyProfileWakeup);
}
if (state->stop != PROGRESS_RUNNING) { // We might have been woken up to stop.
if (state->stop) { // We might have been woken up to stop.
pthread_mutex_unlock(&pool->mutex);
return ncclSuccess;
}
@@ -864,7 +856,7 @@ void* ncclProxyProgress(void *proxyState_) {
* frequency of calling ncclProxyGetPostedOps() and reduce the perf impact. */
int proxyOpAppendCounter = 0;
struct ncclProxyArgs profArgs; // Only used for profiling purposes
while (state->stop == PROGRESS_RUNNING || (state->stop == PROGRESS_REQUEST_STOP && state->active)) {
while ((state->stop == 0 || (state->stop == 1 && state->active)) && *proxyState->abortFlag == 0) {
int idle = 1;
ncclResult_t ret = progressOps(proxyState, state, state->active, &idle);
if (ret != ncclSuccess) {
@@ -878,7 +870,7 @@ void* ncclProxyProgress(void *proxyState_) {
int added = 0;
proxyOpAppendCounter = 0;
TIME_START(3);
if (state->stop == PROGRESS_RUNNING)
if (state->stop == 0)
ret = ncclProxyGetPostedOps(proxyState, &added);
if (added) { TIME_STOP(3); } else { TIME_CANCEL(3); }
if (ret != ncclSuccess) {
@@ -891,9 +883,6 @@ void* ncclProxyProgress(void *proxyState_) {
}
lastIdle = idle;
}
/* progress serive thread should be waiting for me, I need to notify it. */
__atomic_store_n(&state->stop, PROGRESS_COMPLETE, __ATOMIC_RELEASE);
return NULL;
}
@@ -916,11 +905,7 @@ ncclResult_t ncclProxyStart(struct ncclComm* comm) {
static ncclResult_t ncclProxyProgressCreate(struct ncclProxyState* proxyState) {
struct ncclProxyProgressState* state = &proxyState->progressState;
if (!state->thread) {
pthread_attr_t attr;
SYSCHECK(pthread_attr_init(&attr), "pthread_attr_init");
SYSCHECK(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED), "pthread_attr_setdetachstate");
SYSCHECK(pthread_create(&state->thread, &attr, ncclProxyProgress, proxyState), "pthread_create");
SYSCHECK(pthread_attr_destroy(&attr), "pthread_attr_destroy");
pthread_create(&state->thread, NULL, ncclProxyProgress, proxyState);
ncclSetThreadName(state->thread, "NCCL Progress%2d", proxyState->tpLocalnRanks);
}
return ncclSuccess;
@@ -932,17 +917,10 @@ ncclResult_t ncclProxyProgressDestroy(struct ncclProxyState* proxyState) {
// Request the proxy to stop and then wake it
if (state->opsPool) {
pthread_mutex_lock(&state->opsPool->mutex);
if (*proxyState->abortFlag == 0)
state->stop = PROGRESS_REQUEST_STOP;
else
state->stop = PROGRESS_ABORT;
state->stop = 1;
pthread_cond_signal(&state->opsPool->cond);
pthread_mutex_unlock(&state->opsPool->mutex);
/* progress thread is always detached, wait for it to exit. */
uint64_t t0 = clockNano();
while (__atomic_load_n(&state->stop, __ATOMIC_ACQUIRE) != PROGRESS_COMPLETE) {
if (clockNano() - t0 >= 1000) sched_yield();
}
pthread_join(state->thread, NULL);
}
// Free off any memory allocated for the proxy arg pools
@@ -1582,19 +1560,6 @@ void* ncclProxyService(void* _args) {
ncclSocketClose(proxyState->listenSock);
free(proxyState->listenSock);
proxyOpsFree(proxyState);
if (*proxyState->abortFlag) {
/* abort happened, need to notify main thread I am done. */
__atomic_store_n(&proxyState->stop, SERVICE_COMPLETE, __ATOMIC_RELEASE);
}
if (ncclAtomicRefCountDecrement(proxyState->abortFlagRefCount) == 0) {
ncclCudaHostFree((void *)proxyState->abortFlag);
free((void*)proxyState->abortFlagRefCount);
}
/* proxy itself holds one internal ref count, needs to call ncclProxyDestroy */
ncclProxyDestroy(proxyState);
return NULL;
}
@@ -1603,8 +1568,6 @@ ncclResult_t ncclProxyInit(struct ncclComm* comm, struct ncclSocket* sock, union
NCCLCHECK(ncclCalloc(&comm->sharedRes->proxyState, 1));
comm->proxyState = comm->sharedRes->proxyState;
comm->proxyState->refCount = 1;
/* ref count for communicator and proxy service thread. */
comm->proxyState->internalRefCount = 2;
comm->proxyState->listenSock = sock;
comm->proxyState->peerAddresses = peerAddresses;
// Seed the random number generator for UDS filename generation
@@ -1627,8 +1590,6 @@ ncclResult_t ncclProxyCreate(struct ncclComm* comm) {
proxyState->tpLocalnRanks = comm->localRanks;
proxyState->cudaDev = comm->cudaDev;
proxyState->abortFlag = comm->abortFlag;
proxyState->abortFlagRefCount = comm->abortFlagRefCount;
ncclAtomicRefCountIncrement(comm->abortFlagRefCount);
proxyState->p2pnChannels = comm->p2pnChannels;
proxyState->p2pChunkSize = comm->p2pChunkSize;
proxyState->nChannels = comm->nChannels;
@@ -1686,41 +1647,15 @@ ncclResult_t ncclProxyStop(struct ncclComm* comm) {
return ncclSuccess;
}
ncclResult_t ncclProxyDestroy(struct ncclProxyState *proxyState) {
if (__atomic_sub_fetch(&proxyState->internalRefCount, 1, __ATOMIC_ACQ_REL) == 0) {
free(proxyState->peerAddresses);
free(proxyState->peerSocks);
free(proxyState->proxyOps);
free(proxyState->sharedDevMems);
expectedProxyResponseFree(proxyState);
free(proxyState);
}
return ncclSuccess;
}
ncclResult_t ncclProxyDestroy(struct ncclComm* comm) {
struct ncclProxyState* sharedProxyState = comm->sharedRes->proxyState;
/* detach all proxy threads in case of abort */
ncclResult_t ncclProxyTryDetach(struct ncclProxyState *proxyState) {
if (proxyState && proxyState->thread) {
/* proxy service thread can call cudaFreeHost to free pinned host mem, but
* it can cause a hang if main thread is issuing other cuda calls. To solution
* should be allocate/free pinned host mem using cuMem* driver API, this waiting
* 5 secs is just a workaround for now. */
bool join = false;
struct timespec start, now;
clock_gettime(CLOCK_MONOTONIC, &start);
do {
clock_gettime(CLOCK_MONOTONIC, &now);
if (__atomic_load_n(&proxyState->stop, __ATOMIC_ACQUIRE) == SERVICE_COMPLETE) {
/* proxy thread is done, join it. */
pthread_join(proxyState->thread, NULL);
join = true;
break;
}
} while(now.tv_sec - start.tv_sec < 5);
if (join == false) {
pthread_detach(proxyState->thread);
}
}
assert(sharedProxyState->refCount == 0);
free(sharedProxyState->peerAddresses);
free(sharedProxyState->peerSocks);
free(sharedProxyState->proxyOps);
free(sharedProxyState->sharedDevMems);
expectedProxyResponseFree(sharedProxyState);
free(sharedProxyState);
return ncclSuccess;
}