AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'nccl/master' into develop

Browse Source 
[ROCm/rccl commit: a6bf9bfc9e]
This commit is contained in:
BertanDogancay
2025-04-23 20:46:36 -07:00
parent 07620c7efd 8f0723e904
commit d045d0ca23
118 changed files with 12789 additions and 3358 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/misc/cudawrap.cc
+26 -11
View File
@@ -11,7 +11,7 @@
// This env var (NCCL_CUMEM_ENABLE) toggles cuMem API usage
NCCL_PARAM(CuMemEnable, "CUMEM_ENABLE", -2);
NCCL_PARAM(CuMemHostEnable, "CUMEM_HOST_ENABLE", 0);
NCCL_PARAM(CuMemHostEnable, "CUMEM_HOST_ENABLE", -1);
// Handle type used for cuMemCreate()
CUmemAllocationHandleType ncclCuMemHandleType = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR;
@@ -35,9 +35,6 @@ int ncclIsCuMemSupported() {
// Query device to see if CUMEM VMM support is available
CUCHECKGOTO(cuDeviceGetAttribute(&flag, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, currentDev), ret, error);
if (!flag) return 0;
// Query device to see if CUMEM RDMA support is available
CUCHECKGOTO(cuDeviceGetAttribute(&flag, CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED, currentDev), ret, error);
if (!flag) return 0;
error:
return (ret == ncclSuccess);
#endif
@@ -49,11 +46,31 @@ int ncclCuMemEnable() {
return param >= 0 ? param : (param == -2 && ncclCuMemSupported);
}
static int ncclCumemHostEnable = -1;
int ncclCuMemHostEnable() {
if (ncclCumemHostEnable != -1)
return ncclCumemHostEnable;
#if CUDART_VERSION < 12020
return 0;
ncclCumemHostEnable = 0;
return ncclCumemHostEnable;
#else
return ncclParamCuMemHostEnable();
ncclResult_t ret = ncclSuccess;
int cudaDriverVersion;
int paramValue = -1;
CUDACHECKGOTO(cudaDriverGetVersion(&cudaDriverVersion), ret, error);
if (cudaDriverVersion < 12020) {
ncclCumemHostEnable = 0;
}
else {
paramValue = ncclParamCuMemHostEnable();
if (paramValue != -1)
ncclCumemHostEnable = paramValue;
else
ncclCumemHostEnable = (cudaDriverVersion >= 12060) ? 1 : 0;
}
return ncclCumemHostEnable;
error:
return (ret == ncclSuccess);
#endif
}
@@ -218,10 +235,9 @@ static void initOnceFunc() {
// Determine whether we support the cuMem APIs or not
ncclCuMemSupported = ncclIsCuMemSupported();
#if 12020 <= CUDART_VERSION && CUDART_VERSION <= 12030
/* To use cuMem* for host memory allocation, we need to create context on each
* visible device. This is workaround needed in CUDA 12.3 which is fixed in 12.4. */
if (ncclCuMemSupported && ncclCuMemHostEnable()) {
/* To use cuMem* for host memory allocation, we need to create context on each visible device.
* This is a workaround needed in CUDA 12.2 and CUDA 12.3 which is fixed in 12.4. */
if (ncclCuMemSupported && ncclCuMemHostEnable() && 12020 <= driverVersion && driverVersion <= 12030) {
int deviceCnt, saveDevice;
cudaGetDevice(&saveDevice);
cudaGetDeviceCount(&deviceCnt);
@@ -231,7 +247,6 @@ static void initOnceFunc() {
}
cudaSetDevice(saveDevice);
}
#endif
initResult = ret;
return;
error:
projects/rccl/src/misc/ibvwrap.cc
+91 -3
View File
@@ -8,6 +8,7 @@
#include <sys/types.h>
#include <unistd.h>
#include "ibvcore.h"
#include "ibvsymbols.h"
static pthread_once_t initOnceControl = PTHREAD_ONCE_INIT;
@@ -53,7 +54,7 @@ ncclResult_t wrap_ibv_symbols(void) {
} \
int ret = container.call; \
if (ret == ENOTSUP || ret == EOPNOTSUPP) { \
INFO(NCCL_NET, "Call to " name " failed with error %s errno %d", strerror(ret), ret); \
INFO(NCCL_NET, "Call to " name " not supported"); \
*supported = 0; \
return ncclSuccess; \
} else if (ret != success_retval) { \
@@ -87,6 +88,14 @@ ncclResult_t wrap_ibv_symbols(void) {
container.call; \
return ncclSuccess;
NCCL_PARAM(IbMQpRetryAll, "IB_MQP_RETRY_ALL", 0);
NCCL_PARAM(IbMQpRetryCnt, "IB_MQP_RETRY_CNT", 34);
NCCL_PARAM(IbMQpRetryTimeout, "IB_MQP_RETRY_SLEEP_MSEC", 100); // in milliseconds
#define IBV_ERR_EQ(e, code) (e == code || e == (-code))
#define IBV_MQP_RETRY_ERRNO(e) (IBV_ERR_EQ(e, ETIMEDOUT))
#define IBV_MQP_RETRY_ERRNO_ALL(e) (ncclParamIbMQpRetryAll() ? (e != 0) : IBV_MQP_RETRY_ERRNO(e))
ncclResult_t wrap_ibv_fork_init() {
IBV_INT_CHECK(ibvSymbols, ibv_internal_fork_init, ibv_internal_fork_init(), -1, "ibv_fork_init");
}
@@ -202,8 +211,87 @@ ncclResult_t wrap_ibv_create_qp(struct ibv_qp **ret, struct ibv_pd *pd, struct i
IBV_PTR_CHECK_ERRNO(ibvSymbols, ibv_internal_create_qp, ibv_internal_create_qp(pd, qp_init_attr), *ret, NULL, "ibv_create_qp");
}
ncclResult_t wrap_ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) { /*returns 0 on success, or the value of errno on failure (which indicates the failure reason)*/
IBV_INT_CHECK_RET_ERRNO(ibvSymbols, ibv_internal_modify_qp, ibv_internal_modify_qp(qp, attr, attr_mask), 0, "ibv_modify_qp");
static void ibvQpStateName(enum ibv_qp_state state, char* msg, const size_t len) {
switch (state) {
case (IBV_QPS_RESET): snprintf(msg, len, "RESET"); break;
case (IBV_QPS_INIT): snprintf(msg, len, "INIT"); break;
case (IBV_QPS_RTR): snprintf(msg, len, "RTR"); break;
case (IBV_QPS_RTS): snprintf(msg, len, "RTS"); break;
case (IBV_QPS_SQD): snprintf(msg, len, "SQD"); break;
case (IBV_QPS_SQE): snprintf(msg, len, "SQE"); break;
case (IBV_QPS_ERR): snprintf(msg, len, "ERR"); break;
case (IBV_QPS_UNKNOWN): snprintf(msg, len, "UNKNOWN"); break;
default: snprintf(msg, len, "NOT RECOGNIZED (%d)", state); break;
}
}
#define QP_ATTR(attr, userAttr, userFlag, mask) ((userFlag & mask) ? (userAttr) : (attr))
static void ibvModifyQpLog(struct ibv_qp* qp, enum ibv_qp_state qpState, struct ibv_qp_attr* userAttr, int userFlag, char* msg, size_t msgLen) {
ncclResult_t res;
int portNum = -1, gidIndex = -1;
char localGidName[INET6_ADDRSTRLEN], remoteGidName[INET6_ADDRSTRLEN];
const char *localGidRes = NULL, *remoteGidRes = NULL;
char nextState[32], currState[32];
ibvQpStateName(qp->state, currState, sizeof(currState));
ibvQpStateName(qpState, nextState, sizeof(nextState));
char devName[IBV_SYSFS_NAME_MAX] = "";
snprintf(devName, sizeof(devName), "%s", (qp->pd->context) ? wrap_ibv_get_device_name(qp->pd->context->device) : "N/A");
struct ibv_qp_attr attr;
struct ibv_qp_init_attr init_attr;
int attr_mask = IBV_QP_PORT | IBV_QP_AV;
res = wrap_ibv_query_qp(qp, &attr, attr_mask, &init_attr);
struct ibv_qp_attr *qpAttr = (res == ncclSuccess) ? &attr : NULL;
// port info, portAttr can be NULL if not given by the user and query_qp failed
struct ibv_qp_attr *portAttr = QP_ATTR(qpAttr, userAttr, userFlag, IBV_QP_PORT);
portNum = portAttr ? portAttr->port_num : -1;
// address info, avAttr can be NULL if not given by the user and query_qp failed
struct ibv_qp_attr *avAttr = QP_ATTR(qpAttr, userAttr, userFlag, IBV_QP_AV);
if (avAttr && avAttr->ah_attr.is_global) {
union ibv_gid *remoteGid = &avAttr->ah_attr.grh.dgid;
remoteGidRes = ibvGetGidStr(remoteGid, remoteGidName, sizeof(remoteGidName));
// we need pd->context to retrieve local GID, skip if not there
if (!qp->pd->context) goto print;
gidIndex = avAttr->ah_attr.grh.sgid_index;
union ibv_gid localGid;
NCCLCHECKGOTO(wrap_ibv_query_gid(qp->pd->context, portNum, gidIndex, &localGid), res, print);
localGidRes = ibvGetGidStr(&localGid, localGidName, sizeof(localGidName));
}
print:
snprintf(msg, msgLen, "on dev %s:%d, curr state %s, next state %s, local GID index %d, local GID %s, remote GID %s",
devName, portNum, currState, nextState, gidIndex, localGidRes ? localGidName : "N/A", remoteGidRes ? remoteGidName : "N/A");
return;
}
ncclResult_t wrap_ibv_modify_qp(struct ibv_qp* qp, struct ibv_qp_attr* attr, int attr_mask) {
char qpMsg[1024];
int ret = 0, attempts = 0;
int maxCnt = (int)ncclParamIbMQpRetryCnt() + 1; // number of attempts = number of retry + 1
int timeOut = (int)ncclParamIbMQpRetryTimeout();
CHECK_NOT_NULL(ibvSymbols, ibv_internal_modify_qp);
do {
if (attempts > 0) {
unsigned int sleepTime = timeOut * attempts;
ibvModifyQpLog(qp, attr->qp_state, attr, attr_mask, qpMsg, sizeof(qpMsg));
INFO(NCCL_NET, "Call to ibv_modify_qp failed with %d %s, %s, retrying %d/%d after %u msec of sleep", ret, strerror(ret), qpMsg, attempts, maxCnt, sleepTime);
// sleep before retrying
struct timespec tv = {.tv_sec = sleepTime / 1000, .tv_nsec = (sleepTime % 1000) * ((long)1e6)};
nanosleep(&tv, NULL);
}
ret = ibvSymbols.ibv_internal_modify_qp(qp, attr, attr_mask);
attempts++;
} while (IBV_MQP_RETRY_ERRNO_ALL(ret) && attempts < maxCnt);
if (ret != 0) {
ibvModifyQpLog(qp, attr->qp_state, attr, attr_mask, qpMsg, sizeof(qpMsg));
WARN("Call to ibv_modify_qp failed with %d %s, %s", ret, strerror(ret), qpMsg);
return ncclSystemError;
}
return ncclSuccess;
}
ncclResult_t wrap_ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece, int* supported) { /*returns 0 on success, or the value of errno on failure (which indicates the failure reason)*/
projects/rccl/src/misc/ipcsocket.cc
+9 -7
View File
@@ -189,14 +189,16 @@ ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen,
TRACE(NCCL_INIT, "UDS: Sending hdr %p len %d fd %d to UDS socket %s", hdr, hdrLen, sendFd, temp);
msg.msg_control = control_un.control;
msg.msg_controllen = sizeof(control_un.control);
if (sendFd != -1) {
msg.msg_control = control_un.control;
msg.msg_controllen = sizeof(control_un.control);
cmptr = CMSG_FIRSTHDR(&msg);
cmptr->cmsg_len = CMSG_LEN(sizeof(int));
cmptr->cmsg_level = SOL_SOCKET;
cmptr->cmsg_type = SCM_RIGHTS;
memmove(CMSG_DATA(cmptr), &sendFd, sizeof(sendFd));
cmptr = CMSG_FIRSTHDR(&msg);
cmptr->cmsg_len = CMSG_LEN(sizeof(int));
cmptr->cmsg_level = SOL_SOCKET;
cmptr->cmsg_type = SCM_RIGHTS;
memmove(CMSG_DATA(cmptr), &sendFd, sizeof(sendFd));
}
msg.msg_name = (void *)&cliaddr;
msg.msg_namelen = sizeof(struct sockaddr_un);
projects/rccl/src/misc/msccl/msccl_setup.cc
+3 -4
View File
@@ -112,9 +112,8 @@ ncclResult_t mscclSetupConnections(struct mscclAlgo* hostAlgo, ncclComm_t comm)
// Connect MSCCL connections
mscclSetIsCallerFlag();
int highestTransportType = TRANSPORT_P2P;
bool needsProxy = false;
NCCLCHECK(ncclTransportP2pSetup(comm, NULL, 0, &highestTransportType, &needsProxy));
NCCLCHECK(ncclTransportP2pSetup(comm, NULL, 0, &needsProxy));
status.needsProxy |= needsProxy;
mscclClearIsCallerFlag();
@@ -273,11 +272,11 @@ static ncclResult_t hostToDevRedOp(
break;
#endif
#if defined(RCCL_FLOAT8)
case ncclFp8E4M3:
case ncclFloat8e4m3:
opFull->op = ncclDevPreMulSum;
fp8_e4m3 = (rccl_float8)(float(1.0/comm->nRanks));
break;
case ncclFp8E5M2:
case ncclFloat8e5m2:
opFull->op = ncclDevPreMulSum;
fp8_e5m2 = (rccl_bfloat8)(float(1.0/comm->nRanks));
break;
projects/rccl/src/misc/nvmlwrap.cc
+4 -4
View File
@@ -311,19 +311,19 @@ ncclResult_t ncclNvmlGetCCStatus(struct ncclNvmlCCStatus *status) {
status->CCEnabled = false;
if (ccInfo.settingV12040.multiGpuMode == NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE)
status->multiGpuCCEnabled = true;
status->multiGpuProtectedPCIE = true;
else
status->multiGpuCCEnabled = false;
status->multiGpuProtectedPCIE = false;
} else if (pfn_nvmlSystemGetConfComputeState != NULL) {
NVMLTRY(nvmlSystemGetConfComputeState, &ccInfo.settingV12020);
if (ccInfo.settingV12020.ccFeature == NVML_CC_SYSTEM_FEATURE_ENABLED)
status->CCEnabled = true;
else
status->CCEnabled = false;
status->multiGpuCCEnabled = false;
status->multiGpuProtectedPCIE = false;
} else {
status->CCEnabled = false;
status->multiGpuCCEnabled = false;
status->multiGpuProtectedPCIE = false;
}
return ncclSuccess;
}
projects/rccl/src/misc/profiler.cc
+165 -55
View File
@@ -17,9 +17,110 @@ static pthread_mutex_t profilerLock = PTHREAD_MUTEX_INITIALIZER;
static int profilerPluginRefCount;
static void* profilerPluginLib;
static ncclProfiler_t* ncclProfiler;
static ncclProfiler_v2_t ncclProfiler_v1_as_v2;
static ncclProfiler_v1_t* ncclProfiler_v1;
static uint8_t ncclStringToFunc(const char* func) {
if (0 == strcmp(func, "AllGather")) return ncclFuncAllGather;
if (0 == strcmp(func, "AllReduce")) return ncclFuncAllReduce;
if (0 == strcmp(func, "Broadcast")) return ncclFuncBroadcast;
if (0 == strcmp(func, "Recv")) return ncclFuncRecv;
if (0 == strcmp(func, "Reduce")) return ncclFuncReduce;
if (0 == strcmp(func, "ReduceScatter")) return ncclFuncReduceScatter;
if (0 == strcmp(func, "SendRecv")) return ncclFuncSendRecv;
return ncclFuncSend;
}
static uint8_t ncclStringToAlgo(const char* algo) {
if (0 == strcmp(algo, "TREE")) return NCCL_ALGO_TREE;
if (0 == strcmp(algo, "RING")) return NCCL_ALGO_RING;
if (0 == strcmp(algo, "COLLNET_DIRECT")) return NCCL_ALGO_COLLNET_DIRECT;
if (0 == strcmp(algo, "COLLNET_CHAIN")) return NCCL_ALGO_COLLNET_CHAIN;
if (0 == strcmp(algo, "NVLS")) return NCCL_ALGO_NVLS;
if (0 == strcmp(algo, "NVLS_TREE")) return NCCL_ALGO_NVLS_TREE;
return NCCL_ALGO_PAT;
}
static uint8_t ncclStringToProto(const char* proto) {
if (0 == strcmp(proto, "LL")) return NCCL_PROTO_LL;
if (0 == strcmp(proto, "LL128")) return NCCL_PROTO_LL128;
return NCCL_PROTO_SIMPLE;
}
static uint8_t ncclStringToDatatype(const char* dt) {
if (0 == strcmp(dt, "ncclInt8")) return ncclInt8;
if (0 == strcmp(dt, "ncclInt32")) return ncclInt32;
if (0 == strcmp(dt, "ncclUint32")) return ncclUint32;
if (0 == strcmp(dt, "ncclInt64")) return ncclInt64;
if (0 == strcmp(dt, "ncclUint64")) return ncclUint64;
if (0 == strcmp(dt, "ncclFloat16")) return ncclFloat16;
if (0 == strcmp(dt, "ncclFloat32")) return ncclFloat32;
#if defined(__CUDA_BF16_TYPES_EXIST__)
if (0 == strcmp(dt, "ncclBfloat16")) return ncclBfloat16;
#endif
return ncclFloat64;
}
static ncclResult_t ncclProfiler_v1_as_v2_startEvent(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr) {
ncclProfilerEventDescr_v1_t eDescr_v1 = { 0 };
eDescr_v1.type = eDescr->type;
eDescr_v1.parentObj = eDescr->parentObj;
eDescr_v1.rank = eDescr->rank;
switch(eDescr->type) {
case ncclProfileGroup: break;
case ncclProfileColl: {
eDescr_v1.coll.name = eDescr->coll.name;
eDescr_v1.coll.commHash = eDescr->coll.commHash;
eDescr_v1.coll.seqNumber = eDescr->coll.seqNumber;
eDescr_v1.coll.func = ncclStringToFunc(eDescr->coll.func);
eDescr_v1.coll.sendBuff = eDescr->coll.sendBuff;
eDescr_v1.coll.recvBuff = eDescr->coll.recvBuff;
eDescr_v1.coll.count = eDescr->coll.count;
eDescr_v1.coll.root = eDescr->coll.root;
eDescr_v1.coll.datatype = ncclStringToDatatype(eDescr->coll.datatype);
eDescr_v1.coll.op = 0; // removed in v2
eDescr_v1.coll.trafficBytes = eDescr->coll.trafficBytes;
eDescr_v1.coll.nMaxChannels = eDescr->coll.nMaxChannels;
eDescr_v1.coll.nWarps = eDescr->coll.nWarps;
eDescr_v1.coll.algo = ncclStringToAlgo(eDescr->coll.algo);
eDescr_v1.coll.proto = ncclStringToProto(eDescr->coll.proto);
} break;
case ncclProfileP2p: {
eDescr_v1.p2p.name = eDescr->p2p.name;
eDescr_v1.p2p.commHash = eDescr->p2p.commHash;
eDescr_v1.p2p.func = ncclStringToFunc(eDescr->p2p.func);
eDescr_v1.p2p.buff = eDescr->p2p.buff;
eDescr_v1.p2p.count = eDescr->p2p.count;
eDescr_v1.p2p.datatype = ncclStringToDatatype(eDescr->p2p.datatype);
eDescr_v1.p2p.peer = eDescr->p2p.peer;
} break;
case ncclProfileProxyOp: {
eDescr_v1.proxyOp.pid = eDescr->proxyOp.pid;
eDescr_v1.proxyOp.channelId = eDescr->proxyOp.channelId;
eDescr_v1.proxyOp.peer = eDescr->proxyOp.peer;
eDescr_v1.proxyOp.nSteps = eDescr->proxyOp.nSteps;
eDescr_v1.proxyOp.chunkSize = eDescr->proxyOp.chunkSize;
eDescr_v1.proxyOp.isSend = eDescr->proxyOp.isSend;
} break;
case ncclProfileProxyStep: {
eDescr_v1.proxyStep.step = eDescr->proxyStep.step;
} break;
case ncclProfileProxyCtrl: break;
default:;
}
return ncclProfiler_v1->startEvent(context, eHandle, &eDescr_v1);
}
static ncclResult_t ncclProfiler_v1_as_v2_init(void** context, int* eActivationMask) {
ncclProfiler_v1->init(context, eActivationMask);
ncclProfiler_v1_as_v2.startEvent = ncclProfiler_v1_as_v2_startEvent;
ncclProfiler_v1_as_v2.stopEvent = ncclProfiler_v1->stopEvent;
ncclProfiler_v1_as_v2.recordEventState = ncclProfiler_v1->recordEventState;
ncclProfiler_v1_as_v2.finalize = ncclProfiler_v1->finalize;
return ncclSuccess;
}
#define MAX_STR_LEN 256
#define NCCL_PROFILER_PLUGIN_SYMBOL "ncclProfiler_v1"
static void* tryOpenLib(char* name, int *err, char* errStr) {
if (nullptr == name || strlen(name) == 0) {
@@ -34,7 +135,7 @@ static void* tryOpenLib(char* name, int *err, char* errStr) {
if (nullptr == handle) {
strncpy(errStr, dlerror(), MAX_STR_LEN);
errStr[MAX_STR_LEN] = 0;
if (strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
if (name && strstr(errStr, name) && strstr(errStr, "No such file or directory")) {
*err = ENOENT;
}
}
@@ -117,10 +218,21 @@ static ncclResult_t ncclProfilerPluginLoad(void) {
goto fail;
}
ncclProfiler = (ncclProfiler_t*)dlsym(profilerPluginLib, NCCL_PROFILER_PLUGIN_SYMBOL);
ncclProfiler = (ncclProfiler_v2_t*)dlsym(profilerPluginLib, "ncclProfiler_v2");
if (ncclProfiler == nullptr) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find " NCCL_PROFILER_PLUGIN_SYMBOL ".");
goto fail;
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v2.");
ncclProfiler_v1 = (ncclProfiler_v1_t*)dlsym(profilerPluginLib, "ncclProfiler_v1");
if (ncclProfiler_v1 == nullptr) {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: failed to find ncclProfiler_v1.");
goto fail;
} else {
ncclProfiler = &ncclProfiler_v1_as_v2;
ncclProfiler_v1_as_v2.name = ncclProfiler_v1->name;
ncclProfiler_v1_as_v2.init = ncclProfiler_v1_as_v2_init;
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded ncclProfiler_v1.");
}
} else {
INFO(NCCL_INIT|NCCL_ENV, "PROFILER/Plugin: loaded ncclProfiler_v2.");
}
++profilerPluginRefCount;
@@ -248,7 +360,7 @@ ncclResult_t ncclProfilerStartGroupEvent(struct ncclKernelPlan* plan) {
eActivationMaskGroup = __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED);
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (eActivationMaskGroup & (ncclProfileColl | ncclProfileP2p | ncclProfileProxyOp | ncclProfileProxyStep)) {
ncclProfilerEventDescr_v1_t eDescr = { 0 };
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileGroup;
ncclProfiler->startEvent(plan->comm->profilerContext, &plan->groupEventHandle, &eDescr);
}
@@ -280,20 +392,17 @@ ncclResult_t ncclProfilerStartTaskEvents(struct ncclKernelPlan* plan) {
eDescr.coll.name = plan->comm->commName;
eDescr.coll.commHash = plan->comm->commHash;
eDescr.coll.seqNumber = plan->comm->seqNumber[ct->func]++;
eDescr.coll.func = ct->func;
eDescr.coll.func = ncclFuncToString(ct->func);
eDescr.coll.sendBuff = ct->sendbuff;
eDescr.coll.recvBuff = ct->recvbuff;
eDescr.coll.count = ct->count;
eDescr.coll.root = ct->root;
eDescr.coll.datatype = ct->datatype;
eDescr.coll.op = ct->opHost;
eDescr.coll.datatype = ncclDatatypeToString(ct->datatype);
eDescr.coll.trafficBytes = ct->trafficBytes;
eDescr.coll.nMaxChannels = ct->nMaxChannels;
eDescr.coll.nWarps = ct->nWarps;
eDescr.coll.algo = ct->algorithm;
eDescr.coll.proto = ct->protocol;
eDescr.coll.isCollnet = ct->isCollnet;
eDescr.coll.isNvls = ct->isNvls;
eDescr.coll.algo = ncclAlgoToString(ct->algorithm);
eDescr.coll.proto = ncclProtoToString(ct->protocol);
ncclProfiler->startEvent(plan->comm->profilerContext, &ct->eventHandle, &eDescr);
// update collective task with group event activation mask
@@ -308,10 +417,10 @@ ncclResult_t ncclProfilerStartTaskEvents(struct ncclKernelPlan* plan) {
eDescr.rank = plan->comm->rank;
eDescr.p2p.name = plan->comm->commName;
eDescr.p2p.commHash = plan->comm->commHash;
eDescr.p2p.func = pt->func;
eDescr.p2p.func = ncclFuncToString(pt->func);
eDescr.p2p.buff = pt->buff;
eDescr.p2p.count = pt->count;
eDescr.p2p.datatype = pt->datatype;
eDescr.p2p.datatype = ncclDatatypeToString(pt->datatype);
eDescr.p2p.peer = pt->root;
ncclProfiler->startEvent(plan->comm->profilerContext, &pt->eventHandle, &eDescr);
@@ -346,6 +455,11 @@ ncclResult_t ncclProfilerStopTaskEvents(struct ncclKernelPlan* plan) {
return ncclSuccess;
}
// Bellow we set the proxy descriptor step number to DIVUP(step, args->sliceSteps).
// The reason is that for some ncclOp (e.g. AllReduce) one network transfer is
// made of sliceSteps steps rather than one step. In the profiler we are still
// interested in whole network transfers though, so we account for this when
// computing the actual network step number.
ncclResult_t ncclProfilerStartSendProxyOpEvent(int s, struct ncclProxyArgs* args) {
TIME_START_EVENT(proxyOpStart);
struct ncclProxySubArgs* sub = &args->subs[s];
@@ -355,13 +469,13 @@ ncclResult_t ncclProfilerStartSendProxyOpEvent(int s, struct ncclProxyArgs* args
eDescr.type = ncclProfileProxyOp;
eDescr.parentObj = sub->taskEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyOp.pid = args->pid;
eDescr.proxyOp.pid = sub->pid;
eDescr.proxyOp.channelId = sub->channelId;
eDescr.proxyOp.peer = sub->peer;
eDescr.proxyOp.nSteps = sub->nsteps;
eDescr.proxyOp.chunkSize = args->chunkSize;
eDescr.proxyOp.nSteps = DIVUP(sub->nsteps, args->sliceSteps);
eDescr.proxyOp.chunkSize = args->chunkSize * args->sliceSteps;
eDescr.proxyOp.isSend = 1;
ncclProfiler->startEvent(args->profilerContext, &sub->opEventHandle, &eDescr);
ncclProfiler->startEvent(sub->profilerContext, &sub->opEventHandle, &eDescr);
}
}
TIME_STOP_EVENT(proxyOpStart);
@@ -377,13 +491,13 @@ ncclResult_t ncclProfilerStartRecvProxyOpEvent(int s, struct ncclProxyArgs* args
eDescr.type = ncclProfileProxyOp;
eDescr.parentObj = sub->taskEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyOp.pid = args->pid;
eDescr.proxyOp.pid = sub->pid;
eDescr.proxyOp.channelId = sub->channelId;
eDescr.proxyOp.peer = sub->peer;
eDescr.proxyOp.nSteps = sub->nsteps;
eDescr.proxyOp.chunkSize = args->chunkSize;
eDescr.proxyOp.nSteps = DIVUP(sub->nsteps, args->sliceSteps);
eDescr.proxyOp.chunkSize = args->chunkSize * args->sliceSteps;
eDescr.proxyOp.isSend = 0;
ncclProfiler->startEvent(args->profilerContext, &sub->opEventHandle, &eDescr);
ncclProfiler->startEvent(sub->profilerContext, &sub->opEventHandle, &eDescr);
}
}
TIME_STOP_EVENT(proxyOpStart);
@@ -401,53 +515,50 @@ ncclResult_t ncclProfilerStopProxyOpEvent(int s, struct ncclProxyArgs* args) {
return ncclSuccess;
}
ncclResult_t ncclProfilerStartSendProxyStepEvents(int s, struct ncclProxyArgs* args, uint64_t stepLo, uint64_t stepHi) {
ncclResult_t ncclProfilerStartSendProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
TIME_START_EVENT(proxyStepStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->opEventHandle && (sub->eActivationMask & ncclProfileProxyStep)) {
for (uint64_t step = stepLo; step < stepHi; step++) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
eDescr.parentObj = sub->opEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyStep.step = step;
ncclProfiler->startEvent(args->profilerContext, &sub->stepEventHandles[step%NCCL_STEPS], &eDescr);
}
int step_ = DIVUP(stepId, args->sliceSteps);
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
eDescr.parentObj = sub->opEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyStep.step = step_;
ncclProfiler->startEvent(sub->profilerContext, &sub->stepEventHandles[step_%NCCL_STEPS], &eDescr);
}
}
TIME_STOP_EVENT(proxyStepStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStartRecvProxyStepEvents(int s, struct ncclProxyArgs* args, uint64_t stepLo, uint64_t stepHi) {
ncclResult_t ncclProfilerStartRecvProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
TIME_START_EVENT(proxyStepStart);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
if (sub->opEventHandle && (sub->eActivationMask & ncclProfileProxyStep)) {
for (uint64_t step = stepLo; step < stepHi; step++) {
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
eDescr.parentObj = sub->opEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyStep.step = step;
ncclProfiler->startEvent(args->profilerContext, &sub->stepEventHandles[step%NCCL_STEPS], &eDescr);
}
int step_ = DIVUP(stepId, args->sliceSteps);
ncclProfilerEventDescr_t eDescr = { 0 };
eDescr.type = ncclProfileProxyStep;
eDescr.parentObj = sub->opEventHandle;
eDescr.rank = sub->rank;
eDescr.proxyStep.step = step_;
ncclProfiler->startEvent(sub->profilerContext, &sub->stepEventHandles[step_%NCCL_STEPS], &eDescr);
}
}
TIME_STOP_EVENT(proxyStepStart);
return ncclSuccess;
}
ncclResult_t ncclProfilerStopProxyStepEvents(int s, struct ncclProxyArgs* args, uint64_t stepLo, uint64_t stepHi) {
ncclResult_t ncclProfilerStopProxyStepEvent(int s, struct ncclProxyArgs* args, int stepId) {
TIME_START_EVENT(proxyStepStop);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0)) {
for (uint64_t step = stepLo; step < stepHi; step++) {
if (sub->stepEventHandles[step%NCCL_STEPS]) {
ncclProfiler->stopEvent(sub->stepEventHandles[step%NCCL_STEPS]);
sub->stepEventHandles[step%NCCL_STEPS] = NULL;
}
int step_ = DIVUP(stepId, args->sliceSteps);
if (sub->stepEventHandles[step_%NCCL_STEPS]) {
ncclProfiler->stopEvent(sub->stepEventHandles[step_%NCCL_STEPS]);
sub->stepEventHandles[step_%NCCL_STEPS] = NULL;
}
}
TIME_STOP_EVENT(proxyStepStop);
@@ -485,8 +596,8 @@ ncclResult_t ncclProfilerRecordProxyOpEventState(int s, struct ncclProxyArgs* ar
TIME_START_EVENT(proxyOpRecord);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
ncclProfilerEventStateArgs_t a = { 0 };
a.proxyOp.steps = steps;
ncclProfilerEventStateArgs_t a = { };
a.proxyOp.steps = DIVUP(steps, args->sliceSteps);
a.proxyOp.transSize = transSize;
ncclProfiler->recordEventState(sub->opEventHandle, eState, &a);
}
@@ -494,14 +605,13 @@ ncclResult_t ncclProfilerRecordProxyOpEventState(int s, struct ncclProxyArgs* ar
return ncclSuccess;
}
ncclResult_t ncclProfilerRecordProxyStepEventStates(int s, struct ncclProxyArgs* args, uint64_t stepLo, uint64_t stepHi, ncclProfilerEventState_t eState) {
ncclResult_t ncclProfilerRecordProxyStepEventState(int s, struct ncclProxyArgs* args, int stepId, ncclProfilerEventState_t eState) {
TIME_START_EVENT(proxyStepRecord);
struct ncclProxySubArgs* sub = &args->subs[s];
if (__builtin_expect(ncclProfiler != NULL, 0) && sub->opEventHandle) {
for (uint64_t step = stepLo; step < stepHi; step++) {
if (sub->stepEventHandles[step%NCCL_STEPS]) {
ncclProfiler->recordEventState(sub->stepEventHandles[step%NCCL_STEPS], eState, 0);
}
int step_ = DIVUP(stepId, args->sliceSteps);
if (sub->stepEventHandles[step_%NCCL_STEPS]) {
ncclProfiler->recordEventState(sub->stepEventHandles[step_%NCCL_STEPS], eState, 0);
}
}
TIME_STOP_EVENT(proxyStepRecord);
@@ -511,7 +621,7 @@ ncclResult_t ncclProfilerRecordProxyStepEventStates(int s, struct ncclProxyArgs*
ncclResult_t ncclProfilerRecordProxyCtrlEventState(void* eHandle, int appended, ncclProfilerEventState_t eState) {
TIME_START_EVENT(proxyCtrlRecord);
if (__builtin_expect(ncclProfiler != NULL, 0) && eHandle && __atomic_load_n(&eActivationMask, __ATOMIC_RELAXED) & ncclProfileProxyCtrl) {
ncclProfilerEventStateArgs_t args = { 0 };
ncclProfilerEventStateArgs_t args = { };
args.proxyCtrl.appendedProxyOps = appended;
ncclProfiler->recordEventState(eHandle, eState, &args);
}
projects/rccl/src/misc/shmutils.cc
+3 -3
View File
@@ -45,7 +45,7 @@ static void shmHandleInit(int fd, char* shmPath, size_t shmSize, size_t realShmS
return;
}
ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** devShmPtr, int refcount, ncclShmHandle_t* handle) {
ncclResult_t ncclShmOpen(char* shmPath, size_t shmPathSize, size_t shmSize, void** shmPtr, void** devShmPtr, int refcount, ncclShmHandle_t* handle) {
int fd = -1;
char* hptr = NULL;
void* dptr = NULL;
@@ -62,7 +62,7 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
* refcount references; when the peer attaches, it should pass -1 to reduce one reference count. When it
* goes down to 0, unlink should be called in order to delete shared memory file. */
if (shmPath[0] == '\0') {
sprintf(shmPath, "/dev/shm/nccl-XXXXXX");
snprintf(shmPath, shmPathSize, "/dev/shm/nccl-XXXXXX");
retry_mkstemp:
fd = mkstemp(shmPath);
if (fd < 0) {
@@ -70,7 +70,7 @@ ncclResult_t ncclShmOpen(char* shmPath, size_t shmSize, void** shmPtr, void** de
INFO(NCCL_ALL, "mkstemp: Failed to create %s, error: %s (%d) - retrying", shmPath, strerror(errno), errno);
goto retry_mkstemp;
}
WARN("Error: failed to create shared memory file %p, error %s (%d)", shmPath, strerror(errno), errno);
WARN("Error: failed to create shared memory file %s, error %s (%d)", shmPath, strerror(errno), errno);
ret = ncclSystemError;
goto fail;
}
projects/rccl/src/misc/socket.cc
+178 -119
View File
@@ -14,6 +14,18 @@
#include <unistd.h>
#include <sys/syscall.h>
#include "param.h"
#include <time.h>
NCCL_PARAM(RetryCnt, "SOCKET_RETRY_CNT", 34);
NCCL_PARAM(RetryTimeOut, "SOCKET_RETRY_SLEEP_MSEC", 100);
static void msleep(unsigned int time_msec) {
const long c_1e6 = 1e6;
struct timespec tv = (struct timespec){
.tv_sec = time_msec / 1000,
.tv_nsec = (time_msec % 1000) * c_1e6,
};
nanosleep(&tv, NULL);
}
RCCL_PARAM(SocketReuseAddr, "SOCKET_REUSEADDR", 0);
RCCL_PARAM(SocketLinger, "SOCKET_LINGER", -1);
@@ -31,8 +43,13 @@ static ncclResult_t socketProgressOpt(int op, struct ncclSocket* sock, void* ptr
return ncclSuccess;
}
if (bytes == -1) {
if ((op == NCCL_SOCKET_SEND && errno == EPIPE) || (op == NCCL_SOCKET_RECV && errno == ECONNRESET)) {
*closed = 1;
return ncclSuccess;
}
if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN) {
WARN("socketProgressOpt: Call to recv from %s failed : %s", ncclSocketToString(&sock->addr, line), strerror(errno));
WARN("socketProgressOpt: Call to %s %s failed : %s", (op == NCCL_SOCKET_RECV ? "recv from" : "send to"),
ncclSocketToString(&sock->addr, line), strerror(errno));
return ncclRemoteError;
} else {
bytes = 0;
@@ -43,17 +60,22 @@ static ncclResult_t socketProgressOpt(int op, struct ncclSocket* sock, void* ptr
INFO(NCCL_NET, "socketProgressOpt: abort called");
return ncclInternalError;
}
} while (bytes > 0 && (*offset) < size);
} while (sock->asyncFlag == 0 && bytes > 0 && (*offset) < size);
return ncclSuccess;
}
static ncclResult_t socketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset) {
static ncclResult_t socketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset, int* pclosed = NULL) {
int closed;
NCCLCHECK(socketProgressOpt(op, sock, ptr, size, offset, 0 /*block*/, &closed));
if (closed) {
char line[SOCKET_NAME_MAXLEN+1];
WARN("socketProgress: Connection closed by remote peer %s", ncclSocketToString(&sock->addr, line, 0));
return ncclRemoteError;
if (pclosed) {
*pclosed = closed;
return ncclSuccess;
} else {
char line[SOCKET_NAME_MAXLEN+1];
WARN("socketProgress: Connection closed by remote peer %s", ncclSocketToString(&sock->addr, line, 0));
return ncclRemoteError;
}
}
return ncclSuccess;
}
@@ -68,9 +90,9 @@ static ncclResult_t socketWait(int op, struct ncclSocket* sock, void* ptr, int s
*
* Output: "IPv4/IPv6 address<port>"
*/
const char *ncclSocketToString(union ncclSocketAddress *addr, char *buf, const int numericHostForm /*= 1*/) {
const char *ncclSocketToString(const union ncclSocketAddress *addr, char *buf, const int numericHostForm /*= 1*/) {
if (buf == NULL || addr == NULL) return NULL;
struct sockaddr *saddr = &addr->sa;
const struct sockaddr *saddr = &addr->sa;
if (saddr->sa_family != AF_INET && saddr->sa_family != AF_INET6) { buf[0]='\0'; return buf; }
char host[NI_MAXHOST], service[NI_MAXSERV];
/* NI_NUMERICHOST: If set, then the numeric form of the hostname is returned.
@@ -375,10 +397,9 @@ ncclResult_t ncclSocketListen(struct ncclSocket* sock) {
if (socketToPort(&sock->addr)) {
// Port is forced by env. Make sure we get the port.
int opt = 1;
#if defined(SO_REUSEPORT)
SYSCHECK(setsockopt(sock->fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, sizeof(opt)), "setsockopt");
#else
SYSCHECK(setsockopt(sock->fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)), "setsockopt");
#if defined(SO_REUSEPORT)
SYSCHECK(setsockopt(sock->fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)), "setsockopt");
#endif
}
@@ -417,6 +438,15 @@ static ncclResult_t socketTryAccept(struct ncclSocket* sock) {
sock->fd = accept(sock->acceptFd, (struct sockaddr*)&sock->addr, &socklen);
if (sock->fd != -1) {
sock->state = ncclSocketStateAccepted;
} else if (errno == ENETDOWN || errno == EPROTO || errno == ENOPROTOOPT || errno == EHOSTDOWN ||
errno == ENONET || errno == EHOSTUNREACH || errno == EOPNOTSUPP || errno == ENETUNREACH) {
/* per accept's man page, for linux sockets, the following errors might be already pending errors
* and should be considered as EAGAIN. To avoid infinite loop in case of errors, we use the retry count*/
if (++sock->errorRetries == ncclParamRetryCnt()) {
WARN("socketTryAccept: exceeded error retry count (%d), %s", sock->errorRetries, strerror(errno));
return ncclSystemError;
}
INFO(NCCL_ALL, "Call to accept returned %s, retrying", strerror(errno));
} else if (errno != EAGAIN && errno != EWOULDBLOCK) {
WARN("socketTryAccept: Accept failed: %s", strerror(errno));
return ncclSystemError;
@@ -424,72 +454,118 @@ static ncclResult_t socketTryAccept(struct ncclSocket* sock) {
return ncclSuccess;
}
static ncclResult_t socketSetFlags(struct ncclSocket* sock) {
const int one = 1;
/* Set socket as non-blocking if async or if we need to be able to abort */
if ((sock->asyncFlag || sock->abortFlag) && sock->fd >= 0) {
int flags;
SYSCHECK(flags = fcntl(sock->fd, F_GETFL), "fcntl");
SYSCHECK(fcntl(sock->fd, F_SETFL, flags | O_NONBLOCK), "fcntl");
}
SYSCHECK(setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
return ncclSuccess;
}
static ncclResult_t socketFinalizeAccept(struct ncclSocket* sock) {
uint64_t magic;
enum ncclSocketType type;
int received = 0;
const int one = 1;
SYSCHECK(setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
int received;
// once accepted, linux sockets do NOT inherit file status flags such as O_NONBLOCK (BSD ones do)
NCCLCHECK(socketSetFlags(sock));
NCCLCHECK(ncclSocketProgress(NCCL_SOCKET_RECV, sock, &magic, sizeof(magic), &received));
if (received == 0) return ncclSuccess;
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &magic, sizeof(magic), &received));
if (magic != sock->magic) {
WARN("socketFinalizeAccept: wrong magic %lx != %lx", magic, sock->magic);
close(sock->fd);
sock->fd = -1;
// Ignore spurious connection and accept again
sock->state = ncclSocketStateAccepting;
return ncclSuccess;
} else {
received = 0;
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &type, sizeof(type), &received));
if (type != sock->type) {
WARN("socketFinalizeAccept: wrong type %d != %d", type, sock->type);
sock->state = ncclSocketStateError;
if (sock->asyncFlag == 0 || sock->finalizeCounter < sizeof(magic)) {
if (sock->asyncFlag == 0) {
received = 0;
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &magic, sizeof(magic), &received));
} else {
received = sock->finalizeCounter;
NCCLCHECK(socketProgress(NCCL_SOCKET_RECV, sock, sock->finalizeBuffer, sizeof(magic), &received));
sock->finalizeCounter = received;
if (received < sizeof(magic)) return ncclSuccess;
memcpy(&magic, sock->finalizeBuffer, sizeof(magic));
}
if (magic != sock->magic) {
WARN("socketFinalizeAccept: wrong magic %lx != %lx", magic, sock->magic);
close(sock->fd);
sock->fd = -1;
return ncclInternalError;
} else {
sock->state = ncclSocketStateReady;
// Ignore spurious connection and accept again
sock->state = ncclSocketStateAccepting;
return ncclSuccess;
}
}
if (sock->asyncFlag == 0) {
received = 0;
NCCLCHECK(socketWait(NCCL_SOCKET_RECV, sock, &type, sizeof(type), &received));
} else {
received = sock->finalizeCounter - sizeof(magic);
NCCLCHECK(socketProgress(NCCL_SOCKET_RECV, sock, sock->finalizeBuffer, sizeof(type), &received));
sock->finalizeCounter = received + sizeof(magic);
if (received < sizeof(type)) return ncclSuccess;
memcpy(&type, sock->finalizeBuffer, sizeof(type));
}
if (type != sock->type) {
WARN("socketFinalizeAccept: wrong type %d != %d", type, sock->type);
sock->state = ncclSocketStateError;
close(sock->fd);
sock->fd = -1;
return ncclInternalError;
} else {
sock->state = ncclSocketStateReady;
}
return ncclSuccess;
}
static ncclResult_t socketStartConnect(struct ncclSocket* sock) {
/* blocking/non-blocking connect() is determined by asyncFlag. */
int ret = connect(sock->fd, &sock->addr.sa, sock->salen);
if (ret == 0) {
static ncclResult_t socketResetFd(struct ncclSocket* sock) {
ncclResult_t ret = ncclSuccess;
int fd = -1;
SYSCHECKGOTO(fd = socket(sock->addr.sa.sa_family, SOCK_STREAM, 0), "socket", ret, cleanup);
// if sock->fd is valid, close it and reuse its number
if (sock->fd != -1) {
SYSCHECKGOTO(dup2(fd, sock->fd), "dup2", ret, cleanup);
SYSCHECKGOTO(close(fd), "close", ret, cleanup);
} else {
sock->fd = fd;
}
NCCLCHECKGOTO(socketSetFlags(sock), ret, exit);
exit:
return ret;
cleanup:
// cleanup fd, leave sock->fd untouched
if (fd != -1) {
(void)close(fd);
}
goto exit;
}
static ncclResult_t socketConnectCheck(struct ncclSocket* sock, int errCode, const char funcName[]) {
if (errCode == 0) {
sock->state = ncclSocketStateConnected;
return ncclSuccess;
} else if (errno == EINPROGRESS) {
} else if (errCode == EINPROGRESS) {
sock->state = ncclSocketStateConnectPolling;
return ncclSuccess;
} else if (errno == ECONNREFUSED) {
if (++sock->refusedRetries == RETRY_REFUSED_TIMES) {
sock->state = ncclSocketStateError;
WARN("socketStartConnect: exceeded retries (%d)", sock->refusedRetries);
return ncclRemoteError;
} else if (errCode == ETIMEDOUT || errCode == EHOSTUNREACH || errCode == ECONNREFUSED) {
if (sock->customRetry == 0) {
if (sock->errorRetries++ == ncclParamRetryCnt()) {
sock->state = ncclSocketStateError;
WARN("%s: connect returned %s, exceeded error retry count (%d)", funcName, strerror(errCode), sock->errorRetries);
return ncclRemoteError;
}
unsigned int sleepTime = sock->errorRetries * ncclParamRetryTimeOut();
INFO(NCCL_ALL, "%s: connect returned %s, retrying (%d/%ld) after sleep for %u msec", funcName, strerror(errCode), sock->errorRetries, ncclParamRetryCnt(), sleepTime);
msleep(sleepTime);
}
usleep(SLEEP_INT);
if (sock->refusedRetries % 1000 == 0) INFO(NCCL_ALL, "Call to connect returned %s, retrying", strerror(errno));
return ncclSuccess;
} else if (errno == ETIMEDOUT) {
if (++sock->timedOutRetries == RETRY_TIMEDOUT_TIMES) {
sock->state = ncclSocketStateError;
WARN("socketStartConnect: exceeded timeouts (%d)", sock->timedOutRetries);
return ncclRemoteError;
}
usleep(SLEEP_INT);
return ncclSuccess;
NCCLCHECK(socketResetFd(sock)); /* in case of failure in connect, socket state is unspecified */
sock->state = ncclSocketStateConnecting;
} else {
char line[SOCKET_NAME_MAXLEN+1];
sock->state = ncclSocketStateError;
WARN("socketStartConnect: Connect to %s failed : %s", ncclSocketToString(&sock->addr, line), strerror(errno));
WARN("%s: Connect to %s failed : %s", funcName, ncclSocketToString(&sock->addr, line), strerror(errCode));
return ncclSystemError;
}
return ncclSuccess;
}
static ncclResult_t socketStartConnect(struct ncclSocket* sock) {
/* blocking/non-blocking connect() is determined by asyncFlag. */
int ret = connect(sock->fd, &sock->addr.sa, sock->salen);
return socketConnectCheck(sock, (ret == -1) ? errno : 0, __func__);
}
static ncclResult_t socketPollConnect(struct ncclSocket* sock) {
@@ -514,33 +590,7 @@ static ncclResult_t socketPollConnect(struct ncclSocket* sock) {
/* check socket status */
SYSCHECK(getsockopt(sock->fd, SOL_SOCKET, SO_ERROR, (void*)&ret, &rlen), "getsockopt");
if (ret == 0) {
sock->state = ncclSocketStateConnected;
} else if (ret == ECONNREFUSED) {
if (++sock->refusedRetries == RETRY_REFUSED_TIMES) {
sock->state = ncclSocketStateError;
WARN("socketPollConnect: exceeded retries (%d)", sock->refusedRetries);
return ncclRemoteError;
}
if (sock->refusedRetries % 1000 == 0) INFO(NCCL_ALL, "Call to connect returned %s, retrying", strerror(errno));
usleep(SLEEP_INT);
sock->state = ncclSocketStateConnecting;
} else if (ret == ETIMEDOUT) {
if (++sock->timedOutRetries == RETRY_TIMEDOUT_TIMES) {
sock->state = ncclSocketStateError;
WARN("socketPollConnect: exceeded timeouts (%d)", sock->timedOutRetries);
return ncclRemoteError;
}
usleep(SLEEP_INT);
sock->state = ncclSocketStateConnecting;
} else if (ret != EINPROGRESS) {
sock->state = ncclSocketStateError;
char line[SOCKET_NAME_MAXLEN+1];
WARN("socketPollConnect: Connect to %s returned %d(%s) errno %d(%s)", ncclSocketToString(&sock->addr, line), ret, strerror(ret), errno, strerror(errno));
return ncclSystemError;
}
return ncclSuccess;
return socketConnectCheck(sock, ret, __func__);
}
ncclResult_t ncclSocketPollConnect(struct ncclSocket* sock) {
@@ -553,12 +603,24 @@ ncclResult_t ncclSocketPollConnect(struct ncclSocket* sock) {
}
static ncclResult_t socketFinalizeConnect(struct ncclSocket* sock) {
int sent = 0;
NCCLCHECK(socketProgress(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
if (sent == 0) return ncclSuccess;
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
sent = 0;
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->type, sizeof(sock->type), &sent));
int sent;
if (sock->asyncFlag == 0) {
sent = 0;
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
sent = 0;
NCCLCHECK(socketWait(NCCL_SOCKET_SEND, sock, &sock->type, sizeof(sock->type), &sent));
} else {
if (sock->finalizeCounter < sizeof(sock->magic)) {
sent = sock->finalizeCounter;
NCCLCHECK(socketProgress(NCCL_SOCKET_SEND, sock, &sock->magic, sizeof(sock->magic), &sent));
sock->finalizeCounter = sent;
if (sent < sizeof(sock->magic)) return ncclSuccess;
}
sent = sock->finalizeCounter - sizeof(sock->magic);
NCCLCHECK(socketProgress(NCCL_SOCKET_SEND, sock, &sock->type, sizeof(sock->type), &sent));
sock->finalizeCounter = sent + sizeof(sock->magic);
if (sent < sizeof(sock->type)) return ncclSuccess;
}
sock->state = ncclSocketStateReady;
return ncclSuccess;
}
@@ -601,7 +663,6 @@ ncclResult_t ncclSocketReady(struct ncclSocket* sock, int *running) {
ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
char line[SOCKET_NAME_MAXLEN+1];
const int one = 1;
if (sock == NULL) {
WARN("ncclSocketConnect: pass NULL socket");
@@ -619,9 +680,8 @@ ncclResult_t ncclSocketConnect(struct ncclSocket* sock) {
}
TRACE(NCCL_INIT|NCCL_NET,"Connecting to socket %s", ncclSocketToString(&sock->addr, line));
SYSCHECK(setsockopt(sock->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)), "setsockopt");
sock->state = ncclSocketStateConnecting;
sock->finalizeCounter = 0;
do {
NCCLCHECK(socketProgressState(sock));
} while (sock->asyncFlag == 0 &&
@@ -667,6 +727,7 @@ ncclResult_t ncclSocketAccept(struct ncclSocket* sock, struct ncclSocket* listen
memcpy(sock, listenSock, sizeof(struct ncclSocket));
sock->acceptFd = listenSock->fd;
sock->state = ncclSocketStateAccepting;
sock->finalizeCounter = 0;
}
do {
@@ -697,12 +758,11 @@ exit:
return ret;
}
ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* addr, uint64_t magic, enum ncclSocketType type, volatile uint32_t* abortFlag, int asyncFlag) {
ncclResult_t ncclSocketInit(struct ncclSocket* sock, const union ncclSocketAddress* addr, uint64_t magic, enum ncclSocketType type, volatile uint32_t* abortFlag, int asyncFlag, int customRetry) {
ncclResult_t ret = ncclSuccess;
if (sock == NULL) goto exit;
sock->timedOutRetries = 0;
sock->refusedRetries = 0;
sock->errorRetries = 0;
sock->abortFlag = abortFlag;
sock->asyncFlag = asyncFlag;
sock->state = ncclSocketStateInitialized;
@@ -710,6 +770,7 @@ ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* ad
sock->type = type;
sock->fd = -1;
sock->acceptFd = -1;
sock->customRetry = customRetry;
if (addr) {
/* IPv4/IPv6 support */
@@ -721,17 +782,11 @@ ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* ad
WARN("ncclSocketInit: connecting to address %s with family %d is neither AF_INET(%d) nor AF_INET6(%d)",
ncclSocketToString(&sock->addr, line), family, AF_INET, AF_INET6);
ret = ncclInternalError;
goto fail;
goto exit;
}
sock->salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
/* Connect to a hostname / port */
sock->fd = socket(family, SOCK_STREAM, 0);
if (sock->fd == -1) {
WARN("ncclSocketInit: Socket creation failed : %s", strerror(errno));
ret = ncclSystemError;
goto fail;
}
// in case of error, we close the fd before returning as it's unclear if the caller has to use ncclSocketClose for cleanup
NCCLCHECKGOTO(socketResetFd(sock), ret, fail);
// [RCCL] Runtime socket options
if (rcclParamSocketReuseAddr()) {
@@ -746,14 +801,6 @@ ncclResult_t ncclSocketInit(struct ncclSocket* sock, union ncclSocketAddress* ad
} else {
memset(&sock->addr, 0, sizeof(union ncclSocketAddress));
}
/* Set socket as non-blocking if async or if we need to be able to abort */
if ((sock->asyncFlag || sock->abortFlag) && sock->fd >= 0) {
int flags;
SYSCHECKGOTO(flags = fcntl(sock->fd, F_GETFL), "fcntl", ret, fail);
SYSCHECKGOTO(fcntl(sock->fd, F_SETFL, flags | O_NONBLOCK), "fcntl", ret, fail);
}
exit:
return ret;
fail:
@@ -764,12 +811,12 @@ fail:
goto exit;
}
ncclResult_t ncclSocketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset) {
ncclResult_t ncclSocketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset, int* closed) {
if (sock == NULL) {
WARN("ncclSocketProgress: pass NULL socket");
return ncclInvalidArgument;
}
NCCLCHECK(socketProgress(op, sock, ptr, size, offset));
NCCLCHECK(socketProgress(op, sock, ptr, size, offset, closed));
return ncclSuccess;
}
@@ -802,7 +849,7 @@ ncclResult_t ncclSocketRecv(struct ncclSocket* sock, void* ptr, int size) {
WARN("ncclSocketRecv: pass NULL socket");
return ncclInvalidArgument;
}
if (sock->state != ncclSocketStateReady) {
if (sock->state != ncclSocketStateReady && sock->state != ncclSocketStateTerminating) {
WARN("ncclSocketRecv: socket state (%d) is not ready", sock->state);
return ncclInternalError;
}
@@ -816,7 +863,8 @@ ncclResult_t ncclSocketSendRecv(struct ncclSocket* sendSock, void* sendPtr, int
WARN("ncclSocketSendRecv: invalid socket %p/%p", sendSock, recvSock);
return ncclInternalError;
}
if (sendSock->state != ncclSocketStateReady || recvSock->state != ncclSocketStateReady) {
if (sendSock->state != ncclSocketStateReady ||
(recvSock->state != ncclSocketStateReady && recvSock->state != ncclSocketStateTerminating)) {
WARN("ncclSocketSendRecv: socket state (%d/%d) is not ready", sendSock->state, recvSock->state);
return ncclInternalError;
}
@@ -860,9 +908,20 @@ ncclResult_t ncclSocketTryRecv(struct ncclSocket* sock, void* ptr, int size, int
return ncclSuccess;
}
ncclResult_t ncclSocketClose(struct ncclSocket* sock) {
// Make it possible to close just one part of a socket.
ncclResult_t ncclSocketShutdown(struct ncclSocket* sock, int how) {
if (sock != NULL) {
if (sock->fd >= 0) {
shutdown(sock->fd, how);
}
sock->state = ncclSocketStateTerminating;
}
return ncclSuccess;
}
ncclResult_t ncclSocketClose(struct ncclSocket* sock) {
if (sock != NULL) {
if (sock->state > ncclSocketStateNone && sock->state < ncclSocketStateNum && sock->fd >= 0) {
/* shutdown() is needed to send FIN packet to proxy thread; shutdown() is not affected
* by refcount of fd, but close() is. close() won't close a fd and send FIN packet if
* the fd is duplicated (e.g. fork()). So shutdown() guarantees the correct and graceful
projects/rccl/src/misc/tuner.cc
+40 -17
View File
@@ -16,9 +16,11 @@
pthread_mutex_t tunerPluginLock = PTHREAD_MUTEX_INITIALIZER;
static int tunerPluginRefCount;
static void* tunerPluginLib = nullptr;
static ncclTuner_v3_t* tunerSymbol = nullptr;
static ncclTuner_v4_t* tunerSymbol = nullptr;
static ncclTuner_v3_t* ncclTuner_v3 = nullptr;
static ncclTuner_v2_t* ncclTuner_v2 = nullptr;
static ncclTuner_v3_t ncclTuner_v2_as_v3;
static ncclTuner_v4_t ncclTuner_v2_as_v4;
static ncclTuner_v4_t ncclTuner_v3_as_v4;
static int hasNvlsSupport(float** collCostTable) {
// Requirements for support of different algorithms:
@@ -39,7 +41,20 @@ static int hasCollNetSupport(float** collCostTable) {
return (table[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] == NCCL_ALGO_PROTO_IGNORE) ? 0 : 1;
}
static ncclResult_t ncclTuner_v2_as_v3_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int* nChannels) {
static ncclResult_t ncclTuner_v3_as_v4_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff __attribute__((unused)), int* nChannels) {
NCCLCHECK(ncclTuner_v3->getCollInfo(context, collType, nBytes, numPipeOps, collCostTable, numAlgo, numProto, nChannels));
return ncclSuccess;
}
static ncclResult_t ncclTuner_v3_as_v4_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
NCCLCHECK(ncclTuner_v3->init(nRanks, nNodes, logFunction, context));
ncclTuner_v3_as_v4.name = ncclTuner_v3->name;
ncclTuner_v3_as_v4.getCollInfo = ncclTuner_v3_as_v4_getCollInfo;
ncclTuner_v3_as_v4.destroy = ncclTuner_v3->destroy;
return ncclSuccess;
}
static ncclResult_t ncclTuner_v2_as_v4_getCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo __attribute__((unused)), int numProto __attribute__((unused)), int regBuff __attribute__((unused)), int* nChannels) {
int algorithm = NCCL_ALGO_UNDEF;
int protocol = NCCL_PROTO_UNDEF;
int nvlsSupport = hasNvlsSupport(collCostTable);
@@ -53,11 +68,11 @@ static ncclResult_t ncclTuner_v2_as_v3_getCollInfo(void* context, ncclFunc_t col
return ncclSuccess;
}
static ncclResult_t ncclTuner_v2_as_v3_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
static ncclResult_t ncclTuner_v2_as_v4_init(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void** context) {
NCCLCHECK(ncclTuner_v2->init(nRanks, nNodes, logFunction, context));
ncclTuner_v2_as_v3.name = ncclTuner_v2->name;
ncclTuner_v2_as_v3.getCollInfo = ncclTuner_v2_as_v3_getCollInfo;
ncclTuner_v2_as_v3.destroy = ncclTuner_v2->destroy;
ncclTuner_v2_as_v4.name = ncclTuner_v2->name;
ncclTuner_v2_as_v4.getCollInfo = ncclTuner_v2_as_v4_getCollInfo;
ncclTuner_v2_as_v4.destroy = ncclTuner_v2->destroy;
return ncclSuccess;
}
@@ -198,18 +213,26 @@ ncclResult_t ncclTunerPluginLoad(struct ncclComm* comm) {
goto fail;
}
tunerSymbol = (ncclTuner_v3_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v3");
tunerSymbol = (ncclTuner_v4_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v4");
if (tunerSymbol == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v2");
if (ncclTuner_v2 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
dlclose(tunerPluginLib);
goto fail;
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.");
ncclTuner_v3 = (ncclTuner_v3_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v3");
if (ncclTuner_v3 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.");
ncclTuner_v2 = (ncclTuner_v2_t*)dlsym(tunerPluginLib, "ncclTunerPlugin_v2");
if (ncclTuner_v2 == nullptr) {
INFO(NCCL_ENV|NCCL_TUNING, "TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.");
dlclose(tunerPluginLib);
goto fail;
} else {
ncclTuner_v2_as_v4.init = ncclTuner_v2_as_v4_init;
ncclTuner_v2_as_v4.name = ncclTuner_v2->name;
tunerSymbol = &ncclTuner_v2_as_v4;
}
} else {
ncclTuner_v2_as_v3.init = ncclTuner_v2_as_v3_init;
ncclTuner_v2_as_v3.name = ncclTuner_v2->name;
tunerSymbol = &ncclTuner_v2_as_v3;
ncclTuner_v3_as_v4.init = ncclTuner_v3_as_v4_init;
ncclTuner_v3_as_v4.name = ncclTuner_v3->name;
tunerSymbol = &ncclTuner_v3_as_v4;
}
}