2018-12-13 15:56:12 -08:00
|
|
|
/*************************************************************************
|
2019-03-14 19:39:20 -07:00
|
|
|
* Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
|
2018-12-13 15:56:12 -08:00
|
|
|
*
|
|
|
|
|
* See LICENSE.txt for license information
|
|
|
|
|
************************************************************************/
|
|
|
|
|
|
2019-03-14 19:39:20 -07:00
|
|
|
#ifndef NCCL_CHECKS_H_
|
|
|
|
|
#define NCCL_CHECKS_H_
|
2018-12-13 15:56:12 -08:00
|
|
|
|
2019-03-14 19:39:20 -07:00
|
|
|
#include "debug.h"
|
|
|
|
|
|
|
|
|
|
// Check CUDA calls
|
|
|
|
|
#define CUDACHECK(cmd) do { \
|
|
|
|
|
cudaError_t e = cmd; \
|
|
|
|
|
if( e != cudaSuccess ) { \
|
|
|
|
|
WARN("Cuda failure '%s'", cudaGetErrorString(e)); \
|
|
|
|
|
return ncclUnhandledCudaError; \
|
|
|
|
|
} \
|
|
|
|
|
} while(false)
|
|
|
|
|
|
|
|
|
|
#define CUDACHECKGOTO(cmd, res, label) do { \
|
|
|
|
|
cudaError_t e = cmd; \
|
|
|
|
|
if( e != cudaSuccess ) { \
|
|
|
|
|
WARN("Cuda failure '%s'", cudaGetErrorString(e)); \
|
|
|
|
|
res = ncclUnhandledCudaError; \
|
|
|
|
|
goto label; \
|
|
|
|
|
} \
|
|
|
|
|
} while(false)
|
|
|
|
|
|
|
|
|
|
#include <errno.h>
|
|
|
|
|
// Check system calls
|
|
|
|
|
#define SYSCHECK(call, name) do { \
|
|
|
|
|
int retval; \
|
|
|
|
|
SYSCHECKVAL(call, name, retval); \
|
|
|
|
|
} while (false)
|
|
|
|
|
|
|
|
|
|
#define SYSCHECKVAL(call, name, retval) do { \
|
|
|
|
|
SYSCHECKSYNC(call, name, retval); \
|
|
|
|
|
if (retval == -1) { \
|
|
|
|
|
WARN("Call to " name " failed : %s", strerror(errno)); \
|
|
|
|
|
return ncclSystemError; \
|
|
|
|
|
} \
|
|
|
|
|
} while (false)
|
|
|
|
|
|
|
|
|
|
#define SYSCHECKSYNC(call, name, retval) do { \
|
|
|
|
|
retval = call; \
|
|
|
|
|
if (retval == -1 && (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN)) { \
|
|
|
|
|
INFO(NCCL_ALL,"Call to " name " returned %s, retrying", strerror(errno)); \
|
|
|
|
|
} else { \
|
|
|
|
|
break; \
|
|
|
|
|
} \
|
|
|
|
|
} while(true)
|
|
|
|
|
|
|
|
|
|
// Propagate errors up
|
|
|
|
|
#define NCCLCHECK(call) do { \
|
|
|
|
|
ncclResult_t res = call; \
|
|
|
|
|
if (res != ncclSuccess) { \
|
|
|
|
|
/* Print the back trace*/ \
|
|
|
|
|
INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, res); \
|
|
|
|
|
return res; \
|
|
|
|
|
} \
|
|
|
|
|
} while (0);
|
|
|
|
|
|
|
|
|
|
#define NCCLCHECKGOTO(call, res, label) do { \
|
|
|
|
|
res = call; \
|
|
|
|
|
if (res != ncclSuccess) { \
|
|
|
|
|
/* Print the back trace*/ \
|
|
|
|
|
INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, res); \
|
|
|
|
|
goto label; \
|
|
|
|
|
} \
|
|
|
|
|
} while (0);
|
|
|
|
|
|
|
|
|
|
#endif
|
