AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for external network.

Browse Source 
Dynamically load external network from libnccl-net.so.
Add init function in networks.
Move PCI scoring to net.cu, only ask transport to provide a path.
Simplify CUDA PCI path detection.
Add dummy external network
This commit is contained in:
Sylvain Jeaugey
2018-11-13 10:37:20 -08:00
parent d7a58cfa58
commit 0d3a20f96d
29 changed files with 437 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/include/debug.h
+21 -62
View File
@@ -16,65 +16,24 @@
#include <limits.h>
#include <string.h>
#include "nccl.h"
#include "nccl_net.h"
#define gettid() (pid_t) syscall(SYS_gettid)
typedef enum {NONE=0, VERSION=1, WARN=2, INFO=3, ABORT=4, TRACE=5} DebugLevel;
typedef enum {INIT=1, COLL=2, P2P=4, SHM=8, NET=16, ALL=~0} SubSys;
extern DebugLevel ncclDebugLevel;
extern int ncclDebugLevel;
extern uint64_t ncclDebugMask;
extern pthread_mutex_t ncclDebugOutputLock;
extern FILE *ncclDebugFile;
extern ncclResult_t getHostName(char* hostname, int maxlen);
#define WARN(...) do { \
if (ncclDebugLevel >= WARN) { \
char hostname[1024]; \
getHostName(hostname, 1024); \
int cudaDev; \
cudaGetDevice(&cudaDev); \
pthread_mutex_lock(&ncclDebugOutputLock); \
fprintf(ncclDebugFile,"\n%s:%d:%d [%d] %s:%d NCCL WARN ", hostname, getpid(), gettid(), cudaDev, __FILE__, __LINE__); \
fprintf(ncclDebugFile,__VA_ARGS__); \
fprintf(ncclDebugFile,"\n"); \
fflush(ncclDebugFile); \
pthread_mutex_unlock(&ncclDebugOutputLock); \
if (ncclDebugLevel == ABORT) { fprintf(stderr,"\n%s:%d:%d [%d] %s:%d NCCL ABORT\n", hostname, getpid(), gettid(), cudaDev, __FILE__, __LINE__); abort(); } \
} \
} while(0)
extern void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *filefunc, int line, const char *fmt, ...);
#define INFO(FLAGS, ...) do { \
if (ncclDebugLevel >= INFO && ((FLAGS) & ncclDebugMask)) { \
char hostname[1024]; \
getHostName(hostname, 1024); \
int cudaDev; \
cudaGetDevice(&cudaDev); \
pthread_mutex_lock(&ncclDebugOutputLock); \
fprintf(ncclDebugFile,"%s:%d:%d [%d] NCCL INFO ", hostname, getpid(), gettid(), cudaDev); \
fprintf(ncclDebugFile,__VA_ARGS__);fprintf(ncclDebugFile,"\n"); \
fflush(ncclDebugFile); \
pthread_mutex_unlock(&ncclDebugOutputLock); \
} \
} while(0)
#define WARN(...) ncclDebugLog(NCCL_LOG_WARN, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__)
#define INFO(FLAGS, ...) ncclDebugLog(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__)
#ifdef ENABLE_TRACE
#define TRACE(FLAGS, ...) do { \
if (ncclDebugLevel == TRACE && ((FLAGS) & ncclDebugMask)) { \
char hostname[1024]; \
getHostName(hostname, 1024); \
int cudaDev; \
cudaGetDevice(&cudaDev); \
pthread_mutex_lock(&ncclDebugOutputLock); \
auto delta = std::chrono::high_resolution_clock::now() - ncclEpoch; \
double timestamp = std::chrono::duration_cast<std::chrono::duration<double>>(delta).count()*1000; \
fprintf(ncclDebugFile,"%s:%d:%d [%d] %f %s:%d NCCL TRACE ", hostname, getpid(), gettid(), cudaDev, timestamp, __func__, __LINE__); \
fprintf(ncclDebugFile,__VA_ARGS__);fprintf(ncclDebugFile,"\n"); \
fflush(ncclDebugFile); \
pthread_mutex_unlock(&ncclDebugOutputLock); \
} \
} while(0)
#define TRACE(FLAGS, ...) ncclDebugLog(NCCL_LOG_TRACE, (FLAGS), __func__, __LINE__, __VA_ARGS__)
extern std::chrono::high_resolution_clock::time_point ncclEpoch;
#else
#define TRACE(...)
#endif
@@ -84,17 +43,17 @@ extern std::chrono::high_resolution_clock::time_point ncclEpoch;
static inline void initDebug() {
const char* nccl_debug = getenv("NCCL_DEBUG");
if (nccl_debug == NULL) {
ncclDebugLevel = NONE;
ncclDebugLevel = NCCL_LOG_NONE;
} else if (strcasecmp(nccl_debug, "VERSION") == 0) {
ncclDebugLevel = VERSION;
ncclDebugLevel = NCCL_LOG_VERSION;
} else if (strcasecmp(nccl_debug, "WARN") == 0) {
ncclDebugLevel = WARN;
ncclDebugLevel = NCCL_LOG_WARN;
} else if (strcasecmp(nccl_debug, "INFO") == 0) {
ncclDebugLevel = INFO;
ncclDebugLevel = NCCL_LOG_INFO;
} else if (strcasecmp(nccl_debug, "ABORT") == 0) {
ncclDebugLevel = ABORT;
ncclDebugLevel = NCCL_LOG_ABORT;
} else if (strcasecmp(nccl_debug, "TRACE") == 0) {
ncclDebugLevel = TRACE;
ncclDebugLevel = NCCL_LOG_TRACE;
}
/* Parse the NCCL_DEBUG_SUBSYS env var
@@ -109,17 +68,17 @@ static inline void initDebug() {
uint64_t mask = 0;
if (subsys[0] == '^') { invert = 1; subsys++; }
if (strcasecmp(subsys, "INIT") == 0) {
mask = INIT;
mask = NCCL_INIT;
} else if (strcasecmp(subsys, "COLL") == 0) {
mask = COLL;
mask = NCCL_COLL;
} else if (strcasecmp(subsys, "P2P") == 0) {
mask = P2P;
mask = NCCL_P2P;
} else if (strcasecmp(subsys, "SHM") == 0) {
mask = SHM;
mask = NCCL_SHM;
} else if (strcasecmp(subsys, "NET") == 0) {
mask = NET;
mask = NCCL_NET;
} else if (strcasecmp(subsys, "ALL") == 0) {
mask = ALL;
mask = NCCL_ALL;
}
if (mask) {
if (invert) ncclDebugMask &= ~mask; else ncclDebugMask |= mask;
@@ -133,7 +92,7 @@ static inline void initDebug() {
* NCCL_DEBUG level is > VERSION
*/
const char* nccl_debug_file = getenv("NCCL_DEBUG_FILE");
if (ncclDebugLevel > VERSION && nccl_debug_file != NULL) {
if (ncclDebugLevel > NCCL_LOG_VERSION && nccl_debug_file != NULL) {
int c = 0;
char debug_fn[PATH_MAX+1] = "";
char *dfn = debug_fn;
@@ -164,7 +123,7 @@ static inline void initDebug() {
if (debug_fn[0] != '\0') {
FILE *file = fopen(debug_fn, "w");
if (file != NULL) {
INFO(ALL,"DEBUG file is '%s'", debug_fn);
INFO(NCCL_ALL,"DEBUG file is '%s'", debug_fn);
ncclDebugFile = file;
}
}