AI/rocm-systems
2
0
Bifurcation 0
Code Tickets Demandes d'ajout Actions Paquets Projets Publications Wiki Activité

Add support for external network.

Parcourir la source 
Dynamically load external network from libnccl-net.so.
Add init function in networks.
Move PCI scoring to net.cu, only ask transport to provide a path.
Simplify CUDA PCI path detection.
Add dummy external network
Cette révision appartient à :
Sylvain Jeaugey
2018-11-13 10:37:20 -08:00
Parent d7a58cfa58
révision 0d3a20f96d
29 fichiers modifiés avec 437 ajouts et 361 suppressions
Télécharger le Fichier Patch Télécharger le Fichier des Différences Développer tous les fichiers Réduire tous les fichiers
src/include/core.h
+3 -3
Voir le fichier
@@ -271,7 +271,7 @@ struct ncclComm {
while (ret == -1) { \
SYSCHECKVAL(call, name, ret); \
if (ret == -1) { \
INFO(ALL,"Got %s, retrying", strerror(errno)); \
INFO(NCCL_ALL,"Got %s, retrying", strerror(errno)); \
}\
} \
} while (0);
@@ -313,7 +313,7 @@ struct ncclComm {
ncclResult_t res = call; \
if (res != ncclSuccess) { \
/* Print the back trace*/ \
INFO(ALL,"%s:%d -> %d", __FILE__, __LINE__, res); \
INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, res); \
return res; \
} \
} while (0);
@@ -322,7 +322,7 @@ struct ncclComm {
res = call; \
if (res != ncclSuccess) { \
/* Print the back trace*/ \
INFO(ALL,"%s:%d -> %d", __FILE__, __LINE__, res); \
INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, res); \
goto label; \
} \
} while (0);
src/include/debug.h
+21 -62
Voir le fichier
@@ -16,65 +16,24 @@
#include <limits.h>
#include <string.h>
#include "nccl.h"
#include "nccl_net.h"
#define gettid() (pid_t) syscall(SYS_gettid)
typedef enum {NONE=0, VERSION=1, WARN=2, INFO=3, ABORT=4, TRACE=5} DebugLevel;
typedef enum {INIT=1, COLL=2, P2P=4, SHM=8, NET=16, ALL=~0} SubSys;
extern DebugLevel ncclDebugLevel;
extern int ncclDebugLevel;
extern uint64_t ncclDebugMask;
extern pthread_mutex_t ncclDebugOutputLock;
extern FILE *ncclDebugFile;
extern ncclResult_t getHostName(char* hostname, int maxlen);
#define WARN(...) do { \
if (ncclDebugLevel >= WARN) { \
char hostname[1024]; \
getHostName(hostname, 1024); \
int cudaDev; \
cudaGetDevice(&cudaDev); \
pthread_mutex_lock(&ncclDebugOutputLock); \
fprintf(ncclDebugFile,"\n%s:%d:%d [%d] %s:%d NCCL WARN ", hostname, getpid(), gettid(), cudaDev, __FILE__, __LINE__); \
fprintf(ncclDebugFile,__VA_ARGS__); \
fprintf(ncclDebugFile,"\n"); \
fflush(ncclDebugFile); \
pthread_mutex_unlock(&ncclDebugOutputLock); \
if (ncclDebugLevel == ABORT) { fprintf(stderr,"\n%s:%d:%d [%d] %s:%d NCCL ABORT\n", hostname, getpid(), gettid(), cudaDev, __FILE__, __LINE__); abort(); } \
} \
} while(0)
extern void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *filefunc, int line, const char *fmt, ...);
#define INFO(FLAGS, ...) do { \
if (ncclDebugLevel >= INFO && ((FLAGS) & ncclDebugMask)) { \
char hostname[1024]; \
getHostName(hostname, 1024); \
int cudaDev; \
cudaGetDevice(&cudaDev); \
pthread_mutex_lock(&ncclDebugOutputLock); \
fprintf(ncclDebugFile,"%s:%d:%d [%d] NCCL INFO ", hostname, getpid(), gettid(), cudaDev); \
fprintf(ncclDebugFile,__VA_ARGS__);fprintf(ncclDebugFile,"\n"); \
fflush(ncclDebugFile); \
pthread_mutex_unlock(&ncclDebugOutputLock); \
} \
} while(0)
#define WARN(...) ncclDebugLog(NCCL_LOG_WARN, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__)
#define INFO(FLAGS, ...) ncclDebugLog(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__)
#ifdef ENABLE_TRACE
#define TRACE(FLAGS, ...) do { \
if (ncclDebugLevel == TRACE && ((FLAGS) & ncclDebugMask)) { \
char hostname[1024]; \
getHostName(hostname, 1024); \
int cudaDev; \
cudaGetDevice(&cudaDev); \
pthread_mutex_lock(&ncclDebugOutputLock); \
auto delta = std::chrono::high_resolution_clock::now() - ncclEpoch; \
double timestamp = std::chrono::duration_cast<std::chrono::duration<double>>(delta).count()*1000; \
fprintf(ncclDebugFile,"%s:%d:%d [%d] %f %s:%d NCCL TRACE ", hostname, getpid(), gettid(), cudaDev, timestamp, __func__, __LINE__); \
fprintf(ncclDebugFile,__VA_ARGS__);fprintf(ncclDebugFile,"\n"); \
fflush(ncclDebugFile); \
pthread_mutex_unlock(&ncclDebugOutputLock); \
} \
} while(0)
#define TRACE(FLAGS, ...) ncclDebugLog(NCCL_LOG_TRACE, (FLAGS), __func__, __LINE__, __VA_ARGS__)
extern std::chrono::high_resolution_clock::time_point ncclEpoch;
#else
#define TRACE(...)
#endif
@@ -84,17 +43,17 @@ extern std::chrono::high_resolution_clock::time_point ncclEpoch;
static inline void initDebug() {
const char* nccl_debug = getenv("NCCL_DEBUG");
if (nccl_debug == NULL) {
ncclDebugLevel = NONE;
ncclDebugLevel = NCCL_LOG_NONE;
} else if (strcasecmp(nccl_debug, "VERSION") == 0) {
ncclDebugLevel = VERSION;
ncclDebugLevel = NCCL_LOG_VERSION;
} else if (strcasecmp(nccl_debug, "WARN") == 0) {
ncclDebugLevel = WARN;
ncclDebugLevel = NCCL_LOG_WARN;
} else if (strcasecmp(nccl_debug, "INFO") == 0) {
ncclDebugLevel = INFO;
ncclDebugLevel = NCCL_LOG_INFO;
} else if (strcasecmp(nccl_debug, "ABORT") == 0) {
ncclDebugLevel = ABORT;
ncclDebugLevel = NCCL_LOG_ABORT;
} else if (strcasecmp(nccl_debug, "TRACE") == 0) {
ncclDebugLevel = TRACE;
ncclDebugLevel = NCCL_LOG_TRACE;
}
/* Parse the NCCL_DEBUG_SUBSYS env var
@@ -109,17 +68,17 @@ static inline void initDebug() {
uint64_t mask = 0;
if (subsys[0] == '^') { invert = 1; subsys++; }
if (strcasecmp(subsys, "INIT") == 0) {
mask = INIT;
mask = NCCL_INIT;
} else if (strcasecmp(subsys, "COLL") == 0) {
mask = COLL;
mask = NCCL_COLL;
} else if (strcasecmp(subsys, "P2P") == 0) {
mask = P2P;
mask = NCCL_P2P;
} else if (strcasecmp(subsys, "SHM") == 0) {
mask = SHM;
mask = NCCL_SHM;
} else if (strcasecmp(subsys, "NET") == 0) {
mask = NET;
mask = NCCL_NET;
} else if (strcasecmp(subsys, "ALL") == 0) {
mask = ALL;
mask = NCCL_ALL;
}
if (mask) {
if (invert) ncclDebugMask &= ~mask; else ncclDebugMask |= mask;
@@ -133,7 +92,7 @@ static inline void initDebug() {
* NCCL_DEBUG level is > VERSION
*/
const char* nccl_debug_file = getenv("NCCL_DEBUG_FILE");
if (ncclDebugLevel > VERSION && nccl_debug_file != NULL) {
if (ncclDebugLevel > NCCL_LOG_VERSION && nccl_debug_file != NULL) {
int c = 0;
char debug_fn[PATH_MAX+1] = "";
char *dfn = debug_fn;
@@ -164,7 +123,7 @@ static inline void initDebug() {
if (debug_fn[0] != '\0') {
FILE *file = fopen(debug_fn, "w");
if (file != NULL) {
INFO(ALL,"DEBUG file is '%s'", debug_fn);
INFO(NCCL_ALL,"DEBUG file is '%s'", debug_fn);
ncclDebugFile = file;
}
}
src/include/nccl_net.h
+14 -16
Voir le fichier
@@ -9,25 +9,25 @@
#include "nccl.h"
#define NCCL_NET_MAJOR 1
#define NCCL_NET_MINOR 0
#define NCCL_NET_HANDLE_MAXSIZE 64
#define NCCL_PTR_HOST 0x1
#define NCCL_PTR_CUDA 0x2
#define NCCL_MAX_SCORE 0x7
typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel;
typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_ALL=~0} ncclDebugLogSubSys;
typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...);
typedef struct {
// Name of the network (mainly for logs)
const char* name;
// Return the number of network devices along with their scores relative to the
// current CUDA device. The per device score should be a value from 1-7 with a
// higher score representing a better choice for performance.
// This call should allocate the 'scores' array using malloc(3), and it
// will then be freed automatically by NCCL.
ncclResult_t (*devices)(int* ndev, int** scores);
// Initialize the network.
ncclResult_t (*init)(ncclDebugLogger_t logFunction);
// Return the number of adapters.
ncclResult_t (*devices)(int* ndev);
// Return the device path in /sys. NCCL will call free on this path.
ncclResult_t (*pciPath)(int dev, char** path);
// Return whether this device supports host pointers and/or CUDA pointers
// as data from the current GPU. Supported types should be composed with
// NCCL_PTR_HOST and NCCL_PTR_CUDA.
@@ -53,12 +53,10 @@ typedef struct {
ncclResult_t (*closeSend)(void* sendComm);
ncclResult_t (*closeRecv)(void* recvComm);
ncclResult_t (*closeListen)(void* listenComm);
} ncclNet_t;
} ncclNet_v1_t;
extern
#ifdef __cplusplus
"C"
#endif
ncclNet_t* ncclNet;
typedef ncclNet_v1_t ncclNet_t;
#define NCCL_PLUGIN_SYMBOL ncclNetPlugin_v1
#endif // end include guard
src/include/net.h
+3 -2
Voir le fichier
@@ -10,6 +10,7 @@
#include "nccl.h"
#include "nccl_net.h"
extern ncclNet_t* ncclNet;
typedef char ncclNetHandle_t[NCCL_NET_HANDLE_MAXSIZE];
/* Socket Interface Selection type */
@@ -19,7 +20,8 @@ typedef enum { findSubnetIf = -1,
// Translation to external API
static const char* ncclNetName() { return ncclNet->name; }
static ncclResult_t ncclNetDevices(int* ndev, int** scores) { NCCLCHECK(ncclNet->devices(ndev, scores)); return ncclSuccess; }
static ncclResult_t ncclNetDevices(int* ndev) { NCCLCHECK(ncclNet->devices(ndev)); return ncclSuccess; }
static ncclResult_t ncclNetPciPath(int dev, char** path) { NCCLCHECK(ncclNet->pciPath(dev, path)); return ncclSuccess; }
static ncclResult_t ncclNetPtrSupport(int dev, int* supportedTypes) { NCCLCHECK(ncclNet->ptrSupport(dev, supportedTypes)); return ncclSuccess; }
static ncclResult_t ncclNetListen(int dev, void* handle, void** listenComm) { NCCLCHECK(ncclNet->listen(dev, handle, listenComm)); return ncclSuccess; }
static ncclResult_t ncclNetConnect(int dev, void* handle, void** sendComm) { NCCLCHECK(ncclNet->connect(dev, handle, sendComm)); return ncclSuccess; }
@@ -32,7 +34,6 @@ static ncclResult_t ncclNetCloseSend(void* sendComm) { NCCLCHECK(ncclNet->closeS
static ncclResult_t ncclNetCloseRecv(void* recvComm) { NCCLCHECK(ncclNet->closeRecv(recvComm)); return ncclSuccess; }
static ncclResult_t ncclNetCloseListen(void* listenComm) { NCCLCHECK(ncclNet->closeListen(listenComm)); return ncclSuccess; }
extern bool ncclIbSupport();
extern ncclResult_t ncclSocketCreateHandle(void* opaqueHandle, const char* str);
extern ncclNet_t ncclNetIb;
extern ncclNet_t ncclNetSocket;
src/include/param.h
+2 -2
Voir le fichier
@@ -67,10 +67,10 @@ int64_t ncclParam##name() { \
errno = 0; \
int64_t v = strtoll(str, NULL, 0); \
if (errno) { \
INFO(ALL,"Invalid value %s for %s, using default %lu.", str, "NCCL_" env, value); \
INFO(NCCL_ALL,"Invalid value %s for %s, using default %lu.", str, "NCCL_" env, value); \
} else { \
value = v; \
INFO(ALL,"%s set by environment to %lu.", "NCCL_" env, value); \
INFO(NCCL_ALL,"%s set by environment to %lu.", "NCCL_" env, value); \
} \
} \
} \
src/include/socket.h
+7 -7
Voir le fichier
@@ -76,7 +76,7 @@ static int findInterfaces(const char* prefixList, char* names, union socketAddre
if (family != AF_INET && family != AF_INET6)
continue;
TRACE(INIT|NET,"Found interface %s:%s", interface->ifa_name, socketToString(interface->ifa_addr, line));
TRACE(NCCL_INIT|NCCL_NET,"Found interface %s:%s", interface->ifa_name, socketToString(interface->ifa_addr, line));
/* Allow the caller to force the socket family type */
if (sock_family != -1 && family != sock_family)
@@ -106,7 +106,7 @@ static int findInterfaces(const char* prefixList, char* names, union socketAddre
// Store the IP address
int salen = (family == AF_INET) ? sizeof(sockaddr_in) : sizeof(sockaddr_in6);
memcpy(addrs+found, interface->ifa_addr, salen);
INFO(INIT|NET,"NET : Using interface %s:%s", interface->ifa_name, socketToString(interface->ifa_addr, line));
INFO(NCCL_INIT|NCCL_NET,"NET : Using interface %s:%s", interface->ifa_name, socketToString(interface->ifa_addr, line));
found++;
}
}
@@ -183,7 +183,7 @@ static int findInterfaceMatchSubnet(char* ifNames, union socketAddress* localAdd
// Store the interface name
strncpy(ifNames+found*ifNameMaxSize, interface->ifa_name, ifNameMaxSize);
INFO(INIT|NET,"NET : Found interface %s:%s in the same subnet as remote address %s", interface->ifa_name, socketToString(&(localAddrs[found].sa), line), socketToString(&(remoteAddr.sa), line_a));
INFO(NCCL_INIT|NCCL_NET,"NET : Found interface %s:%s in the same subnet as remote address %s", interface->ifa_name, socketToString(&(localAddrs[found].sa), line), socketToString(&(remoteAddr.sa), line_a));
found++;
if (found == maxIfs) break;
}
@@ -333,7 +333,7 @@ static ncclResult_t createListenSocket(int *fd, union socketAddress *localAddr)
#ifdef ENABLE_TRACE
char line[1024];
TRACE(INIT|NET,"Listening on socket %s", socketToString(&localAddr->sa, line));
TRACE(NCCL_INIT|NCCL_NET,"Listening on socket %s", socketToString(&localAddr->sa, line));
#endif
/* Put the socket in listen mode */
@@ -363,7 +363,7 @@ static ncclResult_t connectAddress(int* fd, union socketAddress* remoteAddr) {
#ifdef ENABLE_TRACE
char line[1024];
TRACE(INIT|NET,"Connecting to socket %s", socketToString(&remoteAddr->sa, line));
TRACE(NCCL_INIT|NCCL_NET,"Connecting to socket %s", socketToString(&remoteAddr->sa, line));
#endif
SYSCHECKNTIMES(connect(*fd, &remoteAddr->sa, salen), "connect", RETRY_TIMES, SLEEP_INT, ECONNREFUSED);
@@ -381,7 +381,7 @@ static ncclResult_t socketReceive(int fd, void* ptr, int size) {
return ncclSystemError;
}
if (recvsize == -1) {
INFO(NET,"Recv : got retcode %d, retrying", errno);
INFO(NCCL_NET,"Recv : got retcode %d, retrying", errno);
continue;
}
data += recvsize;
@@ -397,7 +397,7 @@ static ncclResult_t socketSend(int fd, void* ptr, int size) {
int sendsize;
SYSCHECKVAL(write(fd, data, size-offset), "write", sendsize);
if (sendsize == -1) {
INFO(NET,"Send : got retcode %d, retrying", errno);
INFO(NCCL_NET,"Send : got retcode %d, retrying", errno);
continue;
}
data += sendsize;
src/include/topo.h
+8 -35
Voir le fichier
@@ -8,53 +8,26 @@
#define NCCL_TOPO_H_
#include "nccl.h"
#include <limits.h>
#include <stdlib.h>
#include <ctype.h>
#define MAXPATHSIZE 1024
static ncclResult_t getCudaPath(int cudaDev, char** path) {
char busId[16];
CUDACHECK(cudaDeviceGetPCIBusId(busId, 16, cudaDev));
for (int i=0; i<16; i++) busId[i] = tolower(busId[i]);
char busPath[] = "/sys/class/pci_bus/0000:00/device";
char busPath[] = "/sys/class/pci_bus/0000:00/../../0000:00:00.0";
memcpy(busPath+sizeof("/sys/class/pci_bus/")-1, busId, sizeof("0000:00")-1);
char* cudaRpath = realpath(busPath, NULL);
char pathname[MAXPATHSIZE];
strncpy(pathname, cudaRpath, MAXPATHSIZE);
strncpy(pathname+strlen(pathname), "/", MAXPATHSIZE-strlen(pathname));
strncpy(pathname+strlen(pathname), busId, MAXPATHSIZE-strlen(pathname));
free(cudaRpath);
*path = realpath(pathname, NULL);
memcpy(busPath+sizeof("/sys/class/pci_bus/0000:00/../../")-1, busId, sizeof("0000:00:00.0")-1);
*path = realpath(busPath, NULL);
if (*path == NULL) {
WARN("Could not find real path of %s", pathname);
WARN("Could not find real path of %s", busPath);
return ncclSystemError;
}
return ncclSuccess;
}
static ncclResult_t getMlxPath(char* ibName, char** path) {
char devicepath[MAXPATHSIZE];
snprintf(devicepath, MAXPATHSIZE, "/sys/class/infiniband/%s/device", ibName);
*path = realpath(devicepath, NULL);
if (*path == NULL) {
WARN("Could not find real path of %s", devicepath);
return ncclSystemError;
}
return ncclSuccess;
}
static ncclResult_t getSockPath(char* ifName, char** path) {
char devicepath[MAXPATHSIZE];
snprintf(devicepath, MAXPATHSIZE, "/sys/class/net/%s/device", ifName);
*path = realpath(devicepath, NULL);
if (*path == NULL) {
INFO(NET|INIT, "Could not find real path of %s", devicepath);
return ncclSystemError;
}
return ncclSuccess;
}
enum ncclIbPathDist {
enum ncclPathDist {
PATH_PIX = 0,
PATH_PXB = 1,
PATH_PHB = 2,
@@ -74,7 +47,7 @@ static int pciDistance(char* path1, char* path2) {
if (same == 1) score++;
}
}
if (score == 3) return PATH_SOC;
if (score <= 3) return PATH_SOC;
if (score == 4) return PATH_PHB;
if (score == depth-1) return PATH_PIX;
return PATH_PXB;