AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2.5.6-1 (#255)

Browse Source 
Add LL128 Protocol.

Rewrite the topology detection and tree/ring creation (#179). Improve
tree performance by sending/receiving from different GPUs. Add
model-based tuning to switch between the different algorithms and
protocols.

Rework P2P/SHM detection in containers (#155, #248).

Detect duplicated devices and return an error (#231).

Add tuning for GCP

[ROCm/rccl commit: 299c554dcc]
This commit is contained in:
Sylvain Jeaugey
2019-11-19 14:57:39 -08:00
committed by GitHub
parent 221b65bee1
commit 71560fd67b
65 changed files with 4783 additions and 2832 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/include/net.h
+31 -1
View File
@@ -17,7 +17,6 @@ typedef char ncclNetHandle_t[NCCL_NET_HANDLE_MAXSIZE];
static const char* ncclNetName() { return ncclNet->name; }
static ncclResult_t ncclNetDevices(int* ndev) { NCCLCHECK(ncclNet->devices(ndev)); return ncclSuccess; }
static ncclResult_t ncclNetPciPath(int dev, char** path) { NCCLCHECK(ncclNet->pciPath(dev, path)); return ncclSuccess; }
static ncclResult_t ncclNetPtrSupport(int dev, int* supportedTypes) { NCCLCHECK(ncclNet->ptrSupport(dev, supportedTypes)); return ncclSuccess; }
static ncclResult_t ncclNetListen(int dev, void* handle, void** listenComm) { NCCLCHECK(ncclNet->listen(dev, handle, listenComm)); return ncclSuccess; }
static ncclResult_t ncclNetConnect(int dev, void* handle, void** sendComm) { NCCLCHECK(ncclNet->connect(dev, handle, sendComm)); return ncclSuccess; }
static ncclResult_t ncclNetAccept(void* listenComm, void** recvComm) { NCCLCHECK(ncclNet->accept(listenComm, recvComm)); return ncclSuccess; }
@@ -31,6 +30,37 @@ static ncclResult_t ncclNetCloseSend(void* sendComm) { NCCLCHECK(ncclNet->closeS
static ncclResult_t ncclNetCloseRecv(void* recvComm) { NCCLCHECK(ncclNet->closeRecv(recvComm)); return ncclSuccess; }
static ncclResult_t ncclNetCloseListen(void* listenComm) { NCCLCHECK(ncclNet->closeListen(listenComm)); return ncclSuccess; }
#define GPU_BUF_SIZE (2*1024*1024)
static ncclResult_t ncclNetPtrSupport(int dev, int* supportedTypes) {
int support;
NCCLCHECK(ncclNet->ptrSupport(dev, &support));
*supportedTypes = support & ~NCCL_PTR_CUDA;
// The network supports GPU Direct RDMA ; verify the GPU supports it as well.
if (support & NCCL_PTR_CUDA) {
void *lComm = NULL, *sComm = NULL, *rComm = NULL;
ncclNetHandle_t handle;
void* gpuPtr = NULL;
void* mHandle = NULL;
ncclResult_t res;
NCCLCHECKGOTO(ncclNetListen(dev, &handle, &lComm), res, cleanup);
NCCLCHECKGOTO(ncclNetConnect(dev, &handle, &sComm), res, cleanup);
NCCLCHECKGOTO(ncclNetAccept(lComm, &rComm), res, cleanup);
CUDACHECKGOTO(cudaMalloc(&gpuPtr, GPU_BUF_SIZE), res, cleanup);
NOWARN(ncclNetRegMr(sComm, gpuPtr, GPU_BUF_SIZE, NCCL_PTR_CUDA, &mHandle), res);
if (res != ncclSuccess) goto cleanup;
NCCLCHECKGOTO(ncclNetDeregMr(sComm, mHandle), res, cleanup);
NCCLCHECKGOTO(ncclNetRegMr(rComm, gpuPtr, GPU_BUF_SIZE, NCCL_PTR_CUDA, &mHandle), res, cleanup);
NCCLCHECKGOTO(ncclNetDeregMr(rComm, mHandle), res, cleanup);
*supportedTypes |= NCCL_PTR_CUDA;
cleanup:
if (gpuPtr) cudaFree(gpuPtr);
if (rComm) ncclNetCloseRecv(rComm);
if (sComm) ncclNetCloseSend(sComm);
if (lComm) ncclNetCloseListen(lComm);
}
return ncclSuccess;
}
extern ncclNet_t ncclNetIb;
extern ncclNet_t ncclNetSocket;