From ca7a565236ce9353d1fe56026afa5a2b0e7bb9f1 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Fri, 16 Aug 2019 09:06:28 -0700 Subject: [PATCH 01/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a4bbbc6ca..b8b65676b0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # NCCL Tests -These tests check both the performance and the correctness of NCCL operations. They can be compiled against [NCCL](http://github.com/nvidia/nccl) +These tests check both the performance and the correctness of [NCCL](http://github.com/nvidia/nccl) operations. ## Build From a2af1d959dd01a68b55a8f31aa44538d58dc0c35 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Thu, 10 Oct 2019 10:51:05 -0700 Subject: [PATCH 02/10] Update README.md Checks are now fully local, no need to disable them at scale. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8b65676b0..791bed2599 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,9 @@ Run on 8 GPUs (`-g 8`), scanning from 8 Bytes to 128MBytes : $ ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 8 ``` -Run with MPI on 40 processes (potentially on multiple nodes) with 4 GPUs each, disabling checks : +Run with MPI on 40 processes (potentially on multiple nodes) with 4 GPUs each : ```shell -$ mpirun -np 40 ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4 -c 0 +$ mpirun -np 40 ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4 ``` ### Performance From 0f173234bb2837327d806e9e4de9af3dda9a7043 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 16 Dec 2019 16:18:22 -0800 Subject: [PATCH 03/10] Add -L$(MPI_HOME)/lib64 to NVLDFLAGS In some cases, the MPI library is not in $(MPI_HOME)/lib but in $(MPI_HOME)/lib64. For example, on RedHat like Linux system (CentOS, Amazon Linux), and MPI is installed by yum or rpm. Under such circumstance, the current make file will cause failure. This patch address this issue by adding -L$(MPI_HOME)/lib64 to NVLDFLAGS in src/Makefile. Signed-off-by: Wei Zhang --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 034cc672fa..ed723d4210 100644 --- a/src/Makefile +++ b/src/Makefile @@ -52,7 +52,7 @@ endif ifeq ($(MPI), 1) NVCUFLAGS += -DMPI_SUPPORT -I$(MPI_HOME)/include -NVLDFLAGS += -L$(MPI_HOME)/lib -lmpi +NVLDFLAGS += -L$(MPI_HOME)/lib -L$(MPI_HOME)/lib64 -lmpi endif LIBRARIES += curand nccl nvToolsExt NVLDFLAGS += $(LIBRARIES:%=-l%) From ba924dac95c794540b582cdbc480398fb3f64930 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Wed, 3 Jun 2020 15:07:51 -0700 Subject: [PATCH 04/10] Fix #43 : Add .gitignore for build dir --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..a0a013e438 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# See LICENCE.txt for license information +/build From 7a833631b2ba685627aec257627a966f58e26bd4 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Mon, 15 Jun 2020 08:54:21 -0700 Subject: [PATCH 05/10] Remove sm_30 --- src/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index ed723d4210..56d2e6345d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -15,8 +15,7 @@ NVCC = $(CUDA_HOME)/bin/nvcc # Better define NVCC_GENCODE in your environment to the minimal set # of archs to reduce compile time. -NVCC_GENCODE ?= -gencode=arch=compute_30,code=sm_30 \ - -gencode=arch=compute_35,code=sm_35 \ +NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \ -gencode=arch=compute_50,code=sm_50 \ -gencode=arch=compute_60,code=sm_60 \ -gencode=arch=compute_61,code=sm_61 \ From af4fa0f4cf7c2c3db0540da7ac8d6efc1d526635 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Tue, 7 Jan 2020 13:30:19 -0800 Subject: [PATCH 06/10] Fix some memory leaks --- src/common.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common.cu b/src/common.cu index 5a3ae529d6..2c5e38eca3 100644 --- a/src/common.cu +++ b/src/common.cu @@ -302,7 +302,8 @@ testResult_t CheckData(struct threadArgs* args, ncclDataType_t type, ncclRedOp_t printf("%d:%d ", j, dataHost[j]); } printf("\n"); - free(temp); + free(expectedHost); + free(dataHost); } #endif } @@ -351,6 +352,7 @@ testResult_t testStreamSynchronize(int ngpus, cudaStream_t* streams, ncclComm_t* // We might want to let other threads (including NCCL threads) use the CPU. if (idle) pthread_yield(); } + free(done); return testSuccess; } From 07ac716c1ac5999964bd583806ec37e928251119 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Thu, 18 Jun 2020 15:00:05 -0700 Subject: [PATCH 07/10] Fix #47 : compilation error on NCCL<2.7 Return an error when trying to run alltoall test when compiled against NCCL<2.7. --- src/alltoall.cu | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/alltoall.cu b/src/alltoall.cu index aea9370f65..31cfca090d 100644 --- a/src/alltoall.cu +++ b/src/alltoall.cu @@ -64,14 +64,18 @@ testResult_t AlltoAllRunColl(void* sendbuff, void* recvbuff, size_t count, ncclD size_t rankOffset = count * wordSize(type); if (count == 0) return testSuccess; +#if NCCL_MAJOR < 2 || NCCL_MINOR < 7 + printf("NCCL 2.7 or later is needed for alltoall. This test was compiled with %d.%d.\n", NCCL_MAJOR, NCCL_MINOR); + return testNcclError; +#else NCCLCHECK(ncclGroupStart()); for (int r=0; r Date: Fri, 19 Jun 2020 10:40:33 -0700 Subject: [PATCH 08/10] Change all_gather/reduce_scatter algbw to match the documentation. Fix #45 : All_gather and reduce_scatter algorithm bandwidth was computed as time/count*(nranks-1) which is not consistent with the way we compute it for other collectives. This change makes algbw higher; busbw is unchanged. --- src/all_gather.cu | 4 ++-- src/reduce_scatter.cu | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/all_gather.cu b/src/all_gather.cu index cfb2ec356b..f5bc44c57d 100644 --- a/src/all_gather.cu +++ b/src/all_gather.cu @@ -48,10 +48,10 @@ testResult_t AllGatherInitData(struct threadArgs* args, ncclDataType_t type, ncc } void AllGatherGetBw(size_t count, int typesize, double sec, double* algBw, double* busBw, int nranks) { - double baseBw = (double)(count * typesize * (nranks - 1)) / 1.0E9 / sec; + double baseBw = (double)(count * typesize * nranks) / 1.0E9 / sec; *algBw = baseBw; - double factor = 1; + double factor = ((double)(nranks - 1))/((double)nranks); *busBw = baseBw * factor; } diff --git a/src/reduce_scatter.cu b/src/reduce_scatter.cu index 0b1d986952..86e789c15d 100644 --- a/src/reduce_scatter.cu +++ b/src/reduce_scatter.cu @@ -47,10 +47,10 @@ testResult_t ReduceScatterInitData(struct threadArgs* args, ncclDataType_t type, } void ReduceScatterGetBw(size_t count, int typesize, double sec, double* algBw, double* busBw, int nranks) { - double baseBw = (double)(count * typesize * (nranks - 1)) / 1.0E9 / sec; + double baseBw = (double)(count * typesize * nranks) / 1.0E9 / sec; *algBw = baseBw; - double factor = 1; + double factor = ((double)(nranks - 1))/((double)nranks); *busBw = baseBw * factor; } From b2603a2e85436b63b80a02b5fc45df84fe42be7b Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Tue, 23 Jun 2020 18:16:46 -0700 Subject: [PATCH 09/10] Add gencode for CUDA11 --- src/Makefile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Makefile b/src/Makefile index 2440db1672..0770f080ed 100644 --- a/src/Makefile +++ b/src/Makefile @@ -13,14 +13,25 @@ CUDA_LIB ?= $(CUDA_HOME)/lib64 CUDA_INC ?= $(CUDA_HOME)/include NVCC = $(CUDA_HOME)/bin/nvcc +CUDA_VERSION = $(strip $(shell which $(NVCC) >/dev/null && $(NVCC) --version | grep release | sed 's/.*release //' | sed 's/\,.*//')) +CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) + # Better define NVCC_GENCODE in your environment to the minimal set # of archs to reduce compile time. +ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0) +NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ + -gencode=arch=compute_61,code=sm_61 \ + -genncode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_80,code=sm_80 \ + -gencode=arch=compute_80,code=compute_80 +else NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \ -gencode=arch=compute_50,code=sm_50 \ - -gencode=arch=compute_60,code=sm_60 \ + -gencode=arch=compute_60,code=sm_60 \ -gencode=arch=compute_61,code=sm_61 \ - -gencode=arch=compute_70,code=compute_70 \ - -gencode=arch=compute_70,code=sm_70 + -gencode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_70,code=compute_70 +endif NVCUFLAGS := -ccbin $(CXX) $(NVCC_GENCODE) -std=c++11 From afdaf59b3b179af51553614c85925dd2ab0a39a4 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 24 Jun 2020 14:39:22 -0700 Subject: [PATCH 10/10] Fix typo in src/Makefile --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 0770f080ed..52169bb3e1 100644 --- a/src/Makefile +++ b/src/Makefile @@ -21,7 +21,7 @@ CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0) NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ -gencode=arch=compute_61,code=sm_61 \ - -genncode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_70,code=sm_70 \ -gencode=arch=compute_80,code=sm_80 \ -gencode=arch=compute_80,code=compute_80 else