From 6e12e2d6658b86ff5b49affb37c4c632cb94edcd Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Fri, 16 Aug 2019 09:06:28 -0700 Subject: [PATCH 01/10] Update README.md [ROCm/rccl-tests commit: ca7a565236ce9353d1fe56026afa5a2b0e7bb9f1] --- projects/rccl-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rccl-tests/README.md b/projects/rccl-tests/README.md index 7a4bbbc6ca..b8b65676b0 100644 --- a/projects/rccl-tests/README.md +++ b/projects/rccl-tests/README.md @@ -1,6 +1,6 @@ # NCCL Tests -These tests check both the performance and the correctness of NCCL operations. They can be compiled against [NCCL](http://github.com/nvidia/nccl) +These tests check both the performance and the correctness of [NCCL](http://github.com/nvidia/nccl) operations. ## Build From 23326c8d3447d8b53b9418ca7e13d4621ec7f6da Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Thu, 10 Oct 2019 10:51:05 -0700 Subject: [PATCH 02/10] Update README.md Checks are now fully local, no need to disable them at scale. [ROCm/rccl-tests commit: a2af1d959dd01a68b55a8f31aa44538d58dc0c35] --- projects/rccl-tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/rccl-tests/README.md b/projects/rccl-tests/README.md index b8b65676b0..791bed2599 100644 --- a/projects/rccl-tests/README.md +++ b/projects/rccl-tests/README.md @@ -29,9 +29,9 @@ Run on 8 GPUs (`-g 8`), scanning from 8 Bytes to 128MBytes : $ ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 8 ``` -Run with MPI on 40 processes (potentially on multiple nodes) with 4 GPUs each, disabling checks : +Run with MPI on 40 processes (potentially on multiple nodes) with 4 GPUs each : ```shell -$ mpirun -np 40 ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4 -c 0 +$ mpirun -np 40 ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4 ``` ### Performance From c76094c70406c67961782cd9ba25b8e9a87b5eac Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 16 Dec 2019 16:18:22 -0800 Subject: [PATCH 03/10] Add -L$(MPI_HOME)/lib64 to NVLDFLAGS In some cases, the MPI library is not in $(MPI_HOME)/lib but in $(MPI_HOME)/lib64. For example, on RedHat like Linux system (CentOS, Amazon Linux), and MPI is installed by yum or rpm. Under such circumstance, the current make file will cause failure. This patch address this issue by adding -L$(MPI_HOME)/lib64 to NVLDFLAGS in src/Makefile. Signed-off-by: Wei Zhang [ROCm/rccl-tests commit: 0f173234bb2837327d806e9e4de9af3dda9a7043] --- projects/rccl-tests/src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rccl-tests/src/Makefile b/projects/rccl-tests/src/Makefile index 034cc672fa..ed723d4210 100644 --- a/projects/rccl-tests/src/Makefile +++ b/projects/rccl-tests/src/Makefile @@ -52,7 +52,7 @@ endif ifeq ($(MPI), 1) NVCUFLAGS += -DMPI_SUPPORT -I$(MPI_HOME)/include -NVLDFLAGS += -L$(MPI_HOME)/lib -lmpi +NVLDFLAGS += -L$(MPI_HOME)/lib -L$(MPI_HOME)/lib64 -lmpi endif LIBRARIES += curand nccl nvToolsExt NVLDFLAGS += $(LIBRARIES:%=-l%) From e260c673fe5cf5d35df8c1fe2d93bf8e2085f9d5 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Wed, 3 Jun 2020 15:07:51 -0700 Subject: [PATCH 04/10] Fix #43 : Add .gitignore for build dir [ROCm/rccl-tests commit: ba924dac95c794540b582cdbc480398fb3f64930] --- projects/rccl-tests/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 projects/rccl-tests/.gitignore diff --git a/projects/rccl-tests/.gitignore b/projects/rccl-tests/.gitignore new file mode 100644 index 0000000000..a0a013e438 --- /dev/null +++ b/projects/rccl-tests/.gitignore @@ -0,0 +1,4 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# See LICENCE.txt for license information +/build From 0dfae3da2801cecac1faf603e9f5b794a616de10 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Mon, 15 Jun 2020 08:54:21 -0700 Subject: [PATCH 05/10] Remove sm_30 [ROCm/rccl-tests commit: 7a833631b2ba685627aec257627a966f58e26bd4] --- projects/rccl-tests/src/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/projects/rccl-tests/src/Makefile b/projects/rccl-tests/src/Makefile index ed723d4210..56d2e6345d 100644 --- a/projects/rccl-tests/src/Makefile +++ b/projects/rccl-tests/src/Makefile @@ -15,8 +15,7 @@ NVCC = $(CUDA_HOME)/bin/nvcc # Better define NVCC_GENCODE in your environment to the minimal set # of archs to reduce compile time. -NVCC_GENCODE ?= -gencode=arch=compute_30,code=sm_30 \ - -gencode=arch=compute_35,code=sm_35 \ +NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \ -gencode=arch=compute_50,code=sm_50 \ -gencode=arch=compute_60,code=sm_60 \ -gencode=arch=compute_61,code=sm_61 \ From 3a6293b748d5328265d74f0633e2d224af327f21 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Tue, 7 Jan 2020 13:30:19 -0800 Subject: [PATCH 06/10] Fix some memory leaks [ROCm/rccl-tests commit: af4fa0f4cf7c2c3db0540da7ac8d6efc1d526635] --- projects/rccl-tests/src/common.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/rccl-tests/src/common.cu b/projects/rccl-tests/src/common.cu index 5a3ae529d6..2c5e38eca3 100644 --- a/projects/rccl-tests/src/common.cu +++ b/projects/rccl-tests/src/common.cu @@ -302,7 +302,8 @@ testResult_t CheckData(struct threadArgs* args, ncclDataType_t type, ncclRedOp_t printf("%d:%d ", j, dataHost[j]); } printf("\n"); - free(temp); + free(expectedHost); + free(dataHost); } #endif } @@ -351,6 +352,7 @@ testResult_t testStreamSynchronize(int ngpus, cudaStream_t* streams, ncclComm_t* // We might want to let other threads (including NCCL threads) use the CPU. if (idle) pthread_yield(); } + free(done); return testSuccess; } From fcaaf2c4a133e593d73c3ad5ff54845a18527511 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Thu, 18 Jun 2020 15:00:05 -0700 Subject: [PATCH 07/10] Fix #47 : compilation error on NCCL<2.7 Return an error when trying to run alltoall test when compiled against NCCL<2.7. [ROCm/rccl-tests commit: 07ac716c1ac5999964bd583806ec37e928251119] --- projects/rccl-tests/src/alltoall.cu | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/rccl-tests/src/alltoall.cu b/projects/rccl-tests/src/alltoall.cu index aea9370f65..31cfca090d 100644 --- a/projects/rccl-tests/src/alltoall.cu +++ b/projects/rccl-tests/src/alltoall.cu @@ -64,14 +64,18 @@ testResult_t AlltoAllRunColl(void* sendbuff, void* recvbuff, size_t count, ncclD size_t rankOffset = count * wordSize(type); if (count == 0) return testSuccess; +#if NCCL_MAJOR < 2 || NCCL_MINOR < 7 + printf("NCCL 2.7 or later is needed for alltoall. This test was compiled with %d.%d.\n", NCCL_MAJOR, NCCL_MINOR); + return testNcclError; +#else NCCLCHECK(ncclGroupStart()); for (int r=0; r Date: Fri, 19 Jun 2020 10:40:33 -0700 Subject: [PATCH 08/10] Change all_gather/reduce_scatter algbw to match the documentation. Fix #45 : All_gather and reduce_scatter algorithm bandwidth was computed as time/count*(nranks-1) which is not consistent with the way we compute it for other collectives. This change makes algbw higher; busbw is unchanged. [ROCm/rccl-tests commit: ec1b5e22e618d342698fda659efdd5918da6bd9f] --- projects/rccl-tests/src/all_gather.cu | 4 ++-- projects/rccl-tests/src/reduce_scatter.cu | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/rccl-tests/src/all_gather.cu b/projects/rccl-tests/src/all_gather.cu index cfb2ec356b..f5bc44c57d 100644 --- a/projects/rccl-tests/src/all_gather.cu +++ b/projects/rccl-tests/src/all_gather.cu @@ -48,10 +48,10 @@ testResult_t AllGatherInitData(struct threadArgs* args, ncclDataType_t type, ncc } void AllGatherGetBw(size_t count, int typesize, double sec, double* algBw, double* busBw, int nranks) { - double baseBw = (double)(count * typesize * (nranks - 1)) / 1.0E9 / sec; + double baseBw = (double)(count * typesize * nranks) / 1.0E9 / sec; *algBw = baseBw; - double factor = 1; + double factor = ((double)(nranks - 1))/((double)nranks); *busBw = baseBw * factor; } diff --git a/projects/rccl-tests/src/reduce_scatter.cu b/projects/rccl-tests/src/reduce_scatter.cu index 0b1d986952..86e789c15d 100644 --- a/projects/rccl-tests/src/reduce_scatter.cu +++ b/projects/rccl-tests/src/reduce_scatter.cu @@ -47,10 +47,10 @@ testResult_t ReduceScatterInitData(struct threadArgs* args, ncclDataType_t type, } void ReduceScatterGetBw(size_t count, int typesize, double sec, double* algBw, double* busBw, int nranks) { - double baseBw = (double)(count * typesize * (nranks - 1)) / 1.0E9 / sec; + double baseBw = (double)(count * typesize * nranks) / 1.0E9 / sec; *algBw = baseBw; - double factor = 1; + double factor = ((double)(nranks - 1))/((double)nranks); *busBw = baseBw * factor; } From 0624d2cedefec44fd11a186c32100b3fc21ae842 Mon Sep 17 00:00:00 2001 From: Sylvain Jeaugey Date: Tue, 23 Jun 2020 18:16:46 -0700 Subject: [PATCH 09/10] Add gencode for CUDA11 [ROCm/rccl-tests commit: b2603a2e85436b63b80a02b5fc45df84fe42be7b] --- projects/rccl-tests/src/Makefile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/projects/rccl-tests/src/Makefile b/projects/rccl-tests/src/Makefile index 2440db1672..0770f080ed 100644 --- a/projects/rccl-tests/src/Makefile +++ b/projects/rccl-tests/src/Makefile @@ -13,14 +13,25 @@ CUDA_LIB ?= $(CUDA_HOME)/lib64 CUDA_INC ?= $(CUDA_HOME)/include NVCC = $(CUDA_HOME)/bin/nvcc +CUDA_VERSION = $(strip $(shell which $(NVCC) >/dev/null && $(NVCC) --version | grep release | sed 's/.*release //' | sed 's/\,.*//')) +CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) + # Better define NVCC_GENCODE in your environment to the minimal set # of archs to reduce compile time. +ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0) +NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ + -gencode=arch=compute_61,code=sm_61 \ + -genncode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_80,code=sm_80 \ + -gencode=arch=compute_80,code=compute_80 +else NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \ -gencode=arch=compute_50,code=sm_50 \ - -gencode=arch=compute_60,code=sm_60 \ + -gencode=arch=compute_60,code=sm_60 \ -gencode=arch=compute_61,code=sm_61 \ - -gencode=arch=compute_70,code=compute_70 \ - -gencode=arch=compute_70,code=sm_70 + -gencode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_70,code=compute_70 +endif NVCUFLAGS := -ccbin $(CXX) $(NVCC_GENCODE) -std=c++11 From 8b83a414c5221f9d9a29bde83ba97fa859aec22f Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 24 Jun 2020 14:39:22 -0700 Subject: [PATCH 10/10] Fix typo in src/Makefile [ROCm/rccl-tests commit: afdaf59b3b179af51553614c85925dd2ab0a39a4] --- projects/rccl-tests/src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rccl-tests/src/Makefile b/projects/rccl-tests/src/Makefile index 0770f080ed..52169bb3e1 100644 --- a/projects/rccl-tests/src/Makefile +++ b/projects/rccl-tests/src/Makefile @@ -21,7 +21,7 @@ CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0) NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ -gencode=arch=compute_61,code=sm_61 \ - -genncode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_70,code=sm_70 \ -gencode=arch=compute_80,code=sm_80 \ -gencode=arch=compute_80,code=compute_80 else