2.3.5-5
Add support for inter-node communication using sockets and InfiniBand/RoCE. Improve latency. Add support for aggregation. Improve LL/regular tuning. Remove tests as those are now at github.com/nvidia/nccl-tests .
This commit is contained in:
Sylvain Jeaugey
@@ -0,0 +1,26 @@
|
||||
#
|
||||
# Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# See LICENSE.txt for license information
|
||||
#
|
||||
.PHONY : all clean
|
||||
|
||||
default : build
|
||||
build : debian.build txz.build
|
||||
|
||||
BUILDDIR ?= $(abspath ../build)
|
||||
ABSBUILDDIR := $(abspath $(BUILDDIR))
|
||||
TARGETS := debian txz
|
||||
all: ${TARGETS:%=%.build}
|
||||
prep: ${TARGETS:%=%.prep}
|
||||
build: ${TARGETS:%=%.build}
|
||||
clean: ${TARGETS:%=%.clean}
|
||||
|
||||
%.prep:
|
||||
${MAKE} -C $* prep BUILDDIR=${ABSBUILDDIR}
|
||||
|
||||
%.build:
|
||||
${MAKE} -C $* build BUILDDIR=${ABSBUILDDIR}
|
||||
|
||||
%.clean:
|
||||
${MAKE} -C $* clean
|
||||
@@ -0,0 +1,7 @@
|
||||
/*.debhelper.log
|
||||
/*.debhelper
|
||||
/*.substvars
|
||||
/tmp/
|
||||
/files
|
||||
/libnccl1/
|
||||
/libnccl-dev/
|
||||
@@ -0,0 +1,58 @@
|
||||
#
|
||||
# Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# See LICENSE.txt for license information
|
||||
#
|
||||
|
||||
include ../../makefiles/common.mk
|
||||
include ../../makefiles/version.mk
|
||||
BUILDDIR ?= $(abspath ../../build)
|
||||
DEBPREPDIR := $(BUILDDIR)/debian
|
||||
PKGDIR := $(BUILDDIR)/pkg/deb/
|
||||
|
||||
DEBGEN_IN := $(wildcard *.in)
|
||||
DEBGEN := $(DEBGEN_IN:.in=)
|
||||
DEBFILES := compat copyright libnccl-dev.install rules $(DEBGEN)
|
||||
DEBTARGETS := $(patsubst %, $(DEBPREPDIR)/%, $(DEBFILES))
|
||||
|
||||
PKG_TIMESTAMP := $(shell date -R)
|
||||
ARCH := $(shell uname -m)
|
||||
PKG_ARCH ?= $(shell uname -m | sed -e "s/x86_64/amd64/g" | sed -e "s/ppc64le/ppc64el/g")
|
||||
PKG_MULTIARCH ?= $(shell $(CXX) -print-multiarch)
|
||||
ifeq ($(PKG_MULTIARCH),)
|
||||
# Hardwire the PKG_MULTIARCH directory as the RHEL6 distribution agnostic compiler (gcc 4.8.3) doesn't set it
|
||||
PKG_MULTIARCH := $(ARCH)-linux-gnu
|
||||
endif
|
||||
|
||||
prep : $(DEBTARGETS)
|
||||
$(MAKE) -C ../.. lic BUILDDIR=$(BUILDDIR)
|
||||
|
||||
build : prep
|
||||
$(MAKE) -C ../.. src.build BUILDDIR=$(BUILDDIR)
|
||||
@printf "Building Debian package\n"
|
||||
(cd $(BUILDDIR); debuild -eLD_LIBRARY_PATH -uc -us -d -b)
|
||||
mkdir -p $(PKGDIR)
|
||||
mv $(BUILDDIR)/../libnccl*.deb $(PKGDIR)/
|
||||
|
||||
clean:
|
||||
rm -Rf $(DEBPREPDIR) $(PKGDIR)
|
||||
|
||||
$(DEBPREPDIR)/% : %.in
|
||||
@printf "Generating %-35s > %s\n" $< $@
|
||||
mkdir -p $(DEBPREPDIR)
|
||||
sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
||||
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
||||
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
||||
-e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \
|
||||
-e "s/\$${cuda:Major}/$(CUDA_MAJOR)/g" \
|
||||
-e "s/\$${cuda:Minor}/$(CUDA_MINOR)/g" \
|
||||
-e "s/\$${pkg:Revision}/$(PKG_REVISION)/g" \
|
||||
-e "s/\$${pkg:Timestamp}/$(PKG_TIMESTAMP)/g" \
|
||||
-e "s/\$${pkg:Arch}/$(PKG_ARCH)/g" \
|
||||
-e "s/\$${pkg:MultiArch}/$(PKG_MULTIARCH)/g" \
|
||||
$< > $@
|
||||
|
||||
$(DEBPREPDIR)/% : %
|
||||
@printf "Grabbing %-35s > %s\n" $< $@
|
||||
mkdir -p $(DEBPREPDIR)
|
||||
cp -f $< $@
|
||||
@@ -0,0 +1,5 @@
|
||||
nccl (${nccl:Major}.${nccl:Minor}.${nccl:Patch}${nccl:Suffix}-${pkg:Revision}+cuda${cuda:Major}.${cuda:Minor}) trusty; urgency=medium
|
||||
|
||||
* Automatic Debian package from build
|
||||
|
||||
-- cudatools <cudatools@nvidia.com> ${pkg:Timestamp}
|
||||
@@ -0,0 +1 @@
|
||||
9
|
||||
@@ -0,0 +1,30 @@
|
||||
Source: nccl
|
||||
Section: libs
|
||||
Maintainer: cudatools <cudatools@nvidia.com>
|
||||
Priority: optional
|
||||
Build-depends: debhelper(>=9)
|
||||
Standards-Version: 3.9.5
|
||||
|
||||
Package: libnccl${nccl:Major}
|
||||
Section: libs
|
||||
Architecture: ${pkg:Arch}
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}
|
||||
Description: NVIDIA Collectives Communication Library (NCCL) Runtime
|
||||
NCCL (pronounced "Nickel") is a stand-alone library of standard collective
|
||||
communication routines for GPUs, implementing all-reduce, all-gather, reduce,
|
||||
broadcast, and reduce-scatter.
|
||||
It has been optimized to achieve high bandwidth on any platform using PCIe,
|
||||
NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP
|
||||
sockets.
|
||||
|
||||
Package: libnccl-dev
|
||||
Section: libdevel
|
||||
Architecture: ${pkg:Arch}
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}, libnccl${nccl:Major} (= ${binary:Version})
|
||||
Description: NVIDIA Collectives Communication Library (NCCL) Development Files
|
||||
NCCL (pronounced "Nickel") is a stand-alone library of standard collective
|
||||
communication routines for GPUs, implementing all-reduce, all-gather, reduce,
|
||||
broadcast, and reduce-scatter.
|
||||
It has been optimized to achieve high bandwidth on any platform using PCIe,
|
||||
NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP
|
||||
sockets.
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../LICENSE.txt
|
||||
@@ -0,0 +1,9 @@
|
||||
[DEFAULT]
|
||||
debian-branch = master
|
||||
upstream-branch = master
|
||||
|
||||
ignore-new = True
|
||||
|
||||
[git-buildpackage]
|
||||
|
||||
no-purge = True
|
||||
@@ -0,0 +1,3 @@
|
||||
include/nccl.h /usr/include
|
||||
lib/libnccl.so /usr/lib/${pkg:MultiArch}
|
||||
lib/libnccl_static.a /usr/lib/${pkg:MultiArch}
|
||||
@@ -0,0 +1,2 @@
|
||||
lib/libnccl.so.${nccl:Major} /usr/lib/${pkg:MultiArch}
|
||||
lib/libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch} /usr/lib/${pkg:MultiArch}
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/make -f
|
||||
|
||||
%:
|
||||
dh $@ --parallel
|
||||
|
||||
override_dh_auto_install:
|
||||
PREFIX=debian/tmp dh_auto_install
|
||||
|
||||
override_dh_auto_test:
|
||||
# Do not make test
|
||||
|
||||
override_dh_auto_clean:
|
||||
# Do not make clean
|
||||
@@ -0,0 +1 @@
|
||||
3.0 (native)
|
||||
@@ -0,0 +1,62 @@
|
||||
#
|
||||
# Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# See LICENSE.txt for license information
|
||||
#
|
||||
|
||||
include ../../makefiles/common.mk
|
||||
include ../../makefiles/version.mk
|
||||
BUILDDIR ?= $(abspath ../../build)
|
||||
RPMPREPDIR := $(BUILDDIR)/redhat
|
||||
PKGDIR := $(BUILDDIR)/pkg/rpm/
|
||||
|
||||
RPMGEN_IN := $(wildcard *.in)
|
||||
RPMGEN := $(RPMGEN_IN:.in=)
|
||||
RPMFILES := $(RPMGEN)
|
||||
RPMTARGETS := $(patsubst %, $(RPMPREPDIR)/%, $(RPMFILES))
|
||||
|
||||
PKG_TIMESTAMP := $(shell date -R)
|
||||
ARCH := $(shell uname -m)
|
||||
PKG_ARCH ?= $(shell uname -m)
|
||||
PKG_MULTIARCH ?= $(shell $(CXX) -print-multiarch)
|
||||
ifeq ($(PKG_MULTIARCH),)
|
||||
# Hardwire the PKG_MULTIARCH directory as the RHEL6 distribution agnostic compiler (gcc 4.8.3) doesn't set it
|
||||
PKG_MULTIARCH := $(ARCH)-linux-gnu
|
||||
endif
|
||||
|
||||
prep : $(RPMTARGETS)
|
||||
$(MAKE) -C ../.. lic BUILDDIR=$(BUILDDIR)
|
||||
|
||||
build : prep
|
||||
$(MAKE) -C ../.. src.build BUILDDIR=$(BUILDDIR)
|
||||
$(MAKE) -C ../txz build BUILDDIR=$(BUILDDIR)
|
||||
@printf "Building Redhat package\n"
|
||||
mkdir -p $(PKGDIR)
|
||||
rpmbuild --define "_sourcedir $(BUILDDIR)/pkg/txz" \
|
||||
--define "_rpmdir $(PKGDIR)" \
|
||||
--define "_builddir $(PKGDIR)/build/" \
|
||||
--define "_buildrootdir $(PKGDIR)/buildroot/" \
|
||||
-bb $(BUILDDIR)/redhat/nccl.spec
|
||||
|
||||
clean:
|
||||
rm -Rf $(RPMPREPDIR) $(PKGDIR)
|
||||
|
||||
$(RPMPREPDIR)/% : %.in
|
||||
@printf "Generating %-35s > %s\n" $< $@
|
||||
mkdir -p $(RPMPREPDIR)
|
||||
sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
||||
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
||||
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
||||
-e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \
|
||||
-e "s/\$${cuda:Major}/$(CUDA_MAJOR)/g" \
|
||||
-e "s/\$${cuda:Minor}/$(CUDA_MINOR)/g" \
|
||||
-e "s/\$${pkg:Revision}/$(PKG_REVISION)/g" \
|
||||
-e "s/\$${pkg:Timestamp}/$(PKG_TIMESTAMP)/g" \
|
||||
-e "s/\$${pkg:Arch}/$(PKG_ARCH)/g" \
|
||||
-e "s/\$${pkg:MultiArch}/$(PKG_MULTIARCH)/g" \
|
||||
$< > $@
|
||||
|
||||
$(RPMPREPDIR)/% : %
|
||||
@printf "Grabbing %-35s > %s\n" $< $@
|
||||
mkdir -p $(RPMPREPDIR)
|
||||
cp -f $< $@
|
||||
@@ -0,0 +1,73 @@
|
||||
Name: libnccl
|
||||
Version: ${nccl:Major}.${nccl:Minor}.${nccl:Patch}
|
||||
Release: ${pkg:Revision}
|
||||
Summary: NVIDIA Collectives Communication Library (NCCL) Runtime
|
||||
|
||||
Group: Development/Libraries
|
||||
License: BSD
|
||||
URL: http://developer.nvidia.com/nccl
|
||||
Source0: nccl_${nccl:Major}.${nccl:Minor}.${nccl:Patch}${nccl:Suffix}-${pkg:Revision}+cuda${cuda:Major}.${cuda:Minor}_${pkg:Arch}.txz
|
||||
|
||||
%description
|
||||
NCCL (pronounced "Nickel") is a stand-alone library of standard collective
|
||||
communication routines for GPUs, implementing all-reduce, all-gather, reduce,
|
||||
broadcast, and reduce-scatter.
|
||||
It has been optimized to achieve high bandwidth on any platform using PCIe,
|
||||
NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP
|
||||
sockets.
|
||||
|
||||
%package devel
|
||||
Summary: NVIDIA Collectives Communication Library (NCCL) Runtime
|
||||
Group: Development/Libraries
|
||||
%description devel
|
||||
NCCL development files
|
||||
|
||||
%package static
|
||||
Summary: NVIDIA Collectives Communication Library (NCCL) Runtime
|
||||
Group: Development/Libraries
|
||||
%description static
|
||||
NCCL static library
|
||||
|
||||
%define debug_package %{nil}
|
||||
|
||||
%prep
|
||||
%setup -n nccl_${nccl:Major}.${nccl:Minor}.${nccl:Patch}${nccl:Suffix}-${pkg:Revision}+cuda${cuda:Major}.${cuda:Minor}_${pkg:Arch} -q
|
||||
|
||||
%build
|
||||
|
||||
%install
|
||||
rm -rf $RPM_BUILD_ROOT
|
||||
install -m 755 -d $RPM_BUILD_ROOT
|
||||
install -m 755 -d $RPM_BUILD_ROOT/%{_libdir}
|
||||
install -m 755 lib/libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch} $RPM_BUILD_ROOT/%{_libdir}
|
||||
ln -s libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch} $RPM_BUILD_ROOT/%{_libdir}/libnccl.so.${nccl:Major}
|
||||
|
||||
# devel
|
||||
install -m 755 -d $RPM_BUILD_ROOT/%{_includedir}
|
||||
install -m 644 include/nccl.h $RPM_BUILD_ROOT/%{_includedir}
|
||||
ln -s libnccl.so.${nccl:Major} $RPM_BUILD_ROOT/%{_libdir}/libnccl.so
|
||||
|
||||
# static
|
||||
install -m 644 lib/libnccl_static.a $RPM_BUILD_ROOT/%{_libdir}
|
||||
|
||||
%clean
|
||||
rm -rf $RPM_BUILD_ROOT
|
||||
|
||||
%files devel
|
||||
%doc LICENSE.txt
|
||||
%defattr(-,root,root,-)
|
||||
%{_includedir}/nccl.h
|
||||
%{_libdir}/libnccl.so
|
||||
|
||||
%files static
|
||||
%doc LICENSE.txt
|
||||
%defattr(-,root,root,-)
|
||||
%{_libdir}/libnccl_static.a
|
||||
|
||||
%files
|
||||
%doc LICENSE.txt
|
||||
%defattr(-,root,root,-)
|
||||
%{_libdir}/libnccl.so.${nccl:Major}
|
||||
%{_libdir}/libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch}
|
||||
|
||||
%changelog
|
||||
@@ -0,0 +1,39 @@
|
||||
#
|
||||
# Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# See LICENSE.txt for license information
|
||||
#
|
||||
|
||||
include ../../makefiles/common.mk
|
||||
include ../../makefiles/version.mk
|
||||
BUILDDIR ?= $(abspath ../../build)
|
||||
TXZPREPDIR := $(BUILDDIR)/srctxz
|
||||
PKGDIR := $(BUILDDIR)/pkg/srctxz/
|
||||
|
||||
TXZGEN_IN := $(wildcard *.in)
|
||||
TXZGEN := $(TXZGEN_IN:.in=)
|
||||
TXZTARGETS := $(patsubst %, $(TXZPREPDIR)/%, $(TXZGEN))
|
||||
|
||||
PKG_REVISION ?= 3
|
||||
PKG_ARCH := $(shell uname -m)
|
||||
|
||||
prep: $(TXZTARGETS)
|
||||
|
||||
build: prep
|
||||
$(MAKE) -C ../../src clean
|
||||
@printf "Building source tar.xz package\n"
|
||||
(cd $(BUILDDIR); bash srctxz/create_srctxz.sh)
|
||||
mkdir -p $(PKGDIR)
|
||||
mv $(BUILDDIR)/../../nccl-src*.txz $(PKGDIR)
|
||||
|
||||
clean:
|
||||
rm -Rf $(TXZPREPDIR) $(PKGDIR)
|
||||
|
||||
$(TXZPREPDIR)/% : %.in
|
||||
@printf "Generating %-35s > %s\n" $< $@
|
||||
mkdir -p $(TXZPREPDIR)
|
||||
sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
||||
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
||||
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
||||
-e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \
|
||||
$< > $@
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# See LICENSE.txt for license information
|
||||
#
|
||||
|
||||
# To run from $BUILDDIR/
|
||||
|
||||
cd ..
|
||||
NCCLDIR=`basename $PWD`
|
||||
|
||||
echo "Checking for unclean directory ..."
|
||||
git clean -x -i
|
||||
echo "Clean done."
|
||||
echo "Checking for uncommited files ..."
|
||||
if [ "`git status -s | wc -l`" != "0" ]; then
|
||||
git status -s
|
||||
echo "Some changes are not committed yet. Continue ? (Ctrl-C to abort)"
|
||||
read
|
||||
fi
|
||||
|
||||
cd ..
|
||||
NCCL_MAJOR=${nccl:Major}
|
||||
NCCL_MINOR=${nccl:Minor}
|
||||
NCCL_PATCH=${nccl:Patch}
|
||||
NCCL_SUFFIX=${nccl:Suffix}
|
||||
|
||||
NCCLNAME="nccl-src_${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH}${NCCL_SUFFIX}"
|
||||
|
||||
tar --exclude build \
|
||||
--exclude ".git*" \
|
||||
--exclude pkg/srctxz \
|
||||
--transform "s/^$NCCLDIR/$NCCLNAME/" -Jcf $NCCLNAME.txz --owner=0 --group=0 $NCCLDIR
|
||||
@@ -0,0 +1,43 @@
|
||||
#
|
||||
# Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# See LICENSE.txt for license information
|
||||
#
|
||||
|
||||
include ../../makefiles/common.mk
|
||||
include ../../makefiles/version.mk
|
||||
BUILDDIR ?= $(abspath ../../build)
|
||||
TXZPREPDIR := $(BUILDDIR)/txz
|
||||
PKGDIR := $(BUILDDIR)/pkg/txz/
|
||||
|
||||
TXZGEN_IN := $(wildcard *.in)
|
||||
TXZGEN := $(TXZGEN_IN:.in=)
|
||||
TXZTARGETS := $(patsubst %, $(TXZPREPDIR)/%, $(TXZGEN))
|
||||
|
||||
PKG_ARCH := $(shell uname -m)
|
||||
|
||||
prep: $(TXZTARGETS)
|
||||
$(MAKE) -C ../.. lic BUILDDIR=$(BUILDDIR)
|
||||
|
||||
build: prep
|
||||
$(MAKE) -C ../.. src.build BUILDDIR=$(BUILDDIR)
|
||||
@printf "Building tar.xz package\n"
|
||||
(cd $(BUILDDIR); bash txz/create_txz.sh)
|
||||
mkdir -p $(PKGDIR)
|
||||
mv $(BUILDDIR)/../nccl*.txz $(PKGDIR)
|
||||
|
||||
clean:
|
||||
rm -Rf $(TXZPREPDIR) $(PKGDIR)
|
||||
|
||||
$(TXZPREPDIR)/% : %.in
|
||||
@printf "Generating %-35s > %s\n" $< $@
|
||||
mkdir -p $(TXZPREPDIR)
|
||||
sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
||||
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
||||
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
||||
-e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \
|
||||
-e "s/\$${cuda:Major}/$(CUDA_MAJOR)/g" \
|
||||
-e "s/\$${cuda:Minor}/$(CUDA_MINOR)/g" \
|
||||
-e "s/\$${pkg:Revision}/$(PKG_REVISION)/g" \
|
||||
-e "s/\$${pkg:Arch}/$(PKG_ARCH)/g" \
|
||||
$< > $@
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# See LICENSE.txt for license information
|
||||
#
|
||||
|
||||
# To run from $BUILDDIR/
|
||||
|
||||
BUILDDIR=`basename $PWD`
|
||||
|
||||
cd ..
|
||||
NCCL_MAJOR=${nccl:Major}
|
||||
NCCL_MINOR=${nccl:Minor}
|
||||
NCCL_PATCH=${nccl:Patch}
|
||||
NCCL_SUFFIX=${nccl:Suffix}
|
||||
CUDA_MAJOR=${cuda:Major}
|
||||
CUDA_MINOR=${cuda:Minor}
|
||||
PKG_REVISION=${pkg:Revision}
|
||||
PKG_ARCH=${pkg:Arch}
|
||||
|
||||
NCCLNAME="nccl_${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH}${NCCL_SUFFIX}-${PKG_REVISION}+cuda${CUDA_MAJOR}.${CUDA_MINOR}_${PKG_ARCH}"
|
||||
|
||||
tar --transform "s/^$BUILDDIR/$NCCLNAME/" -Jcf $NCCLNAME.txz --owner=0 --group=0 $BUILDDIR/include $BUILDDIR/lib $BUILDDIR/*.txt
|
||||
Reference in New Issue
Block a user