Riatre Foo
2d8601701d
Add $(INCTARGETS) to build dependencies of %.o and $(DEVICELIB). As there were no dep files during the first build, Make may kick off source compilation before nccl.h got generated, which leads to occasional build failures on systems with high core count. The build failure could be reproduced reliably with a `sleep 5` in $(INCDIR)/nccl.h rule.
137 righe
4.7 KiB
Makefile
137 righe
4.7 KiB
Makefile
#
|
|
# Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved.
|
|
#
|
|
# See LICENSE.txt for license information
|
|
#
|
|
|
|
include ../makefiles/common.mk
|
|
include ../makefiles/version.mk
|
|
|
|
##### src files
|
|
INCEXPORTS := nccl.h nccl_net.h
|
|
LIBSRCFILES := init.cc channel.cc bootstrap.cc transport.cc enqueue.cc group.cc debug.cc proxy.cc \
|
|
misc/nvmlwrap.cc misc/ibvwrap.cc misc/utils.cc misc/argcheck.cc \
|
|
transport/p2p.cc transport/shm.cc transport/net.cc transport/net_socket.cc transport/net_ib.cc transport/coll_net.cc \
|
|
collectives/sendrecv.cc collectives/all_reduce.cc collectives/all_gather.cc collectives/broadcast.cc collectives/reduce.cc collectives/reduce_scatter.cc \
|
|
graph/topo.cc graph/paths.cc graph/search.cc graph/connect.cc graph/rings.cc graph/trees.cc graph/tuning.cc graph/xml.cc
|
|
|
|
##### lib files
|
|
LIBNAME := libnccl.so
|
|
STATICLIBNAME := libnccl_static.a
|
|
##### pkgconfig files
|
|
PKGCONFIGFILE := nccl.pc
|
|
##### dirs
|
|
BUILDDIR ?= $(abspath ../build)
|
|
INCDIR := $(BUILDDIR)/include
|
|
LIBDIR := $(BUILDDIR)/lib
|
|
OBJDIR := $(BUILDDIR)/obj
|
|
PKGDIR := $(BUILDDIR)/lib/pkgconfig
|
|
##### target files
|
|
CUDARTLIB ?= cudart_static
|
|
INCTARGETS := $(INCEXPORTS:%=$(INCDIR)/%)
|
|
LIBSONAME := $(LIBNAME:%=%.$(NCCL_MAJOR))
|
|
LIBTARGET := $(LIBNAME:%=%.$(NCCL_MAJOR).$(NCCL_MINOR).$(NCCL_PATCH))
|
|
STATICLIBTARGET := $(STATICLIBNAME)
|
|
PKGTARGET := $(PKGCONFIGFILE)
|
|
LIBOBJ := $(LIBSRCFILES:%.cc=$(OBJDIR)/%.o)
|
|
DEPFILES := $(LIBOBJ:%.o=%.d)
|
|
LDFLAGS += -L${CUDA_LIB} -l$(CUDARTLIB) -lpthread -lrt -ldl
|
|
|
|
DEVICELIB := $(BUILDDIR)/obj/collectives/device/colldevice.a
|
|
|
|
##### rules
|
|
build : lib staticlib
|
|
|
|
lib : $(INCTARGETS) $(LIBDIR)/$(LIBTARGET) $(PKGDIR)/$(PKGTARGET)
|
|
|
|
staticlib : $(LIBDIR)/$(STATICLIBTARGET)
|
|
|
|
$(DEVICELIB): ALWAYS_REBUILD $(INCTARGETS)
|
|
$(MAKE) -C collectives/device
|
|
|
|
# Empty target to force rebuild
|
|
ALWAYS_REBUILD:
|
|
|
|
-include $(DEPFILES)
|
|
$(LIBDIR)/$(LIBTARGET) $(LIBDIR)/$(STATICLIBTARGET) : $(LIBOBJ)
|
|
|
|
$(INCDIR)/nccl.h : nccl.h.in
|
|
# NCCL_VERSION(X,Y,Z) ((X) * 1000 + (Y) * 100 + (Z))
|
|
@$(eval NCCL_VERSION := $(shell printf "%d%d%02d" $(NCCL_MAJOR) $(NCCL_MINOR) $(NCCL_PATCH)))
|
|
mkdir -p $(INCDIR)
|
|
@printf "Generating %-35s > %s\n" $< $@
|
|
sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
|
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
|
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
|
-e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \
|
|
-e "s/\$${nccl:Version}/$(NCCL_VERSION)/g" \
|
|
$< > $@
|
|
|
|
$(LIBDIR)/$(LIBTARGET): $(LIBOBJ) $(DEVICELIB)
|
|
@printf "Linking %-35s > %s\n" $(LIBTARGET) $@
|
|
mkdir -p $(LIBDIR)
|
|
$(CXX) $(CXXFLAGS) -shared -Wl,--no-as-needed -Wl,-soname,$(LIBSONAME) -o $@ $(LIBOBJ) $(DEVICELIB) $(LDFLAGS)
|
|
ln -sf $(LIBSONAME) $(LIBDIR)/$(LIBNAME)
|
|
ln -sf $(LIBTARGET) $(LIBDIR)/$(LIBSONAME)
|
|
|
|
$(LIBDIR)/$(STATICLIBTARGET): $(LIBOBJ) $(DEVICELIB)
|
|
@printf "Archiving %-35s > %s\n" $(STATICLIBTARGET) $@
|
|
mkdir -p $(LIBDIR)
|
|
$(eval TMP := $(shell mktemp -d))
|
|
cp $(LIBOBJ) $(TMP)
|
|
cd $(TMP) && ar x $(DEVICELIB) && cd -
|
|
ar cr $@ $(LIBOBJ) $(TMP)/*.o
|
|
rm -Rf $(TMP)
|
|
|
|
$(PKGDIR)/nccl.pc : nccl.pc.in
|
|
mkdir -p $(PKGDIR)
|
|
@printf "Generating %-35s > %s\n" $< $@
|
|
sed -e 's|$${nccl:Prefix}|\$(PREFIX)|g' \
|
|
-e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \
|
|
-e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \
|
|
-e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \
|
|
$< > $@
|
|
|
|
$(INCDIR)/%.h : %.h
|
|
@printf "Grabbing %-35s > %s\n" $< $@
|
|
mkdir -p $(INCDIR)
|
|
install -m 644 $< $@
|
|
|
|
$(INCDIR)/nccl_%.h : include/nccl_%.h
|
|
@printf "Grabbing %-35s > %s\n" $< $@
|
|
mkdir -p $(INCDIR)
|
|
install -m 644 $< $@
|
|
|
|
$(PKGDIR)/%.pc : %.pc
|
|
@printf "Grabbing %-35s > %s\n" $< $@
|
|
mkdir -p $(PKGDIR)
|
|
install -m 644 $< $@
|
|
|
|
$(OBJDIR)/%.o : %.cc $(INCTARGETS)
|
|
@printf "Compiling %-35s > %s\n" $< $@
|
|
mkdir -p `dirname $@`
|
|
$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -c $< -o $@
|
|
@$(CXX) -I. -I$(INCDIR) $(CXXFLAGS) -Iinclude -M $< > $(@:%.o=%.d.tmp)
|
|
@sed "0,/^.*:/s//$(subst /,\/,$@):/" $(@:%.o=%.d.tmp) > $(@:%.o=%.d)
|
|
@sed -e 's/.*://' -e 's/\\$$//' < $(@:%.o=%.d.tmp) | fmt -1 | \
|
|
sed -e 's/^ *//' -e 's/$$/:/' >> $(@:%.o=%.d)
|
|
@rm -f $(@:%.o=%.d.tmp)
|
|
|
|
clean :
|
|
$(MAKE) -C collectives/device clean
|
|
rm -rf ${INCDIR} ${LIBDIR} ${PKGDIR} ${OBJDIR}
|
|
|
|
install : lib
|
|
mkdir -p $(PREFIX)/lib
|
|
mkdir -p $(PREFIX)/lib/pkgconfig
|
|
mkdir -p $(PREFIX)/include
|
|
cp -P -v $(BUILDDIR)/lib/lib* $(PREFIX)/lib/
|
|
cp -P -v $(BUILDDIR)/lib/pkgconfig/* $(PREFIX)/lib/pkgconfig/
|
|
cp -v $(BUILDDIR)/include/* $(PREFIX)/include/
|
|
|
|
FILESTOFORMAT := $(shell find . -name ".\#*" -prune -o \( -name "*.cc" -o -name "*.h" \) -print | grep -v -E 'ibvwrap.h|nvmlwrap.h|nccl.h')
|
|
# Note that formatting.mk defines a new target so in order to not overwrite the default target,
|
|
# it shouldn't be included at the top. Also, it uses the above definition of FILESTOFORMAT as well
|
|
# as the BUILDDIR variable.
|
|
include ../makefiles/formatting.mk
|