From 3eda60a031fa03fc3247cecf5c800719ffef25ba Mon Sep 17 00:00:00 2001 From: Sean Karlage Date: Wed, 16 Oct 2024 09:58:50 -0400 Subject: [PATCH] static: Enable true rccl static library build (#1379) * static: Enable true rccl static library build Rccl uses `-fgpu-rdc` to compile, which requires a specialized link command in order to produce a true static library. When "linking" with `amdclang++`, you need to use `--emit-static-lib` and `--hip-link` to get a static library with all gpu code generated. Subsequent links with binaries do not need any special flags to generate gpu code.` Building a static library: ``` $ cmake -DROCM_PATH=$ROCM_PATH -DCMAKE_PREFIX_PATH=$ROCM_PATH -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=off -DCMAKE_POSITION_INDEPENDENT_CODE=on -DAMDGPU_TARGETS=gfx942 -DCMAKE_CXX_COMPILER=$ROCM_PATH/lib/llvm/bin/amdclang++ -DCMAKE_C_COMPILER=$ROCM_PATH/lib/llvm/bin/amdclang .. 2>&1 | tee -a /tmp/build.txt -- Could NOT find GTest (missing: GTEST_LIBRARY GTEST_INCLUDE_DIR GTEST_MAIN_LIBRARY) (Required is at least version "1.11") -- Checking for ROCm support for GPU targets: gfx942 -- Compiling for gfx942 -- Could NOT find GTest (missing: GTEST_LIBRARY GTEST_INCLUDE_DIR GTEST_MAIN_LIBRARY) (Required is at least version "1.11") -- ROCM_PATH found: /opt/rocm -- Compiling with amdclang++ -- HIP compiler: clang -- HIP runtime: rocclr -- amdclang++ executable: /opt/rocm/llvm/bin/amdclang++ -- amdclang++ version: 18.0.0git -- hipconfig executable: /opt/rocm/bin/hipconfig -- amdclang++ HIP version: 6.2.41133 -- ROCm version: 6.2.0 ... $ make -j 32 [ 0%] Updating git_version.cpp if necessary -- Updating git_version.cpp [ 0%] Built target git_version_check [ 0%] Hipifying src/transport/shm.cc -> /home/skarlage/local/rccl/build/hipify/src/transport/shm.cc [ 0%] Hipifying src/bootstrap.cc -> /home/skarlage/local/rccl/build/hipify/src/bootstrap.cc [ 0%] Hipifying src/channel.cc -> /home/skarlage/local/rccl/build/hipify/src/channel.cc [ 1%] Hipifying src/device/all_reduce.h -> /home/skarlage/local/rccl/build/hipify/src/device/all_reduce.h [ 1%] Hipifying src/device/broadcast.h -> /home/skarlage/local/rccl/build/hipify/src/device/broadcast.h [ 1%] Hipifying src/device/all_gather.h -> /home/skarlage/local/rccl/build/hipify/src/device/all_gather.h [ 1%] Hipifying src/device/common.cu -> /home/skarlage/local/rccl/build/hipify/src/device/common.cu.cpp [ 1%] Hipifying src/debug.cc -> /home/skarlage/local/rccl/build/hipify/src/debug.cc [ 1%] Hipifying src/device/alltoall_pivot.h -> /home/skarlage/local/rccl/build/hipify/src/device/alltoall_pivot.h [ 1%] Hipifying src/device/network/unpack/unpack.h -> /home/skarlage/local/rccl/build/hipify/src/device/network/unpack/unpack.h [ 4%] Hipifying src/collectives.cc -> /home/skarlage/local/rccl/build/hipify/src/collectives.cc [ 4%] Hipifying src/device/msccl_kernel_impl.h -> /home/skarlage/local/rccl/build/hipify/src/device/msccl_kernel_impl.h [ 4%] Hipifying src/device/network/unpack/unpack_defs.h -> /home/skarlage/local/rccl/build/hipify/src/device/network/unpack/unpack_defs.h [ 4%] Hipifying src/device/op128.h -> /home/skarlage/local/rccl/build/hipify/src/device/op128.h [ 4%] Hipifying src/device/onerank.cu -> /home/skarlage/local/rccl/build/hipify/src/device/onerank.cu.cpp [ 4%] Hipifying src/device/common.h -> /home/skarlage/local/rccl/build/hipify/src/device/common.h [ 6%] Hipifying src/device/prims_ll.h -> /home/skarlage/local/rccl/build/hipify/src/device/prims_ll.h [ 6%] Hipifying src/device/primitives.h -> /home/skarlage/local/rccl/build/hipify/src/device/primitives.h [ 6%] Hipifying src/device/prims_ll128.h -> /home/skarlage/local/rccl/build/hipify/src/device/prims_ll128.h [ 6%] Hipifying src/device/reduce.h -> /home/skarlage/local/rccl/build/hipify/src/device/reduce.h [ 7%] Hipifying src/device/common_kernel.h -> /home/skarlage/local/rccl/build/hipify/src/device/common_kernel.h [ 7%] Hipifying src/device/reduce_scatter.h -> /home/skarlage/local/rccl/build/hipify/src/device/reduce_scatter.h [ 7%] Hipifying src/device/sendrecv.h -> /home/skarlage/local/rccl/build/hipify/src/device/sendrecv.h [ 7%] Hipifying src/device/prims_simple.h -> /home/skarlage/local/rccl/build/hipify/src/device/prims_simple.h [ 7%] Hipifying src/enqueue.cc -> /home/skarlage/local/rccl/build/hipify/src/enqueue.cc [ 7%] Hipifying src/device/reduce_kernel.h -> /home/skarlage/local/rccl/build/hipify/src/device/reduce_kernel.h [ 7%] Hipifying src/graph/connect.cc -> /home/skarlage/local/rccl/build/hipify/src/graph/connect.cc [ 7%] Hipifying src/graph/rings.h -> /home/skarlage/local/rccl/build/hipify/src/graph/rings.h [ 8%] Hipifying src/graph/rings.cc -> /home/skarlage/local/rccl/build/hipify/src/graph/rings.cc [ 8%] Hipifying src/graph/rome_models.cc -> /home/skarlage/local/rccl/build/hipify/src/graph/rome_models.cc [ 8%] Hipifying src/graph/rome_models.h -> /home/skarlage/local/rccl/build/hipify/src/graph/rome_models.h [ 8%] Hipifying src/graph/paths.cc -> /home/skarlage/local/rccl/build/hipify/src/graph/paths.cc [ 9%] Hipifying src/graph/search.cc -> /home/skarlage/local/rccl/build/hipify/src/graph/search.cc [ 9%] Hipifying src/graph/topo.cc -> /home/skarlage/local/rccl/build/hipify/src/graph/topo.cc ... [100%] Linking CXX static library librccl.a Elapsed time: 270 s. (time), 0.00046 s. (clock) Elapsed time: 0 s. (time), 0.000342 s. (clock) [100%] Built target rccl ``` Static rccl exists: ``` $ file librccl.a librccl.a: current ar archive ``` * Fix up tests Cmake for static builds We also need to fix up the tests CMakeLists.txt to: * Remove the unused `BUILD_STATIC` option * Use `SHARED_LIBS` as a definition of whether we're building static or not. [ROCm/rccl commit: bdf9544c8134c3206671b1ed225c4e52f72bff61] --- projects/rccl/CMakeLists.txt | 31 +++++++++++++++++++++++++++++++ projects/rccl/test/CMakeLists.txt | 10 ++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index 4cdb9bf5a1..3fac74cedc 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -837,6 +837,37 @@ set_property(TARGET rccl PROPERTY RULE_LAUNCH_LINK "${CMAKE_COMMAND} -E time") ## Setup librccl.so version rocm_set_soversion(rccl "1.0") +if(NOT BUILD_SHARED_LIBS) + # To create a static lib with `-fgpu-rdc`, you need `--emit-static-lib` and `--hip-link`. + # You also need to invoke amdclang++ again to trigger GPU code generation. + set(static_link_flags + ${CXXFLAGS} + --hip-link + -fgpu-rdc + --emit-static-lib + ) + + # Find all the libraries we need to link at link time to include them in the clang link + # command line. + get_target_property(rccl_libs rccl LINK_LIBRARIES) + foreach(target ${rccl_libs}) + if(TARGET ${target}) + get_target_property(location ${target} LOCATION) + if(location) + LIST(APPEND static_link_flags -l${location}) + endif() + endif() + endforeach() + + foreach(target ${GPU_TARGETS}) + list(APPEND static_link_flags --offload-arch=${target}) + endforeach() + list(JOIN static_link_flags " " flags_str) + + # Invoking amdclang++ this way will produce a static archive, so just override ARCHIVE_CREATE. + set(CMAKE_CXX_ARCHIVE_CREATE " ${flags_str} -o ") +endif() + # Install settings #================================================================================================== ## Specify install targets diff --git a/projects/rccl/test/CMakeLists.txt b/projects/rccl/test/CMakeLists.txt index 7fcbea1c08..48fcfe2746 100644 --- a/projects/rccl/test/CMakeLists.txt +++ b/projects/rccl/test/CMakeLists.txt @@ -86,14 +86,12 @@ if(BUILD_TESTS) # rccl-UnitTests using static library of rccl requires passing rccl # through -l and -L instead of command line input. - if(BUILD_STATIC) + if(BUILD_SHARED_LIBS) + target_link_libraries(rccl-UnitTests PRIVATE rccl) + set_property(TARGET rccl-UnitTests PROPERTY INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${ROCM_PATH}/lib;${CMAKE_BINARY_DIR}") + else() add_dependencies(rccl-UnitTests rccl) target_link_libraries(rccl-UnitTests PRIVATE dl rt numa -lrccl -L${CMAKE_BINARY_DIR} -lrocm_smi64 -L${ROCM_PATH}/lib -L${ROCM_PATH}/rocm_smi/lib) - else() - target_link_libraries(rccl-UnitTests PRIVATE rccl) - endif() - if(BUILD_SHARED_LIBS) - set_property(TARGET rccl-UnitTests PROPERTY INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${ROCM_PATH}/lib;${CMAKE_BINARY_DIR}") endif() set_property(TARGET rccl-UnitTests PROPERTY BUILD_RPATH "${CMAKE_BINARY_DIR};${ROCM_PATH}/lib") rocm_install(TARGETS rccl-UnitTests COMPONENT tests)