From 0a3a397481be861f3096cecde07e1e252ab65ca4 Mon Sep 17 00:00:00 2001
From: Aaron Enye Shi <enye.shi@gmail.com>
Date: Mon, 31 Aug 2020 16:10:09 +0000
Subject: [PATCH] Add RCCL Static Lib Creation with -fgpu-rdc

RCCL uses -fgpu-rdc to compile its source objects. When linking
the RCCL static library, the link and archive step must do through
hipcc and uses the flag --emit-static-lib. When compiling
UnitTests, the librccl.a must be consumed through -l and -L.


[ROCm/rccl commit: 958b213428779ef136195058ffad7f0617e65ec4]
---
 projects/rccl/CMakeLists.txt      |  9 +++++++++
 projects/rccl/test/CMakeLists.txt | 12 +++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt
index 3bd1ce0c74..a1d3db7d5d 100644
--- a/projects/rccl/CMakeLists.txt
+++ b/projects/rccl/CMakeLists.txt
@@ -176,6 +176,15 @@ if("${HIP_COMPILER}" MATCHES "clang")
     target_compile_options(rccl PRIVATE -parallel-jobs=4 PRIVATE -Wno-format-nonliteral)
     target_link_libraries(rccl PRIVATE -parallel-jobs=4)
   endif()
+
+  # RCCL static lib uses -fgpu-rdc which requires hipcc as the linker and archiver
+  if(BUILD_STATIC)
+    target_link_libraries(rccl PRIVATE --emit-static-lib)
+    set(CMAKE_AR "${hipcc_executable}")
+    get_property(link_libraries TARGET rccl PROPERTY LINK_LIBRARIES)
+    string (REPLACE ";" " " LINK_PROPS "${link_libraries}")
+    set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -o <TARGET> ${LINK_PROPS} <LINK_FLAGS> <OBJECTS>")
+  endif()
 endif()
 
 if("${HIP_COMPILER}" MATCHES "hcc")
diff --git a/projects/rccl/test/CMakeLists.txt b/projects/rccl/test/CMakeLists.txt
index 456b13f25b..c012f32d76 100644
--- a/projects/rccl/test/CMakeLists.txt
+++ b/projects/rccl/test/CMakeLists.txt
@@ -25,7 +25,17 @@ if(BUILD_TESTS)
 
   add_executable(UnitTests ${TEST_SOURCES})
   target_include_directories(UnitTests PRIVATE /opt/rocm ${GTEST_INCLUDE_DIRS})
-  target_link_libraries(UnitTests PRIVATE ${GTEST_BOTH_LIBRARIES}  PRIVATE rccl)
+  target_link_libraries(UnitTests PRIVATE ${GTEST_BOTH_LIBRARIES})
+
+  # UnitTests using static library of rccl requires passing rccl
+  # through -l and -L instead of command line input.
+  if(BUILD_STATIC)
+    add_dependencies(UnitTests rccl)
+    target_link_libraries(UnitTests PRIVATE dl rt numa -lrccl -L${CMAKE_BINARY_DIR})
+    target_link_libraries(UnitTests PRIVATE amdhip64 amd_comgr hsa-runtime64::hsa-runtime64)
+  else()
+    target_link_libraries(UnitTests PRIVATE rccl)
+  endif()
 else()
   message("Not building unit tests")
 endif()