RCCL support (#93)

* Initial support for RCCL

* OMNITRACE_USE_RCCLP + sampling tweaks

- also OMNITRACE_SAMPLING_KEEP_INTERNAL option
- minor modifications to sampling to use keep internal option + discard funlockfile

* Update docker and workflows to download RCCL

* Update CPack DEB with rocprofiler dependency

* Rework rccl into library and library/components folder

- add tpls/rccl/rccl/rccl.h

* Fix timemory includes

* rcclp inline definitions when disabled

* Tweaks to ubuntu-focal-external-rocm

- disable ompt
- enable building testing

* Tweaks to ubuntu-focal-external-rocm

- ctest exclude

* Tweak ubuntu-focal.yml

- remove source /.../setup-env.sh, replace with $GITHUB_ENV

* Fix ubuntu-focal-rocm + OMPI + root

* Improved rocm-smi error handling

- Recover from rocm-smi errors
- Disabling rocm-smi after recovering from errors
- Werror in developer mode
- Remove State::DelayedInit
- Add State::Disabled

* formatting

* Fix merge of OMNITRACE_SAMPLING_KEEP_INTERNAL

* Update RCCL include directory

- based on ROCm version we need with <rccl/rccl.h> or <rccl.h>

* RCCL Testing

- updated tests to use configuration files
- many tests generate a configuration file
- tests how have GPU option
- enable ncclCommCount, disable ncclGetVersion
- add testing for RCCLP via rccl-tests
- working directory of tests is PROJECT_BINARY_DIR
- add nccl/rccl functions to get_whole_function_names
- some clang compiler fixes

* Handle RCCL include w/o HIP

* RCCL requires HIP

* Update OMNITRACE_SAMPLING_CPUS for testing

* Update tests/CMakeLists.txt

* Debug settings

* Install MPI even when USE_MPI=OFF

* exclude printf

* skip mpi tests w/o USE_MPI or USE_MPI_HEADERS

* update ubuntu rocm workflow

* Fix configure env step for ubuntu rocm

[ROCm/rocprofiler-systems commit: 45be03906a]

projects/rocprofiler-systems/examples/CMakeLists.txt

@@ -32,3 +32,4 @@ add_subdirectory(openmp)
 add_subdirectory(mpi)
 add_subdirectory(python)
 add_subdirectory(lulesh)
+add_subdirectory(rccl)

projects/rocprofiler-systems/examples/rccl/CMakeLists.txt

@@ -0,0 +1,61 @@
+cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
+
+project(omnitrace-rccl-example LANGUAGES CXX)
+
+find_package(rccl)
+find_package(hip HINTS ${ROCmVersion_DIR} PATHS ${ROCmVersion_DIR})
+
+function(rccl_message)
+    if("${CMAKE_PROJECT_NAME}" STREQUAL "omnitrace")
+        omnitrace_message(${ARGN})
+    else()
+        message(${ARGN})
+    endif()
+endfunction()
+
+if(hip_FOUND AND rccl_FOUND)
+    include(FetchContent)
+    fetchcontent_declare(
+        rccl-tests GIT_REPOSITORY https://github.com/ROCmSoftwarePlatform/rccl-tests.git)
+
+    # After the following call, the CMake targets defined by googletest and Catch2 will be
+    # available to the rest of the build
+    fetchcontent_makeavailable(rccl-tests)
+
+    get_filename_component(rccl_ROOT_DIR "${rccl_INCLUDE_DIR}" DIRECTORY)
+
+    rccl_message(STATUS "Building rccl-tests...")
+    execute_process(
+        COMMAND make HIP_HOME=${ROCM_PATH} RCCL_HOME=${rccl_ROOT_DIR}
+        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/_deps/rccl-tests-src
+        RESULT_VARIABLE _RCCL_BUILD_RET
+        ERROR_VARIABLE _RCCL_BUILD_ERR
+        OUTPUT_VARIABLE _RCCL_BUILD_OUT
+        OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
+
+    if(NOT _RCCL_BUILD_RET EQUAL 0)
+        rccl_message(${_RCCL_BUILD_OUT})
+        rccl_message(AUTHOR_WARNING "Failed to build rccl-tests: ${_RCCL_BUILD_ERR}")
+    else()
+        file(GLOB RCCL_TEST_EXECUTABLES
+             ${CMAKE_BINARY_DIR}/_deps/rccl-tests-src/build/*_perf)
+        set(_RCCL_TEST_TARGETS)
+
+        foreach(_EXE ${RCCL_TEST_EXECUTABLES})
+            get_filename_component(_EXE_NAME "${_EXE}" NAME)
+            execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${_EXE}
+                                    ${CMAKE_CURRENT_BINARY_DIR}/${_EXE_NAME})
+            add_executable(rccl-tests::${_EXE_NAME} IMPORTED GLOBAL)
+            set_property(
+                TARGET rccl-tests::${_EXE_NAME}
+                PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/${_EXE_NAME})
+            list(APPEND _RCCL_TEST_TARGETS "rccl-tests::${_EXE_NAME}")
+        endforeach()
+
+        set(RCCL_TEST_TARGETS
+            "${_RCCL_TEST_TARGETS}"
+            CACHE INTERNAL "rccl-test targets")
+    endif()
+else()
+    rccl_message(AUTHOR_WARNING "${PROJECT_NAME} skipped. Missing RCCL and/or HIP...")
+endif()

projects/rocprofiler-systems/examples/transpose/CMakeLists.txt

@@ -40,13 +40,16 @@ endif()
 
 add_executable(transpose transpose.cpp)
 
-if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+if(CMAKE_CXX_COMPILER_ID MATCHES "Clang"
+   AND NOT CMAKE_CXX_COMPILER_IS_HIPCC
+   AND NOT HIPCC_EXECUTABLE)
     target_link_libraries(
         transpose
-        PRIVATE
-            $<IF:$<TARGET_EXISTS:omnitrace::omnitrace-compile-options>,omnitrace::omnitrace-compile-options,>
-            $<IF:$<TARGET_EXISTS:hip::host>,hip::host,>
-            $<IF:$<TARGET_EXISTS:hip::device>,hip::device,>)
+        PRIVATE $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>
+                $<TARGET_NAME_IF_EXISTS:hip::host> $<TARGET_NAME_IF_EXISTS:hip::device>)
+elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    target_link_libraries(
+        transpose PRIVATE $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>)
 else()
     target_compile_options(transpose PRIVATE -W -Wall)
 endif()