Yiltan Temucin
@@ -25,30 +25,15 @@ cmake_minimum_required(VERSION 3.16.3 FATAL_ERROR)
|
||||
###############################################################################
|
||||
# AVOID IN SOURCE BUILD
|
||||
###############################################################################
|
||||
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND CMAKE_CURRENT_SOURCE_DIR STREQUAL
|
||||
CMAKE_SOURCE_DIR)
|
||||
set(MSG "")
|
||||
message(STATUS "Warning! Building from the source directory is not recommended")
|
||||
message(STATUS "If unintended, please remove 'CMakeCache.txt' and 'CMakeFiles'")
|
||||
message(STATUS "and build from a separate directory")
|
||||
message(FATAL_ERROR "In-source build")
|
||||
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND
|
||||
CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
|
||||
set(MSG "")
|
||||
message(STATUS "Warning! Building from the source directory is not recommended")
|
||||
message(STATUS "If unintended, please remove 'CMakeCache.txt' and 'CMakeFiles'")
|
||||
message(STATUS "and build from a separate directory")
|
||||
message(FATAL_ERROR "In-source build")
|
||||
endif()
|
||||
|
||||
###############################################################################
|
||||
# SYSTEM INFO
|
||||
###############################################################################
|
||||
cmake_host_system_information(RESULT LOCALHOST QUERY FQDN)
|
||||
message(STATUS "Hostname: ${LOCALHOST}")
|
||||
|
||||
###############################################################################
|
||||
# VERSIONING INFO DERIVED FROM FILE
|
||||
###############################################################################
|
||||
file(READ "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" FULL_VERSION_STRING LIMIT_COUNT 1)
|
||||
string(REGEX REPLACE "(\n|\r)" "" FULL_VERSION_STRING "${FULL_VERSION_STRING}")
|
||||
set(ROCSHMEM_FULL_VERSION "${FULL_VERSION_STRING}")
|
||||
string(REGEX REPLACE "([0-9]+)\.([0-9]+)\.([0-9]+)(.*)" "\\1.\\2.\\3" ROCSHMEM_VERSION
|
||||
"${FULL_VERSION_STRING}")
|
||||
|
||||
###############################################################################
|
||||
# CONFIGURATION OPTIONS
|
||||
###############################################################################
|
||||
@@ -71,106 +56,85 @@ option(USE_HOST_SIDE_HDP_FLUSH "Use a polling thread to flush the HDP cache on t
|
||||
option(BUILD_FUNCTIONAL_TESTS "Build the functional tests" ON)
|
||||
option(BUILD_SOS_TESTS "Build the host-facing tests" OFF)
|
||||
option(BUILD_UNIT_TESTS "Build the unit tests" ON)
|
||||
|
||||
set(ROCM_PATH "" CACHE PATH "ROCm path to use")
|
||||
option(BUILD_LOCAL_GPU_TARGET_ONLY "Build only for GPUs detected on this machine" OFF)
|
||||
|
||||
configure_file(cmake/rocshmem_config.h.in rocshmem_config.h)
|
||||
|
||||
###############################################################################
|
||||
# Validate user passed options
|
||||
###############################################################################
|
||||
if(ROCM_PATH)
|
||||
set(HIPCONFIG_PATH "${ROCM_PATH}/bin/hipconfig")
|
||||
execute_process(
|
||||
COMMAND "${HIPCONFIG_PATH}" --version
|
||||
RESULT_VARIABLE HIPCONFIG_RETURN
|
||||
OUTPUT_VARIABLE ROCM_VERSION
|
||||
ERROR_QUIET
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
if(HIPCONFIG_RETURN EQUAL 0)
|
||||
message(STATUS "Using ROCm with version: ${ROCM_VERSION}")
|
||||
else()
|
||||
message(FATAL_ERROR "ROCm at ${ROCM_PATH} is missing the hipconfig binary. "
|
||||
"Use -DROCM_PATH=/path/to/rocm or ensure ROCm's hipconfig is in your path.")
|
||||
endif()
|
||||
else()
|
||||
set(HIPCONFIG_PATH "hipconfig")
|
||||
execute_process(
|
||||
COMMAND "${HIPCONFIG_PATH}" --rocmpath
|
||||
RESULT_VARIABLE HIPCONFIG_RETURN
|
||||
OUTPUT_VARIABLE ROCM_PATH
|
||||
ERROR_QUIET
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
if(HIPCONFIG_RETURN EQUAL 0)
|
||||
message(STATUS "Found ROCm at: ${ROCM_PATH}")
|
||||
else()
|
||||
message(FATAL_ERROR "Ensure hipconfig is in your path or use -DROCM_PATH=/path/to/rocm.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
###############################################################################
|
||||
# GLOBAL COMPILE FLAGS
|
||||
###############################################################################
|
||||
if (DEFINED ENV{ROCM_PATH})
|
||||
set(ROCM_PATH "$ENV{ROCM_PATH}" CACHE STRING "ROCm install directory")
|
||||
else()
|
||||
set(ROCM_PATH "/opt/rocm" CACHE STRING "ROCm install directory")
|
||||
endif()
|
||||
|
||||
if (NOT DEFINED CMAKE_CXX_COMPILER)
|
||||
set(CMAKE_CXX_COMPILER ${ROCM_PATH}/bin/hipcc)
|
||||
set(CMAKE_CXX_COMPILER ${ROCM_PATH}/bin/hipcc)
|
||||
endif()
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -ggdb")
|
||||
|
||||
###############################################################################
|
||||
# MODULE SEARCH PATH
|
||||
###############################################################################
|
||||
set(
|
||||
CMAKE_MODULE_PATH
|
||||
${CMAKE_MODULE_PATH}
|
||||
"${CMAKE_SOURCE_DIR}/cmake/Modules"
|
||||
"{ROCM_PATH}/hip/cmake"
|
||||
"{ROCM_PATH}/rocclr/lib/cmake/rocclr"
|
||||
)
|
||||
find_package(ROCM 0.8 REQUIRED PATHS ${ROCM_PATH})
|
||||
set(ROCMCHECKS_WARN_TOOLCHAIN_VAR OFF)
|
||||
|
||||
find_package(ROCmCMakeBuildTools)
|
||||
include(ROCMCreatePackage)
|
||||
include(ROCMInstallTargets)
|
||||
include(ROCMCheckTargetIds)
|
||||
|
||||
###############################################################################
|
||||
# PROJECT
|
||||
###############################################################################
|
||||
project(
|
||||
rocshmem
|
||||
VERSION ${ROCSHMEM_VERSION}
|
||||
LANGUAGES CXX
|
||||
DESCRIPTION "rocSHMEM"
|
||||
HOMEPAGE_URL "https://github.com/ROCm/rocSHMEM")
|
||||
rocm_setup_version(VERSION 1.6.0)
|
||||
|
||||
project(rocshmem CXX)
|
||||
|
||||
###############################################################################
|
||||
# DEFAULT BUILD TYPE
|
||||
# SET GPU ARCHITECTURES
|
||||
###############################################################################
|
||||
if(NOT CMAKE_BUILD_TYPE)
|
||||
set(
|
||||
CMAKE_BUILD_TYPE
|
||||
"Release"
|
||||
CACHE
|
||||
STRING
|
||||
"build type: Release, Debug, RelWithDebInfo, MinSizeRel"
|
||||
FORCE
|
||||
)
|
||||
message(STATUS "CMAKE_BUILD_TYPE unspecified: using ${CMAKE_BUILD_TYPE}")
|
||||
include(cmake/rocm_local_targets.cmake)
|
||||
|
||||
set(DEFAULT_GPUS
|
||||
gfx90a
|
||||
gfx942)
|
||||
|
||||
if (BUILD_LOCAL_GPU_TARGET_ONLY)
|
||||
message(STATUS "Building only for local GPU target")
|
||||
if (COMMAND rocm_local_targets)
|
||||
rocm_local_targets(DEFAULT_GPUS)
|
||||
else()
|
||||
message(WARNING "Unable to determine local GPU targets. Falling back to default GPUs.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
|
||||
set(CMAKE_INSTALL_PREFIX
|
||||
"/opt/lib/rocshmem"
|
||||
CACHE PATH "default install path" FORCE)
|
||||
set(GPU_TARGETS "${DEFAULT_GPUS}" CACHE STRING
|
||||
"Target default GPUs if GPU_TARGETS is not defined.")
|
||||
|
||||
if (COMMAND rocm_check_target_ids)
|
||||
message(STATUS "Checking for ROCm support for GPU targets: " "${GPU_TARGETS}")
|
||||
rocm_check_target_ids(SUPPORTED_GPUS TARGETS ${GPU_TARGETS})
|
||||
else()
|
||||
message(WARNING "Unable to check for supported GPU targets. Falling back to default GPUs.")
|
||||
set(SUPPORTED_GPUS ${DEFAULT_GPUS})
|
||||
endif()
|
||||
message(STATUS "Installation path: ${CMAKE_INSTALL_PREFIX}")
|
||||
|
||||
set(COMPILING_TARGETS "${SUPPORTED_GPUS}" CACHE STRING "GPU targets to compile for.")
|
||||
message(STATUS "Compiling for ${COMPILING_TARGETS}")
|
||||
|
||||
foreach(target ${COMPILING_TARGETS})
|
||||
list(APPEND static_link_flags --offload-arch=${target})
|
||||
endforeach()
|
||||
list(JOIN static_link_flags " " flags_str)
|
||||
add_compile_options(${flags_str})
|
||||
|
||||
###############################################################################
|
||||
# CREATE ROCSHMEM LIBRARY
|
||||
###############################################################################
|
||||
add_library(
|
||||
${PROJECT_NAME}
|
||||
STATIC
|
||||
""
|
||||
)
|
||||
add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
|
||||
add_library(${PROJECT_NAME})
|
||||
add_library(roc::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
|
||||
|
||||
###############################################################################
|
||||
# INCLUDE DIRECTORIES
|
||||
@@ -183,21 +147,10 @@ target_include_directories(
|
||||
$<INSTALL_INTERFACE:include>
|
||||
)
|
||||
|
||||
###############################################################################
|
||||
# SPECIFY PUBLIC HEADER FILES
|
||||
###############################################################################
|
||||
set_target_properties(
|
||||
${PROJECT_NAME}
|
||||
PROPERTIES
|
||||
PUBLIC_HEADER
|
||||
"${CMAKE_BINARY_DIR}/rocshmem_config.h"
|
||||
)
|
||||
|
||||
###############################################################################
|
||||
# SUBDIRECTORY TARGETS
|
||||
###############################################################################
|
||||
add_subdirectory(src)
|
||||
add_subdirectory(docs)
|
||||
add_subdirectory(tests)
|
||||
add_subdirectory(examples)
|
||||
|
||||
@@ -241,19 +194,19 @@ target_link_libraries(
|
||||
# IBVERBS
|
||||
###############################################################################
|
||||
IF (USE_GPU_IB)
|
||||
find_package(Ibverbs REQUIRED)
|
||||
find_package(Ibverbs REQUIRED)
|
||||
|
||||
target_include_directories(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
${IBVERBS_INCLUDE_DIRS}
|
||||
)
|
||||
target_include_directories(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
${IBVERBS_INCLUDE_DIRS}
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
${IBVERBS_LIBRARIES}
|
||||
)
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
${IBVERBS_LIBRARIES}
|
||||
)
|
||||
ENDIF()
|
||||
|
||||
###############################################################################
|
||||
@@ -274,165 +227,33 @@ target_link_libraries(
|
||||
${MPI_mpicxx_LIBRARY}
|
||||
)
|
||||
|
||||
###############################################################################
|
||||
# MODULEFILE
|
||||
###############################################################################
|
||||
set(MOD_INSTALL_PATH
|
||||
"${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_DATAROOTDIR}/modulefiles"
|
||||
CACHE STRING "Install path for modulefile")
|
||||
message(STATUS "Modulefile install path: ${MOD_INSTALL_PATH}")
|
||||
|
||||
set(moduleFileTemplate "${PROJECT_NAME}.lua.in")
|
||||
|
||||
configure_file(
|
||||
${PROJECT_SOURCE_DIR}/cmake/${moduleFileTemplate}
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}/${ROCSHMEM_FULL_VERSION}.lua
|
||||
@ONLY)
|
||||
|
||||
# Crusher
|
||||
if(LOCALHOST MATCHES ".*\.crusher\.olcf\.ornl\.gov")
|
||||
list(APPEND CMAKE_MESSAGE_INDENT " ")
|
||||
message(STATUS "Using crusher-specific modulefile modification")
|
||||
file(READ ${PROJECT_SOURCE_DIR}/cmake/modfile.crusher.mod mod_additions)
|
||||
file(
|
||||
APPEND
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}/${ROCSHMEM_FULL_VERSION}.lua
|
||||
${mod_additions})
|
||||
list(POP_BACK CMAKE_MESSAGE_INDENT)
|
||||
endif()
|
||||
|
||||
# Frontier
|
||||
if(LOCALHOST MATCHES ".*\.frontier\.olcf\.ornl\.gov")
|
||||
list(APPEND CMAKE_MESSAGE_INDENT " ")
|
||||
message(STATUS "Using frontier-specific modulefile modification")
|
||||
file(READ ${PROJECT_SOURCE_DIR}/cmake/modfile.frontier.mod mod_additions)
|
||||
file(
|
||||
APPEND
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}/${ROCSHMEM_FULL_VERSION}.lua
|
||||
${mod_additions})
|
||||
list(POP_BACK CMAKE_MESSAGE_INDENT)
|
||||
endif()
|
||||
|
||||
# hpcfund
|
||||
if(LOCALHOST MATCHES ".*\.hpcfund")
|
||||
list(APPEND CMAKE_MESSAGE_INDENT " ")
|
||||
message(STATUS "Using hpcfund-specific modulefile modification")
|
||||
file(READ ${PROJECT_SOURCE_DIR}/cmake/modfile.hpcfund.mod mod_additions)
|
||||
file(
|
||||
APPEND
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}/${ROCSHMEM_FULL_VERSION}.lua
|
||||
${mod_additions})
|
||||
list(POP_BACK CMAKE_MESSAGE_INDENT)
|
||||
endif()
|
||||
|
||||
###############################################################################
|
||||
# INSTALL
|
||||
###############################################################################
|
||||
include(GNUInstallDirs)
|
||||
include(ROCMInstallTargets)
|
||||
include(ROCMCreatePackage)
|
||||
|
||||
# Specify layout within the build directory
|
||||
set(
|
||||
CMAKE_ARCHIVE_OUTPUT_DIRECTORY
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}
|
||||
)
|
||||
set(
|
||||
CMAKE_LIBRARY_OUTPUT_DIRECTORY
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}
|
||||
)
|
||||
set(
|
||||
CMAKE_RUNTIME_OUTPUT_DIRECTORY
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}
|
||||
rocm_install(TARGETS rocshmem)
|
||||
|
||||
rocm_install(
|
||||
DIRECTORY ${CMAKE_SOURCE_DIR}/include/
|
||||
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
||||
)
|
||||
|
||||
# Offer alternatives for default installation subdirectories
|
||||
set(
|
||||
INSTALL_LIBDIR
|
||||
${CMAKE_INSTALL_LIBDIR} CACHE PATH
|
||||
"Installation directory for libraries"
|
||||
)
|
||||
set(
|
||||
INSTALL_BINDIR
|
||||
${CMAKE_INSTALL_BINDIR} CACHE PATH
|
||||
"Installation directory for executables"
|
||||
)
|
||||
set(
|
||||
INSTALL_INCLUDEDIR
|
||||
${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH
|
||||
"Installation directory for header files"
|
||||
)
|
||||
set(
|
||||
DEF_INSTALL_CMAKEDIR
|
||||
share/cmake/${PROJECT_NAME}
|
||||
)
|
||||
set(
|
||||
INSTALL_CMAKEDIR
|
||||
${DEF_INSTALL_CMAKEDIR} CACHE PATH
|
||||
"Installation directory for CMake files"
|
||||
rocm_install(
|
||||
FILES "${CMAKE_BINARY_DIR}/rocshmem_config.h"
|
||||
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rocshmem
|
||||
)
|
||||
|
||||
install(
|
||||
TARGETS ${PROJECT_NAME} EXPORT ${PROJECT_NAME}Targets
|
||||
ARCHIVE
|
||||
DESTINATION ${INSTALL_LIBDIR}
|
||||
COMPONENT lib
|
||||
LIBRARY
|
||||
DESTINATION ${INSTALL_LIBDIR}
|
||||
COMPONENT lib
|
||||
PUBLIC_HEADER
|
||||
DESTINATION ${INSTALL_INCLUDEDIR}
|
||||
COMPONENT dev
|
||||
RUNTIME
|
||||
DESTINATION ${INSTALL_BINDIR}
|
||||
COMPONENT bin
|
||||
rocm_package_add_dependencies(
|
||||
DEPENDS
|
||||
hsa-rocr
|
||||
"hip-runtime-amd >= 6.2.2"
|
||||
"rocm-dev >= 6.2.2"
|
||||
)
|
||||
|
||||
install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/
|
||||
DESTINATION ${INSTALL_INCLUDEDIR}
|
||||
COMPONENT dev
|
||||
)
|
||||
|
||||
install(
|
||||
EXPORT
|
||||
${PROJECT_NAME}Targets
|
||||
FILE
|
||||
${PROJECT_NAME}Targets.cmake
|
||||
NAMESPACE
|
||||
${PROJECT_NAME}::
|
||||
DESTINATION
|
||||
${INSTALL_CMAKEDIR}
|
||||
COMPONENT
|
||||
dev
|
||||
)
|
||||
|
||||
install(
|
||||
FILES
|
||||
${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
|
||||
${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
|
||||
DESTINATION
|
||||
${INSTALL_CMAKEDIR}
|
||||
)
|
||||
|
||||
install(
|
||||
FILES
|
||||
${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}/${ROCSHMEM_FULL_VERSION}.lua
|
||||
DESTINATION ${MOD_INSTALL_PATH}/${PROJECT_NAME})
|
||||
|
||||
###############################################################################
|
||||
# PACKAGE
|
||||
###############################################################################
|
||||
include(CMakePackageConfigHelpers)
|
||||
|
||||
write_basic_package_version_file(
|
||||
${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
|
||||
VERSION
|
||||
${PROJECT_VERSION}
|
||||
COMPATIBILITY
|
||||
SameMajorVersion
|
||||
)
|
||||
|
||||
configure_package_config_file(
|
||||
${PROJECT_SOURCE_DIR}/cmake/${PROJECT_NAME}Config.cmake.in
|
||||
${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
|
||||
INSTALL_DESTINATION
|
||||
${INSTALL_CMAKEDIR}
|
||||
rocm_create_package(
|
||||
NAME "rocSHMEM"
|
||||
DESCRIPTION "ROCm OpenSHMEM (rocSHMEM)"
|
||||
MAINTAINER "rocSHMEM Maintainer <rocshmem-maintainer@amd.com>"
|
||||
)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
1.6.0
|
||||
@@ -1 +0,0 @@
|
||||
@ROCSHMEM_GIT_REV@
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Crusher-specific additions
|
||||
depends_on "craype-accel-amd-gfx90a"
|
||||
depends_on "amd-mixed"
|
||||
depends_on "cray-mpich"
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Crusher-specific additions
|
||||
depends_on "rocm"
|
||||
prereq(atleast("rocm","5.3.0"))
|
||||
@@ -1,4 +0,0 @@
|
||||
-- hpcfund-specific additions
|
||||
depends_on "rocm"
|
||||
prereq(atleast("rocm","6.0.2"))
|
||||
|
||||
+27
-16
@@ -20,20 +20,31 @@
|
||||
# IN THE SOFTWARE.
|
||||
###############################################################################
|
||||
|
||||
find_package(Doxygen)
|
||||
if (DOXYGEN_FOUND)
|
||||
set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in)
|
||||
set(DOXYGEN_SOURCE_BROWSER YES)
|
||||
set(DOXYGEN_EXTRACT_PRIVATE YES)
|
||||
set(DOXYGEN_MACRO_EXPANSION YES)
|
||||
set(DOXYGEN_GENERATE_LATEX YES)
|
||||
set(DOXYGEN_USE_PDFLATEX YES)
|
||||
# Find available local ROCM targets
|
||||
# NOTE: This will eventually be part of ROCm-CMake and should be removed at that time
|
||||
function(rocm_local_targets VARIABLE)
|
||||
set(${VARIABLE} "NOTFOUND" PARENT_SCOPE)
|
||||
find_program(_rocm_agent_enumerator rocm_agent_enumerator HINTS /opt/rocm/bin ENV ROCM_PATH)
|
||||
if(NOT _rocm_agent_enumerator STREQUAL "_rocm_agent_enumerator-NOTFOUND")
|
||||
execute_process(
|
||||
COMMAND "${_rocm_agent_enumerator}"
|
||||
RESULT_VARIABLE _found_agents
|
||||
OUTPUT_VARIABLE _rocm_agents
|
||||
ERROR_QUIET
|
||||
)
|
||||
if (_found_agents EQUAL 0)
|
||||
string(REPLACE "\n" ";" _rocm_agents "${_rocm_agents}")
|
||||
unset(result)
|
||||
foreach (agent IN LISTS _rocm_agents)
|
||||
if (NOT agent STREQUAL "gfx000")
|
||||
list(APPEND result "${agent}")
|
||||
endif()
|
||||
endforeach()
|
||||
if(result)
|
||||
list(REMOVE_DUPLICATES result)
|
||||
set(${VARIABLE} "${result}" PARENT_SCOPE)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
|
||||
doxygen_add_docs(
|
||||
docs
|
||||
${PROJECT_SOURCE_DIR}
|
||||
COMMENT "Generate man pages"
|
||||
MACRO_EXPANSION YES
|
||||
)
|
||||
endif()
|
||||
@@ -1,33 +0,0 @@
|
||||
local help_message = [[
|
||||
|
||||
rocSHMEM is an open-source GPU initiated networking library
|
||||
for High Performance Computing and Machine Learning workloads.
|
||||
|
||||
Version @ROCSHMEM_FULL_VERSION@
|
||||
]]
|
||||
|
||||
help(help_message,"\n")
|
||||
|
||||
whatis("Name: rocshmem")
|
||||
whatis("Version: @ROCSHMEM_FULL_VERSION@")
|
||||
whatis("Keywords: GPU, PGAS, RMA, HPC")
|
||||
whatis("Description: tool for GPU initiated networking")
|
||||
whatis("URL: https://github.com/ROCm/rocSHMEM")
|
||||
|
||||
-- Export environmental variables
|
||||
local topDir="@CMAKE_INSTALL_PREFIX@"
|
||||
local binDir="@CMAKE_INSTALL_FULL_BINDIR@"
|
||||
local shareDir="@CMAKE_INSTALL_FULL_DATADIR@"
|
||||
local pythonDeps="@PYTHON_DEPS@"
|
||||
|
||||
setenv("ROCSHMEM_DIR",topDir)
|
||||
setenv("ROCSHMEM_BIN",binDir)
|
||||
setenv("ROCSHMEM_SHARE",shareDir)
|
||||
setenv("MPICH_GPU_SUPPORT_ENABLED",1)
|
||||
|
||||
-- Update relevant PATH variables
|
||||
prepend_path("PATH",binDir)
|
||||
if ( pythonDeps ~= "" ) then
|
||||
prepend_path("PYTHONPATH",pythonDeps)
|
||||
end
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
@PACKAGE_INIT@
|
||||
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/rocshmemTargets.cmake")
|
||||
|
||||
check_required_components(
|
||||
"rocshmem"
|
||||
)
|
||||
Dosya farkı çok büyük olduğundan ihmal edildi
Fark Yükle
@@ -1,59 +0,0 @@
|
||||
# Builds for host-facing support
|
||||
|
||||
As of July 29, 2021 (when support for host-facing functions was merged in),
|
||||
we are running it with MPICH-3.4 and UCX-1.10.0.
|
||||
|
||||
## Background
|
||||
|
||||
MPICH-3.4 does not have support for HIP (GPU support in MPICH checks whether
|
||||
the buffer is on the host or the device).
|
||||
|
||||
UCX-1.10.0 claims it does not support GPU-aware communication for RMA
|
||||
operations. As of the time of this writing, this claim remains even on the
|
||||
latest UCX version (1.11.0 and the master branch). UCX developers to merge
|
||||
in stable GPU-aware support by the end of 2021.
|
||||
|
||||
A side note: OSU microbechmarks with RoCM memory hang with UCX-1.10.0
|
||||
(during ucp_mem_map() which is called by MPI_Win_create()).
|
||||
I don't see this hang with the latest version of UCX.
|
||||
|
||||
So, for RoCM memory, MPICH-3.4 still offloads RMA operations to UCX.
|
||||
|
||||
## So, how does it work with the current builds?
|
||||
|
||||
Theoretically, there are no limitations preventing GPU-aware RDMA
|
||||
communicaiton. As long as the GPU memory is registered with the NIC,
|
||||
the NIC can perform operations on device memory.
|
||||
|
||||
Even though UCX claims to not support GPU-aware RMA communication, it does
|
||||
not check whether or not the buffer being passed in is a device or host
|
||||
buffer. So, as long as the device memory being used is registered with the
|
||||
NIC (this does occur during MPI_Win_create), we are good.
|
||||
|
||||
UCX claims to not support GPU-aware communication because they have
|
||||
not added in support for the different types of scenarios that could
|
||||
exist in a system (eg, when a system does not have GPU-direct). The
|
||||
scope of rocSHMEM is currently limited to configurations that UCX
|
||||
already supports.
|
||||
|
||||
## But the main branch of MPICH does support HIP now?
|
||||
|
||||
Since MPICH is going off of UCX's claim that it does not support
|
||||
GPU-aware RMA communication, MPICH executes its RMA operations
|
||||
using active messages when it notices that the buffer is a GPU
|
||||
buffer. So, if we use MPICH with HIP support, we end up using
|
||||
active-message implementations unnecessarily, and hence lose
|
||||
a lot of performance.
|
||||
|
||||
## Moving forward
|
||||
|
||||
We should switch to using MPICH "correctly" (i.e. with HIP support)
|
||||
only when UCX officially claims to support GPU-aware RMA
|
||||
communication because that is when MPICH will offload MPI
|
||||
RMA operations to UCX RMA operations.
|
||||
|
||||
But if there is a need for MPICH's HIP support for GPU IPC (unsure
|
||||
if this is needed for now), we will need an alternative. In the
|
||||
current MPICH configuration, communication between processes on
|
||||
the same node are funneled through the netmod (UCX in our case) as
|
||||
well.
|
||||
@@ -45,7 +45,7 @@ foreach(SOURCE_FILE IN LISTS EXAMPLE_SOURCES)
|
||||
target_link_libraries(
|
||||
${EXECUTABLE_NAME}
|
||||
PRIVATE
|
||||
rocshmem::rocshmem
|
||||
roc::rocshmem
|
||||
-fgpu-rdc
|
||||
# xnack allows address translation fault recovery
|
||||
# required option for managed heap configs
|
||||
|
||||
@@ -67,13 +67,13 @@ target_sources(
|
||||
target_include_directories(
|
||||
${TESTS_NAME}
|
||||
PRIVATE
|
||||
rocshmem::rocshmem
|
||||
roc::rocshmem
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
${TESTS_NAME}
|
||||
PRIVATE
|
||||
rocshmem::rocshmem
|
||||
roc::rocshmem
|
||||
-fgpu-rdc
|
||||
# xnack allows address translation fault recovery
|
||||
# required option for managed heap configs
|
||||
|
||||
@@ -22,6 +22,8 @@
|
||||
|
||||
cmake_minimum_required(VERSION 3.16.3 FATAL_ERROR)
|
||||
|
||||
set(INSTALL_GTEST OFF CACHE BOOL "" FORCE)
|
||||
|
||||
###############################################################################
|
||||
# GLOBAL COMPILE FLAGS
|
||||
###############################################################################
|
||||
@@ -102,13 +104,13 @@ find_package(hip REQUIRED)
|
||||
target_include_directories(
|
||||
${PROJECT_NAME}
|
||||
PRIVATE
|
||||
rocshmem::rocshmem
|
||||
roc::rocshmem
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
PRIVATE
|
||||
rocshmem::rocshmem
|
||||
roc::rocshmem
|
||||
hip::host
|
||||
-fgpu-rdc
|
||||
)
|
||||
|
||||
Yeni konuda referans
Bir kullanıcı engelle