diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index 5f4b13d715..611e501750 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -53,10 +53,13 @@ option(USE_FUNC_CALL "Force compiler to use function calls on library API" OFF) option(USE_SHARED_CTX "Request support for shared ctx between WG" OFF) option(USE_SINGLE_NODE "Enable single node support only." OFF) option(USE_HOST_SIDE_HDP_FLUSH "Use a polling thread to flush the HDP cache on the host." OFF) + option(BUILD_FUNCTIONAL_TESTS "Build the functional tests" ON) option(BUILD_EXAMPLES "Build the examples" ON) option(BUILD_SOS_TESTS "Build the host-facing tests" OFF) option(BUILD_UNIT_TESTS "Build the unit tests" ON) +option(BUILD_TESTS_ONLY "Build only tests. Used to link agains rocSHMEM in a ROCm Release" OFF) + option(BUILD_LOCAL_GPU_TARGET_ONLY "Build only for GPUs detected on this machine" OFF) configure_file(cmake/rocshmem_config.h.in rocshmem_config.h) @@ -78,191 +81,170 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_FLAGS_DEBUG "-O0 -ggdb") +if (BUILD_TESTS_ONLY) + if (DEFINED ENV{ROCSHMEM_HOME}) + set(ROCSHMEM_HOME "$ENV{ROCSHMEM_HOME}") + else() + message("Environment variable ROCSHMEM_HOME is not set.") + message("Assuming that rocSHMEM is installed at ${ROCM_PATH}.") + set(ROCSHMEM_HOME "${ROCM_PATH}") + endif() +endif() + find_package(ROCM 0.8 REQUIRED PATHS ${ROCM_PATH}) set(ROCMCHECKS_WARN_TOOLCHAIN_VAR OFF) -find_package(ROCmCMakeBuildTools) -include(ROCMCreatePackage) -include(ROCMInstallTargets) -include(ROCMCheckTargetIds) - -############################################################################### -# PROJECT -############################################################################### -rocm_setup_version(VERSION 2.0.0) - -project(rocshmem VERSION 2.0.0 LANGUAGES CXX) - -############################################################################### -# SET GPU ARCHITECTURES -############################################################################### include(cmake/rocm_local_targets.cmake) set(DEFAULT_GPUS gfx90a gfx942) -if (BUILD_LOCAL_GPU_TARGET_ONLY) - message(STATUS "Building only for local GPU target") - if (COMMAND rocm_local_targets) - rocm_local_targets(DEFAULT_GPUS) - else() - message(WARNING "Unable to determine local GPU targets. Falling back to default GPUs.") - endif() +############################################################################### +# PROJECT +############################################################################### +find_package(ROCmCMakeBuildTools) +include(ROCMCreatePackage) +include(ROCMInstallTargets) +include(ROCMCheckTargetIds) + +rocm_setup_version(VERSION 2.0.0) +project(rocshmem VERSION 2.0.0 LANGUAGES CXX) + +############################################################################### +# TEST SUBDIRECTORIES +############################################################################### +add_subdirectory(tests) + +if (BUILD_EXAMPLES) + add_subdirectory(examples) endif() -set(GPU_TARGETS "${DEFAULT_GPUS}" CACHE STRING - "Target default GPUs if GPU_TARGETS is not defined.") - -if (COMMAND rocm_check_target_ids) - message(STATUS "Checking for ROCm support for GPU targets: " "${GPU_TARGETS}") - rocm_check_target_ids(SUPPORTED_GPUS TARGETS ${GPU_TARGETS}) -else() - message(WARNING "Unable to check for supported GPU targets. Falling back to default GPUs.") - set(SUPPORTED_GPUS ${DEFAULT_GPUS}) -endif() - -set(COMPILING_TARGETS "${SUPPORTED_GPUS}" CACHE STRING "GPU targets to compile for.") -message(STATUS "Compiling for ${COMPILING_TARGETS}") - -foreach(target ${COMPILING_TARGETS}) - list(APPEND static_link_flags --offload-arch=${target}) -endforeach() -list(JOIN static_link_flags " " flags_str) -add_compile_options(${flags_str}) - ############################################################################### # CREATE ROCSHMEM LIBRARY ############################################################################### -add_library(${PROJECT_NAME}) -add_library(roc::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) +if (NOT BUILD_TESTS_ONLY) + add_library(${PROJECT_NAME}) + add_library(roc::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) + add_subdirectory(src) -############################################################################### -# INCLUDE DIRECTORIES -############################################################################### -target_include_directories( - ${PROJECT_NAME} - PUBLIC - $ - $ # rocshmem_config.h - $ -) + ############################################################################# + # SET GPU ARCHITECTURES + ############################################################################# + if (BUILD_LOCAL_GPU_TARGET_ONLY) + message(STATUS "Building only for local GPU target") + if (COMMAND rocm_local_targets) + rocm_local_targets(DEFAULT_GPUS) + else() + message(WARNING "Unable to determine local GPU targets. Falling back to default GPUs.") + endif() + endif() -############################################################################### -# SUBDIRECTORY TARGETS -############################################################################### -add_subdirectory(src) -add_subdirectory(tests) + set(GPU_TARGETS "${DEFAULT_GPUS}" CACHE STRING + "Target default GPUs if GPU_TARGETS is not defined.") -IF (BUILD_EXAMPLES) - add_subdirectory(examples) -ENDIF() + if (COMMAND rocm_check_target_ids) + message(STATUS "Checking for ROCm support for GPU targets: " "${GPU_TARGETS}") + rocm_check_target_ids(SUPPORTED_GPUS TARGETS ${GPU_TARGETS}) + else() + message(WARNING "Unable to check for supported GPU targets. Falling back to default GPUs.") + set(SUPPORTED_GPUS ${DEFAULT_GPUS}) + endif() -############################################################################### -# HIP -############################################################################### -find_package(hip REQUIRED) + set(COMPILING_TARGETS "${SUPPORTED_GPUS}" CACHE STRING "GPU targets to compile for.") + message(STATUS "Compiling for ${COMPILING_TARGETS}") -target_link_libraries( - ${PROJECT_NAME} - PUBLIC - hip::device - hip::host -) + foreach (target ${COMPILING_TARGETS}) + list(APPEND static_link_flags --offload-arch=${target}) + endforeach() + list(JOIN static_link_flags " " flags_str) + add_compile_options(${flags_str}) -############################################################################### -# HSA-RUNTIME64 -############################################################################### -find_package(hsa-runtime64 REQUIRED) + ############################################################################# + # PACKAGE DEPENDENCIES + ############################################################################# + find_package(MPI REQUIRED) + find_package(hip REQUIRED) + find_package(hsa-runtime64 REQUIRED) -target_link_libraries( - ${PROJECT_NAME} - PUBLIC - hsa-runtime64::hsa-runtime64 -) + if (USE_GPU_IB) + find_package(Ibverbs REQUIRED) + endif() -############################################################################### -# PTHREADS -############################################################################### -set(CMAKE_THREAD_PREFER_PTHREAD TRUE) -set(THREADS_PREFER_PTHREAD_FLAG TRUE) -find_package(Threads REQUIRED) - -target_link_libraries( - ${PROJECT_NAME} - PUBLIC - Threads::Threads -) - -############################################################################### -# IBVERBS -############################################################################### -IF (USE_GPU_IB) - find_package(Ibverbs REQUIRED) + set(CMAKE_THREAD_PREFER_PTHREAD TRUE) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) + find_package(Threads REQUIRED) + ############################################################################# + # LINKING AND INCLUDE DIRECTORIES + ############################################################################# target_include_directories( ${PROJECT_NAME} PUBLIC - ${IBVERBS_INCLUDE_DIRS} + $ + $ # rocshmem_config.h + $ + ${MPI_CXX_HEADER_DIR} ) target_link_libraries( ${PROJECT_NAME} PUBLIC - ${IBVERBS_LIBRARIES} + Threads::Threads + ${MPI_mpi_LIBRARY} + ${MPI_mpicxx_LIBRARY} + hip::device + hip::host + hsa-runtime64::hsa-runtime64 ) -ENDIF() -############################################################################### -# MPI -############################################################################### -find_package(MPI REQUIRED) + if (USE_GPU_IB) + target_include_directories( + ${PROJECT_NAME} + PUBLIC + ${IBVERBS_INCLUDE_DIRS} + ) -target_include_directories( - ${PROJECT_NAME} - PUBLIC - ${MPI_CXX_HEADER_DIR} -) + target_link_libraries( + ${PROJECT_NAME} + PUBLIC + ${IBVERBS_LIBRARIES} + ) + endif() -target_link_libraries( - ${PROJECT_NAME} - PUBLIC - ${MPI_mpi_LIBRARY} - ${MPI_mpicxx_LIBRARY} -) + ############################################################################# + # INSTALL + ############################################################################# + include(ROCMInstallTargets) + include(ROCMCreatePackage) -############################################################################### -# INSTALL -############################################################################### -include(ROCMInstallTargets) -include(ROCMCreatePackage) + rocm_install(TARGETS rocshmem) -rocm_install(TARGETS rocshmem) + rocm_install( + DIRECTORY ${CMAKE_SOURCE_DIR}/include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) -rocm_install( - DIRECTORY ${CMAKE_SOURCE_DIR}/include/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) + rocm_install( + FILES "${CMAKE_BINARY_DIR}/rocshmem_config.h" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rocshmem + ) -rocm_install( - FILES "${CMAKE_BINARY_DIR}/rocshmem_config.h" - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rocshmem -) + rocm_package_add_dependencies( + DEPENDS + hsa-rocr + hip-runtime-amd + rocm-dev + ) -rocm_package_add_dependencies( - DEPENDS - hsa-rocr - hip-runtime-amd - rocm-dev -) + rocm_export_targets( + TARGETS roc::rocshmem + NAMESPACE roc:: + ) -rocm_export_targets( - TARGETS roc::rocshmem - NAMESPACE roc:: -) - -rocm_create_package( - NAME "rocSHMEM" - DESCRIPTION "ROCm OpenSHMEM (rocSHMEM)" - MAINTAINER "rocSHMEM Maintainer " -) + rocm_create_package( + NAME "rocSHMEM" + DESCRIPTION "ROCm OpenSHMEM (rocSHMEM)" + MAINTAINER "rocSHMEM Maintainer " + ) +endif() diff --git a/projects/rocshmem/examples/CMakeLists.txt b/projects/rocshmem/examples/CMakeLists.txt index 29122baa63..d720dc32d3 100644 --- a/projects/rocshmem/examples/CMakeLists.txt +++ b/projects/rocshmem/examples/CMakeLists.txt @@ -36,19 +36,51 @@ foreach(SOURCE_FILE IN LISTS EXAMPLE_SOURCES) add_executable(${EXECUTABLE_NAME} ${SOURCE_FILE}) - target_include_directories( - ${EXECUTABLE_NAME} - PRIVATE - rocshmem::rocshmem - ) + if (BUILD_TESTS_ONLY) + find_package(MPI REQUIRED) - target_link_libraries( - ${EXECUTABLE_NAME} - PRIVATE - roc::rocshmem - -fgpu-rdc - # xnack allows address translation fault recovery - # required option for managed heap configs - # -mxnack - ) + target_include_directories( + ${EXECUTABLE_NAME} + PRIVATE + $ + $ + ) + + foreach (target ${DEFAULT_GPUS}) + list(APPEND static_link_flags --offload-arch=${target}) + endforeach() + list(JOIN static_link_flags " " flags_str) + + target_compile_options( + ${EXECUTABLE_NAME} + PRIVATE + ${flags_str} + -fgpu-rdc + ) + + target_link_libraries( + ${EXECUTABLE_NAME} + PRIVATE + ${MPI_mpi_LIBRARY} + ${MPI_mpicxx_LIBRARY} + -L${ROCSHMEM_HOME}/lib + -lamdhip64 + -lhsa-runtime64 + -lrocshmem + -fgpu-rdc + ) + else() + target_include_directories( + ${EXECUTABLE_NAME} + PRIVATE + roc::rocshmem + ) + + target_link_libraries( + ${EXECUTABLE_NAME} + PRIVATE + roc::rocshmem + -fgpu-rdc + ) + endif() endforeach() diff --git a/projects/rocshmem/scripts/build_configs/ipc_single b/projects/rocshmem/scripts/build_configs/ipc_single index 7c76bad6be..dc6b0c522b 100755 --- a/projects/rocshmem/scripts/build_configs/ipc_single +++ b/projects/rocshmem/scripts/build_configs/ipc_single @@ -26,7 +26,7 @@ cmake \ -DUSE_WF_COAL=OFF \ -DUSE_SINGLE_NODE=ON \ -DUSE_HOST_SIDE_HDP_FLUSH=OFF \ - -DBUILD_LOCAL_GPU_TARGET_ONLY=ON \ + -DBUILD_LOCAL_GPU_TARGET_ONLY=OFF \ $src_path cmake --build . --parallel 8 cmake --install . diff --git a/projects/rocshmem/scripts/build_configs/ipc_tests_only b/projects/rocshmem/scripts/build_configs/ipc_tests_only new file mode 100755 index 0000000000..5be1ecbfe3 --- /dev/null +++ b/projects/rocshmem/scripts/build_configs/ipc_tests_only @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +set -e + +if [ -z $1 ] +then + install_path=~/rocshmem +else + install_path=$1 +fi + +if [ -z $1 ] && [ -z $ROCSHMEM_HOME ] +then + export ROCSHMEM_HOME=/opt/rocm/ +fi + +src_path=$(dirname "$(realpath $0)")/../../ + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$install_path \ + -DCMAKE_VERBOSE_MAKEFILE=OFF \ + -DDEBUG=OFF \ + -DPROFILE=OFF \ + -DUSE_GPU_IB=OFF \ + -DUSE_RO=OFF \ + -DUSE_DC=OFF \ + -DUSE_IPC=ON \ + -DUSE_COHERENT_HEAP=ON \ + -DUSE_THREADS=OFF \ + -DUSE_WF_COAL=OFF \ + -DUSE_SINGLE_NODE=ON \ + -DUSE_HOST_SIDE_HDP_FLUSH=OFF \ + -DBUILD_LOCAL_GPU_TARGET_ONLY=OFF \ + -DBUILD_TESTS_ONLY=ON \ + $src_path +cmake --build . --parallel 8 diff --git a/projects/rocshmem/scripts/functional_tests/driver.sh b/projects/rocshmem/scripts/functional_tests/driver.sh index cb31c766f9..cdd83e949a 100755 --- a/projects/rocshmem/scripts/functional_tests/driver.sh +++ b/projects/rocshmem/scripts/functional_tests/driver.sh @@ -322,9 +322,9 @@ TestOther() { ValidateInput() { INPUT_COUNT=$1 - if [ $INPUT_COUNT -eq 0 ] ; then - echo "This script must be run with at least 2 arguments." - echo 'Usage: ${0} argument1 argument2 [argument3] [argument4]' + if [ $INPUT_COUNT -lt 3 ] ; then + echo "This script must be run with at least 3 arguments." + echo 'Usage: ${0} argument1 argument2 argument3 [argument4]' echo " argument1 : path to the tester driver" echo " argument2 : test type to run, e.g put" echo " argument3 : directory to put the output logs" @@ -333,6 +333,14 @@ ValidateInput() { fi } +ValidateLogDir() { + if [ ! -d $1 ]; then + echo "LOG_DIR=$1 does not exist" + mkdir -p $1 + echo "Created $1" + fi +} + APP=$1 TEST=$2 LOG_DIR=$3 @@ -341,6 +349,7 @@ HOSTFILE=$4 DRIVER_RETURN_STATUS=0 ValidateInput $# +ValidateLogDir $LOG_DIR case $TEST in *"all") diff --git a/projects/rocshmem/tests/functional_tests/CMakeLists.txt b/projects/rocshmem/tests/functional_tests/CMakeLists.txt index 1ad791bb2c..fcf6a63ba3 100644 --- a/projects/rocshmem/tests/functional_tests/CMakeLists.txt +++ b/projects/rocshmem/tests/functional_tests/CMakeLists.txt @@ -63,19 +63,52 @@ target_sources( ############################################################################### # ROCSHMEM ############################################################################### +if (BUILD_TESTS_ONLY) + find_package(MPI REQUIRED) -target_include_directories( - ${TESTS_NAME} - PRIVATE - roc::rocshmem -) + target_include_directories( + ${TESTS_NAME} + PRIVATE + $ + $ + $ + $ + ) -target_link_libraries( - ${TESTS_NAME} - PRIVATE - roc::rocshmem - -fgpu-rdc -# xnack allows address translation fault recovery -# required option for managed heap configs -# -mxnack -) + foreach (target ${DEFAULT_GPUS}) + list(APPEND static_link_flags --offload-arch=${target}) + endforeach() + list(JOIN static_link_flags " " flags_str) + + target_compile_options( + ${TESTS_NAME} + PRIVATE + ${flags_str} + -fgpu-rdc + ) + + target_link_libraries( + ${TESTS_NAME} + PRIVATE + ${MPI_mpi_LIBRARY} + ${MPI_mpicxx_LIBRARY} + -L${ROCSHMEM_HOME}/lib + -lamdhip64 + -lhsa-runtime64 + -lrocshmem + -fgpu-rdc + ) +else() + target_include_directories( + ${TESTS_NAME} + PRIVATE + roc::rocshmem + ) + + target_link_libraries( + ${TESTS_NAME} + PRIVATE + roc::rocshmem + -fgpu-rdc + ) +endif() diff --git a/projects/rocshmem/tests/unit_tests/CMakeLists.txt b/projects/rocshmem/tests/unit_tests/CMakeLists.txt index aab5d1a5e5..688917c3a6 100644 --- a/projects/rocshmem/tests/unit_tests/CMakeLists.txt +++ b/projects/rocshmem/tests/unit_tests/CMakeLists.txt @@ -102,19 +102,56 @@ target_sources( ############################################################################### find_package(hip REQUIRED) -target_include_directories( - ${PROJECT_NAME} - PRIVATE - roc::rocshmem -) +if (BUILD_TESTS_ONLY) + find_package(MPI REQUIRED) -target_link_libraries( - ${PROJECT_NAME} - PRIVATE - roc::rocshmem - hip::host - -fgpu-rdc -) + target_include_directories( + ${PROJECT_NAME} + PRIVATE + $ + $ + $ + $ + ) + + foreach (target ${DEFAULT_GPUS}) + list(APPEND static_link_flags --offload-arch=${target}) + endforeach() + list(JOIN static_link_flags " " flags_str) + + target_compile_options( + ${PROJECT_NAME} + PRIVATE + ${flags_str} + -fgpu-rdc + ) + + target_link_libraries( + ${PROJECT_NAME} + PRIVATE + ${MPI_mpi_LIBRARY} + ${MPI_mpicxx_LIBRARY} + -L${ROCSHMEM_HOME}/lib + -lamdhip64 + -lhsa-runtime64 + -lrocshmem + -fgpu-rdc + ) +else() + target_include_directories( + ${PROJECT_NAME} + PRIVATE + roc::rocshmem + ) + + target_link_libraries( + ${PROJECT_NAME} + PRIVATE + roc::rocshmem + hip::host + -fgpu-rdc + ) +endif() ############################################################################### # GTEST DEPENDENCY