From b0963b2f2cb7e813fc03d0844cd6bb576425ba17 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Mon, 15 Jul 2024 10:18:20 -0500 Subject: [PATCH 01/46] Remove CACHED_HEAP option [ROCm/rocshmem commit: 39c5be18b1f8205c15cab789c0df0423ae3b41b5] --- projects/rocshmem/CMakeLists.txt | 1 - projects/rocshmem/README.md | 2 +- projects/rocshmem/cmake/config.h.in | 3 +-- projects/rocshmem/scripts/build_configs/rc_single | 1 - projects/rocshmem/scripts/build_configs/rc_single_single_node | 2 -- .../rocshmem/scripts/build_configs/rc_single_single_node_debug | 1 - projects/rocshmem/src/gpu_ib/connection.cpp | 2 +- projects/rocshmem/src/memory/heap_type.hpp | 2 +- projects/rocshmem/src/util.hpp | 2 +- 9 files changed, 5 insertions(+), 11 deletions(-) diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index ee7f42fb12..8603737308 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -60,7 +60,6 @@ option(USE_IPC "Enable IPC support (using HIP)" OFF) option(USE_THREADS "Enable workgroup threads to share network queues" OFF) option(USE_WF_COAL "Enable wavefront message coalescing" OFF) option(USE_COHERENT_HEAP "Enable support for coherent systems" OFF) -option(USE_CACHED_HEAP "Enable support for cached systems" OFF) option(USE_MANAGED_HEAP "Enable managed memory" OFF) option(USE_HOST_HEAP "Enable host memory using malloc/free" OFF) option(USE_HIP_HOST_HEAP "Enable host memory using hip api" OFF) diff --git a/projects/rocshmem/README.md b/projects/rocshmem/README.md index c974fd7052..ba671ff51a 100644 --- a/projects/rocshmem/README.md +++ b/projects/rocshmem/README.md @@ -45,7 +45,7 @@ ROC_SHMEM base requirements: * May work with other versions, but not tested * AMD GFX9 GPUs (e.g.: MI25, Vega 56, Vega 64, MI50, MI60, MI100, Radeon VII) * AMD MI200 GPUs: To enable the support on MI200, please configure the library - with USE_CACHED_HEAP + with USE_COHERENT_HEAP * ROCm-aware MPI as described in [Building the Dependencies](#building-the-dependencies) * InfiniBand adaptor compatable with ROCm RDMA technology diff --git a/projects/rocshmem/cmake/config.h.in b/projects/rocshmem/cmake/config.h.in index 6ec07c2853..f409c10ba3 100644 --- a/projects/rocshmem/cmake/config.h.in +++ b/projects/rocshmem/cmake/config.h.in @@ -7,10 +7,9 @@ #cmakedefine USE_SHARED_CTX #cmakedefine USE_WF_COAL #cmakedefine USE_COHERENT_HEAP -#cmakedefine USE_CACHED_HEAP #cmakedefine USE_MANAGED_HEAP #cmakedefine USE_HOST_HEAP #cmakedefine USE_HIP_HOST_HEAP #cmakedefine USE_FUNC_CALL #cmakedefine USE_SINGLE_NODE -#cmakedefine USE_HOST_SIDE_HDP_FLUSH \ No newline at end of file +#cmakedefine USE_HOST_SIDE_HDP_FLUSH diff --git a/projects/rocshmem/scripts/build_configs/rc_single b/projects/rocshmem/scripts/build_configs/rc_single index 31eb7283c6..aceca7dbed 100755 --- a/projects/rocshmem/scripts/build_configs/rc_single +++ b/projects/rocshmem/scripts/build_configs/rc_single @@ -20,7 +20,6 @@ cmake \ -DUSE_DC=OFF \ -DUSE_IPC=OFF \ -DUSE_COHERENT_HEAP=OFF \ - -DUSE_CACHED_HEAP=OFF \ -DUSE_THREADS=OFF \ -DUSE_WF_COAL=OFF \ $src_path diff --git a/projects/rocshmem/scripts/build_configs/rc_single_single_node b/projects/rocshmem/scripts/build_configs/rc_single_single_node index 8bcf9fcb0d..a5223c379f 100755 --- a/projects/rocshmem/scripts/build_configs/rc_single_single_node +++ b/projects/rocshmem/scripts/build_configs/rc_single_single_node @@ -20,12 +20,10 @@ cmake \ -DUSE_DC=OFF \ -DUSE_IPC=ON \ -DUSE_COHERENT_HEAP=OFF \ - -DUSE_CACHED_HEAP=OFF \ -DUSE_THREADS=OFF \ -DUSE_WF_COAL=OFF \ -DUSE_SINGLE_NODE=ON \ -DUSE_HOST_SIDE_HDP_FLUSH=ON\ - -DROCM_PATH="/opt/rocm-5.4.2/"\ $src_path cmake --build . --parallel 8 cmake --install . diff --git a/projects/rocshmem/scripts/build_configs/rc_single_single_node_debug b/projects/rocshmem/scripts/build_configs/rc_single_single_node_debug index b535f1b853..c4e2477da9 100755 --- a/projects/rocshmem/scripts/build_configs/rc_single_single_node_debug +++ b/projects/rocshmem/scripts/build_configs/rc_single_single_node_debug @@ -20,7 +20,6 @@ cmake \ -DUSE_DC=OFF \ -DUSE_IPC=ON \ -DUSE_COHERENT_HEAP=OFF \ - -DUSE_CACHED_HEAP=OFF \ -DUSE_THREADS=OFF \ -DUSE_WF_COAL=OFF \ -DUSE_SINGLE_NODE=ON \ diff --git a/projects/rocshmem/src/gpu_ib/connection.cpp b/projects/rocshmem/src/gpu_ib/connection.cpp index 631f72f324..b6b2ae4dcd 100644 --- a/projects/rocshmem/src/gpu_ib/connection.cpp +++ b/projects/rocshmem/src/gpu_ib/connection.cpp @@ -256,7 +256,7 @@ void* Connection::buf_alloc([[maybe_unused]] struct ibv_pd* pd, if (use_gpu_mem) { void* dev_ptr; if (coherent_cq == 1) { -#if defined USE_COHERENT_HEAP || defined USE_CACHED_HEAP +#if defined USE_COHERENT_HEAP CHECK_HIP(hipMalloc(reinterpret_cast(&dev_ptr), size)); #else #ifdef HIP_SUPPORTS_MALLOC_UNCACHED diff --git a/projects/rocshmem/src/memory/heap_type.hpp b/projects/rocshmem/src/memory/heap_type.hpp index 0008317498..3f48edd11e 100644 --- a/projects/rocshmem/src/memory/heap_type.hpp +++ b/projects/rocshmem/src/memory/heap_type.hpp @@ -40,7 +40,7 @@ namespace rocshmem { #if defined USE_MANAGED_HEAP using HEAP_T = HeapMemory; -#elif defined USE_COHERENT_HEAP || defined USE_CACHED_HEAP +#elif defined USE_COHERENT_HEAP using HEAP_T = HeapMemory; #elif defined USE_HOST_HEAP using HEAP_T = HeapMemory; diff --git a/projects/rocshmem/src/util.hpp b/projects/rocshmem/src/util.hpp index 0d4dad806a..1d5cd9f9a6 100644 --- a/projects/rocshmem/src/util.hpp +++ b/projects/rocshmem/src/util.hpp @@ -126,7 +126,7 @@ extern const int gpu_clock_freq_mhz; __device__ __forceinline__ void __roc_inv() { asm volatile("buffer_wbinvl1;"); } __device__ __forceinline__ void __roc_flush() { -#if defined USE_COHERENT_HEAP || defined USE_CACHED_HEAP +#if defined USE_COHERENT_HEAP #if __gfx90a__ asm volatile("s_dcache_wb;"); asm volatile("buffer_wbl2;"); From aff61dc498ad22b28bac08e25a4f58b2ec512983 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Mon, 15 Jul 2024 10:38:38 -0500 Subject: [PATCH 02/46] Create new ro_ipc config [ROCm/rocshmem commit: 23f2983214ec9ab8215364cb63f366f68d7735e1] --- .../rocshmem/scripts/build_configs/ro_ipc | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 projects/rocshmem/scripts/build_configs/ro_ipc diff --git a/projects/rocshmem/scripts/build_configs/ro_ipc b/projects/rocshmem/scripts/build_configs/ro_ipc new file mode 100755 index 0000000000..d57404b88c --- /dev/null +++ b/projects/rocshmem/scripts/build_configs/ro_ipc @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + +if [ -z $1 ] +then + install_path=~/rocshmem +else + install_path=$1 +fi + +src_path=$(dirname "$(realpath $0)")/../../ + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$install_path \ + -DCMAKE_VERBOSE_MAKEFILE=OFF \ + -DDEBUG=OFF \ + -DPROFILE=OFF \ + -DUSE_GPU_IB=OFF \ + -DUSE_DC=OFF \ + -DUSE_IPC=ON \ + -DUSE_COHERENT_HEAP=ON \ + -DUSE_THREADS=OFF \ + -DUSE_WF_COAL=OFF \ + -DUSE_SINGLE_NODE=ON \ + -DUSE_HOST_SIDE_HDP_FLUSH=OFF\ + $src_path +cmake --build . --parallel 8 +cmake --install . From 13ba4a18944517a2843260aa7eee959d25d53439 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Thu, 11 Jul 2024 08:37:52 -0700 Subject: [PATCH 03/46] Update functional_tests driver.sh script [ROCm/rocshmem commit: ca36cc249e92bc38429fbb8cc6a410e1b071d4c8] --- .../scripts/functional_tests/driver.sh | 785 +++++++++--------- 1 file changed, 414 insertions(+), 371 deletions(-) diff --git a/projects/rocshmem/scripts/functional_tests/driver.sh b/projects/rocshmem/scripts/functional_tests/driver.sh index a7a3e869e5..5adb033166 100755 --- a/projects/rocshmem/scripts/functional_tests/driver.sh +++ b/projects/rocshmem/scripts/functional_tests/driver.sh @@ -22,417 +22,460 @@ if [ $# -eq 0 ] ; then echo "This script must be run with at least 2 arguments." - echo 'Usage: ${0} argument1 argument2 [argument3] [argument4] [argument5]' + echo 'Usage: ${0} argument1 argument2 [argument3]' echo " argument1 : path to the tester driver" echo " argument2 : test type to run, e.g put" echo " argument3 : directory to put the output logs" - echo " argument4 : enable gdb debug" - echo " argument5 : shmem context type" exit 1 fi -shm_ctx=4 # run tests with SHMEM_CONTEXT_WG_PRIVATE -if [ $# -eq 5 ] ; then - shm_ctx=$5 -fi - -ENABLE_DBG=false -if [ $# -ge 4 ] ; then - ENABLE_DBG=$4 -fi - -if [ "$ENABLE_DBG" = true ] -then - gdb_cmd="xterm -e gdb -x gdbscript --args" -else - gdb_cmd="" -fi - -echo "Test Name ${2} (with shmem context: ${shm_ctx})" +echo "Test Name ${2}" check() { if [ $? -ne 0 ] then echo "Failed $1" >&2 - exit 1 fi } case $2 in - *"single_thread") - echo "get" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 0 -x ${shm_ctx} > $3/get.log - check get - echo "get_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 0 -x ${shm_ctx} > $3/get_th.log - check get_th - echo "get_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 1 -x ${shm_ctx} > $3/get_nbi.log - check get_nbi - echo "get_nbi_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 1 -x ${shm_ctx} > $3/get_nbi_th.log - check get_nbi_th - echo "put" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 2 -x ${shm_ctx} > $3/put.log - check put - echo "put_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 2 -x ${shm_ctx} > $3/put_th.log - check put_th - echo "put_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 3 -x ${shm_ctx} > $3/put_nbi.log - check put_nbi - echo "put_nbi_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 3 -x ${shm_ctx} > $3/put_nbi_th.log - check put_nbi_th -# echo "team_ctx_infra" -# mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 42 -x ${shm_ctx} > $3/team_ctx_infra.log -# check team_ctx_infra - echo "team_ctx_put_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 41 -x ${shm_ctx} > $3/team_ctx_put_nbi.log - check team_ctx_put_nbi - echo "team_ctx_put_nbi_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 41 -x ${shm_ctx} > $3/team_ctx_put_nbi_th.log - check team_ctx_put_nbi_th - echo "team_ctx_get_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 39 -x ${shm_ctx} > $3/team_ctx_get_nbi.log - check team_ctx_get_nbi - echo "team_ctx_get_nbi_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 39 -x ${shm_ctx} > $3/team_ctx_get_nbi_th.log - check team_ctx_get_nbi_th - echo "reduction" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 5 -x ${shm_ctx} > $3/reduction.log - check reduction - echo "amo_fadd" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 6 -x ${shm_ctx} > $3/amo_fadd.log - check amo_fadd - echo "amo_fadd_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 6 -x ${shm_ctx} > $3/amo_fadd_th.log - check amo_fadd_th - echo "amo_finc" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 7 -x ${shm_ctx} > $3/amo_finc.log - check amo_finc - echo "amo_finc_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 7 -x ${shm_ctx} > $3/amo_finc_th.log - check amo_finc_th - echo "amo_fetch" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 8 -x ${shm_ctx} > $3/amo_fetch.log - check amo_fetch - echo "amo_fetch_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 8 -x ${shm_ctx} > $3/amo_fetch_th.log - check amo_fetch_th - echo "amo_fcswap" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 9 -x ${shm_ctx} > $3/amo_fcswap.log - check amo_fcswap - echo "amo_fcswap_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 9 -x ${shm_ctx} > $3/amo_fcswap_th.log - check amo_fcswap_th - echo "amo_add" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 10 -x ${shm_ctx} > $3/amo_add.log - check amo_add - echo "amo_add_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 10 -x ${shm_ctx} > $3/amo_add_th.log - check amo_add_th - echo "amo_set" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 44 -x ${shm_ctx} > $3/amo_set.log - check amo_set - echo "amo_set_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 44 -x ${shm_ctx} > $3/amo_set_th.log - check amo_set_th - echo "amo_inc" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 11 -x ${shm_ctx} > $3/amo_inc.log - check amo_inc - echo "amo_inc_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 11 -x ${shm_ctx} > $3/amo_inc_th.log - check amo_inc_th - echo "init" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 13 -x ${shm_ctx} > $3/init.log - check init - echo "ping_pong" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 14 -x ${shm_ctx} > $3/ping_pong.log - check ping_pong - echo "ping_pong_th" - mpirun -np 2 ${gdb_cmd} $1 -t 1024 -w 25 -s 32768 -a 14 -x ${shm_ctx} > $3/ping_pong_th.log - check ping_pong_th - echo "barrier_all" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 17 -x ${shm_ctx} > $3/barrier_all.log - check barrier_all - echo "sync_all" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 18 -x ${shm_ctx} > $3/sync_all.log - check sync_all - echo "sync" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 19 -x ${shm_ctx} > $3/sync.log - check sync - echo "alltoall" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 23 -x ${shm_ctx} > $3/alltoall.log - check alltoall - echo "fcollect" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 22 -x ${shm_ctx} > $3/fcollect.log - check fcollect + ########################################################################### + ############################## SERIAL TESTS ############################### + ########################################################################### + *"serial") + echo "get_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 0 > $3/get_n2_w1_z1_1MB.log + check get_n2_w1_z1_1MB + echo "getnbi_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 1 > $3/getnbi_n2_w1_z1_1MB.log + check getnbi_n2_w1_z1_1MB + echo "put_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 2 > $3/put_n2_w1_z1_1MB.log + check put_n2_w1_z1_1MB + echo "putnbi_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 3 > $3/putnbi_n2_w1_z1_1MB.log + check putnbi_n2_w1_z1_1MB + echo "amofadd_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 6 > $3/amofadd_n2_w1_z1.log + check amofadd_n2_w1_z1 + echo "amofinc_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 7 > $3/amofinc_n2_w1_z1.log + check amofinc_n2_w1_z1 + echo "amofetch_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 8 > $3/amofetch_n2_w1_z1.log + check amofetch_n2_w1_z1 + echo "amofcswap_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 9 > $3/amofcswap_n2_w1_z1.log + check amofcswap_n2_w1_z1 + echo "amoadd_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 10 > $3/amoadd_n2_w1_z1.log + check amoadd_n2_w1_z1 + echo "amoinc_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 11 > $3/amoinc_n2_w1_z1.log + check amoinc_n2_w1_z1 + echo "pingpong_n2_w1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 14 > $3/pingpong_n2_w1.log + check pingpong_n2_w1 + echo "amoset_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 44 > $3/amoset_n2_w1_z1.log + check amoset_n2_w1_z1 ;; - *"multi_thread") - echo "get" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 0 -x ${shm_ctx} > $3/get.log - check get - echo "get_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 1 -x ${shm_ctx} > $3/get_nbi.log - check get_nbi - echo "put" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 2 -x ${shm_ctx} > $3/put.log - check put - echo "put_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 3 -x ${shm_ctx} > $3/put_nbi.log - check put_nbi -# echo "team_ctx_infra" -# mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 42 -x ${shm_ctx} > $3/team_ctx_infra.log -# check team_ctx_infra - echo "team_ctx_put_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 41 -x ${shm_ctx} > $3/team_ctx_put_nbi.log - check team_ctx_put_nbi - echo "team_ctx_get_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 39 -x ${shm_ctx} > $3/team_ctx_get_nbi.log - check team_ctx_get_nbi - echo "get_swarm" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 4 -x ${shm_ctx} > $3/get_swarm.log - check get_swarm - echo "reduction" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 5 -x ${shm_ctx} > $3/reduction.log - check reduction - echo "amo_fadd" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 6 -x ${shm_ctx} > $3/amo_fadd.log - check amo_fadd - echo "amo_finc" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 7 -x ${shm_ctx} > $3/amo_finc.log - check amo_finc - echo "amo_fetch" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 8 -x ${shm_ctx} > $3/amo_fetch.log - check amo_fetch - echo "amo_fcswap" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 9 -x ${shm_ctx} > $3/amo_fcswap.log - check amo_fcswap - echo "amo_add" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 10 -x ${shm_ctx} > $3/amo_add.log - check amo_add - echo "amo_set" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 44 -x ${shm_ctx} > $3/amo_set.log - check amo_set - echo "amo_inc" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 11 -x ${shm_ctx} > $3/amo_inc.log - check amo_inc - echo "init" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 13 -x ${shm_ctx} > $3/init.log - check init - echo "ping_pong" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 14 -x ${shm_ctx} > $3/ping_pong.log - check ping_pong - echo "barrier_all" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 17 -x ${shm_ctx} > $3/barrier_all.log - check barrier_all - echo "sync_all" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 18 -x ${shm_ctx} > $3/sync_all.log - check sync_all - echo "sync" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 19 -x ${shm_ctx} > $3/sync.log - check sync - echo "alltoall" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 23 -x ${shm_ctx} > $3/alltoall.log - check alltoall - echo "fcollect" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 22 -x ${shm_ctx} > $3/fcollect.log - check fcollect + + ########################################################################### + ############################### SHORT TESTS ############################### + ########################################################################### + *"short") + echo "get_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 0 > $3/get_n2_w16_z128_8B.log + check get_n2_w16_z128_8B + echo "getnbi_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 1 > $3/getnbi_n2_w16_z128_8B.log + check getnbi_n2_w16_z128_8B + echo "put_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 2 > $3/put_n2_w16_z128_8B.log + check put_n2_w16_z128_8B + echo "putnbi_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 3 > $3/putnbi_n2_w16_z128_8B.log + check putnbi_n2_w16_z128_8B + echo "amofadd_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 6 > $3/amofadd_n2_w8_z1.log + check amofadd_n2_w8_z1 + echo "amofinc_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 7 > $3/amofinc_n2_w8_z1.log + check amofinc_n2_w8_z1 + echo "amofetch_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 8 > $3/amofetch_n2_w8_z1.log + check amofetch_n2_w8_z1 + echo "amofcswap_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 9 > $3/amofcswap_n2_w8_z1.log + check amofcswap_n2_w8_z1 + echo "amoadd_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 10 > $3/amoadd_n2_w8_z1.log + check amoadd_n2_w8_z1 + echo "amoinc_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 11 > $3/amoinc_n2_w8_z1.log + check amoinc_n2_w8_z1 + echo "pingpong_n2_w1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 14 > $3/pingpong_n2_w1.log + check pingpong_n2_w1 + echo "amoset_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 44 > $3/amoset_n2_w8_z1.log + check amoset_n2_w8_z1 ;; - *"ro") - echo "get" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 0 -x ${shm_ctx} > $3/get.log - check get - echo "get_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 1 -x ${shm_ctx} > $3/get_nbi.log - check get_nbi - echo "put" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 2 -x ${shm_ctx} > $3/put.log - check put - echo "put_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 3 -x ${shm_ctx} > $3/put_nbi.log - check put_nbi -# echo "team_ctx_infra" -# mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 42 -x ${shm_ctx} > $3/team_ctx_infra.log -# check team_ctx_infra - echo "team_ctx_put_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 41 -x ${shm_ctx} > $3/team_ctx_put_nbi.log - check team_ctx_put_nbi - echo "team_ctx_get_nbi" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 39 -x ${shm_ctx} > $3/team_ctx_get_nbi.log - check team_ctx_get_nbi -# mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 4 -x ${shm_ctx} > $3/get_swarm.log -# check get_swarm - #mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 5 -x ${shm_ctx} > $3/reduction.log - echo "amo_fadd" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 6 -x ${shm_ctx} > $3/amo_fadd.log - check amo_fadd - echo "amo_finc" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 7 -x ${shm_ctx} > $3/amo_finc.log - check amo_finc - echo "amo_fetch" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 8 -x ${shm_ctx} > $3/amo_fetch.log - check amo_fetch - echo "amo_fcswap" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 9 -x ${shm_ctx} > $3/amo_fcswap.log - check amo_fcswap - echo "amo_add" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 10 -x ${shm_ctx} > $3/amo_add.log - check amo_add - echo "amo_set" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 44 -x ${shm_ctx} > $3/amo_set.log - check amo_set - echo "amo_swap" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 45 -x ${shm_ctx} > $3/amo_swap.log - check amo_swap - echo "amo_fetch_and" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 46 -x ${shm_ctx} > $3/amo_fetch_and.log - check amo_fetch_and - echo "amo_and" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 49 -x ${shm_ctx} > $3/amo_and.log - check amo_and - echo "amo_fetch_or" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 47 -x ${shm_ctx} > $3/amo_fetch_or.log - check amo_fetch_or - echo "amo_or" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 50 -x ${shm_ctx} > $3/amo_or.log - check amo_or - echo "amo_fetch_xor" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 47 -x ${shm_ctx} > $3/amo_fetch_xor.log - check amo_fetch_xor - echo "amo_xor" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 50 -x ${shm_ctx} > $3/amo_xor.log - check amo_xor - echo "amo_inc" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 11 -x ${shm_ctx} > $3/amo_inc.log - check amo_inc - echo "init" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 13 -x ${shm_ctx} > $3/init.log - check init - echo "ping_pong" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 14 -x ${shm_ctx} > $3/ping_pong.log - check ping_pong - echo "barrier_all" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 17 -x ${shm_ctx} > $3/barrier_all.log - check barrier_all - echo "sync_all" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 18 -x ${shm_ctx} > $3/sync_all.log - check sync_all - echo "sync" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 19 -x ${shm_ctx} > $3/sync.log - check sync - echo "alltoall" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 23 -x ${shm_ctx} > $3/alltoall.log - check alltoall - echo "fcollect" - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 22 -x ${shm_ctx} > $3/fcollect.log - check fcollect - ;; - *"team_ctx_get") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 38 -x ${shm_ctx} - ;; - *"team_ctx_get_nbi") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 39 -x ${shm_ctx} - ;; - *"team_ctx_put") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 40 -x ${shm_ctx} - ;; - *"team_ctx_put_nbi") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 41 -x ${shm_ctx} + + ########################################################################### + ############################# EXHAUSTIVE TESTS ############################ + ########################################################################### + *"exhaustive") + ############################### GET ################################### + echo "get_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 0 > $3/get_n2_w1_z1_1MB.log + check get_n2_w1_z1_1MB + echo "get_n2_w1_z1024_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -s 512 -a 0 > $3/get_n2_w1_z1024_512B.log + check get_n2_w1_z1024_512B + echo "get_n2_w8_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -s 1048576 -a 0 > $3/get_n2_w8_z1_1MB.log + check get_n2_w8_z1_1MB + echo "get_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 0 > $3/get_n2_w16_z128_8B.log + check get_n2_w16_z128_8B + echo "get_n2_w32_z256_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 256 -s 512 -a 0 > $3/get_n2_w32_z256_512B.log + check get_n2_w32_z256_512B + echo "get_n2_w64_z1024_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=64 mpirun -np 2 $1 -w 64 -z 1024 -s 8 -a 0 > $3/get_n2_w64_z1024_8B.log + check get_n2_w64_z1024_8B + ############################### GETNBI ################################ + echo "getnbi_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 1 > $3/getnbi_n2_w1_z1_1MB.log + check getnbi_n2_w1_z1_1MB + echo "getnbi_n2_w1_z1024_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -s 512 -a 1 > $3/getnbi_n2_w1_z1024_512B.log + check getnbi_n2_w1_z1024_512B + echo "getnbi_n2_w8_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -s 1048576 -a 1 > $3/getnbi_n2_w8_z1_1MB.log + check getnbi_n2_w8_z1_1MB + echo "getnbi_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 1 > $3/getnbi_n2_w16_z128_8B.log + check getnbi_n2_w16_z128_8B + echo "getnbi_n2_w32_z256_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 256 -s 512 -a 1 > $3/getnbi_n2_w32_z256_512B.log + check getnbi_n2_w32_z256_512B + echo "getnbi_n2_w64_z1024_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=64 mpirun -np 2 $1 -w 64 -z 1024 -s 8 -a 1 > $3/getnbi_n2_w64_z1024_8B.log + check getnbi_n2_w64_z1024_8B + ############################### PUT ################################### + echo "put_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 2 > $3/put_n2_w1_z1_1MB.log + check put_n2_w1_z1_1MB + echo "put_n2_w1_z1024_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -s 512 -a 2 > $3/put_n2_w1_z1024_512B.log + check put_n2_w1_z1024_512B + echo "put_n2_w8_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -s 1048576 -a 2 > $3/put_n2_w8_z1_1MB.log + check put_n2_w8_z1_1MB + echo "put_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 2 > $3/put_n2_w16_z128_8B.log + check put_n2_w16_z128_8B + echo "put_n2_w32_z256_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 256 -s 512 -a 2 > $3/put_n2_w32_z256_512B.log + check put_n2_w32_z256_512B + echo "put_n2_w64_z1024_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=64 mpirun -np 2 $1 -w 64 -z 1024 -s 8 -a 2 > $3/put_n2_w64_z1024_8B.log + check put_n2_w64_z1024_8B + ############################### PUTNBI ################################ + echo "putnbi_n2_w1_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 3 > $3/putnbi_n2_w1_z1_1MB.log + check putnbi_n2_w1_z1_1MB + echo "putnbi_n2_w1_z1024_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -s 512 -a 3 > $3/putnbi_n2_w1_z1024_512B.log + check putnbi_n2_w1_z1024_512B + echo "putnbi_n2_w8_z1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -s 1048576 -a 3 > $3/putnbi_n2_w8_z1_1MB.log + check putnbi_n2_w8_z1_1MB + echo "putnbi_n2_w16_z128_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 3 > $3/putnbi_n2_w16_z128_8B.log + check putnbi_n2_w16_z128_8B + echo "putnbi_n2_w32_z256_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 256 -s 512 -a 3 > $3/putnbi_n2_w32_z256_512B.log + check putnbi_n2_w32_z256_512B + echo "putnbi_n2_w64_z1024_8B" + ROC_SHMEM_MAX_NUM_CONTEXTS=64 mpirun -np 2 $1 -w 64 -z 1024 -s 8 -a 3 > $3/putnbi_n2_w64_z1024_8B.log + check putnbi_n2_w64_z1024_8B + ############################# REDUCTION ############################## + echo "reduction_n2_w1_z1_32K" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 32768 -a 5 > $3/reduction_n2_w1_z1_32K.log + check reduction_n2_w1_z1_32K + echo "reduction_n2_w8_z1_32K" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -s 32768 -a 5 > $3/reduction_n2_w8_z1_32K.log + check reduction_n2_w8_z1_32K + echo "reduction_n2_w32_z1_32K" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 1 -s 32768 -a 5 > $3/reduction_n2_w32_z1_32K.log + check reduction_n2_w32_z1_32K + ############################## AMOFADD ############################### + echo "amofadd_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 6 > $3/amofadd_n2_w1_z1.log + check amofadd_n2_w1_z1 + echo "amofadd_n2_w1_z1024" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -a 6 > $3/amofadd_n2_w1_z1024.log + check amofadd_n2_w1_z1024 + echo "amofadd_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 6 > $3/amofadd_n2_w8_z1.log + check amofadd_n2_w8_z1 + echo "amofadd_n2_w32_z128" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 128 -a 6 > $3/amofadd_n2_w32_z128.log + check amofadd_n2_w32_z128 + ############################## AMOFINC ############################### + echo "amofinc_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 7 > $3/amofinc_n2_w1_z1.log + check amofinc_n2_w1_z1 + echo "amofinc_n2_w1_z1024" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -a 7 > $3/amofinc_n2_w1_z1024.log + check amofinc_n2_w1_z1024 + echo "amofinc_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 7 > $3/amofinc_n2_w8_z1.log + check amofinc_n2_w8_z1 + echo "amofinc_n2_w32_z128" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 128 -a 7 > $3/amofinc_n2_w32_z128.log + check amofinc_n2_w32_z128 + ############################ AMOFETCH ################################ + echo "amofetch_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 8 > $3/amofetch_n2_w1_z1.log + check amofetch_n2_w1_z1 + echo "amofetch_n2_w1_z1024" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -a 8 > $3/amofetch_n2_w1_z1024.log + check amofetch_n2_w1_z1024 + echo "amofetch_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 8 > $3/amofetch_n2_w8_z1.log + check amofetch_n2_w8_z1 + echo "amofetch_n2_w32_z128" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 128 -a 8 > $3/amofetch_n2_w32_z128.log + check amofetch_n2_w32_z128 + ########################### AMOFCSWAP ################################ + echo "amofcswap_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 9 > $3/amofcswap_n2_w1_z1.log + check amofcswap_n2_w1_z1 + echo "amofcswap_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 9 > $3/amofcswap_n2_w8_z1.log + check amofcswap_n2_w8_z1 + echo "amofcswap_n2_w32_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 1 -a 9 > $3/amofcswap_n2_w32_z1.log + check amofcswap_n2_w32_z1 + ############################# AMOADD ################################ + echo "amoadd_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 10 > $3/amoadd_n2_w1_z1.log + check amoadd_n2_w1_z1 + echo "amoadd_n2_w1_z1024" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -a 10 > $3/amoadd_n2_w1_z1024.log + check amoadd_n2_w1_z1024 + echo "amoadd_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 10 > $3/amoadd_n2_w8_z1.log + check amoadd_n2_w8_z1 + echo "amoadd_n2_w32_z128" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 128 -a 10 > $3/amoadd_n2_w32_z128.log + check amoadd_n2_w32_z128 + ############################# AMOINC ################################ + echo "amoinc_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 11 > $3/amoinc_n2_w1_z1.log + check amoinc_n2_w1_z1 + echo "amoinc_n2_w1_z1024" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1024 -a 11 > $3/amoinc_n2_w1_z1024.log + check amoinc_n2_w1_z1024 + echo "amoinc_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 11 > $3/amoinc_n2_w8_z1.log + check amoinc_n2_w8_z1 + echo "amoinc_n2_w32_z128" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 128 -a 11 > $3/amoinc_n2_w32_z128.log + check amoinc_n2_w32_z128 + ############################## INIT ################################# + echo "init_n2" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -a 13 > $3/init_n2.log + check init_n2 + ########################### PINGPONG ################################ + echo "pingpong_n2_w1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 14 > $3/pingpong_n2_w1.log + check pingpong_n2_w1 + echo "pingpong_n2_w8" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -a 14 > $3/pingpong_n2_w8.log + check pingpong_n2_w8 + echo "pingpong_n2_w32" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -a 14 > $3/pingpong_n2_w32.log + check pingpong_n2_w32 + ############################ BARRIER ################################ + echo "barrier_n2_w1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 17 > $3/barrier_n2_w1.log + check barrier_n2_w1 + echo "barrier_n2_w8" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -a 17 > $3/barrier_n2_w8.log + check barrier_n2_w8 + echo "barrier_n2_w32" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -a 17 > $3/barrier_n2_w32.log + check barrier_n2_w32 + ############################ SYNCALL ################################ + echo "syncall_n2_w1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 18 > $3/syncall_n2_w1.log + check syncall_n2_w1 + echo "syncall_n2_w8" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -a 18 > $3/syncall_n2_w8.log + check syncall_n2_w8 + echo "syncall_n2_w32" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -a 18 > $3/syncall_n2_w32.log + check syncall_n2_w32 + ############################# SYNC ################################## + echo "sync_n2_w1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 19 > $3/sync_n2_w1.log + check sync_n2_w1 + echo "sync_n2_w8" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -a 19 > $3/sync_n2_w8.log + check sync_n2_w8 + echo "sync_n2_w32" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -a 19 > $3/sync_n2_w32.log + check sync_n2_w32 + ########################### FCOLLECT ################################ + echo "fcollect_n2_w1_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -s 512 -a 22 > $3/fcollect_n2_w1_512B.log + check fcollect_n2_w1_512B + echo "fcollect_n2_w8_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -s 512 -a 22 > $3/fcollect_n2_w8_512B.log + check fcollect_n2_w8_512B + echo "fcollect_n2_w32_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -s 512 -a 22 > $3/fcollect_n2_w32_512B.log + check fcollect_n2_w32_512B + ########################### ALLTOALL ################################ + echo "alltoall_n2_w1_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -s 512 -a 23 > $3/alltoall_n2_w1_512B.log + check alltoall_n2_w1_512B + echo "alltoall_n2_w8_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -s 512 -a 23 > $3/alltoall_n2_w8_512B.log + check alltoall_n2_w8_512B + echo "alltoall_n2_w32_512B" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -s 512 -a 23 > $3/alltoall_n2_w32_512B.log + check alltoall_n2_w32_512B + ########################## TEAMGETNBI ############################### + echo "teamgetnbi_n2_w1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -s 1048576 -a 39 > $3/teamgetnbi_n2_w1_1MB.log + check teamgetnbi_n2_w1_1MB + ########################## TEAMPUTNBI ############################### + echo "teamputnbi_n2_w1_1MB" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -s 1048576 -a 41 > $3/teamputnbi_n2_w1_1MB.log + check teamputnbi_n2_w1_1MB + ############################ AMOSET ################################# + echo "amoset_n2_w1_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 44 > $3/amoset_n2_w1_z1.log + check amoset_n2_w1_z1 + echo "amoset_n2_w8_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 44 > $3/amoset_n2_w8_z1.log + check amoset_n2_w8_z1 + echo "amoset_n2_w32_z1" + ROC_SHMEM_MAX_NUM_CONTEXTS=32 mpirun -np 2 $1 -w 32 -z 1 -a 44 > $3/amoset_n2_w32_z1.log + check amoset_n2_w32_z1 ;; + + ########################################################################### + ############################# INDIVIDUAL TESTS ############################ + ########################################################################### *"get") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 0 -x ${shm_ctx} + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 0 ;; - *"get_nbi") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 1 -x ${shm_ctx} + *"getnbi") + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 1 ;; *"put") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 2 -x ${shm_ctx} + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 2 ;; - *"put_nbi") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 3 -x ${shm_ctx} - ;; - *"get_swarm") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 4 -x ${shm_ctx} - ;; - *"team_reduction") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 37 -x ${shm_ctx} + *"putnbi") + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 3 ;; *"reduction") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 5 -x ${shm_ctx} + mpirun -np 2 $1 -w 1 -z 1 -s 32768 -a 5 ;; - *"amo_fadd") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 6 -x ${shm_ctx} + *"amofadd") + mpirun -np 2 $1 -w 1 -z 1 -a 6 ;; - *"amo_finc") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 7 -x ${shm_ctx} + *"amofinc") + mpirun -np 2 $1 -w 1 -z 1 -a 7 ;; - *"amo_fetch") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 8 -x ${shm_ctx} + *"amofetch") + mpirun -np 2 $1 -w 1 -z 1 -a 8 ;; - *"amo_fcswap") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 9 -x ${shm_ctx} + *"amofcswap") + mpirun -np 2 $1 -w 1 -z 1 -a 9 ;; - *"amo_add") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 10 -x ${shm_ctx} + *"amoadd") + mpirun -np 2 $1 -w 1 -z 1 -a 10 ;; - *"amo_set") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 44 -x ${shm_ctx} - ;; - *"amo_swap") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 45 -x ${shm_ctx} - ;; - *"amo_fetch_and") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 46 -x ${shm_ctx} - ;; - *"amo_and") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 49 -x ${shm_ctx} - ;; - *"amo_fetch_or") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 47 -x ${shm_ctx} - ;; - *"amo_or") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 50 -x ${shm_ctx} - ;; - *"amo_fetch_xor") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 48 -x ${shm_ctx} - ;; - *"amo_xor") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 51 -x ${shm_ctx} - ;; - *"amo_inc") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 11 -x ${shm_ctx} + *"amoinc") + mpirun -np 2 $1 -w 1 -z 1 -a 11 ;; *"init") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 13 -x ${shm_ctx} + mpirun -np 2 $1 -a 13 ;; - *"ping_pong") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 32768 -a 14 -x ${shm_ctx} + *"pingpong") + mpirun -np 2 $1 -w 1 -z 1 -a 14 ;; - *"team_broadcast") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 36 -x ${shm_ctx} + *"barrier") + mpirun -np 2 $1 -w 1 -z 1 -a 17 ;; - *"alltoall") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 23 -x ${shm_ctx} - ;; - *"fcollect") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 22 -x ${shm_ctx} - ;; - *"broadcast") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 512 -a 20 -x ${shm_ctx} - ;; - *"barrier_all") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 17 -x ${shm_ctx} - ;; - *"sync_all") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 18 -x ${shm_ctx} + *"syncall") + mpirun -np 2 $1 -w 1 -z 1 -a 18 ;; *"sync") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 19 -x ${shm_ctx} + mpirun -np 2 $1 -w 1 -z 1 -s 8 -a 19 + ;; + *"broadcast") + mpirun -np 2 $1 -w 1 -z 1 -s 32768 -a 20 + ;; + *"fcollect") + mpirun -np 2 $1 -w 1 -z 1 -s 32768 -a 22 + ;; + *"alltoall") + mpirun -np 2 $1 -w 1 -z 1 -s 32768 -a 23 + ;; + *"team_broadcast") + mpirun -np 2 $1 -w 1 -z 1 -s 32768 -a 36 + ;; + *"team_reduction") + mpirun -np 2 $1 -w 1 -z 1 -s 32768 -a 37 + ;; + *"team_get") + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 38 + ;; + *"team_getnbi") + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 39 + ;; + *"team_put") + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 40 + ;; + *"team_putnbi") + mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 41 ;; *"ctx_infra") - mpirun -np 2 ${gdb_cmd} $1 -t 1 -w 1 -s 8 -a 42 -x ${shm_ctx} + mpirun -np 2 $1 -w 1 -z 1 -a 42 + ;; + *"amoset") + mpirun -np 2 $1 -w 1 -z 1 -a 44 + ;; + *"amoswap") + mpirun -np 2 $1 -w 1 -z 1 -a 45 + ;; + *"amofetchand") + mpirun -np 2 $1 -w 1 -z 1 -a 46 + ;; + *"amofetchor") + mpirun -np 2 $1 -w 1 -z 1 -a 47 + ;; + *"amofetchxor") + mpirun -np 2 $1 -w 1 -z 1 -a 48 + ;; + *"amoand") + mpirun -np 2 $1 -w 1 -z 1 -a 49 + ;; + *"amoor") + mpirun -np 2 $1 -w 1 -z 1 -a 50 + ;; + *"amoxor") + mpirun -np 2 $1 -w 1 -z 1 -a 51 ;; *) echo "UNKNOWN TEST TYPE: $2" From 6880f8e741233baeeca6dcc7965e19a4bf1a2d15 Mon Sep 17 00:00:00 2001 From: Muhammad Awad Date: Tue, 30 Jul 2024 12:52:06 -0500 Subject: [PATCH 04/46] Remove `dev_mtx_` that is no longer in the backend [ROCm/rocshmem commit: ebcfdb2f75c6e94c0dcc114c9c2de33aedcf8b52] --- projects/rocshmem/src/gpu_ib/backend_ib.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/rocshmem/src/gpu_ib/backend_ib.cpp b/projects/rocshmem/src/gpu_ib/backend_ib.cpp index 9ae3acd482..1357aea78d 100644 --- a/projects/rocshmem/src/gpu_ib/backend_ib.cpp +++ b/projects/rocshmem/src/gpu_ib/backend_ib.cpp @@ -133,8 +133,6 @@ __device__ bool GPUIBBackend::create_ctx(int64_t options, return false; } ctx_ = pop_result.value; - - ctx_->dev_mtx_.shareable_ = static_cast(options >> 3); ctx->ctx_opaque = ctx_; return true; } From 61140c9a10849730b2b09e16704a75ee351784ba Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 30 Jul 2024 12:56:32 -0500 Subject: [PATCH 05/46] Move inline assembly into arch defines blocks [ROCm/rocshmem commit: 73303ca2d26658bb41a63224fea5a41545753fad] --- .../src/reverse_offload/context_ro_device.cpp | 8 +++-- projects/rocshmem/src/util.cpp | 2 ++ projects/rocshmem/src/util.hpp | 30 ++++++++++++++++--- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/projects/rocshmem/src/reverse_offload/context_ro_device.cpp b/projects/rocshmem/src/reverse_offload/context_ro_device.cpp index 6182ce0917..6c6ea8f809 100644 --- a/projects/rocshmem/src/reverse_offload/context_ro_device.cpp +++ b/projects/rocshmem/src/reverse_offload/context_ro_device.cpp @@ -391,20 +391,24 @@ __device__ bool enough_space(BlockHandle *h, uint64_t required) { __device__ void refresh_volatile_dwordx2(volatile uint64_t *assigned_value, volatile uint64_t *read_value) { - __asm__ volatile( +#if __gfx90a__ + asm volatile( "global_load_dwordx2 %0 %1 off glc slc\n " "s_waitcnt vmcnt(0)" : "=v"(*assigned_value) : "v"(read_value)); +#endif } __device__ void refresh_volatile_sbyte(volatile int *assigned_value, volatile char *read_value) { - __asm__ volatile( +#if __gfx90a__ + asm volatile( "global_load_sbyte %0 %1 off glc slc\n " "s_waitcnt vmcnt(0)" : "=v"(*assigned_value) : "v"(read_value)); +#endif } __device__ void acquire_lock(BlockHandle *handle) { diff --git a/projects/rocshmem/src/util.cpp b/projects/rocshmem/src/util.cpp index 11122d7b16..c68bfc55c1 100644 --- a/projects/rocshmem/src/util.cpp +++ b/projects/rocshmem/src/util.cpp @@ -42,10 +42,12 @@ std::vector cpu_agents; __device__ uint64_t __read_clock() { uint64_t clock{}; +#if __gfx90a__ asm volatile( "s_memrealtime %0\n" "s_waitcnt lgkmcnt(0)\n" : "=s"(clock)); +#endif return clock; } diff --git a/projects/rocshmem/src/util.hpp b/projects/rocshmem/src/util.hpp index 1d5cd9f9a6..1fa9330509 100644 --- a/projects/rocshmem/src/util.hpp +++ b/projects/rocshmem/src/util.hpp @@ -43,11 +43,13 @@ namespace rocshmem { __device__ __forceinline__ int uncached_load_ubyte(uint8_t* src) { int ret; - __asm__ volatile( +#if __gfx90a__ + asm volatile( "global_load_ubyte %0 %1 off glc slc \n" "s_waitcnt vmcnt(0)" : "=v"(ret) : "v"(src)); +#endif return ret; } @@ -62,18 +64,22 @@ NOWARN(-Wdeprecated-volatile, T ret; switch (sizeof(T)) { case 4: - __asm__ volatile( +#if __gfx90a__ + asm volatile( "global_load_dword %0 %1 off glc slc \n" "s_waitcnt vmcnt(0)" : "=v"(ret) : "v"(src)); +#endif break; case 8: - __asm__ volatile( +#if __gfx90a__ + asm volatile( "global_load_dwordx2 %0 %1 off glc slc \n" "s_waitcnt vmcnt(0)" : "=v"(ret) : "v"(src)); +#endif break; default: break; @@ -123,7 +129,13 @@ NOWARN(-Wdeprecated-volatile, extern const int gpu_clock_freq_mhz; /* Device-side internal functions */ -__device__ __forceinline__ void __roc_inv() { asm volatile("buffer_wbinvl1;"); } +__device__ __forceinline__ void __roc_inv() { +#if defined USE_COHERENT_HEAP +#if __gfx90a__ + asm volatile("buffer_wbinvl1;"); +#endif +#endif +} __device__ __forceinline__ void __roc_flush() { #if defined USE_COHERENT_HEAP @@ -131,6 +143,10 @@ __device__ __forceinline__ void __roc_flush() { asm volatile("s_dcache_wb;"); asm volatile("buffer_wbl2;"); #endif +#if __gfx90a__ + asm volatile("s_dcache_wb;"); + asm volatile("buffer_wbl2;"); +#endif #endif } @@ -227,17 +243,23 @@ __device__ __forceinline__ void store_asm(uint8_t* val, uint8_t* dst, switch (size) { case 2: { int16_t val16{*(reinterpret_cast(val))}; +#if __gfx90a__ asm volatile("flat_store_short %0 %1 glc slc" : : "v"(dst), "v"(val16)); +#endif break; } case 4: { int32_t val32{*(reinterpret_cast(val))}; +#if __gfx90a__ asm volatile("flat_store_dword %0 %1 glc slc" : : "v"(dst), "v"(val32)); +#endif break; } case 8: { int64_t val64{*(reinterpret_cast(val))}; +#if __gfx90a__ asm volatile("flat_store_dwordx2 %0 %1 glc slc" : : "v"(dst), "v"(val64)); +#endif break; } default: From 21d8301c837af05ed6eb2f8f4da2a09d12859cba Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 30 Jul 2024 14:53:19 -0500 Subject: [PATCH 06/46] Move inline asm into separate file [ROCm/rocshmem commit: 862ef5713f0aebbba5292fc6bd23d6ab33eb1b14] --- projects/rocshmem/src/assembly.hpp | 236 ++++++++++++++++++ .../src/reverse_offload/context_ro_device.cpp | 22 -- projects/rocshmem/src/util.cpp | 11 - projects/rocshmem/src/util.hpp | 110 +------- 4 files changed, 237 insertions(+), 142 deletions(-) create mode 100644 projects/rocshmem/src/assembly.hpp diff --git a/projects/rocshmem/src/assembly.hpp b/projects/rocshmem/src/assembly.hpp new file mode 100644 index 0000000000..8d69249561 --- /dev/null +++ b/projects/rocshmem/src/assembly.hpp @@ -0,0 +1,236 @@ +/****************************************************************************** + * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef LIBRARY_SRC_ASSEMBLY_HPP_ +#define LIBRARY_SRC_ASSEMBLY_HPP_ + +#include +#include +#include + +namespace rocshmem { + +#define DO_PRAGMA(x) _Pragma(#x) +#define NOWARN(warnoption, ...) \ + DO_PRAGMA(GCC diagnostic push) \ + DO_PRAGMA(GCC diagnostic ignored #warnoption) \ + __VA_ARGS__ \ + DO_PRAGMA(GCC diagnostic pop) + +#define SFENCE() asm volatile("sfence" ::: "memory") + +__device__ __forceinline__ int uncached_load_ubyte(uint8_t* src) { + int ret; +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile( + "global_load_ubyte %0 %1 off glc slc \n" + "s_waitcnt vmcnt(0)" + : "=v"(ret) + : "v"(src)); +#endif +#if __gfx940__ +#endif + return ret; +} + +__device__ __forceinline__ void refresh_volatile_sbyte(volatile int *assigned_value, + volatile char *read_value) { +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile( + "global_load_sbyte %0 %1 off glc slc\n " + "s_waitcnt vmcnt(0)" + : "=v"(*assigned_value) + : "v"(read_value)); +#endif +#if __gfx940__ +#endif +} + +__device__ __forceinline__ void refresh_volatile_dwordx2(volatile uint64_t *assigned_value, + volatile uint64_t *read_value) { +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile( + "global_load_dwordx2 %0 %1 off glc slc\n " + "s_waitcnt vmcnt(0)" + : "=v"(*assigned_value) + : "v"(read_value)); +#endif +#if __gfx940__ +#endif +} + +/* Ignore the warning about deprecated volatile. + * The only usage of volatile is to force the compiler to generate + * the assembly instruction. If volatile is omitted, the compiler + * will NOT generate the non-temporal load or the waitcnt. + */ +// clang-format off +NOWARN(-Wdeprecated-volatile, + template __device__ __forceinline__ T uncached_load(T* src) { + T ret; + switch (sizeof(T)) { + case 4: +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile( + "global_load_dword %0 %1 off glc slc \n" + "s_waitcnt vmcnt(0)" + : "=v"(ret) + : "v"(src)); +#endif +#if __gfx940__ +#endif + break; + case 8: +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile( + "global_load_dwordx2 %0 %1 off glc slc \n" + "s_waitcnt vmcnt(0)" + : "=v"(ret) + : "v"(src)); +#endif +#if __gfx940__ +#endif + break; + default: + break; + } + return ret; + } +) +// clang-format on + +__device__ __forceinline__ void __roc_inv() { +#if defined USE_COHERENT_HEAP +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile("buffer_wbinvl1;"); +#endif +#if __gfx940__ +#endif +#endif +} + +__device__ __forceinline__ void __roc_flush() { +#if defined USE_COHERENT_HEAP +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile("s_dcache_wb;"); + asm volatile("buffer_wbl2;"); +#endif +#if __gfx940__ +#endif +#endif +} + +__device__ __forceinline__ void store_asm(uint8_t* val, uint8_t* dst, + int size) { + switch (size) { + case 2: { + int16_t val16{*(reinterpret_cast(val))}; +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile("flat_store_short %0 %1 glc slc" : : "v"(dst), "v"(val16)); +#endif +#if __gfx940__ +#endif + break; + } + case 4: { + int32_t val32{*(reinterpret_cast(val))}; +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile("flat_store_dword %0 %1 glc slc" : : "v"(dst), "v"(val32)); +#endif +#if __gfx940__ +#endif + break; + } + case 8: { + int64_t val64{*(reinterpret_cast(val))}; +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile("flat_store_dwordx2 %0 %1 glc slc" : : "v"(dst), "v"(val64)); +#endif +#if __gfx940__ +#endif + break; + } + default: + break; + } +} + +__device__ __forceinline__ uint64_t __read_clock() { + uint64_t clock{}; +#if __gfx906__ +#endif +#if __gfx908__ +#endif +#if __gfx90a__ + asm volatile( + "s_memrealtime %0\n" + "s_waitcnt lgkmcnt(0)\n" + : "=s"(clock)); +#endif +#if __gfx940__ +#endif + return clock; +} + +} // namespace rocshmem + +#endif // LIBRARY_SRC_ASSEMBLY_HPP_ diff --git a/projects/rocshmem/src/reverse_offload/context_ro_device.cpp b/projects/rocshmem/src/reverse_offload/context_ro_device.cpp index 6c6ea8f809..8e933e9777 100644 --- a/projects/rocshmem/src/reverse_offload/context_ro_device.cpp +++ b/projects/rocshmem/src/reverse_offload/context_ro_device.cpp @@ -389,28 +389,6 @@ __device__ bool enough_space(BlockHandle *h, uint64_t required) { return (h->queue_size - (h->write_index - h->read_index)) >= required; } -__device__ void refresh_volatile_dwordx2(volatile uint64_t *assigned_value, - volatile uint64_t *read_value) { -#if __gfx90a__ - asm volatile( - "global_load_dwordx2 %0 %1 off glc slc\n " - "s_waitcnt vmcnt(0)" - : "=v"(*assigned_value) - : "v"(read_value)); -#endif -} - -__device__ void refresh_volatile_sbyte(volatile int *assigned_value, - volatile char *read_value) { -#if __gfx90a__ - asm volatile( - "global_load_sbyte %0 %1 off glc slc\n " - "s_waitcnt vmcnt(0)" - : "=v"(*assigned_value) - : "v"(read_value)); -#endif -} - __device__ void acquire_lock(BlockHandle *handle) { while(atomicCAS((uint64_t *)&handle->lock, 0, 1) == 1) ; } diff --git a/projects/rocshmem/src/util.cpp b/projects/rocshmem/src/util.cpp index c68bfc55c1..7318b14208 100644 --- a/projects/rocshmem/src/util.cpp +++ b/projects/rocshmem/src/util.cpp @@ -40,17 +40,6 @@ typedef struct device_agent { std::vector gpu_agents; std::vector cpu_agents; -__device__ uint64_t __read_clock() { - uint64_t clock{}; -#if __gfx90a__ - asm volatile( - "s_memrealtime %0\n" - "s_waitcnt lgkmcnt(0)\n" - : "=s"(clock)); -#endif - return clock; -} - hsa_status_t rocm_hsa_amd_memory_pool_callback( hsa_amd_memory_pool_t memory_pool, void* data) { hsa_amd_memory_pool_global_flag_t pool_flag{}; diff --git a/projects/rocshmem/src/util.hpp b/projects/rocshmem/src/util.hpp index 1fa9330509..c967ec1a9c 100644 --- a/projects/rocshmem/src/util.hpp +++ b/projects/rocshmem/src/util.hpp @@ -29,66 +29,12 @@ #include +#include "assembly.hpp" #include "config.h" // NOLINT(build/include_subdir) #include "constants.hpp" namespace rocshmem { -#define DO_PRAGMA(x) _Pragma(#x) -#define NOWARN(warnoption, ...) \ - DO_PRAGMA(GCC diagnostic push) \ - DO_PRAGMA(GCC diagnostic ignored #warnoption) \ - __VA_ARGS__ \ - DO_PRAGMA(GCC diagnostic pop) - -__device__ __forceinline__ int uncached_load_ubyte(uint8_t* src) { - int ret; -#if __gfx90a__ - asm volatile( - "global_load_ubyte %0 %1 off glc slc \n" - "s_waitcnt vmcnt(0)" - : "=v"(ret) - : "v"(src)); -#endif - return ret; -} - -/* Ignore the warning about deprecated volatile. - * The only usage of volatile is to force the compiler to generate - * the assembly instruction. If volatile is omitted, the compiler - * will NOT generate the non-temporal load or the waitcnt. - */ -// clang-format off -NOWARN(-Wdeprecated-volatile, - template __device__ __forceinline__ T uncached_load(T* src) { - T ret; - switch (sizeof(T)) { - case 4: -#if __gfx90a__ - asm volatile( - "global_load_dword %0 %1 off glc slc \n" - "s_waitcnt vmcnt(0)" - : "=v"(ret) - : "v"(src)); -#endif - break; - case 8: -#if __gfx90a__ - asm volatile( - "global_load_dwordx2 %0 %1 off glc slc \n" - "s_waitcnt vmcnt(0)" - : "=v"(ret) - : "v"(src)); -#endif - break; - default: - break; - } - return ret; - } -) -// clang-format on - #define LOAD(VAR) __atomic_load_n((VAR), __ATOMIC_SEQ_CST) #define STORE(DST, SRC) __atomic_store_n((DST), (SRC), __ATOMIC_SEQ_CST) @@ -102,8 +48,6 @@ NOWARN(-Wdeprecated-volatile, } \ } -#define SFENCE() asm volatile("sfence" ::: "memory") - #ifdef DEBUG #define DPRINTF(...) \ do { \ @@ -129,29 +73,6 @@ NOWARN(-Wdeprecated-volatile, extern const int gpu_clock_freq_mhz; /* Device-side internal functions */ -__device__ __forceinline__ void __roc_inv() { -#if defined USE_COHERENT_HEAP -#if __gfx90a__ - asm volatile("buffer_wbinvl1;"); -#endif -#endif -} - -__device__ __forceinline__ void __roc_flush() { -#if defined USE_COHERENT_HEAP -#if __gfx90a__ - asm volatile("s_dcache_wb;"); - asm volatile("buffer_wbl2;"); -#endif -#if __gfx90a__ - asm volatile("s_dcache_wb;"); - asm volatile("buffer_wbl2;"); -#endif -#endif -} - -__device__ uint64_t __read_clock(); - __device__ __forceinline__ uint32_t lowerID() { return __ffsll(__ballot(1)) - 1; } @@ -238,35 +159,6 @@ __device__ void gpu_dprintf(const char* fmt, const Args&... args) { } } -__device__ __forceinline__ void store_asm(uint8_t* val, uint8_t* dst, - int size) { - switch (size) { - case 2: { - int16_t val16{*(reinterpret_cast(val))}; -#if __gfx90a__ - asm volatile("flat_store_short %0 %1 glc slc" : : "v"(dst), "v"(val16)); -#endif - break; - } - case 4: { - int32_t val32{*(reinterpret_cast(val))}; -#if __gfx90a__ - asm volatile("flat_store_dword %0 %1 glc slc" : : "v"(dst), "v"(val32)); -#endif - break; - } - case 8: { - int64_t val64{*(reinterpret_cast(val))}; -#if __gfx90a__ - asm volatile("flat_store_dwordx2 %0 %1 glc slc" : : "v"(dst), "v"(val64)); -#endif - break; - } - default: - break; - } -} - __device__ __forceinline__ void memcpy(void* dst, void* src, size_t size) { uint8_t* dst_bytes{static_cast(dst)}; uint8_t* src_bytes{static_cast(src)}; From 962f4bedf8a43d9022c02b8677af0f0881638f5e Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Wed, 31 Jul 2024 14:14:24 -0500 Subject: [PATCH 07/46] Add assembly for gfx94x [ROCm/rocshmem commit: bb6e48dded1a6088c2bd0404faf9f059cac80e90] --- projects/rocshmem/src/assembly.hpp | 133 ++++++++++++++++++----------- 1 file changed, 84 insertions(+), 49 deletions(-) diff --git a/projects/rocshmem/src/assembly.hpp b/projects/rocshmem/src/assembly.hpp index 8d69249561..076600067e 100644 --- a/projects/rocshmem/src/assembly.hpp +++ b/projects/rocshmem/src/assembly.hpp @@ -40,53 +40,68 @@ namespace rocshmem { __device__ __forceinline__ int uncached_load_ubyte(uint8_t* src) { int ret; -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile( "global_load_ubyte %0 %1 off glc slc \n" "s_waitcnt vmcnt(0)" : "=v"(ret) : "v"(src)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile( + "global_load_ubyte %0 %1 off sc0 sc1 \n" + "s_waitcnt vmcnt(0)" + : "=v"(ret) + : "v"(src)); #endif return ret; } __device__ __forceinline__ void refresh_volatile_sbyte(volatile int *assigned_value, volatile char *read_value) { -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile( "global_load_sbyte %0 %1 off glc slc\n " "s_waitcnt vmcnt(0)" : "=v"(*assigned_value) : "v"(read_value)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile( + "global_load_sbyte %0 %1 off sc0 sc1\n " + "s_waitcnt vmcnt(0)" + : "=v"(*assigned_value) + : "v"(read_value)); #endif } __device__ __forceinline__ void refresh_volatile_dwordx2(volatile uint64_t *assigned_value, volatile uint64_t *read_value) { -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile( "global_load_dwordx2 %0 %1 off glc slc\n " "s_waitcnt vmcnt(0)" : "=v"(*assigned_value) : "v"(read_value)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile( + "global_load_dwordx2 %0 %1 off sc0 sc1\n " + "s_waitcnt vmcnt(0)" + : "=v"(*assigned_value) + : "v"(read_value)); #endif } @@ -101,33 +116,43 @@ NOWARN(-Wdeprecated-volatile, T ret; switch (sizeof(T)) { case 4: -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile( "global_load_dword %0 %1 off glc slc \n" "s_waitcnt vmcnt(0)" : "=v"(ret) : "v"(src)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile( + "global_load_dword %0 %1 off sc0 sc1 \n" + "s_waitcnt vmcnt(0)" + : "=v"(ret) + : "v"(src)); #endif break; +#if defined(__gfx906__) +#endif +#if defined(__gfx908__) +#endif +#if defined(__gfx90a__) case 8: -#if __gfx906__ -#endif -#if __gfx908__ -#endif -#if __gfx90a__ asm volatile( "global_load_dwordx2 %0 %1 off glc slc \n" "s_waitcnt vmcnt(0)" : "=v"(ret) : "v"(src)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile( + "global_load_dwordx2 %0 %1 off sc0 sc1 \n" + "s_waitcnt vmcnt(0)" + : "=v"(ret) + : "v"(src)); #endif break; default: @@ -140,29 +165,32 @@ NOWARN(-Wdeprecated-volatile, __device__ __forceinline__ void __roc_inv() { #if defined USE_COHERENT_HEAP -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ - asm volatile("buffer_wbinvl1;"); +#if defined(__gfx90a__) +// asm volatile("buffer_wbinvl1;"); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) +// asm volatile("buffer_inv sc0 sc1;"); #endif #endif } __device__ __forceinline__ void __roc_flush() { #if defined USE_COHERENT_HEAP -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ - asm volatile("s_dcache_wb;"); - asm volatile("buffer_wbl2;"); +#if defined(__gfx90a__) +// asm volatile("s_dcache_wb;"); +// asm volatile("buffer_wbl2;"); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) +// asm volatile("s_dcache_wb;"); +// asm volatile("buffer_wbl2;"); #endif #endif } @@ -172,40 +200,43 @@ __device__ __forceinline__ void store_asm(uint8_t* val, uint8_t* dst, switch (size) { case 2: { int16_t val16{*(reinterpret_cast(val))}; -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile("flat_store_short %0 %1 glc slc" : : "v"(dst), "v"(val16)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile("flat_store_short %0 %1 sc0 sc1" : : "v"(dst), "v"(val16)); #endif break; } case 4: { int32_t val32{*(reinterpret_cast(val))}; -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile("flat_store_dword %0 %1 glc slc" : : "v"(dst), "v"(val32)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile("flat_store_dword %0 %1 sc0 sc1" : : "v"(dst), "v"(val32)); #endif break; } case 8: { int64_t val64{*(reinterpret_cast(val))}; -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile("flat_store_dwordx2 %0 %1 glc slc" : : "v"(dst), "v"(val64)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile("flat_store_dwordx2 %0 %1 sc0 sc1" : : "v"(dst), "v"(val64)); #endif break; } @@ -216,17 +247,21 @@ __device__ __forceinline__ void store_asm(uint8_t* val, uint8_t* dst, __device__ __forceinline__ uint64_t __read_clock() { uint64_t clock{}; -#if __gfx906__ +#if defined(__gfx906__) #endif -#if __gfx908__ +#if defined(__gfx908__) #endif -#if __gfx90a__ +#if defined(__gfx90a__) asm volatile( "s_memrealtime %0\n" "s_waitcnt lgkmcnt(0)\n" : "=s"(clock)); #endif -#if __gfx940__ +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + asm volatile( + "s_memrealtime %0\n" + "s_waitcnt lgkmcnt(0)\n" + : "=s"(clock)); #endif return clock; } From f0127c9e211a580f96c41d0e8cf9320ab6b30ea6 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Tue, 27 Aug 2024 06:25:58 -0700 Subject: [PATCH 08/46] remove device wait_until functions adding the device versions of the wait_until* and test functions in the ipc folder leads to linking errors of the functional tests. Remove them and use for now the upper level versions of the functions, similarly to the RO conduit. Might have to revisit this later again. [ROCm/rocshmem commit: e2e30b53395cdc605d4d4d99dc983f15cf8fcebf] --- .../rocshmem/src/ipc/context_ipc_device.hpp | 38 -------------- .../src/ipc/context_ipc_tmpl_device.hpp | 52 ------------------- 2 files changed, 90 deletions(-) diff --git a/projects/rocshmem/src/ipc/context_ipc_device.hpp b/projects/rocshmem/src/ipc/context_ipc_device.hpp index afadb9828c..a8cefb446a 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.hpp @@ -194,44 +194,6 @@ class IPCContext : public Context { template __device__ void get_nbi_wave(T *dest, const T *source, size_t nelems, int pe); - // Wait / Test functions - template - __device__ void wait_until(T* ptr, roc_shmem_cmps cmp, T val); - - template - __device__ void wait_until_all(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T val); - - template - __device__ size_t wait_until_any(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T val); - - template - __device__ size_t wait_until_some(T* ptr, size_t nelems, - size_t* indices, - const int *status, - roc_shmem_cmps cmp, T val); - - template - __device__ void wait_until_all_vector(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T* vals); - - template - __device__ size_t wait_until_any_vector(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T* vals); - template - __device__ size_t wait_until_some_vector(T* ptr, size_t nelems, - size_t* indices, - const int *status, - roc_shmem_cmps cmp, T* vals); - - template - __device__ int test(T* ptr, roc_shmem_cmps cmp, T val); - private: //context class has IpcImpl object (ipcImpl_) diff --git a/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp b/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp index 28cd718f0a..4bf652781e 100644 --- a/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp @@ -223,58 +223,6 @@ __device__ void IPCContext::get_nbi_wave(T *dest, const T *source, getmem_nbi_wave(dest, source, nelems * sizeof(T), pe); } - -//Wait/test functions -template -__device__ void wait_until(T* ptr, roc_shmem_cmps cmp, T val) { -} - -template -__device__ void wait_until_all(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T val) { -} - -template -__device__ size_t wait_until_any(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T val) { - return 0; -} - -template -__device__ size_t wait_until_some(T* ptr, size_t nelems, - size_t* indices, - const int *status, - roc_shmem_cmps cmp, T val){ - return 0; -} - -template -__device__ void wait_until_all_vector(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T* vals) { -} - -template -__device__ size_t wait_until_any_vector(T* ptr, size_t nelems, - const int *status, - roc_shmem_cmps cmp, T* vals){ - return 0; -} - -template -__device__ size_t wait_until_some_vector(T* ptr, size_t nelems, - size_t* indices, - const int *status, - roc_shmem_cmps cmp, T* vals) { -} - -template -__device__ int test(T* ptr, roc_shmem_cmps cmp, T val) { - return 0; -} - } // namespace rocshmem #endif // LIBRARY_SRC_IPC_CONTEXT_TMPL_DEVICE_HPP_ From 9a380a65ff6f2e117c8efabd69d9fe0fea51e2e8 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Tue, 13 Aug 2024 14:09:15 -0700 Subject: [PATCH 09/46] first cut on collectives and sync code is based on the GPUIB implementations of the routines, which seem however generic enough to work also for the IPC conduit. Some code is in for broadcast, fcollect, and alltoall. [ROCm/rocshmem commit: 0de3b5e6fcdde7413053a55d9ac86b2e4782d2c0] --- projects/rocshmem/src/ipc/CMakeLists.txt | 1 + .../rocshmem/src/ipc/context_ipc_device.cpp | 12 -- .../rocshmem/src/ipc/context_ipc_device.hpp | 39 +++++- .../src/ipc/context_ipc_device_coll.cpp | 120 ++++++++++++++++++ .../src/ipc/context_ipc_tmpl_device.hpp | 114 ++++++++++++++++- 5 files changed, 268 insertions(+), 18 deletions(-) create mode 100644 projects/rocshmem/src/ipc/context_ipc_device_coll.cpp diff --git a/projects/rocshmem/src/ipc/CMakeLists.txt b/projects/rocshmem/src/ipc/CMakeLists.txt index e8d64ff523..07a40538a5 100644 --- a/projects/rocshmem/src/ipc/CMakeLists.txt +++ b/projects/rocshmem/src/ipc/CMakeLists.txt @@ -30,4 +30,5 @@ target_sources( context_ipc_host.cpp backend_ipc.cpp ipc_team.cpp + context_ipc_device_coll.cpp ) diff --git a/projects/rocshmem/src/ipc/context_ipc_device.cpp b/projects/rocshmem/src/ipc/context_ipc_device.cpp index 7b9f3dd469..48d7dd9b4f 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.cpp @@ -100,18 +100,6 @@ __device__ void *IPCContext::shmem_ptr(const void *dest, int pe) { return ret; } -__device__ void IPCContext::barrier_all() { - __syncthreads(); -} - -__device__ void IPCContext::sync_all() { - __syncthreads(); -} - -__device__ void IPCContext::sync(roc_shmem_team_t team) { - __syncthreads(); -} - __device__ void IPCContext::putmem_wg(void *dest, const void *source, size_t nelems, int pe) { // TODO (Avinash) check if PE is available for IPC using (isIpcAvailable) diff --git a/projects/rocshmem/src/ipc/context_ipc_device.hpp b/projects/rocshmem/src/ipc/context_ipc_device.hpp index a8cefb446a..01fb1c1fb8 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.hpp @@ -196,13 +196,44 @@ class IPCContext : public Context { private: - //context class has IpcImpl object (ipcImpl_) - IpcImpl *ipcImpl{nullptr}; + //context class has IpcImpl object (ipcImpl_) + IpcImpl *ipcImpl{nullptr}; - uint64_t* atomic_base_ptr{nullptr}; + uint64_t* atomic_base_ptr{nullptr}; - char* g_ret; + char* g_ret; + //internal functions used by collective operations + template + __device__ void internal_put_broadcast(T *dst, const T *src, int nelems, + int pe_root, int PE_start, + int logPE_stride, int PE_size, + long *pSync); // NOLINT(runtime/int) + + template + __device__ void internal_get_broadcast(T *dst, const T *src, int nelems, + int pe_root, + long *pSync); // NOLINT(runtime/int) + + template + __device__ void fcollect_linear(roc_shmem_team_t team, T *dest, + const T *source, int nelems); + + template + __device__ void alltoall_linear(roc_shmem_team_t team, T *dest, + const T *source, int nelems); + + __device__ void internal_sync(int pe, int PE_start, int stride, int PE_size, + int64_t *pSync); + + __device__ void internal_direct_barrier(int pe, int PE_start, int stride, + int n_pes, int64_t *pSync); + + __device__ void internal_atomic_barrier(int pe, int PE_start, int stride, + int n_pes, int64_t *pSync); + + //Temporary scratchpad memory used by internal barrier algorithms. + int64_t *barrier_sync{nullptr}; }; } // namespace rocshmem diff --git a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp new file mode 100644 index 0000000000..804be2efbf --- /dev/null +++ b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp @@ -0,0 +1,120 @@ +/****************************************************************************** + * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + *****************************************************************************/ + +#include "roc_shmem/roc_shmem.hpp" +#include "../context_incl.hpp" +#include "context_ipc_tmpl_device.hpp" +#include "../util.hpp" +#include "ipc_team.hpp" + +namespace rocshmem { + +__device__ void IPCContext::internal_direct_barrier(int pe, int PE_start, + int stride, int n_pes, + int64_t *pSync) { + int64_t flag_val = 1; + if (pe == PE_start) { + // Go through all PE offsets (except current offset = 0) + // and wait until they all reach + for (size_t i = 1; i < n_pes; i++) { + wait_until(&pSync[i], ROC_SHMEM_CMP_EQ, flag_val); + pSync[i] = ROC_SHMEM_SYNC_VALUE; + } + threadfence_system(); + + // Announce to other PEs that all have reached + for (size_t i = 1, j = PE_start + stride; i < n_pes; ++i, j += stride) { + put_nbi(&pSync[0], &flag_val, 1, j); + } + } else { + // Mark current PE offset as reached + size_t pe_offset = (pe - PE_start) / stride; + put_nbi(&pSync[pe_offset], &flag_val, 1, PE_start); + wait_until(&pSync[0], ROC_SHMEM_CMP_EQ, flag_val); + pSync[0] = ROC_SHMEM_SYNC_VALUE; + threadfence_system(); + } +} + +__device__ void IPCContext::internal_atomic_barrier(int pe, int PE_start, + int stride, int n_pes, + int64_t *pSync) { + int64_t flag_val = 1; + if (pe == PE_start) { + wait_until(&pSync[0], ROC_SHMEM_CMP_EQ, (int64_t)(n_pes - 1)); + pSync[0] = ROC_SHMEM_SYNC_VALUE; + threadfence_system(); + + for (size_t i = 1, j = PE_start + stride; i < n_pes; ++i, j += stride) { + put_nbi(&pSync[0], &flag_val, 1, j); + } + } else { + amo_add(&pSync[0], flag_val, PE_start); + wait_until(&pSync[0], ROC_SHMEM_CMP_EQ, flag_val); + pSync[0] = ROC_SHMEM_SYNC_VALUE; + threadfence_system(); + } +} + +// Uses PE values that are relative to world +__device__ void IPCContext::internal_sync(int pe, int PE_start, int stride, + int PE_size, int64_t *pSync) { + __syncthreads(); + if (is_thread_zero_in_block()) { + if (PE_size < 64) { + internal_direct_barrier(pe, PE_start, stride, PE_size, pSync); + } else { + internal_atomic_barrier(pe, PE_start, stride, PE_size, pSync); + } + } + __threadfence(); + __syncthreads(); +} + +__device__ void IPCContext::sync(roc_shmem_team_t team) { + IPCTeam *team_obj = reinterpret_cast(team); + + /** + * Ensure that the stride is a multiple of 2. + */ + int log_pe_stride = static_cast(team_obj->tinfo_wrt_world->log_stride); + int pe = team_obj->my_pe_in_world; + int pe_start = team_obj->tinfo_wrt_world->pe_start; + int pe_stride = (1 << log_pe_stride); + int pe_size = team_obj->num_pes; + + internal_sync(pe, pe_start, pe_stride, pe_size, barrier_sync); +} + +__device__ void IPCContext::sync_all() { + internal_sync(my_pe, 0, 1, num_pes, barrier_sync); +} + +__device__ void IPCContext::barrier_all() { + if (is_thread_zero_in_block()) { + quiet(); + } + sync_all(); + __syncthreads(); +} + +} // namespace rocshmem diff --git a/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp b/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp index 4bf652781e..af79cd20b1 100644 --- a/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp @@ -27,6 +27,7 @@ #include "roc_shmem/roc_shmem.hpp" #include "context_ipc_device.hpp" #include "../util.hpp" +#include "ipc_team.hpp" namespace rocshmem { @@ -150,11 +151,48 @@ __device__ void IPCContext::to_all(T *dest, const T *source, int nreduce, long *pSync) { // NOLINT(runtime/int) } +template +__device__ void IPCContext::internal_put_broadcast( + T *dst, const T *src, int nelems, int pe_root, int pe_start, + int log_pe_stride, int pe_size, + [[maybe_unused]] long *p_sync) { // NOLINT(runtime/int) + if (my_pe == pe_root) { + int stride = 1 << log_pe_stride; + int finish = pe_start + stride * pe_size; + for (int i = pe_start; i < finish; i += stride) { + if (i != my_pe) { + put_nbi_wg(dst, src, nelems, i); + } + } + } +} + +template +__device__ void IPCContext::internal_get_broadcast( + T *dst, const T *src, int nelems, int pe_root, + [[maybe_unused]] long *pSync) { // NOLINT(runtime/int) + if (my_pe != pe_root) { + get_wg(dst, src, nelems, pe_root); + } +} + template __device__ void IPCContext::broadcast(roc_shmem_team_t team, T *dst, const T *src, int nelems, int pe_root) { - //broadcast(dst, src, nelems, pe_root_world, pe_start, log_pe_stride, - // pe_size, p_sync); + IPCTeam *team_obj = reinterpret_cast(team); + + /** + * Ensure that the stride is a multiple of 2 . + */ + int log_pe_stride = static_cast(team_obj->tinfo_wrt_world->log_stride); + int pe_start = team_obj->tinfo_wrt_world->pe_start; + int pe_size = team_obj->tinfo_wrt_world->size; + long *p_sync = team_obj->bcast_pSync; + + // Passed pe_root is relative to team, convert to world root + int pe_root_world = team_obj->get_pe_in_world(pe_root); + broadcast(dst, src, nelems, pe_root_world, pe_start, log_pe_stride, + pe_size, p_sync); } template @@ -162,16 +200,88 @@ __device__ void IPCContext::broadcast(T *dst, const T *src, int nelems, int pe_root, int pe_start, int log_pe_stride, int pe_size, long *p_sync) { // NOLINT(runtime/int) + if (num_pes < 4) { + internal_put_broadcast(dst, src, nelems, pe_root, pe_start, log_pe_stride, + pe_size, p_sync); + } else { + internal_get_broadcast(dst, src, nelems, pe_root, p_sync); + } + + // Synchronize on completion of broadcast + internal_sync(my_pe, pe_start, (1 << log_pe_stride), pe_size, p_sync); } template __device__ void IPCContext::alltoall(roc_shmem_team_t team, T *dst, const T *src, int nelems) { + alltoall_linear(team, dst, src, nelems); +} + +template +__device__ void IPCContext::alltoall_linear(roc_shmem_team_t team, T *dst, + const T *src, int nelems) { + IPCTeam *team_obj = reinterpret_cast(team); + + /** + * Ensure that the stride is a multiple of 2 + */ + int log_pe_stride = static_cast(team_obj->tinfo_wrt_world->log_stride); + int pe_start = team_obj->tinfo_wrt_world->pe_start; + int pe_size = team_obj->num_pes; + int stride = 1 << log_pe_stride; + long *pSync = team_obj->alltoall_pSync; + int my_pe_in_team = team_obj->my_pe; + + /* TODO (egabriel): shouldn't the loop handle pe_start, pe_size, and stride? + * otherwise what is the point of the few lines above? + */ + // Have each PE put their designated data to the other PEs + for (int j = 0; j < pe_size; j++) { + int dest_pe = team_obj->get_pe_in_world(j); + put_nbi_wg(&dst[my_pe_in_team * nelems], &src[j * nelems], nelems, dest_pe); + } + if (is_thread_zero_in_block()) { + quiet(); + } + // wait until everyone has obtained their designated data + internal_sync(my_pe, pe_start, stride, pe_size, pSync); } template __device__ void IPCContext::fcollect(roc_shmem_team_t team, T *dst, const T *src, int nelems) { + fcollect_linear(team, dst, src, nelems); +} + +template +__device__ void IPCContext::fcollect_linear(roc_shmem_team_t team, T *dst, + const T *src, int nelems) { + IPCTeam *team_obj = reinterpret_cast(team); + + /** + * Ensure that the stride is a multiple of 2. + */ + int log_pe_stride = static_cast(team_obj->tinfo_wrt_world->log_stride); + int pe_start = team_obj->tinfo_wrt_world->pe_start; + int pe_size = team_obj->num_pes; + int stride = 1 << log_pe_stride; + long *pSync = team_obj->alltoall_pSync; + int my_pe_in_team = team_obj->my_pe; + + /* TODO (egabriel): shouldn't the loop handle pe_start, pe_size, and stride? + * otherwise what is the point of the few lines above? + */ + // Have each PE put their designated data to the other PEs + for (int j = 0; j < pe_size; j++) { + int dest_pe = team_obj->get_pe_in_world(j); + put_nbi_wg(&dst[my_pe_in_team * nelems], src, nelems, dest_pe); + } + + if (is_thread_zero_in_block()) { + quiet(); + } + // wait until everyone has obtained their designated data + internal_sync(my_pe, pe_start, stride, pe_size, pSync); } // Block/wave functions From 6f93dd9585a8df47fdcf411d300e72d221f44829 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Wed, 28 Aug 2024 09:41:05 -0700 Subject: [PATCH 10/46] ipc/context_ipc_device.cpp: set barrier_sync set the barrier_sync variable on the context during object creation [ROCm/rocshmem commit: aae62954602f134d6ed92dabda96731dc4683374] --- projects/rocshmem/src/ipc/context_ipc_device.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rocshmem/src/ipc/context_ipc_device.cpp b/projects/rocshmem/src/ipc/context_ipc_device.cpp index 48d7dd9b4f..ede7633b06 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.cpp @@ -43,6 +43,7 @@ __host__ IPCContext::IPCContext(Backend *b) auto *bp{backend->ipc_backend_proxy.get()}; + barrier_sync = backend->barrier_sync; g_ret = bp->g_ret; atomic_base_ptr = bp->atomic_ret->atomic_base_ptr; } From caf0a7dfadc9673c1a7e44719944f35d729cddef Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Wed, 28 Aug 2024 09:42:14 -0700 Subject: [PATCH 11/46] utils.hpp: silence warnings silence some warnings when compiling with debug flags. [ROCm/rocshmem commit: 1ffe62bb2e747f7136a0c1c555fca93c77599be8] --- projects/rocshmem/src/util.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rocshmem/src/util.hpp b/projects/rocshmem/src/util.hpp index 67fdac6786..75687a0ac8 100644 --- a/projects/rocshmem/src/util.hpp +++ b/projects/rocshmem/src/util.hpp @@ -212,7 +212,7 @@ __device__ void gpu_dprintf(const char* fmt, const Args&... args) { while (atomicCAS(print_lock, 0, 1) == 1) { } - printf("WG (%lu, %lu, %lu) TH (%lu, %lu, %lu) ", hipBlockIdx_x, + printf("WG (%u, %u, %u) TH (%u, %u, %u) ", hipBlockIdx_x, hipBlockIdx_y, hipBlockIdx_z, hipThreadIdx_x, hipThreadIdx_y, hipThreadIdx_z); printf(fmt, args...); From d71286ace412d54958148b87b5e4cf5f0b1d36c9 Mon Sep 17 00:00:00 2001 From: avinashkethineedi Date: Wed, 28 Aug 2024 12:31:01 -0700 Subject: [PATCH 12/46] Update IPC object * Update the state of the IPC object in the RO context class with the instance created in the RO backend [ROCm/rocshmem commit: 862421e0ca5b00e52ba11c86637b099a9cc7447c] --- projects/rocshmem/src/reverse_offload/context_ro_device.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/rocshmem/src/reverse_offload/context_ro_device.cpp b/projects/rocshmem/src/reverse_offload/context_ro_device.cpp index 8e933e9777..86e12c7c38 100644 --- a/projects/rocshmem/src/reverse_offload/context_ro_device.cpp +++ b/projects/rocshmem/src/reverse_offload/context_ro_device.cpp @@ -50,6 +50,9 @@ __host__ ROContext::ROContext(Backend *b, size_t block_id) block_handle = &block_base[block_id]; } ro_net_win_id = block_id % backend->ro_window_proxy_->MAX_NUM_WINDOWS; + + ipcImpl_.ipc_bases = b->ipcImpl.ipc_bases; + ipcImpl_.shm_size = b->ipcImpl.shm_size; } __device__ void ROContext::putmem(void *dest, const void *source, size_t nelems, From 46176eeeffdd07d5e994ffd7b1e3af5d53ee9889 Mon Sep 17 00:00:00 2001 From: avinashkethineedi Date: Wed, 28 Aug 2024 12:40:51 -0700 Subject: [PATCH 13/46] Comment out ping pong test * ping pong test fails sporadically * issues with roc_shmem_wait_until [ROCm/rocshmem commit: 9c9ef4ffd337030e058aa687774533cc2337dc19] --- projects/rocshmem/scripts/functional_tests/driver.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/rocshmem/scripts/functional_tests/driver.sh b/projects/rocshmem/scripts/functional_tests/driver.sh index 5adb033166..f65a1aeadd 100755 --- a/projects/rocshmem/scripts/functional_tests/driver.sh +++ b/projects/rocshmem/scripts/functional_tests/driver.sh @@ -73,9 +73,9 @@ case $2 in echo "amoinc_n2_w1_z1" ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 11 > $3/amoinc_n2_w1_z1.log check amoinc_n2_w1_z1 - echo "pingpong_n2_w1" - ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 14 > $3/pingpong_n2_w1.log - check pingpong_n2_w1 + # echo "pingpong_n2_w1" + # ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 14 > $3/pingpong_n2_w1.log + # check pingpong_n2_w1 echo "amoset_n2_w1_z1" ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 44 > $3/amoset_n2_w1_z1.log check amoset_n2_w1_z1 @@ -115,9 +115,9 @@ case $2 in echo "amoinc_n2_w8_z1" ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 11 > $3/amoinc_n2_w8_z1.log check amoinc_n2_w8_z1 - echo "pingpong_n2_w1" - ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 14 > $3/pingpong_n2_w1.log - check pingpong_n2_w1 + # echo "pingpong_n2_w1" + # ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -a 14 > $3/pingpong_n2_w1.log + # check pingpong_n2_w1 echo "amoset_n2_w8_z1" ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 44 > $3/amoset_n2_w8_z1.log check amoset_n2_w8_z1 From 2a87c646f50a60d09cc67d65754e50e62710e4ad Mon Sep 17 00:00:00 2001 From: Muhammad Awad Date: Thu, 5 Sep 2024 18:39:02 -0500 Subject: [PATCH 14/46] Fix bad auto Signed-off-by: Muhammad Awad [ROCm/rocshmem commit: b811b2b683194953aeea29e72964542c2884a891] --- projects/rocshmem/src/backend_type.hpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/projects/rocshmem/src/backend_type.hpp b/projects/rocshmem/src/backend_type.hpp index ad9e992636..ce6ccb64cc 100644 --- a/projects/rocshmem/src/backend_type.hpp +++ b/projects/rocshmem/src/backend_type.hpp @@ -67,13 +67,11 @@ enum class BackendType { RO_BACKEND, GPU_IB_BACKEND }; */ #ifdef USE_GPU_IB #define DISPATCH_RET(Func) \ - auto ret_val{0}; \ - ret_val = static_cast(this)->Func; \ + auto ret_val = static_cast(this)->Func; \ return ret_val; #else #define DISPATCH_RET(Func) \ - auto ret_val{0}; \ - ret_val = static_cast(this)->Func; \ + auto ret_val = static_cast(this)->Func; \ return ret_val; #endif /** @@ -113,13 +111,11 @@ enum class BackendType { RO_BACKEND, GPU_IB_BACKEND }; #ifdef USE_GPU_IB #define HOST_DISPATCH_RET(Func) \ - auto ret_val{0}; \ - ret_val = static_cast(this)->Func; \ + auto ret_val = static_cast(this)->Func; \ return ret_val; #else #define HOST_DISPATCH_RET(Func) \ - auto ret_val{0}; \ - ret_val = static_cast(this)->Func; \ + auto ret_val = static_cast(this)->Func; \ return ret_val; #endif From 1ddc82fc28599999e9cf41c49d3f3081729a862a Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Mon, 9 Sep 2024 07:01:20 -0700 Subject: [PATCH 15/46] remove pSync from internal_bcast functions remove the pSync arguments from the internal_broadcast functions, they are not used anyway. [ROCm/rocshmem commit: dfcacdc4a389a0bc32a28438a293981736127f7c] --- projects/rocshmem/src/ipc/context_ipc_device.hpp | 6 ++---- .../rocshmem/src/ipc/context_ipc_tmpl_device.hpp | 16 ++++------------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/projects/rocshmem/src/ipc/context_ipc_device.hpp b/projects/rocshmem/src/ipc/context_ipc_device.hpp index 01fb1c1fb8..4cc6d08d56 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.hpp @@ -207,13 +207,11 @@ class IPCContext : public Context { template __device__ void internal_put_broadcast(T *dst, const T *src, int nelems, int pe_root, int PE_start, - int logPE_stride, int PE_size, - long *pSync); // NOLINT(runtime/int) + int logPE_stride, int PE_size); // NOLINT(runtime/int) template __device__ void internal_get_broadcast(T *dst, const T *src, int nelems, - int pe_root, - long *pSync); // NOLINT(runtime/int) + int pe_root); // NOLINT(runtime/int) template __device__ void fcollect_linear(roc_shmem_team_t team, T *dest, diff --git a/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp b/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp index af79cd20b1..81ec0b2b97 100644 --- a/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_tmpl_device.hpp @@ -154,8 +154,7 @@ __device__ void IPCContext::to_all(T *dest, const T *source, int nreduce, template __device__ void IPCContext::internal_put_broadcast( T *dst, const T *src, int nelems, int pe_root, int pe_start, - int log_pe_stride, int pe_size, - [[maybe_unused]] long *p_sync) { // NOLINT(runtime/int) + int log_pe_stride, int pe_size) { // NOLINT(runtime/int) if (my_pe == pe_root) { int stride = 1 << log_pe_stride; int finish = pe_start + stride * pe_size; @@ -169,8 +168,7 @@ __device__ void IPCContext::internal_put_broadcast( template __device__ void IPCContext::internal_get_broadcast( - T *dst, const T *src, int nelems, int pe_root, - [[maybe_unused]] long *pSync) { // NOLINT(runtime/int) + T *dst, const T *src, int nelems, int pe_root) { // NOLINT(runtime/int) if (my_pe != pe_root) { get_wg(dst, src, nelems, pe_root); } @@ -202,9 +200,9 @@ __device__ void IPCContext::broadcast(T *dst, const T *src, int nelems, long *p_sync) { // NOLINT(runtime/int) if (num_pes < 4) { internal_put_broadcast(dst, src, nelems, pe_root, pe_start, log_pe_stride, - pe_size, p_sync); + pe_size); } else { - internal_get_broadcast(dst, src, nelems, pe_root, p_sync); + internal_get_broadcast(dst, src, nelems, pe_root); } // Synchronize on completion of broadcast @@ -232,9 +230,6 @@ __device__ void IPCContext::alltoall_linear(roc_shmem_team_t team, T *dst, long *pSync = team_obj->alltoall_pSync; int my_pe_in_team = team_obj->my_pe; - /* TODO (egabriel): shouldn't the loop handle pe_start, pe_size, and stride? - * otherwise what is the point of the few lines above? - */ // Have each PE put their designated data to the other PEs for (int j = 0; j < pe_size; j++) { int dest_pe = team_obj->get_pe_in_world(j); @@ -268,9 +263,6 @@ __device__ void IPCContext::fcollect_linear(roc_shmem_team_t team, T *dst, long *pSync = team_obj->alltoall_pSync; int my_pe_in_team = team_obj->my_pe; - /* TODO (egabriel): shouldn't the loop handle pe_start, pe_size, and stride? - * otherwise what is the point of the few lines above? - */ // Have each PE put their designated data to the other PEs for (int j = 0; j < pe_size; j++) { int dest_pe = team_obj->get_pe_in_world(j); From 4610659bf1b07e1a22e9c6e8a2e5f6cb12f61778 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 6 Aug 2024 14:32:14 -0700 Subject: [PATCH 16/46] Add simple fine test [ROCm/rocshmem commit: da93542c40b9a7eb60ea69ca577173cbd3abc84b] --- .../rocshmem/tests/unit_tests/CMakeLists.txt | 1 + .../unit_tests/ipc_impl_simple_fine_gtest.cpp | 1030 +++++++++++++++++ .../unit_tests/ipc_impl_simple_fine_gtest.hpp | 236 ++++ 3 files changed, 1267 insertions(+) create mode 100644 projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp create mode 100644 projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp diff --git a/projects/rocshmem/tests/unit_tests/CMakeLists.txt b/projects/rocshmem/tests/unit_tests/CMakeLists.txt index 349bb2c2bf..b47da1c154 100644 --- a/projects/rocshmem/tests/unit_tests/CMakeLists.txt +++ b/projects/rocshmem/tests/unit_tests/CMakeLists.txt @@ -90,6 +90,7 @@ target_sources( free_list_gtest.cpp #context_ipc_gtest.cpp ipc_impl_simple_coarse_gtest.cpp + ipc_impl_simple_fine_gtest.cpp ) ############################################################################### diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp new file mode 100644 index 0000000000..1c4c409f59 --- /dev/null +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp @@ -0,0 +1,1030 @@ +/****************************************************************************** + * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + *****************************************************************************/ + +#include "ipc_impl_simple_fine_gtest.hpp" + +using namespace rocshmem; + +TEST_F(IPCImplSimpleFineTestFixture, ptr_check) { + ASSERT_NE(heap_mem_.get_ptr(), nullptr); +} + +TEST_F(IPCImplSimpleFineTestFixture, MPI_num_pes) { + ASSERT_EQ(mpi_.num_pes(), 2); +} + +TEST_F(IPCImplSimpleFineTestFixture, IPC_bases) { + for(int i{0}; i < mpi_.num_pes(); i++) { + ASSERT_NE(ipc_impl_.ipc_bases[i], nullptr); + } +} + +TEST_F(IPCImplSimpleFineTestFixture, golden_1048576_int) { + iota_golden(1048576); + validate_golden(1048576); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_1024x1x1_32_int) { + dim3 grid {1,1,1}; + dim3 block {1024,1,1}; + write_wg(grid, block, 32); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_1024x1x1_32_int) { + dim3 grid {1,1,1}; + dim3 block {1024,1,1}; + read_wg(grid, block, 32); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_1x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_2x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {2,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_4x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {4,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_8x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {8,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_16x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {16,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_32x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {32,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_64x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_128x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {128,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_256x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {256,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_512x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {512,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_768x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {768,1,1}; + write_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_1024x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1024,1,1}; + write_wg(grid, block, 1048576); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_1x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_2x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {2,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_4x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {4,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_8x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {8,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_16x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {16,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_32x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {32,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_64x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_128x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {128,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_256x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {256,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_512x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {512,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_768x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {768,1,1}; + read_wg(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wg_1x1x1_1024x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1024,1,1}; + read_wg(grid, block, 1048576); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_64x1x1_1_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + write_wave(grid, block, 1); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_64x1x1_1_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + read_wave(grid, block, 1); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_64x1x1_32_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + write_wave(grid, block, 32); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_64x1x1_32_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + read_wave(grid, block, 32); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_1x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_2x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {2,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_3x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {3,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_4x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {4,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_5x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {5,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_6x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {6,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_7x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {7,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_8x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {8,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_9x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {9,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_10x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {10,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_11x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {11,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_12x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {12,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_13x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {13,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_14x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {14,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_15x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {15,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_16x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {16,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_17x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {17,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_18x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {18,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_19x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {19,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_20x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {20,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_21x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {21,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_22x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {22,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_23x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {23,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_24x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {24,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_25x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {25,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_26x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {26,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_27x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {27,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_28x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {28,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_29x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {29,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_30x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {30,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_31x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {31,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_32x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {32,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_33x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {33,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_34x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {34,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_35x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {35,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_36x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {36,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_37x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {37,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_38x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {38,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_39x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {39,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_40x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {40,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_41x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {41,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_42x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {42,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_43x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {43,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_44x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {44,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_45x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {45,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_46x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {46,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_47x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {47,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_48x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {48,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_49x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {49,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_50x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {50,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_51x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {51,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_52x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {52,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_53x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {53,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_54x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {54,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_55x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {55,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_56x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {56,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_57x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {57,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_58x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {58,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_59x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {59,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_60x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {60,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_61x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {61,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_62x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {62,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_63x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {63,1,1}; + write_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, write_wave_1x1x1_64x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + write_wave(grid, block, 1048576); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_1x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_2x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {2,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_3x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {3,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_4x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {4,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_5x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {5,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_6x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {6,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_7x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {7,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_8x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {8,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_9x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {9,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_10x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {10,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_11x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {11,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_12x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {12,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_13x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {13,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_14x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {14,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_15x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {15,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_16x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {16,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_17x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {17,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_18x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {18,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_19x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {19,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_20x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {20,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_21x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {21,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_22x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {22,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_23x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {23,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_24x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {24,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_25x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {25,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_26x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {26,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_27x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {27,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_28x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {28,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_29x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {29,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_30x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {30,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_31x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {31,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_32x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {32,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_33x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {33,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_34x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {34,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_35x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {35,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_36x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {36,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_37x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {37,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_38x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {38,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_39x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {39,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_40x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {40,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_41x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {41,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_42x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {42,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_43x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {43,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_44x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {44,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_45x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {45,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_46x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {46,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_47x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {47,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_48x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {48,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_49x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {49,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_50x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {50,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_51x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {51,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_52x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {52,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_53x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {53,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_54x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {54,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_55x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {55,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_56x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {56,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_57x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {57,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_58x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {58,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_59x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {59,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_60x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {60,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_61x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {61,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_62x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {62,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_63x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {63,1,1}; + read_wave(grid, block, 1048576); +} + +TEST_F(IPCImplSimpleFineTestFixture, read_wave_1x1x1_64x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {64,1,1}; + read_wave(grid, block, 1048576); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, write_1x1x1_1x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1,1,1}; + write(grid, block, 1048576); +} + +//============================================================================= + +TEST_F(IPCImplSimpleFineTestFixture, read_1x1x1_1x1x1_1048576_int) { + dim3 grid {1,1,1}; + dim3 block {1,1,1}; + read(grid, block, 1048576); +} + + diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp new file mode 100644 index 0000000000..3f85528e7d --- /dev/null +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp @@ -0,0 +1,236 @@ +/****************************************************************************** + * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef ROCSHMEM_IPC_IMPL_SIMPLE_FINE_GTEST_HPP +#define ROCSHMEM_IPC_IMPL_SIMPLE_FINE_GTEST_HPP + +#include "gtest/gtest.h" + +#include + +#include +#include "../src/memory/symmetric_heap.hpp" +#include "../src/ipc/ipc_policy.hpp" + +namespace rocshmem { + +__global__ +void +kernel_simple_fine_copy(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { + if (!threadIdx.x) { + ipc_impl->ipcCopy(dest, src, bytes); + ipc_impl->ipcFence(); + } + __syncthreads(); +} + +__global__ +void +kernel_simple_fine_copy_wg(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { + ipc_impl->ipcCopy_wg(dest, src, bytes); + ipc_impl->ipcFence(); + __syncthreads(); +} + +__global__ +void +kernel_simple_fine_copy_wave(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { + ipc_impl->ipcCopy_wave(dest, src, bytes); + ipc_impl->ipcFence(); + __syncthreads(); +} + +class IPCImplSimpleFineTestFixture : public ::testing::Test { + + using HEAP_T = HeapMemory; + + using MPI_T = RemoteHeapInfo; + + using FN_T = void (*)(IpcImpl*, int*, int*, size_t); + + public: + IPCImplSimpleFineTestFixture() { + ipc_impl_.ipcHostInit(mpi_.my_pe(), mpi_.get_heap_bases() , MPI_COMM_WORLD); + + assert(ipc_impl_dptr_ == nullptr); + hip_allocator_.allocate((void**)&ipc_impl_dptr_, sizeof(IpcImpl)); + + CHECK_HIP(hipMemcpy(ipc_impl_dptr_, &ipc_impl_, + sizeof(IpcImpl), hipMemcpyHostToDevice)); + } + + ~IPCImplSimpleFineTestFixture() { + if (ipc_impl_dptr_) { + hip_allocator_.deallocate(ipc_impl_dptr_); + } + + ipc_impl_.ipcHostStop(); + } + + void launch(FN_T f, const dim3 grid, const dim3 block, int* src, int* dest, size_t bytes) { + f<<>>(ipc_impl_dptr_, src, dest, bytes); + CHECK_HIP(hipStreamSynchronize(nullptr)); + } + + enum TestType { + READ = 0, + WRITE = 1 + }; + + void write(const dim3 grid, const dim3 block, size_t elems) { + iota_golden(elems); + initialize_src_buffer(WRITE); + copy(WRITE, grid, block); + validate_dest_buffer(WRITE); + } + + void write_wg(const dim3 grid, const dim3 block, size_t elems) { + iota_golden(elems); + initialize_src_buffer(WRITE); + copy_wg(WRITE, grid, block); + validate_dest_buffer(WRITE); + } + + void write_wave(const dim3 grid, const dim3 block, size_t elems) { + iota_golden(elems); + initialize_src_buffer(WRITE); + copy_wave(WRITE, grid, block); + validate_dest_buffer(WRITE); + } + + void read(const dim3 grid, const dim3 block, size_t elems) { + iota_golden(elems); + initialize_src_buffer(READ); + copy(READ, grid, block); + validate_dest_buffer(READ); + } + + void read_wg(const dim3 grid, const dim3 block, size_t elems) { + iota_golden(elems); + initialize_src_buffer(READ); + copy_wg(READ, grid, block); + validate_dest_buffer(READ); + } + + void read_wave(const dim3 grid, const dim3 block, size_t elems) { + iota_golden(elems); + initialize_src_buffer(READ); + copy_wave(READ, grid, block); + validate_dest_buffer(READ); + } + + void iota_golden(size_t elems) { + golden_.resize(elems); + std::iota(golden_.begin(), golden_.end(), 0); + } + + void validate_golden(size_t elems) { + ASSERT_EQ(golden_.size(), elems); + for (int i{0}; i < golden_.size(); i++) { + ASSERT_EQ(golden_[i], i); + } + } + + void initialize_src_buffer(TestType test) { + if (!pe_initializes_src_buffer(test)) { + return; + } + size_t bytes = golden_.size() * sizeof(int); + auto dev_src = reinterpret_cast(ipc_impl_.ipc_bases[mpi_.my_pe()]); + CHECK_HIP(hipMemcpy(dev_src, golden_.data(), bytes, hipMemcpyHostToDevice)); + CHECK_HIP(hipStreamSynchronize(nullptr)); + } + + bool pe_initializes_src_buffer(TestType test) { + bool is_write_test = test; + bool is_read_test = !test; + return (is_write_test && mpi_.my_pe() == 0) || + (is_read_test && mpi_.my_pe() == 1); + } + + void execute(TestType test, FN_T fn, const dim3 grid, const dim3 block) { + if (mpi_.my_pe()) { + mpi_.barrier(); + mpi_.barrier(); + return; + } + int *src{nullptr}; + int *dest{nullptr}; + if (test == WRITE) { + src = reinterpret_cast(ipc_impl_.ipc_bases[0]); + dest = reinterpret_cast(ipc_impl_.ipc_bases[1]); + } else { + src = reinterpret_cast(ipc_impl_.ipc_bases[1]); + dest = reinterpret_cast(ipc_impl_.ipc_bases[0]); + } + size_t bytes = golden_.size() * sizeof(int); + mpi_.barrier(); + launch(fn, grid, block, src, dest, bytes); + mpi_.barrier(); + } + + void copy(TestType test, dim3 grid, dim3 block) { + execute(test, kernel_simple_fine_copy, grid, block); + } + + void copy_wg(TestType test, dim3 grid, dim3 block) { + execute(test, kernel_simple_fine_copy_wg, grid, block); + } + + void copy_wave(TestType test, dim3 grid, dim3 block) { + execute(test, kernel_simple_fine_copy_wave, grid, block); + } + + void validate_dest_buffer(TestType test) { + if (!pe_validates_dest_buffer(test)) { + return; + } + + auto dev_dest = reinterpret_cast(ipc_impl_.ipc_bases[mpi_.my_pe()]); + for (int i{0}; i < golden_.size(); i++) { + ASSERT_EQ(golden_[i], dev_dest[i]); + } + } + + bool pe_validates_dest_buffer(TestType test) { + return !pe_initializes_src_buffer(test); + } + + protected: + std::vector golden_; + + std::vector output_; + + HEAP_T heap_mem_ {}; + + MPI_T mpi_ {heap_mem_.get_ptr(), heap_mem_.get_size()}; + + IpcImpl ipc_impl_ {}; + + IpcImpl *ipc_impl_dptr_ {nullptr}; + + HIPAllocator hip_allocator_ {}; +}; + +} // namespace rocshmem + +#endif // ROCSHMEM_IPC_IMPL_SIMPLE_FINE_GTEST_HPP From cec6ba91a0a4c79ffe9db7f3824a7d6c948d8650 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Wed, 7 Aug 2024 11:03:18 -0700 Subject: [PATCH 17/46] Partial notifier [ROCm/rocshmem commit: c4b7e0d91becd8bc20c4b3eefac6146f76d2a82f] --- .../rocshmem/scripts/build_configs/ro_net | 2 +- projects/rocshmem/src/atomic.hpp | 131 ++++++++++++++++++ projects/rocshmem/src/memory/notifier.hpp | 25 ++-- 3 files changed, 140 insertions(+), 18 deletions(-) create mode 100644 projects/rocshmem/src/atomic.hpp diff --git a/projects/rocshmem/scripts/build_configs/ro_net b/projects/rocshmem/scripts/build_configs/ro_net index 17809fa0a9..47690bb2d9 100755 --- a/projects/rocshmem/scripts/build_configs/ro_net +++ b/projects/rocshmem/scripts/build_configs/ro_net @@ -24,4 +24,4 @@ cmake \ -DUSE_COHERENT_HEAP=ON \ $src_path cmake --build . --parallel 8 -cmake --install . +#cmake --install . diff --git a/projects/rocshmem/src/atomic.hpp b/projects/rocshmem/src/atomic.hpp new file mode 100644 index 0000000000..85a140f75a --- /dev/null +++ b/projects/rocshmem/src/atomic.hpp @@ -0,0 +1,131 @@ +/****************************************************************************** + * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef LIBRARY_SRC_ATOMIC_HPP +#define LIBRARY_SRC_ATOMIC_HPP + +namespace rocshmem { +namespace detail { +namespace atomic { + +typedef enum rocshmem_memory_scope { + memory_scope_thread = __HIP_MEMORY_SCOPE_SINGLETHREAD, + memory_scope_wavefront = __HIP_MEMORY_SCOPE_WAVEFRONT, + memory_scope_workgroup = __HIP_MEMORY_SCOPE_WORKGROUP, + memory_scope_agent = __HIP_MEMORY_SCOPE_AGENT, + memory_scope_system = __HIP_MEMORY_SCOPE_SYSTEM, +} rocshmem_memory_scope; + +typedef enum rocshmem_memory_order { + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_consume = __ATOMIC_CONSUME, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, + memory_order_seq_cst = __ATOMIC_SEQ_CST +} rocshmem_memory_order; + +template +__host__ __device__ +T load(const T* address, rocshmem_memory_order order) { + return __hip_atomic_load(address, order, Scope); +} + +template +__host__ __device__ +void store(const T value, const T* address, rocshmem_memory_order order) { + return __hip_atomic_store(value, address, order, Scope); +} + +template +__host__ __device__ +bool compare_exchange_weak(T& expected, T desired, rocshmem_memory_order success, + rocshmem_memory_order failure) { + return __hip_atomic_compare_exchange_weak(expected, desired, success, failure, Scope); +} + +template +__host__ __device__ +bool compare_exchange_strong(T& expected, T desired, rocshmem_memory_order success, + rocshmem_memory_order failure) { + return __hip_atomic_compare_exchange_strong(expected, desired, success, failure, Scope); +} + +template +__host__ __device__ +T fetch_add(T* obj, U arg, rocshmem_memory_order order) { + return __hip_atomic_fetch_add(obj, arg, order, Scope); +} + +template +__host__ __device__ +T fetch_sub(T* obj, U arg, rocshmem_memory_order order) { + return __hip_atomic_fetch_sub(obj, arg, order, Scope); +} + +template +__host__ __device__ +T fetch_and(T* obj, U arg, rocshmem_memory_order order) { + return __hip_atomic_fetch_and(obj, arg, order, Scope); +} + +template +__host__ __device__ +T fetch_or(T* obj, U arg, rocshmem_memory_order order) { + return __hip_atomic_fetch_or(obj, arg, order, Scope); +} + +template +__host__ __device__ +T fetch_xor(T* obj, U arg, rocshmem_memory_order order) { + return __hip_atomic_fetch_xor(obj, arg, order, Scope); +} + +template +__host__ __device__ +T fetch_max(T* obj, U arg, rocshmem_memory_order order) { + return __hip_atomic_fetch_max(obj, arg, order, Scope); +} + +template +__host__ __device__ +T fetch_min(T* obj, U arg, rocshmem_memory_order order) { + return __hip_atomic_fetch_min(obj, arg, order, Scope); +} + +template +__device__ +void thread_fence([[maybe_unused]] rocshmem_memory_order order) { + if constexpr (Scope == memory_scope_system) { + __threadfence_system(); + } else if constexpr (Scope == memory_scope_agent) { + __threadfence(); + } else if constexpr (Scope == memory_scope_workgroup) { + __threadfence_block(); + } +} + +} // namespace atomic +} // namespace detail +} // namespace rocshmem + +#endif // LIBRARY_SRC_ATOMIC_HPP_ diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index 0d8cd92f2f..d398110e9a 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -20,30 +20,21 @@ * IN THE SOFTWARE. *****************************************************************************/ -/** - * @file notifier.hpp - * - * @brief Contains the notification memory space for threads to communicate - * results with one another. - * - * Assume one thread does work on behalf of other threads (as a leader) and - * that work needs to be communicated to the other threads as a result. - * To expose the result, the threads need to share a memory space where the - * result can be written. The leader thread writes the result out to this - * memory space and all threads synchronize on it. - * - * This class allows the leader thread to notify other threads of the update. - */ - #ifndef LIBRARY_SRC_MEMORY_NOTIFIER_HPP_ #define LIBRARY_SRC_MEMORY_NOTIFIER_HPP_ #include "../device_proxy.hpp" #include "../util.hpp" +#include "../atomic.hpp" namespace rocshmem { +template class Notifier { +}; + +template +class Notifier { public: __device__ uint64_t read() { return value_; } @@ -67,9 +58,9 @@ class Notifier { uint64_t value_{}; }; -template +template class NotifierProxy { - using ProxyT = DeviceProxy; + using ProxyT = DeviceProxy, 1>; public: __host__ __device__ Notifier* get() { return proxy_.get(); } From ef4a14e9478f15ff89c8f75546fc76763822fc06 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Mon, 12 Aug 2024 11:29:31 -0700 Subject: [PATCH 18/46] Fix problems with Notifier [ROCm/rocshmem commit: 0c53a075f2faaf5717ac12d32473b59b68160c81] --- projects/rocshmem/src/atomic.hpp | 87 ++++++++++--------- projects/rocshmem/src/memory/notifier.hpp | 23 +++-- projects/rocshmem/src/memory/slab_heap.hpp | 2 +- .../tests/unit_tests/notifier_gtest.hpp | 4 +- 4 files changed, 61 insertions(+), 55 deletions(-) diff --git a/projects/rocshmem/src/atomic.hpp b/projects/rocshmem/src/atomic.hpp index 85a140f75a..7b1ce9300a 100644 --- a/projects/rocshmem/src/atomic.hpp +++ b/projects/rocshmem/src/atomic.hpp @@ -44,82 +44,91 @@ typedef enum rocshmem_memory_order { memory_order_seq_cst = __ATOMIC_SEQ_CST } rocshmem_memory_order; -template +struct rocshmem_memory_orders { + rocshmem_memory_order load {memory_order_acquire}; + rocshmem_memory_order store {memory_order_release}; + rocshmem_memory_order fence {memory_order_acq_rel}; + rocshmem_memory_order atomic {memory_order_acq_rel}; + rocshmem_memory_order weak_cas_success {memory_order_acq_rel}; + rocshmem_memory_order weak_cas_failure {memory_order_acq_rel}; + rocshmem_memory_order strong_cas_success {memory_order_acq_rel}; + rocshmem_memory_order strong_cas_failure {memory_order_acq_rel}; +}; + +template __host__ __device__ -T load(const T* address, rocshmem_memory_order order) { - return __hip_atomic_load(address, order, Scope); +T load(const T* address, rocshmem_memory_orders o) { + return __hip_atomic_load(address, o.load, s); } -template +template __host__ __device__ -void store(const T value, const T* address, rocshmem_memory_order order) { - return __hip_atomic_store(value, address, order, Scope); +void store(T* address, const T value, rocshmem_memory_orders o) { + return __hip_atomic_store(address, value, o.store, s); } -template +template __host__ __device__ -bool compare_exchange_weak(T& expected, T desired, rocshmem_memory_order success, - rocshmem_memory_order failure) { - return __hip_atomic_compare_exchange_weak(expected, desired, success, failure, Scope); +bool compare_exchange_weak(T& expected, T desired, rocshmem_memory_orders o) { + return __hip_atomic_compare_exchange_weak(expected, desired, o.weak_cas_success, o.weak_cas_failure, s); } -template +template __host__ __device__ -bool compare_exchange_strong(T& expected, T desired, rocshmem_memory_order success, - rocshmem_memory_order failure) { - return __hip_atomic_compare_exchange_strong(expected, desired, success, failure, Scope); +bool compare_exchange_strong(T& expected, T desired, rocshmem_memory_orders o) { + return __hip_atomic_compare_exchange_strong(expected, desired, o.strong_cas_success, o.strong_cas_failure, s); } -template +template __host__ __device__ -T fetch_add(T* obj, U arg, rocshmem_memory_order order) { - return __hip_atomic_fetch_add(obj, arg, order, Scope); +T fetch_add(T* obj, U arg, rocshmem_memory_orders o) { + return __hip_atomic_fetch_add(obj, arg, o.atomic, s); } -template +template __host__ __device__ -T fetch_sub(T* obj, U arg, rocshmem_memory_order order) { - return __hip_atomic_fetch_sub(obj, arg, order, Scope); +T fetch_sub(T* obj, U arg, rocshmem_memory_orders o) { + return __hip_atomic_fetch_sub(obj, arg, o.atomic, s); } -template +template __host__ __device__ -T fetch_and(T* obj, U arg, rocshmem_memory_order order) { - return __hip_atomic_fetch_and(obj, arg, order, Scope); +T fetch_and(T* obj, U arg, rocshmem_memory_orders o) { + return __hip_atomic_fetch_and(obj, arg, o.atomic, s); } -template +template __host__ __device__ -T fetch_or(T* obj, U arg, rocshmem_memory_order order) { - return __hip_atomic_fetch_or(obj, arg, order, Scope); +T fetch_or(T* obj, U arg, rocshmem_memory_orders o) { + return __hip_atomic_fetch_or(obj, arg, o, s); } -template +template __host__ __device__ -T fetch_xor(T* obj, U arg, rocshmem_memory_order order) { - return __hip_atomic_fetch_xor(obj, arg, order, Scope); +T fetch_xor(T* obj, U arg, rocshmem_memory_orders o) { + return __hip_atomic_fetch_xor(obj, arg, o.atomic, s); } -template +template __host__ __device__ -T fetch_max(T* obj, U arg, rocshmem_memory_order order) { - return __hip_atomic_fetch_max(obj, arg, order, Scope); +T fetch_max(T* obj, U arg, rocshmem_memory_orders o) { + return __hip_atomic_fetch_max(obj, arg, o.atomic, s); } -template +template __host__ __device__ -T fetch_min(T* obj, U arg, rocshmem_memory_order order) { - return __hip_atomic_fetch_min(obj, arg, order, Scope); +T fetch_min(T* obj, U arg, rocshmem_memory_orders o) { + return __hip_atomic_fetch_min(obj, arg, o.atomic, s); } -template +template __device__ void thread_fence([[maybe_unused]] rocshmem_memory_order order) { - if constexpr (Scope == memory_scope_system) { + if constexpr (s == memory_scope_system) { __threadfence_system(); - } else if constexpr (Scope == memory_scope_agent) { + } else if constexpr (s == memory_scope_agent) { __threadfence(); - } else if constexpr (Scope == memory_scope_workgroup) { + } else if constexpr (s == memory_scope_workgroup) { __threadfence_block(); } } diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index d398110e9a..e6364a26cf 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -29,20 +29,15 @@ namespace rocshmem { -template +template class Notifier { -}; - -template -class Notifier { public: - __device__ uint64_t read() { return value_; } + __device__ uint64_t read() { + return detail::atomic::load(&value_, orders); + } __device__ void write(uint64_t val) { - if (is_thread_zero_in_block()) { - value_ = val; - } - publish(); + detail::atomic::store(&value_, val, orders); } __device__ void done() { __syncthreads(); } @@ -55,15 +50,17 @@ class Notifier { __syncthreads(); } + detail::atomic::rocshmem_memory_orders orders; + uint64_t value_{}; }; -template +template class NotifierProxy { - using ProxyT = DeviceProxy, 1>; + using ProxyT = DeviceProxy, 1>; public: - __host__ __device__ Notifier* get() { return proxy_.get(); } + __host__ __device__ Notifier* get() { return proxy_.get(); } private: ProxyT proxy_{}; diff --git a/projects/rocshmem/src/memory/slab_heap.hpp b/projects/rocshmem/src/memory/slab_heap.hpp index a3655b27c1..171332bee3 100644 --- a/projects/rocshmem/src/memory/slab_heap.hpp +++ b/projects/rocshmem/src/memory/slab_heap.hpp @@ -48,7 +48,7 @@ class SlabHeap { /** * @brief Helper type for notifier */ - using NOTIFIER_PROXY_T = NotifierProxy; + using NOTIFIER_PROXY_T = NotifierProxy; /** * @brief Helper type for notifier diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp index ecb2e7a619..4a72212f5d 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp @@ -52,7 +52,7 @@ write_to_memory(uint8_t* raw_memory) { __global__ void all_threads_once(uint8_t* raw_memory, - Notifier* notifier) { + Notifier * notifier) { notifier->write(NOTIFIER_OFFSET); uint64_t offset_u64 {notifier->read()}; notifier->done(); @@ -65,7 +65,7 @@ all_threads_once(uint8_t* raw_memory, } class NotifierTestFixture : public ::testing::Test { - using NotifierProxyT = NotifierProxy; + using NotifierProxyT = NotifierProxy; public: NotifierTestFixture() { From 4896235adad2db9ea3bbe4051bffa396072eb1c4 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Mon, 12 Aug 2024 14:37:47 -0700 Subject: [PATCH 19/46] Change read/write to load/store in Nofitier API [ROCm/rocshmem commit: 039ea82777df665645ae630c51572958bd963903] --- projects/rocshmem/src/memory/notifier.hpp | 4 ++-- projects/rocshmem/src/memory/slab_heap.cpp | 4 ++-- projects/rocshmem/tests/unit_tests/notifier_gtest.hpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index e6364a26cf..b2211f0cc5 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -32,11 +32,11 @@ namespace rocshmem { template class Notifier { public: - __device__ uint64_t read() { + __device__ uint64_t load() { return detail::atomic::load(&value_, orders); } - __device__ void write(uint64_t val) { + __device__ void store(uint64_t val) { detail::atomic::store(&value_, val, orders); } diff --git a/projects/rocshmem/src/memory/slab_heap.cpp b/projects/rocshmem/src/memory/slab_heap.cpp index 89067b78da..80b8a8cef9 100644 --- a/projects/rocshmem/src/memory/slab_heap.cpp +++ b/projects/rocshmem/src/memory/slab_heap.cpp @@ -75,8 +75,8 @@ __device__ void SlabHeap::malloc(void** ptr, size_t size) { * Notify other threads in block about the allocation result. */ auto notifier{notifier_.get()}; - notifier->write(ptr_deref_u64); - uint64_t notification_u64{notifier->read()}; + notifier->store(ptr_deref_u64); + uint64_t notification_u64{notifier->load()}; notifier->done(); /* diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp index 4a72212f5d..b914ca4f22 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp @@ -53,8 +53,8 @@ __global__ void all_threads_once(uint8_t* raw_memory, Notifier * notifier) { - notifier->write(NOTIFIER_OFFSET); - uint64_t offset_u64 {notifier->read()}; + notifier->store(NOTIFIER_OFFSET); + uint64_t offset_u64 {notifier->load()}; notifier->done(); uint64_t raw_memory_u64 {reinterpret_cast(raw_memory)}; From 13ec689cdf6cd838b673f57b968c12b1e3d84805 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Mon, 12 Aug 2024 15:11:48 -0700 Subject: [PATCH 20/46] Updates to Notifier [ROCm/rocshmem commit: 51c33b2a6670b7e287169d30d73e739f65902ff3] --- projects/rocshmem/src/atomic.hpp | 3 +-- projects/rocshmem/src/memory/notifier.hpp | 10 +++------- projects/rocshmem/src/memory/slab_heap.cpp | 7 +++++-- projects/rocshmem/tests/unit_tests/notifier_gtest.hpp | 8 +++++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/projects/rocshmem/src/atomic.hpp b/projects/rocshmem/src/atomic.hpp index 7b1ce9300a..5d9dc30886 100644 --- a/projects/rocshmem/src/atomic.hpp +++ b/projects/rocshmem/src/atomic.hpp @@ -47,7 +47,6 @@ typedef enum rocshmem_memory_order { struct rocshmem_memory_orders { rocshmem_memory_order load {memory_order_acquire}; rocshmem_memory_order store {memory_order_release}; - rocshmem_memory_order fence {memory_order_acq_rel}; rocshmem_memory_order atomic {memory_order_acq_rel}; rocshmem_memory_order weak_cas_success {memory_order_acq_rel}; rocshmem_memory_order weak_cas_failure {memory_order_acq_rel}; @@ -123,7 +122,7 @@ T fetch_min(T* obj, U arg, rocshmem_memory_orders o) { template __device__ -void thread_fence([[maybe_unused]] rocshmem_memory_order order) { +void thread_fence() { if constexpr (s == memory_scope_system) { __threadfence_system(); } else if constexpr (s == memory_scope_agent) { diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index b2211f0cc5..1f9f6efe76 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -40,15 +40,11 @@ class Notifier { detail::atomic::store(&value_, val, orders); } - __device__ void done() { __syncthreads(); } + __device__ void fence() { + detail::atomic::thread_fence(); + } private: - __device__ void publish() { - if (is_thread_zero_in_block()) { - __threadfence(); - } - __syncthreads(); - } detail::atomic::rocshmem_memory_orders orders; diff --git a/projects/rocshmem/src/memory/slab_heap.cpp b/projects/rocshmem/src/memory/slab_heap.cpp index 80b8a8cef9..faf3a84b1b 100644 --- a/projects/rocshmem/src/memory/slab_heap.cpp +++ b/projects/rocshmem/src/memory/slab_heap.cpp @@ -75,9 +75,12 @@ __device__ void SlabHeap::malloc(void** ptr, size_t size) { * Notify other threads in block about the allocation result. */ auto notifier{notifier_.get()}; - notifier->store(ptr_deref_u64); + if (!threadIdx.x) { + notifier->store(ptr_deref_u64); + notifier->fence(); + } + __syncthreads(); uint64_t notification_u64{notifier->load()}; - notifier->done(); /* * Write to the ptr parameter (to return it back up the call stack). diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp index b914ca4f22..303f1932c9 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp @@ -53,10 +53,12 @@ __global__ void all_threads_once(uint8_t* raw_memory, Notifier * notifier) { - notifier->store(NOTIFIER_OFFSET); + if (!threadIdx.x) { + notifier->store(NOTIFIER_OFFSET); + notifier->fence(); + } + __syncthreads(); uint64_t offset_u64 {notifier->load()}; - notifier->done(); - uint64_t raw_memory_u64 {reinterpret_cast(raw_memory)}; uint64_t address_u64 {raw_memory_u64 + offset_u64}; uint8_t* address {reinterpret_cast(address_u64)}; From 68384da0199c3ff5bbe87fdb9210a5b8b67df3c5 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 13 Aug 2024 12:01:24 -0700 Subject: [PATCH 21/46] Update Notifier fixture to Block [ROCm/rocshmem commit: 5b42cff96cea239991c144cfbb84f1abd4d5c5bb] --- projects/rocshmem/src/atomic.hpp | 12 +++++++- projects/rocshmem/src/memory/notifier.hpp | 10 +++++-- .../tests/unit_tests/notifier_gtest.cpp | 28 +++++++++---------- .../tests/unit_tests/notifier_gtest.hpp | 23 ++++++++------- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/projects/rocshmem/src/atomic.hpp b/projects/rocshmem/src/atomic.hpp index 5d9dc30886..330bd82f86 100644 --- a/projects/rocshmem/src/atomic.hpp +++ b/projects/rocshmem/src/atomic.hpp @@ -122,7 +122,7 @@ T fetch_min(T* obj, U arg, rocshmem_memory_orders o) { template __device__ -void thread_fence() { +void threadfence() { if constexpr (s == memory_scope_system) { __threadfence_system(); } else if constexpr (s == memory_scope_agent) { @@ -132,6 +132,16 @@ void thread_fence() { } } +template +__device__ +void sync() { + if constexpr (s == memory_scope_workgroup) { + __syncthreads(); + } else { + assert(false); + } +} + } // namespace atomic } // namespace detail } // namespace rocshmem diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index 1f9f6efe76..12f53a4c8b 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -33,15 +33,19 @@ template class Notifier { public: __device__ uint64_t load() { - return detail::atomic::load(&value_, orders); + return detail::atomic::load(&value_, orders); } __device__ void store(uint64_t val) { - detail::atomic::store(&value_, val, orders); + detail::atomic::store(&value_, val, orders); } __device__ void fence() { - detail::atomic::thread_fence(); + detail::atomic::threadfence(); + } + + __device__ void sync() { + detail::atomic::sync(); } private: diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp index 9f79c62397..e1ebea0210 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp @@ -28,30 +28,30 @@ using namespace rocshmem; ******************************* Fixture Tests ******************************* *****************************************************************************/ -TEST_F(NotifierTestFixture, run_all_threads_once_1_1) { - run_all_threads_once(1, 1); +TEST_F(NotifierBlockTestFixture, run_all_threads_once_1_1) { + run_all_threads_once_block(1, 1); } -TEST_F(NotifierTestFixture, run_all_threads_once_2_1) { - run_all_threads_once(2, 1); +TEST_F(NotifierBlockTestFixture, run_all_threads_once_2_1) { + run_all_threads_once_block(2, 1); } -TEST_F(NotifierTestFixture, run_all_threads_once_64_1) { - run_all_threads_once(64, 1); +TEST_F(NotifierBlockTestFixture, run_all_threads_once_64_1) { + run_all_threads_once_block(64, 1); } -TEST_F(NotifierTestFixture, run_all_threads_once_128_1) { - run_all_threads_once(128, 1); +TEST_F(NotifierBlockTestFixture, run_all_threads_once_128_1) { + run_all_threads_once_block(128, 1); } -TEST_F(NotifierTestFixture, run_all_threads_once_256_1) { - run_all_threads_once(256, 1); +TEST_F(NotifierBlockTestFixture, run_all_threads_once_256_1) { + run_all_threads_once_block(256, 1); } -TEST_F(NotifierTestFixture, run_all_threads_once_512_1) { - run_all_threads_once(512, 1); +TEST_F(NotifierBlockTestFixture, run_all_threads_once_512_1) { + run_all_threads_once_block(512, 1); } -TEST_F(NotifierTestFixture, run_all_threads_once_1024_1) { - run_all_threads_once(1024, 1); +TEST_F(NotifierBlockTestFixture, run_all_threads_once_1024_1) { + run_all_threads_once_block(1024, 1); } diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp index 303f1932c9..453b93680d 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp @@ -43,7 +43,7 @@ static const uint64_t NOTIFIER_OFFSET {0x100B00}; inline __device__ void -write_to_memory(uint8_t* raw_memory) { +write_to_memory_block(uint8_t* raw_memory) { auto thread_idx {get_flat_block_id()}; raw_memory[thread_idx] = THREAD_VALUE; __threadfence(); @@ -51,44 +51,43 @@ write_to_memory(uint8_t* raw_memory) { __global__ void -all_threads_once(uint8_t* raw_memory, - Notifier * notifier) { +all_threads_once_block(uint8_t* raw_memory, + Notifier * notifier) { if (!threadIdx.x) { notifier->store(NOTIFIER_OFFSET); notifier->fence(); } - __syncthreads(); + notifier->sync(); uint64_t offset_u64 {notifier->load()}; uint64_t raw_memory_u64 {reinterpret_cast(raw_memory)}; uint64_t address_u64 {raw_memory_u64 + offset_u64}; uint8_t* address {reinterpret_cast(address_u64)}; - write_to_memory(address); - __syncthreads(); + write_to_memory_block(address); } -class NotifierTestFixture : public ::testing::Test { +class NotifierBlockTestFixture : public ::testing::Test { using NotifierProxyT = NotifierProxy; public: - NotifierTestFixture() { + NotifierBlockTestFixture() { assert(raw_memory_ == nullptr); hip_allocator_.allocate((void**)&raw_memory_, GIBIBYTE_); assert(raw_memory_); } - ~NotifierTestFixture() { + ~NotifierBlockTestFixture() { if (raw_memory_) { hip_allocator_.deallocate(raw_memory_); } } void - run_all_threads_once(uint32_t x_block_dim, - uint32_t x_grid_dim) { + run_all_threads_once_block(uint32_t x_block_dim, + uint32_t x_grid_dim) { const dim3 hip_blocksize(x_block_dim, 1, 1); const dim3 hip_gridsize(x_grid_dim, 1, 1); - hipLaunchKernelGGL(all_threads_once, + hipLaunchKernelGGL(all_threads_once_block, hip_gridsize, hip_blocksize, 0, From 9b0e4dc05dd187f61daca4805d5928b402b4fe00 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Thu, 15 Aug 2024 12:22:38 -0700 Subject: [PATCH 22/46] Change notifier fixture to prep for other fixtures [ROCm/rocshmem commit: 1289d50be5e213a10269926cdf5219ab9dca5a82] --- .../tests/unit_tests/notifier_gtest.cpp | 14 ++-- .../tests/unit_tests/notifier_gtest.hpp | 64 +++++++++---------- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp index e1ebea0210..073c37a495 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp @@ -29,29 +29,29 @@ using namespace rocshmem; *****************************************************************************/ TEST_F(NotifierBlockTestFixture, run_all_threads_once_1_1) { - run_all_threads_once_block(1, 1); + run_all_threads_once(1, 1); } TEST_F(NotifierBlockTestFixture, run_all_threads_once_2_1) { - run_all_threads_once_block(2, 1); + run_all_threads_once(2, 1); } TEST_F(NotifierBlockTestFixture, run_all_threads_once_64_1) { - run_all_threads_once_block(64, 1); + run_all_threads_once(64, 1); } TEST_F(NotifierBlockTestFixture, run_all_threads_once_128_1) { - run_all_threads_once_block(128, 1); + run_all_threads_once(128, 1); } TEST_F(NotifierBlockTestFixture, run_all_threads_once_256_1) { - run_all_threads_once_block(256, 1); + run_all_threads_once(256, 1); } TEST_F(NotifierBlockTestFixture, run_all_threads_once_512_1) { - run_all_threads_once_block(512, 1); + run_all_threads_once(512, 1); } TEST_F(NotifierBlockTestFixture, run_all_threads_once_1024_1) { - run_all_threads_once_block(1024, 1); + run_all_threads_once(1024, 1); } diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp index 453b93680d..b82f47b1d9 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp @@ -43,17 +43,18 @@ static const uint64_t NOTIFIER_OFFSET {0x100B00}; inline __device__ void -write_to_memory_block(uint8_t* raw_memory) { - auto thread_idx {get_flat_block_id()}; +write_to_memory(uint8_t* raw_memory) { + auto thread_idx {get_flat_id()}; raw_memory[thread_idx] = THREAD_VALUE; __threadfence(); } +template __global__ void -all_threads_once_block(uint8_t* raw_memory, - Notifier * notifier) { - if (!threadIdx.x) { +all_threads_once(uint8_t* raw_memory, + Notifier * notifier) { + if (!get_flat_id()) { notifier->store(NOTIFIER_OFFSET); notifier->fence(); } @@ -62,49 +63,26 @@ all_threads_once_block(uint8_t* raw_memory, uint64_t raw_memory_u64 {reinterpret_cast(raw_memory)}; uint64_t address_u64 {raw_memory_u64 + offset_u64}; uint8_t* address {reinterpret_cast(address_u64)}; - write_to_memory_block(address); + write_to_memory(address); } -class NotifierBlockTestFixture : public ::testing::Test { - using NotifierProxyT = NotifierProxy; - +class NotifierBase : public ::testing::Test { public: - NotifierBlockTestFixture() { + NotifierBase() { assert(raw_memory_ == nullptr); hip_allocator_.allocate((void**)&raw_memory_, GIBIBYTE_); assert(raw_memory_); } - ~NotifierBlockTestFixture() { + ~NotifierBase() { if (raw_memory_) { hip_allocator_.deallocate(raw_memory_); } } void - run_all_threads_once_block(uint32_t x_block_dim, - uint32_t x_grid_dim) { - const dim3 hip_blocksize(x_block_dim, 1, 1); - const dim3 hip_gridsize(x_grid_dim, 1, 1); - - hipLaunchKernelGGL(all_threads_once_block, - hip_gridsize, - hip_blocksize, - 0, - nullptr, - raw_memory_, - notifier_.get()); - - hipError_t return_code = hipStreamSynchronize(nullptr); - if (return_code != hipSuccess) { - printf("Failed in stream synchronize\n"); - assert(return_code == hipSuccess); - } - - size_t number_threads {x_block_dim * x_grid_dim}; - + verify(size_t number_threads) { uint8_t* offset_addr {compute_offset_addr()}; - for (size_t i {0}; i < number_threads; i++) { ASSERT_EQ(offset_addr[i], THREAD_VALUE); } @@ -137,6 +115,26 @@ class NotifierBlockTestFixture : public ::testing::Test { */ uint8_t *raw_memory_ {nullptr}; +}; + +class NotifierBlockTestFixture : public NotifierBase { + using NotifierT = Notifier; + using NotifierProxyT = NotifierProxy; + + public: + void + run_all_threads_once(uint32_t x_block_dim, + uint32_t x_grid_dim) { + const dim3 block(x_block_dim, 1, 1); + const dim3 grid(x_grid_dim, 1, 1); + + all_threads_once<<>>(raw_memory_, notifier_.get()); + + CHECK_HIP(hipStreamSynchronize(nullptr)); + + verify(x_block_dim * x_grid_dim); + } + /** * @brief Used to broadcast base offset for writing. */ From aed0da61d083b684e7fea601666ab0b711ba7e20 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Fri, 16 Aug 2024 10:45:33 -0700 Subject: [PATCH 23/46] Add sync method to notifier class [ROCm/rocshmem commit: 359d6be797bcafd6e5766990d171535bf89b96be] --- projects/rocshmem/src/atomic.hpp | 10 --- projects/rocshmem/src/memory/notifier.hpp | 50 ++++++++++++-- .../tests/unit_tests/notifier_gtest.cpp | 68 +++++++++++++++++++ .../tests/unit_tests/notifier_gtest.hpp | 27 ++++++-- 4 files changed, 135 insertions(+), 20 deletions(-) diff --git a/projects/rocshmem/src/atomic.hpp b/projects/rocshmem/src/atomic.hpp index 330bd82f86..eadacf7284 100644 --- a/projects/rocshmem/src/atomic.hpp +++ b/projects/rocshmem/src/atomic.hpp @@ -132,16 +132,6 @@ void threadfence() { } } -template -__device__ -void sync() { - if constexpr (s == memory_scope_workgroup) { - __syncthreads(); - } else { - assert(false); - } -} - } // namespace atomic } // namespace detail } // namespace rocshmem diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index 12f53a4c8b..f01f7f8301 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -31,13 +31,14 @@ namespace rocshmem { template class Notifier { + public: __device__ uint64_t load() { - return detail::atomic::load(&value_, orders); + return detail::atomic::load(&value_, orders_); } __device__ void store(uint64_t val) { - detail::atomic::store(&value_, val, orders); + detail::atomic::store(&value_, val, orders_); } __device__ void fence() { @@ -45,19 +46,56 @@ class Notifier { } __device__ void sync() { - detail::atomic::sync(); + if constexpr (scope == detail::atomic::memory_scope_thread || + scope == detail::atomic::memory_scope_wavefront) { + return; + } + if constexpr (scope == detail::atomic::memory_scope_workgroup) { + __syncthreads(); + return; + } + if constexpr (scope == detail::atomic::memory_scope_system) { + assert(false); + return; + } + + uint32_t done = signal_ + 1; + __syncthreads(); + + uint32_t retval = 0; + bool executor {!threadIdx.x && !threadIdx.y && !threadIdx.z}; + if (executor) { + retval = detail::atomic::fetch_add(&count_, 1, orders_); + detail::atomic::threadfence(); + } + __syncthreads(); + + if (retval == ((gridDim.x * gridDim.y * gridDim.z) - 1)) { + if (executor) { + detail::atomic::store(&count_, 0, orders_); + detail::atomic::threadfence(); + auto x = detail::atomic::fetch_add(&signal_, 1, orders_); + detail::atomic::threadfence(); + } + } + while (detail::atomic::load(&signal_, orders_) != done) { + ; + } } private: - - detail::atomic::rocshmem_memory_orders orders; + detail::atomic::rocshmem_memory_orders orders_{}; uint64_t value_{}; + + uint32_t signal_ {}; + + uint32_t count_ {}; }; template class NotifierProxy { - using ProxyT = DeviceProxy, 1>; + using ProxyT = DeviceProxy>; public: __host__ __device__ Notifier* get() { return proxy_.get(); } diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp index 073c37a495..e6275b10a1 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp @@ -55,3 +55,71 @@ TEST_F(NotifierBlockTestFixture, run_all_threads_once_512_1) { TEST_F(NotifierBlockTestFixture, run_all_threads_once_1024_1) { run_all_threads_once(1024, 1); } + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_1) { + run_all_threads_once(1, 1); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_2_1) { + run_all_threads_once(2, 1); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_64_1) { + run_all_threads_once(64, 1); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_128_1) { + run_all_threads_once(128, 1); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_256_1) { + run_all_threads_once(256, 1); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_512_1) { + run_all_threads_once(512, 1); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_1) { + run_all_threads_once(1024, 1); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_2) { + run_all_threads_once(1, 2); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_2) { + run_all_threads_once(1024, 2); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_4) { + run_all_threads_once(1, 4); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_4) { + run_all_threads_once(1024, 4); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_8) { + run_all_threads_once(1, 8); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_8) { + run_all_threads_once(1024, 8); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_32) { + run_all_threads_once(1, 32); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_32) { + run_all_threads_once(1024, 32); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_64) { + run_all_threads_once(1, 64); +} + +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_64) { + run_all_threads_once(1024, 64); +} diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp index b82f47b1d9..e130159b2e 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.hpp @@ -53,7 +53,7 @@ template __global__ void all_threads_once(uint8_t* raw_memory, - Notifier * notifier) { + NotifierT * notifier) { if (!get_flat_id()) { notifier->store(NOTIFIER_OFFSET); notifier->fence(); @@ -125,13 +125,11 @@ class NotifierBlockTestFixture : public NotifierBase { void run_all_threads_once(uint32_t x_block_dim, uint32_t x_grid_dim) { + new (notifier_.get()) NotifierT(); const dim3 block(x_block_dim, 1, 1); const dim3 grid(x_grid_dim, 1, 1); - all_threads_once<<>>(raw_memory_, notifier_.get()); - CHECK_HIP(hipStreamSynchronize(nullptr)); - verify(x_block_dim * x_grid_dim); } @@ -141,6 +139,27 @@ class NotifierBlockTestFixture : public NotifierBase { NotifierProxyT notifier_ {}; }; +class NotifierAgentTestFixture : public NotifierBase { + using NotifierT = Notifier; + using NotifierProxyT = NotifierProxy; + + public: + void + run_all_threads_once(uint32_t x_block_dim, + uint32_t x_grid_dim) { + new (notifier_.get()) NotifierT(); + const dim3 block(x_block_dim, 1, 1); + const dim3 grid(x_grid_dim, 1, 1); + all_threads_once<<>>(raw_memory_, notifier_.get()); + CHECK_HIP(hipStreamSynchronize(nullptr)); + verify(x_block_dim * x_grid_dim); + } + + /** + * @brief Used to broadcast base offset for writing. + */ + NotifierProxyT notifier_ {}; +}; } // namespace rocshmem From 688826937ffc55304bccadc1fbfd6613b7b871e7 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Fri, 16 Aug 2024 12:33:13 -0700 Subject: [PATCH 24/46] Minor updates to Nofifier sync method [ROCm/rocshmem commit: 45c29e7734385ceb5de6ae26db197e208c543752] --- projects/rocshmem/src/memory/notifier.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index f01f7f8301..bcc030ce0a 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -59,25 +59,25 @@ class Notifier { return; } - uint32_t done = signal_ + 1; + uint32_t done {signal_ + 1}; __syncthreads(); - uint32_t retval = 0; + uint32_t retval {0}; bool executor {!threadIdx.x && !threadIdx.y && !threadIdx.z}; if (executor) { retval = detail::atomic::fetch_add(&count_, 1, orders_); - detail::atomic::threadfence(); + fence(); } __syncthreads(); if (retval == ((gridDim.x * gridDim.y * gridDim.z) - 1)) { if (executor) { detail::atomic::store(&count_, 0, orders_); - detail::atomic::threadfence(); - auto x = detail::atomic::fetch_add(&signal_, 1, orders_); - detail::atomic::threadfence(); + fence(); + detail::atomic::fetch_add(&signal_, 1, orders_); } } + while (detail::atomic::load(&signal_, orders_) != done) { ; } From 74c4a248cce362bdcad8912b70f1d27e6a27602f Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Mon, 9 Sep 2024 10:14:06 -0700 Subject: [PATCH 25/46] Add an extra assertion check for nullptr [ROCm/rocshmem commit: 678564ba3ce1d772745b16c9d1c4b2908a7351c0] --- .../rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.cpp | 1 + .../rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.cpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.cpp index dbf72923b5..37833f9cfe 100644 --- a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.cpp @@ -33,6 +33,7 @@ TEST_F(IPCImplSimpleCoarseTestFixture, MPI_num_pes) { } TEST_F(IPCImplSimpleCoarseTestFixture, IPC_bases) { + ASSERT_NE(ipc_impl_.ipc_bases, nullptr); for(int i{0}; i < mpi_.num_pes(); i++) { ASSERT_NE(ipc_impl_.ipc_bases[i], nullptr); } diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp index 1c4c409f59..7c752e2f6a 100644 --- a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp @@ -33,6 +33,7 @@ TEST_F(IPCImplSimpleFineTestFixture, MPI_num_pes) { } TEST_F(IPCImplSimpleFineTestFixture, IPC_bases) { + ASSERT_NE(ipc_impl_.ipc_bases, nullptr); for(int i{0}; i < mpi_.num_pes(); i++) { ASSERT_NE(ipc_impl_.ipc_bases[i], nullptr); } From 10d351b6a19b9f21b662f2dc5ca9b21f7efc1fd0 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 10 Sep 2024 07:08:56 -0700 Subject: [PATCH 26/46] Intermediate commit for rebase [ROCm/rocshmem commit: 2806e1be79a6de385d0db21d526a0a3ad402a95e] --- .../scripts/build_configs/ro_net_debug | 2 +- projects/rocshmem/src/util.hpp | 8 ++ .../ipc_impl_simple_coarse_gtest.hpp | 2 - .../unit_tests/ipc_impl_simple_fine_gtest.hpp | 82 +++++++++++++++---- 4 files changed, 77 insertions(+), 17 deletions(-) diff --git a/projects/rocshmem/scripts/build_configs/ro_net_debug b/projects/rocshmem/scripts/build_configs/ro_net_debug index 67c3f2d0a5..e8309b06d8 100755 --- a/projects/rocshmem/scripts/build_configs/ro_net_debug +++ b/projects/rocshmem/scripts/build_configs/ro_net_debug @@ -18,7 +18,7 @@ cmake \ -DPROFILE=OFF \ -DUSE_GPU_IB=OFF \ -DUSE_DC=OFF \ - -DUSE_IPC=OFF \ + -DUSE_IPC=ON \ -DUSE_THREADS=ON \ -DUSE_WF_COAL=OFF \ -DUSE_COHERENT_HEAP=ON \ diff --git a/projects/rocshmem/src/util.hpp b/projects/rocshmem/src/util.hpp index c209750d49..c02f891dc4 100644 --- a/projects/rocshmem/src/util.hpp +++ b/projects/rocshmem/src/util.hpp @@ -93,6 +93,7 @@ __device__ __forceinline__ bool is_thread_zero_in_block() { __device__ __forceinline__ bool is_block_zero_in_grid() { return hipBlockIdx_x == 0 && hipBlockIdx_y == 0 && hipBlockIdx_z == 0; } + /* * Returns the number of threads in the caller's flattened thread block. */ @@ -100,6 +101,13 @@ __device__ __forceinline__ int get_flat_block_size() { return hipBlockDim_x * hipBlockDim_y * hipBlockDim_z; } +/* + * Returns the number of threads in the caller's flattened grid. + */ +__device__ __forceinline__ int get_flat_grid_size() { + return get_flat_block_size() * hipGridDim_x * hipGridDim_y * hipGridDim_z; +} + /* * Returns the flattened thread index of the calling thread within its * thread block. diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.hpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.hpp index 02dfd8c55a..083d73f418 100644 --- a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_coarse_gtest.hpp @@ -218,8 +218,6 @@ class IPCImplSimpleCoarseTestFixture : public ::testing::Test { protected: std::vector golden_; - std::vector output_; - HEAP_T heap_mem_ {}; MPI_T mpi_ {heap_mem_.get_ptr(), heap_mem_.get_size()}; diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp index 3f85528e7d..fb60acdec5 100644 --- a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp @@ -33,9 +33,27 @@ namespace rocshmem { +enum TestType { + READ = 0, + WRITE = 1 +}; + __global__ void -kernel_simple_fine_copy(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { +kernel_simple_fine_copy(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes, TestType test) { + if (!threadIdx.x) { + ipc_impl->ipcCopy(dest, src, bytes); + ipc_impl->ipcFence(); + if (test == WRITE) { + ipc_impl->ipc + } + } + __syncthreads(); +} + +__global__ +void +kernel_simple_fine_copy_signal_validate(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { if (!threadIdx.x) { ipc_impl->ipcCopy(dest, src, bytes); ipc_impl->ipcFence(); @@ -51,6 +69,14 @@ kernel_simple_fine_copy_wg(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) __syncthreads(); } +__global__ +void +kernel_simple_fine_copy_wg_signal_validate(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { + ipc_impl->ipcCopy_wg(dest, src, bytes); + ipc_impl->ipcFence(); + __syncthreads(); +} + __global__ void kernel_simple_fine_copy_wave(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { @@ -59,6 +85,14 @@ kernel_simple_fine_copy_wave(IpcImpl *ipc_impl, int *src, int *dest, size_t byte __syncthreads(); } +__global__ +void +kernel_simple_fine_copy_wave_signal_validate(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { + ipc_impl->ipcCopy_wave(dest, src, bytes); + ipc_impl->ipcFence(); + __syncthreads(); +} + class IPCImplSimpleFineTestFixture : public ::testing::Test { using HEAP_T = HeapMemory; @@ -91,51 +125,46 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { CHECK_HIP(hipStreamSynchronize(nullptr)); } - enum TestType { - READ = 0, - WRITE = 1 - }; - void write(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); initialize_src_buffer(WRITE); copy(WRITE, grid, block); - validate_dest_buffer(WRITE); + check_device_validation_errors(WRITE); } void write_wg(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); initialize_src_buffer(WRITE); copy_wg(WRITE, grid, block); - validate_dest_buffer(WRITE); + check_device_validation_errors(WRITE); } void write_wave(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); initialize_src_buffer(WRITE); copy_wave(WRITE, grid, block); - validate_dest_buffer(WRITE); + check_device_validation_errors(WRITE); } void read(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); initialize_src_buffer(READ); copy(READ, grid, block); - validate_dest_buffer(READ); + check_device_validation_errors(READ); } void read_wg(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); initialize_src_buffer(READ); copy_wg(READ, grid, block); - validate_dest_buffer(READ); + check_device_validation_errors(READ); } void read_wave(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); initialize_src_buffer(READ); copy_wave(READ, grid, block); - validate_dest_buffer(READ); + check_device_validation_errors(READ); } void iota_golden(size_t elems) { @@ -160,6 +189,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { CHECK_HIP(hipStreamSynchronize(nullptr)); } + __host__ __device__ bool pe_initializes_src_buffer(TestType test) { bool is_write_test = test; bool is_read_test = !test; @@ -184,7 +214,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { } size_t bytes = golden_.size() * sizeof(int); mpi_.barrier(); - launch(fn, grid, block, src, dest, bytes); + launch(fn, grid, block, src, dest, bytes, test); mpi_.barrier(); } @@ -200,6 +230,13 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { execute(test, kernel_simple_fine_copy_wave, grid, block); } + void check_device_validation_errors(TestType test) { + if (!pe_validates_dest_buffer(test)) { + return; + } + ASSERT_EQ(validation_error, false); + } + void validate_dest_buffer(TestType test) { if (!pe_validates_dest_buffer(test)) { return; @@ -211,6 +248,21 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { } } + __device__ + void validate_dest_buffer(TestType test) { + if (!pe_validates_dest_buffer(test)) { + return; + } + + auto dev_dest = reinterpret_cast(ipc_impl_.ipc_bases[mpi_.my_pe()]); + for (int i {get_flat_id()}; i < golden_.size(); i += get_flat_grid_size()) { + if (dev_golden_[i] != dev_dest[i]) { + validation_error = true; + } + } + } + + __host__ __device__ bool pe_validates_dest_buffer(TestType test) { return !pe_initializes_src_buffer(test); } @@ -218,7 +270,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { protected: std::vector golden_; - std::vector output_; + std::vector device_golden_; HEAP_T heap_mem_ {}; @@ -229,6 +281,8 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { IpcImpl *ipc_impl_dptr_ {nullptr}; HIPAllocator hip_allocator_ {}; + + bool validation_error {false}; }; } // namespace rocshmem From bf79c21ea833334d56352e18997a0b25d65dd172 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 10 Sep 2024 09:34:45 -0700 Subject: [PATCH 27/46] Conservatively use SEQ_CST atomics in IPC conduit [ROCm/rocshmem commit: 7411c4559151e963c173b89910b89ffb3eb03b35] --- projects/rocshmem/src/ipc_policy.hpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/projects/rocshmem/src/ipc_policy.hpp b/projects/rocshmem/src/ipc_policy.hpp index c0190198ca..48b7b7fd40 100644 --- a/projects/rocshmem/src/ipc_policy.hpp +++ b/projects/rocshmem/src/ipc_policy.hpp @@ -68,34 +68,34 @@ class IpcOnImpl { template __device__ T ipcAMOFetchAdd(T *val, T value) { - return __hip_atomic_fetch_add(val, value, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); + return __hip_atomic_fetch_add(val, value, __ATOMIC_SEQ_CST, + __HIP_MEMORY_SCOPE_SYSTEM); } template __device__ T ipcAMOFetchCas(T *val, T cond, T value) { - __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_RELAXED, - __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); + __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST, + __HIP_MEMORY_SCOPE_SYSTEM); return cond; } template __device__ void ipcAMOAdd(T *val, T value) { - __hip_atomic_fetch_add(val, value, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); + __hip_atomic_fetch_add(val, value, __ATOMIC_SEQ_CST, + __HIP_MEMORY_SCOPE_SYSTEM); } template __device__ void ipcAMOCas(T *val, T cond, T value) { - __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_RELAXED, - __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); + __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST, + __HIP_MEMORY_SCOPE_SYSTEM); } template __device__ void ipcAMOSet(T *val, T value) { - __hip_atomic_store(val, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + __hip_atomic_store(val, value, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SYSTEM); } __device__ void zero_byte_read(int pe) { From e64264d233db80d6b666037b82854134edbbd9b5 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 10 Sep 2024 09:35:02 -0700 Subject: [PATCH 28/46] Add missing header file [ROCm/rocshmem commit: 86a2f34539c327ad106efb1ae722c4f84ebe40b8] --- projects/rocshmem/src/atomic.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rocshmem/src/atomic.hpp b/projects/rocshmem/src/atomic.hpp index eadacf7284..f0828e4145 100644 --- a/projects/rocshmem/src/atomic.hpp +++ b/projects/rocshmem/src/atomic.hpp @@ -23,6 +23,8 @@ #ifndef LIBRARY_SRC_ATOMIC_HPP #define LIBRARY_SRC_ATOMIC_HPP +#include + namespace rocshmem { namespace detail { namespace atomic { From 56c1626df190236a5373d1a89efbdf0f5daf8094 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Tue, 10 Sep 2024 09:35:41 -0700 Subject: [PATCH 29/46] Update fine-grained simple tests [ROCm/rocshmem commit: 46fdb1851c067267ad40208508b6ed5b82d40005] --- .../unit_tests/ipc_impl_simple_fine_gtest.hpp | 205 +++++++++++------- 1 file changed, 122 insertions(+), 83 deletions(-) diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp index fb60acdec5..8e9c28ac46 100644 --- a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp @@ -26,10 +26,13 @@ #include "gtest/gtest.h" #include - #include + +#include "../src/atomic.hpp" +#include "../src/ipc_policy.hpp" +#include "../src/memory/notifier.hpp" #include "../src/memory/symmetric_heap.hpp" -#include "../src/ipc/ipc_policy.hpp" +#include "../src/util.hpp" namespace rocshmem { @@ -38,68 +41,96 @@ enum TestType { WRITE = 1 }; +const uint32_t SIGNAL_OFFSET {67108864}; + +__device__ +void +validator(bool *error, int *golden, int *dest, size_t bytes) { + size_t elements {bytes / sizeof(int)}; + for (int i {get_flat_id()}; i < elements; i += get_flat_grid_size()) { + if (golden[i] != dest[i]) { + *error = true; + } + } +} + +template __global__ void -kernel_simple_fine_copy(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes, TestType test) { - if (!threadIdx.x) { - ipc_impl->ipcCopy(dest, src, bytes); - ipc_impl->ipcFence(); - if (test == WRITE) { - ipc_impl->ipc - } +kernel_put_with_signal_validator(bool *error, int *golden, int *dest, size_t bytes, NotifierT *notifier) { + detail::atomic::rocshmem_memory_orders orders{}; + if (!get_flat_id()) { + while (detail::atomic::load(dest + SIGNAL_OFFSET, orders) == 0) { + ; + } + } + notifier->sync(); + validator(error, golden, dest, bytes); +} + +template +__global__ +void +kernel_simple_fine_copy(IpcImpl *ipc_impl, bool *error, int *golden, int *src, int *dest, size_t bytes, TestType test, NotifierT *notifier) { + if (!get_flat_id()) { + ipc_impl->ipcCopy(dest, src, bytes); + ipc_impl->ipcFence(); + if (test == WRITE) { + ipc_impl->ipcAMOFetchAdd(dest + SIGNAL_OFFSET, 1); + } + } + if (test == READ) { + notifier->sync(); + validator(error, golden, dest, bytes); + } +} + +template +__global__ +void +kernel_simple_fine_copy_wg(IpcImpl *ipc_impl, bool *error, int *golden, int *src, int *dest, size_t bytes, TestType test, NotifierT *notifier) { + if (!blockIdx.x) { + ipc_impl->ipcCopy_wg(dest, src, bytes); + ipc_impl->ipcFence(); + if (test == WRITE) { + if (!threadIdx.x) { + ipc_impl->ipcAMOFetchAdd(dest + SIGNAL_OFFSET, 1); + } + } + } + if (test == READ) { + notifier->sync(); + validator(error, golden, dest, bytes); + } +} + +template +__global__ +void +kernel_simple_fine_copy_wave(IpcImpl *ipc_impl, bool *error, int *golden, int *src, int *dest, size_t bytes, TestType test, NotifierT *notifier) { + if (!blockIdx.x && threadIdx.x < 64) { + ipc_impl->ipcCopy_wave(dest, src, bytes); + ipc_impl->ipcFence(); + if (test == WRITE) { + if (!threadIdx.x) { + ipc_impl->ipcAMOFetchAdd(dest + SIGNAL_OFFSET, 1); + } + } } __syncthreads(); -} - -__global__ -void -kernel_simple_fine_copy_signal_validate(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { - if (!threadIdx.x) { - ipc_impl->ipcCopy(dest, src, bytes); - ipc_impl->ipcFence(); + if (test == READ) { + notifier->sync(); + validator(error, golden, dest, bytes); } - __syncthreads(); -} - -__global__ -void -kernel_simple_fine_copy_wg(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { - ipc_impl->ipcCopy_wg(dest, src, bytes); - ipc_impl->ipcFence(); - __syncthreads(); -} - -__global__ -void -kernel_simple_fine_copy_wg_signal_validate(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { - ipc_impl->ipcCopy_wg(dest, src, bytes); - ipc_impl->ipcFence(); - __syncthreads(); -} - -__global__ -void -kernel_simple_fine_copy_wave(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { - ipc_impl->ipcCopy_wave(dest, src, bytes); - ipc_impl->ipcFence(); - __syncthreads(); -} - -__global__ -void -kernel_simple_fine_copy_wave_signal_validate(IpcImpl *ipc_impl, int *src, int *dest, size_t bytes) { - ipc_impl->ipcCopy_wave(dest, src, bytes); - ipc_impl->ipcFence(); - __syncthreads(); } class IPCImplSimpleFineTestFixture : public ::testing::Test { - using HEAP_T = HeapMemory; - using MPI_T = RemoteHeapInfo; - - using FN_T = void (*)(IpcImpl*, int*, int*, size_t); + using NotifierT = Notifier; + using NotifierProxyT = NotifierProxy; + using FN_T1 = void (*)(IpcImpl*, bool*, int*, int*, int*, size_t, TestType, NotifierT*); + using FN_T2 = void (*)(bool*, int*, int*, size_t, NotifierT*); public: IPCImplSimpleFineTestFixture() { @@ -107,21 +138,34 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { assert(ipc_impl_dptr_ == nullptr); hip_allocator_.allocate((void**)&ipc_impl_dptr_, sizeof(IpcImpl)); + CHECK_HIP(hipMemcpy(ipc_impl_dptr_, &ipc_impl_, sizeof(IpcImpl), hipMemcpyHostToDevice)); - CHECK_HIP(hipMemcpy(ipc_impl_dptr_, &ipc_impl_, - sizeof(IpcImpl), hipMemcpyHostToDevice)); + assert(error_dptr_ == nullptr); + hip_allocator_.allocate((void**)&error_dptr_, sizeof(bool)); + *error_dptr_ = false; } ~IPCImplSimpleFineTestFixture() { if (ipc_impl_dptr_) { hip_allocator_.deallocate(ipc_impl_dptr_); } + if (error_dptr_) { + hip_allocator_.deallocate(error_dptr_); + } + if (golden_dptr_) { + hip_allocator_.deallocate(golden_dptr_); + } ipc_impl_.ipcHostStop(); } - void launch(FN_T f, const dim3 grid, const dim3 block, int* src, int* dest, size_t bytes) { - f<<>>(ipc_impl_dptr_, src, dest, bytes); + void launch(FN_T1 f, const dim3 grid, const dim3 block, int* src, int* dest, size_t bytes, TestType test) { + f<<>>(ipc_impl_dptr_, error_dptr_, golden_dptr_, src, dest, bytes, test, notifier_.get()); + CHECK_HIP(hipStreamSynchronize(nullptr)); + } + + void launch(FN_T2 f, const dim3 grid, const dim3 block, int* dest, size_t bytes) { + f<<>>(error_dptr_, golden_dptr_, dest, bytes, notifier_.get()); CHECK_HIP(hipStreamSynchronize(nullptr)); } @@ -170,6 +214,11 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { void iota_golden(size_t elems) { golden_.resize(elems); std::iota(golden_.begin(), golden_.end(), 0); + + assert(golden_dptr_ == nullptr); + size_t golden_dptr_bytes {golden_.size() * sizeof(int)}; + hip_allocator_.allocate((void**)&golden_dptr_, golden_dptr_bytes); + CHECK_HIP(hipMemcpy(golden_dptr_, golden_.data(), golden_dptr_bytes, hipMemcpyHostToDevice)); } void validate_golden(size_t elems) { @@ -186,10 +235,8 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { size_t bytes = golden_.size() * sizeof(int); auto dev_src = reinterpret_cast(ipc_impl_.ipc_bases[mpi_.my_pe()]); CHECK_HIP(hipMemcpy(dev_src, golden_.data(), bytes, hipMemcpyHostToDevice)); - CHECK_HIP(hipStreamSynchronize(nullptr)); } - __host__ __device__ bool pe_initializes_src_buffer(TestType test) { bool is_write_test = test; bool is_read_test = !test; @@ -197,9 +244,15 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { (is_read_test && mpi_.my_pe() == 1); } - void execute(TestType test, FN_T fn, const dim3 grid, const dim3 block) { + void execute(TestType test, FN_T1 fn, const dim3 grid, const dim3 block) { + size_t bytes = golden_.size() * sizeof(int); if (mpi_.my_pe()) { mpi_.barrier(); + if (test == WRITE) { + int *dest = reinterpret_cast(ipc_impl_.ipc_bases[1]); + FN_T2 val_fn = kernel_put_with_signal_validator; + launch(val_fn, grid, block, dest, bytes); + } mpi_.barrier(); return; } @@ -212,7 +265,6 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { src = reinterpret_cast(ipc_impl_.ipc_bases[1]); dest = reinterpret_cast(ipc_impl_.ipc_bases[0]); } - size_t bytes = golden_.size() * sizeof(int); mpi_.barrier(); launch(fn, grid, block, src, dest, bytes, test); mpi_.barrier(); @@ -234,7 +286,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { if (!pe_validates_dest_buffer(test)) { return; } - ASSERT_EQ(validation_error, false); + ASSERT_EQ(*error_dptr_, false); } void validate_dest_buffer(TestType test) { @@ -248,41 +300,28 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { } } - __device__ - void validate_dest_buffer(TestType test) { - if (!pe_validates_dest_buffer(test)) { - return; - } - - auto dev_dest = reinterpret_cast(ipc_impl_.ipc_bases[mpi_.my_pe()]); - for (int i {get_flat_id()}; i < golden_.size(); i += get_flat_grid_size()) { - if (dev_golden_[i] != dev_dest[i]) { - validation_error = true; - } - } - } - - __host__ __device__ bool pe_validates_dest_buffer(TestType test) { return !pe_initializes_src_buffer(test); } protected: - std::vector golden_; + HIPDefaultFinegrainedAllocator hip_allocator_ {}; - std::vector device_golden_; + NotifierProxyT notifier_ {}; HEAP_T heap_mem_ {}; MPI_T mpi_ {heap_mem_.get_ptr(), heap_mem_.get_size()}; + std::vector golden_; + + int *golden_dptr_ {nullptr}; + IpcImpl ipc_impl_ {}; IpcImpl *ipc_impl_dptr_ {nullptr}; - HIPAllocator hip_allocator_ {}; - - bool validation_error {false}; + bool *error_dptr_ {nullptr}; }; } // namespace rocshmem From fe3ecde6f6803f4bbd3f1d43c551b8c5b0b3bda7 Mon Sep 17 00:00:00 2001 From: Muhammad Awad Date: Tue, 17 Sep 2024 20:34:18 -0500 Subject: [PATCH 30/46] Vectorize WQe segments writes Signed-off-by: Muhammad Awad [ROCm/rocshmem commit: 3162d49b5610be4604f591648d7ac4c66d9dcdab] --- .../rocshmem/src/gpu_ib/segment_builder.cpp | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/projects/rocshmem/src/gpu_ib/segment_builder.cpp b/projects/rocshmem/src/gpu_ib/segment_builder.cpp index 205c890861..6f2a77a23e 100644 --- a/projects/rocshmem/src/gpu_ib/segment_builder.cpp +++ b/projects/rocshmem/src/gpu_ib/segment_builder.cpp @@ -22,6 +22,7 @@ #include "segment_builder.hpp" +#include "../util.hpp" #include "endian.hpp" namespace rocshmem { @@ -35,9 +36,9 @@ __device__ SegmentBuilder::SegmentBuilder(uint64_t wqe_idx, void *base) { __device__ void SegmentBuilder::update_cntrl_seg( uint8_t opcode, uint16_t wqe_idx, uint32_t ctrl_qp_sq, uint64_t ctrl_sig, ConnectionImpl *connection_policy, bool zero_byte_rd) { - mlx5_wqe_ctrl_seg *ctrl_seg = &seg_ptr->ctrl_seg; + mlx5_wqe_ctrl_seg ctrl_seg; - ctrl_seg->opmod_idx_opcode = (opcode << 24) | (wqe_idx << 8); + ctrl_seg.opmod_idx_opcode = (opcode << 24) | (wqe_idx << 8); uint32_t DS = 2; if (zero_byte_rd == false) { @@ -48,39 +49,43 @@ __device__ void SegmentBuilder::update_cntrl_seg( DS += connection_policy->wqeCntrlOffset(); - ctrl_seg->qpn_ds = (DS << 24) | ctrl_qp_sq; + ctrl_seg.qpn_ds = (DS << 24) | ctrl_qp_sq; - ctrl_seg->signature = ctrl_sig; + ctrl_seg.signature = ctrl_sig; - ctrl_seg->fm_ce_se = ctrl_sig >> 24; + ctrl_seg.fm_ce_se = ctrl_sig >> 24; - ctrl_seg->imm = ctrl_sig >> 32; + ctrl_seg.imm = ctrl_sig >> 32; + + memcpy(&seg_ptr->ctrl_seg, &ctrl_seg, sizeof(mlx5_wqe_ctrl_seg)); seg_ptr++; } __device__ void SegmentBuilder::update_atomic_data_seg(uint64_t atomic_data, uint64_t atomic_cmp) { - mlx5_wqe_atomic_seg *atomic_seg = &seg_ptr->atomic_seg; + mlx5_wqe_atomic_seg atomic_seg; - swap_endian_store(reinterpret_cast(&atomic_seg->swap_add), + swap_endian_store(reinterpret_cast(&atomic_seg.swap_add), atomic_data); - swap_endian_store(reinterpret_cast(&atomic_seg->compare), + swap_endian_store(reinterpret_cast(&atomic_seg.compare), atomic_cmp); + memcpy(&seg_ptr->atomic_seg, &atomic_seg, sizeof(mlx5_wqe_atomic_seg)); seg_ptr++; } __device__ void SegmentBuilder::update_rdma_seg(uintptr_t *raddr, uint32_t rkey) { - mlx5_wqe_raddr_seg *raddr_seg = &seg_ptr->raddr_seg; + mlx5_wqe_raddr_seg raddr_seg; - raddr_seg->rkey = rkey; + raddr_seg.rkey = rkey; - swap_endian_store(reinterpret_cast(&raddr_seg->raddr), + swap_endian_store(reinterpret_cast(&raddr_seg.raddr), reinterpret_cast(raddr)); + memcpy(&seg_ptr->raddr_seg, &raddr_seg, sizeof(mlx5_wqe_raddr_seg)); seg_ptr++; } @@ -90,33 +95,36 @@ __device__ void SegmentBuilder::update_data_seg(uintptr_t *laddr, int32_t size, return; } - mlx5_wqe_data_seg *data_seg = &seg_ptr->data_seg; + mlx5_wqe_data_seg data_seg; + data_seg.lkey = lkey; - data_seg->lkey = lkey; - - swap_endian_store(&data_seg->byte_count, size & 0x7FFFFFFFU); - - swap_endian_store(reinterpret_cast(&data_seg->addr), + swap_endian_store(&data_seg.byte_count, size & 0x7FFFFFFFU); + swap_endian_store(reinterpret_cast(&data_seg.addr), reinterpret_cast(laddr)); + memcpy(&seg_ptr->data_seg, &data_seg, sizeof(mlx5_wqe_data_seg)); seg_ptr++; } __device__ void SegmentBuilder::update_inl_data_seg(uintptr_t *laddr, int32_t size) { - mlx5_wqe_inl_data_seg *inl_data_seg = &seg_ptr->inl_data_seg; + mlx5_wqe_inl_data_seg inl_data_seg; - swap_endian_store(&inl_data_seg->byte_count, (size & 0x3FF) | 0x80000000); + swap_endian_store(&inl_data_seg.byte_count, (size & 0x3FF) | 0x80000000); // Assume fence HDP flush // TODO(khamidou): Rework fence interface to avoid this + size_t field_size{sizeof(mlx5_wqe_inl_data_seg)}; if (!laddr) { uint8_t flush_val = 1; - memcpy(inl_data_seg + 1, &flush_val, sizeof(flush_val)); + memcpy(&inl_data_seg + 1, &flush_val, sizeof(flush_val)); + field_size += sizeof(flush_val); } else { - memcpy(inl_data_seg + 1, laddr, size); + memcpy(&inl_data_seg + 1, laddr, size); + field_size += size; } + memcpy(&seg_ptr->inl_data_seg, &inl_data_seg, field_size); seg_ptr++; } From bed676f89d2952d50e78d5173f89ed8ecc782112 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Fri, 20 Sep 2024 20:25:40 +0000 Subject: [PATCH 31/46] fix assembly switch/case instruction move the case statement out of the architecture specific section. [ROCm/rocshmem commit: c133ea18a500d28722b9a2a1cb642b1b87984a8d] --- projects/rocshmem/src/assembly.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rocshmem/src/assembly.hpp b/projects/rocshmem/src/assembly.hpp index 076600067e..a14907a0bd 100644 --- a/projects/rocshmem/src/assembly.hpp +++ b/projects/rocshmem/src/assembly.hpp @@ -135,12 +135,12 @@ NOWARN(-Wdeprecated-volatile, : "v"(src)); #endif break; + case 8: #if defined(__gfx906__) #endif #if defined(__gfx908__) #endif #if defined(__gfx90a__) - case 8: asm volatile( "global_load_dwordx2 %0 %1 off glc slc \n" "s_waitcnt vmcnt(0)" From 325ce3cba74e68345d027bb1798bb819b7a60fc0 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Thu, 26 Sep 2024 13:40:05 -0500 Subject: [PATCH 32/46] Bugfixes for the ipc unit tests [ROCm/rocshmem commit: f85c46ec0a4c34ac599949c09c65da9fa4d279e3] --- projects/rocshmem/src/ipc_policy.hpp | 2 +- projects/rocshmem/src/memory/notifier.hpp | 8 ++++++++ .../unit_tests/ipc_impl_simple_fine_gtest.cpp | 6 ++++++ .../unit_tests/ipc_impl_simple_fine_gtest.hpp | 15 +++++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/projects/rocshmem/src/ipc_policy.hpp b/projects/rocshmem/src/ipc_policy.hpp index 48b7b7fd40..1b84c52613 100644 --- a/projects/rocshmem/src/ipc_policy.hpp +++ b/projects/rocshmem/src/ipc_policy.hpp @@ -64,7 +64,7 @@ class IpcOnImpl { __device__ void ipcCopy_wave(void *dst, void *src, size_t size); - __device__ void ipcFence() { __threadfence(); } + __device__ void ipcFence() { __threadfence_system(); } template __device__ T ipcAMOFetchAdd(T *val, T value) { diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index bcc030ce0a..b72a256f15 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -98,6 +98,14 @@ class NotifierProxy { using ProxyT = DeviceProxy>; public: + NotifierProxy() { + new (proxy_.get()) Notifier(); + } + + ~NotifierProxy() { + proxy_.get()->~Notifier(); + } + __host__ __device__ Notifier* get() { return proxy_.get(); } private: diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp index 7c752e2f6a..71cb85dc9e 100644 --- a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.cpp @@ -46,6 +46,12 @@ TEST_F(IPCImplSimpleFineTestFixture, golden_1048576_int) { //============================================================================= +TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_1x1x1_32_int) { + dim3 grid {1,1,1}; + dim3 block {1,1,1}; + write_wg(grid, block, 32); +} + TEST_F(IPCImplSimpleFineTestFixture, write_wg_1x1x1_1024x1x1_32_int) { dim3 grid {1,1,1}; dim3 block {1024,1,1}; diff --git a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp index 8e9c28ac46..ba4c20e288 100644 --- a/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp +++ b/projects/rocshmem/tests/unit_tests/ipc_impl_simple_fine_gtest.hpp @@ -49,6 +49,7 @@ validator(bool *error, int *golden, int *dest, size_t bytes) { size_t elements {bytes / sizeof(int)}; for (int i {get_flat_id()}; i < elements; i += get_flat_grid_size()) { if (golden[i] != dest[i]) { + printf("golden[%d] %d != dest[%d] %d\n", i, golden[i], i, dest[i]); *error = true; } } @@ -171,6 +172,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { void write(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); + initialize_signal(WRITE); initialize_src_buffer(WRITE); copy(WRITE, grid, block); check_device_validation_errors(WRITE); @@ -178,6 +180,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { void write_wg(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); + initialize_signal(WRITE); initialize_src_buffer(WRITE); copy_wg(WRITE, grid, block); check_device_validation_errors(WRITE); @@ -185,6 +188,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { void write_wave(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); + initialize_signal(WRITE); initialize_src_buffer(WRITE); copy_wave(WRITE, grid, block); check_device_validation_errors(WRITE); @@ -192,6 +196,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { void read(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); + initialize_signal(READ); initialize_src_buffer(READ); copy(READ, grid, block); check_device_validation_errors(READ); @@ -199,6 +204,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { void read_wg(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); + initialize_signal(READ); initialize_src_buffer(READ); copy_wg(READ, grid, block); check_device_validation_errors(READ); @@ -206,6 +212,7 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { void read_wave(const dim3 grid, const dim3 block, size_t elems) { iota_golden(elems); + initialize_signal(READ); initialize_src_buffer(READ); copy_wave(READ, grid, block); check_device_validation_errors(READ); @@ -228,6 +235,14 @@ class IPCImplSimpleFineTestFixture : public ::testing::Test { } } + void initialize_signal(TestType test) { + bool is_write_test = test; + if (is_write_test && mpi_.my_pe() == 0) { + int *dest = reinterpret_cast(ipc_impl_.ipc_bases[1]); + *(dest + SIGNAL_OFFSET) = 0; + } + } + void initialize_src_buffer(TestType test) { if (!pe_initializes_src_buffer(test)) { return; From 24a527dcda14d11ccbff1caf1d25c31b3f0a4310 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Fri, 27 Sep 2024 11:16:55 -0500 Subject: [PATCH 33/46] Reset config options to original values [ROCm/rocshmem commit: 56b2ed699bd04c3bf7fe3cee03c4b6bc46ad1182] --- projects/rocshmem/scripts/build_configs/ro_net | 4 ++-- projects/rocshmem/scripts/build_configs/ro_net_debug | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/rocshmem/scripts/build_configs/ro_net b/projects/rocshmem/scripts/build_configs/ro_net index 47690bb2d9..95ea2950a8 100755 --- a/projects/rocshmem/scripts/build_configs/ro_net +++ b/projects/rocshmem/scripts/build_configs/ro_net @@ -21,7 +21,7 @@ cmake \ -DUSE_IPC=ON \ -DUSE_THREADS=ON \ -DUSE_WF_COAL=OFF \ - -DUSE_COHERENT_HEAP=ON \ + -DUSE_COHERENT_HEAP=OFF \ $src_path cmake --build . --parallel 8 -#cmake --install . +cmake --install . diff --git a/projects/rocshmem/scripts/build_configs/ro_net_debug b/projects/rocshmem/scripts/build_configs/ro_net_debug index e8309b06d8..c2b8afdc00 100755 --- a/projects/rocshmem/scripts/build_configs/ro_net_debug +++ b/projects/rocshmem/scripts/build_configs/ro_net_debug @@ -18,10 +18,10 @@ cmake \ -DPROFILE=OFF \ -DUSE_GPU_IB=OFF \ -DUSE_DC=OFF \ - -DUSE_IPC=ON \ + -DUSE_IPC=OFF \ -DUSE_THREADS=ON \ -DUSE_WF_COAL=OFF \ - -DUSE_COHERENT_HEAP=ON \ + -DUSE_COHERENT_HEAP=OFF \ $src_path cmake --build . --parallel 8 cmake --install . From 25d7d7fccd41295c6f54e578a067e4437cabc584 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Fri, 27 Sep 2024 11:17:53 -0500 Subject: [PATCH 34/46] Change notifier max thread block value to account for MI300 CPX [ROCm/rocshmem commit: db221b022abb18139f85142102005104afb2e692] --- projects/rocshmem/tests/unit_tests/notifier_gtest.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp index e6275b10a1..d3b699e3d9 100644 --- a/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/notifier_gtest.cpp @@ -116,10 +116,10 @@ TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_32) { run_all_threads_once(1024, 32); } -TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_64) { - run_all_threads_once(1, 64); +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1_38) { + run_all_threads_once(1, 38); // MI300 CPX } -TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_64) { - run_all_threads_once(1024, 64); +TEST_F(NotifierAgentTestFixture, run_all_threads_once_1024_38) { + run_all_threads_once(1024, 38); // MI300 CPX } From 0641a4a29e9c7263249f70ac626274eb83d83718 Mon Sep 17 00:00:00 2001 From: avinashkethineedi Date: Tue, 1 Oct 2024 18:33:36 +0000 Subject: [PATCH 35/46] make MPI_Init and MPI_Finalize independent of the test fixtures [ROCm/rocshmem commit: 0f7dc708940563eaa245ae2e67a3f5ff1f869a3b] --- .../rocshmem/tests/unit_tests/shmem_gtest.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp b/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp index 3115ef5e52..d5a982f4c7 100644 --- a/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp @@ -21,8 +21,24 @@ *****************************************************************************/ #include "gtest/gtest.h" +#include int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + + int initialized; + MPI_Initialized(&initialized); + if (!initialized) { + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); + } + + int ret_val = RUN_ALL_TESTS(); + + int finalized{0}; + MPI_Finalized(&finalized); + if (!finalized) { + MPI_Finalize(); + } + return ret_val; } From cd44115728d14fcb9ff4656e15f7a7f9dfc75701 Mon Sep 17 00:00:00 2001 From: Brandon Potter Date: Fri, 27 Sep 2024 15:17:57 -0500 Subject: [PATCH 36/46] Poll the signal from one thread instead of all [ROCm/rocshmem commit: 24b928a0075ca9425028d719b41262f14aa1135a] --- projects/rocshmem/src/memory/notifier.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/projects/rocshmem/src/memory/notifier.hpp b/projects/rocshmem/src/memory/notifier.hpp index b72a256f15..946a9c291e 100644 --- a/projects/rocshmem/src/memory/notifier.hpp +++ b/projects/rocshmem/src/memory/notifier.hpp @@ -78,9 +78,12 @@ class Notifier { } } - while (detail::atomic::load(&signal_, orders_) != done) { - ; + if (executor) { + while (detail::atomic::load(&signal_, orders_) != done) { + ; + } } + __syncthreads(); } private: From 285ac5cab6c0412d3a9e33f0f5e17d23deb6eb78 Mon Sep 17 00:00:00 2001 From: avinashkethineedi Date: Tue, 1 Oct 2024 20:05:15 +0000 Subject: [PATCH 37/46] Add MPI_THREAD_MULTIPLE check [ROCm/rocshmem commit: 2f0739d8234eafe8852309cc806dfd8f802bd381] --- projects/rocshmem/tests/unit_tests/shmem_gtest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp b/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp index d5a982f4c7..8536ca638a 100644 --- a/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp +++ b/projects/rocshmem/tests/unit_tests/shmem_gtest.cpp @@ -31,6 +31,9 @@ int main(int argc, char **argv) { if (!initialized) { int provided; MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); + if (provided != MPI_THREAD_MULTIPLE) { + std::cerr << "MPI_THREAD_MULTIPLE support disabled.\n"; + } } int ret_val = RUN_ALL_TESTS(); From 69784a7423b2d2647fdae315338663e26d1e989e Mon Sep 17 00:00:00 2001 From: avinashkethineedi Date: Thu, 3 Oct 2024 06:28:12 +0000 Subject: [PATCH 38/46] Add fence and quiet functionality * Perform atomic stores to enforce memory ordering [ROCm/rocshmem commit: 979aed105a88d3df0d8b8e23340c3058f6822d87] --- projects/rocshmem/src/ipc/backend_ipc.cpp | 10 ++++++++++ projects/rocshmem/src/ipc/backend_ipc.hpp | 10 ++++++++++ projects/rocshmem/src/ipc/context_ipc_device.cpp | 8 ++++++++ projects/rocshmem/src/ipc/context_ipc_device.hpp | 7 +++++++ 4 files changed, 35 insertions(+) diff --git a/projects/rocshmem/src/ipc/backend_ipc.cpp b/projects/rocshmem/src/ipc/backend_ipc.cpp index 58caaf53f7..b09adee2ff 100644 --- a/projects/rocshmem/src/ipc/backend_ipc.cpp +++ b/projects/rocshmem/src/ipc/backend_ipc.cpp @@ -88,6 +88,8 @@ IPCBackend::IPCBackend(MPI_Comm comm) roc_shmem_collective_init(); + setup_fence_buffer(); + teams_init(); setup_ctxs(); @@ -287,6 +289,14 @@ void IPCBackend::teams_destroy() { free(reduced_bitmask_); } +void IPCBackend::setup_fence_buffer() { + /* + * Allocate heap space for fence + */ + fence_pool = reinterpret_cast(roc_shmem_malloc( + sizeof(int) * num_pes)); +} + void IPCBackend::roc_shmem_collective_init() { /* * Allocate heap space for barrier_sync diff --git a/projects/rocshmem/src/ipc/backend_ipc.hpp b/projects/rocshmem/src/ipc/backend_ipc.hpp index cc385f16ee..7d00e6cf07 100644 --- a/projects/rocshmem/src/ipc/backend_ipc.hpp +++ b/projects/rocshmem/src/ipc/backend_ipc.hpp @@ -156,6 +156,11 @@ class IPCBackend : public Backend { */ void *pAta_pool{nullptr}; + /** + * @brief Handle for raw memory for fence/quiet + */ + int *fence_pool{nullptr}; + protected: /** * @copydoc Backend::dump_backend_stats() @@ -203,6 +208,11 @@ class IPCBackend : public Backend { */ void roc_shmem_collective_init(); + /** + * @brief Allocate buffer for fence/quiet operation + */ + void setup_fence_buffer(); + private: /** * @brief Proxy for the default context diff --git a/projects/rocshmem/src/ipc/context_ipc_device.cpp b/projects/rocshmem/src/ipc/context_ipc_device.cpp index 6fbb362d73..036f5cf77b 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.cpp @@ -47,6 +47,9 @@ __host__ IPCContext::IPCContext(Backend *b) barrier_sync = backend->barrier_sync; g_ret = bp->g_ret; atomic_base_ptr = bp->atomic_ret->atomic_base_ptr; + fence_pool = backend->fence_pool; + + orders_.store = detail::atomic::rocshmem_memory_order::memory_order_seq_cst; } __device__ void IPCContext::threadfence_system() { @@ -85,12 +88,17 @@ __device__ void IPCContext::getmem_nbi(void *dest, const void *source, } __device__ void IPCContext::fence() { + for (int i{0}; i < num_pes; i++) { + detail::atomic::store(&fence_pool[i], 1, orders_); + } } __device__ void IPCContext::fence(int pe) { + detail::atomic::store(&fence_pool[pe], 1, orders_); } __device__ void IPCContext::quiet() { + fence(); } __device__ void *IPCContext::shmem_ptr(const void *dest, int pe) { diff --git a/projects/rocshmem/src/ipc/context_ipc_device.hpp b/projects/rocshmem/src/ipc/context_ipc_device.hpp index 4cc6d08d56..450bb2a3cc 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.hpp @@ -24,6 +24,7 @@ #define LIBRARY_SRC_IPC_CONTEXT_DEVICE_HPP_ #include "../context.hpp" +#include "../atomic.hpp" namespace rocshmem { @@ -232,6 +233,12 @@ class IPCContext : public Context { //Temporary scratchpad memory used by internal barrier algorithms. int64_t *barrier_sync{nullptr}; + + //Struct defining memory ordering for atomic operations. + detail::atomic::rocshmem_memory_orders orders_{}; + + //Buffer to perform Atomic store to enforce memory ordering + int *fence_pool{nullptr}; }; } // namespace rocshmem From 37b1de86cd894e4e7e886b0377c472b59381ad0c Mon Sep 17 00:00:00 2001 From: avinashkethineedi Date: Fri, 4 Oct 2024 17:56:15 +0000 Subject: [PATCH 39/46] Add team information to the context * Update roc_shmem_ctx_fence API to use team-relative PE numbering * Update backend to populate team_opaque member of ROC_SHMEM_CTX_DEFAULT (used to store information about the team wrt TEAM_WORLD) [ROCm/rocshmem commit: 92fb1abaf2cb2a8a72332a6dc12bc604167f2931] --- projects/rocshmem/src/gpu_ib/backend_ib.cpp | 3 ++- projects/rocshmem/src/ipc/backend_ipc.cpp | 7 +++++-- projects/rocshmem/src/ipc/context_ipc_device.cpp | 4 ++-- projects/rocshmem/src/ipc/context_ipc_device.hpp | 9 +++++++++ projects/rocshmem/src/ipc/ipc_context_proxy.hpp | 5 +++-- projects/rocshmem/src/reverse_offload/backend_ro.cpp | 4 +++- .../rocshmem/src/reverse_offload/context_proxy.hpp | 5 +++-- projects/rocshmem/src/roc_shmem_gpu.cpp | 10 ++++++---- 8 files changed, 33 insertions(+), 14 deletions(-) diff --git a/projects/rocshmem/src/gpu_ib/backend_ib.cpp b/projects/rocshmem/src/gpu_ib/backend_ib.cpp index 9ae3acd482..87b02473d3 100644 --- a/projects/rocshmem/src/gpu_ib/backend_ib.cpp +++ b/projects/rocshmem/src/gpu_ib/backend_ib.cpp @@ -296,7 +296,8 @@ void GPUIBBackend::setup_default_ctx() { CHECK_HIP(hipGetSymbolAddress(reinterpret_cast(&symbol_address), HIP_SYMBOL(ROC_SHMEM_CTX_DEFAULT))); - roc_shmem_ctx_t ctx_default_host{default_ctx_, nullptr}; + TeamInfo *tinfo = team_tracker.get_team_world()->tinfo_wrt_world; + roc_shmem_ctx_t ctx_default_host{default_ctx_, tinfo}; hipStream_t stream; CHECK_HIP(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); diff --git a/projects/rocshmem/src/ipc/backend_ipc.cpp b/projects/rocshmem/src/ipc/backend_ipc.cpp index b09adee2ff..70a9295486 100644 --- a/projects/rocshmem/src/ipc/backend_ipc.cpp +++ b/projects/rocshmem/src/ipc/backend_ipc.cpp @@ -82,10 +82,11 @@ IPCBackend::IPCBackend(MPI_Comm comm) allocate_atomic_region(&bp->atomic_ret, MAX_NUM_BLOCKS); - default_context_proxy_ = IPCDefaultContextProxyT(this); - setup_team_world(); + TeamInfo *tinfo = team_tracker.get_team_world()->tinfo_wrt_world; + default_context_proxy_ = IPCDefaultContextProxyT(this, tinfo); + roc_shmem_collective_init(); setup_fence_buffer(); @@ -143,6 +144,8 @@ __device__ bool IPCBackend::create_ctx(int64_t options, roc_shmem_ctx_t *ctx) { ctx_ = pop_result.value; ctx->ctx_opaque = ctx_; + + ctx_->tinfo = reinterpret_cast(ctx->team_opaque); return true; } diff --git a/projects/rocshmem/src/ipc/context_ipc_device.cpp b/projects/rocshmem/src/ipc/context_ipc_device.cpp index 036f5cf77b..4d891cbc53 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.cpp @@ -88,8 +88,8 @@ __device__ void IPCContext::getmem_nbi(void *dest, const void *source, } __device__ void IPCContext::fence() { - for (int i{0}; i < num_pes; i++) { - detail::atomic::store(&fence_pool[i], 1, orders_); + for (int i{0}, j{tinfo->pe_start}; i < tinfo->size; i++, j += tinfo->stride) { + detail::atomic::store(&fence_pool[j], 1, orders_); } } diff --git a/projects/rocshmem/src/ipc/context_ipc_device.hpp b/projects/rocshmem/src/ipc/context_ipc_device.hpp index 450bb2a3cc..48fe5acbdf 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.hpp @@ -25,6 +25,7 @@ #include "../context.hpp" #include "../atomic.hpp" +#include "../team.hpp" namespace rocshmem { @@ -239,6 +240,14 @@ class IPCContext : public Context { //Buffer to perform Atomic store to enforce memory ordering int *fence_pool{nullptr}; + + public: + //TODO(Avinash): + //Make tinfo private variable, it requires changes to the context + //creation API in backend + + //Team information for the team associated with the context + TeamInfo *tinfo{nullptr}; }; } // namespace rocshmem diff --git a/projects/rocshmem/src/ipc/ipc_context_proxy.hpp b/projects/rocshmem/src/ipc/ipc_context_proxy.hpp index 867f199094..87ca4371e0 100644 --- a/projects/rocshmem/src/ipc/ipc_context_proxy.hpp +++ b/projects/rocshmem/src/ipc/ipc_context_proxy.hpp @@ -41,10 +41,11 @@ class IPCDefaultContextProxy { /* * Placement new the memory which is allocated by proxy_ */ - explicit IPCDefaultContextProxy(IPCBackend* backend) : constructed_{true} { + explicit IPCDefaultContextProxy(IPCBackend* backend, TeamInfo *tinfo) + : constructed_{true} { auto ctx{proxy_.get()}; new (ctx) IPCContext(reinterpret_cast(backend)); - roc_shmem_ctx_t local{ctx, nullptr}; + roc_shmem_ctx_t local{ctx, tinfo}; set_internal_ctx(&local); } diff --git a/projects/rocshmem/src/reverse_offload/backend_ro.cpp b/projects/rocshmem/src/reverse_offload/backend_ro.cpp index 1c1bf4645c..96471181e3 100644 --- a/projects/rocshmem/src/reverse_offload/backend_ro.cpp +++ b/projects/rocshmem/src/reverse_offload/backend_ro.cpp @@ -94,7 +94,9 @@ ROBackend::ROBackend(MPI_Comm comm) default_block_handle_proxy_ = DefaultBlockHandleProxyT( bp->g_ret, bp->atomic_ret, &queue_, &ipcImpl, hdp_proxy_.get()); - default_context_proxy_ = DefaultContextProxyT(this); + + TeamInfo *tinfo = team_tracker.get_team_world()->tinfo_wrt_world; + default_context_proxy_ = DefaultContextProxyT(this, tinfo); block_handle_proxy_ = BlockHandleProxyT(bp->g_ret, bp->atomic_ret, &queue_, &ipcImpl, hdp_proxy_.get()); diff --git a/projects/rocshmem/src/reverse_offload/context_proxy.hpp b/projects/rocshmem/src/reverse_offload/context_proxy.hpp index d89281d921..4ae94d3851 100644 --- a/projects/rocshmem/src/reverse_offload/context_proxy.hpp +++ b/projects/rocshmem/src/reverse_offload/context_proxy.hpp @@ -42,10 +42,11 @@ class DefaultContextProxy { /* * Placement new the memory which is allocated by proxy_ */ - explicit DefaultContextProxy(ROBackend* backend) : constructed_{true} { + explicit DefaultContextProxy(ROBackend* backend, TeamInfo *tinfo) + : constructed_{true} { auto ctx{proxy_.get()}; new (ctx) ROContext(reinterpret_cast(backend), -1); - roc_shmem_ctx_t local{ctx, nullptr}; + roc_shmem_ctx_t local{ctx, tinfo}; set_internal_ctx(&local); } diff --git a/projects/rocshmem/src/roc_shmem_gpu.cpp b/projects/rocshmem/src/roc_shmem_gpu.cpp index 086a6deed1..1afd5b10ff 100644 --- a/projects/rocshmem/src/roc_shmem_gpu.cpp +++ b/projects/rocshmem/src/roc_shmem_gpu.cpp @@ -262,9 +262,9 @@ __device__ int roc_shmem_wg_ctx_create(long option, roc_shmem_ctx_t *ctx) { GPU_DPRINTF("Function: roc_shmem_ctx_create\n"); bool result{true}; if (get_flat_block_id() == 0) { + ctx->team_opaque = reinterpret_cast(ROC_SHMEM_CTX_DEFAULT.team_opaque); device_backend_proxy->create_ctx(option, ctx); reinterpret_cast(ctx->ctx_opaque)->setFence(option); - ctx->team_opaque = nullptr; } __syncthreads(); return result == true ? 0 : -1; @@ -279,11 +279,11 @@ __device__ int roc_shmem_wg_team_create_ctx(roc_shmem_team_t team, long options, bool result{true}; if (get_flat_block_id() == 0) { - result = device_backend_proxy->create_ctx(options, ctx); - reinterpret_cast(ctx->ctx_opaque)->setFence(options); Team *team_obj{get_internal_team(team)}; TeamInfo *info_wrt_world = team_obj->tinfo_wrt_world; ctx->team_opaque = info_wrt_world; + result = device_backend_proxy->create_ctx(options, ctx); + reinterpret_cast(ctx->ctx_opaque)->setFence(options); } __syncthreads(); @@ -412,7 +412,9 @@ __device__ void roc_shmem_ctx_fence(roc_shmem_ctx_t ctx) { __device__ void roc_shmem_ctx_fence(roc_shmem_ctx_t ctx, int pe) { GPU_DPRINTF("Function: roc_shmem_ctx_fence\n"); - get_internal_ctx(ctx)->fence(pe); + int pe_in_world = translate_pe(ctx, pe); + + get_internal_ctx(ctx)->fence(pe_in_world); } __device__ void roc_shmem_ctx_quiet(roc_shmem_ctx_t ctx) { From 7eec77ea17d03c2ba892cca6f2ccce73e60bc409 Mon Sep 17 00:00:00 2001 From: avinashkethineedi Date: Tue, 8 Oct 2024 18:12:07 +0000 Subject: [PATCH 40/46] Add script to run unit tests [ROCm/rocshmem commit: c1bcf336b431d3dce6dfd77d2774ecc207f562fd] --- .../rocshmem/scripts/unit_tests/driver.sh | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 projects/rocshmem/scripts/unit_tests/driver.sh diff --git a/projects/rocshmem/scripts/unit_tests/driver.sh b/projects/rocshmem/scripts/unit_tests/driver.sh new file mode 100755 index 0000000000..8eea7daef7 --- /dev/null +++ b/projects/rocshmem/scripts/unit_tests/driver.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Function to display help information +function display_help { + echo "Usage:" + echo " $0 binary_name all # Runs all standard tests" + echo " $0 binary_name custom # Runs custom test configuration" + echo + echo "Arguments:" + echo " binary_name: Name of the binary to run." + echo " all: Executes predefined test configurations." + echo " custom: Executes a test with custom MPI ranks and GTest filter." + echo " ranks: Number of MPI ranks (required for custom mode)." + echo " filter: GTest filter string (required for custom mode)." + echo +} + +# Validate number of arguments for each mode +if [[ "$#" -lt 2 ]] || + { [[ "$2" == "all" ]] && [[ "$#" -ne 2 ]]; } || + { [[ "$2" == "custom" ]] && [[ "$#" -ne 4 ]]; }; then + display_help + exit 1 +fi + +binary_name=$1 +mode=$2 +timestamp=$(date "+%Y-%m-%d-%H:%M:%S") +log_file="unit_tests_${timestamp}.log" + +# Function to execute mpirun command +function run_mpirun { + local np=$1 + local gtest_filter=$2 + echo "mpirun -np $np $binary_name --gtest_filter='$gtest_filter'" | tee -a "$log_file" + mpirun -np "$np" "$binary_name" --gtest_filter="$gtest_filter" >> "$log_file" 2>&1 +} + +# Processing modes +case $mode in + all) + run_mpirun 4 "-IPCImplSimpleCoarseTestFixture.*:IPCImplSimpleFineTestFixture.*" + run_mpirun 2 "IPCImplSimpleCoarseTestFixture.*:IPCImplSimpleFineTestFixture.*" + ;; + custom) + # Check if ranks is a positive integer + if [[ "$3" -le 1 ]]; then + echo "Error: 'ranks' must be a positive integer." + display_help + exit 1 + fi + run_mpirun $3 $4 + ;; + *) + echo "Error: Invalid mode '$mode'." | tee -a "$log_file" + display_help + exit 1 + ;; +esac + +echo "Tests Completed" +echo "log file: '$log_file'" From 3cba9ccd42b21621cab1adef9ed9490ae8c6b929 Mon Sep 17 00:00:00 2001 From: Yiltan Hassan Temucin Date: Tue, 1 Oct 2024 13:27:45 -0700 Subject: [PATCH 41/46] added .gitignore, we do not want to include the build directory in our commits [ROCm/rocshmem commit: 120453c75c890c324f8d90d0678fd9ae7b9dc095] --- projects/rocshmem/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 projects/rocshmem/.gitignore diff --git a/projects/rocshmem/.gitignore b/projects/rocshmem/.gitignore new file mode 100644 index 0000000000..567609b123 --- /dev/null +++ b/projects/rocshmem/.gitignore @@ -0,0 +1 @@ +build/ From ef571f58632752859fbd402d272de4a164dbe98b Mon Sep 17 00:00:00 2001 From: Yiltan Hassan Temucin Date: Wed, 2 Oct 2024 09:20:07 -0700 Subject: [PATCH 42/46] fixed barrier issue on MI250X [ROCm/rocshmem commit: e2f6a65284bc335ef9a684e21bbf02828b17433e] --- projects/rocshmem/src/ipc/context_ipc_device_coll.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp index 804be2efbf..1465b8ffe0 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp @@ -35,6 +35,9 @@ __device__ void IPCContext::internal_direct_barrier(int pe, int PE_start, if (pe == PE_start) { // Go through all PE offsets (except current offset = 0) // and wait until they all reach +#if defined(__gfx90a__) + __threadfence_system(); +#endif /* __gfx90a__ */ for (size_t i = 1; i < n_pes; i++) { wait_until(&pSync[i], ROC_SHMEM_CMP_EQ, flag_val); pSync[i] = ROC_SHMEM_SYNC_VALUE; @@ -49,6 +52,9 @@ __device__ void IPCContext::internal_direct_barrier(int pe, int PE_start, // Mark current PE offset as reached size_t pe_offset = (pe - PE_start) / stride; put_nbi(&pSync[pe_offset], &flag_val, 1, PE_start); +#if defined(__gfx90a__) + __threadfence_system(); +#endif /* __gfx90a__ */ wait_until(&pSync[0], ROC_SHMEM_CMP_EQ, flag_val); pSync[0] = ROC_SHMEM_SYNC_VALUE; threadfence_system(); From 45976b23ae5b73fe469ec46fac26e5fc9f92be10 Mon Sep 17 00:00:00 2001 From: Yiltan Hassan Temucin Date: Wed, 9 Oct 2024 13:10:50 -0700 Subject: [PATCH 43/46] Fix initialization order bug [ROCm/rocshmem commit: 1baa071edf525572fe4de6c03883ed8fb69047a1] --- projects/rocshmem/src/ipc/backend_ipc.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/rocshmem/src/ipc/backend_ipc.cpp b/projects/rocshmem/src/ipc/backend_ipc.cpp index 70a9295486..7475fb2c8b 100644 --- a/projects/rocshmem/src/ipc/backend_ipc.cpp +++ b/projects/rocshmem/src/ipc/backend_ipc.cpp @@ -56,18 +56,18 @@ int get_ls_non_zero_bit(char *bitmask, int mask_length) { IPCBackend::IPCBackend(MPI_Comm comm) : Backend() { type = BackendType::IPC_BACKEND; - + if (auto maximum_num_contexts_str = getenv("ROC_SHMEM_MAX_NUM_CONTEXTS")) { std::stringstream sstream(maximum_num_contexts_str); sstream >> maximum_num_contexts_; } init_mpi_once(comm); - + initIPC(); - + auto *bp{ipc_backend_proxy.get()}; - + bp->heap_ptr = &heap; /* Initialize the host interface */ @@ -85,7 +85,6 @@ IPCBackend::IPCBackend(MPI_Comm comm) setup_team_world(); TeamInfo *tinfo = team_tracker.get_team_world()->tinfo_wrt_world; - default_context_proxy_ = IPCDefaultContextProxyT(this, tinfo); roc_shmem_collective_init(); @@ -93,8 +92,9 @@ IPCBackend::IPCBackend(MPI_Comm comm) teams_init(); + default_context_proxy_ = IPCDefaultContextProxyT(this, tinfo); + setup_ctxs(); - } IPCBackend::~IPCBackend() { @@ -108,7 +108,7 @@ IPCBackend::~IPCBackend() { * Free the atomic_ret array. */ CHECK_HIP(hipFree(bp->atomic_ret->atomic_base_ptr)); - + // TODO(Avinash) Free g_ret // delete host_interface; @@ -402,4 +402,4 @@ void IPCBackend::teams_init() { NET_CHECK(MPI_Barrier(thread_comm)); } -} // namespace rocshmem \ No newline at end of file +} // namespace rocshmem From caa6d356c05b157dcf3b166dc78ce80bd03cbe98 Mon Sep 17 00:00:00 2001 From: Yiltan Hassan Temucin Date: Tue, 1 Oct 2024 13:26:43 -0700 Subject: [PATCH 44/46] Added Cooperative Groups configure option and header [ROCm/rocshmem commit: 63667a3167cd4c1a774331ee8985a708faa77e87] --- projects/rocshmem/CMakeLists.txt | 1 + projects/rocshmem/cmake/config.h.in | 1 + .../scripts/build_configs/ipc_single_cg | 31 +++++++++++++++++++ .../src/ipc/context_ipc_device_coll.cpp | 15 +++++++++ .../tests/functional_tests/sync_tester.cpp | 12 +++++++ 5 files changed, 60 insertions(+) create mode 100755 projects/rocshmem/scripts/build_configs/ipc_single_cg diff --git a/projects/rocshmem/CMakeLists.txt b/projects/rocshmem/CMakeLists.txt index f9c3d83d10..5cf74fb6cb 100644 --- a/projects/rocshmem/CMakeLists.txt +++ b/projects/rocshmem/CMakeLists.txt @@ -68,6 +68,7 @@ option(USE_FUNC_CALL "Force compiler to use function calls on library API" OFF) option(USE_SHARED_CTX "Request support for shared ctx between WG" OFF) option(USE_SINGLE_NODE "Enable single node support only." OFF) option(USE_HOST_SIDE_HDP_FLUSH "Use a polling thread to flush the HDP cache on the host." OFF) +option(USE_COOPERATIVE_GROUPS "Use cooperative groups for internal syncronization" OFF) option(BUILD_FUNCTIONAL_TESTS "Build the functional tests" ON) option(BUILD_SOS_TESTS "Build the host-facing tests" OFF) option(BUILD_UNIT_TESTS "Build the unit tests" ON) diff --git a/projects/rocshmem/cmake/config.h.in b/projects/rocshmem/cmake/config.h.in index 8067651a01..21619a6c3e 100644 --- a/projects/rocshmem/cmake/config.h.in +++ b/projects/rocshmem/cmake/config.h.in @@ -14,3 +14,4 @@ #cmakedefine USE_FUNC_CALL #cmakedefine USE_SINGLE_NODE #cmakedefine USE_HOST_SIDE_HDP_FLUSH +#cmakedefine USE_COOPERATIVE_GROUPS diff --git a/projects/rocshmem/scripts/build_configs/ipc_single_cg b/projects/rocshmem/scripts/build_configs/ipc_single_cg new file mode 100755 index 0000000000..c0904bad93 --- /dev/null +++ b/projects/rocshmem/scripts/build_configs/ipc_single_cg @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. + +if [ -z $1 ] +then + install_path=~/rocshmem +else + install_path=$1 +fi + +src_path=$(dirname "$(realpath $0)")/../../ + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$install_path \ + -DCMAKE_VERBOSE_MAKEFILE=OFF \ + -DDEBUG=OFF \ + -DPROFILE=OFF \ + -DUSE_GPU_IB=OFF \ + -DUSE_RO=OFF \ + -DUSE_DC=OFF \ + -DUSE_IPC=ON \ + -DUSE_COHERENT_HEAP=ON \ + -DUSE_THREADS=OFF \ + -DUSE_WF_COAL=OFF \ + -DUSE_SINGLE_NODE=ON \ + -DUSE_HOST_SIDE_HDP_FLUSH=OFF \ + -DUSE_COOPERATIVE_GROUPS=ON \ + $src_path +cmake --build . --parallel 8 +cmake --install . diff --git a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp index 1465b8ffe0..5e1f950c3b 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp @@ -26,6 +26,11 @@ #include "../util.hpp" #include "ipc_team.hpp" +#ifdef USE_COOPERATIVE_GROUPS +#include +namespace cg = cooperative_groups; +#endif /* USE_COOPERATIVE_GROUPS */ + namespace rocshmem { __device__ void IPCContext::internal_direct_barrier(int pe, int PE_start, @@ -84,8 +89,14 @@ __device__ void IPCContext::internal_atomic_barrier(int pe, int PE_start, // Uses PE values that are relative to world __device__ void IPCContext::internal_sync(int pe, int PE_start, int stride, int PE_size, int64_t *pSync) { +#ifdef USE_COOPERATIVE_GROUPS + cg::grid_group grid = cg::this_grid(); + grid.sync(); + if (0 == grid.thread_rank()) { +#else __syncthreads(); if (is_thread_zero_in_block()) { +#endif /* USE_COOPERATIVE_GROUPS */ if (PE_size < 64) { internal_direct_barrier(pe, PE_start, stride, PE_size, pSync); } else { @@ -93,7 +104,11 @@ __device__ void IPCContext::internal_sync(int pe, int PE_start, int stride, } } __threadfence(); +#ifdef USE_COOPERATIVE_GROUPS + grid.sync(); +#else __syncthreads(); +#endif /* USE_COOPERATIVE_GROUPS */ } __device__ void IPCContext::sync(roc_shmem_team_t team) { diff --git a/projects/rocshmem/tests/functional_tests/sync_tester.cpp b/projects/rocshmem/tests/functional_tests/sync_tester.cpp index 54d4a99f99..1afecaa45c 100644 --- a/projects/rocshmem/tests/functional_tests/sync_tester.cpp +++ b/projects/rocshmem/tests/functional_tests/sync_tester.cpp @@ -83,9 +83,21 @@ void SyncTester::launchKernel(dim3 gridSize, dim3 blockSize, int loop, roc_shmem_team_split_strided(ROC_SHMEM_TEAM_WORLD, 0, 1, n_pes, nullptr, 0, &team_sync_world_dup); +#ifdef USE_COOPERATIVE_GROUPS + void* kernelParams[] = {(void*)&loop, + (void*)&args.skip, + (void*)&timer, + (void*)&_type, + (void*)&_shmem_context, + (void*)&team_sync_world_dup}; + + CHECK_HIP(hipLaunchCooperativeKernel(SyncTest, gridSize, blockSize, + kernelParams, shared_bytes, stream)); +#else hipLaunchKernelGGL(SyncTest, gridSize, blockSize, shared_bytes, stream, loop, args.skip, timer, _type, _shmem_context, team_sync_world_dup); +#endif /* USE_COOPERATIVE_GROUPS */ num_msgs = (loop + args.skip) * gridSize.x; num_timed_msgs = loop; From 8334214b98fd91a4c75c24d1141f6cc7774755f7 Mon Sep 17 00:00:00 2001 From: Yiltan Hassan Temucin Date: Mon, 7 Oct 2024 06:06:42 -0700 Subject: [PATCH 45/46] added notifier->sync() when we are not using cooperative groups updated scope bug [ROCm/rocshmem commit: b1134e8633ecb1f9670bad7540b22857f1d87892] --- projects/rocshmem/src/ipc/context_ipc_device.hpp | 12 ++++++++++++ .../rocshmem/src/ipc/context_ipc_device_coll.cpp | 15 ++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/projects/rocshmem/src/ipc/context_ipc_device.hpp b/projects/rocshmem/src/ipc/context_ipc_device.hpp index 48fe5acbdf..8046fa2088 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.hpp @@ -27,6 +27,13 @@ #include "../atomic.hpp" #include "../team.hpp" +#ifdef USE_COOPERATIVE_GROUPS +#include +namespace cg = cooperative_groups; +#else +#include "../memory/notifier.hpp" +#endif /* USE_COOPERATIVE_GROUPS */ + namespace rocshmem { class IPCContext : public Context { @@ -235,6 +242,11 @@ class IPCContext : public Context { //Temporary scratchpad memory used by internal barrier algorithms. int64_t *barrier_sync{nullptr}; +#ifndef USE_COOPERATIVE_GROUPS + using NOTIFIER_PROXY_T = NotifierProxy; + NOTIFIER_PROXY_T notifier_{}; +#endif /* NOT DEFINED: USE_COOPERATIVE_GROUPS */ + //Struct defining memory ordering for atomic operations. detail::atomic::rocshmem_memory_orders orders_{}; diff --git a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp index 5e1f950c3b..02bb3ffcec 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp @@ -26,11 +26,6 @@ #include "../util.hpp" #include "ipc_team.hpp" -#ifdef USE_COOPERATIVE_GROUPS -#include -namespace cg = cooperative_groups; -#endif /* USE_COOPERATIVE_GROUPS */ - namespace rocshmem { __device__ void IPCContext::internal_direct_barrier(int pe, int PE_start, @@ -92,11 +87,13 @@ __device__ void IPCContext::internal_sync(int pe, int PE_start, int stride, #ifdef USE_COOPERATIVE_GROUPS cg::grid_group grid = cg::this_grid(); grid.sync(); - if (0 == grid.thread_rank()) { + if (0 == grid.thread_rank()) #else - __syncthreads(); - if (is_thread_zero_in_block()) { + auto notifier{notifier_.get()}; + notifier->sync(); + if (is_thread_zero_in_block() && is_block_zero_in_grid()) #endif /* USE_COOPERATIVE_GROUPS */ + { if (PE_size < 64) { internal_direct_barrier(pe, PE_start, stride, PE_size, pSync); } else { @@ -107,7 +104,7 @@ __device__ void IPCContext::internal_sync(int pe, int PE_start, int stride, #ifdef USE_COOPERATIVE_GROUPS grid.sync(); #else - __syncthreads(); + notifier->sync(); #endif /* USE_COOPERATIVE_GROUPS */ } From 17323323f81f14cbcb14c2caf4181dfe0425d832 Mon Sep 17 00:00:00 2001 From: Yiltan Hassan Temucin Date: Thu, 10 Oct 2024 06:45:43 -0700 Subject: [PATCH 46/46] fixed notifier bug [ROCm/rocshmem commit: 509277c034a5def013b89b35ff4d8ec1638447e1] --- projects/rocshmem/src/ipc/backend_ipc.hpp | 12 ++++++++++-- projects/rocshmem/src/ipc/context_ipc_device.cpp | 4 ++++ projects/rocshmem/src/ipc/context_ipc_device.hpp | 3 +-- .../rocshmem/src/ipc/context_ipc_device_coll.cpp | 7 +++---- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/projects/rocshmem/src/ipc/backend_ipc.hpp b/projects/rocshmem/src/ipc/backend_ipc.hpp index 7d00e6cf07..de8cb9c617 100644 --- a/projects/rocshmem/src/ipc/backend_ipc.hpp +++ b/projects/rocshmem/src/ipc/backend_ipc.hpp @@ -32,6 +32,10 @@ #include "ipc_context_proxy.hpp" #include "../ipc_policy.hpp" +#ifndef USE_COOPERATIVE_GROUPS +#include "../memory/notifier.hpp" +#endif /* NOT DEFINED: USE_COOPERATIVE_GROUPS */ + namespace rocshmem { class IPCBackend : public Backend { @@ -104,7 +108,7 @@ class IPCBackend : public Backend { TeamInfo *team_info_wrt_world, int num_pes, int my_pe_in_new_team, MPI_Comm team_comm, roc_shmem_team_t *new_team) override; - + /** * @copydoc Backend::team_destroy(roc_shmem_team_t) */ @@ -161,6 +165,10 @@ class IPCBackend : public Backend { */ int *fence_pool{nullptr}; +#ifndef USE_COOPERATIVE_GROUPS + NotifierProxy notifier_{}; +#endif /* NOT DEFINED: USE_COOPERATIVE_GROUPS */ + protected: /** * @copydoc Backend::dump_backend_stats() @@ -259,4 +267,4 @@ class IPCBackend : public Backend { } // namespace rocshmem -#endif // LIBRARY_SRC_IPC_BACKEND_HPP_ \ No newline at end of file +#endif // LIBRARY_SRC_IPC_BACKEND_HPP_ diff --git a/projects/rocshmem/src/ipc/context_ipc_device.cpp b/projects/rocshmem/src/ipc/context_ipc_device.cpp index 4d891cbc53..d208a070fb 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.cpp @@ -50,6 +50,10 @@ __host__ IPCContext::IPCContext(Backend *b) fence_pool = backend->fence_pool; orders_.store = detail::atomic::rocshmem_memory_order::memory_order_seq_cst; + +#ifndef USE_COOPERATIVE_GROUPS + notifier_ = backend->notifier_.get(); +#endif /* NOT DEFINED: USE_COOPERATIVE_GROUPS */ } __device__ void IPCContext::threadfence_system() { diff --git a/projects/rocshmem/src/ipc/context_ipc_device.hpp b/projects/rocshmem/src/ipc/context_ipc_device.hpp index 8046fa2088..065d37cec4 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device.hpp +++ b/projects/rocshmem/src/ipc/context_ipc_device.hpp @@ -243,8 +243,7 @@ class IPCContext : public Context { int64_t *barrier_sync{nullptr}; #ifndef USE_COOPERATIVE_GROUPS - using NOTIFIER_PROXY_T = NotifierProxy; - NOTIFIER_PROXY_T notifier_{}; + Notifier *notifier_{nullptr}; #endif /* NOT DEFINED: USE_COOPERATIVE_GROUPS */ //Struct defining memory ordering for atomic operations. diff --git a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp index 02bb3ffcec..f128d76d3f 100644 --- a/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp +++ b/projects/rocshmem/src/ipc/context_ipc_device_coll.cpp @@ -89,9 +89,8 @@ __device__ void IPCContext::internal_sync(int pe, int PE_start, int stride, grid.sync(); if (0 == grid.thread_rank()) #else - auto notifier{notifier_.get()}; - notifier->sync(); - if (is_thread_zero_in_block() && is_block_zero_in_grid()) + notifier_->sync(); + if (0 == get_flat_id()) #endif /* USE_COOPERATIVE_GROUPS */ { if (PE_size < 64) { @@ -104,7 +103,7 @@ __device__ void IPCContext::internal_sync(int pe, int PE_start, int stride, #ifdef USE_COOPERATIVE_GROUPS grid.sync(); #else - notifier->sync(); + notifier_->sync(); #endif /* USE_COOPERATIVE_GROUPS */ }