Add 'projects/rocshmem/' from commit '0496586829058af5cfd7f23acda2a6d0040da584'

git-subtree-dir: projects/rocshmem
git-subtree-mainline: 5fd976da70
git-subtree-split: 0496586829

projects/rocshmem/examples/CMakeLists.txt

@@ -0,0 +1,65 @@
+###############################################################################
+# Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+###############################################################################
+cmake_minimum_required(VERSION 3.16.3 FATAL_ERROR)
+
+###############################################################################
+# PROJECT
+###############################################################################
+include(${CMAKE_SOURCE_DIR}/cmake/setup_project.cmake)
+project(rocshmem_examples VERSION 1.0.0 LANGUAGES CXX)
+
+find_package(MPI)
+find_package(hip REQUIRED PATHS /opt/rocm)
+if (NOT TARGET roc::rocshmem)
+  find_package(rocshmem REQUIRED PATHS /opt/rocm)
+endif()
+
+###############################################################################
+# SOURCES
+###############################################################################
+set(EXAMPLE_SOURCES
+  rocshmem_allreduce_test.cc
+  rocshmem_alltoall_test.cc
+  rocshmem_broadcast_test.cc
+  rocshmem_getmem_test.cc
+  rocshmem_put_signal_test.cc
+)
+
+if (MPI_CXX_FOUND)
+  list(APPEND EXAMPLE_SOURCES
+    rocshmem_init_attr_test.cc)
+endif()
+
+foreach(SOURCE_FILE IN LISTS EXAMPLE_SOURCES)
+  get_filename_component(EXECUTABLE_NAME ${SOURCE_FILE} NAME_WE)
+
+  add_executable(${EXECUTABLE_NAME} ${SOURCE_FILE})
+
+  target_link_libraries(
+    ${EXECUTABLE_NAME}
+    PRIVATE
+      roc::rocshmem
+      $<TARGET_NAME_IF_EXISTS:MPI::MPI_CXX>
+  )
+endforeach()

projects/rocshmem/examples/rocshmem_allreduce_test.cc

@@ -0,0 +1,155 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+/*
+ * First find your offload target, and if xnack is enabled/disabled using
+
+   rocminfo | grep amdgcn
+
+ * It should output a string like so:
+
+   "Name:                    amdgcn-amd-amdhsa--gfx942:sramecc+:xnack-"
+
+ * This lists the offload taret (gfx942) and that xnack is disabled (xnack-).
+ * Therefore, we need to specify --offload-arch=gfx942:xnack- to our link and compile commands.
+ * Please modify the compile and link commands to suit your system
+
+ * To compile:
+   hipcc -c -fgpu-rdc -x hip rocshmem_allreduce_test.cc \
+         --offload-arch=<target>:<xnack>                \
+         -I/opt/rocm/include                            \
+         -I$ROCSHMEM_INSTALL_DIR/include                \
+         -I$OPENMPI_UCX_INSTALL_DIR/include/
+
+ * To link:
+   hipcc -fgpu-rdc --hip-link rocshmem_allreduce_test.o -o rocshmem_allreduce_test  \
+          --offload-arch=<target>:<xnack>                                           \
+          $ROCSHMEM_INSTALL_DIR/lib/librocshmem.a                                   \
+          $OPENMPI_UCX_INSTALL_DIR/lib/libmpi.so                                    \
+          -L/opt/rocm/lib -lamdhip64 -lhsa-runtime64
+
+ * To run:
+   mpirun -np 8 -x ROCSHMEM_MAX_NUM_CONTEXTS=2 ./rocshmem_allreduce_test
+
+ */
+
+#include <rocshmem/rocshmem.hpp>
+
+#include "util.h"
+
+using namespace rocshmem;
+
+__global__ void allreduce_test(int *source, int *dest, size_t nelem,
+        rocshmem_team_t team) {
+    __shared__ rocshmem_ctx_t ctx;
+    int64_t ctx_type = 0;
+
+    rocshmem_wg_ctx_create(ctx_type, &ctx);
+    int num_pes = rocshmem_ctx_n_pes(ctx);
+
+    rocshmem_ctx_int_sum_reduce_wg(ctx, team, dest, source, nelem);
+
+    rocshmem_ctx_quiet(ctx);
+    __syncthreads();
+
+    rocshmem_wg_ctx_destroy(&ctx);
+}
+
+static void init_sendbuf (int *source, int nelem, int my_pe)
+{
+    for (int i = 0; i < nelem; i++) {
+        source[i] = my_pe + i%9;
+    }
+}
+
+static bool check_recvbuf(int *dest, int nelem, int my_pe, int npes)
+{
+    bool res=true;
+    int expected = npes * (npes -1) / 2;
+
+    for (int i = 0; i < nelem; i++) {
+        int result = expected + npes * (i%9);
+        if (dest[i] != result) {
+            res = false;
+#ifdef VERBOSE
+            printf("recvbuf[%d] = %d expected %d \n", i, dest[i], result);
+#endif
+        }
+    }
+
+    return res;
+}
+
+#define MAX_ELEM 256
+
+int main (int argc, char **argv)
+{
+    int nelem = MAX_ELEM;
+
+    if (argc > 1) {
+        nelem = atoi(argv[1]);
+    }
+
+    CHECK_HIP(hipSetDevice(get_launcher_local_rank()));
+
+    rocshmem_init();
+
+    int my_pe = rocshmem_my_pe();
+    int npes =  rocshmem_n_pes();
+
+    int *source = (int *)rocshmem_malloc(nelem * sizeof(int));
+    int *dest = (int *)rocshmem_malloc(nelem * sizeof(int));
+    if (NULL == source || NULL == dest) {
+        std::cout << "Error allocating memory from symmetric heap" << std::endl;
+        std::cout << "source: " << source << ", dest: " << dest << ", size: "
+          << sizeof(int) * nelem << std::endl;
+        rocshmem_global_exit(1);
+    }
+
+    init_sendbuf(source, nelem, my_pe);
+    for (int i=0; i<nelem; i++) {
+        dest[i] = -1;
+    }
+
+    rocshmem_team_t team_reduce_world_dup;
+    team_reduce_world_dup = ROCSHMEM_TEAM_INVALID;
+    rocshmem_team_split_strided(ROCSHMEM_TEAM_WORLD, 0, 1, npes, nullptr, 0,
+                               &team_reduce_world_dup);
+
+    CHECK_HIP(hipDeviceSynchronize());
+
+    int threadsPerBlock=256;
+    allreduce_test<<<dim3(1), dim3(threadsPerBlock), 0, 0>>>(source, dest,
+                        nelem, team_reduce_world_dup);
+    CHECK_HIP(hipDeviceSynchronize());
+
+    bool pass = check_recvbuf(dest, nelem, my_pe, npes);
+    printf("Test %s \t nelem %d %s\n", argv[0], nelem, pass ? "[PASS]" : "[FAIL]");
+
+    rocshmem_free(source);
+    rocshmem_free(dest);
+
+    rocshmem_finalize();
+    return 0;
+}

projects/rocshmem/examples/rocshmem_alltoall_test.cc

@@ -0,0 +1,161 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+/*
+ * First find your offload target, and if xnack is enabled/disabled using
+
+   rocminfo | grep amdgcn
+
+ * It should output a string like so:
+
+   "Name:                    amdgcn-amd-amdhsa--gfx942:sramecc+:xnack-"
+
+ * This lists the offload taret (gfx942) and that xnack is disabled (xnack-).
+ * Therefore, we need to specify --offload-arch=gfx942:xnack- to our link and compile commands.
+ * Please modify the compile and link commands to suit your system
+
+ * To compile:
+   hipcc -c -fgpu-rdc -x hip rocshmem_alltoall_test.cc  \
+         --offload-arch=<target>:<xnack>                \
+         -I/opt/rocm/include                            \
+         -I$ROCSHMEM_INSTALL_DIR/include                \
+         -I$OPENMPI_UCX_INSTALL_DIR/include/
+
+ * To link:
+   hipcc -fgpu-rdc --hip-link rocshmem_alltoall_test.o -o rocshmem_alltoall_test   \
+         --offload-arch=<target>:<xnack>                                           \
+         $ROCSHMEM_INSTALL_DIR/lib/librocshmem.a                                   \
+         $OPENMPI_UCX_INSTALL_DIR/lib/libmpi.so                                    \
+         -L/opt/rocm/lib -lamdhip64 -lhsa-runtime64
+
+ * To run:
+    mpirun -np 8 -x ROCSHMEM_MAX_NUM_CONTEXTS=2 ./rocshmem_alltoall_test
+
+ */
+
+#include <rocshmem/rocshmem.hpp>
+
+#include "util.h"
+
+using namespace rocshmem;
+
+__global__ void alltoall_test(int *source, int *dest, size_t nelem,
+        rocshmem_team_t team) {
+    __shared__ rocshmem_ctx_t ctx;
+    int64_t ctx_type = 0;
+
+    rocshmem_wg_ctx_create(ctx_type, &ctx);
+    int num_pes = rocshmem_ctx_n_pes(ctx);
+
+    rocshmem_ctx_int_alltoall_wg(ctx, team, dest, source, nelem);
+
+    rocshmem_ctx_quiet(ctx);
+    __syncthreads();
+
+    rocshmem_wg_ctx_destroy(&ctx);
+}
+
+static void init_sendbuf (int *source, int nelem, int my_pe, int npes)
+{
+    for (int pe = 0; pe < npes; pe++) {
+        for (int i = 0; i < nelem; i++) {
+            int idx = (pe * nelem) + i;
+            source[idx] = my_pe + pe;
+        }
+    }
+}
+
+static bool check_recvbuf(int *dest, int nelem, int my_pe, int npes)
+{
+    bool res=true;
+
+    for(int pe = 0; pe < npes; pe++) {
+        for(int i = 0; i < nelem; i++) {
+            int idx = (pe * nelem) + i;
+            int result = my_pe + pe;
+            if (dest[idx] != result) {
+                res = false;
+#ifdef VERBOSE
+                printf("recvbuf[%d] = %d expected %d \n", i, dest[i], result);
+#endif
+            }
+        }
+    }
+
+    return res;
+}
+
+#define MAX_ELEM 256
+
+int main (int argc, char **argv)
+{
+    int nelem = MAX_ELEM;
+
+    if (argc > 1) {
+        nelem = atoi(argv[1]);
+    }
+
+    CHECK_HIP(hipSetDevice(get_launcher_local_rank()));
+
+    rocshmem_init();
+
+    int my_pe = rocshmem_my_pe();
+    int npes =  rocshmem_n_pes();
+
+    int *source = (int *)rocshmem_malloc(nelem * npes * sizeof(int));
+    int *dest = (int *)rocshmem_malloc(nelem * npes * sizeof(int));
+    if (NULL == source || NULL == dest) {
+        std::cout << "Error allocating memory from symmetric heap" << std::endl;
+        std::cout << "source: " << source << ", dest: " << dest << ", size: "
+          << sizeof(int) * nelem * npes << std::endl;
+        rocshmem_global_exit(1);
+    }
+
+    init_sendbuf(source, nelem, my_pe, npes);
+    for (int i = 0; i < nelem * npes; i++) {
+        dest[i] = -1;
+    }
+
+    rocshmem_team_t team_reduce_world_dup;
+    team_reduce_world_dup = ROCSHMEM_TEAM_INVALID;
+    rocshmem_team_split_strided(ROCSHMEM_TEAM_WORLD, 0, 1, npes, nullptr, 0,
+                               &team_reduce_world_dup);
+
+    CHECK_HIP(hipDeviceSynchronize());
+
+    int threadsPerBlock=256;
+    alltoall_test<<<dim3(1), dim3(threadsPerBlock), 0, 0>>>(source, dest,
+                        nelem, team_reduce_world_dup);
+    CHECK_HIP(hipDeviceSynchronize());
+
+    bool pass = check_recvbuf(dest, nelem, my_pe, npes);
+
+    printf("Test %s \t nelem %d %s\n", argv[0], nelem, pass ? "[PASS]" : "[FAIL]");
+
+    rocshmem_free(source);
+    rocshmem_free(dest);
+
+    rocshmem_finalize();
+    return 0;
+}

projects/rocshmem/examples/rocshmem_broadcast_test.cc

@@ -0,0 +1,156 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+/*
+ * First find your offload target, and if xnack is enabled/disabled using
+
+   rocminfo | grep amdgcn
+
+ * It should output a string like so:
+
+   "Name:                    amdgcn-amd-amdhsa--gfx942:sramecc+:xnack-"
+
+ * This lists the offload taret (gfx942) and that xnack is disabled (xnack-).
+ * Therefore, we need to specify --offload-arch=gfx942:xnack- to our link and compile commands.
+ * Please modify the compile and link commands to suit your system
+
+ * To compile:
+   hipcc -c -fgpu-rdc -x hip rocshmem_broadcast_test.cc \
+         --offload-arch=<target>:<xnack>                \
+         -I/opt/rocm/include                            \
+         -I$ROCSHMEM_INSTALL_DIR/include                \
+         -I$OPENMPI_UCX_INSTALL_DIR/include/
+
+ * To link:
+   hipcc -fgpu-rdc --hip-link rocshmem_broadcast_test.o -o rocshmem_broadcast_test \
+         --offload-arch=<target>:<xnack>                                           \
+         $ROCSHMEM_INSTALL_DIR/lib/librocshmem.a                                   \
+         $OPENMPI_UCX_INSTALL_DIR/lib/libmpi.so                                    \
+         -L/opt/rocm/lib -lamdhip64 -lhsa-runtime64
+
+ * To run:
+   mpirun -np 8 -x ROCSHMEM_MAX_NUM_CONTEXTS=2 ./rocshmem_broadcast_test
+
+ */
+
+#include <rocshmem/rocshmem.hpp>
+
+#include "util.h"
+
+using namespace rocshmem;
+
+__global__ void broadcast_test(int *source, int *dest, size_t nelem,
+        int root, rocshmem_team_t team) {
+    __shared__ rocshmem_ctx_t ctx;
+    int64_t ctx_type = 0;
+
+    rocshmem_wg_ctx_create(ctx_type, &ctx);
+    int num_pes = rocshmem_ctx_n_pes(ctx);
+
+    rocshmem_ctx_int_broadcast_wg(ctx, team, dest, source, nelem, root);
+
+    rocshmem_ctx_quiet(ctx);
+    __syncthreads();
+
+    rocshmem_wg_ctx_destroy(&ctx);
+}
+
+static void init_sendbuf(int *source, int nelem, int my_pe)
+{
+    for (int i = 0; i < nelem; i++) {
+        source[i] = i;
+    }
+}
+
+static bool check_recvbuf(int *dest, int nelem, int my_pe, int npes)
+{
+    bool res=true;
+
+    for (int i = 0; i < npes; i++) {
+        if (dest[i] != i) {
+            res = false;
+#ifdef VERBOSE
+            printf("PE: %d, dest[%d] = %d, expected %d \n", my_pe, i, dest[i], i);
+#endif
+        }
+    }
+
+    return res;
+}
+
+#define MAX_ELEM 256
+
+int main(int argc, char **argv)
+{
+    int nelem = MAX_ELEM;
+
+    if (argc > 1) {
+        nelem = atoi(argv[1]);
+    }
+
+    CHECK_HIP(hipSetDevice(get_launcher_local_rank()));
+
+    rocshmem_init();
+
+    int my_pe = rocshmem_my_pe();
+    int npes =  rocshmem_n_pes();
+
+    int *source = (int *)rocshmem_malloc(nelem * sizeof(int));
+    int *dest = (int *)rocshmem_malloc(nelem * sizeof(int));
+    if (NULL == source || NULL == dest) {
+        std::cout << "Error allocating memory from symmetric heap" << std::endl;
+        std::cout << "source: " << source << ", dest: " << dest << ", size: "
+          << sizeof(int) * nelem << std::endl;
+        rocshmem_global_exit(1);
+    }
+
+    init_sendbuf(source, nelem, my_pe);
+    for (int i=0; i<nelem; i++) {
+        dest[i] = -1;
+    }
+
+    int root = 0;
+    rocshmem_team_t team_reduce_world_dup;
+    team_reduce_world_dup = ROCSHMEM_TEAM_INVALID;
+    rocshmem_team_split_strided(ROCSHMEM_TEAM_WORLD, 0, 1, npes, nullptr, 0,
+                               &team_reduce_world_dup);
+
+    CHECK_HIP(hipDeviceSynchronize());
+
+    int threadsPerBlock=256;
+    broadcast_test<<<dim3(1), dim3(threadsPerBlock), 0, 0>>>(source, dest,
+                        nelem, root, team_reduce_world_dup);
+    CHECK_HIP(hipDeviceSynchronize());
+
+    if(my_pe != root) {
+        bool pass = check_recvbuf(dest, nelem, my_pe, npes);
+        printf("Test %s \t nelem %d %s\n", argv[0], nelem, pass ? "[PASS]" : "[FAIL]");
+    }
+
+    rocshmem_free(source);
+    rocshmem_free(dest);
+
+    rocshmem_finalize();
+    return 0;
+}

projects/rocshmem/examples/rocshmem_getmem_test.cc

@@ -0,0 +1,129 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+/*
+ * First find your offload target, and if xnack is enabled/disabled using
+
+   rocminfo | grep amdgcn
+
+ * It should output a string like so:
+
+   "Name:                    amdgcn-amd-amdhsa--gfx942:sramecc+:xnack-"
+
+ * This lists the offload taret (gfx942) and that xnack is disabled (xnack-).
+ * Therefore, we need to specify --offload-arch=gfx942:xnack- to our link and compile commands.
+ * Please modify the compile and link commands to suit your system
+
+ * To compile:
+   hipcc -c -fgpu-rdc -x hip rocshmem_getmem_test.cc \
+         --offload-arch=<target>:<xnack>             \
+         -I/opt/rocm/include                         \
+         -I$ROCSHMEM_INSTALL_DIR/include             \
+         -I$OPENMPI_UCX_INSTALL_DIR/include/
+
+ * To link:
+   hipcc -fgpu-rdc --hip-link rocshmem_getmem_test.o -o rocshmem_getmem_test \
+         --offload-arch=<target>:<xnack>                                     \
+         $ROCSHMEM_INSTALL_DIR/lib/librocshmem.a                             \
+         $OPENMPI_UCX_INSTALL_DIR/lib/libmpi.so                              \
+         -L/opt/rocm/lib -lamdhip64 -lhsa-runtime64
+
+ * To run:
+   mpirun -np 8 -x ROCSHMEM_MAX_NUM_CONTEXTS=2 ./rocshmem_getmem_test
+
+ */
+
+#include <rocshmem/rocshmem.hpp>
+
+#include "util.h"
+
+using namespace rocshmem;
+
+__global__ void simple_getmem_test(int *src, int *dst, size_t nelem)
+{
+
+    int threadId = blockIdx.x * blockDim.x + threadIdx.x;
+    if (threadId == 0) {
+        int my_pe = rocshmem_my_pe();
+        int peer =  my_pe ? 0 : 1;
+        rocshmem_getmem(dst, src, nelem * sizeof(int), peer);
+        rocshmem_quiet();
+    }
+
+    __syncthreads();
+}
+
+#define MAX_ELEM 256
+
+int main (int argc, char **argv)
+{
+    int nelem = MAX_ELEM;
+
+    if (argc > 1) {
+        nelem = atoi(argv[1]);
+    }
+
+    CHECK_HIP(hipSetDevice(get_launcher_local_rank()));
+
+    rocshmem_init();
+
+    int my_pe = rocshmem_my_pe();
+    int npes =  rocshmem_n_pes();
+
+    int *src = (int *)rocshmem_malloc(nelem * sizeof(int));
+    int *dst = (int *)rocshmem_malloc(nelem * sizeof(int));
+    if (NULL == src || NULL == dst) {
+        std::cout << "Error allocating memory from symmetric heap" << std::endl;
+        std::cout << "source: " << src << ", dest: " << dst << ", size: "
+          << sizeof(int) * nelem << std::endl;
+        rocshmem_global_exit(1);
+    }
+
+    for (int i=0; i<nelem; i++) {
+        src[i] = 0;
+        dst[i] = 1;
+    }
+    CHECK_HIP(hipDeviceSynchronize());
+
+    int threadsPerBlock=256;
+    simple_getmem_test<<<dim3(1), dim3(threadsPerBlock), 0, 0>>>(src, dst, nelem);
+    rocshmem_barrier_all();
+    CHECK_HIP(hipDeviceSynchronize());
+
+    bool pass = true;
+    for (int i=0; i<nelem; i++) {
+        if (dst[i] != 0) {
+            pass = false;
+#if VERBOSE
+            printf("[%d] Error in element %d expected 0 got %d\n", my_pe, i, dst[i]);
+#endif
+        }
+    }
+    printf("Test %s \t %s\n", argv[0], pass ? "[PASS]" : "[FAIL]");
+
+    rocshmem_free(src);
+    rocshmem_free(dst);
+    rocshmem_finalize();
+    return 0;
+}

projects/rocshmem/examples/rocshmem_init_attr_test.cc

@@ -0,0 +1,121 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+/*
+ * First find your offload target, and if xnack is enabled/disabled using
+
+   rocminfo | grep amdgcn
+
+ * It should output a string like so:
+
+   "Name:                    amdgcn-amd-amdhsa--gfx942:sramecc+:xnack-"
+
+ * This lists the offload taret (gfx942) and that xnack is disabled (xnack-).
+ * Therefore, we need to specify --offload-arch=gfx942:xnack- to our link and compile commands.
+ * Please modify the compile and link commands to suit your system
+
+ * To compile:
+   hipcc -c -fgpu-rdc -x hip rocshmem_init_attr_test.cc \
+         --offload-arch=<target>:<xnack>                \
+         -I/opt/rocm/include                            \
+         -I$ROCSHMEM_INSTALL_DIR/include                \
+         -I$OPENMPI_UCX_INSTALL_DIR/include/
+
+ * To link:
+   hipcc -fgpu-rdc --hip-link rocshmem_init_attr_test.o -o rocshmem_init_attr_test \
+         --offload-arch=<target>:<xnack>                                           \
+         $ROCSHMEM_INSTALL_DIR/lib/librocshmem.a                                   \
+         $OPENMPI_UCX_INSTALL_DIR/lib/libmpi.so                                    \
+         -L/opt/rocm/lib -lamdhip64 -lhsa-runtime64
+
+ * To run:
+   mpirun -np 8 -x ROCSHMEM_MAX_NUM_CONTEXTS=2 ./rocshmem_init_attr_test
+
+ * Note:
+    running this test with the Reverse Offload (RO) conduit requires setting
+                  ROCSHMEM_UNIQUEID_WITH_MPI=1
+
+ */
+
+#include <rocshmem/rocshmem.hpp>
+#include <mpi.h>
+
+#include "util.h"
+
+using namespace rocshmem;
+
+int main (int argc, char **argv)
+{
+    int world_rank, world_nranks;
+    int ret;
+    rocshmem_uniqueid_t uid;
+    rocshmem_init_attr_t attr;
+    int provided;
+
+    MPI_Init_thread (&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+    if (provided != MPI_THREAD_MULTIPLE) {
+      std::cerr << "MPI_THREAD_MULTIPLE support disabled.\n";
+    }
+    MPI_Comm_rank (MPI_COMM_WORLD, &world_rank);
+    MPI_Comm_size (MPI_COMM_WORLD, &world_nranks);
+
+    // Create two disjoint groups of processes, each
+    // one creating a unique rocshmem environment independent
+    // of the other group
+    MPI_Comm newcomm;
+    int color = world_rank %2;
+    int rank, nranks;
+
+    MPI_Comm_split(MPI_COMM_WORLD, color, world_rank, &newcomm);
+    MPI_Comm_rank (newcomm, &rank);
+    MPI_Comm_size (newcomm, &nranks);
+
+    if (rank == 0) {
+      ret = rocshmem_get_uniqueid (&uid);
+      if (ret != ROCSHMEM_SUCCESS) {
+        std::cout << rank << ": Error in rocshmem_get_uniqueid. Aborting.\n";
+        MPI_Abort (MPI_COMM_WORLD, ret);
+      }
+    }
+
+    MPI_Bcast (&uid, sizeof(rocshmem_uniqueid_t), MPI_BYTE, 0, newcomm);
+    ret = rocshmem_set_attr_uniqueid_args(rank, nranks, &uid, &attr);
+    if (ret != ROCSHMEM_SUCCESS) {
+      std::cout << rank << ": Error in rocshmem_set_attr_uniqueid_args. Aborting.\n";
+      MPI_Abort (MPI_COMM_WORLD, ret);
+    }
+
+    ret = rocshmem_init_attr(ROCSHMEM_INIT_WITH_UNIQUEID, &attr);
+    if (ret != ROCSHMEM_SUCCESS) {
+      std::cout << rank << ": Error in rocshmem_init_attr. Aborting.\n";
+      MPI_Abort (MPI_COMM_WORLD, ret);
+    }
+
+    std::cout << rank << ": rocshmem_init_attr SUCCESS\n";
+
+    rocshmem_finalize();
+    MPI_Comm_free (&newcomm);
+    MPI_Finalize();
+    return 0;
+}

projects/rocshmem/examples/rocshmem_put_signal_test.cc

@@ -0,0 +1,140 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+/*
+ * First find your offload target, and if xnack is enabled/disabled using
+
+   rocminfo | grep amdgcn
+
+ * It should output a string like so:
+
+   "Name:                    amdgcn-amd-amdhsa--gfx942:sramecc+:xnack-"
+
+ * This lists the offload taret (gfx942) and that xnack is disabled (xnack-).
+ * Therefore, we need to specify --offload-arch=gfx942:xnack- to our link and compile commands.
+ * Please modify the compile and link commands to suit your system
+
+ * To compile:
+   hipcc -c -fgpu-rdc -x hip rocshmem_put_signal_test.cc \
+         --offload-arch=<target>:<xnack>                 \
+         -I/opt/rocm/include                             \
+         -I$ROCSHMEM_INSTALL_DIR/include                 \
+         -I$OPENMPI_UCX_INSTALL_DIR/include/
+
+ * To link:
+   hipcc -fgpu-rdc --hip-link rocshmem_put_signal_test.o -o rocshmem_put_signal_test \
+         --offload-arch=<target>:<xnack>                                             \
+         $ROCSHMEM_INSTALL_DIR/lib/librocshmem.a                                     \
+         $OPENMPI_UCX_INSTALL_DIR/lib/libmpi.so                                      \
+         -L/opt/rocm/lib -lamdhip64 -lhsa-runtime64
+
+ * To run:
+   mpirun -np 8 -x ROCSHMEM_MAX_NUM_CONTEXTS=2 ./rocshmem_put_signal_test
+
+ */
+
+#include <rocshmem/rocshmem.hpp>
+
+#include "util.h"
+
+using namespace rocshmem;
+
+__global__ void simple_put_signal_test(uint64_t *data, uint64_t *message, size_t nelem,
+                                       uint64_t *sig_addr, int my_pe, int dst_pe)
+{
+
+    int threadId = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (threadId == 0) {
+        if (my_pe == 0) {
+            rocshmem_ulong_put_signal(data, message, nelem, sig_addr, 1, ROCSHMEM_SIGNAL_SET, dst_pe);
+        }
+        else {
+            rocshmem_ulong_wait_until(sig_addr, ROCSHMEM_CMP_EQ, 1);
+            rocshmem_ulong_put_signal(data, data, nelem, sig_addr, 1, ROCSHMEM_SIGNAL_SET, dst_pe);
+        }
+    }
+
+    __syncthreads();
+}
+
+#define MAX_ELEM 256
+
+int main (int argc, char **argv)
+{
+    int nelem = MAX_ELEM;
+
+    if (argc > 1) {
+        nelem = atoi(argv[1]);
+    }
+
+    CHECK_HIP(hipSetDevice(get_launcher_local_rank()));
+
+    rocshmem_init();
+
+    int my_pe = rocshmem_my_pe();
+    int npes =  rocshmem_n_pes();
+
+    int dst_pe = (my_pe + 1) % npes;
+    uint64_t *message = (uint64_t*)rocshmem_malloc(nelem * sizeof(uint64_t));
+    uint64_t *data = (uint64_t*)rocshmem_malloc(nelem * sizeof(uint64_t));
+    uint64_t *sig_addr = (uint64_t*)rocshmem_malloc(sizeof(uint64_t));
+    if (NULL == data || NULL == message || NULL == sig_addr) {
+        std::cout << "Error allocating memory from symmetric heap" << std::endl;
+        std::cout << "data: " << data
+                  << ", message: " << message
+                  << ", size: " << sizeof(uint64_t) * nelem
+                  << ", sig_addr: " << sig_addr
+                  << std::endl;
+        rocshmem_global_exit(1);
+    }
+
+    for (int i=0; i<nelem; i++) {
+        message[i] = my_pe;
+    }
+
+    CHECK_HIP(hipMemset(data, 0, (nelem * sizeof(uint64_t))));
+    CHECK_HIP(hipDeviceSynchronize());
+
+    int threadsPerBlock=256;
+    simple_put_signal_test<<<dim3(1), dim3(threadsPerBlock), 0, 0>>>(data, message, nelem, sig_addr, my_pe, dst_pe);
+    rocshmem_barrier_all();
+    CHECK_HIP(hipDeviceSynchronize());
+
+    bool pass = true;
+    for (int i=0; i<nelem; i++) {
+        if (data[i] != 0) {
+            pass = false;
+#if VERBOSE
+            printf("[%d] Error in element %d expected 0 got %d\n", my_pe, i, dst[i]);
+#endif
+        }
+    }
+    printf("[%d] Test %s \t %s\n", my_pe, argv[0], pass ? "[PASS]" : "[FAIL]");
+
+    rocshmem_free(data);
+    rocshmem_free(message);
+    rocshmem_finalize();
+    return 0;
+}

projects/rocshmem/examples/util.h

@@ -0,0 +1,52 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifndef __ROCSHMEM_EXAMPLES_UTIL_H__
+#define __ROCSHMEM_EXAMPLES_UTIL_H__
+
+#include <iostream>
+
+#include <hip/hip_runtime_api.h>
+#include <hip/hip_runtime.h>
+
+#define CHECK_HIP(condition) {                                            \
+        hipError_t error = condition;                                     \
+        if(error != hipSuccess){                                          \
+            fprintf(stderr,"HIP error: %d line: %d\n", error,  __LINE__); \
+            exit(error);                                                  \
+        }                                                                 \
+    }
+
+static int get_launcher_local_rank() {
+    char *local_rank_str = nullptr;
+
+    local_rank_str = getenv("OMPI_COMM_WORLD_LOCAL_RANK");
+    if (nullptr != local_rank_str) {
+        return atoi(local_rank_str);
+    }
+
+    return -1;
+}
+
+#endif /* __ROCSHMEM_EXAMPLES_UTIL_H__ */