From ff954237dd3eee17352753cb2c60947e2f14dff1 Mon Sep 17 00:00:00 2001
From: avinashkethineedi <avinash.kethineedi@amd.com>
Date: Tue, 3 Sep 2024 14:19:01 -0700
Subject: [PATCH 1/4] add functional tests for puts and gets at wavefront level

* These functional tests are simple puts and gets, where every wave will get/put the same amount of data
* Enabled workgroup level puts and gets tests
---
 scripts/functional_tests/driver.sh            |  24 ++++
 tests/functional_tests/CMakeLists.txt         |   1 +
 tests/functional_tests/tester.cpp             |  17 +++
 .../wave_level_primitives.cpp                 | 118 ++++++++++++++++++
 .../wave_level_primitives.hpp                 |  48 +++++++
 5 files changed, 208 insertions(+)
 create mode 100644 tests/functional_tests/wave_level_primitives.cpp
 create mode 100644 tests/functional_tests/wave_level_primitives.hpp

diff --git a/scripts/functional_tests/driver.sh b/scripts/functional_tests/driver.sh
index 5adb033166..556681ec66 100755
--- a/scripts/functional_tests/driver.sh
+++ b/scripts/functional_tests/driver.sh
@@ -55,6 +55,30 @@ case $2 in
         echo "putnbi_n2_w1_z1_1MB"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 3 > $3/putnbi_n2_w1_z1_1MB.log
         check putnbi_n2_w1_z1_1MB
+        echo "wg_get_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 28 > $3/wg_get_n2_w1_z1_1MB.log
+        check wg_get_n2_w1_z1_1MB
+        echo "wg_getnbi_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 29 > $3/wg_getnbi_n2_w1_z1_1MB.log
+        check wg_getnbi_n2_w1_z1_1MB
+        echo "wg_put_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 30 > $3/wg_put_n2_w1_z1_1MB.log
+        check wg_put_n2_w1_z1_1MB
+        echo "wg_putnbi_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_n2_w1_z1_1MB.log
+        check wg_putnbi_n2_w1_z1_1MB
+        echo "wave_get_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 32 > $3/wave_get_n2_w1_z1_1MB.log
+        check wave_get_n2_w1_z1_1MB
+        echo "wave_getnbi_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 33 > $3/wave_getnbi_n2_w1_z1_1MB.log
+        check wave_getnbi_n2_w1_z1_1MB
+        echo "wave_put_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 34 > $3/wave_put_n2_w1_z1_1MB.log
+        check wave_put_n2_w1_z1_1MB
+        echo "wave_putnbi_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 35 > $3/wave_putnbi_n2_w1_z1_1MB.log
+        check wave_putnbi_n2_w1_z1_1MB
         echo "amofadd_n2_w1_z1"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 6 > $3/amofadd_n2_w1_z1.log
         check amofadd_n2_w1_z1
diff --git a/tests/functional_tests/CMakeLists.txt b/tests/functional_tests/CMakeLists.txt
index a6176e402e..542462261d 100644
--- a/tests/functional_tests/CMakeLists.txt
+++ b/tests/functional_tests/CMakeLists.txt
@@ -55,6 +55,7 @@ target_sources(
     shmem_ptr_tester.cpp
     extended_primitives.cpp
     empty_tester.cpp
+    wave_level_primitives.cpp
 )
 
 ###############################################################################
diff --git a/tests/functional_tests/tester.cpp b/tests/functional_tests/tester.cpp
index f35928eb53..58d40cb4f2 100644
--- a/tests/functional_tests/tester.cpp
+++ b/tests/functional_tests/tester.cpp
@@ -52,6 +52,7 @@
 #include "team_ctx_infra_tester.hpp"
 #include "team_ctx_primitive_tester.hpp"
 #include "team_reduction_tester.hpp"
+#include "wave_level_primitives.hpp"
 
 Tester::Tester(TesterArguments args) : args(args) {
   _type = (TestType)args.algorithm;
@@ -488,6 +489,22 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
         std::cout << "Non-Blocking Put message rate***" << std::endl;
       testers.push_back(new PrimitiveMRTester(args));
       return testers;
+    case WAVEGetTestType:
+      if (rank == 0) std::cout << "WAVE Blocking Gets***" << std::endl;
+      testers.push_back(new WaveLevelPrimitiveTester(args));
+      return testers;
+    case WAVEGetNBITestType:
+      if (rank == 0) std::cout << "WAVE Non-Blocking Gets***" << std::endl;
+      testers.push_back(new WaveLevelPrimitiveTester(args));
+      return testers;
+    case WAVEPutTestType:
+      if (rank == 0) std::cout << "WAVE Blocking Puts***" << std::endl;
+      testers.push_back(new WaveLevelPrimitiveTester(args));
+      return testers;
+    case WAVEPutNBITestType:
+      if (rank == 0) std::cout << "WAVE Non-Blocking Puts***" << std::endl;
+      testers.push_back(new WaveLevelPrimitiveTester(args));
+      return testers;
     default:
       if (rank == 0) std::cout << "Unknown***" << std::endl;
       testers.push_back(new PrimitiveTester(args));
diff --git a/tests/functional_tests/wave_level_primitives.cpp b/tests/functional_tests/wave_level_primitives.cpp
new file mode 100644
index 0000000000..6cb2ee8a80
--- /dev/null
+++ b/tests/functional_tests/wave_level_primitives.cpp
@@ -0,0 +1,118 @@
+/******************************************************************************
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+#include "wave_level_primitives.hpp"
+
+#include <roc_shmem/roc_shmem.hpp>
+
+using namespace rocshmem;
+
+/******************************************************************************
+ * DEVICE TEST KERNEL
+ *****************************************************************************/
+__global__ void WaveLevelPrimitiveTest(int loop, int skip, uint64_t *timer,
+                                      char *s_buf, char *r_buf, int size,
+                                      TestType type,
+                                      ShmemContextType ctx_type) {
+  __shared__ roc_shmem_ctx_t ctx;
+  roc_shmem_wg_init();
+  roc_shmem_wg_ctx_create(ctx_type, &ctx);
+
+  uint64_t start;
+
+  for (int i = 0; i < loop + skip; i++) {
+    if (i == skip) start = roc_shmem_timer();
+
+    switch (type) {
+      case WAVEGetTestType:
+        roc_shmemx_ctx_getmem_wave(ctx, r_buf, s_buf, size, 1);
+        break;
+      case WAVEGetNBITestType:
+        roc_shmemx_ctx_getmem_nbi_wave(ctx, r_buf, s_buf, size, 1);
+        break;
+      case WAVEPutTestType:
+        roc_shmemx_ctx_putmem_wave(ctx, r_buf, s_buf, size, 1);
+        break;
+      case WAVEPutNBITestType:
+        roc_shmemx_ctx_putmem_nbi_wave(ctx, r_buf, s_buf, size, 1);
+        break;
+      default:
+        break;
+    }
+  }
+
+  roc_shmem_ctx_quiet(ctx);
+
+  if (hipThreadIdx_x == 0) {
+    timer[hipBlockIdx_x] = roc_shmem_timer() - start;
+  }
+
+  roc_shmem_wg_ctx_destroy(&ctx);
+  roc_shmem_wg_finalize();
+}
+
+/******************************************************************************
+ * HOST TESTER CLASS METHODS
+ *****************************************************************************/
+WaveLevelPrimitiveTester::WaveLevelPrimitiveTester(TesterArguments args)
+    : Tester(args) {
+  s_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.wg_size);
+  r_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.wg_size);
+}
+
+WaveLevelPrimitiveTester::~WaveLevelPrimitiveTester() {
+  roc_shmem_free(s_buf);
+  roc_shmem_free(r_buf);
+}
+
+void WaveLevelPrimitiveTester::resetBuffers(uint64_t size) {
+  memset(s_buf, '0', args.max_msg_size * args.wg_size);
+  memset(r_buf, '1', args.max_msg_size * args.wg_size);
+}
+
+void WaveLevelPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
+                                           int loop, uint64_t size) {
+  size_t shared_bytes = 0;
+
+  hipLaunchKernelGGL(WaveLevelPrimitiveTest, gridSize, blockSize, shared_bytes,
+                     stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
+                     _shmem_context);
+
+  num_msgs = (loop + args.skip) * gridSize.x;
+  num_timed_msgs = loop;
+}
+
+void WaveLevelPrimitiveTester::verifyResults(uint64_t size) {
+  int check_id = (_type == WAVEGetTestType || _type == WAVEGetNBITestType)
+                     ? 0
+                     : 1;
+
+  if (args.myid == check_id) {
+    for (int i = 0; i < size; i++) {
+      if (r_buf[i] != '0') {
+        fprintf(stderr, "Data validation error at idx %d\n", i);
+        fprintf(stderr, "Got %c, Expected %c \n", r_buf[i], '0');
+        exit(-1);
+      }
+    }
+  }
+}
diff --git a/tests/functional_tests/wave_level_primitives.hpp b/tests/functional_tests/wave_level_primitives.hpp
new file mode 100644
index 0000000000..8c59d81350
--- /dev/null
+++ b/tests/functional_tests/wave_level_primitives.hpp
@@ -0,0 +1,48 @@
+/******************************************************************************
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifndef _WAVE_LEVEL_PRIMITIVE_TEST_HPP_
+#define _WAVE_LEVEL_PRIMITIVE_TEST_HPP_
+
+#include "tester.hpp"
+
+/******************************************************************************
+ * HOST TESTER CLASS
+ *****************************************************************************/
+class WaveLevelPrimitiveTester : public Tester {
+ public:
+  explicit WaveLevelPrimitiveTester(TesterArguments args);
+  virtual ~WaveLevelPrimitiveTester();
+
+ protected:
+  virtual void resetBuffers(uint64_t size) override;
+
+  virtual void launchKernel(dim3 gridSize, dim3 blockSize, int loop,
+                            uint64_t size) override;
+
+  virtual void verifyResults(uint64_t size) override;
+
+  char *s_buf = nullptr;
+  char *r_buf = nullptr;
+};
+
+#endif

From d226922733dc081187824c7ca90a7e035ea6e773 Mon Sep 17 00:00:00 2001
From: avinashkethineedi <avinash.kethineedi@amd.com>
Date: Wed, 4 Sep 2024 12:55:10 -0700
Subject: [PATCH 2/4] Add tilled version of puts and gets at the workgroup
 level to the functional test suite

---
 scripts/functional_tests/driver.sh            | 12 ++++
 .../functional_tests/extended_primitives.cpp  | 60 +++++++++++++++++--
 .../functional_tests/extended_primitives.hpp  |  1 +
 tests/functional_tests/tester.cpp             | 20 +++++--
 tests/functional_tests/tester_arguments.cpp   |  4 ++
 tests/functional_tests/tester_arguments.hpp   |  1 +
 6 files changed, 89 insertions(+), 9 deletions(-)

diff --git a/scripts/functional_tests/driver.sh b/scripts/functional_tests/driver.sh
index 556681ec66..0acfd26090 100755
--- a/scripts/functional_tests/driver.sh
+++ b/scripts/functional_tests/driver.sh
@@ -67,6 +67,18 @@ case $2 in
         echo "wg_putnbi_n2_w1_z1_1MB"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_n2_w1_z1_1MB.log
         check wg_putnbi_n2_w1_z1_1MB
+        echo "wg_get_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 28 -ti 1 > $3/wg_get_tiled_n2_w1_z1_1MB.log
+        check wg_get_tiled_n2_w1_z1_1MB
+        echo "wg_getnbi_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 29 -ti 1 > $3/wg_getnbi_tiled_n2_w1_z1_1MB.log
+        check wg_getnbi_tiled_n2_w1_z1_1MB
+        echo "wg_put_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 30 -ti 1 > $3/wg_put_tiled_n2_w1_z1_1MB.log
+        check wg_put_tiled_n2_w1_z1_1MB
+        echo "wg_putnbi_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 31 -ti 1 > $3/wg_putnbi_tiled_n2_w1_z1_1MB.log
+        check wg_putnbi_tiled_n2_w1_z1_1MB
         echo "wave_get_n2_w1_z1_1MB"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 32 > $3/wave_get_n2_w1_z1_1MB.log
         check wave_get_n2_w1_z1_1MB
diff --git a/tests/functional_tests/extended_primitives.cpp b/tests/functional_tests/extended_primitives.cpp
index affb6b3e14..8eb569cd32 100644
--- a/tests/functional_tests/extended_primitives.cpp
+++ b/tests/functional_tests/extended_primitives.cpp
@@ -27,7 +27,7 @@
 using namespace rocshmem;
 
 /******************************************************************************
- * DEVICE TEST KERNEL
+ * DEVICE TEST KERNELS
  *****************************************************************************/
 __global__ void ExtendedPrimitiveTest(int loop, int skip, uint64_t *timer,
                                       char *s_buf, char *r_buf, int size,
@@ -70,6 +70,50 @@ __global__ void ExtendedPrimitiveTest(int loop, int skip, uint64_t *timer,
   roc_shmem_wg_finalize();
 }
 
+__global__ void ExtendedPrimitiveTestTiled(int loop, int skip, uint64_t *timer,
+                                      char *s_buf, char *r_buf, int size,
+                                      TestType type,
+                                      ShmemContextType ctx_type) {
+  __shared__ roc_shmem_ctx_t ctx;
+  roc_shmem_wg_init();
+  roc_shmem_wg_ctx_create(ctx_type, &ctx);
+
+  uint64_t start;
+  uint64_t idx = size * get_flat_grid_id();
+  s_buf += idx;
+  r_buf += idx;
+
+  for (int i = 0; i < loop + skip; i++) {
+    if (i == skip) start = roc_shmem_timer();
+
+    switch (type) {
+      case WGGetTestType:
+        roc_shmemx_ctx_getmem_wg(ctx, r_buf, s_buf, size, 1);
+        break;
+      case WGGetNBITestType:
+        roc_shmemx_ctx_getmem_nbi_wg(ctx, r_buf, s_buf, size, 1);
+        break;
+      case WGPutTestType:
+        roc_shmemx_ctx_putmem_wg(ctx, r_buf, s_buf, size, 1);
+        break;
+      case WGPutNBITestType:
+        roc_shmemx_ctx_putmem_nbi_wg(ctx, r_buf, s_buf, size, 1);
+        break;
+      default:
+        break;
+    }
+  }
+
+  roc_shmem_ctx_quiet(ctx);
+
+  if (hipThreadIdx_x == 0) {
+    timer[hipBlockIdx_x] = roc_shmem_timer() - start;
+  }
+
+  roc_shmem_wg_ctx_destroy(&ctx);
+  roc_shmem_wg_finalize();
+}
+
 /******************************************************************************
  * HOST TESTER CLASS METHODS
  *****************************************************************************/
@@ -93,17 +137,23 @@ void ExtendedPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
                                            int loop, uint64_t size) {
   size_t shared_bytes = 0;
 
-  hipLaunchKernelGGL(ExtendedPrimitiveTest, gridSize, blockSize, shared_bytes,
+  if (args.tiled){
+    hipLaunchKernelGGL(ExtendedPrimitiveTestTiled, gridSize, blockSize, shared_bytes,
                      stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
                      _shmem_context);
+  }
+  else {
+    hipLaunchKernelGGL(ExtendedPrimitiveTest, gridSize, blockSize, shared_bytes,
+                     stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
+                     _shmem_context);
+  }
 
   num_msgs = (loop + args.skip) * gridSize.x;
-  num_timed_msgs = loop;
+  num_timed_msgs = loop * gridSize.x;
 }
 
 void ExtendedPrimitiveTester::verifyResults(uint64_t size) {
-  int check_id = (_type == WGGetTestType || _type == WGGetNBITestType ||
-                  _type == WAVEGetTestType)
+  int check_id = (_type == WGGetTestType || _type == WGGetNBITestType)
                      ? 0
                      : 1;
 
diff --git a/tests/functional_tests/extended_primitives.hpp b/tests/functional_tests/extended_primitives.hpp
index dbad6a48f3..33fdac2f8f 100644
--- a/tests/functional_tests/extended_primitives.hpp
+++ b/tests/functional_tests/extended_primitives.hpp
@@ -24,6 +24,7 @@
 #define _EXTENDED_PRIMITIVES_HPP_
 
 #include "tester.hpp"
+#include "../src/util.hpp"
 
 /******************************************************************************
  * HOST TESTER CLASS
diff --git a/tests/functional_tests/tester.cpp b/tests/functional_tests/tester.cpp
index 58d40cb4f2..22ab55f4e1 100644
--- a/tests/functional_tests/tester.cpp
+++ b/tests/functional_tests/tester.cpp
@@ -469,19 +469,31 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
       testers.push_back(new ShmemPtrTester(args));
       return testers;
     case WGGetTestType:
-      if (rank == 0) std::cout << "Blocking WG level Gets***" << std::endl;
+      if (rank == 0) {
+        if (args.tiled) std::cout << "Tiled Blocking WG level Gets***" << std::endl;
+        else std::cout << "Blocking WG level Gets***" << std::endl;
+      }
       testers.push_back(new ExtendedPrimitiveTester(args));
       return testers;
     case WGGetNBITestType:
-      if (rank == 0) std::cout << "Non-Blocking WG level Gets***" << std::endl;
+      if (rank == 0) {
+        if (args.tiled) std::cout << "Tiled Non-Blocking WG level Gets***" << std::endl;
+        else std::cout << "Non-Blocking WG level Gets***" << std::endl;
+      }
       testers.push_back(new ExtendedPrimitiveTester(args));
       return testers;
     case WGPutTestType:
-      if (rank == 0) std::cout << "Blocking WG level Puts***" << std::endl;
+      if (rank == 0) {
+        if (args.tiled) std::cout << "Tiled Blocking WG level Puts***" << std::endl;
+        else std::cout << "Blocking WG level Puts***" << std::endl;
+      }
       testers.push_back(new ExtendedPrimitiveTester(args));
       return testers;
     case WGPutNBITestType:
-      if (rank == 0) std::cout << "Non-Blocking WG level Puts***" << std::endl;
+      if (rank == 0) {
+        if(args.tiled) std::cout << "Tiled Non-Blocking WG level Puts***" << std::endl;
+        else std::cout << "Non-Blocking WG level Puts***" << std::endl;
+      }
       testers.push_back(new ExtendedPrimitiveTester(args));
       return testers;
     case PutNBIMRTestType:
diff --git a/tests/functional_tests/tester_arguments.cpp b/tests/functional_tests/tester_arguments.cpp
index 88e4b12cd6..014d5d1388 100644
--- a/tests/functional_tests/tester_arguments.cpp
+++ b/tests/functional_tests/tester_arguments.cpp
@@ -60,6 +60,9 @@ TesterArguments::TesterArguments(int argc, char *argv[]) {
     } else if (arg == "-x") {
       i++;
       shmem_context = atoi(argv[i]);
+    } else if (arg == "-ti") {
+      i++;
+      tiled = (atoi(argv[i]) == 1);
     } else {
       show_usage(argv[0]);
       exit(-1);
@@ -119,6 +122,7 @@ void TesterArguments::show_usage(std::string executable_name) {
   std::cout << "\t-o <Operation type for the random_access test>\n";
   std::cout << "\t-ta <Number of Thread Accessing the communication>\n";
   std::cout << "\t-x <shmem context>\n";
+  std::cout << "\t-ti <Tiled version>\n";
 }
 
 void TesterArguments::get_rocshmem_arguments() {
diff --git a/tests/functional_tests/tester_arguments.hpp b/tests/functional_tests/tester_arguments.hpp
index 88ff6a5537..810d00d825 100644
--- a/tests/functional_tests/tester_arguments.hpp
+++ b/tests/functional_tests/tester_arguments.hpp
@@ -58,6 +58,7 @@ class TesterArguments {
   unsigned coal_coef = 64;
   unsigned op_type = 0;
   unsigned shmem_context = rocshmem::ROC_SHMEM_CTX_WG_PRIVATE;
+  bool tiled = false;
 
   /**
    * Arguments obtained from rocshmem

From b6d31ac7ef13187fe12736b683c5eb51411c4941 Mon Sep 17 00:00:00 2001
From: avinashkethineedi <avinash.kethineedi@amd.com>
Date: Sat, 7 Sep 2024 16:06:36 -0700
Subject: [PATCH 3/4] Add tilled version of puts and gets at wavefront level to
 the functional test suite

* Implemented tiled version of put*_wave and get*_wave functions
* Maintain single kernel that supports both tiled and untiled versions
* Disable IPC in the default RO build script
---
 scripts/build_configs/ro_net                  |  3 +-
 scripts/functional_tests/driver.sh            | 20 +++++--
 .../functional_tests/extended_primitives.cpp  | 60 ++-----------------
 tests/functional_tests/tester.cpp             | 41 ++++++++++---
 tests/functional_tests/tester.hpp             |  3 +
 tests/functional_tests/tester_arguments.cpp   |  4 --
 tests/functional_tests/tester_arguments.hpp   |  1 -
 .../wave_level_primitives.cpp                 | 27 +++++----
 .../wave_level_primitives.hpp                 |  1 +
 9 files changed, 78 insertions(+), 82 deletions(-)

diff --git a/scripts/build_configs/ro_net b/scripts/build_configs/ro_net
index 17809fa0a9..c1d09194f7 100755
--- a/scripts/build_configs/ro_net
+++ b/scripts/build_configs/ro_net
@@ -17,8 +17,9 @@ cmake \
     -DDEBUG=OFF \
     -DPROFILE=OFF \
     -DUSE_GPU_IB=OFF \
+    -DUSE_RO=ON \
     -DUSE_DC=OFF \
-    -DUSE_IPC=ON \
+    -DUSE_IPC=OFF \
     -DUSE_THREADS=ON \
     -DUSE_WF_COAL=OFF \
     -DUSE_COHERENT_HEAP=ON \
diff --git a/scripts/functional_tests/driver.sh b/scripts/functional_tests/driver.sh
index 0acfd26090..5e6bc53ae3 100755
--- a/scripts/functional_tests/driver.sh
+++ b/scripts/functional_tests/driver.sh
@@ -68,16 +68,16 @@ case $2 in
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_n2_w1_z1_1MB.log
         check wg_putnbi_n2_w1_z1_1MB
         echo "wg_get_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 28 -ti 1 > $3/wg_get_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 28 > $3/wg_get_tiled_n2_w1_z1_1MB.log
         check wg_get_tiled_n2_w1_z1_1MB
         echo "wg_getnbi_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 29 -ti 1 > $3/wg_getnbi_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 29 > $3/wg_getnbi_tiled_n2_w1_z1_1MB.log
         check wg_getnbi_tiled_n2_w1_z1_1MB
         echo "wg_put_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 30 -ti 1 > $3/wg_put_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 30 > $3/wg_put_tiled_n2_w1_z1_1MB.log
         check wg_put_tiled_n2_w1_z1_1MB
         echo "wg_putnbi_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 31 -ti 1 > $3/wg_putnbi_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_tiled_n2_w1_z1_1MB.log
         check wg_putnbi_tiled_n2_w1_z1_1MB
         echo "wave_get_n2_w1_z1_1MB"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 32 > $3/wave_get_n2_w1_z1_1MB.log
@@ -91,6 +91,18 @@ case $2 in
         echo "wave_putnbi_n2_w1_z1_1MB"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 35 > $3/wave_putnbi_n2_w1_z1_1MB.log
         check wave_putnbi_n2_w1_z1_1MB
+        echo "wave_get_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 32 > $3/wave_get_tiled_n2_w1_z1_1MB.log
+        check wave_get_tiled_n2_w1_z1_1MB
+        echo "wave_getnbi_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 33 > $3/wave_getnbi_tiled_n2_w1_z1_1MB.log
+        check wave_getnbi_tiled_n2_w1_z1_1MB
+        echo "wave_put_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 34 > $3/wave_put_tiled_n2_w1_z1_1MB.log
+        check wave_put_tiled_n2_w1_z1_1MB
+        echo "wave_putnbi_tiled_n2_w1_z1_1MB"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 35 > $3/wave_putnbi_tiled_n2_w1_z1_1MB.log
+        check wave_putnbi_tiled_n2_w1_z1_1MB
         echo "amofadd_n2_w1_z1"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 6 > $3/amofadd_n2_w1_z1.log
         check amofadd_n2_w1_z1
diff --git a/tests/functional_tests/extended_primitives.cpp b/tests/functional_tests/extended_primitives.cpp
index 8eb569cd32..cc73d9c502 100644
--- a/tests/functional_tests/extended_primitives.cpp
+++ b/tests/functional_tests/extended_primitives.cpp
@@ -37,47 +37,6 @@ __global__ void ExtendedPrimitiveTest(int loop, int skip, uint64_t *timer,
   roc_shmem_wg_init();
   roc_shmem_wg_ctx_create(ctx_type, &ctx);
 
-  uint64_t start;
-
-  for (int i = 0; i < loop + skip; i++) {
-    if (i == skip) start = roc_shmem_timer();
-
-    switch (type) {
-      case WGGetTestType:
-        roc_shmemx_ctx_getmem_wg(ctx, r_buf, s_buf, size, 1);
-        break;
-      case WGGetNBITestType:
-        roc_shmemx_ctx_getmem_nbi_wg(ctx, r_buf, s_buf, size, 1);
-        break;
-      case WGPutTestType:
-        roc_shmemx_ctx_putmem_wg(ctx, r_buf, s_buf, size, 1);
-        break;
-      case WGPutNBITestType:
-        roc_shmemx_ctx_putmem_nbi_wg(ctx, r_buf, s_buf, size, 1);
-        break;
-      default:
-        break;
-    }
-  }
-
-  roc_shmem_ctx_quiet(ctx);
-
-  if (hipThreadIdx_x == 0) {
-    timer[hipBlockIdx_x] = roc_shmem_timer() - start;
-  }
-
-  roc_shmem_wg_ctx_destroy(&ctx);
-  roc_shmem_wg_finalize();
-}
-
-__global__ void ExtendedPrimitiveTestTiled(int loop, int skip, uint64_t *timer,
-                                      char *s_buf, char *r_buf, int size,
-                                      TestType type,
-                                      ShmemContextType ctx_type) {
-  __shared__ roc_shmem_ctx_t ctx;
-  roc_shmem_wg_init();
-  roc_shmem_wg_ctx_create(ctx_type, &ctx);
-
   uint64_t start;
   uint64_t idx = size * get_flat_grid_id();
   s_buf += idx;
@@ -119,8 +78,8 @@ __global__ void ExtendedPrimitiveTestTiled(int loop, int skip, uint64_t *timer,
  *****************************************************************************/
 ExtendedPrimitiveTester::ExtendedPrimitiveTester(TesterArguments args)
     : Tester(args) {
-  s_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.wg_size);
-  r_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.wg_size);
+  s_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs);
+  r_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs);
 }
 
 ExtendedPrimitiveTester::~ExtendedPrimitiveTester() {
@@ -129,24 +88,17 @@ ExtendedPrimitiveTester::~ExtendedPrimitiveTester() {
 }
 
 void ExtendedPrimitiveTester::resetBuffers(uint64_t size) {
-  memset(s_buf, '0', args.max_msg_size * args.wg_size);
-  memset(r_buf, '1', args.max_msg_size * args.wg_size);
+  memset(s_buf, '0', size * args.num_wgs);
+  memset(r_buf, '1', size * args.num_wgs);
 }
 
 void ExtendedPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
                                            int loop, uint64_t size) {
   size_t shared_bytes = 0;
 
-  if (args.tiled){
-    hipLaunchKernelGGL(ExtendedPrimitiveTestTiled, gridSize, blockSize, shared_bytes,
+  hipLaunchKernelGGL(ExtendedPrimitiveTest, gridSize, blockSize, shared_bytes,
                      stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
                      _shmem_context);
-  }
-  else {
-    hipLaunchKernelGGL(ExtendedPrimitiveTest, gridSize, blockSize, shared_bytes,
-                     stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
-                     _shmem_context);
-  }
 
   num_msgs = (loop + args.skip) * gridSize.x;
   num_timed_msgs = loop * gridSize.x;
@@ -158,7 +110,7 @@ void ExtendedPrimitiveTester::verifyResults(uint64_t size) {
                      : 1;
 
   if (args.myid == check_id) {
-    for (int i = 0; i < size; i++) {
+    for (int i = 0; i < size * args.num_wgs; i++) {
       if (r_buf[i] != '0') {
         fprintf(stderr, "Data validation error at idx %d\n", i);
         fprintf(stderr, "Got %c, Expected %c \n", r_buf[i], '0');
diff --git a/tests/functional_tests/tester.cpp b/tests/functional_tests/tester.cpp
index 22ab55f4e1..cbd1ba89a5 100644
--- a/tests/functional_tests/tester.cpp
+++ b/tests/functional_tests/tester.cpp
@@ -57,6 +57,9 @@
 Tester::Tester(TesterArguments args) : args(args) {
   _type = (TestType)args.algorithm;
   _shmem_context = args.shmem_context;
+  CHECK_HIP(hipGetDevice(&device_id));
+  CHECK_HIP(hipGetDeviceProperties(&deviceProps, device_id));
+  num_warps = args.wg_size / deviceProps.warpSize;
   CHECK_HIP(hipStreamCreate(&stream));
   CHECK_HIP(hipEventCreate(&start_event));
   CHECK_HIP(hipEventCreate(&stop_event));
@@ -470,28 +473,32 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
       return testers;
     case WGGetTestType:
       if (rank == 0) {
-        if (args.tiled) std::cout << "Tiled Blocking WG level Gets***" << std::endl;
+        if (args.num_wgs > 1)
+          std::cout << "Tiled Blocking WG level Gets***" << std::endl;
         else std::cout << "Blocking WG level Gets***" << std::endl;
       }
       testers.push_back(new ExtendedPrimitiveTester(args));
       return testers;
     case WGGetNBITestType:
       if (rank == 0) {
-        if (args.tiled) std::cout << "Tiled Non-Blocking WG level Gets***" << std::endl;
+        if (args.num_wgs > 1)
+          std::cout << "Tiled Non-Blocking WG level Gets***" << std::endl;
         else std::cout << "Non-Blocking WG level Gets***" << std::endl;
       }
       testers.push_back(new ExtendedPrimitiveTester(args));
       return testers;
     case WGPutTestType:
       if (rank == 0) {
-        if (args.tiled) std::cout << "Tiled Blocking WG level Puts***" << std::endl;
+        if (args.num_wgs > 1)
+          std::cout << "Tiled Blocking WG level Puts***" << std::endl;
         else std::cout << "Blocking WG level Puts***" << std::endl;
       }
       testers.push_back(new ExtendedPrimitiveTester(args));
       return testers;
     case WGPutNBITestType:
       if (rank == 0) {
-        if(args.tiled) std::cout << "Tiled Non-Blocking WG level Puts***" << std::endl;
+        if (args.num_wgs > 1)
+          std::cout << "Tiled Non-Blocking WG level Puts***" << std::endl;
         else std::cout << "Non-Blocking WG level Puts***" << std::endl;
       }
       testers.push_back(new ExtendedPrimitiveTester(args));
@@ -502,19 +509,35 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
       testers.push_back(new PrimitiveMRTester(args));
       return testers;
     case WAVEGetTestType:
-      if (rank == 0) std::cout << "WAVE Blocking Gets***" << std::endl;
+      if (rank == 0) {
+        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+          std::cout << "Tiled Blocking WAVE level Gets***" << std::endl;
+        else std::cout << "Blocking WAVE level Gets***" << std::endl;
+      }
       testers.push_back(new WaveLevelPrimitiveTester(args));
       return testers;
     case WAVEGetNBITestType:
-      if (rank == 0) std::cout << "WAVE Non-Blocking Gets***" << std::endl;
+      if (rank == 0) {
+        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+          std::cout << "Tiled Non-Blocking WAVE level Gets***" << std::endl;
+        else std::cout << "Non-Blocking WAVE level Gets***" << std::endl;
+      }
       testers.push_back(new WaveLevelPrimitiveTester(args));
       return testers;
     case WAVEPutTestType:
-      if (rank == 0) std::cout << "WAVE Blocking Puts***" << std::endl;
+      if (rank == 0) {
+        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+          std::cout << "Tiled Blocking WAVE level Puts***" << std::endl;
+        else std::cout << "Blocking WAVE level Puts***" << std::endl;
+      }
       testers.push_back(new WaveLevelPrimitiveTester(args));
       return testers;
     case WAVEPutNBITestType:
-      if (rank == 0) std::cout << "WAVE Non-Blocking Puts***" << std::endl;
+      if (rank == 0) {
+        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+          std::cout << "Tiled Non-Blocking WAVE level Puts***" << std::endl;
+        else std::cout << "Non-Blocking WAVE level Puts***" << std::endl;
+      }
       testers.push_back(new WaveLevelPrimitiveTester(args));
       return testers;
     default:
@@ -684,6 +707,8 @@ uint64_t Tester::gpuCyclesToMicroseconds(uint64_t cycles) {
 uint64_t Tester::timerAvgInMicroseconds() {
   uint64_t sum = 0;
 
+  //TODO: Modify the calcuation for the Tiled version of puts and gets at 
+  //      wavefront level (bpotter/avinash)
   for (int i = 0; i < args.num_wgs; i++) {
     sum += gpuCyclesToMicroseconds(timer[i]);
   }
diff --git a/tests/functional_tests/tester.hpp b/tests/functional_tests/tester.hpp
index 14a91435e5..e0482868e8 100644
--- a/tests/functional_tests/tester.hpp
+++ b/tests/functional_tests/tester.hpp
@@ -117,7 +117,9 @@ class Tester {
 
   int num_msgs = 0;
   int num_timed_msgs = 0;
+  int num_warps = 0;
   int bw_factor = 1;
+  int device_id = 0;
 
   TesterArguments args;
 
@@ -125,6 +127,7 @@ class Tester {
   ShmemContextType _shmem_context = 8;  // SHMEM_CTX_WP_PRIVATE
 
   hipStream_t stream;
+  hipDeviceProp_t deviceProps;
 
   uint64_t *timer = nullptr;
 
diff --git a/tests/functional_tests/tester_arguments.cpp b/tests/functional_tests/tester_arguments.cpp
index 014d5d1388..88e4b12cd6 100644
--- a/tests/functional_tests/tester_arguments.cpp
+++ b/tests/functional_tests/tester_arguments.cpp
@@ -60,9 +60,6 @@ TesterArguments::TesterArguments(int argc, char *argv[]) {
     } else if (arg == "-x") {
       i++;
       shmem_context = atoi(argv[i]);
-    } else if (arg == "-ti") {
-      i++;
-      tiled = (atoi(argv[i]) == 1);
     } else {
       show_usage(argv[0]);
       exit(-1);
@@ -122,7 +119,6 @@ void TesterArguments::show_usage(std::string executable_name) {
   std::cout << "\t-o <Operation type for the random_access test>\n";
   std::cout << "\t-ta <Number of Thread Accessing the communication>\n";
   std::cout << "\t-x <shmem context>\n";
-  std::cout << "\t-ti <Tiled version>\n";
 }
 
 void TesterArguments::get_rocshmem_arguments() {
diff --git a/tests/functional_tests/tester_arguments.hpp b/tests/functional_tests/tester_arguments.hpp
index 810d00d825..88ff6a5537 100644
--- a/tests/functional_tests/tester_arguments.hpp
+++ b/tests/functional_tests/tester_arguments.hpp
@@ -58,7 +58,6 @@ class TesterArguments {
   unsigned coal_coef = 64;
   unsigned op_type = 0;
   unsigned shmem_context = rocshmem::ROC_SHMEM_CTX_WG_PRIVATE;
-  bool tiled = false;
 
   /**
    * Arguments obtained from rocshmem
diff --git a/tests/functional_tests/wave_level_primitives.cpp b/tests/functional_tests/wave_level_primitives.cpp
index 6cb2ee8a80..bb7c299ef9 100644
--- a/tests/functional_tests/wave_level_primitives.cpp
+++ b/tests/functional_tests/wave_level_primitives.cpp
@@ -27,17 +27,22 @@
 using namespace rocshmem;
 
 /******************************************************************************
- * DEVICE TEST KERNEL
+ * DEVICE TEST KERNELS
  *****************************************************************************/
 __global__ void WaveLevelPrimitiveTest(int loop, int skip, uint64_t *timer,
                                       char *s_buf, char *r_buf, int size,
-                                      TestType type,
-                                      ShmemContextType ctx_type) {
+                                      TestType type, ShmemContextType ctx_type,
+                                      int wf_size) {
   __shared__ roc_shmem_ctx_t ctx;
   roc_shmem_wg_init();
   roc_shmem_wg_ctx_create(ctx_type, &ctx);
 
   uint64_t start;
+  int wf_id = get_flat_block_id() / wf_size;
+  int offset = size * get_flat_grid_id() * (get_flat_block_size() / wf_size);
+  int idx = wf_id * size + offset;
+  s_buf += idx;
+  r_buf += idx;
 
   for (int i = 0; i < loop + skip; i++) {
     if (i == skip) start = roc_shmem_timer();
@@ -75,8 +80,10 @@ __global__ void WaveLevelPrimitiveTest(int loop, int skip, uint64_t *timer,
  *****************************************************************************/
 WaveLevelPrimitiveTester::WaveLevelPrimitiveTester(TesterArguments args)
     : Tester(args) {
-  s_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.wg_size);
-  r_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.wg_size);
+  s_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs
+           * num_warps);
+  r_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs
+           * num_warps);
 }
 
 WaveLevelPrimitiveTester::~WaveLevelPrimitiveTester() {
@@ -85,8 +92,8 @@ WaveLevelPrimitiveTester::~WaveLevelPrimitiveTester() {
 }
 
 void WaveLevelPrimitiveTester::resetBuffers(uint64_t size) {
-  memset(s_buf, '0', args.max_msg_size * args.wg_size);
-  memset(r_buf, '1', args.max_msg_size * args.wg_size);
+  memset(s_buf, '0', size * args.num_wgs * num_warps);
+  memset(r_buf, '1', size * args.num_wgs * num_warps);
 }
 
 void WaveLevelPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
@@ -95,10 +102,10 @@ void WaveLevelPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
 
   hipLaunchKernelGGL(WaveLevelPrimitiveTest, gridSize, blockSize, shared_bytes,
                      stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
-                     _shmem_context);
+                     _shmem_context, deviceProps.warpSize);
 
   num_msgs = (loop + args.skip) * gridSize.x;
-  num_timed_msgs = loop;
+  num_timed_msgs = loop * gridSize.x;
 }
 
 void WaveLevelPrimitiveTester::verifyResults(uint64_t size) {
@@ -107,7 +114,7 @@ void WaveLevelPrimitiveTester::verifyResults(uint64_t size) {
                      : 1;
 
   if (args.myid == check_id) {
-    for (int i = 0; i < size; i++) {
+    for (int i = 0; i < size * args.num_wgs * num_warps; i++) {
       if (r_buf[i] != '0') {
         fprintf(stderr, "Data validation error at idx %d\n", i);
         fprintf(stderr, "Got %c, Expected %c \n", r_buf[i], '0');
diff --git a/tests/functional_tests/wave_level_primitives.hpp b/tests/functional_tests/wave_level_primitives.hpp
index 8c59d81350..3b8a98541d 100644
--- a/tests/functional_tests/wave_level_primitives.hpp
+++ b/tests/functional_tests/wave_level_primitives.hpp
@@ -24,6 +24,7 @@
 #define _WAVE_LEVEL_PRIMITIVE_TEST_HPP_
 
 #include "tester.hpp"
+#include "../src/util.hpp"
 
 /******************************************************************************
  * HOST TESTER CLASS

From 18a1bdd0ac17e9a665ea31463831bbbbd49678f6 Mon Sep 17 00:00:00 2001
From: avinashkethineedi <avinash.kethineedi@amd.com>
Date: Mon, 14 Oct 2024 23:49:40 +0000
Subject: [PATCH 4/4] Use C++ iota function to reset buffers and use its values
 for verification

* Update functional test script to include new tests
---
 scripts/build_configs/ro_net                  |  3 +-
 scripts/functional_tests/driver.sh            | 80 +++++++++++++++----
 .../functional_tests/extended_primitives.cpp  | 28 ++++---
 .../functional_tests/extended_primitives.hpp  |  5 +-
 tests/functional_tests/tester.cpp             | 45 +++++++++--
 tests/functional_tests/tester_arguments.cpp   | 10 +++
 .../wave_level_primitives.cpp                 | 41 ++++++----
 .../wave_level_primitives.hpp                 |  5 +-
 8 files changed, 161 insertions(+), 56 deletions(-)

diff --git a/scripts/build_configs/ro_net b/scripts/build_configs/ro_net
index 6abc528ca6..95ea2950a8 100755
--- a/scripts/build_configs/ro_net
+++ b/scripts/build_configs/ro_net
@@ -17,9 +17,8 @@ cmake \
     -DDEBUG=OFF \
     -DPROFILE=OFF \
     -DUSE_GPU_IB=OFF \
-    -DUSE_RO=ON \
     -DUSE_DC=OFF \
-    -DUSE_IPC=OFF \
+    -DUSE_IPC=ON \
     -DUSE_THREADS=ON \
     -DUSE_WF_COAL=OFF \
     -DUSE_COHERENT_HEAP=OFF \
diff --git a/scripts/functional_tests/driver.sh b/scripts/functional_tests/driver.sh
index f93045fae8..e0e0d45fa6 100755
--- a/scripts/functional_tests/driver.sh
+++ b/scripts/functional_tests/driver.sh
@@ -56,52 +56,52 @@ case $2 in
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -s 1048576 -a 3 > $3/putnbi_n2_w1_z1_1MB.log
         check putnbi_n2_w1_z1_1MB
         echo "wg_get_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 28 > $3/wg_get_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 28 > $3/wg_get_n2_w1_z64_1MB.log
         check wg_get_n2_w1_z1_1MB
         echo "wg_getnbi_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 29 > $3/wg_getnbi_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 29 > $3/wg_getnbi_n2_w1_z64_1MB.log
         check wg_getnbi_n2_w1_z1_1MB
         echo "wg_put_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 30 > $3/wg_put_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 30 > $3/wg_put_n2_w1_z64_1MB.log
         check wg_put_n2_w1_z1_1MB
         echo "wg_putnbi_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_n2_w1_z64_1MB.log
         check wg_putnbi_n2_w1_z1_1MB
         echo "wg_get_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 28 > $3/wg_get_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 28 > $3/wg_get_tiled_n2_w2_z64_1MB.log
         check wg_get_tiled_n2_w1_z1_1MB
         echo "wg_getnbi_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 29 > $3/wg_getnbi_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 29 > $3/wg_getnbi_tiled_n2_w2_z64_1MB.log
         check wg_getnbi_tiled_n2_w1_z1_1MB
         echo "wg_put_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 30 > $3/wg_put_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 30 > $3/wg_put_tiled_n2_w2_z64_1MB.log
         check wg_put_tiled_n2_w1_z1_1MB
         echo "wg_putnbi_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 64 -s 1048576 -a 31 > $3/wg_putnbi_tiled_n2_w2_z64_1MB.log
         check wg_putnbi_tiled_n2_w1_z1_1MB
         echo "wave_get_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 32 > $3/wave_get_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 32 > $3/wave_get_n2_w1_z64_1MB.log
         check wave_get_n2_w1_z1_1MB
         echo "wave_getnbi_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 33 > $3/wave_getnbi_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 33 > $3/wave_getnbi_n2_w1_z64_1MB.log
         check wave_getnbi_n2_w1_z1_1MB
         echo "wave_put_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 34 > $3/wave_put_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 34 > $3/wave_put_n2_w1_z64_1MB.log
         check wave_put_n2_w1_z1_1MB
         echo "wave_putnbi_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 35 > $3/wave_putnbi_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 1048576 -a 35 > $3/wave_putnbi_n2_w1_z64_1MB.log
         check wave_putnbi_n2_w1_z1_1MB
         echo "wave_get_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 32 > $3/wave_get_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 32 > $3/wave_get_tiled_n2_w2_z128_1MB.log
         check wave_get_tiled_n2_w1_z1_1MB
         echo "wave_getnbi_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 33 > $3/wave_getnbi_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 33 > $3/wave_getnbi_tiled_n2_w2_z128_1MB.log
         check wave_getnbi_tiled_n2_w1_z1_1MB
         echo "wave_put_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 34 > $3/wave_put_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 34 > $3/wave_put_tiled_n2_w2_z128_1MB.log
         check wave_put_tiled_n2_w1_z1_1MB
         echo "wave_putnbi_tiled_n2_w1_z1_1MB"
-        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 35 > $3/wave_putnbi_tiled_n2_w1_z1_1MB.log
+        ROC_SHMEM_MAX_NUM_CONTEXTS=2 mpirun -np 2 $1 -w 2 -z 128 -s 1048576 -a 35 > $3/wave_putnbi_tiled_n2_w2_z128_1MB.log
         check wave_putnbi_tiled_n2_w1_z1_1MB
         echo "amofadd_n2_w1_z1"
         ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 1 -a 6 > $3/amofadd_n2_w1_z1.log
@@ -145,6 +145,54 @@ case $2 in
         echo "putnbi_n2_w16_z128_8B"
         ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 3 > $3/putnbi_n2_w16_z128_8B.log
         check putnbi_n2_w16_z128_8B
+        echo "wg_get_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 28 > $3/wg_get_n2_w1_z64_8B.log
+        check wg_get_n2_w1_z64_8B
+        echo "wg_getnbi_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 29 > $3/wg_getnbi_n2_w1_z64_8B.log
+        check wg_getnbi_n2_w1_z64_8B
+        echo "wg_put_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 30 > $3/wg_put_n2_w1_z64_8B.log
+        check wg_put_n2_w1_z64_8B
+        echo "wg_putnbi_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 31 > $3/wg_putnbi_n2_w1_z64_8B.log
+        check wg_putnbi_n2_w1_z64_8B
+        echo "wg_get_tiled_n2_w16_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 64 -s 8 -a 28 > $3/wg_get_tiled_n2_w16_z64_8B.log
+        check wg_get_tiled_n2_w16_z64_8B
+        echo "wg_getnbi_tiled_n2_w16_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 64 -s 8 -a 29 > $3/wg_getnbi_tiled_n2_w16_z64_8B.log
+        check wg_getnbi_tiled_n2_w16_z64_8B
+        echo "wg_put_tiled_n2_w16_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 64 -s 8 -a 30 > $3/wg_put_tiled_n2_w16_z64_8B.log
+        check wg_put_tiled_n2_w16_z64_8B
+        echo "wg_putnbi_tiled_n2_w16_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 64 -s 8 -a 31 > $3/wg_putnbi_tiled_n2_w16_z64_8B.log
+        check wg_putnbi_tiled_n2_w16_z64_8B
+        echo "wave_get_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 32 > $3/wave_get_n2_w1_z64_8B.log
+        check wave_get_n2_w1_z64_8B
+        echo "wave_getnbi_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 33 > $3/wave_getnbi_n2_w1_z64_8B.log
+        check wave_getnbi_n2_w1_z64_8B
+        echo "wave_put_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 34 > $3/wave_put_n2_w1_z64_8B.log
+        check wave_put_n2_w1_z64_8B
+        echo "wave_putnbi_n2_w1_z64_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=1 mpirun -np 2 $1 -w 1 -z 64 -s 8 -a 35 > $3/wave_putnbi_n2_w1_z64_8B.log
+        check wave_putnbi_n2_w1_z64_8B
+        echo "wave_get_tiled_n2_w16_z128_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 32 > $3/wave_get_tiled_n2_w16_z128_8B.log
+        check wave_get_tiled_n2_w16_z128_8B
+        echo "wave_getnbi_tiled_n2_w16_z128_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 33 > $3/wave_getnbi_tiled_n2_w16_z128_8B.log
+        check wave_getnbi_tiled_n2_w16_z128_8B
+        echo "wave_put_tiled_n2_w16_z128_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 34 > $3/wave_put_tiled_n2_w16_z128_8B.log
+        check wave_put_tiled_n2_w16_z128_8B
+        echo "wave_putnbi_tiled_n2_w16_z128_8B"
+        ROC_SHMEM_MAX_NUM_CONTEXTS=16 mpirun -np 2 $1 -w 16 -z 128 -s 8 -a 35 > $3/wave_putnbi_tiled_n2_w16_z128_8B.log
+        check wave_putnbi_tiled_n2_w16_z128_8B
         echo "amofadd_n2_w8_z1"
         ROC_SHMEM_MAX_NUM_CONTEXTS=8 mpirun -np 2 $1 -w 8 -z 1 -a 6 > $3/amofadd_n2_w8_z1.log
         check amofadd_n2_w8_z1
diff --git a/tests/functional_tests/extended_primitives.cpp b/tests/functional_tests/extended_primitives.cpp
index cc73d9c502..d6c6042faa 100644
--- a/tests/functional_tests/extended_primitives.cpp
+++ b/tests/functional_tests/extended_primitives.cpp
@@ -24,10 +24,12 @@
 
 #include <roc_shmem/roc_shmem.hpp>
 
+#include <numeric>
+
 using namespace rocshmem;
 
 /******************************************************************************
- * DEVICE TEST KERNELS
+ * DEVICE TEST KERNEL
  *****************************************************************************/
 __global__ void ExtendedPrimitiveTest(int loop, int skip, uint64_t *timer,
                                       char *s_buf, char *r_buf, int size,
@@ -37,6 +39,11 @@ __global__ void ExtendedPrimitiveTest(int loop, int skip, uint64_t *timer,
   roc_shmem_wg_init();
   roc_shmem_wg_ctx_create(ctx_type, &ctx);
 
+  /**
+   * Calculate start index for each work group for tiled version
+   * If the number of work groups is greater than 1, this kernel performs a
+   * tiled functional test
+  */
   uint64_t start;
   uint64_t idx = size * get_flat_grid_id();
   s_buf += idx;
@@ -78,8 +85,8 @@ __global__ void ExtendedPrimitiveTest(int loop, int skip, uint64_t *timer,
  *****************************************************************************/
 ExtendedPrimitiveTester::ExtendedPrimitiveTester(TesterArguments args)
     : Tester(args) {
-  s_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs);
-  r_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs);
+  s_buf = static_cast<int*>(roc_shmem_malloc(args.max_msg_size * args.num_wgs));
+  r_buf = static_cast<int*>(roc_shmem_malloc(args.max_msg_size * args.num_wgs));
 }
 
 ExtendedPrimitiveTester::~ExtendedPrimitiveTester() {
@@ -88,8 +95,9 @@ ExtendedPrimitiveTester::~ExtendedPrimitiveTester() {
 }
 
 void ExtendedPrimitiveTester::resetBuffers(uint64_t size) {
-  memset(s_buf, '0', size * args.num_wgs);
-  memset(r_buf, '1', size * args.num_wgs);
+  num_elems = (size * args.num_wgs) / sizeof(int);
+  std::iota(s_buf, s_buf + num_elems, 0);
+  memset(r_buf, 0, size * args.num_wgs);
 }
 
 void ExtendedPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
@@ -97,8 +105,8 @@ void ExtendedPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
   size_t shared_bytes = 0;
 
   hipLaunchKernelGGL(ExtendedPrimitiveTest, gridSize, blockSize, shared_bytes,
-                     stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
-                     _shmem_context);
+                     stream, loop, args.skip, timer, (char*)s_buf,
+                     (char*)r_buf, size, _type, _shmem_context);
 
   num_msgs = (loop + args.skip) * gridSize.x;
   num_timed_msgs = loop * gridSize.x;
@@ -110,10 +118,10 @@ void ExtendedPrimitiveTester::verifyResults(uint64_t size) {
                      : 1;
 
   if (args.myid == check_id) {
-    for (int i = 0; i < size * args.num_wgs; i++) {
-      if (r_buf[i] != '0') {
+    for (int i = 0; i < num_elems; i++) {
+      if (r_buf[i] != i) {
         fprintf(stderr, "Data validation error at idx %d\n", i);
-        fprintf(stderr, "Got %c, Expected %c \n", r_buf[i], '0');
+        fprintf(stderr, "Got %d, Expected %d \n", r_buf[i], i);
         exit(-1);
       }
     }
diff --git a/tests/functional_tests/extended_primitives.hpp b/tests/functional_tests/extended_primitives.hpp
index 33fdac2f8f..76225f07b0 100644
--- a/tests/functional_tests/extended_primitives.hpp
+++ b/tests/functional_tests/extended_primitives.hpp
@@ -42,8 +42,9 @@ class ExtendedPrimitiveTester : public Tester {
 
   virtual void verifyResults(uint64_t size) override;
 
-  char *s_buf = nullptr;
-  char *r_buf = nullptr;
+  int *s_buf = nullptr;
+  int *r_buf = nullptr;
+  int num_elems = 0;
 };
 
 #endif
diff --git a/tests/functional_tests/tester.cpp b/tests/functional_tests/tester.cpp
index cbd1ba89a5..e802c0b289 100644
--- a/tests/functional_tests/tester.cpp
+++ b/tests/functional_tests/tester.cpp
@@ -59,7 +59,7 @@ Tester::Tester(TesterArguments args) : args(args) {
   _shmem_context = args.shmem_context;
   CHECK_HIP(hipGetDevice(&device_id));
   CHECK_HIP(hipGetDeviceProperties(&deviceProps, device_id));
-  num_warps = args.wg_size / deviceProps.warpSize;
+  num_warps = (args.wg_size - 1) / deviceProps.warpSize + 1;
   CHECK_HIP(hipStreamCreate(&stream));
   CHECK_HIP(hipEventCreate(&start_event));
   CHECK_HIP(hipEventCreate(&stop_event));
@@ -76,6 +76,11 @@ Tester::~Tester() {
 std::vector<Tester*> Tester::create(TesterArguments args) {
   int rank = args.myid;
   std::vector<Tester*> testers;
+  hipDeviceProp_t deviceProps;
+  int device_id, numWarps;
+  CHECK_HIP(hipGetDevice(&device_id));
+  CHECK_HIP(hipGetDeviceProperties(&deviceProps, device_id));
+  numWarps = (args.wg_size - 1) / deviceProps.warpSize + 1;
 
   if (rank == 0) std::cout << "*** Creating Test: ";
 
@@ -510,7 +515,7 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
       return testers;
     case WAVEGetTestType:
       if (rank == 0) {
-        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+        if (args.num_wgs > 1 || numWarps > 1)
           std::cout << "Tiled Blocking WAVE level Gets***" << std::endl;
         else std::cout << "Blocking WAVE level Gets***" << std::endl;
       }
@@ -518,7 +523,7 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
       return testers;
     case WAVEGetNBITestType:
       if (rank == 0) {
-        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+        if (args.num_wgs > 1 || numWarps > 1)
           std::cout << "Tiled Non-Blocking WAVE level Gets***" << std::endl;
         else std::cout << "Non-Blocking WAVE level Gets***" << std::endl;
       }
@@ -526,7 +531,7 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
       return testers;
     case WAVEPutTestType:
       if (rank == 0) {
-        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+        if (args.num_wgs > 1 || numWarps > 1)
           std::cout << "Tiled Blocking WAVE level Puts***" << std::endl;
         else std::cout << "Blocking WAVE level Puts***" << std::endl;
       }
@@ -534,7 +539,7 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
       return testers;
     case WAVEPutNBITestType:
       if (rank == 0) {
-        if (args.num_wgs > 1 || args.wg_size / 64 > 1)
+        if (args.num_wgs > 1 || numWarps > 1)
           std::cout << "Tiled Non-Blocking WAVE level Puts***" << std::endl;
         else std::cout << "Non-Blocking WAVE level Puts***" << std::endl;
       }
@@ -612,10 +617,32 @@ void Tester::execute() {
     // data validation
     verifyResults(size);
 
+    /**
+     * Adjust size for *_wg and *_wave functions
+    */
+    uint64_t size_ = size;
+    TestType type = (TestType)args.algorithm;
+    switch (type) {
+      case WAVEGetTestType:
+      case WAVEGetNBITestType:
+      case WAVEPutTestType:
+      case WAVEPutNBITestType:
+        size_ *= (args.num_wgs * num_warps);
+        break;
+      case WGGetTestType:
+      case WGGetNBITestType:
+      case WGPutTestType:
+      case WGPutNBITestType:
+        size_ *= args.num_wgs;
+        break;
+      default:
+        break;
+    }
+
     barrier();
 
     if (_type != TeamCtxInfraTestType) {
-      print(size);
+      print(size_);
     }
   }
 }
@@ -707,8 +734,10 @@ uint64_t Tester::gpuCyclesToMicroseconds(uint64_t cycles) {
 uint64_t Tester::timerAvgInMicroseconds() {
   uint64_t sum = 0;
 
-  //TODO: Modify the calcuation for the Tiled version of puts and gets at 
-  //      wavefront level (bpotter/avinash)
+  /**
+   * TODO: (bpotter/avinash) Modify the calcuation for the Tiled version of
+   *       puts and gets at wavefront level
+  */
   for (int i = 0; i < args.num_wgs; i++) {
     sum += gpuCyclesToMicroseconds(timer[i]);
   }
diff --git a/tests/functional_tests/tester_arguments.cpp b/tests/functional_tests/tester_arguments.cpp
index 88e4b12cd6..1167fafaeb 100644
--- a/tests/functional_tests/tester_arguments.cpp
+++ b/tests/functional_tests/tester_arguments.cpp
@@ -103,6 +103,16 @@ TesterArguments::TesterArguments(int argc, char *argv[]) {
     case PutNBIMRTestType:
       min_msg_size = max_msg_size;
       break;
+    case WAVEGetTestType:
+    case WAVEGetNBITestType:
+    case WAVEPutTestType:
+    case WAVEPutNBITestType:
+    case WGGetTestType:
+    case WGGetNBITestType:
+    case WGPutTestType:
+    case WGPutNBITestType:
+      min_msg_size = 4;
+      break;
     default:
       break;
   }
diff --git a/tests/functional_tests/wave_level_primitives.cpp b/tests/functional_tests/wave_level_primitives.cpp
index bb7c299ef9..56b263ebf4 100644
--- a/tests/functional_tests/wave_level_primitives.cpp
+++ b/tests/functional_tests/wave_level_primitives.cpp
@@ -24,10 +24,12 @@
 
 #include <roc_shmem/roc_shmem.hpp>
 
+#include <numeric>
+
 using namespace rocshmem;
 
 /******************************************************************************
- * DEVICE TEST KERNELS
+ * DEVICE TEST KERNEL
  *****************************************************************************/
 __global__ void WaveLevelPrimitiveTest(int loop, int skip, uint64_t *timer,
                                       char *s_buf, char *r_buf, int size,
@@ -37,10 +39,15 @@ __global__ void WaveLevelPrimitiveTest(int loop, int skip, uint64_t *timer,
   roc_shmem_wg_init();
   roc_shmem_wg_ctx_create(ctx_type, &ctx);
 
+  /**
+   * Calculate start index for each wavefront for tiled version
+   * If the number of wavefronts is greater than 1, this kernel performs a
+   * tiled functional test
+  */
   uint64_t start;
   int wf_id = get_flat_block_id() / wf_size;
-  int offset = size * get_flat_grid_id() * (get_flat_block_size() / wf_size);
-  int idx = wf_id * size + offset;
+  int wg_offset = size * get_flat_grid_id() * (get_flat_block_size() / wf_size);
+  int idx = wf_id * size + wg_offset;
   s_buf += idx;
   r_buf += idx;
 
@@ -80,10 +87,10 @@ __global__ void WaveLevelPrimitiveTest(int loop, int skip, uint64_t *timer,
  *****************************************************************************/
 WaveLevelPrimitiveTester::WaveLevelPrimitiveTester(TesterArguments args)
     : Tester(args) {
-  s_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs
-           * num_warps);
-  r_buf = (char *)roc_shmem_malloc(args.max_msg_size * args.num_wgs
-           * num_warps);
+  s_buf = static_cast<int*>(
+      roc_shmem_malloc(args.max_msg_size * args.num_wgs * num_warps));
+  r_buf = static_cast<int*>(
+      roc_shmem_malloc(args.max_msg_size * args.num_wgs * num_warps));
 }
 
 WaveLevelPrimitiveTester::~WaveLevelPrimitiveTester() {
@@ -92,8 +99,9 @@ WaveLevelPrimitiveTester::~WaveLevelPrimitiveTester() {
 }
 
 void WaveLevelPrimitiveTester::resetBuffers(uint64_t size) {
-  memset(s_buf, '0', size * args.num_wgs * num_warps);
-  memset(r_buf, '1', size * args.num_wgs * num_warps);
+  num_elems = (size * args.num_wgs * num_warps) / sizeof(int);
+  std::iota(s_buf, s_buf + num_elems, 0);
+  memset(r_buf, 0, size * args.num_wgs * num_warps);
 }
 
 void WaveLevelPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
@@ -101,11 +109,12 @@ void WaveLevelPrimitiveTester::launchKernel(dim3 gridSize, dim3 blockSize,
   size_t shared_bytes = 0;
 
   hipLaunchKernelGGL(WaveLevelPrimitiveTest, gridSize, blockSize, shared_bytes,
-                     stream, loop, args.skip, timer, s_buf, r_buf, size, _type,
-                     _shmem_context, deviceProps.warpSize);
+                     stream, loop, args.skip, timer, (char*)s_buf,
+                     (char*)r_buf, size, _type, _shmem_context,
+                     deviceProps.warpSize);
 
-  num_msgs = (loop + args.skip) * gridSize.x;
-  num_timed_msgs = loop * gridSize.x;
+  num_msgs = (loop + args.skip) * gridSize.x * num_warps;
+  num_timed_msgs = loop * gridSize.x * num_warps;
 }
 
 void WaveLevelPrimitiveTester::verifyResults(uint64_t size) {
@@ -114,10 +123,10 @@ void WaveLevelPrimitiveTester::verifyResults(uint64_t size) {
                      : 1;
 
   if (args.myid == check_id) {
-    for (int i = 0; i < size * args.num_wgs * num_warps; i++) {
-      if (r_buf[i] != '0') {
+    for (int i = 0; i < num_elems; i++) {
+      if (r_buf[i] != i) {
         fprintf(stderr, "Data validation error at idx %d\n", i);
-        fprintf(stderr, "Got %c, Expected %c \n", r_buf[i], '0');
+        fprintf(stderr, "Got %d, Expected %d \n", r_buf[i], i);
         exit(-1);
       }
     }
diff --git a/tests/functional_tests/wave_level_primitives.hpp b/tests/functional_tests/wave_level_primitives.hpp
index 3b8a98541d..af65ac0a30 100644
--- a/tests/functional_tests/wave_level_primitives.hpp
+++ b/tests/functional_tests/wave_level_primitives.hpp
@@ -42,8 +42,9 @@ class WaveLevelPrimitiveTester : public Tester {
 
   virtual void verifyResults(uint64_t size) override;
 
-  char *s_buf = nullptr;
-  char *r_buf = nullptr;
+  int *s_buf = nullptr;
+  int *r_buf = nullptr;
+  int num_elems = 0;
 };
 
 #endif