Update Barrier_All and Sync_All APIs (#72)

* Fix deadlock in `rocshmem_ctx_wg_barrier_all` API in IPC conduit by adding per-context pSync buffers and context IDs - Added separate pSync buffers for each device context - Resolved deadlock when invoking barrier API (`rocshmem_ctx_wg_barrier_all`) concurrently from multiple contexts * Update barrier_all functional tests for multi-context support * Add thread, wavefront, and workgroup-level barrier_all APIs in IPC and RO conduits - Implemented barrier_all APIs at thread, wavefront, and workgroup granularity - Added support in both IPC and RO conduits - Updated functional tests to cover all `barrier_all` APIs * Add thread, wavefront, and workgroup-level sync_all APIs in IPC and RO conduits - Implemented sync_all APIs for thread, wavefront, and workgroup scopes - Added support into both IPC and RO conduits - Added functional tests to cover all `sync_all` APIs [ROCm/rocshmem commit: c652f58cef]
2025-04-02 11:58:55 -05:00
parent 0cde5f53dc
commit 426bbf525b
22 changed files with 508 additions and 53 deletions
@@ -30,9 +30,12 @@ using namespace rocshmem;
 * DEVICE TEST KERNEL
 *****************************************************************************/
 __global__ void BarrierAllTest(int loop, int skip, long long int *start_time,
-                               long long int *end_time) {
+                               long long int *end_time, TestType type,
+                               int wf_size) {
  __shared__ rocshmem_ctx_t ctx;
+  int t_id  = get_flat_block_id();
  int wg_id = get_flat_grid_id();
+  int wf_id = t_id / wf_size;

  rocshmem_wg_init();
  rocshmem_wg_ctx_create(ROCSHMEM_CTX_WG_PRIVATE, &ctx);
@@ -42,17 +45,25 @@ __global__ void BarrierAllTest(int loop, int skip, long long int *start_time,
      start_time[wg_id] = wall_clock64();
    }

-    __syncthreads();
-
-    /**
-     * The function `rocshmem_ctx_wg_barrier_all` should be called from only
-     * one group within the grid to avoid unintended behavior.
-     */
-    if (is_block_zero_in_grid()) {
-      rocshmem_ctx_wg_barrier_all(ctx);
+    switch (type) {
+      case BarrierAllTestType:
+        if(t_id == 0) {
+          rocshmem_ctx_barrier_all(ctx);
+        }
+        break;
+      case WAVEBarrierAllTestType:
+        if(wf_id == 0) {
+          rocshmem_ctx_wave_barrier_all(ctx);
+        }
+        break;
+      case WGBarrierAllTestType:
+        rocshmem_ctx_wg_barrier_all(ctx);
+        break;
+      default:
+        break;
    }
+    __syncthreads();
  }
-  __syncthreads();

  if (hipThreadIdx_x == 0) {
    end_time[wg_id] = wall_clock64();
@@ -74,10 +85,10 @@ void BarrierAllTester::launchKernel(dim3 gridSize, dim3 blockSize, int loop,
  size_t shared_bytes = 0;

  hipLaunchKernelGGL(BarrierAllTest, gridSize, blockSize, shared_bytes, stream,
-                     loop, args.skip, start_time, end_time);
+                     loop, args.skip, start_time, end_time, _type, wf_size);

-  num_msgs = loop + args.skip;
-  num_timed_msgs = loop;
+  num_msgs = (loop + args.skip) * gridSize.x;
+  num_timed_msgs = loop * gridSize.x;
 }

 void BarrierAllTester::resetBuffers(uint64_t size) {}
@@ -0,0 +1,96 @@
+/******************************************************************************
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+#include "sync_all_tester.hpp"
+
+#include <rocshmem/rocshmem.hpp>
+
+using namespace rocshmem;
+
+/******************************************************************************
+ * DEVICE TEST KERNEL
+ *****************************************************************************/
+__global__ void SyncAllTest(int loop, int skip, long long int *start_time,
+                               long long int *end_time, TestType type,
+                               int wf_size) {
+  __shared__ rocshmem_ctx_t ctx;
+  int t_id  = get_flat_block_id();
+  int wg_id = get_flat_grid_id();
+  int wf_id = t_id / wf_size;
+
+  rocshmem_wg_init();
+  rocshmem_wg_ctx_create(ROCSHMEM_CTX_WG_PRIVATE, &ctx);
+
+  for (int i = 0; i < loop + skip; i++) {
+    if (hipThreadIdx_x == 0 && i == skip) {
+      start_time[wg_id] = wall_clock64();
+    }
+
+    switch (type) {
+      case SyncAllTestType:
+        if(t_id == 0) {
+          rocshmem_ctx_sync_all(ctx);
+        }
+        break;
+      case WAVESyncAllTestType:
+        if(wf_id == 0) {
+          rocshmem_ctx_wave_sync_all(ctx);
+        }
+        break;
+      case WGSyncAllTestType:
+        rocshmem_ctx_wg_sync_all(ctx);
+        break;
+      default:
+        break;
+    }
+    __syncthreads();
+  }
+
+  if (hipThreadIdx_x == 0) {
+    end_time[wg_id] = wall_clock64();
+  }
+
+  rocshmem_wg_ctx_destroy(&ctx);
+  rocshmem_wg_finalize();
+}
+
+/******************************************************************************
+ * HOST TESTER CLASS METHODS
+ *****************************************************************************/
+SyncAllTester::SyncAllTester(TesterArguments args) : Tester(args) {}
+
+SyncAllTester::~SyncAllTester() {}
+
+void SyncAllTester::launchKernel(dim3 gridSize, dim3 blockSize, int loop,
+                                    uint64_t size) {
+  size_t shared_bytes = 0;
+
+  hipLaunchKernelGGL(SyncAllTest, gridSize, blockSize, shared_bytes, stream,
+                     loop, args.skip, start_time, end_time, _type, wf_size);
+
+  num_msgs = (loop + args.skip) * gridSize.x;
+  num_timed_msgs = loop * gridSize.x;
+}
+
+void SyncAllTester::resetBuffers(uint64_t size) {}
+
+void SyncAllTester::verifyResults(uint64_t size) {}
@@ -0,0 +1,50 @@
+/******************************************************************************
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifndef _BARRIER_ALL_TESTER_HPP_
+#define _BARRIER_ALL_TESTER_HPP_
+
+#include "tester.hpp"
+
+/******************************************************************************
+ * DEVICE TEST KERNEL
+ *****************************************************************************/
+__global__ void SyncAllTest(TestType type);
+
+/******************************************************************************
+ * HOST TESTER CLASS
+ *****************************************************************************/
+class SyncAllTester : public Tester {
+ public:
+  explicit SyncAllTester(TesterArguments args);
+  virtual ~SyncAllTester();
+
+ protected:
+  virtual void resetBuffers(uint64_t size) override;
+
+  virtual void launchKernel(dim3 gridSize, dim3 blockSize, int loop,
+                            uint64_t size) override;
+
+  virtual void verifyResults(uint64_t size) override;
+};
+
+#endif
@@ -41,15 +41,6 @@ __global__ void SyncTest(int loop, int skip, long long int *start_time,

    __syncthreads();
    switch (type) {
-      case SyncAllTestType:
-        /**
-        * The function `rocshmem_ctx_wg_sync_all` should be called from only
-        * one group within the grid to avoid unintended behavior.
-        */
-        if (is_block_zero_in_grid()) {
-          rocshmem_ctx_wg_sync_all(ctx);
-        }
-        break;
      case SyncTestType:
        rocshmem_ctx_wg_team_sync(ctx, teams[wg_id]);
        break;
@@ -43,10 +43,11 @@
 #include "random_access_tester.hpp"
 #include "shmem_ptr_tester.hpp"
 #include "signaling_operations_tester.hpp"
+#include "sync_all_tester.hpp"
 #include "sync_tester.hpp"
 #include "team_alltoall_tester.hpp"
-#include "team_broadcast_tester.hpp"
 #include "team_barrier_tester.hpp"
+#include "team_broadcast_tester.hpp"
 #include "team_ctx_infra_tester.hpp"
 #include "team_ctx_primitive_tester.hpp"
 #include "team_fcollect_tester.hpp"
@@ -317,6 +318,14 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
      if (rank == 0) std::cout << "Barrier_All ###" << std::endl;
      testers.push_back(new BarrierAllTester(args));
      return testers;
+    case WAVEBarrierAllTestType:
+      if (rank == 0) std::cout << "WAVE Barrier_All ###" << std::endl;
+      testers.push_back(new BarrierAllTester(args));
+      return testers;
+    case WGBarrierAllTestType:
+      if (rank == 0) std::cout << "WG Barrier_All ###" << std::endl;
+      testers.push_back(new BarrierAllTester(args));
+      return testers;
    case TeamBarrierTestType:
      if (rank == 0) std::cout << "Team Barrier Test ###" << std::endl;
      testers.push_back(new TeamBarrierTester(args));
@@ -325,6 +334,14 @@ std::vector<Tester*> Tester::create(TesterArguments args) {
      if (rank == 0) std::cout << "SyncAll ###" << std::endl;
      testers.push_back(new SyncTester(args));
      return testers;
+    case WAVESyncAllTestType:
+      if (rank == 0) std::cout << "WAVE SyncAll ###" << std::endl;
+      testers.push_back(new SyncTester(args));
+      return testers;
+    case WGSyncAllTestType:
+      if (rank == 0) std::cout << "WG SyncAll ###" << std::endl;
+      testers.push_back(new SyncTester(args));
+      return testers;
    case SyncTestType:
      if (rank == 0) std::cout << "Sync ###" << std::endl;
      testers.push_back(new SyncTester(args));
@@ -510,7 +527,9 @@ bool Tester::peLaunchesKernel() {
                (_type == TeamBroadcastTestType) || (_type == TeamCtxInfraTestType) ||
                (_type == TeamAllToAllTestType) || (_type == TeamFCollectTestType) ||
                (_type == PingPongTestType) || (_type == BarrierAllTestType) ||
+                (_type == WAVEBarrierAllTestType) || (_type == WGBarrierAllTestType) ||
                (_type == SyncTestType) || (_type == SyncAllTestType) ||
+                (_type == WAVESyncAllTestType) || (_type == WGSyncAllTestType) ||
                (_type == RandomAccessTestType) || (_type == PingAllTestType) ||
                (_type == TeamBarrierTestType);

@@ -99,6 +99,10 @@ enum TestType {
  DefaultCTXPutNBITestType = 62,
  DefaultCTXPTestType = 63,
  DefaultCTXGTestType = 64,
+  WAVEBarrierAllTestType = 65,
+  WGBarrierAllTestType = 66,
+  WAVESyncAllTestType = 67,
+  WGSyncAllTestType = 68,
 };

 enum OpType { PutType = 0, GetType = 1 };
@@ -85,8 +85,12 @@ TesterArguments::TesterArguments(int argc, char *argv[]) {
    case AMO_IncTestType:
    case AMO_FetchTestType:
    case BarrierAllTestType:
+    case WAVEBarrierAllTestType:
+    case WGBarrierAllTestType:
    case TeamBarrierTestType:
    case SyncAllTestType:
+    case WAVESyncAllTestType:
+    case WGSyncAllTestType:
    case SyncTestType:
    case ShmemPtrTestType:
      min_msg_size = 8;
@@ -133,7 +137,9 @@ void TesterArguments::get_rocshmem_arguments() {
  myid = rocshmem_my_pe();

  TestType type = (TestType)algorithm;
-  if ((type != BarrierAllTestType) && (type != SyncAllTestType) &&
+  if ((type != BarrierAllTestType) && (type != WAVEBarrierAllTestType) &&
+      (type != WGBarrierAllTestType) && (type != SyncAllTestType) &&
+      (type != WAVESyncAllTestType) && (type != WGSyncAllTestType) &&
      (type != SyncTestType) && (type != TeamAllToAllTestType) &&
      (type != TeamFCollectTestType) && (type != TeamReductionTestType) &&
      (type != TeamBroadcastTestType) && (type != PingAllTestType) &&