Merge pull request #31 from BKP/ipc_bringup_fine_unit_09-26-24

Add IPC Simple Buffer Fine-grained Unit Tests

src/atomic.hpp

@@ -0,0 +1,141 @@
+/******************************************************************************
+ * Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifndef LIBRARY_SRC_ATOMIC_HPP
+#define LIBRARY_SRC_ATOMIC_HPP
+
+#include <hip/hip_runtime.h>
+
+namespace rocshmem {
+namespace detail {
+namespace atomic {
+
+typedef enum rocshmem_memory_scope {
+  memory_scope_thread = __HIP_MEMORY_SCOPE_SINGLETHREAD,
+  memory_scope_wavefront = __HIP_MEMORY_SCOPE_WAVEFRONT,
+  memory_scope_workgroup = __HIP_MEMORY_SCOPE_WORKGROUP,
+  memory_scope_agent = __HIP_MEMORY_SCOPE_AGENT,
+  memory_scope_system = __HIP_MEMORY_SCOPE_SYSTEM,
+} rocshmem_memory_scope;
+
+typedef enum rocshmem_memory_order {
+  memory_order_relaxed = __ATOMIC_RELAXED,
+  memory_order_consume = __ATOMIC_CONSUME,
+  memory_order_acquire = __ATOMIC_ACQUIRE,
+  memory_order_release = __ATOMIC_RELEASE,
+  memory_order_acq_rel = __ATOMIC_ACQ_REL,
+  memory_order_seq_cst = __ATOMIC_SEQ_CST
+} rocshmem_memory_order;
+
+struct rocshmem_memory_orders {
+  rocshmem_memory_order load {memory_order_acquire};
+  rocshmem_memory_order store {memory_order_release};
+  rocshmem_memory_order atomic {memory_order_acq_rel};
+  rocshmem_memory_order weak_cas_success {memory_order_acq_rel};
+  rocshmem_memory_order weak_cas_failure {memory_order_acq_rel};
+  rocshmem_memory_order strong_cas_success {memory_order_acq_rel};
+  rocshmem_memory_order strong_cas_failure {memory_order_acq_rel};
+};
+
+template <typename T, rocshmem_memory_scope s>
+__host__ __device__
+T load(const T* address, rocshmem_memory_orders o) {
+  return __hip_atomic_load(address, o.load, s);
+}
+
+template <typename T, rocshmem_memory_scope s>
+__host__ __device__
+void store(T* address, const T value, rocshmem_memory_orders o) {
+  return __hip_atomic_store(address, value, o.store, s);
+}
+
+template <typename T, rocshmem_memory_scope s>
+__host__ __device__
+bool compare_exchange_weak(T& expected, T desired, rocshmem_memory_orders o) {
+  return __hip_atomic_compare_exchange_weak(expected, desired, o.weak_cas_success, o.weak_cas_failure, s);
+}
+
+template <typename T, rocshmem_memory_scope s>
+__host__ __device__
+bool compare_exchange_strong(T& expected, T desired, rocshmem_memory_orders o) {
+  return __hip_atomic_compare_exchange_strong(expected, desired, o.strong_cas_success, o.strong_cas_failure, s);
+}
+
+template <class T, class U, rocshmem_memory_scope s>
+__host__ __device__
+T fetch_add(T* obj, U arg, rocshmem_memory_orders o) {
+  return __hip_atomic_fetch_add(obj, arg, o.atomic, s);
+}
+
+template <class T, class U, rocshmem_memory_scope s>
+__host__ __device__
+T fetch_sub(T* obj, U arg, rocshmem_memory_orders o) {
+  return __hip_atomic_fetch_sub(obj, arg, o.atomic, s);
+}
+
+template <class T, class U, rocshmem_memory_scope s>
+__host__ __device__
+T fetch_and(T* obj, U arg, rocshmem_memory_orders o) {
+  return __hip_atomic_fetch_and(obj, arg, o.atomic, s);
+}
+
+template <class T, class U, rocshmem_memory_scope s>
+__host__ __device__
+T fetch_or(T* obj, U arg, rocshmem_memory_orders o) {
+  return __hip_atomic_fetch_or(obj, arg, o, s);
+}
+
+template <class T, class U, rocshmem_memory_scope s>
+__host__ __device__
+T fetch_xor(T* obj, U arg, rocshmem_memory_orders o) {
+  return __hip_atomic_fetch_xor(obj, arg, o.atomic, s);
+}
+
+template <class T, class U, rocshmem_memory_scope s>
+__host__ __device__
+T fetch_max(T* obj, U arg, rocshmem_memory_orders o) {
+  return __hip_atomic_fetch_max(obj, arg, o.atomic, s);
+}
+
+template <class T, class U, rocshmem_memory_scope s>
+__host__ __device__
+T fetch_min(T* obj, U arg, rocshmem_memory_orders o) {
+  return __hip_atomic_fetch_min(obj, arg, o.atomic, s);
+}
+
+template <rocshmem_memory_scope s>
+__device__
+void threadfence() {
+  if constexpr (s == memory_scope_system) {
+    __threadfence_system();
+  } else if constexpr (s == memory_scope_agent) {
+    __threadfence();
+  } else if constexpr (s == memory_scope_workgroup) {
+    __threadfence_block();
+  }
+}
+
+} // namespace atomic
+} // namespace detail
+} // namespace rocshmem
+
+#endif  // LIBRARY_SRC_ATOMIC_HPP_

src/ipc_policy.hpp

@@ -64,38 +64,38 @@ class IpcOnImpl {
 
   __device__ void ipcCopy_wave(void *dst, void *src, size_t size);
 
-  __device__ void ipcFence() { __threadfence(); }
+  __device__ void ipcFence() { __threadfence_system(); }
 
   template <typename T>
   __device__ T ipcAMOFetchAdd(T *val, T value) {
-    return __hip_atomic_fetch_add(val, value, __ATOMIC_RELAXED,
-                                  __HIP_MEMORY_SCOPE_AGENT);
+    return __hip_atomic_fetch_add(val, value, __ATOMIC_SEQ_CST,
+                                  __HIP_MEMORY_SCOPE_SYSTEM);
   }
 
   template <typename T>
   __device__ T ipcAMOFetchCas(T *val, T cond, T value) {
-    __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_RELAXED,
-                                         __ATOMIC_RELAXED,
-                                         __HIP_MEMORY_SCOPE_AGENT);
+    __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_SEQ_CST,
+                                         __ATOMIC_SEQ_CST,
+                                         __HIP_MEMORY_SCOPE_SYSTEM);
     return cond;
   }
 
   template <typename T>
   __device__ void ipcAMOAdd(T *val, T value) {
-    __hip_atomic_fetch_add(val, value, __ATOMIC_RELAXED,
-                           __HIP_MEMORY_SCOPE_AGENT);
+    __hip_atomic_fetch_add(val, value, __ATOMIC_SEQ_CST,
+                           __HIP_MEMORY_SCOPE_SYSTEM);
   }
 
   template <typename T>
   __device__ void ipcAMOCas(T *val, T cond, T value) {
-    __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_RELAXED,
-                                         __ATOMIC_RELAXED,
-                                         __HIP_MEMORY_SCOPE_AGENT);
+    __hip_atomic_compare_exchange_strong(val, &cond, value, __ATOMIC_SEQ_CST,
+                                         __ATOMIC_SEQ_CST,
+                                         __HIP_MEMORY_SCOPE_SYSTEM);
   }
 
   template <typename T>
   __device__ void ipcAMOSet(T *val, T value) {
-    __hip_atomic_store(val, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    __hip_atomic_store(val, value, __ATOMIC_SEQ_CST, __HIP_MEMORY_SCOPE_SYSTEM);
   }
 
   __device__ void zero_byte_read(int pe) {

src/memory/notifier.hpp

@@ -20,59 +20,96 @@
  * IN THE SOFTWARE.
  *****************************************************************************/
 
-/**
- * @file notifier.hpp
- *
- * @brief Contains the notification memory space for threads to communicate
- * results with one another.
- *
- * Assume one thread does work on behalf of other threads (as a leader) and
- * that work needs to be communicated to the other threads as a result.
- * To expose the result, the threads need to share a memory space where the
- * result can be written. The leader thread writes the result out to this
- * memory space and all threads synchronize on it.
- *
- * This class allows the leader thread to notify other threads of the update.
- */
-
 #ifndef LIBRARY_SRC_MEMORY_NOTIFIER_HPP_
 #define LIBRARY_SRC_MEMORY_NOTIFIER_HPP_
 
 #include "../device_proxy.hpp"
 #include "../util.hpp"
+#include "../atomic.hpp"
 
 namespace rocshmem {
 
+template<detail::atomic::rocshmem_memory_scope scope>
 class Notifier {
- public:
-  __device__ uint64_t read() { return value_; }
 
-  __device__ void write(uint64_t val) {
-    if (is_thread_zero_in_block()) {
-      value_ = val;
-    }
-    publish();
+ public:
+  __device__ uint64_t load() {
+    return detail::atomic::load<uint64_t, scope>(&value_, orders_);
   }
 
-  __device__ void done() { __syncthreads(); }
+  __device__ void store(uint64_t val) {
+    detail::atomic::store<uint64_t, scope>(&value_, val, orders_);
+  }
 
- private:
-  __device__ void publish() {
-    if (is_thread_zero_in_block()) {
-      __threadfence();
+  __device__ void fence() {
+    detail::atomic::threadfence<scope>();
+  }
+
+  __device__ void sync() {
+    if constexpr (scope == detail::atomic::memory_scope_thread ||
+                  scope == detail::atomic::memory_scope_wavefront) {
+      return;
+    }
+    if constexpr (scope == detail::atomic::memory_scope_workgroup) {
+      __syncthreads();
+      return;
+    }
+    if constexpr (scope == detail::atomic::memory_scope_system) {
+      assert(false);
+      return;
+    }
+
+    uint32_t done {signal_ + 1};
+    __syncthreads();
+
+    uint32_t retval {0};
+    bool executor {!threadIdx.x && !threadIdx.y && !threadIdx.z};
+    if (executor) {
+      retval = detail::atomic::fetch_add<uint32_t, uint32_t, scope>(&count_, 1, orders_);
+      fence();
+    }
+    __syncthreads();
+
+    if (retval == ((gridDim.x * gridDim.y * gridDim.z) - 1)) {
+      if (executor) {
+        detail::atomic::store<uint32_t, scope>(&count_, 0, orders_);
+        fence();
+        detail::atomic::fetch_add<uint32_t, uint32_t, scope>(&signal_, 1, orders_);
+      }
+    }
+
+    if (executor) {
+      while (detail::atomic::load<uint32_t, scope>(&signal_, orders_) != done) {
+        ;
+      }
     }
     __syncthreads();
   }
 
+ private:
+  detail::atomic::rocshmem_memory_orders orders_{};
+
   uint64_t value_{};
+
+  uint32_t signal_ {};
+
+  uint32_t count_ {};
 };
 
-template <typename ALLOCATOR>
+template <typename ALLOCATOR, detail::atomic::rocshmem_memory_scope scope>
 class NotifierProxy {
-  using ProxyT = DeviceProxy<ALLOCATOR, Notifier, 1>;
+  using ProxyT = DeviceProxy<ALLOCATOR, Notifier<scope>>;
 
  public:
-  __host__ __device__ Notifier* get() { return proxy_.get(); }
+  NotifierProxy() {
+    new (proxy_.get()) Notifier<scope>();
+  }
+
+  ~NotifierProxy() {
+    proxy_.get()->~Notifier<scope>();
+  }
+
+  __host__ __device__ Notifier<scope>* get() { return proxy_.get(); }
 
  private:
   ProxyT proxy_{};

src/memory/slab_heap.cpp

@@ -75,9 +75,12 @@ __device__ void SlabHeap::malloc(void** ptr, size_t size) {
    * Notify other threads in block about the allocation result.
    */
   auto notifier{notifier_.get()};
-  notifier->write(ptr_deref_u64);
-  uint64_t notification_u64{notifier->read()};
-  notifier->done();
+  if (!threadIdx.x) {
+    notifier->store(ptr_deref_u64);
+    notifier->fence();
+  }
+  __syncthreads();
+  uint64_t notification_u64{notifier->load()};
 
   /*
    * Write to the ptr parameter (to return it back up the call stack).

src/memory/slab_heap.hpp

@@ -48,7 +48,7 @@ class SlabHeap {
   /**
    * @brief Helper type for notifier
    */
-  using NOTIFIER_PROXY_T = NotifierProxy<HIPAllocator>;
+  using NOTIFIER_PROXY_T = NotifierProxy<HIPAllocator, detail::atomic::memory_scope_workgroup>;
 
   /**
    * @brief Helper type for notifier

src/util.hpp

@@ -93,6 +93,7 @@ __device__ __forceinline__ bool is_thread_zero_in_block() {
 __device__ __forceinline__ bool is_block_zero_in_grid() {
   return hipBlockIdx_x == 0 && hipBlockIdx_y == 0 && hipBlockIdx_z == 0;
 }
+
 /*
  * Returns the number of threads in the caller's flattened thread block.
  */
@@ -100,6 +101,13 @@ __device__ __forceinline__ int get_flat_block_size() {
   return hipBlockDim_x * hipBlockDim_y * hipBlockDim_z;
 }
 
+/*
+ * Returns the number of threads in the caller's flattened grid.
+ */
+__device__ __forceinline__ int get_flat_grid_size() {
+  return get_flat_block_size() * hipGridDim_x * hipGridDim_y * hipGridDim_z;
+}
+
 /*
  * Returns the flattened thread index of the calling thread within its
  * thread block.