Substitute pow2bin allocator with a dlmalloc based allocator (#71)

* Add dlmalloc_strat allocator strategy
 - Use mspace variant to ease encapsulation
 - Make pow2bins and dlmalloc cmake selectable
* Add unit tester for dlmalloc, rework single_heap, pow2bins unit testers
accordingly
 - add dlmalloc get_used/get_avail, and have all strats allocators also have a get_used
 - Rework memallocator unit tests: bin size is per strat, alignment is verified in singleheap
* bugfix: dlmalloc exposed that the pingpong test would write past end of
allocation with -w 32
* iostream leakage/mixed usage of cerr and fprintf(stderr

---------

Signed-off-by: Aurelien Bouteiller <aurelien.bouteiller@amd.com>

src/memory/CMakeLists.txt

@@ -30,4 +30,5 @@ target_sources(
   PRIVATE
     single_heap.cpp
     memory_allocator.cpp
+    dlmalloc.cpp
 )

src/memory/dlmalloc.hpp

@@ -0,0 +1,186 @@
+/******************************************************************************
+ * Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifndef LIBRARY_SRC_MEMORY_DLMALLOC_HPP_
+#define LIBRARY_SRC_MEMORY_DLMALLOC_HPP_
+
+#include <cassert>
+#include <map>
+
+#include "../constants.hpp"
+#include "shmem_allocator_strategy.hpp"
+
+/**
+ * @file dlmalloc.hpp
+ *
+ * @brief Contains an allocator strategy for the heap using dlmalloc.
+ *
+ * This strategy uses DLMalloc to allocate/free from the symmetric heap
+ *
+ */
+
+namespace rocshmem {
+
+
+/*
+ * @brief an encapsulation class for the C-static functions inherited from dlmalloc
+ *
+ * @note only a subset of dlmalloc is exposed, not intended for external use
+ *
+ * @rationale static functions cannot be defined in the templated class DLAllocatorStategy
+ */
+class DLMalloc {
+ public:
+  typedef void* mspace;
+  static size_t destroy_mspace(mspace msp);
+  static mspace create_mspace_with_base(void* base, size_t capacity, int locked);
+  static void* mspace_malloc(mspace msp, size_t bytes);
+  static void mspace_free(mspace msp, void* mem);
+  static void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
+  static size_t mspace_footprint(mspace msp);
+  static size_t mspace_max_footprint(mspace msp);
+  static size_t mspace_avail(mspace msp);
+  static size_t mspace_used(mspace msp);
+};
+
+template <typename HM_T>
+class DLAllocatorStrategy : public ShmemAllocatorStrategy {
+
+public:
+  /**
+   * @brief Required for default construction of other objects
+   *
+   * @note Not intended for direct usage.
+   */
+  DLAllocatorStrategy() = default;
+
+  /**
+   * @brief Primary constructor type
+   *
+   * Construct the dlmalloc mspace
+   *
+   * @param[in] Raw pointer to heap memory type
+   */
+  explicit DLAllocatorStrategy(HM_T* heap_mem) {
+    mspace_ = DLMalloc::create_mspace_with_base(heap_mem->get_ptr(), heap_mem->get_size(), false);
+  }
+
+  /**
+   * @brief Primary destructor
+   *
+   * Destroy the dlmalloc mspace
+   */
+  ~DLAllocatorStrategy() {
+    if(mspace_) {
+      DLMalloc::destroy_mspace(mspace_);
+    }
+  }
+
+  /**
+   * @brief Allocates memory from the heap
+   *
+   * @param[in, out] Address of raw pointer (&pointer_to_char)
+   * @param[in] Size in bytes of memory allocation
+   */
+  void alloc(char** ptr, size_t request_size) override {
+    assert(ptr);
+    *ptr = nullptr;
+
+    if (!request_size) {
+      return;
+    }
+    *ptr = static_cast<char*>(DLMalloc::mspace_malloc(mspace_, request_size));
+  }
+
+  /**
+   * @brief Allocates memory from the heap
+   *
+   * @param[in, out] Address of raw pointer (&pointer_to_char)
+   * @param[in] Size in bytes of memory allocation
+   *
+   * @note Not implemented
+   */
+  __device__ void alloc([[maybe_unused]] char** ptr,
+                        [[maybe_unused]] size_t request_size) override {}
+
+  /**
+   * @brief Frees memory from the heap
+   *
+   * Released memory is tracked by bookkeeping structures within this class.
+   *
+   * @param[in] Raw pointer to heap memory
+   *
+   */
+  void free(char* ptr) override {
+    DLMalloc::mspace_free(mspace_, ptr);
+  }
+
+  /**
+   * @brief Frees memory from the heap
+   *
+   * Released memory is tracked by bookkeeping structures within this class.
+   *
+   * @param[in] Raw pointer to heap memory
+   *
+   * @note Not implemented
+   */
+  __device__ void free([[maybe_unused]] char* ptr) override {}
+
+  /**
+   * @brief Used heap memory
+   *
+   * @return memory size
+   *
+   * @note The used size may be larger than the sum of the user allocation sizes
+   * (due to chunk tracking overhead and alignment).
+   *
+   */
+  size_t get_used() override {
+    size_t size{0};
+    size = DLMalloc::mspace_used(mspace_);
+    return size;
+  }
+
+  /**
+   * @brief Available heap memory
+   *
+   * @return memory size
+   *
+   * @note The available size may be smaller than the total heap size minus the sum
+   * of user allocation sizes (due to chunk tracking overhead and alignment).
+   */
+  size_t get_avail() {
+    size_t size{0};
+    size = DLMalloc::mspace_avail(mspace_);
+    return size;
+  }
+
+ private:
+  DLMalloc::mspace mspace_{nullptr};
+};
+
+
+}  // namespace rocshmem
+
+#endif  // LIBRARY_SRC_MEMORY_DLMALLOC_HPP_

src/memory/pow2_bins.hpp

@@ -164,6 +164,17 @@ class Pow2Bins : public ShmemAllocatorStrategy {
    */
   __device__ void free([[maybe_unused]] char* ptr) override {}
 
+  /**
+   * @brief Used memory from the heap
+   *
+   * Sum of all proffered_ memory sizes
+   *
+   * @return memory size
+   */
+  size_t get_used() override {
+    return amount_proffered();
+  }
+
   /**
    * @brief Sum of all proffered_ memory sizes
    *

src/memory/shmem_allocator_strategy.hpp

@@ -85,6 +85,17 @@ class ShmemAllocatorStrategy {
    * @param[in] Raw pointer to symmetric heap memory
    */
   __device__ virtual void free(char* ptr) = 0;
+
+  /**
+   * @brief Used heap memory
+   *
+   * @return memory size
+   *
+   * @note The used size may be larger than the sum of the user allocation sizes
+   * (due to chunk tracking overhead and alignment).
+   *
+   */
+  virtual size_t get_used() = 0;
 };
 
 }  // namespace rocshmem

src/memory/single_heap.cpp

@@ -61,7 +61,7 @@ char* SingleHeap::get_base_ptr() { return heap_mem_.get_ptr(); }
 
 size_t SingleHeap::get_size() { return heap_mem_.get_size(); }
 
-size_t SingleHeap::get_used() { return strat_.amount_proffered(); }
+size_t SingleHeap::get_used() { return strat_.get_used(); }
 
 size_t SingleHeap::get_avail() { return get_size() - get_used(); }
 

src/memory/single_heap.hpp

@@ -25,10 +25,16 @@
 #ifndef LIBRARY_SRC_MEMORY_SINGLE_HEAP_HPP_
 #define LIBRARY_SRC_MEMORY_SINGLE_HEAP_HPP_
 
-#include "address_record.hpp"
 #include "heap_memory.hpp"
 #include "heap_type.hpp"
+#if defined USE_ALLOC_DLMALLOC
+#include "dlmalloc.hpp"
+#elif defined USE_ALLOC_POW2BINS
+#include "address_record.hpp"
 #include "pow2_bins.hpp"
+#else
+#error "You need to have one of USE_ALLOC_DLMALLOC, USE_ALLOC_POW2BINS set to ON"
+#endif
 
 /**
  * @file single_heap.hpp
@@ -42,15 +48,21 @@
 namespace rocshmem {
 
 class SingleHeap {
+#if defined USE_ALLOC_DLMALLOC
+  /**
+   * @brief Helper type for allocation strategy
+   */
+  using STRAT_T = DLAllocatorStrategy<HEAP_T>;
+#elif defined USE_ALLOC_POW2BINS
   /**
    * @brief Helper type for address records
    */
   using AR_T = AddressRecord;
-
   /**
    * @brief Helper type for allocation strategy
    */
   using STRAT_T = Pow2Bins<AR_T, HEAP_T>;
+#endif // defined USE_ALLOC_POW2BINS
 
  public:
   /**