Add pooling for Signal ABI blocks (SharedSignal).

Makes better use of memory and greatly reduces mmap count. Change-Id: Ib444cd1ccd144986adbcc7cec297a966e2c08bc7 [ROCm/ROCR-Runtime commit: 8323b2e1d7]
2018-11-08 19:09:24 -06:00
commit ed18ee7f38
@@ -68,35 +68,89 @@ class BaseShared {
  static std::function<void(void*)> free_;
 };

-/// @brief Container for object located in GPU visible host memory.
-/// Alignment defaults to __alignof(T) but may be increased.
-template <typename T, size_t Align = 0> class Shared final : private BaseShared {
+/// @brief Default Allocator for Shared.  Ensures allocations are whole pages.
+template <typename T> class PageAllocator : private BaseShared {
 public:
-  Shared() {
-    assert(allocate_ != nullptr && free_ != nullptr &&
-           "Shared object allocator is not set");
-    static_assert((__alignof(T) <= Align) || (Align == 0),
-                  "Align is less than alignof(T)");
+  __forceinline static T* alloc() {
+    T* ret = reinterpret_cast<T*>(allocate_(AlignUp(sizeof(T), 4096), 4096, 0));
+    if (ret == nullptr) throw std::bad_alloc();

-    shared_object_ =
-        reinterpret_cast<T*>(allocate_(sizeof(T), Max(__alignof(T), Align), 0));
-    if (shared_object_ == nullptr) throw std::bad_alloc();
+    MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(ret); });

-    MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(shared_object_); });
-
-    new (shared_object_) T;
+    new (ret) T;

    throwGuard.Dismiss();
+    return ret;
+  }
+
+  __forceinline static void free(T* ptr) {
+    if (ptr != nullptr) {
+      ptr->~T();
+      free_(ptr);
+    }
+  }
+};
+
+/// @brief Container for object located in GPU visible host memory.
+/// If a custom allocator is not given then data will be placed in dedicated pages.
+template <typename T, typename Allocator = PageAllocator<T>>
+class Shared final : private BaseShared {
+ public:
+  explicit Shared(Allocator* pool = nullptr) : pool_(pool) {
+    assert(allocate_ != nullptr && free_ != nullptr &&
+           "Shared object allocator is not set");
+
+    if (pool_)
+      shared_object_ = pool_->alloc();
+    else
+      shared_object_ = PageAllocator<T>::alloc();
+  }
+
+  ~Shared() {
+    assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set");
+
+    if (pool_)
+      pool_->free(shared_object_);
+    else
+      PageAllocator<T>::free(shared_object_);
+  }
+
+  Shared(Shared&& rhs) {
+    this->~Shared();
+    shared_object_ = rhs.shared_object_;
+    rhs.shared_object_ = nullptr;
+    pool_ = rhs.pool_;
+    rhs.pool_ = nullptr;
+  }
+  Shared& operator=(Shared&& rhs) {
+    this->~Shared();
+    shared_object_ = rhs.shared_object_;
+    rhs.shared_object_ = nullptr;
+    pool_ = rhs.pool_;
+    rhs.pool_ = nullptr;
+    return *this;
+  }
+
+  T* shared_object() const { return shared_object_; }
+
+ private:
+  T* shared_object_;
+  Allocator* pool_;
+};
+
+template <typename T> class Shared<T, PageAllocator<T>> final : private BaseShared {
+ public:
+  Shared() {
+    assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set");
+
+    shared_object_ = PageAllocator<T>::alloc();
  }

  ~Shared() {
    assert(allocate_ != nullptr && free_ != nullptr &&
           "Shared object allocator is not set");

-    if (shared_object_ != nullptr) {
-      shared_object_->~T();
-      free_(shared_object_);
-    }
+    PageAllocator<T>::free(shared_object_);
  }

  Shared(Shared&& rhs) {
@@ -170,7 +170,7 @@ class DefaultSignal : private LocalSignal, public BusyWaitSignal {

  /// @brief See base class Signal.
  explicit DefaultSignal(hsa_signal_value_t initial_value, bool enableIPC = false)
-      : LocalSignal(initial_value), BusyWaitSignal(signal(), enableIPC) {}
+      : LocalSignal(initial_value, enableIPC), BusyWaitSignal(signal(), enableIPC) {}

 protected:
  bool _IsA(rtti_t id) const {
@@ -130,7 +130,7 @@ class LocalQueue {
  SharedQueue* queue() const { return local_queue_.shared_object(); }

 private:
-  Shared<SharedQueue, AMD_QUEUE_ALIGN_BYTES> local_queue_;
+  Shared<SharedQueue> local_queue_;
 };

 /// @brief Class Queue which encapsulate user mode queues and
@@ -333,6 +333,8 @@ class Runtime {

  void InternalQueueCreateNotify(const hsa_queue_t* queue, hsa_agent_t agent);

+  SharedSignalPool_t* GetSharedSignalPool() { return &SharedSignalPool; }
+
 protected:
  static void AsyncEventsLoop(void*);

@@ -505,6 +507,9 @@ class Runtime {
  // Track environment variables.
  Flag flag_;

+  // Pools memory for SharedSignal (Signal ABI blocks)
+  SharedSignalPool_t SharedSignalPool;
+
  // Frees runtime memory when the runtime library is unloaded if safe to do so.
  // Failure to release the runtime indicates an incorrect application but is
  // common (example: calls library routines at process exit).
@@ -48,12 +48,13 @@
 #include <map>
 #include <functional>
 #include <memory>
+#include <vector>
+#include <utility>

 #include "hsakmt.h"

 #include "core/common/shared.h"

-#include "core/inc/runtime.h"
 #include "core/inc/checked.h"
 #include "core/inc/exceptions.h"

@@ -112,16 +113,36 @@ static_assert(std::is_standard_layout<SharedSignal>::value,
 static_assert(std::is_trivially_destructible<SharedSignal>::value,
              "SharedSignal must not be modified on delete for IPC use.");

+/// @brief Pool class for SharedSignal suitable for use with Shared.
+class SharedSignalPool_t : private BaseShared {
+ public:
+  SharedSignalPool_t() : block_size_(minblock_) {}
+  ~SharedSignalPool_t() { clear(); }
+
+  SharedSignal* alloc();
+  void free(SharedSignal* ptr);
+  void clear();
+
+ private:
+  static const size_t minblock_ = 4096 / sizeof(SharedSignal);
+  KernelMutex lock_;
+  std::vector<SharedSignal*> free_list_;
+  std::vector<std::pair<void*, size_t>> block_list_;
+  size_t block_size_;
+};
+
 class LocalSignal {
 public:
+  // Temporary, for legacy tools lib support.
  explicit LocalSignal(hsa_signal_value_t initial_value) {
    local_signal_.shared_object()->amd_signal.value = initial_value;
  }
+  LocalSignal(hsa_signal_value_t initial_value, bool exportable);

  SharedSignal* signal() const { return local_signal_.shared_object(); }

 private:
-  Shared<SharedSignal, AMD_SIGNAL_ALIGN_BYTES> local_signal_;
+  Shared<SharedSignal, SharedSignalPool_t> local_signal_;
 };

 /// @brief An abstract base class which helps implement the public hsa_signal_t
@@ -80,7 +80,7 @@ int AqlQueue::rtti_id_ = 0;
 AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, ScratchInfo& scratch,
                   core::HsaEventCallback callback, void* err_data, bool is_kv)
    : Queue(),
-      LocalSignal(0),
+      LocalSignal(0, false),
      DoorbellSignal(signal()),
      ring_buf_(nullptr),
      ring_buf_alloc_bytes_(0),
@@ -69,7 +69,7 @@ int InterceptQueue::rtti_id_ = 0;

 InterceptQueue::InterceptQueue(std::unique_ptr<Queue> queue)
    : QueueProxy(std::move(queue)),
-      LocalSignal(0),
+      LocalSignal(0, false),
      DoorbellSignal(signal()),
      next_packet_(0),
      retry_index_(0),
@@ -69,7 +69,7 @@ int InterruptSignal::rtti_id_ = 0;
 void InterruptSignal::DestroyEvent(HsaEvent* evt) { hsaKmtDestroyEvent(evt); }

 InterruptSignal::InterruptSignal(hsa_signal_value_t initial_value, HsaEvent* use_event)
-    : LocalSignal(initial_value), Signal(signal()) {
+    : LocalSignal(initial_value, false), Signal(signal()) {
  if (use_event != NULL) {
    event_ = use_event;
    free_event_ = false;
@@ -1219,6 +1219,8 @@ void Runtime::Unload() {
  core::InterruptSignal::DestroyEvent(vm_fault_event_);
  vm_fault_event_ = nullptr;

+  SharedSignalPool.clear();
+
  DestroyAgents();

  CloseTools();
@@ -47,12 +47,84 @@

 #include <algorithm>
 #include "core/util/timer.h"
+#include "core/inc/runtime.h"

 namespace core {

 KernelMutex Signal::ipcLock_;
 std::map<decltype(hsa_signal_t::handle), Signal*> Signal::ipcMap_;

+void SharedSignalPool_t::clear() {
+  ifdebug {
+    size_t capacity = 0;
+    for (auto& block : block_list_) capacity += block.second;
+    if (capacity != free_list_.size())
+      debug_print("Warning: Resource leak detected by SharedSignalPool, %ld Signals leaked.\n",
+                  capacity - free_list_.size());
+  }
+
+  for (auto& block : block_list_) free_(block.first);
+  block_list_.clear();
+  free_list_.clear();
+}
+
+SharedSignal* SharedSignalPool_t::alloc() {
+  ScopedAcquire<KernelMutex> lock(&lock_);
+  if (free_list_.empty()) {
+    SharedSignal* block = reinterpret_cast<SharedSignal*>(
+        allocate_(block_size_ * sizeof(SharedSignal), __alignof(SharedSignal), 0));
+    if (block == nullptr) {
+      block_size_ = minblock_;
+      block = reinterpret_cast<SharedSignal*>(
+          allocate_(block_size_ * sizeof(SharedSignal), __alignof(SharedSignal), 0));
+      if (block == nullptr) throw std::bad_alloc();
+    }
+
+    MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(block); });
+    block_list_.push_back(std::make_pair(block, block_size_));
+    throwGuard.Dismiss();
+
+
+    for (int i = 0; i < block_size_; i++) {
+      free_list_.push_back(&block[i]);
+    }
+
+    block_size_ *= 2;
+  }
+
+  SharedSignal* ret = free_list_.back();
+  new (ret) SharedSignal();
+  free_list_.pop_back();
+  return ret;
+}
+
+void SharedSignalPool_t::free(SharedSignal* ptr) {
+  if (ptr == nullptr) return;
+
+  ptr->~SharedSignal();
+  ScopedAcquire<KernelMutex> lock(&lock_);
+
+  ifdebug {
+    bool valid = false;
+    for (auto& block : block_list_) {
+      if ((block.first <= ptr) &&
+          (uintptr_t(ptr) < uintptr_t(block.first) + block.second * sizeof(SharedSignal))) {
+        valid = true;
+        break;
+      }
+    }
+    assert(valid && "Object does not belong to pool.");
+  }
+
+  free_list_.push_back(ptr);
+}
+
+LocalSignal::LocalSignal(hsa_signal_value_t initial_value, bool exportable)
+    : local_signal_(exportable ? nullptr
+                               : core::Runtime::runtime_singleton_->GetSharedSignalPool()) {
+  local_signal_.shared_object()->amd_signal.value = initial_value;
+}
+
 void Signal::registerIpc() {
  ScopedAcquire<KernelMutex> lock(&ipcLock_);
  auto handle = Convert(this);
@@ -86,11 +86,12 @@ class Flag {
    tools_lib_names_ = os::GetEnvVar("HSA_TOOLS_LIB");

    var = os::GetEnvVar("HSA_TOOLS_REPORT_LOAD_FAILURE");
-#ifdef NDEBUG
-    report_tool_load_failures_ = (var == "1") ? true : false;
-#else
-    report_tool_load_failures_ = (var == "0") ? false : true;
-#endif
+
+    ifdebug {
+      report_tool_load_failures_ = (var == "1") ? true : false;
+    } else {
+      report_tool_load_failures_ = (var == "0") ? false : true;
+    }

    var = os::GetEnvVar("HSA_DISABLE_FRAGMENT_ALLOCATOR");
    disable_fragment_alloc_ = (var == "1") ? true : false;
@@ -121,6 +121,12 @@ static __forceinline unsigned long long int strtoull(const char* str,
  } while (false)
 #endif

+#ifdef NDEBUG
+#define ifdebug if (false)
+#else
+#define ifdebug if (true)
+#endif
+
 // A macro to disallow the copy and move constructor and operator= functions
 #define DISALLOW_COPY_AND_ASSIGN(TypeName)                                                         \
  TypeName(const TypeName&) = delete;                                                              \