Support large scratch allocations and reclaim.

Also improve small_heap used for scratch region allocation.

Change-Id: Ib7311b663b38968d88ebc355b81e12c0863dc541

runtime/hsa-runtime/core/util/small_heap.cpp

@@ -42,25 +42,47 @@
 
 #include "small_heap.h"
 
-SmallHeap::memory_t::iterator SmallHeap::merge(
-    SmallHeap::memory_t::iterator& keep,
-    SmallHeap::memory_t::iterator& destroy) {
-  assert((char*)keep->first + keep->second.len == (char*)destroy->first &&
-         "Invalid merge");
-  assert(keep->second.isfree() && "Merge with allocated block");
-  assert(destroy->second.isfree() && "Merge with allocated block");
+// Inserts node into freelist after place.
+// Assumes node will not be an end of the list (list has guard nodes).
+void SmallHeap::insertafter(SmallHeap::iterator_t place, SmallHeap::iterator_t node) {
+  assert(place->first < node->first && "Order violation");
+  assert(isfree(place->second) && "Freelist operation error.");
+  iterator_t next = place->second.next;
+  node->second.next = next;
+  node->second.prior = place;
+  place->second.next = node;
+  next->second.prior = node;
+}
 
-  keep->second.len += destroy->second.len;
-  keep->second.next_free = destroy->second.next_free;
-  if (!destroy->second.islastfree())
-    memory[destroy->second.next_free].prior_free = keep->first;
+// Removes node from freelist.
+// Assumes node will not be an end of the list (list has guard nodes).
+void SmallHeap::remove(SmallHeap::iterator_t node) {
+  assert(isfree(node->second) && "Freelist operation error.");
+  node->second.prior->second.next = node->second.next;
+  node->second.next->second.prior = node->second.prior;
+  setused(node->second);
+}
 
-  memory.erase(destroy);
-  return keep;
+// Returns high if merge failed or the merged node.
+SmallHeap::memory_t::iterator SmallHeap::merge(SmallHeap::memory_t::iterator low,
+                                               SmallHeap::memory_t::iterator high) {
+  assert(isfree(low->second) && "Merge with allocated block");
+  assert(isfree(high->second) && "Merge with allocated block");
+
+  if ((char*)low->first + low->second.len != (char*)high->first) return high;
+
+  assert(!islastfree(high->second) && "Illegal merge.");
+
+  low->second.len += high->second.len;
+  low->second.next = high->second.next;
+  high->second.next->second.prior = low;
+
+  memory.erase(high);
+  return low;
 }
 
 void SmallHeap::free(void* ptr) {
-  if (ptr == NULL) return;
+  if (ptr == nullptr) return;
 
   auto iterator = memory.find(ptr);
 
@@ -70,105 +92,90 @@ void SmallHeap::free(void* ptr) {
     return;
   }
 
-  const auto start_guard = memory.find(0);
-  const auto end_guard = memory.find((void*)0xFFFFFFFFFFFFFFFFull);
-
   // Return memory to total and link node into free list
   total_free += iterator->second.len;
-  if (first_free < iterator->first) {
-    auto before = iterator;
-    before--;
-    while (before != start_guard && !before->second.isfree()) before--;
-    assert(before->second.next_free > iterator->first &&
-           "Inconsistency in small heap.");
-    iterator->second.prior_free = before->first;
-    iterator->second.next_free = before->second.next_free;
-    before->second.next_free = iterator->first;
-    if (!iterator->second.islastfree())
-      memory[iterator->second.next_free].prior_free = iterator->first;
-  } else {
-    iterator->second.setfirstfree();
-    iterator->second.next_free = first_free;
-    first_free = iterator->first;
-    if (!iterator->second.islastfree())
-      memory[iterator->second.next_free].prior_free = iterator->first;
-  }
 
-  // Attempt compaction
+  // Could also traverse the free list which might be faster in some cases.
   auto before = iterator;
   before--;
-  if (before != start_guard) {
-    if (before->second.isfree()) {
-      iterator = merge(before, iterator);
-    }
-  }
+  while (!isfree(before->second)) before--;
+  assert(before->second.next->first > iterator->first && "Inconsistency in small heap.");
+  insertafter(before, iterator);
 
-  auto after = iterator;
-  after++;
-  if (after != end_guard) {
-    if (after->second.isfree()) {
-      iterator = merge(iterator, after);
-    }
-  }
+  // Attempt compaction
+  iterator = merge(before, iterator);
+  merge(iterator, iterator->second.next);
+
+  // Update lowHighBondary
+  high.erase(ptr);
 }
 
 void* SmallHeap::alloc(size_t bytes) {
   // Is enough memory available?
-  if ((bytes > total_free) || (bytes == 0)) return NULL;
+  if ((bytes > total_free) || (bytes == 0)) return nullptr;
 
-  memory_t::iterator current;
-  memory_t::iterator prior;
+  iterator_t current;
 
   // Walk the free list and allocate at first fitting location
-  prior = current = memory.find(first_free);
-  while (true) {
+  current = firstfree();
+  while (!islastfree(current->second)) {
     if (bytes <= current->second.len) {
       // Decrement from total
       total_free -= bytes;
 
-      // Is allocation an exact fit?
-      if (bytes == current->second.len) {
-        if (prior == current) {
-          first_free = current->second.next_free;
-          if (!current->second.islastfree())
-            memory[current->second.next_free].setfirstfree();
-        } else {
-          prior->second.next_free = current->second.next_free;
-          if (!current->second.islastfree())
-            memory[current->second.next_free].prior_free = prior->first;
-        }
-        current->second.next_free = NULL;
-        return current->first;
-      } else {
-        // Split current node
+      // Split node
+      if (bytes != current->second.len) {
         void* remaining = (char*)current->first + bytes;
         Node& node = memory[remaining];
-        node.next_free = current->second.next_free;
-        node.prior_free = current->second.prior_free;
         node.len = current->second.len - bytes;
         current->second.len = bytes;
-
-        if (prior == current) {
-          first_free = remaining;
-          node.setfirstfree();
-        } else {
-          prior->second.next_free = remaining;
-          node.prior_free = prior->first;
-        }
-        if (!node.islastfree()) memory[node.next_free].prior_free = remaining;
-
-        current->second.next_free = NULL;
-        return current->first;
+        insertafter(current, memory.find(remaining));
       }
+
+      remove(current);
+      return current->first;
     }
-
-    // End of free list?
-    if (current->second.islastfree()) break;
-
-    prior = current;
-    current = memory.find(current->second.next_free);
+    current = current->second.next;
   }
+  assert(current->second.len == 0 && "Freelist corruption.");
 
   // Can't service the request due to fragmentation
-  return NULL;
+  return nullptr;
+}
+
+void* SmallHeap::alloc_high(size_t bytes) {
+  // Is enough memory available?
+  if ((bytes > total_free) || (bytes == 0)) return nullptr;
+
+  iterator_t current;
+
+  // Walk the free list and allocate at first fitting location
+  current = lastfree();
+  while (!isfirstfree(current->second)) {
+    if (bytes <= current->second.len) {
+      // Decrement from total
+      total_free -= bytes;
+
+      void* alloc;
+      // Split node
+      if (bytes != current->second.len) {
+        alloc = (char*)current->first + current->second.len - bytes;
+        current->second.len -= bytes;
+        Node& node = memory[alloc];
+        node.len = bytes;
+        setused(node);
+      } else {
+        alloc = current->first;
+        remove(current);
+      }
+
+      high.insert(alloc);
+      return alloc;
+    }
+    current = current->second.prior;
+  }
+  assert(current->second.len == 0 && "Freelist corruption.");
+
+  // Can't service the request due to fragmentation
+  return nullptr;
 }

runtime/hsa-runtime/core/util/small_heap.h

@@ -47,68 +47,81 @@
 #ifndef HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_
 #define HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_
 
-#include "utils.h"
-
 #include <map>
+#include <set>
 
+#include "utils.h"
+  
 class SmallHeap {
- public:
-  class Node {
-   public:
-    size_t len;
-    void* next_free;
-    void* prior_free;
-    static const intptr_t END = -1;
+ private:
+  struct Node;
+  typedef std::map<void*, Node> memory_t;
+  typedef memory_t::iterator iterator_t;
 
-    __forceinline bool isfree() const { return next_free != NULL; }
-    __forceinline bool islastfree() const { return intptr_t(next_free) == END; }
-    __forceinline bool isfirstfree() const {
-      return intptr_t(prior_free) == END;
-    }
-    __forceinline void setlastfree() {
-      *reinterpret_cast<intptr_t*>(&next_free) = END;
-    }
-    __forceinline void setfirstfree() {
-      *reinterpret_cast<intptr_t*>(&prior_free) = END;
-    }
+  struct Node {
+    size_t len;
+    iterator_t next;
+    iterator_t prior;
   };
 
- private:
-  SmallHeap(const SmallHeap& rhs);
-  SmallHeap& operator=(const SmallHeap& rhs);
+  SmallHeap(const SmallHeap& rhs) = delete;
+  SmallHeap& operator=(const SmallHeap& rhs) = delete;
 
   void* const pool;
   const size_t length;
 
   size_t total_free;
-  void* first_free;
-  std::map<void*, Node> memory;
+  memory_t memory;
+  std::set<void*> high;
 
-  typedef decltype(memory) memory_t;
-  memory_t::iterator merge(memory_t::iterator& keep,
-                           memory_t::iterator& destroy);
+  __forceinline bool isfree(const Node& node) const { return node.next != memory.begin(); }
+  __forceinline bool islastfree(const Node& node) const { return node.next == memory.end(); }
+  __forceinline bool isfirstfree(const Node& node) const { return node.prior == memory.end(); }
+  __forceinline void setlastfree(Node& node) { node.next = memory.end(); }
+  __forceinline void setfirstfree(Node& node) { node.prior = memory.end(); }
+  __forceinline void setused(Node& node) { node.next = memory.begin(); }
+
+  __forceinline iterator_t firstfree() { return memory.begin()->second.next; }
+  __forceinline iterator_t lastfree() { return memory.rbegin()->second.prior; }
+  void insertafter(iterator_t place, iterator_t node);
+  void remove(iterator_t node);
+  iterator_t merge(iterator_t low, iterator_t high);
 
  public:
-  SmallHeap() : pool(NULL), length(0), total_free(0) {}
+  SmallHeap() : pool(nullptr), length(0), total_free(0) {}
   SmallHeap(void* base, size_t length)
       : pool(base), length(length), total_free(length) {
-    first_free = pool;
+    assert(pool != nullptr && "Invalid base address.");
+    assert(pool != (void*)0xFFFFFFFFFFFFFFFFull && "Invalid base address.");
+    assert((char*)pool + length != (char*)0xFFFFFFFFFFFFFFFFull && "Invalid pool bounds.");
+
+    Node& start = memory[0];
+    Node& node = memory[pool];
+    Node& end = memory[(void*)0xFFFFFFFFFFFFFFFFull];
+
+    start.len = 0;
+    start.next = memory.find(pool);
+    setfirstfree(start);
 
-    Node& node = memory[first_free];
     node.len = length;
-    node.setlastfree();
-    node.setfirstfree();
+    node.prior = memory.begin();
+    node.next = --memory.end();
 
-    memory[0].len = 0;
-    memory[(void*)0xFFFFFFFFFFFFFFFFull].len = 0;
+    end.len = 0;
+    end.prior = start.next;
+    setlastfree(end);
+
+    high.insert((void*)0xFFFFFFFFFFFFFFFFull);
   }
 
   void* alloc(size_t bytes);
+  void* alloc_high(size_t bytes);
   void free(void* ptr);
 
   void* base() const { return pool; }
   size_t size() const { return length; }
   size_t remaining() const { return total_free; }
+  void* high_split() const { return *high.begin(); }
 };
 
 #endif