use rbtree instead of vm_objects list

simple test of mapping many system memory to gpu.
before
[ RUN      ] KFDMemoryTest.MMap
[          ] Using ISA for GFXIP 9.0
[          ] successfully register/map 32GB system memory to gpu
[       OK ] KFDMemoryTest.MMap (36932 ms)

after
[ RUN      ] KFDMemoryTest.MMap
[          ] Using ISA for GFXIP 9.0
[          ] successfully register/map 32GB system memory to gpu
[       OK ] KFDMemoryTest.MMap (11441 ms)

So there is 11s VS 36s improvement.

Looks like we can do something similar with vm_area too.

Change-Id: I0349aacdeddec3534016d28176f0fabf632c61fc
Signed-off-by: xinhui pan <xinhui.pan@amd.com>


[ROCm/ROCR-Runtime commit: ab9017715f]

projects/rocr-runtime/CMakeLists.txt

@@ -88,6 +88,7 @@ set ( HSAKMT_SRC "src/debug.c"
                  "src/queues.c"
                  "src/time.c"
                  "src/topology.c"
+                 "src/rbtree.c"
                  "src/version.c")
 
 ## Include paths

projects/rocr-runtime/LICENSE.md

@@ -21,3 +21,30 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This product contains software provided by Nginx, Inc. and its contributors.
+
+Copyright (C) 2002-2018 Igor Sysoev
+Copyright (C) 2011-2018 Nginx, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.

projects/rocr-runtime/src/fmm.c

@@ -38,6 +38,7 @@
 #include <errno.h>
 #include <pci/pci.h>
 #include <numaif.h>
+#include "rbtree.h"
 #ifndef MPOL_F_STATIC_NODES
 /* Bug in numaif.h, this should be defined in there. Definition copied
  * from linux/mempolicy.h.
@@ -53,11 +54,25 @@
 	.align = 0,						\
 	.guard_pages = 1,					\
 	.vm_ranges = NULL,					\
-	.vm_objects = NULL,					\
 	.fmm_mutex = PTHREAD_MUTEX_INITIALIZER,			\
 	.is_coherent = false					\
 	}
 
+#define container_of(ptr, type, member) ({			\
+		char *__mptr = (void *)(ptr);			\
+		((type *)(__mptr - offsetof(type, member))); })
+
+#define rb_entry(ptr, type, member)				\
+		container_of(ptr, type, member)
+
+#define vm_object_entry(n, is_userptr) ({			\
+		(is_userptr) == 0 ?				\
+		rb_entry(n, vm_object_t, node) :		\
+		rb_entry(n, vm_object_t, user_node); })
+
+#define vm_object_tree(app, is_userptr)				\
+		((is_userptr) ? &(app)->user_tree : &(app)->tree)
+
 struct vm_object {
 	void *start;
 	void *userptr;
@@ -68,8 +83,9 @@ struct vm_object {
 			*/
 	uint64_t handle; /* opaque */
 	uint32_t node_id;
-	struct vm_object *next;
-	struct vm_object *prev;
+	rbtree_node_t node;
+	rbtree_node_t user_node;
+
 	uint32_t flags; /* memory allocation flags */
 	/* Registered nodes to map on SVM mGPU */
 	uint32_t *registered_device_id_array;
@@ -105,7 +121,8 @@ typedef struct {
 	uint64_t align;
 	uint32_t guard_pages;
 	vm_area_t *vm_ranges;
-	vm_object_t *vm_objects;
+	rbtree_t tree;
+	rbtree_t user_tree;
 	pthread_mutex_t fmm_mutex;
 	bool is_coherent;
 } manageable_aperture_t;
@@ -263,7 +280,6 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size,
 		object->userptr_size = 0;
 		object->size = size;
 		object->handle = handle;
-		object->next = object->prev = NULL;
 		object->registered_device_id_array_size = 0;
 		object->mapped_device_id_array_size = 0;
 		object->registered_device_id_array = NULL;
@@ -276,6 +292,8 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size,
 		object->metadata = NULL;
 		object->user_data = NULL;
 		object->is_imported_kfd_bo = false;
+		object->node.key = rbtree_key((unsigned long)start, size);
+		object->user_node.key = rbtree_key(0, 0);
 	}
 
 	return object;
@@ -303,9 +321,6 @@ static void vm_remove_area(manageable_aperture_t *app, vm_area_t *area)
 
 static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object)
 {
-	vm_object_t *next;
-	vm_object_t *prev;
-
 	/* Free allocations inside the object */
 	if (object->registered_device_id_array)
 		free(object->registered_device_id_array);
@@ -321,16 +336,9 @@ static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object)
 	if (object->mapped_node_id_array)
 		free(object->mapped_node_id_array);
 
-	next = object->next;
-	prev = object->prev;
-
-	if (!prev) /* The first element */
-		app->vm_objects = next;
-	else
-		prev->next = next;
-
-	if (next) /* If not the last element */
-		next->prev = prev;
+	rbtree_delete(&app->tree, &object->node);
+	if (object->userptr)
+		rbtree_delete(&app->user_tree, &object->user_node);
 
 	free(object);
 }
@@ -347,19 +355,6 @@ static void vm_add_area_after(vm_area_t *after_this, vm_area_t *new_area)
 		next->prev = new_area;
 }
 
-static void vm_add_object_before(vm_object_t *before_this,
-				vm_object_t *new_object)
-{
-	vm_object_t *prev = before_this->prev;
-
-	before_this->prev = new_object;
-	new_object->next = before_this;
-
-	new_object->prev = prev;
-	if (prev)
-		prev->next = new_object;
-}
-
 static void vm_split_area(manageable_aperture_t *app, vm_area_t *area,
 				void *address, uint64_t MemorySizeInBytes)
 {
@@ -377,87 +372,129 @@ static void vm_split_area(manageable_aperture_t *app, vm_area_t *area,
 	vm_add_area_after(area, new_area);
 }
 
-static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app,
-					const void *address, uint64_t size)
+static vm_object_t *vm_find_object_by_address_userptr(manageable_aperture_t *app,
+					const void *address, uint64_t size, int is_userptr)
 {
-	vm_object_t *cur = app->vm_objects;
+	vm_object_t *cur = NULL;
 
-	size = ALIGN_UP(size, app->align);
+	if (is_userptr == 0)
+		size = ALIGN_UP(size, app->align);
+	rbtree_t *tree = vm_object_tree(app, is_userptr);
+	rbtree_key_t key = rbtree_key((unsigned long)address, size);
+	void *start;
+	uint64_t s;
 
-	/* Look up the appropriate address range containing the given address */
-	while (cur) {
-		if (cur->start == address && (cur->size == size || size == 0))
-			break;
-		cur = cur->next;
+	/* rbtree_lookup_nearest(,,,RIGHT) will return a node with
+	 * its size >= key.size and its address >= key.address
+	 * if there are two nodes with format(address, size),
+	 * (0x100, 16) and (0x110, 8). the key is (0x100, 0),
+	 * then node (0x100, 16) will be returned.
+	 */
+	rbtree_node_t *n = rbtree_lookup_nearest(tree, &key, LKP_ALL, RIGHT);
+
+	if (n) {
+		cur = vm_object_entry(n, is_userptr);
+		if (is_userptr == 0) {
+			start = cur->start;
+			s = cur->size;
+		} else {
+			start = cur->userptr;
+			s = cur->userptr_size;
+		}
+
+		if (start != address)
+			return NULL;
+
+		if (size)
+			return size == s ? cur : NULL;
+
+		/* size is 0, make sure there is only one node whose address == key.address*/
+		key = rbtree_key((unsigned long)address, (unsigned long)-1);
+		rbtree_node_t *rn = rbtree_lookup_nearest(tree, &key, LKP_ALL, LEFT);
+
+		if (rn != n)
+			return NULL;
 	}
 
 	return cur; /* NULL if not found */
 }
 
+
+static vm_object_t *vm_find_object_by_address_userptr_range(manageable_aperture_t *app,
+						    const void *address, int is_userptr)
+{
+	vm_object_t *cur = NULL;
+	rbtree_t *tree = vm_object_tree(app, is_userptr);
+	rbtree_key_t key = rbtree_key((unsigned long)address, 0);
+	rbtree_node_t *ln = rbtree_lookup_nearest(tree, &key,
+			LKP_ALL, LEFT);
+	rbtree_node_t *rn = rbtree_lookup_nearest(tree, &key,
+			LKP_ALL, RIGHT);
+	void *start;
+	uint64_t size;
+	int bad = 0;
+
+loop:
+	while (ln) {
+		cur = vm_object_entry(ln, is_userptr);
+		if (is_userptr == 0) {
+			start = cur->start;
+			size = cur->size;
+		} else {
+			start = cur->userptr;
+			size = cur->userptr_size;
+		}
+
+		if (address >= start &&
+				(uint64_t)address < ((uint64_t)start + size))
+			break;
+
+		cur = NULL;
+
+		if (ln == rn)
+			break;
+
+		ln = rbtree_next(tree, ln);
+	}
+
+	if (cur == NULL && bad == 0) {
+		/* As there is area overlap, say, (address, size) like
+		 * (0x100, 32), (0x108, 8), and the key.address is 0x118
+		 * The lookup above only find (0x108, 8), but the correct node should
+		 * be (0x100, 16). So try to walk though the tree to find the node.
+		 */
+		rn = ln;
+		key = rbtree_key(0, 0);
+		ln = rbtree_lookup_nearest(tree, &key, LKP_ALL, RIGHT);
+		bad = 1;
+		goto loop;
+	}
+
+	return cur; /* NULL if not found */
+}
+
+static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app,
+					const void *address, uint64_t size)
+{
+	return vm_find_object_by_address_userptr(app, address, size, 0);
+}
+
 static vm_object_t *vm_find_object_by_address_range(manageable_aperture_t *app,
 						    const void *address)
 {
-	vm_object_t *cur = app->vm_objects;
-
-	while (cur) {
-		if (address >= cur->start &&
-			(uint64_t)address < ((uint64_t)cur->start + cur->size))
-			break;
-		cur = cur->next;
-	}
-
-	return cur; /* NULL if not found */
+	return vm_find_object_by_address_userptr_range(app, address, 0);
 }
 
 static vm_object_t *vm_find_object_by_userptr(manageable_aperture_t *app,
 					const void *address, HSAuint64 size)
 {
-	vm_object_t *cur = app->vm_objects, *obj;
-	uint32_t found = 0;
-
-	/* Look up the userptr that matches the address. If size is specified,
-	 * the size needs to match too.
-	 */
-	while (cur) {
-		if ((cur->userptr == address) &&
-				((cur->userptr_size == size) || !size)) {
-			found = 1;
-			break;
-		}
-		cur = cur->next;
-	}
-
-	/* If size is not specified, we need to ensure the vm_obj found is the
-	 * only obj having this address.
-	 */
-	if (found && !size) {
-		obj = cur->next;
-		while (obj) {
-			if (obj->userptr == address) {
-				cur = NULL;
-				break;
-			}
-			obj = obj->next;
-		}
-	}
-
-	return cur; /* NULL if any look-up failure */
+	return vm_find_object_by_address_userptr(app, address, size, 1);
 }
 
 static vm_object_t *vm_find_object_by_userptr_range(manageable_aperture_t *app,
 						const void *address)
 {
-	vm_object_t *cur = app->vm_objects;
-
-	/* Look up the appropriate address range containing the given address */
-	while (cur) {
-		if (address >= cur->userptr &&
-		(uint64_t)address < (uint64_t)cur->userptr + cur->userptr_size)
-			break;
-		cur = cur->next;
-	}
-
-	return cur; /* NULL if not found */
+	return vm_find_object_by_address_userptr_range(app, address, 1);
 }
 
 static vm_area_t *vm_find(manageable_aperture_t *app, void *address)
@@ -615,12 +652,7 @@ static vm_object_t *aperture_allocate_object(manageable_aperture_t *app,
 	if (!new_object)
 		return NULL;
 
-	/* check for non-empty list */
-	if (app->vm_objects)
-		/* Add it before the first element */
-		vm_add_object_before(app->vm_objects, new_object);
-
-	app->vm_objects = new_object; /* Update head */
+	rbtree_insert(&app->tree, &new_object->node);
 
 	return new_object;
 }
@@ -784,7 +816,8 @@ static void aperture_print(aperture_t *app)
 static void manageable_aperture_print(manageable_aperture_t *app)
 {
 	vm_area_t *cur = app->vm_ranges;
-	vm_object_t *object = app->vm_objects;
+	rbtree_node_t *n = rbtree_node_any(&app->tree, LEFT);
+	vm_object_t *object;
 
 	pr_info("\t Base: %p\n", app->base);
 	pr_info("\t Limit: %p\n", app->limit);
@@ -794,11 +827,12 @@ static void manageable_aperture_print(manageable_aperture_t *app)
 		cur = cur->next;
 	};
 	pr_info("\t Objects:\n");
-	while (object) {
+	while (n) {
+		object = vm_object_entry(n, 0);
 		pr_info("\t\t Object [%p - %" PRIu64 "]\n",
 				object->start, object->size);
-		object = object->next;
-	};
+		n = rbtree_next(&app->tree, n);
+	}
 }
 
 void fmm_print(uint32_t gpu_id)
@@ -834,6 +868,7 @@ static void fmm_release_scratch(uint32_t gpu_id)
 	uint64_t size;
 	vm_object_t *obj;
 	manageable_aperture_t *aperture;
+	rbtree_node_t *n;
 
 	gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id);
 	if (gpu_mem_id < 0)
@@ -846,7 +881,9 @@ static void fmm_release_scratch(uint32_t gpu_id)
 	if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) {
 		/* unmap and remove all remaining objects */
 		pthread_mutex_lock(&aperture->fmm_mutex);
-		while ((obj = aperture->vm_objects)) {
+		while ((n = rbtree_node_any(&aperture->tree, MID))) {
+			obj = vm_object_entry(n, 0);
+
 			void *obj_addr = obj->start;
 
 			pthread_mutex_unlock(&aperture->fmm_mutex);
@@ -1643,6 +1680,30 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit,
 	return HSAKMT_STATUS_SUCCESS;
 }
 
+static void fmm_init_rbtree(void)
+{
+	static int once;
+	int i = gpu_mem_count;
+
+	if (once++ == 0) {
+		rbtree_init(&svm.dgpu_aperture.tree);
+		rbtree_init(&svm.dgpu_aperture.user_tree);
+		rbtree_init(&svm.dgpu_alt_aperture.tree);
+		rbtree_init(&svm.dgpu_alt_aperture.user_tree);
+		rbtree_init(&cpuvm_aperture.tree);
+		rbtree_init(&cpuvm_aperture.user_tree);
+	}
+
+	while (i--) {
+		rbtree_init(&gpu_mem[i].scratch_aperture.tree);
+		rbtree_init(&gpu_mem[i].scratch_aperture.user_tree);
+		rbtree_init(&gpu_mem[i].scratch_physical.tree);
+		rbtree_init(&gpu_mem[i].scratch_physical.user_tree);
+		rbtree_init(&gpu_mem[i].gpuvm_aperture.tree);
+		rbtree_init(&gpu_mem[i].gpuvm_aperture.user_tree);
+	}
+}
+
 HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 {
 	uint32_t i = 0;
@@ -1854,6 +1915,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
 	cpuvm_aperture.align = PAGE_SIZE;
 	cpuvm_aperture.limit = (void *)0x7FFFFFFFFFFF; /* 2^47 - 1 */
 
+	fmm_init_rbtree();
+
 	free(process_apertures);
 	return ret;
 
@@ -2570,6 +2633,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj
 		gpuid_to_nodeid(gpu_id, &obj->node_id);
 		obj->userptr_size = size;
 		obj->registration_count = 1;
+		obj->user_node.key = rbtree_key((unsigned long)addr, size);
+		rbtree_insert(&aperture->user_tree, &obj->user_node);
 		pthread_mutex_unlock(&aperture->fmm_mutex);
 	} else
 		return HSAKMT_STATUS_ERROR;
@@ -3234,8 +3299,10 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data)
 
 static void fmm_clear_aperture(manageable_aperture_t *app)
 {
-	while (app->vm_objects)
-		vm_remove_object(app, app->vm_objects);
+	rbtree_node_t *n;
+
+	while ((n = rbtree_node_any(&app->tree, MID)))
+		vm_remove_object(app, vm_object_entry(n, 0));
 
 	while (app->vm_ranges)
 		vm_remove_area(app, app->vm_ranges);

projects/rocr-runtime/src/rbtree.c

@@ -0,0 +1,374 @@
+/*
+ * Copyright (C) 2002-2018 Igor Sysoev
+ * Copyright (C) 2011-2018 Nginx, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "rbtree.h"
+
+static inline void rbtree_left_rotate(rbtree_node_t **root,
+		rbtree_node_t *sentinel, rbtree_node_t *node);
+static inline void rbtree_right_rotate(rbtree_node_t **root,
+		rbtree_node_t *sentinel, rbtree_node_t *node);
+
+static void
+rbtree_insert_value(rbtree_node_t *temp, rbtree_node_t *node,
+		rbtree_node_t *sentinel)
+{
+	rbtree_node_t  **p;
+
+	for ( ;; ) {
+
+		p = rbtree_key_compare(LKP_ALL, &node->key, &temp->key) < 0 ?
+			&temp->left : &temp->right;
+
+		if (*p == sentinel) {
+			break;
+		}
+
+		temp = *p;
+	}
+
+	*p = node;
+	node->parent = temp;
+	node->left = sentinel;
+	node->right = sentinel;
+	rbt_red(node);
+}
+
+
+void
+rbtree_insert(rbtree_t *tree, rbtree_node_t *node)
+{
+	rbtree_node_t  **root, *temp, *sentinel;
+
+	/* a binary tree insert */
+
+	root = &tree->root;
+	sentinel = &tree->sentinel;
+
+	if (*root == sentinel) {
+		node->parent = NULL;
+		node->left = sentinel;
+		node->right = sentinel;
+		rbt_black(node);
+		*root = node;
+
+		return;
+	}
+
+	rbtree_insert_value(*root, node, sentinel);
+
+	/* re-balance tree */
+
+	while (node != *root && rbt_is_red(node->parent)) {
+
+		if (node->parent == node->parent->parent->left) {
+			temp = node->parent->parent->right;
+
+			if (rbt_is_red(temp)) {
+				rbt_black(node->parent);
+				rbt_black(temp);
+				rbt_red(node->parent->parent);
+				node = node->parent->parent;
+
+			} else {
+				if (node == node->parent->right) {
+					node = node->parent;
+					rbtree_left_rotate(root, sentinel, node);
+				}
+
+				rbt_black(node->parent);
+				rbt_red(node->parent->parent);
+				rbtree_right_rotate(root, sentinel, node->parent->parent);
+			}
+
+		} else {
+			temp = node->parent->parent->left;
+
+			if (rbt_is_red(temp)) {
+				rbt_black(node->parent);
+				rbt_black(temp);
+				rbt_red(node->parent->parent);
+				node = node->parent->parent;
+
+			} else {
+				if (node == node->parent->left) {
+					node = node->parent;
+					rbtree_right_rotate(root, sentinel, node);
+				}
+
+				rbt_black(node->parent);
+				rbt_red(node->parent->parent);
+				rbtree_left_rotate(root, sentinel, node->parent->parent);
+			}
+		}
+	}
+
+	rbt_black(*root);
+}
+
+
+void
+rbtree_delete(rbtree_t *tree, rbtree_node_t *node)
+{
+	unsigned int red;
+	rbtree_node_t  **root, *sentinel, *subst, *temp, *w;
+
+	/* a binary tree delete */
+
+	root = &tree->root;
+	sentinel = &tree->sentinel;
+
+	if (node->left == sentinel) {
+		temp = node->right;
+		subst = node;
+
+	} else if (node->right == sentinel) {
+		temp = node->left;
+		subst = node;
+
+	} else {
+		subst = rbtree_min(node->right, sentinel);
+
+		if (subst->left != sentinel) {
+			temp = subst->left;
+		} else {
+			temp = subst->right;
+		}
+	}
+
+	if (subst == *root) {
+		*root = temp;
+		rbt_black(temp);
+
+		return;
+	}
+
+	red = rbt_is_red(subst);
+
+	if (subst == subst->parent->left) {
+		subst->parent->left = temp;
+
+	} else {
+		subst->parent->right = temp;
+	}
+
+	if (subst == node) {
+
+		temp->parent = subst->parent;
+
+	} else {
+
+		if (subst->parent == node) {
+			temp->parent = subst;
+
+		} else {
+			temp->parent = subst->parent;
+		}
+
+		subst->left = node->left;
+		subst->right = node->right;
+		subst->parent = node->parent;
+		rbt_copy_color(subst, node);
+
+		if (node == *root) {
+			*root = subst;
+
+		} else {
+			if (node == node->parent->left) {
+				node->parent->left = subst;
+			} else {
+				node->parent->right = subst;
+			}
+		}
+
+		if (subst->left != sentinel) {
+			subst->left->parent = subst;
+		}
+
+		if (subst->right != sentinel) {
+			subst->right->parent = subst;
+		}
+	}
+
+	if (red) {
+		return;
+	}
+
+	/* a delete fixup */
+
+	while (temp != *root && rbt_is_black(temp)) {
+
+		if (temp == temp->parent->left) {
+			w = temp->parent->right;
+
+			if (rbt_is_red(w)) {
+				rbt_black(w);
+				rbt_red(temp->parent);
+				rbtree_left_rotate(root, sentinel, temp->parent);
+				w = temp->parent->right;
+			}
+
+			if (rbt_is_black(w->left) && rbt_is_black(w->right)) {
+				rbt_red(w);
+				temp = temp->parent;
+
+			} else {
+				if (rbt_is_black(w->right)) {
+					rbt_black(w->left);
+					rbt_red(w);
+					rbtree_right_rotate(root, sentinel, w);
+					w = temp->parent->right;
+				}
+
+				rbt_copy_color(w, temp->parent);
+				rbt_black(temp->parent);
+				rbt_black(w->right);
+				rbtree_left_rotate(root, sentinel, temp->parent);
+				temp = *root;
+			}
+
+		} else {
+			w = temp->parent->left;
+
+			if (rbt_is_red(w)) {
+				rbt_black(w);
+				rbt_red(temp->parent);
+				rbtree_right_rotate(root, sentinel, temp->parent);
+				w = temp->parent->left;
+			}
+
+			if (rbt_is_black(w->left) && rbt_is_black(w->right)) {
+				rbt_red(w);
+				temp = temp->parent;
+
+			} else {
+				if (rbt_is_black(w->left)) {
+					rbt_black(w->right);
+					rbt_red(w);
+					rbtree_left_rotate(root, sentinel, w);
+					w = temp->parent->left;
+				}
+
+				rbt_copy_color(w, temp->parent);
+				rbt_black(temp->parent);
+				rbt_black(w->left);
+				rbtree_right_rotate(root, sentinel, temp->parent);
+				temp = *root;
+			}
+		}
+	}
+
+	rbt_black(temp);
+}
+
+
+static inline void
+rbtree_left_rotate(rbtree_node_t **root, rbtree_node_t *sentinel,
+		rbtree_node_t *node)
+{
+	rbtree_node_t  *temp;
+
+	temp = node->right;
+	node->right = temp->left;
+
+	if (temp->left != sentinel) {
+		temp->left->parent = node;
+	}
+
+	temp->parent = node->parent;
+
+	if (node == *root) {
+		*root = temp;
+
+	} else if (node == node->parent->left) {
+		node->parent->left = temp;
+
+	} else {
+		node->parent->right = temp;
+	}
+
+	temp->left = node;
+	node->parent = temp;
+}
+
+
+static inline void
+rbtree_right_rotate(rbtree_node_t **root, rbtree_node_t *sentinel,
+		rbtree_node_t *node)
+{
+	rbtree_node_t  *temp;
+
+	temp = node->left;
+	node->left = temp->right;
+
+	if (temp->right != sentinel) {
+		temp->right->parent = node;
+	}
+
+	temp->parent = node->parent;
+
+	if (node == *root) {
+		*root = temp;
+
+	} else if (node == node->parent->right) {
+		node->parent->right = temp;
+
+	} else {
+		node->parent->left = temp;
+	}
+
+	temp->right = node;
+	node->parent = temp;
+}
+
+
+rbtree_node_t *
+rbtree_next(rbtree_t *tree, rbtree_node_t *node)
+{
+	rbtree_node_t  *root, *sentinel, *parent;
+
+	sentinel = &tree->sentinel;
+
+	if (node->right != sentinel) {
+		return rbtree_min(node->right, sentinel);
+	}
+
+	root = tree->root;
+
+	for ( ;; ) {
+		parent = node->parent;
+
+		if (node == root) {
+			return NULL;
+		}
+
+		if (node == parent->left) {
+			return parent;
+		}
+
+		node = parent;
+	}
+}

projects/rocr-runtime/src/rbtree.h

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2002-2018 Igor Sysoev
+ * Copyright (C) 2011-2018 Nginx, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _RBTREE_H_
+#define _RBTREE_H_
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <errno.h>
+#include "rbtree_amd.h"
+
+typedef struct rbtree_node_s rbtree_node_t;
+
+struct rbtree_node_s {
+	rbtree_key_t    key;
+	rbtree_node_t   *left;
+	rbtree_node_t   *right;
+	rbtree_node_t   *parent;
+	unsigned char   color;
+	unsigned char   data;
+};
+
+typedef struct rbtree_s rbtree_t;
+
+struct rbtree_s {
+	rbtree_node_t   *root;
+	rbtree_node_t   sentinel;
+};
+
+#define rbtree_init(tree)				\
+	rbtree_sentinel_init(&(tree)->sentinel);	\
+	(tree)->root = &(tree)->sentinel;
+
+void rbtree_insert(rbtree_t *tree, rbtree_node_t *node);
+void rbtree_delete(rbtree_t *tree, rbtree_node_t *node);
+rbtree_node_t *rbtree_next(rbtree_t *tree,
+		rbtree_node_t *node);
+
+#define rbt_red(node)			((node)->color = 1)
+#define rbt_black(node)			((node)->color = 0)
+#define rbt_is_red(node)		((node)->color)
+#define rbt_is_black(node)		(!rbt_is_red(node))
+#define rbt_copy_color(n1, n2)		(n1->color = n2->color)
+
+/* a sentinel must be black */
+
+#define rbtree_sentinel_init(node)	rbt_black(node)
+
+static inline rbtree_node_t *
+rbtree_min(rbtree_node_t *node, rbtree_node_t *sentinel)
+{
+	while (node->left != sentinel) {
+		node = node->left;
+	}
+
+	return node;
+}
+
+#include "rbtree_amd.h"
+
+#endif

projects/rocr-runtime/src/rbtree_amd.h

@@ -0,0 +1,155 @@
+/*
+ * Copyright © 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy,
+ * modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including
+ * the next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _RBTREE_AMD_H_
+#define _RBTREE_AMD_H_
+
+typedef struct rbtree_key_s rbtree_key_t;
+struct rbtree_key_s {
+#define ADDR_BIT 0
+#define SIZE_BIT 1
+	unsigned long addr;
+	unsigned long size;
+};
+#define BIT(x) (1<<(x))
+#define LKP_ALL (BIT(ADDR_BIT) | BIT(SIZE_BIT))
+#define LKP_ADDR (BIT(ADDR_BIT))
+#define LKP_ADDR_SIZE (BIT(ADDR_BIT) | BIT(SIZE_BIT))
+
+static inline rbtree_key_t
+rbtree_key(unsigned long addr, unsigned long size)
+{
+	return (rbtree_key_t){addr, size};
+}
+
+/*
+ * compare addr, size one by one
+ */
+static inline int
+rbtree_key_compare(unsigned int type, rbtree_key_t *key1, rbtree_key_t *key2)
+{
+	if ((type & 1 << ADDR_BIT) && (key1->addr != key2->addr))
+		return key1->addr > key2->addr ? 1 : -1;
+
+	if ((type & 1 << SIZE_BIT) && (key1->size != key2->size))
+		return key1->size > key2->size ? 1 : -1;
+
+	return 0;
+}
+#endif /*_RBTREE_AMD_H_*/
+
+/*inlcude this file again with RBTREE_HELPER defined*/
+#ifndef RBTREE_HELPER
+#define RBTREE_HELPER
+#else
+#ifndef _RBTREE_AMD_H_HELPER_
+#define _RBTREE_AMD_H_HELPER_
+static inline rbtree_node_t *
+rbtree_max(rbtree_node_t *node, rbtree_node_t *sentinel)
+{
+	while (node->right != sentinel)
+		node = node->right;
+
+	return node;
+}
+
+#define LEFT 0
+#define RIGHT 1
+#define MID 2
+static inline rbtree_node_t *
+rbtree_min_max(rbtree_t *tree, int lr)
+{
+	rbtree_node_t *sentinel = &tree->sentinel;
+	rbtree_node_t *node = tree->root;
+
+	if (node == sentinel)
+		return NULL;
+
+	if (lr == LEFT)
+		node = rbtree_min(node, sentinel);
+	else if (lr == RIGHT)
+		node = rbtree_max(node, sentinel);
+
+	return node;
+}
+
+static inline rbtree_node_t *
+rbtree_node_any(rbtree_t *tree, int lmr)
+{
+	rbtree_node_t *sentinel = &tree->sentinel;
+	rbtree_node_t *node = tree->root;
+
+	if (node == sentinel)
+		return NULL;
+
+	if (lmr == MID)
+		return node;
+
+	return rbtree_min_max(tree, lmr);
+}
+
+static inline rbtree_node_t *
+rbtree_lookup_nearest(rbtree_t *rbtree, rbtree_key_t *key,
+		unsigned int type, int lr)
+{
+	int rc;
+	rbtree_node_t *node, *sentinel, *n = NULL;
+
+	node = rbtree->root;
+	sentinel = &rbtree->sentinel;
+
+	while (node != sentinel) {
+		rc = rbtree_key_compare(type, key, &node->key);
+
+		if (rc < 0) {
+			if (lr == RIGHT)
+				n = node;
+			node = node->left;
+			continue;
+		}
+
+		if (rc > 0) {
+			if (lr == LEFT)
+				n = node;
+			node = node->right;
+			continue;
+		}
+
+		return node;
+	}
+
+	return n;
+}
+
+static inline rbtree_node_t *
+rbtree_lookup(rbtree_t *rbtree, rbtree_key_t *key,
+		unsigned int type)
+{
+	return rbtree_lookup_nearest(rbtree, key, type, -1);
+}
+#endif /*_RBTREE_AMD_H_HELPER_*/
+
+#endif /*RBTREE_HELPER*/
+