Red-black tree based physical memory management

lib/page_alloc.c

@@ -18,6 +18,15 @@
 #include <ihk/page_alloc.h>
 #include <memory.h>
 #include <bitops.h>
+#include <errno.h>
+
+//#define DEBUG_PRINT_PAGE_ALLOC
+
+#ifdef DEBUG_PRINT_PAGE_ALLOC
+#define dkprintf kprintf
+#else
+#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
+#endif
 
 void free_pages(void *, int npages);
 
@@ -301,3 +310,345 @@ kprintf("\nzeroing done\n");
 }
 
 
+#ifdef IHK_RBTREE_ALLOCATOR
+
+/*
+ * Simple red-black tree based physical memory management routines.
+ *
+ * Allocation grabs first suitable chunk (splits chunk if alignment requires it).
+ * Deallocation merges with immediate neighbours.
+ *
+ * NOTE: invariant property: free_chunk structures are placed in the very front
+ * of their corresponding memory (i.e., they are on the free memory chunk itself).
+ */
+
+/*
+ * Free pages.
+ * NOTE: locking must be managed by the caller.
+ */
+static int __page_alloc_rbtree_free_range(struct rb_root *root,
+		unsigned long addr, unsigned long size)
+{
+	struct rb_node **iter = &(root->rb_node), *parent = NULL;
+	struct free_chunk *new_chunk;
+
+	/* Figure out where to put new node */
+	while (*iter) {
+		struct free_chunk *ichunk = container_of(*iter, struct free_chunk, node);
+		parent = *iter;
+
+		if ((addr >= ichunk->addr) && (addr < ichunk->addr + ichunk->size)) {
+			kprintf("%s: ERROR: free memory chunk: 0x%lx:%lu"
+					" and requested range to be freed: 0x%lx:%lu are "
+					"overlapping (double-free?)\n",
+					__FUNCTION__,
+					ichunk->addr, ichunk->size, addr, size);
+			return EINVAL;
+		}
+
+		/* Is ichunk contigous from the left? */
+		if (ichunk->addr + ichunk->size == addr) {
+			struct rb_node *right;
+			/* Extend it to the right */
+			ichunk->size += size;
+			dkprintf("%s: chunk extended to right: 0x%lx:%lu\n",
+					__FUNCTION__, ichunk->addr, ichunk->size);
+
+			/* Have the right chunk of ichunk and ichunk become contigous? */
+			right = rb_next(*iter);
+			if (right) {
+				struct free_chunk *right_chunk =
+					container_of(right, struct free_chunk, node);
+
+				if (ichunk->addr + ichunk->size == right_chunk->addr) {
+					ichunk->size += right_chunk->size;
+					rb_erase(right, root);
+					dkprintf("%s: chunk merged to right: 0x%lx:%lu\n",
+							__FUNCTION__, ichunk->addr, ichunk->size);
+				}
+			}
+
+			return 0;
+		}
+
+		/* Is ichunk contigous from the right? */
+		if (addr + size == ichunk->addr) {
+			struct rb_node *left;
+			/* Extend it to the left */
+			ichunk->addr -= size;
+			ichunk->size += size;
+			dkprintf("%s: chunk extended to left: 0x%lx:%lu\n",
+					__FUNCTION__, ichunk->addr, ichunk->size);
+
+			/* Have the left chunk of ichunk and ichunk become contigous? */
+			left = rb_prev(*iter);
+			if (left) {
+				struct free_chunk *left_chunk =
+					container_of(left, struct free_chunk, node);
+
+				if (left_chunk->addr + left_chunk->size == ichunk->addr) {
+					ichunk->addr -= left_chunk->size;
+					ichunk->size += left_chunk->size;
+					rb_erase(left, root);
+					dkprintf("%s: chunk merged to left: 0x%lx:%lu\n",
+							__FUNCTION__, ichunk->addr, ichunk->size);
+				}
+			}
+
+			/* Move chunk structure to the front */
+			new_chunk = (struct free_chunk *)phys_to_virt(ichunk->addr);
+			*new_chunk = *ichunk;
+			rb_replace_node(&ichunk->node, &new_chunk->node, root);
+			dkprintf("%s: chunk moved to front: 0x%lx:%lu\n",
+					__FUNCTION__, new_chunk->addr, new_chunk->size);
+
+			return 0;
+		}
+
+		if (addr < ichunk->addr)
+			iter = &((*iter)->rb_left);
+		else
+			iter = &((*iter)->rb_right);
+	}
+
+	new_chunk = (struct free_chunk *)phys_to_virt(addr);
+	new_chunk->addr = addr;
+	new_chunk->size = size;
+	dkprintf("%s: new chunk: 0x%lx:%lu\n",
+		__FUNCTION__, new_chunk->addr, new_chunk->size);
+
+	/* Add new node and rebalance tree. */
+	rb_link_node(&new_chunk->node, parent, iter);
+	rb_insert_color(&new_chunk->node, root);
+
+	return 0;
+}
+
+/*
+ * Mark address range as used (i.e., allocated).
+ *
+ * chunk is the free memory chunk in which
+ * [aligned_addr, aligned_addr + size] resides.
+ *
+ * NOTE: locking must be managed by the caller.
+ */
+static int __page_alloc_rbtree_mark_range_allocated(struct rb_root *root,
+		struct free_chunk *chunk,
+		unsigned long aligned_addr, unsigned long size)
+{
+	struct free_chunk *left_chunk = NULL, *right_chunk = NULL;
+
+	/* Is there leftover on the right? */
+	if ((aligned_addr + size) < (chunk->addr + chunk->size)) {
+		right_chunk = (struct free_chunk *)phys_to_virt(aligned_addr + size);
+		right_chunk->addr = aligned_addr + size;
+		right_chunk->size = (chunk->addr + chunk->size) - (aligned_addr + size);
+	}
+
+	/* Is there leftover on the left? */
+	if (aligned_addr != chunk->addr) {
+		left_chunk = chunk;
+	}
+
+	/* Update chunk's size, possibly becomes zero */
+	chunk->size = (aligned_addr - chunk->addr);
+
+	if (left_chunk) {
+		/* Left chunk reuses chunk, add right chunk */
+		if (right_chunk) {
+			dkprintf("%s: adding right chunk: 0x%lx:%lu\n",
+					__FUNCTION__, right_chunk->addr, right_chunk->size);
+			if (__page_alloc_rbtree_free_range(root,
+					right_chunk->addr, right_chunk->size)) {
+				kprintf("%s: ERROR: adding right chunk: 0x%lx:%lu\n",
+						__FUNCTION__, right_chunk->addr, right_chunk->size);
+				return EINVAL;
+			}
+		}
+	}
+	else {
+		/* Replace left with right */
+		if (right_chunk) {
+			rb_replace_node(&chunk->node, &right_chunk->node, root);
+			dkprintf("%s: chunk replaced with right: 0x%lx:%lu\n",
+					__FUNCTION__, right_chunk->addr, right_chunk->size);
+		}
+		/* No left chunk and no right chunk => chunk was exact match, delete it */
+		else {
+			rb_erase(&chunk->node, root);
+			dkprintf("%s: chunk deleted: 0x%lx:%lu\n",
+					__FUNCTION__, chunk->addr, chunk->size);
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Allocate pages.
+ *
+ * NOTE: locking must be managed by the caller.
+ */
+static unsigned long __page_alloc_rbtree_alloc_pages(struct rb_root *root,
+		int npages, int p2align)
+{
+	struct free_chunk *chunk;
+	struct rb_node *node;
+	unsigned long size = PAGE_SIZE * npages;
+	unsigned long align_size = (PAGE_SIZE << p2align);
+	unsigned long align_mask = ~(align_size - 1);
+	unsigned long aligned_addr = 0;
+
+	for (node = rb_first(root); node; node = rb_next(node)) {
+		chunk = container_of(node, struct free_chunk, node);
+		aligned_addr = (chunk->addr + (align_size - 1)) & align_mask;
+
+		/* Is this a suitable chunk? */
+		if ((aligned_addr + size) <= (chunk->addr + chunk->size)) {
+			break;
+		}
+	}
+
+	/* No matching chunk at all? */
+	if (!node) {
+		return 0;
+	}
+
+	dkprintf("%s: allocating: 0x%lx:%lu\n",
+			__FUNCTION__, aligned_addr, size);
+	if (__page_alloc_rbtree_mark_range_allocated(root, chunk,
+			aligned_addr, size)) {
+		kprintf("%s: ERROR: allocating 0x%lx:%lu\n",
+			__FUNCTION__, aligned_addr, size);
+		return 0;
+	}
+
+	return aligned_addr;
+}
+
+/*
+ * Reserve pages.
+ *
+ * NOTE: locking must be managed by the caller.
+ */
+static unsigned long __page_alloc_rbtree_reserve_pages(struct rb_root *root,
+		unsigned long aligned_addr, int npages)
+{
+	struct free_chunk *chunk;
+	struct rb_node *node;
+	unsigned long size = PAGE_SIZE * npages;
+
+	for (node = rb_first(root); node; node = rb_next(node)) {
+		chunk = container_of(node, struct free_chunk, node);
+
+		/* Is this the containing chunk? */
+		if (aligned_addr >= chunk->addr &&
+				(aligned_addr + size) <= (chunk->addr + chunk->size)) {
+			break;
+		}
+	}
+
+	/* No matching chunk at all? */
+	if (!node) {
+		kprintf("%s: WARNING: attempted to reserve non-free"
+				" physical range: 0x%lx:%lu\n",
+				__FUNCTION__,
+				aligned_addr, size);
+		return 0;
+	}
+
+	dkprintf("%s: reserving: 0x%lx:%lu\n",
+			__FUNCTION__, aligned_addr, size);
+	if (__page_alloc_rbtree_mark_range_allocated(root, chunk,
+			aligned_addr, size)) {
+		kprintf("%s: ERROR: reserving 0x%lx:%lu\n",
+			__FUNCTION__, aligned_addr, size);
+		return 0;
+	}
+
+	return aligned_addr;
+}
+
+
+/*
+ * External routines.
+ */
+int ihk_numa_add_free_pages(struct ihk_mc_numa_node *node,
+		unsigned long addr, unsigned long size)
+{
+	if (__page_alloc_rbtree_free_range(&node->free_chunks, addr, size)) {
+		kprintf("%s: ERROR: adding 0x%lx:%lu\n",
+			__FUNCTION__, addr, size);
+		return EINVAL;
+	}
+
+	if (addr < node->min_addr)
+		node->min_addr = addr;
+
+	if (addr + size > node->max_addr)
+		node->max_addr = addr + size;
+
+	node->nr_free_pages += (size >> PAGE_SHIFT);
+	dkprintf("%s: added free pages 0x%lx:%lu\n",
+		__FUNCTION__, addr, size);
+	return 0;
+}
+
+
+unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
+	int npages, int p2align)
+{
+	unsigned long addr = 0;
+	mcs_lock_node_t mcs_node;
+
+	mcs_lock_lock(&node->lock, &mcs_node);
+
+	if (node->nr_free_pages < npages) {
+		goto unlock_out;
+	}
+
+	addr = __page_alloc_rbtree_alloc_pages(&node->free_chunks,
+			npages, p2align);
+
+	/* Does not necessarily succeed due to alignment */
+	if (addr) {
+		node->nr_free_pages -= npages;
+		dkprintf("%s: allocated pages 0x%lx:%lu\n",
+				__FUNCTION__, addr, npages << PAGE_SHIFT);
+	}
+
+unlock_out:
+	mcs_lock_unlock(&node->lock, &mcs_node);
+
+	return addr;
+}
+
+void ihk_numa_free_pages(struct ihk_mc_numa_node *node,
+	unsigned long addr, int npages)
+{
+	mcs_lock_node_t mcs_node;
+
+	if (addr < node->min_addr ||
+			(addr + (npages << PAGE_SHIFT)) > node->max_addr) {
+		return;
+	}
+
+	if (npages <= 0) {
+		return;
+	}
+
+	mcs_lock_lock(&node->lock, &mcs_node);
+	if (__page_alloc_rbtree_free_range(&node->free_chunks, addr,
+				npages << PAGE_SHIFT)) {
+		kprintf("%s: ERROR: freeing 0x%lx:%lu\n",
+				__FUNCTION__, addr, npages << PAGE_SHIFT);
+	}
+	else {
+		node->nr_free_pages += npages;
+		dkprintf("%s: freed pages 0x%lx:%lu\n",
+				__FUNCTION__, addr, npages << PAGE_SHIFT);
+	}
+	mcs_lock_unlock(&node->lock, &mcs_node);
+}
+
+#endif // IHK_RBTREE_ALLOCATOR