MM: deferred zero cleaning on Linux CPUs

Change-Id: Icdb8ac807688533be7a95b7101edfd904250cd02

kernel/include/rbtree.h

@@ -108,4 +108,6 @@ static inline void rb_link_node(struct rb_node * node, struct rb_node * parent,
 			typeof(*pos), field); 1; }); \
 	     pos = n)
 
+struct rb_node *rb_preorder_dfs_search(const struct rb_root *root,
+	bool (*__cond)(struct rb_node *, void *arg), void *__cond_arg);
 #endif	/* _LINUX_RBTREE_H */

kernel/mem.c

@@ -44,6 +44,7 @@
 #include <limits.h>
 #include <sysfs.h>
 #include <ihk/debug.h>
+#include <llist.h>
 #include <bootparam.h>
 
 //#define DEBUG_PRINT_MEM
@@ -749,7 +750,6 @@ distance_based:
 						ihk_mc_get_numa_id(),
 						npages,
 						memory_nodes[node].nodes_by_distance[i].id);
-
 			}
 		}
 
@@ -1546,13 +1546,15 @@ static void numa_init(void)
 		INIT_LIST_HEAD(&memory_nodes[i].allocators);
 		memory_nodes[i].nodes_by_distance = 0;
 #ifdef IHK_RBTREE_ALLOCATOR
-		memory_nodes[i].zeroed_chunks.rb_node = 0;
+		ihk_atomic_set(&memory_nodes[i].zeroing_workers, 0);
+		ihk_atomic_set(&memory_nodes[i].nr_to_zero_pages, 0);
 		memory_nodes[i].free_chunks.rb_node = 0;
+		init_llist_head(&memory_nodes[i].zeroed_list);
+		init_llist_head(&memory_nodes[i].to_zero_list);
 		mcs_lock_init(&memory_nodes[i].lock);
 		memory_nodes[i].min_addr = 0xFFFFFFFFFFFFFFFF;
 		memory_nodes[i].max_addr = 0;
 		memory_nodes[i].nr_pages = 0;
-		memory_nodes[i].nr_zeroed_pages = 0;
 		memory_nodes[i].nr_free_pages = 0;
 #endif
 	}

kernel/rbtree.c

@@ -432,6 +432,43 @@ struct rb_node *rb_first(const struct rb_root *root)
 }
 EXPORT_SYMBOL(rb_first);
 
+/*
+ * Pre-order depth first search.
+ * Return a node where __cond is true.
+ */
+static struct rb_node *__rb_preorder_dfs(struct rb_node *n,
+	bool (*__cond)(struct rb_node *, void *arg), void *__cond_arg)
+{
+	struct rb_node *left_res = NULL;
+
+	if (__cond(n, __cond_arg))
+		return n;
+
+	if (n->rb_left) {
+		left_res = __rb_preorder_dfs(n->rb_left, __cond, __cond_arg);
+		if (left_res) {
+			return left_res;
+		}
+	}
+	if (n->rb_right)
+		return __rb_preorder_dfs(n->rb_right, __cond, __cond_arg);
+
+	return NULL;
+}
+
+struct rb_node *rb_preorder_dfs_search(const struct rb_root *root,
+	bool (*__cond)(struct rb_node *, void *arg), void *__cond_arg)
+{
+	struct rb_node *n;
+
+	n = root->rb_node;
+	if (!n)
+		return NULL;
+
+	return __rb_preorder_dfs(n, __cond, __cond_arg);
+}
+
+
 struct rb_node *rb_first_safe(const struct rb_root *root)
 {
 	struct rb_node	*n;