gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Red-black tree based physical memory management

Browse Source
This commit is contained in:
Balazs Gerofi
2017-06-30 22:02:37 +09:00
parent 5cdd194856
commit 201a7e2595
3 changed files with 462 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
lib/include/ihk/page_alloc.h
View File

@@ -5,16 +5,19 @@
* Declare functions acquire physical pages and assign virtual addresses
* to them.
* \author Taku Shimosawa <shimosawa@is.s.u-tokyo.ac.jp> \par
* Copyright (C) 2011 - 2012 Taku Shimosawa
* \author Balazs Gerofi <bgerofi@riken.jp> \par
*/
/*
* HISTORY
* 2016/12 - bgerofi - NUMA support
* 2017/06 - bgerofi - rewrite physical memory mngt for red-black trees
*/
#ifndef __HEADER_GENERIC_IHK_PAGE_ALLOC
#define __HEADER_GENERIC_IHK_PAGE_ALLOC
#include <list.h>
#include <rbtree.h>
/* XXX: Physical memory management shouldn't be part of IHK */
struct node_distance {
@@ -22,14 +25,40 @@ struct node_distance {
int distance;
};
#define IHK_RBTREE_ALLOCATOR
#ifdef IHK_RBTREE_ALLOCATOR
struct free_chunk {
unsigned long addr, size;
struct rb_node node;
};
#endif
struct ihk_mc_numa_node {
int id;
int linux_numa_id;
int type;
struct list_head allocators;
struct node_distance *nodes_by_distance;
#ifdef IHK_RBTREE_ALLOCATOR
struct rb_root free_chunks;
mcs_lock_node_t lock;
unsigned long nr_free_pages;
unsigned long min_addr;
unsigned long max_addr;
#endif
};
#ifdef IHK_RBTREE_ALLOCATOR
unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
int npages, int p2align);
void ihk_numa_free_pages(struct ihk_mc_numa_node *node,
unsigned long addr, int npages);
int ihk_numa_add_free_pages(struct ihk_mc_numa_node *node,
unsigned long addr, unsigned long size);
#endif
struct ihk_page_allocator_desc {
unsigned long start, end;
unsigned int last;

351
lib/page_alloc.c
View File

@@ -18,6 +18,15 @@
#include <ihk/page_alloc.h>
#include <memory.h>
#include <bitops.h>
#include <errno.h>
//#define DEBUG_PRINT_PAGE_ALLOC
#ifdef DEBUG_PRINT_PAGE_ALLOC
#define dkprintf kprintf
#else
#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
#endif
void free_pages(void *, int npages);
@@ -301,3 +310,345 @@ kprintf("\nzeroing done\n");
}
#ifdef IHK_RBTREE_ALLOCATOR
/*
* Simple red-black tree based physical memory management routines.
*
* Allocation grabs first suitable chunk (splits chunk if alignment requires it).
* Deallocation merges with immediate neighbours.
*
* NOTE: invariant property: free_chunk structures are placed in the very front
* of their corresponding memory (i.e., they are on the free memory chunk itself).
*/
/*
* Free pages.
* NOTE: locking must be managed by the caller.
*/
static int __page_alloc_rbtree_free_range(struct rb_root *root,
unsigned long addr, unsigned long size)
{
struct rb_node **iter = &(root->rb_node), *parent = NULL;
struct free_chunk *new_chunk;
/* Figure out where to put new node */
while (*iter) {
struct free_chunk *ichunk = container_of(*iter, struct free_chunk, node);
parent = *iter;
if ((addr >= ichunk->addr) && (addr < ichunk->addr + ichunk->size)) {
kprintf("%s: ERROR: free memory chunk: 0x%lx:%lu"
" and requested range to be freed: 0x%lx:%lu are "
"overlapping (double-free?)\n",
__FUNCTION__,
ichunk->addr, ichunk->size, addr, size);
return EINVAL;
}
/* Is ichunk contigous from the left? */
if (ichunk->addr + ichunk->size == addr) {
struct rb_node *right;
/* Extend it to the right */
ichunk->size += size;
dkprintf("%s: chunk extended to right: 0x%lx:%lu\n",
__FUNCTION__, ichunk->addr, ichunk->size);
/* Have the right chunk of ichunk and ichunk become contigous? */
right = rb_next(*iter);
if (right) {
struct free_chunk *right_chunk =
container_of(right, struct free_chunk, node);
if (ichunk->addr + ichunk->size == right_chunk->addr) {
ichunk->size += right_chunk->size;
rb_erase(right, root);
dkprintf("%s: chunk merged to right: 0x%lx:%lu\n",
__FUNCTION__, ichunk->addr, ichunk->size);
}
}
return 0;
}
/* Is ichunk contigous from the right? */
if (addr + size == ichunk->addr) {
struct rb_node *left;
/* Extend it to the left */
ichunk->addr -= size;
ichunk->size += size;
dkprintf("%s: chunk extended to left: 0x%lx:%lu\n",
__FUNCTION__, ichunk->addr, ichunk->size);
/* Have the left chunk of ichunk and ichunk become contigous? */
left = rb_prev(*iter);
if (left) {
struct free_chunk *left_chunk =
container_of(left, struct free_chunk, node);
if (left_chunk->addr + left_chunk->size == ichunk->addr) {
ichunk->addr -= left_chunk->size;
ichunk->size += left_chunk->size;
rb_erase(left, root);
dkprintf("%s: chunk merged to left: 0x%lx:%lu\n",
__FUNCTION__, ichunk->addr, ichunk->size);
}
}
/* Move chunk structure to the front */
new_chunk = (struct free_chunk *)phys_to_virt(ichunk->addr);
*new_chunk = *ichunk;
rb_replace_node(&ichunk->node, &new_chunk->node, root);
dkprintf("%s: chunk moved to front: 0x%lx:%lu\n",
__FUNCTION__, new_chunk->addr, new_chunk->size);
return 0;
}
if (addr < ichunk->addr)
iter = &((*iter)->rb_left);
else
iter = &((*iter)->rb_right);
}
new_chunk = (struct free_chunk *)phys_to_virt(addr);
new_chunk->addr = addr;
new_chunk->size = size;
dkprintf("%s: new chunk: 0x%lx:%lu\n",
__FUNCTION__, new_chunk->addr, new_chunk->size);
/* Add new node and rebalance tree. */
rb_link_node(&new_chunk->node, parent, iter);
rb_insert_color(&new_chunk->node, root);
return 0;
}
/*
* Mark address range as used (i.e., allocated).
*
* chunk is the free memory chunk in which
* [aligned_addr, aligned_addr + size] resides.
*
* NOTE: locking must be managed by the caller.
*/
static int __page_alloc_rbtree_mark_range_allocated(struct rb_root *root,
struct free_chunk *chunk,
unsigned long aligned_addr, unsigned long size)
{
struct free_chunk *left_chunk = NULL, *right_chunk = NULL;
/* Is there leftover on the right? */
if ((aligned_addr + size) < (chunk->addr + chunk->size)) {
right_chunk = (struct free_chunk *)phys_to_virt(aligned_addr + size);
right_chunk->addr = aligned_addr + size;
right_chunk->size = (chunk->addr + chunk->size) - (aligned_addr + size);
}
/* Is there leftover on the left? */
if (aligned_addr != chunk->addr) {
left_chunk = chunk;
}
/* Update chunk's size, possibly becomes zero */
chunk->size = (aligned_addr - chunk->addr);
if (left_chunk) {
/* Left chunk reuses chunk, add right chunk */
if (right_chunk) {
dkprintf("%s: adding right chunk: 0x%lx:%lu\n",
__FUNCTION__, right_chunk->addr, right_chunk->size);
if (__page_alloc_rbtree_free_range(root,
right_chunk->addr, right_chunk->size)) {
kprintf("%s: ERROR: adding right chunk: 0x%lx:%lu\n",
__FUNCTION__, right_chunk->addr, right_chunk->size);
return EINVAL;
}
}
}
else {
/* Replace left with right */
if (right_chunk) {
rb_replace_node(&chunk->node, &right_chunk->node, root);
dkprintf("%s: chunk replaced with right: 0x%lx:%lu\n",
__FUNCTION__, right_chunk->addr, right_chunk->size);
}
/* No left chunk and no right chunk => chunk was exact match, delete it */
else {
rb_erase(&chunk->node, root);
dkprintf("%s: chunk deleted: 0x%lx:%lu\n",
__FUNCTION__, chunk->addr, chunk->size);
}
}
return 0;
}
/*
* Allocate pages.
*
* NOTE: locking must be managed by the caller.
*/
static unsigned long __page_alloc_rbtree_alloc_pages(struct rb_root *root,
int npages, int p2align)
{
struct free_chunk *chunk;
struct rb_node *node;
unsigned long size = PAGE_SIZE * npages;
unsigned long align_size = (PAGE_SIZE << p2align);
unsigned long align_mask = ~(align_size - 1);
unsigned long aligned_addr = 0;
for (node = rb_first(root); node; node = rb_next(node)) {
chunk = container_of(node, struct free_chunk, node);
aligned_addr = (chunk->addr + (align_size - 1)) & align_mask;
/* Is this a suitable chunk? */
if ((aligned_addr + size) <= (chunk->addr + chunk->size)) {
break;
}
}
/* No matching chunk at all? */
if (!node) {
return 0;
}
dkprintf("%s: allocating: 0x%lx:%lu\n",
__FUNCTION__, aligned_addr, size);
if (__page_alloc_rbtree_mark_range_allocated(root, chunk,
aligned_addr, size)) {
kprintf("%s: ERROR: allocating 0x%lx:%lu\n",
__FUNCTION__, aligned_addr, size);
return 0;
}
return aligned_addr;
}
/*
* Reserve pages.
*
* NOTE: locking must be managed by the caller.
*/
static unsigned long __page_alloc_rbtree_reserve_pages(struct rb_root *root,
unsigned long aligned_addr, int npages)
{
struct free_chunk *chunk;
struct rb_node *node;
unsigned long size = PAGE_SIZE * npages;
for (node = rb_first(root); node; node = rb_next(node)) {
chunk = container_of(node, struct free_chunk, node);
/* Is this the containing chunk? */
if (aligned_addr >= chunk->addr &&
(aligned_addr + size) <= (chunk->addr + chunk->size)) {
break;
}
}
/* No matching chunk at all? */
if (!node) {
kprintf("%s: WARNING: attempted to reserve non-free"
" physical range: 0x%lx:%lu\n",
__FUNCTION__,
aligned_addr, size);
return 0;
}
dkprintf("%s: reserving: 0x%lx:%lu\n",
__FUNCTION__, aligned_addr, size);
if (__page_alloc_rbtree_mark_range_allocated(root, chunk,
aligned_addr, size)) {
kprintf("%s: ERROR: reserving 0x%lx:%lu\n",
__FUNCTION__, aligned_addr, size);
return 0;
}
return aligned_addr;
}
/*
* External routines.
*/
int ihk_numa_add_free_pages(struct ihk_mc_numa_node *node,
unsigned long addr, unsigned long size)
{
if (__page_alloc_rbtree_free_range(&node->free_chunks, addr, size)) {
kprintf("%s: ERROR: adding 0x%lx:%lu\n",
__FUNCTION__, addr, size);
return EINVAL;
}
if (addr < node->min_addr)
node->min_addr = addr;
if (addr + size > node->max_addr)
node->max_addr = addr + size;
node->nr_free_pages += (size >> PAGE_SHIFT);
dkprintf("%s: added free pages 0x%lx:%lu\n",
__FUNCTION__, addr, size);
return 0;
}
unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
int npages, int p2align)
{
unsigned long addr = 0;
mcs_lock_node_t mcs_node;
mcs_lock_lock(&node->lock, &mcs_node);
if (node->nr_free_pages < npages) {
goto unlock_out;
}
addr = __page_alloc_rbtree_alloc_pages(&node->free_chunks,
npages, p2align);
/* Does not necessarily succeed due to alignment */
if (addr) {
node->nr_free_pages -= npages;
dkprintf("%s: allocated pages 0x%lx:%lu\n",
__FUNCTION__, addr, npages << PAGE_SHIFT);
}
unlock_out:
mcs_lock_unlock(&node->lock, &mcs_node);
return addr;
}
void ihk_numa_free_pages(struct ihk_mc_numa_node *node,
unsigned long addr, int npages)
{
mcs_lock_node_t mcs_node;
if (addr < node->min_addr ||
(addr + (npages << PAGE_SHIFT)) > node->max_addr) {
return;
}
if (npages <= 0) {
return;
}
mcs_lock_lock(&node->lock, &mcs_node);
if (__page_alloc_rbtree_free_range(&node->free_chunks, addr,
npages << PAGE_SHIFT)) {
kprintf("%s: ERROR: freeing 0x%lx:%lu\n",
__FUNCTION__, addr, npages << PAGE_SHIFT);
}
else {
node->nr_free_pages += npages;
dkprintf("%s: freed pages 0x%lx:%lu\n",
__FUNCTION__, addr, npages << PAGE_SHIFT);
}
mcs_lock_unlock(&node->lock, &mcs_node);
}
#endif // IHK_RBTREE_ALLOCATOR