gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Include mbind support (this is a rebase commit to merge into development)

Browse Source
This commit is contained in:
Katsuya Horigome
2017-11-06 12:47:56 +09:00
parent bad487cc07
commit d649d6fc2d
124 changed files with 1458 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kernel/devobj.c
View File

@@ -225,7 +225,7 @@ static void devobj_release(struct memobj *memobj)
return;
}
static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintptr_t *physp, unsigned long *flag)
static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintptr_t *physp, unsigned long *flag, uintptr_t virt_addr)
{
const off_t pgoff = off / PAGE_SIZE;
struct devobj *obj = to_devobj(memobj);

30
kernel/fileobj.c
View File

@@ -191,7 +191,7 @@ static struct fileobj *obj_list_lookup(uintptr_t handle)
/***********************************************************************
* fileobj
*/
int fileobj_create(int fd, struct memobj **objp, int *maxprotp)
int fileobj_create(int fd, struct memobj **objp, int *maxprotp, uintptr_t virt_addr)
{
ihk_mc_user_context_t ctx;
struct pager_create_result result __attribute__((aligned(64)));
@@ -265,7 +265,7 @@ int fileobj_create(int fd, struct memobj **objp, int *maxprotp)
/* Get the actual pages NUMA interleaved */
for (j = 0; j < nr_pages; ++j) {
mo->pages[j] = ihk_mc_alloc_aligned_pages_node_user(1,
PAGE_P2ALIGN, IHK_MC_AP_NOWAIT, node);
PAGE_P2ALIGN, IHK_MC_AP_NOWAIT, node, virt_addr);
if (!mo->pages[j]) {
kprintf("%s: ERROR: allocating pages[%d]\n",
__FUNCTION__, j);
@@ -558,7 +558,7 @@ out:
}
static int fileobj_get_page(struct memobj *memobj, off_t off,
int p2align, uintptr_t *physp, unsigned long *pflag)
int p2align, uintptr_t *physp, unsigned long *pflag, uintptr_t virt_addr)
{
struct thread *proc = cpu_local_var(current);
struct fileobj *obj = to_fileobj(memobj);
@@ -571,7 +571,7 @@ static int fileobj_get_page(struct memobj *memobj, off_t off,
struct mcs_rwlock_node mcs_node;
int hash = (off >> PAGE_SHIFT) & FILEOBJ_PAGE_HASH_MASK;
dkprintf("fileobj_get_page(%p,%lx,%x,%p)\n", obj, off, p2align, physp);
dkprintf("fileobj_get_page(%p,%lx,%x,%x,%p)\n", obj, off, p2align, virt_addr, physp);
if (p2align != PAGE_P2ALIGN) {
return -ENOMEM;
}
@@ -584,13 +584,13 @@ static int fileobj_get_page(struct memobj *memobj, off_t off,
int page_ind = off >> PAGE_SHIFT;
if (!memobj->pages[page_ind]) {
virt = ihk_mc_alloc_pages_user(1, IHK_MC_AP_NOWAIT | IHK_MC_AP_USER);
virt = ihk_mc_alloc_pages_user(1, IHK_MC_AP_NOWAIT | IHK_MC_AP_USER, virt_addr);
if (!virt) {
error = -ENOMEM;
kprintf("fileobj_get_page(%p,%lx,%x,%p):"
kprintf("fileobj_get_page(%p,%lx,%x,%x,%x,%p):"
"alloc failed. %d\n",
obj, off, p2align, physp,
obj, off, p2align, virt_addr, physp,
error);
goto out_nolock;
}
@@ -627,22 +627,22 @@ static int fileobj_get_page(struct memobj *memobj, off_t off,
args = kmalloc(sizeof(*args), IHK_MC_AP_NOWAIT);
if (!args) {
error = -ENOMEM;
kprintf("fileobj_get_page(%p,%lx,%x,%p):"
kprintf("fileobj_get_page(%p,%lx,%x,%x,%p):"
"kmalloc failed. %d\n",
obj, off, p2align, physp, error);
obj, off, p2align, virt_addr, physp, error);
goto out;
}
if (!page) {
npages = 1 << p2align;
virt = ihk_mc_alloc_pages_user(npages, IHK_MC_AP_NOWAIT |
(to_memobj(obj)->flags & MF_ZEROFILL) ? IHK_MC_AP_USER : 0);
virt = ihk_mc_alloc_pages_user(npages, (IHK_MC_AP_NOWAIT |
(to_memobj(obj)->flags & MF_ZEROFILL) ? IHK_MC_AP_USER : 0), virt_addr);
if (!virt) {
error = -ENOMEM;
kprintf("fileobj_get_page(%p,%lx,%x,%p):"
kprintf("fileobj_get_page(%p,%lx,%x,%x,%p):"
"alloc failed. %d\n",
obj, off, p2align, physp,
obj, off, p2align, virt_addr, physp,
error);
goto out;
}
@@ -707,8 +707,8 @@ out_nolock:
if (args) {
kfree(args);
}
dkprintf("fileobj_get_page(%p,%lx,%x,%p): %d %lx\n",
obj, off, p2align, physp, error, phys);
dkprintf("fileobj_get_page(%p,%lx,%x,%x,%p): %d %lx\n",
obj, off, p2align, virt_addr, physp, error, phys);
return error;
}

4
kernel/host.c
View File

@@ -148,7 +148,7 @@ int prepare_process_ranges_args_envs(struct thread *thread,
}
if ((up_v = ihk_mc_alloc_pages_user(range_npages,
IHK_MC_AP_NOWAIT | ap_flags)) == NULL) {
IHK_MC_AP_NOWAIT | ap_flags, s)) == NULL) {
kprintf("ERROR: alloc pages for ELF section %i\n", i);
goto err;
}
@@ -259,7 +259,7 @@ int prepare_process_ranges_args_envs(struct thread *thread,
e = addr + PAGE_SIZE * ARGENV_PAGE_COUNT;
if((args_envs = ihk_mc_alloc_pages_user(ARGENV_PAGE_COUNT,
IHK_MC_AP_NOWAIT)) == NULL){
IHK_MC_AP_NOWAIT, -1)) == NULL){
kprintf("ERROR: allocating pages for args/envs\n");
goto err;
}

8
kernel/include/memobj.h
View File

@@ -65,7 +65,7 @@ struct memobj {
typedef void memobj_release_func_t(struct memobj *obj);
typedef void memobj_ref_func_t(struct memobj *obj);
typedef int memobj_get_page_func_t(struct memobj *obj, off_t off, int p2align, uintptr_t *physp, unsigned long *flag);
typedef int memobj_get_page_func_t(struct memobj *obj, off_t off, int p2align, uintptr_t *physp, unsigned long *flag, uintptr_t virt_addr);
typedef uintptr_t memobj_copy_page_func_t(struct memobj *obj, uintptr_t orgphys, int p2align);
typedef int memobj_flush_page_func_t(struct memobj *obj, uintptr_t phys, size_t pgsize);
typedef int memobj_invalidate_page_func_t(struct memobj *obj, uintptr_t phys, size_t pgsize);
@@ -96,10 +96,10 @@ static inline void memobj_ref(struct memobj *obj)
}
static inline int memobj_get_page(struct memobj *obj, off_t off,
int p2align, uintptr_t *physp, unsigned long *pflag)
int p2align, uintptr_t *physp, unsigned long *pflag, uintptr_t virt_addr)
{
if (obj->ops->get_page) {
return (*obj->ops->get_page)(obj, off, p2align, physp, pflag);
return (*obj->ops->get_page)(obj, off, p2align, physp, pflag, virt_addr);
}
return -ENXIO;
}
@@ -159,7 +159,7 @@ static inline int memobj_is_removable(struct memobj *obj)
return !!(obj->flags & MF_IS_REMOVABLE);
}
int fileobj_create(int fd, struct memobj **objp, int *maxprotp);
int fileobj_create(int fd, struct memobj **objp, int *maxprotp, uintptr_t virt_addr);
struct shmid_ds;
int shmobj_create(struct shmid_ds *ds, struct memobj **objp);
int zeroobj_create(struct memobj **objp);

4
kernel/include/process.h
View File

@@ -390,7 +390,7 @@ struct vm_range {
};
struct vm_range_numa_policy {
struct list_head list;
struct rb_node policy_rb_node;
unsigned long start, end;
DECLARE_BITMAP(numa_mask, PROCESS_NUMA_MASK_BITS);
int numa_mem_policy;
@@ -725,7 +725,7 @@ struct process_vm {
DECLARE_BITMAP(numa_mask, PROCESS_NUMA_MASK_BITS);
int numa_mem_policy;
/* Protected by memory_range_lock */
struct list_head vm_range_numa_policy_list;
struct rb_root vm_range_numa_policy_tree;
struct vm_range *range_cache[VM_RANGE_CACHE_SIZE];
int range_cache_ind;
struct swapinfo *swapinfo;

1
kernel/include/syscall.h
View File

@@ -471,6 +471,7 @@ int arch_map_vdso(struct process_vm *vm); /* arch dependent */
int arch_setup_vdso(void);
int arch_cpu_read_write_register(struct ihk_os_cpu_register *desc,
enum mcctrl_os_cpu_operation op);
struct vm_range_numa_policy *vm_range_policy_search(struct process_vm *vm, uintptr_t addr);
#ifndef POSTK_DEBUG_ARCH_DEP_52
#define VDSO_MAXPAGES 2

61
kernel/mem.c
View File

@@ -76,7 +76,7 @@ static void *___kmalloc(int size, ihk_mc_ap_flag flag);
static void ___kfree(void *ptr);
static void *___ihk_mc_alloc_aligned_pages_node(int npages,
int p2align, ihk_mc_ap_flag flag, int node, int is_user);
int p2align, ihk_mc_ap_flag flag, int node, int is_user, uintptr_t virt_addr);
static void *___ihk_mc_alloc_pages(int npages, ihk_mc_ap_flag flag, int is_user);
static void ___ihk_mc_free_pages(void *p, int npages, int is_user);
@@ -193,7 +193,7 @@ struct pagealloc_track_entry *__pagealloc_track_find_entry(
/* Top level routines called from macros */
void *_ihk_mc_alloc_aligned_pages_node(int npages, int p2align,
ihk_mc_ap_flag flag, int node, int is_user,
ihk_mc_ap_flag flag, int node, int is_user, uintptr_t virt_addr,
char *file, int line)
{
unsigned long irqflags;
@@ -201,7 +201,7 @@ void *_ihk_mc_alloc_aligned_pages_node(int npages, int p2align,
struct pagealloc_track_addr_entry *addr_entry;
int hash, addr_hash;
void *r = ___ihk_mc_alloc_aligned_pages_node(npages,
p2align, flag, node, is_user);
p2align, flag, node, is_user, virt_addr);
if (!memdebug || !pagealloc_track_initialized)
return r;
@@ -497,10 +497,10 @@ void pagealloc_memcheck(void)
/* Actual allocation routines */
static void *___ihk_mc_alloc_aligned_pages_node(int npages, int p2align,
ihk_mc_ap_flag flag, int node, int is_user)
ihk_mc_ap_flag flag, int node, int is_user, uintptr_t virt_addr)
{
if (pa_ops)
return pa_ops->alloc_page(npages, p2align, flag, node, is_user);
return pa_ops->alloc_page(npages, p2align, flag, node, is_user, virt_addr);
else
return early_alloc_pages(npages);
}
@@ -508,7 +508,7 @@ static void *___ihk_mc_alloc_aligned_pages_node(int npages, int p2align,
static void *___ihk_mc_alloc_pages(int npages, ihk_mc_ap_flag flag,
int is_user)
{
return ___ihk_mc_alloc_aligned_pages_node(npages, PAGE_P2ALIGN, flag, -1, is_user);
return ___ihk_mc_alloc_aligned_pages_node(npages, PAGE_P2ALIGN, flag, -1, is_user, -1);
}
static void ___ihk_mc_free_pages(void *p, int npages, int is_user)
@@ -544,7 +544,7 @@ static void reserve_pages(struct ihk_page_allocator_desc *pa_allocator,
extern int cpu_local_var_initialized;
static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
ihk_mc_ap_flag flag, int pref_node, int is_user)
ihk_mc_ap_flag flag, int pref_node, int is_user, uintptr_t virt_addr)
{
unsigned long pa = 0;
int i, node;
@@ -553,6 +553,12 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
#endif
int numa_id;
struct vm_range_numa_policy *range_policy_iter = NULL;
int numa_mem_policy = -1;
struct process_vm *vm;
struct vm_range *range = NULL;
int chk_shm = 0;
if(npages <= 0)
return NULL;
@@ -565,7 +571,23 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
/* No explicitly requested NUMA or user policy? */
if ((pref_node == -1) && (!(flag & IHK_MC_AP_USER) ||
cpu_local_var(current)->vm->numa_mem_policy == MPOL_DEFAULT)) {
goto distance_based;
if (virt_addr != -1) {
vm = cpu_local_var(current)->vm;
range_policy_iter = vm_range_policy_search(vm, virt_addr);
if (range_policy_iter) {
range = lookup_process_memory_range(vm, (uintptr_t)virt_addr, ((uintptr_t)virt_addr) + 1);
if (range) {
if( (range->memobj) && (range->memobj->flags == MF_SHM)) {
chk_shm = 1;
}
}
}
}
if ((!((range_policy_iter) && (range_policy_iter->numa_mem_policy != MPOL_DEFAULT))) && (chk_shm == 0))
goto distance_based;
}
node = ihk_mc_get_numa_id();
@@ -611,7 +633,28 @@ static void *mckernel_allocate_aligned_pages_node(int npages, int p2align,
}
}
switch (cpu_local_var(current)->vm->numa_mem_policy) {
if ((virt_addr != -1) && (chk_shm == 0)) {
vm = cpu_local_var(current)->vm;
if (!(range_policy_iter)) {
range_policy_iter = vm_range_policy_search(vm, virt_addr);
}
if (range_policy_iter) {
range = lookup_process_memory_range(vm, (uintptr_t)virt_addr, ((uintptr_t)virt_addr) + 1);
if ((range && (range->memobj->flags == MF_SHM))) {
chk_shm = 1;
} else {
numa_mem_policy = range_policy_iter->numa_mem_policy;
}
}
}
if (numa_mem_policy == -1)
numa_mem_policy = cpu_local_var(current)->vm->numa_mem_policy;
switch (numa_mem_policy) {
case MPOL_BIND:
case MPOL_PREFERRED:

16
kernel/process.c
View File

@@ -256,7 +256,7 @@ init_process_vm(struct process *owner, struct address_space *asp, struct process
ihk_atomic_set(&vm->refcount, 1);
vm->vm_range_tree = RB_ROOT;
INIT_LIST_HEAD(&vm->vm_range_numa_policy_list);
vm->vm_range_numa_policy_tree = RB_ROOT;
vm->address_space = asp;
vm->proc = owner;
vm->exiting = 0;
@@ -646,7 +646,7 @@ static int copy_user_pte(void *arg0, page_table_t src_pt, pte_t *src_ptep, void
npages = pgsize / PAGE_SIZE;
virt = ihk_mc_alloc_aligned_pages_user(npages, pgalign,
IHK_MC_AP_NOWAIT);
IHK_MC_AP_NOWAIT, (uintptr_t)pgaddr);
if (!virt) {
kprintf("ERROR: copy_user_pte() allocating new page\n");
error = -ENOMEM;
@@ -1765,7 +1765,7 @@ static int page_fault_process_memory_range(struct process_vm *vm, struct vm_rang
off = pte_get_off(ptep, pgsize);
}
error = memobj_get_page(range->memobj, off, p2align,
&phys, &memobj_flag);
&phys, &memobj_flag, fault_addr);
if (error) {
struct memobj *obj;
@@ -1787,7 +1787,7 @@ retry:
npages = pgsize / PAGE_SIZE;
virt = ihk_mc_alloc_aligned_pages_user(npages, p2align,
IHK_MC_AP_NOWAIT |
(range->flag & VR_AP_USER) ? IHK_MC_AP_USER : 0);
((range->flag & VR_AP_USER) ? IHK_MC_AP_USER : 0), fault_addr);
if (!virt && !range->pgshift && (pgsize != PAGE_SIZE)) {
error = arch_get_smaller_page_size(NULL, pgsize, &pgsize, &p2align);
if (error) {
@@ -1847,7 +1847,7 @@ retry:
npages = pgsize / PAGE_SIZE;
virt = ihk_mc_alloc_aligned_pages_user(npages, p2align,
IHK_MC_AP_NOWAIT);
IHK_MC_AP_NOWAIT, fault_addr);
if (!virt) {
error = -ENOMEM;
kprintf("page_fault_process_memory_range(%p,%lx-%lx %lx,%lx,%lx):cannot allocate copy page. %d\n", vm, range->start, range->end, range->flag, fault_addr, reason, error);
@@ -2156,7 +2156,7 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
ap_flag ? "(IHK_MC_AP_USER)" : "");
stack = ihk_mc_alloc_aligned_pages_user(minsz >> PAGE_SHIFT,
LARGE_PAGE_P2ALIGN, IHK_MC_AP_NOWAIT | ap_flag);
LARGE_PAGE_P2ALIGN, IHK_MC_AP_NOWAIT | ap_flag, start);
if (!stack) {
kprintf("%s: error: couldn't allocate initial stack\n",
@@ -2300,7 +2300,7 @@ unsigned long extend_process_region(struct process_vm *vm,
p = ihk_mc_alloc_aligned_pages_user(
(new_end_allocated - end_allocated) >> PAGE_SHIFT,
align_p2align, IHK_MC_AP_NOWAIT |
(!(vm->proc->mpol_flags & MPOL_NO_HEAP) ? IHK_MC_AP_USER : 0));
(!(vm->proc->mpol_flags & MPOL_NO_HEAP) ? IHK_MC_AP_USER : 0), end_allocated);
if (!p) {
return end_allocated;
@@ -2891,7 +2891,7 @@ void sched_init(void)
ihk_mc_init_context(&idle_thread->ctx, NULL, idle);
ihk_mc_spinlock_init(&idle_thread->vm->memory_range_lock);
idle_thread->vm->vm_range_tree = RB_ROOT;
INIT_LIST_HEAD(&idle_thread->vm->vm_range_numa_policy_list);
idle_thread->vm->vm_range_numa_policy_tree = RB_ROOT;
idle_thread->proc->pid = 0;
idle_thread->tid = ihk_mc_get_processor_id();

4
kernel/shmobj.c
View File

@@ -374,7 +374,7 @@ static void shmobj_ref(struct memobj *memobj)
}
static int shmobj_get_page(struct memobj *memobj, off_t off, int p2align,
uintptr_t *physp, unsigned long *pflag)
uintptr_t *physp, unsigned long *pflag, uintptr_t virt_addr)
{
struct shmobj *obj = to_shmobj(memobj);
int error;
@@ -415,7 +415,7 @@ static int shmobj_get_page(struct memobj *memobj, off_t off, int p2align,
if (!page) {
npages = 1 << p2align;
virt = ihk_mc_alloc_aligned_pages_user(npages, p2align,
IHK_MC_AP_NOWAIT);
IHK_MC_AP_NOWAIT, virt_addr);
if (!virt) {
error = -ENOMEM;
ekprintf("shmobj_get_page(%p,%#lx,%d,%p):"

130
kernel/syscall.c
View File

@@ -1473,7 +1473,7 @@ do_mmap(const intptr_t addr0, const size_t len0, const int prot,
maxprot = PROT_READ | PROT_WRITE | PROT_EXEC;
if (!(flags & MAP_ANONYMOUS)) {
off = off0;
error = fileobj_create(fd, &memobj, &maxprot);
error = fileobj_create(fd, &memobj, &maxprot, addr0);
#ifdef ATTACHED_MIC
/*
* XXX: refuse device mapping in attached-mic now:
@@ -1536,7 +1536,7 @@ do_mmap(const intptr_t addr0, const size_t len0, const int prot,
}
p = ihk_mc_alloc_aligned_pages_user(npages, p2align,
IHK_MC_AP_NOWAIT | ap_flag);
IHK_MC_AP_NOWAIT | ap_flag, addr0);
if (p == NULL) {
dkprintf("%s: warning: failed to allocate %d contiguous pages "
" (bytes: %lu, pgshift: %d), enabling demand paging\n",
@@ -3771,6 +3771,57 @@ perf_mmap(struct mckfd *sfd, ihk_mc_user_context_t *ctx)
return rc;
}
struct vm_range_numa_policy *vm_range_policy_search(struct process_vm *vm, uintptr_t addr)
{
struct rb_root *root = &vm->vm_range_numa_policy_tree;
struct rb_node *node = root->rb_node;
struct vm_range_numa_policy *numa_policy = NULL;
while (node) {
numa_policy = rb_entry(node, struct vm_range_numa_policy, policy_rb_node);
if (addr < numa_policy->start) {
node = node->rb_left;
} else if (addr >= numa_policy->end) {
node = node->rb_right;
} else {
return numa_policy;
}
}
return NULL;
}
static int vm_policy_insert(struct process_vm *vm, struct vm_range_numa_policy *newrange)
{
struct rb_root *root = &vm->vm_range_numa_policy_tree;
struct rb_node **new = &(root->rb_node), *parent = NULL;
struct vm_range_numa_policy *range;
while (*new) {
range = rb_entry(*new, struct vm_range_numa_policy, policy_rb_node);
parent = *new;
if (newrange->end <= range->start) {
new = &((*new)->rb_left);
} else if (newrange->start >= range->end) {
new = &((*new)->rb_right);
} else {
ekprintf("vm_range_insert(%p,%lx-%lx (nodemask)%lx (policy)%d): overlap %lx-%lx (nodemask)%lx (policy)%d\n",
vm, newrange->start, newrange->end, newrange->numa_mask, newrange->numa_mem_policy,
range->start, range->end, range->numa_mask, range->numa_mem_policy);
return -EFAULT;
}
}
dkprintf("vm_range_insert: %p,%p: %lx-%lx (nodemask)%lx (policy)%d\n",
vm, newrange, newrange->start, newrange->end, newrange->numa_mask, newrange->numa_mem_policy);
rb_link_node(&newrange->policy_rb_node, parent, new);
rb_insert_color(&newrange->policy_rb_node, root);
return 0;
}
struct mc_perf_event*
mc_perf_event_alloc(struct perf_event_attr *attr)
{
@@ -8063,8 +8114,7 @@ SYSCALL_DECLARE(mbind)
int error = 0;
int bit;
struct vm_range *range;
struct vm_range_numa_policy *range_policy, *range_policy_iter;
struct vm_range_numa_policy *range_policy_next = NULL;
struct vm_range_numa_policy *range_policy, *range_policy_iter = NULL;
DECLARE_BITMAP(numa_mask, PROCESS_NUMA_MASK_BITS);
dkprintf("%s: addr: 0x%lx, len: %lu, mode: 0x%x, "
@@ -8275,17 +8325,12 @@ SYSCALL_DECLARE(mbind)
case MPOL_INTERLEAVE:
case MPOL_PREFERRED:
/* Adjust any overlapping range settings and add new one */
range_policy_next = NULL;
list_for_each_entry(range_policy_iter,
&vm->vm_range_numa_policy_list, list) {
range_policy_iter = vm_range_policy_search(vm, addr);
if (range_policy_iter) {
int adjusted = 0;
unsigned long orig_end = range_policy_iter->end;
if (range_policy_iter->end < addr)
continue;
/* Special case of entirely overlapping */
if (range_policy_iter->start == addr &&
if (range_policy_iter->start == addr &&
range_policy_iter->end == addr + len) {
range_policy = range_policy_iter;
goto mbind_update_only;
@@ -8302,7 +8347,7 @@ SYSCALL_DECLARE(mbind)
if (orig_end > addr + len) {
if (adjusted) {
/* Add a new entry after */
range_policy = kmalloc(sizeof(*range_policy),
range_policy = kmalloc(sizeof(struct vm_range_numa_policy),
IHK_MC_AP_NOWAIT);
if (!range_policy) {
dkprintf("%s: error allocating range_policy\n",
@@ -8311,31 +8356,24 @@ SYSCALL_DECLARE(mbind)
goto unlock_out;
}
memcpy(range_policy, range_policy_iter,
sizeof(*range_policy));
RB_CLEAR_NODE(&range_policy->policy_rb_node);
range_policy->start = addr + len;
range_policy->end = orig_end;
list_add(&range_policy->list,
&range_policy_iter->list);
range_policy_next = range_policy;
break;
error = vm_policy_insert(vm, range_policy);
if (error) {
kprintf("%s: ERROR: could not insert range: %d\n",__FUNCTION__, error);
return error;
}
}
else {
range_policy_iter->start = addr + len;
range_policy_next = range_policy_iter;
break;
}
}
/* Next one in ascending address order? */
if (range_policy_iter->start >= addr + len) {
range_policy_next = range_policy_iter;
break;
}
}
/* Add a new entry */
range_policy = kmalloc(sizeof(*range_policy),
range_policy = kmalloc(sizeof(struct vm_range_numa_policy),
IHK_MC_AP_NOWAIT);
if (!range_policy) {
dkprintf("%s: error allocating range_policy\n",
@@ -8344,17 +8382,14 @@ SYSCALL_DECLARE(mbind)
goto unlock_out;
}
memset(range_policy, 0, sizeof(*range_policy));
RB_CLEAR_NODE(&range_policy->policy_rb_node);
range_policy->start = addr;
range_policy->end = addr + len;
if (range_policy_next) {
list_add_tail(&range_policy->list,
&range_policy_next->list);
}
else {
list_add_tail(&range_policy->list,
&vm->vm_range_numa_policy_list);
error = vm_policy_insert(vm, range_policy);
if (error) {
kprintf("%s: ERROR: could not insert range: %d\n",__FUNCTION__, error);
return error;
}
mbind_update_only:
@@ -8395,8 +8430,6 @@ SYSCALL_DECLARE(set_mempolicy)
struct process_vm *vm = cpu_local_var(current)->vm;
int error = 0;
int bit, valid_mask;
struct vm_range_numa_policy *range_policy_iter;
struct vm_range_numa_policy *range_policy_next = NULL;
DECLARE_BITMAP(numa_mask, PROCESS_NUMA_MASK_BITS);
memset(numa_mask, 0, sizeof(numa_mask));
@@ -8417,6 +8450,8 @@ SYSCALL_DECLARE(set_mempolicy)
}
}
mode &= ~MPOL_MODE_FLAGS;
switch (mode) {
case MPOL_DEFAULT:
if (nodemask && nodemask_bits) {
@@ -8440,6 +8475,13 @@ SYSCALL_DECLARE(set_mempolicy)
set_bit(bit, vm->numa_mask);
}
#if 0
/* In man, "MPOL_DEFAULT mode deletes a process memory policy
other than the default and interprets that the memory policy"
falls back to the system default policy ", but not to delete
the NUMA memory policy.
There was no processing of Linux's same name command. */
/* Delete all range settings */
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
list_for_each_entry_safe(range_policy_iter, range_policy_next,
@@ -8448,6 +8490,7 @@ SYSCALL_DECLARE(set_mempolicy)
kfree(range_policy_iter);
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
#endif
vm->numa_mem_policy = mode;
error = 0;
@@ -8595,7 +8638,6 @@ SYSCALL_DECLARE(get_mempolicy)
/* Address range specific? */
if (flags & MPOL_F_ADDR) {
struct vm_range_numa_policy *range_policy_iter;
struct vm_range *range;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
@@ -8607,16 +8649,8 @@ SYSCALL_DECLARE(get_mempolicy)
goto out;
}
list_for_each_entry(range_policy_iter,
&vm->vm_range_numa_policy_list, list) {
if (range_policy_iter->start > addr ||
range_policy_iter->end <= addr) {
continue;
}
range_policy = vm_range_policy_search(vm, addr);
range_policy = range_policy_iter;
break;
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
}

2
kernel/zeroobj.c
View File

@@ -169,7 +169,7 @@ out:
}
static int zeroobj_get_page(struct memobj *memobj, off_t off, int p2align,
uintptr_t *physp, unsigned long *pflag)
uintptr_t *physp, unsigned long *pflag, uintptr_t virt_addr)
{
int error;
struct zeroobj *obj = to_zeroobj(memobj);