memory_range_lock: Enable interrupt when trylock fails
Also use read-write-lock Change-Id: I03150b7208325ec1fe422dcd5f931e4e41c8e40e Refs: #452
This commit is contained in:
committed by
Masamichi Takagi
parent
258156b57e
commit
0d3ef65092
@@ -2097,6 +2097,7 @@ int do_process_vm_read_writev(int pid,
|
||||
struct vm_range *range;
|
||||
struct mcs_rwlock_node_irqsave lock;
|
||||
struct mcs_rwlock_node update_lock;
|
||||
unsigned long irqflags;
|
||||
|
||||
/* Sanity checks */
|
||||
if (flags) {
|
||||
@@ -2108,7 +2109,7 @@ int do_process_vm_read_writev(int pid,
|
||||
}
|
||||
|
||||
/* Check if parameters are okay */
|
||||
ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_lock(lthread->vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(lthread->vm,
|
||||
(uintptr_t)local_iov,
|
||||
@@ -2130,7 +2131,7 @@ int do_process_vm_read_writev(int pid,
|
||||
|
||||
ret = 0;
|
||||
arg_out:
|
||||
ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_unlock(lthread->vm, &irqflags);
|
||||
|
||||
if (ret != 0) {
|
||||
goto out;
|
||||
@@ -2199,7 +2200,7 @@ arg_out:
|
||||
if (pli != li) {
|
||||
struct vm_range *range;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_lock(lthread->vm, &irqflags);
|
||||
|
||||
/* Is base valid? */
|
||||
range = lookup_process_memory_range(lthread->vm,
|
||||
@@ -2229,7 +2230,7 @@ arg_out:
|
||||
|
||||
ret = 0;
|
||||
pli_out:
|
||||
ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_unlock(lthread->vm, &irqflags);
|
||||
|
||||
if (ret != 0) {
|
||||
goto out;
|
||||
@@ -2242,7 +2243,7 @@ pli_out:
|
||||
if (pri != ri) {
|
||||
struct vm_range *range;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock);
|
||||
memory_range_read_lock(rvm, &irqflags);
|
||||
|
||||
/* Is base valid? */
|
||||
range = lookup_process_memory_range(rvm,
|
||||
@@ -2272,7 +2273,7 @@ pli_out:
|
||||
|
||||
ret = 0;
|
||||
pri_out:
|
||||
ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock);
|
||||
memory_range_read_unlock(rvm, &irqflags);
|
||||
|
||||
if (ret != 0) {
|
||||
goto out;
|
||||
|
||||
@@ -2234,6 +2234,7 @@ int do_process_vm_read_writev(int pid,
|
||||
struct vm_range *range;
|
||||
struct mcs_rwlock_node_irqsave lock;
|
||||
struct mcs_rwlock_node update_lock;
|
||||
unsigned long irqflags;
|
||||
|
||||
/* Sanity checks */
|
||||
if (flags) {
|
||||
@@ -2245,7 +2246,7 @@ int do_process_vm_read_writev(int pid,
|
||||
}
|
||||
|
||||
/* Check if parameters are okay */
|
||||
ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_lock(lthread->vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(lthread->vm,
|
||||
(uintptr_t)local_iov,
|
||||
@@ -2267,7 +2268,7 @@ int do_process_vm_read_writev(int pid,
|
||||
|
||||
ret = 0;
|
||||
arg_out:
|
||||
ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_unlock(lthread->vm, &irqflags);
|
||||
|
||||
if (ret != 0) {
|
||||
goto out;
|
||||
@@ -2336,7 +2337,7 @@ arg_out:
|
||||
if (pli != li) {
|
||||
struct vm_range *range;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_lock(lthread->vm, &irqflags);
|
||||
|
||||
/* Is base valid? */
|
||||
range = lookup_process_memory_range(lthread->vm,
|
||||
@@ -2366,7 +2367,7 @@ arg_out:
|
||||
|
||||
ret = 0;
|
||||
pli_out:
|
||||
ihk_mc_spinlock_unlock_noirq(<hread->vm->memory_range_lock);
|
||||
memory_range_read_unlock(lthread->vm, &irqflags);
|
||||
|
||||
if (ret != 0) {
|
||||
goto out;
|
||||
@@ -2379,7 +2380,7 @@ pli_out:
|
||||
if (pri != ri) {
|
||||
struct vm_range *range;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&rvm->memory_range_lock);
|
||||
memory_range_read_lock(rvm, &irqflags);
|
||||
|
||||
/* Is base valid? */
|
||||
range = lookup_process_memory_range(rvm,
|
||||
@@ -2409,7 +2410,7 @@ pli_out:
|
||||
|
||||
ret = 0;
|
||||
pri_out:
|
||||
ihk_mc_spinlock_unlock_noirq(&rvm->memory_range_lock);
|
||||
memory_range_read_unlock(rvm, &irqflags);
|
||||
|
||||
if (ret != 0) {
|
||||
goto out;
|
||||
|
||||
@@ -737,7 +737,7 @@ struct process_vm {
|
||||
void *vvar_addr;
|
||||
|
||||
ihk_spinlock_t page_table_lock;
|
||||
ihk_spinlock_t memory_range_lock;
|
||||
struct ihk_rwlock memory_range_lock;
|
||||
// to protect the followings:
|
||||
// 1. addition of process "memory range" (extend_process_region, add_process_memory_range)
|
||||
// 2. addition of process page table (allocate_pages, update_process_page_table)
|
||||
@@ -771,6 +771,46 @@ static inline int has_cap_sys_admin(struct thread *th)
|
||||
return !(th->proc->euid);
|
||||
}
|
||||
|
||||
static inline void memory_range_read_lock(struct process_vm *vm,
|
||||
unsigned long *flags)
|
||||
{
|
||||
for (;;) {
|
||||
*flags = cpu_disable_interrupt_save();
|
||||
if (ihk_mc_read_trylock(&vm->memory_range_lock)) {
|
||||
break;
|
||||
}
|
||||
cpu_restore_interrupt(*flags);
|
||||
cpu_pause();
|
||||
}
|
||||
}
|
||||
|
||||
static inline void memory_range_write_lock(struct process_vm *vm,
|
||||
unsigned long *flags)
|
||||
{
|
||||
for (;;) {
|
||||
*flags = cpu_disable_interrupt_save();
|
||||
if (ihk_mc_write_trylock(&vm->memory_range_lock)) {
|
||||
break;
|
||||
}
|
||||
cpu_restore_interrupt(*flags);
|
||||
cpu_pause();
|
||||
}
|
||||
}
|
||||
|
||||
static inline void memory_range_read_unlock(struct process_vm *vm,
|
||||
unsigned long *flags)
|
||||
{
|
||||
ihk_mc_read_unlock(&vm->memory_range_lock);
|
||||
cpu_restore_interrupt(*flags);
|
||||
}
|
||||
|
||||
static inline void memory_range_write_unlock(struct process_vm *vm,
|
||||
unsigned long *flags)
|
||||
{
|
||||
ihk_mc_write_unlock(&vm->memory_range_lock);
|
||||
cpu_restore_interrupt(*flags);
|
||||
}
|
||||
|
||||
void hold_address_space(struct address_space *);
|
||||
void release_address_space(struct address_space *);
|
||||
struct thread *create_thread(unsigned long user_pc,
|
||||
|
||||
@@ -239,7 +239,7 @@ static int
|
||||
init_process_vm(struct process *owner, struct address_space *asp, struct process_vm *vm)
|
||||
{
|
||||
int i;
|
||||
ihk_mc_spinlock_init(&vm->memory_range_lock);
|
||||
ihk_mc_rwlock_init(&vm->memory_range_lock);
|
||||
ihk_mc_spinlock_init(&vm->page_table_lock);
|
||||
|
||||
ihk_atomic_set(&vm->refcount, 1);
|
||||
@@ -740,8 +740,9 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm)
|
||||
struct vm_range *src_range;
|
||||
struct vm_range *range;
|
||||
struct copy_args args;
|
||||
unsigned long irqflags;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&orgvm->memory_range_lock);
|
||||
memory_range_write_lock(orgvm, &irqflags);
|
||||
|
||||
/* Iterate original process' vm_range list and take a copy one-by-one */
|
||||
src_range = NULL;
|
||||
@@ -801,7 +802,7 @@ static int copy_user_ranges(struct process_vm *vm, struct process_vm *orgvm)
|
||||
vm_range_insert(vm, range);
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&orgvm->memory_range_lock);
|
||||
memory_range_write_unlock(orgvm, &irqflags);
|
||||
|
||||
return 0;
|
||||
|
||||
@@ -813,7 +814,7 @@ err_rollback:
|
||||
/* TODO: implement rollback */
|
||||
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&orgvm->memory_range_lock);
|
||||
memory_range_write_unlock(orgvm, &irqflags);
|
||||
|
||||
return -1;
|
||||
}
|
||||
@@ -2102,6 +2103,7 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
|
||||
struct vm_range *range;
|
||||
struct thread *thread = cpu_local_var(current);
|
||||
int locked = 0;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("[%d]do_page_fault_process_vm(%p,%lx,%lx)\n",
|
||||
ihk_mc_get_processor_id(), vm, fault_addr0, reason);
|
||||
@@ -2112,10 +2114,12 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
|
||||
if (thread->vm->is_memory_range_lock_taken) {
|
||||
goto skip;
|
||||
}
|
||||
if (ihk_mc_spinlock_trylock_noirq(&vm->memory_range_lock)) {
|
||||
irqflags = cpu_disable_interrupt_save();
|
||||
if (ihk_mc_read_trylock(&vm->memory_range_lock)) {
|
||||
locked = 1;
|
||||
break;
|
||||
}
|
||||
cpu_restore_interrupt(irqflags);
|
||||
}
|
||||
} else {
|
||||
skip:;
|
||||
@@ -2231,7 +2235,8 @@ skip:;
|
||||
error = 0;
|
||||
out:
|
||||
if (locked) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
ihk_mc_read_unlock(&vm->memory_range_lock);
|
||||
cpu_restore_interrupt(irqflags);
|
||||
}
|
||||
dkprintf("[%d]do_page_fault_process_vm(%p,%lx,%lx): %d\n",
|
||||
ihk_mc_get_processor_id(), vm, fault_addr0,
|
||||
@@ -2532,9 +2537,10 @@ void flush_process_memory(struct process_vm *vm)
|
||||
struct vm_range *range;
|
||||
struct rb_node *node, *next = rb_first(&vm->vm_range_tree);
|
||||
int error;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("flush_process_memory(%p)\n", vm);
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
/* Let concurrent page faults know the VM will be gone */
|
||||
vm->exiting = 1;
|
||||
while ((node = next)) {
|
||||
@@ -2552,7 +2558,7 @@ void flush_process_memory(struct process_vm *vm)
|
||||
}
|
||||
}
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
dkprintf("flush_process_memory(%p):\n", vm);
|
||||
return;
|
||||
}
|
||||
@@ -2562,12 +2568,13 @@ void free_process_memory_ranges(struct process_vm *vm)
|
||||
int error;
|
||||
struct vm_range *range;
|
||||
struct rb_node *node, *next = rb_first(&vm->vm_range_tree);
|
||||
unsigned long irqflags;
|
||||
|
||||
if (vm == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
while ((node = next)) {
|
||||
range = rb_entry(node, struct vm_range, vm_rb_node);
|
||||
next = rb_next(node);
|
||||
@@ -2580,7 +2587,7 @@ void free_process_memory_ranges(struct process_vm *vm)
|
||||
/* through */
|
||||
}
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
}
|
||||
|
||||
static void free_thread_pages(struct thread *thread)
|
||||
@@ -2655,8 +2662,9 @@ free_all_process_memory_range(struct process_vm *vm)
|
||||
struct vm_range *range;
|
||||
struct rb_node *node, *next = rb_first(&vm->vm_range_tree);
|
||||
int error;
|
||||
unsigned long irqflags;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
while ((node = next)) {
|
||||
range = rb_entry(node, struct vm_range, vm_rb_node);
|
||||
next = rb_next(node);
|
||||
@@ -2669,7 +2677,7 @@ free_all_process_memory_range(struct process_vm *vm)
|
||||
/* through */
|
||||
}
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
}
|
||||
|
||||
void
|
||||
@@ -3084,7 +3092,7 @@ void sched_init(void)
|
||||
&idle_thread->proc->children_list);
|
||||
|
||||
ihk_mc_init_context(&idle_thread->ctx, NULL, idle);
|
||||
ihk_mc_spinlock_init(&idle_thread->vm->memory_range_lock);
|
||||
ihk_mc_rwlock_init(&idle_thread->vm->memory_range_lock);
|
||||
idle_thread->vm->vm_range_tree = RB_ROOT;
|
||||
idle_thread->vm->vm_range_numa_policy_tree = RB_ROOT;
|
||||
idle_thread->proc->pid = 0;
|
||||
|
||||
@@ -174,6 +174,7 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
int err = -EIO;
|
||||
struct mckernel_procfs_buffer *buf_top = NULL;
|
||||
struct mckernel_procfs_buffer *buf_cur = NULL;
|
||||
unsigned long irqflags;
|
||||
|
||||
dprintf("process_procfs_request: invoked.\n");
|
||||
|
||||
@@ -419,7 +420,7 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
if (strcmp(p, "maps") == 0) {
|
||||
struct vm_range *range;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(vm, 0, -1);
|
||||
while (range) {
|
||||
@@ -455,13 +456,13 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
|
||||
if (ans < 0 || ans > count ||
|
||||
buf_add(&buf_top, &buf_cur, buf, ans) < 0) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
goto err;
|
||||
}
|
||||
range = next_process_memory_range(vm, range);
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
|
||||
ans = 0;
|
||||
goto end;
|
||||
@@ -484,7 +485,7 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
start = (offset / sizeof(uint64_t)) << PAGE_SHIFT;
|
||||
end = start + ((count / sizeof(uint64_t)) << PAGE_SHIFT);
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
|
||||
while (start < end) {
|
||||
*_buf = ihk_mc_pt_virt_to_pagemap(proc->vm->address_space->page_table, start);
|
||||
@@ -494,7 +495,7 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
++_buf;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
|
||||
dprintf("/proc/pagemap: 0x%lx - 0x%lx, count: %d\n",
|
||||
start, end, count);
|
||||
@@ -526,14 +527,14 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
goto err;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&proc->vm->memory_range_lock);
|
||||
memory_range_read_lock(proc->vm, &irqflags);
|
||||
range = lookup_process_memory_range(vm, 0, -1);
|
||||
while (range) {
|
||||
if(range->flag & VR_LOCKED)
|
||||
lockedsize += range->end - range->start;
|
||||
range = next_process_memory_range(vm, range);
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&proc->vm->memory_range_lock);
|
||||
memory_range_read_unlock(proc->vm, &irqflags);
|
||||
|
||||
cpu_bitmask = &bitmasks[bitmasks_offset];
|
||||
bitmasks_offset += bitmap_scnprintf(cpu_bitmask,
|
||||
|
||||
105
kernel/syscall.c
105
kernel/syscall.c
@@ -1602,6 +1602,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
|
||||
struct mckfd *fdp = NULL;
|
||||
int pgshift;
|
||||
struct vm_range *range = NULL;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("do_mmap(%lx,%lx,%x,%x,%d,%lx)\n",
|
||||
addr0, len0, prot, flags, fd, off0);
|
||||
@@ -1633,6 +1634,8 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
|
||||
|
||||
flush_nfo_tlb();
|
||||
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
|
||||
if (flags & MAP_HUGETLB) {
|
||||
pgshift = (flags >> MAP_HUGE_SHIFT) & 0x3F;
|
||||
p2align = pgshift - PAGE_SHIFT;
|
||||
@@ -1655,8 +1658,6 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
|
||||
p2align = PAGE_P2ALIGN;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
|
||||
|
||||
if (flags & MAP_FIXED) {
|
||||
/* clear specified address range */
|
||||
error = do_munmap((void *)addr, len, 1/* holding memory_range_lock */);
|
||||
@@ -1945,7 +1946,7 @@ out:
|
||||
if (ro_vma_mapped) {
|
||||
(void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */);
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_unlock(thread->vm, &irqflags);
|
||||
|
||||
if (!error && populated_mapping && !((vrflags & VR_PROT_MASK) == VR_PROT_NONE)) {
|
||||
error = populate_process_memory(thread->vm,
|
||||
@@ -1998,6 +1999,7 @@ SYSCALL_DECLARE(munmap)
|
||||
struct vm_regions *region = &thread->vm->region;
|
||||
size_t len;
|
||||
int error;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("[%d]sys_munmap(%lx,%lx)\n",
|
||||
ihk_mc_get_processor_id(), addr, len0);
|
||||
@@ -2013,9 +2015,9 @@ SYSCALL_DECLARE(munmap)
|
||||
goto out;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
error = do_munmap((void *)addr, len, 1/* holding memory_range_lock */);
|
||||
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_unlock(thread->vm, &irqflags);
|
||||
|
||||
out:
|
||||
dkprintf("[%d]sys_munmap(%lx,%lx): %d\n",
|
||||
@@ -2040,6 +2042,7 @@ SYSCALL_DECLARE(mprotect)
|
||||
const unsigned long protflags = PROT_TO_VR_FLAG(prot);
|
||||
unsigned long denied;
|
||||
int ro_changed = 0;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("[%d]sys_mprotect(%lx,%lx,%x)\n",
|
||||
ihk_mc_get_processor_id(), start, len0, prot);
|
||||
@@ -2069,7 +2072,7 @@ SYSCALL_DECLARE(mprotect)
|
||||
|
||||
flush_nfo_tlb();
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
|
||||
first = lookup_process_memory_range(thread->vm, start, start+PAGE_SIZE);
|
||||
|
||||
@@ -2159,7 +2162,7 @@ out:
|
||||
/* through */
|
||||
}
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_unlock(thread->vm, &irqflags);
|
||||
dkprintf("[%d]sys_mprotect(%lx,%lx,%x): %d\n",
|
||||
ihk_mc_get_processor_id(), start, len0, prot, error);
|
||||
return error;
|
||||
@@ -2172,6 +2175,7 @@ SYSCALL_DECLARE(brk)
|
||||
unsigned long r;
|
||||
unsigned long vrflag;
|
||||
unsigned long old_brk_end_allocated = 0;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("SC(%d)[sys_brk] brk_start=%lx,end=%lx\n",
|
||||
ihk_mc_get_processor_id(), region->brk_start, region->brk_end);
|
||||
@@ -2206,11 +2210,11 @@ SYSCALL_DECLARE(brk)
|
||||
vrflag |= VR_PRIVATE;
|
||||
vrflag |= VRFLAG_PROT_TO_MAXPROT(vrflag);
|
||||
old_brk_end_allocated = region->brk_end_allocated;
|
||||
ihk_mc_spinlock_lock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
|
||||
memory_range_write_lock(cpu_local_var(current)->vm, &irqflags);
|
||||
region->brk_end_allocated =
|
||||
extend_process_region(cpu_local_var(current)->vm,
|
||||
region->brk_end_allocated, address, vrflag);
|
||||
ihk_mc_spinlock_unlock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
|
||||
memory_range_write_unlock(cpu_local_var(current)->vm, &irqflags);
|
||||
|
||||
if (old_brk_end_allocated == region->brk_end_allocated) {
|
||||
r = old_brk_end_allocated;
|
||||
@@ -2432,8 +2436,9 @@ static void munmap_all(void)
|
||||
void *addr;
|
||||
size_t size;
|
||||
int error;
|
||||
unsigned long irqflags;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
next = lookup_process_memory_range(vm, 0, -1);
|
||||
while ((range = next)) {
|
||||
next = next_process_memory_range(vm, range);
|
||||
@@ -2447,7 +2452,7 @@ static void munmap_all(void)
|
||||
/* through */
|
||||
}
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
|
||||
/* free vm_ranges which do_munmap() failed to remove. */
|
||||
free_process_memory_ranges(thread->vm);
|
||||
@@ -2478,19 +2483,20 @@ SYSCALL_DECLARE(execve)
|
||||
struct vm_range *range;
|
||||
struct process *proc = thread->proc;
|
||||
int i;
|
||||
unsigned long irqflags;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(vm, (unsigned long)filename,
|
||||
(unsigned long)filename+1);
|
||||
|
||||
if (range == NULL || !(range->flag & VR_PROT_READ)) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
kprintf("execve(): ERROR: filename is bad address\n");
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
|
||||
desc = ihk_mc_alloc_pages(4, IHK_MC_AP_NOWAIT);
|
||||
if (!desc) {
|
||||
@@ -4573,6 +4579,7 @@ SYSCALL_DECLARE(mincore)
|
||||
uint8_t value;
|
||||
int error;
|
||||
pte_t *ptep;
|
||||
unsigned long irqflags;
|
||||
|
||||
if (start & (PAGE_SIZE - 1)) {
|
||||
dkprintf("mincore(0x%lx,0x%lx,%p): EINVAL\n", start, len, vec);
|
||||
@@ -4589,10 +4596,10 @@ SYSCALL_DECLARE(mincore)
|
||||
range = NULL;
|
||||
up = vec;
|
||||
for (addr = start; addr < end; addr += PAGE_SIZE) {
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
range = lookup_process_memory_range(vm, addr, addr+1);
|
||||
if (!range) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
dkprintf("mincore(0x%lx,0x%lx,%p):lookup failed. ENOMEM\n",
|
||||
start, len, vec);
|
||||
return -ENOMEM;
|
||||
@@ -4614,7 +4621,7 @@ SYSCALL_DECLARE(mincore)
|
||||
value = 0;
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock);
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
|
||||
error = copy_to_user(up, &value, sizeof(value));
|
||||
if (error) {
|
||||
@@ -4737,6 +4744,7 @@ SYSCALL_DECLARE(madvise)
|
||||
int error;
|
||||
uintptr_t s;
|
||||
uintptr_t e;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("[%d]sys_madvise(%lx,%lx,%x)\n",
|
||||
ihk_mc_get_processor_id(), start, len0, advice);
|
||||
@@ -4796,7 +4804,7 @@ SYSCALL_DECLARE(madvise)
|
||||
goto out2;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
/* check contiguous map */
|
||||
first = NULL;
|
||||
range = NULL; /* for avoidance of warning */
|
||||
@@ -4921,7 +4929,7 @@ SYSCALL_DECLARE(madvise)
|
||||
|
||||
error = 0;
|
||||
out:
|
||||
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_unlock(thread->vm, &irqflags);
|
||||
|
||||
out2:
|
||||
dkprintf("[%d]sys_madvise(%lx,%lx,%x): %d\n",
|
||||
@@ -5184,6 +5192,7 @@ SYSCALL_DECLARE(shmat)
|
||||
int req;
|
||||
struct shmobj *obj;
|
||||
size_t pgsize;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("shmat(%#x,%p,%#x)\n", shmid, shmaddr, shmflg);
|
||||
|
||||
@@ -5233,11 +5242,11 @@ SYSCALL_DECLARE(shmat)
|
||||
return -EACCES;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
|
||||
if (addr) {
|
||||
if (lookup_process_memory_range(vm, addr, addr+len)) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
shmobj_list_unlock();
|
||||
memobj_unref(&obj->memobj);
|
||||
dkprintf("shmat(%#x,%p,%#x):lookup_process_memory_range succeeded. -ENOMEM\n", shmid, shmaddr, shmflg);
|
||||
@@ -5247,7 +5256,7 @@ SYSCALL_DECLARE(shmat)
|
||||
else {
|
||||
error = search_free_space(len, obj->pgshift, &addr);
|
||||
if (error) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
shmobj_list_unlock();
|
||||
memobj_unref(&obj->memobj);
|
||||
dkprintf("shmat(%#x,%p,%#x):search_free_space failed. %d\n", shmid, shmaddr, shmflg, error);
|
||||
@@ -5263,7 +5272,7 @@ SYSCALL_DECLARE(shmat)
|
||||
if (!(prot & PROT_WRITE)) {
|
||||
error = set_host_vma(addr, len, PROT_READ | PROT_EXEC, 1/* holding memory_range_lock */);
|
||||
if (error) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
shmobj_list_unlock();
|
||||
memobj_unref(&obj->memobj);
|
||||
dkprintf("shmat(%#x,%p,%#x):set_host_vma failed. %d\n", shmid, shmaddr, shmflg, error);
|
||||
@@ -5278,13 +5287,13 @@ SYSCALL_DECLARE(shmat)
|
||||
(void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */);
|
||||
}
|
||||
memobj_unref(&obj->memobj);
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
shmobj_list_unlock();
|
||||
dkprintf("shmat(%#x,%p,%#x):add_process_memory_range failed. %d\n", shmid, shmaddr, shmflg, error);
|
||||
return error;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
shmobj_list_unlock();
|
||||
|
||||
dkprintf("shmat(%#x,%p,%#x): 0x%lx. %d\n", shmid, shmaddr, shmflg, addr);
|
||||
@@ -5569,25 +5578,26 @@ SYSCALL_DECLARE(shmdt)
|
||||
struct process_vm *vm = thread->vm;
|
||||
struct vm_range *range;
|
||||
int error;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("shmdt(%p)\n", shmaddr);
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
range = lookup_process_memory_range(vm, (uintptr_t)shmaddr, (uintptr_t)shmaddr+1);
|
||||
if (!range || (range->start != (uintptr_t)shmaddr) || !range->memobj
|
||||
|| !(range->memobj->flags & MF_SHMDT_OK)) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
dkprintf("shmdt(%p): -EINVAL\n", shmaddr);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
error = do_munmap((void *)range->start, (range->end - range->start), 1/* holding memory_range_lock */);
|
||||
if (error) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
dkprintf("shmdt(%p): %d\n", shmaddr, error);
|
||||
return error;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
dkprintf("shmdt(%p): 0\n", shmaddr);
|
||||
return 0;
|
||||
} /* sys_shmdt() */
|
||||
@@ -7450,6 +7460,7 @@ SYSCALL_DECLARE(mlock)
|
||||
struct vm_range *range;
|
||||
int error;
|
||||
struct vm_range *changed;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("[%d]sys_mlock(%lx,%lx)\n",
|
||||
ihk_mc_get_processor_id(), start0, len0);
|
||||
@@ -7477,7 +7488,7 @@ SYSCALL_DECLARE(mlock)
|
||||
goto out2;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
|
||||
/* check contiguous map */
|
||||
first = NULL;
|
||||
@@ -7582,7 +7593,7 @@ SYSCALL_DECLARE(mlock)
|
||||
|
||||
error = 0;
|
||||
out:
|
||||
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_unlock(thread->vm, &irqflags);
|
||||
|
||||
if (!error) {
|
||||
error = populate_process_memory(thread->vm, (void *)start, len);
|
||||
@@ -7625,6 +7636,7 @@ SYSCALL_DECLARE(munlock)
|
||||
struct vm_range *range;
|
||||
int error;
|
||||
struct vm_range *changed;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("[%d]sys_munlock(%lx,%lx)\n",
|
||||
ihk_mc_get_processor_id(), start0, len0);
|
||||
@@ -7652,7 +7664,7 @@ SYSCALL_DECLARE(munlock)
|
||||
goto out2;
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
|
||||
/* check contiguous map */
|
||||
first = NULL;
|
||||
@@ -7757,7 +7769,7 @@ SYSCALL_DECLARE(munlock)
|
||||
|
||||
error = 0;
|
||||
out:
|
||||
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_unlock(thread->vm, &irqflags);
|
||||
out2:
|
||||
dkprintf("[%d]sys_munlock(%lx,%lx): %d\n",
|
||||
ihk_mc_get_processor_id(), start0, len0, error);
|
||||
@@ -7810,10 +7822,11 @@ SYSCALL_DECLARE(remap_file_pages)
|
||||
struct vm_range *range;
|
||||
int er;
|
||||
int need_populate = 0;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("sys_remap_file_pages(%#lx,%#lx,%#x,%#lx,%#x)\n",
|
||||
start0, size, prot, pgoff, flags);
|
||||
ihk_mc_spinlock_lock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
#define PGOFF_LIMIT ((off_t)1 << ((8*sizeof(off_t) - 1) - PAGE_SHIFT))
|
||||
if ((size <= 0) || (size & (PAGE_SIZE - 1)) || (prot != 0)
|
||||
|| (PGOFF_LIMIT <= pgoff)
|
||||
@@ -7857,7 +7870,7 @@ SYSCALL_DECLARE(remap_file_pages)
|
||||
}
|
||||
error = 0;
|
||||
out:
|
||||
ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
|
||||
memory_range_write_unlock(thread->vm, &irqflags);
|
||||
|
||||
if (need_populate
|
||||
&& (er = populate_process_memory(
|
||||
@@ -7895,10 +7908,11 @@ SYSCALL_DECLARE(mremap)
|
||||
uintptr_t ret;
|
||||
uintptr_t lckstart = -1;
|
||||
uintptr_t lckend = -1;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx)\n",
|
||||
oldaddr, oldsize0, newsize0, flags, newaddr);
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
|
||||
/* check arguments */
|
||||
if ((oldaddr & ~PAGE_MASK)
|
||||
@@ -8094,7 +8108,7 @@ SYSCALL_DECLARE(mremap)
|
||||
|
||||
error = 0;
|
||||
out:
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
if (!error && (lckstart < lckend)) {
|
||||
error = populate_process_memory(thread->vm, (void *)lckstart, (lckend - lckstart));
|
||||
if (error) {
|
||||
@@ -8127,9 +8141,10 @@ SYSCALL_DECLARE(msync)
|
||||
struct vm_range *range;
|
||||
uintptr_t s;
|
||||
uintptr_t e;
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("sys_msync(%#lx,%#lx,%#x)\n", start0, len0, flags);
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
|
||||
if ((start0 & ~PAGE_MASK)
|
||||
|| (flags & ~(MS_ASYNC|MS_INVALIDATE|MS_SYNC))
|
||||
@@ -8223,7 +8238,7 @@ SYSCALL_DECLARE(msync)
|
||||
|
||||
error = 0;
|
||||
out:
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
dkprintf("sys_msync(%#lx,%#lx,%#x):%d\n", start0, len0, flags, error);
|
||||
return error;
|
||||
} /* sys_msync() */
|
||||
@@ -8272,6 +8287,7 @@ SYSCALL_DECLARE(mbind)
|
||||
struct vm_range *range;
|
||||
struct vm_range_numa_policy *range_policy, *range_policy_iter = NULL;
|
||||
DECLARE_BITMAP(numa_mask, PROCESS_NUMA_MASK_BITS);
|
||||
unsigned long irqflags;
|
||||
|
||||
dkprintf("%s: addr: 0x%lx, len: %lu, mode: 0x%x, "
|
||||
"nodemask: 0x%lx, flags: %lx\n",
|
||||
@@ -8404,7 +8420,7 @@ SYSCALL_DECLARE(mbind)
|
||||
}
|
||||
|
||||
/* Validate address range */
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(vm, addr, addr + len);
|
||||
if (!range) {
|
||||
@@ -8571,7 +8587,7 @@ mbind_update_only:
|
||||
error = 0;
|
||||
|
||||
unlock_out:
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
out:
|
||||
return error;
|
||||
} /* sys_mbind() */
|
||||
@@ -8800,19 +8816,20 @@ SYSCALL_DECLARE(get_mempolicy)
|
||||
/* Address range specific? */
|
||||
if (flags & MPOL_F_ADDR) {
|
||||
struct vm_range *range;
|
||||
unsigned long irqflags;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
range = lookup_process_memory_range(vm, addr, addr + 1);
|
||||
if (!range) {
|
||||
dkprintf("%s: ERROR: range is invalid\n", __FUNCTION__);
|
||||
error = -EFAULT;
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
goto out;
|
||||
}
|
||||
|
||||
range_policy = vm_range_policy_search(vm, addr);
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
}
|
||||
|
||||
/* Return policy */
|
||||
|
||||
@@ -1011,6 +1011,7 @@ static int xpmem_attach(
|
||||
struct mcs_rwlock_node_irqsave at_lock;
|
||||
struct vm_range *vmr;
|
||||
struct process_vm *vm = cpu_local_var(current)->vm;
|
||||
unsigned long irqflags;
|
||||
|
||||
XPMEM_DEBUG("call: apid=0x%lx, offset=0x%lx, size=0x%lx, vaddr=0x%lx, "
|
||||
"fd=%d, att_flags=%d",
|
||||
@@ -1103,12 +1104,12 @@ static int xpmem_attach(
|
||||
if (flags & MAP_FIXED) {
|
||||
struct vm_range *existing_vmr;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
|
||||
existing_vmr = lookup_process_memory_range(vm, vaddr,
|
||||
vaddr + size);
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_read_unlock(vm, &irqflags);
|
||||
|
||||
for (; existing_vmr && existing_vmr->start < vaddr + size;
|
||||
existing_vmr = next_process_memory_range(vm,
|
||||
@@ -1134,7 +1135,7 @@ static int xpmem_attach(
|
||||
XPMEM_DEBUG("at_vaddr=0x%lx", at_vaddr);
|
||||
att->at_vaddr = at_vaddr;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
|
||||
vmr = lookup_process_memory_range(vm, at_vaddr, at_vaddr + 1);
|
||||
|
||||
@@ -1145,7 +1146,7 @@ static int xpmem_attach(
|
||||
ekprintf("%s: vmr->memobj equals to NULL\n", __FUNCTION__);
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
|
||||
if (!vmr) {
|
||||
ret = -ENOENT;
|
||||
@@ -1190,21 +1191,22 @@ static int xpmem_detach(
|
||||
struct mcs_rwlock_node_irqsave at_lock;
|
||||
struct vm_range *range;
|
||||
struct process_vm *vm = cpu_local_var(current)->vm;
|
||||
unsigned long irqflags;
|
||||
|
||||
XPMEM_DEBUG("call: at_vaddr=0x%lx", at_vaddr);
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(vm, at_vaddr, at_vaddr + 1);
|
||||
|
||||
if (!range || range->start > at_vaddr) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
return 0;
|
||||
}
|
||||
|
||||
att = (struct xpmem_attachment *)range->private_data;
|
||||
if (att == NULL) {
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
@@ -1214,7 +1216,7 @@ static int xpmem_detach(
|
||||
|
||||
if (att->flags & XPMEM_FLAG_DESTROYING) {
|
||||
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
xpmem_att_deref(att);
|
||||
return 0;
|
||||
}
|
||||
@@ -1227,7 +1229,7 @@ static int xpmem_detach(
|
||||
att->flags &= ~XPMEM_FLAG_DESTROYING;
|
||||
xpmem_ap_deref(ap);
|
||||
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
xpmem_att_deref(att);
|
||||
return -EACCES;
|
||||
}
|
||||
@@ -1247,7 +1249,7 @@ static int xpmem_detach(
|
||||
ekprintf("%s: ERROR: xpmem_vm_munmap() failed %d\n",
|
||||
__FUNCTION__, ret);
|
||||
}
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
DBUG_ON(ret != 0);
|
||||
|
||||
att->flags &= ~XPMEM_FLAG_VALIDPTEs;
|
||||
@@ -1407,6 +1409,7 @@ static void xpmem_detach_att(
|
||||
struct vm_range *range;
|
||||
struct process_vm *vm;
|
||||
struct mcs_rwlock_node_irqsave at_lock;
|
||||
unsigned long irqflags;
|
||||
|
||||
XPMEM_DEBUG("call: apid=0x%lx, att=0x%p", ap->apid, att);
|
||||
|
||||
@@ -1415,13 +1418,13 @@ static void xpmem_detach_att(
|
||||
|
||||
vm = cpu_local_var(current)->vm ? cpu_local_var(current)->vm : att->vm;
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_lock(vm, &irqflags);
|
||||
|
||||
mcs_rwlock_writer_lock(&att->at_lock, &at_lock);
|
||||
|
||||
if (att->flags & XPMEM_FLAG_DESTROYING) {
|
||||
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
XPMEM_DEBUG("return: XPMEM_FLAG_DESTROYING");
|
||||
return;
|
||||
}
|
||||
@@ -1435,7 +1438,7 @@ static void xpmem_detach_att(
|
||||
list_del_init(&att->att_list);
|
||||
ihk_mc_spinlock_unlock_noirq(&ap->lock);
|
||||
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
xpmem_att_destroyable(att);
|
||||
XPMEM_DEBUG("return: range=%p");
|
||||
return;
|
||||
@@ -1471,7 +1474,7 @@ static void xpmem_detach_att(
|
||||
__FUNCTION__, ret);
|
||||
}
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
|
||||
memory_range_write_unlock(vm, &irqflags);
|
||||
|
||||
xpmem_att_destroyable(att);
|
||||
|
||||
@@ -1570,11 +1573,12 @@ static void xpmem_clear_PTEs_of_att(
|
||||
{
|
||||
int ret;
|
||||
struct mcs_rwlock_node_irqsave at_lock;
|
||||
unsigned long irqflags;
|
||||
|
||||
XPMEM_DEBUG("call: att=0x%p, start=0x%lx, end=0x%lx",
|
||||
att, start, end);
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&att->vm->memory_range_lock);
|
||||
memory_range_write_lock(att->vm, &irqflags);
|
||||
mcs_rwlock_writer_lock(&att->at_lock, &at_lock);
|
||||
|
||||
if (att->flags & XPMEM_FLAG_VALIDPTEs) {
|
||||
@@ -1634,7 +1638,7 @@ static void xpmem_clear_PTEs_of_att(
|
||||
}
|
||||
out:
|
||||
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
|
||||
ihk_mc_spinlock_unlock_noirq(&att->vm->memory_range_lock);
|
||||
memory_range_write_unlock(att->vm, &irqflags);
|
||||
|
||||
XPMEM_DEBUG("return: ");
|
||||
}
|
||||
@@ -1649,6 +1653,7 @@ int xpmem_remove_process_memory_range(
|
||||
struct xpmem_access_permit *ap;
|
||||
struct xpmem_attachment *att;
|
||||
struct mcs_rwlock_node_irqsave at_lock;
|
||||
unsigned long irqflags;
|
||||
|
||||
XPMEM_DEBUG("call: vmr=0x%p, att=0x%p", vmr, vmr->private_data);
|
||||
|
||||
@@ -1662,8 +1667,7 @@ int xpmem_remove_process_memory_range(
|
||||
|
||||
xpmem_att_ref(att);
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(
|
||||
&cpu_local_var(current)->vm->memory_range_lock);
|
||||
memory_range_write_lock(cpu_local_var(current)->vm, &irqflags);
|
||||
|
||||
mcs_rwlock_writer_lock(&att->at_lock, &at_lock);
|
||||
|
||||
@@ -1745,8 +1749,7 @@ int xpmem_remove_process_memory_range(
|
||||
out:
|
||||
mcs_rwlock_writer_unlock(&att->at_lock, &at_lock);
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(
|
||||
&cpu_local_var(current)->vm->memory_range_lock);
|
||||
memory_range_write_unlock(cpu_local_var(current)->vm, &irqflags);
|
||||
|
||||
xpmem_att_deref(att);
|
||||
|
||||
@@ -1898,17 +1901,18 @@ static int xpmem_remap_pte(
|
||||
size_t att_pgsize;
|
||||
int att_p2align;
|
||||
enum ihk_mc_pt_attribute att_attr;
|
||||
unsigned long irqflags;
|
||||
|
||||
XPMEM_DEBUG("call: vmr=0x%p, vaddr=0x%lx, reason=0x%lx, segid=0x%lx, "
|
||||
"seg_vaddr=0x%lx",
|
||||
vmr, vaddr, reason, seg->segid, seg_vaddr);
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&seg_tg->vm->memory_range_lock);
|
||||
memory_range_read_lock(seg_tg->vm, &irqflags);
|
||||
|
||||
seg_vmr = lookup_process_memory_range(seg_tg->vm, seg_vaddr,
|
||||
seg_vaddr + 1);
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&seg_tg->vm->memory_range_lock);
|
||||
memory_range_read_unlock(seg_tg->vm, &irqflags);
|
||||
|
||||
if (!seg_vmr) {
|
||||
ret = -EFAULT;
|
||||
@@ -2037,14 +2041,15 @@ static int xpmem_pin_page(
|
||||
{
|
||||
int ret;
|
||||
struct vm_range *range;
|
||||
unsigned long irqflags;
|
||||
|
||||
XPMEM_DEBUG("call: tgid=%d, vaddr=0x%lx", tg->tgid, vaddr);
|
||||
|
||||
ihk_mc_spinlock_lock_noirq(&src_vm->memory_range_lock);
|
||||
memory_range_read_lock(src_vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(src_vm, vaddr, vaddr + 1);
|
||||
|
||||
ihk_mc_spinlock_unlock_noirq(&src_vm->memory_range_lock);
|
||||
memory_range_read_unlock(src_vm, &irqflags);
|
||||
|
||||
if (!range || range->start > vaddr) {
|
||||
return -ENOENT;
|
||||
|
||||
107
test/issues/452/C452.patch
Normal file
107
test/issues/452/C452.patch
Normal file
@@ -0,0 +1,107 @@
|
||||
diff --git a/arch/arm64/kernel/include/syscall_list.h b/arch/arm64/kernel/include/syscall_list.h
|
||||
index f911674..52c164f 100644
|
||||
--- a/arch/arm64/kernel/include/syscall_list.h
|
||||
+++ b/arch/arm64/kernel/include/syscall_list.h
|
||||
@@ -134,6 +134,8 @@ SYSCALL_HANDLED(731, util_indicate_clone)
|
||||
SYSCALL_HANDLED(732, get_system)
|
||||
SYSCALL_HANDLED(733, util_register_desc)
|
||||
|
||||
+SYSCALL_HANDLED(740, setkdebug)
|
||||
+
|
||||
/* McKernel Specific */
|
||||
SYSCALL_HANDLED(801, swapout)
|
||||
SYSCALL_HANDLED(802, linux_mlock)
|
||||
diff --git a/arch/x86_64/kernel/include/syscall_list.h b/arch/x86_64/kernel/include/syscall_list.h
|
||||
index 79eda7f..1f81b0a 100644
|
||||
--- a/arch/x86_64/kernel/include/syscall_list.h
|
||||
+++ b/arch/x86_64/kernel/include/syscall_list.h
|
||||
@@ -174,6 +174,8 @@ SYSCALL_HANDLED(731, util_indicate_clone)
|
||||
SYSCALL_HANDLED(732, get_system)
|
||||
SYSCALL_HANDLED(733, util_register_desc)
|
||||
|
||||
+SYSCALL_HANDLED(740, setkdebug)
|
||||
+
|
||||
/* McKernel Specific */
|
||||
SYSCALL_HANDLED(801, swapout)
|
||||
SYSCALL_HANDLED(802, linux_mlock)
|
||||
diff --git a/kernel/include/process.h b/kernel/include/process.h
|
||||
index 0a9ff47..ecb464f 100644
|
||||
--- a/kernel/include/process.h
|
||||
+++ b/kernel/include/process.h
|
||||
@@ -573,6 +573,7 @@ struct process {
|
||||
#endif // PROFILE_ENABLE
|
||||
int nr_processes; /* For partitioned execution */
|
||||
int process_rank; /* Rank in partition */
|
||||
+ int debug_flags;
|
||||
};
|
||||
|
||||
/*
|
||||
diff --git a/kernel/procfs.c b/kernel/procfs.c
|
||||
index 5f9675c..a1b6d22 100644
|
||||
--- a/kernel/procfs.c
|
||||
+++ b/kernel/procfs.c
|
||||
@@ -420,6 +420,7 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
if (strcmp(p, "maps") == 0) {
|
||||
struct vm_range *range;
|
||||
|
||||
+ kprintf("read /proc/*/maps\n");
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
|
||||
range = lookup_process_memory_range(vm, 0, -1);
|
||||
@@ -485,6 +486,7 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
start = (offset / sizeof(uint64_t)) << PAGE_SHIFT;
|
||||
end = start + ((count / sizeof(uint64_t)) << PAGE_SHIFT);
|
||||
|
||||
+ kprintf("read /proc/*/pagemap\n");
|
||||
memory_range_read_lock(vm, &irqflags);
|
||||
|
||||
while (start < end) {
|
||||
@@ -527,6 +529,7 @@ int process_procfs_request(struct ikc_scd_packet *rpacket)
|
||||
goto err;
|
||||
}
|
||||
|
||||
+ kprintf("read /proc/*/status\n");
|
||||
memory_range_read_lock(proc->vm, &irqflags);
|
||||
range = lookup_process_memory_range(vm, 0, -1);
|
||||
while (range) {
|
||||
diff --git a/kernel/syscall.c b/kernel/syscall.c
|
||||
index 012ef13..9a34984 100644
|
||||
--- a/kernel/syscall.c
|
||||
+++ b/kernel/syscall.c
|
||||
@@ -1635,6 +1635,18 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
|
||||
flush_nfo_tlb();
|
||||
|
||||
memory_range_write_lock(thread->vm, &irqflags);
|
||||
+ if(thread->proc->debug_flags) {
|
||||
+ // sleep 5 sec
|
||||
+ unsigned long ts = rdtsc();
|
||||
+ unsigned long nanosecs = 5000000000L;
|
||||
+ unsigned long tscs = nanosecs * 1000 / ihk_mc_get_ns_per_tsc();
|
||||
+
|
||||
+ kprintf("kernel debug sleep 5sec...\n");
|
||||
+ while (rdtsc() - ts < tscs) {
|
||||
+ cpu_pause();
|
||||
+ }
|
||||
+ kprintf("kernel debug wake up\n");
|
||||
+ }
|
||||
|
||||
if (flags & MAP_HUGETLB) {
|
||||
pgshift = (flags >> MAP_HUGE_SHIFT) & 0x3F;
|
||||
@@ -9482,6 +9494,17 @@ SYSCALL_DECLARE(util_register_desc)
|
||||
return 0;
|
||||
}
|
||||
|
||||
+SYSCALL_DECLARE(setkdebug)
|
||||
+{
|
||||
+ int flags = ihk_mc_syscall_arg0(ctx);
|
||||
+ struct thread *mythread = cpu_local_var(current);
|
||||
+ struct process *proc = mythread->proc;
|
||||
+
|
||||
+ kprintf("set kernel debug flag pid=%d val=%d\n", proc->pid, flags);
|
||||
+ proc->debug_flags = flags;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
void
|
||||
reset_cputime()
|
||||
{
|
||||
22
test/issues/452/C452.sh
Normal file
22
test/issues/452/C452.sh
Normal file
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
USELTP=1
|
||||
USEOSTEST=0
|
||||
|
||||
. ../../common.sh
|
||||
|
||||
################################################################################
|
||||
$MCEXEC ./C452T01
|
||||
|
||||
for i in mmap01:02 mmap02:03 mmap03:04 mmap04:05 mmap12:06 brk01:07 fork01:08 \
|
||||
fork02:09 fork03:10; do
|
||||
tp=`echo $i|sed 's/:.*//'`
|
||||
id=`echo $i|sed 's/.*://'`
|
||||
$MCEXEC $LTPBIN/$tp 2>&1 | tee $tp.txt
|
||||
ok=`grep TPASS $tp.txt | wc -l`
|
||||
ng=`grep TFAIL $tp.txt | wc -l`
|
||||
if [ $ng = 0 ]; then
|
||||
echo "*** C452T$id: $tp PASS ($ok)"
|
||||
else
|
||||
echo "*** C452T$id: $tp FAIL (ok=$ok ng=%ng)"
|
||||
fi
|
||||
done
|
||||
52
test/issues/452/C452.txt
Normal file
52
test/issues/452/C452.txt
Normal file
@@ -0,0 +1,52 @@
|
||||
Script started on Wed Jul 3 13:39:36 2019
|
||||
bash-4.2$ make test
|
||||
sh ./C452.sh
|
||||
mcstop+release.sh ... done
|
||||
mcreboot.sh -c 1-7,17-23,9-15,25-31 -m 10G@0,10G@1 ... done
|
||||
*** C452T01 test start
|
||||
000000400000-000000401000 r-xs 0 0:0 0
|
||||
000000600000-000000601000 r--s 0 0:0 0
|
||||
000000601000-000000602000 rw-s 0 0:0 0
|
||||
155555600000-155555622000 r-xs 0 0:0 0
|
||||
155555821000-155555822000 r--s 0 0:0 0
|
||||
155555822000-155555824000 rw-s 0 0:0 0
|
||||
2aaaaa9f8000-2aaaaaa00000 rw-p 0 0:0 0
|
||||
2aaaaaa00000-2aaaaaa02000 r-xs 0 0:0 0 [vdso]
|
||||
2aaaaaa02000-2aaaaaa03000 rw-p 0 0:0 0
|
||||
2aaaaaa1a000-2aaaaaa1b000 rw-p 0 0:0 0
|
||||
2aaaaaa1b000-2aaaaabde000 r-xp 0 0:0 0 /usr/lib64/libc-2.17.so
|
||||
2aaaaabde000-2aaaaaddd000 ---p 0 0:0 0 /usr/lib64/libc-2.17.so
|
||||
2aaaaaddd000-2aaaaade1000 r--p 0 0:0 0 /usr/lib64/libc-2.17.so
|
||||
2aaaaade1000-2aaaaade3000 rw-p 0 0:0 0 /usr/lib64/libc-2.17.so
|
||||
2aaaaade3000-2aaaaade8000 rw-p 0 0:0 0
|
||||
2aaaaade8000-2aaaaadea000 rw-p 0 0:0 0
|
||||
2aaaaadea000-2aaaaadeb000 rw-p 0 0:0 0
|
||||
547fff800000-548000000000 rw-p 0 0:0 0 [stack]
|
||||
*** C452T01 PASS
|
||||
mmap01 1 TPASS : Functionality of mmap() successful
|
||||
*** C452T02: mmap01 PASS (1)
|
||||
mmap02 1 TPASS : Functionality of mmap() successful
|
||||
*** C452T03: mmap02 PASS (1)
|
||||
mmap03 1 TPASS : mmap() functionality is correct
|
||||
*** C452T04: mmap03 PASS (1)
|
||||
mmap04 1 TPASS : Functionality of mmap() successful
|
||||
*** C452T05: mmap04 PASS (1)
|
||||
mmap12 1 TPASS : Functionality of mmap() successful
|
||||
*** C452T06: mmap12 PASS (1)
|
||||
brk01 1 TPASS : brk(8626060) returned 0, new size verified by sbrk
|
||||
*** C452T07: brk01 PASS (1)
|
||||
fork01 1 TPASS : fork() returned 19074
|
||||
fork01 2 TPASS : child pid and fork() return agree: 19074
|
||||
*** C452T08: fork01 PASS (2)
|
||||
fork02 0 TINFO : Inside parent
|
||||
fork02 0 TINFO : Inside child
|
||||
fork02 0 TINFO : exit status of wait 0
|
||||
fork02 1 TPASS : test 1 PASSED
|
||||
*** C452T09: fork02 PASS (1)
|
||||
fork03 0 TINFO : process id in parent of child from fork : 19223
|
||||
fork03 1 TPASS : test 1 PASSED
|
||||
*** C452T10: fork03 PASS (1)
|
||||
bash-4.2$ exit
|
||||
exit
|
||||
|
||||
Script done on Wed Jul 3 13:39:53 2019
|
||||
40
test/issues/452/C452T01.c
Normal file
40
test/issues/452/C452T01.c
Normal file
@@ -0,0 +1,40 @@
|
||||
#define _GNU_SOURCE
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/wait.h>
|
||||
#include <signal.h>
|
||||
#include <errno.h>
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
pid_t pid;
|
||||
int st;
|
||||
|
||||
fprintf(stderr, "*** C452T01 test start\n");
|
||||
fflush(stderr);
|
||||
pid = fork();
|
||||
if (pid == 0) {
|
||||
char file[32];
|
||||
|
||||
sleep(1);
|
||||
sprintf(file, "/proc/%d/maps", getppid());
|
||||
execlp("cat", "cat", file, NULL);
|
||||
exit(1);
|
||||
}
|
||||
fflush(stdout);
|
||||
if (syscall(740, 1) == -1) {
|
||||
fprintf(stderr, "*** C452T01 FAIL no patched kernel\n");
|
||||
exit(1);
|
||||
}
|
||||
mmap(NULL, 4096, PROT_READ|PROT_WRITE,
|
||||
MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
|
||||
syscall(740, 0);
|
||||
while (waitpid(pid, &st, 0) == -1 && errno == EINTR)
|
||||
;
|
||||
fprintf(stderr, "*** C452T01 PASS\n");
|
||||
exit(0);
|
||||
}
|
||||
13
test/issues/452/Makefile
Normal file
13
test/issues/452/Makefile
Normal file
@@ -0,0 +1,13 @@
|
||||
CC = gcc
|
||||
TARGET = C452T01
|
||||
|
||||
all:: $(TARGET)
|
||||
|
||||
C452T01: C452T01.c
|
||||
$(CC) -g -Wall -o $@ $^
|
||||
|
||||
test:: all
|
||||
sh ./C452.sh
|
||||
|
||||
clean::
|
||||
rm -f $(TARGET) *.o
|
||||
36
test/issues/452/README
Normal file
36
test/issues/452/README
Normal file
@@ -0,0 +1,36 @@
|
||||
【Issue#452 動作確認】
|
||||
□ テスト内容
|
||||
1. Issue 指摘事項の再現確認
|
||||
以下のパッチ (C452.patch) を McKernel に適用し、意図的に memory_range_lock の
|
||||
競合を起こすことでテストを容易にする。
|
||||
- mmap処理のmemory_range_lock取得中にsleep可能とする
|
||||
- mmap処理のmemory_range_lock取得中にsleepするかどうかを制御する
|
||||
システムコールを追加する
|
||||
このパッチ適用カーネルを使ってテストする。
|
||||
|
||||
C452T01 memory_range_lock 取得中に /proc/*/maps を参照し、PASS すること
|
||||
|
||||
2. LTP を用いて既存処理に影響しないことを確認
|
||||
メモリ関連処理を変更したため、関連するシステムコールのテストを選定した。
|
||||
C452T02 mmap01: mmap の基本機能の確認
|
||||
C452T03 mmap02: mmap の基本機能の確認
|
||||
C452T04 mmap03: mmap の基本機能の確認
|
||||
C452T05 mmap04: mmap の基本機能の確認
|
||||
C452T06 mmap12: mmap の基本機能の確認
|
||||
C452T07 brk01: brk の基本機能の確認
|
||||
C452T08 fork01: fork の基本機能の確認 (fork時メモリがコピーされる)
|
||||
C452T09 fork02: fork の基本機能の確認
|
||||
C452T10 fork03: fork の基本機能の確認
|
||||
|
||||
□ 実行手順
|
||||
$ make test
|
||||
|
||||
McKernelのインストール先や LTP の配置場所は、$HOME/.mck_test_config を
|
||||
参照する。.mck_test_config は、McKernel をビルドした際に生成される
|
||||
mck_test_config.sample ファイルを $HOME にコピーし、適宜編集すること。
|
||||
|
||||
尚、テスト実行には C452.patch を適用した McKernel を使用すること。
|
||||
|
||||
□ 実行結果
|
||||
C452.txt 参照。
|
||||
全ての項目が PASS していることを確認。
|
||||
Reference in New Issue
Block a user