gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite page_fault_handler()

Browse Source
This commit is contained in:
NAKAMURA Gou
2013-08-08 12:43:00 +09:00
parent b0de24f13e
commit 480f6d4c2f
9 changed files with 738 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
arch/x86/kernel/cpu.c
View File

@@ -494,7 +494,7 @@ int ihk_mc_unregister_interrupt_handler(int vector,
extern unsigned long __page_fault_handler_address;
void ihk_mc_set_page_fault_handler(void (*h)(unsigned long, void *))
void ihk_mc_set_page_fault_handler(void (*h)(void *, uint64_t, void *))
{
__page_fault_handler_address = (unsigned long)h;
}

2
arch/x86/kernel/include/arch-memory.h
View File

@@ -57,6 +57,8 @@
#define PFL3_PRESENT ((pte_t)0x01)
#define PFL3_WRITABLE ((pte_t)0x02)
#define PFL3_USER ((pte_t)0x04)
#define PFL3_PWT ((pte_t)0x08)
#define PFL3_PCD ((pte_t)0x10)
#define PFL3_ACCESSED ((pte_t)0x20)
#define PFL3_DIRTY ((pte_t)0x40)
#define PFL3_SIZE ((pte_t)0x80) /* Used in 1G page */

14
arch/x86/kernel/include/ihk/types.h
View File

@@ -5,16 +5,16 @@ typedef unsigned char uint8_t;
typedef unsigned short uint16_t;
typedef unsigned int uint32_t;
typedef unsigned long long uint64_t;
typedef char int8_t;
typedef short int16_t;
typedef int int32_t;
typedef long long int64_t;
typedef signed char int8_t;
typedef signed short int16_t;
typedef signed int int32_t;
typedef signed long long int64_t;
typedef long long ptrdiff_t;
typedef int64_t ptrdiff_t;
typedef int64_t intptr_t;
typedef uint64_t uintptr_t;
typedef unsigned long long size_t;
typedef long long ssize_t;
typedef uint64_t size_t;
typedef int64_t ssize_t;
typedef int64_t off_t;
#define NULL ((void *)0)

4
arch/x86/kernel/interrupt.S
View File

@@ -66,8 +66,8 @@ page_fault:
cld
PUSH_ALL_REGS
movq %cr2, %rdi
movq %rsp, %rsi
movq %rbp, %rdx
movq 80(%rsp),%rsi
movq %rsp,%rdx
movq __page_fault_handler_address(%rip), %rax
andq %rax, %rax
jz 1f

475
arch/x86/kernel/memory.c
View File

@@ -8,6 +8,7 @@
#include <list.h>
#include <process.h>
#define dkprintf(...)
#define ekprintf(...) kprintf(__VA_ARGS__)
static char *last_page;
@@ -106,6 +107,23 @@ struct page_table {
static struct page_table *init_pt;
#ifdef USE_LARGE_PAGES
static int use_1gb_page = 0;
#endif
#ifdef USE_LARGE_PAGES
static void check_available_page_size(void)
{
uint32_t edx;
asm ("cpuid" : "=d" (edx) : "a" (0x80000001) : "%rbx", "%rcx");
use_1gb_page = (edx & (1 << 26))? 1: 0;
kprintf("use_1gb_page: %d\n", use_1gb_page);
return;
}
#endif
static unsigned long setup_l2(struct page_table *pt,
unsigned long page_head, unsigned long start,
unsigned long end)
@@ -194,11 +212,16 @@ static unsigned long attr_to_l4attr(enum ihk_mc_pt_attribute attr)
{
return (attr & ATTR_MASK) | PFL4_PRESENT;
}
#endif
static unsigned long attr_to_l3attr(enum ihk_mc_pt_attribute attr)
{
return (attr & ATTR_MASK) | PFL3_PRESENT;
unsigned long r = (attr & (ATTR_MASK | PTATTR_LARGEPAGE));
if ((attr & PTATTR_UNCACHABLE) && (attr & PTATTR_LARGEPAGE)) {
return r | PFL3_PCD | PFL3_PWT;
}
return r;
}
#endif
static unsigned long attr_to_l2attr(enum ihk_mc_pt_attribute attr)
{
unsigned long r = (attr & (ATTR_MASK | PTATTR_LARGEPAGE));
@@ -792,7 +815,8 @@ struct clear_range_args {
int free_physical;
};
static int clear_range_l1(void *args0, pte_t *ptep, uint64_t base, uint64_t start, uint64_t end)
static int clear_range_l1(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct clear_range_args *args = args0;
uint64_t phys;
@@ -811,7 +835,8 @@ static int clear_range_l1(void *args0, pte_t *ptep, uint64_t base, uint64_t star
return 0;
}
static int clear_range_l2(void *args0, pte_t *ptep, uint64_t base, uint64_t start, uint64_t end)
static int clear_range_l2(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct clear_range_args *args = args0;
uint64_t phys;
@@ -862,7 +887,8 @@ static int clear_range_l2(void *args0, pte_t *ptep, uint64_t base, uint64_t star
return 0;
}
static int clear_range_l3(void *args0, pte_t *ptep, uint64_t base, uint64_t start, uint64_t end)
static int clear_range_l3(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct page_table *pt;
@@ -874,7 +900,8 @@ static int clear_range_l3(void *args0, pte_t *ptep, uint64_t base, uint64_t star
return walk_pte_l2(pt, base, start, end, &clear_range_l2, args0);
}
static int clear_range_l4(void *args0, pte_t *ptep, uint64_t base, uint64_t start, uint64_t end)
static int clear_range_l4(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct page_table *pt;
@@ -886,16 +913,16 @@ static int clear_range_l4(void *args0, pte_t *ptep, uint64_t base, uint64_t star
return walk_pte_l3(pt, base, start, end, &clear_range_l3, args0);
}
static int clear_range(page_table_t pt, void *start0, void *end0, int free_physical)
static int clear_range(struct page_table *pt, uintptr_t start, uintptr_t end,
int free_physical)
{
const uint64_t start = (uint64_t)start0;
const uint64_t end = (uint64_t)end0;
int error;
struct clear_range_args args;
if ((USER_END <= start) || (USER_END < end) || (end <= start)) {
ekprintf("clear_range(%p,%p,%p,%x):invalid start and/or end.\n",
pt, start0, end0, free_physical);
ekprintf("clear_range(%p,%p,%p,%x):"
"invalid start and/or end.\n",
pt, start, end, free_physical);
return -EINVAL;
}
@@ -904,16 +931,18 @@ static int clear_range(page_table_t pt, void *start0, void *end0, int free_physi
return error;
}
int ihk_mc_pt_clear_range(page_table_t pt, void *start0, void *end0)
int ihk_mc_pt_clear_range(page_table_t pt, void *start, void *end)
{
#define KEEP_PHYSICAL 0
return clear_range(pt, start0, end0, KEEP_PHYSICAL);
return clear_range(pt, (uintptr_t)start, (uintptr_t)end,
KEEP_PHYSICAL);
}
int ihk_mc_pt_free_range(page_table_t pt, void *start0, void *end0)
int ihk_mc_pt_free_range(page_table_t pt, void *start, void *end)
{
#define FREE_PHYSICAL 1
return clear_range(pt, start0, end0, FREE_PHYSICAL);
return clear_range(pt, (uintptr_t)start, (uintptr_t)end,
FREE_PHYSICAL);
}
struct change_attr_args {
@@ -1133,6 +1162,419 @@ int ihk_mc_pt_alloc_range(page_table_t pt, void *start, void *end,
&alloc_range_l4, &attr);
}
static pte_t *lookup_pte(struct page_table *pt, uintptr_t virt,
uintptr_t *basep, size_t *sizep, int *p2alignp)
{
int l4idx, l3idx, l2idx, l1idx;
pte_t *ptep;
uintptr_t base;
size_t size;
int p2align;
GET_VIRT_INDICES(virt, l4idx, l3idx, l2idx, l1idx);
#ifdef USE_LARGE_PAGES
if (use_1gb_page) {
ptep = NULL;
base = GET_INDICES_VIRT(l4idx, 0, 0, 0);
size = PTL3_SIZE;
p2align = PTL3_SHIFT - PTL1_SHIFT;
}
else {
ptep = NULL;
base = GET_INDICES_VIRT(l4idx, l3idx, 0, 0);
size = PTL2_SIZE;
p2align = PTL2_SHIFT - PTL1_SHIFT;
}
#else
ptep = NULL;
base = GET_INDICES_VIRT(l4idx, l3idx, l2idx, l1idx);
size = PTL1_SIZE;
p2align = PTL1_SHIFT - PTL1_SHIFT;
#endif
if (pt->entry[l4idx] == PTE_NULL) {
goto out;
}
pt = phys_to_virt(pt->entry[l4idx] & PT_PHYSMASK);
if ((pt->entry[l3idx] == PTE_NULL)
|| (pt->entry[l3idx] & PFL3_SIZE)) {
#ifdef USE_LARGE_PAGES
if (use_1gb_page) {
ptep = &pt->entry[l3idx];
base = GET_INDICES_VIRT(l4idx, l3idx, 0, 0);
size = PTL3_SIZE;
p2align = PTL3_SHIFT - PTL1_SHIFT;
}
#endif
goto out;
}
pt = phys_to_virt(pt->entry[l3idx] & PT_PHYSMASK);
if ((pt->entry[l2idx] == PTE_NULL)
|| (pt->entry[l2idx] & PFL2_SIZE)) {
#ifdef USE_LARGE_PAGES
ptep = &pt->entry[l2idx];
base = GET_INDICES_VIRT(l4idx, l3idx, l2idx, 0);
size = PTL2_SIZE;
p2align = PTL2_SHIFT - PTL1_SHIFT;
#endif
goto out;
}
pt = phys_to_virt(pt->entry[l2idx] & PT_PHYSMASK);
ptep = &pt->entry[l1idx];
base = GET_INDICES_VIRT(l4idx, l3idx, l2idx, l1idx);
size = PTL1_SIZE;
p2align = PTL1_SHIFT - PTL1_SHIFT;
out:
if (basep) *basep = base;
if (sizep) *sizep = size;
if (p2alignp) *p2alignp = p2align;
return ptep;
}
pte_t *ihk_mc_pt_lookup_pte(page_table_t pt, void *virt, void **basep,
size_t *sizep, int *p2alignp)
{
pte_t *ptep;
uintptr_t base;
size_t size;
int p2align;
dkprintf("ihk_mc_pt_lookup_pte(%p,%p)\n", pt, virt);
ptep = lookup_pte(pt, (uintptr_t)virt, &base, &size, &p2align);
if (basep) *basep = (void *)base;
if (sizep) *sizep = size;
if (p2alignp) *p2alignp = p2align;
dkprintf("ihk_mc_pt_lookup_pte(%p,%p): %p %lx %lx %d\n",
pt, virt, ptep, base, size, p2align);
return ptep;
}
struct set_range_args {
page_table_t pt;
uintptr_t phys;
enum ihk_mc_pt_attribute attr;
int padding;
uintptr_t diff;
};
int set_range_l1(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct set_range_args *args = args0;
int error;
uintptr_t phys;
dkprintf("set_range_l1(%lx,%lx,%lx)\n", base, start, end);
if (*ptep != PTE_NULL) {
error = -EBUSY;
ekprintf("set_range_l1(%lx,%lx,%lx):page exists. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, start, base, KEEP_PHYSICAL);
goto out;
}
phys = args->phys + (base - start);
*ptep = phys | attr_to_l1attr(args->attr);
error = 0;
out:
dkprintf("set_range_l1(%lx,%lx,%lx): %d %lx\n",
base, start, end, error, *ptep);
return error;
}
int set_range_l2(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct set_range_args *args = args0;
int error;
struct page_table *pt;
#ifdef USE_LARGE_PAGES
uintptr_t phys;
#endif
dkprintf("set_range_l2(%lx,%lx,%lx)\n", base, start, end);
if (*ptep == PTE_NULL) {
#ifdef USE_LARGE_PAGES
if ((start <= base) && ((base + PTL2_SIZE) <= end)
&& ((args->diff & (PTL2_SIZE - 1)) == 0)) {
phys = args->phys + (base - start);
*ptep = phys | attr_to_l2attr(
args->attr|PTATTR_LARGEPAGE);
error = 0;
dkprintf("set_range_l2(%lx,%lx,%lx):"
"large page. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
#endif
pt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (pt == NULL) {
error = -ENOMEM;
ekprintf("set_range_l2(%lx,%lx,%lx):"
"__alloc_new_pt failed. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, start, base,
KEEP_PHYSICAL);
goto out;
}
*ptep = virt_to_phys(pt) | PFL2_PDIR_ATTR;
}
else if (*ptep & PFL2_SIZE) {
error = -EBUSY;
ekprintf("set_range_l2(%lx,%lx,%lx):"
"page exists. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, start, base, KEEP_PHYSICAL);
goto out;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l1(pt, base, start, end, &set_range_l1, args0);
if (error) {
ekprintf("set_range_l2(%lx,%lx,%lx):"
"walk_pte_l1 failed. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
error = 0;
out:
dkprintf("set_range_l2(%lx,%lx,%lx): %d %lx\n",
base, start, end, error, *ptep);
return error;
}
int set_range_l3(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct set_range_args *args = args0;
struct page_table *pt;
int error;
#ifdef USE_LARGE_PAGES
uintptr_t phys;
#endif
dkprintf("set_range_l3(%lx,%lx,%lx)\n", base, start, end);
if (*ptep == PTE_NULL) {
#ifdef USE_LARGE_PAGES
if ((start <= base) && ((base + PTL3_SIZE) <= end)
&& ((args->diff & (PTL3_SIZE - 1)) == 0)
&& use_1gb_page) {
phys = args->phys + (base - start);
*ptep = phys | attr_to_l3attr(
args->attr|PTATTR_LARGEPAGE);
error = 0;
dkprintf("set_range_l3(%lx,%lx,%lx):"
"1GiB page. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
#endif
pt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (pt == NULL) {
error = -ENOMEM;
ekprintf("set_range_l3(%lx,%lx,%lx):"
"__alloc_new_pt failed. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, start, base,
KEEP_PHYSICAL);
goto out;
}
*ptep = virt_to_phys(pt) | PFL3_PDIR_ATTR;
}
else if (*ptep & PFL3_SIZE) {
error = -EBUSY;
ekprintf("set_range_l3(%lx,%lx,%lx):"
"page exists. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, start, base, KEEP_PHYSICAL);
goto out;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l2(pt, base, start, end, &set_range_l2, args0);
if (error) {
ekprintf("set_range_l3(%lx,%lx,%lx):"
"walk_pte_l2 failed. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
error = 0;
out:
dkprintf("set_range_l3(%lx,%lx,%lx): %d\n",
base, start, end, error, *ptep);
return error;
}
int set_range_l4(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct set_range_args *args = args0;
struct page_table *pt;
int error;
dkprintf("set_range_l4(%lx,%lx,%lx)\n", base, start, end);
if (*ptep == PTE_NULL) {
pt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (pt == NULL) {
error = -ENOMEM;
ekprintf("set_range_l4(%lx,%lx,%lx):"
"__alloc_new_pt failed. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, start, base,
KEEP_PHYSICAL);
goto out;
}
*ptep = virt_to_phys(pt) | PFL4_PDIR_ATTR;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l3(pt, base, start, end, &set_range_l3, args0);
if (error) {
ekprintf("set_range_l4(%lx,%lx,%lx):"
"walk_pte_l3 failed. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
error = 0;
out:
dkprintf("set_range_l4(%lx,%lx,%lx): %d %lx\n",
base, start, end, error, *ptep);
return error;
}
int ihk_mc_pt_set_range(page_table_t pt, void *start, void *end,
uintptr_t phys, enum ihk_mc_pt_attribute attr)
{
int error;
struct set_range_args args;
dkprintf("ihk_mc_pt_set_range(%p,%p,%p,%lx,%x)\n",
pt, start, end, phys, attr);
args.pt = pt;
args.phys = phys;
args.attr = attr;
args.diff = (uintptr_t)start ^ phys;
error = walk_pte_l4(pt, 0, (uintptr_t)start, (uintptr_t)end,
&set_range_l4, &args);
if (error) {
ekprintf("ihk_mc_pt_set_range(%p,%p,%p,%lx,%x):"
"walk_pte_l4 failed. %d\n",
pt, start, end, phys, attr, error);
goto out;
}
error = 0;
out:
dkprintf("ihk_mc_pt_set_range(%p,%p,%p,%lx,%x): %d\n",
pt, start, end, phys, attr, error);
return error;
}
int ihk_mc_pt_set_pte(page_table_t pt, pte_t *ptep, size_t pgsize,
uintptr_t phys, enum ihk_mc_pt_attribute attr)
{
int error;
dkprintf("ihk_mc_pt_set_pte(%p,%p,%lx,%lx,%x)\n",
pt, ptep, pgsize, phys, attr);
if (pgsize == PTL1_SIZE) {
*ptep = phys | attr_to_l1attr(attr);
}
#ifdef USE_LARGE_PAGES
else if (pgsize == PTL2_SIZE) {
*ptep = phys | attr_to_l2attr(attr | PTATTR_LARGEPAGE);
}
else if ((pgsize == PTL3_SIZE) && (use_1gb_page)) {
*ptep = phys | attr_to_l3attr(attr | PTATTR_LARGEPAGE);
}
#endif
else {
error = -EINVAL;
ekprintf("ihk_mc_pt_set_pte(%p,%p,%lx,%lx,%x):"
"page size. %d %lx\n",
pt, ptep, pgsize, phys, attr, error, *ptep);
panic("ihk_mc_pt_set_pte:page size");
goto out;
}
error = 0;
out:
dkprintf("ihk_mc_pt_set_pte(%p,%p,%lx,%lx,%x): %d %lx\n",
pt, ptep, pgsize, phys, attr, error, *ptep);
return error;
}
int arch_get_smaller_page_size(void *args, size_t cursize, size_t *newsizep,
int *p2alignp)
{
size_t newsize;
int p2align;
int error;
if (0) {
/* dummy */
panic("not reached");
}
#ifdef USE_LARGE_PAGES
else if ((cursize > PTL3_SIZE) && use_1gb_page) {
/* 1GiB */
newsize = PTL3_SIZE;
p2align = PTL3_SHIFT - PTL1_SHIFT;
}
else if (cursize > PTL2_SIZE) {
/* 2MiB */
newsize = PTL2_SIZE;
p2align = PTL2_SHIFT - PTL1_SHIFT;
}
#endif
else if (cursize > PTL1_SIZE) {
/* 4KiB : basic page size */
newsize = PTL1_SIZE;
p2align = PTL1_SHIFT - PTL1_SHIFT;
}
else {
error = -ENOMEM;
newsize = 0;
p2align = -1;
goto out;
}
error = 0;
if (newsizep) *newsizep = newsize;
if (p2alignp) *p2alignp = p2align;
out:
dkprintf("arch_get_smaller_page_size(%p,%lx): %d %lx %d\n",
args, cursize, error, newsize, p2align);
return error;
}
void load_page_table(struct page_table *pt)
{
unsigned long pt_addr;
@@ -1224,6 +1666,9 @@ void init_low_area(struct page_table *pt)
void init_page_table(void)
{
#ifdef USE_LARGE_PAGES
check_available_page_size();
#endif
init_pt = arch_alloc_page(IHK_MC_AP_CRITICAL);
memset(init_pt, 0, sizeof(PAGE_SIZE));

2
kernel/include/process.h
View File

@@ -133,6 +133,8 @@ struct vm_range *next_process_memory_range(
struct process_vm *vm, struct vm_range *range);
struct vm_range *previous_process_memory_range(
struct process_vm *vm, struct vm_range *range);
int page_fault_process(struct process *proc, void *fault_addr, uint64_t reason);
int remove_process_region(struct process *proc,
unsigned long start, unsigned long end);
struct program_load_desc;

115
kernel/mem.c
View File

@@ -143,72 +143,40 @@ static struct ihk_mc_interrupt_handler query_free_mem_handler = {
void sigsegv(void *);
static void page_fault_handler(unsigned long address, void *regs,
unsigned long rbp)
static void unhandled_page_fault(struct process *proc, void *fault_addr, void *regs)
{
struct vm_range *range, *next;
char found = 0;
const uintptr_t address = (uintptr_t)fault_addr;
struct process_vm *vm = proc->vm;
struct vm_range *range;
char found;
int irqflags;
unsigned long error = ((struct x86_regs *)regs)->error;
irqflags = kprintf_lock();
__kprintf("[%d] Page fault for 0x%lX, (rbp: 0x%lX)\n",
ihk_mc_get_processor_id(), address, rbp);
__kprintf("[%d] Page fault for 0x%lX\n",
ihk_mc_get_processor_id(), address);
__kprintf("%s for %s access in %s mode (reserved bit %s set), "
"it %s an instruction fetch\n",
(error & PF_PROT ? "protection fault" : "no page found"),
(error & PF_WRITE ? "write" : "read"),
(error & PF_USER ? "user" : "kernel"),
(error & PF_RSVD ? "was" : "wasn't"),
(error & PF_INSTR ? "was" : "wasn't"));
__kprintf("%s for %s access in %s mode (reserved bit %s set), it %s an instruction fetch\n",
(error & PF_PROT ? "protection fault" : "no page found"),
(error & PF_WRITE ? "write" : "read"),
(error & PF_USER ? "user" : "kernel"),
(error & PF_RSVD ? "was" : "wasn't"),
(error & PF_INSTR ? "was" : "wasn't"));
list_for_each_entry_safe(range, next,
&cpu_local_var(current)->vm->vm_range_list,
list) {
found = 0;
list_for_each_entry(range, &vm->vm_range_list, list) {
if (range->start <= address && range->end > address) {
__kprintf("address is in range, flag: 0x%X! \n", range->flag);
if(range->flag & VR_DEMAND_PAGING){
//allocate page for demand paging
__kprintf("demand paging\n");
void* pa = allocate_pages(1, IHK_MC_AP_CRITICAL);
if(!pa){
kprintf_unlock(irqflags);
panic("allocate_pages failed");
}
__kprintf("physical memory area obtained %lx\n", virt_to_phys(pa));
{
enum ihk_mc_pt_attribute flag = 0;
struct process *process = cpu_local_var(current);
unsigned long flags = ihk_mc_spinlock_lock(&process->vm->page_table_lock);
const enum ihk_mc_pt_attribute attr = flag | PTATTR_WRITABLE | PTATTR_USER | PTATTR_FOR_USER;
int rc = ihk_mc_pt_set_page(process->vm->page_table, (void*)(address & PAGE_MASK), virt_to_phys(pa), attr);
if(rc != 0) {
ihk_mc_spinlock_unlock(&process->vm->page_table_lock, flags);
__kprintf("ihk_mc_pt_set_page failed,rc=%d,%p,%lx,%08x\n", rc, (void*)(address & PAGE_MASK), virt_to_phys(pa), attr);
ihk_mc_pt_print_pte(process->vm->page_table, (void*)address);
goto fn_fail;
}
ihk_mc_spinlock_unlock(&process->vm->page_table_lock, flags);
__kprintf("update_process_page_table success\n");
}
kprintf_unlock(irqflags);
memset(pa, 0, PAGE_SIZE);
return;
}
found = 1;
ihk_mc_pt_print_pte(cpu_local_var(current)->vm->page_table,
(void*)address);
__kprintf("address is in range, flag: 0x%X! \n",
range->flag);
ihk_mc_pt_print_pte(vm->page_table, (void*)address);
break;
}
}
if (!found)
if (!found) {
__kprintf("address is out of range! \n");
}
fn_fail:
kprintf_unlock(irqflags);
/* TODO */
@@ -216,19 +184,44 @@ static void page_fault_handler(unsigned long address, void *regs,
#ifdef DEBUG_PRINT_MEM
{
const struct x86_regs *_regs = regs;
dkprintf("*rsp:%lx,*rsp+8:%lx,*rsp+16:%lx,*rsp+24:%lx,\n",
*((unsigned long*)_regs->rsp),
*((unsigned long*)_regs->rsp+8),
*((unsigned long*)_regs->rsp+16),
*((unsigned long*)_regs->rsp+24)
);
uint64_t *sp = (void *)REGS_GET_STACK_POINTER(regs);
kprintf("*rsp:%lx,*rsp+8:%lx,*rsp+16:%lx,*rsp+24:%lx,\n",
sp[0], sp[1], sp[2], sp[3]);
}
#endif
#if 0
panic("mem fault");
#endif
sigsegv(regs);
return;
}
//panic("mem fault");
static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
{
struct process *proc = cpu_local_var(current);
int error;
dkprintf("[%d]page_fault_handler(%p,%lx,%p)\n",
ihk_mc_get_processor_id(), fault_addr, reason, regs);
error = page_fault_process(proc, fault_addr, reason);
if (error) {
kprintf("[%d]page_fault_handler(%p,%lx,%p):"
"fault proc failed. %d\n",
ihk_mc_get_processor_id(), fault_addr,
reason, regs, error);
unhandled_page_fault(proc, fault_addr, regs);
goto out;
}
error = 0;
out:
dkprintf("[%d]page_fault_handler(%p,%lx,%p): (%d)\n",
ihk_mc_get_processor_id(), fault_addr, reason,
regs, error);
return;
}
static void page_allocator_init(void)

203
kernel/process.c
View File

@@ -636,6 +636,209 @@ out:
return error;
}
static int page_fault_process_memory_range(struct process_vm *vm,
struct vm_range *range, uintptr_t fault_addr)
{
int error;
int npages;
void *virt = NULL;
void *ptepgaddr;
size_t ptepgsize;
int ptep2align;
void *pgaddr;
size_t pgsize;
int p2align;
uintptr_t phys;
enum ihk_mc_pt_attribute attr;
pte_t *ptep;
dkprintf("[%d]page_fault_process_memory_range(%p,%lx-%lx %lx,%lx)\n",
ihk_mc_get_processor_id(), vm, range->start,
range->end, range->flag, fault_addr);
ihk_mc_spinlock_lock_noirq(&vm->page_table_lock);
/* (1) check PTE */
ptep = ihk_mc_pt_lookup_pte(vm->page_table, (void *)fault_addr,
&ptepgaddr, &ptepgsize, &ptep2align);
if (ptep && (*ptep != PTE_NULL)) {
if (!(*ptep & PF_PRESENT)) {
error = -EFAULT;
kprintf("[%d]page_fault_process_memory_range"
"(%p,%lx-%lx %lx,%lx):"
"disabled page. %d\n",
ihk_mc_get_processor_id(), vm,
range->start, range->end,
range->flag, fault_addr, error);
goto out;
}
error = 0;
kprintf("[%d]page_fault_process_memory_range"
"(%p,%lx-%lx %lx,%lx):already mapped. %d\n",
ihk_mc_get_processor_id(), vm, range->start,
range->end, range->flag, fault_addr, error);
goto out;
}
/* (2) select page size */
#ifdef USE_LARGE_PAGES
if (!ptep) {
/* get largest page size */
error = arch_get_smaller_page_size(NULL, -1, &ptepgsize, &ptep2align);
if (error) {
kprintf("[%d]page_fault_process_memory_range"
"(%p,%lx-%lx %lx,%lx):"
"get pgsize failed. %d\n",
ihk_mc_get_processor_id(), vm,
range->start, range->end,
range->flag, fault_addr, error);
goto out;
}
}
#else
if (!ptep || (ptepgsize != PAGE_SIZE)) {
ptep = NULL;
ptepgsize = PAGE_SIZE;
ptep2align = PAGE_P2ALIGN;
}
#endif
pgsize = ptepgsize;
p2align = ptep2align;
/* (3) get physical page */
for (;;) {
pgaddr = (void *)(fault_addr & ~(pgsize - 1));
if ((range->start <= (uintptr_t)pgaddr)
&& (((uintptr_t)pgaddr + pgsize) <= range->end)) {
npages = pgsize / PAGE_SIZE;
virt = ihk_mc_alloc_aligned_pages(npages, p2align,
IHK_MC_AP_NOWAIT);
if (virt) {
phys = virt_to_phys(virt);
memset(virt, 0, pgsize);
break;
}
}
/* (4) if failed, select smaller page size, and retry */
ptep = NULL;
error = arch_get_smaller_page_size(
vm, pgsize, &pgsize, &p2align);
if (error) {
kprintf("[%d]page_fault_process_memory_range"
"(%p,%lx-%lx %lx,%lx):"
"get pgsize failed. %d\n",
ihk_mc_get_processor_id(), vm,
range->start, range->end,
range->flag, fault_addr, error);
goto out;
}
}
/* (5) mapping */
attr = vrflag_to_ptattr(range->flag);
if (ptep) {
error = ihk_mc_pt_set_pte(vm->page_table, ptep, pgsize, phys, attr);
if (error) {
kprintf("[%d]page_fault_process_memory_range"
"(%p,%lx-%lx %lx,%lx):"
"set pte failed. %d\n",
ihk_mc_get_processor_id(), vm,
range->start, range->end,
range->flag, fault_addr, error);
goto out;
}
}
else {
error = ihk_mc_pt_set_range(vm->page_table, pgaddr,
pgaddr+pgsize, phys, attr);
if (error) {
kprintf("[%d]page_fault_process_memory_range"
"(%p,%lx-%lx %lx,%lx):"
"set range failed. %d\n",
ihk_mc_get_processor_id(), vm,
range->start, range->end,
range->flag, fault_addr, error);
goto out;
}
}
virt = NULL;
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock);
if (virt != NULL) {
ihk_mc_free_pages(virt, npages);
}
dkprintf("[%d]page_fault_process_memory_range(%p,%lx-%lx %lx,%lx): %d\n",
ihk_mc_get_processor_id(), vm, range->start,
range->end, range->flag, fault_addr, error);
return error;
}
int page_fault_process(struct process *proc, void *fault_addr0, uint64_t reason)
{
struct process_vm *vm = proc->vm;
int error;
const uintptr_t fault_addr = (uintptr_t)fault_addr0;
struct vm_range *range;
dkprintf("[%d]page_fault_process(%p,%lx,%lx)\n",
ihk_mc_get_processor_id(), proc, fault_addr0, reason);
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
/* NYI: page proctection fault */
if (reason & PF_PROT) {
error = -EFAULT;
kprintf("[%d]page_fault_process(%p,%lx,%lx):"
"protection fault. %d\n",
ihk_mc_get_processor_id(), proc,
fault_addr0, reason, error);
goto out;
}
range = lookup_process_memory_range(vm, fault_addr, fault_addr+1);
if (range == NULL) {
error = -EFAULT;
kprintf("[%d]page_fault_process(%p,%lx,%lx):"
"out of range. %d\n",
ihk_mc_get_processor_id(), proc,
fault_addr0, reason, error);
goto out;
}
if (((range->flag & VR_PROT_MASK) == VR_PROT_NONE)
|| ((reason & PF_WRITE)
&& !(range->flag & VR_PROT_WRITE))) {
error = -EFAULT;
kprintf("[%d]page_fault_process(%p,%lx,%lx):"
"access denied. %d\n",
ihk_mc_get_processor_id(), proc,
fault_addr0, reason, error);
goto out;
}
error = page_fault_process_memory_range(vm, range, fault_addr);
if (error) {
kprintf("[%d]page_fault_process(%p,%lx,%lx):"
"fault range failed. %d\n",
ihk_mc_get_processor_id(), proc,
fault_addr0, reason, error);
goto out;
}
error = 0;
out:
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
dkprintf("[%d]page_fault_process(%p,%lx,%lx): %d\n",
ihk_mc_get_processor_id(), proc, fault_addr0,
reason, error);
return error;
}
int init_process_stack(struct process *process, struct program_load_desc *pn,
int argc, char **argv,
int envc, char **env)

8
lib/include/ihk/mm.h
View File

@@ -1,6 +1,7 @@
#ifndef __HEADER_GENERIC_IHK_MM_H
#define __HEADER_GENERIC_IHK_MM_H
#include <ihk/types.h>
#include <memory.h>
enum ihk_mc_gma_type {
@@ -58,7 +59,7 @@ struct ihk_mc_pa_ops {
};
void ihk_mc_set_page_allocator(struct ihk_mc_pa_ops *);
void ihk_mc_set_page_fault_handler(void (*h)(unsigned long, void *, unsigned long));
void ihk_mc_set_page_fault_handler(void (*h)(void *, uint64_t, void *));
unsigned long ihk_mc_map_memory(void *os, unsigned long phys,
unsigned long size);
@@ -82,6 +83,7 @@ void ihk_mc_free(void *p);
void *arch_alloc_page(enum ihk_mc_ap_flag flag);
void arch_free_page(void *ptr);
int arch_get_smaller_page_size(void *args, size_t origsize, size_t *sizep, int *p2alignp);
typedef void *page_table_t;
@@ -100,6 +102,10 @@ int ihk_mc_pt_change_attr_range(page_table_t pt, void *start, void *end,
enum ihk_mc_pt_attribute setattr);
int ihk_mc_pt_alloc_range(page_table_t pt, void *start, void *end,
enum ihk_mc_pt_attribute attr);
pte_t *ihk_mc_pt_lookup_pte(page_table_t pt, void *virt, void **pgbasep, size_t *pgsizep, int *p2alignp);
int ihk_mc_pt_set_range(page_table_t pt, void *start, void *end,
uintptr_t phys, enum ihk_mc_pt_attribute attr);
int ihk_mc_pt_set_pte(page_table_t pt, pte_t *ptep, size_t pgsize, uintptr_t phys, enum ihk_mc_pt_attribute attr);
int ihk_mc_pt_prepare_map(page_table_t pt, void *virt, unsigned long size,
enum ihk_mc_pt_prepare_flag);