From 55faba77a551b6a542f71634398c9e94b2e064b9 Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Tue, 4 Feb 2020 23:35:30 +0000 Subject: [PATCH] dump: rewrite NMI handling (for resume) and fix PANIC register saving Change-Id: I360e9aa8efa64b6ebd99b209a5dd4ee0dc7806cf --- arch/arm64/kernel/cpu.c | 92 +++++++++---------- arch/x86_64/kernel/cpu.c | 90 +++++++++++++++++- arch/x86_64/kernel/interrupt.S | 69 ++------------ .../user/arch/x86_64/include/arch-eclair.h | 5 +- ihk | 2 +- kernel/freeze.c | 46 ++++++++++ kernel/mem.c | 35 +++---- lib/include/ihk/mm.h | 1 + 8 files changed, 202 insertions(+), 138 deletions(-) diff --git a/arch/arm64/kernel/cpu.c b/arch/arm64/kernel/cpu.c index 9e1faef0..6730b9f4 100644 --- a/arch/arm64/kernel/cpu.c +++ b/arch/arm64/kernel/cpu.c @@ -33,6 +33,7 @@ #include #include #include +#include //#define DEBUG_PRINT_CPU @@ -119,64 +120,49 @@ static struct ihk_mc_interrupt_handler cpu_stop_handler = { }; extern long freeze_thaw(void *nmi_ctx); -static void multi_interrupt_handler(void *priv) -{ - switch (multi_intr_mode) { - case 1: - case 2: /* mode == 1or2, for FREEZER intr */ - dkprintf("%s: freeze mode intr catch. (multi_intr_mode=%d)\n", - __func__, multi_intr_mode); - freeze_thaw(NULL); - break; - default: - ekprintf("%s: Unknown multi-intr-mode(%d) detected.\n", - __func__, multi_intr_mode); - break; - } -} -static void multi_nm_interrupt_handler(void *priv) +void arch_save_panic_regs(void *irq_regs) { - extern int nmi_mode; - struct pt_regs *regs = (struct pt_regs *)priv; + struct pt_regs *regs = (struct pt_regs *)irq_regs; union arm64_cpu_local_variables *clv; - switch (nmi_mode) { - case 0: - /* mode == 0, for MEMDUMP NMI */ - clv = get_arm64_this_cpu_local(); + clv = get_arm64_this_cpu_local(); - if (regs) { - memcpy(clv->arm64_cpu_local_thread.panic_regs, - regs->regs, sizeof(regs->regs)); - clv->arm64_cpu_local_thread.panic_regs[31] = regs->sp; - clv->arm64_cpu_local_thread.panic_regs[32] = regs->pc; - clv->arm64_cpu_local_thread.panic_regs[33] = - regs->pstate; - } - clv->arm64_cpu_local_thread.paniced = 1; - ihk_mc_query_mem_areas(); - /* memdump-nmi is halted McKernel, break is unnecessary. */ - /* fall through */ - case 3: - /* mode == 3, for SHUTDOWN-WAIT NMI */ - while (1) { - cpu_halt(); - } - break; - - default: - ekprintf("%s: Unknown nmi-mode(%d) detected.\n", - __func__, nmi_mode); - break; + /* For user-space, use saved kernel context */ + if (regs->pc < USER_END) { + memset(clv->arm64_cpu_local_thread.panic_regs, + 0, sizeof(clv->arm64_cpu_local_thread.panic_regs)); + clv->arm64_cpu_local_thread.panic_regs[29] = + current_thread_info()->cpu_context.fp; + clv->arm64_cpu_local_thread.panic_regs[31] = + current_thread_info()->cpu_context.sp; + clv->arm64_cpu_local_thread.panic_regs[32] = + current_thread_info()->cpu_context.pc; + clv->arm64_cpu_local_thread.panic_regs[33] = + PSR_MODE_EL1h; } + else { + memcpy(clv->arm64_cpu_local_thread.panic_regs, + regs->regs, sizeof(regs->regs)); + clv->arm64_cpu_local_thread.panic_regs[31] = regs->sp; + clv->arm64_cpu_local_thread.panic_regs[32] = regs->pc; + clv->arm64_cpu_local_thread.panic_regs[33] = + regs->pstate; + + } + + clv->arm64_cpu_local_thread.paniced = 1; } -static struct ihk_mc_interrupt_handler multi_intr_handler = { - .func = multi_interrupt_handler, - .priv = NULL, -}; +void arch_clear_panic(void) +{ + union arm64_cpu_local_variables *clv; + clv = get_arm64_this_cpu_local(); + clv->arm64_cpu_local_thread.paniced = 0; +} + +extern void multi_nm_interrupt_handler(void *irq_regs); static struct ihk_mc_interrupt_handler multi_nmi_handler = { .func = multi_nm_interrupt_handler, .priv = NULL, @@ -441,8 +427,6 @@ void ihk_mc_init_ap(void) ihk_mc_register_interrupt_handler(INTRID_CPU_STOP, &cpu_stop_handler); ihk_mc_register_interrupt_handler(INTRID_MULTI_NMI, &multi_nmi_handler); - ihk_mc_register_interrupt_handler(INTRID_MULTI_INTR, - &multi_intr_handler); ihk_mc_register_interrupt_handler( ihk_mc_get_vector(IHK_TLB_FLUSH_IRQ_VECTOR_START), &remote_tlb_flush_handler); @@ -1035,6 +1019,9 @@ void ihk_mc_init_context(ihk_mc_kernel_context_t *new_ctx, /* branch in ret_from_fork */ new_ctx->thread->cpu_context.x19 = (unsigned long)next_function; + sp -= 16; + new_ctx->thread->cpu_context.fp = sp; + /* set stack_pointer */ new_ctx->thread->cpu_context.sp = sp - sizeof(ihk_mc_user_context_t); @@ -1073,6 +1060,9 @@ void ihk_mc_init_context(ihk_mc_kernel_context_t *new_ctx, /* set stack_pointer */ new_ctx->thread->cpu_context.sp = sp; + /* use the 16 bytes padding in ihk_mc_init_user_process() + * as closing frame in the frame chain */ + new_ctx->thread->cpu_context.fp = sp + sizeof(ihk_mc_user_context_t); /* clear pt_regs area */ new_uctx = (ihk_mc_user_context_t *)new_ctx->thread->cpu_context.sp; diff --git a/arch/x86_64/kernel/cpu.c b/arch/x86_64/kernel/cpu.c index 9a7b7cee..b1bc901d 100644 --- a/arch/x86_64/kernel/cpu.c +++ b/arch/x86_64/kernel/cpu.c @@ -145,7 +145,7 @@ void reload_idt(void) } static struct list_head handlers[256 - 32]; -extern char nmi[]; +extern char nmi_handler[]; extern char page_fault[], general_protection_exception[]; extern char debug_exception[], int3_exception[]; @@ -172,7 +172,7 @@ static void init_idt(void) set_idt_entry(i, generic_common_handlers[i]); } - set_idt_entry(2, (uintptr_t)nmi); + set_idt_entry(2, (uintptr_t)nmi_handler); set_idt_entry(13, (unsigned long)general_protection_exception); set_idt_entry(14, (unsigned long)page_fault); @@ -2003,6 +2003,92 @@ mod_nmi_ctx(void *nmi_ctx, void (*func)()) l[i++] = 0x28; // KERNEL DS } +void arch_save_panic_regs(void *irq_regs) +{ + struct thread *current = cpu_local_var(current); + struct x86_user_context *regs = + (struct x86_user_context *)irq_regs; + struct x86_cpu_local_variables *x86v = + get_x86_cpu_local_variable(ihk_mc_get_processor_id()); + struct segment_regs { + uint32_t rflags; + uint32_t cs; + uint32_t ss; + uint32_t ds; + uint32_t es; + uint32_t fs; + uint32_t gs; + } *sregs; + + /* Kernel space? */ + if (regs->gpr.rip > USER_END) { + x86v->panic_regs[0] = regs->gpr.rax; + x86v->panic_regs[1] = regs->gpr.rbx; + x86v->panic_regs[2] = regs->gpr.rcx; + x86v->panic_regs[3] = regs->gpr.rdx; + x86v->panic_regs[4] = regs->gpr.rsi; + x86v->panic_regs[5] = regs->gpr.rdi; + x86v->panic_regs[6] = regs->gpr.rbp; + x86v->panic_regs[7] = regs->gpr.rsp; + x86v->panic_regs[8] = regs->gpr.r8; + x86v->panic_regs[9] = regs->gpr.r9; + x86v->panic_regs[10] = regs->gpr.r10; + x86v->panic_regs[11] = regs->gpr.r11; + x86v->panic_regs[12] = regs->gpr.r12; + x86v->panic_regs[13] = regs->gpr.r13; + x86v->panic_regs[14] = regs->gpr.r14; + x86v->panic_regs[15] = regs->gpr.r15; + x86v->panic_regs[16] = regs->gpr.rip; + sregs = (struct segment_regs *)&x86v->panic_regs[17]; + sregs->rflags = regs->gpr.rflags; + sregs->cs = regs->gpr.cs; + sregs->ss = regs->gpr.ss; + sregs->ds = regs->sr.ds; + sregs->es = regs->sr.es; + sregs->fs = regs->sr.fs; + sregs->gs = regs->sr.gs; + } + /* User-space, show kernel context */ + else { + kprintf("%s: in user-space: %p\n", __func__, regs->gpr.rip); + x86v->panic_regs[0] = 0; + x86v->panic_regs[1] = current->ctx.rbx; + x86v->panic_regs[2] = 0; + x86v->panic_regs[3] = 0; + x86v->panic_regs[4] = current->ctx.rsi; + x86v->panic_regs[5] = current->ctx.rdi; + x86v->panic_regs[6] = current->ctx.rbp; + x86v->panic_regs[7] = current->ctx.rsp; + x86v->panic_regs[8] = 0; + x86v->panic_regs[9] = 0; + x86v->panic_regs[10] = 0; + x86v->panic_regs[11] = 0; + x86v->panic_regs[12] = regs->gpr.r12; + x86v->panic_regs[13] = regs->gpr.r13; + x86v->panic_regs[14] = regs->gpr.r14; + x86v->panic_regs[15] = regs->gpr.r15; + x86v->panic_regs[16] = (unsigned long)enter_user_mode; + sregs = (struct segment_regs *)&x86v->panic_regs[17]; + sregs->rflags = regs->gpr.rflags; + sregs->cs = regs->gpr.cs; + sregs->ss = regs->gpr.ss; + sregs->ds = regs->sr.ds; + sregs->es = regs->sr.es; + sregs->fs = regs->sr.fs; + sregs->gs = regs->sr.gs; + } + + x86v->paniced = 1; +} + +void arch_clear_panic(void) +{ + struct x86_cpu_local_variables *x86v = + get_x86_cpu_local_variable(ihk_mc_get_processor_id()); + + x86v->paniced = 0; +} + int arch_cpu_read_write_register( struct ihk_os_cpu_register *desc, enum mcctrl_os_cpu_operation op) diff --git a/arch/x86_64/kernel/interrupt.S b/arch/x86_64/kernel/interrupt.S index 3d07ea79..a535954d 100644 --- a/arch/x86_64/kernel/interrupt.S +++ b/arch/x86_64/kernel/interrupt.S @@ -102,24 +102,20 @@ common_interrupt: POP_ALL_REGS addq $8, %rsp iretq -1: -#define PANIC_REGS 240 - movq %rax,%gs:PANIC_REGS+0x00 - movq %rsp,%gs:PANIC_REGS+0x08 + + +.globl nmi_handler +nmi_handler: cld - movq %gs:PANIC_REGS+0x00,%rax + pushq $0 /* error field of x86_basic_regs */ PUSH_ALL_REGS - subq $40, %rsp - movq %rsp,%gs:PANIC_REGS+0x10 movq %rsp, %rdi - call freeze_thaw - cmpq $0, %rax - jnz 2f - addq $40, %rsp -2: + call multi_nm_interrupt_handler /* Enter C code */ POP_ALL_REGS + addq $8, %rsp iretq + .globl __page_fault_handler_address __page_fault_handler_address: .quad 0 @@ -158,55 +154,6 @@ __freeze: POP_ALL_REGS iretq -.globl nmi -nmi: -#define PANICED 232 - movq %rax,%gs:PANIC_REGS+0x00 - movq %rsp,%gs:PANIC_REGS+0x08 - - movl nmi_mode(%rip),%eax - cmp $3,%rax - je 4f -3: - movq %rbx,%gs:PANIC_REGS+0x08 - movq %rcx,%gs:PANIC_REGS+0x10 - movq %rdx,%gs:PANIC_REGS+0x18 - movq %rsi,%gs:PANIC_REGS+0x20 - movq %rdi,%gs:PANIC_REGS+0x28 - movq %rbp,%gs:PANIC_REGS+0x30 - movq 0x18(%rsp),%rax /* rsp */ - movq %rax,%gs:PANIC_REGS+0x38 - movq %r8, %gs:PANIC_REGS+0x40 - movq %r9, %gs:PANIC_REGS+0x48 - movq %r10,%gs:PANIC_REGS+0x50 - movq %r11,%gs:PANIC_REGS+0x58 - movq %r12,%gs:PANIC_REGS+0x60 - movq %r13,%gs:PANIC_REGS+0x68 - movq %r14,%gs:PANIC_REGS+0x70 - movq %r15,%gs:PANIC_REGS+0x78 - movq 0x00(%rsp),%rax /* rip */ - movq %rax,%gs:PANIC_REGS+0x80 - movq 0x10(%rsp),%rax /* rflags */ - movl %eax,%gs:PANIC_REGS+0x88 - movq 0x08(%rsp),%rax /* cs */ - movl %eax,%gs:PANIC_REGS+0x8C - movq 0x20(%rsp),%rax /* ss */ - movl %eax,%gs:PANIC_REGS+0x90 - xorq %rax,%rax - movw %ds,%ax - movl %eax,%gs:PANIC_REGS+0x94 - movw %es,%ax - movl %eax,%gs:PANIC_REGS+0x98 - movw %fs,%ax - movl %eax,%gs:PANIC_REGS+0x9C - movw %gs,%ax - movl %eax,%gs:PANIC_REGS+0xA0 - movq $1,%gs:PANICED - call ihk_mc_query_mem_areas -4: - hlt - jmp 4b - .globl x86_syscall x86_syscall: cld diff --git a/executer/user/arch/x86_64/include/arch-eclair.h b/executer/user/arch/x86_64/include/arch-eclair.h index 1a0f056b..3f94204c 100644 --- a/executer/user/arch/x86_64/include/arch-eclair.h +++ b/executer/user/arch/x86_64/include/arch-eclair.h @@ -11,9 +11,8 @@ extern unsigned long linux_page_offset; #define ARCH_REGS 21 -#define PANIC_REGS_OFFSET 240 - -#define MAP_KERNEL_TEXT "0xffffffff80001000" +/* See struct x86_cpu_local_variables */ +#define PANIC_REGS_OFFSET 288 struct arch_kregs { uintptr_t rsp, rbp, rbx, rsi; diff --git a/ihk b/ihk index b053576c..f0ef7f64 160000 --- a/ihk +++ b/ihk @@ -1 +1 @@ -Subproject commit b053576c89b428e0e6a8f75412d85909a7259b27 +Subproject commit f0ef7f641f7c41eebbeaeb9f2086c77cdd594529 diff --git a/kernel/freeze.c b/kernel/freeze.c index 677fca5b..c0abe762 100644 --- a/kernel/freeze.c +++ b/kernel/freeze.c @@ -7,6 +7,7 @@ #include #include +extern int nmi_mode; extern void mod_nmi_ctx(void *, void(*)()); extern void lapic_ack(); extern void __freeze(); @@ -70,3 +71,48 @@ freeze_thaw(void *nmi_ctx) } return 0; } + +extern void arch_save_panic_regs(void *irq_regs); +extern void arch_clear_panic(void); + +void +multi_nm_interrupt_handler(void *irq_regs) +{ + dkprintf("%s: ...\n", __func__); + switch (nmi_mode) { + case 1: + case 2: + /* mode == 1 or 2, for FREEZER NMI */ + dkprintf("%s: freeze mode NMI catch. (nmi_mode=%d)\n", + __func__, nmi_mode); + freeze_thaw(NULL); + break; + + case 0: + /* mode == 0, for MEMDUMP NMI */ + arch_save_panic_regs(irq_regs); + ihk_mc_query_mem_areas(); + /* memdump-nmi is halted McKernel, break is unnecessary. */ + /* fall through */ + case 3: + /* mode == 3, for SHUTDOWN-WAIT NMI */ + kprintf("%s: STOP\n", __func__); + while (nmi_mode != 4) + cpu_halt(); + break; + + case 4: + /* mode == 4, continue NMI */ + arch_clear_panic(); + if (!ihk_mc_get_processor_id()) { + ihk_mc_clear_dump_page_completion(); + } + kprintf("%s: RESUME, nmi_mode: %d\n", __func__, nmi_mode); + break; + + default: + ekprintf("%s: Unknown nmi-mode(%d) detected.\n", + __func__, nmi_mode); + break; + } +} diff --git a/kernel/mem.c b/kernel/mem.c index 702f0999..270d8016 100644 --- a/kernel/mem.c +++ b/kernel/mem.c @@ -44,6 +44,7 @@ #include #include #include +#include //#define DEBUG_PRINT_MEM @@ -114,29 +115,11 @@ struct pagealloc_track_entry { ihk_spinlock_t addr_list_lock; }; -struct ihk_dump_page { - unsigned long start; - unsigned long map_count; - unsigned long map[0]; -}; - -struct ihk_dump_page_set { - volatile unsigned int completion_flag; - unsigned int count; - unsigned long page_size; - unsigned long phy_page; -}; - struct dump_pase_info { struct ihk_dump_page_set *dump_page_set; struct ihk_dump_page *dump_pages; }; -#define IHK_DUMP_PAGE_SET_INCOMPLETE 0 -#define IHK_DUMP_PAGE_SET_COMPLETED 1 -#define DUMP_LEVEL_ALL 0 -#define DUMP_LEVEL_USER_UNUSED_EXCLUDE 24 - /** Get the index in the map array */ #define MAP_INDEX(n) ((n) >> 6) /** Get the bit number in a map element */ @@ -2512,10 +2495,13 @@ void ihk_mc_query_mem_areas(void){ struct ihk_dump_page_set *dump_page_set; struct dump_pase_info dump_pase_info; - /* Performed only for CPU 0 */ + /* + * Performed only on the last CPU to make sure + * all other cores are already stopped. + */ cpu_id = ihk_mc_get_processor_id(); - if (0 != cpu_id) + if (num_processors - 1 != cpu_id) return; dump_page_set = ihk_mc_get_dump_page_set(); @@ -2534,10 +2520,19 @@ void ihk_mc_query_mem_areas(void){ } dump_page_set->completion_flag = IHK_DUMP_PAGE_SET_COMPLETED; + dkprintf("%s: IHK_DUMP_PAGE_SET_COMPLETED\n", __func__); return; } +void ihk_mc_clear_dump_page_completion(void) +{ + struct ihk_dump_page_set *dump_page_set; + + dump_page_set = ihk_mc_get_dump_page_set(); + dump_page_set->completion_flag = IHK_DUMP_PAGE_SET_INCOMPLETE; +} + void ihk_mc_query_mem_user_page(void *dump_pase_info) { struct resource_set *rset = cpu_local_var(resource_set); diff --git a/lib/include/ihk/mm.h b/lib/include/ihk/mm.h index 58fba77f..e5b94c2f 100644 --- a/lib/include/ihk/mm.h +++ b/lib/include/ihk/mm.h @@ -251,6 +251,7 @@ unsigned int ihk_mc_get_dump_level(void); struct ihk_dump_page_set *ihk_mc_get_dump_page_set(void); struct ihk_dump_page *ihk_mc_get_dump_page(void); void ihk_mc_query_mem_areas(void); +void ihk_mc_clear_dump_page_completion(void); void ihk_mc_query_mem_user_page(void *dump_page_set); void ihk_mc_query_mem_free_page(void *dump_page_set); int ihk_mc_chk_page_address(pte_t mem_addr);