dump: rewrite NMI handling (for resume) and fix PANIC register saving

Change-Id: I360e9aa8efa64b6ebd99b209a5dd4ee0dc7806cf

arch/arm64/kernel/cpu.c

@@ -33,6 +33,7 @@
 #include <hwcap.h>
 #include <virt.h>
 #include <init.h>
+#include <bootparam.h>
 
 //#define DEBUG_PRINT_CPU
 
@@ -119,64 +120,49 @@ static struct ihk_mc_interrupt_handler cpu_stop_handler = {
 };
 
 extern long freeze_thaw(void *nmi_ctx);
-static void multi_interrupt_handler(void *priv)
-{
-	switch (multi_intr_mode) {
-	case 1:
-	case 2: /* mode == 1or2, for FREEZER intr */
-		dkprintf("%s: freeze mode intr catch. (multi_intr_mode=%d)\n",
-			 __func__, multi_intr_mode);
-		freeze_thaw(NULL);
-		break;
-	default:
-		ekprintf("%s: Unknown multi-intr-mode(%d) detected.\n",
-			 __func__, multi_intr_mode);
-		break;
-	}
-}
 
-static void multi_nm_interrupt_handler(void *priv)
+void arch_save_panic_regs(void *irq_regs)
 {
-	extern int nmi_mode;
+	struct pt_regs *regs = (struct pt_regs *)irq_regs;
-	struct pt_regs *regs = (struct pt_regs *)priv;
 	union arm64_cpu_local_variables *clv;
 
-	switch (nmi_mode) {
-	case 0:
-		/* mode == 0,    for MEMDUMP NMI */
 	clv = get_arm64_this_cpu_local();
 
-		if (regs) {
+	/* For user-space, use saved kernel context */
+	if (regs->pc < USER_END) {
+		memset(clv->arm64_cpu_local_thread.panic_regs,
+				0, sizeof(clv->arm64_cpu_local_thread.panic_regs));
+		clv->arm64_cpu_local_thread.panic_regs[29] =
+			current_thread_info()->cpu_context.fp;
+		clv->arm64_cpu_local_thread.panic_regs[31] =
+			current_thread_info()->cpu_context.sp;
+		clv->arm64_cpu_local_thread.panic_regs[32] =
+			current_thread_info()->cpu_context.pc;
+		clv->arm64_cpu_local_thread.panic_regs[33] =
+			PSR_MODE_EL1h;
+	}
+	else {
 		memcpy(clv->arm64_cpu_local_thread.panic_regs,
 				regs->regs, sizeof(regs->regs));
 		clv->arm64_cpu_local_thread.panic_regs[31] = regs->sp;
 		clv->arm64_cpu_local_thread.panic_regs[32] = regs->pc;
 		clv->arm64_cpu_local_thread.panic_regs[33] =
 			regs->pstate;
+
 	}
+
 	clv->arm64_cpu_local_thread.paniced = 1;
-		ihk_mc_query_mem_areas();
-		/* memdump-nmi is halted McKernel, break is unnecessary. */
-		/* fall through */
-	case 3:
-		/* mode == 3,    for SHUTDOWN-WAIT NMI */
-		while (1) {
-			cpu_halt();
-		}
-		break;
-
-	default:
-		ekprintf("%s: Unknown nmi-mode(%d) detected.\n",
-			 __func__, nmi_mode);
-		break;
-	}
 }
 
-static struct ihk_mc_interrupt_handler multi_intr_handler = {
+void arch_clear_panic(void)
-	.func = multi_interrupt_handler,
+{
-	.priv = NULL,
+	union arm64_cpu_local_variables *clv;
-};
 
+	clv = get_arm64_this_cpu_local();
+	clv->arm64_cpu_local_thread.paniced = 0;
+}
+
+extern void multi_nm_interrupt_handler(void *irq_regs);
 static struct ihk_mc_interrupt_handler multi_nmi_handler = {
 	.func = multi_nm_interrupt_handler,
 	.priv = NULL,
@@ -441,8 +427,6 @@ void ihk_mc_init_ap(void)
 
 	ihk_mc_register_interrupt_handler(INTRID_CPU_STOP, &cpu_stop_handler);
 	ihk_mc_register_interrupt_handler(INTRID_MULTI_NMI, &multi_nmi_handler);
-	ihk_mc_register_interrupt_handler(INTRID_MULTI_INTR,
-					&multi_intr_handler);
 	ihk_mc_register_interrupt_handler(
 		ihk_mc_get_vector(IHK_TLB_FLUSH_IRQ_VECTOR_START),
 		&remote_tlb_flush_handler);
@@ -1035,6 +1019,9 @@ void ihk_mc_init_context(ihk_mc_kernel_context_t *new_ctx,
 		/* branch in ret_from_fork */
 		new_ctx->thread->cpu_context.x19 = (unsigned long)next_function;
 
+		sp -= 16;
+		new_ctx->thread->cpu_context.fp = sp;
+
 		/* set stack_pointer */
 		new_ctx->thread->cpu_context.sp = sp - sizeof(ihk_mc_user_context_t);
 
@@ -1073,6 +1060,9 @@ void ihk_mc_init_context(ihk_mc_kernel_context_t *new_ctx,
 
 		/* set stack_pointer */
 		new_ctx->thread->cpu_context.sp = sp;
+		/* use the 16 bytes padding in ihk_mc_init_user_process()
+		 * as closing frame in the frame chain */
+		new_ctx->thread->cpu_context.fp = sp + sizeof(ihk_mc_user_context_t);
 
 		/* clear pt_regs area */
 		new_uctx = (ihk_mc_user_context_t *)new_ctx->thread->cpu_context.sp;

arch/x86_64/kernel/cpu.c

@@ -145,7 +145,7 @@ void reload_idt(void)
 }
 
 static struct list_head handlers[256 - 32];
-extern char nmi[];
+extern char nmi_handler[];
 extern char page_fault[], general_protection_exception[];
 extern char debug_exception[], int3_exception[];
 
@@ -172,7 +172,7 @@ static void init_idt(void)
 		set_idt_entry(i, generic_common_handlers[i]);
 	}
 
-	set_idt_entry(2, (uintptr_t)nmi);
+	set_idt_entry(2, (uintptr_t)nmi_handler);
 	set_idt_entry(13, (unsigned long)general_protection_exception);
 	set_idt_entry(14, (unsigned long)page_fault);
 
@@ -2003,6 +2003,92 @@ mod_nmi_ctx(void *nmi_ctx, void (*func)())
 	l[i++] = 0x28;				// KERNEL DS
 }
 
+void arch_save_panic_regs(void *irq_regs)
+{
+	struct thread *current = cpu_local_var(current);
+	struct x86_user_context *regs =
+		(struct x86_user_context *)irq_regs;
+	struct x86_cpu_local_variables *x86v =
+		get_x86_cpu_local_variable(ihk_mc_get_processor_id());
+	struct segment_regs {
+		uint32_t rflags;
+		uint32_t cs;
+		uint32_t ss;
+		uint32_t ds;
+		uint32_t es;
+		uint32_t fs;
+		uint32_t gs;
+	} *sregs;
+
+	/* Kernel space? */
+	if (regs->gpr.rip > USER_END) {
+		x86v->panic_regs[0] = regs->gpr.rax;
+		x86v->panic_regs[1] = regs->gpr.rbx;
+		x86v->panic_regs[2] = regs->gpr.rcx;
+		x86v->panic_regs[3] = regs->gpr.rdx;
+		x86v->panic_regs[4] = regs->gpr.rsi;
+		x86v->panic_regs[5] = regs->gpr.rdi;
+		x86v->panic_regs[6] = regs->gpr.rbp;
+		x86v->panic_regs[7] = regs->gpr.rsp;
+		x86v->panic_regs[8] = regs->gpr.r8;
+		x86v->panic_regs[9] = regs->gpr.r9;
+		x86v->panic_regs[10] = regs->gpr.r10;
+		x86v->panic_regs[11] = regs->gpr.r11;
+		x86v->panic_regs[12] = regs->gpr.r12;
+		x86v->panic_regs[13] = regs->gpr.r13;
+		x86v->panic_regs[14] = regs->gpr.r14;
+		x86v->panic_regs[15] = regs->gpr.r15;
+		x86v->panic_regs[16] = regs->gpr.rip;
+		sregs = (struct segment_regs *)&x86v->panic_regs[17];
+		sregs->rflags = regs->gpr.rflags;
+		sregs->cs = regs->gpr.cs;
+		sregs->ss = regs->gpr.ss;
+		sregs->ds = regs->sr.ds;
+		sregs->es = regs->sr.es;
+		sregs->fs = regs->sr.fs;
+		sregs->gs = regs->sr.gs;
+	}
+	/* User-space, show kernel context */
+	else {
+		kprintf("%s: in user-space: %p\n", __func__, regs->gpr.rip);
+		x86v->panic_regs[0] = 0;
+		x86v->panic_regs[1] = current->ctx.rbx;
+		x86v->panic_regs[2] = 0;
+		x86v->panic_regs[3] = 0;
+		x86v->panic_regs[4] = current->ctx.rsi;
+		x86v->panic_regs[5] = current->ctx.rdi;
+		x86v->panic_regs[6] = current->ctx.rbp;
+		x86v->panic_regs[7] = current->ctx.rsp;
+		x86v->panic_regs[8] = 0;
+		x86v->panic_regs[9] = 0;
+		x86v->panic_regs[10] = 0;
+		x86v->panic_regs[11] = 0;
+		x86v->panic_regs[12] = regs->gpr.r12;
+		x86v->panic_regs[13] = regs->gpr.r13;
+		x86v->panic_regs[14] = regs->gpr.r14;
+		x86v->panic_regs[15] = regs->gpr.r15;
+		x86v->panic_regs[16] = (unsigned long)enter_user_mode;
+		sregs = (struct segment_regs *)&x86v->panic_regs[17];
+		sregs->rflags = regs->gpr.rflags;
+		sregs->cs = regs->gpr.cs;
+		sregs->ss = regs->gpr.ss;
+		sregs->ds = regs->sr.ds;
+		sregs->es = regs->sr.es;
+		sregs->fs = regs->sr.fs;
+		sregs->gs = regs->sr.gs;
+	}
+
+	x86v->paniced = 1;
+}
+
+void arch_clear_panic(void)
+{
+	struct x86_cpu_local_variables *x86v =
+		get_x86_cpu_local_variable(ihk_mc_get_processor_id());
+
+	x86v->paniced = 0;
+}
+
 int arch_cpu_read_write_register(
 		struct ihk_os_cpu_register *desc,
 		enum mcctrl_os_cpu_operation op)

arch/x86_64/kernel/interrupt.S

@@ -102,24 +102,20 @@ common_interrupt:
 		POP_ALL_REGS
 		addq	$8, %rsp
 		iretq
-1:
+
-#define PANIC_REGS 240
+
-		movq	%rax,%gs:PANIC_REGS+0x00
+.globl  nmi_handler
-		movq	%rsp,%gs:PANIC_REGS+0x08
+nmi_handler:
 		cld
-		movq	%gs:PANIC_REGS+0x00,%rax
+		pushq   $0 /* error field of x86_basic_regs */
 		PUSH_ALL_REGS
-		subq	$40, %rsp
-		movq	%rsp,%gs:PANIC_REGS+0x10
 		movq	%rsp, %rdi
-		call	freeze_thaw
+		call 	multi_nm_interrupt_handler /* Enter C code */
-		cmpq	$0, %rax
-		jnz	2f
-		addq	$40, %rsp
-2:
 		POP_ALL_REGS
+		addq	$8, %rsp
 		iretq
 
+
 .globl __page_fault_handler_address
 __page_fault_handler_address:
 		.quad	0
@@ -158,55 +154,6 @@ __freeze:
 		POP_ALL_REGS
 		iretq
 
-.globl nmi
-nmi:
-#define PANICED 232
-		movq	%rax,%gs:PANIC_REGS+0x00
-		movq	%rsp,%gs:PANIC_REGS+0x08
-
-		movl	nmi_mode(%rip),%eax
-		cmp $3,%rax
-		je  4f
-3:
-		movq	%rbx,%gs:PANIC_REGS+0x08
-		movq	%rcx,%gs:PANIC_REGS+0x10
-		movq	%rdx,%gs:PANIC_REGS+0x18
-		movq	%rsi,%gs:PANIC_REGS+0x20
-		movq	%rdi,%gs:PANIC_REGS+0x28
-		movq	%rbp,%gs:PANIC_REGS+0x30
-		movq	0x18(%rsp),%rax		/* rsp */
-		movq	%rax,%gs:PANIC_REGS+0x38
-		movq	%r8, %gs:PANIC_REGS+0x40
-		movq	%r9, %gs:PANIC_REGS+0x48
-		movq	%r10,%gs:PANIC_REGS+0x50
-		movq	%r11,%gs:PANIC_REGS+0x58
-		movq	%r12,%gs:PANIC_REGS+0x60
-		movq	%r13,%gs:PANIC_REGS+0x68
-		movq	%r14,%gs:PANIC_REGS+0x70
-		movq	%r15,%gs:PANIC_REGS+0x78
-		movq	0x00(%rsp),%rax		/* rip */
-		movq	%rax,%gs:PANIC_REGS+0x80
-		movq	0x10(%rsp),%rax		/* rflags */
-		movl	%eax,%gs:PANIC_REGS+0x88
-		movq	0x08(%rsp),%rax		/* cs */
-		movl	%eax,%gs:PANIC_REGS+0x8C
-		movq	0x20(%rsp),%rax		/* ss */
-		movl	%eax,%gs:PANIC_REGS+0x90
-		xorq	%rax,%rax
-		movw	%ds,%ax
-		movl	%eax,%gs:PANIC_REGS+0x94
-		movw	%es,%ax
-		movl	%eax,%gs:PANIC_REGS+0x98
-		movw	%fs,%ax
-		movl	%eax,%gs:PANIC_REGS+0x9C
-		movw	%gs,%ax
-		movl	%eax,%gs:PANIC_REGS+0xA0
-		movq	$1,%gs:PANICED
-		call	ihk_mc_query_mem_areas
-4:
-		hlt
-		jmp	4b
-
 .globl x86_syscall
 x86_syscall:
 		cld

executer/user/arch/x86_64/include/arch-eclair.h

@@ -11,9 +11,8 @@ extern unsigned long linux_page_offset;
 
 #define ARCH_REGS	21
 
-#define PANIC_REGS_OFFSET	240
+/* See struct x86_cpu_local_variables */
-
+#define PANIC_REGS_OFFSET	288
-#define MAP_KERNEL_TEXT	"0xffffffff80001000"
 
 struct arch_kregs {
 	uintptr_t rsp, rbp, rbx, rsi;

kernel/freeze.c

@@ -7,6 +7,7 @@
 #include <ihk/monitor.h>
 #include <init.h>
 
+extern int nmi_mode;
 extern void mod_nmi_ctx(void *, void(*)());
 extern void lapic_ack();
 extern void __freeze();
@@ -70,3 +71,48 @@ freeze_thaw(void *nmi_ctx)
 	}
 	return 0;
 }
+
+extern void arch_save_panic_regs(void *irq_regs);
+extern void arch_clear_panic(void);
+
+void
+multi_nm_interrupt_handler(void *irq_regs)
+{
+	dkprintf("%s: ...\n", __func__);
+	switch (nmi_mode) {
+	case 1:
+	case 2:
+		/* mode == 1 or 2, for FREEZER NMI */
+		dkprintf("%s: freeze mode NMI catch. (nmi_mode=%d)\n",
+			 __func__, nmi_mode);
+		freeze_thaw(NULL);
+		break;
+
+	case 0:
+		/* mode == 0, for MEMDUMP NMI */
+		arch_save_panic_regs(irq_regs);
+		ihk_mc_query_mem_areas();
+		/* memdump-nmi is halted McKernel, break is unnecessary. */
+		/* fall through */
+	case 3:
+		/* mode == 3, for SHUTDOWN-WAIT NMI */
+		kprintf("%s: STOP\n", __func__);
+		while (nmi_mode != 4)
+			cpu_halt();
+		break;
+
+	case 4:
+		/* mode == 4, continue NMI */
+		arch_clear_panic();
+		if (!ihk_mc_get_processor_id()) {
+			ihk_mc_clear_dump_page_completion();
+		}
+		kprintf("%s: RESUME, nmi_mode: %d\n", __func__, nmi_mode);
+		break;
+
+	default:
+		ekprintf("%s: Unknown nmi-mode(%d) detected.\n",
+			 __func__, nmi_mode);
+		break;
+	}
+}

kernel/mem.c

@@ -44,6 +44,7 @@
 #include <limits.h>
 #include <sysfs.h>
 #include <ihk/debug.h>
+#include <bootparam.h>
 
 //#define DEBUG_PRINT_MEM
 
@@ -114,29 +115,11 @@ struct pagealloc_track_entry {
 	ihk_spinlock_t addr_list_lock;
 };
 
-struct ihk_dump_page {
-	unsigned long start;
-	unsigned long map_count;
-	unsigned long map[0];
-};
-
-struct ihk_dump_page_set {
-	volatile unsigned int completion_flag;
-	unsigned int count;
-	unsigned long page_size;
-	unsigned long phy_page;
-};
-
 struct dump_pase_info {
 	struct ihk_dump_page_set *dump_page_set;
 	struct ihk_dump_page *dump_pages;
 };
 
-#define IHK_DUMP_PAGE_SET_INCOMPLETE 0
-#define IHK_DUMP_PAGE_SET_COMPLETED  1
-#define DUMP_LEVEL_ALL 0
-#define DUMP_LEVEL_USER_UNUSED_EXCLUDE 24
-
 /** Get the index in the map array */
 #define MAP_INDEX(n)    ((n) >> 6)
 /** Get the bit number in a map element */
@@ -2512,10 +2495,13 @@ void ihk_mc_query_mem_areas(void){
 	struct ihk_dump_page_set *dump_page_set;
 	struct dump_pase_info dump_pase_info;
 
-	/* Performed only for CPU 0 */
+	/*
+	 * Performed only on the last CPU to make sure
+	 * all other cores are already stopped.
+	 */
 	cpu_id = ihk_mc_get_processor_id();
 
-	if (0 != cpu_id)
+	if (num_processors - 1 != cpu_id)
 		return;
 
 	dump_page_set = ihk_mc_get_dump_page_set();
@@ -2534,10 +2520,19 @@ void ihk_mc_query_mem_areas(void){
 	}
 
 	dump_page_set->completion_flag = IHK_DUMP_PAGE_SET_COMPLETED;
+	dkprintf("%s: IHK_DUMP_PAGE_SET_COMPLETED\n", __func__);
 
 	return;
 }
 
+void ihk_mc_clear_dump_page_completion(void)
+{
+	struct ihk_dump_page_set *dump_page_set;
+
+	dump_page_set = ihk_mc_get_dump_page_set();
+	dump_page_set->completion_flag = IHK_DUMP_PAGE_SET_INCOMPLETE;
+}
+
 void ihk_mc_query_mem_user_page(void *dump_pase_info) {
 
 	struct resource_set *rset = cpu_local_var(resource_set);

lib/include/ihk/mm.h

@@ -251,6 +251,7 @@ unsigned int ihk_mc_get_dump_level(void);
 struct ihk_dump_page_set *ihk_mc_get_dump_page_set(void);
 struct ihk_dump_page *ihk_mc_get_dump_page(void);
 void ihk_mc_query_mem_areas(void);
+void ihk_mc_clear_dump_page_completion(void);
 void ihk_mc_query_mem_user_page(void *dump_page_set);
 void ihk_mc_query_mem_free_page(void *dump_page_set);
 int ihk_mc_chk_page_address(pte_t mem_addr);