diff --git a/arch/x86/kernel/context.S b/arch/x86/kernel/context.S index 0551f76e..c94216d9 100644 --- a/arch/x86/kernel/context.S +++ b/arch/x86/kernel/context.S @@ -10,7 +10,7 @@ * HISTORY */ -#define X86_CPU_LOCAL_OFFSET_TSS 128 +#define X86_CPU_LOCAL_OFFSET_TSS 176 #define X86_TSS_OFFSET_SP0 4 #define X86_CPU_LOCAL_OFFSET_SP0 \ (X86_CPU_LOCAL_OFFSET_TSS + X86_TSS_OFFSET_SP0) diff --git a/arch/x86/kernel/cpu.c b/arch/x86/kernel/cpu.c index 3b74bec8..4d329f1e 100644 --- a/arch/x86/kernel/cpu.c +++ b/arch/x86/kernel/cpu.c @@ -103,6 +103,12 @@ static uint64_t gdt[] __attribute__((aligned(16))) = { 0x00aff3000000ffff, /* 56 : USER_DS */ 0x0000890000000067, /* 64 : TSS */ 0, /* (72: TSS) */ + 0, /* 80 */ + 0, /* 88 */ + 0, /* 96 */ + 0, /* 104 */ + 0, /* 112 */ + 0x0000f10000000000, /* 120 : GETCPU */ }; struct tss64 tss __attribute__((aligned(16))); @@ -570,6 +576,7 @@ static void init_smp_processor(void) { struct x86_cpu_local_variables *v; unsigned long tss_addr; + unsigned node_cpu; v = get_x86_this_cpu_local(); tss_addr = (unsigned long)&v->tss; @@ -590,6 +597,9 @@ static void init_smp_processor(void) | (0x89UL << 40) | ((tss_addr & 0xff000000) << 32); v->gdt[GLOBAL_TSS_ENTRY + 1] = (tss_addr >> 32); + node_cpu = v->processor_id; /* assumes NUMA node 0 */ + v->gdt[GETCPU_ENTRY] |= node_cpu; + v->gdt_ptr.size = sizeof(v->gdt) - 1; v->gdt_ptr.address = (unsigned long)v->gdt; @@ -597,6 +607,8 @@ static void init_smp_processor(void) reload_gdt(&v->gdt_ptr); set_kstack((unsigned long)get_x86_this_cpu_kstack()); +#define MSR_IA32_TSC_AUX 0xc0000103 + wrmsr(MSR_IA32_TSC_AUX, node_cpu); } static char *trampoline_va, *first_page_va; diff --git a/arch/x86/kernel/include/arch-memory.h b/arch/x86/kernel/include/arch-memory.h index bdb70dfc..055c6784 100644 --- a/arch/x86/kernel/include/arch-memory.h +++ b/arch/x86/kernel/include/arch-memory.h @@ -22,6 +22,7 @@ #define USER_CS_ENTRY 6 #define USER_DS_ENTRY 7 #define GLOBAL_TSS_ENTRY 8 +#define GETCPU_ENTRY 15 #define KERNEL_CS (KERNEL_CS_ENTRY * 8) #define KERNEL_DS (KERNEL_DS_ENTRY * 8) diff --git a/arch/x86/kernel/include/cpulocal.h b/arch/x86/kernel/include/cpulocal.h index 76c2fbbe..8c4eb1e7 100644 --- a/arch/x86/kernel/include/cpulocal.h +++ b/arch/x86/kernel/include/cpulocal.h @@ -22,7 +22,7 @@ * - 4096 : kernel stack */ -#define X86_CPU_LOCAL_OFFSET_TSS 128 +#define X86_CPU_LOCAL_OFFSET_TSS 176 #define X86_CPU_LOCAL_OFFSET_KSTACK 16 #define X86_CPU_LOCAL_OFFSET_USTACK 24 @@ -39,13 +39,13 @@ struct x86_cpu_local_variables { struct x86_desc_ptr gdt_ptr; unsigned short pad[3]; /* 48 */ - uint64_t gdt[10]; -/* 128 */ + uint64_t gdt[16]; +/* 176 */ struct tss64 tss; -/* 232 */ +/* 280 */ unsigned long paniced; uint64_t panic_regs[21]; -/* 408 */ +/* 456 */ } __attribute__((packed)); struct x86_cpu_local_variables *get_x86_cpu_local_variable(int id); diff --git a/arch/x86/kernel/interrupt.S b/arch/x86/kernel/interrupt.S index 74f88fd7..3537468b 100644 --- a/arch/x86/kernel/interrupt.S +++ b/arch/x86/kernel/interrupt.S @@ -13,7 +13,7 @@ * 2013/?? - bgerofi + shimosawa: handle rsp correctly for nested interrupts */ -#define X86_CPU_LOCAL_OFFSET_TSS 128 +#define X86_CPU_LOCAL_OFFSET_TSS 176 #define X86_TSS_OFFSET_SP0 4 #define X86_CPU_LOCAL_OFFSET_SP0 \ (X86_CPU_LOCAL_OFFSET_TSS + X86_TSS_OFFSET_SP0) diff --git a/arch/x86/kernel/syscall.c b/arch/x86/kernel/syscall.c index e11b6ae3..248fd9fb 100644 --- a/arch/x86/kernel/syscall.c +++ b/arch/x86/kernel/syscall.c @@ -216,7 +216,9 @@ SYSCALL_DECLARE(rt_sigreturn) struct sigsp ksigsp; struct sigsp *sigsp; - asm("movq %%gs:132, %0" : "=r" (regs)); + asm ("movq %%gs:(%1),%0" + : "=r"(regs) + : "r"(offsetof(struct x86_cpu_local_variables, tss.rsp0))); --regs; sigsp = (struct sigsp *)regs->gpr.rsp; @@ -680,7 +682,9 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi } if(regs == NULL){ /* call from syscall */ - asm("movq %%gs:132, %0" : "=r" (regs)); + asm ("movq %%gs:(%1),%0" + : "=r"(regs) + : "r"(offsetof(struct x86_cpu_local_variables, tss.rsp0))); --regs; } else{