gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IPI: use logical CPU ids in ihk_mc_interrupt_cpu()

Browse Source 
Also make remote TLB invalidation arch independent,
removes POSTK_DEBUG_ARCH_DEP_8.

Change-Id: I2b0fbcfa2bfe5da07607863e3e772d8e892e8525
This commit is contained in:
Balazs Gerofi
2019-03-19 16:33:28 +09:00
committed by Dominique Martinet
parent 2cee82673b
commit de042b2cb2
9 changed files with 29 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
arch/arm64/kernel/cpu.c
View File

@@ -32,6 +32,7 @@
#include <ihk/debug.h> #include <ihk/debug.h>
#include <hwcap.h> #include <hwcap.h>
#include <virt.h> #include <virt.h>
#include <init.h>
//#define DEBUG_PRINT_CPU //#define DEBUG_PRINT_CPU
@@ -1346,11 +1347,15 @@ int ihk_mc_arch_get_special_register(enum ihk_asr_type type,
} }
/*@ /*@
@ requires \valid_apicid(cpu); // valid APIC ID or not @ requires \valid_cpuid(cpu); // valid CPU logical ID
@ ensures \result == 0 @ ensures \result == 0
@*/ @*/
int ihk_mc_interrupt_cpu(int cpu, int vector) int ihk_mc_interrupt_cpu(int cpu, int vector)
{ {
if (cpu < 0 || cpu >= num_processors) {
kprintf("%s: invalid CPU id: %d\n", __func__, cpu);
return -1;
}
dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu); dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu);
(*arm64_issue_ipi)(cpu, vector); (*arm64_issue_ipi)(cpu, vector);
return 0; return 0;

38
arch/arm64/kernel/memory.c
View File

@@ -3715,44 +3715,6 @@ translation_table_t* get_translation_table_as_paddr(const struct page_table *pt)
return pt->tt_pa; return pt->tt_pa;
} }
#ifdef POSTK_DEBUG_ARCH_DEP_8
void remote_flush_tlb_cpumask(struct process_vm *vm,
unsigned long addr, int cpu_id)
{
unsigned long cpu;
cpu_set_t _cpu_set;
int flush_ind;
if (addr) {
flush_ind = (addr >> PAGE_SHIFT) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE;
}
/* Zero address denotes full TLB flush */
else {
/* Random.. */
flush_ind = (rdtsc()) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE;
}
/* Take a copy of the cpu set so that we don't hold the lock
* all the way while interrupting other cores */
ihk_mc_spinlock_lock_noirq(&vm->address_space->cpu_set_lock);
memcpy(&_cpu_set, &vm->address_space->cpu_set, sizeof(cpu_set_t));
ihk_mc_spinlock_unlock_noirq(&vm->address_space->cpu_set_lock);
/* Loop through CPUs in this address space and interrupt them for
* TLB flush on the specified address */
for_each_set_bit(cpu, (const unsigned long*)&_cpu_set.__bits, CPU_SETSIZE) {
if (ihk_mc_get_processor_id() == cpu)
continue;
dkprintf("remote_flush_tlb_cpumask: flush_ind: %d, addr: 0x%lX, interrupting cpu: %d\n",
flush_ind, addr, cpu);
ihk_mc_interrupt_cpu(cpu,
ihk_mc_get_vector(flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START));
}
}
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
void arch_adjust_allocate_page_size(struct page_table *pt, void arch_adjust_allocate_page_size(struct page_table *pt,
uintptr_t fault_addr, uintptr_t fault_addr,
pte_t *ptep, pte_t *ptep,

12
arch/x86_64/kernel/cpu.c
View File

@@ -1601,14 +1601,18 @@ int ihk_mc_arch_get_special_register(enum ihk_asr_type type,
} }
/*@ /*@
@ requires \valid_apicid(cpu); // valid APIC ID or not @ requires \valid_cpuid(cpu); // valid CPU logical ID
@ ensures \result == 0 @ ensures \result == 0
@*/ @*/
int ihk_mc_interrupt_cpu(int cpu, int vector) int ihk_mc_interrupt_cpu(int cpu, int vector)
{ {
if (cpu < 0 || cpu >= num_processors) {
kprintf("%s: invalid CPU id: %d\n", __func__, cpu);
return -1;
}
dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu); dkprintf("[%d] ihk_mc_interrupt_cpu: %d\n", ihk_mc_get_processor_id(), cpu);
x86_issue_ipi(cpu, vector); x86_issue_ipi(get_x86_cpu_local_variable(cpu)->apic_id, vector);
return 0; return 0;
} }
@@ -2095,9 +2099,7 @@ int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg)
ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock, ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock,
irq_flags); irq_flags);
ihk_mc_interrupt_cpu( ihk_mc_interrupt_cpu(cpu, LOCAL_SMP_FUNC_CALL_VECTOR);
get_x86_cpu_local_variable(cpu)->apic_id,
LOCAL_SMP_FUNC_CALL_VECTOR);
++cpu_index; ++cpu_index;
} }

103
arch/x86_64/kernel/memory.c
View File

@@ -1354,109 +1354,6 @@ struct clear_range_args {
int max_nr_addr; int max_nr_addr;
}; };
#ifdef POSTK_DEBUG_ARCH_DEP_8
void remote_flush_tlb_cpumask(struct process_vm *vm,
unsigned long addr, int cpu_id)
{
unsigned long __addr = addr;
return remote_flush_tlb_array_cpumask(vm, &__addr, 1, cpu_id);
}
void remote_flush_tlb_array_cpumask(struct process_vm *vm,
unsigned long *addr,
int nr_addr,
int cpu_id)
{
unsigned long cpu;
int flush_ind;
struct tlb_flush_entry *flush_entry;
cpu_set_t _cpu_set;
if (addr[0]) {
flush_ind = (addr[0] >> PAGE_SHIFT) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE;
}
/* Zero address denotes full TLB flush */
else {
/* Random.. */
flush_ind = (rdtsc()) % IHK_TLB_FLUSH_IRQ_VECTOR_SIZE;
}
flush_entry = &tlb_flush_vector[flush_ind];
/* Take a copy of the cpu set so that we don't hold the lock
* all the way while interrupting other cores */
ihk_mc_spinlock_lock_noirq(&vm->address_space->cpu_set_lock);
memcpy(&_cpu_set, &vm->address_space->cpu_set, sizeof(cpu_set_t));
ihk_mc_spinlock_unlock_noirq(&vm->address_space->cpu_set_lock);
dkprintf("trying to aquire flush_entry->lock flush_ind: %d\n", flush_ind);
ihk_mc_spinlock_lock_noirq(&flush_entry->lock);
flush_entry->vm = vm;
flush_entry->addr = addr;
flush_entry->nr_addr = nr_addr;
ihk_atomic_set(&flush_entry->pending, 0);
dkprintf("lock aquired, iterating cpu mask.. flush_ind: %d\n", flush_ind);
/* Loop through CPUs in this address space and interrupt them for
* TLB flush on the specified address */
for_each_set_bit(cpu, (const unsigned long*)&_cpu_set.__bits, CPU_SETSIZE) {
if (ihk_mc_get_processor_id() == cpu)
continue;
ihk_atomic_inc(&flush_entry->pending);
dkprintf("remote_flush_tlb_cpumask: flush_ind: %d, addr: 0x%lX, interrupting cpu: %d\n",
flush_ind, addr, cpu);
#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */
/* TODO(pka_idke) Interim support */
ihk_mc_interrupt_cpu(cpu,
ihk_mc_get_vector(flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START));
#else /* POSTK_DEBUG_ARCH_DEP_8 */
ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(cpu)->apic_id,
flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START);
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
}
#ifdef DEBUG_IC_TLB
{
unsigned long tsc;
tsc = rdtsc() + 12884901888; /* 1.2GHz =>10 sec */
#endif
if (flush_entry->addr[0]) {
int i;
for (i = 0; i < flush_entry->nr_addr; ++i) {
flush_tlb_single(flush_entry->addr[i] & PAGE_MASK);
}
}
/* Zero address denotes full TLB flush */
else {
flush_tlb();
}
/* Wait for all cores */
while (ihk_atomic_read(&flush_entry->pending) != 0) {
cpu_pause();
#ifdef DEBUG_IC_TLB
if (rdtsc() > tsc) {
kprintf("waited 10 secs for remote TLB!! -> panic_all()\n");
panic_all_cores("waited 10 secs for remote TLB!!\n");
}
#endif
}
#ifdef DEBUG_IC_TLB
}
#endif
ihk_mc_spinlock_unlock_noirq(&flush_entry->lock);
}
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
static void remote_flush_tlb_add_addr(struct clear_range_args *args, static void remote_flush_tlb_add_addr(struct clear_range_args *args,
unsigned long addr) unsigned long addr)
{ {

3
arch/x86_64/kernel/syscall.c
View File

@@ -1570,7 +1570,8 @@ done:
if (thread != tthread) { if (thread != tthread) {
dkprintf("do_kill,ipi,pid=%d,cpu_id=%d\n", dkprintf("do_kill,ipi,pid=%d,cpu_id=%d\n",
tproc->pid, tthread->cpu_id); tproc->pid, tthread->cpu_id);
ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(tthread->cpu_id)->apic_id, 0xd0); ihk_mc_interrupt_cpu(tthread->cpu_id,
ihk_mc_get_vector(IHK_GV_IKC));
} }
if (status != PS_RUNNING) { if (status != PS_RUNNING) {

2
ihk

Submodule ihk updated: 0681d7a055...70adc3dcfd

8
kernel/mem.c
View File

@@ -996,7 +996,6 @@ void coredump(struct thread *thread, void *regs)
freecore(&coretable); freecore(&coretable);
} }
#ifndef POSTK_DEBUG_ARCH_DEP_8
void remote_flush_tlb_cpumask(struct process_vm *vm, void remote_flush_tlb_cpumask(struct process_vm *vm,
unsigned long addr, int cpu_id) unsigned long addr, int cpu_id)
{ {
@@ -1053,14 +1052,8 @@ void remote_flush_tlb_array_cpumask(struct process_vm *vm,
dkprintf("remote_flush_tlb_cpumask: flush_ind: %d, addr: 0x%lX, interrupting cpu: %d\n", dkprintf("remote_flush_tlb_cpumask: flush_ind: %d, addr: 0x%lX, interrupting cpu: %d\n",
flush_ind, addr, cpu); flush_ind, addr, cpu);
#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */
/* TODO(pka_idke) Interim support */
ihk_mc_interrupt_cpu(cpu, ihk_mc_interrupt_cpu(cpu,
ihk_mc_get_vector(flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START)); ihk_mc_get_vector(flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START));
#else /* POSTK_DEBUG_ARCH_DEP_8 */
ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(cpu)->apic_id,
flush_ind + IHK_TLB_FLUSH_IRQ_VECTOR_START);
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
} }
#ifdef DEBUG_IC_TLB #ifdef DEBUG_IC_TLB
@@ -1097,7 +1090,6 @@ void remote_flush_tlb_array_cpumask(struct process_vm *vm,
ihk_mc_spinlock_unlock_noirq(&flush_entry->lock); ihk_mc_spinlock_unlock_noirq(&flush_entry->lock);
} }
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
void tlb_flush_handler(int vector) void tlb_flush_handler(int vector)
{ {

33
kernel/process.c
View File

@@ -3518,14 +3518,8 @@ int __sched_wakeup_thread(struct thread *thread,
if (!status && (thread->cpu_id != ihk_mc_get_processor_id())) { if (!status && (thread->cpu_id != ihk_mc_get_processor_id())) {
dkprintf("%s: issuing IPI, thread->cpu_id=%d\n", dkprintf("%s: issuing IPI, thread->cpu_id=%d\n",
__FUNCTION__, thread->cpu_id); __FUNCTION__, thread->cpu_id);
#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */
ihk_mc_interrupt_cpu(thread->cpu_id, ihk_mc_interrupt_cpu(thread->cpu_id,
ihk_mc_get_vector(IHK_GV_IKC)); ihk_mc_get_vector(IHK_GV_IKC));
#else /* POSTK_DEBUG_ARCH_DEP_8 */
ihk_mc_interrupt_cpu(
get_x86_cpu_local_variable(thread->cpu_id)->apic_id,
0xd1);
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
} }
return status; return status;
@@ -3579,15 +3573,11 @@ void sched_request_migrate(int cpu_id, struct thread *thread)
v->status = CPU_STATUS_RUNNING; v->status = CPU_STATUS_RUNNING;
ihk_mc_spinlock_unlock(&v->runq_lock, irqstate); ihk_mc_spinlock_unlock(&v->runq_lock, irqstate);
#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ if (cpu_id != ihk_mc_get_processor_id()) {
if (cpu_id != ihk_mc_get_processor_id()) /* Kick scheduler */
ihk_mc_interrupt_cpu(/* Kick scheduler */ ihk_mc_interrupt_cpu(thread->cpu_id,
thread->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); ihk_mc_get_vector(IHK_GV_IKC));
#else /* POSTK_DEBUG_ARCH_DEP_8 */ }
if (cpu_id != ihk_mc_get_processor_id())
ihk_mc_interrupt_cpu(/* Kick scheduler */
get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1);
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
dkprintf("%s: tid: %d -> cpu: %d\n", dkprintf("%s: tid: %d -> cpu: %d\n",
__FUNCTION__, thread->tid, cpu_id); __FUNCTION__, thread->tid, cpu_id);
@@ -3640,15 +3630,10 @@ void runq_add_thread(struct thread *thread, int cpu_id)
#endif #endif
/* Kick scheduler */ /* Kick scheduler */
#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ if (cpu_id != ihk_mc_get_processor_id()) {
if (cpu_id != ihk_mc_get_processor_id()) ihk_mc_interrupt_cpu(thread->cpu_id,
ihk_mc_interrupt_cpu( ihk_mc_get_vector(IHK_GV_IKC));
thread->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); }
#else /* POSTK_DEBUG_ARCH_DEP_8 */
if (cpu_id != ihk_mc_get_processor_id())
ihk_mc_interrupt_cpu(
get_x86_cpu_local_variable(cpu_id)->apic_id, 0xd1);
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
} }
/* NOTE: shouldn't remove a running process! */ /* NOTE: shouldn't remove a running process! */

14
kernel/syscall.c
View File

@@ -5988,11 +5988,8 @@ SYSCALL_DECLARE(getrusage)
child->status == PS_RUNNING && child->status == PS_RUNNING &&
!child->in_kernel){ !child->in_kernel){
child->times_update = 0; child->times_update = 0;
#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ ihk_mc_interrupt_cpu(child->cpu_id,
ihk_mc_interrupt_cpu(child->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); ihk_mc_get_vector(IHK_GV_IKC));
#else /* POSTK_DEBUG_ARCH_DEP_8 */
ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(child->cpu_id)->apic_id, 0xd1);
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
} }
else else
child->times_update = 1; child->times_update = 1;
@@ -7231,11 +7228,8 @@ SYSCALL_DECLARE(clock_gettime)
child->status == PS_RUNNING && child->status == PS_RUNNING &&
!child->in_kernel){ !child->in_kernel){
child->times_update = 0; child->times_update = 0;
#ifdef POSTK_DEBUG_ARCH_DEP_8 /* arch depend hide */ ihk_mc_interrupt_cpu(child->cpu_id,
ihk_mc_interrupt_cpu(child->cpu_id, ihk_mc_get_vector(IHK_GV_IKC)); ihk_mc_get_vector(IHK_GV_IKC));
#else /* POSTK_DEBUG_ARCH_DEP_8 */
ihk_mc_interrupt_cpu(get_x86_cpu_local_variable(child->cpu_id)->apic_id, 0xd1);
#endif /* POSTK_DEBUG_ARCH_DEP_8 */
} }
} }
ats.tv_sec = proc->utime.tv_sec; ats.tv_sec = proc->utime.tv_sec;