diff --git a/arch/arm64/kernel/cpu.c b/arch/arm64/kernel/cpu.c index bb1767ee..2b5a0896 100644 --- a/arch/arm64/kernel/cpu.c +++ b/arch/arm64/kernel/cpu.c @@ -1972,15 +1972,15 @@ int arch_cpu_read_write_register( return ret; } -int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg) -{ - /* TODO: skeleton for smp_call_func */ - return -1; -} - void arch_flush_icache_all(void) { asm("ic ialluis"); dsb(ish); } + +int ihk_mc_get_smp_handler_irq(void) +{ + return LOCAL_SMP_FUNC_CALL_VECTOR; +} + /*** end of file ***/ diff --git a/arch/arm64/kernel/include/irq.h b/arch/arm64/kernel/include/irq.h index 7d43216a..abc67012 100644 --- a/arch/arm64/kernel/include/irq.h +++ b/arch/arm64/kernel/include/irq.h @@ -17,6 +17,7 @@ #define INTRID_STACK_TRACE 5 #define INTRID_MULTI_INTR 6 #define INTRID_MULTI_NMI 7 +#define LOCAL_SMP_FUNC_CALL_VECTOR 1 /* same as IKC */ /* use PPI interrupt number */ #define INTRID_PERF_OVF 23 diff --git a/arch/x86_64/kernel/cpu.c b/arch/x86_64/kernel/cpu.c index d3b3157b..8d9ea707 100644 --- a/arch/x86_64/kernel/cpu.c +++ b/arch/x86_64/kernel/cpu.c @@ -80,7 +80,11 @@ static void (*lapic_icr_write)(unsigned int h, unsigned int l); static void (*lapic_wait_icr_idle)(void); void (*x86_issue_ipi)(unsigned int apicid, unsigned int low); int running_on_kvm(void); -static void smp_func_call_handler(void); +void smp_func_call_handler(void); +int ihk_mc_get_smp_handler_irq(void) +{ + return LOCAL_SMP_FUNC_CALL_VECTOR; +} void init_processors_local(int max_id); void assign_processor_id(void); @@ -2170,144 +2174,6 @@ int arch_cpu_read_write_register( return 0; } -/* - * Generic remote CPU function invocation facility. - */ -static void smp_func_call_handler(void) -{ - int irq_flags; - struct smp_func_call_request *req; - int reqs_left; - -reiterate: - req = NULL; - reqs_left = 0; - - irq_flags = ihk_mc_spinlock_lock( - &cpu_local_var(smp_func_req_lock)); - - /* Take requests one-by-one */ - if (!list_empty(&cpu_local_var(smp_func_req_list))) { - req = list_first_entry(&cpu_local_var(smp_func_req_list), - struct smp_func_call_request, list); - list_del(&req->list); - - reqs_left = !list_empty(&cpu_local_var(smp_func_req_list)); - } - - ihk_mc_spinlock_unlock(&cpu_local_var(smp_func_req_lock), - irq_flags); - - if (req) { - req->ret = req->sfcd->func(req->cpu_index, - req->sfcd->nr_cpus, req->sfcd->arg); - ihk_atomic_dec(&req->sfcd->cpus_left); - } - - if (reqs_left) - goto reiterate; -} - -int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg) -{ - int cpu, nr_cpus = 0; - int cpu_index = 0; - int this_cpu_index = 0; - struct smp_func_call_data sfcd; - struct smp_func_call_request *reqs; - int ret = 0; - int call_on_this_cpu = 0; - cpu_set_t cpu_set; - - /* Sanity checks */ - if (!__cpu_set || !__func) { - return -EINVAL; - } - - /* Make sure it won't change in between */ - cpu_set = *__cpu_set; - - for_each_set_bit(cpu, (unsigned long *)&cpu_set, - sizeof(cpu_set) * BITS_PER_BYTE) { - - if (cpu == ihk_mc_get_processor_id()) { - call_on_this_cpu = 1; - } - ++nr_cpus; - } - - if (!nr_cpus) { - return -EINVAL; - } - - reqs = kmalloc(sizeof(*reqs) * nr_cpus, IHK_MC_AP_NOWAIT); - if (!reqs) { - ret = -ENOMEM; - goto free_out; - } - - sfcd.nr_cpus = nr_cpus; - sfcd.func = __func; - sfcd.arg = __arg; - ihk_atomic_set(&sfcd.cpus_left, - call_on_this_cpu ? nr_cpus - 1 : nr_cpus); - - /* Add requests and send IPIs */ - cpu_index = 0; - for_each_set_bit(cpu, (unsigned long *)&cpu_set, - sizeof(cpu_set) * BITS_PER_BYTE) { - unsigned long irq_flags; - - reqs[cpu_index].cpu_index = cpu_index; - reqs[cpu_index].ret = 0; - - if (cpu == ihk_mc_get_processor_id()) { - this_cpu_index = cpu_index; - ++cpu_index; - continue; - } - - reqs[cpu_index].sfcd = &sfcd; - - irq_flags = - ihk_mc_spinlock_lock(&get_cpu_local_var(cpu)->smp_func_req_lock); - list_add_tail(&reqs[cpu_index].list, - &get_cpu_local_var(cpu)->smp_func_req_list); - ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock, - irq_flags); - - ihk_mc_interrupt_cpu(cpu, LOCAL_SMP_FUNC_CALL_VECTOR); - - ++cpu_index; - } - - /* Is this CPU involved? */ - if (call_on_this_cpu) { - reqs[this_cpu_index].ret = - __func(this_cpu_index, nr_cpus, __arg); - } - - /* Wait for the rest of the CPUs */ - while (ihk_atomic_read(&sfcd.cpus_left) > 0) { - cpu_pause(); - } - - /* Check return values, if error, report the first non-zero */ - for (cpu_index = 0; cpu_index < nr_cpus; ++cpu_index) { - if (reqs[cpu_index].ret != 0) { - ret = reqs[cpu_index].ret; - goto free_out; - } - } - - ret = 0; - -free_out: - kfree(reqs); - - return ret; -} - extern int nmi_mode; extern long freeze_thaw(void *nmi_ctx); diff --git a/ihk b/ihk index a66df509..823ede5e 160000 --- a/ihk +++ b/ihk @@ -1 +1 @@ -Subproject commit a66df50991d1cbdac7d74cba77906719d913bf8f +Subproject commit 823ede5e9ad7a0effd418c5077b373b2f90dc792 diff --git a/kernel/ap.c b/kernel/ap.c index 926d1a73..57184d1e 100644 --- a/kernel/ap.c +++ b/kernel/ap.c @@ -267,3 +267,154 @@ cpu_sysfs_setup(void) return; } /* cpu_sysfs_setup() */ + +/* + * Generic remote CPU function invocation facility. + */ +void smp_func_call_handler(void) +{ + unsigned long irq_flags; + struct smp_func_call_request *req; + int reqs_left; + +reiterate: + req = NULL; + reqs_left = 0; + + irq_flags = ihk_mc_spinlock_lock( + &cpu_local_var(smp_func_req_lock)); + + /* Take requests one-by-one */ + if (!list_empty(&cpu_local_var(smp_func_req_list))) { + req = list_first_entry(&cpu_local_var(smp_func_req_list), + struct smp_func_call_request, list); + list_del(&req->list); + + reqs_left = !list_empty(&cpu_local_var(smp_func_req_list)); + } + + ihk_mc_spinlock_unlock(&cpu_local_var(smp_func_req_lock), + irq_flags); + + if (req) { + req->ret = req->sfcd->func(req->cpu_index, + req->sfcd->nr_cpus, req->sfcd->arg); + ihk_atomic_dec(&req->sfcd->cpus_left); + } + + if (reqs_left) + goto reiterate; +} + +int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg) +{ + int cpu, nr_cpus = 0; + int cpu_index = 0; + int this_cpu_index = 0; + struct smp_func_call_data sfcd; + struct smp_func_call_request *reqs; + int ret = 0; + int call_on_this_cpu = 0; + cpu_set_t cpu_set; + int max_nr_cpus = 4; + + /* Sanity checks */ + if (!__cpu_set || !__func) { + return -EINVAL; + } + + /* Make sure it won't change in between */ + cpu_set = *__cpu_set; + + for_each_set_bit(cpu, (unsigned long *)&cpu_set, + sizeof(cpu_set) * BITS_PER_BYTE) { + + if (cpu == ihk_mc_get_processor_id()) { + call_on_this_cpu = 1; + } + ++nr_cpus; + + if (nr_cpus == max_nr_cpus) + break; + } + + if (!nr_cpus) { + return -EINVAL; + } + + reqs = kmalloc(sizeof(*reqs) * nr_cpus, IHK_MC_AP_NOWAIT); + if (!reqs) { + ret = -ENOMEM; + goto free_out; + } + + kprintf("%s: interrupting %d CPUs for SMP call..\n", __func__, nr_cpus); + sfcd.nr_cpus = nr_cpus; + sfcd.func = __func; + sfcd.arg = __arg; + ihk_atomic_set(&sfcd.cpus_left, + call_on_this_cpu ? nr_cpus - 1 : nr_cpus); + smp_wmb(); + + /* Add requests and send IPIs */ + cpu_index = 0; + for_each_set_bit(cpu, (unsigned long *)&cpu_set, + sizeof(cpu_set) * BITS_PER_BYTE) { + unsigned long irq_flags; + + reqs[cpu_index].cpu_index = cpu_index; + reqs[cpu_index].ret = 0; + + if (cpu == ihk_mc_get_processor_id()) { + this_cpu_index = cpu_index; + ++cpu_index; + continue; + } + + reqs[cpu_index].sfcd = &sfcd; + + irq_flags = + ihk_mc_spinlock_lock(&get_cpu_local_var(cpu)->smp_func_req_lock); + list_add_tail(&reqs[cpu_index].list, + &get_cpu_local_var(cpu)->smp_func_req_list); + ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock, + irq_flags); + + dkprintf("%s: interrupting IRQ: %d -> CPU: %d\n", __func__, + ihk_mc_get_smp_handler_irq(), cpu); + ihk_mc_interrupt_cpu(cpu, ihk_mc_get_smp_handler_irq()); + + ++cpu_index; + if (cpu_index == max_nr_cpus) + break; + } + + /* Is this CPU involved? */ + if (call_on_this_cpu) { + reqs[this_cpu_index].ret = + __func(this_cpu_index, nr_cpus, __arg); + } + + dkprintf("%s: waiting for remote CPUs..\n", __func__); + /* Wait for the rest of the CPUs */ + while (smp_load_acquire(&sfcd.cpus_left.counter) > 0) { + cpu_pause(); + } + + /* Check return values, if error, report the first non-zero */ + for (cpu_index = 0; cpu_index < nr_cpus; ++cpu_index) { + if (reqs[cpu_index].ret != 0) { + ret = reqs[cpu_index].ret; + goto free_out; + } + } + + kprintf("%s: all CPUs finished SMP call successfully\n", __func__); + ret = 0; + +free_out: + kfree(reqs); + + return ret; +} + diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h index 10ebc7ba..dd8ef7ed 100644 --- a/lib/include/ihk/cpu.h +++ b/lib/include/ihk/cpu.h @@ -173,4 +173,7 @@ struct cpu_mapping; int arch_get_cpu_mapping(struct cpu_mapping **buf, int *nelemsp); int ihk_mc_ikc_arch_issue_host_ipi(int cpu, int vector); +void smp_func_call_handler(void); +int ihk_mc_get_smp_handler_irq(void); + #endif