smp: make smp_call_func() arch independent

Change-Id: Ib60604ceb3274b173bd7f96cf57c8c35c1889e44

arch/arm64/kernel/cpu.c

@@ -1972,15 +1972,15 @@ int arch_cpu_read_write_register(
 	return ret;
 }
 
-int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg)
-{
-	/* TODO: skeleton for smp_call_func */
-	return -1;
-}
-
 void arch_flush_icache_all(void)
 {
 	asm("ic	ialluis");
 	dsb(ish);
 }
+
+int ihk_mc_get_smp_handler_irq(void)
+{
+	return LOCAL_SMP_FUNC_CALL_VECTOR;
+}
+
 /*** end of file ***/

arch/arm64/kernel/include/irq.h

@@ -17,6 +17,7 @@
 #define INTRID_STACK_TRACE	5
 #define INTRID_MULTI_INTR	6
 #define INTRID_MULTI_NMI	7
+#define LOCAL_SMP_FUNC_CALL_VECTOR   1 /* same as IKC */
 
 /* use PPI interrupt number */
 #define INTRID_PERF_OVF		23

arch/x86_64/kernel/cpu.c

@@ -80,7 +80,11 @@ static void (*lapic_icr_write)(unsigned int h, unsigned int l);
 static void (*lapic_wait_icr_idle)(void);
 void (*x86_issue_ipi)(unsigned int apicid, unsigned int low);
 int running_on_kvm(void);
-static void smp_func_call_handler(void);
+void smp_func_call_handler(void);
+int ihk_mc_get_smp_handler_irq(void)
+{
+	return LOCAL_SMP_FUNC_CALL_VECTOR;
+}
 
 void init_processors_local(int max_id);
 void assign_processor_id(void);
@@ -2170,144 +2174,6 @@ int arch_cpu_read_write_register(
 	return 0;
 }
 
-/*
- * Generic remote CPU function invocation facility.
- */
-static void smp_func_call_handler(void)
-{
-	int irq_flags;
-	struct smp_func_call_request *req;
-	int reqs_left;
-
-reiterate:
-	req = NULL;
-	reqs_left = 0;
-
-	irq_flags = ihk_mc_spinlock_lock(
-			&cpu_local_var(smp_func_req_lock));
-
-	/* Take requests one-by-one */
-	if (!list_empty(&cpu_local_var(smp_func_req_list))) {
-		req = list_first_entry(&cpu_local_var(smp_func_req_list),
-			struct smp_func_call_request, list);
-		list_del(&req->list);
-
-		reqs_left = !list_empty(&cpu_local_var(smp_func_req_list));
-	}
-
-	ihk_mc_spinlock_unlock(&cpu_local_var(smp_func_req_lock),
-			irq_flags);
-
-	if (req) {
-		req->ret = req->sfcd->func(req->cpu_index,
-				req->sfcd->nr_cpus, req->sfcd->arg);
-		ihk_atomic_dec(&req->sfcd->cpus_left);
-	}
-
-	if (reqs_left)
-		goto reiterate;
-}
-
-int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg)
-{
-	int cpu, nr_cpus = 0;
-	int cpu_index = 0;
-	int this_cpu_index = 0;
-	struct smp_func_call_data sfcd;
-	struct smp_func_call_request *reqs;
-	int ret = 0;
-	int call_on_this_cpu = 0;
-	cpu_set_t cpu_set;
-
-	/* Sanity checks */
-	if (!__cpu_set || !__func) {
-		return -EINVAL;
-	}
-
-	/* Make sure it won't change in between */
-	cpu_set = *__cpu_set;
-
-	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
-			sizeof(cpu_set) * BITS_PER_BYTE) {
-
-		if (cpu == ihk_mc_get_processor_id()) {
-			call_on_this_cpu = 1;
-		}
-		++nr_cpus;
-	}
-
-	if (!nr_cpus) {
-		return -EINVAL;
-	}
-
-	reqs = kmalloc(sizeof(*reqs) * nr_cpus, IHK_MC_AP_NOWAIT);
-	if (!reqs) {
-		ret = -ENOMEM;
-		goto free_out;
-	}
-
-	sfcd.nr_cpus = nr_cpus;
-	sfcd.func = __func;
-	sfcd.arg = __arg;
-	ihk_atomic_set(&sfcd.cpus_left,
-			call_on_this_cpu ? nr_cpus - 1 : nr_cpus);
-
-	/* Add requests and send IPIs */
-	cpu_index = 0;
-	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
-			sizeof(cpu_set) * BITS_PER_BYTE) {
-		unsigned long irq_flags;
-
-		reqs[cpu_index].cpu_index = cpu_index;
-		reqs[cpu_index].ret = 0;
-
-		if (cpu == ihk_mc_get_processor_id()) {
-			this_cpu_index = cpu_index;
-			++cpu_index;
-			continue;
-		}
-
-		reqs[cpu_index].sfcd = &sfcd;
-
-		irq_flags =
-			ihk_mc_spinlock_lock(&get_cpu_local_var(cpu)->smp_func_req_lock);
-		list_add_tail(&reqs[cpu_index].list,
-				&get_cpu_local_var(cpu)->smp_func_req_list);
-		ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock,
-			irq_flags);
-
-		ihk_mc_interrupt_cpu(cpu, LOCAL_SMP_FUNC_CALL_VECTOR);
-
-		++cpu_index;
-	}
-
-	/* Is this CPU involved? */
-	if (call_on_this_cpu) {
-		reqs[this_cpu_index].ret =
-			__func(this_cpu_index, nr_cpus, __arg);
-	}
-
-	/* Wait for the rest of the CPUs */
-	while (ihk_atomic_read(&sfcd.cpus_left) > 0) {
-		cpu_pause();
-	}
-
-	/* Check return values, if error, report the first non-zero */
-	for (cpu_index = 0; cpu_index < nr_cpus; ++cpu_index) {
-		if (reqs[cpu_index].ret != 0) {
-			ret = reqs[cpu_index].ret;
-			goto free_out;
-		}
-	}
-
-	ret = 0;
-
-free_out:
-	kfree(reqs);
-
-	return ret;
-}
-
 extern int nmi_mode;
 extern long freeze_thaw(void *nmi_ctx);
 

kernel/ap.c

@@ -267,3 +267,154 @@ cpu_sysfs_setup(void)
 
 	return;
 } /* cpu_sysfs_setup() */
+
+/*
+ * Generic remote CPU function invocation facility.
+ */
+void smp_func_call_handler(void)
+{
+	unsigned long irq_flags;
+	struct smp_func_call_request *req;
+	int reqs_left;
+
+reiterate:
+	req = NULL;
+	reqs_left = 0;
+
+	irq_flags = ihk_mc_spinlock_lock(
+			&cpu_local_var(smp_func_req_lock));
+
+	/* Take requests one-by-one */
+	if (!list_empty(&cpu_local_var(smp_func_req_list))) {
+		req = list_first_entry(&cpu_local_var(smp_func_req_list),
+			struct smp_func_call_request, list);
+		list_del(&req->list);
+
+		reqs_left = !list_empty(&cpu_local_var(smp_func_req_list));
+	}
+
+	ihk_mc_spinlock_unlock(&cpu_local_var(smp_func_req_lock),
+			irq_flags);
+
+	if (req) {
+		req->ret = req->sfcd->func(req->cpu_index,
+				req->sfcd->nr_cpus, req->sfcd->arg);
+		ihk_atomic_dec(&req->sfcd->cpus_left);
+	}
+
+	if (reqs_left)
+		goto reiterate;
+}
+
+int smp_call_func(cpu_set_t *__cpu_set, smp_func_t __func, void *__arg)
+{
+	int cpu, nr_cpus = 0;
+	int cpu_index = 0;
+	int this_cpu_index = 0;
+	struct smp_func_call_data sfcd;
+	struct smp_func_call_request *reqs;
+	int ret = 0;
+	int call_on_this_cpu = 0;
+	cpu_set_t cpu_set;
+	int max_nr_cpus = 4;
+
+	/* Sanity checks */
+	if (!__cpu_set || !__func) {
+		return -EINVAL;
+	}
+
+	/* Make sure it won't change in between */
+	cpu_set = *__cpu_set;
+
+	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
+			sizeof(cpu_set) * BITS_PER_BYTE) {
+
+		if (cpu == ihk_mc_get_processor_id()) {
+			call_on_this_cpu = 1;
+		}
+		++nr_cpus;
+
+		if (nr_cpus == max_nr_cpus)
+			break;
+	}
+
+	if (!nr_cpus) {
+		return -EINVAL;
+	}
+
+	reqs = kmalloc(sizeof(*reqs) * nr_cpus, IHK_MC_AP_NOWAIT);
+	if (!reqs) {
+		ret = -ENOMEM;
+		goto free_out;
+	}
+
+	kprintf("%s: interrupting %d CPUs for SMP call..\n", __func__, nr_cpus);
+	sfcd.nr_cpus = nr_cpus;
+	sfcd.func = __func;
+	sfcd.arg = __arg;
+	ihk_atomic_set(&sfcd.cpus_left,
+			call_on_this_cpu ? nr_cpus - 1 : nr_cpus);
+	smp_wmb();
+
+	/* Add requests and send IPIs */
+	cpu_index = 0;
+	for_each_set_bit(cpu, (unsigned long *)&cpu_set,
+			sizeof(cpu_set) * BITS_PER_BYTE) {
+		unsigned long irq_flags;
+
+		reqs[cpu_index].cpu_index = cpu_index;
+		reqs[cpu_index].ret = 0;
+
+		if (cpu == ihk_mc_get_processor_id()) {
+			this_cpu_index = cpu_index;
+			++cpu_index;
+			continue;
+		}
+
+		reqs[cpu_index].sfcd = &sfcd;
+
+		irq_flags =
+			ihk_mc_spinlock_lock(&get_cpu_local_var(cpu)->smp_func_req_lock);
+		list_add_tail(&reqs[cpu_index].list,
+				&get_cpu_local_var(cpu)->smp_func_req_list);
+		ihk_mc_spinlock_unlock(&get_cpu_local_var(cpu)->smp_func_req_lock,
+			irq_flags);
+
+		dkprintf("%s: interrupting IRQ: %d -> CPU: %d\n", __func__,
+			ihk_mc_get_smp_handler_irq(), cpu);
+		ihk_mc_interrupt_cpu(cpu, ihk_mc_get_smp_handler_irq());
+
+		++cpu_index;
+		if (cpu_index == max_nr_cpus)
+			break;
+	}
+
+	/* Is this CPU involved? */
+	if (call_on_this_cpu) {
+		reqs[this_cpu_index].ret =
+			__func(this_cpu_index, nr_cpus, __arg);
+	}
+
+	dkprintf("%s: waiting for remote CPUs..\n", __func__);
+	/* Wait for the rest of the CPUs */
+	while (smp_load_acquire(&sfcd.cpus_left.counter) > 0) {
+		cpu_pause();
+	}
+
+	/* Check return values, if error, report the first non-zero */
+	for (cpu_index = 0; cpu_index < nr_cpus; ++cpu_index) {
+		if (reqs[cpu_index].ret != 0) {
+			ret = reqs[cpu_index].ret;
+			goto free_out;
+		}
+	}
+
+	kprintf("%s: all CPUs finished SMP call successfully\n", __func__);
+	ret = 0;
+
+free_out:
+	kfree(reqs);
+
+	return ret;
+}
+

lib/include/ihk/cpu.h

@@ -173,4 +173,7 @@ struct cpu_mapping;
 int arch_get_cpu_mapping(struct cpu_mapping **buf, int *nelemsp);
 int ihk_mc_ikc_arch_issue_host_ipi(int cpu, int vector);
 
+void smp_func_call_handler(void);
+int ihk_mc_get_smp_handler_irq(void);
+
 #endif