From a697f5e98df494aafd11e42eacd5449714103b4a Mon Sep 17 00:00:00 2001
From: Balazs Gerofi <bgerofi@riken.jp>
Date: Wed, 23 Aug 2017 10:16:22 +0900
Subject: [PATCH] partitioned execution: pass process rank to LWK

Cherry-pick of d2d134d5e6a4b16a34d55d31b14614a2a91ecf47

Conflicts:
	kernel/include/process.h
---
 executer/include/uprotocol.h     |  2 ++
 executer/kernel/mcctrl/control.c | 11 +++++++++++
 executer/kernel/mcctrl/mcctrl.h  |  1 +
 executer/user/mcexec.c           |  3 +++
 kernel/host.c                    |  1 +
 kernel/include/process.h         |  1 +
 kernel/include/syscall.h         |  1 +
 7 files changed, 20 insertions(+)

diff --git a/executer/include/uprotocol.h b/executer/include/uprotocol.h
index e590fd22..d1722bb1 100644
--- a/executer/include/uprotocol.h
+++ b/executer/include/uprotocol.h
@@ -91,6 +91,7 @@ struct program_image_section {
 
 struct get_cpu_set_arg {
 	int nr_processes;
+	int *process_rank;
 	void *cpu_set;
 	size_t cpu_set_size;	// Size in bytes
 	int *target_core;
@@ -141,6 +142,7 @@ struct program_load_desc {
 	long stack_premap;
 	unsigned long mpol_bind_mask;
 	int nr_processes;
+	int process_rank;
 	char shell_path[SHELL_PATH_MAX_LEN];
 	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];
 	int profile;
diff --git a/executer/kernel/mcctrl/control.c b/executer/kernel/mcctrl/control.c
index 5b4483fb..b6ca3577 100644
--- a/executer/kernel/mcctrl/control.c
+++ b/executer/kernel/mcctrl/control.c
@@ -654,6 +654,7 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 		wake_up_interruptible(&pli_next->pli_wq);
 		/* Reset process counter */
 		pe->nr_processes_left = pe->nr_processes;
+		pe->process_rank = 0;
 	}
 
 	/* Wait for the rest if not the last or if the last but
@@ -885,6 +886,15 @@ next_cpu:
 		goto put_and_unlock_out;
 	}
 
+	/* Copy rank */
+	if (copy_to_user(req.process_rank, &pe->process_rank,
+				sizeof(int))) {
+		printk("%s: error copying process rank to user\n",
+				__FUNCTION__);
+		ret = -EINVAL;
+		goto put_and_unlock_out;
+	}
+
 	/* mcexec NUMA to bind to */
 	mcexec_linux_numa = cpu_to_node(mckernel_cpu_2_linux_cpu(udp, cpu));
 	if (copy_to_user(req.mcexec_linux_numa, &mcexec_linux_numa,
@@ -932,6 +942,7 @@ next_cpu:
 	}
 	/* Otherwise wake up next process in list */
 	else {
+		++pe->process_rank;
 		pli_next = list_first_entry(&pe->pli_list,
 			struct process_list_item, list);
 		list_del(&pli_next->list);
diff --git a/executer/kernel/mcctrl/mcctrl.h b/executer/kernel/mcctrl/mcctrl.h
index b4ade3dc..08cab6ed 100644
--- a/executer/kernel/mcctrl/mcctrl.h
+++ b/executer/kernel/mcctrl/mcctrl.h
@@ -316,6 +316,7 @@ struct mcctrl_part_exec {
 	struct mutex lock;	
 	int nr_processes;
 	int nr_processes_left;
+	int process_rank;
 	cpumask_t cpus_used;
 	struct list_head pli_list;
 };
diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c
index b0ab99cb..81a73dfc 100644
--- a/executer/user/mcexec.c
+++ b/executer/user/mcexec.c
@@ -2456,6 +2456,7 @@ int main(int argc, char **argv)
 		struct get_cpu_set_arg cpu_set_arg;
 		int mcexec_linux_numa = 0;
 		int ikc_mapped = 0;
+		int process_rank = -1;
 		cpu_set_t mcexec_cpu_set;
 
 		CPU_ZERO(&mcexec_cpu_set);
@@ -2464,6 +2465,7 @@ int main(int argc, char **argv)
 		cpu_set_arg.cpu_set_size = sizeof(desc->cpu_set);
 		cpu_set_arg.nr_processes = nr_processes;
 		cpu_set_arg.target_core = &target_core;
+		cpu_set_arg.process_rank = &process_rank;
 		cpu_set_arg.mcexec_linux_numa = &mcexec_linux_numa;
 		cpu_set_arg.mcexec_cpu_set = &mcexec_cpu_set;
 		cpu_set_arg.mcexec_cpu_set_size = sizeof(mcexec_cpu_set);
@@ -2476,6 +2478,7 @@ int main(int argc, char **argv)
 		}
 
 		desc->cpu = target_core;
+		desc->process_rank = process_rank;
 
 		/* Bind to CPU cores where the LWK process' IKC target maps to */
 		if (ikc_mapped && !no_bind_ikc_map) {
diff --git a/kernel/host.c b/kernel/host.c
index ca948ca4..1b460e69 100644
--- a/kernel/host.c
+++ b/kernel/host.c
@@ -482,6 +482,7 @@ static int process_msg_prepare_process(unsigned long rphys)
 	proc->mpol_flags = pn->mpol_flags;
 	proc->mpol_threshold = pn->mpol_threshold;
 	proc->nr_processes = pn->nr_processes;
+	proc->process_rank = pn->process_rank;
 	proc->heap_extension = pn->heap_extension;
 
 	/* Update NUMA binding policy if requested */
diff --git a/kernel/include/process.h b/kernel/include/process.h
index 1c25c86f..2295757a 100644
--- a/kernel/include/process.h
+++ b/kernel/include/process.h
@@ -570,6 +570,7 @@ struct process {
 	unsigned long profile_elapsed_ts;
 #endif // PROFILE_ENABLE
 	int nr_processes; /* For partitioned execution */
+	int process_rank; /* Rank in partition */
 };
 
 /*
diff --git a/kernel/include/syscall.h b/kernel/include/syscall.h
index 1517f175..e04ac921 100644
--- a/kernel/include/syscall.h
+++ b/kernel/include/syscall.h
@@ -199,6 +199,7 @@ struct program_load_desc {
 	long stack_premap;
 	unsigned long mpol_bind_mask;
 	int nr_processes;
+	int process_rank;
 	char shell_path[SHELL_PATH_MAX_LEN];
 	__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];
 	int profile;