uti: Identify uti thread by clone count

--uti-thread-count <count> is added to mcexec.

Change-Id: Id9ec464412a5bb71e4d9e87d05f79de22d35b067

executer/include/uprotocol.h

@@ -142,6 +142,7 @@ struct program_load_desc {
 	unsigned long heap_extension;
 	long stack_premap;
 	unsigned long mpol_bind_mask;
+	int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */
 	int nr_processes;
 	int process_rank;
 	char shell_path[SHELL_PATH_MAX_LEN];

executer/user/mcexec.c

@@ -233,6 +233,7 @@ static long stack_premap = (2ULL << 20);
 static long stack_max = -1;
 static struct rlimit rlim_stack;
 static char *mpol_bind_nodes = NULL;
+static int uti_thread_rank = 0;
 
 /* Partitioned execution (e.g., for MPI) */
 static int nr_processes = 0;
@@ -1787,6 +1788,12 @@ static struct option mcexec_options[] = {
 		.flag =		NULL,
 		.val =		's',
 	},
+	{
+		.name =		"uti-thread-rank",
+		.has_arg =	required_argument,
+		.flag =		NULL,
+		.val =		'u',
+	},
 	/* end */
 	{ NULL, 0, NULL, 0, },
 };
@@ -2049,9 +2056,9 @@ int main(int argc, char **argv)
 
 	/* Parse options ("+" denotes stop at the first non-option) */
 #ifdef ADD_ENVS_OPTION
-	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:", mcexec_options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:", mcexec_options, NULL)) != -1) {
 #else /* ADD_ENVS_OPTION */
-	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:", mcexec_options, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:", mcexec_options, NULL)) != -1) {
 #endif /* ADD_ENVS_OPTION */
 		switch (opt) {
 			char *tmp;
@@ -2114,6 +2121,10 @@ int main(int argc, char **argv)
 			__dprintf("stack_premap=%ld,stack_max=%ld\n", stack_premap, stack_max);
 			break; }
 
+			case 'u':
+				uti_thread_rank = atoi(optarg);
+				break;
+
 			case 0:	/* long opt */
 				break;
 
@@ -2598,6 +2609,9 @@ int main(int argc, char **argv)
 		}
 	}
 
+	desc->uti_thread_rank = uti_thread_rank;
+
+	/* user_start and user_end are set by this call */
 	if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
 		perror("prepare");
 		close(fd);

kernel/host.c

@@ -505,6 +505,8 @@ static int process_msg_prepare_process(unsigned long rphys)
 		vm->numa_mem_policy = MPOL_BIND;
 	}
 
+	proc->uti_thread_rank = pn->uti_thread_rank;
+
 #ifdef PROFILE_ENABLE
 	proc->profile = pn->profile;
 	thread->profile = pn->profile;

kernel/include/process.h

@@ -556,6 +556,8 @@ struct process {
 	size_t mpol_threshold;
 	unsigned long heap_extension;
 	unsigned long mpol_bind_mask;
+	int uti_thread_rank; /* Spawn on Linux CPU when clone_count reaches this */
+	int clone_count;
 
 	// perf_event
 	int perf_status;

kernel/include/syscall.h

@@ -200,6 +200,7 @@ struct program_load_desc {
 	unsigned long heap_extension;
 	long stack_premap;
 	unsigned long mpol_bind_mask;
+	int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */
 	int nr_processes;
 	int process_rank;
 	char shell_path[SHELL_PATH_MAX_LEN];

kernel/process.c

@@ -3455,6 +3455,8 @@ void runq_add_thread(struct thread *thread, int cpu_id)
 
 	procfs_create_thread(thread);
 
+	__sync_add_and_fetch(&thread->proc->clone_count, 1);
+	dkprintf("%s: clone_count is %d\n", __FUNCTION__, thread->proc->clone_count);
 	rusage_num_threads_inc();
 #ifdef RUSAGE_DEBUG
 	if (rusage->num_threads == 1) {

kernel/syscall.c

@@ -2451,6 +2451,18 @@ unsigned long do_fork(int clone_flags, unsigned long newsp,
         return -EAGAIN;
     }
 
+	/* N-th creation put the new on Linux CPU. It's turned off when zero is 
+	   set to uti_thread_rank. */
+	if (oldproc->uti_thread_rank) {
+		if (oldproc->clone_count + 1 == oldproc->uti_thread_rank) {
+			old->mod_clone = SPAWN_TO_REMOTE;
+			kprintf("%s: mod_clone is set to %d\n", __FUNCTION__, old->mod_clone);
+		} else {
+			old->mod_clone = SPAWN_TO_LOCAL;
+			kprintf("%s: mod_clone is set to %d\n", __FUNCTION__, old->mod_clone);
+		}
+	}
+
 	new = clone_thread(old, curpc,
 	                    newsp ? newsp : cursp, clone_flags);