diff --git a/executer/include/uprotocol.h b/executer/include/uprotocol.h index fa45832a..dece625e 100644 --- a/executer/include/uprotocol.h +++ b/executer/include/uprotocol.h @@ -142,6 +142,7 @@ struct program_load_desc { unsigned long heap_extension; long stack_premap; unsigned long mpol_bind_mask; + int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */ int nr_processes; int process_rank; char shell_path[SHELL_PATH_MAX_LEN]; diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c index 25b10cec..be372a14 100644 --- a/executer/user/mcexec.c +++ b/executer/user/mcexec.c @@ -233,6 +233,7 @@ static long stack_premap = (2ULL << 20); static long stack_max = -1; static struct rlimit rlim_stack; static char *mpol_bind_nodes = NULL; +static int uti_thread_rank = 0; /* Partitioned execution (e.g., for MPI) */ static int nr_processes = 0; @@ -1787,6 +1788,12 @@ static struct option mcexec_options[] = { .flag = NULL, .val = 's', }, + { + .name = "uti-thread-rank", + .has_arg = required_argument, + .flag = NULL, + .val = 'u', + }, /* end */ { NULL, 0, NULL, 0, }, }; @@ -2049,9 +2056,9 @@ int main(int argc, char **argv) /* Parse options ("+" denotes stop at the first non-option) */ #ifdef ADD_ENVS_OPTION - while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:", mcexec_options, NULL)) != -1) { + while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:", mcexec_options, NULL)) != -1) { #else /* ADD_ENVS_OPTION */ - while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:", mcexec_options, NULL)) != -1) { + while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:", mcexec_options, NULL)) != -1) { #endif /* ADD_ENVS_OPTION */ switch (opt) { char *tmp; @@ -2114,6 +2121,10 @@ int main(int argc, char **argv) __dprintf("stack_premap=%ld,stack_max=%ld\n", stack_premap, stack_max); break; } + case 'u': + uti_thread_rank = atoi(optarg); + break; + case 0: /* long opt */ break; @@ -2598,6 +2609,9 @@ int main(int argc, char **argv) } } + desc->uti_thread_rank = uti_thread_rank; + + /* user_start and user_end are set by this call */ if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) { perror("prepare"); close(fd); diff --git a/kernel/host.c b/kernel/host.c index 1b460e69..4a891a67 100644 --- a/kernel/host.c +++ b/kernel/host.c @@ -505,6 +505,8 @@ static int process_msg_prepare_process(unsigned long rphys) vm->numa_mem_policy = MPOL_BIND; } + proc->uti_thread_rank = pn->uti_thread_rank; + #ifdef PROFILE_ENABLE proc->profile = pn->profile; thread->profile = pn->profile; diff --git a/kernel/include/process.h b/kernel/include/process.h index a2db49c8..86cedd37 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -556,6 +556,8 @@ struct process { size_t mpol_threshold; unsigned long heap_extension; unsigned long mpol_bind_mask; + int uti_thread_rank; /* Spawn on Linux CPU when clone_count reaches this */ + int clone_count; // perf_event int perf_status; diff --git a/kernel/include/syscall.h b/kernel/include/syscall.h index d578d686..046d083d 100644 --- a/kernel/include/syscall.h +++ b/kernel/include/syscall.h @@ -200,6 +200,7 @@ struct program_load_desc { unsigned long heap_extension; long stack_premap; unsigned long mpol_bind_mask; + int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */ int nr_processes; int process_rank; char shell_path[SHELL_PATH_MAX_LEN]; diff --git a/kernel/process.c b/kernel/process.c index c3217e31..7e7ba63e 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -3455,6 +3455,8 @@ void runq_add_thread(struct thread *thread, int cpu_id) procfs_create_thread(thread); + __sync_add_and_fetch(&thread->proc->clone_count, 1); + dkprintf("%s: clone_count is %d\n", __FUNCTION__, thread->proc->clone_count); rusage_num_threads_inc(); #ifdef RUSAGE_DEBUG if (rusage->num_threads == 1) { diff --git a/kernel/syscall.c b/kernel/syscall.c index 4b954318..7401d1f4 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -2451,6 +2451,18 @@ unsigned long do_fork(int clone_flags, unsigned long newsp, return -EAGAIN; } + /* N-th creation put the new on Linux CPU. It's turned off when zero is + set to uti_thread_rank. */ + if (oldproc->uti_thread_rank) { + if (oldproc->clone_count + 1 == oldproc->uti_thread_rank) { + old->mod_clone = SPAWN_TO_REMOTE; + kprintf("%s: mod_clone is set to %d\n", __FUNCTION__, old->mod_clone); + } else { + old->mod_clone = SPAWN_TO_LOCAL; + kprintf("%s: mod_clone is set to %d\n", __FUNCTION__, old->mod_clone); + } + } + new = clone_thread(old, curpc, newsp ? newsp : cursp, clone_flags);