uti: Replace dead uti thread with new mcexec thread in proc->tids

Change-Id: Ic6e906dd1bfac1b07f1317732cbe0a5191831cd8

arch/arm64/kernel/syscall.c

@@ -1575,7 +1575,7 @@ done:
 		return 0;
 	}
 
-	if (tthread->thread_offloaded) {
+	if (tthread->uti_state == UTI_STATE_RUNNING_IN_LINUX) {
 		interrupt_syscall(tthread, sig);
 		release_thread(tthread);
 		return 0;

arch/x86_64/kernel/syscall.c

@@ -1370,7 +1370,7 @@ done:
 	}
 
 	/* Forward signal to Linux by interrupt_syscall mechanism */
-	if (tthread->thread_offloaded) {
+	if (tthread->uti_state == UTI_STATE_RUNNING_IN_LINUX) {
 		if (!tthread->proc->nohost) {
 			interrupt_syscall(tthread, sig);
 		}

executer/kernel/mcctrl/control.c

@@ -2391,7 +2391,7 @@ long
 mcexec_util_thread1(ihk_os_t os, unsigned long arg, struct file *file)
 {
 	void **__user uparam = (void ** __user)arg;
-	void *param[6];
+	void *param[7];
 	unsigned long p_rctx;
 	unsigned long phys;
 	void *__user u_rctx;
@@ -2401,7 +2401,7 @@ mcexec_util_thread1(ihk_os_t os, unsigned long arg, struct file *file)
 	unsigned long free_size;
 	unsigned long icurrent = (unsigned long)current;
 
-	if(copy_from_user(param, uparam, sizeof(void *) * 6)) {
+	if(copy_from_user(param, uparam, sizeof(void *) * 7)) {
 		return -EFAULT;
 	}
 	p_rctx = (unsigned long)param[0];
@@ -2422,6 +2422,7 @@ mcexec_util_thread1(ihk_os_t os, unsigned long arg, struct file *file)
 
 	((unsigned long *)rctx)[0] = free_address;
 	((unsigned long *)rctx)[1] = free_size;
+	((unsigned long *)rctx)[2] = (unsigned long)param[6];
 
 #ifdef CONFIG_MIC
 	iounmap(rctx);

executer/user/mcexec.c

@@ -1040,6 +1040,7 @@ pid_t master_tid;
 
 pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
 pthread_barrier_t init_ready;
+pthread_barrier_t uti_init_ready;
 
 pthread_attr_t watchdog_thread_attr;
 pthread_t watchdog_thread;
@@ -1390,8 +1391,7 @@ void init_sigaction(void)
 
 static int max_cpuid;
 
-static int
+static int create_worker_thread(struct thread_data_s **tp_out, pthread_barrier_t *init_ready)
-create_worker_thread(pthread_barrier_t *init_ready)
 {
 	struct thread_data_s *tp;
 
@@ -1409,6 +1409,10 @@ create_worker_thread(pthread_barrier_t *init_ready)
 	tp->next = thread_data;
 	thread_data = tp;
 
+	if (tp_out) {
+		*tp_out = tp;
+	}
+
 	return pthread_create(&tp->thread_id, NULL, 
 	                      &main_loop_thread_func, tp);
 }
@@ -1422,7 +1426,7 @@ int init_worker_threads(int fd)
 
 	max_cpuid = 0;
 	for (i = 0; i <= n_threads; ++i) {
-		int ret = create_worker_thread(&init_ready);
+		int ret = create_worker_thread(NULL, &init_ready);
 
 		if (ret) {
 			printf("ERROR: creating syscall threads (%d), check ulimit?\n", ret);
@@ -2882,10 +2886,18 @@ static long util_thread(struct thread_data_s *my_thread, unsigned long uctx_pa,
 	int i;
 	void *lctx;
 	void *rctx;
-	void *param[6];
+	void *param[7];
 	int rc = 0;
 	void *uti_wp = (void*)-1;
 
+	pthread_barrier_init(&uti_init_ready, NULL, 2);
+	if ((rc = create_worker_thread(&tp, &uti_init_ready))) {
+		printf("%s: Error: create_worker_thread failed (%d)\n", __FUNCTION__, rc);
+		goto out;
+	}
+	pthread_barrier_wait(&uti_init_ready);
+	__dprintf("%s: worker tid: %d\n", __FUNCTION__, tp->tid);
+
 	uti_desc = (struct uti_desc *)_uti_desc;
 	if (!uti_desc) {
 		fprintf(stderr, "%s: ERROR: uti_desc isn't set. Use mcexec.sh instead of mcexec\n", __FUNCTION__);
@@ -2916,8 +2928,8 @@ static long util_thread(struct thread_data_s *my_thread, unsigned long uctx_pa,
 	rctx = (char *)lctx + PAGE_SIZE;
 #endif  /* POSTK_DEBUG_ARCH_DEP_35 */
 
-	param[0] = (void *)uctx_pa;
+	param[0] = (void *)uctx_pa; /* Source address, kernel space */
-	param[1] = rctx;
+	param[1] = rctx; /* Destination address, user space */
 	param[2] = lctx;
 	param[4] = uti_desc->wp;
 #ifdef POSTK_DEBUG_ARCH_DEP_35
@@ -2925,12 +2937,19 @@ static long util_thread(struct thread_data_s *my_thread, unsigned long uctx_pa,
 #else	/* POSTK_DEBUG_ARCH_DEP_35 */
 	param[5] = (void *)(PAGE_SIZE * 3);
 #endif	/* POSTK_DEBUG_ARCH_DEP_35 */
+	param[6] = (void *)tp->tid;
+	printf("%s: param[6]=%ld,tid=%ld\n", __FUNCTION__, (long)param[6], (long)((void *)tp->tid));
+
+	__dprintf("%s: before MCEXEC_UP_UTIL_THREAD1\n", __FUNCTION__);
+
+	/* 1. Copy context from kernel space to user space 
+	   2. Write uti_wp addr, its size for McKernel to uctx_pa 
+	      Note that this overwrites context */
 	if ((rc = ioctl(fd, MCEXEC_UP_UTIL_THREAD1, param)) == -1) {
 		fprintf(stderr, "util_thread1: %d errno=%d\n", rc, errno);
 		rc = -errno;
 		goto out;
 	}
-	create_worker_thread(NULL);
 
 	/* Record the info of the thread migrating to Linux */
 	uti_desc->wp = uti_wp;
@@ -3372,7 +3391,7 @@ int main_loop(struct thread_data_s *my_thread)
 					tids[i++] = tp->tid;
 				}
 
-				for (; i < ncpu; ++i) {
+				for (; i < w.sr.args[4]; ++i) {
 					tids[i] = 0;
 				}
 

kernel/include/process.h

@@ -242,6 +242,11 @@ enum mpol_rebind_step {
 #define SPAWN_TO_REMOTE 1
 #define SPAWNING_TO_REMOTE 1001
 
+#define UTI_STATE_DEAD 0
+#define UTI_STATE_PROLOGUE 1
+#define UTI_STATE_RUNNING_IN_LINUX 2
+#define UTI_STATE_EPILOGUE 3
+
 #include <waitq.h>
 #include <futex.h>
 
@@ -690,10 +695,11 @@ struct thread {
 	/* Syscall offload wait queue head */
 	struct waitq scd_wq;
 
-	int thread_offloaded;
+	int uti_state;
 	int mod_clone;
 	struct uti_attr *mod_clone_arg;
 	int parent_cpuid;
+	int uti_refill_tid;
 
 	// for performance counter
 	unsigned long pmc_alloc_map;

kernel/process.c

@@ -2623,6 +2623,21 @@ void __release_tid(struct process *proc, struct thread *thread) {
 	}
 }
 
+/* Replace tid specified by thread with tid specified by new_tid */
+void __find_and_replace_tid(struct process *proc, struct thread *thread, int new_tid) {
+	int i;
+
+	for (i = 0; i < proc->nr_tids; ++i) {
+		if (proc->tids[i].thread != thread) continue;
+
+		proc->tids[i].thread = NULL;
+		proc->tids[i].tid = new_tid;
+		kprintf("%s: tid %d (thread %p) has been relaced with tid %d\n",
+				__FUNCTION__, thread->tid, thread, new_tid);
+		break;
+	}
+}
+
 void destroy_thread(struct thread *thread)
 {
 	struct sig_pending *pending;
@@ -2646,7 +2661,11 @@ void destroy_thread(struct thread *thread)
 
 	mcs_rwlock_writer_lock(&proc->threads_lock, &lock);
 	list_del(&thread->siblings_list);
-	__release_tid(proc, thread);
+	if (thread->uti_state == UTI_STATE_EPILOGUE) {
+		__find_and_replace_tid(proc, thread, thread->uti_refill_tid);
+	} else {
+		__release_tid(proc, thread);
+	}
 	mcs_rwlock_writer_unlock(&proc->threads_lock, &lock);
 
 	mcs_rwlock_writer_unlock(&proc->update_lock, &updatelock);

kernel/syscall.c

@@ -9127,6 +9127,8 @@ int util_thread(struct uti_attr *arg)
 		struct uti_attr attr;
 	} kattr;
 
+	thread->uti_state = UTI_STATE_PROLOGUE;
+
 	context = (volatile unsigned long *)ihk_mc_alloc_pages(1,
 	                                                      IHK_MC_AP_NOWAIT);
 	if (!context) {
@@ -9154,15 +9156,20 @@ int util_thread(struct uti_attr *arg)
 	}
 	request.args[3] = (unsigned long)uti_clv;
 	request.args[4] = uti_desc;
-	thread->thread_offloaded = 1;
+	thread->uti_state = UTI_STATE_RUNNING_IN_LINUX;
 	rc = do_syscall(&request, ihk_mc_get_processor_id(), 0);
 	dkprintf("%s: returned from do_syscall,tid=%d,rc=%lx\n", __FUNCTION__, thread->tid, rc);
 
+	thread->uti_state = UTI_STATE_EPILOGUE;
+
 	util_show_syscall_profile();
 
-	thread->thread_offloaded = 0;
+	/* These are written by mcexec_util_thread1() */
 	free_address = context[0];
 	free_size = context[1];
+	thread->uti_refill_tid = context[2];
+	dkprintf("%s: mcexec worker tid=%d\n", __FUNCTION__, context[2]);
+	
 	ihk_mc_free_pages((void *)context, 1);
 	kfree(uti_clv);