uti: Call into McKernel futex()

(1) Masquerade clv (2) Fix timeout (3) Let mcexec thread with the same tid as McKernel thread migrating to Linux handles the migration request (4) Call create_tracer() before creating proxy related objects Change-Id: I6b2689b70db49827f10aa7d5a4c581aa81319b55
2018-09-02 17:36:28 +09:00
parent 460917c4a0
commit 52afbbbc98
16 changed files with 629 additions and 150 deletions
--- a/executer/include/uprotocol.h
+++ b/executer/include/uprotocol.h
@@ -62,6 +62,7 @@
 #define MCEXEC_UP_TERMINATE_THREAD 0x30a02925
 #define MCEXEC_UP_GET_NUM_POOL_THREADS  0x30a02926
 #define MCEXEC_UP_UTI_ATTR       0x30a02927
+#define MCEXEC_UP_UNMAP_PSEUDO_FILEMAP 0x30a02928

 #define MCEXEC_UP_DEBUG_LOG     0x40000000

--- a/executer/kernel/mcctrl/control.c
+++ b/executer/kernel/mcctrl/control.c
@@ -283,6 +283,8 @@ struct mcos_handler_info {
 	int cpu;
 	struct mcctrl_usrdata *ud;
 	struct file *file;
+	unsigned long user_start;
+	unsigned long user_end;
 };

 struct mcos_handler_info;
@@ -420,6 +422,8 @@ static long mcexec_start_image(ihk_os_t os,

 	info->pid = desc->pid;
 	info->cpu = desc->cpu;
+	info->user_start = desc->user_start;
+	info->user_end = desc->user_end;
 	ihk_os_register_release_handler(file, release_handler, info);
 	ihk_os_set_mcos_private_data(file, info);

@@ -1198,9 +1202,28 @@ int mcexec_syscall(struct mcctrl_usrdata *ud, struct ikc_scd_packet *packet)
 			wqhln = wqhln_iter;
 			break;
 		}
-		if (!wqhln) {
-			printk("%s: WARNING: no target thread found for exact request??\n",
-				__FUNCTION__);
+		/* Find the mcexec thread with the same tid as the requesting McKernel thread 
+		   and let it handle the migrate-to-Linux request */
+		if (packet->req.number == __NR_sched_setaffinity && packet->req.args[0] == 0) {
+			list_for_each_entry(wqhln_iter, &ppd->wq_list, list) {
+				if (packet->req.ttid == wqhln_iter->rtid) {
+					if (!wqhln_iter->task) {
+						printk("%s: ERROR: wqhln_iter->task=%p,rtid=%d,&ppd->wq_list_lock=%p\n", __FUNCTION__, wqhln_iter->task, wqhln_iter->rtid, &ppd->wq_list_lock);
+					} else if(wqhln_iter->req) {
+						/* list_del() is called after woken-up */
+						dprintk("%s: INFO: target thread is busy, wqhln_iter->req=%d,rtid=%d,&ppd->wq_list_lock=%p\n", __FUNCTION__, wqhln_iter->req, wqhln_iter->rtid, &ppd->wq_list_lock);
+					} else {
+						wqhln = wqhln_iter;
+						dprintk("%s: uti, worker with tid of %d found in wq_list\n", __FUNCTION__, packet->req.ttid);
+					}
+					break;
+				}
+			}
+		} else {
+			if (!wqhln) {
+				printk("%s: WARNING: no target thread (tid=%d) found for exact request??\n",
+					   __FUNCTION__, packet->req.ttid);
+			}
 		}
 	}
 	/* Is there any thread available? */
@@ -1225,6 +1248,12 @@ retry_alloc:
 		wqhln = wqhln_alloc;
 		wqhln->req = 0;
 		wqhln->task = NULL;
+		/* Let the mcexec thread to handle migrate-to-Linux request in mcexec_wait_syscall() after finishing the current task */
+		if (packet->req.number == __NR_sched_setaffinity && packet->req.args[0] == 0) {
+			wqhln->rtid = packet->req.ttid;
+		} else {
+			wqhln->rtid = 0;
+		}
 		init_waitqueue_head(&wqhln->wq_syscall);
 		list_add_tail(&wqhln->list, &ppd->wq_req_list);
 	}
@@ -1272,16 +1301,27 @@ int mcexec_wait_syscall(ihk_os_t os, struct syscall_wait_desc *__user req)
 retry:
 	/* Prepare per-thread wait queue head or find a valid request */
 	irqflags = ihk_ikc_spinlock_lock(&ppd->wq_list_lock);
+
+	/* Handle migrate-to-Linux request if any */
+	list_for_each_entry(wqhln_iter, &ppd->wq_req_list, list) {
+		if (wqhln_iter->rtid == task_pid_vnr(current)) {
+			wqhln = wqhln_iter;
+			wqhln->task = current;
+			list_del(&wqhln->list);
+			goto found;
+		}
+	}
+
 	/* First see if there is a valid request already that is not yet taken */
 	list_for_each_entry(wqhln_iter, &ppd->wq_req_list, list) {
-		if (wqhln_iter->task == NULL && wqhln_iter->req) {
+		if (!wqhln_iter->rtid && wqhln_iter->task == NULL && wqhln_iter->req) {
 			wqhln = wqhln_iter;
 			wqhln->task = current;
 			list_del(&wqhln->list);
 			break;
 		}
 	}
-
+ found:
 	if (!wqhln) {
 retry_alloc:
 		wqhln = kmalloc(sizeof(*wqhln), GFP_ATOMIC);
@@ -1293,6 +1333,8 @@ retry_alloc:
 		wqhln->task = current;
 		wqhln->req = 0;
 		wqhln->packet = NULL;
+		/* Let mcexec_syscall() find the mcexec thread to handle migrate-to-Linux request */
+		wqhln->rtid = task_pid_vnr(current);
 		init_waitqueue_head(&wqhln->wq_syscall);

 		list_add(&wqhln->list, &ppd->wq_list);
@@ -2354,6 +2396,8 @@ mcexec_util_thread2(ihk_os_t os, unsigned long arg, struct file *file)
 	void **__user param = (void **__user )arg;
 	void *__user rctx = (void *__user)param[1];
 	void *__user lctx = (void *__user)param[2];
+	struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
+	struct mcctrl_per_proc_data *ppd;

 	save_fs_ctx(lctx);
 	info = ihk_os_get_mcos_private_data(file);
@@ -2451,34 +2495,148 @@ err:
 	kfree(thread);
 	return 0;
 }
+ 
+long mcexec_unmap_pseudo_filemap(ihk_os_t os, struct file *file)
+{
+	long rc = -1;
+	struct mcos_handler_info *info;
+	info = ihk_os_get_mcos_private_data(file);
+	dprintk("%s: clear_pte_range %p-%p\n", __FUNCTION__, (void*)info->user_start, (void*)info->user_end);
+	rc = clear_pte_range(info->user_start, info->user_end - info->user_start);
+	return rc;
+}

-long
-mcexec_syscall_thread(ihk_os_t os, unsigned long arg, struct file *file)
+ static long (*mckernel_do_futex)(int n, unsigned long arg0, unsigned long arg1,
+			  unsigned long arg2, unsigned long arg3,
+			  unsigned long arg4, unsigned long arg5,
+				   unsigned long _uti_clv,
+				   void *uti_futex_resp,
+				   void *_linux_wait_event,
+							  void *_linux_printk,
+							  void *_linux_clock_gettime);
+
+ long uti_wait_event(void *_resp, unsigned long nsec_timeout) {
+	 struct uti_futex_resp *resp = _resp;
+	 if (nsec_timeout) {
+		 return wait_event_interruptible_timeout(resp->wq, resp->done, nsecs_to_jiffies(nsec_timeout));
+	 } else {
+		 return wait_event_interruptible(resp->wq, resp->done);
+	 }
+ }
+
+ int uti_printk(const char *fmt, ...) {
+	 int sum = 0, nwritten;
+	 va_list args;
+	 va_start(args, fmt);
+	 nwritten = vprintk(fmt, args);
+	 sum += nwritten;
+	 va_end(args);
+	 return sum;
+ }
+
+int uti_clock_gettime(clockid_t clk_id, struct timespec *tp) {
+	int ret = 0;
+	struct timespec64 ts64;
+	dprintk("%s: clk_id=%x,REALTIME=%x,MONOTONIC=%x\n", __FUNCTION__, clk_id, CLOCK_REALTIME, CLOCK_MONOTONIC);
+	switch(clk_id) {
+	case CLOCK_REALTIME:
+		getnstimeofday64(&ts64);
+		tp->tv_sec = ts64.tv_sec;
+		tp->tv_nsec = ts64.tv_nsec;
+		dprintk("%s: CLOCK_REALTIME,%ld.%09ld\n", __FUNCTION__, tp->tv_sec, tp->tv_nsec);
+		break;
+	case CLOCK_MONOTONIC: {
+		/* Do not use getrawmonotonic() because it returns different value than clock_gettime() */
+		ktime_get_ts64(&ts64);
+		tp->tv_sec = ts64.tv_sec;
+		tp->tv_nsec = ts64.tv_nsec;
+		dprintk("%s: CLOCK_MONOTONIC,%ld.%09ld\n", __FUNCTION__, tp->tv_sec, tp->tv_nsec);
+		break; }
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+long mcexec_syscall_thread(ihk_os_t os, unsigned long arg, struct file *file)
 {
 	struct syscall_struct {
 		int number;
 		unsigned long args[6];
 		unsigned long ret;
+		unsigned long uti_clv; /* copy of a clv in McKernel */
 	};
 	struct syscall_struct param;
 	struct syscall_struct __user *uparam =
 	                              (struct syscall_struct __user *)arg;
-	int rc;
+	long rc;

 	if (copy_from_user(&param, uparam, sizeof param)) {
 		return -EFAULT;
 	}
-	rc = syscall_backward(ihk_host_os_get_usrdata(os), param.number,
-	                      param.args[0], param.args[1], param.args[2],
-	                      param.args[3], param.args[4], param.args[5],
-	                      &param.ret);
+#if 1 /* debug */
+	if (param.number == __NR_futex) {
+#else
+		if (0) {
+#endif
+		struct uti_futex_resp resp = {
+			.done = 0
+		};
+		init_waitqueue_head(&resp.wq);
+		
+ 		if (!mckernel_do_futex) {
+			if (ihk_os_get_special_address(os, IHK_SPADDR_MCKERNEL_DO_FUTEX,
+										   (unsigned long *)&mckernel_do_futex,
+										   NULL)) {
+				kprintf("%s: ihk_os_get_special_address failed\n", __FUNCTION__);
+				return -EINVAL;
+			}
+			dprintk("%s: mckernel_do_futex=%p\n", __FUNCTION__, mckernel_do_futex);
+		}

+		rc = (*mckernel_do_futex)(param.number, param.args[0], param.args[1], param.args[2],
+							  param.args[3], param.args[4], param.args[5], param.uti_clv, (void *)&resp, (void *)uti_wait_event, (void *)uti_printk, (void *)uti_clock_gettime);
+		param.ret = rc;
+	} else {
+			dprintk("%s: syscall_backward, SC %d, tid %d\n", __FUNCTION__, param.number, task_tgid_vnr(current));
+			rc = syscall_backward(ihk_host_os_get_usrdata(os), param.number,
+								  param.args[0], param.args[1], param.args[2],
+								  param.args[3], param.args[4], param.args[5],
+								  &param.ret);
+		}
 	if (copy_to_user(&uparam->ret, &param.ret, sizeof(unsigned long))) {
 		return -EFAULT;
 	}
 	return rc;
 }

+void mcctrl_futex_wake(struct ikc_scd_packet *pisp)
+{
+	struct uti_futex_resp *resp;
+
+	/* Guard the access to pisp->futex.resp, which is dead out of mcexec_syscall_thread() */
+	if (*pisp->futex.spin_sleep == 0) {
+		dprintk("%s: DEBUG: woken up by someone else\n", __FUNCTION__);
+		return;
+	}
+
+	resp = pisp->futex.resp;
+	if (!resp) {
+		kprintf("%s: ERROR: pisp->futex.resp is NULL\n", __FUNCTION__);
+		return;
+	}
+
+	if (*pisp->futex.spin_sleep == 0) {
+		kprintf("%s: ERROR: resp is dead\n", __FUNCTION__);
+		return;
+	}
+
+	resp->done = 1;
+	wake_up_interruptible(&resp->wq);
+}
+
+
 static struct ihk_cache_topology *
 cache_topo_search(struct ihk_cpu_topology *cpu_topo, int level)
 {
@@ -2838,6 +2996,9 @@ long __mcctrl_control(ihk_os_t os, unsigned int req, unsigned long arg,
 	case MCEXEC_UP_TERMINATE_THREAD:
 		return mcexec_terminate_thread(os, (unsigned long *)arg, file);

+	case MCEXEC_UP_UNMAP_PSEUDO_FILEMAP:
+		return mcexec_unmap_pseudo_filemap(os, file);
+
 	case MCEXEC_UP_GET_NUM_POOL_THREADS:
 		return mcctrl_get_num_pool_threads(os);

--- a/executer/kernel/mcctrl/driver.c
+++ b/executer/kernel/mcctrl/driver.c
@@ -90,6 +90,7 @@ static struct ihk_os_user_call_handler mcctrl_uchs[] = {
 	{ .request = MCEXEC_UP_TERMINATE_THREAD, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_GET_NUM_POOL_THREADS, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_UTI_ATTR, .func = mcctrl_ioctl },
+	{ .request = MCEXEC_UP_UNMAP_PSEUDO_FILEMAP, .func = mcctrl_ioctl },
 	{ .request = MCEXEC_UP_DEBUG_LOG, .func = mcctrl_ioctl },
 	{ .request = IHK_OS_AUX_PERF_NUM, .func = mcctrl_ioctl },
 	{ .request = IHK_OS_AUX_PERF_SET, .func = mcctrl_ioctl },
--- a/executer/kernel/mcctrl/ikc.c
+++ b/executer/kernel/mcctrl/ikc.c
@@ -52,6 +52,8 @@
 static void mcctrl_ikc_init(ihk_os_t os, int cpu, unsigned long rphys, struct ihk_ikc_channel_desc *c);
 int mcexec_syscall(struct mcctrl_usrdata *ud, struct ikc_scd_packet *packet);
 void sig_done(unsigned long arg, int err);
+void mcctrl_perf_ack(ihk_os_t os, struct ikc_scd_packet *packet);
+void mcctrl_futex_wake(struct ikc_scd_packet *pisp);
 void mcctrl_os_read_write_cpu_response(ihk_os_t os,
 		struct ikc_scd_packet *pisp);
 void mcctrl_eventfd(ihk_os_t os, struct ikc_scd_packet *pisp);
@@ -221,6 +223,10 @@ static int syscall_packet_handler(struct ihk_ikc_channel_desc *c,
 		mcctrl_eventfd(__os, pisp);
 		break;

+	case SCD_MSG_FUTEX_WAKE:
+		mcctrl_futex_wake(pisp);
+		break;
+
 	default:
 		printk(KERN_ERR "mcctrl:syscall_packet_handler:"
 				"unknown message (%d.%d.%d.%d.%d.%#lx)\n",
--- a/executer/kernel/mcctrl/mcctrl.h
+++ b/executer/kernel/mcctrl/mcctrl.h
@@ -102,6 +102,8 @@
 #define SCD_MSG_CPU_RW_REG              0x52
 #define SCD_MSG_CPU_RW_REG_RESP         0x53

+#define SCD_MSG_FUTEX_WAKE              0x60
+
 #define DMA_PIN_SHIFT                   21

 #define DO_USER_MODE
@@ -126,6 +128,12 @@ enum mcctrl_os_cpu_operation {
 	MCCTRL_OS_CPU_MAX_OP
 };

+/* Used to wake-up a Linux thread futex_wait()-ing */
+struct uti_futex_resp {
+	int done;
+	wait_queue_head_t wq;
+};
+
 struct ikc_scd_packet {
 	int msg;
 	int err;
@@ -164,6 +172,12 @@ struct ikc_scd_packet {
 		struct {
 			int eventfd_type;
 		};
+
+		/* SCD_MSG_FUTEX_WAKE */
+		struct {
+			void *resp;
+			int *spin_sleep; /* 1: waiting in linux_wait_event() 0: woken up by someone else */
+		} futex;
 	};
 	char padding[8];
 };
@@ -465,6 +479,7 @@ inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(

 void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, 
 		long ret, int stid);
+int clear_pte_range(uintptr_t start, uintptr_t len);

 int mcctrl_os_alive(void);

--- a/executer/kernel/mcctrl/syscall.c
+++ b/executer/kernel/mcctrl/syscall.c
@@ -475,7 +475,7 @@ out_put_ppd:
 	return syscall_ret;
 }

-static int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t reason)
+int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t reason)
 {
 	struct ikc_scd_packet *packet;
 	struct ikc_scd_packet *free_packet = NULL;
@@ -1999,7 +1999,7 @@ out:
 	return (IS_ERR_VALUE(map))? (int)map: 0;
 }

-static int clear_pte_range(uintptr_t start, uintptr_t len)
+int clear_pte_range(uintptr_t start, uintptr_t len)
 {
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
--- a/executer/user/arch/x86_64/arch_args.h
+++ b/executer/user/arch/x86_64/arch_args.h
@@ -67,6 +67,12 @@ get_syscall_arg6(syscall_args *args)
 	return args->r9;
 }

+static inline unsigned long
+get_syscall_rip(syscall_args *args)
+{
+	return args->rip;
+}
+
 static inline void
 set_syscall_number(syscall_args *args, unsigned long value)
 {
--- a/executer/user/mcexec.c
+++ b/executer/user/mcexec.c
@@ -192,6 +192,7 @@ struct syscall_struct {
 	int number;
 	unsigned long args[6];
 	unsigned long ret;
+	unsigned long uti_clv; /* copy of a clv in McKernel */
 };

 #ifdef NCCS
@@ -1951,6 +1952,20 @@ static void ld_preload_init()
 #endif
 }

+struct uti_desc {
+	void *wp;
+	int mck_tid;
+	unsigned long key;
+	int pid, tid; /* Used as the id of tracee when issuing MCEXEC_UP_TERMINATE_THREAD */
+	unsigned long uti_clv;
+	sem_t arg, attach;
+};
+
+static int create_tracer();
+int uti_pfd[2];
+void *uti_wp;
+struct uti_desc *uti_desc;
+
 int main(int argc, char **argv)
 {
 	int ret = 0;
@@ -2129,6 +2144,31 @@ int main(int argc, char **argv)
 	if (opendev() == -1)
 		exit(EXIT_FAILURE);

+	/* Perform mmap() before fork() in create_tracer() */
+	uti_wp = mmap(NULL, PAGE_SIZE * 3, PROT_READ | PROT_WRITE,
+	          MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+	if (uti_wp == (void *)-1) {
+		exit(1);
+	}
+	uti_desc = mmap(NULL, sizeof(struct uti_desc), PROT_READ | PROT_WRITE,
+	          MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+	if (uti_desc == (void *)-1) {
+		exit(1);
+	}
+	sem_init(&uti_desc->arg, 1, 0);
+	sem_init(&uti_desc->attach, 1, 0);
+#if 1
+	/* Create tracer before any proxy VMAs are attached */
+	if ((error = pipe(uti_pfd)) == -1) {
+		fprintf(stderr, "%s: pipe returned %d\n", __FUNCTION__, error);
+		return -1;
+	}
+	if ((error = create_tracer())) {
+		fprintf(stderr, "%s: create tracer returned %d\n", __FUNCTION__, error);
+		return error;
+	}
+#endif
+
 	ld_preload_init();

 #ifdef ADD_ENVS_OPTION
@@ -2853,11 +2893,8 @@ debug_sig(int s)
 #endif

 static int
-create_tracer(void *wp, int mck_tid, unsigned long key)
+create_tracer()
 {
-	int pid = getpid();
-	int tid = gettid();
-	int pfd[2];
 	int tpid;
 	int rc;
 	int st;
@@ -2868,44 +2905,42 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 	unsigned long code = 0;
 	int exited = 0;
 	int mode = 0;
+	//struct tracer_desc desc;
+	unsigned long buf;

-	if (pipe(pfd) == -1)
-		return -1;
 	tpid = fork();
 	if (tpid) {
-		struct timeval tv;
-		fd_set rfd;
-
 		if (tpid == -1)
 			return -1;
-		close(pfd[1]);
+		close(uti_pfd[1]);
 		while ((rc = waitpid(tpid, &st, 0)) == -1 && errno == EINTR);
 		if (rc == -1 || !WIFEXITED(st) || WEXITSTATUS(st)) {
 			fprintf(stderr, "waitpid rc=%d st=%08x\n", rc, st);
 			return -ENOMEM;
 		}
+#if 0
+		struct timeval tv;
+		fd_set rfd;
 		FD_ZERO(&rfd);
-		FD_SET(pfd[0], &rfd);
+		FD_SET(uti_pfd[0], &rfd);
 		tv.tv_sec = 1;
 		tv.tv_usec = 0;
-		while ((rc = select(pfd[0] + 1, &rfd, NULL, NULL, &tv)) == -1 &&
+		while ((rc = select(uti_pfd[0] + 1, &rfd, NULL, NULL, &tv)) == -1 &&
 		       errno == EINTR);
 		if (rc == 0) {
-			close(pfd[0]);
+			fprintf(stderr, "%s: select timed out\n", __FUNCTION__);
+			close(uti_pfd[0]);
 			return -ETIMEDOUT;
 		}
 		if (rc == -1) {
-			close(pfd[0]);
+			fprintf(stderr, "%s: select errno=%d\n", __FUNCTION__, errno);
+			close(uti_pfd[0]);
 			return -errno;
 		}
-		rc = read(pfd[0], &st, 1);
-		close(pfd[0]);
-		if (rc != 1) {
-			return -EAGAIN;
-		}
+#endif
 		return 0;
 	}
-	close(pfd[0]);
+	close(uti_pfd[0]);
 	tpid = fork();
 	if (tpid) {
 		if (tpid == -1) {
@@ -2914,17 +2949,42 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 		}
 		exit(0);
 	}
-	if (ptrace(PTRACE_ATTACH, tid, 0, 0) == -1) {
+
+#if 0
+	/* Reopen device because one process must be managed by one opened-device */
+	close(fd);
+	fd = opendev();
+	if (fd < 0) {
+		fprintf(stderr, "%s: ERROR: opendev returned %d\n", __FUNCTION__, errno);
+		exit(1);
+	}
+	
+	if (ioctl(fd, MCEXEC_UP_CREATE_PPD) != 0) {
+		fprintf(stderr, "%s: ERROR: MCEXEC_UP_CREATE_PPD returned %d\n", __FUNCTION__, errno);
+		exit(1);
+	}
+#endif
+
+	sem_wait(&uti_desc->arg);
+	//close(uti_pfd[0]);
+
+	if (ptrace(PTRACE_ATTACH, uti_desc->tid, 0, 0) == -1) {
 		fprintf(stderr, "PTRACE_ATTACH errno=%d\n", errno);
 		exit(1);
 	}
 	waitpid(-1, &st, __WALL);
-	if (ptrace(PTRACE_SETOPTIONS, tid, 0, PTRACE_O_TRACESYSGOOD) == -1) {
+	if (ptrace(PTRACE_SETOPTIONS, uti_desc->tid, 0, PTRACE_O_TRACESYSGOOD) == -1) {
 		fprintf(stderr, "PTRACE_SETOPTIONS errno=%d\n", errno);
 		exit(1);
 	}
-	write(pfd[1], " ", 1);
-	close(pfd[1]);
+
+	/* Wake up tracee so that it can context-switch to McKernel code */
+    rc = write(uti_pfd[1], &buf, sizeof(unsigned long));
+    if (rc != sizeof(unsigned long)) {
+        fprintf(stderr, "%s: write returned %d\n", __FUNCTION__, rc);
+        exit(1);
+    }
+	close(uti_pfd[1]);

 	for (i = 0; i < 4096; i++)
 		if (i != fd
@@ -2940,33 +3000,42 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 #endif

 	for (i = 1; i <= 10; i++) {
-		param = (struct syscall_struct *)wp + i;
+		param = (struct syscall_struct *)uti_desc->wp + i;
 		*(void **)param = param_top;
 		param_top = param;
 	}
-	memset(wp, '\0', sizeof(long));
+	memset(uti_desc->wp, '\0', sizeof(long));

 #ifdef DEBUG_UTI
 	fprintf(stderr, "tracer PID=%d\n", getpid());
 	signal(SIGINT, debug_sig);
 #endif
 	for (;;) {
-		ptrace(PTRACE_SYSCALL, tid, 0, sig);
+		ptrace(PTRACE_SYSCALL, uti_desc->tid, 0, sig);
 		sig = 0;
 		waitpid(-1, &st, __WALL);
 		if (WIFEXITED(st) || WIFSIGNALED(st)) {
 			unsigned long term_param[4];

-			term_param[0] = pid;
-			term_param[1] = tid;
-			term_param[3] = key;
+			term_param[0] = uti_desc->pid;
+			term_param[1] = uti_desc->tid;
+			term_param[3] = uti_desc->key;
 			code = st;
 			if (exited == 2 || // exit_group
 			    WIFSIGNALED(st)) {
 				code |= 0x0000000100000000;
 			}
 			term_param[2] = code;
-			ioctl(fd, MCEXEC_UP_TERMINATE_THREAD, term_param);
+			if (ioctl(fd, MCEXEC_UP_TERMINATE_THREAD, term_param) != 0) {
+				fprintf(stderr, "%s: ERROR: MCEXEC_UP_TERMINATE_THREAD returned %d\n", __FUNCTION__, errno);
+			}
+			__dprintf("%s:  WIFEXITED=%d,WIFSIGNALED=%d,WTERMSIG=%d,exited=%d\n", __FUNCTION__, WIFEXITED(st), WIFSIGNALED(st), WTERMSIG(st), exited);
+#if 0
+			if (ptrace(PTRACE_DETACH, uti_desc->tid, 0, WIFSIGNALED(st) ? WTERMSIG(st) : 0) && errno != ESRCH) {
+				fprintf(stderr, "PTRACE_DETACH errno=%d\n", errno);
+				exit(1);
+			}
+#endif
 			break;
 		}
 		if (!WIFSTOPPED(st)) {
@@ -2975,7 +3044,7 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 		if (WSTOPSIG(st) & 0x80) { // syscall
 			syscall_args args;

-			get_syscall_args(tid, &args);
+			get_syscall_args(uti_desc->tid, &args);

 #ifdef DEBUG_UTI
 			if (get_syscall_return(&args) == -ENOSYS) {
@@ -3000,8 +3069,8 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 			switch (get_syscall_number(&args)) {
 			    case __NR_gettid:
 				set_syscall_number(&args, -1);
-				set_syscall_return(&args, mck_tid);
-				set_syscall_args(tid, &args);
+				set_syscall_return(&args, uti_desc->mck_tid);
+				set_syscall_args(uti_desc->tid, &args);
 				continue;
 			    case __NR_futex:
 			    case __NR_brk:
@@ -3029,7 +3098,7 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 #endif /* POSTK_DEBUG_ARCH_DEP_78 */
 			    case __NR_execve:
 				set_syscall_number(&args, -1);
-				set_syscall_args(tid, &args);
+				set_syscall_args(uti_desc->tid, &args);
 				continue;
 			    case __NR_ioctl:
 				param = (struct syscall_struct *)
@@ -3038,7 +3107,7 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 				    get_syscall_arg1(&args) == fd &&
 				    get_syscall_arg2(&args) ==
 				                     MCEXEC_UP_SYSCALL_THREAD &&
-				    samepage(wp, param)) {
+				    samepage(uti_desc->wp, param)) {
 					set_syscall_arg1(&args, param->args[0]);
 					set_syscall_arg2(&args, param->args[1]);
 					set_syscall_arg3(&args, param->args[2]);
@@ -3048,7 +3117,7 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 					set_syscall_return(&args, param->ret);
 					*(void **)param = param_top;
 					param_top = param;
-					set_syscall_args(tid, &args);
+					set_syscall_args(uti_desc->tid, &args);
 				}
 				continue;
 			    default:
@@ -3068,6 +3137,7 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 				param->args[3] = get_syscall_arg4(&args);
 				param->args[4] = get_syscall_arg5(&args);
 				param->args[5] = get_syscall_arg6(&args);
+				param->uti_clv = uti_desc->uti_clv;
 				param->ret = -EINVAL;
 				set_syscall_number(&args, __NR_ioctl);
 				set_syscall_arg1(&args, fd);
@@ -3075,7 +3145,7 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 				                      MCEXEC_UP_SYSCALL_THREAD);
 				set_syscall_arg3(&args, (unsigned long)param);
 			}
-			set_syscall_args(tid, &args);
+			set_syscall_args(uti_desc->tid, &args);
 		}
 		else { // signal
 			sig = WSTOPSIG(st) & 0x7f;
@@ -3083,45 +3153,47 @@ create_tracer(void *wp, int mck_tid, unsigned long key)
 	}

 #ifdef DEBUG_UTI
-	fprintf(stderr, "offloaded thread called these syscalls\n");
-	debug_sig(0);
+	//fprintf(stderr, "offloaded thread called these syscalls\n");
+	//debug_sig(0);
 #endif

 	exit(0);
 }

 static long
-util_thread(unsigned long uctx_pa, int remote_tid, unsigned long pattr)
+util_thread(struct thread_data_s *my_thread, unsigned long uctx_pa, int remote_tid, unsigned long pattr, unsigned long uti_clv)
 {
 	void *lctx;
 	void *rctx;
-	void *wp;
 	void *param[6];
 	int rc = 0;
+	unsigned long buf;

-#ifdef POSTK_DEBUG_ARCH_DEP_35
-	wp = mmap(NULL, page_size * 3, PROT_READ | PROT_WRITE,
-	          MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-#else	/* POSTK_DEBUG_ARCH_DEP_35 */
-	wp = mmap(NULL, PAGE_SIZE * 3, PROT_READ | PROT_WRITE,
-	          MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-#endif	/* POSTK_DEBUG_ARCH_DEP_35 */
-	if (wp == (void *)-1) {
-		rc = -errno;
-		goto out;
+#if 0
+	{
+		int error;
+		if ((error = pipe(uti_pfd)) == -1) {
+			fprintf(stderr, "%s: pipe returned %d\n", __FUNCTION__, error);
+			rc = error; goto out;
+		}
+		if ((error = create_tracer())) {
+			fprintf(stderr, "%s: create_tracer returned %d\n", __FUNCTION__, error);
+			rc = error; goto out;
+		}
 	}
+#endif
 #ifdef POSTK_DEBUG_ARCH_DEP_35
-	lctx = (char *)wp + page_size;
+	lctx = (char *)uti_wp + page_size;
 	rctx = (char *)lctx + page_size;
-#else	/* POSTK_DEBUG_ARCH_DEP_35 */
-	lctx = (char *)wp + PAGE_SIZE;
+#else
+	lctx = (char *)uti_wp + PAGE_SIZE;
 	rctx = (char *)lctx + PAGE_SIZE;
 #endif	/* POSTK_DEBUG_ARCH_DEP_35 */

 	param[0] = (void *)uctx_pa;
 	param[1] = rctx;
 	param[2] = lctx;
-	param[4] = wp;
+	param[4] = uti_wp;
 #ifdef POSTK_DEBUG_ARCH_DEP_35
 	param[5] = (void *)(page_size * 3);
 #else	/* POSTK_DEBUG_ARCH_DEP_35 */
@@ -3134,11 +3206,37 @@ util_thread(unsigned long uctx_pa, int remote_tid, unsigned long pattr)
 	}

 	create_worker_thread(NULL);
-	if ((rc = create_tracer(wp, remote_tid, (unsigned long)param[3]))) {
-		fprintf(stderr, "create tracer %d\n", rc);
+
+	/* Pass info to the tracer so that it can masquerade as the tracee */
+	uti_desc->wp = uti_wp;
+	uti_desc->mck_tid = remote_tid;
+	uti_desc->key = (unsigned long)param[3];
+	uti_desc->pid = getpid();
+	uti_desc->tid = gettid();
+	uti_desc->uti_clv = uti_clv;
+	
+#if 0
+	//usleep(100000);
+	ssize_t nwritten;
+	char *cur;
+	for(cur = (char*)&uti_desc; (nwritten = write(uti_pfd[1], cur, sizeof(struct uti_desc) - (cur - (char*)&uti_desc))) > 0; cur += nwritten) { }
+	if (nwritten < 0) {
+		fprintf(stderr, "write returned %ld errno=%d\n", nwritten, errno);
 		rc = -errno;
 		goto out;
 	}
+	close(uti_pfd[1]);
+#endif
+	sem_post(&uti_desc->arg);
+
+    /* Wait until tracer attaches me. We can't use
+       futex because it would be captured and redirected by tracer */
+    rc = read(uti_pfd[0], &buf, sizeof(unsigned long));
+    if (rc != sizeof(unsigned long)) {
+        fprintf(stderr, "%s: write returned %d\n", __FUNCTION__, rc);
+        exit(1);
+    }
+	close(uti_pfd[0]);

 	if (pattr) {
 		struct uti_attr_desc desc;
@@ -3155,11 +3253,11 @@ util_thread(unsigned long uctx_pa, int remote_tid, unsigned long pattr)
 	pthread_exit(NULL);

 out:
-	if (wp)
+	if (uti_wp != (void*)-1)
 #ifdef POSTK_DEBUG_ARCH_DEP_35
-		munmap(wp, page_size * 3);
+		munmap(uti_wp, page_size * 3);
 #else	/* POSTK_DEBUG_ARCH_DEP_35 */
-		munmap(wp, PAGE_SIZE * 3);
+		munmap(uti_wp, PAGE_SIZE * 3);
 #endif	/* POSTK_DEBUG_ARCH_DEP_35 */
 	return rc;
 }
@@ -4225,8 +4323,8 @@ return_execve2:

 		case __NR_sched_setaffinity:
 			if (w.sr.args[0] == 0) {
-				ret = util_thread(w.sr.args[1], w.sr.rtid,
-				                  w.sr.args[2]);
+				ret = util_thread(my_thread, w.sr.args[1], w.sr.rtid,
+				                  w.sr.args[2], w.sr.args[3]);
 			}
 			else {
 				ret = munmap((void *)w.sr.args[1],