diff --git a/executer/kernel/mcctrl/control.c b/executer/kernel/mcctrl/control.c index d578b756..8beb2bcd 100644 --- a/executer/kernel/mcctrl/control.c +++ b/executer/kernel/mcctrl/control.c @@ -58,6 +58,13 @@ #define dprintk(...) #endif +//#define DEBUG_PTD +#ifdef DEBUG_PTD +#define pr_ptd(msg, tid, ptd) do { printk("%s: " msg ",tid=%d,refc=%d\n", __FUNCTION__, tid, atomic_read(&ptd->refcount)); } while(0) +#else +#define pr_ptd(msg, tid, ptd) do { } while(0) +#endif + //extern struct mcctrl_channel *channels; int mcctrl_ikc_set_recv_cpu(ihk_os_t os, int cpu); int syscall_backward(struct mcctrl_usrdata *, int, unsigned long, unsigned long, @@ -1077,6 +1084,8 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd) struct wait_queue_head_list_node *wqhln; struct wait_queue_head_list_node *wqhln_next; struct ikc_scd_packet *packet; + struct mcctrl_per_thread_data *ptd; + struct mcctrl_per_thread_data *next; if (!ppd) return; @@ -1095,25 +1104,32 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd) write_unlock_irqrestore(&ppd->ud->per_proc_data_hash_lock[hash], flags); dprintk("%s: deallocating PPD for pid %d\n", __FUNCTION__, ppd->pid); - for (i = 0; i < MCCTRL_PER_THREAD_DATA_HASH_SIZE; i++) { - struct mcctrl_per_thread_data *ptd; - struct mcctrl_per_thread_data *next; + for (i = 0; i < MCCTRL_PER_THREAD_DATA_HASH_SIZE; i++) { write_lock_irqsave(&ppd->per_thread_data_hash_lock[i], flags); list_for_each_entry_safe(ptd, next, ppd->per_thread_data_hash + i, hash) { - packet = ptd->data; - list_del(&ptd->hash); - kfree(ptd); + /* We use ERESTARTSYS to tell the LWK that the proxy - * process is gone and the application should be terminated */ + process is gone and the application should be terminated. */ + packet = (struct ikc_scd_packet *)ptd->data; + dprintk("%s: calling __return_syscall (hash),target pid=%d,tid=%d\n", __FUNCTION__, ppd->pid, packet->req.rtid); __return_syscall(ppd->ud->os, packet, -ERESTARTSYS, - packet->req.rtid); + packet->req.rtid); ihk_ikc_release_packet( (struct ihk_ikc_free_packet *)packet, (ppd->ud->ikc2linux[smp_processor_id()] ? ppd->ud->ikc2linux[smp_processor_id()] : ppd->ud->ikc2linux[0])); + + /* Note that uti ptd needs another put by mcexec_terminate_thread() + (see mcexec_syscall_wait()). + TODO: Detect tracer has died before calling mcexec_terminate_thread() and put uti ptd */ + if (atomic_read(&ptd->refcount) != 1) { + printk("%s: WARNING: ptd->refcount != 1 but %d\n", __FUNCTION__, atomic_read(&ptd->refcount)); + } + mcctrl_put_per_thread_data_unsafe(ptd); + pr_ptd("put", ptd->tid, ptd); } write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[i], flags); } @@ -1283,6 +1299,7 @@ int mcexec_wait_syscall(ihk_os_t os, struct syscall_wait_desc *__user req) int ret = 0; unsigned long irqflags; struct mcctrl_per_proc_data *ppd; + struct mcctrl_per_thread_data *ptd = NULL; /* Get a reference to per-process structure */ ppd = mcctrl_get_per_proc_data(usrdata, task_tgid_vnr(current)); @@ -1293,12 +1310,13 @@ int mcexec_wait_syscall(ihk_os_t os, struct syscall_wait_desc *__user req) return -EINVAL; } - packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); - if (packet) { + ptd = mcctrl_get_per_thread_data(ppd, current); + if (ptd) { printk("%s: ERROR: packet %p is already registered for thread %d\n", - __FUNCTION__, packet, task_pid_vnr(current)); + __FUNCTION__, ptd->data, task_pid_vnr(current)); + mcctrl_put_per_thread_data(ptd); ret = -EBUSY; - goto put_ppd_out; + goto no_ptd; } retry: @@ -1364,7 +1382,7 @@ retry_alloc: } wqhln = NULL; ret = -EINTR; - goto put_ppd_out; + goto no_ptd; } packet = wqhln->packet; @@ -1400,27 +1418,43 @@ retry_alloc: packet->req.args[4], packet->req.args[5]); - if (mcctrl_add_per_thread_data(ppd, current, packet) < 0) { - kprintf("%s: error adding per-thread data\n", __FUNCTION__); - ret = -EINVAL;; - goto put_ppd_out; + /* Create ptd */ + if ((ret = mcctrl_add_per_thread_data(ppd, packet))) { + kprintf("%s: error adding per-thread data (%d)\n", __FUNCTION__, ret); + ret = -EINVAL; + goto no_ptd; + } + + /* Get a reference valid until offload is done */ + ptd = mcctrl_get_per_thread_data(ppd, current); + if (!ptd) { + kprintf("%s: ERROR: ptd not found\n", __FUNCTION__); + ret = -EINVAL; + goto no_ptd; + } + pr_ptd("get", task_pid_vnr(current), ptd); + + if (packet->req.number == __NR_sched_setaffinity && packet->req.args[0] == 0) { + dprintk("%s: uti,packet=%p,tid=%d\n", __FUNCTION__, packet, task_pid_vnr(current)); + + /* Get a reference valid until thread-offload is done */ + ptd = mcctrl_get_per_thread_data(ppd, current); + if (!ptd) { + kprintf("%s: ptd not found\n", __FUNCTION__); + ret = -EINVAL; + goto no_ptd; + } + pr_ptd("get", task_pid_vnr(current), ptd); } if (__do_in_kernel_syscall(os, packet)) { if (copy_to_user(&req->sr, &packet->req, sizeof(struct syscall_request))) { - - if (mcctrl_delete_per_thread_data(ppd, current) < 0) { - kprintf("%s: error deleting per-thread data\n", __FUNCTION__); - } - ret = -EINVAL;; + ret = -EINVAL; goto put_ppd_out; } if (copy_to_user(&req->cpu, &packet->ref, sizeof(req->cpu))) { - if (mcctrl_delete_per_thread_data(ppd, current) < 0) { - kprintf("%s: error deleting per-thread data\n", __FUNCTION__); - } ret = -EINVAL; goto put_ppd_out; } @@ -1434,15 +1468,18 @@ retry_alloc: usrdata->ikc2linux[smp_processor_id()] : usrdata->ikc2linux[0])); - if (mcctrl_delete_per_thread_data(ppd, current) < 0) { - kprintf("%s: error deleting per-thread data\n", __FUNCTION__); - ret = -EINVAL;; - goto put_ppd_out; - } + /* Drop reference to zero and restart from add */ + mcctrl_put_per_thread_data(ptd); + pr_ptd("put,in_kernel", task_pid_vnr(current), ptd); + mcctrl_put_per_thread_data(ptd); + pr_ptd("put,in_kernel", task_pid_vnr(current), ptd); goto retry; put_ppd_out: + mcctrl_put_per_thread_data(ptd); + pr_ptd("put,in_mcexec", task_pid_vnr(current), ptd); + no_ptd: mcctrl_put_per_proc_data(ppd); return ret; } @@ -1537,6 +1574,7 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg) struct ikc_scd_packet *packet; struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os); struct mcctrl_per_proc_data *ppd; + struct mcctrl_per_thread_data *ptd; int error = 0; if (copy_from_user(&ret, arg, sizeof(struct syscall_ret_desc))) { @@ -1551,16 +1589,22 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg) return -EINVAL; } - packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); + /* Get a reference for this function */ + ptd = mcctrl_get_per_thread_data(ppd, current); + if (!ptd) { + printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__); + error = -EINVAL; + goto no_ptd; + } + pr_ptd("get", task_pid_vnr(current), ptd); + packet = (struct ikc_scd_packet *)ptd->data; if (!packet) { kprintf("%s: ERROR: no packet registered for TID %d\n", __FUNCTION__, task_pid_vnr(current)); error = -EINVAL; - goto out; + goto put_ppd_out; } - mcctrl_delete_per_thread_data(ppd, current); - if (ret.size > 0) { /* Host => Accel. Write is fast. */ unsigned long phys; @@ -1595,7 +1639,15 @@ out: (usrdata->ikc2linux[smp_processor_id()] ? usrdata->ikc2linux[smp_processor_id()] : usrdata->ikc2linux[0])); - + put_ppd_out: + /* Drop a reference for this function */ + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", task_pid_vnr(current), ptd); + + /* Final drop of the reference for non-uti syscall offloading */ + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", task_pid_vnr(current), ptd); + no_ptd: mcctrl_put_per_proc_data(ppd); return error; } @@ -2452,6 +2504,7 @@ mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long sig, struct t struct ikc_scd_packet *packet; struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os); struct mcctrl_per_proc_data *ppd; + struct mcctrl_per_thread_data *ptd; dprintk("%s: target pid=%d,tid=%d,sig=%lx,task=%p\n", __FUNCTION__, pid, tid, sig, tsk); @@ -2461,30 +2514,48 @@ mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long sig, struct t __FUNCTION__, pid); goto no_ppd; } - packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, tsk); + + ptd = mcctrl_get_per_thread_data(ppd, tsk); + if (!ptd) { + printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__); + goto no_ptd; + } + if (ptd->tid != tid) { + printk("%s: ERROR: ptd->tid(%d) != tid(%d)\n", __FUNCTION__, ptd->tid, tid); + goto no_ptd; + } + pr_ptd("get", tid, ptd); + + packet = (struct ikc_scd_packet *)ptd->data; if (!packet) { kprintf("%s: ERROR: no packet registered for TID %d\n", __FUNCTION__, tid); goto no_ptd; } - - if ((rc = mcctrl_delete_per_thread_data(ppd, tsk))) { - kprintf("%s: ERROR: mcctrl_delete_per_thread_data failed (%d)\n", __FUNCTION__, rc); - goto no_ptd; - } - __return_syscall(usrdata->os, packet, sig, tid); ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet, (usrdata->ikc2linux[smp_processor_id()] ? usrdata->ikc2linux[smp_processor_id()] : usrdata->ikc2linux[0])); + + /* Drop reference for this function */ + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", tid, ptd); + + /* Final drop of reference for uti ptd */ + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", tid, ptd); + + if (atomic_read(&ptd->refcount) != 1) { + printk("%s: WARNING: ptd->refcount != 1 but %d\n", __FUNCTION__, atomic_read(&ptd->refcount)); + } + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", tid, ptd); no_ptd: - /* Destroy per_proc_data with the following two puts. Note that - it survived the signal-kill of tracee thanks to the additional put - done in mcexec_util_thread2 */ - mcctrl_put_per_proc_data(ppd); mcctrl_put_per_proc_data(ppd); + /* This is the final drop of uti-ppd */ + mcctrl_put_per_proc_data(ppd); no_ppd: return 0; } @@ -3094,6 +3165,7 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu) { struct mcctrl_usrdata *usrdata; struct mcctrl_per_proc_data *ppd; + struct mcctrl_per_thread_data *ptd; struct ikc_scd_packet *packet; struct ihk_ikc_channel_desc *ch; int ret = 0; @@ -3118,11 +3190,18 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu) } /* Look up per-thread structure */ - packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); - if (!packet) { + ptd = mcctrl_get_per_thread_data(ppd, current); + if (!ptd) { + printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__); ret = -EINVAL; + goto no_ptd; + } + pr_ptd("get", task_pid_vnr(current), ptd); + packet = (struct ikc_scd_packet *)ptd->data; + if (!packet) { printk("%s: ERROR: no packet registered for TID %d\n", __FUNCTION__, task_pid_vnr(current)); + ret = -EINVAL; goto out_put_ppd; } @@ -3135,6 +3214,9 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu) printk("%s: OS: %p, CPU: %d\n", __FUNCTION__, os, *ret_cpu); out_put_ppd: + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", task_pid_vnr(current), ptd); + no_ptd: mcctrl_put_per_proc_data(ppd); return ret; diff --git a/executer/kernel/mcctrl/mcctrl.h b/executer/kernel/mcctrl/mcctrl.h index 178f1d9a..651263eb 100644 --- a/executer/kernel/mcctrl/mcctrl.h +++ b/executer/kernel/mcctrl/mcctrl.h @@ -228,9 +228,12 @@ struct mcctrl_channel { }; struct mcctrl_per_thread_data { + struct mcctrl_per_proc_data *ppd; struct list_head hash; struct task_struct *task; void *data; + int tid; /* debug */ + atomic_t refcount; }; #define MCCTRL_PER_THREAD_DATA_HASH_SHIFT 8 @@ -447,20 +450,18 @@ struct mcctrl_per_proc_data *mcctrl_get_per_proc_data( struct mcctrl_usrdata *ud, int pid); void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd); -int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data* ppd, - struct task_struct *task, void *data); -int mcctrl_delete_per_thread_data(struct mcctrl_per_proc_data* ppd, - struct task_struct *task); +int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data *ppd, void *data); +void mcctrl_put_per_thread_data_unsafe(struct mcctrl_per_thread_data *ptd); +void mcctrl_put_per_thread_data(struct mcctrl_per_thread_data* ptd); #ifdef POSTK_DEBUG_ARCH_DEP_56 /* Strange how to use inline declaration fix. */ -static inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data( - struct mcctrl_per_proc_data *ppd, struct task_struct *task) +inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc_data *ppd, struct task_struct *task) { struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL; int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); unsigned long flags; - /* Check if data for this thread exists and return it */ - read_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); + /* Check if data for this thread exists */ + write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { if (ptd_iter->task == task) { @@ -469,12 +470,21 @@ static inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data( } } - read_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); - return ptd ? ptd->data : NULL; + if (ptd) { + if (atomic_read(&ptd->refcount) <= 0) { + printk("%s: ERROR: use-after-free detected (%d)", __FUNCTION__, atomic_read(&ptd->refcount)); + ptd = NULL; + goto out; + } + atomic_inc(&ptd->refcount); + } + + out: + write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); + return ptd; } #else /* POSTK_DEBUG_ARCH_DEP_56 */ -inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data( - struct mcctrl_per_proc_data *ppd, struct task_struct *task); +inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc_data *ppd, struct task_struct *task); #endif /* POSTK_DEBUG_ARCH_DEP_56 */ void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, diff --git a/executer/kernel/mcctrl/syscall.c b/executer/kernel/mcctrl/syscall.c index a52417f2..15fabb61 100644 --- a/executer/kernel/mcctrl/syscall.c +++ b/executer/kernel/mcctrl/syscall.c @@ -63,6 +63,13 @@ #define dprintk(...) #endif +//#define DEBUG_PTD +#ifdef DEBUG_PTD +#define pr_ptd(msg, tid, ptd) do { printk("%s: " msg ",tid=%d,refc=%d\n", __FUNCTION__, tid, atomic_read(&ptd->refcount)); } while(0) +#else +#define pr_ptd(msg, tid, ptd) do { } while(0) +#endif + static long pager_call_irq(ihk_os_t os, struct syscall_request *req); static long pager_call(ihk_os_t os, struct syscall_request *req); @@ -80,75 +87,90 @@ static void print_dma_lastreq(void) } #endif -int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data* ppd, - struct task_struct *task, void *data) +void mcctrl_put_per_thread_data_unsafe(struct mcctrl_per_thread_data *ptd) { - struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL; - struct mcctrl_per_thread_data *ptd_alloc = NULL; - int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); - int ret = 0; - unsigned long flags; - - ptd_alloc = kmalloc(sizeof(*ptd), GFP_ATOMIC); - if (!ptd_alloc) { - kprintf("%s: error allocate per thread data\n", __FUNCTION__); - ret = -ENOMEM; - goto out_noalloc; - } - - /* Check if data for this thread exists and add if not */ - write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); - list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { - if (ptd_iter->task == task) { - ptd = ptd_iter; - break; + if (!atomic_dec_and_test(&ptd->refcount)) { + int ret = atomic_read(&ptd->refcount); + if (ret < 0) { + printk("%s: ERROR: invalid refcount=%d\n", __FUNCTION__, ret); } + return; } - if (unlikely(ptd)) { - ret = -EBUSY; - kfree(ptd_alloc); - goto out; - } - - ptd = ptd_alloc; - ptd->task = task; - ptd->data = data; - list_add_tail(&ptd->hash, &ppd->per_thread_data_hash[hash]); - -out: - write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); -out_noalloc: - return ret; + list_del(&ptd->hash); + kfree(ptd); } -int mcctrl_delete_per_thread_data(struct mcctrl_per_proc_data* ppd, - struct task_struct *task) +void mcctrl_put_per_thread_data(struct mcctrl_per_thread_data* _ptd) { + struct mcctrl_per_proc_data *ppd = _ptd->ppd; struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL; - int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); - int ret = 0; + int hash = (((uint64_t)_ptd->task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); unsigned long flags; - + /* Check if data for this thread exists and delete it */ write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { - if (ptd_iter->task == task) { + if (ptd_iter->task == _ptd->task) { ptd = ptd_iter; break; } } if (!ptd) { - ret = -EINVAL; + printk("%s: ERROR: ptd not found\n", __FUNCTION__); goto out; } - list_del(&ptd->hash); - kfree(ptd); - + mcctrl_put_per_thread_data_unsafe(ptd); + out: write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); +} + +int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data *ppd, void *data) +{ + struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL; + struct mcctrl_per_thread_data *ptd_alloc = NULL; + int hash = (((uint64_t)current >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); + int ret = 0; + unsigned long flags; + + ptd_alloc = kmalloc(sizeof(struct mcctrl_per_thread_data), GFP_ATOMIC); + if (!ptd_alloc) { + kprintf("%s: error allocate per thread data\n", __FUNCTION__); + ret = -ENOMEM; + goto out_noalloc; + } + memset(ptd_alloc, 0, sizeof(struct mcctrl_per_thread_data)); + + /* Check if data for this thread exists and add if not */ + write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); + list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { + if (ptd_iter->task == current) { + ptd = ptd_iter; + break; + } + } + + if (unlikely(ptd)) { + kprintf("%s: WARNING: ptd of tid: %d exists\n", __FUNCTION__, task_pid_vnr(current)); + ret = -EBUSY; + kfree(ptd_alloc); + goto out; + } + + ptd = ptd_alloc; + ptd->ppd = ppd; + ptd->task = current; + ptd->tid = task_pid_vnr(current); + ptd->data = data; + atomic_set(&ptd->refcount, 1); + list_add_tail(&ptd->hash, &ppd->per_thread_data_hash[hash]); + + out: + write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); + out_noalloc: return ret; } @@ -159,7 +181,7 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); unsigned long flags; - /* Check if data for this thread exists and return it */ + /* Check if data for this thread exists */ read_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { @@ -169,8 +191,18 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc } } + if (ptd) { + if (atomic_read(&ptd->refcount) <= 0) { + printk("%s: ERROR: use-after-free detected (%d)", __FUNCTION__, atomic_read(&ptd->refcount)); + ptd = NULL; + goto out; + } + atomic_inc(&ptd->refcount); + } + + out: read_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); - return ptd ? ptd->data : NULL; + return ptd; } #endif /* !POSTK_DEBUG_ARCH_DEP_56 */ @@ -299,6 +331,7 @@ long syscall_backward(struct mcctrl_usrdata *usrdata, int num, struct wait_queue_head_list_node *wqhln; unsigned long irqflags; struct mcctrl_per_proc_data *ppd; + struct mcctrl_per_thread_data *ptd; unsigned long phys; struct syscall_request _request[2]; struct syscall_request *request; @@ -327,7 +360,14 @@ long syscall_backward(struct mcctrl_usrdata *usrdata, int num, return -EINVAL; } - packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); + ptd = mcctrl_get_per_thread_data(ppd, current); + if (!ptd) { + printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__); + syscall_ret = -ENOENT; + goto no_ptd; + } + pr_ptd("get", task_pid_vnr(current), ptd); + packet = (struct ikc_scd_packet *)ptd->data; if (!packet) { syscall_ret = -ENOENT; printk("%s: no packet registered for TID %d\n", @@ -466,6 +506,9 @@ out: ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp)); out_put_ppd: + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", task_pid_vnr(current), ptd); + no_ptd: dprintk("%s: tid: %d, syscall: %d, syscall_ret: %lx\n", __FUNCTION__, task_pid_vnr(current), num, syscall_ret); @@ -483,6 +526,7 @@ int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t struct wait_queue_head_list_node *wqhln; unsigned long irqflags; struct mcctrl_per_proc_data *ppd; + struct mcctrl_per_thread_data *ptd; unsigned long phys; int retry; @@ -498,11 +542,18 @@ int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t return -EINVAL; } - packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); - if (!packet) { + ptd = mcctrl_get_per_thread_data(ppd, current); + if (!ptd) { + printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__); error = -ENOENT; + goto no_ptd; + } + pr_ptd("get", task_pid_vnr(current), ptd); + packet = (struct ikc_scd_packet *)ptd->data; + if (!packet) { printk("%s: no packet registered for TID %d\n", __FUNCTION__, task_pid_vnr(current)); + error = -ENOENT; goto out_put_ppd; } @@ -669,6 +720,9 @@ out: ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp)); out_put_ppd: + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", task_pid_vnr(current), ptd); + no_ptd: dprintk("%s: tid: %d, fault_addr: %p, reason: %lu, error: %d\n", __FUNCTION__, task_pid_vnr(current), fault_addr, (unsigned long)reason, error); @@ -711,6 +765,7 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) size_t pix; #endif struct mcctrl_per_proc_data *ppd; + struct mcctrl_per_thread_data *ptd; struct ikc_scd_packet *packet; int ret = 0; @@ -740,7 +795,14 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) goto no_ppd; } - packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); + ptd = mcctrl_get_per_thread_data(ppd, current); + if (!ptd) { + printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__); + ret = VM_FAULT_SIGBUS; + goto no_ptd; + } + pr_ptd("get", task_pid_vnr(current), ptd); + packet = (struct ikc_scd_packet *)ptd->data; if (!packet) { ret = VM_FAULT_SIGBUS; printk("%s: no packet registered for TID %d\n", @@ -929,6 +991,9 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) ret = VM_FAULT_NOPAGE; put_and_out: + mcctrl_put_per_thread_data(ptd); + pr_ptd("put", task_pid_vnr(current), ptd); + no_ptd: mcctrl_put_per_proc_data(ppd); no_ppd: return ret;