gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reference counting per-thread data

Browse Source 
It is accompanied by the following fixes:
(1) Fix put ppd locations in mcexec_wait_syscall()
(2) Move put ptd to end of mcexec_terminate_thread_unsafe() and mcexec_ret_syscall()
(3) Add debug messages for ptd add/get/put
(4) Fix ptd-add/get/put matching in mcexec_wait_syscall()
    * Skip put when woken-up from wait_event_interruptible() by signal

Change-Id: Ib9be3f5e62a7a370197cb36c9fa7c4d79f44c314
This commit is contained in:
Masamichi Takagi
2018-09-03 19:36:28 +09:00
parent a121ffc785
commit b8bacdd2de
3 changed files with 269 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
executer/kernel/mcctrl/control.c
View File

@@ -58,6 +58,13 @@
#define dprintk(...) #define dprintk(...)
#endif #endif
//#define DEBUG_PTD
#ifdef DEBUG_PTD
#define pr_ptd(msg, tid, ptd) do { printk("%s: " msg ",tid=%d,refc=%d\n", __FUNCTION__, tid, atomic_read(&ptd->refcount)); } while(0)
#else
#define pr_ptd(msg, tid, ptd) do { } while(0)
#endif
//extern struct mcctrl_channel *channels; //extern struct mcctrl_channel *channels;
int mcctrl_ikc_set_recv_cpu(ihk_os_t os, int cpu); int mcctrl_ikc_set_recv_cpu(ihk_os_t os, int cpu);
int syscall_backward(struct mcctrl_usrdata *, int, unsigned long, unsigned long, int syscall_backward(struct mcctrl_usrdata *, int, unsigned long, unsigned long,
@@ -1077,6 +1084,8 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd)
struct wait_queue_head_list_node *wqhln; struct wait_queue_head_list_node *wqhln;
struct wait_queue_head_list_node *wqhln_next; struct wait_queue_head_list_node *wqhln_next;
struct ikc_scd_packet *packet; struct ikc_scd_packet *packet;
struct mcctrl_per_thread_data *ptd;
struct mcctrl_per_thread_data *next;
if (!ppd) if (!ppd)
return; return;
@@ -1095,25 +1104,32 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd)
write_unlock_irqrestore(&ppd->ud->per_proc_data_hash_lock[hash], flags); write_unlock_irqrestore(&ppd->ud->per_proc_data_hash_lock[hash], flags);
dprintk("%s: deallocating PPD for pid %d\n", __FUNCTION__, ppd->pid); dprintk("%s: deallocating PPD for pid %d\n", __FUNCTION__, ppd->pid);
for (i = 0; i < MCCTRL_PER_THREAD_DATA_HASH_SIZE; i++) {
struct mcctrl_per_thread_data *ptd;
struct mcctrl_per_thread_data *next;
for (i = 0; i < MCCTRL_PER_THREAD_DATA_HASH_SIZE; i++) {
write_lock_irqsave(&ppd->per_thread_data_hash_lock[i], flags); write_lock_irqsave(&ppd->per_thread_data_hash_lock[i], flags);
list_for_each_entry_safe(ptd, next, list_for_each_entry_safe(ptd, next,
ppd->per_thread_data_hash + i, hash) { ppd->per_thread_data_hash + i, hash) {
packet = ptd->data;
list_del(&ptd->hash);
kfree(ptd);
/* We use ERESTARTSYS to tell the LWK that the proxy /* We use ERESTARTSYS to tell the LWK that the proxy
* process is gone and the application should be terminated */ process is gone and the application should be terminated. */
packet = (struct ikc_scd_packet *)ptd->data;
dprintk("%s: calling __return_syscall (hash),target pid=%d,tid=%d\n", __FUNCTION__, ppd->pid, packet->req.rtid);
__return_syscall(ppd->ud->os, packet, -ERESTARTSYS, __return_syscall(ppd->ud->os, packet, -ERESTARTSYS,
packet->req.rtid); packet->req.rtid);
ihk_ikc_release_packet( ihk_ikc_release_packet(
(struct ihk_ikc_free_packet *)packet, (struct ihk_ikc_free_packet *)packet,
(ppd->ud->ikc2linux[smp_processor_id()] ? (ppd->ud->ikc2linux[smp_processor_id()] ?
ppd->ud->ikc2linux[smp_processor_id()] : ppd->ud->ikc2linux[smp_processor_id()] :
ppd->ud->ikc2linux[0])); ppd->ud->ikc2linux[0]));
/* Note that uti ptd needs another put by mcexec_terminate_thread()
(see mcexec_syscall_wait()).
TODO: Detect tracer has died before calling mcexec_terminate_thread() and put uti ptd */
if (atomic_read(&ptd->refcount) != 1) {
printk("%s: WARNING: ptd->refcount != 1 but %d\n", __FUNCTION__, atomic_read(&ptd->refcount));
}
mcctrl_put_per_thread_data_unsafe(ptd);
pr_ptd("put", ptd->tid, ptd);
} }
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[i], flags); write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[i], flags);
} }
@@ -1283,6 +1299,7 @@ int mcexec_wait_syscall(ihk_os_t os, struct syscall_wait_desc *__user req)
int ret = 0; int ret = 0;
unsigned long irqflags; unsigned long irqflags;
struct mcctrl_per_proc_data *ppd; struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd = NULL;
/* Get a reference to per-process structure */ /* Get a reference to per-process structure */
ppd = mcctrl_get_per_proc_data(usrdata, task_tgid_vnr(current)); ppd = mcctrl_get_per_proc_data(usrdata, task_tgid_vnr(current));
@@ -1293,12 +1310,13 @@ int mcexec_wait_syscall(ihk_os_t os, struct syscall_wait_desc *__user req)
return -EINVAL; return -EINVAL;
} }
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); ptd = mcctrl_get_per_thread_data(ppd, current);
if (packet) { if (ptd) {
printk("%s: ERROR: packet %p is already registered for thread %d\n", printk("%s: ERROR: packet %p is already registered for thread %d\n",
__FUNCTION__, packet, task_pid_vnr(current)); __FUNCTION__, ptd->data, task_pid_vnr(current));
mcctrl_put_per_thread_data(ptd);
ret = -EBUSY; ret = -EBUSY;
goto put_ppd_out; goto no_ptd;
} }
retry: retry:
@@ -1364,7 +1382,7 @@ retry_alloc:
} }
wqhln = NULL; wqhln = NULL;
ret = -EINTR; ret = -EINTR;
goto put_ppd_out; goto no_ptd;
} }
packet = wqhln->packet; packet = wqhln->packet;
@@ -1400,27 +1418,43 @@ retry_alloc:
packet->req.args[4], packet->req.args[4],
packet->req.args[5]); packet->req.args[5]);
if (mcctrl_add_per_thread_data(ppd, current, packet) < 0) { /* Create ptd */
kprintf("%s: error adding per-thread data\n", __FUNCTION__); if ((ret = mcctrl_add_per_thread_data(ppd, packet))) {
ret = -EINVAL;; kprintf("%s: error adding per-thread data (%d)\n", __FUNCTION__, ret);
goto put_ppd_out; ret = -EINVAL;
goto no_ptd;
}
/* Get a reference valid until offload is done */
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
kprintf("%s: ERROR: ptd not found\n", __FUNCTION__);
ret = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
if (packet->req.number == __NR_sched_setaffinity && packet->req.args[0] == 0) {
dprintk("%s: uti,packet=%p,tid=%d\n", __FUNCTION__, packet, task_pid_vnr(current));
/* Get a reference valid until thread-offload is done */
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
kprintf("%s: ptd not found\n", __FUNCTION__);
ret = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
} }
if (__do_in_kernel_syscall(os, packet)) { if (__do_in_kernel_syscall(os, packet)) {
if (copy_to_user(&req->sr, &packet->req, if (copy_to_user(&req->sr, &packet->req,
sizeof(struct syscall_request))) { sizeof(struct syscall_request))) {
ret = -EINVAL;
if (mcctrl_delete_per_thread_data(ppd, current) < 0) {
kprintf("%s: error deleting per-thread data\n", __FUNCTION__);
}
ret = -EINVAL;;
goto put_ppd_out; goto put_ppd_out;
} }
if (copy_to_user(&req->cpu, &packet->ref, sizeof(req->cpu))) { if (copy_to_user(&req->cpu, &packet->ref, sizeof(req->cpu))) {
if (mcctrl_delete_per_thread_data(ppd, current) < 0) {
kprintf("%s: error deleting per-thread data\n", __FUNCTION__);
}
ret = -EINVAL; ret = -EINVAL;
goto put_ppd_out; goto put_ppd_out;
} }
@@ -1434,15 +1468,18 @@ retry_alloc:
usrdata->ikc2linux[smp_processor_id()] : usrdata->ikc2linux[smp_processor_id()] :
usrdata->ikc2linux[0])); usrdata->ikc2linux[0]));
if (mcctrl_delete_per_thread_data(ppd, current) < 0) { /* Drop reference to zero and restart from add */
kprintf("%s: error deleting per-thread data\n", __FUNCTION__); mcctrl_put_per_thread_data(ptd);
ret = -EINVAL;; pr_ptd("put,in_kernel", task_pid_vnr(current), ptd);
goto put_ppd_out;
}
mcctrl_put_per_thread_data(ptd);
pr_ptd("put,in_kernel", task_pid_vnr(current), ptd);
goto retry; goto retry;
put_ppd_out: put_ppd_out:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put,in_mcexec", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd); mcctrl_put_per_proc_data(ppd);
return ret; return ret;
} }
@@ -1537,6 +1574,7 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg)
struct ikc_scd_packet *packet; struct ikc_scd_packet *packet;
struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os); struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
struct mcctrl_per_proc_data *ppd; struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
int error = 0; int error = 0;
if (copy_from_user(&ret, arg, sizeof(struct syscall_ret_desc))) { if (copy_from_user(&ret, arg, sizeof(struct syscall_ret_desc))) {
@@ -1551,16 +1589,22 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg)
return -EINVAL; return -EINVAL;
} }
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); /* Get a reference for this function */
ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
error = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) { if (!packet) {
kprintf("%s: ERROR: no packet registered for TID %d\n", kprintf("%s: ERROR: no packet registered for TID %d\n",
__FUNCTION__, task_pid_vnr(current)); __FUNCTION__, task_pid_vnr(current));
error = -EINVAL; error = -EINVAL;
goto out; goto put_ppd_out;
} }
mcctrl_delete_per_thread_data(ppd, current);
if (ret.size > 0) { if (ret.size > 0) {
/* Host => Accel. Write is fast. */ /* Host => Accel. Write is fast. */
unsigned long phys; unsigned long phys;
@@ -1595,7 +1639,15 @@ out:
(usrdata->ikc2linux[smp_processor_id()] ? (usrdata->ikc2linux[smp_processor_id()] ?
usrdata->ikc2linux[smp_processor_id()] : usrdata->ikc2linux[smp_processor_id()] :
usrdata->ikc2linux[0])); usrdata->ikc2linux[0]));
put_ppd_out:
/* Drop a reference for this function */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
/* Final drop of the reference for non-uti syscall offloading */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd); mcctrl_put_per_proc_data(ppd);
return error; return error;
} }
@@ -2452,6 +2504,7 @@ mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long sig, struct t
struct ikc_scd_packet *packet; struct ikc_scd_packet *packet;
struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os); struct mcctrl_usrdata *usrdata = ihk_host_os_get_usrdata(os);
struct mcctrl_per_proc_data *ppd; struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
dprintk("%s: target pid=%d,tid=%d,sig=%lx,task=%p\n", __FUNCTION__, pid, tid, sig, tsk); dprintk("%s: target pid=%d,tid=%d,sig=%lx,task=%p\n", __FUNCTION__, pid, tid, sig, tsk);
@@ -2461,30 +2514,48 @@ mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long sig, struct t
__FUNCTION__, pid); __FUNCTION__, pid);
goto no_ppd; goto no_ppd;
} }
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, tsk);
ptd = mcctrl_get_per_thread_data(ppd, tsk);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
goto no_ptd;
}
if (ptd->tid != tid) {
printk("%s: ERROR: ptd->tid(%d) != tid(%d)\n", __FUNCTION__, ptd->tid, tid);
goto no_ptd;
}
pr_ptd("get", tid, ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) { if (!packet) {
kprintf("%s: ERROR: no packet registered for TID %d\n", kprintf("%s: ERROR: no packet registered for TID %d\n",
__FUNCTION__, tid); __FUNCTION__, tid);
goto no_ptd; goto no_ptd;
} }
if ((rc = mcctrl_delete_per_thread_data(ppd, tsk))) {
kprintf("%s: ERROR: mcctrl_delete_per_thread_data failed (%d)\n", __FUNCTION__, rc);
goto no_ptd;
}
__return_syscall(usrdata->os, packet, sig, tid); __return_syscall(usrdata->os, packet, sig, tid);
ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet, ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet,
(usrdata->ikc2linux[smp_processor_id()] ? (usrdata->ikc2linux[smp_processor_id()] ?
usrdata->ikc2linux[smp_processor_id()] : usrdata->ikc2linux[smp_processor_id()] :
usrdata->ikc2linux[0])); usrdata->ikc2linux[0]));
/* Drop reference for this function */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", tid, ptd);
/* Final drop of reference for uti ptd */
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", tid, ptd);
if (atomic_read(&ptd->refcount) != 1) {
printk("%s: WARNING: ptd->refcount != 1 but %d\n", __FUNCTION__, atomic_read(&ptd->refcount));
}
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", tid, ptd);
no_ptd: no_ptd:
/* Destroy per_proc_data with the following two puts. Note that
it survived the signal-kill of tracee thanks to the additional put
done in mcexec_util_thread2 */
mcctrl_put_per_proc_data(ppd);
mcctrl_put_per_proc_data(ppd); mcctrl_put_per_proc_data(ppd);
/* This is the final drop of uti-ppd */
mcctrl_put_per_proc_data(ppd);
no_ppd: no_ppd:
return 0; return 0;
} }
@@ -3094,6 +3165,7 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
{ {
struct mcctrl_usrdata *usrdata; struct mcctrl_usrdata *usrdata;
struct mcctrl_per_proc_data *ppd; struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
struct ikc_scd_packet *packet; struct ikc_scd_packet *packet;
struct ihk_ikc_channel_desc *ch; struct ihk_ikc_channel_desc *ch;
int ret = 0; int ret = 0;
@@ -3118,11 +3190,18 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
} }
/* Look up per-thread structure */ /* Look up per-thread structure */
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); ptd = mcctrl_get_per_thread_data(ppd, current);
if (!packet) { if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
ret = -EINVAL; ret = -EINVAL;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
printk("%s: ERROR: no packet registered for TID %d\n", printk("%s: ERROR: no packet registered for TID %d\n",
__FUNCTION__, task_pid_vnr(current)); __FUNCTION__, task_pid_vnr(current));
ret = -EINVAL;
goto out_put_ppd; goto out_put_ppd;
} }
@@ -3135,6 +3214,9 @@ int mcctrl_get_request_os_cpu(ihk_os_t os, int *ret_cpu)
printk("%s: OS: %p, CPU: %d\n", __FUNCTION__, os, *ret_cpu); printk("%s: OS: %p, CPU: %d\n", __FUNCTION__, os, *ret_cpu);
out_put_ppd: out_put_ppd:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd); mcctrl_put_per_proc_data(ppd);
return ret; return ret;

34
executer/kernel/mcctrl/mcctrl.h
View File

@@ -228,9 +228,12 @@ struct mcctrl_channel {
}; };
struct mcctrl_per_thread_data { struct mcctrl_per_thread_data {
struct mcctrl_per_proc_data *ppd;
struct list_head hash; struct list_head hash;
struct task_struct *task; struct task_struct *task;
void *data; void *data;
int tid; /* debug */
atomic_t refcount;
}; };
#define MCCTRL_PER_THREAD_DATA_HASH_SHIFT 8 #define MCCTRL_PER_THREAD_DATA_HASH_SHIFT 8
@@ -447,20 +450,18 @@ struct mcctrl_per_proc_data *mcctrl_get_per_proc_data(
struct mcctrl_usrdata *ud, int pid); struct mcctrl_usrdata *ud, int pid);
void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd); void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd);
int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data* ppd, int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data *ppd, void *data);
struct task_struct *task, void *data); void mcctrl_put_per_thread_data_unsafe(struct mcctrl_per_thread_data *ptd);
int mcctrl_delete_per_thread_data(struct mcctrl_per_proc_data* ppd, void mcctrl_put_per_thread_data(struct mcctrl_per_thread_data* ptd);
struct task_struct *task);
#ifdef POSTK_DEBUG_ARCH_DEP_56 /* Strange how to use inline declaration fix. */ #ifdef POSTK_DEBUG_ARCH_DEP_56 /* Strange how to use inline declaration fix. */
static inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data( inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc_data *ppd, struct task_struct *task)
struct mcctrl_per_proc_data *ppd, struct task_struct *task)
{ {
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL; struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL;
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
unsigned long flags; unsigned long flags;
/* Check if data for this thread exists and return it */ /* Check if data for this thread exists */
read_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == task) { if (ptd_iter->task == task) {
@@ -469,12 +470,21 @@ static inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(
} }
} }
read_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); if (ptd) {
return ptd ? ptd->data : NULL; if (atomic_read(&ptd->refcount) <= 0) {
printk("%s: ERROR: use-after-free detected (%d)", __FUNCTION__, atomic_read(&ptd->refcount));
ptd = NULL;
goto out;
}
atomic_inc(&ptd->refcount);
}
out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
return ptd;
} }
#else /* POSTK_DEBUG_ARCH_DEP_56 */ #else /* POSTK_DEBUG_ARCH_DEP_56 */
inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data( inline struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc_data *ppd, struct task_struct *task);
struct mcctrl_per_proc_data *ppd, struct task_struct *task);
#endif /* POSTK_DEBUG_ARCH_DEP_56 */ #endif /* POSTK_DEBUG_ARCH_DEP_56 */
void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet,

165
executer/kernel/mcctrl/syscall.c
View File

@@ -63,6 +63,13 @@
#define dprintk(...) #define dprintk(...)
#endif #endif
//#define DEBUG_PTD
#ifdef DEBUG_PTD
#define pr_ptd(msg, tid, ptd) do { printk("%s: " msg ",tid=%d,refc=%d\n", __FUNCTION__, tid, atomic_read(&ptd->refcount)); } while(0)
#else
#define pr_ptd(msg, tid, ptd) do { } while(0)
#endif
static long pager_call_irq(ihk_os_t os, struct syscall_request *req); static long pager_call_irq(ihk_os_t os, struct syscall_request *req);
static long pager_call(ihk_os_t os, struct syscall_request *req); static long pager_call(ihk_os_t os, struct syscall_request *req);
@@ -80,75 +87,90 @@ static void print_dma_lastreq(void)
} }
#endif #endif
int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data* ppd, void mcctrl_put_per_thread_data_unsafe(struct mcctrl_per_thread_data *ptd)
struct task_struct *task, void *data)
{ {
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL; if (!atomic_dec_and_test(&ptd->refcount)) {
struct mcctrl_per_thread_data *ptd_alloc = NULL; int ret = atomic_read(&ptd->refcount);
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); if (ret < 0) {
int ret = 0; printk("%s: ERROR: invalid refcount=%d\n", __FUNCTION__, ret);
unsigned long flags;
ptd_alloc = kmalloc(sizeof(*ptd), GFP_ATOMIC);
if (!ptd_alloc) {
kprintf("%s: error allocate per thread data\n", __FUNCTION__);
ret = -ENOMEM;
goto out_noalloc;
}
/* Check if data for this thread exists and add if not */
write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == task) {
ptd = ptd_iter;
break;
} }
return;
} }
if (unlikely(ptd)) { list_del(&ptd->hash);
ret = -EBUSY; kfree(ptd);
kfree(ptd_alloc);
goto out;
}
ptd = ptd_alloc;
ptd->task = task;
ptd->data = data;
list_add_tail(&ptd->hash, &ppd->per_thread_data_hash[hash]);
out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
out_noalloc:
return ret;
} }
int mcctrl_delete_per_thread_data(struct mcctrl_per_proc_data* ppd, void mcctrl_put_per_thread_data(struct mcctrl_per_thread_data* _ptd)
struct task_struct *task)
{ {
struct mcctrl_per_proc_data *ppd = _ptd->ppd;
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL; struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL;
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); int hash = (((uint64_t)_ptd->task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
int ret = 0;
unsigned long flags; unsigned long flags;
/* Check if data for this thread exists and delete it */ /* Check if data for this thread exists and delete it */
write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == task) { if (ptd_iter->task == _ptd->task) {
ptd = ptd_iter; ptd = ptd_iter;
break; break;
} }
} }
if (!ptd) { if (!ptd) {
ret = -EINVAL; printk("%s: ERROR: ptd not found\n", __FUNCTION__);
goto out; goto out;
} }
list_del(&ptd->hash); mcctrl_put_per_thread_data_unsafe(ptd);
kfree(ptd);
out: out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
}
int mcctrl_add_per_thread_data(struct mcctrl_per_proc_data *ppd, void *data)
{
struct mcctrl_per_thread_data *ptd_iter, *ptd = NULL;
struct mcctrl_per_thread_data *ptd_alloc = NULL;
int hash = (((uint64_t)current >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
int ret = 0;
unsigned long flags;
ptd_alloc = kmalloc(sizeof(struct mcctrl_per_thread_data), GFP_ATOMIC);
if (!ptd_alloc) {
kprintf("%s: error allocate per thread data\n", __FUNCTION__);
ret = -ENOMEM;
goto out_noalloc;
}
memset(ptd_alloc, 0, sizeof(struct mcctrl_per_thread_data));
/* Check if data for this thread exists and add if not */
write_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
if (ptd_iter->task == current) {
ptd = ptd_iter;
break;
}
}
if (unlikely(ptd)) {
kprintf("%s: WARNING: ptd of tid: %d exists\n", __FUNCTION__, task_pid_vnr(current));
ret = -EBUSY;
kfree(ptd_alloc);
goto out;
}
ptd = ptd_alloc;
ptd->ppd = ppd;
ptd->task = current;
ptd->tid = task_pid_vnr(current);
ptd->data = data;
atomic_set(&ptd->refcount, 1);
list_add_tail(&ptd->hash, &ppd->per_thread_data_hash[hash]);
out:
write_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
out_noalloc:
return ret; return ret;
} }
@@ -159,7 +181,7 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc
int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK); int hash = (((uint64_t)task >> 4) & MCCTRL_PER_THREAD_DATA_HASH_MASK);
unsigned long flags; unsigned long flags;
/* Check if data for this thread exists and return it */ /* Check if data for this thread exists */
read_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags); read_lock_irqsave(&ppd->per_thread_data_hash_lock[hash], flags);
list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) { list_for_each_entry(ptd_iter, &ppd->per_thread_data_hash[hash], hash) {
@@ -169,8 +191,18 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc
} }
} }
if (ptd) {
if (atomic_read(&ptd->refcount) <= 0) {
printk("%s: ERROR: use-after-free detected (%d)", __FUNCTION__, atomic_read(&ptd->refcount));
ptd = NULL;
goto out;
}
atomic_inc(&ptd->refcount);
}
out:
read_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags); read_unlock_irqrestore(&ppd->per_thread_data_hash_lock[hash], flags);
return ptd ? ptd->data : NULL; return ptd;
} }
#endif /* !POSTK_DEBUG_ARCH_DEP_56 */ #endif /* !POSTK_DEBUG_ARCH_DEP_56 */
@@ -299,6 +331,7 @@ long syscall_backward(struct mcctrl_usrdata *usrdata, int num,
struct wait_queue_head_list_node *wqhln; struct wait_queue_head_list_node *wqhln;
unsigned long irqflags; unsigned long irqflags;
struct mcctrl_per_proc_data *ppd; struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
unsigned long phys; unsigned long phys;
struct syscall_request _request[2]; struct syscall_request _request[2];
struct syscall_request *request; struct syscall_request *request;
@@ -327,7 +360,14 @@ long syscall_backward(struct mcctrl_usrdata *usrdata, int num,
return -EINVAL; return -EINVAL;
} }
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
syscall_ret = -ENOENT;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) { if (!packet) {
syscall_ret = -ENOENT; syscall_ret = -ENOENT;
printk("%s: no packet registered for TID %d\n", printk("%s: no packet registered for TID %d\n",
@@ -466,6 +506,9 @@ out:
ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp)); ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp));
out_put_ppd: out_put_ppd:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
dprintk("%s: tid: %d, syscall: %d, syscall_ret: %lx\n", dprintk("%s: tid: %d, syscall: %d, syscall_ret: %lx\n",
__FUNCTION__, task_pid_vnr(current), num, syscall_ret); __FUNCTION__, task_pid_vnr(current), num, syscall_ret);
@@ -483,6 +526,7 @@ int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t
struct wait_queue_head_list_node *wqhln; struct wait_queue_head_list_node *wqhln;
unsigned long irqflags; unsigned long irqflags;
struct mcctrl_per_proc_data *ppd; struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
unsigned long phys; unsigned long phys;
int retry; int retry;
@@ -498,11 +542,18 @@ int remote_page_fault(struct mcctrl_usrdata *usrdata, void *fault_addr, uint64_t
return -EINVAL; return -EINVAL;
} }
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); ptd = mcctrl_get_per_thread_data(ppd, current);
if (!packet) { if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
error = -ENOENT; error = -ENOENT;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) {
printk("%s: no packet registered for TID %d\n", printk("%s: no packet registered for TID %d\n",
__FUNCTION__, task_pid_vnr(current)); __FUNCTION__, task_pid_vnr(current));
error = -ENOENT;
goto out_put_ppd; goto out_put_ppd;
} }
@@ -669,6 +720,9 @@ out:
ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp)); ihk_device_unmap_memory(ihk_os_to_dev(usrdata->os), phys, sizeof(*resp));
out_put_ppd: out_put_ppd:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
dprintk("%s: tid: %d, fault_addr: %p, reason: %lu, error: %d\n", dprintk("%s: tid: %d, fault_addr: %p, reason: %lu, error: %d\n",
__FUNCTION__, task_pid_vnr(current), fault_addr, (unsigned long)reason, error); __FUNCTION__, task_pid_vnr(current), fault_addr, (unsigned long)reason, error);
@@ -711,6 +765,7 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
size_t pix; size_t pix;
#endif #endif
struct mcctrl_per_proc_data *ppd; struct mcctrl_per_proc_data *ppd;
struct mcctrl_per_thread_data *ptd;
struct ikc_scd_packet *packet; struct ikc_scd_packet *packet;
int ret = 0; int ret = 0;
@@ -740,7 +795,14 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
goto no_ppd; goto no_ppd;
} }
packet = (struct ikc_scd_packet *)mcctrl_get_per_thread_data(ppd, current); ptd = mcctrl_get_per_thread_data(ppd, current);
if (!ptd) {
printk("%s: ERROR: mcctrl_get_per_thread_data failed\n", __FUNCTION__);
ret = VM_FAULT_SIGBUS;
goto no_ptd;
}
pr_ptd("get", task_pid_vnr(current), ptd);
packet = (struct ikc_scd_packet *)ptd->data;
if (!packet) { if (!packet) {
ret = VM_FAULT_SIGBUS; ret = VM_FAULT_SIGBUS;
printk("%s: no packet registered for TID %d\n", printk("%s: no packet registered for TID %d\n",
@@ -929,6 +991,9 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
ret = VM_FAULT_NOPAGE; ret = VM_FAULT_NOPAGE;
put_and_out: put_and_out:
mcctrl_put_per_thread_data(ptd);
pr_ptd("put", task_pid_vnr(current), ptd);
no_ptd:
mcctrl_put_per_proc_data(ppd); mcctrl_put_per_proc_data(ppd);
no_ppd: no_ppd:
return ret; return ret;