From bb2589bac4fe3a428038eda19e2f584513741bcc Mon Sep 17 00:00:00 2001 From: "Shiratori, Takehiro" Date: Tue, 19 Mar 2019 18:30:00 +0900 Subject: [PATCH] uti: futex_wait: Use kmalloc area for wait queue Change-Id: Ida994c87334f9613bbf5cbda45b6b5474fd4c6be --- kernel/futex.c | 60 +++++++++++++--------------------------- kernel/include/futex.h | 33 ++++++++++++++++++++++ kernel/include/process.h | 1 + 3 files changed, 53 insertions(+), 41 deletions(-) diff --git a/kernel/futex.c b/kernel/futex.c index c6581168..762d791a 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -87,38 +87,6 @@ unsigned long ihk_mc_get_ns_per_tsc(void); -/** - * struct futex_q - The hashed futex queue entry, one per waiting task - * @task: the task waiting on the futex - * @lock_ptr: the hash bucket lock - * @key: the key the futex is hashed on - * @requeue_pi_key: the requeue_pi target futex key - * @bitset: bitset for the optional bitmasked wakeup - * - * We use this hashed waitqueue, instead of a normal wait_queue_t, so - * we can wake only the relevant ones (hashed queues may be shared). - * - * A futex_q has a woken state, just like tasks have TASK_RUNNING. - * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. - * The order of wakup is always to make the first condition true, then - * the second. - * - * PI futexes are typically woken before they are removed from the hash list via - * the rt_mutex code. See unqueue_me_pi(). - */ -struct futex_q { - struct plist_node list; - - struct thread *task; - ihk_spinlock_t *lock_ptr; - union futex_key key; - union futex_key *requeue_pi_key; - uint32_t bitset; - - /* Used to wake-up a thread running on a Linux CPU */ - void *uti_futex_resp; -}; - /* * Hash buckets are shared by all the futex_keys that hash to the same * location. Each key may have multiple futex_q structures, one for each task @@ -825,13 +793,22 @@ static int futex_wait(uint32_t __user *uaddr, int fshared, struct cpu_local_var *clv_override) { struct futex_hash_bucket *hb; - struct futex_q q; int64_t time_remain; + struct futex_q lq; + struct futex_q *q = NULL; int ret; if (!bitset) return -EINVAL; + if (!clv_override) { + q = &lq; + } + else { + q = &cpu_local_var_with_override(current, + clv_override)->futex_q; + } + #ifdef PROFILE_ENABLE if (cpu_local_var_with_override(current, clv_override)->profile && cpu_local_var_with_override(current, clv_override)->profile_start_ts) { @@ -841,24 +818,25 @@ static int futex_wait(uint32_t __user *uaddr, int fshared, } #endif - q.bitset = bitset; - q.requeue_pi_key = NULL; - q.uti_futex_resp = cpu_local_var_with_override(uti_futex_resp, clv_override); + q->bitset = bitset; + q->requeue_pi_key = NULL; + q->uti_futex_resp = cpu_local_var_with_override(uti_futex_resp, + clv_override); retry: /* Prepare to wait on uaddr. */ - ret = futex_wait_setup(uaddr, val, fshared, &q, &hb, clv_override); + ret = futex_wait_setup(uaddr, val, fshared, q, &hb, clv_override); if (ret) { uti_dkprintf("%s: tid=%d futex_wait_setup returns zero, no need to sleep\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid); goto out; } /* queue_me and wait for wakeup, timeout, or a signal. */ - time_remain = futex_wait_queue_me(hb, &q, timeout, clv_override); + time_remain = futex_wait_queue_me(hb, q, timeout, clv_override); /* If we were woken (and unqueued), we succeeded, whatever. */ ret = 0; - if (!unqueue_me(&q)) { + if (!unqueue_me(q)) { uti_dkprintf("%s: tid=%d unqueued\n", __FUNCTION__, cpu_local_var_with_override(current, clv_override)->tid); goto out_put_key; } @@ -878,11 +856,11 @@ retry: } /* RIKEN: no signals */ - put_futex_key(fshared, &q.key); + put_futex_key(fshared, &q->key); goto retry; out_put_key: - put_futex_key(fshared, &q.key); + put_futex_key(fshared, &q->key); out: #ifdef PROFILE_ENABLE if (cpu_local_var_with_override(current, clv_override)->profile) { diff --git a/kernel/include/futex.h b/kernel/include/futex.h index f09afcf9..8639b268 100644 --- a/kernel/include/futex.h +++ b/kernel/include/futex.h @@ -94,6 +94,7 @@ #include #include #include +#include #ifndef _ASM_X86_FUTEX_H #define _ASM_X86_FUTEX_H @@ -165,5 +166,37 @@ futex( ); +/** + * struct futex_q - The hashed futex queue entry, one per waiting task + * @task: the task waiting on the futex + * @lock_ptr: the hash bucket lock + * @key: the key the futex is hashed on + * @requeue_pi_key: the requeue_pi target futex key + * @bitset: bitset for the optional bitmasked wakeup + * + * We use this hashed waitqueue, instead of a normal wait_queue_t, so + * we can wake only the relevant ones (hashed queues may be shared). + * + * A futex_q has a woken state, just like tasks have TASK_RUNNING. + * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. + * The order of wakup is always to make the first condition true, then + * the second. + * + * PI futexes are typically woken before they are removed from the hash list via + * the rt_mutex code. See unqueue_me_pi(). + */ +struct futex_q { + struct plist_node list; + + struct thread *task; + ihk_spinlock_t *lock_ptr; + union futex_key key; + union futex_key *requeue_pi_key; + uint32_t bitset; + + /* Used to wake-up a thread running on a Linux CPU */ + void *uti_futex_resp; +}; + #endif #endif diff --git a/kernel/include/process.h b/kernel/include/process.h index 3e275843..e587315e 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -717,6 +717,7 @@ struct thread { struct uti_attr *mod_clone_arg; int parent_cpuid; int uti_refill_tid; + struct futex_q futex_q; // for performance counter unsigned long pmc_alloc_map;