futex adaptation from Linux 2.6.34 (Intel MPSS Linux)
This commit is contained in:
Balazs Gerofi bgerofi@riken.jp
@@ -1,24 +1,50 @@
|
||||
/* Kitten LWK futex adaptation */
|
||||
/*
|
||||
* Linux futex adaptation.
|
||||
* (C) Copyright 2013 RIKEN AICS
|
||||
* Balazs Gerofi <bgerofi@riken.jp>
|
||||
*/
|
||||
|
||||
|
||||
#ifndef _LWK_FUTEX_H
|
||||
#define _LWK_FUTEX_H
|
||||
#ifndef _FUTEX_H
|
||||
#define _FUTEX_H
|
||||
|
||||
/** \name Futex Commands
|
||||
* @{
|
||||
*/
|
||||
#define FUTEX_WAIT 0
|
||||
#define FUTEX_WAKE 1
|
||||
#define FUTEX_FD 2
|
||||
#define FUTEX_REQUEUE 3
|
||||
#define FUTEX_CMP_REQUEUE 4
|
||||
#define FUTEX_WAKE_OP 5
|
||||
#define FUTEX_LOCK_PI 6
|
||||
#define FUTEX_UNLOCK_PI 7
|
||||
#define FUTEX_TRYLOCK_PI 8
|
||||
#define FUTEX_WAIT_BITSET 9
|
||||
#define FUTEX_WAKE_BITSET 10
|
||||
#define FUTEX_WAIT_REQUEUE_PI 11
|
||||
#define FUTEX_CMP_REQUEUE_PI 12
|
||||
// @}
|
||||
|
||||
#define FUTEX_PRIVATE_FLAG 128
|
||||
#define FUTEX_CLOCK_REALTIME 256
|
||||
#define FUTEX_CMD_MASK ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME)
|
||||
|
||||
#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_WAIT_BITSET_PRIVATE (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_WAKE_BITSET_PRIVATE (FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | \
|
||||
FUTEX_PRIVATE_FLAG)
|
||||
#define FUTEX_CMP_REQUEUE_PI_PRIVATE (FUTEX_CMP_REQUEUE_PI | \
|
||||
FUTEX_PRIVATE_FLAG)
|
||||
|
||||
|
||||
/** \name Futex Operations, used for FUTEX_WAKE_OP
|
||||
* @{
|
||||
*/
|
||||
@@ -201,30 +227,34 @@ static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
|
||||
|
||||
#define FUTEX_HASHBITS 8 /* 256 entries in each futex hash tbl */
|
||||
|
||||
/** Futex tracking structure.
|
||||
*
|
||||
* A futex has a woken state, just like tasks have TASK_RUNNING.
|
||||
* It is considered woken when list_empty(&futex->link) || futex->lock_ptr == 0.
|
||||
* The order of wakup is always to make the first condition true, then
|
||||
* wake up futex->waitq, then make the second condition true.
|
||||
*/
|
||||
struct futex {
|
||||
struct list_head link;
|
||||
struct waitq waitq;
|
||||
ihk_spinlock_t * lock_ptr;
|
||||
uint32_t __user * uaddr;
|
||||
uint32_t bitset;
|
||||
#define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */
|
||||
#define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */
|
||||
|
||||
struct process_vm;
|
||||
|
||||
union futex_key {
|
||||
#if 0
|
||||
struct {
|
||||
unsigned long pgoff;
|
||||
struct inode *inode;
|
||||
int offset;
|
||||
} shared;
|
||||
#endif
|
||||
struct {
|
||||
unsigned long address;
|
||||
struct process_vm *mm;
|
||||
int offset;
|
||||
} private;
|
||||
struct {
|
||||
unsigned long word;
|
||||
void *ptr;
|
||||
int offset;
|
||||
} both;
|
||||
};
|
||||
|
||||
struct futex_queue {
|
||||
ihk_spinlock_t lock;
|
||||
struct list_head futex_list;
|
||||
};
|
||||
#define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = NULL } }
|
||||
|
||||
extern void
|
||||
futex_queue_init(
|
||||
struct futex_queue * queue
|
||||
);
|
||||
extern int futex_init(void);
|
||||
|
||||
extern int
|
||||
futex(
|
||||
|
||||
145
kernel/include/jhash.h
Normal file
145
kernel/include/jhash.h
Normal file
@@ -0,0 +1,145 @@
|
||||
#ifndef _LINUX_JHASH_H
|
||||
#define _LINUX_JHASH_H
|
||||
|
||||
/* RIKEN: u32 replaced to uint32_t
|
||||
*
|
||||
* jhash.h: Jenkins hash support.
|
||||
*
|
||||
* Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
|
||||
*
|
||||
* http://burtleburtle.net/bob/hash/
|
||||
*
|
||||
* These are the credits from Bob's sources:
|
||||
*
|
||||
* lookup2.c, by Bob Jenkins, December 1996, Public Domain.
|
||||
* hash(), hash2(), hash3, and mix() are externally useful functions.
|
||||
* Routines to test the hash are included if SELF_TEST is defined.
|
||||
* You can use this free for any purpose. It has no warranty.
|
||||
*
|
||||
* Copyright (C) 2003 David S. Miller (davem@redhat.com)
|
||||
*
|
||||
* I've modified Bob's hash to be useful in the Linux kernel, and
|
||||
* any bugs present are surely my fault. -DaveM
|
||||
*/
|
||||
|
||||
/* NOTE: Arguments are modified. */
|
||||
#define __jhash_mix(a, b, c) \
|
||||
{ \
|
||||
a -= b; a -= c; a ^= (c>>13); \
|
||||
b -= c; b -= a; b ^= (a<<8); \
|
||||
c -= a; c -= b; c ^= (b>>13); \
|
||||
a -= b; a -= c; a ^= (c>>12); \
|
||||
b -= c; b -= a; b ^= (a<<16); \
|
||||
c -= a; c -= b; c ^= (b>>5); \
|
||||
a -= b; a -= c; a ^= (c>>3); \
|
||||
b -= c; b -= a; b ^= (a<<10); \
|
||||
c -= a; c -= b; c ^= (b>>15); \
|
||||
}
|
||||
|
||||
/* The golden ration: an arbitrary value */
|
||||
#define JHASH_GOLDEN_RATIO 0x9e3779b9
|
||||
|
||||
/* The most generic version, hashes an arbitrary sequence
|
||||
* of bytes. No alignment or length assumptions are made about
|
||||
* the input key.
|
||||
*/
|
||||
static inline uint32_t jhash(const void *key, uint32_t length, uint32_t initval)
|
||||
{
|
||||
uint32_t a, b, c, len;
|
||||
const uint8_t *k = key;
|
||||
|
||||
len = length;
|
||||
a = b = JHASH_GOLDEN_RATIO;
|
||||
c = initval;
|
||||
|
||||
while (len >= 12) {
|
||||
a += (k[0] +((uint32_t)k[1]<<8) +((uint32_t)k[2]<<16) +((uint32_t)k[3]<<24));
|
||||
b += (k[4] +((uint32_t)k[5]<<8) +((uint32_t)k[6]<<16) +((uint32_t)k[7]<<24));
|
||||
c += (k[8] +((uint32_t)k[9]<<8) +((uint32_t)k[10]<<16)+((uint32_t)k[11]<<24));
|
||||
|
||||
__jhash_mix(a,b,c);
|
||||
|
||||
k += 12;
|
||||
len -= 12;
|
||||
}
|
||||
|
||||
c += length;
|
||||
switch (len) {
|
||||
case 11: c += ((uint32_t)k[10]<<24);
|
||||
case 10: c += ((uint32_t)k[9]<<16);
|
||||
case 9 : c += ((uint32_t)k[8]<<8);
|
||||
case 8 : b += ((uint32_t)k[7]<<24);
|
||||
case 7 : b += ((uint32_t)k[6]<<16);
|
||||
case 6 : b += ((uint32_t)k[5]<<8);
|
||||
case 5 : b += k[4];
|
||||
case 4 : a += ((uint32_t)k[3]<<24);
|
||||
case 3 : a += ((uint32_t)k[2]<<16);
|
||||
case 2 : a += ((uint32_t)k[1]<<8);
|
||||
case 1 : a += k[0];
|
||||
};
|
||||
|
||||
__jhash_mix(a,b,c);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
/* A special optimized version that handles 1 or more of uint32_ts.
|
||||
* The length parameter here is the number of uint32_ts in the key.
|
||||
*/
|
||||
static inline uint32_t jhash2(const uint32_t *k, uint32_t length, uint32_t initval)
|
||||
{
|
||||
uint32_t a, b, c, len;
|
||||
|
||||
a = b = JHASH_GOLDEN_RATIO;
|
||||
c = initval;
|
||||
len = length;
|
||||
|
||||
while (len >= 3) {
|
||||
a += k[0];
|
||||
b += k[1];
|
||||
c += k[2];
|
||||
__jhash_mix(a, b, c);
|
||||
k += 3; len -= 3;
|
||||
}
|
||||
|
||||
c += length * 4;
|
||||
|
||||
switch (len) {
|
||||
case 2 : b += k[1];
|
||||
case 1 : a += k[0];
|
||||
};
|
||||
|
||||
__jhash_mix(a,b,c);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
|
||||
/* A special ultra-optimized versions that knows they are hashing exactly
|
||||
* 3, 2 or 1 word(s).
|
||||
*
|
||||
* NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
|
||||
* done at the end is not done here.
|
||||
*/
|
||||
static inline uint32_t jhash_3words(uint32_t a, uint32_t b, uint32_t c, uint32_t initval)
|
||||
{
|
||||
a += JHASH_GOLDEN_RATIO;
|
||||
b += JHASH_GOLDEN_RATIO;
|
||||
c += initval;
|
||||
|
||||
__jhash_mix(a, b, c);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
static inline uint32_t jhash_2words(uint32_t a, uint32_t b, uint32_t initval)
|
||||
{
|
||||
return jhash_3words(a, b, 0, initval);
|
||||
}
|
||||
|
||||
static inline uint32_t jhash_1word(uint32_t a, uint32_t initval)
|
||||
{
|
||||
return jhash_3words(a, 0, 0, initval);
|
||||
}
|
||||
|
||||
#endif /* _LINUX_JHASH_H */
|
||||
273
kernel/include/plist.h
Normal file
273
kernel/include/plist.h
Normal file
@@ -0,0 +1,273 @@
|
||||
/*
|
||||
* Descending-priority-sorted double-linked list
|
||||
*
|
||||
* (C) 2002-2003 Intel Corp
|
||||
* Inaky Perez-Gonzalez <inaky.perez-gonzalez@intel.com>.
|
||||
*
|
||||
* 2001-2005 (c) MontaVista Software, Inc.
|
||||
* Daniel Walker <dwalker@mvista.com>
|
||||
*
|
||||
* (C) 2005 Thomas Gleixner <tglx@linutronix.de>
|
||||
*
|
||||
* Simplifications of the original code by
|
||||
* Oleg Nesterov <oleg@tv-sign.ru>
|
||||
*
|
||||
* Licensed under the FSF's GNU Public License v2 or later.
|
||||
*
|
||||
* Based on simple lists (include/linux/list.h).
|
||||
*
|
||||
* This is a priority-sorted list of nodes; each node has a
|
||||
* priority from INT_MIN (highest) to INT_MAX (lowest).
|
||||
*
|
||||
* Addition is O(K), removal is O(1), change of priority of a node is
|
||||
* O(K) and K is the number of RT priority levels used in the system.
|
||||
* (1 <= K <= 99)
|
||||
*
|
||||
* This list is really a list of lists:
|
||||
*
|
||||
* - The tier 1 list is the prio_list, different priority nodes.
|
||||
*
|
||||
* - The tier 2 list is the node_list, serialized nodes.
|
||||
*
|
||||
* Simple ASCII art explanation:
|
||||
*
|
||||
* |HEAD |
|
||||
* | |
|
||||
* |prio_list.prev|<------------------------------------|
|
||||
* |prio_list.next|<->|pl|<->|pl|<--------------->|pl|<-|
|
||||
* |10 | |10| |21| |21| |21| |40| (prio)
|
||||
* | | | | | | | | | | | |
|
||||
* | | | | | | | | | | | |
|
||||
* |node_list.next|<->|nl|<->|nl|<->|nl|<->|nl|<->|nl|<-|
|
||||
* |node_list.prev|<------------------------------------|
|
||||
*
|
||||
* The nodes on the prio_list list are sorted by priority to simplify
|
||||
* the insertion of new nodes. There are no nodes with duplicate
|
||||
* priorites on the list.
|
||||
*
|
||||
* The nodes on the node_list are ordered by priority and can contain
|
||||
* entries which have the same priority. Those entries are ordered
|
||||
* FIFO
|
||||
*
|
||||
* Addition means: look for the prio_list node in the prio_list
|
||||
* for the priority of the node and insert it before the node_list
|
||||
* entry of the next prio_list node. If it is the first node of
|
||||
* that priority, add it to the prio_list in the right position and
|
||||
* insert it into the serialized node_list list
|
||||
*
|
||||
* Removal means remove it from the node_list and remove it from
|
||||
* the prio_list if the node_list list_head is non empty. In case
|
||||
* of removal from the prio_list it must be checked whether other
|
||||
* entries of the same priority are on the list or not. If there
|
||||
* is another entry of the same priority then this entry has to
|
||||
* replace the removed entry on the prio_list. If the entry which
|
||||
* is removed is the only entry of this priority then a simple
|
||||
* remove from both list is sufficient.
|
||||
*
|
||||
* INT_MIN is the highest priority, 0 is the medium highest, INT_MAX
|
||||
* is lowest priority.
|
||||
*
|
||||
* No locking is done, up to the caller.
|
||||
*
|
||||
*/
|
||||
#ifndef _LINUX_PLIST_H_
|
||||
#define _LINUX_PLIST_H_
|
||||
|
||||
#include <ihk/lock.h>
|
||||
#include <list.h>
|
||||
|
||||
struct plist_head {
|
||||
struct list_head prio_list;
|
||||
struct list_head node_list;
|
||||
#ifdef CONFIG_DEBUG_PI_LIST
|
||||
raw_spinlock_t *rawlock;
|
||||
spinlock_t *spinlock;
|
||||
#endif
|
||||
};
|
||||
|
||||
struct plist_node {
|
||||
int prio;
|
||||
struct plist_head plist;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_DEBUG_PI_LIST
|
||||
# define PLIST_HEAD_LOCK_INIT(_lock) .spinlock = _lock
|
||||
# define PLIST_HEAD_LOCK_INIT_RAW(_lock) .rawlock = _lock
|
||||
#else
|
||||
# define PLIST_HEAD_LOCK_INIT(_lock)
|
||||
# define PLIST_HEAD_LOCK_INIT_RAW(_lock)
|
||||
#endif
|
||||
|
||||
#define _PLIST_HEAD_INIT(head) \
|
||||
.prio_list = LIST_HEAD_INIT((head).prio_list), \
|
||||
.node_list = LIST_HEAD_INIT((head).node_list)
|
||||
|
||||
/**
|
||||
* PLIST_HEAD_INIT - static struct plist_head initializer
|
||||
* @head: struct plist_head variable name
|
||||
* @_lock: lock to initialize for this list
|
||||
*/
|
||||
#define PLIST_HEAD_INIT(head, _lock) \
|
||||
{ \
|
||||
_PLIST_HEAD_INIT(head), \
|
||||
PLIST_HEAD_LOCK_INIT(&(_lock)) \
|
||||
}
|
||||
|
||||
/**
|
||||
* PLIST_HEAD_INIT_RAW - static struct plist_head initializer
|
||||
* @head: struct plist_head variable name
|
||||
* @_lock: lock to initialize for this list
|
||||
*/
|
||||
#define PLIST_HEAD_INIT_RAW(head, _lock) \
|
||||
{ \
|
||||
_PLIST_HEAD_INIT(head), \
|
||||
PLIST_HEAD_LOCK_INIT_RAW(&(_lock)) \
|
||||
}
|
||||
|
||||
/**
|
||||
* PLIST_NODE_INIT - static struct plist_node initializer
|
||||
* @node: struct plist_node variable name
|
||||
* @__prio: initial node priority
|
||||
*/
|
||||
#define PLIST_NODE_INIT(node, __prio) \
|
||||
{ \
|
||||
.prio = (__prio), \
|
||||
.plist = { _PLIST_HEAD_INIT((node).plist) }, \
|
||||
}
|
||||
|
||||
/**
|
||||
* plist_head_init - dynamic struct plist_head initializer
|
||||
* @head: &struct plist_head pointer
|
||||
* @lock: spinlock protecting the list (debugging)
|
||||
*/
|
||||
static inline void
|
||||
plist_head_init(struct plist_head *head, ihk_spinlock_t *lock)
|
||||
{
|
||||
INIT_LIST_HEAD(&head->prio_list);
|
||||
INIT_LIST_HEAD(&head->node_list);
|
||||
#ifdef CONFIG_DEBUG_PI_LIST
|
||||
head->spinlock = lock;
|
||||
head->rawlock = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* plist_head_init_raw - dynamic struct plist_head initializer
|
||||
* @head: &struct plist_head pointer
|
||||
* @lock: raw_spinlock protecting the list (debugging)
|
||||
*/
|
||||
static inline void
|
||||
plist_head_init_raw(struct plist_head *head, ihk_spinlock_t *lock)
|
||||
{
|
||||
INIT_LIST_HEAD(&head->prio_list);
|
||||
INIT_LIST_HEAD(&head->node_list);
|
||||
#ifdef CONFIG_DEBUG_PI_LIST
|
||||
head->rawlock = lock;
|
||||
head->spinlock = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* plist_node_init - Dynamic struct plist_node initializer
|
||||
* @node: &struct plist_node pointer
|
||||
* @prio: initial node priority
|
||||
*/
|
||||
static inline void plist_node_init(struct plist_node *node, int prio)
|
||||
{
|
||||
node->prio = prio;
|
||||
plist_head_init(&node->plist, NULL);
|
||||
}
|
||||
|
||||
extern void plist_add(struct plist_node *node, struct plist_head *head);
|
||||
extern void plist_del(struct plist_node *node, struct plist_head *head);
|
||||
|
||||
/**
|
||||
* plist_for_each - iterate over the plist
|
||||
* @pos: the type * to use as a loop counter
|
||||
* @head: the head for your list
|
||||
*/
|
||||
#define plist_for_each(pos, head) \
|
||||
list_for_each_entry(pos, &(head)->node_list, plist.node_list)
|
||||
|
||||
/**
|
||||
* plist_for_each_safe - iterate safely over a plist of given type
|
||||
* @pos: the type * to use as a loop counter
|
||||
* @n: another type * to use as temporary storage
|
||||
* @head: the head for your list
|
||||
*
|
||||
* Iterate over a plist of given type, safe against removal of list entry.
|
||||
*/
|
||||
#define plist_for_each_safe(pos, n, head) \
|
||||
list_for_each_entry_safe(pos, n, &(head)->node_list, plist.node_list)
|
||||
|
||||
/**
|
||||
* plist_for_each_entry - iterate over list of given type
|
||||
* @pos: the type * to use as a loop counter
|
||||
* @head: the head for your list
|
||||
* @mem: the name of the list_struct within the struct
|
||||
*/
|
||||
#define plist_for_each_entry(pos, head, mem) \
|
||||
list_for_each_entry(pos, &(head)->node_list, mem.plist.node_list)
|
||||
|
||||
/**
|
||||
* plist_for_each_entry_safe - iterate safely over list of given type
|
||||
* @pos: the type * to use as a loop counter
|
||||
* @n: another type * to use as temporary storage
|
||||
* @head: the head for your list
|
||||
* @m: the name of the list_struct within the struct
|
||||
*
|
||||
* Iterate over list of given type, safe against removal of list entry.
|
||||
*/
|
||||
#define plist_for_each_entry_safe(pos, n, head, m) \
|
||||
list_for_each_entry_safe(pos, n, &(head)->node_list, m.plist.node_list)
|
||||
|
||||
/**
|
||||
* plist_head_empty - return !0 if a plist_head is empty
|
||||
* @head: &struct plist_head pointer
|
||||
*/
|
||||
static inline int plist_head_empty(const struct plist_head *head)
|
||||
{
|
||||
return list_empty(&head->node_list);
|
||||
}
|
||||
|
||||
/**
|
||||
* plist_node_empty - return !0 if plist_node is not on a list
|
||||
* @node: &struct plist_node pointer
|
||||
*/
|
||||
static inline int plist_node_empty(const struct plist_node *node)
|
||||
{
|
||||
return plist_head_empty(&node->plist);
|
||||
}
|
||||
|
||||
/* All functions below assume the plist_head is not empty. */
|
||||
|
||||
/**
|
||||
* plist_first_entry - get the struct for the first entry
|
||||
* @head: the &struct plist_head pointer
|
||||
* @type: the type of the struct this is embedded in
|
||||
* @member: the name of the list_struct within the struct
|
||||
*/
|
||||
#ifdef CONFIG_DEBUG_PI_LIST
|
||||
# define plist_first_entry(head, type, member) \
|
||||
({ \
|
||||
WARN_ON(plist_head_empty(head)); \
|
||||
container_of(plist_first(head), type, member); \
|
||||
})
|
||||
#else
|
||||
# define plist_first_entry(head, type, member) \
|
||||
container_of(plist_first(head), type, member)
|
||||
#endif
|
||||
|
||||
/**
|
||||
* plist_first - return the first node (and thus, highest priority)
|
||||
* @head: the &struct plist_head pointer
|
||||
*
|
||||
* Assumes the plist is _not_ empty.
|
||||
*/
|
||||
static inline struct plist_node *plist_first(const struct plist_head *head)
|
||||
{
|
||||
return list_entry(head->node_list.next,
|
||||
struct plist_node, plist.node_list);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -73,9 +73,6 @@ struct process_vm {
|
||||
struct list_head vm_range_list;
|
||||
struct vm_regions region;
|
||||
|
||||
// Address space private futexes
|
||||
struct futex_queue futex_queues[1 << FUTEX_HASHBITS];
|
||||
|
||||
ihk_spinlock_t page_table_lock;
|
||||
ihk_spinlock_t memory_range_lock;
|
||||
// to protect the followings:
|
||||
|
||||
Reference in New Issue
Block a user