add arm64 support
- add arm64 dependent codes with GICv3 and SVE support - fix bugs based on architecture separation requests
This commit is contained in:
Takayuki Okamoto
363
arch/arm64/kernel/include/ihk/atomic.h
Normal file
363
arch/arm64/kernel/include/ihk/atomic.h
Normal file
@@ -0,0 +1,363 @@
|
||||
/* atomic.h COPYRIGHT FUJITSU LIMITED 2015-2016 */
|
||||
#ifndef __HEADER_ARM64_IHK_ATOMIC_H
|
||||
#define __HEADER_ARM64_IHK_ATOMIC_H
|
||||
|
||||
#include <arch/cpu.h>
|
||||
|
||||
/***********************************************************************
|
||||
* ihk_atomic_t
|
||||
*/
|
||||
|
||||
typedef struct {
|
||||
int counter;
|
||||
} ihk_atomic_t;
|
||||
|
||||
#define IHK_ATOMIC_INIT(i) { (i) }
|
||||
|
||||
static inline int ihk_atomic_read(const ihk_atomic_t *v)
|
||||
{
|
||||
return (*(volatile int *)&(v)->counter);
|
||||
}
|
||||
|
||||
static inline void ihk_atomic_set(ihk_atomic_t *v, int i)
|
||||
{
|
||||
v->counter = i;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_add (atomic_##op) */
|
||||
static inline void ihk_atomic_add(int i, ihk_atomic_t *v)
|
||||
{
|
||||
unsigned long tmp;
|
||||
int result;
|
||||
|
||||
asm volatile("// atomic_add\n"
|
||||
"1: ldxr %w0, %2\n"
|
||||
" add %w0, %w0, %w3\n"
|
||||
" stxr %w1, %w0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
|
||||
: "Ir" (i));
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_sub (atomic_##op) */
|
||||
static inline void ihk_atomic_sub(int i, ihk_atomic_t *v)
|
||||
{
|
||||
unsigned long tmp;
|
||||
int result;
|
||||
|
||||
asm volatile("// atomic_sub\n"
|
||||
"1: ldxr %w0, %2\n"
|
||||
" sub %w0, %w0, %w3\n"
|
||||
" stxr %w1, %w0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
|
||||
: "Ir" (i));
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_inc */
|
||||
#define ihk_atomic_inc(v) ihk_atomic_add(1, v)
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_dec */
|
||||
#define ihk_atomic_dec(v) ihk_atomic_sub(1, v)
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_add_return (atomic_##op##_return) */
|
||||
static inline int ihk_atomic_add_return(int i, ihk_atomic_t *v)
|
||||
{
|
||||
unsigned long tmp;
|
||||
int result;
|
||||
|
||||
asm volatile("// atomic_add_return\n"
|
||||
"1: ldxr %w0, %2\n"
|
||||
" add %w0, %w0, %w3\n"
|
||||
" stlxr %w1, %w0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
|
||||
: "Ir" (i)
|
||||
: "memory");
|
||||
|
||||
smp_mb();
|
||||
return result;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_sub_return (atomic_##op##_return) */
|
||||
static inline int ihk_atomic_sub_return(int i, ihk_atomic_t *v)
|
||||
{
|
||||
unsigned long tmp;
|
||||
int result;
|
||||
|
||||
asm volatile("// atomic_sub_return\n"
|
||||
"1: ldxr %w0, %2\n"
|
||||
" sub %w0, %w0, %w3\n"
|
||||
" stlxr %w1, %w0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
|
||||
: "Ir" (i)
|
||||
: "memory");
|
||||
|
||||
smp_mb();
|
||||
return result;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_inc_and_test */
|
||||
#define ihk_atomic_inc_and_test(v) (ihk_atomic_add_return(1, v) == 0)
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_dec_and_test */
|
||||
#define ihk_atomic_dec_and_test(v) (ihk_atomic_sub_return(1, v) == 0)
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_inc_return */
|
||||
#define ihk_atomic_inc_return(v) (ihk_atomic_add_return(1, v))
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic_dec_return */
|
||||
#define ihk_atomic_dec_return(v) (ihk_atomic_sub_return(1, v))
|
||||
|
||||
/***********************************************************************
|
||||
* ihk_atomic64_t
|
||||
*/
|
||||
typedef struct {
|
||||
long counter64;
|
||||
} ihk_atomic64_t;
|
||||
|
||||
#define IHK_ATOMIC64_INIT(i) { .counter64 = (i) }
|
||||
|
||||
static inline long ihk_atomic64_read(const ihk_atomic64_t *v)
|
||||
{
|
||||
return *(volatile long *)&(v)->counter64;
|
||||
}
|
||||
|
||||
static inline void ihk_atomic64_set(ihk_atomic64_t *v, int i)
|
||||
{
|
||||
v->counter64 = i;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic64_add (atomic64_##op) */
|
||||
static inline void ihk_atomic64_add(long i, ihk_atomic64_t *v)
|
||||
{
|
||||
long result;
|
||||
unsigned long tmp;
|
||||
|
||||
asm volatile("// atomic64_add\n"
|
||||
"1: ldxr %0, %2\n"
|
||||
" add %0, %0, %3\n"
|
||||
" stxr %w1, %0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter64)
|
||||
: "Ir" (i));
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/atomic.h::atomic64_inc */
|
||||
#define ihk_atomic64_inc(v) ihk_atomic64_add(1LL, (v))
|
||||
|
||||
/***********************************************************************
|
||||
* others
|
||||
*/
|
||||
/* @ref.impl arch/arm64/include/asm/cmpxchg.h::__xchg */
|
||||
static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
|
||||
{
|
||||
unsigned long ret = 0, tmp;
|
||||
|
||||
switch (size) {
|
||||
case 1:
|
||||
asm volatile("// __xchg1\n"
|
||||
"1: ldxrb %w0, %2\n"
|
||||
" stlxrb %w1, %w3, %2\n"
|
||||
" cbnz %w1, 1b\n"
|
||||
: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned char *)ptr)
|
||||
: "r" (x)
|
||||
: "memory");
|
||||
break;
|
||||
case 2:
|
||||
asm volatile("// __xchg2\n"
|
||||
"1: ldxrh %w0, %2\n"
|
||||
" stlxrh %w1, %w3, %2\n"
|
||||
" cbnz %w1, 1b\n"
|
||||
: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned short *)ptr)
|
||||
: "r" (x)
|
||||
: "memory");
|
||||
break;
|
||||
case 4:
|
||||
asm volatile("// __xchg4\n"
|
||||
"1: ldxr %w0, %2\n"
|
||||
" stlxr %w1, %w3, %2\n"
|
||||
" cbnz %w1, 1b\n"
|
||||
: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned int *)ptr)
|
||||
: "r" (x)
|
||||
: "memory");
|
||||
break;
|
||||
case 8:
|
||||
asm volatile("// __xchg8\n"
|
||||
"1: ldxr %0, %2\n"
|
||||
" stlxr %w1, %3, %2\n"
|
||||
" cbnz %w1, 1b\n"
|
||||
: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned long *)ptr)
|
||||
: "r" (x)
|
||||
: "memory");
|
||||
break;
|
||||
/*
|
||||
default:
|
||||
BUILD_BUG();
|
||||
*/
|
||||
}
|
||||
|
||||
smp_mb();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/cmpxchg.h::xchg */
|
||||
#define xchg(ptr,x) \
|
||||
({ \
|
||||
__typeof__(*(ptr)) __ret; \
|
||||
__ret = (__typeof__(*(ptr))) \
|
||||
__xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define xchg4(ptr, x) xchg(ptr,x)
|
||||
#define xchg8(ptr, x) xchg(ptr,x)
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/cmpxchg.h::__cmpxchg */
|
||||
static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
|
||||
unsigned long new, int size)
|
||||
{
|
||||
unsigned long oldval = 0, res;
|
||||
|
||||
switch (size) {
|
||||
case 1:
|
||||
do {
|
||||
asm volatile("// __cmpxchg1\n"
|
||||
" ldxrb %w1, %2\n"
|
||||
" mov %w0, #0\n"
|
||||
" cmp %w1, %w3\n"
|
||||
" b.ne 1f\n"
|
||||
" stxrb %w0, %w4, %2\n"
|
||||
"1:\n"
|
||||
: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned char *)ptr)
|
||||
: "Ir" (old), "r" (new) : "cc");
|
||||
} while (res);
|
||||
break;
|
||||
|
||||
case 2:
|
||||
do {
|
||||
asm volatile("// __cmpxchg2\n"
|
||||
" ldxrh %w1, %2\n"
|
||||
" mov %w0, #0\n"
|
||||
" cmp %w1, %w3\n"
|
||||
" b.ne 1f\n"
|
||||
" stxrh %w0, %w4, %2\n"
|
||||
"1:\n"
|
||||
: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned short *)ptr)
|
||||
: "Ir" (old), "r" (new)
|
||||
: "cc");
|
||||
} while (res);
|
||||
break;
|
||||
|
||||
case 4:
|
||||
do {
|
||||
asm volatile("// __cmpxchg4\n"
|
||||
" ldxr %w1, %2\n"
|
||||
" mov %w0, #0\n"
|
||||
" cmp %w1, %w3\n"
|
||||
" b.ne 1f\n"
|
||||
" stxr %w0, %w4, %2\n"
|
||||
"1:\n"
|
||||
: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned int *)ptr)
|
||||
: "Ir" (old), "r" (new)
|
||||
: "cc");
|
||||
} while (res);
|
||||
break;
|
||||
|
||||
case 8:
|
||||
do {
|
||||
asm volatile("// __cmpxchg8\n"
|
||||
" ldxr %1, %2\n"
|
||||
" mov %w0, #0\n"
|
||||
" cmp %1, %3\n"
|
||||
" b.ne 1f\n"
|
||||
" stxr %w0, %4, %2\n"
|
||||
"1:\n"
|
||||
: "=&r" (res), "=&r" (oldval), "+Q" (*(unsigned long *)ptr)
|
||||
: "Ir" (old), "r" (new)
|
||||
: "cc");
|
||||
} while (res);
|
||||
break;
|
||||
/*
|
||||
default:
|
||||
BUILD_BUG();
|
||||
*/
|
||||
}
|
||||
|
||||
return oldval;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/cmpxchg.h::__cmpxchg_mb */
|
||||
static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
|
||||
unsigned long new, int size)
|
||||
{
|
||||
unsigned long ret;
|
||||
|
||||
smp_mb();
|
||||
ret = __cmpxchg(ptr, old, new, size);
|
||||
smp_mb();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/cmpxchg.h::cmpxchg */
|
||||
#define cmpxchg(ptr, o, n) \
|
||||
({ \
|
||||
__typeof__(*(ptr)) __ret; \
|
||||
__ret = (__typeof__(*(ptr))) \
|
||||
__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
|
||||
sizeof(*(ptr))); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define atomic_cmpxchg4(ptr, o, n) cmpxchg(ptr,o,n)
|
||||
#define atomic_cmpxchg8(ptr, o, n) cmpxchg(ptr,o,n)
|
||||
|
||||
static inline void ihk_atomic_add_long(long i, long *v)
|
||||
{
|
||||
long result;
|
||||
unsigned long tmp;
|
||||
|
||||
asm volatile("// atomic64_add\n"
|
||||
"1: ldxr %0, %2\n"
|
||||
" add %0, %0, %3\n"
|
||||
" stxr %w1, %0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (*v)
|
||||
: "Ir" (i));
|
||||
}
|
||||
|
||||
static inline void ihk_atomic_add_ulong(long i, unsigned long *v)
|
||||
{
|
||||
long result;
|
||||
unsigned long tmp;
|
||||
|
||||
asm volatile("// atomic64_add\n"
|
||||
"1: ldxr %0, %2\n"
|
||||
" add %0, %0, %3\n"
|
||||
" stxr %w1, %0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (*v)
|
||||
: "Ir" (i));
|
||||
}
|
||||
|
||||
static inline unsigned long ihk_atomic_add_long_return(long i, long *v)
|
||||
{
|
||||
unsigned long result;
|
||||
unsigned long tmp;
|
||||
|
||||
asm volatile("// atomic64_add_return\n"
|
||||
"1: ldxr %0, %2\n"
|
||||
" add %0, %0, %3\n"
|
||||
" stlxr %w1, %0, %2\n"
|
||||
" cbnz %w1, 1b"
|
||||
: "=&r" (result), "=&r" (tmp), "+Q" (*v)
|
||||
: "Ir" (i)
|
||||
: "memory");
|
||||
|
||||
smp_mb();
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif /* !__HEADER_ARM64_COMMON_IHK_ATOMIC_H */
|
||||
80
arch/arm64/kernel/include/ihk/context.h
Normal file
80
arch/arm64/kernel/include/ihk/context.h
Normal file
@@ -0,0 +1,80 @@
|
||||
/* context.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
|
||||
#ifndef __HEADER_ARM64_IHK_CONTEXT_H
|
||||
#define __HEADER_ARM64_IHK_CONTEXT_H
|
||||
|
||||
#include <registers.h>
|
||||
|
||||
struct thread_info;
|
||||
typedef struct {
|
||||
struct thread_info *thread;
|
||||
} ihk_mc_kernel_context_t;
|
||||
|
||||
struct user_pt_regs {
|
||||
unsigned long regs[31];
|
||||
unsigned long sp;
|
||||
unsigned long pc;
|
||||
unsigned long pstate;
|
||||
};
|
||||
|
||||
struct pt_regs {
|
||||
union {
|
||||
struct user_pt_regs user_regs;
|
||||
struct {
|
||||
unsigned long regs[31];
|
||||
unsigned long sp;
|
||||
unsigned long pc;
|
||||
unsigned long pstate;
|
||||
};
|
||||
};
|
||||
unsigned long orig_x0;
|
||||
unsigned long syscallno;
|
||||
};
|
||||
|
||||
typedef struct pt_regs ihk_mc_user_context_t;
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/ptrace.h */
|
||||
#define GET_IP(regs) ((unsigned long)(regs)->pc)
|
||||
#define SET_IP(regs, value) ((regs)->pc = ((uint64_t) (value)))
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/ptrace.h */
|
||||
/* AArch32 CPSR bits */
|
||||
#define COMPAT_PSR_MODE_MASK 0x0000001f
|
||||
|
||||
/* @ref.impl include/asm-generic/ptrace.h */
|
||||
static inline unsigned long instruction_pointer(struct pt_regs *regs)
|
||||
{
|
||||
return GET_IP(regs);
|
||||
}
|
||||
/* @ref.impl include/asm-generic/ptrace.h */
|
||||
static inline void instruction_pointer_set(struct pt_regs *regs,
|
||||
unsigned long val)
|
||||
{
|
||||
SET_IP(regs, val);
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/include/asm/ptrace.h */
|
||||
/*
|
||||
* Write a register given an architectural register index r.
|
||||
* This handles the common case where 31 means XZR, not SP.
|
||||
*/
|
||||
static inline void pt_regs_write_reg(struct pt_regs *regs, int r,
|
||||
unsigned long val)
|
||||
{
|
||||
if (r != 31)
|
||||
regs->regs[r] = val;
|
||||
}
|
||||
|
||||
/* temp */
|
||||
#define ihk_mc_syscall_arg0(uc) (uc)->regs[0]
|
||||
#define ihk_mc_syscall_arg1(uc) (uc)->regs[1]
|
||||
#define ihk_mc_syscall_arg2(uc) (uc)->regs[2]
|
||||
#define ihk_mc_syscall_arg3(uc) (uc)->regs[3]
|
||||
#define ihk_mc_syscall_arg4(uc) (uc)->regs[4]
|
||||
#define ihk_mc_syscall_arg5(uc) (uc)->regs[5]
|
||||
|
||||
#define ihk_mc_syscall_ret(uc) (uc)->regs[0]
|
||||
|
||||
#define ihk_mc_syscall_pc(uc) (uc)->pc
|
||||
#define ihk_mc_syscall_sp(uc) (uc)->sp
|
||||
|
||||
#endif /* !__HEADER_ARM64_IHK_CONTEXT_H */
|
||||
14
arch/arm64/kernel/include/ihk/ikc.h
Normal file
14
arch/arm64/kernel/include/ihk/ikc.h
Normal file
@@ -0,0 +1,14 @@
|
||||
/* ikc.h COPYRIGHT FUJITSU LIMITED 2015 */
|
||||
#ifndef __HEADER_ARM64_IHK_IKC_H
|
||||
#define __HEADER_ARM64_IHK_IKC_H
|
||||
|
||||
#include <ikc/ihk.h>
|
||||
|
||||
#define IKC_PORT_IKC2MCKERNEL 501
|
||||
#define IKC_PORT_IKC2LINUX 503
|
||||
|
||||
/* manycore side */
|
||||
int ihk_mc_ikc_init_first(struct ihk_ikc_channel_desc *,
|
||||
ihk_ikc_ph_t handler);
|
||||
|
||||
#endif /* !__HEADER_ARM64_IHK_IKC_H */
|
||||
35
arch/arm64/kernel/include/ihk/types.h
Normal file
35
arch/arm64/kernel/include/ihk/types.h
Normal file
@@ -0,0 +1,35 @@
|
||||
/* types.h COPYRIGHT FUJITSU LIMITED 2015-2017 */
|
||||
#ifndef __HEADER_ARM64_IHK_TYPES_H
|
||||
#define __HEADER_ARM64_IHK_TYPES_H
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
typedef unsigned char uint8_t;
|
||||
typedef unsigned short uint16_t;
|
||||
typedef unsigned int uint32_t;
|
||||
typedef unsigned long long uint64_t;
|
||||
typedef signed char int8_t;
|
||||
typedef signed short int16_t;
|
||||
typedef signed int int32_t;
|
||||
typedef signed long long int64_t;
|
||||
|
||||
typedef int64_t ptrdiff_t;
|
||||
typedef int64_t intptr_t;
|
||||
typedef uint64_t uintptr_t;
|
||||
typedef uint64_t size_t;
|
||||
typedef int64_t ssize_t;
|
||||
typedef int64_t off_t;
|
||||
|
||||
#ifdef POSTK_DEBUG_ARCH_DEP_18 /* coredump arch separation. */
|
||||
typedef int32_t key_t;
|
||||
typedef uint32_t uid_t;
|
||||
typedef uint32_t gid_t;
|
||||
typedef int64_t time_t;
|
||||
typedef int32_t pid_t;
|
||||
#endif /* POSTK_DEBUG_ARCH_DEP_18 */
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#define NULL ((void *)0)
|
||||
|
||||
#endif /* !__HEADER_ARM64_IHK_TYPES_H */
|
||||
Reference in New Issue
Block a user