add arm64 support
- add arm64 dependent codes with GICv3 and SVE support - fix bugs based on architecture separation requests
This commit is contained in:
Takayuki Okamoto
325
arch/arm64/kernel/fpsimd.c
Normal file
325
arch/arm64/kernel/fpsimd.c
Normal file
@@ -0,0 +1,325 @@
|
||||
/* fpsimd.c COPYRIGHT FUJITSU LIMITED 2016-2017 */
|
||||
#include <thread_info.h>
|
||||
#include <fpsimd.h>
|
||||
#include <cpuinfo.h>
|
||||
#include <lwk/compiler.h>
|
||||
#include <ikc/ihk.h>
|
||||
#include <hwcap.h>
|
||||
#include <cls.h>
|
||||
#include <prctl.h>
|
||||
#include <cpufeature.h>
|
||||
#include <kmalloc.h>
|
||||
|
||||
//#define DEBUG_PRINT_FPSIMD
|
||||
|
||||
#ifdef DEBUG_PRINT_FPSIMD
|
||||
#define dkprintf kprintf
|
||||
#define ekprintf kprintf
|
||||
#else
|
||||
#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
|
||||
#define ekprintf kprintf
|
||||
#endif
|
||||
|
||||
#define BUG_ON(condition) do { if (condition) { kprintf("PANIC: %s: %s(line:%d)\n",\
|
||||
__FILE__, __FUNCTION__, __LINE__); panic(""); } } while(0)
|
||||
|
||||
#ifdef CONFIG_ARM64_SVE
|
||||
|
||||
/* Maximum supported vector length across all CPUs (initially poisoned) */
|
||||
int sve_max_vl = -1;
|
||||
/* Default VL for tasks that don't set it explicitly: */
|
||||
int sve_default_vl = -1;
|
||||
|
||||
size_t sve_state_size(struct thread const *thread)
|
||||
{
|
||||
unsigned int vl = thread->ctx.thread->sve_vl;
|
||||
|
||||
BUG_ON(!sve_vl_valid(vl));
|
||||
return SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl));
|
||||
}
|
||||
|
||||
void sve_free(struct thread *thread)
|
||||
{
|
||||
if (thread->ctx.thread->sve_state) {
|
||||
kfree(thread->ctx.thread->sve_state);
|
||||
thread->ctx.thread->sve_state = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void sve_alloc(struct thread *thread)
|
||||
{
|
||||
if (thread->ctx.thread->sve_state) {
|
||||
return;
|
||||
}
|
||||
|
||||
thread->ctx.thread->sve_state =
|
||||
kmalloc(sve_state_size(thread), IHK_MC_AP_NOWAIT);
|
||||
BUG_ON(!thread->ctx.thread->sve_state);
|
||||
|
||||
memset(thread->ctx.thread->sve_state, 0, sve_state_size(thread));
|
||||
}
|
||||
|
||||
static int get_nr_threads(struct process *proc)
|
||||
{
|
||||
struct thread *child;
|
||||
struct mcs_rwlock_node_irqsave lock;
|
||||
int nr_threads = 0;
|
||||
|
||||
mcs_rwlock_reader_lock(&proc->threads_lock, &lock);
|
||||
list_for_each_entry(child, &proc->threads_list, siblings_list){
|
||||
nr_threads++;
|
||||
}
|
||||
mcs_rwlock_reader_unlock(&proc->threads_lock, &lock);
|
||||
return nr_threads;
|
||||
}
|
||||
|
||||
extern void save_fp_regs(struct thread *thread);
|
||||
extern void clear_fp_regs(struct thread *thread);
|
||||
extern void restore_fp_regs(struct thread *thread);
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_set_vector_length */
|
||||
int sve_set_vector_length(struct thread *thread,
|
||||
unsigned long vl, unsigned long flags)
|
||||
{
|
||||
struct thread_info *ti = thread->ctx.thread;
|
||||
|
||||
BUG_ON(thread == cpu_local_var(current) && cpu_local_var(no_preempt) == 0);
|
||||
|
||||
/*
|
||||
* To avoid accidents, forbid setting for individual threads of a
|
||||
* multithreaded process. User code that knows what it's doing can
|
||||
* pass PR_SVE_SET_VL_THREAD to override this restriction:
|
||||
*/
|
||||
if (!(flags & PR_SVE_SET_VL_THREAD) && get_nr_threads(thread->proc) != 1) {
|
||||
return -EINVAL;
|
||||
}
|
||||
flags &= ~(unsigned long)PR_SVE_SET_VL_THREAD;
|
||||
|
||||
if (flags & ~(unsigned long)(PR_SVE_SET_VL_INHERIT |
|
||||
PR_SVE_SET_VL_ONEXEC)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (!sve_vl_valid(vl)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (vl > sve_max_vl) {
|
||||
BUG_ON(!sve_vl_valid(sve_max_vl));
|
||||
vl = sve_max_vl;
|
||||
}
|
||||
|
||||
if (flags & (PR_SVE_SET_VL_ONEXEC |
|
||||
PR_SVE_SET_VL_INHERIT)) {
|
||||
ti->sve_vl_onexec = vl;
|
||||
} else {
|
||||
/* Reset VL to system default on next exec: */
|
||||
ti->sve_vl_onexec = 0;
|
||||
}
|
||||
|
||||
/* Only actually set the VL if not deferred: */
|
||||
if (flags & PR_SVE_SET_VL_ONEXEC) {
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (vl != ti->sve_vl) {
|
||||
if ((elf_hwcap & HWCAP_SVE)) {
|
||||
fp_regs_struct fp_regs;
|
||||
memset(&fp_regs, 0, sizeof(fp_regs));
|
||||
|
||||
/* for self at prctl syscall */
|
||||
if (thread == cpu_local_var(current)) {
|
||||
save_fp_regs(thread);
|
||||
clear_fp_regs(thread);
|
||||
thread_sve_to_fpsimd(thread, &fp_regs);
|
||||
sve_free(thread);
|
||||
|
||||
ti->sve_vl = vl;
|
||||
|
||||
sve_alloc(thread);
|
||||
thread_fpsimd_to_sve(thread, &fp_regs);
|
||||
restore_fp_regs(thread);
|
||||
/* for target thread at ptrace */
|
||||
} else {
|
||||
thread_sve_to_fpsimd(thread, &fp_regs);
|
||||
sve_free(thread);
|
||||
|
||||
ti->sve_vl = vl;
|
||||
|
||||
sve_alloc(thread);
|
||||
thread_fpsimd_to_sve(thread, &fp_regs);
|
||||
}
|
||||
}
|
||||
}
|
||||
ti->sve_vl = vl;
|
||||
|
||||
out:
|
||||
ti->sve_flags = flags & PR_SVE_SET_VL_INHERIT;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_prctl_status */
|
||||
/*
|
||||
* Encode the current vector length and flags for return.
|
||||
* This is only required for prctl(): ptrace has separate fields
|
||||
*/
|
||||
static int sve_prctl_status(const struct thread_info *ti)
|
||||
{
|
||||
int ret = ti->sve_vl;
|
||||
|
||||
ret |= ti->sve_flags << 16;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_set_task_vl */
|
||||
int sve_set_thread_vl(struct thread *thread, const unsigned long vector_length,
|
||||
const unsigned long flags)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (!(elf_hwcap & HWCAP_SVE)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
BUG_ON(thread != cpu_local_var(current));
|
||||
|
||||
preempt_disable();
|
||||
ret = sve_set_vector_length(thread, vector_length, flags);
|
||||
preempt_enable();
|
||||
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
return sve_prctl_status(thread->ctx.thread);
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::sve_get_ti_vl */
|
||||
int sve_get_thread_vl(const struct thread *thread)
|
||||
{
|
||||
if (!(elf_hwcap & HWCAP_SVE)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
return sve_prctl_status(thread->ctx.thread);
|
||||
}
|
||||
|
||||
void do_sve_acc(unsigned int esr, struct pt_regs *regs)
|
||||
{
|
||||
kprintf("PANIC: CPU: %d PID: %d ESR: %x Trapped SVE access.\n",
|
||||
ihk_mc_get_processor_id(), cpu_local_var(current)->proc->pid, esr);
|
||||
panic("");
|
||||
}
|
||||
|
||||
void init_sve_vl(void)
|
||||
{
|
||||
extern unsigned long ihk_param_default_vl;
|
||||
uint64_t zcr;
|
||||
|
||||
if (unlikely(!(elf_hwcap & HWCAP_SVE))) {
|
||||
return;
|
||||
}
|
||||
|
||||
zcr = read_system_reg(SYS_ZCR_EL1);
|
||||
BUG_ON(((zcr & ZCR_EL1_LEN_MASK) + 1) * 16 > sve_max_vl);
|
||||
|
||||
sve_max_vl = ((zcr & ZCR_EL1_LEN_MASK) + 1) * 16;
|
||||
sve_default_vl = ihk_param_default_vl;
|
||||
|
||||
if (sve_default_vl == 0) {
|
||||
kprintf("SVE: Getting default VL = 0 from HOST-Linux.\n");
|
||||
sve_default_vl = sve_max_vl > 64 ? 64 : sve_max_vl;
|
||||
kprintf("SVE: Using default vl(%d byte).\n", sve_default_vl);
|
||||
}
|
||||
|
||||
kprintf("SVE: maximum available vector length %u bytes per vector\n",
|
||||
sve_max_vl);
|
||||
kprintf("SVE: default vector length %u bytes per vector\n",
|
||||
sve_default_vl);
|
||||
}
|
||||
|
||||
#else /* CONFIG_ARM64_SVE */
|
||||
|
||||
void init_sve_vl(void)
|
||||
{
|
||||
/* nothing to do. */
|
||||
}
|
||||
|
||||
#endif /* CONFIG_ARM64_SVE */
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::__task_pffr */
|
||||
static void *__thread_pffr(struct thread *thread)
|
||||
{
|
||||
unsigned int vl = thread->ctx.thread->sve_vl;
|
||||
|
||||
BUG_ON(!sve_vl_valid(vl));
|
||||
return (char *)thread->ctx.thread->sve_state + 34 * vl;
|
||||
}
|
||||
|
||||
/* There is a need to call from to check the HWCAP_FP and HWCAP_ASIMD state. */
|
||||
void thread_fpsimd_load(struct thread *thread)
|
||||
{
|
||||
if (likely(elf_hwcap & HWCAP_SVE)) {
|
||||
unsigned int vl = thread->ctx.thread->sve_vl;
|
||||
|
||||
BUG_ON(!sve_vl_valid(vl));
|
||||
sve_load_state(__thread_pffr(thread), &thread->fp_regs->fpsr, sve_vq_from_vl(vl) - 1);
|
||||
dkprintf("sve for TID %d restored\n", thread->tid);
|
||||
} else {
|
||||
// Load the current FPSIMD state to memory.
|
||||
fpsimd_load_state(thread->fp_regs);
|
||||
dkprintf("fp_regs for TID %d restored\n", thread->tid);
|
||||
}
|
||||
}
|
||||
|
||||
/* There is a need to call from to check the HWCAP_FP and HWCAP_ASIMD state. */
|
||||
void thread_fpsimd_save(struct thread *thread)
|
||||
{
|
||||
if (likely(elf_hwcap & HWCAP_SVE)) {
|
||||
sve_save_state(__thread_pffr(thread), &thread->fp_regs->fpsr);
|
||||
dkprintf("sve for TID %d saved\n", thread->tid);
|
||||
} else {
|
||||
// Save the current FPSIMD state to memory.
|
||||
fpsimd_save_state(thread->fp_regs);
|
||||
dkprintf("fp_regs for TID %d saved\n", thread->tid);
|
||||
}
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::__task_fpsimd_to_sve */
|
||||
static void __thread_fpsimd_to_sve(struct thread *thread, fp_regs_struct *fp_regs, unsigned int vq)
|
||||
{
|
||||
struct fpsimd_sve_state(vq) *sst = thread->ctx.thread->sve_state;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < 32; i++) {
|
||||
sst->zregs[i][0] = fp_regs->vregs[i];
|
||||
}
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::task_fpsimd_to_sve */
|
||||
void thread_fpsimd_to_sve(struct thread *thread, fp_regs_struct *fp_regs)
|
||||
{
|
||||
unsigned int vl = thread->ctx.thread->sve_vl;
|
||||
|
||||
BUG_ON(!sve_vl_valid(vl));
|
||||
__thread_fpsimd_to_sve(thread, fp_regs, sve_vq_from_vl(vl));
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::__task_sve_to_fpsimd */
|
||||
static void __thread_sve_to_fpsimd(struct thread *thread, fp_regs_struct *fp_regs, unsigned int vq)
|
||||
{
|
||||
struct fpsimd_sve_state(vq) *sst = thread->ctx.thread->sve_state;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < 32; i++) {
|
||||
fp_regs->vregs[i] = sst->zregs[i][0];
|
||||
}
|
||||
}
|
||||
|
||||
/* @ref.impl arch/arm64/kernel/fpsimd.c::task_sve_to_fpsimd */
|
||||
void thread_sve_to_fpsimd(struct thread *thread, fp_regs_struct *fp_regs)
|
||||
{
|
||||
unsigned int vl = thread->ctx.thread->sve_vl;
|
||||
|
||||
BUG_ON(!sve_vl_valid(vl));
|
||||
__thread_sve_to_fpsimd(thread, fp_regs, sve_vq_from_vl(vl));
|
||||
}
|
||||
Reference in New Issue
Block a user