From 3bd0137c25f64f38dbfb34061dd761f001bf3694 Mon Sep 17 00:00:00 2001 From: Takayuki Okamoto Date: Mon, 25 Sep 2017 17:47:01 +0900 Subject: [PATCH] Fix some race condition on arm64 * move barrier() to architecture depended region * add barrier() in issue_ipi, kprintf, map_virtual * enable the workaround for cavium thunderx --- arch/arm64/kernel/include/arch/cpu.h | 2 ++ arch/arm64/kernel/include/assembler.h | 2 ++ arch/arm64/kernel/include/cputype.h | 2 ++ arch/arm64/kernel/irq-gic-v3.c | 15 ++++++++++++--- arch/x86/kernel/include/arch/cpu.h | 6 +++--- kernel/debug.c | 1 + kernel/mem.c | 1 + lib/include/ihk/cpu.h | 3 ++- 8 files changed, 25 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kernel/include/arch/cpu.h b/arch/arm64/kernel/include/arch/cpu.h index 5f3139ec..5a5161be 100644 --- a/arch/arm64/kernel/include/arch/cpu.h +++ b/arch/arm64/kernel/include/arch/cpu.h @@ -25,6 +25,8 @@ #define smp_rmb() dmb(ishld) #define smp_wmb() dmb(ishst) +#define arch_barrier() smp_mb() + #define smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ diff --git a/arch/arm64/kernel/include/assembler.h b/arch/arm64/kernel/include/assembler.h index 212c366d..f2f33100 100644 --- a/arch/arm64/kernel/include/assembler.h +++ b/arch/arm64/kernel/include/assembler.h @@ -6,6 +6,8 @@ #if defined(CONFIG_HAS_NMI) #include +#else /* defined(CONFIG_HAS_NMI) */ +#include #endif /* defined(CONFIG_HAS_NMI) */ #if defined(CONFIG_HAS_NMI) diff --git a/arch/arm64/kernel/include/cputype.h b/arch/arm64/kernel/include/cputype.h index c578fc2b..8feed533 100644 --- a/arch/arm64/kernel/include/cputype.h +++ b/arch/arm64/kernel/include/cputype.h @@ -35,6 +35,8 @@ #define MIDR_IMPLEMENTOR(midr) \ (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) +#define ARM_CPU_IMP_CAVIUM 0x43 + #ifndef __ASSEMBLY__ static unsigned int read_cpuid_id(void) diff --git a/arch/arm64/kernel/irq-gic-v3.c b/arch/arm64/kernel/irq-gic-v3.c index 12932bee..02e231e6 100644 --- a/arch/arm64/kernel/irq-gic-v3.c +++ b/arch/arm64/kernel/irq-gic-v3.c @@ -10,6 +10,8 @@ //#define DEBUG_GICV3 +#define USE_CAVIUM_THUNDER_X + #ifdef DEBUG_GICV3 #define dkprintf(...) kprintf(__VA_ARGS__) #define ekprintf(...) kprintf(__VA_ARGS__) @@ -18,6 +20,10 @@ #define ekprintf(...) kprintf(__VA_ARGS__) #endif +#ifdef USE_CAVIUM_THUNDER_X +static char is_cavium_thunderx = 0; +#endif + void *dist_base; void *rdist_base[NR_CPUS]; @@ -108,8 +114,8 @@ static uint64_t gic_read_iar_cavium_thunderx(void) asm volatile("nop;nop;nop;nop;"); asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat)); asm volatile("nop;nop;nop;nop;"); - mb(); #endif /* CONFIG_HAS_NMI */ + mb(); return irqstat; } @@ -118,7 +124,7 @@ static uint64_t gic_read_iar_cavium_thunderx(void) static uint64_t gic_read_iar(void) { #ifdef USE_CAVIUM_THUNDER_X - if (static_key_false(&is_cavium_thunderx)) + if (is_cavium_thunderx) return gic_read_iar_cavium_thunderx(); else #endif @@ -266,6 +272,7 @@ void arm64_issue_ipi_gicv3(uint32_t cpuid, uint32_t vector) { dkprintf("Send irq#%d to cpuid=%d\n", vector, cpuid); + barrier(); if(vector < 16){ // send SGI arm64_raise_sgi_gicv3(cpuid, vector); @@ -304,7 +311,9 @@ void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size) #ifdef USE_CAVIUM_THUNDER_X /* Cavium ThunderX erratum 23154 */ - gicv3_check_capabilities(); + if (MIDR_IMPLEMENTOR(read_cpuid_id()) == ARM_CPU_IMP_CAVIUM) { + is_cavium_thunderx = 1; + } #endif } diff --git a/arch/x86/kernel/include/arch/cpu.h b/arch/x86/kernel/include/arch/cpu.h index 90e998ba..5dd04cfc 100644 --- a/arch/x86/kernel/include/arch/cpu.h +++ b/arch/x86/kernel/include/arch/cpu.h @@ -13,16 +13,16 @@ #ifndef ARCH_CPU_H #define ARCH_CPU_H -#include +#define arch_barrier() asm volatile("" : : : "memory") static inline void rmb(void) { - barrier(); + arch_barrier(); } static inline void wmb(void) { - barrier(); + arch_barrier(); } static unsigned long read_tsc(void) diff --git a/kernel/debug.c b/kernel/debug.c index 1d2ab252..a88bc2c3 100644 --- a/kernel/debug.c +++ b/kernel/debug.c @@ -170,6 +170,7 @@ int kprintf(const char *format, ...) ihk_mc_delay_us(IHK_KMSG_NOTIFY_DELAY); } + barrier(); return len; } diff --git a/kernel/mem.c b/kernel/mem.c index d3dc94a6..78671ba2 100644 --- a/kernel/mem.c +++ b/kernel/mem.c @@ -1617,6 +1617,7 @@ void *ihk_mc_map_virtual(unsigned long phys, int npages, return NULL; } } + barrier(); return (char *)p + offset; } diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h index 9dbb3c7a..7a6e2431 100644 --- a/lib/include/ihk/cpu.h +++ b/lib/include/ihk/cpu.h @@ -16,6 +16,7 @@ #include #include +#include void cpu_enable_interrupt(void); void cpu_disable_interrupt(void); @@ -24,7 +25,7 @@ void cpu_safe_halt(void); void cpu_restore_interrupt(unsigned long); void cpu_pause(void); -#define barrier() asm volatile("" : : : "memory") +#define barrier() arch_barrier() unsigned long cpu_disable_interrupt_save(void);