Fix some race condition on arm64

* move barrier() to architecture depended region
* add barrier() in issue_ipi, kprintf, map_virtual
* enable the workaround for cavium thunderx

arch/arm64/kernel/include/arch/cpu.h

@@ -25,6 +25,8 @@
 #define smp_rmb()	dmb(ishld)
 #define smp_wmb()	dmb(ishst)
 
+#define arch_barrier()	smp_mb()
+
 #define smp_store_release(p, v)						\
 do {									\
 	compiletime_assert_atomic_type(*p);				\

arch/arm64/kernel/include/assembler.h

@@ -6,6 +6,8 @@
 
 #if defined(CONFIG_HAS_NMI)
 #include <arm-gic-v3.h>
+#else /* defined(CONFIG_HAS_NMI) */
+#include <sysreg.h>
 #endif /* defined(CONFIG_HAS_NMI) */
 
 #if defined(CONFIG_HAS_NMI)

arch/arm64/kernel/include/cputype.h

@@ -35,6 +35,8 @@
 #define MIDR_IMPLEMENTOR(midr)	\
 	(((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT)
 
+#define ARM_CPU_IMP_CAVIUM	0x43
+
 #ifndef __ASSEMBLY__
 
 static unsigned int read_cpuid_id(void)

arch/arm64/kernel/irq-gic-v3.c

@@ -10,6 +10,8 @@
 
 //#define DEBUG_GICV3
 
+#define USE_CAVIUM_THUNDER_X
+
 #ifdef DEBUG_GICV3
 #define	dkprintf(...)	kprintf(__VA_ARGS__)
 #define	ekprintf(...)	kprintf(__VA_ARGS__)
@@ -18,6 +20,10 @@
 #define	ekprintf(...)	kprintf(__VA_ARGS__)
 #endif
 
+#ifdef USE_CAVIUM_THUNDER_X
+static char is_cavium_thunderx = 0;
+#endif
+
 void *dist_base;
 void *rdist_base[NR_CPUS];
 
@@ -108,8 +114,8 @@ static uint64_t gic_read_iar_cavium_thunderx(void)
 	asm volatile("nop;nop;nop;nop;");
 	asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat));
 	asm volatile("nop;nop;nop;nop;");
-	mb();
 #endif /* CONFIG_HAS_NMI */
+	mb();
 
 	return irqstat;
 }
@@ -118,7 +124,7 @@ static uint64_t gic_read_iar_cavium_thunderx(void)
 static uint64_t gic_read_iar(void)
 {
 #ifdef USE_CAVIUM_THUNDER_X
-	if (static_key_false(&is_cavium_thunderx))
+	if (is_cavium_thunderx)
 		return gic_read_iar_cavium_thunderx();
 	else
 #endif
@@ -266,6 +272,7 @@ void arm64_issue_ipi_gicv3(uint32_t cpuid, uint32_t vector)
 {
 	dkprintf("Send irq#%d to cpuid=%d\n", vector, cpuid);
 
+	barrier();
 	if(vector < 16){
 		// send SGI
 		arm64_raise_sgi_gicv3(cpuid, vector);
@@ -304,7 +311,9 @@ void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size)
 
 #ifdef USE_CAVIUM_THUNDER_X
 	/* Cavium ThunderX erratum 23154 */
-	gicv3_check_capabilities();
+	if (MIDR_IMPLEMENTOR(read_cpuid_id()) == ARM_CPU_IMP_CAVIUM) {
+		is_cavium_thunderx = 1;
+	}
 #endif
 }
 

arch/x86/kernel/include/arch/cpu.h

@@ -13,16 +13,16 @@
 #ifndef ARCH_CPU_H
 #define ARCH_CPU_H
 
-#include <ihk/cpu.h>
+#define arch_barrier()	asm volatile("" : : : "memory")
 
 static inline void rmb(void)
 {
-	barrier();
+	arch_barrier();
 }
 
 static inline void wmb(void)
 {
-	barrier();
+	arch_barrier();
 }
 
 static unsigned long read_tsc(void)

kernel/debug.c

@@ -170,6 +170,7 @@ int kprintf(const char *format, ...)
 		ihk_mc_delay_us(IHK_KMSG_NOTIFY_DELAY);
 	}
 
+	barrier();
 	return len;
 }
 

kernel/mem.c

@@ -1617,6 +1617,7 @@ void *ihk_mc_map_virtual(unsigned long phys, int npages,
 			return NULL;
 		}
 	}
+	barrier();
 	return (char *)p + offset;
 }
 

lib/include/ihk/cpu.h

@@ -16,6 +16,7 @@
 
 #include <list.h>
 #include <ihk/context.h>
+#include <arch/cpu.h>
 
 void cpu_enable_interrupt(void);
 void cpu_disable_interrupt(void);
@@ -24,7 +25,7 @@ void cpu_safe_halt(void);
 void cpu_restore_interrupt(unsigned long);
 void cpu_pause(void);
 
-#define barrier()   asm volatile("" : : : "memory")
+#define barrier()   arch_barrier()
 
 unsigned long cpu_disable_interrupt_save(void);