diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fa4ba42..a5c19553 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,12 @@ if (ENABLE_WERROR) add_compile_options("-Werror") endif(ENABLE_WERROR) +option(ENABLE_LINUX_WORK_IRQ_FOR_IKC "Use Linux work IRQ for IKC IPI" ON) +if (ENABLE_LINUX_WORK_IRQ_FOR_IKC) + set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DIHK_IKC_USE_LINUX_WORK_IRQ") + add_definitions(-DIHK_IKC_USE_LINUX_WORK_IRQ) +endif() + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") set(BUILD_TARGET "smp-x86" CACHE STRING "Build target: smp-x86 | smp-arm64") elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") @@ -186,4 +192,5 @@ message("ENABLE_QLMPI: ${ENABLE_QLMPI}") message("ENABLE_UTI: ${ENABLE_UTI}") message("ENABLE_WERROR: ${ENABLE_WERROR}") message("ENABLE_UBSAN: ${ENABLE_UBSAN}") +message("ENABLE_LINUX_WORK_IRQ_FOR_IKC: ${ENABLE_LINUX_WORK_IRQ_FOR_IKC}") message("-------------------------------") diff --git a/arch/arm64/kernel/cpu.c b/arch/arm64/kernel/cpu.c index 2297bd9e..8f9fca3c 100644 --- a/arch/arm64/kernel/cpu.c +++ b/arch/arm64/kernel/cpu.c @@ -67,6 +67,7 @@ void (*gic_dist_init)(unsigned long dist_base_pa, unsigned long size); void (*gic_cpu_init)(unsigned long cpu_base_pa, unsigned long size); void (*gic_enable)(void); void (*arm64_issue_ipi)(unsigned int cpid, unsigned int vector); +void (*arm64_issue_host_ipi)(unsigned int cpid, unsigned int vector); void (*handle_arch_irq)(struct pt_regs *); static void gic_init(void) @@ -77,14 +78,18 @@ static void gic_init(void) gic_cpu_init = gic_cpu_init_gicv3; gic_enable = gic_enable_gicv3; arm64_issue_ipi = arm64_issue_ipi_gicv3; + arm64_issue_host_ipi = arm64_issue_host_ipi_gicv3; handle_arch_irq = handle_interrupt_gicv3; + kprintf("%: GICv3\n", __func__); } else { /* Setup functions for GICv2 */ gic_dist_init = gic_dist_init_gicv2; gic_cpu_init = gic_cpu_init_gicv2; gic_enable = gic_enable_gicv2; arm64_issue_ipi = arm64_issue_ipi_gicv2; + arm64_issue_host_ipi = arm64_issue_host_ipi_gicv2; handle_arch_irq = handle_interrupt_gicv2; + kprintf("%: GICv2\n", __func__); } gic_dist_init(ihk_param_gic_dist_base_pa, ihk_param_gic_dist_map_size); diff --git a/arch/arm64/kernel/include/irq.h b/arch/arm64/kernel/include/irq.h index 17f641b8..704960e6 100644 --- a/arch/arm64/kernel/include/irq.h +++ b/arch/arm64/kernel/include/irq.h @@ -29,6 +29,7 @@ extern void gic_dist_init_gicv2(unsigned long dist_base_pa, unsigned long size); extern void gic_cpu_init_gicv2(unsigned long cpu_base_pa, unsigned long size); extern void gic_enable_gicv2(void); extern void arm64_issue_ipi_gicv2(unsigned int cpuid, unsigned int vector); +extern void arm64_issue_host_ipi_gicv2(uint32_t cpuid, uint32_t vector); extern void handle_interrupt_gicv2(struct pt_regs *regs); /* Functions for GICv3 */ @@ -36,6 +37,7 @@ extern void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size); extern void gic_cpu_init_gicv3(unsigned long cpu_base_pa, unsigned long size); extern void gic_enable_gicv3(void); extern void arm64_issue_ipi_gicv3(unsigned int cpuid, unsigned int vector); +extern void arm64_issue_host_ipi_gicv3(uint32_t cpuid, uint32_t vector); extern void handle_interrupt_gicv3(struct pt_regs *regs); void handle_IPI(unsigned int vector, struct pt_regs *regs); diff --git a/arch/arm64/kernel/irq-gic-v2.c b/arch/arm64/kernel/irq-gic-v2.c index 9a57a037..f957909d 100644 --- a/arch/arm64/kernel/irq-gic-v2.c +++ b/arch/arm64/kernel/irq-gic-v2.c @@ -31,10 +31,9 @@ void *cpu_base; * function, it is not necessary to perform the disable/enable * interrupts in this function as gic_raise_softirq() . */ -static void arm64_raise_sgi_gicv2(unsigned int cpuid, unsigned int vector) +static void __arm64_raise_sgi_gicv2(unsigned int hw_cpuid, unsigned int vector) { /* Build interrupt destination of the target cpu */ - unsigned int hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid]; uint8_t cpu_target_list = gic_hwid_to_affinity(hw_cpuid); /* @@ -50,6 +49,23 @@ static void arm64_raise_sgi_gicv2(unsigned int cpuid, unsigned int vector) ); } +static void arm64_raise_sgi_gicv2(uint32_t cpuid, uint32_t vector) +{ + /* Build interrupt destination of the target CPU */ + uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid]; + + __arm64_raise_sgi_gicv2(hw_cpuid, vector); +} + +static void arm64_raise_sgi_to_host_gicv2(uint32_t cpuid, uint32_t vector) +{ + /* Build interrupt destination of the target Linux/host CPU */ + uint32_t hw_cpuid = ihk_mc_get_apicid(cpuid); + + __arm64_raise_sgi_gicv2(hw_cpuid, vector); +} + + /** * arm64_raise_spi_gicv2 * @ref.impl nothing. @@ -77,6 +93,11 @@ static void arm64_raise_spi_gicv2(unsigned int cpuid, unsigned int vector) ); } +void arm64_issue_host_ipi_gicv2(uint32_t cpuid, uint32_t vector) +{ + arm64_raise_sgi_to_host_gicv2(cpuid, vector); +} + /** * arm64_issue_ipi_gicv2 * @param cpuid : hardware cpu id diff --git a/arch/arm64/kernel/irq-gic-v3.c b/arch/arm64/kernel/irq-gic-v3.c index 4fa17fad..9f72b4d0 100644 --- a/arch/arm64/kernel/irq-gic-v3.c +++ b/arch/arm64/kernel/irq-gic-v3.c @@ -195,15 +195,12 @@ static inline void gic_write_bpr1(uint32_t val) } #endif -static void arm64_raise_sgi_gicv3(uint32_t cpuid, uint32_t vector) +static void __arm64_raise_sgi_gicv3(uint32_t hw_cpuid, uint32_t vector) { uint64_t mpidr, cluster_id; uint16_t tlist; uint64_t val; - /* Build interrupt destination of the target cpu */ - uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid]; - /* * Ensure that stores to Normal memory are visible to the * other CPUs before issuing the IPI. @@ -239,6 +236,22 @@ static void arm64_raise_sgi_gicv3(uint32_t cpuid, uint32_t vector) } } +static void arm64_raise_sgi_gicv3(uint32_t cpuid, uint32_t vector) +{ + /* Build interrupt destination of the target CPU */ + uint32_t hw_cpuid = ihk_mc_get_cpu_info()->hw_ids[cpuid]; + + __arm64_raise_sgi_gicv3(hw_cpuid, vector); +} + +static void arm64_raise_sgi_to_host_gicv3(uint32_t cpuid, uint32_t vector) +{ + /* Build interrupt destination of the target Linux/host CPU */ + uint32_t hw_cpuid = ihk_mc_get_apicid(cpuid); + + __arm64_raise_sgi_gicv3(hw_cpuid, vector); +} + static void arm64_raise_spi_gicv3(uint32_t cpuid, uint32_t vector) { uint64_t spi_reg_offset; @@ -268,6 +281,11 @@ static void arm64_raise_lpi_gicv3(uint32_t cpuid, uint32_t vector) ekprintf("%s called.\n", __func__); } +void arm64_issue_host_ipi_gicv3(uint32_t cpuid, uint32_t vector) +{ + arm64_raise_sgi_to_host_gicv3(cpuid, vector); +} + void arm64_issue_ipi_gicv3(uint32_t cpuid, uint32_t vector) { dkprintf("Send irq#%d to cpuid=%d\n", vector, cpuid); @@ -344,9 +362,11 @@ static void init_spi_routing(uint32_t irq, uint32_t linux_cpu) void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size) { +#ifndef IHK_IKC_USE_LINUX_WORK_IRQ extern int spi_table[]; extern int nr_spi_table; int i; +#endif // !IHK_IKC_USE_LINUX_WORK_IRQ dist_base = map_fixed_area(dist_base_pa, size, 1 /*non chachable*/); @@ -357,6 +377,7 @@ void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size) } #endif +#ifndef IHK_IKC_USE_LINUX_WORK_IRQ /* initialize spi routing */ for (i = 0; i < nr_spi_table; i++) { if (spi_table[i] == -1) { @@ -364,6 +385,7 @@ void gic_dist_init_gicv3(unsigned long dist_base_pa, unsigned long size) } init_spi_routing(spi_table[i], i); } +#endif // !IHK_IKC_USE_LINUX_WORK_IRQ } void gic_cpu_init_gicv3(unsigned long cpu_base_pa, unsigned long size) diff --git a/ihk b/ihk index 70adc3dc..eb9420bb 160000 --- a/ihk +++ b/ihk @@ -1 +1 @@ -Subproject commit 70adc3dcfd02e878141634a2fe84e859216d431b +Subproject commit eb9420bbbc2a3d9223a8d04e796c4cc15b0db04a diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 545a9e88..608c435e 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -14,6 +14,7 @@ include_directories( "${CMAKE_CURRENT_BINARY_DIR}" "${PROJECT_BINARY_DIR}" "${IHK_FULL_SOURCE_DIR}/cokernel/smp/${ARCH}/include" + "${IHK_FULL_SOURCE_DIR}/cokernel/smp/${ARCH}/" "${IHK_FULL_SOURCE_DIR}/ikc/include" "${IHK_FULL_SOURCE_DIR}/linux/include" "${PROJECT_SOURCE_DIR}/lib/include" @@ -44,8 +45,8 @@ set(MCKERNEL_SRCS ${PROJECT_SOURCE_DIR}/lib/string.c ${PROJECT_SOURCE_DIR}/lib/vsprintf.c + ${IHK_FULL_SOURCE_DIR}/cokernel/smp/ikc.c ${IHK_FULL_SOURCE_DIR}/cokernel/smp/${ARCH}/dma.c - ${IHK_FULL_SOURCE_DIR}/cokernel/smp/${ARCH}/ikc.c ${IHK_FULL_SOURCE_DIR}/cokernel/smp/${ARCH}/setup.c ) diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h index c8292866..27767b9e 100644 --- a/lib/include/ihk/cpu.h +++ b/lib/include/ihk/cpu.h @@ -18,6 +18,8 @@ #include #include +extern int num_processors; + void cpu_enable_interrupt(void); void cpu_disable_interrupt(void); void cpu_halt(void); @@ -151,5 +153,6 @@ void arch_start_pvclock(void); struct cpu_mapping; int arch_get_cpu_mapping(struct cpu_mapping **buf, int *nelemsp); +int ihk_mc_ikc_arch_issue_host_ipi(int cpu, int vector); #endif diff --git a/scripts/mcreboot-smp.sh.in b/scripts/mcreboot-smp.sh.in index 11c87885..93d28ab7 100644 --- a/scripts/mcreboot-smp.sh.in +++ b/scripts/mcreboot-smp.sh.in @@ -26,6 +26,7 @@ ETCDIR=@ETCDIR@ KMODDIR="@KMODDIR@" KERNDIR="@MCKERNELDIR@" MCK_BUILDID=@BUILDID@ +ENABLE_LINUX_WORK_IRQ_FOR_IKC="@ENABLE_LINUX_WORK_IRQ_FOR_IKC@" mem="512M@0" cpus="" @@ -271,7 +272,7 @@ sudo sync # Load IHK-SMP if not loaded and reserve CPUs and memory if ! grep ihk_smp_@ARCH@ /proc/modules &>/dev/null; then - if [ "$ihk_irq" == "" ]; then + if [ "$ihk_irq" == "" ] && [ "$ENABLE_LINUX_WORK_IRQ_FOR_IKC" != "ON" ]; then for i in `seq 64 255`; do if [ ! -d /proc/irq/$i ] && [ "`cat /proc/interrupts | grep ":" | awk '{print $1}' | grep -o '[0-9]*' | grep -e '^$i$'`" == "" ]; then ihk_irq=$i @@ -283,7 +284,13 @@ if ! grep ihk_smp_@ARCH@ /proc/modules &>/dev/null; then error_exit "ihk_loaded" fi fi - if ! taskset -c 0 sudo insmod ${KMODDIR}/ihk-smp-@ARCH@.ko ihk_start_irq=$ihk_irq ihk_ikc_irq_core=$ihk_ikc_irq_core 2>/dev/null; then + + IHK_IRQ_ARG="" + if [ "${ihk_irq}" != "" ]; then + IHK_IRQ_ARG="ihk_start_irq=${ihk_irq}" + fi + + if ! taskset -c 0 sudo insmod ${KMODDIR}/ihk-smp-@ARCH@.ko ${IHK_IRQ_ARG} ihk_ikc_irq_core=$ihk_ikc_irq_core 2>/dev/null; then echo "error: loading ihk-smp-@ARCH@" >&2 error_exit "ihk_loaded" fi