diff --git a/CMakeLists.txt b/CMakeLists.txt index aa22cd67..9cb6e5bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,10 +26,8 @@ endif() if (BUILD_TARGET STREQUAL "smp-x86") set(ARCH "x86_64") - option(ENABLE_TOFU "Built-in tofu driver support" OFF) elseif (BUILD_TARGET STREQUAL "smp-arm64") set(ARCH "arm64") - option(ENABLE_TOFU "Built-in tofu driver support" ON) endif() include(GNUInstallDirs) @@ -52,6 +50,20 @@ if (ENABLE_WERROR) add_compile_options("-Werror") endif(ENABLE_WERROR) +execute_process(COMMAND bash -c "ls -ld /proc/tofu/ 2>/dev/null | wc -l" + OUTPUT_VARIABLE PROC_TOFU OUTPUT_STRIP_TRAILING_WHITESPACE) + +if(PROC_TOFU STREQUAL "1") + option(ENABLE_TOFU "Built-in tofu driver support" ON) +else() + option(ENABLE_TOFU "Built-in tofu driver support" OFF) +endif() + +if(ENABLE_TOFU) + add_definitions(-DENABLE_TOFU) + set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DENABLE_TOFU") +endif() + option(ENABLE_LINUX_WORK_IRQ_FOR_IKC "Use Linux work IRQ for IKC IPI" ON) if (ENABLE_LINUX_WORK_IRQ_FOR_IKC) set(KBUILD_C_FLAGS "${KBUILD_C_FLAGS} -DIHK_IKC_USE_LINUX_WORK_IRQ") diff --git a/arch/arm64/kernel/fault.c b/arch/arm64/kernel/fault.c index 04dbc5ce..ae59bef8 100644 --- a/arch/arm64/kernel/fault.c +++ b/arch/arm64/kernel/fault.c @@ -223,8 +223,12 @@ static int do_translation_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { +#ifdef ENABLE_TOFU // XXX: Handle kernel space page faults for Tofu driver //if (addr < USER_END) +#else + if (addr < USER_END) +#endif return do_page_fault(addr, esr, regs); do_bad_area(addr, esr, regs); diff --git a/arch/arm64/kernel/include/arch-memory.h b/arch/arm64/kernel/include/arch-memory.h index eab5cc18..3ef1bdac 100644 --- a/arch/arm64/kernel/include/arch-memory.h +++ b/arch/arm64/kernel/include/arch-memory.h @@ -94,7 +94,11 @@ extern char _end[]; # define LD_TASK_UNMAPPED_BASE UL(0x0000080000000000) # define TASK_UNMAPPED_BASE UL(0x0000100000000000) # define USER_END UL(0x0000400000000000) +#ifdef ENABLE_TOFU # define MAP_VMAP_START UL(0xffff7bdfffff0000) +#else +# define MAP_VMAP_START UL(0xffff780000000000) +#endif # define MAP_VMAP_SIZE UL(0x0000000100000000) # define MAP_FIXED_START UL(0xffff7ffffbdd0000) # define MAP_ST_START UL(0xffff800000000000) diff --git a/arch/arm64/kernel/memory.c b/arch/arm64/kernel/memory.c index 1274f35c..3173401e 100644 --- a/arch/arm64/kernel/memory.c +++ b/arch/arm64/kernel/memory.c @@ -217,11 +217,13 @@ static inline int ptl4_index(unsigned long addr) int idx = (addr >> PTL4_SHIFT) & PTL4_INDEX_MASK; return idx; } +#ifdef ENABLE_TOFU static inline int ptl3_index_linux(unsigned long addr) { int idx = (addr >> PTL3_SHIFT) & PTL3_INDEX_MASK_LINUX; return idx; } +#endif static inline int ptl3_index(unsigned long addr) { int idx = (addr >> PTL3_SHIFT) & PTL3_INDEX_MASK; @@ -281,6 +283,7 @@ static inline pte_t* ptl4_offset(const translation_table_t* ptl4, unsigned long return ptep; } +#ifdef ENABLE_TOFU static inline pte_t* ptl3_offset_linux(const pte_t* l4p, unsigned long addr) { pte_t* ptep = NULL; @@ -311,6 +314,7 @@ static inline pte_t* ptl3_offset_linux(const pte_t* l4p, unsigned long addr) } return ptep; } +#endif static inline pte_t* ptl3_offset(const pte_t* l4p, unsigned long addr) { @@ -991,10 +995,12 @@ static void init_normal_area(struct page_table *pt) tt = get_translation_table(pt); +#ifdef ENABLE_TOFU setup(tt, arm64_st_phys_base, arm64_st_phys_base + (1UL << 40)); return; +#endif for (i = 0; i < ihk_mc_get_nr_memory_chunks(); i++) { unsigned long map_start, map_end; @@ -1323,6 +1329,7 @@ out: return ret; } +#ifdef ENABLE_TOFU int ihk_mc_linux_pt_virt_to_phys_size(struct page_table *pt, const void *virt, unsigned long *phys, @@ -1373,7 +1380,7 @@ out: if(size) *size = lsize; return 0; } - +#endif int ihk_mc_pt_virt_to_phys_size(struct page_table *pt, const void *virt, diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 852d04b9..99c5b3f7 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -174,13 +174,14 @@ void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) arch_show_interrupt_context(regs); -#if 0 +#ifdef ENABLE_TOFU + info.si_signo = SIGSTOP; + info.si_errno = 0; +#else info.si_signo = SIGILL; info.si_errno = 0; info.si_code = ILL_ILLOPC; #endif - info.si_signo = SIGSTOP; - info.si_errno = 0; info._sifields._sigfault.si_addr = (void*)regs->pc; arm64_notify_die("Oops - bad mode", regs, &info, 0); diff --git a/config.h.in b/config.h.in index 6f35d33f..f0323e1a 100644 --- a/config.h.in +++ b/config.h.in @@ -16,9 +16,6 @@ /* whether perf is enabled */ #cmakedefine ENABLE_PERF 1 -/* whether built-in tofu driver is enabled */ -#cmakedefine ENABLE_TOFU 1 - /* whether qlmpi is enabled */ #cmakedefine ENABLE_QLMPI 1 diff --git a/executer/include/uprotocol.h b/executer/include/uprotocol.h index 2c1fdd4a..7debda29 100644 --- a/executer/include/uprotocol.h +++ b/executer/include/uprotocol.h @@ -168,7 +168,9 @@ struct program_load_desc { int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */ int straight_map; size_t straight_map_threshold; +#ifdef ENABLE_TOFU int enable_tofu; +#endif int nr_processes; int process_rank; __cpu_set_unit cpu_set[PLD_CPU_SET_SIZE]; @@ -215,7 +217,9 @@ struct syscall_response { unsigned long req_thread_status; long ret; unsigned long fault_address; +#ifdef ENABLE_TOFU void *pde_data; +#endif }; struct syscall_ret_desc { diff --git a/executer/kernel/mcctrl/arch/arm64/archdeps.c b/executer/kernel/mcctrl/arch/arm64/archdeps.c index 2d9a7003..2b5f9bf8 100644 --- a/executer/kernel/mcctrl/arch/arm64/archdeps.c +++ b/executer/kernel/mcctrl/arch/arm64/archdeps.c @@ -28,6 +28,7 @@ void *vdso_end; static struct vm_special_mapping (*vdso_spec)[2]; #endif +#ifdef ENABLE_TOFU /* Tofu CQ and barrier gate release functions */ struct file_operations *mcctrl_tof_utofu_procfs_ops_cq; int (*mcctrl_tof_utofu_release_cq)(struct inode *inode, @@ -35,6 +36,7 @@ int (*mcctrl_tof_utofu_release_cq)(struct inode *inode, struct file_operations *mcctrl_tof_utofu_procfs_ops_bch; int (*mcctrl_tof_utofu_release_bch)(struct inode *inode, struct file *filp); +#endif int arch_symbols_init(void) { @@ -52,6 +54,7 @@ int arch_symbols_init(void) return -EFAULT; #endif +#ifdef ENABLE_TOFU mcctrl_tof_utofu_procfs_ops_cq = (void *)kallsyms_lookup_name("tof_utofu_procfs_ops_cq"); if (WARN_ON(!mcctrl_tof_utofu_procfs_ops_cq)) @@ -71,6 +74,7 @@ int arch_symbols_init(void) (void *)kallsyms_lookup_name("tof_utofu_release_bch"); if (WARN_ON(!mcctrl_tof_utofu_release_bch)) return -EFAULT; +#endif return 0; } @@ -448,6 +452,7 @@ out: } +#ifdef ENABLE_TOFU /* * Tofu CQ and BCH release handlers */ @@ -549,3 +554,4 @@ int __mcctrl_tof_utofu_release_bch(struct inode *inode, struct file *filp) return __mcctrl_tof_utofu_release_handler(inode, filp, mcctrl_tof_utofu_release_bch); } +#endif diff --git a/executer/kernel/mcctrl/driver.c b/executer/kernel/mcctrl/driver.c index 39afb7c2..1f9573fc 100644 --- a/executer/kernel/mcctrl/driver.c +++ b/executer/kernel/mcctrl/driver.c @@ -50,7 +50,9 @@ extern void procfs_exit(int); extern void uti_attr_finalize(void); extern void binfmt_mcexec_init(void); extern void binfmt_mcexec_exit(void); +#ifdef ENABLE_TOFU extern void mcctrl_file_to_pidfd_hash_init(void); +#endif extern int mcctrl_os_read_cpu_register(ihk_os_t os, int cpu, struct ihk_os_cpu_register *desc); @@ -325,7 +327,9 @@ static int __init mcctrl_init(void) } binfmt_mcexec_init(); +#ifdef ENABLE_TOFU mcctrl_file_to_pidfd_hash_init(); +#endif if ((ret = symbols_init())) goto error; diff --git a/executer/kernel/mcctrl/mcctrl.h b/executer/kernel/mcctrl/mcctrl.h index ed9daa8c..e9c5c7fa 100644 --- a/executer/kernel/mcctrl/mcctrl.h +++ b/executer/kernel/mcctrl/mcctrl.h @@ -560,6 +560,7 @@ struct uti_futex_resp { wait_queue_head_t wq; }; +#ifdef ENABLE_TOFU /* * Hash table to keep track of files and related processes * and file descriptors. @@ -585,3 +586,4 @@ struct mcctrl_file_to_pidfd *mcctrl_file_to_pidfd_hash_lookup( int mcctrl_file_to_pidfd_hash_remove(struct file *filp, ihk_os_t os, struct task_struct *group_leader, int fd); #endif +#endif diff --git a/executer/kernel/mcctrl/syscall.c b/executer/kernel/mcctrl/syscall.c index 3fc69980..a0a464bf 100644 --- a/executer/kernel/mcctrl/syscall.c +++ b/executer/kernel/mcctrl/syscall.c @@ -1843,6 +1843,7 @@ static long pager_call(ihk_os_t os, struct syscall_request *req) return ret; } +#ifdef ENABLE_TOFU struct list_head mcctrl_file_to_pidfd_hash[MCCTRL_FILE_2_PIDFD_HASH_SIZE]; spinlock_t mcctrl_file_to_pidfd_hash_lock; @@ -1971,7 +1972,7 @@ unlock_out: spin_unlock_irqrestore(&mcctrl_file_to_pidfd_hash_lock, irqflags); return ret; } - +#endif void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, long ret, int stid) @@ -2458,6 +2459,7 @@ int __do_in_kernel_syscall(ihk_os_t os, struct ikc_scd_packet *packet) dprintk("%s: system call: %lx\n", __FUNCTION__, sc->args[0]); switch (sc->number) { +#ifdef ENABLE_TOFU case __NR_close: { struct fd f; int fd; @@ -2478,6 +2480,7 @@ int __do_in_kernel_syscall(ihk_os_t os, struct ikc_scd_packet *packet) break; } +#endif case __NR_mmap: ret = pager_call(os, sc); break; diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c index ec19146f..c48e2450 100644 --- a/executer/user/mcexec.c +++ b/executer/user/mcexec.c @@ -68,13 +68,13 @@ #include #endif /* !__aarch64__ */ #include +#include "../../config.h" #include "../include/uprotocol.h" #include #include "../include/uti.h" #include #include "archdep.h" #include "arch_args.h" -#include "../../config.h" #include #include #include @@ -204,7 +204,9 @@ static char *mpol_bind_nodes = NULL; static int uti_thread_rank = 0; static int uti_use_last_cpu = 0; static int enable_uti = 0; +#ifdef ENABLE_TOFU static int enable_tofu = 0; +#endif /* Partitioned execution (e.g., for MPI) */ static int nr_processes = 0; @@ -1787,12 +1789,14 @@ static struct option mcexec_options[] = { .flag = &enable_uti, .val = 1, }, +#ifdef ENABLE_TOFU { .name = "enable-tofu", .has_arg = no_argument, .flag = &enable_tofu, .val = 1, }, +#endif { .name = "debug-mcexec", .has_arg = no_argument, @@ -2816,7 +2820,9 @@ int main(int argc, char **argv) desc->straight_map = straight_map; desc->straight_map_threshold = straight_map_threshold; +#ifdef ENABLE_TOFU desc->enable_tofu = enable_tofu; +#endif /* user_start and user_end are set by this call */ if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) { diff --git a/ihk b/ihk index bf400518..30e8b79b 160000 --- a/ihk +++ b/ihk @@ -1 +1 @@ -Subproject commit bf40051828ca4a2af4e4139836ed7e46abffab25 +Subproject commit 30e8b79b7c0877dee6415327bab9a16b29932cde diff --git a/kernel/include/process.h b/kernel/include/process.h index 269bffca..1718836b 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -567,7 +567,9 @@ struct process { int thp_disable; int straight_map; +#ifdef ENABLE_TOFU int enable_tofu; +#endif size_t straight_map_threshold; // perf_event @@ -592,9 +594,11 @@ struct process { int coredump_barrier_count, coredump_barrier_count2; mcs_rwlock_lock_t coredump_lock; // lock for coredump +#ifdef ENABLE_TOFU #define MAX_FD_PDE 1024 void *fd_pde_data[MAX_FD_PDE]; char *fd_path[MAX_FD_PDE]; +#endif }; /* @@ -753,8 +757,10 @@ struct thread { struct waitq coredump_wq; int coredump_status; +#ifdef ENABLE_TOFU /* Path of file being opened */ char *fd_path_in_open; +#endif }; #define VM_RANGE_CACHE_SIZE 4 diff --git a/kernel/include/syscall.h b/kernel/include/syscall.h index de1aad48..12e909af 100644 --- a/kernel/include/syscall.h +++ b/kernel/include/syscall.h @@ -238,7 +238,9 @@ struct program_load_desc { int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */ int straight_map; size_t straight_map_threshold; +#ifdef ENABLE_TOFU int enable_tofu; +#endif int nr_processes; int process_rank; __cpu_set_unit cpu_set[PLD_CPU_SET_SIZE]; diff --git a/kernel/syscall.c b/kernel/syscall.c index eac055ff..ff21245f 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -220,7 +220,9 @@ long do_syscall(struct syscall_request *req, int cpu) req->ttid = 0; } res.req_thread_status = IHK_SCD_REQ_THREAD_SPINNING; +#ifdef ENABLE_TOFU res.pde_data = NULL; +#endif send_syscall(req, cpu, &res); if (req->rtid == -1) { @@ -381,6 +383,7 @@ long do_syscall(struct syscall_request *req, int cpu) rc = res.ret; +#ifdef ENABLE_TOFU if ((req->number == __NR_ioctl && rc == 0) || (req->number == __NR_openat && rc > 0)) { int fd = req->number == __NR_ioctl ? req->args[0] : rc; @@ -407,6 +410,7 @@ long do_syscall(struct syscall_request *req, int cpu) res.pde_data); } } +#endif if(req->number != __NR_exit_group){ --thread->in_syscall_offload; @@ -1528,7 +1532,9 @@ int process_cleanup_before_terminate(int pid) { struct process *proc; struct mcs_rwlock_node_irqsave lock; +#ifdef ENABLE_TOFU int fd; +#endif proc = find_process(pid, &lock); if (!proc) { @@ -1536,10 +1542,12 @@ int process_cleanup_before_terminate(int pid) return 0; } +#ifdef ENABLE_TOFU /* Clean up PDE file descriptors */ for (fd = 2; fd < MAX_FD_PDE; ++fd) { __process_cleanup_fd(proc, fd); } +#endif process_unlock(proc, &lock); return 0; @@ -3978,7 +3986,9 @@ SYSCALL_DECLARE(open) goto out; } +#ifdef ENABLE_TOFU cpu_local_var(current)->fd_path_in_open = pathname; +#endif dkprintf("open(): pathname=%s\n", pathname); if (!strncmp(pathname, XPMEM_DEV_PATH, len)) { @@ -3987,15 +3997,21 @@ SYSCALL_DECLARE(open) rc = syscall_generic_forwarding(__NR_open, ctx); } +#ifdef ENABLE_TOFU cpu_local_var(current)->fd_path_in_open = NULL; +#endif out: +#ifdef ENABLE_TOFU if (rc > 0 && rc < MAX_FD_PDE) { cpu_local_var(current)->proc->fd_path[rc] = pathname; } else { kfree(pathname); } +#else + kfree(pathname); +#endif return rc; } @@ -4023,7 +4039,9 @@ SYSCALL_DECLARE(openat) goto out; } +#ifdef ENABLE_TOFU cpu_local_var(current)->fd_path_in_open = pathname; +#endif dkprintf("openat(): pathname=%s\n", pathname); if (!strncmp(pathname, XPMEM_DEV_PATH, len)) { @@ -4032,15 +4050,21 @@ SYSCALL_DECLARE(openat) rc = syscall_generic_forwarding(__NR_openat, ctx); } +#ifdef ENABLE_TOFU cpu_local_var(current)->fd_path_in_open = NULL; +#endif out: +#ifdef ENABLE_TOFU if (rc > 0 && rc < MAX_FD_PDE) { cpu_local_var(current)->proc->fd_path[rc] = pathname; } else { kfree(pathname); } +#else + kfree(pathname); +#endif return rc; } diff --git a/lib/include/ihk/mm.h b/lib/include/ihk/mm.h index 47e79db0..c6e46742 100644 --- a/lib/include/ihk/mm.h +++ b/lib/include/ihk/mm.h @@ -227,9 +227,11 @@ int ihk_mc_get_memory_chunk(int id, unsigned long *start, unsigned long *end, int *numa_id); +#ifdef ENABLE_TOFU int ihk_mc_get_memory_chunk_dma_addr(int id, int tni, int cqid, uintptr_t *dma_addr); +#endif void remote_flush_tlb_cpumask(struct process_vm *vm, unsigned long addr, int cpu_id);