From bf926f234a026c91c89327a2d6371b02a44abf78 Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Tue, 12 Jan 2021 20:36:12 +0900 Subject: [PATCH] Tofu: manage stag ranges in VM range split and misc cleanup Conflicts: kernel/process.c Change-Id: I480850fe93a7963a5bd4d1687fb1e5c43f58057f --- kernel/include/tofu/generate_headers.sh | 23 ++++- kernel/process.c | 14 ++- kernel/tofu/tof_utofu_main.c | 109 +++++++++++++++++++++--- 3 files changed, 127 insertions(+), 19 deletions(-) diff --git a/kernel/include/tofu/generate_headers.sh b/kernel/include/tofu/generate_headers.sh index d43b7730..fba4982f 100755 --- a/kernel/include/tofu/generate_headers.sh +++ b/kernel/include/tofu/generate_headers.sh @@ -6,10 +6,25 @@ CURRENT_DIR=`pwd` cd ${SCRIPT_DIR} -DWARF_TOOL=~/src/mckernel-apollo+a64fx/mckernel/tools/dwarf-extract-struct/dwarf-extract-struct +DWARF_TOOL=${SCRIPT_DIR}/../../../tools/dwarf-extract-struct/dwarf-extract-struct +if [ ! -x ${DWARF_TOOL} ]; then + echo "error: couldn't find DWARF extractor executable (${DWARF_TOOL}), have you compiled it?" + cd - + exit 1 +fi + +echo "Looking for Tofu driver debug symbols..." +if [ "`find /lib/modules/ -name "tof_module.tar.gz" | xargs -r ls -t | head -n 1 | wc -l`" == "0" ]; then + echo "error: couldn't find Tofu modules with debug symbols" + cd - + exit 1 +fi + +MODULE_TAR_GZ=`find /lib/modules/ -name "tof_module.tar.gz" | xargs ls -t | head -n 1` +echo "Using Tofu driver debug symbols: ${MODULE_TAR_GZ}" KMODULE=tof_utofu.ko -if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2>&1 > /dev/null; then +if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then echo "error: uncompressing kernel module with debug symbols" cd - exit 1 @@ -22,7 +37,7 @@ ${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_I rm ${KMODULE} KMODULE=tof_core.ko -if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2>&1 > /dev/null; then +if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then echo "error: uncompressing kernel module with debug symbols" cd - exit 1 @@ -33,4 +48,4 @@ ${DWARF_TOOL} ${KMODULE} tof_core_bg lock reg irq subnet gpid sighandler | sed " rm ${KMODULE} #cat tofu_generated*.h -cd - +cd - > /dev/null diff --git a/kernel/process.c b/kernel/process.c index 51c9750d..270d0781 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -965,8 +965,18 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range, newrange->private_data = range->private_data; #ifdef ENABLE_TOFU - /* TODO: figure out which entries to put on which list! */ INIT_LIST_HEAD(&newrange->tofu_stag_list); + { + extern int tofu_stag_split_vm_range_on_addr(struct process_vm *vm, + struct vm_range *range_low, struct vm_range *range_high, + uintptr_t addr); + + int moved = + tofu_stag_split_vm_range_on_addr(vm, range, newrange, addr); + if (moved > 0) { + kprintf("%s: moved %d stag ranges\n", __func__, moved); + } + } #endif if (range->memobj) { @@ -1182,7 +1192,7 @@ straight_out: entries = tofu_stag_range_remove_overlapping(vm, range); if (entries > 0) { - kprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n", + dkprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n", __func__, entries, range->start, diff --git a/kernel/tofu/tof_utofu_main.c b/kernel/tofu/tof_utofu_main.c index 36da31b3..e6bb8f54 100644 --- a/kernel/tofu/tof_utofu_main.c +++ b/kernel/tofu/tof_utofu_main.c @@ -160,6 +160,83 @@ int tofu_stag_range_remove_overlapping(struct process_vm *vm, return entries; } +void tofu_stag_range_remove_by_addr(struct process_vm *vm, + uintptr_t addr, size_t len) +{ + struct tofu_stag_range *tsr, *next; + int hash; + + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + for (hash = 0; hash < TOFU_STAG_HASH_SIZE; ++hash) { + list_for_each_entry_safe(tsr, next, + &vm->tofu_stag_hash[hash], hash) { + + if (tsr->start >= addr && tsr->end <= (addr + len)) { + linux_spin_lock(&tsr->ucq->trans.mru_lock); + tof_utofu_free_stag(tsr->ucq, tsr->stag); + linux_spin_unlock(&tsr->ucq->trans.mru_lock); + + kprintf("%s: removed stag %d in %p:%lu\n", + __func__, tsr->stag, addr, len); + __tofu_stag_range_remove(vm, tsr); + } + + { + uintptr_t max_start, min_end; + + max_start = addr > tsr->start ? addr : tsr->start; + min_end = (addr + len) < tsr->end ? (addr + len) : tsr->end; + + if ((tsr->start != 0 || vm->proc->status == PS_EXITED) && + (max_start < min_end)) { + linux_spin_lock(&tsr->ucq->trans.mru_lock); + tof_utofu_free_stag(tsr->ucq, tsr->stag); + linux_spin_unlock(&tsr->ucq->trans.mru_lock); + + kprintf("%s: removed stag %p:%lu (overlaps with range %p:%lu)\n", + __func__, tsr->start, (tsr->end - tsr->start), addr, len); + __tofu_stag_range_remove(vm, tsr); + } + } + } + } + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); +} + +int tofu_stag_split_vm_range_on_addr(struct process_vm *vm, + struct vm_range *range_low, struct vm_range *range_high, + uintptr_t addr) +{ + struct tofu_stag_range *tsr, *next; + int moved = 0; + + ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); + + list_for_each_entry_safe(tsr, next, + &range_low->tofu_stag_list, list) { + + if (tsr->start >= addr) { + list_del(&tsr->list); + list_add_tail(&tsr->list, &range_high->tofu_stag_list); + ++moved; + + kprintf("%s: stag: %d @ %p:%lu moved to high range..\n", + __func__, + tsr->stag, + tsr->start, + (unsigned long)(tsr->end - tsr->start)); + } + + if (tsr->start < addr && tsr->end > addr) { + kprintf("%s: WARNING: VM range split in middle of stag range..\n", __func__); + } + } + + ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); + + return moved; +} + #define TOF_UTOFU_VERSION TOF_UAPI_VERSION @@ -1463,7 +1540,8 @@ static int tof_utofu_free_stag(struct tof_utofu_cq *ucq, int stag){ #endif // PROFILE_ENABLE kref_put(&ucq->trans.mru[stag].mbpt->kref, tof_utofu_mbpt_release); ucq->trans.mru[stag].mbpt = NULL; - dkprintf("%s: stag: %d deallocated\n", __func__, stag); + dkprintf("%s: TNI: %d, CQ: %d, STAG: %d deallocated\n", + __func__, ucq->tni, ucq->cqid, stag); #ifdef PROFILE_ENABLE profile_event_add(PROFILE_tofu_stag_free_stag_dealloc, rdtsc() - ts_rolling); profile_event_add(PROFILE_tofu_stag_free_stag, rdtsc() - ts); @@ -1486,7 +1564,7 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon return -EFAULT; } //tof_log_if("[IN] tni=%d cqid=%d num=%u stags=%p\n", ucq->tni, ucq->cqid, req.num, req.stags); - dkprintf("%: [IN] tni=%d cqid=%d num=%u stags=%p\n", + dkprintf("%s: [IN] tni=%d cqid=%d num=%u stags=%p\n", __func__, ucq->tni, ucq->cqid, req.num, req.stags); if(req.num > 1024 || req.stags == NULL){ @@ -1559,10 +1637,8 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon void tof_utofu_release_cq(void *pde_data) { struct tof_utofu_cq *ucq; - //int stag; struct tof_utofu_device *dev; unsigned long irqflags; - struct process_vm *vm = cpu_local_var(current)->vm; int do_free = 1; dev = (struct tof_utofu_device *)pde_data; @@ -1574,15 +1650,10 @@ void tof_utofu_release_cq(void *pde_data) do_free = 0; } -#if 0 - for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) { - linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags); - tof_utofu_free_stag(ucq, stag); - linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags); -#endif { int i; struct tofu_stag_range *tsr, *next; + struct process_vm *vm = cpu_local_var(current)->vm; ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock); for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) { @@ -1610,6 +1681,17 @@ void tof_utofu_release_cq(void *pde_data) ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock); } + /* Loop through as well just to make sure everything is cleaned up */ + if (do_free) { + int stag; + + for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) { + linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags); + tof_utofu_free_stag(ucq, stag); + linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags); + } + } + dkprintf("%s: UCQ (pde: %p) TNI %d, CQ %d\n", __func__, pde_data, ucq->tni, ucq->cqid); } @@ -2356,9 +2438,10 @@ void tof_utofu_finalize(void) list_for_each_entry_safe(tsr, next, &vm->tofu_stag_hash[i], hash) { - dkprintf("%s: WARNING: stray stag %d for TNI %d CQ %d?\n", - __func__, tsr->stag, tsr->ucq->tni, tsr->ucq->cqid); - + dkprintf("%s: WARNING: stray stag %d (%p:%lu) for TNI %d CQ %d?\n", + __func__, tsr->stag, + tsr->start, tsr->end - tsr->start, + tsr->ucq->tni, tsr->ucq->cqid); } } kprintf("%s: STAG processing done\n", __func__);