Tofu: support for barrier gate, kmalloc cache

Change-Id: I6f4cfec2ec404efd03b332fc3f449a775816230e

executer/kernel/mcctrl/syscall.c

@@ -1874,10 +1874,18 @@ void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet,
 	res->stid = stid;
 
 #ifdef ENABLE_TOFU
-	/* Record PDE_DATA after ioctl() calls for Tofu driver */
+	/* Record PDE_DATA after open()/ioctl() calls for Tofu driver */
-	if (packet->req.number == __NR_ioctl && ret == 0) {
+	if ((packet->req.number == __NR_ioctl && ret == 0) ||
+			(packet->req.number == __NR_openat && ret > 1)) {
 		char *pathbuf, *fullpath;
-		struct fd f = fdget(packet->req.args[0]);
+		struct fd f;
+
+		if (packet->req.number == __NR_ioctl) {
+			f = fdget(packet->req.args[0]);
+		}
+		else if (packet->req.number == __NR_openat) {
+			f = fdget(ret);
+		}
 
 		if (!f.file) {
 			goto out_notify;
@@ -1895,18 +1903,19 @@ void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet,
 
 		if (!strncmp("/proc/tofu/dev/", fullpath, 15)) {
 			res->pde_data = PDE_DATA(file_inode(f.file));
-			printk("%s: fd: %ld, path: %s, PDE_DATA: 0x%lx\n",
+			dprintk("%s: %s(): fd: %ld, path: %s, PDE_DATA: 0x%lx\n",
 				__func__,
+				packet->req.number == __NR_ioctl ? "ioctl" : "openat",
 				packet->req.args[0],
 				fullpath,
 				(unsigned long)res->pde_data);
-			printk("%s: pgd_index: %ld, pmd_index: %ld, pte_index: %ld\n",
+			dprintk("%s: pgd_index: %ld, pmd_index: %ld, pte_index: %ld\n",
 				__func__,
 				pgd_index((unsigned long)res->pde_data),
 				pmd_index((unsigned long)res->pde_data),
 				pte_index((unsigned long)res->pde_data));
 #ifdef CONFIG_ARM64
-			printk("CONFIG_ARM64_VA_BITS: %d, PGDIR_SHIFT: %d\n",
+			dprintk("CONFIG_ARM64_VA_BITS: %d, PGDIR_SHIFT: %d\n",
 				CONFIG_ARM64_VA_BITS, PGDIR_SHIFT);
 #endif
 		}

kernel/include/process.h

@@ -750,6 +750,9 @@ struct thread {
 	void *coredump_regs;
 	struct waitq coredump_wq;
 	int coredump_status;
+
+	/* Path of file being opened */
+	char *fd_path_in_open;
 };
 
 #define VM_RANGE_CACHE_SIZE	4

kernel/include/profile.h

@@ -45,6 +45,16 @@ enum profile_event_type {
 	PROFILE_mmap_anon_no_contig_phys,
 	PROFILE_mmap_regular_file,
 	PROFILE_mmap_device_file,
+	PROFILE_tofu_stag_alloc,
+	PROFILE_tofu_stag_alloc_new_steering,
+	PROFILE_tofu_stag_alloc_new_steering_alloc_mbpt,
+	PROFILE_tofu_stag_alloc_new_steering_update_mbpt,
+	PROFILE_tofu_stag_free_stags,
+	PROFILE_tofu_stag_free_stag,
+	PROFILE_tofu_stag_free_stag_pre,
+	PROFILE_tofu_stag_free_stag_cqflush,
+	PROFILE_tofu_stag_free_stag_dealloc,
+	PROFILE_tofu_stag_free_stag_dealloc_free_pages,
 	PROFILE_EVENT_MAX	/* Should be the last event type */
 };
 

kernel/include/tofu/generate_headers.sh

@@ -15,9 +15,10 @@ if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2
 	exit 1
 fi
 
-${DWARF_TOOL} ${KMODULE} tof_utofu_device enabled > tofu_generated-tof_utofu_device.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_device enabled subnet gpid > tofu_generated-tof_utofu_device.h
 ${DWARF_TOOL} ${KMODULE} tof_utofu_cq common tni cqid trans steering mb num_stag | sed "s/struct FILL_IN_MANUALLY trans;/#include \"tof_utofu_cq_trans.h\"/g" > tofu_generated-tof_utofu_cq.h
 ${DWARF_TOOL} ${KMODULE} tof_utofu_mbpt ucq iova sg nsgents mbptstart pgsz kref > tofu_generated-tof_utofu_mbpt.h
+${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_IN_MANUALLY bch;/#include \"tof_utofu_bg_bch.h\"/g" > tofu_generated-tof_utofu_bg.h
 rm ${KMODULE}
 
 KMODULE=tof_core.ko 
@@ -28,6 +29,7 @@ if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2
 fi
 
 ${DWARF_TOOL} ${KMODULE} tof_core_cq reg | sed "s/struct FILL_IN_MANUALLY reg;/#include \"tof_core_cq_reg.h\"/g" > tofu_generated-tof_core_cq.h 
+${DWARF_TOOL} ${KMODULE} tof_core_bg lock reg irq subnet gpid sighandler | sed "s/struct FILL_IN_MANUALLY reg;/#include \"tof_core_bg_reg.h\"/g" > tofu_generated-tof_core_bg.h 
 rm ${KMODULE}
 
 #cat tofu_generated*.h

kernel/include/tofu/tof_core_bg_reg.h

@@ -0,0 +1,4 @@
+	struct {
+		void *bgs;
+		void *bch;
+	} reg;

kernel/include/tofu/tof_utofu_bg_bch.h

@@ -0,0 +1,6 @@
+	struct {
+		bool enabled;
+		uint64_t bgmask[TOF_ICC_NTNIS];
+		uintptr_t iova;
+		void *kaddr;
+	} bch;

kernel/include/tofu/tofu_generated-tof_core_bg.h

@@ -0,0 +1,29 @@
+struct tof_core_bg {
+	union {
+		char whole_struct[120];
+		struct {
+			char padding0[0];
+			spinlock_t lock;
+		};
+		struct {
+			char padding1[8];
+			#include "tof_core_bg_reg.h"
+		};
+		struct {
+			char padding2[24];
+			struct tof_core_irq irq;
+		};
+		struct {
+			char padding3[88];
+			tof_core_signal_handler sighandler;
+		};
+		struct {
+			char padding4[104];
+			uint64_t subnet;
+		};
+		struct {
+			char padding5[112];
+			uint32_t gpid;
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_bg.h

@@ -0,0 +1,21 @@
+struct tof_utofu_bg {
+	union {
+		char whole_struct[160];
+		struct {
+			char padding0[0];
+			struct tof_utofu_device common;
+		};
+		struct {
+			char padding1[80];
+			uint8_t tni;
+		};
+		struct {
+			char padding2[81];
+			uint8_t bgid;
+		};
+		struct {
+			char padding3[88];
+			#include "tof_utofu_bg_bch.h"
+		};
+	};
+};

kernel/include/tofu/tofu_generated-tof_utofu_device.h

@@ -5,5 +5,13 @@ struct tof_utofu_device {
 			char padding0[0];
 			bool enabled;
 		};
+		struct {
+			char padding1[12];
+			uint32_t gpid;
+		};
+		struct {
+			char padding2[24];
+			uint64_t subnet;
+		};
 	};
 };

kernel/include/tofu/tofu_stag_range.h

@@ -0,0 +1,15 @@
+#ifndef __TOFU_STAG_RANGE_HEADER__
+#define __TOFU_STAG_RANGE_HEADER__
+
+struct tof_utofu_cq;
+
+struct tofu_stag_range {
+	uintptr_t start, end;
+	int stag;
+	struct tof_utofu_cq *ucq;
+	struct list_head list; // per-vm_range list
+	struct list_head hash; // per-process stag hash
+};
+
+#endif // __TOFU_STAG_RANGE_HEADER__
+

kernel/profile.c

@@ -68,7 +68,16 @@ char *profile_event_names[] =
 	"mmap_anon_no_contig_phys",
 	"mmap_regular_file",
 	"mmap_device_file",
-	""
+	"tofu_stag_alloc ",
+	"|--new_steering ",
+	"   |-alloc_mbpt ",
+	"   |-update_mbpt",
+	"tofu_stag_free_stags",
+	"tofu_stag_free_stag",
+	"   |--------pre",
+	"   |----cqflush",
+	"   |----dealloc",
+	"      |---free_pages",
 };
 
 mcs_lock_node_t job_profile_lock = { 0 };

kernel/syscall.c

@@ -381,28 +381,30 @@ long do_syscall(struct syscall_request *req, int cpu)
 
 	rc = res.ret;
 
-	if (req->number == __NR_ioctl && rc == 0) {
+	if ((req->number == __NR_ioctl && rc == 0) ||
+			(req->number == __NR_openat && rc > 0)) {
+		int fd = req->number == __NR_ioctl ? req->args[0] : rc;
+		char *path = req->number == __NR_ioctl ?
+			thread->proc->fd_path[fd] : thread->fd_path_in_open;
+
 		if (cpu_local_var(current)->proc->enable_tofu &&
 				res.pde_data &&
-				!thread->proc->fd_pde_data[req->args[0]] &&
+				fd < MAX_FD_PDE &&
-				!strncmp(thread->proc->fd_path[req->args[0]],
+				!thread->proc->fd_pde_data[fd] &&
-					"/proc/tofu/dev/", 15)) {
+				!strncmp(path, "/proc/tofu/dev/", 15)) {
+			unsigned long irqstate;
 
-			if (req->args[0] < MAX_FD_PDE) {
+			irqstate = ihk_mc_spinlock_lock(&thread->proc->mckfd_lock);
-				unsigned long irqstate;
+			thread->proc->fd_pde_data[fd] = res.pde_data;
+			ihk_mc_spinlock_unlock(&thread->proc->mckfd_lock, irqstate);
 
-				irqstate = ihk_mc_spinlock_lock(&thread->proc->mckfd_lock);
+			dkprintf("%s: PID: %d, ioctl fd: %d, filename: "
-				thread->proc->fd_pde_data[req->args[0]] = res.pde_data;
+					"%s, pde_data: 0x%lx\n",
-				ihk_mc_spinlock_unlock(&thread->proc->mckfd_lock, irqstate);
+					__FUNCTION__,
-
+					thread->proc->pid,
-				kprintf("%s: PID: %d, ioctl fd: %d, filename: "
+					fd,
-						"%s, pde_data: 0x%lx\n",
+					path,
-						__FUNCTION__,
+					res.pde_data);
-						thread->proc->pid,
-						req->args[0],
-						thread->proc->fd_path[req->args[0]],
-						res.pde_data);
-			}
 		}
 	}
 
@@ -1292,16 +1294,20 @@ void terminate(int rc, int sig)
 	mcs_rwlock_writer_unlock_noirq(&proc->update_lock, &updatelock);
 
 #ifdef ENABLE_TOFU
-	/* Tofu: clean up stags, must be done before mcexec is gone */
+	/* Tofu: cleanup, must be done before mcexec is gone */
 	if (proc->enable_tofu) {
 		int fd;
 
 		for (fd = 0; fd < MAX_FD_PDE; ++fd) {
 			/* Tofu? */
 			if (proc->enable_tofu && proc->fd_pde_data[fd]) {
-				extern void tof_utofu_release_cq(void *pde_data);
+				extern void tof_utofu_release_fd(struct process *proc, int fd);
 
-				tof_utofu_release_cq(proc->fd_pde_data[fd]);
+				if (proc->fd_path[fd]) {
+					dkprintf("%s: -> tof_utofu_release_fd() @ fd: %d (%s)\n",
+							__func__, fd, proc->fd_path[fd]);
+				}
+				tof_utofu_release_fd(proc, fd);
 				proc->fd_pde_data[fd] = NULL;
 			}
 
@@ -3865,10 +3871,10 @@ SYSCALL_DECLARE(ioctl)
 	/* Tofu? */
 	if (proc->enable_tofu &&
 			fd < MAX_FD_PDE && thread->proc->fd_pde_data[fd]) {
-		extern long tof_utofu_unlocked_ioctl_cq(int fd,
+		extern long tof_utofu_unlocked_ioctl(int fd,
 				unsigned int cmd, unsigned long arg);
 
-		rc = tof_utofu_unlocked_ioctl_cq(fd,
+		rc = tof_utofu_unlocked_ioctl(fd,
 			ihk_mc_syscall_arg1(ctx),
 			ihk_mc_syscall_arg2(ctx));
 
@@ -3913,6 +3919,8 @@ SYSCALL_DECLARE(open)
 		goto out;
 	}
 
+	cpu_local_var(current)->fd_path_in_open = pathname;
+
 	dkprintf("open(): pathname=%s\n", pathname);
 	if (!strncmp(pathname, XPMEM_DEV_PATH, len)) {
 		rc = xpmem_open(pathname, flags, ctx);
@@ -3920,6 +3928,8 @@ SYSCALL_DECLARE(open)
 		rc = syscall_generic_forwarding(__NR_open, ctx);
 	}
 
+	cpu_local_var(current)->fd_path_in_open = NULL;
+
  out:
 	if (rc > 0 && rc < MAX_FD_PDE) {
 		cpu_local_var(current)->proc->fd_path[rc] = pathname;
@@ -3954,6 +3964,8 @@ SYSCALL_DECLARE(openat)
 		goto out;
 	}
 
+	cpu_local_var(current)->fd_path_in_open = pathname;
+
 	dkprintf("openat(): pathname=%s\n", pathname);
 	if (!strncmp(pathname, XPMEM_DEV_PATH, len)) {
 		rc = xpmem_openat(pathname, flags, ctx);
@@ -3961,6 +3973,8 @@ SYSCALL_DECLARE(openat)
 		rc = syscall_generic_forwarding(__NR_openat, ctx);
 	}
 
+	cpu_local_var(current)->fd_path_in_open = NULL;
+
 out:
 	if (rc > 0 && rc < MAX_FD_PDE) {
 		cpu_local_var(current)->proc->fd_path[rc] = pathname;
@@ -4021,9 +4035,13 @@ SYSCALL_DECLARE(close)
 	if (fd >= 0 && fd < MAX_FD_PDE) {
 		/* Tofu? */
 		if (thread->proc->fd_pde_data[fd]) {
-			extern void tof_utofu_release_cq(void *pde_data);
+			extern void tof_utofu_release_fd(struct process *proc, int fd);
 
-			tof_utofu_release_cq(thread->proc->fd_pde_data[fd]);
+			if (thread->proc->fd_path[fd]) {
+				dkprintf("%s: -> tof_utofu_release_fd() @ fd: %d (%s)\n",
+						__func__, fd, thread->proc->fd_path[fd]);
+			}
+			tof_utofu_release_fd(thread->proc, fd);
 			thread->proc->fd_pde_data[fd] = NULL;
 		}