HFI1: hfi1_unmap_device_addresses() at process terminate time

arch/x86_64/kernel/memory.c

@@ -731,6 +731,26 @@ static void destroy_page_table(int level, struct page_table *pt)
 	return;
 }
 
+void ihk_mc_pt_destroy_pgd_subtree(struct page_table *pt, void *virt)
+{
+	int l4idx, l3idx, l2idx, l1idx;
+	unsigned long v = (unsigned long)virt;
+	struct page_table *lower;
+
+	GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx);
+
+	if (!(pt->entry[l4idx] & PF_PRESENT))
+		return;
+
+	lower = (struct page_table *)
+		phys_to_virt(pt->entry[l4idx] & PT_PHYSMASK);
+	destroy_page_table(3, lower);
+
+	pt->entry[l4idx] = 0;
+	dkprintf("%s: virt: 0x%lx, l4idx: %d subtree destroyed\n",
+		__FUNCTION__, virt, l4idx);
+}
+
 void ihk_mc_pt_destroy(struct page_table *pt)
 {
 	const int level = 4;	/* PML4 */

kernel/include/process.h

@@ -575,12 +575,15 @@ struct process {
 	int nr_processes; /* For partitioned execution */
 	int process_rank; /* Rank in partition */
 
-	void *fd_priv_table[265];
+	void *fd_priv_table[256];
 	/* HFI1 specific */
 	void *hfi1_kregbase;
 	void *hfi1_piobase;
 	void *hfi1_rcvarray_wc;
+	size_t hfi1_rcvarray_wc_len;
 	void *hfi1_cq_comps;
+	size_t hfi1_cq_comps_len;
+	ihk_spinlock_t hfi1_lock;
 };
 
 /*

kernel/process.c

@@ -148,6 +148,7 @@ init_process(struct process *proc, struct process *parent)
 	mcs_rwlock_init(&proc->threads_lock);
 	mcs_rwlock_init(&proc->children_lock);
 	ihk_mc_spinlock_init(&proc->mckfd_lock);
+	ihk_mc_spinlock_init(&proc->hfi1_lock);
 	waitq_init(&proc->waitpid_q);
 	ihk_atomic_set(&proc->refcount, 2);
 	proc->monitoring_event = NULL;

kernel/syscall.c

@@ -1078,6 +1078,12 @@ void terminate(int rc, int sig)
 	mcs_rwlock_writer_unlock(&proc->threads_lock, &lock);
 
 	vm = proc->vm;
+
+	{
+		extern int hfi1_unmap_device_addresses(struct process *proc);
+		hfi1_unmap_device_addresses(proc);
+	}
+
 	free_all_process_memory_range(vm);
 
 	if (proc->saved_cmdline) {

kernel/user_exp_rcv.c

@@ -232,13 +232,12 @@ int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd, struct hfi1_tid_info *tinf
 	/* Technically should never be needed (because mapped previously
 	 * on update), but this call is no-op if addresses have been set
 	 * previously
-	 */
 	if (hfi1_map_device_addresses(fd) < 0) {
 		kprintf("%s: Could not map hfi1 device addresses\n",
 			__FUNCTION__);
 		return -EINVAL;
 	}
-
+	*/
 
 	dkprintf("Clear called, cnt %d\n", tinfo->tidcnt);
 	for (tididx = 0; tididx < tinfo->tidcnt; tididx++) {

kernel/user_sdma.c

@@ -51,6 +51,14 @@
 #include <hfi1/user_exp_rcv.h>
 #include <hfi1/common.h> 
 
+//#define DEBUG_PRINT_HFI1_SDMA
+
+#ifdef DEBUG_PRINT_HFI1_SDMA
+#define dkprintf(...) kprintf(__VA_ARGS__)
+#else
+#define dkprintf(...) do { if(0) kprintf(__VA_ARGS__); } while (0)
+#endif
+
 #ifdef __HFI1_ORIG__
 
 #include <linux/mm.h>
@@ -564,12 +572,16 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 #endif
 	unsigned long phys;
 	unsigned long len;
+	unsigned long irqstate;
+	int ret = 0;
 
 	struct process *proc = cpu_local_var(current)->proc;
 	struct process_vm *vm = cpu_local_var(current)->vm;
 	struct hfi1_user_sdma_comp_q *cq = fd->cq;
 	struct hfi1_devdata *dd = fd->dd;
 
+	irqstate = ihk_mc_spinlock_lock(&proc->hfi1_lock);
+
 	/*
 	 * Map device addresses if not mapped or mapping changed.
 	 */
@@ -592,7 +604,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 						virt, phys, attr) < 0) {
 				kprintf("%s: ERROR: failed to map kregbase: 0x%lx -> 0x%lx\n",
 						__FUNCTION__, virt, phys);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
@@ -600,7 +613,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!ptep && !pte_is_present(ptep)) {
 				kprintf("%s: ERROR: no mapping in McKernel for kregbase: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(),
@@ -608,17 +622,19 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!lptep && !pte_is_present(lptep)) {
 				kprintf("%s: ERROR: no mapping in Linux for kregbase: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			*ptep = *lptep;
 		}
 
-		kprintf("%s: hfi1_kregbase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
+		dkprintf("%s: hfi1_kregbase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
 				__FUNCTION__,
 				hfi1_kregbase,
 				hfi1_kregbase + TXE_PIO_SEND,
 				(phys - TXE_PIO_SEND), TXE_PIO_SEND);
+		//ihk_mc_pt_print_pte(vm->address_space->page_table, hfi1_kregbase);
 
 		proc->hfi1_kregbase = hfi1_kregbase;
 	}
@@ -635,7 +651,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 						virt, phys, attr) < 0) {
 				kprintf("%s: ERROR: failed to map piobase: 0x%lx -> 0x%lx\n",
 					__FUNCTION__, virt, phys);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
@@ -643,7 +660,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!ptep && !pte_is_present(ptep)) {
 				kprintf("%s: ERROR: no mapping in McKernel for piobase: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(),
@@ -651,13 +669,14 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!lptep && !pte_is_present(lptep)) {
 				kprintf("%s: ERROR: no mapping in Linux for piobase: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			*ptep = *lptep;
 		}
 
-		kprintf("%s: hfi1_piobase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
+		dkprintf("%s: hfi1_piobase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
 				__FUNCTION__,
 				hfi1_piobase,
 				hfi1_piobase + TXE_PIO_SIZE,
@@ -679,7 +698,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 						virt, phys, attr) < 0) {
 				kprintf("%s: ERROR: failed to map rcvarray_wc: 0x%lx -> 0x%lx\n",
 						__FUNCTION__, virt, phys);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
@@ -687,7 +707,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!ptep && !pte_is_present(ptep)) {
 				kprintf("%s: ERROR: no mapping in McKernel for rcvarray: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(),
@@ -695,13 +716,14 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!lptep && !pte_is_present(lptep)) {
 				kprintf("%s: ERROR: no mapping in Linux for rcvarray: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			*ptep = *lptep;
 		}
 
-		kprintf("%s: hfi1_rcvarray_wc: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
+		dkprintf("%s: hfi1_rcvarray_wc: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
 				__FUNCTION__,
 				hfi1_rcvarray_wc,
 				hfi1_rcvarray_wc + dd->chip_rcv_array_count * 8,
@@ -709,6 +731,7 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 				dd->chip_rcv_array_count * 8);
 
 		proc->hfi1_rcvarray_wc = hfi1_rcvarray_wc;
+		proc->hfi1_rcvarray_wc_len = dd->chip_rcv_array_count * 8;
 	}
 
 	/*
@@ -727,7 +750,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!lptep && !pte_is_present(lptep)) {
 				kprintf("%s: ERROR: no mapping in Linux for cq: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			phys = pte_get_phys(lptep);
@@ -750,20 +774,103 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd)
 			if (!ptep) {
 				kprintf("%s: ERROR: no PTE in McKernel for cq: 0x%lx?\n",
 						__FUNCTION__, virt);
-				return -1;
+				ret = -1;
+				goto unlock_out;
 			}
 
 			*ptep = *lptep;
-			kprintf("%s: cq: 0x%lx -> 0x%lx mapped\n",
-					__FUNCTION__,
-					virt, phys);
 		}
 
+		dkprintf("%s: hfi1_cq_comps: 0x%lx - 0x%lx mapped\n",
+				__FUNCTION__,
+				cq->comps, len);
+
 		proc->hfi1_cq_comps = cq->comps;
+		proc->hfi1_cq_comps_len = len;
 	}
 
-	/* TODO: unmap these at close() time */
+unlock_out:
-	return 0;
+	ihk_mc_spinlock_unlock(&proc->hfi1_lock, irqstate);
+
+	return ret;
+}
+
+
+int hfi1_unmap_device_addresses(struct process *proc)
+{
+	unsigned long irqstate;
+	int ret = 0;
+
+	struct process_vm *vm = proc->vm;
+	extern void ihk_mc_pt_destroy_pgd_subtree(struct page_table *pt,
+			void *virt);
+
+	irqstate = ihk_mc_spinlock_lock(&proc->hfi1_lock);
+
+	/*
+	 * Unmap device addresses if mapped.
+	 */
+	if (proc->hfi1_kregbase) {
+
+		ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table,
+			proc->hfi1_kregbase);
+/*
+		ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm,
+				proc->hfi1_kregbase, proc->hfi1_kregbase + TXE_PIO_SEND);
+
+		kprintf("%s: hfi1_kregbase unmapped\n",
+				__FUNCTION__);
+*/
+		proc->hfi1_kregbase = 0;
+	}
+
+	if (proc->hfi1_piobase) {
+
+		ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table,
+			proc->hfi1_piobase);
+/*
+		ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm,
+				proc->hfi1_piobase, proc->hfi1_piobase + TXE_PIO_SIZE);
+
+		kprintf("%s: hfi1_piobase unmapped\n",
+				__FUNCTION__);
+*/
+		proc->hfi1_piobase = 0;
+	}
+
+	if (proc->hfi1_rcvarray_wc) {
+
+		ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table,
+			proc->hfi1_rcvarray_wc);
+/*
+		ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm,
+				proc->hfi1_rcvarray_wc,
+				proc->hfi1_rcvarray_wc + proc->hfi1_rcvarray_wc_len);
+
+		kprintf("%s: hfi1_rcvarray_wc unmapped\n",
+				__FUNCTION__);
+*/
+		proc->hfi1_rcvarray_wc = 0;
+	}
+
+	if (proc->hfi1_cq_comps) {
+
+		ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table,
+			proc->hfi1_cq_comps);
+/*
+		ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm,
+				proc->hfi1_cq_comps,
+				proc->hfi1_cq_comps + proc->hfi1_cq_comps_len);
+
+		kprintf("%s: hfi1_cq_comps unmapped\n",
+				__FUNCTION__);
+*/
+		proc->hfi1_cq_comps = 0;
+	}
+
+	ihk_mc_spinlock_unlock(&proc->hfi1_lock, irqstate);
+
+	return ret;
 }
 
 struct kmalloc_cache_header tids_cache = {NULL};