HFI1: do device ioremap() mappings in per-process fashion

kernel/file_ops.c

@@ -416,6 +416,8 @@ static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
 }
 #endif /* __HFI1_ORIG__ */
 
+int hfi1_map_device_addresses(struct hfi1_filedata *fd);
+
 #ifdef __HFI1_ORIG__
 static ssize_t hfi1_aio_write(struct kiocb *kiocb, const struct iovec *iovec,
 			      unsigned long dim, loff_t offset)
@@ -452,6 +454,10 @@ ssize_t hfi1_aio_write(void *private_data, const struct iovec *iovec, unsigned l
 	if (!dim)
 		return -EINVAL;
 
+	if (hfi1_map_device_addresses(fd) < 0) {
+		return -EINVAL;
+	}
+
 	hfi1_cdbg(SDMA, "SDMA request from %u:%u (%lu)",
 		  fd->uctxt->ctxt, fd->subctxt, dim);
 

kernel/include/process.h

@@ -575,6 +575,11 @@ struct process {
 	int process_rank; /* Rank in partition */
 
 	void *fd_priv_table[265];
+	/* HFI1 specific */
+	void *hfi1_kregbase;
+	void *hfi1_piobase;
+	void *hfi1_rcvarray_wc;
+	void *hfi1_cq_comps;
 };
 
 /*

kernel/syscall.c

@@ -481,10 +481,13 @@ long do_syscall(struct syscall_request *req, int cpu, int pid)
 #endif // PROFILE_ENABLE
 
 	if (req->number == __NR_open && rc > 0) {
-		if (res.private_data && !strncmp((const char *)req->args[0], "/dev/hfi", 8)) {
+		if (res.private_data &&
+				!strncmp((const char *)req->args[0], "/dev/hfi", 8)) {
 			thread->proc->fd_priv_table[rc] = res.private_data;
-			kprintf("%s: PID: %d, open fd: %d, filename: %s, private_data: 0x%lx\n",
+			kprintf("%s: PID: %d, open fd: %d, filename: "
-				__FUNCTION__, thread->proc->pid, rc, req->args[0], res.private_data);
+					"%s, private_data: 0x%lx\n",
+					__FUNCTION__, thread->proc->pid,
+					rc, req->args[0], res.private_data);
 		}
 	}
 

kernel/user_sdma.c

@@ -532,13 +532,217 @@ static u8 dlid_to_selector(u16 dlid)
 #define TXE_PIO_SEND (TXE + TXE_PIO_SEND_OFFSET)
 #define TXE_PIO_SEND_OFFSET 0x0800000
 #define TXE_PIO_SIZE (32 * 0x100000)	/* 32 MB */
-
-/* HFI1 device address in McKernel */
-void *hfi1_kregbase = 0;
-void *hfi1_piobase = 0;
-void *hfi1_rcvarray_wc = 0;
 #endif
 
+int hfi1_map_device_addresses(struct hfi1_filedata *fd)
+{
+	pte_t *lptep;
+	pte_t *ptep;
+	enum ihk_mc_pt_attribute attr;
+	void *virt;
+	size_t pgsize;
+	unsigned long phys;
+	unsigned long len;
+
+	struct process *proc = cpu_local_var(current)->proc;
+	struct process_vm *vm = cpu_local_var(current)->vm;
+	struct hfi1_user_sdma_comp_q *cq = fd->cq;
+	struct hfi1_user_sdma_pkt_q *pq = fd->pq;
+	struct hfi1_devdata *dd = pq->dd;
+
+	/*
+	 * Map device addresses if not mapped or mapping changed.
+	 */
+	if (proc->hfi1_kregbase != dd->kregbase) {
+		void *hfi1_kregbase = dd->kregbase;
+
+		phys = dd->physaddr;
+		attr = PTATTR_UNCACHABLE | PTATTR_WRITABLE;
+		/*
+		 * No race condition here as ihk_mc_pt_set_page() holds
+		 * the lock to kernel space mapping manipulation
+		 *
+		 * XXX: use large pages?
+		 * XXX: where are we going to unmap this?
+		 */
+
+		for (virt = hfi1_kregbase; virt < (hfi1_kregbase + TXE_PIO_SEND);
+				virt += PAGE_SIZE, phys += PAGE_SIZE) {
+			if (ihk_mc_pt_set_page(vm->address_space->page_table,
+						virt, phys, attr) < 0) {
+				kprintf("%s: WARNING: failed to map kregbase: 0x%lx -> 0x%lx\n",
+						__FUNCTION__, virt, phys);
+			}
+
+			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
+					virt, 0, 0, 0, 0);
+			if (!ptep && !pte_is_present(ptep)) {
+				kprintf("%s: ERROR: no mapping in McKernel for kregbase: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(),
+					virt, 0, 0, 0, 0);
+			if (!lptep && !pte_is_present(lptep)) {
+				kprintf("%s: ERROR: no mapping in Linux for kregbase: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			*ptep = *lptep;
+		}
+
+		kprintf("%s: hfi1_kregbase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
+				__FUNCTION__,
+				hfi1_kregbase,
+				hfi1_kregbase + TXE_PIO_SEND,
+				(phys - TXE_PIO_SEND), TXE_PIO_SEND);
+
+		proc->hfi1_kregbase = hfi1_kregbase;
+	}
+
+	if (proc->hfi1_piobase != dd->piobase) {
+		void *hfi1_piobase = dd->piobase;
+
+		phys = dd->physaddr + TXE_PIO_SEND;
+		attr = PTATTR_WRITE_COMBINED | PTATTR_WRITABLE;
+
+		for (virt = hfi1_piobase; virt < (hfi1_piobase + TXE_PIO_SIZE);
+				virt += PAGE_SIZE, phys += PAGE_SIZE) {
+			if (ihk_mc_pt_set_page(vm->address_space->page_table,
+						virt, phys, attr) < 0) {
+				kprintf("%s: WARNING: failed to map piobase: 0x%lx -> 0x%lx\n",
+					__FUNCTION__, virt, phys);
+			}
+
+			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
+					virt, 0, 0, 0, 0);
+			if (!ptep && !pte_is_present(ptep)) {
+				kprintf("%s: ERROR: no mapping in McKernel for piobase: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(),
+					virt, 0, 0, 0, 0);
+			if (!lptep && !pte_is_present(lptep)) {
+				kprintf("%s: ERROR: no mapping in Linux for piobase: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			*ptep = *lptep;
+		}
+
+		kprintf("%s: hfi1_piobase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
+				__FUNCTION__,
+				hfi1_piobase,
+				hfi1_piobase + TXE_PIO_SIZE,
+				(phys - TXE_PIO_SIZE), TXE_PIO_SIZE);
+
+		proc->hfi1_piobase = hfi1_piobase;
+	}
+
+	if (proc->hfi1_rcvarray_wc != dd->rcvarray_wc) {
+		void *hfi1_rcvarray_wc = dd->rcvarray_wc;
+
+		phys = dd->physaddr + RCV_ARRAY;
+		attr = PTATTR_WRITE_COMBINED | PTATTR_WRITABLE;
+
+		for (virt = hfi1_rcvarray_wc;
+				virt < (hfi1_rcvarray_wc + dd->chip_rcv_array_count * 8);
+				virt += PAGE_SIZE, phys += PAGE_SIZE) {
+			if (ihk_mc_pt_set_page(vm->address_space->page_table,
+						virt, phys, attr) < 0) {
+				kprintf("%s: WARNING: failed to map rcvarray_wc: 0x%lx -> 0x%lx\n",
+						__FUNCTION__, virt, phys);
+			}
+
+			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
+					virt, 0, 0, 0, 0);
+			if (!ptep && !pte_is_present(ptep)) {
+				kprintf("%s: ERROR: no mapping in McKernel for rcvarray: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(),
+					virt, 0, 0, 0, 0);
+			if (!lptep && !pte_is_present(lptep)) {
+				kprintf("%s: ERROR: no mapping in Linux for rcvarray: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			*ptep = *lptep;
+		}
+
+		kprintf("%s: hfi1_rcvarray_wc: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
+				__FUNCTION__,
+				hfi1_rcvarray_wc,
+				hfi1_rcvarray_wc + dd->chip_rcv_array_count * 8,
+				(phys - dd->chip_rcv_array_count * 8),
+				dd->chip_rcv_array_count * 8);
+
+		proc->hfi1_rcvarray_wc = hfi1_rcvarray_wc;
+	}
+
+	/*
+	 * Map in cq->comps, allocated by vmalloc_user() in Linux.
+	 */
+	if (proc->hfi1_cq_comps != cq->comps) {
+		len = ((sizeof(*cq->comps) * cq->nentries)
+				+ PAGE_SIZE - 1) & PAGE_MASK;
+		attr = PTATTR_WRITABLE;
+
+		for (virt = (void *)cq->comps; virt < (((void *)cq->comps) + len);
+				virt += PAGE_SIZE) {
+
+			lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(),
+					virt, 0, 0, 0, 0);
+			if (!lptep && !pte_is_present(lptep)) {
+				kprintf("%s: ERROR: no mapping in Linux for cq: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			phys = pte_get_phys(lptep);
+
+			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
+					virt, 0, 0, 0, 0);
+			if (ptep && pte_is_present(ptep) && pte_get_phys(ptep) == phys) {
+				continue;
+			}
+
+			if (ihk_mc_pt_set_page(vm->address_space->page_table,
+						virt, phys, attr) < 0) {
+				/* Not necessarily an error.. */
+				kprintf("%s: WARNING: mapping cq: 0x%lx -> 0x%lx\n",
+						__FUNCTION__, virt, phys);
+			}
+
+			ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
+					virt, 0, 0, 0, 0);
+			if (!ptep) {
+				kprintf("%s: ERROR: no PTE in McKernel for cq: 0x%lx?\n",
+						__FUNCTION__, virt);
+				return -1;
+			}
+
+			*ptep = *lptep;
+			kprintf("%s: cq: 0x%lx -> 0x%lx mapped\n",
+					__FUNCTION__,
+					virt, phys);
+		}
+
+		proc->hfi1_cq_comps = cq->comps;
+	}
+
+	/* TODO: unmap these at close() time */
+	return 0;
+}
+
 #ifdef __HFI1_ORIG__
 int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec,
 				   unsigned long dim, unsigned long *count)
@@ -565,146 +769,6 @@ int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec,
 	u16 dlid;
 	u32 selector;
 
-#ifndef __HFI1_ORIG__
-	if (!hfi1_kregbase) {
-		struct process_vm *vm = cpu_local_var(current)->vm;
-		enum ihk_mc_pt_attribute attr = PTATTR_UNCACHABLE | PTATTR_WRITABLE;
-		void *virt;
-#if 1
-		unsigned long phys = dd->physaddr;
-		hfi1_kregbase = dd->kregbase;
-		hfi1_piobase = dd->piobase;
-		hfi1_rcvarray_wc = dd->rcvarray_wc;
-#else
-		unsigned long phys = *(uint64_t *)(((void *)dd) + 0xbf8);
-		hfi1_kregbase = *(uint64_t *)(((void *)dd) + 0xbe8);
-#endif
-		/*
-		 * No race condition here as ihk_mc_pt_set_page() holds
-		 * the lock to kernel space mapping manipulation
-		 *
-		 * XXX: use large pages?
-		 * XXX: where are we going to unmap this?
-		 */
-		for (virt = hfi1_kregbase; virt < (hfi1_kregbase + TXE_PIO_SEND);
-				virt += PAGE_SIZE, phys += PAGE_SIZE) {
-			if (ihk_mc_pt_set_page(vm->address_space->page_table,
-						virt, phys, attr) < 0) {
-				kprintf("%s: ERROR: mapping kregbase: 0x%lx -> 0x%lx\n",
-						__FUNCTION__, virt, phys);
-			}
-
-			{
-				pte_t *ptep;
-				struct process_vm *vm = cpu_local_var(current)->vm;
-
-				ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
-						virt, 0, 0, 0, 0);
-				if (ptep && pte_is_present(ptep)) {
-					kprintf("%s: 0x%lx, PTE: 0x%lx\n",
-						__FUNCTION__, virt, *ptep);
-				}
-			}
-		}
-
-		kprintf("%s: hfi1_kregbase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
-				__FUNCTION__,
-				hfi1_kregbase,
-				hfi1_kregbase + TXE_PIO_SEND,
-				(phys - TXE_PIO_SEND), TXE_PIO_SEND);
-		
-		phys = dd->physaddr + TXE_PIO_SEND;
-		attr = PTATTR_WRITE_COMBINED | PTATTR_WRITABLE;
-		for (virt = hfi1_piobase; virt < (hfi1_piobase + TXE_PIO_SIZE);
-				virt += PAGE_SIZE, phys += PAGE_SIZE) {
-			if (ihk_mc_pt_set_page(vm->address_space->page_table,
-						virt, phys, attr) < 0) {
-				kprintf("%s: ERROR: mapping piobase: 0x%lx -> 0x%lx\n",
-					__FUNCTION__, virt, phys);
-			}
-		}
-
-		kprintf("%s: hfi1_piobase: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
-				__FUNCTION__,
-				hfi1_piobase,
-				hfi1_piobase + TXE_PIO_SIZE,
-				(phys - TXE_PIO_SIZE), TXE_PIO_SIZE);	
-
-		phys = dd->physaddr + RCV_ARRAY;
-		attr = PTATTR_WRITE_COMBINED | PTATTR_WRITABLE;
-		for (virt = hfi1_rcvarray_wc; 
-				virt < (hfi1_rcvarray_wc + dd->chip_rcv_array_count * 8);
-				virt += PAGE_SIZE, phys += PAGE_SIZE) {
-			if (ihk_mc_pt_set_page(vm->address_space->page_table,
-						virt, phys, attr) < 0) {
-				kprintf("%s: ERROR: mapping rcvarray_wc: 0x%lx -> 0x%lx\n",
-						__FUNCTION__, virt, phys);
-			}
-		}
-
-		kprintf("%s: hfi1_rcvarray_wc: 0x%lx - 0x%lx -> 0x%lx:%lu\n",
-				__FUNCTION__,
-				hfi1_rcvarray_wc,
-				hfi1_rcvarray_wc + dd->chip_rcv_array_count * 8,
-				(phys - dd->chip_rcv_array_count * 8),
-				dd->chip_rcv_array_count * 8);
-	
-
-		/*
-		* Map in cq->comps, allocated dynamically by
-		* vmalloc_user() in Linux.
-		*/
-		{
-			void *virt;
-			pte_t *ptep;
-			unsigned long phys;
-			size_t pgsize;
-			unsigned long len = ((sizeof(*cq->comps) * cq->nentries)
-					+ PAGE_SIZE - 1) & PAGE_MASK;
-			struct process_vm *vm = cpu_local_var(current)->vm;
-			enum ihk_mc_pt_attribute attr = PTATTR_WRITABLE;
-
-			ihk_mc_spinlock_lock_noirq(&vm->page_table_lock);
-			
-			/* Check if mapping exists already and map if not */
-			for (virt = (void *)cq->comps; virt < (((void *)cq->comps) + len);
-					virt += PAGE_SIZE) {
-				ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
-						virt, 0, 0, &pgsize, 0);
-				if (ptep && pte_is_present(ptep)) {
-					goto cq_map_unlock;
-				}
-
-				if (ihk_mc_pt_virt_to_phys(ihk_mc_get_linux_kernel_pgt(),
-							virt, &phys) < 0) {
-					/* TODO: shall we make this function fail? */
-					kprintf("%s: ERROR: mapping cq, Linux mapping doesn't exist\n",
-							__FUNCTION__);
-					goto cq_map_unlock;
-				}
-
-				if (ihk_mc_pt_set_page(vm->address_space->page_table,
-							virt, phys, attr) < 0) {
-					/* TODO: shall we make this function fail? */
-					kprintf("%s: ERROR: mapping cq: 0x%lx -> 0x%lx\n",
-							__FUNCTION__, virt, phys);
-					goto cq_map_unlock;
-				}
-
-				kprintf("%s: cq: 0x%lx -> 0x%lx mapped\n",
-						__FUNCTION__,
-						virt, phys);
-			}
-		cq_map_unlock:
-			ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock);
-
-			/* TODO: unmap these at close() time */
-		}
-	}
-
-#endif // __HFI1_ORIG__
-
-
 	TP("- kregbase and cq->comps");
 	hfi1_cdbg(AIOWRITE, "+");
 	if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) {