gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

xpmem: Support large page attachment

Browse Source 
Change-Id: I4d672eee1c905160ece204d278f0afd9b6d7dc01
Refs: #1259
This commit is contained in:
Masamichi Takagi
2019-04-04 02:29:34 +00:00
committed by Ken Sato
parent 569dc33a9c
commit a8696d811d
35 changed files with 2394 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
kernel/syscall.c
View File

@@ -1549,6 +1549,12 @@ static int search_free_space(size_t len, int pgshift, uintptr_t *addrp)
/* try given addr first */
addr = *addrp;
if (addr != 0) {
if ((region->user_end <= addr)
|| ((region->user_end - len) < addr)) {
error = -ENOMEM;
goto out;
}
range = lookup_process_memory_range(thread->vm, addr, addr+len);
if (range == NULL)
goto out;
@@ -1584,7 +1590,8 @@ out:
intptr_t
do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
const int flags, const int fd, const off_t off0)
const int flags, const int fd, const off_t off0,
const int vrf0, void *private_data)
{
struct thread *thread = cpu_local_var(current);
struct vm_regions *region = &thread->vm->region;
@@ -1643,7 +1650,8 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
pgshift = (flags >> MAP_HUGE_SHIFT) & 0x3F;
p2align = pgshift - PAGE_SHIFT;
}
else if ((flags & MAP_PRIVATE) && (flags & MAP_ANONYMOUS)
else if ((((flags & MAP_PRIVATE) && (flags & MAP_ANONYMOUS))
|| (vrf0 & VR_XPMEM))
&& !proc->thp_disable) {
pgshift = 0; /* transparent huge page */
p2align = PAGE_P2ALIGN;
@@ -1674,7 +1682,29 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
}
else if (flags & MAP_ANONYMOUS) {
/* Obtain mapping address */
error = search_free_space(len, PAGE_SHIFT + p2align, &addr);
if (vrf0 && VR_XPMEM) {
/* Fit address format to segment area */
struct xpmem_attachment *att;
uintptr_t prev_addr;
att = (struct xpmem_attachment *)private_data;
addr = att->vaddr;
while (!error) {
prev_addr = addr;
error = search_free_space(len,
PAGE_SHIFT + p2align, &addr);
if (prev_addr == addr) {
break;
}
addr = prev_addr +
(1UL << (PAGE_SHIFT + p2align));
}
}
else {
error = search_free_space(len,
PAGE_SHIFT + p2align, &addr);
}
if (error) {
ekprintf("do_mmap:search_free_space(%lx,%lx,%d) failed. %d\n",
len, region->map_end, p2align, error);
@@ -1684,12 +1714,13 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
/* do the map */
vrflags = VR_NONE;
vrflags |= vrf0;
vrflags |= PROT_TO_VR_FLAG(prot);
vrflags |= (flags & MAP_PRIVATE)? VR_PRIVATE: 0;
vrflags |= (flags & MAP_LOCKED)? VR_LOCKED: 0;
vrflags |= VR_DEMAND_PAGING;
if (flags & MAP_ANONYMOUS) {
if (!anon_on_demand && (flags & MAP_PRIVATE)) {
if (flags & MAP_ANONYMOUS && !anon_on_demand) {
if (flags & MAP_PRIVATE || vrflags & VR_XPMEM) {
vrflags &= ~VR_DEMAND_PAGING;
}
}
@@ -1812,6 +1843,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
}
/* Prepopulated ANONYMOUS mapping */
else if (!(vrflags & VR_DEMAND_PAGING)
&& !(flags & MAP_SHARED)
&& ((vrflags & VR_PROT_MASK) != VR_PROT_NONE)) {
npages = len >> PAGE_SHIFT;
/* Small allocations mostly benefit from closest RAM,
@@ -1890,7 +1922,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
vrflags |= VRFLAG_PROT_TO_MAXPROT(PROT_TO_VR_FLAG(maxprot));
error = add_process_memory_range(thread->vm, addr, addr+len, phys,
vrflags, memobj, off, pgshift, &range);
vrflags, memobj, off, pgshift, private_data, &range);
if (error) {
kprintf("%s: add_process_memory_range failed for 0x%lx:%lu"
" flags: %lx, vrflags: %lx, pgshift: %d, error: %d\n",
@@ -4205,7 +4237,7 @@ perf_mmap(struct mckfd *sfd, ihk_mc_user_context_t *ctx)
flags |= MAP_ANONYMOUS;
prot |= PROT_WRITE;
rc = do_mmap(addr0, len0, prot, flags, fd, off0);
rc = do_mmap(addr0, len0, prot, flags, fd, off0, 0, NULL);
// setup perf_event_mmap_page
page = (struct perf_event_mmap_page *)rc;
@@ -5488,7 +5520,7 @@ SYSCALL_DECLARE(shmat)
}
error = add_process_memory_range(vm, addr, addr+len, -1,
vrflags, &obj->memobj, 0, obj->pgshift, NULL);
vrflags, &obj->memobj, 0, obj->pgshift, NULL, NULL);
if (error) {
if (!(prot & PROT_WRITE)) {
(void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */);
@@ -8252,7 +8284,7 @@ SYSCALL_DECLARE(mremap)
error = add_process_memory_range(thread->vm, newstart, newend, -1,
range->flag, range->memobj,
range->objoff + (oldstart - range->start),
range->pgshift, NULL);
range->pgshift, NULL, NULL);
if (error) {
ekprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx):"
"add failed. %d\n",