Merge branch 'master' of postpeta.pccluster.org:mckernel

kernel/Makefile.build

@@ -1,6 +1,7 @@
 IHKDIR=$(IHKBASE)/$(TARGETDIR)
 OBJS = init.o mem.o debug.o mikc.o listeners.o ap.o syscall.o cls.o host.o
-OBJS += process.o copy.o waitq.o futex.o timer.o plist.o fileobj.o
+OBJS += process.o copy.o waitq.o futex.o timer.o plist.o fileobj.o shmobj.o
+OBJS += zeroobj.o
 DEPSRCS=$(wildcard $(SRC)/*.c)
 
 CFLAGS += -I$(SRC)/include -mcmodel=kernel -D__KERNEL__

kernel/fileobj.c

@@ -184,6 +184,7 @@ int fileobj_create(int fd, struct memobj **objp, int *maxprotp)
 
 	memset(newobj, 0, sizeof(*newobj));
 	newobj->memobj.ops = &fileobj_ops;
+	newobj->memobj.flags = MF_HAS_PAGER;
 	newobj->handle = result.handle;
 	newobj->sref = 1;
 	newobj->cref = 1;

kernel/include/memobj.h

@@ -16,10 +16,19 @@
 #include <ihk/types.h>
 #include <ihk/atomic.h>
 #include <ihk/lock.h>
+#include <errno.h>
 #include <list.h>
+#include <shm.h>
+
+enum {
+	/* for memobj.flags */
+	MF_HAS_PAGER	= 0x0001,
+};
 
 struct memobj {
 	struct memobj_ops *	ops;
+	uint32_t		flags;
+	int8_t			padding[4];
 	ihk_spinlock_t		lock;
 };
 
@@ -39,29 +48,42 @@ struct memobj_ops {
 
 static inline void memobj_release(struct memobj *obj)
 {
-	(*obj->ops->release)(obj);
+	if (obj->ops->release) {
+		(*obj->ops->release)(obj);
+	}
 }
 
 static inline void memobj_ref(struct memobj *obj)
 {
-	(*obj->ops->ref)(obj);
+	if (obj->ops->ref) {
+		(*obj->ops->ref)(obj);
+	}
 }
 
 static inline int memobj_get_page(struct memobj *obj, off_t off,
 		int p2align, uintptr_t *physp)
 {
-	return (*obj->ops->get_page)(obj, off, p2align, physp);
+	if (obj->ops->get_page) {
+		return (*obj->ops->get_page)(obj, off, p2align, physp);
+	}
+	return -ENXIO;
 }
 
 static inline uintptr_t memobj_copy_page(struct memobj *obj,
 		uintptr_t orgphys, int p2align)
 {
-	return (*obj->ops->copy_page)(obj, orgphys, p2align);
+	if (obj->ops->copy_page) {
+		return (*obj->ops->copy_page)(obj, orgphys, p2align);
+	}
+	return -ENXIO;
 }
 
 static inline int memobj_flush_page(struct memobj *obj, uintptr_t phys, size_t pgsize)
 {
-	return (*obj->ops->flush_page)(obj, phys, pgsize);
+	if (obj->ops->flush_page) {
+		return (*obj->ops->flush_page)(obj, phys, pgsize);
+	}
+	return 0;
 }
 
 static inline void memobj_lock(struct memobj *obj)
@@ -74,6 +96,13 @@ static inline void memobj_unlock(struct memobj *obj)
 	ihk_mc_spinlock_unlock_noirq(&obj->lock);
 }
 
+static inline int memobj_has_pager(struct memobj *obj)
+{
+	return !!(obj->flags & MF_HAS_PAGER);
+}
+
 int fileobj_create(int fd, struct memobj **objp, int *maxprotp);
+int shmobj_create(struct shmid_ds *ds, struct memobj **objp);
+int zeroobj_create(struct memobj **objp);
 
 #endif /* HEADER_MEMOBJ_H */

kernel/include/shm.h

@@ -0,0 +1,49 @@
+/**
+ * \file shm.h
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  header file for System V shared memory
+ * \author Gou Nakamura  <go.nakamura.yw@hitachi-solutions.com>
+ */
+/*
+ * HISTORY:
+ */
+
+#ifndef HEADER_SHM_H
+#define HEADER_SHM_H
+
+/* begin types.h */
+typedef int32_t key_t;
+typedef uint32_t uid_t;
+typedef uint32_t gid_t;
+typedef int64_t time_t;
+typedef int32_t pid_t;
+/* end types.h */
+
+typedef uint64_t shmatt_t;
+
+struct ipc_perm {
+	key_t		key;
+	uid_t		uid;
+	gid_t		gid;
+	uid_t		cuid;
+	gid_t		cgid;
+	uint16_t	mode;
+	uint8_t		padding[2];
+	uint16_t	seq;
+	uint8_t		padding2[22];
+};
+
+struct shmid_ds {
+	struct ipc_perm	shm_perm;
+	size_t		shm_segsz;
+	time_t		shm_atime;
+	time_t		shm_dtime;
+	time_t		shm_ctime;
+	pid_t		shm_cpid;
+	pid_t		shm_lpid;
+	shmatt_t	shm_nattch;
+	uint8_t		padding[16];
+};
+
+#endif /* HEADER_SHM_H */

kernel/shmobj.c

@@ -0,0 +1,287 @@
+/**
+ * \file shmobj.c
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  shared memory object
+ * \author Gou Nakamura  <go.nakamura.yw@hitachi-solutions.com>
+ */
+/*
+ * HISTORY:
+ */
+
+#include <ihk/atomic.h>
+#include <ihk/debug.h>
+#include <ihk/lock.h>
+#include <ihk/mm.h>
+#include <errno.h>
+#include <kmalloc.h>
+#include <list.h>
+#include <memobj.h>
+#include <memory.h>
+#include <page.h>
+#include <shm.h>
+#include <string.h>
+
+#define	dkprintf(...)	do { if (0) kprintf(__VA_ARGS__); } while (0)
+#define	ekprintf(...)	kprintf(__VA_ARGS__)
+#define	fkprintf(...)	kprintf(__VA_ARGS__)
+
+struct shmobj {
+	struct memobj		memobj;		/* must be first */
+	long			ref;
+	struct shmid_ds		ds;
+	struct list_head	page_list;
+};
+
+static memobj_release_func_t shmobj_release;
+static memobj_ref_func_t shmobj_ref;
+static memobj_get_page_func_t shmobj_get_page;
+
+static struct memobj_ops shmobj_ops = {
+	.release =	&shmobj_release,
+	.ref =		&shmobj_ref,
+	.get_page =	&shmobj_get_page,
+};
+
+static struct shmobj *to_shmobj(struct memobj *memobj)
+{
+	return (struct shmobj *)memobj;
+}
+
+static struct memobj *to_memobj(struct shmobj *shmobj)
+{
+	return &shmobj->memobj;
+}
+
+/***********************************************************************
+ * page_list
+ */
+static void page_list_init(struct shmobj *obj)
+{
+	INIT_LIST_HEAD(&obj->page_list);
+	return;
+}
+
+static void page_list_insert(struct shmobj *obj, struct page *page)
+{
+	list_add(&page->list, &obj->page_list);
+	return;
+}
+
+static void page_list_remove(struct shmobj *obj, struct page *page)
+{
+	list_del(&page->list);
+	return;
+}
+
+static struct page *page_list_lookup(struct shmobj *obj, off_t off)
+{
+	struct page *page;
+
+	list_for_each_entry(page, &obj->page_list, list) {
+		if (page->offset == off) {
+			goto out;
+		}
+	}
+	page = NULL;
+
+out:
+	return page;
+}
+
+static struct page *page_list_first(struct shmobj *obj)
+{
+	if (list_empty(&obj->page_list)) {
+		return NULL;
+	}
+
+	return list_first_entry(&obj->page_list, struct page, list);
+}
+
+int shmobj_create(struct shmid_ds *ds, struct memobj **objp)
+{
+	struct shmobj *obj = NULL;
+	int error;
+
+	dkprintf("shmobj_create(%p %#lx,%p)\n", ds, ds->shm_segsz, objp);
+	obj = kmalloc(sizeof(*obj), IHK_MC_AP_NOWAIT);
+	if (!obj) {
+		error = -ENOMEM;
+		ekprintf("shmobj_create(%p %#lx,%p):kmalloc failed. %d\n",
+				ds, ds->shm_segsz, objp, error);
+		goto out;
+	}
+
+	memset(obj, 0, sizeof(*obj));
+	obj->memobj.ops = &shmobj_ops;
+	obj->ref = 1;
+	obj->ds = *ds;
+	page_list_init(obj);
+	ihk_mc_spinlock_init(&obj->memobj.lock);
+
+	error = 0;
+	*objp = to_memobj(obj);
+	obj = NULL;
+
+out:
+	if (obj) {
+		kfree(obj);
+	}
+	dkprintf("shmobj_create(%p %#lx,%p):%d %p\n",
+			ds, ds->shm_segsz, objp, error, *objp);
+	return error;
+}
+
+static void shmobj_release(struct memobj *memobj)
+{
+	struct shmobj *obj = to_shmobj(memobj);
+	struct shmobj *freeobj = NULL;
+
+	dkprintf("shmobj_release(%p)\n", memobj);
+	memobj_lock(&obj->memobj);
+	--obj->ref;
+	if (obj->ref <= 0) {
+		if (obj->ref < 0) {
+			fkprintf("shmobj_release(%p):ref %ld\n",
+					memobj, obj->ref);
+			panic("shmobj_release:freeing free shmobj");
+		}
+		freeobj = obj;
+	}
+	memobj_unlock(&obj->memobj);
+
+	if (freeobj) {
+		/* zap page_list */
+		for (;;) {
+			struct page *page;
+			int count;
+
+			page = page_list_first(obj);
+			if (!page) {
+				break;
+			}
+			page_list_remove(obj, page);
+
+			dkprintf("shmobj_release(%p):"
+					"release page. %p %#lx %d %d",
+					memobj, page, page_to_phys(page),
+					page->mode, page->count);
+			count = ihk_atomic_sub_return(1, &page->count);
+			if (!((page->mode == PM_MAPPED) && (count == 0))) {
+				fkprintf("shmobj_release(%p): "
+						"page %p phys %#lx mode %#x"
+						" count %d off %#lx\n",
+						memobj, page,
+						page_to_phys(page),
+						page->mode, count,
+						page->offset);
+				panic("shmobj_release");
+			}
+
+			/* XXX:NYI: large pages */
+			page->mode = PM_NONE;
+			free_pages(phys_to_virt(page_to_phys(page)), 1);
+		}
+		dkprintf("shmobj_release(%p):free shmobj", memobj);
+		kfree(freeobj);
+	}
+	dkprintf("shmobj_release(%p):\n", memobj);
+	return;
+}
+
+static void shmobj_ref(struct memobj *memobj)
+{
+	struct shmobj *obj = to_shmobj(memobj);
+	long newref;
+
+	dkprintf("shmobj_ref(%p)\n", memobj);
+	memobj_lock(&obj->memobj);
+	newref = ++obj->ref;
+	memobj_unlock(&obj->memobj);
+	dkprintf("shmobj_ref(%p): newref %ld\n", memobj, newref);
+	return;
+}
+
+static int shmobj_get_page(struct memobj *memobj, off_t off, int p2align,
+		uintptr_t *physp)
+{
+	struct shmobj *obj = to_shmobj(memobj);
+	int error;
+	struct page *page;
+	int npages;
+	void *virt = NULL;
+	uintptr_t phys = -1;
+
+	dkprintf("shmobj_get_page(%p,%#lx,%d,%p)\n",
+			memobj, off, p2align, physp);
+	memobj_lock(&obj->memobj);
+	if (off & ~PAGE_MASK) {
+		error = -EINVAL;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):invalid argument. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+	if (p2align != PAGE_P2ALIGN) {		/* XXX:NYI:large pages */
+		error = -ENOMEM;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):large page. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+	if (obj->ds.shm_segsz <= off) {
+		error = -ERANGE;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):beyond the end. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+	if ((obj->ds.shm_segsz - off) < (PAGE_SIZE << p2align)) {
+		error = -ENOSPC;
+		ekprintf("shmobj_get_page(%p,%#lx,%d,%p):too large. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+
+	page = page_list_lookup(obj, off);
+	if (!page) {
+		npages = 1 << p2align;
+		virt = ihk_mc_alloc_pages(npages, IHK_MC_AP_NOWAIT);
+		if (!virt) {
+			error = -ENOMEM;
+			ekprintf("shmobj_get_page(%p,%#lx,%d,%p):"
+					"alloc failed. %d\n",
+					memobj, off, p2align, physp, error);
+			goto out;
+		}
+		phys = virt_to_phys(virt);
+		page = phys_to_page(phys);
+		if (page->mode != PM_NONE) {
+			fkprintf("shmobj_get_page(%p,%#lx,%d,%p):"
+					"page %p %#lx %d %d %#lx\n",
+					memobj, off, p2align, physp,
+					page, page_to_phys(page), page->mode,
+					page->count, page->offset);
+			panic("shmobj_get_page()");
+		}
+		memset(virt, 0, npages*PAGE_SIZE);
+		page->mode = PM_MAPPED;
+		page->offset = off;
+		ihk_atomic_set(&page->count, 1);
+		page_list_insert(obj, page);
+		virt = NULL;
+		dkprintf("shmobj_get_page(%p,%#lx,%d,%p):alloc page. %p %#lx\n",
+				memobj, off, p2align, physp, page, phys);
+	}
+
+	ihk_atomic_inc(&page->count);
+
+	error = 0;
+	*physp = page_to_phys(page);
+
+out:
+	memobj_unlock(&obj->memobj);
+	if (virt) {
+		ihk_mc_free_pages(virt, npages);
+	}
+	dkprintf("shmobj_get_page(%p,%#lx,%d,%p):%d\n",
+			memobj, off, p2align, physp, error);
+	return error;
+}

kernel/syscall.c

@@ -45,6 +45,7 @@
 #include <mman.h>
 #include <kmalloc.h>
 #include <memobj.h>
+#include <shm.h>
 
 /* Headers taken from kitten LWK */
 #include <lwk/stddef.h>
@@ -215,7 +216,7 @@ long do_syscall(struct syscall_request *req, ihk_mc_user_context_t *ctx,
 					cpu_local_var(current)->pid);		
 			error = page_fault_process(get_cpu_local_var(cpu)->current,
 					(void *)res->fault_address,
-					res->fault_reason);
+					res->fault_reason|PF_POPULATE);
 
 			/* send result */
 			req2.number = __NR_mmap;
@@ -641,12 +642,13 @@ SYSCALL_DECLARE(mmap)
 	const int prot = ihk_mc_syscall_arg2(ctx);
 	const int flags = ihk_mc_syscall_arg3(ctx);
 	const int fd = ihk_mc_syscall_arg4(ctx);
-	const off_t off = ihk_mc_syscall_arg5(ctx);
+	const off_t off0 = ihk_mc_syscall_arg5(ctx);
 
 	struct process *proc = cpu_local_var(current);
 	struct vm_regions *region = &proc->vm->region;
 	intptr_t addr;
 	size_t len;
+	off_t off;
 	int error;
 	intptr_t npages;
 	int p2align;
@@ -657,10 +659,11 @@ SYSCALL_DECLARE(mmap)
 	int maxprot;
 	int denied;
 	int ro_vma_mapped = 0;
+	struct shmid_ds ads;
 
 	dkprintf("[%d]sys_mmap(%lx,%lx,%x,%x,%d,%lx)\n",
 			ihk_mc_get_processor_id(),
-			addr0, len0, prot, flags, fd, off);
+			addr0, len0, prot, flags, fd, off0);
 
 	/* check constants for flags */
 	if (1) {
@@ -692,9 +695,9 @@ SYSCALL_DECLARE(mmap)
 			|| ((region->user_end - len) < addr)
 			|| !(flags & (MAP_SHARED | MAP_PRIVATE))
 			|| ((flags & MAP_SHARED) && (flags & MAP_PRIVATE))
-			|| (off & (PAGE_SIZE - 1))) {
+			|| (off0 & (PAGE_SIZE - 1))) {
 		ekprintf("sys_mmap(%lx,%lx,%x,%x,%x,%lx):EINVAL\n",
-				addr0, len0, prot, flags, fd, off);
+				addr0, len0, prot, flags, fd, off0);
 		error = -EINVAL;
 		goto out2;
 	}
@@ -703,7 +706,7 @@ SYSCALL_DECLARE(mmap)
 	if ((flags & error_flags)
 			|| (flags & ~(supported_flags | ignored_flags))) {
 		ekprintf("sys_mmap(%lx,%lx,%x,%x,%x,%lx):unknown flags %x\n",
-				addr0, len0, prot, flags, fd, off,
+				addr0, len0, prot, flags, fd, off0,
 				(flags & ~(supported_flags | ignored_flags)));
 		error = -EINVAL;
 		goto out2;
@@ -765,8 +768,10 @@ SYSCALL_DECLARE(mmap)
 	}
 
 	phys = 0;
+	off = 0;
 	maxprot = PROT_READ | PROT_WRITE | PROT_EXEC;
 	if (!(flags & MAP_ANONYMOUS)) {
+		off = off0;
 		error = fileobj_create(fd, &memobj, &maxprot);
 		if (error) {
 			ekprintf("sys_mmap:fileobj_create failed. %d\n", error);
@@ -792,6 +797,22 @@ SYSCALL_DECLARE(mmap)
 		}
 		phys = virt_to_phys(p);
 	}
+	else if (flags & MAP_SHARED) {
+		memset(&ads, 0, sizeof(ads));
+		ads.shm_segsz = len;
+		error = shmobj_create(&ads, &memobj);
+		if (error) {
+			ekprintf("sys_mmap:shmobj_create failed. %d\n", error);
+			goto out;
+		}
+	}
+	else {
+		error = zeroobj_create(&memobj);
+		if (error) {
+			ekprintf("sys_mmap:zeroobj_create failed. %d\n", error);
+			goto out;
+		}
+	}
 
 	if ((flags & MAP_PRIVATE) && (maxprot & PROT_READ)) {
 		maxprot |= PROT_WRITE;
@@ -855,7 +876,7 @@ out2:
 	}
 	dkprintf("[%d]sys_mmap(%lx,%lx,%x,%x,%d,%lx): %ld %lx\n",
 			ihk_mc_get_processor_id(),
-			addr0, len0, prot, flags, fd, off, error, addr);
+			addr0, len0, prot, flags, fd, off0, error, addr);
 	return (!error)? addr: error;
 }
 
@@ -1713,15 +1734,14 @@ SYSCALL_DECLARE(madvise)
 			dkprintf("[%d]sys_madvise(%lx,%lx,%x):not contig "
 					"%lx [%lx-%lx)\n",
 					ihk_mc_get_processor_id(), start,
-					len0, advice, addr, range->start,
-					range->end);
+					len0, advice, addr, range?range->start:0,
+					range?range->end:0);
 			error = -ENOMEM;
 			goto out;
 		}
 
-#define	MEMOBJ_IS_FILEOBJ(obj)	((obj) != NULL)
-		if (!MEMOBJ_IS_FILEOBJ(range->memobj)) {
-			dkprintf("[%d]sys_madvise(%lx,%lx,%x):not fileobj "
+		if (!range->memobj || !memobj_has_pager(range->memobj)) {
+			dkprintf("[%d]sys_madvise(%lx,%lx,%x):has not pager"
 					"[%lx-%lx) %lx\n",
 					ihk_mc_get_processor_id(), start,
 					len0, advice, range->start,
@@ -2046,7 +2066,8 @@ SYSCALL_DECLARE(mlock)
 			dkprintf("[%d]sys_mlock(%lx,%lx):not contiguous."
 				       " %lx [%lx-%lx)\n",
 					ihk_mc_get_processor_id(), start0,
-					len0, addr, range->start, range->end);
+					len0, addr, range?range->start:0,
+					range?range->end:0);
 			error = -ENOMEM;
 			goto out;
 		}
@@ -2220,7 +2241,8 @@ SYSCALL_DECLARE(munlock)
 			dkprintf("[%d]sys_munlock(%lx,%lx):not contiguous."
 				       " %lx [%lx-%lx)\n",
 					ihk_mc_get_processor_id(), start0,
-					len0, addr, range->start, range->end);
+					len0, addr, range?range->start:0,
+					range?range->end:0);
 			error = -ENOMEM;
 			goto out;
 		}

kernel/zeroobj.c

@@ -0,0 +1,206 @@
+/**
+ * \file zeroobj.c
+ *  License details are found in the file LICENSE.
+ * \brief
+ *  read-only zeroed page object
+ * \author Gou Nakamura  <go.nakamura.yw@hitachi-solutions.com>
+ */
+/*
+ * HISTORY:
+ */
+
+#include <ihk/atomic.h>
+#include <ihk/debug.h>
+#include <ihk/lock.h>
+#include <ihk/mm.h>
+#include <errno.h>
+#include <kmalloc.h>
+#include <list.h>
+#include <memobj.h>
+#include <memory.h>
+#include <page.h>
+#include <string.h>
+
+#define	dkprintf(...)	do { if (0) kprintf(__VA_ARGS__); } while (0)
+#define	ekprintf(...)	kprintf(__VA_ARGS__)
+#define	fkprintf(...)	kprintf(__VA_ARGS__)
+
+struct zeroobj {
+	struct memobj		memobj;		/* must be first */
+	struct list_head	page_list;
+};
+
+static ihk_spinlock_t the_zeroobj_lock = SPIN_LOCK_UNLOCKED;
+static struct zeroobj *the_zeroobj = NULL;	/* singleton */
+
+static memobj_get_page_func_t zeroobj_get_page;
+
+static struct memobj_ops zeroobj_ops = {
+	.get_page =	&zeroobj_get_page,
+};
+
+static struct zeroobj *to_zeroobj(struct memobj *memobj)
+{
+	return (struct zeroobj *)memobj;
+}
+
+static struct memobj *to_memobj(struct zeroobj *zeroobj)
+{
+	return &zeroobj->memobj;
+}
+
+/***********************************************************************
+ * page_list
+ */
+static void page_list_init(struct zeroobj *obj)
+{
+	INIT_LIST_HEAD(&obj->page_list);
+	return;
+}
+
+static void page_list_insert(struct zeroobj *obj, struct page *page)
+{
+	list_add(&page->list, &obj->page_list);
+	return;
+}
+
+static struct page *page_list_first(struct zeroobj *obj)
+{
+	if (list_empty(&obj->page_list)) {
+		return NULL;
+	}
+
+	return list_first_entry(&obj->page_list, struct page, list);
+}
+
+/***********************************************************************
+ * zeroobj
+ */
+static int alloc_zeroobj(void)
+{
+	int error;
+	struct zeroobj *obj = NULL;
+	void *virt = NULL;
+	uintptr_t phys;
+	struct page *page;
+
+	dkprintf("alloc_zeroobj()\n");
+	ihk_mc_spinlock_lock_noirq(&the_zeroobj_lock);
+	if (the_zeroobj) {
+		error = 0;
+		dkprintf("alloc_zeroobj():already. %d\n", error);
+		goto out;
+	}
+
+	obj = kmalloc(sizeof(*obj), IHK_MC_AP_NOWAIT);
+	if (!obj) {
+		error = -ENOMEM;
+		ekprintf("alloc_zeroobj():kmalloc failed. %d\n", error);
+		goto out;
+	}
+
+	memset(obj, 0, sizeof(*obj));
+	obj->memobj.ops = &zeroobj_ops;
+	page_list_init(obj);
+	ihk_mc_spinlock_init(&obj->memobj.lock);
+
+	virt = ihk_mc_alloc_pages(1, IHK_MC_AP_NOWAIT);	/* XXX:NYI:large page */
+	if (!virt) {
+		error = -ENOMEM;
+		ekprintf("alloc_zeroobj():alloc pages failed. %d\n", error);
+		goto out;
+	}
+	phys = virt_to_phys(virt);
+	page = phys_to_page(phys);
+
+	if (page->mode != PM_NONE) {
+		fkprintf("alloc_zeroobj():"
+				"page %p %#lx %d %d %#lx\n",
+				page, page_to_phys(page), page->mode,
+				page->count, page->offset);
+		panic("alloc_zeroobj:dup alloc");
+	}
+
+	memset(virt, 0, PAGE_SIZE);
+	page->mode = PM_MAPPED;
+	page->offset = 0;
+	ihk_atomic_set(&page->count, 1);
+	page_list_insert(obj, page);
+	virt = NULL;
+
+	error = 0;
+	the_zeroobj = obj;
+	obj = NULL;
+
+out:
+	ihk_mc_spinlock_unlock_noirq(&the_zeroobj_lock);
+	if (virt) {
+		ihk_mc_free_pages(virt, 1);
+	}
+	if (obj) {
+		kfree(obj);
+	}
+	dkprintf("alloc_zeroobj():%d %p\n", error, the_zeroobj);
+	return error;
+}
+
+int zeroobj_create(struct memobj **objp)
+{
+	int error;
+
+	dkprintf("zeroobj_create(%p)\n", objp);
+	if (!the_zeroobj) {
+		error = alloc_zeroobj();
+		if (error) {
+			goto out;
+		}
+	}
+
+	error = 0;
+	*objp = to_memobj(the_zeroobj);
+
+out:
+	dkprintf("zeroobj_create(%p):%d %p\n", objp, error, *objp);
+	return error;
+}
+
+static int zeroobj_get_page(struct memobj *memobj, off_t off, int p2align,
+		uintptr_t *physp)
+{
+	int error;
+	struct zeroobj *obj = to_zeroobj(memobj);
+	struct page *page;
+
+	dkprintf("zeroobj_get_page(%p,%#lx,%d,%p)\n",
+			memobj, off, p2align, physp);
+	if (off & ~PAGE_MASK) {
+		error = -EINVAL;
+		ekprintf("zeroobj_get_page(%p,%#lx,%d,%p):invalid argument. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+	if (p2align != PAGE_P2ALIGN) {		/* XXX:NYI:large pages */
+		error = -ENOMEM;
+		ekprintf("zeroobj_get_page(%p,%#lx,%d,%p):large page. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+
+	page = page_list_first(obj);
+	if (!page) {
+		error = -ENOMEM;
+		ekprintf("zeroobj_get_page(%p,%#lx,%d,%p):page not found. %d\n",
+				memobj, off, p2align, physp, error);
+		goto out;
+	}
+
+	ihk_atomic_inc(&page->count);
+
+	error = 0;
+	*physp = page_to_phys(page);
+
+out:
+	dkprintf("zeroobj_get_page(%p,%#lx,%d,%p):%d\n",
+			memobj, off, p2align, physp, error);
+	return error;
+}