diff --git a/arch/arm64/kernel/include/arch-memory.h b/arch/arm64/kernel/include/arch-memory.h index 2050c198..47bf7ffa 100644 --- a/arch/arm64/kernel/include/arch-memory.h +++ b/arch/arm64/kernel/include/arch-memory.h @@ -3,6 +3,13 @@ #define __HEADER_ARM64_COMMON_ARCH_MEMORY_H #include +#include + +#ifndef __ASSEMBLY__ +#include +#include +void panic(const char *); +#endif /*__ASSEMBLY__*/ #define _SZ4KB (1UL<<12) #define _SZ16KB (1UL<<14) @@ -105,6 +112,10 @@ # define PTL2_INDEX_MASK PTL3_INDEX_MASK # define PTL1_INDEX_MASK PTL2_INDEX_MASK # define FIRST_LEVEL_BLOCK_SUPPORT 1 +# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0) +# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 4) +# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 4) +# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 4) #elif GRANULE_SIZE == _SZ16KB # define __PTL4_SHIFT 47 # define __PTL3_SHIFT 36 @@ -115,6 +126,10 @@ # define PTL2_INDEX_MASK PTL3_INDEX_MASK # define PTL1_INDEX_MASK PTL2_INDEX_MASK # define FIRST_LEVEL_BLOCK_SUPPORT 0 +# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0) +# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0) +# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5) +# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 7) #elif GRANULE_SIZE == _SZ64KB # define __PTL4_SHIFT 0 # define __PTL3_SHIFT 42 @@ -125,6 +140,10 @@ # define PTL2_INDEX_MASK ((UL(1) << 13) - 1) # define PTL1_INDEX_MASK PTL2_INDEX_MASK # define FIRST_LEVEL_BLOCK_SUPPORT 0 +# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0) +# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0) +# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5) +# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 5) #else # error granule size error. #endif @@ -133,10 +152,23 @@ # define __PTL3_SIZE (UL(1) << __PTL3_SHIFT) # define __PTL2_SIZE (UL(1) << __PTL2_SHIFT) # define __PTL1_SIZE (UL(1) << __PTL1_SHIFT) -# define __PTL4_MASK (~__PTL4_SIZE - 1) -# define __PTL3_MASK (~__PTL3_SIZE - 1) -# define __PTL2_MASK (~__PTL2_SIZE - 1) -# define __PTL1_MASK (~__PTL1_SIZE - 1) +# define __PTL4_MASK (~(__PTL4_SIZE - 1)) +# define __PTL3_MASK (~(__PTL3_SIZE - 1)) +# define __PTL2_MASK (~(__PTL2_SIZE - 1)) +# define __PTL1_MASK (~(__PTL1_SIZE - 1)) + +# define __PTL4_CONT_SIZE (UL(1) << __PTL4_CONT_SHIFT) +# define __PTL3_CONT_SIZE (UL(1) << __PTL3_CONT_SHIFT) +# define __PTL2_CONT_SIZE (UL(1) << __PTL2_CONT_SHIFT) +# define __PTL1_CONT_SIZE (UL(1) << __PTL1_CONT_SHIFT) +# define __PTL4_CONT_MASK (~(__PTL4_CONT_SIZE - 1)) +# define __PTL3_CONT_MASK (~(__PTL3_CONT_SIZE - 1)) +# define __PTL2_CONT_MASK (~(__PTL2_CONT_SIZE - 1)) +# define __PTL1_CONT_MASK (~(__PTL1_CONT_SIZE - 1)) +# define __PTL4_CONT_COUNT (UL(1) << (__PTL4_CONT_SHIFT - __PTL4_SHIFT)) +# define __PTL3_CONT_COUNT (UL(1) << (__PTL3_CONT_SHIFT - __PTL3_SHIFT)) +# define __PTL2_CONT_COUNT (UL(1) << (__PTL2_CONT_SHIFT - __PTL2_SHIFT)) +# define __PTL1_CONT_COUNT (UL(1) << (__PTL1_CONT_SHIFT - __PTL1_SHIFT)) /* calculate entries */ #if (CONFIG_ARM64_PGTABLE_LEVELS > 3) && (VA_BITS > __PTL4_SHIFT) @@ -183,6 +215,22 @@ static const unsigned int PTL4_ENTRIES = __PTL4_ENTRIES; static const unsigned int PTL3_ENTRIES = __PTL3_ENTRIES; static const unsigned int PTL2_ENTRIES = __PTL2_ENTRIES; static const unsigned int PTL1_ENTRIES = __PTL1_ENTRIES; +static const unsigned int PTL4_CONT_SHIFT = __PTL4_CONT_SHIFT; +static const unsigned int PTL3_CONT_SHIFT = __PTL3_CONT_SHIFT; +static const unsigned int PTL2_CONT_SHIFT = __PTL2_CONT_SHIFT; +static const unsigned int PTL1_CONT_SHIFT = __PTL1_CONT_SHIFT; +static const unsigned long PTL4_CONT_SIZE = __PTL4_CONT_SIZE; +static const unsigned long PTL3_CONT_SIZE = __PTL3_CONT_SIZE; +static const unsigned long PTL2_CONT_SIZE = __PTL2_CONT_SIZE; +static const unsigned long PTL1_CONT_SIZE = __PTL1_CONT_SIZE; +static const unsigned long PTL4_CONT_MASK = __PTL4_CONT_MASK; +static const unsigned long PTL3_CONT_MASK = __PTL3_CONT_MASK; +static const unsigned long PTL2_CONT_MASK = __PTL2_CONT_MASK; +static const unsigned long PTL1_CONT_MASK = __PTL1_CONT_MASK; +static const unsigned int PTL4_CONT_COUNT = __PTL4_CONT_COUNT; +static const unsigned int PTL3_CONT_COUNT = __PTL3_CONT_COUNT; +static const unsigned int PTL2_CONT_COUNT = __PTL2_CONT_COUNT; +static const unsigned int PTL1_CONT_COUNT = __PTL1_CONT_COUNT; #else # define PTL4_SHIFT __PTL4_SHIFT # define PTL3_SHIFT __PTL3_SHIFT @@ -200,8 +248,26 @@ static const unsigned int PTL1_ENTRIES = __PTL1_ENTRIES; # define PTL3_ENTRIES __PTL3_ENTRIES # define PTL2_ENTRIES __PTL2_ENTRIES # define PTL1_ENTRIES __PTL1_ENTRIES +# define PTL4_CONT_SHIFT __PTL4_CONT_SHIFT +# define PTL3_CONT_SHIFT __PTL3_CONT_SHIFT +# define PTL2_CONT_SHIFT __PTL2_CONT_SHIFT +# define PTL1_CONT_SHIFT __PTL1_CONT_SHIFT +# define PTL4_CONT_SIZE __PTL4_CONT_SIZE +# define PTL3_CONT_SIZE __PTL3_CONT_SIZE +# define PTL2_CONT_SIZE __PTL2_CONT_SIZE +# define PTL1_CONT_SIZE __PTL1_CONT_SIZE +# define PTL4_CONT_MASK __PTL4_CONT_MASK +# define PTL3_CONT_MASK __PTL3_CONT_MASK +# define PTL2_CONT_MASK __PTL2_CONT_MASK +# define PTL1_CONT_MASK __PTL1_CONT_MASK +# define PTL4_CONT_COUNT __PTL4_CONT_COUNT +# define PTL3_CONT_COUNT __PTL3_CONT_COUNT +# define PTL2_CONT_COUNT __PTL2_CONT_COUNT +# define PTL1_CONT_COUNT __PTL1_CONT_COUNT #endif/*__ASSEMBLY__*/ +#define __page_size(pgshift) (UL(1) << (pgshift)) +#define __page_mask(pgsize) (~((pgsize) - 1)) #define __page_offset(addr, size) ((unsigned long)(addr) & ((size) - 1)) #define __page_align(addr, size) ((unsigned long)(addr) & ~((size) - 1)) #define __page_align_up(addr, size) __page_align((unsigned long)(addr) + (size) - 1, size) @@ -210,8 +276,8 @@ static const unsigned int PTL1_ENTRIES = __PTL1_ENTRIES; * nornal page */ #define PAGE_SHIFT __PTL1_SHIFT -#define PAGE_SIZE (UL(1) << __PTL1_SHIFT) -#define PAGE_MASK (~(PTL1_SIZE - 1)) +#define PAGE_SIZE __page_size(PAGE_SHIFT) +#define PAGE_MASK __page_mask(PAGE_SIZE) #define PAGE_P2ALIGN 0 #define page_offset(addr) __page_offset(addr, PAGE_SIZE) #define page_align(addr) __page_align(addr, PAGE_SIZE) @@ -221,8 +287,8 @@ static const unsigned int PTL1_ENTRIES = __PTL1_ENTRIES; * large page */ #define LARGE_PAGE_SHIFT __PTL2_SHIFT -#define LARGE_PAGE_SIZE (UL(1) << __PTL2_SHIFT) -#define LARGE_PAGE_MASK (~(PTL2_SIZE - 1)) +#define LARGE_PAGE_SIZE __page_size(LARGE_PAGE_SHIFT) +#define LARGE_PAGE_MASK __page_mask(LARGE_PAGE_SIZE) #define LARGE_PAGE_P2ALIGN (LARGE_PAGE_SHIFT - PAGE_SHIFT) #define large_page_offset(addr) __page_offset(addr, LARGE_PAGE_SIZE) #define large_page_align(addr) __page_align(addr, LARGE_PAGE_SIZE) @@ -325,18 +391,23 @@ static inline int pfn_is_write_combined(uintptr_t pfn) //共通部と意味がするビット定義 #define attr_flip_bits (PTATTR_WRITABLE | PTATTR_LARGEPAGE) +static inline int pgsize_to_tbllv(size_t pgsize); static inline int pte_is_type_page(const pte_t *ptep, size_t pgsize) { int ret = 0; //default D_TABLE - if ((PTL4_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 3) || - (PTL3_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 2) || - (PTL2_SIZE == pgsize)) { + int level = pgsize_to_tbllv(pgsize); + + switch (level) { + case 4: + case 3: + case 2: // check D_BLOCK ret = ((*ptep & PMD_TYPE_MASK) == PMD_TYPE_SECT); - } - else if (PTL1_SIZE == pgsize) { + break; + case 1: // check D_PAGE ret = ((*ptep & PTE_TYPE_MASK) == PTE_TYPE_PAGE); + break; } return ret; } @@ -415,21 +486,18 @@ static inline enum ihk_mc_pt_attribute pte_get_attr(pte_t *ptep, size_t pgsize) static inline void pte_make_null(pte_t *ptep, size_t pgsize) { - if ((PTL4_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 3) || - (PTL3_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 2) || - (PTL2_SIZE == pgsize) || - (PTL1_SIZE == pgsize)) { - *ptep = PTE_NULL; - } + *ptep = PTE_NULL; } static inline void pte_make_fileoff(off_t off, enum ihk_mc_pt_attribute ptattr, size_t pgsize, pte_t *ptep) { - if ((PTL4_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 3) || - (PTL3_SIZE == pgsize && CONFIG_ARM64_PGTABLE_LEVELS > 2) || - (PTL2_SIZE == pgsize) || - (PTL1_SIZE == pgsize)) { + if (((PTL4_SIZE == pgsize || PTL4_CONT_SIZE == pgsize) + && CONFIG_ARM64_PGTABLE_LEVELS > 3) || + ((PTL3_SIZE == pgsize || PTL3_CONT_SIZE == pgsize) + && CONFIG_ARM64_PGTABLE_LEVELS > 2) || + (PTL2_SIZE == pgsize || PTL2_CONT_SIZE == pgsize) || + (PTL1_SIZE == pgsize || PTL1_CONT_SIZE == pgsize)) { *ptep = PTE_FILEOFF | off | PTE_TYPE_PAGE; } } @@ -459,6 +527,249 @@ static inline void pte_set_dirty(pte_t *ptep, size_t pgsize) } } +static inline int pte_is_contiguous(const pte_t *ptep) +{ + return !!(*ptep & PTE_CONT); +} + +static inline int pgsize_is_contiguous(size_t pgsize) +{ + int ret = 0; + + if ((pgsize == PTL4_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 3) || + (pgsize == PTL3_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 2) || + (pgsize == PTL2_CONT_SIZE) || + (pgsize == PTL1_CONT_SIZE)) { + ret = 1; + } + return ret; +} + +static inline int pgsize_to_tbllv(size_t pgsize) +{ + int level = -EINVAL; + + if ((pgsize == PTL4_CONT_SIZE || pgsize == PTL4_SIZE) + && (CONFIG_ARM64_PGTABLE_LEVELS > 3)) { + level = 4; + } else if ((pgsize == PTL3_CONT_SIZE || pgsize == PTL3_SIZE) + && (CONFIG_ARM64_PGTABLE_LEVELS > 2)) { + level = 3; + } else if (pgsize == PTL2_CONT_SIZE || pgsize == PTL2_SIZE) { + level = 2; + } else if (pgsize == PTL1_CONT_SIZE || pgsize == PTL1_SIZE) { + level = 1; + } + return level; +} + +static inline size_t tbllv_to_pgsize(int level) +{ + size_t pgsize = 0; + + switch (level) { + case 4: + if (CONFIG_ARM64_PGTABLE_LEVELS > 3) { + pgsize = PTL4_SIZE; + } else { + panic("page table level 4 is invalid."); + } + break; + case 3: + if (CONFIG_ARM64_PGTABLE_LEVELS > 2) { + pgsize = PTL3_SIZE; + } else { + panic("page table level 3 is invalid."); + } + break; + case 2: + pgsize = PTL2_SIZE; + break; + case 1: + pgsize = PTL1_SIZE; + break; + default: + panic("page table level is invalid."); + } + return pgsize; +} + +static inline size_t tbllv_to_contpgsize(int level) +{ + size_t pgsize = 0; + + switch (level) { + case 4: + if (CONFIG_ARM64_PGTABLE_LEVELS > 3) { + pgsize = PTL4_CONT_SIZE; + } else { + panic("page table level 4 is invalid."); + } + break; + case 3: + if (CONFIG_ARM64_PGTABLE_LEVELS > 2) { + pgsize = PTL3_CONT_SIZE; + } else { + panic("page table level 3 is invalid."); + } + break; + case 2: + pgsize = PTL2_CONT_SIZE; + break; + case 1: + pgsize = PTL1_CONT_SIZE; + break; + default: + panic("page table level is invalid."); + } + return pgsize; +} + +static inline int tbllv_to_contpgshift(int level) +{ + int ret = 0; + + switch (level) { + case 4: + if (CONFIG_ARM64_PGTABLE_LEVELS > 3) { + ret = PTL4_CONT_SHIFT; + } else { + panic("page table level 4 is invalid."); + } + + break; + case 3: + if (CONFIG_ARM64_PGTABLE_LEVELS > 2) { + ret = PTL3_CONT_SHIFT; + } else { + panic("page table level 3 is invalid."); + } + + break; + case 2: + ret = PTL2_CONT_SHIFT; + break; + case 1: + ret = PTL1_CONT_SHIFT; + break; + default: + panic("page table level is invalid."); + } + return ret; +} + +static inline pte_t *get_contiguous_head(pte_t *__ptep, size_t __pgsize) +{ + unsigned long align; + int shift = 0; + + switch (pgsize_to_tbllv(__pgsize)) { + case 4: + if (CONFIG_ARM64_PGTABLE_LEVELS > 3) { + shift = PTL4_CONT_SHIFT - PTL4_SHIFT; + } else { + panic("page table level 4 is invalid."); + } + break; + case 3: + if (CONFIG_ARM64_PGTABLE_LEVELS > 2) { + shift = PTL3_CONT_SHIFT - PTL3_SHIFT; + } else { + panic("page table level 3 is invalid."); + } + break; + case 2: + shift = PTL2_CONT_SHIFT - PTL2_SHIFT; + break; + case 1: + shift = PTL1_CONT_SHIFT - PTL1_SHIFT; + break; + default: + panic("page table level is invalid."); + } + align = sizeof(*__ptep) << shift; + return (pte_t *)__page_align(__ptep, align); +} + +static inline pte_t *get_contiguous_tail(pte_t *__ptep, size_t __pgsize) +{ + unsigned long align; + int shift = 0; + + switch (pgsize_to_tbllv(__pgsize)) { + case 4: + if (CONFIG_ARM64_PGTABLE_LEVELS > 3) { + shift = PTL4_CONT_SHIFT - PTL4_SHIFT; + } else { + panic("page table level 4 is invalid."); + } + break; + case 3: + if (CONFIG_ARM64_PGTABLE_LEVELS > 2) { + shift = PTL3_CONT_SHIFT - PTL3_SHIFT; + } else { + panic("page table level 3 is invalid."); + } + break; + case 2: + shift = PTL2_CONT_SHIFT - PTL2_SHIFT; + break; + case 1: + shift = PTL1_CONT_SHIFT - PTL1_SHIFT; + break; + default: + panic("page table level is invalid."); + } + align = sizeof(*__ptep) << shift; + return (pte_t *)__page_align_up(__ptep + 1, align) - 1; +} + +static inline int split_contiguous_pages(pte_t *ptep, size_t pgsize) +{ + int ret; + pte_t *head = get_contiguous_head(ptep, pgsize); + pte_t *tail = get_contiguous_tail(ptep, pgsize); + pte_t *ptr; + + uintptr_t phys; + struct page *page; + + phys = pte_get_phys(head); + page = phys_to_page(phys); + if (page && (page_is_in_memobj(page) + || page_is_multi_mapped(page))) { + ret = -EINVAL; + goto out; + } + + for (ptr = head; ptr <= tail; ptr++) { + *ptr &= ~PTE_CONT; + } + + ret = 0; +out: + return ret; +} + +static inline int page_is_contiguous_head(pte_t *ptep, size_t pgsize) +{ + pte_t *ptr = get_contiguous_head(ptep, pgsize); + + return (ptr == ptep); +} + +static inline int page_is_contiguous_tail(pte_t *ptep, size_t pgsize) +{ + pte_t *ptr = get_contiguous_tail(ptep, pgsize); + + return (ptr == ptep); +} + +void arch_adjust_allocate_page_size(uintptr_t fault_addr, + pte_t *ptep, + void **pgaddrp, + size_t *pgsizep); + struct page_table; void set_pte(pte_t *ppte, unsigned long phys, enum ihk_mc_pt_attribute attr); pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr); diff --git a/arch/arm64/kernel/include/pgtable-hwdef.h b/arch/arm64/kernel/include/pgtable-hwdef.h index 599ce6ef..bdd270aa 100644 --- a/arch/arm64/kernel/include/pgtable-hwdef.h +++ b/arch/arm64/kernel/include/pgtable-hwdef.h @@ -72,6 +72,7 @@ #define PMD_SECT_S (UL(3) << 8) #define PMD_SECT_AF (UL(1) << 10) #define PMD_SECT_NG (UL(1) << 11) +#define PMD_SECT_CONT (UL(1) << 52) #define PMD_SECT_PXN (UL(1) << 53) #define PMD_SECT_UXN (UL(1) << 54) @@ -93,6 +94,7 @@ #define PTE_SHARED (UL(3) << 8) /* SH[1:0], inner shareable */ #define PTE_AF (UL(1) << 10) /* Access Flag */ #define PTE_NG (UL(1) << 11) /* nG */ +#define PTE_CONT (UL(1) << 52) /* Contiguous range */ #define PTE_PXN (UL(1) << 53) /* Privileged XN */ #define PTE_UXN (UL(1) << 54) /* User XN */ /* Software defined PTE bits definition.*/ diff --git a/arch/arm64/kernel/memory.c b/arch/arm64/kernel/memory.c index 45e2bab8..bbc8c0a6 100644 --- a/arch/arm64/kernel/memory.c +++ b/arch/arm64/kernel/memory.c @@ -387,6 +387,7 @@ static inline void ptl_set(pte_t* p, pte_t v, int level) panic("ptl_set failed.\n"); } } + /* clear */ static inline void ptl4_clear(pte_t* l4p) { @@ -407,22 +408,23 @@ static inline void ptl1_clear(pte_t* l1p) static inline void ptl_clear(pte_t* p, int level) { switch (level) { - case 4: + case 4: ptl4_clear(p); break; - case 3: + case 3: ptl3_clear(p); break; - case 2: + case 2: ptl2_clear(p); break; - case 1: + case 1: ptl1_clear(p); break; default: panic("ptl_clear failed.\n"); } } + /* null */ static inline int ptl4_null(const pte_t* l4p) { @@ -465,6 +467,7 @@ static inline int ptl_null(const pte_t* p, int level) } return ret; } + /* present */ static inline int ptl4_present(const pte_t* l4p) { @@ -507,6 +510,7 @@ static inline int ptl_present(const pte_t* p, int level) } return ret; } + /* type_block/type_page */ static inline int ptl4_type_block(const pte_t* l4p) { @@ -553,6 +557,55 @@ static inline int ptl_type_page(const pte_t* p, int level) } return ret; } + +/* contiguous */ +static inline int ptl4_is_contiguous(const pte_t *l4p) +{ + pte_t pte = ptl4_val(l4p); + + return pte_is_contiguous(&pte); +} +static inline int ptl3_is_contiguous(const pte_t *l3p) +{ + pte_t pte = ptl3_val(l3p); + + return pte_is_contiguous(&pte); +} +static inline int ptl2_is_contiguous(const pte_t *l2p) +{ + pte_t pte = ptl2_val(l2p); + + return pte_is_contiguous(&pte); +} +static inline int ptl1_is_contiguous(const pte_t *l1p) +{ + pte_t pte = ptl1_val(l1p); + + return pte_is_contiguous(&pte); +} +static inline int ptl_is_contiguous(const pte_t *p, int level) +{ + int ret = 0; + + switch (level) { + case 4: + ret = ptl4_is_contiguous(p); + break; + case 3: + ret = ptl3_is_contiguous(p); + break; + case 2: + ret = ptl2_is_contiguous(p); + break; + case 1: + ret = ptl1_is_contiguous(p); + break; + default: + panic("ptl_is_contiguous failed.\n"); + } + return ret; +} + /* type_table */ static inline int ptl4_type_table(const pte_t* l4p) { @@ -594,6 +647,7 @@ static inline int ptl_type_table(const pte_t* p, int level) } return ret; } + /* phys */ static inline unsigned long ptl4_phys(const pte_t* l4p) { @@ -636,6 +690,7 @@ static inline unsigned long ptl_phys(const pte_t* p, int level) } return ret; } + /* dirty */ static inline int ptl4_dirty(const pte_t* l4p) { @@ -678,6 +733,7 @@ static inline int ptl_dirty(const pte_t* p, int level) } return ret; } + /* fileoff */ static inline int ptl4_fileoff(const pte_t* l4p) { @@ -3156,3 +3212,45 @@ void remote_flush_tlb_cpumask(struct process_vm *vm, } #endif /* POSTK_DEBUG_ARCH_DEP_8 */ +void arch_adjust_allocate_page_size(uintptr_t fault_addr, + pte_t *ptep, + void **pgaddrp, + size_t *pgsizep) +{ + if (ptep == NULL) { + int level = pgsize_to_tbllv(*pgsizep); + *pgsizep = tbllv_to_pgsize(level); + *pgaddrp = (void *)__page_align(fault_addr, *pgsizep); + } else if (pte_is_null(ptep) && pgsize_is_contiguous(*pgsizep)) { + struct memobj *obj; + uintptr_t zeropage = NOPHYS; + pte_t *head; + pte_t *tail; + + if (zeroobj_create(&obj)) { + panic("zeroobj_create"); + } + memobj_get_page(obj, 0, PAGE_P2ALIGN, &zeropage, NULL, 0); + + head = get_contiguous_head(ptep, *pgsizep); + tail = get_contiguous_tail(ptep, *pgsizep); + for (/*nop*/; head <= tail; head++) { + uintptr_t phys; + int level; + + if (pte_is_null(head)) { + continue; + } + + phys = pte_get_phys(head); + if (phys == zeropage) { + continue; + } + + level = pgsize_to_tbllv(*pgsizep); + *pgsizep = tbllv_to_pgsize(level); + *pgaddrp = (void *)__page_align(fault_addr, *pgsizep); + break; + } + } +}