| /mm/kasan/ |
| A D | init.c | 194 p4d_populate(&init_mm, p4d, in zero_p4d_populate()
205 if (p4d_none(*p4d)) { in zero_p4d_populate()
242 p4d_t *p4d; in kasan_populate_early_shadow() local
328 p4d_clear(p4d); in kasan_free_pud()
333 p4d_t *p4d; in kasan_free_p4d() local
337 p4d = p4d_start + i; in kasan_free_p4d()
338 if (!p4d_none(*p4d)) in kasan_free_p4d()
431 if (!p4d_present(*p4d)) in kasan_remove_p4d_table()
437 p4d_clear(p4d); in kasan_remove_p4d_table()
443 kasan_free_pud(pud_offset(p4d, 0), p4d); in kasan_remove_p4d_table()
[all …]
|
| A D | shadow.c | 189 p4d_t *p4d; in shadow_mapped() local
196 p4d = p4d_offset(pgd, addr); in shadow_mapped()
197 if (p4d_none(*p4d)) in shadow_mapped()
199 pud = pud_offset(p4d, addr); in shadow_mapped()
|
| /mm/ |
| A D | sparse-vmemmap.c | 213 pud_t *pud = pud_offset(p4d, addr); in vmemmap_pud_populate()
226 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate() local
227 if (p4d_none(*p4d)) { in vmemmap_p4d_populate()
232 p4d_populate(&init_mm, p4d, p); in vmemmap_p4d_populate()
234 return p4d; in vmemmap_p4d_populate()
255 p4d_t *p4d; in vmemmap_populate_address() local
263 p4d = vmemmap_p4d_populate(pgd, addr, node); in vmemmap_populate_address()
264 if (!p4d) in vmemmap_populate_address()
266 pud = vmemmap_pud_populate(p4d, addr, node); in vmemmap_populate_address()
422 p4d_t *p4d; in vmemmap_populate_hugepages() local
[all …]
|
| A D | pgalloc-track.h | 19 static inline pud_t *pud_alloc_track(struct mm_struct *mm, p4d_t *p4d, in pud_alloc_track() argument
23 if (unlikely(p4d_none(*p4d))) { in pud_alloc_track()
24 if (__pud_alloc(mm, p4d, address)) in pud_alloc_track()
29 return pud_offset(p4d, address); in pud_alloc_track()
|
| A D | memory.c | 269 p4d_t *p4d; in free_p4d_range() local
1860 p4d_t *p4d; in zap_p4d_range() local
2070 p4d_t *p4d; in walk_to_pmd() local
2076 if (!p4d) in walk_to_pmd()
2757 p4d_t *p4d; in remap_p4d_range() local
2763 if (!p4d) in remap_p4d_range()
3109 p4d_t *p4d; in apply_to_p4d_range() local
3115 if (!p4d) in apply_to_p4d_range()
3126 if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) { in apply_to_p4d_range()
6121 if (!p4d) in __handle_mm_fault()
[all …]
|
| A D | pagewalk.c | 167 static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, in walk_pud_range() argument
178 pud = pud_offset(p4d, addr); in walk_pud_range()
235 p4d_t *p4d; in walk_p4d_range() local
243 p4d = p4d_offset(pgd, addr); in walk_p4d_range()
246 if (p4d_none_or_clear_bad(p4d)) { in walk_p4d_range()
248 err = __pud_alloc(walk->mm, p4d, addr); in walk_p4d_range()
257 err = ops->p4d_entry(p4d, addr, next, walk); in walk_p4d_range()
262 err = walk_pud_range(p4d, addr, next, walk); in walk_p4d_range()
265 } while (p4d++, addr = next, addr != end); in walk_p4d_range()
|
| A D | vmalloc.c | 259 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) in vmap_try_huge_p4d()
269 p4d_t *p4d; in vmap_p4d_range() local
273 if (!p4d) in vmap_p4d_range()
440 p4d_t *p4d; in vunmap_p4d_range() local
447 p4d_clear_huge(p4d); in vunmap_p4d_range()
448 if (p4d_bad(*p4d)) in vunmap_p4d_range()
597 p4d_t *p4d; in vmap_pages_p4d_range() local
601 if (!p4d) in vmap_pages_p4d_range()
784 p4d_t *p4d; in vmalloc_to_page() local
803 if (p4d_none(*p4d)) in vmalloc_to_page()
[all …]
|
| A D | mprotect.c | 582 struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr, in change_pud_range() argument
592 pudp = pud_offset(p4d, addr); in change_pud_range()
644 p4d_t *p4d; in change_p4d_range() local
648 p4d = p4d_offset(pgd, addr); in change_p4d_range()
651 ret = change_prepare(vma, p4d, pud, addr, cp_flags); in change_p4d_range()
654 if (p4d_none_or_clear_bad(p4d)) in change_p4d_range()
656 pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, in change_p4d_range()
658 } while (p4d++, addr = next, addr != end); in change_p4d_range()
677 ret = change_prepare(vma, pgd, p4d, addr, cp_flags); in change_protection_range()
|
| A D | page_vma_mapped.c | 191 p4d_t *p4d; in page_vma_mapped_walk() local
230 p4d = p4d_offset(pgd, pvmw->address); in page_vma_mapped_walk()
231 if (!p4d_present(*p4d)) { in page_vma_mapped_walk()
235 pud = pud_offset(p4d, pvmw->address); in page_vma_mapped_walk()
|
| A D | debug_vm_pgtable.c | 467 p4d_t p4d; in p4d_basic_tests() local
470 memset(&p4d, RANDOM_NZVALUE, sizeof(p4d_t)); in p4d_basic_tests()
471 WARN_ON(!p4d_same(p4d, p4d)); in p4d_basic_tests()
522 p4d_t p4d = p4dp_get(args->p4dp); in p4d_clear_tests() local
528 WARN_ON(p4d_none(p4d)); in p4d_clear_tests()
530 p4d = p4dp_get(args->p4dp); in p4d_clear_tests()
531 WARN_ON(!p4d_none(p4d)); in p4d_clear_tests()
536 p4d_t p4d; in p4d_populate_tests() local
549 p4d = p4dp_get(args->p4dp); in p4d_populate_tests()
550 WARN_ON(p4d_bad(p4d)); in p4d_populate_tests()
|
| A D | gup.c | 989 p4d_t *p4dp, p4d; in follow_p4d_mask() local
992 p4d = READ_ONCE(*p4dp); in follow_p4d_mask()
993 BUILD_BUG_ON(p4d_leaf(p4d)); in follow_p4d_mask()
995 if (!p4d_present(p4d) || p4d_bad(p4d)) in follow_p4d_mask()
1053 p4d_t *p4d; in get_gate_page() local
1066 p4d = p4d_offset(pgd, address); in get_gate_page()
1067 if (p4d_none(*p4d)) in get_gate_page()
1069 pud = pud_offset(p4d, address); in get_gate_page()
3101 p4d_t p4d = READ_ONCE(*p4dp); in gup_fast_p4d_range() local
3104 if (!p4d_present(p4d)) in gup_fast_p4d_range()
[all …]
|
| A D | mremap.c | 78 p4d_t *p4d; in get_old_pud() local
85 p4d = p4d_offset(pgd, addr); in get_old_pud()
86 if (p4d_none_or_clear_bad(p4d)) in get_old_pud()
89 pud = pud_offset(p4d, addr); in get_old_pud()
115 p4d_t *p4d; in alloc_new_pud() local
118 p4d = p4d_alloc(mm, pgd, addr); in alloc_new_pud()
119 if (!p4d) in alloc_new_pud()
122 return pud_alloc(mm, p4d, addr); in alloc_new_pud()
|
| A D | pgtable-generic.c | 32 void p4d_clear_bad(p4d_t *p4d) in p4d_clear_bad() argument
34 p4d_ERROR(*p4d); in p4d_clear_bad()
35 p4d_clear(p4d); in p4d_clear_bad()
|
| A D | ptdump.c | 53 static int ptdump_p4d_entry(p4d_t *p4d, unsigned long addr, in ptdump_p4d_entry() argument
57 p4d_t val = READ_ONCE(*p4d); in ptdump_p4d_entry()
|
| A D | percpu.c | 3131 p4d_t *p4d; in pcpu_populate_pte() local
3136 p4d = memblock_alloc_or_panic(P4D_TABLE_SIZE, P4D_TABLE_SIZE); in pcpu_populate_pte()
3137 pgd_populate(&init_mm, pgd, p4d); in pcpu_populate_pte()
3140 p4d = p4d_offset(pgd, addr); in pcpu_populate_pte()
3141 if (p4d_none(*p4d)) { in pcpu_populate_pte()
3143 p4d_populate(&init_mm, p4d, pud); in pcpu_populate_pte()
3146 pud = pud_offset(p4d, addr); in pcpu_populate_pte()
|
| A D | userfaultfd.c | 464 p4d_t *p4d; in mm_alloc_pmd() local
468 p4d = p4d_alloc(mm, pgd, address); in mm_alloc_pmd()
469 if (!p4d) in mm_alloc_pmd()
471 pud = pud_alloc(mm, p4d, address); in mm_alloc_pmd()
|
| A D | swapfile.c | 2178 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, in unuse_pud_range() argument
2186 pud = pud_offset(p4d, addr); in unuse_pud_range()
2202 p4d_t *p4d; in unuse_p4d_range() local
2206 p4d = p4d_offset(pgd, addr); in unuse_p4d_range()
2209 if (p4d_none_or_clear_bad(p4d)) in unuse_p4d_range()
2211 ret = unuse_pud_range(vma, p4d, addr, next, type); in unuse_p4d_range()
2214 } while (p4d++, addr = next, addr != end); in unuse_p4d_range()
|
| A D | memory-failure.c | 406 p4d_t *p4d; in dev_pagemap_mapping_shift() local
416 p4d = p4d_offset(pgd, address); in dev_pagemap_mapping_shift()
417 if (!p4d_present(*p4d)) in dev_pagemap_mapping_shift()
419 pud = pud_offset(p4d, address); in dev_pagemap_mapping_shift()
|
| A D | rmap.c | 818 p4d_t *p4d; in mm_find_pmd() local
826 p4d = p4d_offset(pgd, address); in mm_find_pmd()
827 if (!p4d_present(*p4d)) in mm_find_pmd()
830 pud = pud_offset(p4d, address); in mm_find_pmd()
|
| A D | hugetlb.c | 7595 p4d_t *p4d = p4d_offset(pgd, addr); in huge_pmd_unshare() local
7596 pud_t *pud = pud_offset(p4d, addr); in huge_pmd_unshare()
7648 p4d_t *p4d; in huge_pte_alloc() local
7653 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc()
7654 if (!p4d) in huge_pte_alloc()
7656 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc()
7691 p4d_t *p4d; in huge_pte_offset() local
7698 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
7699 if (!p4d_present(*p4d)) in huge_pte_offset()
7702 pud = pud_offset(p4d, addr); in huge_pte_offset()
|
| A D | vmscan.c | 3761 static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, in walk_pud_range() argument
3770 VM_WARN_ON_ONCE(p4d_leaf(*p4d)); in walk_pud_range()
3772 pud = pud_offset(p4d, start & P4D_MASK); in walk_pud_range()
|