| /mm/kasan/ |
| A D | init.c | 254 pgd_populate(&init_mm, pgd, in kasan_populate_early_shadow()
256 p4d = p4d_offset(pgd, addr); in kasan_populate_early_shadow()
268 if (pgd_none(*pgd)) { in kasan_populate_early_shadow()
343 pgd_clear(pgd); in kasan_free_p4d()
450 pgd_t *pgd; in kasan_remove_zero_shadow() local
464 pgd = pgd_offset_k(addr); in kasan_remove_zero_shadow()
465 if (!pgd_present(*pgd)) in kasan_remove_zero_shadow()
468 if (kasan_p4d_table(*pgd)) { in kasan_remove_zero_shadow()
471 pgd_clear(pgd); in kasan_remove_zero_shadow()
476 p4d = p4d_offset(pgd, addr); in kasan_remove_zero_shadow()
[all …]
|
| A D | shadow.c | 188 pgd_t *pgd = pgd_offset_k(addr); in shadow_mapped() local
194 if (pgd_none(*pgd)) in shadow_mapped()
196 p4d = p4d_offset(pgd, addr); in shadow_mapped()
|
| /mm/ |
| A D | sparse-vmemmap.c | 226 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate()
239 pgd_t *pgd = pgd_offset_k(addr); in vmemmap_pgd_populate() local
240 if (pgd_none(*pgd)) { in vmemmap_pgd_populate()
244 pgd_populate(&init_mm, pgd, p); in vmemmap_pgd_populate()
246 return pgd; in vmemmap_pgd_populate()
254 pgd_t *pgd; in vmemmap_populate_address() local
260 pgd = vmemmap_pgd_populate(addr, node); in vmemmap_populate_address()
261 if (!pgd) in vmemmap_populate_address()
421 pgd_t *pgd; in vmemmap_populate_hugepages() local
429 pgd = vmemmap_pgd_populate(addr, node); in vmemmap_populate_hugepages()
[all …]
|
| A D | pagewalk.c | 243 p4d = p4d_offset(pgd, addr); in walk_p4d_range()
273 pgd_t *pgd; in walk_pgd_range() local
281 if (walk->pgd) in walk_pgd_range()
282 pgd = walk->pgd + pgd_index(addr); in walk_pgd_range()
284 pgd = pgd_offset(walk->mm, addr); in walk_pgd_range()
287 if (pgd_none_or_clear_bad(pgd)) { in walk_pgd_range()
289 err = __p4d_alloc(walk->mm, pgd, addr); in walk_pgd_range()
613 .pgd = pgd, in walk_kernel_page_table_range()
653 pgd_t *pgd, void *private) in walk_page_range_debug() argument
658 .pgd = pgd, in walk_page_range_debug()
[all …]
|
| A D | pgalloc-track.h | 6 static inline p4d_t *p4d_alloc_track(struct mm_struct *mm, pgd_t *pgd, in p4d_alloc_track() argument
10 if (unlikely(pgd_none(*pgd))) { in p4d_alloc_track()
11 if (__p4d_alloc(mm, pgd, address)) in p4d_alloc_track()
16 return p4d_offset(pgd, address); in p4d_alloc_track()
|
| A D | ptdump.c | 30 static int ptdump_pgd_entry(pgd_t *pgd, unsigned long addr, in ptdump_pgd_entry() argument
34 pgd_t val = READ_ONCE(*pgd); in ptdump_pgd_entry()
175 void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) in ptdump_walk_pgd() argument
183 &ptdump_ops, pgd, st); in ptdump_walk_pgd()
|
| A D | page_vma_mapped.c | 190 pgd_t *pgd; in page_vma_mapped_walk() local
225 pgd = pgd_offset(mm, pvmw->address); in page_vma_mapped_walk()
226 if (!pgd_present(*pgd)) { in page_vma_mapped_walk()
230 p4d = p4d_offset(pgd, pvmw->address); in page_vma_mapped_walk()
|
| A D | mprotect.c | 641 struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, in change_p4d_range() argument
648 p4d = p4d_offset(pgd, addr); in change_p4d_range()
668 pgd_t *pgd; in change_protection_range() local
673 pgd = pgd_offset(mm, addr); in change_protection_range()
677 ret = change_prepare(vma, pgd, p4d, addr, cp_flags); in change_protection_range()
682 if (pgd_none_or_clear_bad(pgd)) in change_protection_range()
684 pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot, in change_protection_range()
686 } while (pgd++, addr = next, addr != end); in change_protection_range()
|
| A D | memory.c | 274 p4d = p4d_offset(pgd, addr); in free_p4d_range()
293 p4d = p4d_offset(pgd, start); in free_p4d_range()
294 pgd_clear(pgd); in free_p4d_range()
314 pgd_t *pgd; in free_pgd_range() local
1879 pgd_t *pgd; in unmap_page_range() local
2069 pgd_t *pgd; in walk_to_pmd() local
2778 pgd_t *pgd; in remap_pfn_range_internal() local
3144 pgd_t *pgd; in __apply_to_page_range() local
3162 if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) { in __apply_to_page_range()
3165 pgd_clear_bad(pgd); in __apply_to_page_range()
[all …]
|
| A D | debug_vm_pgtable.c | 476 pgd_t pgd; in pgd_basic_tests() local
479 memset(&pgd, RANDOM_NZVALUE, sizeof(pgd_t)); in pgd_basic_tests()
480 WARN_ON(!pgd_same(pgd, pgd)); in pgd_basic_tests()
555 pgd_t pgd = pgdp_get(args->pgdp); in pgd_clear_tests() local
561 WARN_ON(pgd_none(pgd)); in pgd_clear_tests()
563 pgd = pgdp_get(args->pgdp); in pgd_clear_tests()
564 WARN_ON(!pgd_none(pgd)); in pgd_clear_tests()
569 pgd_t pgd; in pgd_populate_tests() local
582 pgd = pgdp_get(args->pgdp); in pgd_populate_tests()
583 WARN_ON(pgd_bad(pgd)); in pgd_populate_tests()
|
| A D | gup.c | 1029 pgd_t *pgd; in follow_page_mask() local
1036 pgd = pgd_offset(mm, address); in follow_page_mask()
1038 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) in follow_page_mask()
1052 pgd_t *pgd; in get_gate_page() local
1063 pgd = pgd_offset(mm, address); in get_gate_page()
1064 if (pgd_none(*pgd)) in get_gate_page()
1066 p4d = p4d_offset(pgd, address); in get_gate_page()
3099 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in gup_fast_p4d_range()
3123 pgd_t pgd = READ_ONCE(*pgdp); in gup_fast_pgd_range() local
3126 if (pgd_none(pgd)) in gup_fast_pgd_range()
[all …]
|
| A D | vmalloc.c | 295 pgd_t *pgd; in vmap_range_noflush() local
305 pgd = pgd_offset_k(addr); in vmap_range_noflush()
443 p4d = p4d_offset(pgd, addr); in vunmap_p4d_range()
472 pgd_t *pgd; in __vunmap_range_noflush() local
477 pgd = pgd_offset_k(addr); in __vunmap_range_noflush()
480 if (pgd_bad(*pgd)) in __vunmap_range_noflush()
615 pgd_t *pgd; in vmap_small_pages_range_noflush() local
622 pgd = pgd_offset_k(addr); in vmap_small_pages_range_noflush()
625 if (pgd_bad(*pgd)) in vmap_small_pages_range_noflush()
795 if (pgd_none(*pgd)) in vmalloc_to_page()
[all …]
|
| A D | pgtable-generic.c | 25 void pgd_clear_bad(pgd_t *pgd) in pgd_clear_bad() argument
27 pgd_ERROR(*pgd); in pgd_clear_bad()
28 pgd_clear(pgd); in pgd_clear_bad()
|
| A D | mremap.c | 77 pgd_t *pgd; in get_old_pud() local
81 pgd = pgd_offset(mm, addr); in get_old_pud()
82 if (pgd_none_or_clear_bad(pgd)) in get_old_pud()
85 p4d = p4d_offset(pgd, addr); in get_old_pud()
114 pgd_t *pgd; in alloc_new_pud() local
117 pgd = pgd_offset(mm, addr); in alloc_new_pud()
118 p4d = p4d_alloc(mm, pgd, addr); in alloc_new_pud()
|
| A D | init-mm.c | 34 .pgd = swapper_pg_dir,
|
| A D | swapfile.c | 2198 static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, in unuse_p4d_range() argument
2206 p4d = p4d_offset(pgd, addr); in unuse_p4d_range()
2220 pgd_t *pgd; in unuse_vma() local
2227 pgd = pgd_offset(vma->vm_mm, addr); in unuse_vma()
2230 if (pgd_none_or_clear_bad(pgd)) in unuse_vma()
2232 ret = unuse_p4d_range(vma, pgd, addr, next, type); in unuse_vma()
2235 } while (pgd++, addr = next, addr != end); in unuse_vma()
|
| A D | memory-failure.c | 405 pgd_t *pgd; in dev_pagemap_mapping_shift() local
413 pgd = pgd_offset(vma->vm_mm, address); in dev_pagemap_mapping_shift()
414 if (!pgd_present(*pgd)) in dev_pagemap_mapping_shift()
416 p4d = p4d_offset(pgd, address); in dev_pagemap_mapping_shift()
|
| A D | debug.c | 204 mm->pgd, atomic_read(&mm->mm_users), in dump_mm()
|
| A D | rmap.c | 817 pgd_t *pgd; in mm_find_pmd() local
822 pgd = pgd_offset(mm, address); in mm_find_pmd()
823 if (!pgd_present(*pgd)) in mm_find_pmd()
826 p4d = p4d_offset(pgd, address); in mm_find_pmd()
|
| A D | userfaultfd.c | 463 pgd_t *pgd; in mm_alloc_pmd() local
467 pgd = pgd_offset(mm, address); in mm_alloc_pmd()
468 p4d = p4d_alloc(mm, pgd, address); in mm_alloc_pmd()
|
| A D | hugetlb.c | 7594 pgd_t *pgd = pgd_offset(mm, addr); in huge_pmd_unshare() local
7595 p4d_t *p4d = p4d_offset(pgd, addr); in huge_pmd_unshare()
7647 pgd_t *pgd; in huge_pte_alloc() local
7652 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
7653 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc()
7690 pgd_t *pgd; in huge_pte_offset() local
7695 pgd = pgd_offset(mm, addr); in huge_pte_offset()
7696 if (!pgd_present(*pgd)) in huge_pte_offset()
7698 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
|
| A D | percpu.c | 3130 pgd_t *pgd = pgd_offset_k(addr); in pcpu_populate_pte() local
3135 if (pgd_none(*pgd)) { in pcpu_populate_pte()
3137 pgd_populate(&init_mm, pgd, p4d); in pcpu_populate_pte()
3140 p4d = p4d_offset(pgd, addr); in pcpu_populate_pte()
|
| A D | internal.h | 1641 pgd_t *pgd, void *private);
|