| /linux-6.3-rc2/arch/riscv/mm/ |
| A D | kasan_init.c | 224 kasan_populate_p4d(pgdp, vaddr, next, early) : \
226 kasan_populate_pud(pgdp, vaddr, next, early) : \
227 kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))
229 static void __init kasan_populate_pgd(pgd_t *pgdp, in kasan_populate_pgd() argument
244 } else if (pgd_page_vaddr(*pgdp) == in kasan_populate_pgd()
261 kasan_populate_pgd_next(pgdp, vaddr, next, early); in kasan_populate_pgd()
262 } while (pgdp++, vaddr = next, vaddr != end); in kasan_populate_pgd()
330 base_pmd = (pmd_t *)pgd_page_vaddr(*pgdp); in kasan_shallow_populate_pmd()
350 base_pud = (pud_t *)pgd_page_vaddr(*pgdp); in kasan_shallow_populate_pud()
379 base_p4d = (p4d_t *)pgd_page_vaddr(*pgdp); in kasan_shallow_populate_p4d()
[all …]
|
| /linux-6.3-rc2/arch/arm64/mm/ |
| A D | mmu.c | 765 kasan_copy_shadow(pgdp); in map_kernel()
805 map_kernel(pgdp); in paging_init()
806 map_mem(pgdp); in paging_init()
967 pgd_t *pgdp, pgd; in unmap_hotplug_range() local
979 pgdp = pgd_offset_k(addr); in unmap_hotplug_range()
980 pgd = READ_ONCE(*pgdp); in unmap_hotplug_range()
1129 pgd_t *pgdp, pgd; in free_empty_tables() local
1133 pgdp = pgd_offset_k(addr); in free_empty_tables()
1134 pgd = READ_ONCE(*pgdp); in free_empty_tables()
1213 pgd_t *pgdp; in early_fixmap_init() local
[all …]
|
| A D | kasan_init.c | 143 static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr, in kasan_p4d_populate() argument
147 p4d_t *p4dp = p4d_offset(pgdp, addr); in kasan_p4d_populate()
159 pgd_t *pgdp; in kasan_pgd_populate() local
161 pgdp = pgd_offset_k(addr); in kasan_pgd_populate()
164 kasan_p4d_populate(pgdp, addr, next, node, early); in kasan_pgd_populate()
165 } while (pgdp++, addr = next, addr != end); in kasan_pgd_populate()
192 pgd_t *pgdp, *pgdp_new, *pgdp_end; in kasan_copy_shadow() local
194 pgdp = pgd_offset_k(KASAN_SHADOW_START); in kasan_copy_shadow()
198 set_pgd(pgdp_new, READ_ONCE(*pgdp)); in kasan_copy_shadow()
199 } while (pgdp++, pgdp_new++, pgdp != pgdp_end); in kasan_copy_shadow()
|
| A D | hugetlbpage.c | 106 pgd_t *pgdp = pgd_offset(mm, addr); in find_num_contig() local
112 p4dp = p4d_offset(pgdp, addr); in find_num_contig()
290 pgd_t *pgdp; in huge_pte_alloc() local
296 pgdp = pgd_offset(mm, addr); in huge_pte_alloc()
297 p4dp = p4d_offset(pgdp, addr); in huge_pte_alloc()
335 pgd_t *pgdp; in huge_pte_offset() local
340 pgdp = pgd_offset(mm, addr); in huge_pte_offset()
341 if (!pgd_present(READ_ONCE(*pgdp))) in huge_pte_offset()
344 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
|
| A D | pageattr.c | 213 pgd_t *pgdp; in kernel_page_present() local
223 pgdp = pgd_offset_k(addr); in kernel_page_present()
224 if (pgd_none(READ_ONCE(*pgdp))) in kernel_page_present()
227 p4dp = p4d_offset(pgdp, addr); in kernel_page_present()
|
| /linux-6.3-rc2/arch/powerpc/mm/nohash/ |
| A D | book3e_pgtable.c | 76 pgd_t *pgdp; in map_kernel_page() local
84 pgdp = pgd_offset_k(ea); in map_kernel_page()
85 p4dp = p4d_offset(pgdp, ea); in map_kernel_page()
96 pgdp = pgd_offset_k(ea); in map_kernel_page()
97 p4dp = p4d_offset(pgdp, ea); in map_kernel_page()
|
| /linux-6.3-rc2/arch/arm/mm/ |
| A D | kasan_init.c | 122 pgd_t *pgdp; in kasan_pgd_populate() local
126 pgdp = pgd_offset_k(addr); in kasan_pgd_populate()
133 if (!early && pgd_none(*pgdp)) { in kasan_pgd_populate()
141 pgd_populate(&init_mm, pgdp, p); in kasan_pgd_populate()
150 p4dp = p4d_offset(pgdp, addr); in kasan_pgd_populate()
154 } while (pgdp++, addr = next, addr != end); in kasan_pgd_populate()
|
| /linux-6.3-rc2/arch/s390/mm/ |
| A D | hugetlbpage.c | 188 pgd_t *pgdp; in huge_pte_alloc() local
193 pgdp = pgd_offset(mm, addr); in huge_pte_alloc()
194 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
210 pgd_t *pgdp; in huge_pte_offset() local
215 pgdp = pgd_offset(mm, addr); in huge_pte_offset()
216 if (pgd_present(*pgdp)) { in huge_pte_offset()
217 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
|
| A D | pageattr.c | 299 pgd_t *pgdp; in change_page_attr() local
306 pgdp = pgd_offset_k(addr); in change_page_attr()
308 if (pgd_none(*pgdp)) in change_page_attr()
311 rc = walk_p4d_level(pgdp, addr, next, flags); in change_page_attr()
315 } while (pgdp++, addr = next, addr < end && !rc); in change_page_attr()
|
| /linux-6.3-rc2/arch/sparc/mm/ |
| A D | srmmu.c | 496 pgd_t *pgdp; in srmmu_mapioaddr() local
504 pgdp = pgd_offset_k(virt_addr); in srmmu_mapioaddr()
535 pgd_t *pgdp; in srmmu_unmapioaddr() local
542 pgdp = pgd_offset_k(virt_addr); in srmmu_unmapioaddr()
682 pgd_t *pgdp; in srmmu_early_allocate_ptable_skeleton() local
689 pgdp = pgd_offset_k(start); in srmmu_early_allocate_ptable_skeleton()
717 pgd_t *pgdp; in srmmu_allocate_ptable_skeleton() local
724 pgdp = pgd_offset_k(start); in srmmu_allocate_ptable_skeleton()
732 pud_set((pud_t *)pgdp, pmdp); in srmmu_allocate_ptable_skeleton()
776 pgd_t *pgdp; in srmmu_inherit_prom_mappings() local
[all …]
|
| A D | fault_64.c | 81 pgd_t *pgdp = pgd_offset(current->mm, tpc); in get_user_insn() local
89 if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) in get_user_insn()
91 p4dp = p4d_offset(pgdp, tpc); in get_user_insn()
|
| /linux-6.3-rc2/arch/x86/mm/ |
| A D | pgtable.c | 262 static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp) in mop_up_one_pmd() argument
264 pgd_t pgd = *pgdp; in mop_up_one_pmd()
269 pgd_clear(pgdp); in mop_up_one_pmd()
277 static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) in pgd_mop_up_pmds() argument
282 mop_up_one_pmd(mm, &pgdp[i]); in pgd_mop_up_pmds()
289 pgdp = kernel_to_user_pgdp(pgdp); in pgd_mop_up_pmds()
292 mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]); in pgd_mop_up_pmds()
|
| A D | pti.c | 124 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in __pti_set_user_pgtbl() argument
135 if (!pgdp_maps_userspace(pgdp)) in __pti_set_user_pgtbl()
142 kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; in __pti_set_user_pgtbl()
|
| /linux-6.3-rc2/arch/powerpc/mm/kasan/ |
| A D | init_book3e_64.c | 34 pgd_t *pgdp; in kasan_map_kernel_page() local
40 pgdp = pgd_offset_k(ea); in kasan_map_kernel_page()
41 p4dp = p4d_offset(pgdp, ea); in kasan_map_kernel_page()
|
| /linux-6.3-rc2/arch/arm64/include/asm/ |
| A D | mmu_context.h | 151 static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) in cpu_replace_ttbr1() argument
159 phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); in cpu_replace_ttbr1()
161 if (system_supports_cnp() && !WARN_ON(pgdp != lm_alias(swapper_pg_dir))) { in cpu_replace_ttbr1()
|
| /linux-6.3-rc2/arch/x86/include/asm/ |
| A D | pgtable_64.h | 161 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument
163 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); in native_set_pgd()
|
| A D | paravirt.h | 514 static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) in __set_pgd() argument
516 PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd)); in __set_pgd()
519 #define set_pgd(pgdp, pgdval) do { \ argument
521 __set_pgd(pgdp, pgdval); \
523 set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd }); \
526 #define pgd_clear(pgdp) do { \ argument
528 set_pgd(pgdp, native_make_pgd(0)); \
|
| A D | pgtable.h | 76 #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) argument
705 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
712 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument
716 return __pti_set_user_pgtbl(pgdp, pgd); in pti_set_user_pgtbl()
719 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument
1229 static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) in kernel_to_user_pgdp() argument
1231 return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in kernel_to_user_pgdp()
1234 static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) in user_to_kernel_pgdp() argument
1236 return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in user_to_kernel_pgdp()
|
| /linux-6.3-rc2/arch/powerpc/mm/book3s64/ |
| A D | radix_pgtable.c | 77 pgd_t *pgdp; in early_map_kernel_page() local
83 pgdp = pgd_offset_k(ea); in early_map_kernel_page()
84 p4dp = p4d_offset(pgdp, ea); in early_map_kernel_page()
129 pgd_t *pgdp; in __map_kernel_page() local
152 pgdp = pgd_offset_k(ea); in __map_kernel_page()
153 p4dp = p4d_offset(pgdp, ea); in __map_kernel_page()
190 pgd_t *pgdp; in radix__change_memory_range() local
203 pgdp = pgd_offset_k(idx); in radix__change_memory_range()
204 p4dp = p4d_offset(pgdp, idx); in radix__change_memory_range()
|
| A D | hash_pgtable.c | 149 pgd_t *pgdp; in hash__map_kernel_page() local
157 pgdp = pgd_offset_k(ea); in hash__map_kernel_page()
158 p4dp = p4d_offset(pgdp, ea); in hash__map_kernel_page()
|
| /linux-6.3-rc2/arch/powerpc/lib/ |
| A D | code-patching.c | 252 pgd_t *pgdp; in unmap_patch_area() local
254 pgdp = pgd_offset_k(addr); in unmap_patch_area()
255 if (WARN_ON(pgd_none(*pgdp))) in unmap_patch_area()
258 p4dp = p4d_offset(pgdp, addr); in unmap_patch_area()
|
| /linux-6.3-rc2/mm/ |
| A D | debug_vm_pgtable.c | 64 pgd_t *pgdp; member
567 pgd_t pgd = READ_ONCE(*(args->pgdp)); in pgd_clear_tests()
574 WRITE_ONCE(*args->pgdp, pgd); in pgd_clear_tests()
575 pgd_clear(args->pgdp); in pgd_clear_tests()
576 pgd = READ_ONCE(*args->pgdp); in pgd_clear_tests()
593 pgd_clear(args->pgdp); in pgd_populate_tests()
594 pgd_populate(args->mm, args->pgdp, args->start_p4dp); in pgd_populate_tests()
595 pgd = READ_ONCE(*args->pgdp); in pgd_populate_tests()
1227 args->pgdp = pgd_offset(args->mm, args->vaddr); in init_args()
1228 args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr); in init_args()
[all …]
|
| /linux-6.3-rc2/arch/riscv/include/asm/ |
| A D | pgtable-64.h | 313 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) in set_pgd() argument
316 *pgdp = pgd; in set_pgd()
318 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); in set_pgd()
|
| A D | mmu.h | 27 void __init create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa,
|
| /linux-6.3-rc2/arch/m68k/include/asm/ |
| A D | pgtable_no.h | 22 #define pgd_clear(pgdp) argument
|