Lines Matching refs:addr
530 static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, unsigned long addr) in query_vma_find_by_addr() argument
532 return find_vma(mm, addr); in query_vma_find_by_addr()
536 unsigned long addr, u32 flags) in query_matching_vma() argument
541 vma = query_vma_find_by_addr(mm, addr); in query_matching_vma()
567 if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr) in query_matching_vma()
574 addr = vma->vm_end; in query_matching_vma()
907 static int smaps_pte_hole(unsigned long addr, unsigned long end, in smaps_pte_hole() argument
914 linear_page_index(vma, addr), in smaps_pte_hole()
923 static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) in smaps_pte_hole_lookup() argument
928 smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); in smaps_pte_hole_lookup()
933 static void smaps_pte_entry(pte_t *pte, unsigned long addr, in smaps_pte_entry() argument
944 page = vm_normal_page(vma, addr, ptent); in smaps_pte_entry()
970 smaps_pte_hole_lookup(addr, walk); in smaps_pte_entry()
981 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, in smaps_pmd_entry() argument
992 page = vm_normal_page_pmd(vma, addr, *pmd); in smaps_pmd_entry()
1016 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, in smaps_pmd_entry() argument
1022 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, in smaps_pte_range() argument
1031 smaps_pmd_entry(pmd, addr, walk); in smaps_pte_range()
1036 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); in smaps_pte_range()
1041 for (; addr != end; pte++, addr += PAGE_SIZE) in smaps_pte_range()
1042 smaps_pte_entry(pte, addr, walk); in smaps_pte_range()
1146 unsigned long addr, unsigned long end, in smaps_hugetlb_range() argument
1157 ptent = huge_ptep_get(walk->mm, addr, pte); in smaps_hugetlb_range()
1515 static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) in pte_is_pinned() argument
1525 folio = vm_normal_folio(vma, addr, pte); in pte_is_pinned()
1532 unsigned long addr, pte_t *pte) in clear_soft_dirty() argument
1545 if (pte_is_pinned(vma, addr, ptent)) in clear_soft_dirty()
1547 old_pte = ptep_modify_prot_start(vma, addr, pte); in clear_soft_dirty()
1550 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); in clear_soft_dirty()
1553 set_pte_at(vma->vm_mm, addr, pte, ptent); in clear_soft_dirty()
1558 unsigned long addr, pte_t *pte) in clear_soft_dirty() argument
1565 unsigned long addr, pmd_t *pmdp) in clear_soft_dirty_pmd() argument
1571 old = pmdp_invalidate(vma, addr, pmdp); in clear_soft_dirty_pmd()
1580 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); in clear_soft_dirty_pmd()
1583 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); in clear_soft_dirty_pmd()
1588 unsigned long addr, pmd_t *pmdp) in clear_soft_dirty_pmd() argument
1593 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, in clear_refs_pte_range() argument
1605 clear_soft_dirty_pmd(vma, addr, pmd); in clear_refs_pte_range()
1615 pmdp_test_and_clear_young(vma, addr, pmd); in clear_refs_pte_range()
1623 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); in clear_refs_pte_range()
1628 for (; addr != end; pte++, addr += PAGE_SIZE) { in clear_refs_pte_range()
1632 clear_soft_dirty(vma, addr, pte); in clear_refs_pte_range()
1639 folio = vm_normal_folio(vma, addr, ptent); in clear_refs_pte_range()
1644 ptep_test_and_clear_young(vma, addr, pte); in clear_refs_pte_range()
1811 unsigned long addr = start; in pagemap_pte_hole() local
1814 while (addr < end) { in pagemap_pte_hole()
1815 struct vm_area_struct *vma = find_vma(walk->mm, addr); in pagemap_pte_hole()
1825 for (; addr < hole_end; addr += PAGE_SIZE) { in pagemap_pte_hole()
1837 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { in pagemap_pte_hole()
1848 struct vm_area_struct *vma, unsigned long addr, pte_t pte) in pte_to_pagemap_entry() argument
1858 page = vm_normal_page(vma, addr, pte); in pte_to_pagemap_entry()
1906 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, in pagemap_pmd_range() argument
1918 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; in pagemap_pmd_range()
1967 for (; addr != end; addr += PAGE_SIZE, idx++) { in pagemap_pmd_range()
1995 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); in pagemap_pmd_range()
2000 for (; addr < end; pte++, addr += PAGE_SIZE) { in pagemap_pmd_range()
2003 pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); in pagemap_pmd_range()
2018 unsigned long addr, unsigned long end, in pagemap_hugetlb_range() argument
2032 pte = huge_ptep_get(walk->mm, addr, ptep); in pagemap_hugetlb_range()
2049 ((addr & ~hmask) >> PAGE_SHIFT); in pagemap_hugetlb_range()
2054 for (; addr != end; addr += PAGE_SIZE) { in pagemap_hugetlb_range()
2239 unsigned long addr, pte_t pte) in pagemap_page_category() argument
2251 page = vm_normal_page(vma, addr, pte); in pagemap_page_category()
2283 unsigned long addr, pte_t *pte, pte_t ptent) in make_uffd_wp_pte() argument
2288 old_pte = ptep_modify_prot_start(vma, addr, pte); in make_uffd_wp_pte()
2290 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); in make_uffd_wp_pte()
2293 set_pte_at(vma->vm_mm, addr, pte, ptent); in make_uffd_wp_pte()
2295 set_pte_at(vma->vm_mm, addr, pte, in make_uffd_wp_pte()
2303 unsigned long addr, pmd_t pmd) in pagemap_thp_category() argument
2315 page = vm_normal_page_pmd(vma, addr, pmd); in pagemap_thp_category()
2345 unsigned long addr, pmd_t *pmdp) in make_uffd_wp_pmd() argument
2350 old = pmdp_invalidate_ad(vma, addr, pmdp); in make_uffd_wp_pmd()
2352 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); in make_uffd_wp_pmd()
2355 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); in make_uffd_wp_pmd()
2392 unsigned long addr, pte_t *ptep, in make_uffd_wp_huge_pte() argument
2403 set_huge_pte_at(vma->vm_mm, addr, ptep, in make_uffd_wp_huge_pte()
2406 huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, in make_uffd_wp_huge_pte()
2409 set_huge_pte_at(vma->vm_mm, addr, ptep, in make_uffd_wp_huge_pte()
2416 unsigned long addr, unsigned long end) in pagemap_scan_backout_range() argument
2420 if (cur_buf->start != addr) in pagemap_scan_backout_range()
2421 cur_buf->end = addr; in pagemap_scan_backout_range()
2425 p->found_pages -= (end - addr) / PAGE_SIZE; in pagemap_scan_backout_range()
2497 unsigned long addr, unsigned long end) in pagemap_scan_push_range() argument
2506 if (addr == cur_buf->end && categories == cur_buf->categories) { in pagemap_scan_push_range()
2518 cur_buf->start = addr; in pagemap_scan_push_range()
2527 unsigned long addr, unsigned long *end) in pagemap_scan_output() argument
2537 n_pages = (*end - addr) / PAGE_SIZE; in pagemap_scan_output()
2546 if (!pagemap_scan_push_range(categories, p, addr, *end)) { in pagemap_scan_output()
2547 *end = addr; in pagemap_scan_output()
2615 unsigned long addr, flush_end = 0; in pagemap_scan_pmd_entry() local
2635 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { in pagemap_scan_pmd_entry()
2641 make_uffd_wp_pte(vma, addr, pte, ptent); in pagemap_scan_pmd_entry()
2643 start = addr; in pagemap_scan_pmd_entry()
2644 flush_end = addr + PAGE_SIZE; in pagemap_scan_pmd_entry()
2652 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { in pagemap_scan_pmd_entry()
2653 unsigned long next = addr + PAGE_SIZE; in pagemap_scan_pmd_entry()
2660 p, addr, &next); in pagemap_scan_pmd_entry()
2661 if (next == addr) in pagemap_scan_pmd_entry()
2665 make_uffd_wp_pte(vma, addr, pte, ptent); in pagemap_scan_pmd_entry()
2667 start = addr; in pagemap_scan_pmd_entry()
2673 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { in pagemap_scan_pmd_entry()
2676 pagemap_page_category(p, vma, addr, ptent); in pagemap_scan_pmd_entry()
2677 unsigned long next = addr + PAGE_SIZE; in pagemap_scan_pmd_entry()
2682 ret = pagemap_scan_output(categories, p, addr, &next); in pagemap_scan_pmd_entry()
2683 if (next == addr) in pagemap_scan_pmd_entry()
2691 make_uffd_wp_pte(vma, addr, pte, ptent); in pagemap_scan_pmd_entry()
2693 start = addr; in pagemap_scan_pmd_entry()
2699 flush_tlb_range(vma, start, addr); in pagemap_scan_pmd_entry()
2769 static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, in pagemap_scan_pte_hole() argument
2779 ret = pagemap_scan_output(p->cur_vma_category, p, addr, &end); in pagemap_scan_pte_hole()
2780 if (addr == end) in pagemap_scan_pte_hole()
2786 err = uffd_wp_range(vma, addr, end - addr, true); in pagemap_scan_pte_hole()
3047 unsigned long addr) in can_gather_numa_stats() argument
3055 page = vm_normal_page(vma, addr, pte); in can_gather_numa_stats()
3072 unsigned long addr) in can_gather_numa_stats_pmd() argument
3080 page = vm_normal_page_pmd(vma, addr, pmd); in can_gather_numa_stats_pmd()
3095 static int gather_pte_stats(pmd_t *pmd, unsigned long addr, in gather_pte_stats() argument
3109 page = can_gather_numa_stats_pmd(*pmd, vma, addr); in gather_pte_stats()
3117 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); in gather_pte_stats()
3124 struct page *page = can_gather_numa_stats(ptent, vma, addr); in gather_pte_stats()
3129 } while (pte++, addr += PAGE_SIZE, addr != end); in gather_pte_stats()
3136 unsigned long addr, unsigned long end, struct mm_walk *walk) in gather_hugetlb_stats() argument
3144 huge_pte = huge_ptep_get(walk->mm, addr, pte); in gather_hugetlb_stats()
3159 unsigned long addr, unsigned long end, struct mm_walk *walk) in gather_hugetlb_stats() argument