Lines Matching refs:tlb
224 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
228 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
233 static inline void tlb_remove_table(struct mmu_gather *tlb, void *table) in tlb_remove_table() argument
238 tlb_remove_page(tlb, ptdesc_page(ptdesc)); in tlb_remove_table()
289 extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
291 bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page,
299 #define tlb_delay_rmap(tlb) (((tlb)->delayed_rmap = 1), true) argument
300 extern void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma);
313 #define tlb_delay_rmap(tlb) (false) argument
314 static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) { } in tlb_flush_rmaps() argument
380 void tlb_flush_mmu(struct mmu_gather *tlb);
382 static inline void __tlb_adjust_range(struct mmu_gather *tlb, in __tlb_adjust_range() argument
386 tlb->start = min(tlb->start, address); in __tlb_adjust_range()
387 tlb->end = max(tlb->end, address + range_size); in __tlb_adjust_range()
390 static inline void __tlb_reset_range(struct mmu_gather *tlb) in __tlb_reset_range() argument
392 if (tlb->fullmm) { in __tlb_reset_range()
393 tlb->start = tlb->end = ~0; in __tlb_reset_range()
395 tlb->start = TASK_SIZE; in __tlb_reset_range()
396 tlb->end = 0; in __tlb_reset_range()
398 tlb->freed_tables = 0; in __tlb_reset_range()
399 tlb->cleared_ptes = 0; in __tlb_reset_range()
400 tlb->cleared_pmds = 0; in __tlb_reset_range()
401 tlb->cleared_puds = 0; in __tlb_reset_range()
402 tlb->cleared_p4ds = 0; in __tlb_reset_range()
424 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
426 if (tlb->end) in tlb_flush()
427 flush_tlb_mm(tlb->mm); in tlb_flush()
438 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
440 if (tlb->fullmm || tlb->need_flush_all) { in tlb_flush()
441 flush_tlb_mm(tlb->mm); in tlb_flush()
442 } else if (tlb->end) { in tlb_flush()
444 .vm_mm = tlb->mm, in tlb_flush()
445 .vm_flags = (tlb->vma_exec ? VM_EXEC : 0) | in tlb_flush()
446 (tlb->vma_huge ? VM_HUGETLB : 0), in tlb_flush()
449 flush_tlb_range(&vma, tlb->start, tlb->end); in tlb_flush()
457 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_update_vma_flags() argument
470 tlb->vma_huge = is_vm_hugetlb_page(vma); in tlb_update_vma_flags()
471 tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); in tlb_update_vma_flags()
477 tlb->vma_pfn |= !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)); in tlb_update_vma_flags()
480 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) in tlb_flush_mmu_tlbonly() argument
486 if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds || in tlb_flush_mmu_tlbonly()
487 tlb->cleared_puds || tlb->cleared_p4ds)) in tlb_flush_mmu_tlbonly()
490 tlb_flush(tlb); in tlb_flush_mmu_tlbonly()
491 __tlb_reset_range(tlb); in tlb_flush_mmu_tlbonly()
494 static inline void tlb_remove_page_size(struct mmu_gather *tlb, in tlb_remove_page_size() argument
497 if (__tlb_remove_page_size(tlb, page, false, page_size)) in tlb_remove_page_size()
498 tlb_flush_mmu(tlb); in tlb_remove_page_size()
501 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) in tlb_remove_page() argument
503 return tlb_remove_page_size(tlb, page, PAGE_SIZE); in tlb_remove_page()
506 static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) in tlb_remove_ptdesc() argument
508 tlb_remove_table(tlb, pt); in tlb_remove_ptdesc()
511 static inline void tlb_change_page_size(struct mmu_gather *tlb, in tlb_change_page_size() argument
515 if (tlb->page_size && tlb->page_size != page_size) { in tlb_change_page_size()
516 if (!tlb->fullmm && !tlb->need_flush_all) in tlb_change_page_size()
517 tlb_flush_mmu(tlb); in tlb_change_page_size()
520 tlb->page_size = page_size; in tlb_change_page_size()
524 static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) in tlb_get_unmap_shift() argument
526 if (tlb->cleared_ptes) in tlb_get_unmap_shift()
528 if (tlb->cleared_pmds) in tlb_get_unmap_shift()
530 if (tlb->cleared_puds) in tlb_get_unmap_shift()
532 if (tlb->cleared_p4ds) in tlb_get_unmap_shift()
538 static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) in tlb_get_unmap_size() argument
540 return 1UL << tlb_get_unmap_shift(tlb); in tlb_get_unmap_size()
548 static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_start_vma() argument
550 if (tlb->fullmm) in tlb_start_vma()
553 tlb_update_vma_flags(tlb, vma); in tlb_start_vma()
559 static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_end_vma() argument
561 if (tlb->fullmm || IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) in tlb_end_vma()
570 tlb_flush_mmu_tlbonly(tlb); in tlb_end_vma()
573 static inline void tlb_free_vmas(struct mmu_gather *tlb) in tlb_free_vmas() argument
575 if (tlb->fullmm) in tlb_free_vmas()
591 if (tlb->vma_pfn) in tlb_free_vmas()
592 tlb_flush_mmu_tlbonly(tlb); in tlb_free_vmas()
599 static inline void tlb_flush_pte_range(struct mmu_gather *tlb, in tlb_flush_pte_range() argument
602 __tlb_adjust_range(tlb, address, size); in tlb_flush_pte_range()
603 tlb->cleared_ptes = 1; in tlb_flush_pte_range()
606 static inline void tlb_flush_pmd_range(struct mmu_gather *tlb, in tlb_flush_pmd_range() argument
609 __tlb_adjust_range(tlb, address, size); in tlb_flush_pmd_range()
610 tlb->cleared_pmds = 1; in tlb_flush_pmd_range()
613 static inline void tlb_flush_pud_range(struct mmu_gather *tlb, in tlb_flush_pud_range() argument
616 __tlb_adjust_range(tlb, address, size); in tlb_flush_pud_range()
617 tlb->cleared_puds = 1; in tlb_flush_pud_range()
620 static inline void tlb_flush_p4d_range(struct mmu_gather *tlb, in tlb_flush_p4d_range() argument
623 __tlb_adjust_range(tlb, address, size); in tlb_flush_p4d_range()
624 tlb->cleared_p4ds = 1; in tlb_flush_p4d_range()
628 static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addres… in __tlb_remove_tlb_entry() argument
640 #define tlb_remove_tlb_entry(tlb, ptep, address) \ argument
642 tlb_flush_pte_range(tlb, address, PAGE_SIZE); \
643 __tlb_remove_tlb_entry(tlb, ptep, address); \
653 static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, in tlb_remove_tlb_entries() argument
656 tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr); in tlb_remove_tlb_entries()
658 __tlb_remove_tlb_entry(tlb, ptep, address); in tlb_remove_tlb_entries()
666 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ argument
670 tlb_flush_p4d_range(tlb, address, _sz); \
672 tlb_flush_pud_range(tlb, address, _sz); \
674 tlb_flush_pmd_range(tlb, address, _sz); \
676 tlb_flush_pte_range(tlb, address, _sz); \
677 __tlb_remove_tlb_entry(tlb, ptep, address); \
685 #define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0) argument
688 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \ argument
690 tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE); \
691 __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
699 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0) argument
702 #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ argument
704 tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE); \
705 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \
727 #define pte_free_tlb(tlb, ptep, address) \ argument
729 tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \
730 tlb->freed_tables = 1; \
731 __pte_free_tlb(tlb, ptep, address); \
736 #define pmd_free_tlb(tlb, pmdp, address) \ argument
738 tlb_flush_pud_range(tlb, address, PAGE_SIZE); \
739 tlb->freed_tables = 1; \
740 __pmd_free_tlb(tlb, pmdp, address); \
745 #define pud_free_tlb(tlb, pudp, address) \ argument
747 tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \
748 tlb->freed_tables = 1; \
749 __pud_free_tlb(tlb, pudp, address); \
754 #define p4d_free_tlb(tlb, pudp, address) \ argument
756 __tlb_adjust_range(tlb, address, PAGE_SIZE); \
757 tlb->freed_tables = 1; \
758 __p4d_free_tlb(tlb, pudp, address); \