Lines Matching refs:tlb
18 static bool tlb_next_batch(struct mmu_gather *tlb) in tlb_next_batch() argument
23 if (tlb->delayed_rmap && tlb->active != &tlb->local) in tlb_next_batch()
26 batch = tlb->active; in tlb_next_batch()
28 tlb->active = batch->next; in tlb_next_batch()
32 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) in tlb_next_batch()
39 tlb->batch_count++; in tlb_next_batch()
44 tlb->active->next = batch; in tlb_next_batch()
45 tlb->active = batch; in tlb_next_batch()
72 void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_flush_rmaps() argument
74 if (!tlb->delayed_rmap) in tlb_flush_rmaps()
77 tlb_flush_rmap_batch(&tlb->local, vma); in tlb_flush_rmaps()
78 if (tlb->active != &tlb->local) in tlb_flush_rmaps()
79 tlb_flush_rmap_batch(tlb->active, vma); in tlb_flush_rmaps()
80 tlb->delayed_rmap = 0; in tlb_flush_rmaps()
84 static void tlb_batch_pages_flush(struct mmu_gather *tlb) in tlb_batch_pages_flush() argument
88 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { in tlb_batch_pages_flush()
104 tlb->active = &tlb->local; in tlb_batch_pages_flush()
107 static void tlb_batch_list_free(struct mmu_gather *tlb) in tlb_batch_list_free() argument
111 for (batch = tlb->local.next; batch; batch = next) { in tlb_batch_list_free()
115 tlb->local.next = NULL; in tlb_batch_list_free()
118 bool __tlb_remove_page_size(struct mmu_gather *tlb, struct encoded_page *page, int page_size) in __tlb_remove_page_size() argument
122 VM_BUG_ON(!tlb->end); in __tlb_remove_page_size()
125 VM_WARN_ON(tlb->page_size != page_size); in __tlb_remove_page_size()
128 batch = tlb->active; in __tlb_remove_page_size()
135 if (!tlb_next_batch(tlb)) in __tlb_remove_page_size()
137 batch = tlb->active; in __tlb_remove_page_size()
228 static inline void tlb_table_invalidate(struct mmu_gather *tlb) in tlb_table_invalidate() argument
236 tlb_flush_mmu_tlbonly(tlb); in tlb_table_invalidate()
246 static void tlb_table_flush(struct mmu_gather *tlb) in tlb_table_flush() argument
248 struct mmu_table_batch **batch = &tlb->batch; in tlb_table_flush()
251 tlb_table_invalidate(tlb); in tlb_table_flush()
257 void tlb_remove_table(struct mmu_gather *tlb, void *table) in tlb_remove_table() argument
259 struct mmu_table_batch **batch = &tlb->batch; in tlb_remove_table()
264 tlb_table_invalidate(tlb); in tlb_remove_table()
273 tlb_table_flush(tlb); in tlb_remove_table()
276 static inline void tlb_table_init(struct mmu_gather *tlb) in tlb_table_init() argument
278 tlb->batch = NULL; in tlb_table_init()
283 static inline void tlb_table_flush(struct mmu_gather *tlb) { } in tlb_table_flush() argument
284 static inline void tlb_table_init(struct mmu_gather *tlb) { } in tlb_table_init() argument
288 static void tlb_flush_mmu_free(struct mmu_gather *tlb) in tlb_flush_mmu_free() argument
290 tlb_table_flush(tlb); in tlb_flush_mmu_free()
292 tlb_batch_pages_flush(tlb); in tlb_flush_mmu_free()
296 void tlb_flush_mmu(struct mmu_gather *tlb) in tlb_flush_mmu() argument
298 tlb_flush_mmu_tlbonly(tlb); in tlb_flush_mmu()
299 tlb_flush_mmu_free(tlb); in tlb_flush_mmu()
302 static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, in __tlb_gather_mmu() argument
305 tlb->mm = mm; in __tlb_gather_mmu()
306 tlb->fullmm = fullmm; in __tlb_gather_mmu()
309 tlb->need_flush_all = 0; in __tlb_gather_mmu()
310 tlb->local.next = NULL; in __tlb_gather_mmu()
311 tlb->local.nr = 0; in __tlb_gather_mmu()
312 tlb->local.max = ARRAY_SIZE(tlb->__pages); in __tlb_gather_mmu()
313 tlb->active = &tlb->local; in __tlb_gather_mmu()
314 tlb->batch_count = 0; in __tlb_gather_mmu()
316 tlb->delayed_rmap = 0; in __tlb_gather_mmu()
318 tlb_table_init(tlb); in __tlb_gather_mmu()
320 tlb->page_size = 0; in __tlb_gather_mmu()
323 __tlb_reset_range(tlb); in __tlb_gather_mmu()
324 inc_tlb_flush_pending(tlb->mm); in __tlb_gather_mmu()
335 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm) in tlb_gather_mmu() argument
337 __tlb_gather_mmu(tlb, mm, false); in tlb_gather_mmu()
351 void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) in tlb_gather_mmu_fullmm() argument
353 __tlb_gather_mmu(tlb, mm, true); in tlb_gather_mmu_fullmm()
363 void tlb_finish_mmu(struct mmu_gather *tlb) in tlb_finish_mmu() argument
377 if (mm_tlb_flush_nested(tlb->mm)) { in tlb_finish_mmu()
386 tlb->fullmm = 1; in tlb_finish_mmu()
387 __tlb_reset_range(tlb); in tlb_finish_mmu()
388 tlb->freed_tables = 1; in tlb_finish_mmu()
391 tlb_flush_mmu(tlb); in tlb_finish_mmu()
394 tlb_batch_list_free(tlb); in tlb_finish_mmu()
396 dec_tlb_flush_pending(tlb->mm); in tlb_finish_mmu()