Lines Matching refs:batch
20 struct mmu_gather_batch *batch; in tlb_next_batch() local
26 batch = tlb->active; in tlb_next_batch()
27 if (batch->next) { in tlb_next_batch()
28 tlb->active = batch->next; in tlb_next_batch()
35 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); in tlb_next_batch()
36 if (!batch) in tlb_next_batch()
40 batch->next = NULL; in tlb_next_batch()
41 batch->nr = 0; in tlb_next_batch()
42 batch->max = MAX_GATHER_BATCH; in tlb_next_batch()
44 tlb->active->next = batch; in tlb_next_batch()
45 tlb->active = batch; in tlb_next_batch()
51 static void tlb_flush_rmap_batch(struct mmu_gather_batch *batch, struct vm_area_struct *vma) in tlb_flush_rmap_batch() argument
53 for (int i = 0; i < batch->nr; i++) { in tlb_flush_rmap_batch()
54 struct encoded_page *enc = batch->encoded_pages[i]; in tlb_flush_rmap_batch()
86 struct mmu_gather_batch *batch; in tlb_batch_pages_flush() local
88 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { in tlb_batch_pages_flush()
89 struct encoded_page **pages = batch->encoded_pages; in tlb_batch_pages_flush()
95 unsigned int nr = min(512U, batch->nr); in tlb_batch_pages_flush()
99 batch->nr -= nr; in tlb_batch_pages_flush()
102 } while (batch->nr); in tlb_batch_pages_flush()
109 struct mmu_gather_batch *batch, *next; in tlb_batch_list_free() local
111 for (batch = tlb->local.next; batch; batch = next) { in tlb_batch_list_free()
112 next = batch->next; in tlb_batch_list_free()
113 free_pages((unsigned long)batch, 0); in tlb_batch_list_free()
120 struct mmu_gather_batch *batch; in __tlb_remove_page_size() local
128 batch = tlb->active; in __tlb_remove_page_size()
133 batch->encoded_pages[batch->nr++] = page; in __tlb_remove_page_size()
134 if (batch->nr == batch->max) { in __tlb_remove_page_size()
137 batch = tlb->active; in __tlb_remove_page_size()
139 VM_BUG_ON_PAGE(batch->nr > batch->max, encoded_page_ptr(page)); in __tlb_remove_page_size()
148 static void __tlb_remove_table_free(struct mmu_table_batch *batch) in __tlb_remove_table_free() argument
152 for (i = 0; i < batch->nr; i++) in __tlb_remove_table_free()
153 __tlb_remove_table(batch->tables[i]); in __tlb_remove_table_free()
155 free_page((unsigned long)batch); in __tlb_remove_table_free()
211 static void tlb_remove_table_free(struct mmu_table_batch *batch) in tlb_remove_table_free() argument
213 call_rcu(&batch->rcu, tlb_remove_table_rcu); in tlb_remove_table_free()
218 static void tlb_remove_table_free(struct mmu_table_batch *batch) in tlb_remove_table_free() argument
220 __tlb_remove_table_free(batch); in tlb_remove_table_free()
248 struct mmu_table_batch **batch = &tlb->batch; in tlb_table_flush() local
250 if (*batch) { in tlb_table_flush()
252 tlb_remove_table_free(*batch); in tlb_table_flush()
253 *batch = NULL; in tlb_table_flush()
259 struct mmu_table_batch **batch = &tlb->batch; in tlb_remove_table() local
261 if (*batch == NULL) { in tlb_remove_table()
262 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); in tlb_remove_table()
263 if (*batch == NULL) { in tlb_remove_table()
268 (*batch)->nr = 0; in tlb_remove_table()
271 (*batch)->tables[(*batch)->nr++] = table; in tlb_remove_table()
272 if ((*batch)->nr == MAX_TABLE_BATCH) in tlb_remove_table()
278 tlb->batch = NULL; in tlb_table_init()