Lines Matching refs:fbatch
53 struct folio_batch fbatch; member
203 static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) in folio_batch_move_lru() argument
209 for (i = 0; i < folio_batch_count(fbatch); i++) { in folio_batch_move_lru()
210 struct folio *folio = fbatch->folios[i]; in folio_batch_move_lru()
224 folios_put(fbatch->folios, folio_batch_count(fbatch)); in folio_batch_move_lru()
225 folio_batch_init(fbatch); in folio_batch_move_lru()
228 static void folio_batch_add_and_move(struct folio_batch *fbatch, in folio_batch_add_and_move() argument
231 if (folio_batch_add(fbatch, folio) && !folio_test_large(folio) && in folio_batch_add_and_move()
234 folio_batch_move_lru(fbatch, move_fn); in folio_batch_add_and_move()
258 struct folio_batch *fbatch; in folio_rotate_reclaimable() local
263 fbatch = this_cpu_ptr(&lru_rotate.fbatch); in folio_rotate_reclaimable()
264 folio_batch_add_and_move(fbatch, folio, lru_move_tail_fn); in folio_rotate_reclaimable()
346 struct folio_batch *fbatch = &per_cpu(cpu_fbatches.activate, cpu); in folio_activate_drain() local
348 if (folio_batch_count(fbatch)) in folio_activate_drain()
349 folio_batch_move_lru(fbatch, folio_activate_fn); in folio_activate_drain()
356 struct folio_batch *fbatch; in folio_activate() local
360 fbatch = this_cpu_ptr(&cpu_fbatches.activate); in folio_activate()
361 folio_batch_add_and_move(fbatch, folio, folio_activate_fn); in folio_activate()
386 struct folio_batch *fbatch; in __lru_cache_activate_folio() local
390 fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); in __lru_cache_activate_folio()
402 for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) { in __lru_cache_activate_folio()
403 struct folio *batch_folio = fbatch->folios[i]; in __lru_cache_activate_folio()
503 struct folio_batch *fbatch; in folio_add_lru() local
516 fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); in folio_add_lru()
517 folio_batch_add_and_move(fbatch, folio, lru_add_fn); in folio_add_lru()
649 struct folio_batch *fbatch = &fbatches->lru_add; in lru_add_drain_cpu() local
651 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
652 folio_batch_move_lru(fbatch, lru_add_fn); in lru_add_drain_cpu()
654 fbatch = &per_cpu(lru_rotate.fbatch, cpu); in lru_add_drain_cpu()
656 if (data_race(folio_batch_count(fbatch))) { in lru_add_drain_cpu()
661 folio_batch_move_lru(fbatch, lru_move_tail_fn); in lru_add_drain_cpu()
665 fbatch = &fbatches->lru_deactivate_file; in lru_add_drain_cpu()
666 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
667 folio_batch_move_lru(fbatch, lru_deactivate_file_fn); in lru_add_drain_cpu()
669 fbatch = &fbatches->lru_deactivate; in lru_add_drain_cpu()
670 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
671 folio_batch_move_lru(fbatch, lru_deactivate_fn); in lru_add_drain_cpu()
673 fbatch = &fbatches->lru_lazyfree; in lru_add_drain_cpu()
674 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
675 folio_batch_move_lru(fbatch, lru_lazyfree_fn); in lru_add_drain_cpu()
692 struct folio_batch *fbatch; in deactivate_file_folio() local
700 fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate_file); in deactivate_file_folio()
701 folio_batch_add_and_move(fbatch, folio, lru_deactivate_file_fn); in deactivate_file_folio()
717 struct folio_batch *fbatch; in folio_deactivate() local
721 fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate); in folio_deactivate()
722 folio_batch_add_and_move(fbatch, folio, lru_deactivate_fn); in folio_deactivate()
739 struct folio_batch *fbatch; in folio_mark_lazyfree() local
743 fbatch = this_cpu_ptr(&cpu_fbatches.lru_lazyfree); in folio_mark_lazyfree()
744 folio_batch_add_and_move(fbatch, folio, lru_lazyfree_fn); in folio_mark_lazyfree()
796 data_race(folio_batch_count(&per_cpu(lru_rotate.fbatch, cpu))) || in cpu_needs_drain()
1076 void folio_batch_remove_exceptionals(struct folio_batch *fbatch) in folio_batch_remove_exceptionals() argument
1080 for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) { in folio_batch_remove_exceptionals()
1081 struct folio *folio = fbatch->folios[i]; in folio_batch_remove_exceptionals()
1083 fbatch->folios[j++] = folio; in folio_batch_remove_exceptionals()
1085 fbatch->nr = j; in folio_batch_remove_exceptionals()