Lines Matching refs:chunk
219 static bool pcpu_addr_in_chunk(struct pcpu_chunk *chunk, void *addr) in pcpu_addr_in_chunk() argument
223 if (!chunk) in pcpu_addr_in_chunk()
226 start_addr = chunk->base_addr + chunk->start_offset; in pcpu_addr_in_chunk()
227 end_addr = chunk->base_addr + chunk->nr_pages * PAGE_SIZE - in pcpu_addr_in_chunk()
228 chunk->end_offset; in pcpu_addr_in_chunk()
246 static int pcpu_chunk_slot(const struct pcpu_chunk *chunk) in pcpu_chunk_slot() argument
248 const struct pcpu_block_md *chunk_md = &chunk->chunk_md; in pcpu_chunk_slot()
250 if (chunk->free_bytes < PCPU_MIN_ALLOC_SIZE || in pcpu_chunk_slot()
279 static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, in pcpu_chunk_addr() argument
282 return (unsigned long)chunk->base_addr + in pcpu_chunk_addr()
290 static unsigned long *pcpu_index_alloc_map(struct pcpu_chunk *chunk, int index) in pcpu_index_alloc_map() argument
292 return chunk->alloc_map + in pcpu_index_alloc_map()
369 static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off, in pcpu_next_md_free_region() argument
377 for (block = chunk->md_blocks + i; i < pcpu_chunk_nr_blocks(chunk); in pcpu_next_md_free_region()
424 static void pcpu_next_fit_region(struct pcpu_chunk *chunk, int alloc_bits, in pcpu_next_fit_region() argument
432 for (block = chunk->md_blocks + i; i < pcpu_chunk_nr_blocks(chunk); in pcpu_next_fit_region()
472 *bit_off = pcpu_chunk_map_bits(chunk); in pcpu_next_fit_region()
481 #define pcpu_for_each_md_free_region(chunk, bit_off, bits) \ argument
482 for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits)); \
483 (bit_off) < pcpu_chunk_map_bits((chunk)); \
485 pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
487 #define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) \ argument
488 for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
490 (bit_off) < pcpu_chunk_map_bits((chunk)); \
492 pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
530 static void __pcpu_chunk_move(struct pcpu_chunk *chunk, int slot, in __pcpu_chunk_move() argument
533 if (chunk != pcpu_reserved_chunk) { in __pcpu_chunk_move()
535 list_move(&chunk->list, &pcpu_chunk_lists[slot]); in __pcpu_chunk_move()
537 list_move_tail(&chunk->list, &pcpu_chunk_lists[slot]); in __pcpu_chunk_move()
541 static void pcpu_chunk_move(struct pcpu_chunk *chunk, int slot) in pcpu_chunk_move() argument
543 __pcpu_chunk_move(chunk, slot, true); in pcpu_chunk_move()
559 static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) in pcpu_chunk_relocate() argument
561 int nslot = pcpu_chunk_slot(chunk); in pcpu_chunk_relocate()
564 if (chunk->isolated) in pcpu_chunk_relocate()
568 __pcpu_chunk_move(chunk, nslot, oslot < nslot); in pcpu_chunk_relocate()
571 static void pcpu_isolate_chunk(struct pcpu_chunk *chunk) in pcpu_isolate_chunk() argument
575 if (!chunk->isolated) { in pcpu_isolate_chunk()
576 chunk->isolated = true; in pcpu_isolate_chunk()
577 pcpu_nr_empty_pop_pages -= chunk->nr_empty_pop_pages; in pcpu_isolate_chunk()
579 list_move(&chunk->list, &pcpu_chunk_lists[pcpu_to_depopulate_slot]); in pcpu_isolate_chunk()
582 static void pcpu_reintegrate_chunk(struct pcpu_chunk *chunk) in pcpu_reintegrate_chunk() argument
586 if (chunk->isolated) { in pcpu_reintegrate_chunk()
587 chunk->isolated = false; in pcpu_reintegrate_chunk()
588 pcpu_nr_empty_pop_pages += chunk->nr_empty_pop_pages; in pcpu_reintegrate_chunk()
589 pcpu_chunk_relocate(chunk, -1); in pcpu_reintegrate_chunk()
602 static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr) in pcpu_update_empty_pages() argument
604 chunk->nr_empty_pop_pages += nr; in pcpu_update_empty_pages()
605 if (chunk != pcpu_reserved_chunk && !chunk->isolated) in pcpu_update_empty_pages()
716 static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off, in pcpu_block_update_scan() argument
728 block = chunk->md_blocks + s_index; in pcpu_block_update_scan()
731 l_bit = find_last_bit(pcpu_index_alloc_map(chunk, s_index), s_off); in pcpu_block_update_scan()
749 static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan) in pcpu_chunk_refresh_hint() argument
751 struct pcpu_block_md *chunk_md = &chunk->chunk_md; in pcpu_chunk_refresh_hint()
766 pcpu_for_each_md_free_region(chunk, bit_off, bits) in pcpu_chunk_refresh_hint()
778 static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) in pcpu_block_refresh_hint() argument
780 struct pcpu_block_md *block = chunk->md_blocks + index; in pcpu_block_refresh_hint()
781 unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index); in pcpu_block_refresh_hint()
813 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, in pcpu_block_update_hint_alloc() argument
816 struct pcpu_block_md *chunk_md = &chunk->chunk_md; in pcpu_block_update_hint_alloc()
833 s_block = chunk->md_blocks + s_index; in pcpu_block_update_hint_alloc()
834 e_block = chunk->md_blocks + e_index; in pcpu_block_update_hint_alloc()
847 pcpu_index_alloc_map(chunk, s_index), in pcpu_block_update_hint_alloc()
865 pcpu_block_refresh_hint(chunk, s_index); in pcpu_block_update_hint_alloc()
888 pcpu_index_alloc_map(chunk, e_index), in pcpu_block_update_hint_alloc()
901 pcpu_block_refresh_hint(chunk, e_index); in pcpu_block_update_hint_alloc()
920 pcpu_update_empty_pages(chunk, -nr_empty_pages); in pcpu_block_update_hint_alloc()
939 pcpu_chunk_refresh_hint(chunk, false); in pcpu_block_update_hint_alloc()
960 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, in pcpu_block_update_hint_free() argument
980 s_block = chunk->md_blocks + s_index; in pcpu_block_update_hint_free()
981 e_block = chunk->md_blocks + e_index; in pcpu_block_update_hint_free()
1003 int l_bit = find_last_bit(pcpu_index_alloc_map(chunk, s_index), in pcpu_block_update_hint_free()
1012 end = find_next_bit(pcpu_index_alloc_map(chunk, e_index), in pcpu_block_update_hint_free()
1041 pcpu_update_empty_pages(chunk, nr_empty_pages); in pcpu_block_update_hint_free()
1050 pcpu_chunk_refresh_hint(chunk, true); in pcpu_block_update_hint_free()
1052 pcpu_block_update(&chunk->chunk_md, in pcpu_block_update_hint_free()
1070 static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits, in pcpu_is_populated() argument
1079 bitmap_next_clear_region(chunk->populated, &rs, &re, page_end); in pcpu_is_populated()
1106 static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits, in pcpu_find_block_fit() argument
1109 struct pcpu_block_md *chunk_md = &chunk->chunk_md; in pcpu_find_block_fit()
1122 pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) { in pcpu_find_block_fit()
1123 if (!pop_only || pcpu_is_populated(chunk, bit_off, bits, in pcpu_find_block_fit()
1131 if (bit_off == pcpu_chunk_map_bits(chunk)) in pcpu_find_block_fit()
1212 static int pcpu_alloc_area(struct pcpu_chunk *chunk, int alloc_bits, in pcpu_alloc_area() argument
1215 struct pcpu_block_md *chunk_md = &chunk->chunk_md; in pcpu_alloc_area()
1222 oslot = pcpu_chunk_slot(chunk); in pcpu_alloc_area()
1228 pcpu_chunk_map_bits(chunk)); in pcpu_alloc_area()
1229 bit_off = pcpu_find_zero_area(chunk->alloc_map, end, start, alloc_bits, in pcpu_alloc_area()
1235 pcpu_block_update_scan(chunk, area_off, area_bits); in pcpu_alloc_area()
1238 bitmap_set(chunk->alloc_map, bit_off, alloc_bits); in pcpu_alloc_area()
1241 set_bit(bit_off, chunk->bound_map); in pcpu_alloc_area()
1242 bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1); in pcpu_alloc_area()
1243 set_bit(bit_off + alloc_bits, chunk->bound_map); in pcpu_alloc_area()
1245 chunk->free_bytes -= alloc_bits * PCPU_MIN_ALLOC_SIZE; in pcpu_alloc_area()
1250 chunk->alloc_map, in pcpu_alloc_area()
1251 pcpu_chunk_map_bits(chunk), in pcpu_alloc_area()
1254 pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits); in pcpu_alloc_area()
1256 pcpu_chunk_relocate(chunk, oslot); in pcpu_alloc_area()
1272 static int pcpu_free_area(struct pcpu_chunk *chunk, int off) in pcpu_free_area() argument
1274 struct pcpu_block_md *chunk_md = &chunk->chunk_md; in pcpu_free_area()
1278 pcpu_stats_area_dealloc(chunk); in pcpu_free_area()
1280 oslot = pcpu_chunk_slot(chunk); in pcpu_free_area()
1285 end = find_next_bit(chunk->bound_map, pcpu_chunk_map_bits(chunk), in pcpu_free_area()
1288 bitmap_clear(chunk->alloc_map, bit_off, bits); in pcpu_free_area()
1293 chunk->free_bytes += freed; in pcpu_free_area()
1298 pcpu_block_update_hint_free(chunk, bit_off, bits); in pcpu_free_area()
1300 pcpu_chunk_relocate(chunk, oslot); in pcpu_free_area()
1315 static void pcpu_init_md_blocks(struct pcpu_chunk *chunk) in pcpu_init_md_blocks() argument
1320 pcpu_init_md_block(&chunk->chunk_md, pcpu_chunk_map_bits(chunk)); in pcpu_init_md_blocks()
1322 for (md_block = chunk->md_blocks; in pcpu_init_md_blocks()
1323 md_block != chunk->md_blocks + pcpu_chunk_nr_blocks(chunk); in pcpu_init_md_blocks()
1344 struct pcpu_chunk *chunk; in pcpu_alloc_first_chunk() local
1363 alloc_size = struct_size(chunk, populated, in pcpu_alloc_first_chunk()
1365 chunk = memblock_alloc(alloc_size, SMP_CACHE_BYTES); in pcpu_alloc_first_chunk()
1366 if (!chunk) in pcpu_alloc_first_chunk()
1370 INIT_LIST_HEAD(&chunk->list); in pcpu_alloc_first_chunk()
1372 chunk->base_addr = (void *)aligned_addr; in pcpu_alloc_first_chunk()
1373 chunk->start_offset = start_offset; in pcpu_alloc_first_chunk()
1374 chunk->end_offset = region_size - chunk->start_offset - map_size; in pcpu_alloc_first_chunk()
1376 chunk->nr_pages = region_size >> PAGE_SHIFT; in pcpu_alloc_first_chunk()
1377 region_bits = pcpu_chunk_map_bits(chunk); in pcpu_alloc_first_chunk()
1379 alloc_size = BITS_TO_LONGS(region_bits) * sizeof(chunk->alloc_map[0]); in pcpu_alloc_first_chunk()
1380 chunk->alloc_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES); in pcpu_alloc_first_chunk()
1381 if (!chunk->alloc_map) in pcpu_alloc_first_chunk()
1386 BITS_TO_LONGS(region_bits + 1) * sizeof(chunk->bound_map[0]); in pcpu_alloc_first_chunk()
1387 chunk->bound_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES); in pcpu_alloc_first_chunk()
1388 if (!chunk->bound_map) in pcpu_alloc_first_chunk()
1392 alloc_size = pcpu_chunk_nr_blocks(chunk) * sizeof(chunk->md_blocks[0]); in pcpu_alloc_first_chunk()
1393 chunk->md_blocks = memblock_alloc(alloc_size, SMP_CACHE_BYTES); in pcpu_alloc_first_chunk()
1394 if (!chunk->md_blocks) in pcpu_alloc_first_chunk()
1400 chunk->obj_cgroups = NULL; in pcpu_alloc_first_chunk()
1402 pcpu_init_md_blocks(chunk); in pcpu_alloc_first_chunk()
1405 chunk->immutable = true; in pcpu_alloc_first_chunk()
1406 bitmap_fill(chunk->populated, chunk->nr_pages); in pcpu_alloc_first_chunk()
1407 chunk->nr_populated = chunk->nr_pages; in pcpu_alloc_first_chunk()
1408 chunk->nr_empty_pop_pages = chunk->nr_pages; in pcpu_alloc_first_chunk()
1410 chunk->free_bytes = map_size; in pcpu_alloc_first_chunk()
1412 if (chunk->start_offset) { in pcpu_alloc_first_chunk()
1414 offset_bits = chunk->start_offset / PCPU_MIN_ALLOC_SIZE; in pcpu_alloc_first_chunk()
1415 bitmap_set(chunk->alloc_map, 0, offset_bits); in pcpu_alloc_first_chunk()
1416 set_bit(0, chunk->bound_map); in pcpu_alloc_first_chunk()
1417 set_bit(offset_bits, chunk->bound_map); in pcpu_alloc_first_chunk()
1419 chunk->chunk_md.first_free = offset_bits; in pcpu_alloc_first_chunk()
1421 pcpu_block_update_hint_alloc(chunk, 0, offset_bits); in pcpu_alloc_first_chunk()
1424 if (chunk->end_offset) { in pcpu_alloc_first_chunk()
1426 offset_bits = chunk->end_offset / PCPU_MIN_ALLOC_SIZE; in pcpu_alloc_first_chunk()
1427 bitmap_set(chunk->alloc_map, in pcpu_alloc_first_chunk()
1428 pcpu_chunk_map_bits(chunk) - offset_bits, in pcpu_alloc_first_chunk()
1431 chunk->bound_map); in pcpu_alloc_first_chunk()
1432 set_bit(region_bits, chunk->bound_map); in pcpu_alloc_first_chunk()
1434 pcpu_block_update_hint_alloc(chunk, pcpu_chunk_map_bits(chunk) in pcpu_alloc_first_chunk()
1438 return chunk; in pcpu_alloc_first_chunk()
1443 struct pcpu_chunk *chunk; in pcpu_alloc_chunk() local
1446 chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size, gfp); in pcpu_alloc_chunk()
1447 if (!chunk) in pcpu_alloc_chunk()
1450 INIT_LIST_HEAD(&chunk->list); in pcpu_alloc_chunk()
1451 chunk->nr_pages = pcpu_unit_pages; in pcpu_alloc_chunk()
1452 region_bits = pcpu_chunk_map_bits(chunk); in pcpu_alloc_chunk()
1454 chunk->alloc_map = pcpu_mem_zalloc(BITS_TO_LONGS(region_bits) * in pcpu_alloc_chunk()
1455 sizeof(chunk->alloc_map[0]), gfp); in pcpu_alloc_chunk()
1456 if (!chunk->alloc_map) in pcpu_alloc_chunk()
1459 chunk->bound_map = pcpu_mem_zalloc(BITS_TO_LONGS(region_bits + 1) * in pcpu_alloc_chunk()
1460 sizeof(chunk->bound_map[0]), gfp); in pcpu_alloc_chunk()
1461 if (!chunk->bound_map) in pcpu_alloc_chunk()
1464 chunk->md_blocks = pcpu_mem_zalloc(pcpu_chunk_nr_blocks(chunk) * in pcpu_alloc_chunk()
1465 sizeof(chunk->md_blocks[0]), gfp); in pcpu_alloc_chunk()
1466 if (!chunk->md_blocks) in pcpu_alloc_chunk()
1471 chunk->obj_cgroups = in pcpu_alloc_chunk()
1472 pcpu_mem_zalloc(pcpu_chunk_map_bits(chunk) * in pcpu_alloc_chunk()
1474 if (!chunk->obj_cgroups) in pcpu_alloc_chunk()
1479 pcpu_init_md_blocks(chunk); in pcpu_alloc_chunk()
1482 chunk->free_bytes = chunk->nr_pages * PAGE_SIZE; in pcpu_alloc_chunk()
1484 return chunk; in pcpu_alloc_chunk()
1488 pcpu_mem_free(chunk->md_blocks); in pcpu_alloc_chunk()
1491 pcpu_mem_free(chunk->bound_map); in pcpu_alloc_chunk()
1493 pcpu_mem_free(chunk->alloc_map); in pcpu_alloc_chunk()
1495 pcpu_mem_free(chunk); in pcpu_alloc_chunk()
1500 static void pcpu_free_chunk(struct pcpu_chunk *chunk) in pcpu_free_chunk() argument
1502 if (!chunk) in pcpu_free_chunk()
1505 pcpu_mem_free(chunk->obj_cgroups); in pcpu_free_chunk()
1507 pcpu_mem_free(chunk->md_blocks); in pcpu_free_chunk()
1508 pcpu_mem_free(chunk->bound_map); in pcpu_free_chunk()
1509 pcpu_mem_free(chunk->alloc_map); in pcpu_free_chunk()
1510 pcpu_mem_free(chunk); in pcpu_free_chunk()
1523 static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start, in pcpu_chunk_populated() argument
1530 bitmap_set(chunk->populated, page_start, nr); in pcpu_chunk_populated()
1531 chunk->nr_populated += nr; in pcpu_chunk_populated()
1534 pcpu_update_empty_pages(chunk, nr); in pcpu_chunk_populated()
1547 static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk, in pcpu_chunk_depopulated() argument
1554 bitmap_clear(chunk->populated, page_start, nr); in pcpu_chunk_depopulated()
1555 chunk->nr_populated -= nr; in pcpu_chunk_depopulated()
1558 pcpu_update_empty_pages(chunk, -nr); in pcpu_chunk_depopulated()
1577 static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
1579 static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
1581 static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
1584 static void pcpu_destroy_chunk(struct pcpu_chunk *chunk);
1648 struct pcpu_chunk *chunk, int off, in pcpu_memcg_post_alloc_hook() argument
1654 if (likely(chunk && chunk->obj_cgroups)) { in pcpu_memcg_post_alloc_hook()
1655 chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = objcg; in pcpu_memcg_post_alloc_hook()
1667 static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size) in pcpu_memcg_free_hook() argument
1671 if (unlikely(!chunk->obj_cgroups)) in pcpu_memcg_free_hook()
1674 objcg = chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT]; in pcpu_memcg_free_hook()
1677 chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = NULL; in pcpu_memcg_free_hook()
1697 struct pcpu_chunk *chunk, int off, in pcpu_memcg_post_alloc_hook() argument
1702 static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size) in pcpu_memcg_free_hook() argument
1730 struct pcpu_chunk *chunk, *next; in pcpu_alloc() local
1784 chunk = pcpu_reserved_chunk; in pcpu_alloc()
1786 off = pcpu_find_block_fit(chunk, bits, bit_align, is_atomic); in pcpu_alloc()
1792 off = pcpu_alloc_area(chunk, bits, bit_align, off); in pcpu_alloc()
1803 list_for_each_entry_safe(chunk, next, &pcpu_chunk_lists[slot], in pcpu_alloc()
1805 off = pcpu_find_block_fit(chunk, bits, bit_align, in pcpu_alloc()
1809 pcpu_chunk_move(chunk, 0); in pcpu_alloc()
1813 off = pcpu_alloc_area(chunk, bits, bit_align, off); in pcpu_alloc()
1815 pcpu_reintegrate_chunk(chunk); in pcpu_alloc()
1834 chunk = pcpu_create_chunk(pcpu_gfp); in pcpu_alloc()
1835 if (!chunk) { in pcpu_alloc()
1841 pcpu_chunk_relocate(chunk, -1); in pcpu_alloc()
1849 pcpu_stats_area_alloc(chunk, size); in pcpu_alloc()
1859 bitmap_for_each_clear_region(chunk->populated, rs, re, in pcpu_alloc()
1861 WARN_ON(chunk->immutable); in pcpu_alloc()
1863 ret = pcpu_populate_chunk(chunk, rs, re, pcpu_gfp); in pcpu_alloc()
1867 pcpu_free_area(chunk, off); in pcpu_alloc()
1871 pcpu_chunk_populated(chunk, rs, re); in pcpu_alloc()
1883 memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size); in pcpu_alloc()
1885 ptr = __addr_to_pcpu_ptr(chunk->base_addr + off); in pcpu_alloc()
1889 chunk->base_addr, off, ptr); in pcpu_alloc()
1891 pcpu_memcg_post_alloc_hook(objcg, chunk, off, size); in pcpu_alloc()
1990 struct pcpu_chunk *chunk, *next; in pcpu_balance_free() local
1998 list_for_each_entry_safe(chunk, next, free_head, list) { in pcpu_balance_free()
1999 WARN_ON(chunk->immutable); in pcpu_balance_free()
2002 if (chunk == list_first_entry(free_head, struct pcpu_chunk, list)) in pcpu_balance_free()
2005 if (!empty_only || chunk->nr_empty_pop_pages == 0) in pcpu_balance_free()
2006 list_move(&chunk->list, &to_free); in pcpu_balance_free()
2013 list_for_each_entry_safe(chunk, next, &to_free, list) { in pcpu_balance_free()
2016 bitmap_for_each_set_region(chunk->populated, rs, re, 0, in pcpu_balance_free()
2017 chunk->nr_pages) { in pcpu_balance_free()
2018 pcpu_depopulate_chunk(chunk, rs, re); in pcpu_balance_free()
2020 pcpu_chunk_depopulated(chunk, rs, re); in pcpu_balance_free()
2023 pcpu_destroy_chunk(chunk); in pcpu_balance_free()
2045 struct pcpu_chunk *chunk; in pcpu_balance_populated() local
2077 list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) { in pcpu_balance_populated()
2078 nr_unpop = chunk->nr_pages - chunk->nr_populated; in pcpu_balance_populated()
2087 bitmap_for_each_clear_region(chunk->populated, rs, re, 0, in pcpu_balance_populated()
2088 chunk->nr_pages) { in pcpu_balance_populated()
2092 ret = pcpu_populate_chunk(chunk, rs, rs + nr, gfp); in pcpu_balance_populated()
2097 pcpu_chunk_populated(chunk, rs, rs + nr); in pcpu_balance_populated()
2110 chunk = pcpu_create_chunk(gfp); in pcpu_balance_populated()
2113 if (chunk) { in pcpu_balance_populated()
2114 pcpu_chunk_relocate(chunk, -1); in pcpu_balance_populated()
2137 struct pcpu_chunk *chunk; in pcpu_reclaim_populated() local
2152 chunk = list_first_entry(&pcpu_chunk_lists[pcpu_to_depopulate_slot], in pcpu_reclaim_populated()
2154 WARN_ON(chunk->immutable); in pcpu_reclaim_populated()
2160 freed_page_start = chunk->nr_pages; in pcpu_reclaim_populated()
2163 for (i = chunk->nr_pages - 1, end = -1; i >= 0; i--) { in pcpu_reclaim_populated()
2165 if (chunk->nr_empty_pop_pages == 0) in pcpu_reclaim_populated()
2180 block = chunk->md_blocks + i; in pcpu_reclaim_populated()
2182 test_bit(i, chunk->populated)) { in pcpu_reclaim_populated()
2195 pcpu_depopulate_chunk(chunk, i + 1, end + 1); in pcpu_reclaim_populated()
2199 pcpu_chunk_depopulated(chunk, i + 1, end + 1); in pcpu_reclaim_populated()
2211 pcpu_post_unmap_tlb_flush(chunk, in pcpu_reclaim_populated()
2218 if (reintegrate || chunk->free_bytes == pcpu_unit_size) in pcpu_reclaim_populated()
2219 pcpu_reintegrate_chunk(chunk); in pcpu_reclaim_populated()
2221 list_move_tail(&chunk->list, in pcpu_reclaim_populated()
2267 struct pcpu_chunk *chunk; in free_percpu() local
2281 chunk = pcpu_chunk_addr_search(addr); in free_percpu()
2282 off = addr - chunk->base_addr; in free_percpu()
2284 size = pcpu_free_area(chunk, off); in free_percpu()
2286 pcpu_memcg_free_hook(chunk, off, size); in free_percpu()
2293 if (!chunk->isolated && chunk->free_bytes == pcpu_unit_size) { in free_percpu()
2297 if (pos != chunk) { in free_percpu()
2301 } else if (pcpu_should_reclaim_chunk(chunk)) { in free_percpu()
2302 pcpu_isolate_chunk(chunk); in free_percpu()
2306 trace_percpu_free_percpu(chunk->base_addr, off, ptr); in free_percpu()
2594 struct pcpu_chunk *chunk; in pcpu_setup_first_chunk() local
2711 pcpu_chunk_struct_size = struct_size(chunk, populated, in pcpu_setup_first_chunk()
2757 chunk = pcpu_alloc_first_chunk(tmp_addr, map_size); in pcpu_setup_first_chunk()
2761 pcpu_reserved_chunk = chunk; in pcpu_setup_first_chunk()
2766 chunk = pcpu_alloc_first_chunk(tmp_addr, map_size); in pcpu_setup_first_chunk()
2770 pcpu_first_chunk = chunk; in pcpu_setup_first_chunk()