Lines Matching refs:pcp
1435 struct per_cpu_pages *pcp) in free_pcppages_bulk() argument
1441 int prefetch_nr = READ_ONCE(pcp->batch); in free_pcppages_bulk()
1450 count = min(pcp->count, count); in free_pcppages_bulk()
1465 list = &pcp->lists[pindex]; in free_pcppages_bulk()
1505 pcp->count -= nr_freed; in free_pcppages_bulk()
3064 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) in drain_zone_pages() argument
3070 batch = READ_ONCE(pcp->batch); in drain_zone_pages()
3071 to_drain = min(pcp->count, batch); in drain_zone_pages()
3073 free_pcppages_bulk(zone, to_drain, pcp); in drain_zone_pages()
3088 struct per_cpu_pages *pcp; in drain_pages_zone() local
3092 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in drain_pages_zone()
3093 if (pcp->count) in drain_pages_zone()
3094 free_pcppages_bulk(zone, pcp->count, pcp); in drain_pages_zone()
3194 struct per_cpu_pages *pcp; in __drain_all_pages() local
3205 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in __drain_all_pages()
3206 if (pcp->count) in __drain_all_pages()
3210 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); in __drain_all_pages()
3211 if (pcp->count) { in __drain_all_pages()
3317 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch) in nr_pcp_free() argument
3333 batch <<= pcp->free_factor; in nr_pcp_free()
3335 pcp->free_factor++; in nr_pcp_free()
3341 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone) in nr_pcp_high() argument
3343 int high = READ_ONCE(pcp->high); in nr_pcp_high()
3355 return min(READ_ONCE(pcp->batch) << 2, high); in nr_pcp_high()
3362 struct per_cpu_pages *pcp; in free_unref_page_commit() local
3367 pcp = this_cpu_ptr(zone->per_cpu_pageset); in free_unref_page_commit()
3369 list_add(&page->lru, &pcp->lists[pindex]); in free_unref_page_commit()
3370 pcp->count += 1 << order; in free_unref_page_commit()
3371 high = nr_pcp_high(pcp, zone); in free_unref_page_commit()
3372 if (pcp->count >= high) { in free_unref_page_commit()
3373 int batch = READ_ONCE(pcp->batch); in free_unref_page_commit()
3375 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp); in free_unref_page_commit()
3597 struct per_cpu_pages *pcp, in __rmqueue_pcplist() argument
3604 int batch = READ_ONCE(pcp->batch); in __rmqueue_pcplist()
3620 pcp->count += alloced << order; in __rmqueue_pcplist()
3627 pcp->count -= 1 << order; in __rmqueue_pcplist()
3639 struct per_cpu_pages *pcp; in rmqueue_pcplist() local
3651 pcp = this_cpu_ptr(zone->per_cpu_pageset); in rmqueue_pcplist()
3652 pcp->free_factor >>= 1; in rmqueue_pcplist()
3653 list = &pcp->lists[order_to_pindex(migratetype, order)]; in rmqueue_pcplist()
3654 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); in rmqueue_pcplist()
5194 struct per_cpu_pages *pcp; in __alloc_pages_bulk() local
5274 pcp = this_cpu_ptr(zone->per_cpu_pageset); in __alloc_pages_bulk()
5275 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; in __alloc_pages_bulk()
5286 pcp, pcp_list); in __alloc_pages_bulk()
6357 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats);
6880 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, in pageset_update() argument
6883 WRITE_ONCE(pcp->batch, batch); in pageset_update()
6884 WRITE_ONCE(pcp->high, high); in pageset_update()
6887 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) in per_cpu_pages_init() argument
6891 memset(pcp, 0, sizeof(*pcp)); in per_cpu_pages_init()
6895 INIT_LIST_HEAD(&pcp->lists[pindex]); in per_cpu_pages_init()
6903 pcp->high = BOOT_PAGESET_HIGH; in per_cpu_pages_init()
6904 pcp->batch = BOOT_PAGESET_BATCH; in per_cpu_pages_init()
6905 pcp->free_factor = 0; in per_cpu_pages_init()
6911 struct per_cpu_pages *pcp; in __zone_set_pageset_high_and_batch() local
6915 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in __zone_set_pageset_high_and_batch()
6916 pageset_update(pcp, high, batch); in __zone_set_pageset_high_and_batch()
6951 struct per_cpu_pages *pcp; in setup_zone_pageset() local
6954 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in setup_zone_pageset()
6956 per_cpu_pages_init(pcp, pzstats); in setup_zone_pageset()