Lines Matching refs:ai
1583 static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
2431 struct pcpu_alloc_info *ai; in pcpu_alloc_alloc_info() local
2436 base_size = ALIGN(struct_size(ai, groups, nr_groups), in pcpu_alloc_alloc_info()
2437 __alignof__(ai->groups[0].cpu_map[0])); in pcpu_alloc_alloc_info()
2438 ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]); in pcpu_alloc_alloc_info()
2443 ai = ptr; in pcpu_alloc_alloc_info()
2446 ai->groups[0].cpu_map = ptr; in pcpu_alloc_alloc_info()
2449 ai->groups[0].cpu_map[unit] = NR_CPUS; in pcpu_alloc_alloc_info()
2451 ai->nr_groups = nr_groups; in pcpu_alloc_alloc_info()
2452 ai->__ai_size = PFN_ALIGN(ai_size); in pcpu_alloc_alloc_info()
2454 return ai; in pcpu_alloc_alloc_info()
2463 void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai) in pcpu_free_alloc_info() argument
2465 memblock_free(ai, ai->__ai_size); in pcpu_free_alloc_info()
2476 const struct pcpu_alloc_info *ai) in pcpu_dump_alloc_info() argument
2484 v = ai->nr_groups; in pcpu_dump_alloc_info()
2493 upa = ai->alloc_size / ai->unit_size; in pcpu_dump_alloc_info()
2498 lvl, ai->static_size, ai->reserved_size, ai->dyn_size, in pcpu_dump_alloc_info()
2499 ai->unit_size, ai->alloc_size / ai->atom_size, ai->atom_size); in pcpu_dump_alloc_info()
2501 for (group = 0; group < ai->nr_groups; group++) { in pcpu_dump_alloc_info()
2502 const struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_dump_alloc_info()
2579 void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, in pcpu_setup_first_chunk() argument
2582 size_t size_sum = ai->static_size + ai->reserved_size + ai->dyn_size; in pcpu_setup_first_chunk()
2600 pcpu_dump_alloc_info(KERN_EMERG, ai); \ in pcpu_setup_first_chunk()
2606 PCPU_SETUP_BUG_ON(ai->nr_groups <= 0); in pcpu_setup_first_chunk()
2608 PCPU_SETUP_BUG_ON(!ai->static_size); in pcpu_setup_first_chunk()
2613 PCPU_SETUP_BUG_ON(ai->unit_size < size_sum); in pcpu_setup_first_chunk()
2614 PCPU_SETUP_BUG_ON(offset_in_page(ai->unit_size)); in pcpu_setup_first_chunk()
2615 PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); in pcpu_setup_first_chunk()
2616 PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->unit_size, PCPU_BITMAP_BLOCK_SIZE)); in pcpu_setup_first_chunk()
2617 PCPU_SETUP_BUG_ON(ai->dyn_size < PERCPU_DYNAMIC_EARLY_SIZE); in pcpu_setup_first_chunk()
2618 PCPU_SETUP_BUG_ON(!ai->dyn_size); in pcpu_setup_first_chunk()
2619 PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->reserved_size, PCPU_MIN_ALLOC_SIZE)); in pcpu_setup_first_chunk()
2622 PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0); in pcpu_setup_first_chunk()
2625 alloc_size = ai->nr_groups * sizeof(group_offsets[0]); in pcpu_setup_first_chunk()
2631 alloc_size = ai->nr_groups * sizeof(group_sizes[0]); in pcpu_setup_first_chunk()
2655 for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) { in pcpu_setup_first_chunk()
2656 const struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_setup_first_chunk()
2659 group_sizes[group] = gi->nr_units * ai->unit_size; in pcpu_setup_first_chunk()
2671 unit_off[cpu] = gi->base_offset + i * ai->unit_size; in pcpu_setup_first_chunk()
2689 pcpu_dump_alloc_info(KERN_DEBUG, ai); in pcpu_setup_first_chunk()
2691 pcpu_nr_groups = ai->nr_groups; in pcpu_setup_first_chunk()
2698 pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT; in pcpu_setup_first_chunk()
2700 pcpu_atom_size = ai->atom_size; in pcpu_setup_first_chunk()
2704 pcpu_stats_save_ai(ai); in pcpu_setup_first_chunk()
2734 static_size = ALIGN(ai->static_size, PCPU_MIN_ALLOC_SIZE); in pcpu_setup_first_chunk()
2735 dyn_size = ai->dyn_size - (static_size - ai->static_size); in pcpu_setup_first_chunk()
2746 map_size = ai->reserved_size ?: dyn_size; in pcpu_setup_first_chunk()
2750 if (ai->reserved_size) { in pcpu_setup_first_chunk()
2754 ai->reserved_size; in pcpu_setup_first_chunk()
2858 struct pcpu_alloc_info *ai; in pcpu_build_alloc_info() local
2948 ai = pcpu_alloc_alloc_info(nr_groups, nr_units); in pcpu_build_alloc_info()
2949 if (!ai) in pcpu_build_alloc_info()
2951 cpu_map = ai->groups[0].cpu_map; in pcpu_build_alloc_info()
2954 ai->groups[group].cpu_map = cpu_map; in pcpu_build_alloc_info()
2958 ai->static_size = static_size; in pcpu_build_alloc_info()
2959 ai->reserved_size = reserved_size; in pcpu_build_alloc_info()
2960 ai->dyn_size = dyn_size; in pcpu_build_alloc_info()
2961 ai->unit_size = alloc_size / upa; in pcpu_build_alloc_info()
2962 ai->atom_size = atom_size; in pcpu_build_alloc_info()
2963 ai->alloc_size = alloc_size; in pcpu_build_alloc_info()
2966 struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_build_alloc_info()
2973 gi->base_offset = unit * ai->unit_size; in pcpu_build_alloc_info()
2983 return ai; in pcpu_build_alloc_info()
3062 struct pcpu_alloc_info *ai; in pcpu_embed_first_chunk() local
3067 ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size, in pcpu_embed_first_chunk()
3069 if (IS_ERR(ai)) in pcpu_embed_first_chunk()
3070 return PTR_ERR(ai); in pcpu_embed_first_chunk()
3072 size_sum = ai->static_size + ai->reserved_size + ai->dyn_size; in pcpu_embed_first_chunk()
3073 areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *)); in pcpu_embed_first_chunk()
3083 for (group = 0; group < ai->nr_groups; group++) { in pcpu_embed_first_chunk()
3084 struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_embed_first_chunk()
3093 ptr = pcpu_fc_alloc(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn); in pcpu_embed_first_chunk()
3107 max_distance += ai->unit_size * ai->groups[highest_group].nr_units; in pcpu_embed_first_chunk()
3125 for (group = 0; group < ai->nr_groups; group++) { in pcpu_embed_first_chunk()
3126 struct pcpu_group_info *gi = &ai->groups[group]; in pcpu_embed_first_chunk()
3129 for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) { in pcpu_embed_first_chunk()
3132 pcpu_fc_free(ptr, ai->unit_size); in pcpu_embed_first_chunk()
3136 memcpy(ptr, __per_cpu_load, ai->static_size); in pcpu_embed_first_chunk()
3137 pcpu_fc_free(ptr + size_sum, ai->unit_size - size_sum); in pcpu_embed_first_chunk()
3142 for (group = 0; group < ai->nr_groups; group++) { in pcpu_embed_first_chunk()
3143 ai->groups[group].base_offset = areas[group] - base; in pcpu_embed_first_chunk()
3147 PFN_DOWN(size_sum), ai->static_size, ai->reserved_size, in pcpu_embed_first_chunk()
3148 ai->dyn_size, ai->unit_size); in pcpu_embed_first_chunk()
3150 pcpu_setup_first_chunk(ai, base); in pcpu_embed_first_chunk()
3154 for (group = 0; group < ai->nr_groups; group++) in pcpu_embed_first_chunk()
3157 ai->groups[group].nr_units * ai->unit_size); in pcpu_embed_first_chunk()
3159 pcpu_free_alloc_info(ai); in pcpu_embed_first_chunk()
3253 struct pcpu_alloc_info *ai; in pcpu_page_first_chunk() local
3264 ai = pcpu_build_alloc_info(reserved_size, 0, PAGE_SIZE, NULL); in pcpu_page_first_chunk()
3265 if (IS_ERR(ai)) in pcpu_page_first_chunk()
3266 return PTR_ERR(ai); in pcpu_page_first_chunk()
3267 BUG_ON(ai->nr_groups != 1); in pcpu_page_first_chunk()
3268 upa = ai->alloc_size/ai->unit_size; in pcpu_page_first_chunk()
3270 if (WARN_ON(ai->groups[0].nr_units != nr_g0_units)) { in pcpu_page_first_chunk()
3271 pcpu_free_alloc_info(ai); in pcpu_page_first_chunk()
3275 unit_pages = ai->unit_size >> PAGE_SHIFT; in pcpu_page_first_chunk()
3288 unsigned int cpu = ai->groups[0].cpu_map[unit]; in pcpu_page_first_chunk()
3306 vm.size = num_possible_cpus() * ai->unit_size; in pcpu_page_first_chunk()
3311 (unsigned long)vm.addr + unit * ai->unit_size; in pcpu_page_first_chunk()
3331 memcpy((void *)unit_addr, __per_cpu_load, ai->static_size); in pcpu_page_first_chunk()
3336 unit_pages, psize_str, ai->static_size, in pcpu_page_first_chunk()
3337 ai->reserved_size, ai->dyn_size); in pcpu_page_first_chunk()
3339 pcpu_setup_first_chunk(ai, vm.addr); in pcpu_page_first_chunk()
3348 pcpu_free_alloc_info(ai); in pcpu_page_first_chunk()
3404 struct pcpu_alloc_info *ai; in setup_per_cpu_areas() local
3407 ai = pcpu_alloc_alloc_info(1, 1); in setup_per_cpu_areas()
3409 if (!ai || !fc) in setup_per_cpu_areas()
3414 ai->dyn_size = unit_size; in setup_per_cpu_areas()
3415 ai->unit_size = unit_size; in setup_per_cpu_areas()
3416 ai->atom_size = unit_size; in setup_per_cpu_areas()
3417 ai->alloc_size = unit_size; in setup_per_cpu_areas()
3418 ai->groups[0].nr_units = 1; in setup_per_cpu_areas()
3419 ai->groups[0].cpu_map[0] = 0; in setup_per_cpu_areas()
3421 pcpu_setup_first_chunk(ai, fc); in setup_per_cpu_areas()
3422 pcpu_free_alloc_info(ai); in setup_per_cpu_areas()