Lines Matching refs:pg
453 struct ftrace_profile_page *pg; in function_stat_next() local
455 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); in function_stat_next()
461 if ((void *)rec >= (void *)&pg->records[pg->index]) { in function_stat_next()
462 pg = pg->next; in function_stat_next()
463 if (!pg) in function_stat_next()
465 rec = &pg->records[0]; in function_stat_next()
594 struct ftrace_profile_page *pg; in ftrace_profile_reset() local
596 pg = stat->pages = stat->start; in ftrace_profile_reset()
598 while (pg) { in ftrace_profile_reset()
599 memset(pg->records, 0, PROFILE_RECORDS_SIZE); in ftrace_profile_reset()
600 pg->index = 0; in ftrace_profile_reset()
601 pg = pg->next; in ftrace_profile_reset()
610 struct ftrace_profile_page *pg; in ftrace_profile_pages_init() local
636 pg = stat->start = stat->pages; in ftrace_profile_pages_init()
641 pg->next = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
642 if (!pg->next) in ftrace_profile_pages_init()
644 pg = pg->next; in ftrace_profile_pages_init()
650 pg = stat->start; in ftrace_profile_pages_init()
651 while (pg) { in ftrace_profile_pages_init()
652 unsigned long tmp = (unsigned long)pg; in ftrace_profile_pages_init()
654 pg = pg->next; in ftrace_profile_pages_init()
1534 #define do_for_each_ftrace_rec(pg, rec) \ argument
1535 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1537 for (_____i = 0; _____i < pg->index; _____i++) { \
1538 rec = &pg->records[_____i];
1559 struct ftrace_page *pg; in lookup_rec() local
1566 for (pg = ftrace_pages_start; pg; pg = pg->next) { in lookup_rec()
1567 if (end < pg->records[0].ip || in lookup_rec()
1568 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in lookup_rec()
1570 rec = bsearch(&key, pg->records, pg->index, in lookup_rec()
1698 struct ftrace_page *pg; in __ftrace_hash_rec_update() local
1736 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_rec_update()
1941 struct ftrace_page *pg; in __ftrace_hash_update_ipmodify() local
1969 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
2015 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
2730 struct ftrace_page *pg; in ftrace_replace_code() local
2738 do_for_each_ftrace_rec(pg, rec) { in ftrace_replace_code()
2755 struct ftrace_page *pg; member
2777 iter->pg = ftrace_pages_start; in ftrace_rec_iter_start()
2781 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_start()
2782 iter->pg = iter->pg->next; in ftrace_rec_iter_start()
2784 if (!iter->pg) in ftrace_rec_iter_start()
2800 if (iter->index >= iter->pg->index) { in ftrace_rec_iter_next()
2801 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2805 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_next()
2806 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2809 if (!iter->pg) in ftrace_rec_iter_next()
2823 return &iter->pg->records[iter->index]; in ftrace_rec_iter_record()
3160 struct ftrace_page *pg; in ftrace_shutdown() local
3163 do_for_each_ftrace_rec(pg, rec) { in ftrace_shutdown()
3226 struct ftrace_page *pg; in ftrace_update_code() local
3249 for (pg = new_pgs; pg; pg = pg->next) { in ftrace_update_code()
3251 for (i = 0; i < pg->index; i++) { in ftrace_update_code()
3257 p = &pg->records[i]; in ftrace_update_code()
3278 static int ftrace_allocate_records(struct ftrace_page *pg, int count) in ftrace_allocate_records() argument
3292 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); in ftrace_allocate_records()
3294 if (!pg->records) { in ftrace_allocate_records()
3306 pg->order = order; in ftrace_allocate_records()
3318 struct ftrace_page *pg; in ftrace_allocate_pages() local
3324 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); in ftrace_allocate_pages()
3325 if (!pg) in ftrace_allocate_pages()
3334 cnt = ftrace_allocate_records(pg, num_to_init); in ftrace_allocate_pages()
3342 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); in ftrace_allocate_pages()
3343 if (!pg->next) in ftrace_allocate_pages()
3346 pg = pg->next; in ftrace_allocate_pages()
3352 pg = start_pg; in ftrace_allocate_pages()
3353 while (pg) { in ftrace_allocate_pages()
3354 if (pg->records) { in ftrace_allocate_pages()
3355 free_pages((unsigned long)pg->records, pg->order); in ftrace_allocate_pages()
3356 ftrace_number_of_pages -= 1 << pg->order; in ftrace_allocate_pages()
3358 start_pg = pg->next; in ftrace_allocate_pages()
3359 kfree(pg); in ftrace_allocate_pages()
3360 pg = start_pg; in ftrace_allocate_pages()
3373 struct ftrace_page *pg; member
3603 if (iter->idx >= iter->pg->index) { in t_func_next()
3604 if (iter->pg->next) { in t_func_next()
3605 iter->pg = iter->pg->next; in t_func_next()
3610 rec = &iter->pg->records[iter->idx++]; in t_func_next()
3709 iter->pg = ftrace_pages_start; in t_start()
3779 struct ftrace_page *pg; in ftrace_check_work_func() local
3783 do_for_each_ftrace_rec(pg, rec) { in ftrace_check_work_func()
3945 iter->pg = ftrace_pages_start; in ftrace_avail_open()
3969 iter->pg = ftrace_pages_start; in ftrace_enabled_open()
4053 iter->pg = ftrace_pages_start; in ftrace_regex_open()
4181 struct ftrace_page *pg; in add_rec_by_index() local
4188 do_for_each_ftrace_rec(pg, rec) { in add_rec_by_index()
4189 if (pg->index <= index) { in add_rec_by_index()
4190 index -= pg->index; in add_rec_by_index()
4194 rec = &pg->records[index]; in add_rec_by_index()
4265 struct ftrace_page *pg; in match_records() local
4297 do_for_each_ftrace_rec(pg, rec) { in match_records()
6582 struct ftrace_page *pg; in ftrace_graph_set_hash() local
6600 do_for_each_ftrace_rec(pg, rec) { in ftrace_graph_set_hash()
6776 struct ftrace_page *pg; in ftrace_process_locs() local
6830 pg = start_pg; in ftrace_process_locs()
6843 end_offset = (pg->index+1) * sizeof(pg->records[0]); in ftrace_process_locs()
6844 if (end_offset > PAGE_SIZE << pg->order) { in ftrace_process_locs()
6846 if (WARN_ON(!pg->next)) in ftrace_process_locs()
6848 pg = pg->next; in ftrace_process_locs()
6851 rec = &pg->records[pg->index++]; in ftrace_process_locs()
6856 WARN_ON(pg->next); in ftrace_process_locs()
6859 ftrace_pages = pg; in ftrace_process_locs()
6982 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash) in clear_mod_from_hash() argument
6991 for (i = 0; i < pg->index; i++) { in clear_mod_from_hash()
6992 rec = &pg->records[i]; in clear_mod_from_hash()
7005 static void clear_mod_from_hashes(struct ftrace_page *pg) in clear_mod_from_hashes() argument
7014 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); in clear_mod_from_hashes()
7015 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); in clear_mod_from_hashes()
7044 struct ftrace_page *pg; in ftrace_release_mod() local
7064 for (pg = ftrace_pages_start; pg; pg = *last_pg) { in ftrace_release_mod()
7065 rec = &pg->records[0]; in ftrace_release_mod()
7072 if (WARN_ON(pg == ftrace_pages_start)) in ftrace_release_mod()
7076 if (pg == ftrace_pages) in ftrace_release_mod()
7079 ftrace_update_tot_cnt -= pg->index; in ftrace_release_mod()
7080 *last_pg = pg->next; in ftrace_release_mod()
7082 pg->next = tmp_page; in ftrace_release_mod()
7083 tmp_page = pg; in ftrace_release_mod()
7085 last_pg = &pg->next; in ftrace_release_mod()
7090 for (pg = tmp_page; pg; pg = tmp_page) { in ftrace_release_mod()
7093 clear_mod_from_hashes(pg); in ftrace_release_mod()
7095 if (pg->records) { in ftrace_release_mod()
7096 free_pages((unsigned long)pg->records, pg->order); in ftrace_release_mod()
7097 ftrace_number_of_pages -= 1 << pg->order; in ftrace_release_mod()
7099 tmp_page = pg->next; in ftrace_release_mod()
7100 kfree(pg); in ftrace_release_mod()
7108 struct ftrace_page *pg; in ftrace_module_enable() local
7131 do_for_each_ftrace_rec(pg, rec) { in ftrace_module_enable()
7424 struct ftrace_page *pg; in ftrace_free_mem() local
7446 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { in ftrace_free_mem()
7447 if (end < pg->records[0].ip || in ftrace_free_mem()
7448 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in ftrace_free_mem()
7451 rec = bsearch(&key, pg->records, pg->index, in ftrace_free_mem()
7463 pg->index--; in ftrace_free_mem()
7465 if (!pg->index) { in ftrace_free_mem()
7466 *last_pg = pg->next; in ftrace_free_mem()
7467 if (pg->records) { in ftrace_free_mem()
7468 free_pages((unsigned long)pg->records, pg->order); in ftrace_free_mem()
7469 ftrace_number_of_pages -= 1 << pg->order; in ftrace_free_mem()
7472 kfree(pg); in ftrace_free_mem()
7473 pg = container_of(last_pg, struct ftrace_page, next); in ftrace_free_mem()
7475 ftrace_pages = pg; in ftrace_free_mem()
7479 (pg->index - (rec - pg->records)) * sizeof(*rec)); in ftrace_free_mem()