Lines Matching refs:page
368 struct buffer_data_page *page; /* Actual data page */ member
393 return local_read(&bpage->page->commit); in rb_page_commit()
400 free_pages((unsigned long)bpage->page, bpage->order); in free_buffer_page()
629 struct buffer_page *page = cpu_buffer->commit_page; in verify_event() local
639 if (page == tail_page || WARN_ON_ONCE(stop++ > 100)) in verify_event()
641 commit = local_read(&page->page->commit); in verify_event()
642 write = local_read(&page->write); in verify_event()
643 if (addr >= (unsigned long)&page->page->data[commit] && in verify_event()
644 addr < (unsigned long)&page->page->data[write]) in verify_event()
647 next = rb_list_head(page->list.next); in verify_event()
648 page = list_entry(next, struct buffer_page, list); in verify_event()
707 return rb_fix_abs_ts(ts, cpu_buffer->tail_page->page->time_stamp); in ring_buffer_event_time_stamp()
1222 rb_is_head_page(struct buffer_page *page, struct list_head *list) in rb_is_head_page() argument
1228 if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list) in rb_is_head_page()
1241 static bool rb_is_reader_page(struct buffer_page *page) in rb_is_reader_page() argument
1243 struct list_head *list = page->list.prev; in rb_is_reader_page()
1245 return rb_list_head(list->next) != &page->list; in rb_is_reader_page()
1278 meta->head_buffer = (unsigned long)head->page; in rb_head_page_activate()
1372 struct buffer_page *page; in rb_set_head_page() local
1384 page = head = cpu_buffer->head_page; in rb_set_head_page()
1393 if (rb_is_head_page(page, page->list.prev)) { in rb_set_head_page()
1394 cpu_buffer->head_page = page; in rb_set_head_page()
1395 return page; in rb_set_head_page()
1397 rb_inc_page(&page); in rb_set_head_page()
1398 } while (page != head); in rb_set_head_page()
1474 local_set(&next_page->page->commit, 0); in rb_tail_page_update()
1887 ret = rb_validate_buffer(cpu_buffer->reader_page->page, cpu_buffer->cpu); in rb_meta_validate_events()
1893 entry_bytes += local_read(&cpu_buffer->reader_page->page->commit); in rb_meta_validate_events()
1897 ts = head_page->page->time_stamp; in rb_meta_validate_events()
1914 if (ts < head_page->page->time_stamp) in rb_meta_validate_events()
1917 ts = head_page->page->time_stamp; in rb_meta_validate_events()
1923 ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); in rb_meta_validate_events()
1970 meta->head_buffer = (unsigned long)head_page->page; in rb_meta_validate_events()
1974 meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page); in rb_meta_validate_events()
1979 meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page); in rb_meta_validate_events()
1989 if (meta->commit_buffer == (unsigned long)cpu_buffer->reader_page->page) { in rb_meta_validate_events()
2002 ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); in rb_meta_validate_events()
2014 entry_bytes += local_read(&head_page->page->commit); in rb_meta_validate_events()
2040 local_set(&cpu_buffer->reader_page->page->commit, 0); in rb_meta_validate_events()
2045 local_set(&head_page->page->commit, 0); in rb_meta_validate_events()
2191 if (meta->head_buffer == (unsigned long)bpage->page) in rb_meta_buffer_update()
2194 if (meta->commit_buffer == (unsigned long)bpage->page) { in rb_meta_buffer_update()
2244 struct page *page; in __rb_allocate_pages() local
2261 bpage->page = rb_range_buffer(cpu_buffer, i + 1); in __rb_allocate_pages()
2262 if (!bpage->page) in __rb_allocate_pages()
2270 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), in __rb_allocate_pages()
2273 if (!page) in __rb_allocate_pages()
2275 bpage->page = page_address(page); in __rb_allocate_pages()
2276 rb_init_page(bpage->page); in __rb_allocate_pages()
2330 struct page *page; in rb_allocate_cpu_buffer() local
2366 bpage->page = rb_range_buffer(cpu_buffer, 0); in rb_allocate_cpu_buffer()
2367 if (!bpage->page) in rb_allocate_cpu_buffer()
2373 page = alloc_pages_node(cpu_to_node(cpu), in rb_allocate_cpu_buffer()
2376 if (!page) in rb_allocate_cpu_buffer()
2378 bpage->page = page_address(page); in rb_allocate_cpu_buffer()
2379 rb_init_page(bpage->page); in rb_allocate_cpu_buffer()
3146 return bpage->page->data + index; in __rb_page_index()
3200 if (iter->page_stamp != iter_head_page->page->time_stamp || in rb_iter_head_event()
3208 iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; in rb_iter_head_event()
3252 iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; in rb_inc_iter()
3271 unsigned long old_head = (unsigned long)next_page->page; in rb_update_meta_head()
3275 new_head = (unsigned long)next_page->page; in rb_update_meta_head()
3288 void *old_reader = cpu_buffer->reader_page->page; in rb_update_meta_reader()
3289 void *new_reader = reader->page; in rb_update_meta_reader()
3833 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) { in rb_try_to_discard()
3912 local_set(&cpu_buffer->commit_page->page->commit, in rb_set_commit_to_write()
3917 meta->commit_buffer = (unsigned long)cpu_buffer->commit_page->page; in rb_set_commit_to_write()
3927 local_set(&cpu_buffer->commit_page->page->commit, in rb_set_commit_to_write()
3930 local_read(&cpu_buffer->commit_page->page->commit) & in rb_set_commit_to_write()
4370 bpage = info->tail_page->page; in check_buffer()
4556 tail_page->page->time_stamp = info->ts; in __rb_reserve_next()
4720 if (likely(bpage->page == (void *)addr)) { in rb_decrement_entry()
4732 if (bpage->page == (void *)addr) { in rb_decrement_entry()
5061 ret = bpage->page->time_stamp; in ring_buffer_oldest_event_ts()
5251 iter->page_stamp = cpu_buffer->reader_page->page->time_stamp; in rb_iter_reset()
5253 iter->read_stamp = iter->head_page->page->time_stamp; in rb_iter_reset()
5300 commit_ts = commit_page->page->time_stamp; in ring_buffer_iter_empty()
5314 curr_commit_ts = READ_ONCE(curr_commit_page->page->time_stamp); in ring_buffer_iter_empty()
5520 cpu_buffer->read_stamp = reader->page->time_stamp; in rb_get_reader_page()
5666 *ts = rb_fix_abs_ts(*ts, reader->page->time_stamp); in rb_buffer_peek()
5759 *ts = rb_fix_abs_ts(*ts, iter->head_page->page->time_stamp); in rb_iter_peek()
6059 static void rb_clear_buffer_page(struct buffer_page *page) in rb_clear_buffer_page() argument
6061 local_set(&page->write, 0); in rb_clear_buffer_page()
6062 local_set(&page->entries, 0); in rb_clear_buffer_page()
6063 rb_init_page(page->page); in rb_clear_buffer_page()
6064 page->read = 0; in rb_clear_buffer_page()
6093 id = rb_meta_subbuf_idx(cpu_buffer->ring_meta, bpage->page); in rb_page_id()
6124 struct buffer_page *page; in rb_reset_cpu() local
6131 list_for_each_entry(page, cpu_buffer->pages, list) { in rb_reset_cpu()
6132 rb_clear_buffer_page(page); in rb_reset_cpu()
6467 struct page *page; in ring_buffer_alloc_read_page() local
6492 page = alloc_pages_node(cpu_to_node(cpu), in ring_buffer_alloc_read_page()
6495 if (!page) { in ring_buffer_alloc_read_page()
6500 bpage->data = page_address(page); in ring_buffer_alloc_read_page()
6522 struct page *page = virt_to_page(bpage); in ring_buffer_free_read_page() local
6535 if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) in ring_buffer_free_read_page()
6648 struct buffer_data_page *rpage = cpu_buffer->reader_page->page; in ring_buffer_read_page()
6714 bpage = reader->page; in ring_buffer_read_page()
6715 reader->page = data_page->data; in ring_buffer_read_page()
6765 void *ring_buffer_read_page_data(struct buffer_data_read_page *page) in ring_buffer_read_page_data() argument
6767 return page->data; in ring_buffer_read_page_data()
6980 struct page *page; in rb_alloc_meta_page() local
6985 page = alloc_page(GFP_USER | __GFP_ZERO); in rb_alloc_meta_page()
6986 if (!page) in rb_alloc_meta_page()
6989 cpu_buffer->meta_page = page_to_virt(page); in rb_alloc_meta_page()
7012 subbuf_ids[id++] = (unsigned long)cpu_buffer->reader_page->page; in rb_setup_ids_meta_page()
7022 subbuf_ids[id] = (unsigned long)subbuf->page; in rb_setup_ids_meta_page()
7122 struct page **pages __free(kfree) = NULL; in __rb_map_vma()
7187 struct page *page; in __rb_map_vma() local
7193 page = virt_to_page((void *)cpu_buffer->subbuf_ids[s]); in __rb_map_vma()
7195 for (; off < (1 << (subbuf_order)); off++, page++) { in __rb_map_vma()
7199 pages[p++] = page; in __rb_map_vma()
7356 struct buffer_data_page *bpage = reader->page; in ring_buffer_map_get_reader()
7391 cpu, missed_events, cpu_buffer->reader_page->page->time_stamp); in ring_buffer_map_get_reader()
7401 flush_kernel_vmap_range(cpu_buffer->reader_page->page, in ring_buffer_map_get_reader()