Lines Matching refs:io_ctl
398 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode, in io_ctl_init() argument
409 memset(io_ctl, 0, sizeof(struct btrfs_io_ctl)); in io_ctl_init()
411 io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS); in io_ctl_init()
412 if (!io_ctl->pages) in io_ctl_init()
415 io_ctl->num_pages = num_pages; in io_ctl_init()
416 io_ctl->fs_info = btrfs_sb(inode->i_sb); in io_ctl_init()
417 io_ctl->inode = inode; in io_ctl_init()
423 static void io_ctl_free(struct btrfs_io_ctl *io_ctl) in io_ctl_free() argument
425 kfree(io_ctl->pages); in io_ctl_free()
426 io_ctl->pages = NULL; in io_ctl_free()
429 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl) in io_ctl_unmap_page() argument
431 if (io_ctl->cur) { in io_ctl_unmap_page()
432 io_ctl->cur = NULL; in io_ctl_unmap_page()
433 io_ctl->orig = NULL; in io_ctl_unmap_page()
437 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear) in io_ctl_map_page() argument
439 ASSERT(io_ctl->index < io_ctl->num_pages); in io_ctl_map_page()
440 io_ctl->page = io_ctl->pages[io_ctl->index++]; in io_ctl_map_page()
441 io_ctl->cur = page_address(io_ctl->page); in io_ctl_map_page()
442 io_ctl->orig = io_ctl->cur; in io_ctl_map_page()
443 io_ctl->size = PAGE_SIZE; in io_ctl_map_page()
445 clear_page(io_ctl->cur); in io_ctl_map_page()
448 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl) in io_ctl_drop_pages() argument
452 io_ctl_unmap_page(io_ctl); in io_ctl_drop_pages()
454 for (i = 0; i < io_ctl->num_pages; i++) { in io_ctl_drop_pages()
455 if (io_ctl->pages[i]) { in io_ctl_drop_pages()
456 btrfs_page_clear_checked(io_ctl->fs_info, in io_ctl_drop_pages()
457 io_ctl->pages[i], in io_ctl_drop_pages()
458 page_offset(io_ctl->pages[i]), in io_ctl_drop_pages()
460 unlock_page(io_ctl->pages[i]); in io_ctl_drop_pages()
461 put_page(io_ctl->pages[i]); in io_ctl_drop_pages()
466 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate) in io_ctl_prepare_pages() argument
469 struct inode *inode = io_ctl->inode; in io_ctl_prepare_pages()
473 for (i = 0; i < io_ctl->num_pages; i++) { in io_ctl_prepare_pages()
478 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
486 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
490 io_ctl->pages[i] = page; in io_ctl_prepare_pages()
497 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
503 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
509 for (i = 0; i < io_ctl->num_pages; i++) in io_ctl_prepare_pages()
510 clear_page_dirty_for_io(io_ctl->pages[i]); in io_ctl_prepare_pages()
515 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation) in io_ctl_set_generation() argument
517 io_ctl_map_page(io_ctl, 1); in io_ctl_set_generation()
523 io_ctl->cur += (sizeof(u32) * io_ctl->num_pages); in io_ctl_set_generation()
524 io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages); in io_ctl_set_generation()
526 put_unaligned_le64(generation, io_ctl->cur); in io_ctl_set_generation()
527 io_ctl->cur += sizeof(u64); in io_ctl_set_generation()
530 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation) in io_ctl_check_generation() argument
538 io_ctl->cur += sizeof(u32) * io_ctl->num_pages; in io_ctl_check_generation()
539 io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages); in io_ctl_check_generation()
541 cache_gen = get_unaligned_le64(io_ctl->cur); in io_ctl_check_generation()
543 btrfs_err_rl(io_ctl->fs_info, in io_ctl_check_generation()
546 io_ctl_unmap_page(io_ctl); in io_ctl_check_generation()
549 io_ctl->cur += sizeof(u64); in io_ctl_check_generation()
553 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index) in io_ctl_set_crc() argument
560 offset = sizeof(u32) * io_ctl->num_pages; in io_ctl_set_crc()
562 crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); in io_ctl_set_crc()
564 io_ctl_unmap_page(io_ctl); in io_ctl_set_crc()
565 tmp = page_address(io_ctl->pages[0]); in io_ctl_set_crc()
570 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index) in io_ctl_check_crc() argument
577 offset = sizeof(u32) * io_ctl->num_pages; in io_ctl_check_crc()
579 tmp = page_address(io_ctl->pages[0]); in io_ctl_check_crc()
583 io_ctl_map_page(io_ctl, 0); in io_ctl_check_crc()
584 crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); in io_ctl_check_crc()
587 btrfs_err_rl(io_ctl->fs_info, in io_ctl_check_crc()
589 io_ctl_unmap_page(io_ctl); in io_ctl_check_crc()
596 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes, in io_ctl_add_entry() argument
601 if (!io_ctl->cur) in io_ctl_add_entry()
604 entry = io_ctl->cur; in io_ctl_add_entry()
609 io_ctl->cur += sizeof(struct btrfs_free_space_entry); in io_ctl_add_entry()
610 io_ctl->size -= sizeof(struct btrfs_free_space_entry); in io_ctl_add_entry()
612 if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) in io_ctl_add_entry()
615 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_entry()
618 if (io_ctl->index >= io_ctl->num_pages) in io_ctl_add_entry()
622 io_ctl_map_page(io_ctl, 1); in io_ctl_add_entry()
626 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap) in io_ctl_add_bitmap() argument
628 if (!io_ctl->cur) in io_ctl_add_bitmap()
635 if (io_ctl->cur != io_ctl->orig) { in io_ctl_add_bitmap()
636 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_bitmap()
637 if (io_ctl->index >= io_ctl->num_pages) in io_ctl_add_bitmap()
639 io_ctl_map_page(io_ctl, 0); in io_ctl_add_bitmap()
642 copy_page(io_ctl->cur, bitmap); in io_ctl_add_bitmap()
643 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_bitmap()
644 if (io_ctl->index < io_ctl->num_pages) in io_ctl_add_bitmap()
645 io_ctl_map_page(io_ctl, 0); in io_ctl_add_bitmap()
649 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl) in io_ctl_zero_remaining_pages() argument
655 if (io_ctl->cur != io_ctl->orig) in io_ctl_zero_remaining_pages()
656 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_zero_remaining_pages()
658 io_ctl_unmap_page(io_ctl); in io_ctl_zero_remaining_pages()
660 while (io_ctl->index < io_ctl->num_pages) { in io_ctl_zero_remaining_pages()
661 io_ctl_map_page(io_ctl, 1); in io_ctl_zero_remaining_pages()
662 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_zero_remaining_pages()
666 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl, in io_ctl_read_entry() argument
672 if (!io_ctl->cur) { in io_ctl_read_entry()
673 ret = io_ctl_check_crc(io_ctl, io_ctl->index); in io_ctl_read_entry()
678 e = io_ctl->cur; in io_ctl_read_entry()
682 io_ctl->cur += sizeof(struct btrfs_free_space_entry); in io_ctl_read_entry()
683 io_ctl->size -= sizeof(struct btrfs_free_space_entry); in io_ctl_read_entry()
685 if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) in io_ctl_read_entry()
688 io_ctl_unmap_page(io_ctl); in io_ctl_read_entry()
693 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl, in io_ctl_read_bitmap() argument
698 ret = io_ctl_check_crc(io_ctl, io_ctl->index); in io_ctl_read_bitmap()
702 copy_page(entry->bitmap, io_ctl->cur); in io_ctl_read_bitmap()
703 io_ctl_unmap_page(io_ctl); in io_ctl_read_bitmap()
759 struct btrfs_io_ctl io_ctl; in __load_free_space_cache() local
812 ret = io_ctl_init(&io_ctl, inode, 0); in __load_free_space_cache()
818 ret = io_ctl_prepare_pages(&io_ctl, true); in __load_free_space_cache()
822 ret = io_ctl_check_crc(&io_ctl, 0); in __load_free_space_cache()
826 ret = io_ctl_check_generation(&io_ctl, generation); in __load_free_space_cache()
838 ret = io_ctl_read_entry(&io_ctl, e, &type); in __load_free_space_cache()
888 io_ctl_unmap_page(&io_ctl); in __load_free_space_cache()
896 ret = io_ctl_read_bitmap(&io_ctl, e); in __load_free_space_cache()
901 io_ctl_drop_pages(&io_ctl); in __load_free_space_cache()
904 io_ctl_free(&io_ctl); in __load_free_space_cache()
907 io_ctl_drop_pages(&io_ctl); in __load_free_space_cache()
1080 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl, in write_cache_extent_entries() argument
1113 ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes, in write_cache_extent_entries()
1142 ret = io_ctl_add_entry(io_ctl, trim_entry->start, in write_cache_extent_entries()
1212 struct btrfs_io_ctl *io_ctl, in write_pinned_extent_entries() argument
1250 ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL); in write_pinned_extent_entries()
1261 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list) in write_bitmap_entries() argument
1268 ret = io_ctl_add_bitmap(io_ctl, entry->bitmap); in write_bitmap_entries()
1300 struct btrfs_io_ctl *io_ctl, in cleanup_write_cache_enospc() argument
1303 io_ctl_drop_pages(io_ctl); in cleanup_write_cache_enospc()
1311 struct btrfs_io_ctl *io_ctl, in __btrfs_wait_cache_io() argument
1315 struct inode *inode = io_ctl->inode; in __btrfs_wait_cache_io()
1327 io_ctl->entries, io_ctl->bitmaps); in __btrfs_wait_cache_io()
1358 io_ctl->inode = NULL; in __btrfs_wait_cache_io()
1371 block_group, &block_group->io_ctl, in btrfs_wait_cache_io()
1392 struct btrfs_io_ctl *io_ctl, in __btrfs_write_out_cache() argument
1405 WARN_ON(io_ctl->pages); in __btrfs_write_out_cache()
1406 ret = io_ctl_init(io_ctl, inode, 1); in __btrfs_write_out_cache()
1426 ret = io_ctl_prepare_pages(io_ctl, false); in __btrfs_write_out_cache()
1433 io_ctl_set_generation(io_ctl, trans->transid); in __btrfs_write_out_cache()
1438 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1452 ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries); in __btrfs_write_out_cache()
1461 ret = write_bitmap_entries(io_ctl, &bitmap_list); in __btrfs_write_out_cache()
1468 io_ctl_zero_remaining_pages(io_ctl); in __btrfs_write_out_cache()
1471 ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages, in __btrfs_write_out_cache()
1472 io_ctl->num_pages, 0, i_size_read(inode), in __btrfs_write_out_cache()
1483 io_ctl_drop_pages(io_ctl); in __btrfs_write_out_cache()
1484 io_ctl_free(io_ctl); in __btrfs_write_out_cache()
1494 io_ctl->entries = entries; in __btrfs_write_out_cache()
1495 io_ctl->bitmaps = bitmaps; in __btrfs_write_out_cache()
1509 cleanup_write_cache_enospc(inode, io_ctl, &cached_state); in __btrfs_write_out_cache()
1516 io_ctl->inode = NULL; in __btrfs_write_out_cache()
1517 io_ctl_free(io_ctl); in __btrfs_write_out_cache()
1549 block_group, &block_group->io_ctl, trans); in btrfs_write_out_cache()
1558 block_group->io_ctl.inode = NULL; in btrfs_write_out_cache()