Lines Matching refs:io_ctl
377 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode, in io_ctl_init() argument
388 memset(io_ctl, 0, sizeof(struct btrfs_io_ctl)); in io_ctl_init()
390 io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS); in io_ctl_init()
391 if (!io_ctl->pages) in io_ctl_init()
394 io_ctl->num_pages = num_pages; in io_ctl_init()
395 io_ctl->fs_info = inode_to_fs_info(inode); in io_ctl_init()
396 io_ctl->inode = inode; in io_ctl_init()
402 static void io_ctl_free(struct btrfs_io_ctl *io_ctl) in io_ctl_free() argument
404 kfree(io_ctl->pages); in io_ctl_free()
405 io_ctl->pages = NULL; in io_ctl_free()
408 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl) in io_ctl_unmap_page() argument
410 if (io_ctl->cur) { in io_ctl_unmap_page()
411 io_ctl->cur = NULL; in io_ctl_unmap_page()
412 io_ctl->orig = NULL; in io_ctl_unmap_page()
416 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear) in io_ctl_map_page() argument
418 ASSERT(io_ctl->index < io_ctl->num_pages); in io_ctl_map_page()
419 io_ctl->page = io_ctl->pages[io_ctl->index++]; in io_ctl_map_page()
420 io_ctl->cur = page_address(io_ctl->page); in io_ctl_map_page()
421 io_ctl->orig = io_ctl->cur; in io_ctl_map_page()
422 io_ctl->size = PAGE_SIZE; in io_ctl_map_page()
424 clear_page(io_ctl->cur); in io_ctl_map_page()
427 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl) in io_ctl_drop_pages() argument
431 io_ctl_unmap_page(io_ctl); in io_ctl_drop_pages()
433 for (i = 0; i < io_ctl->num_pages; i++) { in io_ctl_drop_pages()
434 if (io_ctl->pages[i]) { in io_ctl_drop_pages()
435 btrfs_folio_clear_checked(io_ctl->fs_info, in io_ctl_drop_pages()
436 page_folio(io_ctl->pages[i]), in io_ctl_drop_pages()
437 page_offset(io_ctl->pages[i]), in io_ctl_drop_pages()
439 unlock_page(io_ctl->pages[i]); in io_ctl_drop_pages()
440 put_page(io_ctl->pages[i]); in io_ctl_drop_pages()
445 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate) in io_ctl_prepare_pages() argument
448 struct inode *inode = io_ctl->inode; in io_ctl_prepare_pages()
452 for (i = 0; i < io_ctl->num_pages; i++) { in io_ctl_prepare_pages()
459 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
467 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
471 io_ctl->pages[i] = &folio->page; in io_ctl_prepare_pages()
478 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
484 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
490 for (i = 0; i < io_ctl->num_pages; i++) in io_ctl_prepare_pages()
491 clear_page_dirty_for_io(io_ctl->pages[i]); in io_ctl_prepare_pages()
496 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation) in io_ctl_set_generation() argument
498 io_ctl_map_page(io_ctl, 1); in io_ctl_set_generation()
504 io_ctl->cur += (sizeof(u32) * io_ctl->num_pages); in io_ctl_set_generation()
505 io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages); in io_ctl_set_generation()
507 put_unaligned_le64(generation, io_ctl->cur); in io_ctl_set_generation()
508 io_ctl->cur += sizeof(u64); in io_ctl_set_generation()
511 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation) in io_ctl_check_generation() argument
519 io_ctl->cur += sizeof(u32) * io_ctl->num_pages; in io_ctl_check_generation()
520 io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages); in io_ctl_check_generation()
522 cache_gen = get_unaligned_le64(io_ctl->cur); in io_ctl_check_generation()
524 btrfs_err_rl(io_ctl->fs_info, in io_ctl_check_generation()
527 io_ctl_unmap_page(io_ctl); in io_ctl_check_generation()
530 io_ctl->cur += sizeof(u64); in io_ctl_check_generation()
534 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index) in io_ctl_set_crc() argument
541 offset = sizeof(u32) * io_ctl->num_pages; in io_ctl_set_crc()
543 crc = crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); in io_ctl_set_crc()
545 io_ctl_unmap_page(io_ctl); in io_ctl_set_crc()
546 tmp = page_address(io_ctl->pages[0]); in io_ctl_set_crc()
551 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index) in io_ctl_check_crc() argument
558 offset = sizeof(u32) * io_ctl->num_pages; in io_ctl_check_crc()
560 tmp = page_address(io_ctl->pages[0]); in io_ctl_check_crc()
564 io_ctl_map_page(io_ctl, 0); in io_ctl_check_crc()
565 crc = crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); in io_ctl_check_crc()
568 btrfs_err_rl(io_ctl->fs_info, in io_ctl_check_crc()
570 io_ctl_unmap_page(io_ctl); in io_ctl_check_crc()
577 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes, in io_ctl_add_entry() argument
582 if (!io_ctl->cur) in io_ctl_add_entry()
585 entry = io_ctl->cur; in io_ctl_add_entry()
590 io_ctl->cur += sizeof(struct btrfs_free_space_entry); in io_ctl_add_entry()
591 io_ctl->size -= sizeof(struct btrfs_free_space_entry); in io_ctl_add_entry()
593 if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) in io_ctl_add_entry()
596 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_entry()
599 if (io_ctl->index >= io_ctl->num_pages) in io_ctl_add_entry()
603 io_ctl_map_page(io_ctl, 1); in io_ctl_add_entry()
607 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap) in io_ctl_add_bitmap() argument
609 if (!io_ctl->cur) in io_ctl_add_bitmap()
616 if (io_ctl->cur != io_ctl->orig) { in io_ctl_add_bitmap()
617 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_bitmap()
618 if (io_ctl->index >= io_ctl->num_pages) in io_ctl_add_bitmap()
620 io_ctl_map_page(io_ctl, 0); in io_ctl_add_bitmap()
623 copy_page(io_ctl->cur, bitmap); in io_ctl_add_bitmap()
624 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_bitmap()
625 if (io_ctl->index < io_ctl->num_pages) in io_ctl_add_bitmap()
626 io_ctl_map_page(io_ctl, 0); in io_ctl_add_bitmap()
630 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl) in io_ctl_zero_remaining_pages() argument
636 if (io_ctl->cur != io_ctl->orig) in io_ctl_zero_remaining_pages()
637 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_zero_remaining_pages()
639 io_ctl_unmap_page(io_ctl); in io_ctl_zero_remaining_pages()
641 while (io_ctl->index < io_ctl->num_pages) { in io_ctl_zero_remaining_pages()
642 io_ctl_map_page(io_ctl, 1); in io_ctl_zero_remaining_pages()
643 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_zero_remaining_pages()
647 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl, in io_ctl_read_entry() argument
653 if (!io_ctl->cur) { in io_ctl_read_entry()
654 ret = io_ctl_check_crc(io_ctl, io_ctl->index); in io_ctl_read_entry()
659 e = io_ctl->cur; in io_ctl_read_entry()
663 io_ctl->cur += sizeof(struct btrfs_free_space_entry); in io_ctl_read_entry()
664 io_ctl->size -= sizeof(struct btrfs_free_space_entry); in io_ctl_read_entry()
666 if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) in io_ctl_read_entry()
669 io_ctl_unmap_page(io_ctl); in io_ctl_read_entry()
674 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl, in io_ctl_read_bitmap() argument
679 ret = io_ctl_check_crc(io_ctl, io_ctl->index); in io_ctl_read_bitmap()
683 copy_page(entry->bitmap, io_ctl->cur); in io_ctl_read_bitmap()
684 io_ctl_unmap_page(io_ctl); in io_ctl_read_bitmap()
740 struct btrfs_io_ctl io_ctl; in __load_free_space_cache() local
793 ret = io_ctl_init(&io_ctl, inode, 0); in __load_free_space_cache()
799 ret = io_ctl_prepare_pages(&io_ctl, true); in __load_free_space_cache()
803 ret = io_ctl_check_crc(&io_ctl, 0); in __load_free_space_cache()
807 ret = io_ctl_check_generation(&io_ctl, generation); in __load_free_space_cache()
819 ret = io_ctl_read_entry(&io_ctl, e, &type); in __load_free_space_cache()
871 io_ctl_unmap_page(&io_ctl); in __load_free_space_cache()
879 ret = io_ctl_read_bitmap(&io_ctl, e); in __load_free_space_cache()
884 io_ctl_drop_pages(&io_ctl); in __load_free_space_cache()
887 io_ctl_free(&io_ctl); in __load_free_space_cache()
890 io_ctl_drop_pages(&io_ctl); in __load_free_space_cache()
1069 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl, in write_cache_extent_entries() argument
1101 ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes, in write_cache_extent_entries()
1130 ret = io_ctl_add_entry(io_ctl, trim_entry->start, in write_cache_extent_entries()
1199 struct btrfs_io_ctl *io_ctl, in write_pinned_extent_entries() argument
1236 ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL); in write_pinned_extent_entries()
1247 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list) in write_bitmap_entries() argument
1254 ret = io_ctl_add_bitmap(io_ctl, entry->bitmap); in write_bitmap_entries()
1286 struct btrfs_io_ctl *io_ctl, in cleanup_write_cache_enospc() argument
1289 io_ctl_drop_pages(io_ctl); in cleanup_write_cache_enospc()
1297 struct btrfs_io_ctl *io_ctl, in __btrfs_wait_cache_io() argument
1301 struct inode *inode = io_ctl->inode; in __btrfs_wait_cache_io()
1313 io_ctl->entries, io_ctl->bitmaps); in __btrfs_wait_cache_io()
1344 io_ctl->inode = NULL; in __btrfs_wait_cache_io()
1357 block_group, &block_group->io_ctl, in btrfs_wait_cache_io()
1377 struct btrfs_io_ctl *io_ctl, in __btrfs_write_out_cache() argument
1391 WARN_ON(io_ctl->pages); in __btrfs_write_out_cache()
1392 ret = io_ctl_init(io_ctl, inode, 1); in __btrfs_write_out_cache()
1412 ret = io_ctl_prepare_pages(io_ctl, false); in __btrfs_write_out_cache()
1419 io_ctl_set_generation(io_ctl, trans->transid); in __btrfs_write_out_cache()
1424 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1438 ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries); in __btrfs_write_out_cache()
1447 ret = write_bitmap_entries(io_ctl, &bitmap_list); in __btrfs_write_out_cache()
1454 io_ctl_zero_remaining_pages(io_ctl); in __btrfs_write_out_cache()
1462 ret = btrfs_dirty_folio(BTRFS_I(inode), page_folio(io_ctl->pages[i]), in __btrfs_write_out_cache()
1474 io_ctl_drop_pages(io_ctl); in __btrfs_write_out_cache()
1475 io_ctl_free(io_ctl); in __btrfs_write_out_cache()
1485 io_ctl->entries = entries; in __btrfs_write_out_cache()
1486 io_ctl->bitmaps = bitmaps; in __btrfs_write_out_cache()
1500 cleanup_write_cache_enospc(inode, io_ctl, &cached_state); in __btrfs_write_out_cache()
1507 io_ctl->inode = NULL; in __btrfs_write_out_cache()
1508 io_ctl_free(io_ctl); in __btrfs_write_out_cache()
1540 &block_group->io_ctl, trans); in btrfs_write_out_cache()
1549 block_group->io_ctl.inode = NULL; in btrfs_write_out_cache()