Lines Matching refs:stripe
231 static void release_scrub_stripe(struct scrub_stripe *stripe) in release_scrub_stripe() argument
233 if (!stripe) in release_scrub_stripe()
237 if (stripe->pages[i]) in release_scrub_stripe()
238 __free_page(stripe->pages[i]); in release_scrub_stripe()
239 stripe->pages[i] = NULL; in release_scrub_stripe()
241 kfree(stripe->sectors); in release_scrub_stripe()
242 kfree(stripe->csums); in release_scrub_stripe()
243 stripe->sectors = NULL; in release_scrub_stripe()
244 stripe->csums = NULL; in release_scrub_stripe()
245 stripe->sctx = NULL; in release_scrub_stripe()
246 stripe->state = 0; in release_scrub_stripe()
250 struct scrub_stripe *stripe) in init_scrub_stripe() argument
254 memset(stripe, 0, sizeof(*stripe)); in init_scrub_stripe()
256 stripe->nr_sectors = BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits; in init_scrub_stripe()
257 stripe->state = 0; in init_scrub_stripe()
259 init_waitqueue_head(&stripe->io_wait); in init_scrub_stripe()
260 init_waitqueue_head(&stripe->repair_wait); in init_scrub_stripe()
261 atomic_set(&stripe->pending_io, 0); in init_scrub_stripe()
262 spin_lock_init(&stripe->write_error_lock); in init_scrub_stripe()
264 ret = btrfs_alloc_page_array(SCRUB_STRIPE_PAGES, stripe->pages, false); in init_scrub_stripe()
268 stripe->sectors = kcalloc(stripe->nr_sectors, in init_scrub_stripe()
271 if (!stripe->sectors) in init_scrub_stripe()
274 stripe->csums = kcalloc(BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits, in init_scrub_stripe()
276 if (!stripe->csums) in init_scrub_stripe()
280 release_scrub_stripe(stripe); in init_scrub_stripe()
284 static void wait_scrub_stripe_io(struct scrub_stripe *stripe) in wait_scrub_stripe_io() argument
286 wait_event(stripe->io_wait, atomic_read(&stripe->pending_io) == 0); in wait_scrub_stripe_io()
582 static struct page *scrub_stripe_get_page(struct scrub_stripe *stripe, int sector_nr) in scrub_stripe_get_page() argument
584 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_stripe_get_page()
587 return stripe->pages[page_index]; in scrub_stripe_get_page()
590 static unsigned int scrub_stripe_get_page_offset(struct scrub_stripe *stripe, in scrub_stripe_get_page_offset() argument
593 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_stripe_get_page_offset()
598 static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr) in scrub_verify_one_metadata() argument
600 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_verify_one_metadata()
602 const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits); in scrub_verify_one_metadata()
603 const struct page *first_page = scrub_stripe_get_page(stripe, sector_nr); in scrub_verify_one_metadata()
604 const unsigned int first_off = scrub_stripe_get_page_offset(stripe, sector_nr); in scrub_verify_one_metadata()
619 bitmap_set(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
620 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
623 logical, stripe->mirror_num, in scrub_verify_one_metadata()
629 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
630 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
633 logical, stripe->mirror_num, in scrub_verify_one_metadata()
639 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
640 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
643 logical, stripe->mirror_num, in scrub_verify_one_metadata()
655 struct page *page = scrub_stripe_get_page(stripe, i); in scrub_verify_one_metadata()
656 unsigned int page_off = scrub_stripe_get_page_offset(stripe, i); in scrub_verify_one_metadata()
664 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
665 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
668 logical, stripe->mirror_num, in scrub_verify_one_metadata()
673 if (stripe->sectors[sector_nr].generation != in scrub_verify_one_metadata()
675 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
676 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
679 logical, stripe->mirror_num, in scrub_verify_one_metadata()
681 stripe->sectors[sector_nr].generation); in scrub_verify_one_metadata()
684 bitmap_clear(&stripe->error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
685 bitmap_clear(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
686 bitmap_clear(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree); in scrub_verify_one_metadata()
689 static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr) in scrub_verify_one_sector() argument
691 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_verify_one_sector()
692 struct scrub_sector_verification *sector = &stripe->sectors[sector_nr]; in scrub_verify_one_sector()
694 struct page *page = scrub_stripe_get_page(stripe, sector_nr); in scrub_verify_one_sector()
695 unsigned int pgoff = scrub_stripe_get_page_offset(stripe, sector_nr); in scrub_verify_one_sector()
699 ASSERT(sector_nr >= 0 && sector_nr < stripe->nr_sectors); in scrub_verify_one_sector()
702 if (!test_bit(sector_nr, &stripe->extent_sector_bitmap)) in scrub_verify_one_sector()
706 if (test_bit(sector_nr, &stripe->io_error_bitmap)) in scrub_verify_one_sector()
719 if (unlikely(sector_nr + sectors_per_tree > stripe->nr_sectors)) { in scrub_verify_one_sector()
722 stripe->logical + in scrub_verify_one_sector()
724 stripe->logical); in scrub_verify_one_sector()
727 scrub_verify_one_metadata(stripe, sector_nr); in scrub_verify_one_sector()
736 clear_bit(sector_nr, &stripe->error_bitmap); in scrub_verify_one_sector()
742 set_bit(sector_nr, &stripe->csum_error_bitmap); in scrub_verify_one_sector()
743 set_bit(sector_nr, &stripe->error_bitmap); in scrub_verify_one_sector()
745 clear_bit(sector_nr, &stripe->csum_error_bitmap); in scrub_verify_one_sector()
746 clear_bit(sector_nr, &stripe->error_bitmap); in scrub_verify_one_sector()
751 static void scrub_verify_one_stripe(struct scrub_stripe *stripe, unsigned long bitmap) in scrub_verify_one_stripe() argument
753 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_verify_one_stripe()
757 for_each_set_bit(sector_nr, &bitmap, stripe->nr_sectors) { in scrub_verify_one_stripe()
758 scrub_verify_one_sector(stripe, sector_nr); in scrub_verify_one_stripe()
759 if (stripe->sectors[sector_nr].is_metadata) in scrub_verify_one_stripe()
764 static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *first_bvec) in calc_sector_number() argument
768 for (i = 0; i < stripe->nr_sectors; i++) { in calc_sector_number()
769 if (scrub_stripe_get_page(stripe, i) == first_bvec->bv_page && in calc_sector_number()
770 scrub_stripe_get_page_offset(stripe, i) == first_bvec->bv_offset) in calc_sector_number()
773 ASSERT(i < stripe->nr_sectors); in calc_sector_number()
785 struct scrub_stripe *stripe = bbio->private; in scrub_repair_read_endio() local
786 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_repair_read_endio()
788 int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); in scrub_repair_read_endio()
792 ASSERT(sector_nr < stripe->nr_sectors); in scrub_repair_read_endio()
798 bitmap_set(&stripe->io_error_bitmap, sector_nr, in scrub_repair_read_endio()
800 bitmap_set(&stripe->error_bitmap, sector_nr, in scrub_repair_read_endio()
803 bitmap_clear(&stripe->io_error_bitmap, sector_nr, in scrub_repair_read_endio()
807 if (atomic_dec_and_test(&stripe->pending_io)) in scrub_repair_read_endio()
808 wake_up(&stripe->io_wait); in scrub_repair_read_endio()
817 static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, in scrub_stripe_submit_repair_read() argument
820 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_stripe_submit_repair_read()
822 const unsigned long old_error_bitmap = stripe->error_bitmap; in scrub_stripe_submit_repair_read()
825 ASSERT(stripe->mirror_num >= 1); in scrub_stripe_submit_repair_read()
826 ASSERT(atomic_read(&stripe->pending_io) == 0); in scrub_stripe_submit_repair_read()
828 for_each_set_bit(i, &old_error_bitmap, stripe->nr_sectors) { in scrub_stripe_submit_repair_read()
833 page = scrub_stripe_get_page(stripe, i); in scrub_stripe_submit_repair_read()
834 pgoff = scrub_stripe_get_page_offset(stripe, i); in scrub_stripe_submit_repair_read()
837 if (bbio && ((i > 0 && !test_bit(i - 1, &stripe->error_bitmap)) || in scrub_stripe_submit_repair_read()
840 atomic_inc(&stripe->pending_io); in scrub_stripe_submit_repair_read()
843 wait_scrub_stripe_io(stripe); in scrub_stripe_submit_repair_read()
848 bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ, in scrub_stripe_submit_repair_read()
849 fs_info, scrub_repair_read_endio, stripe); in scrub_stripe_submit_repair_read()
850 bbio->bio.bi_iter.bi_sector = (stripe->logical + in scrub_stripe_submit_repair_read()
859 atomic_inc(&stripe->pending_io); in scrub_stripe_submit_repair_read()
862 wait_scrub_stripe_io(stripe); in scrub_stripe_submit_repair_read()
867 struct scrub_stripe *stripe) in scrub_stripe_report_errors() argument
880 if (test_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state)) in scrub_stripe_report_errors()
889 if (!bitmap_empty(&stripe->init_error_bitmap, stripe->nr_sectors)) { in scrub_stripe_report_errors()
892 int stripe_index = stripe->mirror_num - 1; in scrub_stripe_report_errors()
896 ASSERT(stripe->mirror_num >= 1); in scrub_stripe_report_errors()
898 stripe->logical, &mapped_len, &bioc, in scrub_stripe_report_errors()
912 for_each_set_bit(sector_nr, &stripe->extent_sector_bitmap, stripe->nr_sectors) { in scrub_stripe_report_errors()
915 if (stripe->sectors[sector_nr].is_metadata) { in scrub_stripe_report_errors()
919 if (!stripe->sectors[sector_nr].csum) in scrub_stripe_report_errors()
923 if (test_bit(sector_nr, &stripe->init_error_bitmap) && in scrub_stripe_report_errors()
924 !test_bit(sector_nr, &stripe->error_bitmap)) { in scrub_stripe_report_errors()
930 if (!test_bit(sector_nr, &stripe->init_error_bitmap)) in scrub_stripe_report_errors()
941 stripe->logical, btrfs_dev_name(dev), in scrub_stripe_report_errors()
946 stripe->logical, stripe->mirror_num); in scrub_stripe_report_errors()
955 stripe->logical, btrfs_dev_name(dev), in scrub_stripe_report_errors()
960 stripe->logical, stripe->mirror_num); in scrub_stripe_report_errors()
963 if (test_bit(sector_nr, &stripe->io_error_bitmap)) in scrub_stripe_report_errors()
966 stripe->logical, physical); in scrub_stripe_report_errors()
967 if (test_bit(sector_nr, &stripe->csum_error_bitmap)) in scrub_stripe_report_errors()
970 stripe->logical, physical); in scrub_stripe_report_errors()
971 if (test_bit(sector_nr, &stripe->meta_error_bitmap)) in scrub_stripe_report_errors()
974 stripe->logical, physical); in scrub_stripe_report_errors()
978 sctx->stat.data_extents_scrubbed += stripe->nr_data_extents; in scrub_stripe_report_errors()
979 sctx->stat.tree_extents_scrubbed += stripe->nr_meta_extents; in scrub_stripe_report_errors()
983 sctx->stat.read_errors += stripe->init_nr_io_errors; in scrub_stripe_report_errors()
984 sctx->stat.csum_errors += stripe->init_nr_csum_errors; in scrub_stripe_report_errors()
985 sctx->stat.verify_errors += stripe->init_nr_meta_errors; in scrub_stripe_report_errors()
987 bitmap_weight(&stripe->error_bitmap, stripe->nr_sectors); in scrub_stripe_report_errors()
992 static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe,
1010 struct scrub_stripe *stripe = container_of(work, struct scrub_stripe, work); in scrub_stripe_read_repair_worker() local
1011 struct scrub_ctx *sctx = stripe->sctx; in scrub_stripe_read_repair_worker()
1013 int num_copies = btrfs_num_copies(fs_info, stripe->bg->start, in scrub_stripe_read_repair_worker()
1014 stripe->bg->length); in scrub_stripe_read_repair_worker()
1019 ASSERT(stripe->mirror_num > 0); in scrub_stripe_read_repair_worker()
1021 wait_scrub_stripe_io(stripe); in scrub_stripe_read_repair_worker()
1022 scrub_verify_one_stripe(stripe, stripe->extent_sector_bitmap); in scrub_stripe_read_repair_worker()
1024 stripe->init_error_bitmap = stripe->error_bitmap; in scrub_stripe_read_repair_worker()
1025 stripe->init_nr_io_errors = bitmap_weight(&stripe->io_error_bitmap, in scrub_stripe_read_repair_worker()
1026 stripe->nr_sectors); in scrub_stripe_read_repair_worker()
1027 stripe->init_nr_csum_errors = bitmap_weight(&stripe->csum_error_bitmap, in scrub_stripe_read_repair_worker()
1028 stripe->nr_sectors); in scrub_stripe_read_repair_worker()
1029 stripe->init_nr_meta_errors = bitmap_weight(&stripe->meta_error_bitmap, in scrub_stripe_read_repair_worker()
1030 stripe->nr_sectors); in scrub_stripe_read_repair_worker()
1032 if (bitmap_empty(&stripe->init_error_bitmap, stripe->nr_sectors)) in scrub_stripe_read_repair_worker()
1041 for (mirror = calc_next_mirror(stripe->mirror_num, num_copies); in scrub_stripe_read_repair_worker()
1042 mirror != stripe->mirror_num; in scrub_stripe_read_repair_worker()
1044 const unsigned long old_error_bitmap = stripe->error_bitmap; in scrub_stripe_read_repair_worker()
1046 scrub_stripe_submit_repair_read(stripe, mirror, in scrub_stripe_read_repair_worker()
1048 wait_scrub_stripe_io(stripe); in scrub_stripe_read_repair_worker()
1049 scrub_verify_one_stripe(stripe, old_error_bitmap); in scrub_stripe_read_repair_worker()
1050 if (bitmap_empty(&stripe->error_bitmap, stripe->nr_sectors)) in scrub_stripe_read_repair_worker()
1065 for (i = 0, mirror = stripe->mirror_num; in scrub_stripe_read_repair_worker()
1068 const unsigned long old_error_bitmap = stripe->error_bitmap; in scrub_stripe_read_repair_worker()
1070 scrub_stripe_submit_repair_read(stripe, mirror, in scrub_stripe_read_repair_worker()
1072 wait_scrub_stripe_io(stripe); in scrub_stripe_read_repair_worker()
1073 scrub_verify_one_stripe(stripe, old_error_bitmap); in scrub_stripe_read_repair_worker()
1074 if (bitmap_empty(&stripe->error_bitmap, stripe->nr_sectors)) in scrub_stripe_read_repair_worker()
1082 bitmap_andnot(&repaired, &stripe->init_error_bitmap, &stripe->error_bitmap, in scrub_stripe_read_repair_worker()
1083 stripe->nr_sectors); in scrub_stripe_read_repair_worker()
1084 if (!sctx->readonly && !bitmap_empty(&repaired, stripe->nr_sectors)) { in scrub_stripe_read_repair_worker()
1088 scrub_write_sectors(sctx, stripe, repaired, false); in scrub_stripe_read_repair_worker()
1089 wait_scrub_stripe_io(stripe); in scrub_stripe_read_repair_worker()
1093 scrub_stripe_report_errors(sctx, stripe); in scrub_stripe_read_repair_worker()
1094 set_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state); in scrub_stripe_read_repair_worker()
1095 wake_up(&stripe->repair_wait); in scrub_stripe_read_repair_worker()
1100 struct scrub_stripe *stripe = bbio->private; in scrub_read_endio() local
1102 int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); in scrub_read_endio()
1107 ASSERT(sector_nr < stripe->nr_sectors); in scrub_read_endio()
1110 num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits; in scrub_read_endio()
1113 bitmap_set(&stripe->io_error_bitmap, sector_nr, num_sectors); in scrub_read_endio()
1114 bitmap_set(&stripe->error_bitmap, sector_nr, num_sectors); in scrub_read_endio()
1116 bitmap_clear(&stripe->io_error_bitmap, sector_nr, num_sectors); in scrub_read_endio()
1119 if (atomic_dec_and_test(&stripe->pending_io)) { in scrub_read_endio()
1120 wake_up(&stripe->io_wait); in scrub_read_endio()
1121 INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker); in scrub_read_endio()
1122 queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work); in scrub_read_endio()
1128 struct scrub_stripe *stripe = bbio->private; in scrub_write_endio() local
1129 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_write_endio()
1131 int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); in scrub_write_endio()
1141 spin_lock_irqsave(&stripe->write_error_lock, flags); in scrub_write_endio()
1142 bitmap_set(&stripe->write_error_bitmap, sector_nr, in scrub_write_endio()
1144 spin_unlock_irqrestore(&stripe->write_error_lock, flags); in scrub_write_endio()
1148 if (atomic_dec_and_test(&stripe->pending_io)) in scrub_write_endio()
1149 wake_up(&stripe->io_wait); in scrub_write_endio()
1153 struct scrub_stripe *stripe, in scrub_submit_write_bio() argument
1159 stripe->logical; in scrub_submit_write_bio()
1161 fill_writer_pointer_gap(sctx, stripe->physical + bio_off); in scrub_submit_write_bio()
1162 atomic_inc(&stripe->pending_io); in scrub_submit_write_bio()
1163 btrfs_submit_repair_write(bbio, stripe->mirror_num, dev_replace); in scrub_submit_write_bio()
1170 wait_scrub_stripe_io(stripe); in scrub_submit_write_bio()
1177 &stripe->write_error_bitmap)) in scrub_submit_write_bio()
1194 static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe, in scrub_write_sectors() argument
1197 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_write_sectors()
1201 for_each_set_bit(sector_nr, &write_bitmap, stripe->nr_sectors) { in scrub_write_sectors()
1202 struct page *page = scrub_stripe_get_page(stripe, sector_nr); in scrub_write_sectors()
1203 unsigned int pgoff = scrub_stripe_get_page_offset(stripe, sector_nr); in scrub_write_sectors()
1207 ASSERT(test_bit(sector_nr, &stripe->extent_sector_bitmap)); in scrub_write_sectors()
1211 scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); in scrub_write_sectors()
1215 bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_WRITE, in scrub_write_sectors()
1216 fs_info, scrub_write_endio, stripe); in scrub_write_sectors()
1217 bbio->bio.bi_iter.bi_sector = (stripe->logical + in scrub_write_sectors()
1225 scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); in scrub_write_sectors()
1483 struct scrub_stripe *stripe, in fill_one_extent_info() argument
1487 for (u64 cur_logical = max(stripe->logical, extent_start); in fill_one_extent_info()
1488 cur_logical < min(stripe->logical + BTRFS_STRIPE_LEN, in fill_one_extent_info()
1491 const int nr_sector = (cur_logical - stripe->logical) >> in fill_one_extent_info()
1494 &stripe->sectors[nr_sector]; in fill_one_extent_info()
1496 set_bit(nr_sector, &stripe->extent_sector_bitmap); in fill_one_extent_info()
1504 static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe) in scrub_stripe_reset_bitmaps() argument
1506 stripe->extent_sector_bitmap = 0; in scrub_stripe_reset_bitmaps()
1507 stripe->init_error_bitmap = 0; in scrub_stripe_reset_bitmaps()
1508 stripe->init_nr_io_errors = 0; in scrub_stripe_reset_bitmaps()
1509 stripe->init_nr_csum_errors = 0; in scrub_stripe_reset_bitmaps()
1510 stripe->init_nr_meta_errors = 0; in scrub_stripe_reset_bitmaps()
1511 stripe->error_bitmap = 0; in scrub_stripe_reset_bitmaps()
1512 stripe->io_error_bitmap = 0; in scrub_stripe_reset_bitmaps()
1513 stripe->csum_error_bitmap = 0; in scrub_stripe_reset_bitmaps()
1514 stripe->meta_error_bitmap = 0; in scrub_stripe_reset_bitmaps()
1530 struct scrub_stripe *stripe) in scrub_find_fill_first_stripe() argument
1544 memset(stripe->sectors, 0, sizeof(struct scrub_sector_verification) * in scrub_find_fill_first_stripe()
1545 stripe->nr_sectors); in scrub_find_fill_first_stripe()
1546 scrub_stripe_reset_bitmaps(stripe); in scrub_find_fill_first_stripe()
1559 stripe->nr_meta_extents++; in scrub_find_fill_first_stripe()
1561 stripe->nr_data_extents++; in scrub_find_fill_first_stripe()
1570 stripe->logical = round_down(cur_logical - bg->start, BTRFS_STRIPE_LEN) + in scrub_find_fill_first_stripe()
1572 stripe->physical = physical + stripe->logical - logical_start; in scrub_find_fill_first_stripe()
1573 stripe->dev = dev; in scrub_find_fill_first_stripe()
1574 stripe->bg = bg; in scrub_find_fill_first_stripe()
1575 stripe->mirror_num = mirror_num; in scrub_find_fill_first_stripe()
1576 stripe_end = stripe->logical + BTRFS_STRIPE_LEN - 1; in scrub_find_fill_first_stripe()
1579 fill_one_extent_info(fs_info, stripe, extent_start, extent_len, in scrub_find_fill_first_stripe()
1596 stripe->nr_meta_extents++; in scrub_find_fill_first_stripe()
1598 stripe->nr_data_extents++; in scrub_find_fill_first_stripe()
1599 fill_one_extent_info(fs_info, stripe, extent_start, extent_len, in scrub_find_fill_first_stripe()
1610 ASSERT(stripe->csums); in scrub_find_fill_first_stripe()
1619 stripe->logical, stripe_end, in scrub_find_fill_first_stripe()
1620 stripe->csums, &csum_bitmap); in scrub_find_fill_first_stripe()
1626 for_each_set_bit(sector_nr, &csum_bitmap, stripe->nr_sectors) { in scrub_find_fill_first_stripe()
1627 stripe->sectors[sector_nr].csum = stripe->csums + in scrub_find_fill_first_stripe()
1631 set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state); in scrub_find_fill_first_stripe()
1636 static void scrub_reset_stripe(struct scrub_stripe *stripe) in scrub_reset_stripe() argument
1638 scrub_stripe_reset_bitmaps(stripe); in scrub_reset_stripe()
1640 stripe->nr_meta_extents = 0; in scrub_reset_stripe()
1641 stripe->nr_data_extents = 0; in scrub_reset_stripe()
1642 stripe->state = 0; in scrub_reset_stripe()
1644 for (int i = 0; i < stripe->nr_sectors; i++) { in scrub_reset_stripe()
1645 stripe->sectors[i].is_metadata = false; in scrub_reset_stripe()
1646 stripe->sectors[i].csum = NULL; in scrub_reset_stripe()
1647 stripe->sectors[i].generation = 0; in scrub_reset_stripe()
1651 static u32 stripe_length(const struct scrub_stripe *stripe) in stripe_length() argument
1653 ASSERT(stripe->bg); in stripe_length()
1656 stripe->bg->start + stripe->bg->length - stripe->logical); in stripe_length()
1660 struct scrub_stripe *stripe) in scrub_submit_extent_sector_read() argument
1662 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in scrub_submit_extent_sector_read()
1664 unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits; in scrub_submit_extent_sector_read()
1666 int mirror = stripe->mirror_num; in scrub_submit_extent_sector_read()
1669 atomic_inc(&stripe->pending_io); in scrub_submit_extent_sector_read()
1671 for_each_set_bit(i, &stripe->extent_sector_bitmap, stripe->nr_sectors) { in scrub_submit_extent_sector_read()
1672 struct page *page = scrub_stripe_get_page(stripe, i); in scrub_submit_extent_sector_read()
1673 unsigned int pgoff = scrub_stripe_get_page_offset(stripe, i); in scrub_submit_extent_sector_read()
1682 !test_bit(i - 1, &stripe->extent_sector_bitmap)) || in scrub_submit_extent_sector_read()
1685 atomic_inc(&stripe->pending_io); in scrub_submit_extent_sector_read()
1693 const u64 logical = stripe->logical + in scrub_submit_extent_sector_read()
1707 set_bit(i, &stripe->io_error_bitmap); in scrub_submit_extent_sector_read()
1708 set_bit(i, &stripe->error_bitmap); in scrub_submit_extent_sector_read()
1712 bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ, in scrub_submit_extent_sector_read()
1713 fs_info, scrub_read_endio, stripe); in scrub_submit_extent_sector_read()
1722 atomic_inc(&stripe->pending_io); in scrub_submit_extent_sector_read()
1726 if (atomic_dec_and_test(&stripe->pending_io)) { in scrub_submit_extent_sector_read()
1727 wake_up(&stripe->io_wait); in scrub_submit_extent_sector_read()
1728 INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker); in scrub_submit_extent_sector_read()
1729 queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work); in scrub_submit_extent_sector_read()
1734 struct scrub_stripe *stripe) in scrub_submit_initial_read() argument
1738 unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits; in scrub_submit_initial_read()
1739 int mirror = stripe->mirror_num; in scrub_submit_initial_read()
1741 ASSERT(stripe->bg); in scrub_submit_initial_read()
1742 ASSERT(stripe->mirror_num > 0); in scrub_submit_initial_read()
1743 ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state)); in scrub_submit_initial_read()
1745 if (btrfs_need_stripe_tree_update(fs_info, stripe->bg->flags)) { in scrub_submit_initial_read()
1746 scrub_submit_extent_sector_read(sctx, stripe); in scrub_submit_initial_read()
1751 scrub_read_endio, stripe); in scrub_submit_initial_read()
1753 bbio->bio.bi_iter.bi_sector = stripe->logical >> SECTOR_SHIFT; in scrub_submit_initial_read()
1756 struct page *page = scrub_stripe_get_page(stripe, cur); in scrub_submit_initial_read()
1757 unsigned int pgoff = scrub_stripe_get_page_offset(stripe, cur); in scrub_submit_initial_read()
1764 atomic_inc(&stripe->pending_io); in scrub_submit_initial_read()
1773 !stripe->dev->bdev)) { in scrub_submit_initial_read()
1774 int num_copies = btrfs_num_copies(fs_info, stripe->bg->start, in scrub_submit_initial_read()
1775 stripe->bg->length); in scrub_submit_initial_read()
1782 static bool stripe_has_metadata_error(struct scrub_stripe *stripe) in stripe_has_metadata_error() argument
1786 for_each_set_bit(i, &stripe->error_bitmap, stripe->nr_sectors) { in stripe_has_metadata_error()
1787 if (stripe->sectors[i].is_metadata) { in stripe_has_metadata_error()
1788 struct btrfs_fs_info *fs_info = stripe->bg->fs_info; in stripe_has_metadata_error()
1792 stripe->logical, in stripe_has_metadata_error()
1793 stripe->logical + (i << fs_info->sectorsize_bits)); in stripe_has_metadata_error()
1813 struct scrub_stripe *stripe = &sctx->stripes[first_slot + i]; in submit_initial_group_read() local
1816 ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state)); in submit_initial_group_read()
1817 scrub_submit_initial_read(sctx, stripe); in submit_initial_group_read()
1825 struct scrub_stripe *stripe; in flush_scrub_stripes() local
1842 stripe = &sctx->stripes[i]; in flush_scrub_stripes()
1844 wait_event(stripe->repair_wait, in flush_scrub_stripes()
1845 test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state)); in flush_scrub_stripes()
1863 stripe = &sctx->stripes[i]; in flush_scrub_stripes()
1865 ASSERT(stripe->dev == fs_info->dev_replace.srcdev); in flush_scrub_stripes()
1867 bitmap_andnot(&good, &stripe->extent_sector_bitmap, in flush_scrub_stripes()
1868 &stripe->error_bitmap, stripe->nr_sectors); in flush_scrub_stripes()
1869 scrub_write_sectors(sctx, stripe, good, true); in flush_scrub_stripes()
1875 stripe = &sctx->stripes[i]; in flush_scrub_stripes()
1877 wait_scrub_stripe_io(stripe); in flush_scrub_stripes()
1879 sctx->stat.last_physical = stripe->physical + stripe_length(stripe); in flush_scrub_stripes()
1881 scrub_reset_stripe(stripe); in flush_scrub_stripes()
1898 struct scrub_stripe *stripe; in queue_scrub_stripe() local
1910 stripe = &sctx->stripes[sctx->cur_stripe]; in queue_scrub_stripe()
1911 scrub_reset_stripe(stripe); in queue_scrub_stripe()
1914 mirror_num, logical, length, stripe); in queue_scrub_stripe()
1918 *found_logical_ret = stripe->logical; in queue_scrub_stripe()
1947 struct scrub_stripe *stripe; in scrub_raid56_parity_stripe() local
1971 stripe = &sctx->raid56_data_stripes[i]; in scrub_raid56_parity_stripe()
1978 scrub_reset_stripe(stripe); in scrub_raid56_parity_stripe()
1979 set_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state); in scrub_raid56_parity_stripe()
1983 BTRFS_STRIPE_LEN, stripe); in scrub_raid56_parity_stripe()
1991 stripe->logical = full_stripe_start + in scrub_raid56_parity_stripe()
1993 stripe->dev = map->stripes[stripe_index].dev; in scrub_raid56_parity_stripe()
1994 stripe->mirror_num = 1; in scrub_raid56_parity_stripe()
1995 set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state); in scrub_raid56_parity_stripe()
2001 stripe = &sctx->raid56_data_stripes[i]; in scrub_raid56_parity_stripe()
2002 if (!bitmap_empty(&stripe->extent_sector_bitmap, stripe->nr_sectors)) { in scrub_raid56_parity_stripe()
2013 stripe = &sctx->raid56_data_stripes[i]; in scrub_raid56_parity_stripe()
2014 scrub_submit_initial_read(sctx, stripe); in scrub_raid56_parity_stripe()
2017 stripe = &sctx->raid56_data_stripes[i]; in scrub_raid56_parity_stripe()
2019 wait_event(stripe->repair_wait, in scrub_raid56_parity_stripe()
2020 test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state)); in scrub_raid56_parity_stripe()
2035 stripe = &sctx->raid56_data_stripes[i]; in scrub_raid56_parity_stripe()
2041 bitmap_and(&error, &stripe->error_bitmap, in scrub_raid56_parity_stripe()
2042 &stripe->extent_sector_bitmap, stripe->nr_sectors); in scrub_raid56_parity_stripe()
2043 if (!bitmap_empty(&error, stripe->nr_sectors)) { in scrub_raid56_parity_stripe()
2046 full_stripe_start, i, stripe->nr_sectors, in scrub_raid56_parity_stripe()
2052 &stripe->extent_sector_bitmap, stripe->nr_sectors); in scrub_raid56_parity_stripe()
2079 stripe = &sctx->raid56_data_stripes[i]; in scrub_raid56_parity_stripe()
2081 raid56_parity_cache_data_pages(rbio, stripe->pages, in scrub_raid56_parity_stripe()