Lines Matching refs:bio
110 struct bio *bio; member
137 static void f2fs_finish_read_bio(struct bio *bio, bool in_task) in f2fs_finish_read_bio() argument
140 struct bio_post_read_ctx *ctx = bio->bi_private; in f2fs_finish_read_bio()
142 bio_for_each_folio_all(fi, bio) { in f2fs_finish_read_bio()
154 folio_end_read(folio, bio->bi_status == BLK_STS_OK); in f2fs_finish_read_bio()
159 bio_put(bio); in f2fs_finish_read_bio()
166 struct bio *bio = ctx->bio; in f2fs_verify_bio() local
177 bio->bi_private = NULL; in f2fs_verify_bio()
186 bio_for_each_folio_all(fi, bio) { in f2fs_verify_bio()
191 bio->bi_status = BLK_STS_IOERR; in f2fs_verify_bio()
196 fsverity_verify_bio(bio); in f2fs_verify_bio()
199 f2fs_finish_read_bio(bio, true); in f2fs_verify_bio()
211 static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task) in f2fs_verify_and_finish_bio() argument
213 struct bio_post_read_ctx *ctx = bio->bi_private; in f2fs_verify_and_finish_bio()
219 f2fs_finish_read_bio(bio, in_task); in f2fs_verify_and_finish_bio()
239 bio_for_each_folio_all(fi, ctx->bio) { in f2fs_handle_step_decompress()
266 struct bio *bio = ctx->bio; in f2fs_post_read_work() local
268 if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) { in f2fs_post_read_work()
269 f2fs_finish_read_bio(bio, true); in f2fs_post_read_work()
276 f2fs_verify_and_finish_bio(bio, true); in f2fs_post_read_work()
279 static void f2fs_read_end_io(struct bio *bio) in f2fs_read_end_io() argument
281 struct f2fs_sb_info *sbi = F2FS_F_SB(bio_first_folio_all(bio)); in f2fs_read_end_io()
285 iostat_update_and_unbind_ctx(bio); in f2fs_read_end_io()
286 ctx = bio->bi_private; in f2fs_read_end_io()
289 bio->bi_status = BLK_STS_IOERR; in f2fs_read_end_io()
291 if (bio->bi_status != BLK_STS_OK) { in f2fs_read_end_io()
292 f2fs_finish_read_bio(bio, intask); in f2fs_read_end_io()
314 f2fs_verify_and_finish_bio(bio, intask); in f2fs_read_end_io()
317 static void f2fs_write_end_io(struct bio *bio) in f2fs_write_end_io() argument
322 iostat_update_and_unbind_ctx(bio); in f2fs_write_end_io()
323 sbi = bio->bi_private; in f2fs_write_end_io()
326 bio->bi_status = BLK_STS_IOERR; in f2fs_write_end_io()
328 bio_for_each_folio_all(fi, bio) { in f2fs_write_end_io()
341 f2fs_compress_write_end_io(bio, folio); in f2fs_write_end_io()
348 if (unlikely(bio->bi_status != BLK_STS_OK)) { in f2fs_write_end_io()
368 bio_put(bio); in f2fs_write_end_io()
372 static void f2fs_zone_write_end_io(struct bio *bio) in f2fs_zone_write_end_io() argument
374 struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; in f2fs_zone_write_end_io()
376 bio->bi_private = io->bi_private; in f2fs_zone_write_end_io()
378 f2fs_write_end_io(bio); in f2fs_zone_write_end_io()
453 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) in __bio_alloc()
458 struct bio *bio; in __bio_alloc() local
461 bio = bio_alloc_bioset(bdev, npages, in __bio_alloc()
464 bio->bi_iter.bi_sector = sector; in __bio_alloc()
466 bio->bi_end_io = f2fs_read_end_io; in __bio_alloc()
467 bio->bi_private = NULL; in __bio_alloc()
469 bio->bi_end_io = f2fs_write_end_io; in __bio_alloc()
470 bio->bi_private = sbi; in __bio_alloc()
471 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, in __bio_alloc()
474 iostat_alloc_and_bind_ctx(sbi, bio, NULL); in __bio_alloc()
477 wbc_init_bio(fio->io_wbc, bio); in __bio_alloc()
479 return bio; in __bio_alloc()
482 static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, in f2fs_set_bio_crypt_ctx() argument
492 fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); in f2fs_set_bio_crypt_ctx()
495 static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, in f2fs_crypt_mergeable_bio() argument
504 return !bio_has_crypt_ctx(bio); in f2fs_crypt_mergeable_bio()
506 return fscrypt_mergeable_bio(bio, inode, next_idx); in f2fs_crypt_mergeable_bio()
509 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, in f2fs_submit_read_bio() argument
512 WARN_ON_ONCE(!is_read_io(bio_op(bio))); in f2fs_submit_read_bio()
513 trace_f2fs_submit_read_bio(sbi->sb, type, bio); in f2fs_submit_read_bio()
515 iostat_update_submit_ctx(bio, type); in f2fs_submit_read_bio()
516 submit_bio(bio); in f2fs_submit_read_bio()
519 static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, in f2fs_submit_write_bio() argument
522 WARN_ON_ONCE(is_read_io(bio_op(bio))); in f2fs_submit_write_bio()
523 trace_f2fs_submit_write_bio(sbi->sb, type, bio); in f2fs_submit_write_bio()
524 iostat_update_submit_ctx(bio, type); in f2fs_submit_write_bio()
525 submit_bio(bio); in f2fs_submit_write_bio()
532 if (!io->bio) in __submit_merged_bio()
536 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
537 f2fs_submit_read_bio(io->sbi, io->bio, fio->type); in __submit_merged_bio()
539 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
540 f2fs_submit_write_bio(io->sbi, io->bio, fio->type); in __submit_merged_bio()
542 io->bio = NULL; in __submit_merged_bio()
545 static bool __has_merged_page(struct bio *bio, struct inode *inode, in __has_merged_page() argument
550 if (!bio) in __has_merged_page()
556 bio_for_each_folio_all(fi, bio) { in __has_merged_page()
600 io->bio = NULL; in f2fs_init_write_merge_io()
625 if (!io->bio) in __f2fs_submit_merged_write()
631 io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC; in __f2fs_submit_merged_write()
633 io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA; in __f2fs_submit_merged_write()
653 ret = __has_merged_page(io->bio, inode, folio, ino); in __submit_merged_write_cond()
690 struct bio *bio; in f2fs_submit_page_bio() local
703 bio = __bio_alloc(fio, 1); in f2fs_submit_page_bio()
705 f2fs_set_bio_crypt_ctx(bio, fio_folio->mapping->host, in f2fs_submit_page_bio()
707 bio_add_folio_nofail(bio, data_folio, folio_size(data_folio), 0); in f2fs_submit_page_bio()
715 if (is_read_io(bio_op(bio))) in f2fs_submit_page_bio()
716 f2fs_submit_read_bio(fio->sbi, bio, fio->type); in f2fs_submit_page_bio()
718 f2fs_submit_write_bio(fio->sbi, bio, fio->type); in f2fs_submit_page_bio()
722 static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, in page_is_mergeable() argument
726 bio->bi_iter.bi_size >= sbi->max_io_bytes)) in page_is_mergeable()
730 return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); in page_is_mergeable()
741 static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, in io_is_mergeable() argument
747 if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr)) in io_is_mergeable()
752 static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio, in add_bio_entry() argument
759 be->bio = bio; in add_bio_entry()
760 bio_get(bio); in add_bio_entry()
762 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE) in add_bio_entry()
776 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, in add_ipu_page() argument
792 if (be->bio != *bio) in add_ipu_page()
797 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio, in add_ipu_page()
800 if (f2fs_crypt_mergeable_bio(*bio, in add_ipu_page()
803 bio_add_page(*bio, page, PAGE_SIZE, 0) == in add_ipu_page()
811 f2fs_submit_write_bio(sbi, *bio, DATA); in add_ipu_page()
818 bio_put(*bio); in add_ipu_page()
819 *bio = NULL; in add_ipu_page()
826 struct bio **bio, struct folio *folio) in f2fs_submit_merged_ipu_write() argument
830 struct bio *target = bio ? *bio : NULL; in f2fs_submit_merged_ipu_write()
845 found = (target == be->bio); in f2fs_submit_merged_ipu_write()
847 found = __has_merged_page(be->bio, NULL, in f2fs_submit_merged_ipu_write()
862 found = (target == be->bio); in f2fs_submit_merged_ipu_write()
864 found = __has_merged_page(be->bio, NULL, in f2fs_submit_merged_ipu_write()
867 target = be->bio; in f2fs_submit_merged_ipu_write()
877 if (bio && *bio) { in f2fs_submit_merged_ipu_write()
878 bio_put(*bio); in f2fs_submit_merged_ipu_write()
879 *bio = NULL; in f2fs_submit_merged_ipu_write()
885 struct bio *bio = *fio->bio; in f2fs_merge_page_bio() local
896 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, in f2fs_merge_page_bio()
898 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); in f2fs_merge_page_bio()
900 if (!bio) { in f2fs_merge_page_bio()
901 bio = __bio_alloc(fio, BIO_MAX_VECS); in f2fs_merge_page_bio()
902 f2fs_set_bio_crypt_ctx(bio, folio->mapping->host, in f2fs_merge_page_bio()
905 add_bio_entry(fio->sbi, bio, &data_folio->page, fio->temp); in f2fs_merge_page_bio()
907 if (add_ipu_page(fio, &bio, &data_folio->page)) in f2fs_merge_page_bio()
917 *fio->bio = bio; in f2fs_merge_page_bio()
992 if (io->bio && in f2fs_submit_page_write()
993 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, in f2fs_submit_page_write()
995 !f2fs_crypt_mergeable_bio(io->bio, fio_inode(fio), in f2fs_submit_page_write()
999 if (io->bio == NULL) { in f2fs_submit_page_write()
1000 io->bio = __bio_alloc(fio, BIO_MAX_VECS); in f2fs_submit_page_write()
1001 f2fs_set_bio_crypt_ctx(io->bio, fio_inode(fio), in f2fs_submit_page_write()
1006 if (!bio_add_folio(io->bio, bio_folio, folio_size(bio_folio), 0)) { in f2fs_submit_page_write()
1021 bio_get(io->bio); in f2fs_submit_page_write()
1023 io->bi_private = io->bio->bi_private; in f2fs_submit_page_write()
1024 io->bio->bi_private = io; in f2fs_submit_page_write()
1025 io->bio->bi_end_io = f2fs_zone_write_end_io; in f2fs_submit_page_write()
1026 io->zone_pending_bio = io->bio; in f2fs_submit_page_write()
1039 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, in f2fs_grab_read_bio()
1044 struct bio *bio; in f2fs_grab_read_bio() local
1050 bio = bio_alloc_bioset(bdev, bio_max_segs(nr_pages), in f2fs_grab_read_bio()
1053 bio->bi_iter.bi_sector = sector; in f2fs_grab_read_bio()
1054 f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); in f2fs_grab_read_bio()
1055 bio->bi_end_io = f2fs_read_end_io; in f2fs_grab_read_bio()
1073 ctx->bio = bio; in f2fs_grab_read_bio()
1078 bio->bi_private = ctx; in f2fs_grab_read_bio()
1080 iostat_alloc_and_bind_ctx(sbi, bio, ctx); in f2fs_grab_read_bio()
1082 return bio; in f2fs_grab_read_bio()
1091 struct bio *bio; in f2fs_submit_page_read() local
1093 bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags, in f2fs_submit_page_read()
1095 if (IS_ERR(bio)) in f2fs_submit_page_read()
1096 return PTR_ERR(bio); in f2fs_submit_page_read()
1101 if (!bio_add_folio(bio, folio, PAGE_SIZE, 0)) { in f2fs_submit_page_read()
1102 iostat_update_and_unbind_ctx(bio); in f2fs_submit_page_read()
1103 if (bio->bi_private) in f2fs_submit_page_read()
1104 mempool_free(bio->bi_private, bio_post_read_ctx_pool); in f2fs_submit_page_read()
1105 bio_put(bio); in f2fs_submit_page_read()
1110 f2fs_submit_read_bio(sbi, bio, DATA); in f2fs_submit_page_read()
2072 struct bio **bio_ret, in f2fs_read_single_page()
2076 struct bio *bio = *bio_ret; in f2fs_read_single_page() local
2141 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, in f2fs_read_single_page()
2143 !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) { in f2fs_read_single_page()
2145 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); in f2fs_read_single_page()
2146 bio = NULL; in f2fs_read_single_page()
2148 if (bio == NULL) { in f2fs_read_single_page()
2149 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, in f2fs_read_single_page()
2152 if (IS_ERR(bio)) { in f2fs_read_single_page()
2153 ret = PTR_ERR(bio); in f2fs_read_single_page()
2154 bio = NULL; in f2fs_read_single_page()
2165 if (!bio_add_folio(bio, folio, blocksize, 0)) in f2fs_read_single_page()
2173 *bio_ret = bio; in f2fs_read_single_page()
2178 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, in f2fs_read_multi_pages()
2185 struct bio *bio = *bio_ret; in f2fs_read_multi_pages() local
2298 if (bio && (!page_is_mergeable(sbi, bio, in f2fs_read_multi_pages()
2300 !f2fs_crypt_mergeable_bio(bio, inode, folio->index, NULL))) { in f2fs_read_multi_pages()
2302 f2fs_submit_read_bio(sbi, bio, DATA); in f2fs_read_multi_pages()
2303 bio = NULL; in f2fs_read_multi_pages()
2306 if (!bio) { in f2fs_read_multi_pages()
2307 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages - i, in f2fs_read_multi_pages()
2310 if (IS_ERR(bio)) { in f2fs_read_multi_pages()
2311 ret = PTR_ERR(bio); in f2fs_read_multi_pages()
2319 if (!bio_add_folio(bio, folio, blocksize, 0)) in f2fs_read_multi_pages()
2322 ctx = get_post_read_ctx(bio); in f2fs_read_multi_pages()
2334 *bio_ret = bio; in f2fs_read_multi_pages()
2347 *bio_ret = bio; in f2fs_read_multi_pages()
2359 struct bio *bio = NULL; in f2fs_mpage_readpages() local
2411 ret = f2fs_read_multi_pages(&cc, &bio, in f2fs_mpage_readpages()
2445 &bio, &last_block_in_bio, rac); in f2fs_mpage_readpages()
2461 ret = f2fs_read_multi_pages(&cc, &bio, in f2fs_mpage_readpages()
2470 if (bio) in f2fs_mpage_readpages()
2471 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); in f2fs_mpage_readpages()
2775 struct bio **bio, in f2fs_write_single_data_page() argument
2807 .bio = bio, in f2fs_write_single_data_page()
2910 if (bio && *bio) in f2fs_write_single_data_page()
2911 f2fs_submit_merged_ipu_write(sbi, bio, NULL); in f2fs_write_single_data_page()
2949 struct bio *bio = NULL; in f2fs_write_cache_pages() local
3142 &submitted, &bio, &last_block, in f2fs_write_cache_pages()
3212 if (bio) in f2fs_write_cache_pages()
3213 f2fs_submit_merged_ipu_write(sbi, &bio, NULL); in f2fs_write_cache_pages()