Lines Matching refs:dio
387 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1585 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in submit_flush_bio() argument
1591 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in submit_flush_bio()
1617 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in do_endio_flush() argument
1619 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in do_endio_flush()
1621 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) in do_endio_flush()
1622 submit_flush_bio(ic, dio); in do_endio_flush()
1627 static void dec_in_flight(struct dm_integrity_io *dio) in dec_in_flight() argument
1629 if (atomic_dec_and_test(&dio->in_flight)) { in dec_in_flight()
1630 struct dm_integrity_c *ic = dio->ic; in dec_in_flight()
1633 remove_range(ic, &dio->range); in dec_in_flight()
1635 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD)) in dec_in_flight()
1638 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in dec_in_flight()
1639 if (unlikely(dio->bi_status) && !bio->bi_status) in dec_in_flight()
1640 bio->bi_status = dio->bi_status; in dec_in_flight()
1641 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { in dec_in_flight()
1642 dio->range.logical_sector += dio->range.n_sectors; in dec_in_flight()
1643 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT); in dec_in_flight()
1644 INIT_WORK(&dio->work, integrity_bio_wait); in dec_in_flight()
1645 queue_work(ic->offload_wq, &dio->work); in dec_in_flight()
1648 do_endio_flush(ic, dio); in dec_in_flight()
1654 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in integrity_end_io() local
1656 dm_bio_restore(&dio->bio_details, bio); in integrity_end_io()
1660 if (dio->completion) in integrity_end_io()
1661 complete(dio->completion); in integrity_end_io()
1663 dec_in_flight(dio); in integrity_end_io()
1721 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); in integrity_metadata() local
1722 struct dm_integrity_c *ic = dio->ic; in integrity_metadata()
1730 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in integrity_metadata()
1740 if (likely(dio->op != REQ_OP_DISCARD)) in integrity_metadata()
1754 if (unlikely(dio->op == REQ_OP_DISCARD)) { in integrity_metadata()
1755 unsigned int bi_size = dio->bio_details.bi_iter.bi_size; in integrity_metadata()
1765 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1781 sector = dio->range.logical_sector; in integrity_metadata()
1782 sectors_to_process = dio->range.n_sectors; in integrity_metadata()
1784 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) { in integrity_metadata()
1801 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1802 checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE); in integrity_metadata()
1833 struct bio_integrity_payload *bip = dio->bio_details.bi_integrity; in integrity_metadata()
1838 unsigned int data_to_process = dio->range.n_sectors; in integrity_metadata()
1850 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1851 this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE); in integrity_metadata()
1861 dec_in_flight(dio); in integrity_metadata()
1864 dio->bi_status = errno_to_blk_status(r); in integrity_metadata()
1865 dec_in_flight(dio); in integrity_metadata()
1871 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in dm_integrity_map() local
1876 dio->ic = ic; in dm_integrity_map()
1877 dio->bi_status = 0; in dm_integrity_map()
1878 dio->op = bio_op(bio); in dm_integrity_map()
1880 if (unlikely(dio->op == REQ_OP_DISCARD)) { in dm_integrity_map()
1896 submit_flush_bio(ic, dio); in dm_integrity_map()
1900 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); in dm_integrity_map()
1901 dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA; in dm_integrity_map()
1902 if (unlikely(dio->fua)) { in dm_integrity_map()
1909 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { in dm_integrity_map()
1911 dio->range.logical_sector, bio_sectors(bio), in dm_integrity_map()
1915 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block… in dm_integrity_map()
1918 dio->range.logical_sector, bio_sectors(bio)); in dm_integrity_map()
1922 if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map()
1957 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) in dm_integrity_map()
1960 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in dm_integrity_map()
1961 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in dm_integrity_map()
1964 dm_integrity_map_continue(dio, true); in dm_integrity_map()
1968 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, in __journal_read_write() argument
1971 struct dm_integrity_c *ic = dio->ic; in __journal_read_write()
1975 logical_sector = dio->range.logical_sector; in __journal_read_write()
1976 n_sectors = dio->range.n_sectors; in __journal_read_write()
1987 if (likely(dio->op == REQ_OP_WRITE)) in __journal_read_write()
1993 if (unlikely(dio->op == REQ_OP_READ)) { in __journal_read_write()
2044 if (likely(dio->op == REQ_OP_WRITE)) in __journal_read_write()
2052 } else if (likely(dio->op == REQ_OP_WRITE)) in __journal_read_write()
2056 if (likely(dio->op == REQ_OP_WRITE)) { in __journal_read_write()
2094 if (unlikely(dio->op == REQ_OP_READ)) in __journal_read_write()
2099 if (likely(dio->op == REQ_OP_WRITE)) { in __journal_read_write()
2108 remove_range(ic, &dio->range); in __journal_read_write()
2113 dio->range.logical_sector = logical_sector; in __journal_read_write()
2114 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in __journal_read_write()
2115 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in __journal_read_write()
2122 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map) in dm_integrity_map_continue() argument
2124 struct dm_integrity_c *ic = dio->ic; in dm_integrity_map_continue()
2125 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in dm_integrity_map_continue()
2130 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ; in dm_integrity_map_continue()
2132 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') in dm_integrity_map_continue()
2136 INIT_WORK(&dio->work, integrity_bio_wait); in dm_integrity_map_continue()
2137 queue_work(ic->offload_wq, &dio->work); in dm_integrity_map_continue()
2149 dio->range.n_sectors = bio_sectors(bio); in dm_integrity_map_continue()
2151 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map_continue()
2152 if (dio->op == REQ_OP_WRITE) { in dm_integrity_map_continue()
2156 dio->range.n_sectors = min(dio->range.n_sectors, in dm_integrity_map_continue()
2158 if (unlikely(!dio->range.n_sectors)) { in dm_integrity_map_continue()
2164 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; in dm_integrity_map_continue()
2182 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); in dm_integrity_map_continue()
2196 } while ((i += ic->sectors_per_block) < dio->range.n_sectors); in dm_integrity_map_continue()
2203 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2205 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) in dm_integrity_map_continue()
2206 dio->range.n_sectors = next_sector - dio->range.logical_sector; in dm_integrity_map_continue()
2211 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { in dm_integrity_map_continue()
2212 if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) in dm_integrity_map_continue()
2215 dio->range.n_sectors = i; in dm_integrity_map_continue()
2219 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_continue()
2228 INIT_WORK(&dio->work, integrity_bio_wait); in dm_integrity_map_continue()
2229 queue_work(ic->wait_wq, &dio->work); in dm_integrity_map_continue()
2233 dio->range.n_sectors = ic->sectors_per_block; in dm_integrity_map_continue()
2234 wait_and_add_new_range(ic, &dio->range); in dm_integrity_map_continue()
2244 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2246 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2251 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) { in dm_integrity_map_continue()
2255 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2257 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { in dm_integrity_map_continue()
2258 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2276 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { in dm_integrity_map_continue()
2277 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2278 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { in dm_integrity_map_continue()
2281 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); in dm_integrity_map_continue()
2290 dio->in_flight = (atomic_t)ATOMIC_INIT(2); in dm_integrity_map_continue()
2294 dio->completion = &read_comp; in dm_integrity_map_continue()
2296 dio->completion = NULL; in dm_integrity_map_continue()
2298 dm_bio_record(&dio->bio_details, bio); in dm_integrity_map_continue()
2303 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; in dm_integrity_map_continue()
2305 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { in dm_integrity_map_continue()
2306 integrity_metadata(&dio->work); in dm_integrity_map_continue()
2309 dio->in_flight = (atomic_t)ATOMIC_INIT(1); in dm_integrity_map_continue()
2310 dio->completion = NULL; in dm_integrity_map_continue()
2322 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) in dm_integrity_map_continue()
2325 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2326 dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) in dm_integrity_map_continue()
2331 integrity_metadata(&dio->work); in dm_integrity_map_continue()
2334 dec_in_flight(dio); in dm_integrity_map_continue()
2336 INIT_WORK(&dio->work, integrity_metadata); in dm_integrity_map_continue()
2337 queue_work(ic->metadata_wq, &dio->work); in dm_integrity_map_continue()
2343 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) in dm_integrity_map_continue()
2346 do_endio_flush(ic, dio); in dm_integrity_map_continue()
2352 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); in integrity_bio_wait() local
2354 dm_integrity_map_continue(dio, false); in integrity_bio_wait()
2811 struct dm_integrity_io *dio; in bitmap_block_work() local
2813 dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in bitmap_block_work()
2815 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2816 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { in bitmap_block_work()
2817 remove_range(ic, &dio->range); in bitmap_block_work()
2818 INIT_WORK(&dio->work, integrity_bio_wait); in bitmap_block_work()
2819 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()
2821 block_bitmap_op(ic, ic->journal, dio->range.logical_sector, in bitmap_block_work()
2822 dio->range.n_sectors, BITMAP_OP_SET); in bitmap_block_work()
2835 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in bitmap_block_work() local
2837 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2838 dio->range.n_sectors, BITMAP_OP_SET); in bitmap_block_work()
2840 remove_range(ic, &dio->range); in bitmap_block_work()
2841 INIT_WORK(&dio->work, integrity_bio_wait); in bitmap_block_work()
2842 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()