Lines Matching refs:ic
107 #define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block]) argument
302 struct dm_integrity_c *ic; member
320 struct dm_integrity_c *ic; member
332 struct dm_integrity_c *ic; member
391 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err) in dm_integrity_io_error() argument
394 atomic64_inc(&ic->number_of_mismatches); in dm_integrity_io_error()
395 if (!cmpxchg(&ic->failed, 0, err)) in dm_integrity_io_error()
399 static int dm_integrity_failed(struct dm_integrity_c *ic) in dm_integrity_failed() argument
401 return READ_ONCE(ic->failed); in dm_integrity_failed()
404 static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic) in dm_integrity_disable_recalculate() argument
406 if (ic->legacy_recalculate) in dm_integrity_disable_recalculate()
408 if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ? in dm_integrity_disable_recalculate()
409 ic->internal_hash_alg.key || ic->journal_mac_alg.key : in dm_integrity_disable_recalculate()
410 ic->internal_hash_alg.key && !ic->journal_mac_alg.key) in dm_integrity_disable_recalculate()
415 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned int i, in dm_integrity_commit_id() argument
422 return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j); in dm_integrity_commit_id()
425 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, in get_area_and_offset() argument
428 if (!ic->meta_dev) { in get_area_and_offset()
429 __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors; in get_area_and_offset()
438 #define sector_to_block(ic, n) \ argument
440 BUG_ON((n) & (unsigned int)((ic)->sectors_per_block - 1)); \
441 (n) >>= (ic)->sb->log2_sectors_per_block; \
444 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area, in get_metadata_sector_and_offset() argument
450 ms = area << ic->sb->log2_interleave_sectors; in get_metadata_sector_and_offset()
451 if (likely(ic->log2_metadata_run >= 0)) in get_metadata_sector_and_offset()
452 ms += area << ic->log2_metadata_run; in get_metadata_sector_and_offset()
454 ms += area * ic->metadata_run; in get_metadata_sector_and_offset()
455 ms >>= ic->log2_buffer_sectors; in get_metadata_sector_and_offset()
457 sector_to_block(ic, offset); in get_metadata_sector_and_offset()
459 if (likely(ic->log2_tag_size >= 0)) { in get_metadata_sector_and_offset()
460 ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size); in get_metadata_sector_and_offset()
461 mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); in get_metadata_sector_and_offset()
463 ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors); in get_metadata_sector_and_offset()
464 mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); in get_metadata_sector_and_offset()
470 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset) in get_data_sector() argument
474 if (ic->meta_dev) in get_data_sector()
477 result = area << ic->sb->log2_interleave_sectors; in get_data_sector()
478 if (likely(ic->log2_metadata_run >= 0)) in get_data_sector()
479 result += (area + 1) << ic->log2_metadata_run; in get_data_sector()
481 result += (area + 1) * ic->metadata_run; in get_data_sector()
483 result += (sector_t)ic->initial_sectors + offset; in get_data_sector()
484 result += ic->start; in get_data_sector()
489 static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr) in wraparound_section() argument
491 if (unlikely(*sec_ptr >= ic->journal_sections)) in wraparound_section()
492 *sec_ptr -= ic->journal_sections; in wraparound_section()
495 static void sb_set_version(struct dm_integrity_c *ic) in sb_set_version() argument
497 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) in sb_set_version()
498 ic->sb->version = SB_VERSION_5; in sb_set_version()
499 else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) in sb_set_version()
500 ic->sb->version = SB_VERSION_4; in sb_set_version()
501 else if (ic->mode == 'B' || ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) in sb_set_version()
502 ic->sb->version = SB_VERSION_3; in sb_set_version()
503 else if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) in sb_set_version()
504 ic->sb->version = SB_VERSION_2; in sb_set_version()
506 ic->sb->version = SB_VERSION_1; in sb_set_version()
509 static int sb_mac(struct dm_integrity_c *ic, bool wr) in sb_mac() argument
511 SHASH_DESC_ON_STACK(desc, ic->journal_mac); in sb_mac()
513 unsigned int size = crypto_shash_digestsize(ic->journal_mac); in sb_mac()
516 dm_integrity_io_error(ic, "digest is too long", -EINVAL); in sb_mac()
520 desc->tfm = ic->journal_mac; in sb_mac()
524 dm_integrity_io_error(ic, "crypto_shash_init", r); in sb_mac()
528 r = crypto_shash_update(desc, (__u8 *)ic->sb, (1 << SECTOR_SHIFT) - size); in sb_mac()
530 dm_integrity_io_error(ic, "crypto_shash_update", r); in sb_mac()
535 r = crypto_shash_final(desc, (__u8 *)ic->sb + (1 << SECTOR_SHIFT) - size); in sb_mac()
537 dm_integrity_io_error(ic, "crypto_shash_final", r); in sb_mac()
545 dm_integrity_io_error(ic, "crypto_shash_final", r); in sb_mac()
548 if (memcmp((__u8 *)ic->sb + (1 << SECTOR_SHIFT) - size, result, size)) { in sb_mac()
549 dm_integrity_io_error(ic, "superblock mac", -EILSEQ); in sb_mac()
550 dm_audit_log_target(DM_MSG_PREFIX, "mac-superblock", ic->ti, 0); in sb_mac()
558 static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) in sync_rw_sb() argument
567 io_req.mem.ptr.addr = ic->sb; in sync_rw_sb()
569 io_req.client = ic->io; in sync_rw_sb()
570 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; in sync_rw_sb()
571 io_loc.sector = ic->start; in sync_rw_sb()
575 sb_set_version(ic); in sync_rw_sb()
576 if (ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { in sync_rw_sb()
577 r = sb_mac(ic, true); in sync_rw_sb()
588 if (ic->mode != 'R' && ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { in sync_rw_sb()
589 r = sb_mac(ic, false); in sync_rw_sb()
603 static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap, in block_bitmap_op() argument
609 if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) { in block_bitmap_op()
613 ic->sb->log2_sectors_per_block, in block_bitmap_op()
614 ic->log2_blocks_per_bitmap_bit, in block_bitmap_op()
622 bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in block_bitmap_op()
624 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in block_bitmap_op()
709 static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *s… in block_bitmap_copy() argument
711 unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); in block_bitmap_copy()
722 static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t secto… in sector_to_bitmap_block() argument
724 unsigned int bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in sector_to_bitmap_block()
727 BUG_ON(bitmap_block >= ic->n_bitmap_blocks); in sector_to_bitmap_block()
728 return &ic->bbs[bitmap_block]; in sector_to_bitmap_block()
731 static void access_journal_check(struct dm_integrity_c *ic, unsigned int section, unsigned int offs… in access_journal_check() argument
735 unsigned int limit = e ? ic->journal_section_entries : ic->journal_section_sectors; in access_journal_check()
737 if (unlikely(section >= ic->journal_sections) || in access_journal_check()
740 function, section, offset, ic->journal_sections, limit); in access_journal_check()
746 static void page_list_location(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, in page_list_location() argument
751 access_journal_check(ic, section, offset, false, "page_list_location"); in page_list_location()
753 sector = section * ic->journal_section_sectors + offset; in page_list_location()
759 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl, in access_page_list() argument
765 page_list_location(ic, section, offset, &pl_index, &pl_offset); in access_page_list()
775 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned int section, unsig… in access_journal() argument
777 return access_page_list(ic, ic->journal, section, offset, NULL); in access_journal()
780 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned int section, … in access_journal_entry() argument
785 access_journal_check(ic, section, n, true, "access_journal_entry"); in access_journal_entry()
790 js = access_journal(ic, section, rel_sector); in access_journal_entry()
791 return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size); in access_journal_entry()
794 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned int section, … in access_journal_data() argument
796 n <<= ic->sb->log2_sectors_per_block; in access_journal_data()
800 access_journal_check(ic, section, n, false, "access_journal_data"); in access_journal_data()
802 return access_journal(ic, section, n); in access_journal_data()
805 static void section_mac(struct dm_integrity_c *ic, unsigned int section, __u8 result[JOURNAL_MAC_SI… in section_mac() argument
807 SHASH_DESC_ON_STACK(desc, ic->journal_mac); in section_mac()
811 desc->tfm = ic->journal_mac; in section_mac()
815 dm_integrity_io_error(ic, "crypto_shash_init", r); in section_mac()
819 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { in section_mac()
822 r = crypto_shash_update(desc, (__u8 *)&ic->sb->salt, SALT_SIZE); in section_mac()
824 dm_integrity_io_error(ic, "crypto_shash_update", r); in section_mac()
831 dm_integrity_io_error(ic, "crypto_shash_update", r); in section_mac()
836 for (j = 0; j < ic->journal_section_entries; j++) { in section_mac()
837 struct journal_entry *je = access_journal_entry(ic, section, j); in section_mac()
841 dm_integrity_io_error(ic, "crypto_shash_update", r); in section_mac()
846 size = crypto_shash_digestsize(ic->journal_mac); in section_mac()
851 dm_integrity_io_error(ic, "crypto_shash_final", r); in section_mac()
859 dm_integrity_io_error(ic, "digest_size", -EINVAL); in section_mac()
864 dm_integrity_io_error(ic, "crypto_shash_final", r); in section_mac()
875 static void rw_section_mac(struct dm_integrity_c *ic, unsigned int section, bool wr) in rw_section_mac() argument
880 if (!ic->journal_mac) in rw_section_mac()
883 section_mac(ic, section, result); in rw_section_mac()
886 struct journal_sector *js = access_journal(ic, section, j); in rw_section_mac()
892 dm_integrity_io_error(ic, "journal mac", -EILSEQ); in rw_section_mac()
893 dm_audit_log_target(DM_MSG_PREFIX, "mac-journal", ic->ti, 0); in rw_section_mac()
908 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, in xor_journal() argument
912 size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT; in xor_journal()
917 source_pl = ic->journal; in xor_journal()
918 target_pl = ic->journal_io; in xor_journal()
920 source_pl = ic->journal_io; in xor_journal()
921 target_pl = ic->journal; in xor_journal()
924 page_list_location(ic, section, 0, &pl_index, &pl_offset); in xor_journal()
941 rw_section_mac(ic, section, true); in xor_journal()
946 page_list_location(ic, section, 0, §ion_index, &dummy); in xor_journal()
952 src_pages[1] = ic->journal_xor[pl_index].page; in xor_journal()
972 complete(&comp->ic->crypto_backoff); in complete_journal_encrypt()
975 dm_integrity_io_error(comp->ic, "asynchronous encrypt", err); in complete_journal_encrypt()
995 wait_for_completion(&comp->ic->crypto_backoff); in do_crypt()
996 reinit_completion(&comp->ic->crypto_backoff); in do_crypt()
999 dm_integrity_io_error(comp->ic, "encrypt", r); in do_crypt()
1003 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, in crypt_journal() argument
1012 source_sg = ic->journal_scatterlist; in crypt_journal()
1013 target_sg = ic->journal_io_scatterlist; in crypt_journal()
1015 source_sg = ic->journal_io_scatterlist; in crypt_journal()
1016 target_sg = ic->journal_scatterlist; in crypt_journal()
1025 rw_section_mac(ic, section, true); in crypt_journal()
1027 req = ic->sk_requests[section]; in crypt_journal()
1028 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); in crypt_journal()
1047 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, in encrypt_journal() argument
1050 if (ic->journal_xor) in encrypt_journal()
1051 return xor_journal(ic, encrypt, section, n_sections, comp); in encrypt_journal()
1053 return crypt_journal(ic, encrypt, section, n_sections, comp); in encrypt_journal()
1061 dm_integrity_io_error(comp->ic, "writing journal", -EIO); in complete_journal_io()
1065 static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, in rw_journal_sectors() argument
1074 if (unlikely(dm_integrity_failed(ic))) { in rw_journal_sectors()
1085 if (ic->journal_io) in rw_journal_sectors()
1086 io_req.mem.ptr.pl = &ic->journal_io[pl_index]; in rw_journal_sectors()
1088 io_req.mem.ptr.pl = &ic->journal[pl_index]; in rw_journal_sectors()
1096 io_req.client = ic->io; in rw_journal_sectors()
1097 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; in rw_journal_sectors()
1098 io_loc.sector = ic->start + SB_SECTORS + sector; in rw_journal_sectors()
1103 dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? in rw_journal_sectors()
1112 static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf, in rw_journal() argument
1118 sector = section * ic->journal_section_sectors; in rw_journal()
1119 n_sectors = n_sections * ic->journal_section_sectors; in rw_journal()
1121 rw_journal_sectors(ic, opf, sector, n_sectors, comp); in rw_journal()
1124 static void write_journal(struct dm_integrity_c *ic, unsigned int commit_start, unsigned int commit… in write_journal() argument
1131 io_comp.ic = ic; in write_journal()
1134 if (commit_start + commit_sections <= ic->journal_sections) { in write_journal()
1136 if (ic->journal_io) { in write_journal()
1137 crypt_comp_1.ic = ic; in write_journal()
1140 encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1); in write_journal()
1144 rw_section_mac(ic, commit_start + i, true); in write_journal()
1146 rw_journal(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, commit_start, in write_journal()
1152 to_end = ic->journal_sections - commit_start; in write_journal()
1153 if (ic->journal_io) { in write_journal()
1154 crypt_comp_1.ic = ic; in write_journal()
1157 encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1); in write_journal()
1159 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, in write_journal()
1163 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1); in write_journal()
1166 crypt_comp_2.ic = ic; in write_journal()
1169 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2); in write_journal()
1171 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp); in write_journal()
1176 rw_section_mac(ic, commit_start + i, true); in write_journal()
1177 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp); in write_journal()
1179 rw_section_mac(ic, i, true); in write_journal()
1181 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 0, commit_sections - to_end, &io_comp); in write_journal()
1187 static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, in copy_from_journal() argument
1195 BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1)); in copy_from_journal()
1197 if (unlikely(dm_integrity_failed(ic))) { in copy_from_journal()
1202 sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset; in copy_from_journal()
1209 io_req.mem.ptr.pl = &ic->journal[pl_index]; in copy_from_journal()
1213 io_req.client = ic->io; in copy_from_journal()
1214 io_loc.bdev = ic->dev->bdev; in copy_from_journal()
1231 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool che… in add_new_range() argument
1233 struct rb_node **n = &ic->in_progress.rb_node; in add_new_range()
1236 …BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned int)(ic->sectors_per_block -… in add_new_range()
1241 list_for_each_entry(range, &ic->wait_list, wait_entry) { in add_new_range()
1262 rb_insert_color(&new_range->node, &ic->in_progress); in add_new_range()
1267 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range) in remove_range_unlocked() argument
1269 rb_erase(&range->node, &ic->in_progress); in remove_range_unlocked()
1270 while (unlikely(!list_empty(&ic->wait_list))) { in remove_range_unlocked()
1272 list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry); in remove_range_unlocked()
1277 if (!add_new_range(ic, last_range, false)) { in remove_range_unlocked()
1279 list_add(&last_range->wait_entry, &ic->wait_list); in remove_range_unlocked()
1287 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range) in remove_range() argument
1291 spin_lock_irqsave(&ic->endio_wait.lock, flags); in remove_range()
1292 remove_range_unlocked(ic, range); in remove_range()
1293 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); in remove_range()
1296 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) in wait_and_add_new_range() argument
1299 list_add_tail(&new_range->wait_entry, &ic->wait_list); in wait_and_add_new_range()
1303 spin_unlock_irq(&ic->endio_wait.lock); in wait_and_add_new_range()
1305 spin_lock_irq(&ic->endio_wait.lock); in wait_and_add_new_range()
1309 static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) in add_new_range_and_wait() argument
1311 if (unlikely(!add_new_range(ic, new_range, true))) in add_new_range_and_wait()
1312 wait_and_add_new_range(ic, new_range); in add_new_range_and_wait()
1321 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector) in add_journal_node() argument
1329 link = &ic->journal_tree_root.rb_node; in add_journal_node()
1344 rb_insert_color(&node->node, &ic->journal_tree_root); in add_journal_node()
1347 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node) in remove_journal_node() argument
1350 rb_erase(&node->node, &ic->journal_tree_root); in remove_journal_node()
1356 static unsigned int find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_se… in find_journal_node() argument
1358 struct rb_node *n = ic->journal_tree_root.rb_node; in find_journal_node()
1366 found = j - ic->journal_tree; in find_journal_node()
1378 static bool test_journal_node(struct dm_integrity_c *ic, unsigned int pos, sector_t sector) in test_journal_node() argument
1383 if (unlikely(pos >= ic->journal_entries)) in test_journal_node()
1385 node = &ic->journal_tree[pos]; in test_journal_node()
1399 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node) in find_newer_committed_node() argument
1416 next_section = (unsigned int)(next_node - ic->journal_tree) / ic->journal_section_entries; in find_newer_committed_node()
1417 if (next_section >= ic->committed_section && in find_newer_committed_node()
1418 next_section < ic->committed_section + ic->n_committed_sections) in find_newer_committed_node()
1420 if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections) in find_newer_committed_node()
1430 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_bl… in dm_integrity_rw_tag() argument
1436 unsigned int may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0); in dm_integrity_rw_tag()
1444 r = dm_integrity_failed(ic); in dm_integrity_rw_tag()
1448 data = dm_bufio_read(ic->bufio, *metadata_block, &b); in dm_integrity_rw_tag()
1452 to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size); in dm_integrity_rw_tag()
1464 if (likely(is_power_of_2(ic->tag_size))) { in dm_integrity_rw_tag()
1466 if (unlikely(!ic->discard) || in dm_integrity_rw_tag()
1481 if (unlikely(hash_offset == ic->tag_size)) { in dm_integrity_rw_tag()
1487 may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0); in dm_integrity_rw_tag()
1496 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) { in dm_integrity_rw_tag()
1501 if (unlikely(!is_power_of_2(ic->tag_size))) in dm_integrity_rw_tag()
1502 hash_offset = (hash_offset + to_copy) % ic->tag_size; in dm_integrity_rw_tag()
1515 struct dm_integrity_c *ic; member
1524 dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO); in flush_notify()
1528 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data) in dm_integrity_flush_buffers() argument
1533 if (!ic->meta_dev) in dm_integrity_flush_buffers()
1541 fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio), in dm_integrity_flush_buffers()
1542 fr.io_reg.bdev = ic->dev->bdev, in dm_integrity_flush_buffers()
1545 fr.ic = ic; in dm_integrity_flush_buffers()
1551 r = dm_bufio_write_dirty_buffers(ic->bufio); in dm_integrity_flush_buffers()
1553 dm_integrity_io_error(ic, "writing tags", r); in dm_integrity_flush_buffers()
1559 static void sleep_on_endio_wait(struct dm_integrity_c *ic) in sleep_on_endio_wait() argument
1563 __add_wait_queue(&ic->endio_wait, &wait); in sleep_on_endio_wait()
1565 spin_unlock_irq(&ic->endio_wait.lock); in sleep_on_endio_wait()
1567 spin_lock_irq(&ic->endio_wait.lock); in sleep_on_endio_wait()
1568 __remove_wait_queue(&ic->endio_wait, &wait); in sleep_on_endio_wait()
1573 struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer); in autocommit_fn() local
1575 if (likely(!dm_integrity_failed(ic))) in autocommit_fn()
1576 queue_work(ic->commit_wq, &ic->commit_work); in autocommit_fn()
1579 static void schedule_autocommit(struct dm_integrity_c *ic) in schedule_autocommit() argument
1581 if (!timer_pending(&ic->autocommit_timer)) in schedule_autocommit()
1582 mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies); in schedule_autocommit()
1585 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in submit_flush_bio() argument
1590 spin_lock_irqsave(&ic->endio_wait.lock, flags); in submit_flush_bio()
1592 bio_list_add(&ic->flush_bio_list, bio); in submit_flush_bio()
1593 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); in submit_flush_bio()
1595 queue_work(ic->commit_wq, &ic->commit_work); in submit_flush_bio()
1598 static void do_endio(struct dm_integrity_c *ic, struct bio *bio) in do_endio() argument
1602 r = dm_integrity_failed(ic); in do_endio()
1605 if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) { in do_endio()
1608 spin_lock_irqsave(&ic->endio_wait.lock, flags); in do_endio()
1609 bio_list_add(&ic->synchronous_bios, bio); in do_endio()
1610 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); in do_endio()
1611 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); in do_endio()
1617 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in do_endio_flush() argument
1621 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) in do_endio_flush()
1622 submit_flush_bio(ic, dio); in do_endio_flush()
1624 do_endio(ic, bio); in do_endio_flush()
1630 struct dm_integrity_c *ic = dio->ic; in dec_in_flight() local
1633 remove_range(ic, &dio->range); in dec_in_flight()
1636 schedule_autocommit(ic); in dec_in_flight()
1645 queue_work(ic->offload_wq, &dio->work); in dec_in_flight()
1648 do_endio_flush(ic, dio); in dec_in_flight()
1666 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector, in integrity_sector_checksum() argument
1670 SHASH_DESC_ON_STACK(req, ic->internal_hash); in integrity_sector_checksum()
1674 req->tfm = ic->internal_hash; in integrity_sector_checksum()
1678 dm_integrity_io_error(ic, "crypto_shash_init", r); in integrity_sector_checksum()
1682 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { in integrity_sector_checksum()
1683 r = crypto_shash_update(req, (__u8 *)&ic->sb->salt, SALT_SIZE); in integrity_sector_checksum()
1685 dm_integrity_io_error(ic, "crypto_shash_update", r); in integrity_sector_checksum()
1692 dm_integrity_io_error(ic, "crypto_shash_update", r); in integrity_sector_checksum()
1696 r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT); in integrity_sector_checksum()
1698 dm_integrity_io_error(ic, "crypto_shash_update", r); in integrity_sector_checksum()
1704 dm_integrity_io_error(ic, "crypto_shash_final", r); in integrity_sector_checksum()
1708 digest_size = crypto_shash_digestsize(ic->internal_hash); in integrity_sector_checksum()
1709 if (unlikely(digest_size < ic->tag_size)) in integrity_sector_checksum()
1710 memset(result + digest_size, 0, ic->tag_size - digest_size); in integrity_sector_checksum()
1716 get_random_bytes(result, ic->tag_size); in integrity_sector_checksum()
1722 struct dm_integrity_c *ic = dio->ic; in integrity_metadata() local
1726 if (ic->internal_hash) { in integrity_metadata()
1729 unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash); in integrity_metadata()
1732 unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; in integrity_metadata()
1737 if (unlikely(ic->mode == 'R')) in integrity_metadata()
1741 …checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size +… in integrity_metadata()
1757 unsigned int max_blocks = max_size / ic->tag_size; in integrity_metadata()
1762 unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); in integrity_metadata()
1765 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1766 this_step_blocks * ic->tag_size, TAG_WRITE); in integrity_metadata()
1773 bi_size -= this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); in integrity_metadata()
1793 integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr); in integrity_metadata()
1794 checksums_ptr += ic->tag_size; in integrity_metadata()
1795 sectors_to_process -= ic->sectors_per_block; in integrity_metadata()
1796 pos += ic->sectors_per_block << SECTOR_SHIFT; in integrity_metadata()
1797 sector += ic->sectors_per_block; in integrity_metadata()
1801 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1807 s = sector - ((r + ic->tag_size - 1) / ic->tag_size); in integrity_metadata()
1811 atomic64_inc(&ic->number_of_mismatches); in integrity_metadata()
1840 sector_to_block(ic, data_to_process); in integrity_metadata()
1841 data_to_process *= ic->tag_size; in integrity_metadata()
1850 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1870 struct dm_integrity_c *ic = ti->private; in dm_integrity_map() local
1876 dio->ic = ic; in dm_integrity_map()
1896 submit_flush_bio(ic, dio); in dm_integrity_map()
1909 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { in dm_integrity_map()
1912 ic->provided_data_sectors); in dm_integrity_map()
1915 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block… in dm_integrity_map()
1917 ic->sectors_per_block, in dm_integrity_map()
1922 if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map()
1927 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { in dm_integrity_map()
1929 bv.bv_offset, bv.bv_len, ic->sectors_per_block); in dm_integrity_map()
1936 if (!ic->internal_hash) { in dm_integrity_map()
1938 unsigned int wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block; in dm_integrity_map()
1940 if (ic->log2_tag_size >= 0) in dm_integrity_map()
1941 wanted_tag_size <<= ic->log2_tag_size; in dm_integrity_map()
1943 wanted_tag_size *= ic->tag_size; in dm_integrity_map()
1957 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) in dm_integrity_map()
1960 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in dm_integrity_map()
1961 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in dm_integrity_map()
1962 bio->bi_iter.bi_sector = get_data_sector(ic, area, offset); in dm_integrity_map()
1971 struct dm_integrity_c *ic = dio->ic; in __journal_read_write() local
1991 struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry); in __journal_read_write()
2002 __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); in __journal_read_write()
2007 js = access_journal_data(ic, journal_section, journal_entry); in __journal_read_write()
2015 } while (++s < ic->sectors_per_block); in __journal_read_write()
2017 if (ic->internal_hash) { in __journal_read_write()
2020 integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack); in __journal_read_write()
2021 if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { in __journal_read_write()
2031 if (!ic->internal_hash) { in __journal_read_write()
2033 unsigned int tag_todo = ic->tag_size; in __journal_read_write()
2034 char *tag_ptr = journal_entry_tag(ic, je); in __journal_read_write()
2060 js = access_journal_data(ic, journal_section, journal_entry); in __journal_read_write()
2061 memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT); in __journal_read_write()
2066 } while (++s < ic->sectors_per_block); in __journal_read_write()
2068 if (ic->internal_hash) { in __journal_read_write()
2069 unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash); in __journal_read_write()
2071 if (unlikely(digest_size > ic->tag_size)) { in __journal_read_write()
2074 integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack); in __journal_read_write()
2075 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size); in __journal_read_write()
2077 integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je)); in __journal_read_write()
2082 logical_sector += ic->sectors_per_block; in __journal_read_write()
2085 if (unlikely(journal_entry == ic->journal_section_entries)) { in __journal_read_write()
2088 wraparound_section(ic, &journal_section); in __journal_read_write()
2091 bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT; in __journal_read_write()
2092 } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT); in __journal_read_write()
2101 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait))) in __journal_read_write()
2102 wake_up(&ic->copy_to_journal_wait); in __journal_read_write()
2103 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) in __journal_read_write()
2104 queue_work(ic->commit_wq, &ic->commit_work); in __journal_read_write()
2106 schedule_autocommit(ic); in __journal_read_write()
2108 remove_range(ic, &dio->range); in __journal_read_write()
2114 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in __journal_read_write()
2115 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in __journal_read_write()
2124 struct dm_integrity_c *ic = dio->ic; in dm_integrity_map_continue() local
2130 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ; in dm_integrity_map_continue()
2132 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') in dm_integrity_map_continue()
2137 queue_work(ic->offload_wq, &dio->work); in dm_integrity_map_continue()
2142 spin_lock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2144 if (unlikely(dm_integrity_failed(ic))) { in dm_integrity_map_continue()
2145 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2146 do_endio(ic, bio); in dm_integrity_map_continue()
2151 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map_continue()
2157 (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block); in dm_integrity_map_continue()
2161 sleep_on_endio_wait(ic); in dm_integrity_map_continue()
2164 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; in dm_integrity_map_continue()
2165 ic->free_sectors -= range_sectors; in dm_integrity_map_continue()
2166 journal_section = ic->free_section; in dm_integrity_map_continue()
2167 journal_entry = ic->free_section_entry; in dm_integrity_map_continue()
2169 next_entry = ic->free_section_entry + range_sectors; in dm_integrity_map_continue()
2170 ic->free_section_entry = next_entry % ic->journal_section_entries; in dm_integrity_map_continue()
2171 ic->free_section += next_entry / ic->journal_section_entries; in dm_integrity_map_continue()
2172 ic->n_uncommitted_sections += next_entry / ic->journal_section_entries; in dm_integrity_map_continue()
2173 wraparound_section(ic, &ic->free_section); in dm_integrity_map_continue()
2175 pos = journal_section * ic->journal_section_entries + journal_entry; in dm_integrity_map_continue()
2182 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); in dm_integrity_map_continue()
2184 if (unlikely(pos >= ic->journal_entries)) in dm_integrity_map_continue()
2187 je = access_journal_entry(ic, ws, we); in dm_integrity_map_continue()
2191 if (unlikely(we == ic->journal_section_entries)) { in dm_integrity_map_continue()
2194 wraparound_section(ic, &ws); in dm_integrity_map_continue()
2196 } while ((i += ic->sectors_per_block) < dio->range.n_sectors); in dm_integrity_map_continue()
2198 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2203 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2211 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { in dm_integrity_map_continue()
2212 if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) in dm_integrity_map_continue()
2219 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_continue()
2227 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2229 queue_work(ic->wait_wq, &dio->work); in dm_integrity_map_continue()
2233 dio->range.n_sectors = ic->sectors_per_block; in dm_integrity_map_continue()
2234 wait_and_add_new_range(ic, &dio->range); in dm_integrity_map_continue()
2244 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2246 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2251 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) { in dm_integrity_map_continue()
2255 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2258 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2259 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2260 queue_work(ic->commit_wq, &ic->commit_work); in dm_integrity_map_continue()
2261 flush_workqueue(ic->commit_wq); in dm_integrity_map_continue()
2262 queue_work(ic->writer_wq, &ic->writer_work); in dm_integrity_map_continue()
2263 flush_workqueue(ic->writer_wq); in dm_integrity_map_continue()
2268 spin_unlock_irq(&ic->endio_wait.lock); in dm_integrity_map_continue()
2271 journal_section = journal_read_pos / ic->journal_section_entries; in dm_integrity_map_continue()
2272 journal_entry = journal_read_pos % ic->journal_section_entries; in dm_integrity_map_continue()
2276 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { in dm_integrity_map_continue()
2277 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2281 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); in dm_integrity_map_continue()
2285 queue_work(ic->writer_wq, &bbs->work); in dm_integrity_map_continue()
2299 bio_set_dev(bio, ic->dev->bdev); in dm_integrity_map_continue()
2305 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { in dm_integrity_map_continue()
2307 dm_integrity_flush_buffers(ic, false); in dm_integrity_map_continue()
2321 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && in dm_integrity_map_continue()
2322 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) in dm_integrity_map_continue()
2324 if (ic->mode == 'B') { in dm_integrity_map_continue()
2325 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2337 queue_work(ic->metadata_wq, &dio->work); in dm_integrity_map_continue()
2346 do_endio_flush(ic, dio); in dm_integrity_map_continue()
2357 static void pad_uncommitted(struct dm_integrity_c *ic) in pad_uncommitted() argument
2359 if (ic->free_section_entry) { in pad_uncommitted()
2360 ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry; in pad_uncommitted()
2361 ic->free_section_entry = 0; in pad_uncommitted()
2362 ic->free_section++; in pad_uncommitted()
2363 wraparound_section(ic, &ic->free_section); in pad_uncommitted()
2364 ic->n_uncommitted_sections++; in pad_uncommitted()
2366 if (WARN_ON(ic->journal_sections * ic->journal_section_entries != in pad_uncommitted()
2367 (ic->n_uncommitted_sections + ic->n_committed_sections) * in pad_uncommitted()
2368 ic->journal_section_entries + ic->free_sectors)) { in pad_uncommitted()
2372 ic->journal_sections, ic->journal_section_entries, in pad_uncommitted()
2373 ic->n_uncommitted_sections, ic->n_committed_sections, in pad_uncommitted()
2374 ic->journal_section_entries, ic->free_sectors); in pad_uncommitted()
2380 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work); in integrity_commit() local
2385 del_timer(&ic->autocommit_timer); in integrity_commit()
2387 spin_lock_irq(&ic->endio_wait.lock); in integrity_commit()
2388 flushes = bio_list_get(&ic->flush_bio_list); in integrity_commit()
2389 if (unlikely(ic->mode != 'J')) { in integrity_commit()
2390 spin_unlock_irq(&ic->endio_wait.lock); in integrity_commit()
2391 dm_integrity_flush_buffers(ic, true); in integrity_commit()
2395 pad_uncommitted(ic); in integrity_commit()
2396 commit_start = ic->uncommitted_section; in integrity_commit()
2397 commit_sections = ic->n_uncommitted_sections; in integrity_commit()
2398 spin_unlock_irq(&ic->endio_wait.lock); in integrity_commit()
2403 ic->wrote_to_journal = true; in integrity_commit()
2407 for (j = 0; j < ic->journal_section_entries; j++) { in integrity_commit()
2410 je = access_journal_entry(ic, i, j); in integrity_commit()
2411 io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); in integrity_commit()
2413 for (j = 0; j < ic->journal_section_sectors; j++) { in integrity_commit()
2416 js = access_journal(ic, i, j); in integrity_commit()
2417 js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq); in integrity_commit()
2420 if (unlikely(i >= ic->journal_sections)) in integrity_commit()
2421 ic->commit_seq = next_commit_seq(ic->commit_seq); in integrity_commit()
2422 wraparound_section(ic, &i); in integrity_commit()
2426 write_journal(ic, commit_start, commit_sections); in integrity_commit()
2428 spin_lock_irq(&ic->endio_wait.lock); in integrity_commit()
2429 ic->uncommitted_section += commit_sections; in integrity_commit()
2430 wraparound_section(ic, &ic->uncommitted_section); in integrity_commit()
2431 ic->n_uncommitted_sections -= commit_sections; in integrity_commit()
2432 ic->n_committed_sections += commit_sections; in integrity_commit()
2433 spin_unlock_irq(&ic->endio_wait.lock); in integrity_commit()
2435 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) in integrity_commit()
2436 queue_work(ic->writer_wq, &ic->writer_work); in integrity_commit()
2443 do_endio(ic, flushes); in integrity_commit()
2452 struct dm_integrity_c *ic = comp->ic; in complete_copy_from_journal() local
2454 remove_range(ic, &io->range); in complete_copy_from_journal()
2455 mempool_free(io, &ic->journal_io_mempool); in complete_copy_from_journal()
2457 dm_integrity_io_error(ic, "copying from journal", -EIO); in complete_copy_from_journal()
2461 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js, in restore_last_bytes() argument
2469 } while (++s < ic->sectors_per_block); in restore_last_bytes()
2472 static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start, in do_journal_write() argument
2481 comp.ic = ic; in do_journal_write()
2486 for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) { in do_journal_write()
2490 rw_section_mac(ic, i, false); in do_journal_write()
2491 for (j = 0; j < ic->journal_section_entries; j++) { in do_journal_write()
2492 struct journal_entry *je = access_journal_entry(ic, i, j); in do_journal_write()
2504 if (unlikely(sec & (unsigned int)(ic->sectors_per_block - 1))) { in do_journal_write()
2505 dm_integrity_io_error(ic, "invalid sector in journal", -EIO); in do_journal_write()
2506 sec &= ~(sector_t)(ic->sectors_per_block - 1); in do_journal_write()
2508 if (unlikely(sec >= ic->provided_data_sectors)) { in do_journal_write()
2513 get_area_and_offset(ic, sec, &area, &offset); in do_journal_write()
2514 restore_last_bytes(ic, access_journal_data(ic, i, j), je); in do_journal_write()
2515 for (k = j + 1; k < ic->journal_section_entries; k++) { in do_journal_write()
2516 struct journal_entry *je2 = access_journal_entry(ic, i, k); in do_journal_write()
2523 if (unlikely(sec2 >= ic->provided_data_sectors)) in do_journal_write()
2525 get_area_and_offset(ic, sec2, &area2, &offset2); in do_journal_write()
2526 if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block)) in do_journal_write()
2528 restore_last_bytes(ic, access_journal_data(ic, i, k), je2); in do_journal_write()
2532 io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO); in do_journal_write()
2535 io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block; in do_journal_write()
2537 spin_lock_irq(&ic->endio_wait.lock); in do_journal_write()
2538 add_new_range_and_wait(ic, &io->range); in do_journal_write()
2541 struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries]; in do_journal_write()
2544 while (j < k && find_newer_committed_node(ic, §ion_node[j])) { in do_journal_write()
2545 struct journal_entry *je2 = access_journal_entry(ic, i, j); in do_journal_write()
2548 remove_journal_node(ic, §ion_node[j]); in do_journal_write()
2550 sec += ic->sectors_per_block; in do_journal_write()
2551 offset += ic->sectors_per_block; in do_journal_write()
2553 while (j < k && find_newer_committed_node(ic, §ion_node[k - 1])) { in do_journal_write()
2554 struct journal_entry *je2 = access_journal_entry(ic, i, k - 1); in do_journal_write()
2557 remove_journal_node(ic, §ion_node[k - 1]); in do_journal_write()
2561 remove_range_unlocked(ic, &io->range); in do_journal_write()
2562 spin_unlock_irq(&ic->endio_wait.lock); in do_journal_write()
2563 mempool_free(io, &ic->journal_io_mempool); in do_journal_write()
2567 remove_journal_node(ic, §ion_node[l]); in do_journal_write()
2569 spin_unlock_irq(&ic->endio_wait.lock); in do_journal_write()
2571 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); in do_journal_write()
2574 struct journal_entry *je2 = access_journal_entry(ic, i, l); in do_journal_write()
2580 ic->internal_hash) { in do_journal_write()
2583 integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), in do_journal_write()
2584 (char *)access_journal_data(ic, i, l), test_tag); in do_journal_write()
2585 if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) { in do_journal_write()
2586 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ); in do_journal_write()
2587 dm_audit_log_target(DM_MSG_PREFIX, "integrity-replay-journal", ic->ti, 0); in do_journal_write()
2592 r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset, in do_journal_write()
2593 ic->tag_size, TAG_WRITE); in do_journal_write()
2595 dm_integrity_io_error(ic, "reading tags", r); in do_journal_write()
2599 copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block, in do_journal_write()
2600 (k - j) << ic->sb->log2_sectors_per_block, in do_journal_write()
2601 get_data_sector(ic, area, offset), in do_journal_write()
2608 dm_bufio_write_dirty_buffers_async(ic->bufio); in do_journal_write()
2615 dm_integrity_flush_buffers(ic, true); in do_journal_write()
2620 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work); in integrity_writer() local
2624 spin_lock_irq(&ic->endio_wait.lock); in integrity_writer()
2625 write_start = ic->committed_section; in integrity_writer()
2626 write_sections = ic->n_committed_sections; in integrity_writer()
2627 spin_unlock_irq(&ic->endio_wait.lock); in integrity_writer()
2632 do_journal_write(ic, write_start, write_sections, false); in integrity_writer()
2634 spin_lock_irq(&ic->endio_wait.lock); in integrity_writer()
2636 ic->committed_section += write_sections; in integrity_writer()
2637 wraparound_section(ic, &ic->committed_section); in integrity_writer()
2638 ic->n_committed_sections -= write_sections; in integrity_writer()
2640 prev_free_sectors = ic->free_sectors; in integrity_writer()
2641 ic->free_sectors += write_sections * ic->journal_section_entries; in integrity_writer()
2643 wake_up_locked(&ic->endio_wait); in integrity_writer()
2645 spin_unlock_irq(&ic->endio_wait.lock); in integrity_writer()
2648 static void recalc_write_super(struct dm_integrity_c *ic) in recalc_write_super() argument
2652 dm_integrity_flush_buffers(ic, false); in recalc_write_super()
2653 if (dm_integrity_failed(ic)) in recalc_write_super()
2656 r = sync_rw_sb(ic, REQ_OP_WRITE); in recalc_write_super()
2658 dm_integrity_io_error(ic, "writing superblock", r); in recalc_write_super()
2663 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); in integrity_recalc() local
2676 DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector)); in integrity_recalc()
2678 spin_lock_irq(&ic->endio_wait.lock); in integrity_recalc()
2682 if (unlikely(dm_post_suspending(ic->ti))) in integrity_recalc()
2685 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); in integrity_recalc()
2686 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) { in integrity_recalc()
2687 if (ic->mode == 'B') { in integrity_recalc()
2688 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in integrity_recalc()
2690 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); in integrity_recalc()
2695 get_area_and_offset(ic, range.logical_sector, &area, &offset); in integrity_recalc()
2696 range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector); in integrity_recalc()
2697 if (!ic->meta_dev) in integrity_recalc()
2698 …range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsign… in integrity_recalc()
2700 add_new_range_and_wait(ic, &range); in integrity_recalc()
2701 spin_unlock_irq(&ic->endio_wait.lock); in integrity_recalc()
2705 if (ic->mode == 'B') { in integrity_recalc()
2706 if (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) in integrity_recalc()
2709 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, in integrity_recalc()
2710 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { in integrity_recalc()
2711 logical_sector += ic->sectors_per_block; in integrity_recalc()
2712 n_sectors -= ic->sectors_per_block; in integrity_recalc()
2715 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector + n_sectors - ic->sectors_per_block, in integrity_recalc()
2716 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { in integrity_recalc()
2717 n_sectors -= ic->sectors_per_block; in integrity_recalc()
2720 get_area_and_offset(ic, logical_sector, &area, &offset); in integrity_recalc()
2726 recalc_write_super(ic); in integrity_recalc()
2727 if (ic->mode == 'B') in integrity_recalc()
2728 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); in integrity_recalc()
2733 if (unlikely(dm_integrity_failed(ic))) in integrity_recalc()
2738 io_req.mem.ptr.addr = ic->recalc_buffer; in integrity_recalc()
2740 io_req.client = ic->io; in integrity_recalc()
2741 io_loc.bdev = ic->dev->bdev; in integrity_recalc()
2742 io_loc.sector = get_data_sector(ic, area, offset); in integrity_recalc()
2747 dm_integrity_io_error(ic, "reading data", r); in integrity_recalc()
2751 t = ic->recalc_tags; in integrity_recalc()
2752 for (i = 0; i < n_sectors; i += ic->sectors_per_block) { in integrity_recalc()
2753 integrity_sector_checksum(ic, logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t); in integrity_recalc()
2754 t += ic->tag_size; in integrity_recalc()
2757 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); in integrity_recalc()
2759 …r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tag… in integrity_recalc()
2761 dm_integrity_io_error(ic, "writing tags", r); in integrity_recalc()
2765 if (ic->mode == 'B') { in integrity_recalc()
2769 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << in integrity_recalc()
2770 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in integrity_recalc()
2772 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << in integrity_recalc()
2773 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in integrity_recalc()
2774 block_bitmap_op(ic, ic->recalc_bitmap, start, end - start, BITMAP_OP_CLEAR); in integrity_recalc()
2780 spin_lock_irq(&ic->endio_wait.lock); in integrity_recalc()
2781 remove_range_unlocked(ic, &range); in integrity_recalc()
2782 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); in integrity_recalc()
2786 remove_range(ic, &range); in integrity_recalc()
2790 spin_unlock_irq(&ic->endio_wait.lock); in integrity_recalc()
2792 recalc_write_super(ic); in integrity_recalc()
2798 struct dm_integrity_c *ic = bbs->ic; in bitmap_block_work() local
2815 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2817 remove_range(ic, &dio->range); in bitmap_block_work()
2819 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()
2821 block_bitmap_op(ic, ic->journal, dio->range.logical_sector, in bitmap_block_work()
2830 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, in bitmap_block_work()
2837 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2840 remove_range(ic, &dio->range); in bitmap_block_work()
2842 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()
2845 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); in bitmap_block_work()
2850 struct dm_integrity_c *ic = container_of(work, struct dm_integrity_c, bitmap_flush_work.work); in bitmap_flush_work() local
2855 dm_integrity_flush_buffers(ic, false); in bitmap_flush_work()
2858 range.n_sectors = ic->provided_data_sectors; in bitmap_flush_work()
2860 spin_lock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2861 add_new_range_and_wait(ic, &range); in bitmap_flush_work()
2862 spin_unlock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2864 dm_integrity_flush_buffers(ic, true); in bitmap_flush_work()
2866 limit = ic->provided_data_sectors; in bitmap_flush_work()
2867 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { in bitmap_flush_work()
2868 limit = le64_to_cpu(ic->sb->recalc_sector) in bitmap_flush_work()
2869 >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit) in bitmap_flush_work()
2870 << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); in bitmap_flush_work()
2873 block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR); in bitmap_flush_work()
2874 block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR); in bitmap_flush_work()
2876 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, in bitmap_flush_work()
2877 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in bitmap_flush_work()
2879 spin_lock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2880 remove_range_unlocked(ic, &range); in bitmap_flush_work()
2881 while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) { in bitmap_flush_work()
2883 spin_unlock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2884 spin_lock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2886 spin_unlock_irq(&ic->endio_wait.lock); in bitmap_flush_work()
2890 static void init_journal(struct dm_integrity_c *ic, unsigned int start_section, in init_journal() argument
2900 wraparound_section(ic, &i); in init_journal()
2901 for (j = 0; j < ic->journal_section_sectors; j++) { in init_journal()
2902 struct journal_sector *js = access_journal(ic, i, j); in init_journal()
2906 js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq); in init_journal()
2908 for (j = 0; j < ic->journal_section_entries; j++) { in init_journal()
2909 struct journal_entry *je = access_journal_entry(ic, i, j); in init_journal()
2915 write_journal(ic, start_section, n_sections); in init_journal()
2918 static int find_commit_seq(struct dm_integrity_c *ic, unsigned int i, unsigned int j, commit_id_t i… in find_commit_seq() argument
2923 if (dm_integrity_commit_id(ic, i, j, k) == id) in find_commit_seq()
2926 dm_integrity_io_error(ic, "journal commit id", -EIO); in find_commit_seq()
2930 static void replay_journal(struct dm_integrity_c *ic) in replay_journal() argument
2940 if (ic->mode == 'R') in replay_journal()
2943 if (ic->journal_uptodate) in replay_journal()
2949 if (!ic->just_formatted) { in replay_journal()
2951 rw_journal(ic, REQ_OP_READ, 0, ic->journal_sections, NULL); in replay_journal()
2952 if (ic->journal_io) in replay_journal()
2953 DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal"); in replay_journal()
2954 if (ic->journal_io) { in replay_journal()
2957 crypt_comp.ic = ic; in replay_journal()
2960 encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp); in replay_journal()
2963 DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal"); in replay_journal()
2966 if (dm_integrity_failed(ic)) in replay_journal()
2972 for (i = 0; i < ic->journal_sections; i++) { in replay_journal()
2973 for (j = 0; j < ic->journal_section_sectors; j++) { in replay_journal()
2975 struct journal_sector *js = access_journal(ic, i, j); in replay_journal()
2977 k = find_commit_seq(ic, i, j, js->commit_id); in replay_journal()
2984 for (j = 0; j < ic->journal_section_entries; j++) { in replay_journal()
2985 struct journal_entry *je = access_journal_entry(ic, i, j); in replay_journal()
3004 dm_integrity_io_error(ic, "journal commit ids", -EIO); in replay_journal()
3019 if (unlikely(write_start >= ic->journal_sections)) in replay_journal()
3021 wraparound_section(ic, &write_start); in replay_journal()
3024 for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) { in replay_journal()
3025 for (j = 0; j < ic->journal_section_sectors; j++) { in replay_journal()
3026 struct journal_sector *js = access_journal(ic, i, j); in replay_journal()
3028 if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) { in replay_journal()
3035 i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq); in replay_journal()
3040 if (unlikely(i >= ic->journal_sections)) in replay_journal()
3042 wraparound_section(ic, &i); in replay_journal()
3049 do_journal_write(ic, write_start, write_sections, true); in replay_journal()
3052 if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) { in replay_journal()
3054 ic->commit_seq = want_commit_seq; in replay_journal()
3055 DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq); in replay_journal()
3065 init_journal(ic, s, 1, erase_seq); in replay_journal()
3067 wraparound_section(ic, &s); in replay_journal()
3068 if (ic->journal_sections >= 2) { in replay_journal()
3069 init_journal(ic, s, ic->journal_sections - 2, erase_seq); in replay_journal()
3070 s += ic->journal_sections - 2; in replay_journal()
3071 wraparound_section(ic, &s); in replay_journal()
3072 init_journal(ic, s, 1, erase_seq); in replay_journal()
3076 ic->commit_seq = next_commit_seq(erase_seq); in replay_journal()
3079 ic->committed_section = continue_section; in replay_journal()
3080 ic->n_committed_sections = 0; in replay_journal()
3082 ic->uncommitted_section = continue_section; in replay_journal()
3083 ic->n_uncommitted_sections = 0; in replay_journal()
3085 ic->free_section = continue_section; in replay_journal()
3086 ic->free_section_entry = 0; in replay_journal()
3087 ic->free_sectors = ic->journal_entries; in replay_journal()
3089 ic->journal_tree_root = RB_ROOT; in replay_journal()
3090 for (i = 0; i < ic->journal_entries; i++) in replay_journal()
3091 init_journal_node(&ic->journal_tree[i]); in replay_journal()
3094 static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic) in dm_integrity_enter_synchronous_mode() argument
3098 if (ic->mode == 'B') { in dm_integrity_enter_synchronous_mode()
3099 ic->bitmap_flush_interval = msecs_to_jiffies(10) + 1; in dm_integrity_enter_synchronous_mode()
3100 ic->synchronous_mode = 1; in dm_integrity_enter_synchronous_mode()
3102 cancel_delayed_work_sync(&ic->bitmap_flush_work); in dm_integrity_enter_synchronous_mode()
3103 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); in dm_integrity_enter_synchronous_mode()
3104 flush_workqueue(ic->commit_wq); in dm_integrity_enter_synchronous_mode()
3110 struct dm_integrity_c *ic = container_of(n, struct dm_integrity_c, reboot_notifier); in dm_integrity_reboot() local
3114 dm_integrity_enter_synchronous_mode(ic); in dm_integrity_reboot()
3121 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; in dm_integrity_postsuspend() local
3124 WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier)); in dm_integrity_postsuspend()
3126 del_timer_sync(&ic->autocommit_timer); in dm_integrity_postsuspend()
3128 if (ic->recalc_wq) in dm_integrity_postsuspend()
3129 drain_workqueue(ic->recalc_wq); in dm_integrity_postsuspend()
3131 if (ic->mode == 'B') in dm_integrity_postsuspend()
3132 cancel_delayed_work_sync(&ic->bitmap_flush_work); in dm_integrity_postsuspend()
3134 queue_work(ic->commit_wq, &ic->commit_work); in dm_integrity_postsuspend()
3135 drain_workqueue(ic->commit_wq); in dm_integrity_postsuspend()
3137 if (ic->mode == 'J') { in dm_integrity_postsuspend()
3138 queue_work(ic->writer_wq, &ic->writer_work); in dm_integrity_postsuspend()
3139 drain_workqueue(ic->writer_wq); in dm_integrity_postsuspend()
3140 dm_integrity_flush_buffers(ic, true); in dm_integrity_postsuspend()
3141 if (ic->wrote_to_journal) { in dm_integrity_postsuspend()
3142 init_journal(ic, ic->free_section, in dm_integrity_postsuspend()
3143 ic->journal_sections - ic->free_section, ic->commit_seq); in dm_integrity_postsuspend()
3144 if (ic->free_section) { in dm_integrity_postsuspend()
3145 init_journal(ic, 0, ic->free_section, in dm_integrity_postsuspend()
3146 next_commit_seq(ic->commit_seq)); in dm_integrity_postsuspend()
3151 if (ic->mode == 'B') { in dm_integrity_postsuspend()
3152 dm_integrity_flush_buffers(ic, true); in dm_integrity_postsuspend()
3155 init_journal(ic, 0, ic->journal_sections, 0); in dm_integrity_postsuspend()
3156 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); in dm_integrity_postsuspend()
3157 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); in dm_integrity_postsuspend()
3159 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_postsuspend()
3163 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); in dm_integrity_postsuspend()
3165 ic->journal_uptodate = true; in dm_integrity_postsuspend()
3170 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; in dm_integrity_resume() local
3171 __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); in dm_integrity_resume()
3176 ic->wrote_to_journal = false; in dm_integrity_resume()
3178 if (ic->provided_data_sectors != old_provided_data_sectors) { in dm_integrity_resume()
3179 if (ic->provided_data_sectors > old_provided_data_sectors && in dm_integrity_resume()
3180 ic->mode == 'B' && in dm_integrity_resume()
3181 ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { in dm_integrity_resume()
3182 rw_journal_sectors(ic, REQ_OP_READ, 0, in dm_integrity_resume()
3183 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3184 block_bitmap_op(ic, ic->journal, old_provided_data_sectors, in dm_integrity_resume()
3185 ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3186 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, in dm_integrity_resume()
3187 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3190 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); in dm_integrity_resume()
3191 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); in dm_integrity_resume()
3193 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_resume()
3196 if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) { in dm_integrity_resume()
3198 rw_journal_sectors(ic, REQ_OP_READ, 0, in dm_integrity_resume()
3199 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3200 if (ic->mode == 'B') { in dm_integrity_resume()
3201 if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && in dm_integrity_resume()
3202 !ic->reset_recalculate_flag) { in dm_integrity_resume()
3203 block_bitmap_copy(ic, ic->recalc_bitmap, ic->journal); in dm_integrity_resume()
3204 block_bitmap_copy(ic, ic->may_write_bitmap, ic->journal); in dm_integrity_resume()
3205 if (!block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, in dm_integrity_resume()
3207 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_resume()
3208 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_resume()
3212 ic->sb->log2_blocks_per_bitmap_bit, ic->log2_blocks_per_bitmap_bit); in dm_integrity_resume()
3213 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; in dm_integrity_resume()
3214 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3215 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3216 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET); in dm_integrity_resume()
3217 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, in dm_integrity_resume()
3218 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3219 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_resume()
3220 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_resume()
3223 if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && in dm_integrity_resume()
3224 … block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR)) || in dm_integrity_resume()
3225 ic->reset_recalculate_flag) { in dm_integrity_resume()
3226 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_resume()
3227 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_resume()
3229 init_journal(ic, 0, ic->journal_sections, 0); in dm_integrity_resume()
3230 replay_journal(ic); in dm_integrity_resume()
3231 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); in dm_integrity_resume()
3233 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); in dm_integrity_resume()
3235 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_resume()
3237 replay_journal(ic); in dm_integrity_resume()
3238 if (ic->reset_recalculate_flag) { in dm_integrity_resume()
3239 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_resume()
3240 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_resume()
3242 if (ic->mode == 'B') { in dm_integrity_resume()
3243 ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP); in dm_integrity_resume()
3244 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; in dm_integrity_resume()
3245 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); in dm_integrity_resume()
3247 dm_integrity_io_error(ic, "writing superblock", r); in dm_integrity_resume()
3249 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in dm_integrity_resume()
3250 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in dm_integrity_resume()
3251 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); in dm_integrity_resume()
3252 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && in dm_integrity_resume()
3253 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors) { in dm_integrity_resume()
3254 block_bitmap_op(ic, ic->journal, le64_to_cpu(ic->sb->recalc_sector), in dm_integrity_resume()
3255 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); in dm_integrity_resume()
3256 block_bitmap_op(ic, ic->recalc_bitmap, le64_to_cpu(ic->sb->recalc_sector), in dm_integrity_resume()
3257 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); in dm_integrity_resume()
3258 block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector), in dm_integrity_resume()
3259 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); in dm_integrity_resume()
3261 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, in dm_integrity_resume()
3262 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); in dm_integrity_resume()
3266 DEBUG_print("testing recalc: %x\n", ic->sb->flags); in dm_integrity_resume()
3267 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { in dm_integrity_resume()
3268 __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector); in dm_integrity_resume()
3270 DEBUG_print("recalc pos: %llx / %llx\n", recalc_pos, ic->provided_data_sectors); in dm_integrity_resume()
3271 if (recalc_pos < ic->provided_data_sectors) { in dm_integrity_resume()
3272 queue_work(ic->recalc_wq, &ic->recalc_work); in dm_integrity_resume()
3273 } else if (recalc_pos > ic->provided_data_sectors) { in dm_integrity_resume()
3274 ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors); in dm_integrity_resume()
3275 recalc_write_super(ic); in dm_integrity_resume()
3279 ic->reboot_notifier.notifier_call = dm_integrity_reboot; in dm_integrity_resume()
3280 ic->reboot_notifier.next = NULL; in dm_integrity_resume()
3281 ic->reboot_notifier.priority = INT_MAX - 1; /* be notified after md and before hardware drivers */ in dm_integrity_resume()
3282 WARN_ON(register_reboot_notifier(&ic->reboot_notifier)); in dm_integrity_resume()
3286 dm_integrity_enter_synchronous_mode(ic); in dm_integrity_resume()
3293 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; in dm_integrity_status() local
3300 (unsigned long long)atomic64_read(&ic->number_of_mismatches), in dm_integrity_status()
3301 ic->provided_data_sectors); in dm_integrity_status()
3302 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) in dm_integrity_status()
3303 DMEMIT(" %llu", le64_to_cpu(ic->sb->recalc_sector)); in dm_integrity_status()
3309 __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100; in dm_integrity_status()
3311 watermark_percentage += ic->journal_entries / 2; in dm_integrity_status()
3312 do_div(watermark_percentage, ic->journal_entries); in dm_integrity_status()
3314 arg_count += !!ic->meta_dev; in dm_integrity_status()
3315 arg_count += ic->sectors_per_block != 1; in dm_integrity_status()
3316 arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); in dm_integrity_status()
3317 arg_count += ic->reset_recalculate_flag; in dm_integrity_status()
3318 arg_count += ic->discard; in dm_integrity_status()
3319 arg_count += ic->mode == 'J'; in dm_integrity_status()
3320 arg_count += ic->mode == 'J'; in dm_integrity_status()
3321 arg_count += ic->mode == 'B'; in dm_integrity_status()
3322 arg_count += ic->mode == 'B'; in dm_integrity_status()
3323 arg_count += !!ic->internal_hash_alg.alg_string; in dm_integrity_status()
3324 arg_count += !!ic->journal_crypt_alg.alg_string; in dm_integrity_status()
3325 arg_count += !!ic->journal_mac_alg.alg_string; in dm_integrity_status()
3326 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0; in dm_integrity_status()
3327 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0; in dm_integrity_status()
3328 arg_count += ic->legacy_recalculate; in dm_integrity_status()
3329 DMEMIT("%s %llu %u %c %u", ic->dev->name, ic->start, in dm_integrity_status()
3330 ic->tag_size, ic->mode, arg_count); in dm_integrity_status()
3331 if (ic->meta_dev) in dm_integrity_status()
3332 DMEMIT(" meta_device:%s", ic->meta_dev->name); in dm_integrity_status()
3333 if (ic->sectors_per_block != 1) in dm_integrity_status()
3334 DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT); in dm_integrity_status()
3335 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) in dm_integrity_status()
3337 if (ic->reset_recalculate_flag) in dm_integrity_status()
3339 if (ic->discard) in dm_integrity_status()
3341 DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS); in dm_integrity_status()
3342 DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); in dm_integrity_status()
3343 DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); in dm_integrity_status()
3344 if (ic->mode == 'J') { in dm_integrity_status()
3346 DMEMIT(" commit_time:%u", ic->autocommit_msec); in dm_integrity_status()
3348 if (ic->mode == 'B') { in dm_integrity_status()
3349 …DMEMIT(" sectors_per_bit:%llu", (sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit); in dm_integrity_status()
3350 DMEMIT(" bitmap_flush_interval:%u", jiffies_to_msecs(ic->bitmap_flush_interval)); in dm_integrity_status()
3352 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) in dm_integrity_status()
3354 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) in dm_integrity_status()
3356 if (ic->legacy_recalculate) in dm_integrity_status()
3361 if (ic->a.alg_string) { \ in dm_integrity_status()
3362 DMEMIT(" %s:%s", n, ic->a.alg_string); \ in dm_integrity_status()
3363 if (ic->a.key_string) \ in dm_integrity_status()
3364 DMEMIT(":%s", ic->a.key_string);\ in dm_integrity_status()
3375 ic->dev->name, ic->start, ic->tag_size, ic->mode); in dm_integrity_status()
3377 if (ic->meta_dev) in dm_integrity_status()
3378 DMEMIT(",meta_device=%s", ic->meta_dev->name); in dm_integrity_status()
3379 if (ic->sectors_per_block != 1) in dm_integrity_status()
3380 DMEMIT(",block_size=%u", ic->sectors_per_block << SECTOR_SHIFT); in dm_integrity_status()
3382 DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ? in dm_integrity_status()
3384 DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n'); in dm_integrity_status()
3386 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n'); in dm_integrity_status()
3388 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) ? 'y' : 'n'); in dm_integrity_status()
3389 DMEMIT(",legacy_recalculate=%c", ic->legacy_recalculate ? 'y' : 'n'); in dm_integrity_status()
3391 DMEMIT(",journal_sectors=%u", ic->initial_sectors - SB_SECTORS); in dm_integrity_status()
3392 DMEMIT(",interleave_sectors=%u", 1U << ic->sb->log2_interleave_sectors); in dm_integrity_status()
3393 DMEMIT(",buffer_sectors=%u", 1U << ic->log2_buffer_sectors); in dm_integrity_status()
3402 struct dm_integrity_c *ic = ti->private; in dm_integrity_iterate_devices() local
3404 if (!ic->meta_dev) in dm_integrity_iterate_devices()
3405 return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data); in dm_integrity_iterate_devices()
3407 return fn(ti, ic->dev, 0, ti->len, data); in dm_integrity_iterate_devices()
3412 struct dm_integrity_c *ic = ti->private; in dm_integrity_io_hints() local
3414 if (ic->sectors_per_block > 1) { in dm_integrity_io_hints()
3415 limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT; in dm_integrity_io_hints()
3416 limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT; in dm_integrity_io_hints()
3417 blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT); in dm_integrity_io_hints()
3422 static void calculate_journal_section_size(struct dm_integrity_c *ic) in calculate_journal_section_size() argument
3426 ic->journal_sections = le32_to_cpu(ic->sb->journal_sections); in calculate_journal_section_size()
3427 …ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block])… in calculate_journal_section_size()
3430 if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) in calculate_journal_section_size()
3432 ic->journal_entries_per_sector = sector_space / ic->journal_entry_size; in calculate_journal_section_size()
3433 ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS; in calculate_journal_section_size()
3434 …ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JO… in calculate_journal_section_size()
3435 ic->journal_entries = ic->journal_section_entries * ic->journal_sections; in calculate_journal_section_size()
3438 static int calculate_device_limits(struct dm_integrity_c *ic) in calculate_device_limits() argument
3442 calculate_journal_section_size(ic); in calculate_device_limits()
3443 initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections; in calculate_device_limits()
3444 …if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UIN… in calculate_device_limits()
3446 ic->initial_sectors = initial_sectors; in calculate_device_limits()
3448 if (!ic->meta_dev) { in calculate_device_limits()
3453 ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING) ? in calculate_device_limits()
3457 …ic->metadata_run = round_up((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2… in calculate_device_limits()
3459 if (!(ic->metadata_run & (ic->metadata_run - 1))) in calculate_device_limits()
3460 ic->log2_metadata_run = __ffs(ic->metadata_run); in calculate_device_limits()
3462 ic->log2_metadata_run = -1; in calculate_device_limits()
3464 get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset); in calculate_device_limits()
3465 last_sector = get_data_sector(ic, last_area, last_offset); in calculate_device_limits()
3466 if (last_sector < ic->start || last_sector >= ic->meta_device_sectors) in calculate_device_limits()
3469 __u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; in calculate_device_limits()
3471 meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1)) in calculate_device_limits()
3472 >> (ic->log2_buffer_sectors + SECTOR_SHIFT); in calculate_device_limits()
3473 meta_size <<= ic->log2_buffer_sectors; in calculate_device_limits()
3474 if (ic->initial_sectors + meta_size < ic->initial_sectors || in calculate_device_limits()
3475 ic->initial_sectors + meta_size > ic->meta_device_sectors) in calculate_device_limits()
3477 ic->metadata_run = 1; in calculate_device_limits()
3478 ic->log2_metadata_run = 0; in calculate_device_limits()
3484 static void get_provided_data_sectors(struct dm_integrity_c *ic) in get_provided_data_sectors() argument
3486 if (!ic->meta_dev) { in get_provided_data_sectors()
3489 ic->provided_data_sectors = 0; in get_provided_data_sectors()
3490 for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) { in get_provided_data_sectors()
3491 __u64 prev_data_sectors = ic->provided_data_sectors; in get_provided_data_sectors()
3493 ic->provided_data_sectors |= (sector_t)1 << test_bit; in get_provided_data_sectors()
3494 if (calculate_device_limits(ic)) in get_provided_data_sectors()
3495 ic->provided_data_sectors = prev_data_sectors; in get_provided_data_sectors()
3498 ic->provided_data_sectors = ic->data_device_sectors; in get_provided_data_sectors()
3499 ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1); in get_provided_data_sectors()
3503 static int initialize_superblock(struct dm_integrity_c *ic, in initialize_superblock() argument
3509 memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT); in initialize_superblock()
3510 memcpy(ic->sb->magic, SB_MAGIC, 8); in initialize_superblock()
3511 ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size); in initialize_superblock()
3512 ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block); in initialize_superblock()
3513 if (ic->journal_mac_alg.alg_string) in initialize_superblock()
3514 ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC); in initialize_superblock()
3516 calculate_journal_section_size(ic); in initialize_superblock()
3517 journal_sections = journal_sectors / ic->journal_section_sectors; in initialize_superblock()
3521 if (ic->fix_hmac && (ic->internal_hash_alg.alg_string || ic->journal_mac_alg.alg_string)) { in initialize_superblock()
3522 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_HMAC); in initialize_superblock()
3523 get_random_bytes(ic->sb->salt, SALT_SIZE); in initialize_superblock()
3526 if (!ic->meta_dev) { in initialize_superblock()
3527 if (ic->fix_padding) in initialize_superblock()
3528 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING); in initialize_superblock()
3529 ic->sb->journal_sections = cpu_to_le32(journal_sections); in initialize_superblock()
3532 ic->sb->log2_interleave_sectors = __fls(interleave_sectors); in initialize_superblock()
3533 …ic->sb->log2_interleave_sectors = max_t(__u8, MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave… in initialize_superblock()
3534 …ic->sb->log2_interleave_sectors = min_t(__u8, MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave… in initialize_superblock()
3536 get_provided_data_sectors(ic); in initialize_superblock()
3537 if (!ic->provided_data_sectors) in initialize_superblock()
3540 ic->sb->log2_interleave_sectors = 0; in initialize_superblock()
3542 get_provided_data_sectors(ic); in initialize_superblock()
3543 if (!ic->provided_data_sectors) in initialize_superblock()
3547 ic->sb->journal_sections = cpu_to_le32(0); in initialize_superblock()
3549 __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections); in initialize_superblock()
3554 ic->sb->journal_sections = cpu_to_le32(test_journal_sections); in initialize_superblock()
3555 if (calculate_device_limits(ic)) in initialize_superblock()
3556 ic->sb->journal_sections = cpu_to_le32(prev_journal_sections); in initialize_superblock()
3559 if (!le32_to_cpu(ic->sb->journal_sections)) { in initialize_superblock()
3560 if (ic->log2_buffer_sectors > 3) { in initialize_superblock()
3561 ic->log2_buffer_sectors--; in initialize_superblock()
3568 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); in initialize_superblock()
3570 sb_set_version(ic); in initialize_superblock()
3575 static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic) in dm_integrity_set() argument
3582 bi.tuple_size = ic->tag_size; in dm_integrity_set()
3584 bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT; in dm_integrity_set()
3625 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **s… in dm_integrity_free_journal_scatterlist() argument
3629 for (i = 0; i < ic->journal_sections; i++) in dm_integrity_free_journal_scatterlist()
3634 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, in dm_integrity_alloc_journal_scatterlist() argument
3640 sl = kvmalloc_array(ic->journal_sections, in dm_integrity_alloc_journal_scatterlist()
3646 for (i = 0; i < ic->journal_sections; i++) { in dm_integrity_alloc_journal_scatterlist()
3653 page_list_location(ic, i, 0, &start_index, &start_offset); in dm_integrity_alloc_journal_scatterlist()
3654 page_list_location(ic, i, ic->journal_section_sectors - 1, in dm_integrity_alloc_journal_scatterlist()
3662 dm_integrity_free_journal_scatterlist(ic, sl); in dm_integrity_alloc_journal_scatterlist()
3754 static int create_journal(struct dm_integrity_c *ic, char **error) in create_journal() argument
3762 ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL); in create_journal()
3763 ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL); in create_journal()
3764 ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL); in create_journal()
3765 ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL); in create_journal()
3767 journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors, in create_journal()
3775 ic->journal_pages = journal_pages; in create_journal()
3777 ic->journal = dm_integrity_alloc_page_list(ic->journal_pages); in create_journal()
3778 if (!ic->journal) { in create_journal()
3783 if (ic->journal_crypt_alg.alg_string) { in create_journal()
3787 comp.ic = ic; in create_journal()
3788 …ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, CRYPTO_ALG_ALLOCATE… in create_journal()
3789 if (IS_ERR(ic->journal_crypt)) { in create_journal()
3791 r = PTR_ERR(ic->journal_crypt); in create_journal()
3792 ic->journal_crypt = NULL; in create_journal()
3795 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); in create_journal()
3796 blocksize = crypto_skcipher_blocksize(ic->journal_crypt); in create_journal()
3798 if (ic->journal_crypt_alg.key) { in create_journal()
3799 r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key, in create_journal()
3800 ic->journal_crypt_alg.key_size); in create_journal()
3807 ic->journal_crypt_alg.alg_string, blocksize, ivsize); in create_journal()
3809 ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages); in create_journal()
3810 if (!ic->journal_io) { in create_journal()
3819 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); in create_journal()
3833 ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages); in create_journal()
3834 if (!ic->journal_xor) { in create_journal()
3840 sg = kvmalloc_array(ic->journal_pages + 1, in create_journal()
3848 sg_init_table(sg, ic->journal_pages + 1); in create_journal()
3849 for (i = 0; i < ic->journal_pages; i++) { in create_journal()
3850 char *va = lowmem_page_address(ic->journal_xor[i].page); in create_journal()
3855 sg_set_buf(&sg[i], &ic->commit_ids, sizeof(ic->commit_ids)); in create_journal()
3858 PAGE_SIZE * ic->journal_pages + sizeof(ic->commit_ids), crypt_iv); in create_journal()
3864 r = dm_integrity_failed(ic); in create_journal()
3869 DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data"); in create_journal()
3871 crypto_free_skcipher(ic->journal_crypt); in create_journal()
3872 ic->journal_crypt = NULL; in create_journal()
3876 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); in create_journal()
3897 ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal); in create_journal()
3898 if (!ic->journal_scatterlist) { in create_journal()
3903 ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io); in create_journal()
3904 if (!ic->journal_io_scatterlist) { in create_journal()
3909 ic->sk_requests = kvmalloc_array(ic->journal_sections, in create_journal()
3912 if (!ic->sk_requests) { in create_journal()
3917 for (i = 0; i < ic->journal_sections; i++) { in create_journal()
3933 r = dm_integrity_failed(ic); in create_journal()
3939 section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); in create_journal()
3954 section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT; in create_journal()
3955 ic->sk_requests[i] = section_req; in create_journal()
3966 if (ic->commit_ids[j] == ic->commit_ids[i]) { in create_journal()
3967 ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1); in create_journal()
3971 DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]); in create_journal()
3974 journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node); in create_journal()
3980 ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL); in create_journal()
3981 if (!ic->journal_tree) { in create_journal()
4019 struct dm_integrity_c *ic; in dm_integrity_ctr() local
4043 ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL); in dm_integrity_ctr()
4044 if (!ic) { in dm_integrity_ctr()
4048 ti->private = ic; in dm_integrity_ctr()
4050 ic->ti = ti; in dm_integrity_ctr()
4052 ic->in_progress = RB_ROOT; in dm_integrity_ctr()
4053 INIT_LIST_HEAD(&ic->wait_list); in dm_integrity_ctr()
4054 init_waitqueue_head(&ic->endio_wait); in dm_integrity_ctr()
4055 bio_list_init(&ic->flush_bio_list); in dm_integrity_ctr()
4056 init_waitqueue_head(&ic->copy_to_journal_wait); in dm_integrity_ctr()
4057 init_completion(&ic->crypto_backoff); in dm_integrity_ctr()
4058 atomic64_set(&ic->number_of_mismatches, 0); in dm_integrity_ctr()
4059 ic->bitmap_flush_interval = BITMAP_FLUSH_INTERVAL; in dm_integrity_ctr()
4061 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev); in dm_integrity_ctr()
4072 ic->start = start; in dm_integrity_ctr()
4075 if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) { in dm_integrity_ctr()
4084 ic->mode = argv[3][0]; in dm_integrity_ctr()
4096 ic->sectors_per_block = 1; in dm_integrity_ctr()
4126 if (ic->meta_dev) { in dm_integrity_ctr()
4127 dm_put_device(ti, ic->meta_dev); in dm_integrity_ctr()
4128 ic->meta_dev = NULL; in dm_integrity_ctr()
4131 dm_table_get_mode(ti->table), &ic->meta_dev); in dm_integrity_ctr()
4144 ic->sectors_per_block = val >> SECTOR_SHIFT; in dm_integrity_ctr()
4153 ic->bitmap_flush_interval = msecs_to_jiffies(val); in dm_integrity_ctr()
4155 r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error, in dm_integrity_ctr()
4160 r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error, in dm_integrity_ctr()
4165 r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error, in dm_integrity_ctr()
4170 ic->recalculate_flag = true; in dm_integrity_ctr()
4172 ic->recalculate_flag = true; in dm_integrity_ctr()
4173 ic->reset_recalculate_flag = true; in dm_integrity_ctr()
4175 ic->discard = true; in dm_integrity_ctr()
4177 ic->fix_padding = true; in dm_integrity_ctr()
4179 ic->fix_hmac = true; in dm_integrity_ctr()
4181 ic->legacy_recalculate = true; in dm_integrity_ctr()
4189 ic->data_device_sectors = bdev_nr_sectors(ic->dev->bdev); in dm_integrity_ctr()
4190 if (!ic->meta_dev) in dm_integrity_ctr()
4191 ic->meta_device_sectors = ic->data_device_sectors; in dm_integrity_ctr()
4193 ic->meta_device_sectors = bdev_nr_sectors(ic->meta_dev->bdev); in dm_integrity_ctr()
4197 ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); in dm_integrity_ctr()
4202 ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT); in dm_integrity_ctr()
4204 r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error, in dm_integrity_ctr()
4209 r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error, in dm_integrity_ctr()
4214 if (!ic->tag_size) { in dm_integrity_ctr()
4215 if (!ic->internal_hash) { in dm_integrity_ctr()
4220 ic->tag_size = crypto_shash_digestsize(ic->internal_hash); in dm_integrity_ctr()
4222 if (ic->tag_size > MAX_TAG_SIZE) { in dm_integrity_ctr()
4227 if (!(ic->tag_size & (ic->tag_size - 1))) in dm_integrity_ctr()
4228 ic->log2_tag_size = __ffs(ic->tag_size); in dm_integrity_ctr()
4230 ic->log2_tag_size = -1; in dm_integrity_ctr()
4232 if (ic->mode == 'B' && !ic->internal_hash) { in dm_integrity_ctr()
4238 if (ic->discard && !ic->internal_hash) { in dm_integrity_ctr()
4244 ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); in dm_integrity_ctr()
4245 ic->autocommit_msec = sync_msec; in dm_integrity_ctr()
4246 timer_setup(&ic->autocommit_timer, autocommit_fn, 0); in dm_integrity_ctr()
4248 ic->io = dm_io_client_create(); in dm_integrity_ctr()
4249 if (IS_ERR(ic->io)) { in dm_integrity_ctr()
4250 r = PTR_ERR(ic->io); in dm_integrity_ctr()
4251 ic->io = NULL; in dm_integrity_ctr()
4256 r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache); in dm_integrity_ctr()
4262 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata", in dm_integrity_ctr()
4264 if (!ic->metadata_wq) { in dm_integrity_ctr()
4274 ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1); in dm_integrity_ctr()
4275 if (!ic->wait_wq) { in dm_integrity_ctr()
4281 ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM, in dm_integrity_ctr()
4283 if (!ic->offload_wq) { in dm_integrity_ctr()
4289 ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1); in dm_integrity_ctr()
4290 if (!ic->commit_wq) { in dm_integrity_ctr()
4295 INIT_WORK(&ic->commit_work, integrity_commit); in dm_integrity_ctr()
4297 if (ic->mode == 'J' || ic->mode == 'B') { in dm_integrity_ctr()
4298 ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1); in dm_integrity_ctr()
4299 if (!ic->writer_wq) { in dm_integrity_ctr()
4304 INIT_WORK(&ic->writer_work, integrity_writer); in dm_integrity_ctr()
4307 ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL); in dm_integrity_ctr()
4308 if (!ic->sb) { in dm_integrity_ctr()
4314 r = sync_rw_sb(ic, REQ_OP_READ); in dm_integrity_ctr()
4320 if (memcmp(ic->sb->magic, SB_MAGIC, 8)) { in dm_integrity_ctr()
4321 if (ic->mode != 'R') { in dm_integrity_ctr()
4322 if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) { in dm_integrity_ctr()
4329 r = initialize_superblock(ic, journal_sectors, interleave_sectors); in dm_integrity_ctr()
4334 if (ic->mode != 'R') in dm_integrity_ctr()
4338 if (!ic->sb->version || ic->sb->version > SB_VERSION_5) { in dm_integrity_ctr()
4343 if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) { in dm_integrity_ctr()
4348 if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) { in dm_integrity_ctr()
4353 if (!le32_to_cpu(ic->sb->journal_sections)) { in dm_integrity_ctr()
4359 if (!ic->meta_dev) { in dm_integrity_ctr()
4360 if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS || in dm_integrity_ctr()
4361 ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) { in dm_integrity_ctr()
4367 if (ic->sb->log2_interleave_sectors) { in dm_integrity_ctr()
4373 …if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string)… in dm_integrity_ctr()
4379 get_provided_data_sectors(ic); in dm_integrity_ctr()
4380 if (!ic->provided_data_sectors) { in dm_integrity_ctr()
4387 r = calculate_device_limits(ic); in dm_integrity_ctr()
4389 if (ic->meta_dev) { in dm_integrity_ctr()
4390 if (ic->log2_buffer_sectors > 3) { in dm_integrity_ctr()
4391 ic->log2_buffer_sectors--; in dm_integrity_ctr()
4401 if (log2_sectors_per_bitmap_bit < ic->sb->log2_sectors_per_block) in dm_integrity_ctr()
4402 log2_sectors_per_bitmap_bit = ic->sb->log2_sectors_per_block; in dm_integrity_ctr()
4404 …bits_in_journal = ((__u64)ic->journal_section_sectors * ic->journal_sections) << (SECTOR_SHIFT + 3… in dm_integrity_ctr()
4407 …while (bits_in_journal < (ic->provided_data_sectors + ((sector_t)1 << log2_sectors_per_bitmap_bit)… in dm_integrity_ctr()
4410 log2_blocks_per_bitmap_bit = log2_sectors_per_bitmap_bit - ic->sb->log2_sectors_per_block; in dm_integrity_ctr()
4411 ic->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; in dm_integrity_ctr()
4413 ic->sb->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; in dm_integrity_ctr()
4415 n_bitmap_bits = ((ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) in dm_integrity_ctr()
4417 ic->n_bitmap_blocks = DIV_ROUND_UP(n_bitmap_bits, BITMAP_BLOCK_SIZE * 8); in dm_integrity_ctr()
4419 if (!ic->meta_dev) in dm_integrity_ctr()
4420 ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run)); in dm_integrity_ctr()
4422 if (ti->len > ic->provided_data_sectors) { in dm_integrity_ctr()
4429 threshold = (__u64)ic->journal_entries * (100 - journal_watermark); in dm_integrity_ctr()
4432 ic->free_sectors_threshold = threshold; in dm_integrity_ctr()
4435 DEBUG_print(" integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size)); in dm_integrity_ctr()
4436 DEBUG_print(" journal_entry_size %u\n", ic->journal_entry_size); in dm_integrity_ctr()
4437 DEBUG_print(" journal_entries_per_sector %u\n", ic->journal_entries_per_sector); in dm_integrity_ctr()
4438 DEBUG_print(" journal_section_entries %u\n", ic->journal_section_entries); in dm_integrity_ctr()
4439 DEBUG_print(" journal_section_sectors %u\n", ic->journal_section_sectors); in dm_integrity_ctr()
4440 DEBUG_print(" journal_sections %u\n", (unsigned int)le32_to_cpu(ic->sb->journal_sections)); in dm_integrity_ctr()
4441 DEBUG_print(" journal_entries %u\n", ic->journal_entries); in dm_integrity_ctr()
4442 DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors); in dm_integrity_ctr()
4443 DEBUG_print(" data_device_sectors 0x%llx\n", bdev_nr_sectors(ic->dev->bdev)); in dm_integrity_ctr()
4444 DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors); in dm_integrity_ctr()
4445 DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run); in dm_integrity_ctr()
4446 DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run); in dm_integrity_ctr()
4447 …DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", ic->provided_data_sectors, ic->provided_data… in dm_integrity_ctr()
4448 DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors); in dm_integrity_ctr()
4451 if (ic->recalculate_flag && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) { in dm_integrity_ctr()
4452 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); in dm_integrity_ctr()
4453 ic->sb->recalc_sector = cpu_to_le64(0); in dm_integrity_ctr()
4456 if (ic->internal_hash) { in dm_integrity_ctr()
4459 ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1); in dm_integrity_ctr()
4460 if (!ic->recalc_wq) { in dm_integrity_ctr()
4465 INIT_WORK(&ic->recalc_work, integrity_recalc); in dm_integrity_ctr()
4466 ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT); in dm_integrity_ctr()
4467 if (!ic->recalc_buffer) { in dm_integrity_ctr()
4472 recalc_tags_size = (RECALC_SECTORS >> ic->sb->log2_sectors_per_block) * ic->tag_size; in dm_integrity_ctr()
4473 if (crypto_shash_digestsize(ic->internal_hash) > ic->tag_size) in dm_integrity_ctr()
4474 recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tag_size; in dm_integrity_ctr()
4475 ic->recalc_tags = kvmalloc(recalc_tags_size, GFP_KERNEL); in dm_integrity_ctr()
4476 if (!ic->recalc_tags) { in dm_integrity_ctr()
4482 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { in dm_integrity_ctr()
4489 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && in dm_integrity_ctr()
4490 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors && in dm_integrity_ctr()
4491 dm_integrity_disable_recalculate(ic)) { in dm_integrity_ctr()
4497 ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, in dm_integrity_ctr()
4498 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0); in dm_integrity_ctr()
4499 if (IS_ERR(ic->bufio)) { in dm_integrity_ctr()
4500 r = PTR_ERR(ic->bufio); in dm_integrity_ctr()
4502 ic->bufio = NULL; in dm_integrity_ctr()
4505 dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors); in dm_integrity_ctr()
4507 if (ic->mode != 'R') { in dm_integrity_ctr()
4508 r = create_journal(ic, &ti->error); in dm_integrity_ctr()
4514 if (ic->mode == 'B') { in dm_integrity_ctr()
4516 unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); in dm_integrity_ctr()
4518 ic->recalc_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); in dm_integrity_ctr()
4519 if (!ic->recalc_bitmap) { in dm_integrity_ctr()
4523 ic->may_write_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); in dm_integrity_ctr()
4524 if (!ic->may_write_bitmap) { in dm_integrity_ctr()
4528 ic->bbs = kvmalloc_array(ic->n_bitmap_blocks, sizeof(struct bitmap_block_status), GFP_KERNEL); in dm_integrity_ctr()
4529 if (!ic->bbs) { in dm_integrity_ctr()
4533 INIT_DELAYED_WORK(&ic->bitmap_flush_work, bitmap_flush_work); in dm_integrity_ctr()
4534 for (i = 0; i < ic->n_bitmap_blocks; i++) { in dm_integrity_ctr()
4535 struct bitmap_block_status *bbs = &ic->bbs[i]; in dm_integrity_ctr()
4539 bbs->ic = ic; in dm_integrity_ctr()
4548 bbs->bitmap = lowmem_page_address(ic->journal[pl_index].page) + pl_offset; in dm_integrity_ctr()
4553 init_journal(ic, 0, ic->journal_sections, 0); in dm_integrity_ctr()
4554 r = dm_integrity_failed(ic); in dm_integrity_ctr()
4559 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); in dm_integrity_ctr()
4564 ic->just_formatted = true; in dm_integrity_ctr()
4567 if (!ic->meta_dev) { in dm_integrity_ctr()
4568 r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors); in dm_integrity_ctr()
4572 if (ic->mode == 'B') { in dm_integrity_ctr()
4575 …max_io_len = ((sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit) * (BITMAP_BLOCK_S… in dm_integrity_ctr()
4586 if (!ic->internal_hash) in dm_integrity_ctr()
4587 dm_integrity_set(ti, ic); in dm_integrity_ctr()
4591 if (ic->discard) in dm_integrity_ctr()
4605 struct dm_integrity_c *ic = ti->private; in dm_integrity_dtr() local
4607 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); in dm_integrity_dtr()
4608 BUG_ON(!list_empty(&ic->wait_list)); in dm_integrity_dtr()
4610 if (ic->mode == 'B') in dm_integrity_dtr()
4611 cancel_delayed_work_sync(&ic->bitmap_flush_work); in dm_integrity_dtr()
4612 if (ic->metadata_wq) in dm_integrity_dtr()
4613 destroy_workqueue(ic->metadata_wq); in dm_integrity_dtr()
4614 if (ic->wait_wq) in dm_integrity_dtr()
4615 destroy_workqueue(ic->wait_wq); in dm_integrity_dtr()
4616 if (ic->offload_wq) in dm_integrity_dtr()
4617 destroy_workqueue(ic->offload_wq); in dm_integrity_dtr()
4618 if (ic->commit_wq) in dm_integrity_dtr()
4619 destroy_workqueue(ic->commit_wq); in dm_integrity_dtr()
4620 if (ic->writer_wq) in dm_integrity_dtr()
4621 destroy_workqueue(ic->writer_wq); in dm_integrity_dtr()
4622 if (ic->recalc_wq) in dm_integrity_dtr()
4623 destroy_workqueue(ic->recalc_wq); in dm_integrity_dtr()
4624 vfree(ic->recalc_buffer); in dm_integrity_dtr()
4625 kvfree(ic->recalc_tags); in dm_integrity_dtr()
4626 kvfree(ic->bbs); in dm_integrity_dtr()
4627 if (ic->bufio) in dm_integrity_dtr()
4628 dm_bufio_client_destroy(ic->bufio); in dm_integrity_dtr()
4629 mempool_exit(&ic->journal_io_mempool); in dm_integrity_dtr()
4630 if (ic->io) in dm_integrity_dtr()
4631 dm_io_client_destroy(ic->io); in dm_integrity_dtr()
4632 if (ic->dev) in dm_integrity_dtr()
4633 dm_put_device(ti, ic->dev); in dm_integrity_dtr()
4634 if (ic->meta_dev) in dm_integrity_dtr()
4635 dm_put_device(ti, ic->meta_dev); in dm_integrity_dtr()
4636 dm_integrity_free_page_list(ic->journal); in dm_integrity_dtr()
4637 dm_integrity_free_page_list(ic->journal_io); in dm_integrity_dtr()
4638 dm_integrity_free_page_list(ic->journal_xor); in dm_integrity_dtr()
4639 dm_integrity_free_page_list(ic->recalc_bitmap); in dm_integrity_dtr()
4640 dm_integrity_free_page_list(ic->may_write_bitmap); in dm_integrity_dtr()
4641 if (ic->journal_scatterlist) in dm_integrity_dtr()
4642 dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist); in dm_integrity_dtr()
4643 if (ic->journal_io_scatterlist) in dm_integrity_dtr()
4644 dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist); in dm_integrity_dtr()
4645 if (ic->sk_requests) { in dm_integrity_dtr()
4648 for (i = 0; i < ic->journal_sections; i++) { in dm_integrity_dtr()
4651 req = ic->sk_requests[i]; in dm_integrity_dtr()
4657 kvfree(ic->sk_requests); in dm_integrity_dtr()
4659 kvfree(ic->journal_tree); in dm_integrity_dtr()
4660 if (ic->sb) in dm_integrity_dtr()
4661 free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT); in dm_integrity_dtr()
4663 if (ic->internal_hash) in dm_integrity_dtr()
4664 crypto_free_shash(ic->internal_hash); in dm_integrity_dtr()
4665 free_alg(&ic->internal_hash_alg); in dm_integrity_dtr()
4667 if (ic->journal_crypt) in dm_integrity_dtr()
4668 crypto_free_skcipher(ic->journal_crypt); in dm_integrity_dtr()
4669 free_alg(&ic->journal_crypt_alg); in dm_integrity_dtr()
4671 if (ic->journal_mac) in dm_integrity_dtr()
4672 crypto_free_shash(ic->journal_mac); in dm_integrity_dtr()
4673 free_alg(&ic->journal_mac_alg); in dm_integrity_dtr()
4675 kfree(ic); in dm_integrity_dtr()