Lines Matching refs:t
60 static inline sector_t *get_node(struct dm_table *t, in get_node() argument
63 return t->index[l] + (n * KEYS_PER_NODE); in get_node()
70 static sector_t high(struct dm_table *t, unsigned int l, unsigned int n) in high() argument
72 for (; l < t->depth - 1; l++) in high()
75 if (n >= t->counts[l]) in high()
78 return get_node(t, l, n)[KEYS_PER_NODE - 1]; in high()
85 static int setup_btree_index(unsigned int l, struct dm_table *t) in setup_btree_index() argument
90 for (n = 0U; n < t->counts[l]; n++) { in setup_btree_index()
91 node = get_node(t, l, n); in setup_btree_index()
94 node[k] = high(t, l + 1, get_child(n, k)); in setup_btree_index()
104 static int alloc_targets(struct dm_table *t, unsigned int num) in alloc_targets() argument
120 kvfree(t->highs); in alloc_targets()
122 t->num_allocated = num; in alloc_targets()
123 t->highs = n_highs; in alloc_targets()
124 t->targets = n_targets; in alloc_targets()
132 struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL); in dm_table_create() local
134 if (!t) in dm_table_create()
137 INIT_LIST_HEAD(&t->devices); in dm_table_create()
145 kfree(t); in dm_table_create()
149 if (alloc_targets(t, num_targets)) { in dm_table_create()
150 kfree(t); in dm_table_create()
154 t->type = DM_TYPE_NONE; in dm_table_create()
155 t->mode = mode; in dm_table_create()
156 t->md = md; in dm_table_create()
157 *result = t; in dm_table_create()
175 static void dm_table_destroy_crypto_profile(struct dm_table *t);
177 void dm_table_destroy(struct dm_table *t) in dm_table_destroy() argument
179 if (!t) in dm_table_destroy()
183 if (t->depth >= 2) in dm_table_destroy()
184 kvfree(t->index[t->depth - 2]); in dm_table_destroy()
187 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_destroy()
188 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_destroy()
196 kvfree(t->highs); in dm_table_destroy()
199 free_devices(&t->devices, t->md); in dm_table_destroy()
201 dm_free_md_mempools(t->mempools); in dm_table_destroy()
203 dm_table_destroy_crypto_profile(t); in dm_table_destroy()
205 kfree(t); in dm_table_destroy()
351 struct dm_table *t = ti->table; in dm_get_device() local
353 BUG_ON(!t); in dm_get_device()
365 if (dev == disk_devt(t->md->disk)) in dm_get_device()
368 dd = find_device(&t->devices, dev); in dm_get_device()
374 r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev); in dm_get_device()
381 list_add(&dd->list, &t->devices); in dm_get_device()
385 r = upgrade_mode(dd, mode, t->md); in dm_get_device()
453 static int adjoin(struct dm_table *t, struct dm_target *ti) in adjoin() argument
457 if (!t->num_targets) in adjoin()
460 prev = &t->targets[t->num_targets - 1]; in adjoin()
567 static int validate_hardware_logical_block_alignment(struct dm_table *t, in validate_hardware_logical_block_alignment() argument
595 for (i = 0; i < t->num_targets; i++) { in validate_hardware_logical_block_alignment()
596 ti = dm_table_get_target(t, i); in validate_hardware_logical_block_alignment()
624 dm_device_name(t->md), i, in validate_hardware_logical_block_alignment()
634 int dm_table_add_target(struct dm_table *t, const char *type, in dm_table_add_target() argument
641 if (t->singleton) { in dm_table_add_target()
643 dm_device_name(t->md), t->targets->type->name); in dm_table_add_target()
647 BUG_ON(t->num_targets >= t->num_allocated); in dm_table_add_target()
649 ti = t->targets + t->num_targets; in dm_table_add_target()
653 DMERR("%s: zero-length target", dm_device_name(t->md)); in dm_table_add_target()
659 DMERR("%s: %s: unknown target type", dm_device_name(t->md), type); in dm_table_add_target()
664 if (t->num_targets) { in dm_table_add_target()
668 t->singleton = true; in dm_table_add_target()
671 if (dm_target_always_writeable(ti->type) && !(t->mode & FMODE_WRITE)) { in dm_table_add_target()
676 if (t->immutable_target_type) { in dm_table_add_target()
677 if (t->immutable_target_type != ti->type) { in dm_table_add_target()
682 if (t->num_targets) { in dm_table_add_target()
686 t->immutable_target_type = ti->type; in dm_table_add_target()
690 t->integrity_added = 1; in dm_table_add_target()
692 ti->table = t; in dm_table_add_target()
700 if (!adjoin(t, ti)) { in dm_table_add_target()
716 t->highs[t->num_targets++] = ti->begin + ti->len - 1; in dm_table_add_target()
720 dm_device_name(t->md), type); in dm_table_add_target()
728 DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r)); in dm_table_add_target()
802 void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type) in dm_table_set_type() argument
804 t->type = type; in dm_table_set_type()
826 static bool dm_table_supports_dax(struct dm_table *t, in dm_table_supports_dax() argument
830 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_dax()
831 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_dax()
857 static int dm_table_determine_type(struct dm_table *t) in dm_table_determine_type() argument
861 struct list_head *devices = dm_table_get_devices(t); in dm_table_determine_type()
862 enum dm_queue_mode live_md_type = dm_get_md_type(t->md); in dm_table_determine_type()
864 if (t->type != DM_TYPE_NONE) { in dm_table_determine_type()
866 if (t->type == DM_TYPE_BIO_BASED) { in dm_table_determine_type()
870 BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED); in dm_table_determine_type()
874 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_determine_type()
875 ti = dm_table_get_target(t, i); in dm_table_determine_type()
904 t->type = DM_TYPE_BIO_BASED; in dm_table_determine_type()
905 if (dm_table_supports_dax(t, device_not_dax_capable) || in dm_table_determine_type()
907 t->type = DM_TYPE_DAX_BIO_BASED; in dm_table_determine_type()
914 t->type = DM_TYPE_REQUEST_BASED; in dm_table_determine_type()
923 if (t->num_targets > 1) { in dm_table_determine_type()
930 struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx); in dm_table_determine_type()
934 t->type = live_table->type; in dm_table_determine_type()
935 dm_put_live_table(t->md, srcu_idx); in dm_table_determine_type()
939 ti = dm_table_get_immutable_target(t); in dm_table_determine_type()
958 enum dm_queue_mode dm_table_get_type(struct dm_table *t) in dm_table_get_type() argument
960 return t->type; in dm_table_get_type()
963 struct target_type *dm_table_get_immutable_target_type(struct dm_table *t) in dm_table_get_immutable_target_type() argument
965 return t->immutable_target_type; in dm_table_get_immutable_target_type()
968 struct dm_target *dm_table_get_immutable_target(struct dm_table *t) in dm_table_get_immutable_target() argument
971 if (t->num_targets > 1 || in dm_table_get_immutable_target()
972 !dm_target_is_immutable(t->targets[0].type)) in dm_table_get_immutable_target()
975 return t->targets; in dm_table_get_immutable_target()
978 struct dm_target *dm_table_get_wildcard_target(struct dm_table *t) in dm_table_get_wildcard_target() argument
980 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_get_wildcard_target()
981 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_get_wildcard_target()
990 bool dm_table_bio_based(struct dm_table *t) in dm_table_bio_based() argument
992 return __table_type_bio_based(dm_table_get_type(t)); in dm_table_bio_based()
995 bool dm_table_request_based(struct dm_table *t) in dm_table_request_based() argument
997 return __table_type_request_based(dm_table_get_type(t)); in dm_table_request_based()
1000 static bool dm_table_supports_poll(struct dm_table *t);
1002 static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md) in dm_table_alloc_md_mempools() argument
1004 enum dm_queue_mode type = dm_table_get_type(t); in dm_table_alloc_md_mempools()
1024 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_alloc_md_mempools()
1025 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_alloc_md_mempools()
1037 dm_table_supports_poll(t) ? BIOSET_PERCPU_CACHE : 0)) in dm_table_alloc_md_mempools()
1039 if (t->integrity_supported && in dm_table_alloc_md_mempools()
1045 if (t->integrity_supported && in dm_table_alloc_md_mempools()
1049 t->mempools = pools; in dm_table_alloc_md_mempools()
1057 static int setup_indexes(struct dm_table *t) in setup_indexes() argument
1064 for (i = t->depth - 2; i >= 0; i--) { in setup_indexes()
1065 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE); in setup_indexes()
1066 total += t->counts[i]; in setup_indexes()
1074 for (i = t->depth - 2; i >= 0; i--) { in setup_indexes()
1075 t->index[i] = indexes; in setup_indexes()
1076 indexes += (KEYS_PER_NODE * t->counts[i]); in setup_indexes()
1077 setup_btree_index(i, t); in setup_indexes()
1086 static int dm_table_build_index(struct dm_table *t) in dm_table_build_index() argument
1092 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); in dm_table_build_index()
1093 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); in dm_table_build_index()
1096 t->counts[t->depth - 1] = leaf_nodes; in dm_table_build_index()
1097 t->index[t->depth - 1] = t->highs; in dm_table_build_index()
1099 if (t->depth >= 2) in dm_table_build_index()
1100 r = setup_indexes(t); in dm_table_build_index()
1114 static struct gendisk *dm_table_get_integrity_disk(struct dm_table *t) in dm_table_get_integrity_disk() argument
1116 struct list_head *devices = dm_table_get_devices(t); in dm_table_get_integrity_disk()
1120 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_get_integrity_disk()
1121 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_get_integrity_disk()
1142 dm_device_name(t->md), in dm_table_get_integrity_disk()
1158 static int dm_table_register_integrity(struct dm_table *t) in dm_table_register_integrity() argument
1160 struct mapped_device *md = t->md; in dm_table_register_integrity()
1164 if (t->integrity_added) in dm_table_register_integrity()
1167 template_disk = dm_table_get_integrity_disk(t); in dm_table_register_integrity()
1172 t->integrity_supported = true; in dm_table_register_integrity()
1188 dm_device_name(t->md), in dm_table_register_integrity()
1194 t->integrity_supported = true; in dm_table_register_integrity()
1233 struct dm_table *t; in dm_keyslot_evict() local
1236 t = dm_get_live_table(md, &srcu_idx); in dm_keyslot_evict()
1237 if (!t) in dm_keyslot_evict()
1240 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_keyslot_evict()
1241 struct dm_target *ti = dm_table_get_target(t, i); in dm_keyslot_evict()
1277 static void dm_table_destroy_crypto_profile(struct dm_table *t) in dm_table_destroy_crypto_profile() argument
1279 dm_destroy_crypto_profile(t->crypto_profile); in dm_table_destroy_crypto_profile()
1280 t->crypto_profile = NULL; in dm_table_destroy_crypto_profile()
1292 static int dm_table_construct_crypto_profile(struct dm_table *t) in dm_table_construct_crypto_profile() argument
1302 dmcp->md = t->md; in dm_table_construct_crypto_profile()
1311 for (i = 0; i < t->num_targets; i++) { in dm_table_construct_crypto_profile()
1312 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_construct_crypto_profile()
1325 if (t->md->queue && in dm_table_construct_crypto_profile()
1327 t->md->queue->crypto_profile)) { in dm_table_construct_crypto_profile()
1354 t->crypto_profile = profile; in dm_table_construct_crypto_profile()
1360 struct dm_table *t) in dm_update_crypto_profile() argument
1362 if (!t->crypto_profile) in dm_update_crypto_profile()
1367 blk_crypto_register(t->crypto_profile, q); in dm_update_crypto_profile()
1370 t->crypto_profile); in dm_update_crypto_profile()
1371 dm_destroy_crypto_profile(t->crypto_profile); in dm_update_crypto_profile()
1373 t->crypto_profile = NULL; in dm_update_crypto_profile()
1378 static int dm_table_construct_crypto_profile(struct dm_table *t) in dm_table_construct_crypto_profile() argument
1387 static void dm_table_destroy_crypto_profile(struct dm_table *t) in dm_table_destroy_crypto_profile() argument
1392 struct dm_table *t) in dm_update_crypto_profile() argument
1402 int dm_table_complete(struct dm_table *t) in dm_table_complete() argument
1406 r = dm_table_determine_type(t); in dm_table_complete()
1412 r = dm_table_build_index(t); in dm_table_complete()
1418 r = dm_table_register_integrity(t); in dm_table_complete()
1424 r = dm_table_construct_crypto_profile(t); in dm_table_complete()
1430 r = dm_table_alloc_md_mempools(t, t->md); in dm_table_complete()
1438 void dm_table_event_callback(struct dm_table *t, in dm_table_event_callback() argument
1442 t->event_fn = fn; in dm_table_event_callback()
1443 t->event_context = context; in dm_table_event_callback()
1447 void dm_table_event(struct dm_table *t) in dm_table_event() argument
1450 if (t->event_fn) in dm_table_event()
1451 t->event_fn(t->event_context); in dm_table_event()
1456 inline sector_t dm_table_get_size(struct dm_table *t) in dm_table_get_size() argument
1458 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0; in dm_table_get_size()
1468 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector) in dm_table_find_target() argument
1473 if (unlikely(sector >= dm_table_get_size(t))) in dm_table_find_target()
1476 for (l = 0; l < t->depth; l++) { in dm_table_find_target()
1478 node = get_node(t, l, n); in dm_table_find_target()
1485 return &t->targets[(KEYS_PER_NODE * n) + k]; in dm_table_find_target()
1519 static bool dm_table_any_dev_attr(struct dm_table *t, in dm_table_any_dev_attr() argument
1522 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_any_dev_attr()
1523 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_any_dev_attr()
1543 static bool dm_table_supports_poll(struct dm_table *t) in dm_table_supports_poll() argument
1545 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_poll()
1546 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_poll()
1562 bool dm_table_has_no_data_devices(struct dm_table *t) in dm_table_has_no_data_devices() argument
1564 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_has_no_data_devices()
1565 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_has_no_data_devices()
1595 static bool dm_table_supports_zoned_model(struct dm_table *t, in dm_table_supports_zoned_model() argument
1598 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_zoned_model()
1599 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_zoned_model()
1630 static int validate_hardware_zoned_model(struct dm_table *t, in validate_hardware_zoned_model() argument
1637 if (!dm_table_supports_zoned_model(t, zoned_model)) { in validate_hardware_zoned_model()
1639 dm_device_name(t->md)); in validate_hardware_zoned_model()
1647 if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) { in validate_hardware_zoned_model()
1649 dm_device_name(t->md)); in validate_hardware_zoned_model()
1659 int dm_calculate_queue_limits(struct dm_table *t, in dm_calculate_queue_limits() argument
1668 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_calculate_queue_limits()
1669 struct dm_target *ti = dm_table_get_target(t, i); in dm_calculate_queue_limits()
1711 dm_device_name(t->md), in dm_calculate_queue_limits()
1731 if (validate_hardware_zoned_model(t, zoned_model, zone_sectors)) in dm_calculate_queue_limits()
1734 return validate_hardware_logical_block_alignment(t, limits); in dm_calculate_queue_limits()
1742 static void dm_table_verify_integrity(struct dm_table *t) in dm_table_verify_integrity() argument
1746 if (t->integrity_added) in dm_table_verify_integrity()
1749 if (t->integrity_supported) { in dm_table_verify_integrity()
1754 template_disk = dm_table_get_integrity_disk(t); in dm_table_verify_integrity()
1756 blk_integrity_compare(dm_disk(t->md), template_disk) >= 0) in dm_table_verify_integrity()
1760 if (integrity_profile_exists(dm_disk(t->md))) { in dm_table_verify_integrity()
1762 dm_device_name(t->md)); in dm_table_verify_integrity()
1763 blk_integrity_unregister(dm_disk(t->md)); in dm_table_verify_integrity()
1776 static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush) in dm_table_supports_flush() argument
1784 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_flush()
1785 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_flush()
1837 static bool dm_table_supports_write_zeroes(struct dm_table *t) in dm_table_supports_write_zeroes() argument
1839 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_write_zeroes()
1840 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_write_zeroes()
1859 static bool dm_table_supports_nowait(struct dm_table *t) in dm_table_supports_nowait() argument
1861 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_nowait()
1862 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_nowait()
1881 static bool dm_table_supports_discards(struct dm_table *t) in dm_table_supports_discards() argument
1883 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_discards()
1884 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_discards()
1910 static bool dm_table_supports_secure_erase(struct dm_table *t) in dm_table_supports_secure_erase() argument
1912 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_secure_erase()
1913 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_secure_erase()
1933 int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, in dm_table_set_restrictions() argument
1944 if (dm_table_supports_nowait(t)) in dm_table_set_restrictions()
1949 if (!dm_table_supports_discards(t)) { in dm_table_set_restrictions()
1957 if (!dm_table_supports_secure_erase(t)) in dm_table_set_restrictions()
1960 if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_WC))) { in dm_table_set_restrictions()
1962 if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_FUA))) in dm_table_set_restrictions()
1967 if (dm_table_supports_dax(t, device_not_dax_capable)) { in dm_table_set_restrictions()
1969 if (dm_table_supports_dax(t, device_not_dax_synchronous_capable)) in dm_table_set_restrictions()
1970 set_dax_synchronous(t->md->dax_dev); in dm_table_set_restrictions()
1974 if (dm_table_any_dev_attr(t, device_dax_write_cache_enabled, NULL)) in dm_table_set_restrictions()
1975 dax_write_cache(t->md->dax_dev, true); in dm_table_set_restrictions()
1978 if (dm_table_any_dev_attr(t, device_is_rotational, NULL)) in dm_table_set_restrictions()
1983 if (!dm_table_supports_write_zeroes(t)) in dm_table_set_restrictions()
1986 dm_table_verify_integrity(t); in dm_table_set_restrictions()
1995 if (dm_table_any_dev_attr(t, device_requires_stable_pages, NULL)) in dm_table_set_restrictions()
2007 dm_table_any_dev_attr(t, device_is_not_random, NULL)) in dm_table_set_restrictions()
2015 r = dm_set_zones_restrictions(t, q); in dm_table_set_restrictions()
2022 dm_update_crypto_profile(q, t); in dm_table_set_restrictions()
2023 disk_update_readahead(t->md->disk); in dm_table_set_restrictions()
2032 if (__table_type_bio_based(t->type)) { in dm_table_set_restrictions()
2033 if (dm_table_supports_poll(t)) in dm_table_set_restrictions()
2042 struct list_head *dm_table_get_devices(struct dm_table *t) in dm_table_get_devices() argument
2044 return &t->devices; in dm_table_get_devices()
2047 fmode_t dm_table_get_mode(struct dm_table *t) in dm_table_get_mode() argument
2049 return t->mode; in dm_table_get_mode()
2059 static void suspend_targets(struct dm_table *t, enum suspend_mode mode) in suspend_targets() argument
2061 lockdep_assert_held(&t->md->suspend_lock); in suspend_targets()
2063 for (unsigned int i = 0; i < t->num_targets; i++) { in suspend_targets()
2064 struct dm_target *ti = dm_table_get_target(t, i); in suspend_targets()
2083 void dm_table_presuspend_targets(struct dm_table *t) in dm_table_presuspend_targets() argument
2085 if (!t) in dm_table_presuspend_targets()
2088 suspend_targets(t, PRESUSPEND); in dm_table_presuspend_targets()
2091 void dm_table_presuspend_undo_targets(struct dm_table *t) in dm_table_presuspend_undo_targets() argument
2093 if (!t) in dm_table_presuspend_undo_targets()
2096 suspend_targets(t, PRESUSPEND_UNDO); in dm_table_presuspend_undo_targets()
2099 void dm_table_postsuspend_targets(struct dm_table *t) in dm_table_postsuspend_targets() argument
2101 if (!t) in dm_table_postsuspend_targets()
2104 suspend_targets(t, POSTSUSPEND); in dm_table_postsuspend_targets()
2107 int dm_table_resume_targets(struct dm_table *t) in dm_table_resume_targets() argument
2112 lockdep_assert_held(&t->md->suspend_lock); in dm_table_resume_targets()
2114 for (i = 0; i < t->num_targets; i++) { in dm_table_resume_targets()
2115 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_resume_targets()
2123 dm_device_name(t->md), ti->type->name, r); in dm_table_resume_targets()
2128 for (i = 0; i < t->num_targets; i++) { in dm_table_resume_targets()
2129 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_resume_targets()
2138 struct mapped_device *dm_table_get_md(struct dm_table *t) in dm_table_get_md() argument
2140 return t->md; in dm_table_get_md()
2144 const char *dm_table_device_name(struct dm_table *t) in dm_table_device_name() argument
2146 return dm_device_name(t->md); in dm_table_device_name()
2150 void dm_table_run_md_queue_async(struct dm_table *t) in dm_table_run_md_queue_async() argument
2152 if (!dm_table_request_based(t)) in dm_table_run_md_queue_async()
2155 if (t->md->queue) in dm_table_run_md_queue_async()
2156 blk_mq_run_hw_queues(t->md->queue, true); in dm_table_run_md_queue_async()