Lines Matching refs:data_vio
1693 static void release_page_lock(struct data_vio *data_vio, char *what) in release_page_lock() argument
1697 struct tree_lock *lock = &data_vio->tree_lock; in release_page_lock()
1703 zone = data_vio->logical.zone->block_map_zone; in release_page_lock()
1711 static void finish_lookup(struct data_vio *data_vio, int result) in finish_lookup() argument
1713 data_vio->tree_lock.height = 0; in finish_lookup()
1715 --data_vio->logical.zone->block_map_zone->active_lookups; in finish_lookup()
1717 set_data_vio_logical_callback(data_vio, continue_data_vio_with_block_map_slot); in finish_lookup()
1718 data_vio->vio.completion.error_handler = handle_data_vio_error; in finish_lookup()
1719 continue_data_vio_with_error(data_vio, result); in finish_lookup()
1724 struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); in abort_lookup_for_waiter() local
1727 if (!data_vio->write) { in abort_lookup_for_waiter()
1734 finish_lookup(data_vio, result); in abort_lookup_for_waiter()
1737 static void abort_lookup(struct data_vio *data_vio, int result, char *what) in abort_lookup() argument
1740 enter_zone_read_only_mode(data_vio->logical.zone->block_map_zone, result); in abort_lookup()
1742 if (data_vio->tree_lock.locked) { in abort_lookup()
1743 release_page_lock(data_vio, what); in abort_lookup()
1744 vdo_waitq_notify_all_waiters(&data_vio->tree_lock.waiters, in abort_lookup()
1749 finish_lookup(data_vio, result); in abort_lookup()
1752 static void abort_load(struct data_vio *data_vio, int result) in abort_load() argument
1754 abort_lookup(data_vio, result, "load"); in abort_load()
1773 static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio);
1775 struct data_vio *data_vio);
1777 static void continue_with_loaded_page(struct data_vio *data_vio, in continue_with_loaded_page() argument
1780 struct tree_lock *lock = &data_vio->tree_lock; in continue_with_loaded_page()
1785 if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { in continue_with_loaded_page()
1791 abort_load(data_vio, VDO_BAD_MAPPING); in continue_with_loaded_page()
1797 allocate_block_map_page(data_vio->logical.zone->block_map_zone, in continue_with_loaded_page()
1798 data_vio); in continue_with_loaded_page()
1804 finish_lookup(data_vio, VDO_SUCCESS); in continue_with_loaded_page()
1809 load_block_map_page(data_vio->logical.zone->block_map_zone, data_vio); in continue_with_loaded_page()
1814 struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); in continue_load_for_waiter() local
1816 data_vio->tree_lock.height--; in continue_load_for_waiter()
1817 continue_with_loaded_page(data_vio, context); in continue_load_for_waiter()
1828 struct data_vio *data_vio = completion->parent; in finish_block_map_page_load() local
1830 struct tree_lock *tree_lock = &data_vio->tree_lock; in finish_block_map_page_load()
1843 release_page_lock(data_vio, "load"); in finish_block_map_page_load()
1845 continue_with_loaded_page(data_vio, page); in finish_block_map_page_load()
1853 struct data_vio *data_vio = completion->parent; in handle_io_error() local
1857 abort_load(data_vio, result); in handle_io_error()
1863 struct data_vio *data_vio = vio->completion.parent; in load_page_endio() local
1866 data_vio->logical.zone->thread_id); in load_page_endio()
1872 struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); in load_page() local
1873 struct tree_lock *lock = &data_vio->tree_lock; in load_page()
1876 pooled->vio.completion.parent = data_vio; in load_page()
1885 static int attempt_page_lock(struct block_map_zone *zone, struct data_vio *data_vio) in attempt_page_lock() argument
1889 struct tree_lock *lock = &data_vio->tree_lock; in attempt_page_lock()
1909 data_vio->tree_lock.locked = true; in attempt_page_lock()
1914 vdo_waitq_enqueue_waiter(&lock_holder->waiters, &data_vio->waiter); in attempt_page_lock()
1919 static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio) in load_block_map_page() argument
1923 result = attempt_page_lock(zone, data_vio); in load_block_map_page()
1925 abort_load(data_vio, result); in load_block_map_page()
1929 if (data_vio->tree_lock.locked) { in load_block_map_page()
1930 data_vio->waiter.callback = load_page; in load_block_map_page()
1931 acquire_vio_from_pool(zone->vio_pool, &data_vio->waiter); in load_block_map_page()
1937 struct data_vio *data_vio = as_data_vio(completion); in allocation_failure() local
1940 data_vio->logical.zone->thread_id)) in allocation_failure()
1943 abort_lookup(data_vio, completion->result, "allocation"); in allocation_failure()
1948 struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); in continue_allocation_for_waiter() local
1949 struct tree_lock *tree_lock = &data_vio->tree_lock; in continue_allocation_for_waiter()
1953 data_vio->tree_lock.tree_slots[tree_lock->height].block_map_slot.pbn = pbn; in continue_allocation_for_waiter()
1956 finish_lookup(data_vio, VDO_SUCCESS); in continue_allocation_for_waiter()
1960 allocate_block_map_page(data_vio->logical.zone->block_map_zone, data_vio); in continue_allocation_for_waiter()
2070 struct data_vio *data_vio = as_data_vio(completion); in finish_block_map_allocation() local
2071 struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; in finish_block_map_allocation()
2072 struct tree_lock *tree_lock = &data_vio->tree_lock; in finish_block_map_allocation()
2075 assert_data_vio_in_logical_zone(data_vio); in finish_block_map_allocation()
2083 vdo_update_block_map_page(page, data_vio, pbn, in finish_block_map_allocation()
2114 release_page_lock(data_vio, "allocation"); in finish_block_map_allocation()
2118 finish_lookup(data_vio, VDO_SUCCESS); in finish_block_map_allocation()
2122 allocate_block_map_page(zone, data_vio); in finish_block_map_allocation()
2127 struct data_vio *data_vio = as_data_vio(completion); in release_block_map_write_lock() local
2129 assert_data_vio_in_allocated_zone(data_vio); in release_block_map_write_lock()
2131 release_data_vio_allocation_lock(data_vio, true); in release_block_map_write_lock()
2132 launch_data_vio_logical_callback(data_vio, finish_block_map_allocation); in release_block_map_write_lock()
2142 struct data_vio *data_vio = as_data_vio(completion); in set_block_map_page_reference_count() local
2144 assert_data_vio_in_allocated_zone(data_vio); in set_block_map_page_reference_count()
2147 vdo_modify_reference_count(completion, &data_vio->increment_updater); in set_block_map_page_reference_count()
2152 struct data_vio *data_vio = as_data_vio(completion); in journal_block_map_allocation() local
2154 assert_data_vio_in_journal_zone(data_vio); in journal_block_map_allocation()
2156 set_data_vio_allocated_zone_callback(data_vio, in journal_block_map_allocation()
2158 vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio); in journal_block_map_allocation()
2163 struct data_vio *data_vio = as_data_vio(completion); in allocate_block() local
2164 struct tree_lock *lock = &data_vio->tree_lock; in allocate_block()
2167 assert_data_vio_in_allocated_zone(data_vio); in allocate_block()
2169 if (!vdo_allocate_block_in_zone(data_vio)) in allocate_block()
2172 pbn = data_vio->allocation.pbn; in allocate_block()
2174 data_vio->increment_updater = (struct reference_updater) { in allocate_block()
2181 .lock = data_vio->allocation.lock, in allocate_block()
2184 launch_data_vio_journal_callback(data_vio, journal_block_map_allocation); in allocate_block()
2188 struct data_vio *data_vio) in allocate_block_map_page() argument
2192 if (!data_vio->write || data_vio->is_discard) { in allocate_block_map_page()
2194 finish_lookup(data_vio, VDO_SUCCESS); in allocate_block_map_page()
2198 result = attempt_page_lock(zone, data_vio); in allocate_block_map_page()
2200 abort_lookup(data_vio, result, "allocation"); in allocate_block_map_page()
2204 if (!data_vio->tree_lock.locked) in allocate_block_map_page()
2207 data_vio_allocate_data_block(data_vio, VIO_BLOCK_MAP_WRITE_LOCK, in allocate_block_map_page()
2217 void vdo_find_block_map_slot(struct data_vio *data_vio) in vdo_find_block_map_slot() argument
2223 struct tree_lock *lock = &data_vio->tree_lock; in vdo_find_block_map_slot()
2224 struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; in vdo_find_block_map_slot()
2228 finish_lookup(data_vio, VDO_SHUTTING_DOWN); in vdo_find_block_map_slot()
2233 data_vio->logical.lbn % VDO_BLOCK_MAP_ENTRIES_PER_PAGE; in vdo_find_block_map_slot()
2262 if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { in vdo_find_block_map_slot()
2268 abort_load(data_vio, VDO_BAD_MAPPING); in vdo_find_block_map_slot()
2274 allocate_block_map_page(zone, data_vio); in vdo_find_block_map_slot()
2281 finish_lookup(data_vio, VDO_SUCCESS); in vdo_find_block_map_slot()
2286 load_block_map_page(zone, data_vio); in vdo_find_block_map_slot()
2945 zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio) in vdo_compute_logical_zone() argument
2947 struct block_map *map = vdo_from_data_vio(data_vio)->block_map; in vdo_compute_logical_zone()
2948 struct tree_lock *tree_lock = &data_vio->tree_lock; in vdo_compute_logical_zone()
2949 page_number_t page_number = data_vio->logical.lbn / VDO_BLOCK_MAP_ENTRIES_PER_PAGE; in vdo_compute_logical_zone()
3076 static void fetch_mapping_page(struct data_vio *data_vio, bool modifiable, in fetch_mapping_page() argument
3079 struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; in fetch_mapping_page()
3082 continue_data_vio_with_error(data_vio, VDO_SHUTTING_DOWN); in fetch_mapping_page()
3086 vdo_get_page(&data_vio->page_completion, zone, in fetch_mapping_page()
3087 data_vio->tree_lock.tree_slots[0].block_map_slot.pbn, in fetch_mapping_page()
3088 modifiable, &data_vio->vio.completion, in fetch_mapping_page()
3097 static void clear_mapped_location(struct data_vio *data_vio) in clear_mapped_location() argument
3099 data_vio->mapped = (struct zoned_pbn) { in clear_mapped_location()
3111 static int __must_check set_mapped_location(struct data_vio *data_vio, in set_mapped_location() argument
3120 result = vdo_get_physical_zone(vdo_from_data_vio(data_vio), in set_mapped_location()
3121 mapped.pbn, &data_vio->mapped.zone); in set_mapped_location()
3123 data_vio->mapped.pbn = mapped.pbn; in set_mapped_location()
3124 data_vio->mapped.state = mapped.state; in set_mapped_location()
3148 if (!data_vio->write) in set_mapped_location()
3155 clear_mapped_location(data_vio); in set_mapped_location()
3166 struct data_vio *data_vio = as_data_vio(completion->parent); in get_mapping_from_fetched_page() local
3181 tree_slot = &data_vio->tree_lock.tree_slots[0]; in get_mapping_from_fetched_page()
3184 result = set_mapped_location(data_vio, entry); in get_mapping_from_fetched_page()
3188 void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio, in vdo_update_block_map_page() argument
3193 struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; in vdo_update_block_map_page()
3197 struct tree_lock *tree_lock = &data_vio->tree_lock; in vdo_update_block_map_page()
3205 new_locked = data_vio->recovery_sequence_number; in vdo_update_block_map_page()
3226 data_vio->recovery_sequence_number = 0; in vdo_update_block_map_page()
3231 struct data_vio *data_vio = as_data_vio(completion->parent); in put_mapping_in_fetched_page() local
3252 data_vio, data_vio->new_mapped.pbn, in put_mapping_in_fetched_page()
3253 data_vio->new_mapped.state, &info->recovery_lock); in put_mapping_in_fetched_page()
3261 void vdo_get_mapped_block(struct data_vio *data_vio) in vdo_get_mapped_block() argument
3263 if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) { in vdo_get_mapped_block()
3268 clear_mapped_location(data_vio); in vdo_get_mapped_block()
3269 continue_data_vio(data_vio); in vdo_get_mapped_block()
3273 fetch_mapping_page(data_vio, false, get_mapping_from_fetched_page); in vdo_get_mapped_block()
3277 void vdo_put_mapped_block(struct data_vio *data_vio) in vdo_put_mapped_block() argument
3279 fetch_mapping_page(data_vio, true, put_mapping_in_fetched_page); in vdo_put_mapped_block()