Lines Matching refs:cache
158 struct vdo_page_cache *cache = info->cache; in get_page_buffer() local
160 return &cache->pages[(info - cache->infos) * VDO_BLOCK_SIZE]; in get_page_buffer()
180 static int initialize_info(struct vdo_page_cache *cache) in initialize_info() argument
184 INIT_LIST_HEAD(&cache->free_list); in initialize_info()
185 for (info = cache->infos; info < cache->infos + cache->page_count; info++) { in initialize_info()
188 info->cache = cache; in initialize_info()
192 result = create_metadata_vio(cache->vdo, VIO_TYPE_BLOCK_MAP, in initialize_info()
199 info->vio->completion.callback_thread_id = cache->zone->thread_id; in initialize_info()
202 list_add_tail(&info->state_entry, &cache->free_list); in initialize_info()
217 static int __must_check allocate_cache_components(struct vdo_page_cache *cache) in allocate_cache_components() argument
219 u64 size = cache->page_count * (u64) VDO_BLOCK_SIZE; in allocate_cache_components()
222 result = vdo_allocate(cache->page_count, struct page_info, "page infos", in allocate_cache_components()
223 &cache->infos); in allocate_cache_components()
227 result = vdo_allocate_memory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages); in allocate_cache_components()
231 result = vdo_int_map_create(cache->page_count, &cache->page_map); in allocate_cache_components()
235 return initialize_info(cache); in allocate_cache_components()
242 static inline void assert_on_cache_thread(struct vdo_page_cache *cache, in assert_on_cache_thread() argument
247 VDO_ASSERT_LOG_ONLY((thread_id == cache->zone->thread_id), in assert_on_cache_thread()
249 function_name, cache->zone->thread_id, thread_id); in assert_on_cache_thread()
253 static inline void assert_io_allowed(struct vdo_page_cache *cache) in assert_io_allowed() argument
255 VDO_ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&cache->zone->state), in assert_io_allowed()
260 static void report_cache_pressure(struct vdo_page_cache *cache) in report_cache_pressure() argument
262 ADD_ONCE(cache->stats.cache_pressure, 1); in report_cache_pressure()
263 if (cache->waiter_count > cache->page_count) { in report_cache_pressure()
264 if ((cache->pressure_report % LOG_INTERVAL) == 0) in report_cache_pressure()
265 vdo_log_info("page cache pressure %u", cache->stats.cache_pressure); in report_cache_pressure()
267 if (++cache->pressure_report >= DISPLAY_INTERVAL) in report_cache_pressure()
268 cache->pressure_report = 0; in report_cache_pressure()
303 struct block_map_statistics *stats = &info->cache->stats; in update_counter()
338 if (info->cache->lru_list.prev != &info->lru_entry) in update_lru()
339 list_move_tail(&info->lru_entry, &info->cache->lru_list); in update_lru()
358 list_move_tail(&info->state_entry, &info->cache->free_list); in set_info_state()
362 list_move_tail(&info->state_entry, &info->cache->outgoing_list); in set_info_state()
376 struct vdo_page_cache *cache = info->cache; in set_info_pbn() local
385 vdo_int_map_remove(cache->page_map, info->pbn); in set_info_pbn()
390 result = vdo_int_map_put(cache->page_map, pbn, info, true, NULL); in set_info_pbn()
422 static struct page_info * __must_check find_free_page(struct vdo_page_cache *cache) in find_free_page() argument
426 info = list_first_entry_or_null(&cache->free_list, struct page_info, in find_free_page()
440 static struct page_info * __must_check find_page(struct vdo_page_cache *cache, in find_page() argument
443 if ((cache->last_found != NULL) && (cache->last_found->pbn == pbn)) in find_page()
444 return cache->last_found; in find_page()
446 cache->last_found = vdo_int_map_get(cache->page_map, pbn); in find_page()
447 return cache->last_found; in find_page()
460 static struct page_info * __must_check select_lru_page(struct vdo_page_cache *cache) in select_lru_page() argument
464 list_for_each_entry(info, &cache->lru_list, lru_entry) in select_lru_page()
556 static void set_persistent_error(struct vdo_page_cache *cache, const char *context, in set_persistent_error() argument
561 struct vdo *vdo = cache->vdo; in set_persistent_error()
569 assert_on_cache_thread(cache, __func__); in set_persistent_error()
571 vdo_waitq_notify_all_waiters(&cache->free_waiters, in set_persistent_error()
573 cache->waiter_count = 0; in set_persistent_error()
575 for (info = cache->infos; info < cache->infos + cache->page_count; info++) { in set_persistent_error()
657 enter_zone_read_only_mode(completion->info->cache->zone, result); in validate_completed_page_or_enter_read_only_mode()
669 struct vdo_page_cache *cache = info->cache; in handle_load_error() local
671 assert_on_cache_thread(cache, __func__); in handle_load_error()
673 vdo_enter_read_only_mode(cache->zone->block_map->vdo, result); in handle_load_error()
674 ADD_ONCE(cache->stats.failed_reads, 1); in handle_load_error()
683 cache->outstanding_reads--; in handle_load_error()
684 check_for_drain_complete(cache->zone); in handle_load_error()
694 struct vdo_page_cache *cache = info->cache; in page_is_loaded() local
695 nonce_t nonce = info->cache->zone->block_map->nonce; in page_is_loaded()
699 assert_on_cache_thread(cache, __func__); in page_is_loaded()
725 cache->outstanding_reads--; in page_is_loaded()
726 check_for_drain_complete(cache->zone); in page_is_loaded()
736 struct vdo_page_cache *cache = info->cache; in handle_rebuild_read_error() local
738 assert_on_cache_thread(cache, __func__); in handle_rebuild_read_error()
745 ADD_ONCE(cache->stats.failed_reads, 1); in handle_rebuild_read_error()
756 continue_vio_after_io(vio, page_is_loaded, info->cache->zone->thread_id); in load_cache_page_endio()
769 struct vdo_page_cache *cache = info->cache; in launch_page_load() local
771 assert_io_allowed(cache); in launch_page_load()
782 cache->outstanding_reads++; in launch_page_load()
783 ADD_ONCE(cache->stats.pages_loaded, 1); in launch_page_load()
784 callback = (cache->rebuilding ? handle_rebuild_read_error : handle_load_error); in launch_page_load()
798 set_persistent_error(info->cache, "flush failed", completion->result); in handle_flush_error()
807 continue_vio_after_io(vio, write_pages, info->cache->zone->thread_id); in flush_endio()
811 static void save_pages(struct vdo_page_cache *cache) in save_pages() argument
816 if ((cache->pages_in_flush > 0) || (cache->pages_to_flush == 0)) in save_pages()
819 assert_io_allowed(cache); in save_pages()
821 info = list_first_entry(&cache->outgoing_list, struct page_info, state_entry); in save_pages()
823 cache->pages_in_flush = cache->pages_to_flush; in save_pages()
824 cache->pages_to_flush = 0; in save_pages()
825 ADD_ONCE(cache->stats.flush_count, 1); in save_pages()
849 info->cache->pages_to_flush++; in schedule_page_save()
850 info->cache->outstanding_writes++; in schedule_page_save()
861 save_pages(info->cache); in launch_page_save()
889 struct vdo_page_cache *cache = info->cache; in allocate_free_page() local
891 assert_on_cache_thread(cache, __func__); in allocate_free_page()
893 if (!vdo_waitq_has_waiters(&cache->free_waiters)) { in allocate_free_page()
894 if (cache->stats.cache_pressure > 0) { in allocate_free_page()
896 WRITE_ONCE(cache->stats.cache_pressure, 0); in allocate_free_page()
904 set_persistent_error(cache, "cannot reset page info", result); in allocate_free_page()
908 oldest_waiter = vdo_waitq_get_first_waiter(&cache->free_waiters); in allocate_free_page()
915 vdo_waitq_dequeue_matching_waiters(&cache->free_waiters, completion_needs_page, in allocate_free_page()
917 cache->waiter_count -= vdo_waitq_num_waiters(&info->waiting); in allocate_free_page()
936 static void discard_a_page(struct vdo_page_cache *cache) in discard_a_page() argument
938 struct page_info *info = select_lru_page(cache); in discard_a_page()
941 report_cache_pressure(cache); in discard_a_page()
953 cache->discard_count++; in discard_a_page()
964 struct vdo_page_cache *cache = vdo_page_comp->cache; in discard_page_for_completion() local
966 cache->waiter_count++; in discard_page_for_completion()
967 vdo_waitq_enqueue_waiter(&cache->free_waiters, &vdo_page_comp->waiter); in discard_page_for_completion()
968 discard_a_page(cache); in discard_page_for_completion()
976 static void discard_page_if_needed(struct vdo_page_cache *cache) in discard_page_if_needed() argument
978 if (cache->waiter_count > cache->discard_count) in discard_page_if_needed()
979 discard_a_page(cache); in discard_page_if_needed()
992 assert_on_cache_thread(info->cache, __func__); in write_has_finished()
993 info->cache->outstanding_writes--; in write_has_finished()
1007 struct vdo_page_cache *cache = info->cache; in handle_page_write_error() local
1019 ADD_ONCE(cache->stats.failed_writes, 1); in handle_page_write_error()
1020 set_persistent_error(cache, "cannot write page", result); in handle_page_write_error()
1023 discard_page_if_needed(cache); in handle_page_write_error()
1025 check_for_drain_complete(cache->zone); in handle_page_write_error()
1035 continue_vio_after_io(vio, page_is_written_out, info->cache->zone->thread_id); in write_cache_page_endio()
1047 struct vdo_page_cache *cache = info->cache; in page_is_written_out() local
1060 vdo_release_recovery_journal_block_reference(cache->zone->block_map->journal, in page_is_written_out()
1063 cache->zone->zone_number); in page_is_written_out()
1071 ADD_ONCE(cache->stats.reclaimed, reclamations); in page_is_written_out()
1074 cache->discard_count--; in page_is_written_out()
1077 discard_page_if_needed(cache); in page_is_written_out()
1081 check_for_drain_complete(cache->zone); in page_is_written_out()
1093 struct vdo_page_cache *cache = ((struct page_info *) flush_completion->parent)->cache; in write_pages() local
1100 bool has_unflushed_pages = (cache->pages_to_flush > 0); in write_pages()
1101 page_count_t pages_in_flush = cache->pages_in_flush; in write_pages()
1103 cache->pages_in_flush = 0; in write_pages()
1106 list_first_entry(&cache->outgoing_list, struct page_info, in write_pages()
1110 if (vdo_is_read_only(info->cache->vdo)) { in write_pages()
1119 ADD_ONCE(info->cache->stats.pages_saved, 1); in write_pages()
1129 save_pages(cache); in write_pages()
1144 struct vdo_page_cache *cache; in vdo_release_page_completion() local
1158 cache = page_completion->cache; in vdo_release_page_completion()
1159 assert_on_cache_thread(cache, __func__); in vdo_release_page_completion()
1171 discard_page_if_needed(cache); in vdo_release_page_completion()
1213 struct vdo_page_cache *cache = &zone->page_cache; in vdo_get_page() local
1217 assert_on_cache_thread(cache, __func__); in vdo_get_page()
1224 .cache = cache, in vdo_get_page()
1227 vdo_initialize_completion(completion, cache->vdo, VDO_PAGE_COMPLETION); in vdo_get_page()
1229 cache->zone->thread_id, parent); in vdo_get_page()
1232 if (page_completion->writable && vdo_is_read_only(cache->vdo)) { in vdo_get_page()
1238 ADD_ONCE(cache->stats.write_count, 1); in vdo_get_page()
1240 ADD_ONCE(cache->stats.read_count, 1); in vdo_get_page()
1242 info = find_page(cache, page_completion->pbn); in vdo_get_page()
1249 ADD_ONCE(cache->stats.wait_for_page, 1); in vdo_get_page()
1256 ADD_ONCE(cache->stats.found_in_cache, 1); in vdo_get_page()
1258 ADD_ONCE(cache->stats.read_outgoing, 1); in vdo_get_page()
1270 info = find_free_page(cache); in vdo_get_page()
1272 ADD_ONCE(cache->stats.fetch_required, 1); in vdo_get_page()
1278 ADD_ONCE(cache->stats.discard_required, 1); in vdo_get_page()
1327 int vdo_invalidate_page_cache(struct vdo_page_cache *cache) in vdo_invalidate_page_cache() argument
1331 assert_on_cache_thread(cache, __func__); in vdo_invalidate_page_cache()
1334 for (info = cache->infos; info < cache->infos + cache->page_count; info++) { in vdo_invalidate_page_cache()
1342 vdo_int_map_free(vdo_forget(cache->page_map)); in vdo_invalidate_page_cache()
1343 return vdo_int_map_create(cache->page_count, &cache->page_map); in vdo_invalidate_page_cache()
2820 struct vdo_page_cache *cache = &zone->page_cache; in uninitialize_block_map_zone() local
2825 if (cache->infos != NULL) { in uninitialize_block_map_zone()
2828 for (info = cache->infos; info < cache->infos + cache->page_count; info++) in uninitialize_block_map_zone()
2832 vdo_int_map_free(vdo_forget(cache->page_map)); in uninitialize_block_map_zone()
2833 vdo_free(vdo_forget(cache->infos)); in uninitialize_block_map_zone()
2834 vdo_free(vdo_forget(cache->pages)); in uninitialize_block_map_zone()
3255 add_to_dirty_lists(info->cache->zone, &info->state_entry, in put_mapping_in_fetched_page()