Lines Matching refs:rgn
67 static int ufshpb_is_valid_srgn(struct ufshpb_region *rgn, in ufshpb_is_valid_srgn() argument
70 return rgn->rgn_state != HPB_RGN_INACTIVE && in ufshpb_is_valid_srgn()
149 struct ufshpb_region *rgn; in ufshpb_iterate_rgn() local
156 rgn = hpb->rgn_tbl + rgn_idx; in ufshpb_iterate_rgn()
157 srgn = rgn->srgn_tbl + srgn_idx; in ufshpb_iterate_rgn()
170 if (rgn->rgn_state != HPB_RGN_INACTIVE) { in ufshpb_iterate_rgn()
177 rgn->read_timeout = ktime_add_ms(ktime_get(), in ufshpb_iterate_rgn()
178 rgn->hpb->params.read_timeout_ms); in ufshpb_iterate_rgn()
179 rgn->read_timeout_expiries = in ufshpb_iterate_rgn()
180 rgn->hpb->params.read_timeout_expiries; in ufshpb_iterate_rgn()
188 spin_lock(&rgn->rgn_lock); in ufshpb_iterate_rgn()
190 rgn->reads -= srgn->reads; in ufshpb_iterate_rgn()
192 set_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); in ufshpb_iterate_rgn()
195 rgn->reads++; in ufshpb_iterate_rgn()
199 spin_unlock(&rgn->rgn_lock); in ufshpb_iterate_rgn()
202 test_and_clear_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags)) { in ufshpb_iterate_rgn()
227 struct ufshpb_region *rgn; in ufshpb_test_ppn_dirty() local
233 rgn = hpb->rgn_tbl + rgn_idx; in ufshpb_test_ppn_dirty()
234 srgn = rgn->srgn_tbl + srgn_idx; in ufshpb_test_ppn_dirty()
236 if (!ufshpb_is_valid_srgn(rgn, srgn)) in ufshpb_test_ppn_dirty()
278 static inline bool is_rgn_dirty(struct ufshpb_region *rgn) in is_rgn_dirty() argument
280 return test_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); in is_rgn_dirty()
349 struct ufshpb_region *rgn; in ufshpb_prep() local
383 rgn = hpb->rgn_tbl + rgn_idx; in ufshpb_prep()
384 srgn = rgn->srgn_tbl + srgn_idx; in ufshpb_prep()
405 if (rgn->reads > hpb->entries_per_srgn) in ufshpb_prep()
530 struct ufshpb_region *rgn; in ufshpb_clear_dirty_bitmap() local
545 rgn = hpb->rgn_tbl + srgn->rgn_idx; in ufshpb_clear_dirty_bitmap()
546 clear_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); in ufshpb_clear_dirty_bitmap()
554 struct ufshpb_region *rgn; in ufshpb_update_active_info() local
557 rgn = hpb->rgn_tbl + rgn_idx; in ufshpb_update_active_info()
558 srgn = rgn->srgn_tbl + srgn_idx; in ufshpb_update_active_info()
560 list_del_init(&rgn->list_inact_rgn); in ufshpb_update_active_info()
570 struct ufshpb_region *rgn; in ufshpb_update_inactive_info() local
574 rgn = hpb->rgn_tbl + rgn_idx; in ufshpb_update_inactive_info()
576 for_each_sub_region(rgn, srgn_idx, srgn) in ufshpb_update_inactive_info()
579 if (list_empty(&rgn->list_inact_rgn)) in ufshpb_update_inactive_info()
580 list_add_tail(&rgn->list_inact_rgn, &hpb->lh_inact_rgn); in ufshpb_update_inactive_info()
588 struct ufshpb_region *rgn; in ufshpb_activate_subregion() local
604 rgn = hpb->rgn_tbl + srgn->rgn_idx; in ufshpb_activate_subregion()
606 if (unlikely(rgn->rgn_state == HPB_RGN_INACTIVE)) { in ufshpb_activate_subregion()
645 static void ufshpb_set_unmap_cmd(unsigned char *cdb, struct ufshpb_region *rgn) in ufshpb_set_unmap_cmd() argument
648 cdb[1] = rgn ? UFSHPB_WRITE_BUFFER_INACT_SINGLE_ID : in ufshpb_set_unmap_cmd()
650 if (rgn) in ufshpb_set_unmap_cmd()
651 put_unaligned_be16(rgn->rgn_idx, &cdb[2]); in ufshpb_set_unmap_cmd()
670 struct ufshpb_region *rgn) in ufshpb_execute_umap_req() argument
679 ufshpb_set_unmap_cmd(scmd->cmnd, rgn); in ufshpb_execute_umap_req()
787 struct ufshpb_region *rgn) in ufshpb_check_srgns_issue_state() argument
792 for_each_sub_region(rgn, srgn_idx, srgn) in ufshpb_check_srgns_issue_state()
804 struct ufshpb_region *rgn, *next_rgn; in ufshpb_read_to_handler() local
814 list_for_each_entry_safe(rgn, next_rgn, &lru_info->lh_lru_rgn, in ufshpb_read_to_handler()
816 bool timedout = ktime_after(ktime_get(), rgn->read_timeout); in ufshpb_read_to_handler()
819 rgn->read_timeout_expiries--; in ufshpb_read_to_handler()
820 if (is_rgn_dirty(rgn) || in ufshpb_read_to_handler()
821 rgn->read_timeout_expiries == 0) in ufshpb_read_to_handler()
822 list_add(&rgn->list_expired_rgn, &expired_list); in ufshpb_read_to_handler()
824 rgn->read_timeout = ktime_add_ms(ktime_get(), in ufshpb_read_to_handler()
831 list_for_each_entry_safe(rgn, next_rgn, &expired_list, in ufshpb_read_to_handler()
833 list_del_init(&rgn->list_expired_rgn); in ufshpb_read_to_handler()
835 ufshpb_update_inactive_info(hpb, rgn->rgn_idx); in ufshpb_read_to_handler()
849 struct ufshpb_region *rgn) in ufshpb_add_lru_info() argument
851 rgn->rgn_state = HPB_RGN_ACTIVE; in ufshpb_add_lru_info()
852 list_add_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); in ufshpb_add_lru_info()
854 if (rgn->hpb->is_hcm) { in ufshpb_add_lru_info()
855 rgn->read_timeout = in ufshpb_add_lru_info()
857 rgn->hpb->params.read_timeout_ms); in ufshpb_add_lru_info()
858 rgn->read_timeout_expiries = in ufshpb_add_lru_info()
859 rgn->hpb->params.read_timeout_expiries; in ufshpb_add_lru_info()
864 struct ufshpb_region *rgn) in ufshpb_hit_lru_info() argument
866 list_move_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); in ufshpb_hit_lru_info()
872 struct ufshpb_region *rgn, *victim_rgn = NULL; in ufshpb_victim_lru_info() local
874 list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) { in ufshpb_victim_lru_info()
875 if (ufshpb_check_srgns_issue_state(hpb, rgn)) in ufshpb_victim_lru_info()
883 rgn->reads > hpb->params.eviction_thld_exit) in ufshpb_victim_lru_info()
886 victim_rgn = rgn; in ufshpb_victim_lru_info()
899 struct ufshpb_region *rgn) in ufshpb_cleanup_lru_info() argument
901 list_del_init(&rgn->list_lru_rgn); in ufshpb_cleanup_lru_info()
902 rgn->rgn_state = HPB_RGN_INACTIVE; in ufshpb_cleanup_lru_info()
917 struct ufshpb_region *rgn, in ufshpb_issue_umap_req() argument
921 int rgn_idx = rgn ? rgn->rgn_idx : 0; in ufshpb_issue_umap_req()
927 ufshpb_execute_umap_req(hpb, umap_req, rgn); in ufshpb_issue_umap_req()
933 struct ufshpb_region *rgn) in ufshpb_issue_umap_single_req() argument
935 return ufshpb_issue_umap_req(hpb, rgn, true); in ufshpb_issue_umap_single_req()
939 struct ufshpb_region *rgn) in __ufshpb_evict_region() argument
947 dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "evict region %d\n", rgn->rgn_idx); in __ufshpb_evict_region()
949 ufshpb_cleanup_lru_info(lru_info, rgn); in __ufshpb_evict_region()
951 for_each_sub_region(rgn, srgn_idx, srgn) in __ufshpb_evict_region()
955 static int ufshpb_evict_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) in ufshpb_evict_region() argument
961 if (rgn->rgn_state == HPB_RGN_PINNED) { in ufshpb_evict_region()
964 rgn->rgn_idx); in ufshpb_evict_region()
968 if (!list_empty(&rgn->list_lru_rgn)) { in ufshpb_evict_region()
969 if (ufshpb_check_srgns_issue_state(hpb, rgn)) { in ufshpb_evict_region()
976 ret = ufshpb_issue_umap_single_req(hpb, rgn); in ufshpb_evict_region()
982 __ufshpb_evict_region(hpb, rgn); in ufshpb_evict_region()
990 struct ufshpb_region *rgn, in ufshpb_issue_map_req() argument
1008 if ((rgn->rgn_state == HPB_RGN_INACTIVE) && in ufshpb_issue_map_req()
1036 rgn->rgn_idx, srgn->srgn_idx); in ufshpb_issue_map_req()
1066 static int ufshpb_add_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) in ufshpb_add_region() argument
1079 if (!list_empty(&rgn->list_lru_rgn)) { in ufshpb_add_region()
1080 ufshpb_hit_lru_info(lru_info, rgn); in ufshpb_add_region()
1084 if (rgn->rgn_state == HPB_RGN_INACTIVE) { in ufshpb_add_region()
1100 rgn->reads < hpb->params.eviction_thld_enter) { in ufshpb_add_region()
1139 ufshpb_add_lru_info(lru_info, rgn); in ufshpb_add_region()
1153 struct ufshpb_region *rgn; in ufshpb_submit_region_inactive() local
1163 rgn = hpb->rgn_tbl + region_index; in ufshpb_submit_region_inactive()
1169 if (rgn->rgn_state != HPB_RGN_INACTIVE) { in ufshpb_submit_region_inactive()
1170 for (subregion_index = 0; subregion_index < rgn->srgn_cnt; subregion_index++) { in ufshpb_submit_region_inactive()
1171 srgn = rgn->srgn_tbl + subregion_index; in ufshpb_submit_region_inactive()
1182 struct ufshpb_region *rgn; in ufshpb_rsp_req_region_update() local
1199 rgn = hpb->rgn_tbl + rgn_i; in ufshpb_rsp_req_region_update()
1201 (rgn->rgn_state != HPB_RGN_ACTIVE || is_rgn_dirty(rgn))) { in ufshpb_rsp_req_region_update()
1218 srgn = rgn->srgn_tbl + srgn_i; in ufshpb_rsp_req_region_update()
1255 struct ufshpb_region *rgn; in ufshpb_set_regions_update() local
1260 list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) in ufshpb_set_regions_update()
1261 set_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags); in ufshpb_set_regions_update()
1291 struct ufshpb_region *rgn; in ufshpb_dev_reset_handler() local
1293 list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) in ufshpb_dev_reset_handler()
1294 ufshpb_submit_region_inactive(hpb, rgn->rgn_idx); in ufshpb_dev_reset_handler()
1383 struct ufshpb_region *rgn, in ufshpb_add_active_list() argument
1386 if (!list_empty(&rgn->list_inact_rgn)) in ufshpb_add_active_list()
1398 struct ufshpb_region *rgn, in ufshpb_add_pending_evict_list() argument
1404 if (!list_empty(&rgn->list_inact_rgn)) in ufshpb_add_pending_evict_list()
1407 for_each_sub_region(rgn, srgn_idx, srgn) in ufshpb_add_pending_evict_list()
1411 list_add_tail(&rgn->list_inact_rgn, pending_list); in ufshpb_add_pending_evict_list()
1416 struct ufshpb_region *rgn; in ufshpb_run_active_subregion_list() local
1431 rgn = hpb->rgn_tbl + srgn->rgn_idx; in ufshpb_run_active_subregion_list()
1432 ret = ufshpb_add_region(hpb, rgn); in ufshpb_run_active_subregion_list()
1436 ret = ufshpb_issue_map_req(hpb, rgn, srgn); in ufshpb_run_active_subregion_list()
1440 ret, rgn->rgn_idx, srgn->srgn_idx); in ufshpb_run_active_subregion_list()
1450 rgn->rgn_idx, srgn->srgn_idx); in ufshpb_run_active_subregion_list()
1452 ufshpb_add_active_list(hpb, rgn, srgn); in ufshpb_run_active_subregion_list()
1458 struct ufshpb_region *rgn; in ufshpb_run_inactive_region_list() local
1464 while ((rgn = list_first_entry_or_null(&hpb->lh_inact_rgn, in ufshpb_run_inactive_region_list()
1470 list_del_init(&rgn->list_inact_rgn); in ufshpb_run_inactive_region_list()
1473 ret = ufshpb_evict_region(hpb, rgn); in ufshpb_run_inactive_region_list()
1476 ufshpb_add_pending_evict_list(hpb, rgn, &pending_list); in ufshpb_run_inactive_region_list()
1495 struct ufshpb_region *rgn = hpb->rgn_tbl + rgn_idx; in ufshpb_normalization_work_handler() local
1498 spin_lock(&rgn->rgn_lock); in ufshpb_normalization_work_handler()
1499 rgn->reads = 0; in ufshpb_normalization_work_handler()
1501 struct ufshpb_subregion *srgn = rgn->srgn_tbl + srgn_idx; in ufshpb_normalization_work_handler()
1504 rgn->reads += srgn->reads; in ufshpb_normalization_work_handler()
1506 spin_unlock(&rgn->rgn_lock); in ufshpb_normalization_work_handler()
1508 if (rgn->rgn_state != HPB_RGN_ACTIVE || rgn->reads) in ufshpb_normalization_work_handler()
1513 ufshpb_update_inactive_info(hpb, rgn->rgn_idx); in ufshpb_normalization_work_handler()
1538 struct ufshpb_region *rgn) in ufshpb_init_pinned_active_region() argument
1544 for_each_sub_region(rgn, srgn_idx, srgn) { in ufshpb_init_pinned_active_region()
1557 rgn->rgn_state = HPB_RGN_PINNED; in ufshpb_init_pinned_active_region()
1562 srgn = rgn->srgn_tbl + i; in ufshpb_init_pinned_active_region()
1569 struct ufshpb_region *rgn, bool last) in ufshpb_init_subregion_tbl() argument
1574 for_each_sub_region(rgn, srgn_idx, srgn) { in ufshpb_init_subregion_tbl()
1577 srgn->rgn_idx = rgn->rgn_idx; in ufshpb_init_subregion_tbl()
1587 struct ufshpb_region *rgn, int srgn_cnt) in ufshpb_alloc_subregion_tbl() argument
1589 rgn->srgn_tbl = kvcalloc(srgn_cnt, sizeof(struct ufshpb_subregion), in ufshpb_alloc_subregion_tbl()
1591 if (!rgn->srgn_tbl) in ufshpb_alloc_subregion_tbl()
1594 rgn->srgn_cnt = srgn_cnt; in ufshpb_alloc_subregion_tbl()
1653 struct ufshpb_region *rgn_table, *rgn; in ufshpb_alloc_region_tbl() local
1666 rgn = rgn_table + rgn_idx; in ufshpb_alloc_region_tbl()
1667 rgn->rgn_idx = rgn_idx; in ufshpb_alloc_region_tbl()
1669 spin_lock_init(&rgn->rgn_lock); in ufshpb_alloc_region_tbl()
1671 INIT_LIST_HEAD(&rgn->list_inact_rgn); in ufshpb_alloc_region_tbl()
1672 INIT_LIST_HEAD(&rgn->list_lru_rgn); in ufshpb_alloc_region_tbl()
1673 INIT_LIST_HEAD(&rgn->list_expired_rgn); in ufshpb_alloc_region_tbl()
1681 ret = ufshpb_alloc_subregion_tbl(hpb, rgn, srgn_cnt); in ufshpb_alloc_region_tbl()
1684 ufshpb_init_subregion_tbl(hpb, rgn, last_srgn); in ufshpb_alloc_region_tbl()
1687 ret = ufshpb_init_pinned_active_region(hba, hpb, rgn); in ufshpb_alloc_region_tbl()
1691 rgn->rgn_state = HPB_RGN_INACTIVE; in ufshpb_alloc_region_tbl()
1694 rgn->rgn_flags = 0; in ufshpb_alloc_region_tbl()
1695 rgn->hpb = hpb; in ufshpb_alloc_region_tbl()
1711 struct ufshpb_region *rgn) in ufshpb_destroy_subregion_tbl() argument
1716 for_each_sub_region(rgn, srgn_idx, srgn) in ufshpb_destroy_subregion_tbl()
1728 struct ufshpb_region *rgn; in ufshpb_destroy_region_tbl() local
1730 rgn = hpb->rgn_tbl + rgn_idx; in ufshpb_destroy_region_tbl()
1731 if (rgn->rgn_state != HPB_RGN_INACTIVE) { in ufshpb_destroy_region_tbl()
1732 rgn->rgn_state = HPB_RGN_INACTIVE; in ufshpb_destroy_region_tbl()
1734 ufshpb_destroy_subregion_tbl(hpb, rgn); in ufshpb_destroy_region_tbl()
1737 kvfree(rgn->srgn_tbl); in ufshpb_destroy_region_tbl()
2254 struct ufshpb_region *rgn, *next_rgn; in ufshpb_discard_rsp_lists() local
2264 list_for_each_entry_safe(rgn, next_rgn, &hpb->lh_inact_rgn, in ufshpb_discard_rsp_lists()
2266 list_del_init(&rgn->list_inact_rgn); in ufshpb_discard_rsp_lists()