Lines Matching refs:level
67 static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level) in kvm_block_mapping_supported() argument
69 u64 granule = kvm_granule_size(level); in kvm_block_mapping_supported()
71 if (!kvm_level_supports_block_mapping(level)) in kvm_block_mapping_supported()
83 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) in kvm_pgtable_idx() argument
85 u64 shift = kvm_granule_shift(level); in kvm_pgtable_idx()
114 static bool kvm_pte_table(kvm_pte_t pte, u32 level) in kvm_pte_table() argument
116 if (level == KVM_PGTABLE_MAX_LEVELS - 1) in kvm_pte_table()
157 static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level) in kvm_init_valid_leaf_pte() argument
160 u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE : in kvm_init_valid_leaf_pte()
176 u32 level, kvm_pte_t *ptep, in kvm_pgtable_visitor_cb() argument
180 return walker->cb(addr, data->end, level, ptep, flag, walker->arg); in kvm_pgtable_visitor_cb()
184 kvm_pte_t *pgtable, u32 level);
187 kvm_pte_t *ptep, u32 level) in __kvm_pgtable_visit() argument
192 bool table = kvm_pte_table(pte, level); in __kvm_pgtable_visit()
196 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
201 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
204 table = kvm_pte_table(pte, level); in __kvm_pgtable_visit()
211 data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level)); in __kvm_pgtable_visit()
212 data->addr += kvm_granule_size(level); in __kvm_pgtable_visit()
217 ret = __kvm_pgtable_walk(data, childp, level + 1); in __kvm_pgtable_visit()
222 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
231 kvm_pte_t *pgtable, u32 level) in __kvm_pgtable_walk() argument
236 if (WARN_ON_ONCE(level >= KVM_PGTABLE_MAX_LEVELS)) in __kvm_pgtable_walk()
239 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) { in __kvm_pgtable_walk()
245 ret = __kvm_pgtable_visit(data, ptep, level); in __kvm_pgtable_walk()
292 u32 level; member
295 static int leaf_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in leaf_walker() argument
301 data->level = level; in leaf_walker()
307 kvm_pte_t *ptep, u32 *level) in kvm_pgtable_get_leaf() argument
322 if (level) in kvm_pgtable_get_leaf()
323 *level = data.level; in kvm_pgtable_get_leaf()
401 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level, in hyp_map_walker_try_leaf() argument
405 u64 granule = kvm_granule_size(level), phys = data->phys; in hyp_map_walker_try_leaf()
407 if (!kvm_block_mapping_supported(addr, end, phys, level)) in hyp_map_walker_try_leaf()
410 new = kvm_init_valid_leaf_pte(phys, data->attr, level); in hyp_map_walker_try_leaf()
418 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_map_walker() argument
425 if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg)) in hyp_map_walker()
428 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in hyp_map_walker()
481 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_free_walker() argument
624 u32 level, struct kvm_pgtable_mm_ops *mm_ops) in stage2_put_pte() argument
632 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level); in stage2_put_pte()
649 static bool stage2_leaf_mapping_allowed(u64 addr, u64 end, u32 level, in stage2_leaf_mapping_allowed() argument
652 if (data->force_pte && (level < (KVM_PGTABLE_MAX_LEVELS - 1))) in stage2_leaf_mapping_allowed()
655 return kvm_block_mapping_supported(addr, end, data->phys, level); in stage2_leaf_mapping_allowed()
658 static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, in stage2_map_walker_try_leaf() argument
663 u64 granule = kvm_granule_size(level), phys = data->phys; in stage2_map_walker_try_leaf()
667 if (!stage2_leaf_mapping_allowed(addr, end, level, data)) in stage2_map_walker_try_leaf()
671 new = kvm_init_valid_leaf_pte(phys, data->attr, level); in stage2_map_walker_try_leaf()
685 stage2_put_pte(ptep, data->mmu, addr, level, mm_ops); in stage2_map_walker_try_leaf()
704 static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, in stage2_map_walk_table_pre() argument
711 if (!stage2_leaf_mapping_allowed(addr, end, level, data)) in stage2_map_walk_table_pre()
727 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walk_leaf() argument
741 ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data); in stage2_map_walk_leaf()
745 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in stage2_map_walk_leaf()
761 stage2_put_pte(ptep, data->mmu, addr, level, mm_ops); in stage2_map_walk_leaf()
769 static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level, in stage2_map_walk_table_post() argument
784 ret = stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walk_table_post()
814 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walker() argument
821 return stage2_map_walk_table_pre(addr, end, level, ptep, data); in stage2_map_walker()
823 return stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walker()
825 return stage2_map_walk_table_post(addr, end, level, ptep, data); in stage2_map_walker()
890 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_unmap_walker() argument
908 if (kvm_pte_table(pte, level)) { in stage2_unmap_walker()
922 stage2_put_pte(ptep, mmu, addr, level, mm_ops); in stage2_unmap_walker()
929 kvm_granule_size(level)); in stage2_unmap_walker()
953 u32 level; member
957 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_attr_walker() argument
968 data->level = level; in stage2_attr_walker()
986 kvm_granule_size(level)); in stage2_attr_walker()
996 u32 *level) in stage2_update_leaf_attrs() argument
1018 if (level) in stage2_update_leaf_attrs()
1019 *level = data.level; in stage2_update_leaf_attrs()
1064 u32 level; in kvm_pgtable_stage2_relax_perms() local
1079 ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level); in kvm_pgtable_stage2_relax_perms()
1081 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level); in kvm_pgtable_stage2_relax_perms()
1085 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_flush_walker() argument
1100 kvm_granule_size(level)); in stage2_flush_walker()
1147 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_free_walker() argument
1159 if (kvm_pte_table(pte, level)) in stage2_free_walker()