1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020 Google LLC
4  * Author: Will Deacon <will@kernel.org>
5  */
6 
7 #ifndef __ARM64_KVM_PGTABLE_H__
8 #define __ARM64_KVM_PGTABLE_H__
9 
10 #include <linux/bits.h>
11 #include <linux/kvm_host.h>
12 #include <linux/types.h>
13 
14 #define KVM_PGTABLE_MAX_LEVELS		4U
15 
16 /*
17  * The largest supported block sizes for KVM (no 52-bit PA support):
18  *  - 4K (level 1):	1GB
19  *  - 16K (level 2):	32MB
20  *  - 64K (level 2):	512MB
21  */
22 #ifdef CONFIG_ARM64_4K_PAGES
23 #define KVM_PGTABLE_MIN_BLOCK_LEVEL	1U
24 #else
25 #define KVM_PGTABLE_MIN_BLOCK_LEVEL	2U
26 #endif
27 
kvm_get_parange(u64 mmfr0)28 static inline u64 kvm_get_parange(u64 mmfr0)
29 {
30 	u64 parange = cpuid_feature_extract_unsigned_field(mmfr0,
31 				ID_AA64MMFR0_EL1_PARANGE_SHIFT);
32 	if (parange > ID_AA64MMFR0_EL1_PARANGE_MAX)
33 		parange = ID_AA64MMFR0_EL1_PARANGE_MAX;
34 
35 	return parange;
36 }
37 
38 typedef u64 kvm_pte_t;
39 
40 #define KVM_PTE_VALID			BIT(0)
41 
42 #define KVM_PTE_ADDR_MASK		GENMASK(47, PAGE_SHIFT)
43 #define KVM_PTE_ADDR_51_48		GENMASK(15, 12)
44 
45 #define KVM_PHYS_INVALID		(-1ULL)
46 
kvm_pte_valid(kvm_pte_t pte)47 static inline bool kvm_pte_valid(kvm_pte_t pte)
48 {
49 	return pte & KVM_PTE_VALID;
50 }
51 
kvm_pte_to_phys(kvm_pte_t pte)52 static inline u64 kvm_pte_to_phys(kvm_pte_t pte)
53 {
54 	u64 pa = pte & KVM_PTE_ADDR_MASK;
55 
56 	if (PAGE_SHIFT == 16)
57 		pa |= FIELD_GET(KVM_PTE_ADDR_51_48, pte) << 48;
58 
59 	return pa;
60 }
61 
kvm_phys_to_pte(u64 pa)62 static inline kvm_pte_t kvm_phys_to_pte(u64 pa)
63 {
64 	kvm_pte_t pte = pa & KVM_PTE_ADDR_MASK;
65 
66 	if (PAGE_SHIFT == 16) {
67 		pa &= GENMASK(51, 48);
68 		pte |= FIELD_PREP(KVM_PTE_ADDR_51_48, pa >> 48);
69 	}
70 
71 	return pte;
72 }
73 
kvm_pte_to_pfn(kvm_pte_t pte)74 static inline kvm_pfn_t kvm_pte_to_pfn(kvm_pte_t pte)
75 {
76 	return __phys_to_pfn(kvm_pte_to_phys(pte));
77 }
78 
kvm_granule_shift(u32 level)79 static inline u64 kvm_granule_shift(u32 level)
80 {
81 	/* Assumes KVM_PGTABLE_MAX_LEVELS is 4 */
82 	return ARM64_HW_PGTABLE_LEVEL_SHIFT(level);
83 }
84 
kvm_granule_size(u32 level)85 static inline u64 kvm_granule_size(u32 level)
86 {
87 	return BIT(kvm_granule_shift(level));
88 }
89 
kvm_level_supports_block_mapping(u32 level)90 static inline bool kvm_level_supports_block_mapping(u32 level)
91 {
92 	return level >= KVM_PGTABLE_MIN_BLOCK_LEVEL;
93 }
94 
95 /**
96  * struct kvm_pgtable_mm_ops - Memory management callbacks.
97  * @zalloc_page:		Allocate a single zeroed memory page.
98  *				The @arg parameter can be used by the walker
99  *				to pass a memcache. The initial refcount of
100  *				the page is 1.
101  * @zalloc_pages_exact:		Allocate an exact number of zeroed memory pages.
102  *				The @size parameter is in bytes, and is rounded
103  *				up to the next page boundary. The resulting
104  *				allocation is physically contiguous.
105  * @free_pages_exact:		Free an exact number of memory pages previously
106  *				allocated by zalloc_pages_exact.
107  * @free_removed_table:		Free a removed paging structure by unlinking and
108  *				dropping references.
109  * @get_page:			Increment the refcount on a page.
110  * @put_page:			Decrement the refcount on a page. When the
111  *				refcount reaches 0 the page is automatically
112  *				freed.
113  * @page_count:			Return the refcount of a page.
114  * @phys_to_virt:		Convert a physical address into a virtual
115  *				address	mapped in the current context.
116  * @virt_to_phys:		Convert a virtual address mapped in the current
117  *				context into a physical address.
118  * @dcache_clean_inval_poc:	Clean and invalidate the data cache to the PoC
119  *				for the	specified memory address range.
120  * @icache_inval_pou:		Invalidate the instruction cache to the PoU
121  *				for the specified memory address range.
122  */
123 struct kvm_pgtable_mm_ops {
124 	void*		(*zalloc_page)(void *arg);
125 	void*		(*zalloc_pages_exact)(size_t size);
126 	void		(*free_pages_exact)(void *addr, size_t size);
127 	void		(*free_removed_table)(void *addr, u32 level);
128 	void		(*get_page)(void *addr);
129 	void		(*put_page)(void *addr);
130 	int		(*page_count)(void *addr);
131 	void*		(*phys_to_virt)(phys_addr_t phys);
132 	phys_addr_t	(*virt_to_phys)(void *addr);
133 	void		(*dcache_clean_inval_poc)(void *addr, size_t size);
134 	void		(*icache_inval_pou)(void *addr, size_t size);
135 };
136 
137 /**
138  * enum kvm_pgtable_stage2_flags - Stage-2 page-table flags.
139  * @KVM_PGTABLE_S2_NOFWB:	Don't enforce Normal-WB even if the CPUs have
140  *				ARM64_HAS_STAGE2_FWB.
141  * @KVM_PGTABLE_S2_IDMAP:	Only use identity mappings.
142  */
143 enum kvm_pgtable_stage2_flags {
144 	KVM_PGTABLE_S2_NOFWB			= BIT(0),
145 	KVM_PGTABLE_S2_IDMAP			= BIT(1),
146 };
147 
148 /**
149  * enum kvm_pgtable_prot - Page-table permissions and attributes.
150  * @KVM_PGTABLE_PROT_X:		Execute permission.
151  * @KVM_PGTABLE_PROT_W:		Write permission.
152  * @KVM_PGTABLE_PROT_R:		Read permission.
153  * @KVM_PGTABLE_PROT_DEVICE:	Device attributes.
154  * @KVM_PGTABLE_PROT_SW0:	Software bit 0.
155  * @KVM_PGTABLE_PROT_SW1:	Software bit 1.
156  * @KVM_PGTABLE_PROT_SW2:	Software bit 2.
157  * @KVM_PGTABLE_PROT_SW3:	Software bit 3.
158  */
159 enum kvm_pgtable_prot {
160 	KVM_PGTABLE_PROT_X			= BIT(0),
161 	KVM_PGTABLE_PROT_W			= BIT(1),
162 	KVM_PGTABLE_PROT_R			= BIT(2),
163 
164 	KVM_PGTABLE_PROT_DEVICE			= BIT(3),
165 
166 	KVM_PGTABLE_PROT_SW0			= BIT(55),
167 	KVM_PGTABLE_PROT_SW1			= BIT(56),
168 	KVM_PGTABLE_PROT_SW2			= BIT(57),
169 	KVM_PGTABLE_PROT_SW3			= BIT(58),
170 };
171 
172 #define KVM_PGTABLE_PROT_RW	(KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W)
173 #define KVM_PGTABLE_PROT_RWX	(KVM_PGTABLE_PROT_RW | KVM_PGTABLE_PROT_X)
174 
175 #define PKVM_HOST_MEM_PROT	KVM_PGTABLE_PROT_RWX
176 #define PKVM_HOST_MMIO_PROT	KVM_PGTABLE_PROT_RW
177 
178 #define PAGE_HYP		KVM_PGTABLE_PROT_RW
179 #define PAGE_HYP_EXEC		(KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_X)
180 #define PAGE_HYP_RO		(KVM_PGTABLE_PROT_R)
181 #define PAGE_HYP_DEVICE		(PAGE_HYP | KVM_PGTABLE_PROT_DEVICE)
182 
183 typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,
184 					   enum kvm_pgtable_prot prot);
185 
186 /**
187  * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk.
188  * @KVM_PGTABLE_WALK_LEAF:		Visit leaf entries, including invalid
189  *					entries.
190  * @KVM_PGTABLE_WALK_TABLE_PRE:		Visit table entries before their
191  *					children.
192  * @KVM_PGTABLE_WALK_TABLE_POST:	Visit table entries after their
193  *					children.
194  * @KVM_PGTABLE_WALK_SHARED:		Indicates the page-tables may be shared
195  *					with other software walkers.
196  * @KVM_PGTABLE_WALK_HANDLE_FAULT:	Indicates the page-table walk was
197  *					invoked from a fault handler.
198  */
199 enum kvm_pgtable_walk_flags {
200 	KVM_PGTABLE_WALK_LEAF			= BIT(0),
201 	KVM_PGTABLE_WALK_TABLE_PRE		= BIT(1),
202 	KVM_PGTABLE_WALK_TABLE_POST		= BIT(2),
203 	KVM_PGTABLE_WALK_SHARED			= BIT(3),
204 	KVM_PGTABLE_WALK_HANDLE_FAULT		= BIT(4),
205 };
206 
207 struct kvm_pgtable_visit_ctx {
208 	kvm_pte_t				*ptep;
209 	kvm_pte_t				old;
210 	void					*arg;
211 	struct kvm_pgtable_mm_ops		*mm_ops;
212 	u64					addr;
213 	u64					end;
214 	u32					level;
215 	enum kvm_pgtable_walk_flags		flags;
216 };
217 
218 typedef int (*kvm_pgtable_visitor_fn_t)(const struct kvm_pgtable_visit_ctx *ctx,
219 					enum kvm_pgtable_walk_flags visit);
220 
kvm_pgtable_walk_shared(const struct kvm_pgtable_visit_ctx * ctx)221 static inline bool kvm_pgtable_walk_shared(const struct kvm_pgtable_visit_ctx *ctx)
222 {
223 	return ctx->flags & KVM_PGTABLE_WALK_SHARED;
224 }
225 
226 /**
227  * struct kvm_pgtable_walker - Hook into a page-table walk.
228  * @cb:		Callback function to invoke during the walk.
229  * @arg:	Argument passed to the callback function.
230  * @flags:	Bitwise-OR of flags to identify the entry types on which to
231  *		invoke the callback function.
232  */
233 struct kvm_pgtable_walker {
234 	const kvm_pgtable_visitor_fn_t		cb;
235 	void * const				arg;
236 	const enum kvm_pgtable_walk_flags	flags;
237 };
238 
239 /*
240  * RCU cannot be used in a non-kernel context such as the hyp. As such, page
241  * table walkers used in hyp do not call into RCU and instead use other
242  * synchronization mechanisms (such as a spinlock).
243  */
244 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)
245 
246 typedef kvm_pte_t *kvm_pteref_t;
247 
kvm_dereference_pteref(struct kvm_pgtable_walker * walker,kvm_pteref_t pteref)248 static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walker,
249 						kvm_pteref_t pteref)
250 {
251 	return pteref;
252 }
253 
kvm_pgtable_walk_begin(struct kvm_pgtable_walker * walker)254 static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
255 {
256 	/*
257 	 * Due to the lack of RCU (or a similar protection scheme), only
258 	 * non-shared table walkers are allowed in the hypervisor.
259 	 */
260 	if (walker->flags & KVM_PGTABLE_WALK_SHARED)
261 		return -EPERM;
262 
263 	return 0;
264 }
265 
kvm_pgtable_walk_end(struct kvm_pgtable_walker * walker)266 static inline void kvm_pgtable_walk_end(struct kvm_pgtable_walker *walker) {}
267 
kvm_pgtable_walk_lock_held(void)268 static inline bool kvm_pgtable_walk_lock_held(void)
269 {
270 	return true;
271 }
272 
273 #else
274 
275 typedef kvm_pte_t __rcu *kvm_pteref_t;
276 
kvm_dereference_pteref(struct kvm_pgtable_walker * walker,kvm_pteref_t pteref)277 static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walker,
278 						kvm_pteref_t pteref)
279 {
280 	return rcu_dereference_check(pteref, !(walker->flags & KVM_PGTABLE_WALK_SHARED));
281 }
282 
kvm_pgtable_walk_begin(struct kvm_pgtable_walker * walker)283 static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
284 {
285 	if (walker->flags & KVM_PGTABLE_WALK_SHARED)
286 		rcu_read_lock();
287 
288 	return 0;
289 }
290 
kvm_pgtable_walk_end(struct kvm_pgtable_walker * walker)291 static inline void kvm_pgtable_walk_end(struct kvm_pgtable_walker *walker)
292 {
293 	if (walker->flags & KVM_PGTABLE_WALK_SHARED)
294 		rcu_read_unlock();
295 }
296 
kvm_pgtable_walk_lock_held(void)297 static inline bool kvm_pgtable_walk_lock_held(void)
298 {
299 	return rcu_read_lock_held();
300 }
301 
302 #endif
303 
304 /**
305  * struct kvm_pgtable - KVM page-table.
306  * @ia_bits:		Maximum input address size, in bits.
307  * @start_level:	Level at which the page-table walk starts.
308  * @pgd:		Pointer to the first top-level entry of the page-table.
309  * @mm_ops:		Memory management callbacks.
310  * @mmu:		Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
311  * @flags:		Stage-2 page-table flags.
312  * @force_pte_cb:	Function that returns true if page level mappings must
313  *			be used instead of block mappings.
314  */
315 struct kvm_pgtable {
316 	u32					ia_bits;
317 	u32					start_level;
318 	kvm_pteref_t				pgd;
319 	struct kvm_pgtable_mm_ops		*mm_ops;
320 
321 	/* Stage-2 only */
322 	struct kvm_s2_mmu			*mmu;
323 	enum kvm_pgtable_stage2_flags		flags;
324 	kvm_pgtable_force_pte_cb_t		force_pte_cb;
325 };
326 
327 /**
328  * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table.
329  * @pgt:	Uninitialised page-table structure to initialise.
330  * @va_bits:	Maximum virtual address bits.
331  * @mm_ops:	Memory management callbacks.
332  *
333  * Return: 0 on success, negative error code on failure.
334  */
335 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
336 			 struct kvm_pgtable_mm_ops *mm_ops);
337 
338 /**
339  * kvm_pgtable_hyp_destroy() - Destroy an unused hypervisor stage-1 page-table.
340  * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
341  *
342  * The page-table is assumed to be unreachable by any hardware walkers prior
343  * to freeing and therefore no TLB invalidation is performed.
344  */
345 void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt);
346 
347 /**
348  * kvm_pgtable_hyp_map() - Install a mapping in a hypervisor stage-1 page-table.
349  * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
350  * @addr:	Virtual address at which to place the mapping.
351  * @size:	Size of the mapping.
352  * @phys:	Physical address of the memory to map.
353  * @prot:	Permissions and attributes for the mapping.
354  *
355  * The offset of @addr within a page is ignored, @size is rounded-up to
356  * the next page boundary and @phys is rounded-down to the previous page
357  * boundary.
358  *
359  * If device attributes are not explicitly requested in @prot, then the
360  * mapping will be normal, cacheable. Attempts to install a new mapping
361  * for a virtual address that is already mapped will be rejected with an
362  * error and a WARN().
363  *
364  * Return: 0 on success, negative error code on failure.
365  */
366 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
367 			enum kvm_pgtable_prot prot);
368 
369 /**
370  * kvm_pgtable_hyp_unmap() - Remove a mapping from a hypervisor stage-1 page-table.
371  * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
372  * @addr:	Virtual address from which to remove the mapping.
373  * @size:	Size of the mapping.
374  *
375  * The offset of @addr within a page is ignored, @size is rounded-up to
376  * the next page boundary and @phys is rounded-down to the previous page
377  * boundary.
378  *
379  * TLB invalidation is performed for each page-table entry cleared during the
380  * unmapping operation and the reference count for the page-table page
381  * containing the cleared entry is decremented, with unreferenced pages being
382  * freed. The unmapping operation will stop early if it encounters either an
383  * invalid page-table entry or a valid block mapping which maps beyond the range
384  * being unmapped.
385  *
386  * Return: Number of bytes unmapped, which may be 0.
387  */
388 u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
389 
390 /**
391  * kvm_get_vtcr() - Helper to construct VTCR_EL2
392  * @mmfr0:	Sanitized value of SYS_ID_AA64MMFR0_EL1 register.
393  * @mmfr1:	Sanitized value of SYS_ID_AA64MMFR1_EL1 register.
394  * @phys_shfit:	Value to set in VTCR_EL2.T0SZ.
395  *
396  * The VTCR value is common across all the physical CPUs on the system.
397  * We use system wide sanitised values to fill in different fields,
398  * except for Hardware Management of Access Flags. HA Flag is set
399  * unconditionally on all CPUs, as it is safe to run with or without
400  * the feature and the bit is RES0 on CPUs that don't support it.
401  *
402  * Return: VTCR_EL2 value
403  */
404 u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift);
405 
406 /**
407  * kvm_pgtable_stage2_pgd_size() - Helper to compute size of a stage-2 PGD
408  * @vtcr:	Content of the VTCR register.
409  *
410  * Return: the size (in bytes) of the stage-2 PGD
411  */
412 size_t kvm_pgtable_stage2_pgd_size(u64 vtcr);
413 
414 /**
415  * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.
416  * @pgt:	Uninitialised page-table structure to initialise.
417  * @mmu:	S2 MMU context for this S2 translation
418  * @mm_ops:	Memory management callbacks.
419  * @flags:	Stage-2 configuration flags.
420  * @force_pte_cb: Function that returns true if page level mappings must
421  *		be used instead of block mappings.
422  *
423  * Return: 0 on success, negative error code on failure.
424  */
425 int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
426 			      struct kvm_pgtable_mm_ops *mm_ops,
427 			      enum kvm_pgtable_stage2_flags flags,
428 			      kvm_pgtable_force_pte_cb_t force_pte_cb);
429 
430 #define kvm_pgtable_stage2_init(pgt, mmu, mm_ops) \
431 	__kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, NULL)
432 
433 /**
434  * kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table.
435  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
436  *
437  * The page-table is assumed to be unreachable by any hardware walkers prior
438  * to freeing and therefore no TLB invalidation is performed.
439  */
440 void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);
441 
442 /**
443  * kvm_pgtable_stage2_free_removed() - Free a removed stage-2 paging structure.
444  * @mm_ops:	Memory management callbacks.
445  * @pgtable:	Unlinked stage-2 paging structure to be freed.
446  * @level:	Level of the stage-2 paging structure to be freed.
447  *
448  * The page-table is assumed to be unreachable by any hardware walkers prior to
449  * freeing and therefore no TLB invalidation is performed.
450  */
451 void kvm_pgtable_stage2_free_removed(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level);
452 
453 /**
454  * kvm_pgtable_stage2_map() - Install a mapping in a guest stage-2 page-table.
455  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
456  * @addr:	Intermediate physical address at which to place the mapping.
457  * @size:	Size of the mapping.
458  * @phys:	Physical address of the memory to map.
459  * @prot:	Permissions and attributes for the mapping.
460  * @mc:		Cache of pre-allocated and zeroed memory from which to allocate
461  *		page-table pages.
462  * @flags:	Flags to control the page-table walk (ex. a shared walk)
463  *
464  * The offset of @addr within a page is ignored, @size is rounded-up to
465  * the next page boundary and @phys is rounded-down to the previous page
466  * boundary.
467  *
468  * If device attributes are not explicitly requested in @prot, then the
469  * mapping will be normal, cacheable.
470  *
471  * Note that the update of a valid leaf PTE in this function will be aborted,
472  * if it's trying to recreate the exact same mapping or only change the access
473  * permissions. Instead, the vCPU will exit one more time from guest if still
474  * needed and then go through the path of relaxing permissions.
475  *
476  * Note that this function will both coalesce existing table entries and split
477  * existing block mappings, relying on page-faults to fault back areas outside
478  * of the new mapping lazily.
479  *
480  * Return: 0 on success, negative error code on failure.
481  */
482 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
483 			   u64 phys, enum kvm_pgtable_prot prot,
484 			   void *mc, enum kvm_pgtable_walk_flags flags);
485 
486 /**
487  * kvm_pgtable_stage2_set_owner() - Unmap and annotate pages in the IPA space to
488  *				    track ownership.
489  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
490  * @addr:	Base intermediate physical address to annotate.
491  * @size:	Size of the annotated range.
492  * @mc:		Cache of pre-allocated and zeroed memory from which to allocate
493  *		page-table pages.
494  * @owner_id:	Unique identifier for the owner of the page.
495  *
496  * By default, all page-tables are owned by identifier 0. This function can be
497  * used to mark portions of the IPA space as owned by other entities. When a
498  * stage 2 is used with identity-mappings, these annotations allow to use the
499  * page-table data structure as a simple rmap.
500  *
501  * Return: 0 on success, negative error code on failure.
502  */
503 int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
504 				 void *mc, u8 owner_id);
505 
506 /**
507  * kvm_pgtable_stage2_unmap() - Remove a mapping from a guest stage-2 page-table.
508  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
509  * @addr:	Intermediate physical address from which to remove the mapping.
510  * @size:	Size of the mapping.
511  *
512  * The offset of @addr within a page is ignored and @size is rounded-up to
513  * the next page boundary.
514  *
515  * TLB invalidation is performed for each page-table entry cleared during the
516  * unmapping operation and the reference count for the page-table page
517  * containing the cleared entry is decremented, with unreferenced pages being
518  * freed. Unmapping a cacheable page will ensure that it is clean to the PoC if
519  * FWB is not supported by the CPU.
520  *
521  * Return: 0 on success, negative error code on failure.
522  */
523 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
524 
525 /**
526  * kvm_pgtable_stage2_wrprotect() - Write-protect guest stage-2 address range
527  *                                  without TLB invalidation.
528  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
529  * @addr:	Intermediate physical address from which to write-protect,
530  * @size:	Size of the range.
531  *
532  * The offset of @addr within a page is ignored and @size is rounded-up to
533  * the next page boundary.
534  *
535  * Note that it is the caller's responsibility to invalidate the TLB after
536  * calling this function to ensure that the updated permissions are visible
537  * to the CPUs.
538  *
539  * Return: 0 on success, negative error code on failure.
540  */
541 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size);
542 
543 /**
544  * kvm_pgtable_stage2_mkyoung() - Set the access flag in a page-table entry.
545  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
546  * @addr:	Intermediate physical address to identify the page-table entry.
547  *
548  * The offset of @addr within a page is ignored.
549  *
550  * If there is a valid, leaf page-table entry used to translate @addr, then
551  * set the access flag in that entry.
552  *
553  * Return: The old page-table entry prior to setting the flag, 0 on failure.
554  */
555 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr);
556 
557 /**
558  * kvm_pgtable_stage2_mkold() - Clear the access flag in a page-table entry.
559  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
560  * @addr:	Intermediate physical address to identify the page-table entry.
561  *
562  * The offset of @addr within a page is ignored.
563  *
564  * If there is a valid, leaf page-table entry used to translate @addr, then
565  * clear the access flag in that entry.
566  *
567  * Note that it is the caller's responsibility to invalidate the TLB after
568  * calling this function to ensure that the updated permissions are visible
569  * to the CPUs.
570  *
571  * Return: The old page-table entry prior to clearing the flag, 0 on failure.
572  */
573 kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr);
574 
575 /**
576  * kvm_pgtable_stage2_relax_perms() - Relax the permissions enforced by a
577  *				      page-table entry.
578  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
579  * @addr:	Intermediate physical address to identify the page-table entry.
580  * @prot:	Additional permissions to grant for the mapping.
581  *
582  * The offset of @addr within a page is ignored.
583  *
584  * If there is a valid, leaf page-table entry used to translate @addr, then
585  * relax the permissions in that entry according to the read, write and
586  * execute permissions specified by @prot. No permissions are removed, and
587  * TLB invalidation is performed after updating the entry. Software bits cannot
588  * be set or cleared using kvm_pgtable_stage2_relax_perms().
589  *
590  * Return: 0 on success, negative error code on failure.
591  */
592 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,
593 				   enum kvm_pgtable_prot prot);
594 
595 /**
596  * kvm_pgtable_stage2_is_young() - Test whether a page-table entry has the
597  *				   access flag set.
598  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
599  * @addr:	Intermediate physical address to identify the page-table entry.
600  *
601  * The offset of @addr within a page is ignored.
602  *
603  * Return: True if the page-table entry has the access flag set, false otherwise.
604  */
605 bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr);
606 
607 /**
608  * kvm_pgtable_stage2_flush_range() - Clean and invalidate data cache to Point
609  * 				      of Coherency for guest stage-2 address
610  *				      range.
611  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
612  * @addr:	Intermediate physical address from which to flush.
613  * @size:	Size of the range.
614  *
615  * The offset of @addr within a page is ignored and @size is rounded-up to
616  * the next page boundary.
617  *
618  * Return: 0 on success, negative error code on failure.
619  */
620 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);
621 
622 /**
623  * kvm_pgtable_walk() - Walk a page-table.
624  * @pgt:	Page-table structure initialised by kvm_pgtable_*_init().
625  * @addr:	Input address for the start of the walk.
626  * @size:	Size of the range to walk.
627  * @walker:	Walker callback description.
628  *
629  * The offset of @addr within a page is ignored and @size is rounded-up to
630  * the next page boundary.
631  *
632  * The walker will walk the page-table entries corresponding to the input
633  * address range specified, visiting entries according to the walker flags.
634  * Invalid entries are treated as leaf entries. Leaf entries are reloaded
635  * after invoking the walker callback, allowing the walker to descend into
636  * a newly installed table.
637  *
638  * Returning a negative error code from the walker callback function will
639  * terminate the walk immediately with the same error code.
640  *
641  * Return: 0 on success, negative error code on failure.
642  */
643 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
644 		     struct kvm_pgtable_walker *walker);
645 
646 /**
647  * kvm_pgtable_get_leaf() - Walk a page-table and retrieve the leaf entry
648  *			    with its level.
649  * @pgt:	Page-table structure initialised by kvm_pgtable_*_init()
650  *		or a similar initialiser.
651  * @addr:	Input address for the start of the walk.
652  * @ptep:	Pointer to storage for the retrieved PTE.
653  * @level:	Pointer to storage for the level of the retrieved PTE.
654  *
655  * The offset of @addr within a page is ignored.
656  *
657  * The walker will walk the page-table entries corresponding to the input
658  * address specified, retrieving the leaf corresponding to this address.
659  * Invalid entries are treated as leaf entries.
660  *
661  * Return: 0 on success, negative error code on failure.
662  */
663 int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
664 			 kvm_pte_t *ptep, u32 *level);
665 
666 /**
667  * kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a
668  *				   stage-2 Page-Table Entry.
669  * @pte:	Page-table entry
670  *
671  * Return: protection attributes of the page-table entry in the enum
672  *	   kvm_pgtable_prot format.
673  */
674 enum kvm_pgtable_prot kvm_pgtable_stage2_pte_prot(kvm_pte_t pte);
675 
676 /**
677  * kvm_pgtable_hyp_pte_prot() - Retrieve the protection attributes of a stage-1
678  *				Page-Table Entry.
679  * @pte:	Page-table entry
680  *
681  * Return: protection attributes of the page-table entry in the enum
682  *	   kvm_pgtable_prot format.
683  */
684 enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte);
685 #endif	/* __ARM64_KVM_PGTABLE_H__ */
686