1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /******************************************************************************
3 * arch/x86/mm/shadow/types.h
4 *
5 * Parts of this code are Copyright (c) 2006 by XenSource Inc.
6 * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
7 * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
8 */
9
10 #ifndef _XEN_SHADOW_TYPES_H
11 #define _XEN_SHADOW_TYPES_H
12
13 /* The number of levels in the shadow pagetable is entirely determined
14 * by the number of levels in the guest pagetable */
15 #if GUEST_PAGING_LEVELS != 2
16 #define SHADOW_PAGING_LEVELS GUEST_PAGING_LEVELS
17 #else
18 #define SHADOW_PAGING_LEVELS 3
19 #endif
20
21 /*
22 * Define various types for handling pagetabels, based on these options:
23 * SHADOW_PAGING_LEVELS : Number of levels of shadow pagetables
24 * GUEST_PAGING_LEVELS : Number of levels of guest pagetables
25 */
26
27 #if SHADOW_PAGING_LEVELS == 3
28 #define SHADOW_L1_PAGETABLE_ENTRIES 512
29 #define SHADOW_L2_PAGETABLE_ENTRIES 512
30 #define SHADOW_L3_PAGETABLE_ENTRIES 4
31 #define SHADOW_L1_PAGETABLE_SHIFT 12
32 #define SHADOW_L2_PAGETABLE_SHIFT 21
33 #define SHADOW_L3_PAGETABLE_SHIFT 30
34 #elif SHADOW_PAGING_LEVELS == 4
35 #define SHADOW_L1_PAGETABLE_ENTRIES 512
36 #define SHADOW_L2_PAGETABLE_ENTRIES 512
37 #define SHADOW_L3_PAGETABLE_ENTRIES 512
38 #define SHADOW_L4_PAGETABLE_ENTRIES 512
39 #define SHADOW_L1_PAGETABLE_SHIFT 12
40 #define SHADOW_L2_PAGETABLE_SHIFT 21
41 #define SHADOW_L3_PAGETABLE_SHIFT 30
42 #define SHADOW_L4_PAGETABLE_SHIFT 39
43 #endif
44
45 /* Types of the shadow page tables */
46 typedef l1_pgentry_t shadow_l1e_t;
47 typedef l2_pgentry_t shadow_l2e_t;
48 typedef l3_pgentry_t shadow_l3e_t;
49 typedef l4_pgentry_t shadow_l4e_t;
50
51 /* Access functions for them */
shadow_l1e_get_paddr(shadow_l1e_t sl1e)52 static inline paddr_t shadow_l1e_get_paddr(shadow_l1e_t sl1e)
53 { return l1e_get_paddr(sl1e); }
shadow_l2e_get_paddr(shadow_l2e_t sl2e)54 static inline paddr_t shadow_l2e_get_paddr(shadow_l2e_t sl2e)
55 { return l2e_get_paddr(sl2e); }
shadow_l3e_get_paddr(shadow_l3e_t sl3e)56 static inline paddr_t shadow_l3e_get_paddr(shadow_l3e_t sl3e)
57 { return l3e_get_paddr(sl3e); }
shadow_l4e_get_paddr(shadow_l4e_t sl4e)58 static inline paddr_t shadow_l4e_get_paddr(shadow_l4e_t sl4e)
59 { return l4e_get_paddr(sl4e); }
60
shadow_l1e_get_mfn(shadow_l1e_t sl1e)61 static inline mfn_t shadow_l1e_get_mfn(shadow_l1e_t sl1e)
62 { return l1e_get_mfn(sl1e); }
shadow_l2e_get_mfn(shadow_l2e_t sl2e)63 static inline mfn_t shadow_l2e_get_mfn(shadow_l2e_t sl2e)
64 { return l2e_get_mfn(sl2e); }
shadow_l3e_get_mfn(shadow_l3e_t sl3e)65 static inline mfn_t shadow_l3e_get_mfn(shadow_l3e_t sl3e)
66 { return l3e_get_mfn(sl3e); }
shadow_l4e_get_mfn(shadow_l4e_t sl4e)67 static inline mfn_t shadow_l4e_get_mfn(shadow_l4e_t sl4e)
68 { return l4e_get_mfn(sl4e); }
69
shadow_l1e_get_flags(shadow_l1e_t sl1e)70 static inline u32 shadow_l1e_get_flags(shadow_l1e_t sl1e)
71 { return l1e_get_flags(sl1e); }
shadow_l2e_get_flags(shadow_l2e_t sl2e)72 static inline u32 shadow_l2e_get_flags(shadow_l2e_t sl2e)
73 { return l2e_get_flags(sl2e); }
shadow_l3e_get_flags(shadow_l3e_t sl3e)74 static inline u32 shadow_l3e_get_flags(shadow_l3e_t sl3e)
75 { return l3e_get_flags(sl3e); }
shadow_l4e_get_flags(shadow_l4e_t sl4e)76 static inline u32 shadow_l4e_get_flags(shadow_l4e_t sl4e)
77 { return l4e_get_flags(sl4e); }
78
79 static inline shadow_l1e_t
shadow_l1e_remove_flags(shadow_l1e_t sl1e,u32 flags)80 shadow_l1e_remove_flags(shadow_l1e_t sl1e, u32 flags)
81 { l1e_remove_flags(sl1e, flags); return sl1e; }
82 static inline shadow_l1e_t
shadow_l1e_flip_flags(shadow_l1e_t sl1e,u32 flags)83 shadow_l1e_flip_flags(shadow_l1e_t sl1e, u32 flags)
84 { l1e_flip_flags(sl1e, flags); return sl1e; }
85
shadow_l1e_empty(void)86 static inline shadow_l1e_t shadow_l1e_empty(void)
87 { return l1e_empty(); }
shadow_l2e_empty(void)88 static inline shadow_l2e_t shadow_l2e_empty(void)
89 { return l2e_empty(); }
shadow_l3e_empty(void)90 static inline shadow_l3e_t shadow_l3e_empty(void)
91 { return l3e_empty(); }
shadow_l4e_empty(void)92 static inline shadow_l4e_t shadow_l4e_empty(void)
93 { return l4e_empty(); }
94
shadow_l1e_from_mfn(mfn_t mfn,u32 flags)95 static inline shadow_l1e_t shadow_l1e_from_mfn(mfn_t mfn, u32 flags)
96 { return l1e_from_mfn(mfn, flags); }
shadow_l2e_from_mfn(mfn_t mfn,u32 flags)97 static inline shadow_l2e_t shadow_l2e_from_mfn(mfn_t mfn, u32 flags)
98 { return l2e_from_mfn(mfn, flags); }
shadow_l3e_from_mfn(mfn_t mfn,u32 flags)99 static inline shadow_l3e_t shadow_l3e_from_mfn(mfn_t mfn, u32 flags)
100 { return l3e_from_mfn(mfn, flags); }
shadow_l4e_from_mfn(mfn_t mfn,u32 flags)101 static inline shadow_l4e_t shadow_l4e_from_mfn(mfn_t mfn, u32 flags)
102 { return l4e_from_mfn(mfn, flags); }
103
104 #define shadow_l1_table_offset(a) l1_table_offset(a)
105 #define shadow_l2_table_offset(a) l2_table_offset(a)
106 #define shadow_l3_table_offset(a) l3_table_offset(a)
107 #define shadow_l4_table_offset(a) l4_table_offset(a)
108
109 /**************************************************************************/
110 /* Access to the linear mapping of shadow page tables. */
111
112 /* Offsets into each level of the linear mapping for a virtual address. */
113 #define shadow_l1_linear_offset(_a) \
114 (((_a) & VADDR_MASK) >> SHADOW_L1_PAGETABLE_SHIFT)
115 #define shadow_l2_linear_offset(_a) \
116 (((_a) & VADDR_MASK) >> SHADOW_L2_PAGETABLE_SHIFT)
117 #define shadow_l3_linear_offset(_a) \
118 (((_a) & VADDR_MASK) >> SHADOW_L3_PAGETABLE_SHIFT)
119 #define shadow_l4_linear_offset(_a) \
120 (((_a) & VADDR_MASK) >> SHADOW_L4_PAGETABLE_SHIFT)
121
122 /* Where to find each level of the linear mapping. For PV guests, we use
123 * the shadow linear-map self-entry as many times as we need. For HVM
124 * guests, the shadow doesn't have a linear-map self-entry so we must use
125 * the monitor-table's linear-map entry N-1 times and then the shadow-map
126 * entry once. */
127 #define __sh_linear_l1_table ((shadow_l1e_t *)(SH_LINEAR_PT_VIRT_START))
128 #define __sh_linear_l2_table ((shadow_l2e_t *) \
129 (__sh_linear_l1_table + shadow_l1_linear_offset(SH_LINEAR_PT_VIRT_START)))
130
131 // shadow linear L3 and L4 tables only exist in 4 level paging...
132 #if SHADOW_PAGING_LEVELS == 4
133 #define __sh_linear_l3_table ((shadow_l3e_t *) \
134 (__sh_linear_l2_table + shadow_l2_linear_offset(SH_LINEAR_PT_VIRT_START)))
135 #define __sh_linear_l4_table ((shadow_l4e_t *) \
136 (__sh_linear_l3_table + shadow_l3_linear_offset(SH_LINEAR_PT_VIRT_START)))
137 #endif
138
139 #define sh_linear_l1_table(v) ({ \
140 ASSERT(current == (v)); \
141 __sh_linear_l1_table; \
142 })
143
144 // XXX -- these should not be conditional on is_hvm_vcpu(v), but rather on
145 // shadow_mode_external(d)...
146 //
147 #define sh_linear_l2_table(v) ({ \
148 ASSERT(current == (v)); \
149 ((shadow_l2e_t *) \
150 (is_hvm_vcpu(v) ? __linear_l1_table : __sh_linear_l1_table) + \
151 shadow_l1_linear_offset(SH_LINEAR_PT_VIRT_START)); \
152 })
153
154 #if SHADOW_PAGING_LEVELS >= 4
155 #define sh_linear_l3_table(v) ({ \
156 ASSERT(current == (v)); \
157 ((shadow_l3e_t *) \
158 (is_hvm_vcpu(v) ? __linear_l2_table : __sh_linear_l2_table) + \
159 shadow_l2_linear_offset(SH_LINEAR_PT_VIRT_START)); \
160 })
161
162 // we use l4_pgentry_t instead of shadow_l4e_t below because shadow_l4e_t is
163 // not defined for when xen_levels==4 & shadow_levels==3...
164 #define sh_linear_l4_table(v) ({ \
165 ASSERT(current == (v)); \
166 ((l4_pgentry_t *) \
167 (is_hvm_vcpu(v) ? __linear_l3_table : __sh_linear_l3_table) + \
168 shadow_l3_linear_offset(SH_LINEAR_PT_VIRT_START)); \
169 })
170 #endif
171
172 /* Override get_gfn to work with gfn_t */
173 #undef get_gfn_query
174 #define get_gfn_query(d, g, t) get_gfn_type((d), gfn_x(g), (t), 0)
175
176 /* The shadow types needed for the various levels. */
177
178 #if GUEST_PAGING_LEVELS == 2
179 #define SH_type_l1_shadow SH_type_l1_32_shadow
180 #define SH_type_l2_shadow SH_type_l2_32_shadow
181 #define SH_type_fl1_shadow SH_type_fl1_32_shadow
182 #elif GUEST_PAGING_LEVELS == 3
183 #define SH_type_l1_shadow SH_type_l1_pae_shadow
184 #define SH_type_fl1_shadow SH_type_fl1_pae_shadow
185 #define SH_type_l2_shadow SH_type_l2_pae_shadow
186 #elif GUEST_PAGING_LEVELS == 4
187 #define SH_type_l1_shadow SH_type_l1_64_shadow
188 #define SH_type_fl1_shadow SH_type_fl1_64_shadow
189 #define SH_type_l2_shadow SH_type_l2_64_shadow
190 #define SH_type_l2h_shadow SH_type_l2h_64_shadow
191 #define SH_type_l3_shadow SH_type_l3_64_shadow
192 #define SH_type_l4_shadow SH_type_l4_64_shadow
193 #endif
194
195 #if GUEST_PAGING_LEVELS
196
197 /* macros for dealing with the naming of the internal function names of the
198 * shadow code's external entry points.
199 */
200 #define INTERNAL_NAME(name) SHADOW_INTERNAL_NAME(name, GUEST_PAGING_LEVELS)
201
202 /* macros for renaming the primary entry points, so that they are more
203 * easily distinguished from a debugger
204 */
205 #define sh_page_fault INTERNAL_NAME(sh_page_fault)
206 #define sh_invlpg INTERNAL_NAME(sh_invlpg)
207 #define sh_gva_to_gfn INTERNAL_NAME(sh_gva_to_gfn)
208 #define sh_update_cr3 INTERNAL_NAME(sh_update_cr3)
209 #define sh_rm_write_access_from_l1 INTERNAL_NAME(sh_rm_write_access_from_l1)
210 #define sh_rm_mappings_from_l1 INTERNAL_NAME(sh_rm_mappings_from_l1)
211 #define sh_remove_l1_shadow INTERNAL_NAME(sh_remove_l1_shadow)
212 #define sh_remove_l2_shadow INTERNAL_NAME(sh_remove_l2_shadow)
213 #define sh_remove_l3_shadow INTERNAL_NAME(sh_remove_l3_shadow)
214 #define sh_map_and_validate_gl4e INTERNAL_NAME(sh_map_and_validate_gl4e)
215 #define sh_map_and_validate_gl3e INTERNAL_NAME(sh_map_and_validate_gl3e)
216 #define sh_map_and_validate_gl2e INTERNAL_NAME(sh_map_and_validate_gl2e)
217 #define sh_map_and_validate_gl2he INTERNAL_NAME(sh_map_and_validate_gl2he)
218 #define sh_map_and_validate_gl1e INTERNAL_NAME(sh_map_and_validate_gl1e)
219 #define sh_destroy_l4_shadow INTERNAL_NAME(sh_destroy_l4_shadow)
220 #define sh_destroy_l3_shadow INTERNAL_NAME(sh_destroy_l3_shadow)
221 #define sh_destroy_l2_shadow INTERNAL_NAME(sh_destroy_l2_shadow)
222 #define sh_destroy_l1_shadow INTERNAL_NAME(sh_destroy_l1_shadow)
223 #define sh_unhook_l2_mappings INTERNAL_NAME(sh_unhook_l2_mappings)
224 #define sh_unhook_l4_mappings INTERNAL_NAME(sh_unhook_l4_mappings)
225 #define sh_paging_mode INTERNAL_NAME(sh_paging_mode)
226 #define sh_audit_l1_table INTERNAL_NAME(sh_audit_l1_table)
227 #define sh_audit_fl1_table INTERNAL_NAME(sh_audit_fl1_table)
228 #define sh_audit_l2_table INTERNAL_NAME(sh_audit_l2_table)
229 #define sh_audit_l3_table INTERNAL_NAME(sh_audit_l3_table)
230 #define sh_audit_l4_table INTERNAL_NAME(sh_audit_l4_table)
231 #define sh_guess_wrmap INTERNAL_NAME(sh_guess_wrmap)
232 #define sh_clear_shadow_entry INTERNAL_NAME(sh_clear_shadow_entry)
233
234 #if SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC
235 #define sh_resync_l1 INTERNAL_NAME(sh_resync_l1)
236 #define sh_safe_not_to_sync INTERNAL_NAME(sh_safe_not_to_sync)
237 #define sh_rm_write_access_from_sl1p INTERNAL_NAME(sh_rm_write_access_from_sl1p)
238 #endif
239
240 #endif /* GUEST_PAGING_LEVELS */
241
242 #if SHADOW_PAGING_LEVELS == 3
243 #define MFN_FITS_IN_HVM_CR3(_MFN) !(mfn_x(_MFN) >> 20)
244 #endif
245
246 #define SH_PRI_pte PRIpte
247 #define SH_PRI_gpte PRI_gpte
248 #define SH_PRI_gfn PRI_gfn
249
250 int shadow_set_l1e(struct domain *d, shadow_l1e_t *sl1e,
251 shadow_l1e_t new_sl1e, p2m_type_t new_type,
252 mfn_t sl1mfn);
253 int shadow_set_l2e(struct domain *d, shadow_l2e_t *sl2e,
254 shadow_l2e_t new_sl2e, mfn_t sl2mfn,
255 unsigned int type_fl1_shadow,
256 mfn_t (*next_page)(mfn_t smfn));
257 int shadow_set_l3e(struct domain *d, shadow_l3e_t *sl3e,
258 shadow_l3e_t new_sl3e, mfn_t sl3mfn);
259 int shadow_set_l4e(struct domain *d, shadow_l4e_t *sl4e,
260 shadow_l4e_t new_sl4e, mfn_t sl4mfn);
261
262 static void inline
shadow_put_page_from_l1e(shadow_l1e_t sl1e,struct domain * d)263 shadow_put_page_from_l1e(shadow_l1e_t sl1e, struct domain *d)
264 {
265 if ( !shadow_mode_refcounts(d) )
266 return;
267
268 put_page_from_l1e(sl1e, d);
269 }
270
271 #if (SHADOW_OPTIMIZATIONS & SHOPT_FAST_FAULT_PATH)
272 /******************************************************************************
273 * We implement a "fast path" for two special cases: faults that require
274 * MMIO emulation, and faults where the guest PTE is not present. We
275 * record these as shadow l1 entries that have reserved bits set in
276 * them, so we can spot them immediately in the fault handler and handle
277 * them without needing to hold the paging lock or walk the guest
278 * pagetables.
279 *
280 * This is only feasible for PAE and 64bit Xen: 32-bit non-PAE PTEs don't
281 * have reserved bits that we can use for this. And even there it can only
282 * be used if we can be certain the processor doesn't use all 52 address bits.
283 *
284 * For the MMIO encoding (see below) we need the bottom 4 bits for
285 * identifying the kind of entry and a full GFN's worth of bits to encode
286 * the originating frame number. Set all remaining bits to trigger
287 * reserved bit faults, if (see above) the hardware permits triggering such.
288 */
289
290 #ifdef CONFIG_BIGMEM
291 # define SH_L1E_MAGIC_MASK (0xfffff00000000000UL | _PAGE_PRESENT)
292 #else
293 # define SH_L1E_MAGIC_MASK (0xfffffff000000000UL | _PAGE_PRESENT)
294 #endif
295
sh_have_pte_rsvd_bits(void)296 static inline bool sh_have_pte_rsvd_bits(void)
297 {
298 return paddr_bits < PADDR_BITS && !cpu_has_hypervisor;
299 }
300
sh_l1e_is_magic(shadow_l1e_t sl1e)301 static inline bool sh_l1e_is_magic(shadow_l1e_t sl1e)
302 {
303 BUILD_BUG_ON(!(PADDR_MASK & PAGE_MASK & SH_L1E_MAGIC_MASK));
304 return (sl1e.l1 & SH_L1E_MAGIC_MASK) == SH_L1E_MAGIC_MASK;
305 }
306
307 /* Guest not present: a single magic value */
sh_l1e_gnp_raw(void)308 static inline shadow_l1e_t sh_l1e_gnp_raw(void)
309 {
310 return (shadow_l1e_t){ -1ULL };
311 }
312
sh_l1e_gnp(void)313 static inline shadow_l1e_t sh_l1e_gnp(void)
314 {
315 /*
316 * On systems with no reserved physical address bits we can't engage the
317 * fast fault path.
318 */
319 return sh_have_pte_rsvd_bits() ? sh_l1e_gnp_raw()
320 : shadow_l1e_empty();
321 }
322
sh_l1e_is_gnp(shadow_l1e_t sl1e)323 static inline bool sh_l1e_is_gnp(shadow_l1e_t sl1e)
324 {
325 return sl1e.l1 == sh_l1e_gnp_raw().l1;
326 }
327
328 /*
329 * MMIO: an invalid PTE that contains the GFN of the equivalent guest l1e.
330 * We store the GFN in bits 4:35 (BIGMEM: 4:43) of the entry.
331 * The present bit is set, and the U/S and R/W bits are taken from the guest.
332 * Bit 3 is always 0, to differentiate from gnp above.
333 */
334 #define SH_L1E_MMIO_MAGIC SH_L1E_MAGIC_MASK
335 #define SH_L1E_MMIO_MAGIC_BIT 8
336 #define SH_L1E_MMIO_MAGIC_MASK (SH_L1E_MMIO_MAGIC | SH_L1E_MMIO_MAGIC_BIT)
337 #define SH_L1E_MMIO_GFN_MASK ~(SH_L1E_MMIO_MAGIC_MASK | _PAGE_RW | _PAGE_USER)
338
sh_l1e_mmio(gfn_t gfn,u32 gflags)339 static inline shadow_l1e_t sh_l1e_mmio(gfn_t gfn, u32 gflags)
340 {
341 unsigned long gfn_val = MASK_INSR(gfn_x(gfn), SH_L1E_MMIO_GFN_MASK);
342 shadow_l1e_t sl1e = { (SH_L1E_MMIO_MAGIC | gfn_val |
343 (gflags & (_PAGE_USER | _PAGE_RW))) };
344
345 BUILD_BUG_ON(SH_L1E_MMIO_MAGIC_BIT <= _PAGE_RW);
346 BUILD_BUG_ON(SH_L1E_MMIO_MAGIC_BIT <= _PAGE_USER);
347
348 if ( !sh_have_pte_rsvd_bits() ||
349 (cpu_has_bug_l1tf &&
350 !is_l1tf_safe_maddr(shadow_l1e_get_paddr(sl1e))) )
351 sl1e = shadow_l1e_empty();
352
353 return sl1e;
354 }
355
sh_l1e_is_mmio(shadow_l1e_t sl1e)356 static inline bool sh_l1e_is_mmio(shadow_l1e_t sl1e)
357 {
358 return (sl1e.l1 & SH_L1E_MMIO_MAGIC_MASK) == SH_L1E_MMIO_MAGIC;
359 }
360
sh_l1e_mmio_get_gfn(shadow_l1e_t sl1e)361 static inline gfn_t sh_l1e_mmio_get_gfn(shadow_l1e_t sl1e)
362 {
363 return _gfn(MASK_EXTR(sl1e.l1, SH_L1E_MMIO_GFN_MASK));
364 }
365
sh_l1e_mmio_get_flags(shadow_l1e_t sl1e)366 static inline uint32_t sh_l1e_mmio_get_flags(shadow_l1e_t sl1e)
367 {
368 return sl1e.l1 & (_PAGE_USER | _PAGE_RW);
369 }
370
371 #else
372
373 #define sh_l1e_gnp() shadow_l1e_empty()
374 #define sh_l1e_mmio(_gfn, _flags) shadow_l1e_empty()
375 #define sh_l1e_is_magic(_e) (0)
376
377 #endif /* SHOPT_FAST_FAULT_PATH */
378
379
380 #endif /* _XEN_SHADOW_TYPES_H */
381
382 /*
383 * Local variables:
384 * mode: C
385 * c-file-style: "BSD"
386 * c-basic-offset: 4
387 * indent-tabs-mode: nil
388 * End:
389 */
390