1 /*
2 * ept-p2m.c: use the EPT page table as p2m
3 * Copyright (c) 2007, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 #include <xen/domain_page.h>
19 #include <xen/sched.h>
20 #include <asm/current.h>
21 #include <asm/paging.h>
22 #include <asm/types.h>
23 #include <asm/domain.h>
24 #include <asm/p2m.h>
25 #include <asm/hvm/vmx/vmx.h>
26 #include <asm/hvm/vmx/vmcs.h>
27 #include <asm/hvm/nestedhvm.h>
28 #include <xen/iommu.h>
29 #include <asm/mtrr.h>
30 #include <asm/hvm/cacheattr.h>
31 #include <xen/keyhandler.h>
32 #include <xen/softirq.h>
33
34 #include "mm-locks.h"
35
36 #define atomic_read_ept_entry(__pepte) \
37 ( (ept_entry_t) { .epte = read_atomic(&(__pepte)->epte) } )
38
39 #define is_epte_present(ept_entry) ((ept_entry)->epte & 0x7)
40 #define is_epte_superpage(ept_entry) ((ept_entry)->sp)
is_epte_valid(ept_entry_t * e)41 static inline bool_t is_epte_valid(ept_entry_t *e)
42 {
43 /* suppress_ve alone is not considered valid, so mask it off */
44 return ((e->epte & ~(1ul << 63)) != 0 && e->sa_p2mt != p2m_invalid);
45 }
46
47 /* returns : 0 for success, -errno otherwise */
atomic_write_ept_entry(ept_entry_t * entryptr,ept_entry_t new,int level)48 static int atomic_write_ept_entry(ept_entry_t *entryptr, ept_entry_t new,
49 int level)
50 {
51 int rc;
52 unsigned long oldmfn = mfn_x(INVALID_MFN);
53 bool_t check_foreign = (new.mfn != entryptr->mfn ||
54 new.sa_p2mt != entryptr->sa_p2mt);
55
56 if ( level )
57 {
58 ASSERT(!is_epte_superpage(&new) || !p2m_is_foreign(new.sa_p2mt));
59 write_atomic(&entryptr->epte, new.epte);
60 return 0;
61 }
62
63 if ( unlikely(p2m_is_foreign(new.sa_p2mt)) )
64 {
65 rc = -EINVAL;
66 if ( !is_epte_present(&new) )
67 goto out;
68
69 if ( check_foreign )
70 {
71 struct domain *fdom;
72
73 if ( !mfn_valid(_mfn(new.mfn)) )
74 goto out;
75
76 rc = -ESRCH;
77 fdom = page_get_owner(mfn_to_page(new.mfn));
78 if ( fdom == NULL )
79 goto out;
80
81 /* get refcount on the page */
82 rc = -EBUSY;
83 if ( !get_page(mfn_to_page(new.mfn), fdom) )
84 goto out;
85 }
86 }
87
88 if ( unlikely(p2m_is_foreign(entryptr->sa_p2mt)) && check_foreign )
89 oldmfn = entryptr->mfn;
90
91 write_atomic(&entryptr->epte, new.epte);
92
93 if ( unlikely(oldmfn != mfn_x(INVALID_MFN)) )
94 put_page(mfn_to_page(oldmfn));
95
96 rc = 0;
97
98 out:
99 if ( rc )
100 gdprintk(XENLOG_ERR, "epte o:%"PRIx64" n:%"PRIx64" rc:%d\n",
101 entryptr->epte, new.epte, rc);
102 return rc;
103 }
104
ept_p2m_type_to_flags(struct p2m_domain * p2m,ept_entry_t * entry,p2m_type_t type,p2m_access_t access)105 static void ept_p2m_type_to_flags(struct p2m_domain *p2m, ept_entry_t *entry,
106 p2m_type_t type, p2m_access_t access)
107 {
108 /*
109 * First apply type permissions.
110 *
111 * A/D bits are also manually set to avoid overhead of MMU having to set
112 * them later. Both A/D bits are safe to be updated directly as they are
113 * ignored by processor if EPT A/D bits is not turned on.
114 *
115 * A bit is set for all present p2m types in middle and leaf EPT entries.
116 * D bit is set for all writable types in EPT leaf entry, except for
117 * log-dirty type with PML.
118 */
119 switch(type)
120 {
121 case p2m_invalid:
122 case p2m_mmio_dm:
123 case p2m_populate_on_demand:
124 case p2m_ram_paging_out:
125 case p2m_ram_paged:
126 case p2m_ram_paging_in:
127 default:
128 entry->r = entry->w = entry->x = 0;
129 break;
130 case p2m_ram_rw:
131 entry->r = entry->w = entry->x = 1;
132 entry->a = entry->d = !!cpu_has_vmx_ept_ad;
133 break;
134 case p2m_ioreq_server:
135 entry->r = 1;
136 entry->w = !(p2m->ioreq.flags & XEN_DMOP_IOREQ_MEM_ACCESS_WRITE);
137 entry->x = 0;
138 entry->a = !!cpu_has_vmx_ept_ad;
139 entry->d = entry->w && entry->a;
140 break;
141 case p2m_mmio_direct:
142 entry->r = entry->x = 1;
143 entry->w = !rangeset_contains_singleton(mmio_ro_ranges,
144 entry->mfn);
145 ASSERT(entry->w || !is_epte_superpage(entry));
146 entry->a = !!cpu_has_vmx_ept_ad;
147 entry->d = entry->w && cpu_has_vmx_ept_ad;
148 break;
149 case p2m_ram_logdirty:
150 entry->r = entry->x = 1;
151 /*
152 * In case of PML, we don't have to write protect 4K page, but
153 * only need to clear D-bit for it, but we still need to write
154 * protect super page in order to split it to 4K pages in EPT
155 * violation.
156 */
157 if ( vmx_domain_pml_enabled(p2m->domain) &&
158 !is_epte_superpage(entry) )
159 entry->w = 1;
160 else
161 entry->w = 0;
162 entry->a = !!cpu_has_vmx_ept_ad;
163 /* For both PML or non-PML cases we clear D bit anyway */
164 entry->d = 0;
165 break;
166 case p2m_ram_ro:
167 case p2m_ram_shared:
168 entry->r = entry->x = 1;
169 entry->w = 0;
170 entry->a = !!cpu_has_vmx_ept_ad;
171 entry->d = 0;
172 break;
173 case p2m_grant_map_rw:
174 case p2m_map_foreign:
175 entry->r = entry->w = 1;
176 entry->x = 0;
177 entry->a = entry->d = !!cpu_has_vmx_ept_ad;
178 break;
179 case p2m_grant_map_ro:
180 entry->r = 1;
181 entry->w = entry->x = 0;
182 entry->a = !!cpu_has_vmx_ept_ad;
183 entry->d = 0;
184 break;
185 }
186
187
188 /* Then restrict with access permissions */
189 switch (access)
190 {
191 case p2m_access_n:
192 case p2m_access_n2rwx:
193 entry->r = entry->w = entry->x = 0;
194 break;
195 case p2m_access_r:
196 entry->w = entry->x = 0;
197 break;
198 case p2m_access_w:
199 entry->r = entry->x = 0;
200 break;
201 case p2m_access_x:
202 entry->r = entry->w = 0;
203 break;
204 case p2m_access_rx:
205 case p2m_access_rx2rw:
206 entry->w = 0;
207 break;
208 case p2m_access_wx:
209 entry->r = 0;
210 break;
211 case p2m_access_rw:
212 entry->x = 0;
213 break;
214 case p2m_access_rwx:
215 break;
216 }
217
218 }
219
220 #define GUEST_TABLE_MAP_FAILED 0
221 #define GUEST_TABLE_NORMAL_PAGE 1
222 #define GUEST_TABLE_SUPER_PAGE 2
223 #define GUEST_TABLE_POD_PAGE 3
224
225 /* Fill in middle levels of ept table */
ept_set_middle_entry(struct p2m_domain * p2m,ept_entry_t * ept_entry)226 static int ept_set_middle_entry(struct p2m_domain *p2m, ept_entry_t *ept_entry)
227 {
228 mfn_t mfn;
229 ept_entry_t *table;
230 unsigned int i;
231
232 mfn = p2m_alloc_ptp(p2m, 0);
233 if ( mfn_eq(mfn, INVALID_MFN) )
234 return 0;
235
236 ept_entry->epte = 0;
237 ept_entry->mfn = mfn_x(mfn);
238 ept_entry->access = p2m->default_access;
239
240 ept_entry->r = ept_entry->w = ept_entry->x = 1;
241 /* Manually set A bit to avoid overhead of MMU having to write it later. */
242 ept_entry->a = !!cpu_has_vmx_ept_ad;
243
244 ept_entry->suppress_ve = 1;
245
246 table = map_domain_page(mfn);
247
248 for ( i = 0; i < EPT_PAGETABLE_ENTRIES; i++ )
249 table[i].suppress_ve = 1;
250
251 unmap_domain_page(table);
252
253 return 1;
254 }
255
256 /* free ept sub tree behind an entry */
ept_free_entry(struct p2m_domain * p2m,ept_entry_t * ept_entry,int level)257 static void ept_free_entry(struct p2m_domain *p2m, ept_entry_t *ept_entry, int level)
258 {
259 /* End if the entry is a leaf entry. */
260 if ( level == 0 || !is_epte_present(ept_entry) ||
261 is_epte_superpage(ept_entry) )
262 return;
263
264 if ( level > 1 )
265 {
266 ept_entry_t *epte = map_domain_page(_mfn(ept_entry->mfn));
267 for ( int i = 0; i < EPT_PAGETABLE_ENTRIES; i++ )
268 ept_free_entry(p2m, epte + i, level - 1);
269 unmap_domain_page(epte);
270 }
271
272 p2m_tlb_flush_sync(p2m);
273 p2m_free_ptp(p2m, mfn_to_page(ept_entry->mfn));
274 }
275
ept_split_super_page(struct p2m_domain * p2m,ept_entry_t * ept_entry,unsigned int level,unsigned int target)276 static bool_t ept_split_super_page(struct p2m_domain *p2m,
277 ept_entry_t *ept_entry,
278 unsigned int level, unsigned int target)
279 {
280 ept_entry_t new_ept, *table;
281 uint64_t trunk;
282 unsigned int i;
283 bool_t rv = 1;
284
285 /* End if the entry is a leaf entry or reaches the target level. */
286 if ( level <= target )
287 return 1;
288
289 ASSERT(is_epte_superpage(ept_entry));
290
291 if ( !ept_set_middle_entry(p2m, &new_ept) )
292 return 0;
293
294 table = map_domain_page(_mfn(new_ept.mfn));
295 trunk = 1UL << ((level - 1) * EPT_TABLE_ORDER);
296
297 for ( i = 0; i < EPT_PAGETABLE_ENTRIES; i++ )
298 {
299 ept_entry_t *epte = table + i;
300
301 *epte = *ept_entry;
302 epte->sp = (level > 1);
303 epte->mfn += i * trunk;
304 epte->snp = (iommu_enabled && iommu_snoop);
305 epte->suppress_ve = 1;
306
307 ept_p2m_type_to_flags(p2m, epte, epte->sa_p2mt, epte->access);
308
309 if ( (level - 1) == target )
310 continue;
311
312 ASSERT(is_epte_superpage(epte));
313
314 if ( !(rv = ept_split_super_page(p2m, epte, level - 1, target)) )
315 break;
316 }
317
318 unmap_domain_page(table);
319
320 /* Even failed we should install the newly allocated ept page. */
321 *ept_entry = new_ept;
322
323 return rv;
324 }
325
326 /* Take the currently mapped table, find the corresponding gfn entry,
327 * and map the next table, if available. If the entry is empty
328 * and read_only is set,
329 * Return values:
330 * 0: Failed to map. Either read_only was set and the entry was
331 * empty, or allocating a new page failed.
332 * GUEST_TABLE_NORMAL_PAGE: next level mapped normally
333 * GUEST_TABLE_SUPER_PAGE:
334 * The next entry points to a superpage, and caller indicates
335 * that they are going to the superpage level, or are only doing
336 * a read.
337 * GUEST_TABLE_POD:
338 * The next entry is marked populate-on-demand.
339 */
ept_next_level(struct p2m_domain * p2m,bool_t read_only,ept_entry_t ** table,unsigned long * gfn_remainder,int next_level)340 static int ept_next_level(struct p2m_domain *p2m, bool_t read_only,
341 ept_entry_t **table, unsigned long *gfn_remainder,
342 int next_level)
343 {
344 unsigned long mfn;
345 ept_entry_t *ept_entry, e;
346 u32 shift, index;
347
348 shift = next_level * EPT_TABLE_ORDER;
349
350 index = *gfn_remainder >> shift;
351
352 /* index must be falling into the page */
353 ASSERT(index < EPT_PAGETABLE_ENTRIES);
354
355 ept_entry = (*table) + index;
356
357 /* ept_next_level() is called (sometimes) without a lock. Read
358 * the entry once, and act on the "cached" entry after that to
359 * avoid races. */
360 e = atomic_read_ept_entry(ept_entry);
361
362 if ( !is_epte_present(&e) )
363 {
364 if ( e.sa_p2mt == p2m_populate_on_demand )
365 return GUEST_TABLE_POD_PAGE;
366
367 if ( read_only )
368 return GUEST_TABLE_MAP_FAILED;
369
370 if ( !ept_set_middle_entry(p2m, ept_entry) )
371 return GUEST_TABLE_MAP_FAILED;
372 else
373 e = atomic_read_ept_entry(ept_entry); /* Refresh */
374 }
375
376 /* The only time sp would be set here is if we had hit a superpage */
377 if ( is_epte_superpage(&e) )
378 return GUEST_TABLE_SUPER_PAGE;
379
380 mfn = e.mfn;
381 unmap_domain_page(*table);
382 *table = map_domain_page(_mfn(mfn));
383 *gfn_remainder &= (1UL << shift) - 1;
384 return GUEST_TABLE_NORMAL_PAGE;
385 }
386
387 /*
388 * Invalidate (via setting the EMT field to an invalid value) all valid
389 * present entries in the given page table, optionally marking the entries
390 * also for their subtrees needing P2M type re-calculation.
391 */
ept_invalidate_emt(mfn_t mfn,bool_t recalc,int level)392 static bool_t ept_invalidate_emt(mfn_t mfn, bool_t recalc, int level)
393 {
394 int rc;
395 ept_entry_t *epte = map_domain_page(mfn);
396 unsigned int i;
397 bool_t changed = 0;
398
399 for ( i = 0; i < EPT_PAGETABLE_ENTRIES; i++ )
400 {
401 ept_entry_t e = atomic_read_ept_entry(&epte[i]);
402
403 if ( !is_epte_valid(&e) || !is_epte_present(&e) ||
404 (e.emt == MTRR_NUM_TYPES && (e.recalc || !recalc)) )
405 continue;
406
407 e.emt = MTRR_NUM_TYPES;
408 if ( recalc )
409 e.recalc = 1;
410 rc = atomic_write_ept_entry(&epte[i], e, level);
411 ASSERT(rc == 0);
412 changed = 1;
413 }
414
415 unmap_domain_page(epte);
416
417 return changed;
418 }
419
420 /*
421 * Just like ept_invalidate_emt() except that
422 * - not all entries at the targeted level may need processing,
423 * - the re-calculation flag gets always set.
424 * The passed in range is guaranteed to not cross a page (table)
425 * boundary at the targeted level.
426 */
ept_invalidate_emt_range(struct p2m_domain * p2m,unsigned int target,unsigned long first_gfn,unsigned long last_gfn)427 static int ept_invalidate_emt_range(struct p2m_domain *p2m,
428 unsigned int target,
429 unsigned long first_gfn,
430 unsigned long last_gfn)
431 {
432 ept_entry_t *table;
433 unsigned long gfn_remainder = first_gfn;
434 unsigned int i, index;
435 int wrc, rc = 0, ret = GUEST_TABLE_MAP_FAILED;
436
437 table = map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
438 for ( i = p2m->ept.wl; i > target; --i )
439 {
440 ret = ept_next_level(p2m, 1, &table, &gfn_remainder, i);
441 if ( ret == GUEST_TABLE_MAP_FAILED )
442 goto out;
443 if ( ret != GUEST_TABLE_NORMAL_PAGE )
444 break;
445 }
446
447 if ( i > target )
448 {
449 /* We need to split the original page. */
450 ept_entry_t split_ept_entry;
451
452 index = gfn_remainder >> (i * EPT_TABLE_ORDER);
453 split_ept_entry = atomic_read_ept_entry(&table[index]);
454 ASSERT(is_epte_superpage(&split_ept_entry));
455 if ( !ept_split_super_page(p2m, &split_ept_entry, i, target) )
456 {
457 ept_free_entry(p2m, &split_ept_entry, i);
458 rc = -ENOMEM;
459 goto out;
460 }
461 wrc = atomic_write_ept_entry(&table[index], split_ept_entry, i);
462 ASSERT(wrc == 0);
463
464 for ( ; i > target; --i )
465 if ( !ept_next_level(p2m, 1, &table, &gfn_remainder, i) )
466 break;
467 ASSERT(i == target);
468 }
469
470 index = gfn_remainder >> (i * EPT_TABLE_ORDER);
471 i = (last_gfn >> (i * EPT_TABLE_ORDER)) & (EPT_PAGETABLE_ENTRIES - 1);
472 for ( ; index <= i; ++index )
473 {
474 ept_entry_t e = atomic_read_ept_entry(&table[index]);
475
476 if ( is_epte_valid(&e) && is_epte_present(&e) &&
477 (e.emt != MTRR_NUM_TYPES || !e.recalc) )
478 {
479 e.emt = MTRR_NUM_TYPES;
480 e.recalc = 1;
481 wrc = atomic_write_ept_entry(&table[index], e, target);
482 ASSERT(wrc == 0);
483 rc = 1;
484 }
485 }
486
487 out:
488 unmap_domain_page(table);
489
490 return rc;
491 }
492
493 /*
494 * Resolve deliberately mis-configured (EMT field set to an invalid value)
495 * entries in the page table hierarchy for the given GFN:
496 * - calculate the correct value for the EMT field,
497 * - if marked so, re-calculate the P2M type,
498 * - propagate EMT and re-calculation flag down to the next page table level
499 * for entries not involved in the translation of the given GFN.
500 * Returns:
501 * - negative errno values in error,
502 * - zero if no adjustment was done,
503 * - a positive value if at least one adjustment was done.
504 */
resolve_misconfig(struct p2m_domain * p2m,unsigned long gfn)505 static int resolve_misconfig(struct p2m_domain *p2m, unsigned long gfn)
506 {
507 struct ept_data *ept = &p2m->ept;
508 unsigned int level = ept->wl;
509 unsigned long mfn = ept->mfn;
510 ept_entry_t *epte;
511 int wrc, rc = 0;
512
513 if ( !mfn )
514 return 0;
515
516 for ( ; ; --level )
517 {
518 ept_entry_t e;
519 unsigned int i;
520
521 epte = map_domain_page(_mfn(mfn));
522 i = (gfn >> (level * EPT_TABLE_ORDER)) & (EPT_PAGETABLE_ENTRIES - 1);
523 e = atomic_read_ept_entry(&epte[i]);
524
525 if ( level == 0 || is_epte_superpage(&e) )
526 {
527 uint8_t ipat = 0;
528
529 if ( e.emt != MTRR_NUM_TYPES )
530 break;
531
532 if ( level == 0 )
533 {
534 for ( gfn -= i, i = 0; i < EPT_PAGETABLE_ENTRIES; ++i )
535 {
536 p2m_type_t nt;
537
538 e = atomic_read_ept_entry(&epte[i]);
539 if ( e.emt == MTRR_NUM_TYPES )
540 e.emt = 0;
541 if ( !is_epte_valid(&e) || !is_epte_present(&e) )
542 continue;
543 e.emt = epte_get_entry_emt(p2m->domain, gfn + i,
544 _mfn(e.mfn), 0, &ipat,
545 e.sa_p2mt == p2m_mmio_direct);
546 e.ipat = ipat;
547
548 nt = p2m_recalc_type(e.recalc, e.sa_p2mt, p2m, gfn + i);
549 if ( nt != e.sa_p2mt )
550 {
551 if ( e.sa_p2mt == p2m_ioreq_server )
552 {
553 ASSERT(p2m->ioreq.entry_count > 0);
554 p2m->ioreq.entry_count--;
555 }
556
557 e.sa_p2mt = nt;
558 ept_p2m_type_to_flags(p2m, &e, e.sa_p2mt, e.access);
559 }
560 e.recalc = 0;
561 wrc = atomic_write_ept_entry(&epte[i], e, level);
562 ASSERT(wrc == 0);
563 }
564 }
565 else
566 {
567 int emt = epte_get_entry_emt(p2m->domain, gfn, _mfn(e.mfn),
568 level * EPT_TABLE_ORDER, &ipat,
569 e.sa_p2mt == p2m_mmio_direct);
570 bool_t recalc = e.recalc;
571
572 if ( recalc && p2m_is_changeable(e.sa_p2mt) )
573 {
574 unsigned long mask = ~0UL << (level * EPT_TABLE_ORDER);
575
576 ASSERT(e.sa_p2mt != p2m_ioreq_server);
577 switch ( p2m_is_logdirty_range(p2m, gfn & mask,
578 gfn | ~mask) )
579 {
580 case 0:
581 e.sa_p2mt = p2m_ram_rw;
582 e.recalc = 0;
583 break;
584 case 1:
585 e.sa_p2mt = p2m_ram_logdirty;
586 e.recalc = 0;
587 break;
588 default: /* Force split. */
589 emt = -1;
590 break;
591 }
592 }
593 if ( unlikely(emt < 0) )
594 {
595 if ( ept_split_super_page(p2m, &e, level, level - 1) )
596 {
597 wrc = atomic_write_ept_entry(&epte[i], e, level);
598 ASSERT(wrc == 0);
599 unmap_domain_page(epte);
600 mfn = e.mfn;
601 continue;
602 }
603 ept_free_entry(p2m, &e, level);
604 rc = -ENOMEM;
605 break;
606 }
607 e.emt = emt;
608 e.ipat = ipat;
609 e.recalc = 0;
610 if ( recalc && p2m_is_changeable(e.sa_p2mt) )
611 ept_p2m_type_to_flags(p2m, &e, e.sa_p2mt, e.access);
612 wrc = atomic_write_ept_entry(&epte[i], e, level);
613 ASSERT(wrc == 0);
614 }
615
616 rc = 1;
617 break;
618 }
619
620 if ( e.emt == MTRR_NUM_TYPES )
621 {
622 ASSERT(is_epte_present(&e));
623 ept_invalidate_emt(_mfn(e.mfn), e.recalc, level);
624 smp_wmb();
625 e.emt = 0;
626 e.recalc = 0;
627 wrc = atomic_write_ept_entry(&epte[i], e, level);
628 ASSERT(wrc == 0);
629 unmap_domain_page(epte);
630 rc = 1;
631 }
632 else if ( is_epte_present(&e) && !e.emt )
633 unmap_domain_page(epte);
634 else
635 break;
636
637 mfn = e.mfn;
638 }
639
640 unmap_domain_page(epte);
641 if ( rc )
642 {
643 struct vcpu *v;
644
645 for_each_vcpu ( p2m->domain, v )
646 v->arch.hvm_vmx.ept_spurious_misconfig = 1;
647 }
648
649 return rc;
650 }
651
ept_handle_misconfig(uint64_t gpa)652 bool_t ept_handle_misconfig(uint64_t gpa)
653 {
654 struct vcpu *curr = current;
655 struct p2m_domain *p2m = p2m_get_hostp2m(curr->domain);
656 bool_t spurious;
657 int rc;
658
659 p2m_lock(p2m);
660
661 spurious = curr->arch.hvm_vmx.ept_spurious_misconfig;
662 rc = resolve_misconfig(p2m, PFN_DOWN(gpa));
663 curr->arch.hvm_vmx.ept_spurious_misconfig = 0;
664
665 p2m_unlock(p2m);
666
667 return spurious ? (rc >= 0) : (rc > 0);
668 }
669
670 /*
671 * ept_set_entry() computes 'need_modify_vtd_table' for itself,
672 * by observing whether any gfn->mfn translations are modified.
673 *
674 * Returns: 0 for success, -errno for failure
675 */
676 static int
ept_set_entry(struct p2m_domain * p2m,gfn_t gfn_,mfn_t mfn,unsigned int order,p2m_type_t p2mt,p2m_access_t p2ma,int sve)677 ept_set_entry(struct p2m_domain *p2m, gfn_t gfn_, mfn_t mfn,
678 unsigned int order, p2m_type_t p2mt, p2m_access_t p2ma,
679 int sve)
680 {
681 ept_entry_t *table, *ept_entry = NULL;
682 unsigned long gfn = gfn_x(gfn_);
683 unsigned long gfn_remainder = gfn;
684 unsigned int i, target = order / EPT_TABLE_ORDER;
685 unsigned long fn_mask = !mfn_eq(mfn, INVALID_MFN) ? (gfn | mfn_x(mfn)) : gfn;
686 int ret, rc = 0;
687 bool_t entry_written = 0;
688 bool_t direct_mmio = (p2mt == p2m_mmio_direct);
689 uint8_t ipat = 0;
690 bool_t need_modify_vtd_table = 1;
691 bool_t vtd_pte_present = 0;
692 unsigned int iommu_flags = p2m_get_iommu_flags(p2mt, mfn);
693 bool_t needs_sync = 1;
694 ept_entry_t old_entry = { .epte = 0 };
695 ept_entry_t new_entry = { .epte = 0 };
696 struct ept_data *ept = &p2m->ept;
697 struct domain *d = p2m->domain;
698
699 ASSERT(ept);
700 /*
701 * the caller must make sure:
702 * 1. passing valid gfn and mfn at order boundary.
703 * 2. gfn not exceeding guest physical address width.
704 * 3. passing a valid order.
705 */
706 if ( (fn_mask & ((1UL << order) - 1)) ||
707 ((u64)gfn >> ((ept->wl + 1) * EPT_TABLE_ORDER)) ||
708 (order % EPT_TABLE_ORDER) )
709 return -EINVAL;
710
711 /* Carry out any eventually pending earlier changes first. */
712 ret = resolve_misconfig(p2m, gfn);
713 if ( ret < 0 )
714 return ret;
715
716 ASSERT((target == 2 && hap_has_1gb) ||
717 (target == 1 && hap_has_2mb) ||
718 (target == 0));
719 ASSERT(!p2m_is_foreign(p2mt) || target == 0);
720
721 table = map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
722
723 ret = GUEST_TABLE_MAP_FAILED;
724 for ( i = ept->wl; i > target; i-- )
725 {
726 ret = ept_next_level(p2m, 0, &table, &gfn_remainder, i);
727 if ( !ret )
728 {
729 rc = -ENOENT;
730 goto out;
731 }
732 else if ( ret != GUEST_TABLE_NORMAL_PAGE )
733 break;
734 }
735
736 ASSERT(ret != GUEST_TABLE_POD_PAGE || i != target);
737
738 ept_entry = table + (gfn_remainder >> (i * EPT_TABLE_ORDER));
739
740 /* In case VT-d uses same page table, this flag is needed by VT-d */
741 vtd_pte_present = is_epte_present(ept_entry);
742
743 /*
744 * If we're here with i > target, we must be at a leaf node, and
745 * we need to break up the superpage.
746 *
747 * If we're here with i == target and i > 0, we need to check to see
748 * if we're replacing a non-leaf entry (i.e., pointing to an N-1 table)
749 * with a leaf entry (a 1GiB or 2MiB page), and handle things appropriately.
750 */
751
752 if ( i == target )
753 {
754 /* We reached the target level. */
755
756 /* No need to flush if the old entry wasn't valid */
757 if ( !is_epte_present(ept_entry) )
758 needs_sync = 0;
759
760 /* If we're replacing a non-leaf entry with a leaf entry (1GiB or 2MiB),
761 * the intermediate tables will be freed below after the ept flush
762 *
763 * Read-then-write is OK because we hold the p2m lock. */
764 old_entry = *ept_entry;
765 }
766 else
767 {
768 /* We need to split the original page. */
769 ept_entry_t split_ept_entry;
770
771 ASSERT(is_epte_superpage(ept_entry));
772
773 split_ept_entry = atomic_read_ept_entry(ept_entry);
774
775 if ( !ept_split_super_page(p2m, &split_ept_entry, i, target) )
776 {
777 ept_free_entry(p2m, &split_ept_entry, i);
778 rc = -ENOMEM;
779 goto out;
780 }
781
782 /* now install the newly split ept sub-tree */
783 /* NB: please make sure domian is paused and no in-fly VT-d DMA. */
784 rc = atomic_write_ept_entry(ept_entry, split_ept_entry, i);
785 ASSERT(rc == 0);
786
787 /* then move to the level we want to make real changes */
788 for ( ; i > target; i-- )
789 if ( !ept_next_level(p2m, 0, &table, &gfn_remainder, i) )
790 break;
791 /* We just installed the pages we need. */
792 ASSERT(i == target);
793
794 ept_entry = table + (gfn_remainder >> (i * EPT_TABLE_ORDER));
795 }
796
797 if ( mfn_valid(mfn) || p2m_allows_invalid_mfn(p2mt) )
798 {
799 int emt = epte_get_entry_emt(p2m->domain, gfn, mfn,
800 i * EPT_TABLE_ORDER, &ipat, direct_mmio);
801
802 if ( emt >= 0 )
803 new_entry.emt = emt;
804 else /* ept_handle_misconfig() will need to take care of this. */
805 new_entry.emt = MTRR_NUM_TYPES;
806
807 new_entry.ipat = ipat;
808 new_entry.sp = !!i;
809 new_entry.sa_p2mt = p2mt;
810 new_entry.access = p2ma;
811 new_entry.snp = (iommu_enabled && iommu_snoop);
812
813 /* the caller should take care of the previous page */
814 new_entry.mfn = mfn_x(mfn);
815
816 /* Safe to read-then-write because we hold the p2m lock */
817 if ( ept_entry->mfn == new_entry.mfn &&
818 p2m_get_iommu_flags(ept_entry->sa_p2mt, _mfn(ept_entry->mfn)) ==
819 iommu_flags )
820 need_modify_vtd_table = 0;
821
822 ept_p2m_type_to_flags(p2m, &new_entry, p2mt, p2ma);
823 }
824
825 if ( sve != -1 )
826 new_entry.suppress_ve = !!sve;
827 else
828 new_entry.suppress_ve = is_epte_valid(&old_entry) ?
829 old_entry.suppress_ve : 1;
830
831 /*
832 * p2m_ioreq_server is only used for 4K pages, so the
833 * count is only done on ept page table entries.
834 */
835 if ( p2mt == p2m_ioreq_server )
836 {
837 ASSERT(i == 0);
838 p2m->ioreq.entry_count++;
839 }
840
841 if ( ept_entry->sa_p2mt == p2m_ioreq_server )
842 {
843 ASSERT(i == 0);
844 ASSERT(p2m->ioreq.entry_count > 0);
845 p2m->ioreq.entry_count--;
846 }
847
848 rc = atomic_write_ept_entry(ept_entry, new_entry, target);
849 if ( unlikely(rc) )
850 old_entry.epte = 0;
851 else
852 {
853 entry_written = 1;
854
855 if ( p2mt != p2m_invalid &&
856 (gfn + (1UL << order) - 1 > p2m->max_mapped_pfn) )
857 /* Track the highest gfn for which we have ever had a valid mapping */
858 p2m->max_mapped_pfn = gfn + (1UL << order) - 1;
859 }
860
861 out:
862 if ( needs_sync )
863 ept_sync_domain(p2m);
864
865 /* For host p2m, may need to change VT-d page table.*/
866 if ( rc == 0 && p2m_is_hostp2m(p2m) && need_iommu(d) &&
867 need_modify_vtd_table )
868 {
869 if ( iommu_hap_pt_share )
870 rc = iommu_pte_flush(d, gfn, &ept_entry->epte, order, vtd_pte_present);
871 else
872 {
873 if ( iommu_flags )
874 for ( i = 0; i < (1 << order); i++ )
875 {
876 rc = iommu_map_page(d, gfn + i, mfn_x(mfn) + i, iommu_flags);
877 if ( unlikely(rc) )
878 {
879 while ( i-- )
880 /* If statement to satisfy __must_check. */
881 if ( iommu_unmap_page(p2m->domain, gfn + i) )
882 continue;
883
884 break;
885 }
886 }
887 else
888 for ( i = 0; i < (1 << order); i++ )
889 {
890 ret = iommu_unmap_page(d, gfn + i);
891 if ( !rc )
892 rc = ret;
893 }
894 }
895 }
896
897 unmap_domain_page(table);
898
899 /* Release the old intermediate tables, if any. This has to be the
900 last thing we do, after the ept_sync_domain() and removal
901 from the iommu tables, so as to avoid a potential
902 use-after-free. */
903 if ( is_epte_present(&old_entry) )
904 ept_free_entry(p2m, &old_entry, target);
905
906 if ( entry_written && p2m_is_hostp2m(p2m) )
907 p2m_altp2m_propagate_change(d, _gfn(gfn), mfn, order, p2mt, p2ma);
908
909 return rc;
910 }
911
912 /* Read ept p2m entries */
ept_get_entry(struct p2m_domain * p2m,gfn_t gfn_,p2m_type_t * t,p2m_access_t * a,p2m_query_t q,unsigned int * page_order,bool_t * sve)913 static mfn_t ept_get_entry(struct p2m_domain *p2m,
914 gfn_t gfn_, p2m_type_t *t, p2m_access_t* a,
915 p2m_query_t q, unsigned int *page_order,
916 bool_t *sve)
917 {
918 ept_entry_t *table =
919 map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
920 unsigned long gfn = gfn_x(gfn_);
921 unsigned long gfn_remainder = gfn;
922 ept_entry_t *ept_entry;
923 u32 index;
924 int i;
925 int ret = 0;
926 bool_t recalc = 0;
927 mfn_t mfn = INVALID_MFN;
928 struct ept_data *ept = &p2m->ept;
929
930 *t = p2m_mmio_dm;
931 *a = p2m_access_n;
932 if ( sve )
933 *sve = 1;
934
935 /* This pfn is higher than the highest the p2m map currently holds */
936 if ( gfn > p2m->max_mapped_pfn )
937 {
938 for ( i = ept->wl; i > 0; --i )
939 if ( (gfn & ~((1UL << (i * EPT_TABLE_ORDER)) - 1)) >
940 p2m->max_mapped_pfn )
941 break;
942 goto out;
943 }
944
945 /* Should check if gfn obeys GAW here. */
946
947 for ( i = ept->wl; i > 0; i-- )
948 {
949 retry:
950 if ( table[gfn_remainder >> (i * EPT_TABLE_ORDER)].recalc )
951 recalc = 1;
952 ret = ept_next_level(p2m, 1, &table, &gfn_remainder, i);
953 if ( !ret )
954 goto out;
955 else if ( ret == GUEST_TABLE_POD_PAGE )
956 {
957 if ( !(q & P2M_ALLOC) )
958 {
959 *t = p2m_populate_on_demand;
960 goto out;
961 }
962
963 /* Populate this superpage */
964 ASSERT(i <= 2);
965
966 index = gfn_remainder >> ( i * EPT_TABLE_ORDER);
967 ept_entry = table + index;
968
969 if ( p2m_pod_demand_populate(p2m, gfn_, i * EPT_TABLE_ORDER) )
970 goto retry;
971 else
972 goto out;
973 }
974 else if ( ret == GUEST_TABLE_SUPER_PAGE )
975 break;
976 }
977
978 index = gfn_remainder >> (i * EPT_TABLE_ORDER);
979 ept_entry = table + index;
980
981 if ( ept_entry->sa_p2mt == p2m_populate_on_demand )
982 {
983 if ( !(q & P2M_ALLOC) )
984 {
985 *t = p2m_populate_on_demand;
986 goto out;
987 }
988
989 ASSERT(i == 0);
990
991 if ( !p2m_pod_demand_populate(p2m, gfn_, PAGE_ORDER_4K) )
992 goto out;
993 }
994
995 if ( is_epte_valid(ept_entry) )
996 {
997 *t = p2m_recalc_type(recalc || ept_entry->recalc,
998 ept_entry->sa_p2mt, p2m, gfn);
999 *a = ept_entry->access;
1000 if ( sve )
1001 *sve = ept_entry->suppress_ve;
1002
1003 mfn = _mfn(ept_entry->mfn);
1004 if ( i )
1005 {
1006 /*
1007 * We may meet super pages, and to split into 4k pages
1008 * to emulate p2m table
1009 */
1010 unsigned long split_mfn = mfn_x(mfn) +
1011 (gfn_remainder &
1012 ((1 << (i * EPT_TABLE_ORDER)) - 1));
1013 mfn = _mfn(split_mfn);
1014 }
1015 }
1016
1017 out:
1018 if ( page_order )
1019 *page_order = i * EPT_TABLE_ORDER;
1020
1021 unmap_domain_page(table);
1022 return mfn;
1023 }
1024
ept_walk_table(struct domain * d,unsigned long gfn)1025 void ept_walk_table(struct domain *d, unsigned long gfn)
1026 {
1027 struct p2m_domain *p2m = p2m_get_hostp2m(d);
1028 struct ept_data *ept = &p2m->ept;
1029 ept_entry_t *table =
1030 map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
1031 unsigned long gfn_remainder = gfn;
1032
1033 int i;
1034
1035 gprintk(XENLOG_ERR, "Walking EPT tables for GFN %lx:\n", gfn);
1036
1037 /* This pfn is higher than the highest the p2m map currently holds */
1038 if ( gfn > p2m->max_mapped_pfn )
1039 {
1040 gprintk(XENLOG_ERR, " gfn exceeds max_mapped_pfn %lx\n",
1041 p2m->max_mapped_pfn);
1042 goto out;
1043 }
1044
1045 for ( i = ept->wl; i >= 0; i-- )
1046 {
1047 ept_entry_t *ept_entry, *next;
1048 u32 index;
1049
1050 /* Stolen from ept_next_level */
1051 index = gfn_remainder >> (i*EPT_TABLE_ORDER);
1052 ept_entry = table + index;
1053
1054 gprintk(XENLOG_ERR, " epte %"PRIx64"\n", ept_entry->epte);
1055
1056 if ( (i == 0) || !is_epte_present(ept_entry) ||
1057 is_epte_superpage(ept_entry) )
1058 goto out;
1059 else
1060 {
1061 gfn_remainder &= (1UL << (i*EPT_TABLE_ORDER)) - 1;
1062
1063 next = map_domain_page(_mfn(ept_entry->mfn));
1064
1065 unmap_domain_page(table);
1066
1067 table = next;
1068 }
1069 }
1070
1071 out:
1072 unmap_domain_page(table);
1073 return;
1074 }
1075
ept_change_entry_type_global(struct p2m_domain * p2m,p2m_type_t ot,p2m_type_t nt)1076 static void ept_change_entry_type_global(struct p2m_domain *p2m,
1077 p2m_type_t ot, p2m_type_t nt)
1078 {
1079 unsigned long mfn = p2m->ept.mfn;
1080
1081 if ( !mfn )
1082 return;
1083
1084 if ( ept_invalidate_emt(_mfn(mfn), 1, p2m->ept.wl) )
1085 ept_sync_domain(p2m);
1086 }
1087
ept_change_entry_type_range(struct p2m_domain * p2m,p2m_type_t ot,p2m_type_t nt,unsigned long first_gfn,unsigned long last_gfn)1088 static int ept_change_entry_type_range(struct p2m_domain *p2m,
1089 p2m_type_t ot, p2m_type_t nt,
1090 unsigned long first_gfn,
1091 unsigned long last_gfn)
1092 {
1093 unsigned int i, wl = p2m->ept.wl;
1094 unsigned long mask = (1 << EPT_TABLE_ORDER) - 1;
1095 int rc = 0, sync = 0;
1096
1097 if ( !p2m->ept.mfn )
1098 return -EINVAL;
1099
1100 for ( i = 0; i <= wl; )
1101 {
1102 if ( first_gfn & mask )
1103 {
1104 unsigned long end_gfn = min(first_gfn | mask, last_gfn);
1105
1106 rc = ept_invalidate_emt_range(p2m, i, first_gfn, end_gfn);
1107 sync |= rc;
1108 if ( rc < 0 || end_gfn >= last_gfn )
1109 break;
1110 first_gfn = end_gfn + 1;
1111 }
1112 else if ( (last_gfn & mask) != mask )
1113 {
1114 unsigned long start_gfn = max(first_gfn, last_gfn & ~mask);
1115
1116 rc = ept_invalidate_emt_range(p2m, i, start_gfn, last_gfn);
1117 sync |= rc;
1118 if ( rc < 0 || start_gfn <= first_gfn )
1119 break;
1120 last_gfn = start_gfn - 1;
1121 }
1122 else
1123 {
1124 ++i;
1125 mask |= mask << EPT_TABLE_ORDER;
1126 }
1127 }
1128
1129 if ( sync )
1130 ept_sync_domain(p2m);
1131
1132 return rc < 0 ? rc : 0;
1133 }
1134
ept_memory_type_changed(struct p2m_domain * p2m)1135 static void ept_memory_type_changed(struct p2m_domain *p2m)
1136 {
1137 unsigned long mfn = p2m->ept.mfn;
1138
1139 if ( !mfn )
1140 return;
1141
1142 if ( ept_invalidate_emt(_mfn(mfn), 0, p2m->ept.wl) )
1143 ept_sync_domain(p2m);
1144 }
1145
__ept_sync_domain(void * info)1146 static void __ept_sync_domain(void *info)
1147 {
1148 /*
1149 * The invalidation will be done before VMENTER (see
1150 * vmx_vmenter_helper()).
1151 */
1152 }
1153
ept_sync_domain_prepare(struct p2m_domain * p2m)1154 static void ept_sync_domain_prepare(struct p2m_domain *p2m)
1155 {
1156 struct domain *d = p2m->domain;
1157 struct ept_data *ept = &p2m->ept;
1158
1159 if ( nestedhvm_enabled(d) )
1160 {
1161 if ( p2m_is_nestedp2m(p2m) )
1162 ept = &p2m_get_hostp2m(d)->ept;
1163 else
1164 p2m_flush_nestedp2m(d);
1165 }
1166
1167 /*
1168 * Need to invalidate on all PCPUs because either:
1169 *
1170 * a) A VCPU has run and some translations may be cached.
1171 * b) A VCPU has not run and and the initial invalidation in case
1172 * of an EP4TA reuse is still needed.
1173 */
1174 cpumask_setall(ept->invalidate);
1175 }
1176
ept_sync_domain_mask(struct p2m_domain * p2m,const cpumask_t * mask)1177 static void ept_sync_domain_mask(struct p2m_domain *p2m, const cpumask_t *mask)
1178 {
1179 on_selected_cpus(mask, __ept_sync_domain, p2m, 1);
1180 }
1181
ept_sync_domain(struct p2m_domain * p2m)1182 void ept_sync_domain(struct p2m_domain *p2m)
1183 {
1184 struct domain *d = p2m->domain;
1185
1186 /* Only if using EPT and this domain has some VCPUs to dirty. */
1187 if ( !paging_mode_hap(d) || !d->vcpu || !d->vcpu[0] )
1188 return;
1189
1190 ept_sync_domain_prepare(p2m);
1191
1192 if ( p2m->defer_flush )
1193 {
1194 p2m->need_flush = 1;
1195 return;
1196 }
1197
1198 ept_sync_domain_mask(p2m, d->domain_dirty_cpumask);
1199 }
1200
ept_tlb_flush(struct p2m_domain * p2m)1201 static void ept_tlb_flush(struct p2m_domain *p2m)
1202 {
1203 ept_sync_domain_mask(p2m, p2m->domain->domain_dirty_cpumask);
1204 }
1205
ept_enable_pml(struct p2m_domain * p2m)1206 static void ept_enable_pml(struct p2m_domain *p2m)
1207 {
1208 /* Domain must have been paused */
1209 ASSERT(atomic_read(&p2m->domain->pause_count));
1210
1211 /*
1212 * No need to return whether vmx_domain_enable_pml has succeeded, as
1213 * ept_p2m_type_to_flags will do the check, and write protection will be
1214 * used if PML is not enabled.
1215 */
1216 if ( vmx_domain_enable_pml(p2m->domain) )
1217 return;
1218
1219 /* Enable EPT A/D bit for PML */
1220 p2m->ept.ad = 1;
1221 vmx_domain_update_eptp(p2m->domain);
1222 }
1223
ept_disable_pml(struct p2m_domain * p2m)1224 static void ept_disable_pml(struct p2m_domain *p2m)
1225 {
1226 /* Domain must have been paused */
1227 ASSERT(atomic_read(&p2m->domain->pause_count));
1228
1229 vmx_domain_disable_pml(p2m->domain);
1230
1231 /* Disable EPT A/D bit */
1232 p2m->ept.ad = 0;
1233 vmx_domain_update_eptp(p2m->domain);
1234 }
1235
ept_flush_pml_buffers(struct p2m_domain * p2m)1236 static void ept_flush_pml_buffers(struct p2m_domain *p2m)
1237 {
1238 /* Domain must have been paused */
1239 ASSERT(atomic_read(&p2m->domain->pause_count));
1240
1241 vmx_domain_flush_pml_buffers(p2m->domain);
1242 }
1243
ept_p2m_init(struct p2m_domain * p2m)1244 int ept_p2m_init(struct p2m_domain *p2m)
1245 {
1246 struct ept_data *ept = &p2m->ept;
1247
1248 p2m->set_entry = ept_set_entry;
1249 p2m->get_entry = ept_get_entry;
1250 p2m->recalc = resolve_misconfig;
1251 p2m->change_entry_type_global = ept_change_entry_type_global;
1252 p2m->change_entry_type_range = ept_change_entry_type_range;
1253 p2m->memory_type_changed = ept_memory_type_changed;
1254 p2m->audit_p2m = NULL;
1255 p2m->tlb_flush = ept_tlb_flush;
1256
1257 /* Set the memory type used when accessing EPT paging structures. */
1258 ept->mt = EPT_DEFAULT_MT;
1259
1260 /* set EPT page-walk length, now it's actual walk length - 1, i.e. 3 */
1261 ept->wl = 3;
1262
1263 if ( cpu_has_vmx_pml )
1264 {
1265 p2m->enable_hardware_log_dirty = ept_enable_pml;
1266 p2m->disable_hardware_log_dirty = ept_disable_pml;
1267 p2m->flush_hardware_cached_dirty = ept_flush_pml_buffers;
1268 }
1269
1270 if ( !zalloc_cpumask_var(&ept->invalidate) )
1271 return -ENOMEM;
1272
1273 /*
1274 * Assume an initial invalidation is required, in case an EP4TA is
1275 * reused.
1276 */
1277 cpumask_setall(ept->invalidate);
1278
1279 return 0;
1280 }
1281
ept_p2m_uninit(struct p2m_domain * p2m)1282 void ept_p2m_uninit(struct p2m_domain *p2m)
1283 {
1284 struct ept_data *ept = &p2m->ept;
1285 free_cpumask_var(ept->invalidate);
1286 }
1287
memory_type_to_str(unsigned int x)1288 static const char *memory_type_to_str(unsigned int x)
1289 {
1290 static const char memory_types[8][3] = {
1291 [MTRR_TYPE_UNCACHABLE] = "UC",
1292 [MTRR_TYPE_WRCOMB] = "WC",
1293 [MTRR_TYPE_WRTHROUGH] = "WT",
1294 [MTRR_TYPE_WRPROT] = "WP",
1295 [MTRR_TYPE_WRBACK] = "WB",
1296 [MTRR_NUM_TYPES] = "??"
1297 };
1298
1299 ASSERT(x < ARRAY_SIZE(memory_types));
1300 return memory_types[x][0] ? memory_types[x] : "?";
1301 }
1302
ept_dump_p2m_table(unsigned char key)1303 static void ept_dump_p2m_table(unsigned char key)
1304 {
1305 struct domain *d;
1306 ept_entry_t *table, *ept_entry;
1307 int order;
1308 int i;
1309 int ret = 0;
1310 unsigned long gfn, gfn_remainder;
1311 unsigned long record_counter = 0;
1312 struct p2m_domain *p2m;
1313 struct ept_data *ept;
1314
1315 for_each_domain(d)
1316 {
1317 if ( !hap_enabled(d) )
1318 continue;
1319
1320 p2m = p2m_get_hostp2m(d);
1321 ept = &p2m->ept;
1322 printk("\ndomain%d EPT p2m table:\n", d->domain_id);
1323
1324 for ( gfn = 0; gfn <= p2m->max_mapped_pfn; gfn += 1UL << order )
1325 {
1326 char c = 0;
1327
1328 gfn_remainder = gfn;
1329 table = map_domain_page(pagetable_get_mfn(p2m_get_pagetable(p2m)));
1330
1331 for ( i = ept->wl; i > 0; i-- )
1332 {
1333 ept_entry = table + (gfn_remainder >> (i * EPT_TABLE_ORDER));
1334 if ( ept_entry->emt == MTRR_NUM_TYPES )
1335 c = '?';
1336 ret = ept_next_level(p2m, 1, &table, &gfn_remainder, i);
1337 if ( ret != GUEST_TABLE_NORMAL_PAGE )
1338 break;
1339 }
1340
1341 order = i * EPT_TABLE_ORDER;
1342 ept_entry = table + (gfn_remainder >> order);
1343 if ( ret != GUEST_TABLE_MAP_FAILED && is_epte_valid(ept_entry) )
1344 {
1345 if ( ept_entry->sa_p2mt == p2m_populate_on_demand )
1346 printk("gfn: %13lx order: %2d PoD\n", gfn, order);
1347 else
1348 printk("gfn: %13lx order: %2d mfn: %13lx %c%c%c %c%c%c\n",
1349 gfn, order, ept_entry->mfn + 0UL,
1350 ept_entry->r ? 'r' : ' ',
1351 ept_entry->w ? 'w' : ' ',
1352 ept_entry->x ? 'x' : ' ',
1353 memory_type_to_str(ept_entry->emt)[0],
1354 memory_type_to_str(ept_entry->emt)[1]
1355 ?: ept_entry->emt + '0',
1356 c ?: ept_entry->ipat ? '!' : ' ');
1357
1358 if ( !(record_counter++ % 100) )
1359 process_pending_softirqs();
1360 }
1361 unmap_domain_page(table);
1362 }
1363 }
1364 }
1365
setup_ept_dump(void)1366 void setup_ept_dump(void)
1367 {
1368 register_keyhandler('D', ept_dump_p2m_table, "dump VT-x EPT tables", 0);
1369 }
1370
p2m_init_altp2m_ept(struct domain * d,unsigned int i)1371 void p2m_init_altp2m_ept(struct domain *d, unsigned int i)
1372 {
1373 struct p2m_domain *p2m = d->arch.altp2m_p2m[i];
1374 struct ept_data *ept;
1375
1376 p2m->min_remapped_gfn = gfn_x(INVALID_GFN);
1377 p2m->max_remapped_gfn = 0;
1378 ept = &p2m->ept;
1379 ept->mfn = pagetable_get_pfn(p2m_get_pagetable(p2m));
1380 d->arch.altp2m_eptp[i] = ept->eptp;
1381 }
1382
p2m_find_altp2m_by_eptp(struct domain * d,uint64_t eptp)1383 unsigned int p2m_find_altp2m_by_eptp(struct domain *d, uint64_t eptp)
1384 {
1385 struct p2m_domain *p2m;
1386 struct ept_data *ept;
1387 unsigned int i;
1388
1389 altp2m_list_lock(d);
1390
1391 for ( i = 0; i < MAX_ALTP2M; i++ )
1392 {
1393 if ( d->arch.altp2m_eptp[i] == mfn_x(INVALID_MFN) )
1394 continue;
1395
1396 p2m = d->arch.altp2m_p2m[i];
1397 ept = &p2m->ept;
1398
1399 if ( eptp == ept->eptp )
1400 goto out;
1401 }
1402
1403 i = INVALID_ALTP2M;
1404
1405 out:
1406 altp2m_list_unlock(d);
1407 return i;
1408 }
1409
1410 /*
1411 * Local variables:
1412 * mode: C
1413 * c-file-style: "BSD"
1414 * c-basic-offset: 4
1415 * tab-width: 4
1416 * indent-tabs-mode: nil
1417 * End:
1418 */
1419