1 /*
2 * Copyright (C) 2018-2024 Intel Corporation.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6 #ifndef PGTABLE_H
7 #define PGTABLE_H
8
9 #include <asm/page.h>
10
11 /**
12 * @addtogroup hwmgmt_page
13 *
14 * @{
15 */
16
17 /**
18 * @file
19 * @brief All APIs to support page table management
20 *
21 * This file defines macros, structures, declarations and functions related for managing page tables.
22 *
23 */
24
25 #define PAGE_PRESENT (1UL << 0U)
26 #define PAGE_RW (1UL << 1U)
27 #define PAGE_USER (1UL << 2U)
28 #define PAGE_PWT (1UL << 3U)
29 #define PAGE_PCD (1UL << 4U)
30 #define PAGE_ACCESSED (1UL << 5U)
31 #define PAGE_DIRTY (1UL << 6U)
32 #define PAGE_PSE (1UL << 7U)
33 #define PAGE_GLOBAL (1UL << 8U)
34 #define PAGE_PAT_LARGE (1UL << 12U)
35 #define PAGE_NX (1UL << 63U)
36
37 #define PAGE_CACHE_MASK (PAGE_PCD | PAGE_PWT)
38 #define PAGE_CACHE_WB 0UL
39 #define PAGE_CACHE_WT PAGE_PWT
40 #define PAGE_CACHE_UC_MINUS PAGE_PCD
41 #define PAGE_CACHE_UC (PAGE_PCD | PAGE_PWT)
42
43 #define PAGE_ATTR_USER (PAGE_PRESENT | PAGE_RW | PAGE_USER | PAGE_NX)
44
45 /**
46 * @defgroup ept_mem_access_right EPT Memory Access Right
47 *
48 * This is a group that includes EPT Memory Access Right Definitions.
49 *
50 * @{
51 */
52
53 /**
54 * @brief EPT memory access right is read-only.
55 */
56 #define EPT_RD (1UL << 0U)
57
58 /**
59 * @brief EPT memory access right is read/write.
60 */
61 #define EPT_WR (1UL << 1U)
62
63 /**
64 * @brief EPT memory access right is executable.
65 */
66 #define EPT_EXE (1UL << 2U)
67
68 /**
69 * @brief EPT memory access right is read/write and executable.
70 */
71 #define EPT_RWX (EPT_RD | EPT_WR | EPT_EXE)
72
73 /**
74 * @}
75 */
76 /* End of ept_mem_access_right */
77
78 /**
79 * @defgroup ept_mem_type EPT Memory Type
80 *
81 * This is a group that includes EPT Memory Type Definitions.
82 *
83 * @{
84 */
85
86 /**
87 * @brief EPT memory type is specified in bits 5:3 of the EPT paging-structure entry.
88 */
89 #define EPT_MT_SHIFT 3U
90
91 /**
92 * @brief EPT memory type is uncacheable.
93 */
94 #define EPT_UNCACHED (0UL << EPT_MT_SHIFT)
95
96 /**
97 * @brief EPT memory type is write combining.
98 */
99 #define EPT_WC (1UL << EPT_MT_SHIFT)
100
101 /**
102 * @brief EPT memory type is write through.
103 */
104 #define EPT_WT (4UL << EPT_MT_SHIFT)
105
106 /**
107 * @brief EPT memory type is write protected.
108 */
109 #define EPT_WP (5UL << EPT_MT_SHIFT)
110
111 /**
112 * @brief EPT memory type is write back.
113 */
114 #define EPT_WB (6UL << EPT_MT_SHIFT)
115
116 /**
117 * @brief Ignore PAT memory type.
118 */
119 #define EPT_IGNORE_PAT (1UL << 6U)
120
121 /**
122 * @}
123 */
124 /* End of ept_mem_type */
125
126 #define EPT_MT_MASK (7UL << EPT_MT_SHIFT)
127 #define EPT_VE (1UL << 63U)
128 /* EPT leaf entry bits (bit 52 - bit 63) should be maksed when calculate PFN */
129 #define EPT_PFN_HIGH_MASK 0xFFF0000000000000UL
130
131 #define PML4E_SHIFT 39U
132 #define PTRS_PER_PML4E 512UL
133 #define PML4E_SIZE (1UL << PML4E_SHIFT)
134 #define PML4E_MASK (~(PML4E_SIZE - 1UL))
135
136 #define PDPTE_SHIFT 30U
137 #define PTRS_PER_PDPTE 512UL
138 #define PDPTE_SIZE (1UL << PDPTE_SHIFT)
139 #define PDPTE_MASK (~(PDPTE_SIZE - 1UL))
140
141 #define PDE_SHIFT 21U
142 #define PTRS_PER_PDE 512UL
143 #define PDE_SIZE (1UL << PDE_SHIFT)
144 #define PDE_MASK (~(PDE_SIZE - 1UL))
145
146 #define PTE_SHIFT 12U
147 #define PTRS_PER_PTE 512UL
148 #define PTE_SIZE (1UL << PTE_SHIFT)
149 #define PTE_MASK (~(PTE_SIZE - 1UL))
150
151 /* TODO: PAGE_MASK & PHYSICAL_MASK */
152 #define PML4E_PFN_MASK 0x0000FFFFFFFFF000UL
153 #define PDPTE_PFN_MASK 0x0000FFFFFFFFF000UL
154 #define PDE_PFN_MASK 0x0000FFFFFFFFF000UL
155
156 #define EPT_ENTRY_PFN_MASK ((~EPT_PFN_HIGH_MASK) & PAGE_MASK)
157
158 /**
159 * @brief Page tables level in IA32 paging mode
160 *
161 * 4-level paging in IA32 mode may map linear addresses to 4-KByte pages, 2-MByte pages, or 1-GByte pages. The 4 levels
162 * are PML4, PDPT, PD, and PT. The value to present each level is fixed.
163 */
164 enum _page_table_level {
165 IA32E_PML4 = 0, /**< The Page-Map-Level-4(PML4) level in the page tables.
166 * The value is fixed to 0. */
167 IA32E_PDPT = 1, /**< The Page-Directory-Pointer-Table(PDPT) level in the page tables. */
168 IA32E_PD = 2, /**< The Page-Directory(PD) level in the page tables. */
169 IA32E_PT = 3, /**< The Page-Table(PT) level in the page tables. */
170 };
171
172 /**
173 * @brief Data structure that contains the related operations and properties of page table.
174 *
175 * This structure is used to add/modify/delete page table.
176 *
177 * @consistency N/A
178 * @alignment N/A
179 *
180 * @remark N/A
181 */
182 struct pgtable {
183 /**
184 * @brief Default memory access rights.
185 *
186 * A linear address can be translated to a physical address by the page tables. The translation is controlled by
187 * the memory access rights, as defined by the architecture's memory system design. The default memory access
188 * rights can be used to set the memory access rights for a page table entry when the page table is created.
189 */
190 uint64_t default_access_right;
191 /**
192 * @brief Mask to check if the page referenced by entry is present.
193 *
194 * The presence of a page is indicated by specific bits in the entry, as defined by the architecture's memory
195 * system design. For example, in ept table entry it's indicated by bit0|bit1|bit2, and in mmu table entry it's
196 * indicated by bit 0.
197 */
198 uint64_t pgentry_present_mask;
199 struct page_pool *pool; /**< Pointer to the page pool used for managing pages. */
200 /**
201 * @brief Function to check if large pages are supported.
202 *
203 * This function is used to check if large pages are supported for a specific page table level and memory access
204 * rights.
205 */
206 bool (*large_page_support)(enum _page_table_level level, uint64_t prot);
207 void (*clflush_pagewalk)(const void *p); /**< Function to flush a page table entry from the cache. */
208 void (*tweak_exe_right)(uint64_t *entry); /**< Function to tweak execution rights for an entry. */
209 void (*recover_exe_right)(uint64_t *entry); /**< Function to recover execution rights for an entry. */
210 };
211
212 /**
213 * @brief Check whether the page referenced by the specified paging-structure entry is present or not.
214 *
215 * This function is used to check if the page referenced is present. A paging-structure entry references a page. The
216 * presence of a page is indicated by specific bits in the entry, as defined by the architecture's memory system design.
217 * For example, in ept table entry it's indicated by bit0|bit1|bit2, and in mmu table entry it's indicated by bit 0.
218 *
219 * This function checks whether the page referenced exists based on specific bits.
220 *
221 * @param[in] table A pointer to the structure pgtable which provides the mask to check whether page referenced is
222 * present or not.
223 * @param[in] pte The paging-structure entry to check.
224 *
225 * @return A boolean value indicating if the page referenced by the specified paging-structure entry is present
226 *
227 * @retval true Indicates the page referenced is present.
228 * @retval false Indicates the page referenced is not present.
229 *
230 * @pre table != NULL
231 *
232 * @post N/A
233 */
pgentry_present(const struct pgtable * table,uint64_t pte)234 static inline bool pgentry_present(const struct pgtable *table, uint64_t pte)
235 {
236 return ((table->pgentry_present_mask & (pte)) != 0UL);
237 }
238
239 /**
240 * @brief Translate a host physical address to a host virtual address before paging mode enabled.
241 *
242 * This function is used to translate a host physical address to a host virtual address before paging mode enabled. HPA
243 * is 1:1 mapping to HVA.
244 *
245 * It returns the host virtual address that corresponds to the given host physical address.
246 *
247 * @param[in] x The host physical address
248 *
249 * @return The translated host virtual address
250 *
251 * @retval NULL if x == 0
252 *
253 * @pre N/A
254 *
255 * @post N/A
256 *
257 * @remark This function is used before paging mode enabled.
258 */
hpa2hva_early(uint64_t x)259 static inline void *hpa2hva_early(uint64_t x)
260 {
261 return (void *)x;
262 }
263
264 /**
265 * @brief Translate a host virtual address to a host physical address before paging mode enabled.
266 *
267 * This function is used to translate a host virtual address to a host physical address before paging mode enabled. HVA
268 * is 1:1 mapping to HPA.
269 *
270 * It returns the host physical address that corresponds to the given host virtual address.
271 *
272 * @param[in] x The host virtual address to be translated
273 *
274 * @return The translated host physical address
275 *
276 * @retval 0 if x == NULL
277 *
278 * @pre N/A
279 *
280 * @post N/A
281 *
282 * @remark This function is used before paging mode enabled.
283 */
hva2hpa_early(void * x)284 static inline uint64_t hva2hpa_early(void *x)
285 {
286 return (uint64_t)x;
287 }
288
289 /**
290 * @brief Translate a host physical address to a host virtual address.
291 *
292 * This function is used to translate a host physical address to a host virtual address. HPA is 1:1 mapping to HVA.
293 *
294 * It returns the host virtual address that corresponds to the given host physical address.
295 *
296 * @param[in] x The host physical address to be translated.
297 *
298 * @return The translated host virtual address
299 *
300 * @retval NULL if x == 0
301 *
302 * @pre N/A
303 *
304 * @post N/A
305 *
306 * @remark This function is used after paging mode enabled.
307 */
hpa2hva(uint64_t x)308 static inline void *hpa2hva(uint64_t x)
309 {
310 return (void *)x;
311 }
312
313 /**
314 * @brief Translate a host virtual address to a host physical address.
315 *
316 * This function is used to translate a host virtual address to a host physical address. HVA is 1:1 mapping to HPA.
317 *
318 * It returns the host physical address that corresponds to the given host virtual address.
319 *
320 * @param[in] x The host virtual address to be translated.
321 *
322 * @return The translated host physical address.
323 *
324 * @retval 0 if x == NULL
325 *
326 * @pre N/A
327 *
328 * @post N/A
329 *
330 * @remark This function is used after paging mode enabled.
331 */
hva2hpa(const void * x)332 static inline uint64_t hva2hpa(const void *x)
333 {
334 return (uint64_t)x;
335 }
336
pml4e_index(uint64_t address)337 static inline uint64_t pml4e_index(uint64_t address)
338 {
339 return (address >> PML4E_SHIFT) & (PTRS_PER_PML4E - 1UL);
340 }
341
pdpte_index(uint64_t address)342 static inline uint64_t pdpte_index(uint64_t address)
343 {
344 return (address >> PDPTE_SHIFT) & (PTRS_PER_PDPTE - 1UL);
345 }
346
pde_index(uint64_t address)347 static inline uint64_t pde_index(uint64_t address)
348 {
349 return (address >> PDE_SHIFT) & (PTRS_PER_PDE - 1UL);
350 }
351
pte_index(uint64_t address)352 static inline uint64_t pte_index(uint64_t address)
353 {
354 return (address >> PTE_SHIFT) & (PTRS_PER_PTE - 1UL);
355 }
356
pml4e_page_vaddr(uint64_t pml4e)357 static inline uint64_t *pml4e_page_vaddr(uint64_t pml4e)
358 {
359 return hpa2hva(pml4e & PML4E_PFN_MASK);
360 }
361
pdpte_page_vaddr(uint64_t pdpte)362 static inline uint64_t *pdpte_page_vaddr(uint64_t pdpte)
363 {
364 return hpa2hva(pdpte & PDPTE_PFN_MASK);
365 }
366
pde_page_vaddr(uint64_t pde)367 static inline uint64_t *pde_page_vaddr(uint64_t pde)
368 {
369 return hpa2hva(pde & PDE_PFN_MASK);
370 }
371
372 /**
373 * @brief Calculate the page map level-4 table entry(PML4E) for a specified input address.
374 *
375 * The page map level-4 table(PML4T) contains 512 entries, each of which points to a page directory pointer table(PDPT).
376 * Address has the index to the PML4E in PML4T. This function is used to calculate the address of PML4E. It is typically
377 * used during the page translation process.
378 *
379 * It will return a pointer to the page map level-4 table entry(PML4E).
380 *
381 * @param[in] pml4_page A pointer to a page map level-4 table(PML4T) page.
382 * @param[in] addr The address value for which the page map level-4 table entry(PML4E) address is to be calculated.
383 * For hypervisor's MMU, it is the host virtual address.
384 * For each VM's EPT, it is the guest physical address.
385 *
386 * @return A pointer to the PML4E.
387 *
388 * @pre pml4_page != NULL
389 *
390 * @post N/A
391 */
pml4e_offset(uint64_t * pml4_page,uint64_t addr)392 static inline uint64_t *pml4e_offset(uint64_t *pml4_page, uint64_t addr)
393 {
394 return pml4_page + pml4e_index(addr);
395 }
396
397 /**
398 * @brief Calculate the page directory pointer table entry(PDPTE) for a specified input address.
399 *
400 * The page directory pointer table(PDPT) is referenced by a page map level-4 table entry(PML4E) and echo entry(PDPTE)
401 * in PDPT points to a page directory table(PDT). Address has the index to the PDPTE in PDPT. This function is used to
402 * calculate the address of PDPTE. It is typically used during the page translation process.
403 *
404 * It will return a pointer to the page directory pointer table entry(PDPTE).
405 *
406 * @param[in] pml4e A pointer to a page map level-4 table entry(PML4E).
407 * @param[in] addr The address for which the page directory pointer table entry(PDPTE) address is to be calculated.
408 * For hypervisor's MMU, it is the host virtual address.
409 * For each VM's EPT, it is the guest physical address.
410 *
411 * @return A pointer to the PDPTE.
412 *
413 * @pre pml4e != NULL
414 *
415 * @post N/A
416 */
pdpte_offset(const uint64_t * pml4e,uint64_t addr)417 static inline uint64_t *pdpte_offset(const uint64_t *pml4e, uint64_t addr)
418 {
419 return pml4e_page_vaddr(*pml4e) + pdpte_index(addr);
420 }
421
422 /**
423 * @brief Calculate the page directory table entry(PDE) for a specified input address.
424 *
425 * The page directory table(PDT) is referenced by a page directory pointer table entry(PDPTE) and echo entry(PDE) in PDT
426 * points to a page table(PT). Address has the index to the PDE in PDT. This function is used to calculate the address
427 * of PDE. It is typically used during the page translation process.
428 *
429 * It will return a pointer to the page directory table entry(PDE).
430 *
431 * @param[in] pdpte A pointer to a page directory pointer table entry(PDPTE).
432 * @param[in] addr The address for which the page directory table entry(PDE) address is to be calculated.
433 * For hypervisor's MMU, it is the host virtual address.
434 * For each VM's EPT, it is the guest physical address.
435 *
436 * @return A pointer to the PDE.
437 *
438 * @pre pdpte != NULL
439 *
440 * @post N/A
441 */
pde_offset(const uint64_t * pdpte,uint64_t addr)442 static inline uint64_t *pde_offset(const uint64_t *pdpte, uint64_t addr)
443 {
444 return pdpte_page_vaddr(*pdpte) + pde_index(addr);
445 }
446
447 /**
448 * @brief Calculate the page table entry(PTE) for a specified input address.
449 *
450 * The page table entry(PTE) is the entry that maps a page. This function is used to calculate the address of the PTE.
451 * It is typically used during the page translation process. The function is essential for managing memory access
452 * permissions and for implementing memory systems.
453 *
454 * It will return the address of a page table entry(PTE).
455 *
456 * @param[in] pde A pointer to a page directory entry(PDE).
457 * @param[in] addr The address for which the page table entry(PTE) address is to be calculated.
458 * For hypervisor's MMU, it is the host virtual address.
459 * For each VM's EPT, it is the guest physical address.
460 *
461 * @return A pointer to the page table entry(PTE).
462 *
463 * @pre pde != NULL
464 *
465 * @post N/A
466 */
pte_offset(const uint64_t * pde,uint64_t addr)467 static inline uint64_t *pte_offset(const uint64_t *pde, uint64_t addr)
468 {
469 return pde_page_vaddr(*pde) + pte_index(addr);
470 }
471
472 /*
473 * pgentry may means pml4e/pdpte/pde/pte
474 */
get_pgentry(const uint64_t * pte)475 static inline uint64_t get_pgentry(const uint64_t *pte)
476 {
477 return *pte;
478 }
479
480 /*
481 * pgentry may means pml4e/pdpte/pde/pte
482 */
set_pgentry(uint64_t * ptep,uint64_t pte,const struct pgtable * table)483 static inline void set_pgentry(uint64_t *ptep, uint64_t pte, const struct pgtable *table)
484 {
485 *ptep = pte;
486 table->clflush_pagewalk(ptep);
487 }
488
489 /**
490 * @brief Check whether the PS flag of the specified page directory table entry(PDE) is 1 or not.
491 *
492 * PS(Page Size) flag in PDE indicates whether maps a 2-MByte page or references a page table. This function checks this
493 * flag. This function is typically used in the context of setting up or modifying page tables where it's necessary to
494 * distinguish between large and regular page mappings.
495 *
496 * It returns the value that bit 7 is 1 if the specified PDE maps a 2-MByte page, or 0 if references a page table.
497 *
498 * @param[in] pde The page directory table entry(PDE) to check.
499 *
500 * @return The value of PS flag in the PDE.
501 *
502 * @retval PAGE_PSE indicating mapping to a 2-MByte page.
503 * @retval 0 indicating reference to a page table.
504 *
505 * @pre N/A
506 *
507 * @post N/A
508 */
pde_large(uint64_t pde)509 static inline uint64_t pde_large(uint64_t pde)
510 {
511 return pde & PAGE_PSE;
512 }
513
514 /**
515 * @brief Check whether the PS flag of the specified page directory pointer table entry(PDPTE) is 1 or not.
516 *
517 * PS(Page Size) flag in PDPTE indicates whether maps a 1-GByte page or references a page directory table. This function
518 * checks this flag. This function is typically used in the context of setting up or modifying page tables where it's
519 * necessary to distinguish between large and regular page mappings.
520 *
521 * It returns the value that bit 7 is 1 if the specified PDPTE maps a 1-GByte page, and 0 if references a page table.
522 *
523 * @param[in] pdpte The page directory pointer table entry(PDPTE) to check.
524 *
525 * @return The value of PS flag in the PDPTE.
526 *
527 * @retval PAGE_PSE indicating mapping to a 1-GByte page.
528 * @retval 0 indicating reference to a page directory table.
529 *
530 * @pre N/A
531 *
532 * @post N/A
533 */
pdpte_large(uint64_t pdpte)534 static inline uint64_t pdpte_large(uint64_t pdpte)
535 {
536 return pdpte & PAGE_PSE;
537 }
538
539 void init_sanitized_page(uint64_t *sanitized_page, uint64_t hpa);
540
541 void *pgtable_create_root(const struct pgtable *table);
542 void *pgtable_create_trusty_root(const struct pgtable *table,
543 void *nworld_pml4_page, uint64_t prot_table_present, uint64_t prot_clr);
544 /**
545 *@pre (pml4_page != NULL) && (pg_size != NULL)
546 */
547 const uint64_t *pgtable_lookup_entry(uint64_t *pml4_page, uint64_t addr,
548 uint64_t *pg_size, const struct pgtable *table);
549
550 void pgtable_add_map(uint64_t *pml4_page, uint64_t paddr_base,
551 uint64_t vaddr_base, uint64_t size,
552 uint64_t prot, const struct pgtable *table);
553 void pgtable_modify_or_del_map(uint64_t *pml4_page, uint64_t vaddr_base,
554 uint64_t size, uint64_t prot_set, uint64_t prot_clr,
555 const struct pgtable *table, uint32_t type);
556 #endif /* PGTABLE_H */
557
558 /**
559 * @}
560 */