1 // Copyright 2018 The Fuchsia Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "trampoline.h"
6
7 #include <cpuid.h>
8 #include <stdalign.h>
9 #include <stdbool.h>
10 #include <stdint.h>
11 #include <string.h>
12
13 #include <arch/x86/page_tables/constants.h>
14 #include <arch/x86/registers.h>
15
16 // Set up minimal page tables for 64-bit mode. These map the low
17 // 4G of address space directly to the low 4G of physical memory.
18
get_cr0(void)19 static uint32_t get_cr0(void) {
20 uint32_t cr0;
21 __asm__("mov %%cr0, %0" : "=r"(cr0));
22 return cr0;
23 }
24
set_cr0(uint32_t cr0)25 static void set_cr0(uint32_t cr0) {
26 __asm__ volatile("mov %0, %%cr0" :: "r"(cr0));
27 }
28
set_cr3(void * cr3)29 static void set_cr3(void* cr3) {
30 __asm__ volatile("mov %0, %%cr3" :: "r"(cr3) : "memory");
31 }
32
get_cr4(void)33 static uint32_t get_cr4(void) {
34 uint32_t cr4;
35 __asm__("mov %%cr4, %0" : "=r"(cr4));
36 return cr4;
37 }
38
set_cr4(uint32_t cr4)39 static void set_cr4(uint32_t cr4) {
40 __asm__ volatile("mov %0, %%cr4" :: "r"(cr4));
41 }
42
read_msr(uint32_t msr)43 static uint64_t read_msr(uint32_t msr) {
44 uint64_t value;
45 __asm__("rdmsr" : "=A"(value) : "c"(msr));
46 return value;
47 }
48
write_msr(uint32_t msr,uint64_t value)49 static void write_msr(uint32_t msr, uint64_t value) {
50 __asm__ volatile("wrmsr" :: "c"(msr), "A"(value));
51 }
52
53 static bool have_page1gb;
54
55 typedef union page_table {
56 alignas(4096) uint64_t pml4e[512];
57 alignas(4096) uint64_t pdpte[512];
58 alignas(4096) uint64_t pde[512];
59 } page_table_t;
60
61 static page_table_t* page_table_memory_start;
62 static page_table_t* page_table_memory_end;
63
get_page_table(void)64 static page_table_t* get_page_table(void) {
65 if (unlikely(page_table_memory_start >= page_table_memory_end)) {
66 panic("ran out of page table memory");
67 }
68 return page_table_memory_start++;
69 }
70
get_pdpte(unsigned int idx)71 static uint64_t get_pdpte(unsigned int idx) {
72 // Each PDPTE covers 1G.
73 if (have_page1gb) {
74 // A single entry direct-maps 1G.
75 return (((uint64_t)idx << 30) |
76 X86_MMU_PG_P | X86_MMU_PG_RW | X86_MMU_PG_PS);
77 } else {
78 // A single entry indirects through a PDE table.
79 page_table_t* pdpt = get_page_table();
80 for (size_t i = 0; i < 512; ++i) {
81 // Each PDE covers 2M.
82 pdpt->pde[i] = (((uint64_t)i << 21) |
83 X86_MMU_PG_P | X86_MMU_PG_RW | X86_MMU_PG_PS);
84 }
85 return ((uint64_t)(uintptr_t)pdpt | X86_MMU_PG_P | X86_MMU_PG_RW);
86 }
87 }
88
89 // The whole PDPT covers 512G, so we only need the one.
get_pml4e(void)90 static uint64_t get_pml4e(void) {
91 page_table_t* pdpt = get_page_table();
92 // Each PDPTE covers 1G, so we need four of those.
93 for (unsigned int i = 0; i < 4; ++i) {
94 pdpt->pdpte[i] = get_pdpte(i);
95 }
96 memset(&pdpt->pdpte[4], 0,
97 sizeof(*pdpt) - offsetof(page_table_t, pdpte[4]));
98 return (uint64_t)(uintptr_t)pdpt | X86_MMU_PG_P | X86_MMU_PG_RW;
99 }
100
get_pml4(void)101 static page_table_t* get_pml4(void) {
102 page_table_t* pml4 = get_page_table();
103 // The top-level PML4 just needs one PML4E to point to the PDPT.
104 pml4->pml4e[0] = get_pml4e();
105 memset(&pml4->pml4e[1], 0,
106 sizeof(*pml4) - offsetof(page_table_t, pml4e[1]));
107 return pml4;
108 }
109
enable_64bit_paging(uintptr_t start,uintptr_t end)110 void enable_64bit_paging(uintptr_t start, uintptr_t end) {
111 // Use available memory for page tables.
112 page_table_memory_start = (void*)((start + 4096 - 1) & -4096u);
113 page_table_memory_end = (void*)(end & -4096u);
114
115 // Determine if 1G pages are available.
116 {
117 uint32_t a, b, c, d;
118 __cpuid(0x80000001u, a, b, c, d);
119 have_page1gb = (d & (1u << 26)) != 0;
120 }
121
122 // Use the 64-bit (PAE) page table format.
123 // This is required in 64-bit (Long) mode.
124 set_cr4(get_cr4() | X86_CR4_PAE);
125
126 // Enable 64-bit (Long) mode.
127 write_msr(X86_MSR_IA32_EFER, read_msr(X86_MSR_IA32_EFER) | X86_EFER_LME);
128
129 // Install the page tables.
130 set_cr3(get_pml4());
131
132 // Enable paging.
133 // Hereafter we're using the direct-mapped page tables just built.
134 set_cr0(get_cr0() | X86_CR0_PG);
135 }
136