1// Copyright 2016 The Fuchsia Authors
2// Copyright (c) 2009 Corey Tabaka
3// Copyright (c) 2015 Intel Corporation
4// Copyright (c) 2016 Travis Geiselbrecht
5//
6// Use of this source code is governed by a MIT-style
7// license that can be found in the LICENSE file or at
8// https://opensource.org/licenses/MIT
9
10#include <asm.h>
11#include <arch/x86/asm.h>
12#include <arch/x86/descriptor.h>
13#include <arch/x86/mmu.h>
14#include <arch/x86/registers.h>
15#include <zircon/tls.h>
16
17#define ADDR_OFFSET_MASK ((1 << ADDR_OFFSET)-1)
18#define SHIFT_OFFSET(_s) ((_s) >> 3)
19#define SHIFT_REMAIN(_s) ((_s) - (SHIFT_OFFSET(_s) << 3))
20
21// Set a page table entry for the kernel module relocated 64-bit virtual
22// address in 32-bit code. Clobbers the %ecx register.
23.macro set_relocated_page_table_entry table, shift, value
24    // Extract 32-bit chunk of kernel_relocated_base containing the index bits
25    // for this page level shift.
26    mov PHYS(kernel_relocated_base + SHIFT_OFFSET(\shift)), %ecx
27
28    // Get the exact portion of the 32-bit value that is the index
29    shrl $SHIFT_REMAIN(\shift), %ecx
30    andl $ADDR_OFFSET_MASK, %ecx
31
32    // Get the address on the page table of index * 8 and set the value
33    shll $3, %ecx
34    addl $PHYS(\table), %ecx
35    movl \value, (%ecx)
36.endm
37
38// This section name is known specially to kernel.ld and gen-kaslr-fixups.sh.
39// This code has relocations for absolute physical addresses, which do not get
40// adjusted by the boot-time fixups (which this code calls at the end).
41.section .text.boot, "ax", @progbits
42.align 8
43FUNCTION_LABEL(_start)
44    /* set up a temporary stack pointer */
45    mov $PHYS(_kstack_end), %rsp
46
47    // Save off the bootdata pointer in a register that won't get clobbered.
48    mov %esi, %ebx
49
50    // The fixup code in image.S runs in 64-bit mode with paging enabled,
51    // so we can't run it too early.  But it overlaps the bss, so we move
52    // it before zeroing the bss.  We can't delay zeroing the bss because
53    // the page tables we're about to set up are themselves in bss.
54
55    // The first word after the kernel image (at __data_end in our view)
56    // gives the size of the following code.  Copy it to _end.
57    mov PHYS(__data_end), %ecx
58    mov $PHYS(__data_end+4), %esi
59    mov $PHYS(_end), %edi
60    rep movsb // while (ecx-- > 0) *edi++ = *esi++;
61
62    // Now it's safe to zero the bss.
63    movl $PHYS(__bss_start), %edi
64    movl $PHYS(_end), %ecx
65    sub %edi, %ecx              // Compute the length of the bss in bytes.
66    xor %eax, %eax
67    rep stosb // while (ecx-- > 0) *edi++ = al;
68
69    // _zbi_base is in bss, so now it's safe to set it.
70    mov %ebx, PHYS(_zbi_base)
71
72    /* give the boot allocator a chance to compute the physical address of the kernel */
73    call boot_alloc_init
74
75.Lpaging_setup64:
76    /* initialize the default page tables */
77    /* Setting the First PML4E with a PDP table reference*/
78    movl $PHYS(pdp), %eax
79    orl  $X86_KERNEL_PD_FLAGS, %eax
80    movl %eax, PHYS(pml4)
81
82    /* Setting the First PDPTE with a Page table reference*/
83    movl $PHYS(pte), %eax
84    orl  $X86_KERNEL_PD_FLAGS, %eax
85    movl %eax, PHYS(pdp)
86
87    /* point the pml4e at the second high PDP (for -2GB mapping) */
88    movl $PHYS(pdp_high),   %eax
89    orl  $X86_KERNEL_PD_FLAGS, %eax
90    set_relocated_page_table_entry pml4, PML4_SHIFT, %eax
91
92    /* point the second pdp at the same low level page table */
93    movl $PHYS(pte), %eax
94    orl  $X86_KERNEL_PD_FLAGS, %eax
95    set_relocated_page_table_entry pdp_high, PDP_SHIFT, %eax
96
97    /* map the first 1GB in this table */
98    movl $PHYS(pte), %esi
99    movl $0x200, %ecx
100    xor  %eax, %eax
101
1020:
103    mov  %eax, %ebx
104    shll $21, %ebx
105    orl  $X86_KERNEL_PD_LP_FLAGS, %ebx
106    movl %ebx, (%esi)
107    addl $8,%esi
108    inc  %eax
109    loop 0b
110
111    /* set up a linear map of the first 64GB at 0xffffff8000000000 */
112    movl $PHYS(linear_map_pdp), %esi
113    movl $32768, %ecx
114    xor  %eax, %eax
115
116    /* loop across these page tables, incrementing the address by 2MB */
1170:
118    mov  %eax, %ebx
119    shll $21, %ebx
120    orl  $X86_KERNEL_PD_LP_FLAGS, %ebx    // lower word of the entry
121    movl %ebx, (%esi)
122    mov  %eax, %ebx
123    shrl $11, %ebx      // upper word of the entry
124    movl %ebx, 4(%esi)
125    addl $8,%esi
126    inc  %eax
127    loop 0b
128
129    /* point the high pdp at our linear mapping page tables */
130    movl $PHYS(pdp_high), %esi
131    movl $64, %ecx
132    movl $PHYS(linear_map_pdp), %eax
133    orl  $X86_KERNEL_PD_FLAGS, %eax
134
1350:
136    movl %eax, (%esi)
137    add  $8, %esi
138    addl $4096, %eax
139    loop 0b
140
141    /*
142     * Set PGE to enable global kernel pages
143     */
144    mov   %cr4, %rax
145    or    $(X86_CR4_PGE), %rax
146    mov   %rax, %cr4
147
148    /* load the physical pointer to the top level page table */
149    mov  $PHYS(pml4), %rax
150    mov  %rax, %cr3
151
152    // Load our new GDT by physical pointer.
153    // _temp_gdtr has it as a virtual pointer, so copy it and adjust.
154    movw PHYS(_temp_gdtr), %ax
155    movl PHYS(_temp_gdtr+2), %ecx
156    sub $PHYS_ADDR_DELTA, %ecx
157    movw %ax, -6(%esp)
158    movl %ecx, -4(%esp)
159    lgdt -6(%esp)
160
161    /* long jump to our code selector and the high address relocated */
162    push  $CODE_64_SELECTOR
163    mov  $PHYS(high_entry), %rax
164    addq PHYS(kernel_relocated_base), %rax
165    pushq %rax
166    lretq
167
168// This code runs at the final virtual address, so it should be pure PIC.
169.text
170high_entry:
171    /* zero our kernel segment data registers */
172    xor %eax, %eax
173    mov %eax, %ds
174    mov %eax, %es
175    mov %eax, %fs
176    mov %eax, %gs
177    mov %eax, %ss
178
179    /* load the high kernel stack */
180    lea _kstack_end(%rip), %rsp
181
182    // move_fixups_and_zero_bss copied the fixup code to _end.
183    // It expects %rdi to contain the actual runtime address of __code_start.
184    lea __code_start(%rip), %rdi
185    call _end
186    // The fixup code won't be used again, so the memory can be reused now.
187
188    /* reload the gdtr after relocations as it relies on relocated VAs */
189    lgdt _temp_gdtr(%rip)
190
191    // Set %gs.base to &bp_percpu.  It's statically initialized
192    // with kernel_unsafe_sp set, so after this it's safe to call
193    // into C code that might use safe-stack and/or stack-protector.
194    lea bp_percpu(%rip), %rax
195    mov %rax, %rdx
196    shr $32, %rdx
197    mov $X86_MSR_IA32_GS_BASE, %ecx
198    wrmsr
199
200    /* set up the idt */
201    lea _idt_startup(%rip), %rdi
202    call idt_setup
203    call load_startup_idt
204
205    /* assign this core CPU# 0 and initialize its per cpu state */
206    xor %edi, %edi
207    call x86_init_percpu
208
209    // Fill the stack canary with a random value as early as possible.
210    // This isn't done in x86_init_percpu because the hw_rng_get_entropy
211    // call would make it eligible for stack-guard checking itself.  But
212    // %gs is not set up yet in the prologue of the function, so it would
213    // crash if it tried to use the stack-guard.
214    call choose_stack_guard
215
216    // Move it into place.
217    mov %rcx, %gs:ZX_TLS_STACK_GUARD_OFFSET
218    // Don't leak that value to other code.
219    xor %ecx, %ecx
220
221    // configure the kernel base address
222    // TODO: dynamically figure this out once we allow the x86 kernel to be loaded anywhere
223    movl $PHYS_LOAD_ADDRESS, kernel_base_phys(%rip)
224
225    /* call the main module */
226    call lk_main
227
2280:                          /* just sit around waiting for interrupts */
229    hlt                     /* interrupts will unhalt the processor */
230    pause
231    jmp 0b                  /* so jump back to halt to conserve power */
232
233.bss
234.align 16
235DATA(_kstack)
236    .skip 4096
237DATA(_kstack_end)
238
239// These symbols are used by image.S
240.global IMAGE_ELF_ENTRY
241IMAGE_ELF_ENTRY = _start
242
243// This symbol is used by gdb python to know the base of the kernel module
244.global KERNEL_BASE_ADDRESS
245KERNEL_BASE_ADDRESS = KERNEL_BASE - KERNEL_LOAD_OFFSET
246