1#include <lk/asm.h>
2#include <arch/arm64/mmu.h>
3#include <arch/asm_macros.h>
4#include <kernel/vm.h>
5
6/*
7 * Register use:
8 *  x0-x3   Arguments
9 *  x9-x15  Scratch
10 *  x19-x28 Globals
11 */
12tmp                     .req x9
13tmp2                    .req x10
14wtmp2                   .req w10
15idx                     .req x11
16idx_shift               .req x12
17page_table              .req x13
18new_page_table          .req x14
19phys_offset             .req x15
20
21cpuid                   .req x19
22page_table1             .req x21
23mmu_initial_mapping     .req x22
24vaddr                   .req x23
25paddr                   .req x24
26mapping_size            .req x25
27size                    .req x26
28attr                    .req x27
29boot_el                 .req x28
30
31.section .text.boot
32FUNCTION(_start)
33.globl arm_reset
34arm_reset:
35    /* keep track of the boot EL */
36    mrs     boot_el, currentel
37
38    /* if we came in at higher than EL1, drop down to EL1 */
39    bl      arm64_elX_to_el1
40
41    /* disable EL1 FPU traps */
42    mov     tmp, #(0b11<<20)
43    msr     cpacr_el1, tmp
44
45    /* enable caches so atomics and spinlocks work */
46    mrs     tmp, sctlr_el1
47    orr     tmp, tmp, #(1<<12) /* Enable icache */
48    orr     tmp, tmp, #(1<<2)  /* Enable dcache/ucache */
49    orr     tmp, tmp, #(1<<3)  /* Enable Stack Alignment Check EL1 */
50    orr     tmp, tmp, #(1<<4)  /* Enable Stack Alignment Check EL0 */
51    bic     tmp, tmp, #(1<<1)  /* Disable Alignment Checking for EL1 EL0 */
52    msr     sctlr_el1, tmp
53
54    /* make sure SP_ELx is being used */
55    msr     spsel, #1
56
57#if WITH_KERNEL_VM
58    /* set up the mmu according to mmu_initial_mappings */
59
60    /* load the base of the translation table and clear the table */
61    adrp    page_table1, arm64_kernel_translation_table
62    add     page_table1, page_table1, #:lo12:arm64_kernel_translation_table
63#endif /* WITH_KERNEL_VM */
64
65#if WITH_SMP
66    /* if the cpu id is != 0 it's a secondary cpu */
67    mrs     cpuid, mpidr_el1
68    ubfx    cpuid, cpuid, #0, #SMP_CPU_ID_BITS
69
70#if WITH_KERNEL_VM
71    cbnz    cpuid, .Lmmu_enable_secondary
72#else
73    cbnz    cpuid, .Lsecondary_boot
74#endif
75#endif /* WITH_SMP */
76
77    /* this path forward until .Lmmu_enable_secondary or .Lsecondary_boot is the primary cpu only */
78
79    /* save a copy of the boot args so x0-x3 are available for use */
80    adrp    tmp, arm64_boot_args
81    add     tmp, tmp, :lo12:arm64_boot_args
82    stp     x0, x1, [tmp], #16
83    stp     x2, x3, [tmp]
84
85    /* save the boot EL */
86    adrp    tmp, arm64_boot_el
87    str     boot_el, [tmp, #:lo12:arm64_boot_el]
88
89#if WITH_KERNEL_VM
90    /* walk through all the entries in the translation table, setting them up */
91    mov     tmp, #0
92.Lclear_top_page_table_loop:
93    str     xzr, [page_table1, tmp, lsl #3]
94    add     tmp, tmp, #1
95    cmp     tmp, #MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP
96    bne     .Lclear_top_page_table_loop
97
98    /* load the address of the mmu_initial_mappings table and start processing */
99    adrp    mmu_initial_mapping, mmu_initial_mappings
100    add     mmu_initial_mapping, mmu_initial_mapping, #:lo12:mmu_initial_mappings
101
102.Linitial_mapping_loop:
103/* Read entry of mmu_initial_mappings (likely defined in platform.c) */
104    ldp     paddr, vaddr, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_PHYS_OFFSET]
105    ldp     size, tmp, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_SIZE_OFFSET]
106
107    tbzmask tmp, MMU_INITIAL_MAPPING_FLAG_DYNAMIC, .Lnot_dynamic
108    adr     paddr, _start
109    mov     size, x0 /* use the arg passed through from platform_reset */
110    str     paddr, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_PHYS_OFFSET]
111    str     size, [mmu_initial_mapping, #__MMU_INITIAL_MAPPING_SIZE_OFFSET]
112
113.Lnot_dynamic:
114    /* if size == 0, end of list, done with initial mapping */
115    cbz     size, .Linitial_mapping_done
116    mov     mapping_size, size
117
118    /* set up the flags */
119    tbzmask tmp, MMU_INITIAL_MAPPING_FLAG_UNCACHED, .Lnot_uncached
120    ldr     attr, =MMU_INITIAL_MAP_STRONGLY_ORDERED
121    b       .Lmem_type_done
122
123.Lnot_uncached:
124    /* is this memory mapped to device/peripherals? */
125    tbzmask tmp, MMU_INITIAL_MAPPING_FLAG_DEVICE, .Lnot_device
126    ldr     attr, =MMU_INITIAL_MAP_DEVICE
127    b       .Lmem_type_done
128.Lnot_device:
129
130/* Determine the segment in which the memory resides and set appropriate
131 *  attributes.  In order to handle offset kernels, the following rules are
132 *  implemented below:
133 *      KERNEL_BASE    to __code_start             -read/write (see note below)
134 *      __code_start   to __rodata_start (.text)   -read only
135 *      __rodata_start to __data_start   (.rodata) -read only, execute never
136 *      __data_start   to .....          (.data)   -read/write
137 *
138 *  The space below __code_start is presently left as read/write (same as .data)
139 *   mainly as a workaround for the raspberry pi boot process.  Boot vectors for
140 *   secondary CPUs are in this area and need to be updated by cpu0 once the system
141 *   is ready to boot the secondary processors.
142 *   TODO: handle this via mmu_initial_mapping entries, which may need to be
143 *         extended with additional flag types
144 */
145.Lmapping_size_loop:
146    ldr     attr, =MMU_PTE_KERNEL_DATA_FLAGS
147    ldr     tmp, =__code_start
148    subs    size, tmp, vaddr
149    /* If page is below  the entry point (_start) mark as kernel data */
150    b.hi    .Lmem_type_done
151
152    ldr     attr, =MMU_PTE_KERNEL_RO_FLAGS
153    ldr     tmp, =__rodata_start
154    subs    size, tmp, vaddr
155    b.hi    .Lmem_type_done
156    orr     attr, attr, #MMU_PTE_ATTR_PXN
157    ldr     tmp, =__data_start
158    subs    size, tmp, vaddr
159    b.hi    .Lmem_type_done
160    ldr     attr, =MMU_PTE_KERNEL_DATA_FLAGS
161    ldr     tmp, =_end
162    subs    size, tmp, vaddr
163    b.lo    . /* Error: _end < vaddr */
164    cmp     mapping_size, size
165    b.lo    . /* Error: mapping_size < size => RAM size too small for data/bss */
166    mov     size, mapping_size
167
168.Lmem_type_done:
169    subs    mapping_size, mapping_size, size
170    b.lo    . /* Error: mapping_size < size (RAM size too small for code/rodata?) */
171
172    /* Check that paddr, vaddr and size are page aligned */
173    orr     tmp, vaddr, paddr
174    orr     tmp, tmp, size
175    tst     tmp, #(1 << MMU_KERNEL_PAGE_SIZE_SHIFT) - 1
176    bne     . /* Error: not page aligned */
177
178    /* Clear top bits of virtual address (should be all set) */
179    eor     vaddr, vaddr, #(~0 << MMU_KERNEL_SIZE_SHIFT)
180
181    /* Check that top bits were all set */
182    tst     vaddr, #(~0 << MMU_KERNEL_SIZE_SHIFT)
183    bne     . /* Error: vaddr out of range */
184
185.Lmap_range_top_loop:
186    /* Select top level page table */
187    mov     page_table, page_table1
188    mov     idx_shift, #MMU_KERNEL_TOP_SHIFT
189
190    lsr     idx, vaddr, idx_shift
191
192
193/* determine the type of page table entry to use given alignment and size
194 *  of the chunk of memory we are mapping
195 */
196.Lmap_range_one_table_loop:
197    /* Check if current level allow block descriptors */
198    cmp     idx_shift, #MMU_PTE_DESCRIPTOR_BLOCK_MAX_SHIFT
199    b.hi    .Lmap_range_need_page_table
200
201    /* Check if paddr and vaddr alignment allows a block descriptor */
202    orr     tmp2, vaddr, paddr
203    lsr     tmp, tmp2, idx_shift
204    lsl     tmp, tmp, idx_shift
205    cmp     tmp, tmp2
206    b.ne    .Lmap_range_need_page_table
207
208    /* Check if size is large enough for a block mapping */
209    lsr     tmp, size, idx_shift
210    cbz     tmp, .Lmap_range_need_page_table
211
212    /* Select descriptor type, page for level 3, block for level 0-2 */
213    orr     tmp, attr, #MMU_PTE_L3_DESCRIPTOR_PAGE
214    cmp     idx_shift, MMU_KERNEL_PAGE_SIZE_SHIFT
215    beq     .Lmap_range_l3
216    orr     tmp, attr, #MMU_PTE_L012_DESCRIPTOR_BLOCK
217.Lmap_range_l3:
218
219    /* Write page table entry */
220    orr     tmp, tmp, paddr
221    str     tmp, [page_table, idx, lsl #3]
222
223    /* Move to next page table entry */
224    mov     tmp, #1
225    lsl     tmp, tmp, idx_shift
226    add     vaddr, vaddr, tmp
227    add     paddr, paddr, tmp
228    subs    size, size, tmp
229    /* TODO: add local loop if next entry is in the same page table */
230    b.ne    .Lmap_range_top_loop /* size != 0 */
231
232    /* Restore top bits of virtual address (should be all set) */
233    eor     vaddr, vaddr, #(~0 << MMU_KERNEL_SIZE_SHIFT)
234    /* Move to next subtype of ram mmu_initial_mappings entry */
235    cbnz     mapping_size, .Lmapping_size_loop
236
237    /* Move to next mmu_initial_mappings entry */
238    add     mmu_initial_mapping, mmu_initial_mapping, __MMU_INITIAL_MAPPING_SIZE
239    b       .Linitial_mapping_loop
240
241.Lmap_range_need_page_table:
242    /* Check if page table entry is unused */
243    ldr     new_page_table, [page_table, idx, lsl #3]
244    cbnz    new_page_table, .Lmap_range_has_page_table
245
246    /* Calculate phys offset (needed for memory allocation) */
247.Lphys_offset:
248    adr     phys_offset, .Lphys_offset /* phys */
249    ldr     tmp, =.Lphys_offset /* virt */
250    sub     phys_offset, tmp, phys_offset
251
252    /* Allocate new page table */
253    calloc_bootmem_aligned new_page_table, tmp, tmp2, MMU_KERNEL_PAGE_SIZE_SHIFT, phys_offset
254
255    /* Write page table entry (with allocated page table) */
256    orr     new_page_table, new_page_table, #MMU_PTE_L012_DESCRIPTOR_TABLE
257    str     new_page_table, [page_table, idx, lsl #3]
258
259.Lmap_range_has_page_table:
260    /* Check descriptor type */
261    and     tmp, new_page_table, #MMU_PTE_DESCRIPTOR_MASK
262    cmp     tmp, #MMU_PTE_L012_DESCRIPTOR_TABLE
263    b.ne    . /* Error: entry already in use (as a block entry) */
264
265    /* switch to next page table level */
266    bic     page_table, new_page_table, #MMU_PTE_DESCRIPTOR_MASK
267    mov     tmp, #~0
268    lsl     tmp, tmp, idx_shift
269    bic     tmp, vaddr, tmp
270    sub     idx_shift, idx_shift, #(MMU_KERNEL_PAGE_SIZE_SHIFT - 3)
271    lsr     idx, tmp, idx_shift
272
273    b       .Lmap_range_one_table_loop
274
275.Linitial_mapping_done:
276    /* compute the base TCR configuration and save away in a global for future use */
277    ldr     tmp, =MMU_TCR_FLAGS_BASE
278
279    /* Set TCR_EL1.IPS to ID_AA64MMFR0_EL1.PARange */
280    mrs     tmp2, id_aa64mmfr0_el1
281    and     tmp2, tmp2, #0xf
282    /*
283     * Give up if we see a reserved value. 52-bit PAs have a different translation
284     * table format that we don't support, so use 48-bit PAs in that case.
285     */
286    cmp     tmp2, #6
287    b.hi    .
288    b.lo    1f
289    mov     tmp2, #5
2901:
291    orr     tmp, tmp, tmp2, lsl #32
292    adrp    tmp2, arm64_mmu_tcr_flags
293    str     tmp, [tmp2, #:lo12:arm64_mmu_tcr_flags]
294
295#if WITH_SMP
296    adrp    tmp, page_tables_not_ready
297    add     tmp, tmp, #:lo12:page_tables_not_ready
298    str     wzr, [tmp]
299    b       .Lpage_tables_ready
300
301.Lmmu_enable_secondary:
302    adrp    tmp, page_tables_not_ready
303    add     tmp, tmp, #:lo12:page_tables_not_ready
304.Lpage_tables_not_ready:
305    ldr     wtmp2, [tmp]
306    cbnz    wtmp2, .Lpage_tables_not_ready
307.Lpage_tables_ready:
308#endif
309
310    /* set up the mmu */
311
312    /* Invalidate TLB */
313    tlbi    vmalle1is
314    dsb     sy
315    isb
316
317    /* Initialize Memory Attribute Indirection Register */
318    ldr     tmp, =MMU_MAIR_VAL
319    msr     mair_el1, tmp
320
321    /* Initialize TCR_EL1 */
322    /* set cacheable attributes on translation walk */
323    /* (SMP extensions) non-shareable, inner write-back write-allocate */
324    adrp    tmp, arm64_mmu_tcr_flags
325    ldr     tmp, [tmp, #:lo12:arm64_mmu_tcr_flags]
326    orr     tmp, tmp, #MMU_TCR_FLAGS_KERNEL
327    msr     tcr_el1, tmp
328
329    isb
330
331    /* Write ttbr with phys addr of the translation table */
332    msr     ttbr0_el1, xzr
333    msr     ttbr1_el1, page_table1
334    isb
335
336    /* Set VBAR to the virtual address of the trampoline VBAR */
337    ldr     tmp, =trampoline_vbar
338    msr     vbar_el1, tmp
339    isb
340
341    /* Read SCTLR */
342    mrs     tmp, sctlr_el1
343
344    /* Turn on the MMU */
345    orr     tmp, tmp, #0x1
346
347    /*
348     * Write back SCTLR. This instruction will cause an exception when fetching
349     * the following instruction, as the PC will contain an unmapped physical
350     * address. This will be handled by the trampoline VBAR which will branch
351     * to that instruction's virtual address.
352     */
353    msr     sctlr_el1, tmp
354.Lmmu_on_pc:
355    isb
356
357    /* Disable the trampoline VBAR */
358    msr     vbar_el1, xzr
359    isb
360
361    /* Invalidate TLB */
362    tlbi    vmalle1
363    dsb     sy
364    isb
365
366#if WITH_SMP
367    cbnz    cpuid, .Lsecondary_boot
368#endif
369#endif /* WITH_KERNEL_VM */
370
371    /* load the stack pointer */
372    ldr tmp, =__stack_end
373    mov sp, tmp
374
375    /* clear bss */
376.L__do_bss:
377    /* clear out the bss excluding the stack and kernel translation table  */
378    /* NOTE: relies on __post_prebss_bss_start and __bss_end being 8 byte aligned */
379    ldr     tmp, =__post_prebss_bss_start
380    ldr     tmp2, =__bss_end
381    sub     tmp2, tmp2, tmp
382    cbz     tmp2, .L__bss_loop_done
383.L__bss_loop:
384    sub     tmp2, tmp2, #8
385    str     xzr, [tmp], #8
386    cbnz    tmp2, .L__bss_loop
387.L__bss_loop_done:
388
389    /* load the boot args we had saved previously */
390    adrp    tmp, arm64_boot_args
391    add     tmp, tmp, :lo12:arm64_boot_args
392    ldp     x0, x1, [tmp], #16
393    ldp     x2, x3, [tmp]
394
395    bl  lk_main
396    b   .
397
398#if WITH_SMP
399.Lsecondary_boot:
400    and     tmp, cpuid, #0xff
401    cmp     tmp, #(1 << SMP_CPU_CLUSTER_SHIFT)
402    bge     .Lunsupported_cpu_trap
403    bic     cpuid, cpuid, #0xff
404    orr     cpuid, tmp, cpuid, LSR #(8 - SMP_CPU_CLUSTER_SHIFT)
405
406    cmp     cpuid, #SMP_MAX_CPUS
407    bge     .Lunsupported_cpu_trap
408
409    /* Set up the stack */
410    ldr     tmp, =__stack_end
411    mov     tmp2, #ARCH_DEFAULT_STACK_SIZE
412    mul     tmp2, tmp2, cpuid
413    sub     sp, tmp, tmp2
414
415    mov     x0, cpuid
416    bl      arm64_secondary_entry
417
418.Lunsupported_cpu_trap:
419    wfe
420    b       .Lunsupported_cpu_trap
421#endif
422
423.ltorg
424
425#if WITH_KERNEL_VM
426.section .text.boot.vectab
427/*
428 * The only type of exception that we expect with the trampoline VBAR active is
429 * sync to current EL. All other exceptions result in infinite loops.
430 */
431LOCAL_FUNCTION(trampoline_vbar)
432.p2align 11
433.org 0x00
434    wfe
435    b       .-4
436
437.org 0x80
438    wfe
439    b       .-4
440
441.org 0x100
442    wfe
443    b       .-4
444
445.org 0x180
446    wfe
447    b       .-4
448
449    /* exception vector for synchronous exceptions from current EL -> current EL */
450.org 0x200
451    b       .Lmmu_on_pc
452
453.org 0x280
454    wfe
455    b       .-4
456
457.org 0x300
458    wfe
459    b       .-4
460
461.org 0x380
462    wfe
463    b       .-4
464
465.org 0x400
466    wfe
467    b       .-4
468
469.org 0x480
470    wfe
471    b       .-4
472
473.org 0x500
474    wfe
475    b       .-4
476
477.org 0x580
478    wfe
479    b       .-4
480
481.org 0x600
482    wfe
483    b       .-4
484
485.org 0x680
486    wfe
487    b       .-4
488
489.org 0x700
490    wfe
491    b       .-4
492
493.org 0x780
494    wfe
495    b       .-4
496END_FUNCTION(trampoline_vbar)
497#endif
498
499.data
500    .balign 8
501LOCAL_DATA(arm64_boot_args)
502    .skip (4 * 8)
503END_DATA(arm64_boot_args)
504DATA(arm64_boot_el)
505    .skip 8
506END_DATA(arm64_boot_el)
507
508#if WITH_SMP
509.data
510DATA(page_tables_not_ready)
511    .long       1
512END_DATA(page_tables_not_ready)
513#endif
514
515.section .bss.prebss.stack
516    .align 4
517DATA(__stack)
518    .skip ARCH_DEFAULT_STACK_SIZE * SMP_MAX_CPUS
519END_DATA(__stack)
520DATA(__stack_end)
521