1/*
2 * Copyright (c) 2008-2015 Travis Geiselbrecht
3 *
4 * Use of this source code is governed by a MIT-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/MIT
7 */
8#include <lk/asm.h>
9#include <arch/arm/cores.h>
10#include <arch/arm/mmu.h>
11
12#if WITH_KERNEL_VM
13#include <kernel/vm.h>
14#endif
15
16.section ".text.boot"
17.globl _start
18_start:
19    b   platform_reset
20    b   arm_undefined
21    b   arm_syscall
22    b   arm_prefetch_abort
23    b   arm_data_abort
24    b   arm_reserved
25    b   arm_irq
26    b   arm_fiq
27#if WITH_SMP
28    b   arm_reset
29#endif
30
31.weak platform_reset
32platform_reset:
33    /* Fall through for the weak symbol */
34
35.globl arm_reset
36arm_reset:
37#if ARM_WITH_HYP
38    /* if in HYP mode, move to SVC */
39    mrs r12, cpsr
40    and r12, r12, #0x1f
41    cmp r12, #0x1a
42    bleq arm32_hyp_to_svc
43#endif // ARM_WITH_HYP
44
45    /* do some early cpu setup */
46    mrc     p15, 0, r12, c1, c0, 0
47    /* i/d cache disable, mmu disabled */
48    bic     r12, #(1<<12)
49    bic     r12, #(1<<2 | 1<<0)
50#if WITH_KERNEL_VM
51    /* enable caches so atomics and spinlocks work */
52    orr     r12, r12, #(1<<12)
53    orr     r12, r12, #(1<<2)
54#endif // WITH_KERNEL_VM
55    mcr     p15, 0, r12, c1, c0, 0
56
57    /* calculate the physical offset from our eventual virtual location */
58.Lphys_offset:
59    ldr     r4, =.Lphys_offset
60    adr     r11, .Lphys_offset
61    sub     r11, r11, r4
62
63#if WITH_SMP
64    /* figure out our cpu number */
65    mrc     p15, 0, r12, c0, c0, 5 /* read MPIDR */
66
67    /* mask off the bottom bits to test cluster number:cpu number */
68    ubfx    r12, r12, #0, #SMP_CPU_ID_BITS
69
70    /* if we're not cpu 0:0, fall into a trap and wait */
71    teq     r12, #0
72    movne   r0, r12
73    bne     arm_secondary_setup
74#endif // WITH_SMP
75
76#if WITH_CPU_EARLY_INIT
77    /* call platform/arch/etc specific init code */
78    bl      __cpu_early_init
79#endif // WITH_CPU_EARLY_INIT
80
81#if WITH_NO_PHYS_RELOCATION
82    /* assume that image is properly loaded in physical memory */
83#else
84    /* see if we need to relocate to our proper location in physical memory */
85    adr     r4, _start                           /* this emits sub r4, pc, #constant */
86    ldr     r5, =(MEMBASE + KERNEL_LOAD_OFFSET)  /* calculate the binary's physical load address */
87    subs    r12, r4, r5                          /* calculate the delta between where we're loaded and the proper spot */
88    beq     .Lrelocate_done
89
90    /* we need to relocate ourselves to the proper spot */
91    ldr     r6, =__data_end
92    ldr     r7, =(KERNEL_BASE - MEMBASE)
93    sub     r6, r7
94    add     r6, r12
95
96.Lrelocate_loop:
97    ldr     r7, [r4], #4
98    str     r7, [r5], #4
99    cmp     r4, r6
100    bne     .Lrelocate_loop
101
102    /* we're relocated, jump to the right address */
103    sub     pc, r12
104    nop     /* skipped in the add to pc */
105
106    /* recalculate the physical offset */
107    sub     r11, r11, r12
108
109.Lrelocate_done:
110#endif // !WITH_NO_PHYS_RELOCATION
111
112#if ARCH_HAS_MMU
113.Lsetup_mmu:
114
115    /* set up the mmu according to mmu_initial_mappings */
116
117    /* load the base of the translation table and clear the table */
118    ldr     r4, =arm_kernel_translation_table
119    add     r4, r4, r11
120        /* r4 = physical address of translation table */
121
122    mov     r5, #0
123    mov     r6, #0
124
125    /* walk through all the entries in the translation table, setting them up */
1260:
127    str     r5, [r4, r6, lsl #2]
128    add     r6, #1
129    cmp     r6, #4096
130    bne     0b
131
132    /* load the address of the mmu_initial_mappings table and start processing */
133    ldr     r5, =mmu_initial_mappings
134    add     r5, r5, r11
135        /* r5 = physical address of mmu initial mapping table */
136
137.Linitial_mapping_loop:
138    ldmia   r5!, { r6-r10 }
139        /* r6 = phys, r7 = virt, r8 = size, r9 = flags, r10 = name */
140
141    /* round size up to 1MB alignment */
142    ubfx        r10, r6, #0, #20
143    add     r8, r8, r10
144    add     r8, r8, #(1 << 20)
145    sub     r8, r8, #1
146
147    /* mask all the addresses and sizes to 1MB boundaries */
148    lsr     r6, #20  /* r6 = physical address / 1MB */
149    lsr     r7, #20  /* r7 = virtual address / 1MB */
150    lsr     r8, #20  /* r8 = size in 1MB chunks */
151
152    /* if size == 0, end of list */
153    cmp     r8, #0
154    beq     .Linitial_mapping_done
155
156    /* set up the flags */
157    ldr     r10, =MMU_KERNEL_L1_PTE_FLAGS
158    teq     r9, #MMU_INITIAL_MAPPING_FLAG_UNCACHED
159    ldreq   r10, =MMU_INITIAL_MAP_STRONGLY_ORDERED
160    beq     0f
161    teq     r9, #MMU_INITIAL_MAPPING_FLAG_DEVICE
162    ldreq   r10, =MMU_INITIAL_MAP_DEVICE
163        /* r10 = mmu entry flags */
164
1650:
166    orr     r12, r10, r6, lsl #20
167        /* r12 = phys addr | flags */
168
169    /* store into appropriate translation table entry */
170    str     r12, [r4, r7, lsl #2]
171
172    /* loop until we're done */
173    add     r6, #1
174    add     r7, #1
175    subs    r8, #1
176    bne     0b
177
178    b       .Linitial_mapping_loop
179
180.Linitial_mapping_done:
181
182#if MMU_WITH_TRAMPOLINE
183    /* move arm_kernel_translation_table address to r8 and
184     * set cacheable attributes on translation walk
185     */
186    orr     r8, r4, #MMU_TTBRx_FLAGS
187
188    /* Prepare tt_trampoline page table */
189    /* Calculate pagetable physical addresses */
190    ldr     r4, =tt_trampoline  /* r4 = tt_trampoline vaddr */
191    add     r4, r4, r11     /* r4 = tt_trampoline paddr */
192
193    /* Zero tt_trampoline translation tables */
194    mov     r6, #0
195    mov     r7, #0
1961:
197    str     r7, [r4, r6, lsl#2]
198    add     r6, #1
199    cmp     r6, #0x1000
200    blt     1b
201
202    /* Setup 1M section mapping at
203     * phys  -> phys   and
204     * virt  -> phys
205     */
206    lsr     r6, pc, #20     /* r6 = paddr index */
207    ldr     r7, =MMU_KERNEL_L1_PTE_FLAGS
208    add     r7, r7, r6, lsl #20 /* r7 = pt entry */
209
210    str     r7, [r4, r6, lsl #2]    /* tt_trampoline[paddr index] = pt entry */
211
212    rsb     r6, r11, r6, lsl #20    /* r6 = vaddr */
213    str     r7, [r4, r6, lsr #(20 - 2)] /* tt_trampoline[vaddr index] = pt entry */
214#endif // MMU_WITH_TRAMPOLINE
215
216    /* set up the mmu */
217    bl      .Lmmu_setup
218#endif // ARCH_HAS_MMU
219
220    /* at this point we're running at our final location in virtual memory (if enabled) */
221.Lstack_setup:
222    /* set up the stack for irq, fiq, abort, undefined, system/user, and lastly supervisor mode */
223    mov     r12, #0
224
225    cpsid   i,#0x12       /* irq */
226    mov     sp, r12
227
228    cpsid   i,#0x11       /* fiq */
229    mov     sp, r12
230
231    cpsid   i,#0x17       /* abort */
232    mov     sp, r12
233
234    cpsid   i,#0x1b       /* undefined */
235    mov     sp, r12
236
237    cpsid   i,#0x1f       /* system */
238    mov     sp, r12
239
240    cpsid   i,#0x13       /* supervisor */
241    ldr     r12, =abort_stack
242    add     r12, #ARCH_DEFAULT_STACK_SIZE
243    mov     sp, r12
244
245    /* stay in supervisor mode from now on out */
246
247    /* copy the initialized data segment out of rom if necessary */
248    ldr     r4, =__data_start_rom
249    ldr     r5, =__data_start
250    ldr     r6, =__data_end
251
252    cmp     r4, r5
253    beq     .L__do_bss
254
255.L__copy_loop:
256    cmp     r5, r6
257    ldrlt   r7, [r4], #4
258    strlt   r7, [r5], #4
259    blt     .L__copy_loop
260
261.L__do_bss:
262    /* clear out the bss */
263    ldr     r4, =__bss_start
264    ldr     r5, =_end
265    mov     r6, #0
266.L__bss_loop:
267    cmp     r4, r5
268    strlt   r6, [r4], #4
269    blt     .L__bss_loop
270
271    bl      lk_main
272    b       .
273
274#if WITH_KERNEL_VM
275    /* per cpu mmu setup, shared between primary and secondary cpus
276       args:
277       r4 == translation table physical
278       r8 == final translation table physical (if using trampoline)
279    */
280.Lmmu_setup:
281    /* Invalidate TLB. The value in r0 is ignored */
282    mcr     p15, 0, r0, c8, c7, 0
283    dsb     sy
284    isb
285
286    /* Write 0 to TTBCR */
287    mov     r12, #0
288    mcr     p15, 0, r12, c2, c0, 2
289    isb
290
291    /* Set cacheable attributes on translation walk */
292    orr     r12, r4, #MMU_TTBRx_FLAGS
293
294    /* Write ttbr with phys addr of the translation table */
295    mcr     p15, 0, r12, c2, c0, 0  // TTBR0
296    isb
297
298    /* Write DACR */
299    mov     r12, #0x1
300    mcr     p15, 0, r12, c3, c0, 0
301    isb
302
303    /* Read SCTLR into r12 */
304    mrc     p15, 0, r12, c1, c0, 0
305
306    /* Disable TRE/AFE */
307    bic     r12, #(1<<29 | 1<<28)
308
309    /* Turn on the MMU */
310    orr     r12, #0x1
311
312    /* Write back SCTLR */
313    mcr     p15, 0, r12, c1, c0, 0
314    isb
315
316    /* Jump to virtual code address */
317    ldr     pc, =1f
3181:
319
320#if MMU_WITH_TRAMPOLINE
321    /* Switch to main page table */
322    mcr     p15, 0, r8, c2, c0, 0
323    isb
324#endif // MMU_WITH_TRAMPOLINE
325
326    /* Invalidate TLB. The value in r0 is ignored */
327    mcr     p15, 0, r0, c8, c7, 0
328    dsb     sy
329    isb
330
331    /* assume lr was in physical memory, adjust it before returning */
332    sub     lr, r11
333    bx      lr
334#endif // WITH_KERNEL_VM
335
336#if WITH_SMP
337    /* secondary cpu entry point */
338    /* r0 holds cpu number */
339    /* r11 hold phys offset */
340FUNCTION(arm_secondary_setup)
341    /* all other cpus, trap and wait to be released */
3421:
343    wfe
344    ldr     r12, =arm_boot_cpu_lock
345    add     r12, r12, r11
346    ldr     r12, [r12]
347    cmp     r12, #0
348    bne     1b
349
350    and     r1, r0, #0xff
351    cmp     r1, #(1 << SMP_CPU_CLUSTER_SHIFT)
352    bge     unsupported_cpu_trap
353    bic     r0, r0, #0xff
354    orr     r0, r1, r0, LSR #(8 - SMP_CPU_CLUSTER_SHIFT)
355
356    cmp     r0, #SMP_MAX_CPUS
357    bge     unsupported_cpu_trap
358    mov     r5, r0 /* save cpu num */
359
360    /* set up the stack for irq, fiq, abort, undefined, system/user, and lastly supervisor mode */
361    mov     r1, #0
362    cpsid   i,#0x12       /* irq */
363    mov     sp, r1
364
365    cpsid   i,#0x11       /* fiq */
366    mov     sp, r1
367
368    cpsid   i,#0x17       /* abort */
369    mov     sp, r1
370
371    cpsid   i,#0x1b       /* undefined */
372    mov     sp, r1
373
374    cpsid   i,#0x1f       /* system */
375    mov     sp, r1
376
377    cpsid   i,#0x13       /* supervisor */
378    ldr     r1, =abort_stack
379    mov     r2, #ARCH_DEFAULT_STACK_SIZE
380    add     r0, #1
381    mul     r2, r2, r0
382    add     r1, r2
383
384    mov     sp, r1
385
386#if WITH_KERNEL_VM
387    /* load the physical base of the translation table and clear the table */
388    ldr     r4, =arm_kernel_translation_table
389    add     r4, r4, r11
390
391#if MMU_WITH_TRAMPOLINE
392    /* move arm_kernel_translation_table address to r8 and
393     * set cacheable attributes on translation walk
394     */
395    orr     r8, r4, #MMU_TTBRx_FLAGS
396
397    /* Prepare tt_trampoline page table */
398    /* Calculate pagetable physical addresses */
399    ldr     r4, =tt_trampoline  /* r4 = tt_trampoline vaddr */
400    add     r4, r4, r11     /* r4 = tt_trampoline paddr */
401#endif // MMU_WITH_TRAMPOLINE
402
403    /* set up the mmu on this cpu and switch to virtual memory */
404    bl      .Lmmu_setup
405#endif // WITH_KERNEL_VM
406
407    /* stay in supervisor and call into arm arch code to continue setup */
408    mov     r0, r5
409    bl      arm_secondary_entry
410
411    /* cpus above the number we claim to support get trapped here */
412unsupported_cpu_trap:
413    wfe
414    b       unsupported_cpu_trap
415#endif // WITH_SMP
416
417#if ARM_WITH_HYP
418arm32_hyp_to_svc:
419    mrs r12, cpsr
420    bic r12, #0x1f      // clear mode bits
421    orr r12, r12, #0x13 // set mode bits to SVC
422    msr SPSR_hyp, r12
423    msr ELR_hyp, lr
424    eret                // "restore" the mode, and return
425#endif // ARM_WITH_HYP
426
427.ltorg
428
429#if WITH_KERNEL_VM && MMU_WITH_TRAMPOLINE
430.section ".bss.prebss.translation_table"
431.align 14
432DATA(tt_trampoline)
433    .skip 16384
434#endif // WITH_KERNEL_VM && MMU_WITH_TRAMPOLINE
435
436.data
437.align 2
438