1/*
2 * Copyright (c) 2008-2012 Travis Geiselbrecht
3 *
4 * Use of this source code is governed by a MIT-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/MIT
7 */
8#include <lk/asm.h>
9#include <arch/ops.h>
10#include <arch/defines.h>
11
12.text
13
14#if ARM_WITH_CACHE
15
16/* low level cache routines for various cpu families */
17
18#if ARM_CPU_ARM1136
19
20/* void arch_disable_cache(uint flags) */
21FUNCTION(arch_disable_cache)
22    mov     r12, #0                     // zero register
23    mrs     r3, cpsr                    // save the old interrupt state
24    cpsid   iaf                         // interrupts disabled
25
26.Ldcache_disable:
27    tst     r0, #ARCH_CACHE_FLAG_DCACHE
28    beq     .Licache_disable
29    mrc     p15, 0, r1, c1, c0, 0       // cr1
30    tst     r1, #(1<<2)                 // is the dcache already disabled?
31    beq     .Licache_disable
32
33    bic     r1, #(1<<2)
34    mcr     p15, 0, r1, c1, c0, 0       // disable dcache
35
36    mcr     p15, 0, r12, c7, c14, 0     // clean & invalidate dcache
37    mcr     p15, 0, r0, c7, c10, 4      // data sync barrier (formerly drain write buffer)
38
39.Licache_disable:
40    tst     r0, #ARCH_CACHE_FLAG_ICACHE
41    beq     .Ldone_disable
42
43    mrc     p15, 0, r1, c1, c0, 0       // cr1
44    bic     r1, #(1<<12)
45    mcr     p15, 0, r1, c1, c0, 0       // disable icache
46
47    mcr     p15, 0, r12, c7, c5, 0      // invalidate icache
48
49.Ldone_disable:
50    msr     cpsr, r3
51    bx      lr
52
53/* void arch_enable_cache(uint flags) */
54FUNCTION(arch_enable_cache)
55    mov     r12, #0                     // zero register
56    mrs     r3, cpsr                    // save the old interrupt state
57    cpsid   iaf                         // interrupts disabled
58
59.Ldcache_enable:
60    tst     r0, #ARCH_CACHE_FLAG_DCACHE
61    beq     .Licache_enable
62    mrc     p15, 0, r1, c1, c0, 0       // cr1
63    tst     r1, #(1<<2)                 // is the dcache already enabled?
64    bne     .Licache_enable
65
66    mcr     p15, 0, r12, c7, c6, 0      // invalidate dcache
67
68    orr     r1, #(1<<2)
69    mcr     p15, 0, r1, c1, c0, 0       // enable dcache
70
71.Licache_enable:
72    tst     r0, #ARCH_CACHE_FLAG_ICACHE
73    beq     .Ldone_enable
74
75    mcr     p15, 0, r12, c7, c5, 0      // invalidate icache
76
77    mrc     p15, 0, r1, c1, c0, 0       // cr1
78    orr     r1, #(1<<12)
79    mcr     p15, 0, r1, c1, c0, 0       // enable icache
80
81.Ldone_enable:
82    msr     cpsr, r3
83    bx      lr
84
85#elif ARM_ISA_ARMV7
86
87/* void arch_disable_cache(uint flags) */
88FUNCTION(arch_disable_cache)
89    stmfd   sp!, {r4-r11, lr}
90
91    mov     r7, r0                      // save flags
92
93    mrs     r8, cpsr                    // save the old interrupt state
94    cpsid   iaf                         // interrupts disabled
95
96.Ldcache_disable:
97    tst     r7, #ARCH_CACHE_FLAG_DCACHE
98    beq     .Licache_disable
99    mrc     p15, 0, r0, c1, c0, 0       // cr1
100    tst     r0, #(1<<2)                 // is the dcache already disabled?
101    beq     .Ldcache_already_disabled
102
103    bic     r0, #(1<<2)
104    mcr     p15, 0, r0, c1, c0, 0       // disable dcache
105
106    // flush and invalidate the dcache
107    // NOTE: trashes a bunch of registers, can't be spilling stuff to the stack
108    bl      flush_invalidate_cache_v7
109
110    b       .Ldcache_disable_L2
111
112.Ldcache_already_disabled:
113    // make sure all of the caches are invalidated
114    // NOTE: trashes a bunch of registers, can't be spilling stuff to the stack
115    bl      invalidate_cache_v7
116
117.Ldcache_disable_L2:
118
119#if ARM_WITH_L2
120    // disable the L2, if present
121    mrc     p15, 0, r0, c1, c0, 1       // aux cr1
122    bic     r0, #(1<<1)
123    mcr     p15, 0, r0, c1, c0, 1       // disable L2 dcache
124#endif
125
126.Licache_disable:
127    tst     r7, #ARCH_CACHE_FLAG_ICACHE
128    beq     .Ldone_disable
129
130    mrc     p15, 0, r0, c1, c0, 0       // cr1
131    bic     r0, #(1<<12)
132    mcr     p15, 0, r0, c1, c0, 0       // disable icache
133
134.Ldone_disable:
135    // make sure the icache is always invalidated
136    mov     r0, #0
137    mcr     p15, 0, r0, c7, c5, 0       // invalidate icache to PoU
138
139    msr     cpsr, r8
140    ldmfd   sp!, {r4-r11, pc}
141
142/* void arch_enable_cache(uint flags) */
143FUNCTION(arch_enable_cache)
144    stmfd   sp!, {r4-r12, lr}
145
146    mov     r7, r0                      // save flags
147
148    mrs     r8, cpsr                    // save the old interrupt state
149    cpsid   iaf                         // interrupts disabled
150
151.Ldcache_enable:
152    tst     r7, #ARCH_CACHE_FLAG_DCACHE
153    beq     .Licache_enable
154    mrc     p15, 0, r0, c1, c0, 0       // cr1
155    tst     r0, #(1<<2)                 // is the dcache already enabled?
156    bne     .Licache_enable
157
158    // invalidate L1 and L2
159    // NOTE: trashes a bunch of registers, can't be spilling stuff to the stack
160    bl      invalidate_cache_v7
161
162#if ARM_WITH_L2
163    // enable the L2, if present
164    mrc     p15, 0, r0, c1, c0, 1       // aux cr1
165    orr     r0, #(1<<1)
166    mcr     p15, 0, r0, c1, c0, 1       // enable L2 dcache
167#endif
168
169    mrc     p15, 0, r0, c1, c0, 0       // cr1
170    orr     r0, #(1<<2)
171    mcr     p15, 0, r0, c1, c0, 0       // enable dcache
172
173.Licache_enable:
174    tst     r7, #ARCH_CACHE_FLAG_ICACHE
175    beq     .Ldone_enable
176
177    mov     r0, #0
178    mcr     p15, 0, r0, c7, c5, 0       // invalidate icache to PoU
179
180    mrc     p15, 0, r0, c1, c0, 0       // cr1
181    orr     r0, #(1<<12)
182    mcr     p15, 0, r0, c1, c0, 0       // enable icache
183
184.Ldone_enable:
185    isb
186    msr     cpsr, r8
187    ldmfd   sp!, {r4-r12, pc}
188
189// flush & invalidate cache routine, trashes r0-r6, r9-r11
190flush_invalidate_cache_v7:
191    /* from ARMv7 manual, B2-17 */
192    dmb
193    MRC     p15, 1, R0, c0, c0, 1       // Read CLIDR
194    ANDS    R3, R0, #0x7000000
195    MOV     R3, R3, LSR #23             // Cache level value (naturally aligned)
196    BEQ     .Lfinished
197    MOV     R10, #0
198.Loop1:
199    ADD     R2, R10, R10, LSR #1        // Work out 3xcachelevel
200    MOV     R1, R0, LSR R2              // bottom 3 bits are the Cache type for this level
201    AND     R1, R1, #7                  // get those 3 bits alone
202    CMP     R1, #2
203    BLT     .Lskip                      // no cache or only instruction cache at this level
204    MCR     p15, 2, R10, c0, c0, 0      // write the Cache Size selection register
205    isb                                 // ISB to sync the change to the CacheSizeID reg
206    MRC     p15, 1, R1, c0, c0, 0       // reads current Cache Size ID register
207    AND     R2, R1, #0x7                // extract the line length field
208    ADD     R2, R2, #4                  // add 4 for the line length offset (log2 16 bytes)
209    LDR     R4, =0x3FF
210    ANDS    R4, R4, R1, LSR #3          // R4 is the max number on the way size (right aligned)
211    CLZ     R5, R4                      // R5 is the bit position of the way size increment
212    LDR     R6, =0x00007FFF
213    ANDS    R6, R6, R1, LSR #13         // R6 is the max number of the index size (right aligned)
214.Loop2:
215    MOV     R9, R4                      // R9 working copy of the max way size (right aligned)
216.Loop3:
217    ORR     R11, R10, R9, LSL R5        // factor in the way number and cache number into R11
218    ORR     R11, R11, R6, LSL R2        // factor in the index number
219    MCR     p15, 0, R11, c7, c14, 2     // clean & invalidate by set/way
220    SUBS    R9, R9, #1                  // decrement the way number
221    BGE     .Loop3
222    SUBS    R6, R6, #1                  // decrement the index
223    BGE     .Loop2
224.Lskip:
225    ADD     R10, R10, #2                    // increment the cache number
226    CMP     R3, R10
227    BGT     .Loop1
228
229.Lfinished:
230    mov     r10, #0
231    mcr     p15, 2, r10, c0, c0, 0      // select cache level 0
232    dsb
233    isb
234
235    bx      lr
236
237// invalidate cache routine, trashes r0-r6, r9-r11
238invalidate_cache_v7:
239    /* from ARMv7 manual, B2-17 */
240    dmb
241    MRC     p15, 1, R0, c0, c0, 1       // Read CLIDR
242    ANDS    R3, R0, #0x7000000
243    MOV     R3, R3, LSR #23             // Cache level value (naturally aligned)
244    BEQ     .Lfinished_invalidate
245    MOV     R10, #0
246.Loop1_invalidate:
247    ADD     R2, R10, R10, LSR #1        // Work out 3xcachelevel
248    MOV     R1, R0, LSR R2              // bottom 3 bits are the Cache type for this level
249    AND     R1, R1, #7                  // get those 3 bits alone
250    CMP     R1, #2
251    BLT     .Lskip_invalidate           // no cache or only instruction cache at this level
252    MCR     p15, 2, R10, c0, c0, 0      // write the Cache Size selection register
253    isb                                 // ISB to sync the change to the CacheSizeID reg
254    MRC     p15, 1, R1, c0, c0, 0       // reads current Cache Size ID register
255    AND     R2, R1, #0x7                // extract the line length field
256    ADD     R2, R2, #4                  // add 4 for the line length offset (log2 16 bytes)
257    LDR     R4, =0x3FF
258    ANDS    R4, R4, R1, LSR #3          // R4 is the max number on the way size (right aligned)
259    CLZ     R5, R4                      // R5 is the bit position of the way size increment
260    LDR     R6, =0x00007FFF
261    ANDS    R6, R6, R1, LSR #13         // R6 is the max number of the index size (right aligned)
262.Loop2_invalidate:
263    MOV     R9, R4                      // R9 working copy of the max way size (right aligned)
264.Loop3_invalidate:
265    ORR     R11, R10, R9, LSL R5        // factor in the way number and cache number into R11
266    ORR     R11, R11, R6, LSL R2        // factor in the index number
267    MCR     p15, 0, R11, c7, c6, 2      // invalidate by set/way
268    SUBS    R9, R9, #1                  // decrement the way number
269    BGE     .Loop3_invalidate
270    SUBS    R6, R6, #1                  // decrement the index
271    BGE     .Loop2_invalidate
272.Lskip_invalidate:
273    ADD     R10, R10, #2                // increment the cache number
274    CMP     R3, R10
275    BGT     .Loop1_invalidate
276
277.Lfinished_invalidate:
278    dsb
279    mov     r10, #0
280    mcr     p15, 2, r10, c0, c0, 0      // select cache level 0
281    isb
282
283    bx      lr
284
285#else
286#error unhandled cpu
287#endif
288
289#if ARM_CPU_ARM926 || ARM_CPU_ARM1136 || ARM_ISA_ARMV7
290/* shared cache flush routines */
291
292    /* void arch_flush_cache_range(addr_t start, size_t len); */
293FUNCTION(arch_clean_cache_range)
294#if ARM_WITH_CP15
295    mov     r3, r0                      // save the start address
296    add     r2, r0, r1                  // calculate the end address
297    bic     r0, #(CACHE_LINE-1)         // align the start with a cache line
2980:
299    mcr     p15, 0, r0, c7, c10, 1      // clean cache to PoC by MVA
300    add     r0, #CACHE_LINE
301    cmp     r0, r2
302    blo     0b
303
304#if ARM_ISA_ARMV7
305    dsb
306#else
307    mov     r0, #0
308    mcr     p15, 0, r0, c7, c10, 4      // data sync barrier
309#endif
310#endif
311#if WITH_DEV_CACHE_PL310
312    mov     r0, r3                      // put the start address back
313    b       pl310_clean_range
314#else
315    bx      lr
316#endif
317
318    /* void arch_flush_invalidate_cache_range(addr_t start, size_t len); */
319FUNCTION(arch_clean_invalidate_cache_range)
320#if ARM_WITH_CP15
321    mov     r3, r0                      // save the start address
322    add     r2, r0, r1                  // calculate the end address
323    bic     r0, #(CACHE_LINE-1)         // align the start with a cache line
3240:
325    mcr     p15, 0, r0, c7, c14, 1      // clean & invalidate dcache to PoC by MVA
326    add     r0, r0, #CACHE_LINE
327    cmp     r0, r2
328    blo     0b
329
330#if ARM_ISA_ARMV7
331    dsb
332#else
333    mov     r0, #0
334    mcr     p15, 0, r0, c7, c10, 4      // data sync barrier
335#endif
336#endif
337#if WITH_DEV_CACHE_PL310
338    mov     r0, r3                      // put the start address back
339    b       pl310_clean_invalidate_range
340#else
341    bx      lr
342#endif
343
344    /* void arch_invalidate_cache_range(addr_t start, size_t len); */
345FUNCTION(arch_invalidate_cache_range)
346#if ARM_WITH_CP15
347    mov     r3, r0                      // save the start address
348    add     r2, r0, r1                  // calculate the end address
349    bic     r0, #(CACHE_LINE-1)         // align the start with a cache line
3500:
351    mcr     p15, 0, r0, c7, c6, 1       // invalidate dcache to PoC by MVA
352    add     r0, r0, #CACHE_LINE
353    cmp     r0, r2
354    blo     0b
355
356#if ARM_ISA_ARMV7
357    dsb
358#else
359    mov     r0, #0
360    mcr     p15, 0, r0, c7, c10, 4      // data sync barrier
361#endif
362#endif
363#if WITH_DEV_CACHE_PL310
364    mov     r0, r3                      // put the start address back
365    b       pl310_invalidate_range
366#else
367    bx      lr
368#endif
369
370    /* void arch_sync_cache_range(addr_t start, size_t len); */
371FUNCTION(arch_sync_cache_range)
372    push    { r14 }
373    bl      arch_clean_cache_range
374
375    mov     r0, #0
376    mcr     p15, 0, r0, c7, c5, 0       // invalidate icache to PoU
377
378    pop     { pc }
379
380#endif // ARM_CPU_...
381
382#else
383
384/* no cache */
385
386FUNCTION(arch_disable_cache)
387    bx      lr
388
389FUNCTION(arch_enable_cache)
390    bx      lr
391
392FUNCTION(arch_clean_cache_range)
393    bx      lr
394
395FUNCTION(arch_clean_invalidate_cache_range)
396    bx      lr
397
398FUNCTION(arch_sync_cache_range)
399    bx      lr
400
401#endif // ARM_WITH_CACHE
402