1/*
2 * Copyright (c) 2017, Linaro Limited. All rights reserved.
3 * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved.
4 *
5 * SPDX-License-Identifier: BSD-3-Clause
6 */
7
8#include <arm.h>
9#include <arm32_macros.S>
10#include <asm.S>
11#include <kernel/cache_helpers.h>
12
13/*
14 * Cache line size helpers
15 */
16.macro  dcache_line_size  reg, tmp
17	read_ctr \tmp
18	ubfx    \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH
19	mov     \reg, #CTR_WORD_SIZE
20	lsl     \reg, \reg, \tmp
21.endm
22
23.macro  icache_line_size  reg, tmp
24	read_ctr \tmp
25	and     \tmp, \tmp, #CTR_IMINLINE_MASK
26	mov     \reg, #CTR_WORD_SIZE
27	lsl     \reg, \reg, \tmp
28.endm
29
30/*
31 * This macro can be used for implementing various data cache operations `op`
32 */
33.macro do_dcache_maintenance_by_mva reg
34	dcache_line_size r2, r3
35	add	r1, r0, r1
36	sub	r3, r2, #1
37	bic	r0, r0, r3
38loop_\reg:
39	write_\reg	r0
40	add	r0, r0, r2
41	cmp	r0, r1
42	blo	loop_\reg
43	dsb	sy
44	bx	lr
45.endm
46
47	/* ------------------------------------------
48	 * Clean+Invalidate from base address till
49	 * size. 'r0' = addr, 'r1' = size
50	 * ------------------------------------------
51	 */
52FUNC dcache_cleaninv_range , :
53	do_dcache_maintenance_by_mva dccimvac
54END_FUNC dcache_cleaninv_range
55
56	/* ------------------------------------------
57	 * Clean from base address till size.
58	 * 'r0' = addr, 'r1' = size
59	 * ------------------------------------------
60	 */
61FUNC dcache_clean_range , :
62	do_dcache_maintenance_by_mva dccmvac
63END_FUNC dcache_clean_range
64
65	/* ------------------------------------------
66	 * Invalidate from base address till
67	 * size. 'r0' = addr, 'r1' = size
68	 * ------------------------------------------
69	 */
70FUNC dcache_inv_range , :
71	do_dcache_maintenance_by_mva dcimvac
72END_FUNC dcache_inv_range
73
74
75	/* ------------------------------------------
76	 * Clean from base address till size to point of unification
77	 * 'r0' = addr, 'r1' = size
78	 * ------------------------------------------
79	 */
80FUNC dcache_clean_range_pou , :
81	do_dcache_maintenance_by_mva dccmvau
82END_FUNC dcache_clean_range_pou
83
84	/* ----------------------------------------------------------------
85	 * Data cache operations by set/way to the level specified
86	 *
87	 * The main function, do_dcsw_op requires:
88	 * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV,
89	 *     DCACHE_OP_CLEAN), as defined in cache_helpers.h
90	 * r1: The cache level to begin operation from
91	 * r2: clidr_el1
92	 * r3: The last cache level to operate on
93	 * and will carry out the operation on each data cache from level 0
94	 * to the level in r3 in sequence
95	 *
96	 * The dcsw_op macro sets up the r2 and r3 parameters based on
97	 * clidr_el1 cache information before invoking the main function
98	 * ----------------------------------------------------------------
99	 */
100
101	.macro	dcsw_op shift, fw, ls
102	read_clidr r2
103	ubfx	r3, r2, \shift, \fw
104	lsl	r3, r3, \ls
105	mov	r1, #0
106	b	do_dcsw_op
107	.endm
108
109LOCAL_FUNC do_dcsw_op , :
110	push	{r4-r12,lr}
111	adr	r11, dcsw_loop_table	// compute cache op based on the operation type
112	add	r6, r11, r0, lsl #3	// cache op is 2x32-bit instructions
113loop1:
114	add	r10, r1, r1, LSR #1	// Work out 3x current cache level
115	mov	r12, r2, LSR r10	// extract cache type bits from clidr
116	and	r12, r12, #7   		// mask the bits for current cache only
117	cmp	r12, #2			// see what cache we have at this level
118	blo	level_done      	// no cache or only instruction cache at this level
119
120	write_csselr r1			// select current cache level in csselr
121	isb				// isb to sych the new cssr&csidr
122	read_ccsidr r12			// read the new ccsidr
123	and	r10, r12, #7   		// extract the length of the cache lines
124	add	r10, r10, #4        	// add 4 (r10 = line length offset)
125	ubfx	r4, r12, #3, #10	// r4 = maximum way number (right aligned)
126	clz	r5, r4            	// r5 = the bit position of the way size increment
127	mov	r9, r4			// r9 working copy of the aligned max way number
128
129loop2:
130	ubfx	r7, r12, #13, #15	// r7 = max set number (right aligned)
131
132loop3:
133	orr	r0, r1, r9, LSL r5	// factor in the way number and cache level into r0
134	orr	r0, r0, r7, LSL r10	// factor in the set number
135
136	blx	r6
137	subs	r7, r7, #1              // decrement the set number
138	bhs	loop3
139	subs	r9, r9, #1              // decrement the way number
140	bhs	loop2
141level_done:
142	add	r1, r1, #2		// increment the cache number
143	cmp	r3, r1
144	dsb	sy			// ensure completion of previous cache maintenance instruction
145	bhi	loop1
146
147	mov	r6, #0
148	write_csselr r6			//select cache level 0 in csselr
149	dsb	sy
150	isb
151	pop	{r4-r12,pc}
152
153dcsw_loop_table:
154	write_dcisw r0
155	bx	lr
156	write_dccisw r0
157	bx	lr
158	write_dccsw r0
159	bx	lr
160END_FUNC do_dcsw_op
161
162	/* ---------------------------------------------------------------
163	 * Data cache operations by set/way till PoU.
164	 *
165	 * The function requires :
166	 * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV,
167	 * DCACHE_OP_CLEAN), as defined in cache_helpers.h
168	 * ---------------------------------------------------------------
169	 */
170FUNC dcache_op_louis , :
171	dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
172END_FUNC dcache_op_louis
173
174	/* ---------------------------------------------------------------
175	 * Data cache operations by set/way till PoC.
176	 *
177	 * The function requires :
178	 * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV,
179	 * DCACHE_OP_CLEAN), as defined in cache_helpers.h
180	 * ---------------------------------------------------------------
181	 */
182FUNC dcache_op_all , :
183	dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
184END_FUNC dcache_op_all
185
186
187	/* ---------------------------------------------------------------
188	 *  Helper macro for data cache operations by set/way for the
189	 *  level specified
190	 * ---------------------------------------------------------------
191	 */
192	.macro	dcsw_op_level level
193	read_clidr r2
194	mov	r3, \level
195	sub	r1, r3, #2
196	b	do_dcsw_op
197	.endm
198
199	/* ---------------------------------------------------------------
200	 * Data cache operations by set/way for level 1 cache
201	 *
202	 * The main function, do_dcsw_op requires:
203	 * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV,
204	 * DCACHE_OP_CLEAN), as defined in cache_helpers.h
205	 * ---------------------------------------------------------------
206	 */
207FUNC dcache_op_level1 , :
208	dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT)
209END_FUNC dcache_op_level1
210
211	/* ---------------------------------------------------------------
212	 * Data cache operations by set/way for level 2 cache
213	 *
214	 * The main function, do_dcsw_op requires:
215	 * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV,
216	 * DCACHE_OP_CLEAN), as defined in cache_helpers.h
217	 * ---------------------------------------------------------------
218	 */
219FUNC dcache_op_level2 , :
220	dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT)
221END_FUNC dcache_op_level2
222
223	/* ---------------------------------------------------------------
224	 * Data cache operations by set/way for level 3 cache
225	 *
226	 * The main function, do_dcsw_op requires:
227	 * r0: The operation type (DCACHE_OP_INV, DCACHE_OP_CLEAN_INV,
228	 * DCACHE_OP_CLEAN), as defined in cache_helpers.h
229	 * ---------------------------------------------------------------
230	 */
231FUNC dcache_op_level3 , :
232	dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT)
233END_FUNC dcache_op_level3
234
235FUNC icache_inv_all , :
236	/* Invalidate Entire Instruction Cache (and branch predictors) */
237	write_icialluis
238
239	dsb	ishst	/* ensure that maintenance operations are seen */
240	isb		/* by the instructions rigth after the isb */
241
242	bx      lr
243END_FUNC icache_inv_all
244
245	/* ------------------------------------------
246	 * Invalidate from base address till
247	 * size. 'r0' = addr, 'r1' = size
248	 * ------------------------------------------
249	 */
250FUNC icache_inv_range , :
251	icache_line_size r2, r3
252	add	r1, r0, r1
253	sub	r3, r2, #1
254	bic	r0, r0, r3
255loop_ic_inv:
256	write_icimvau r0
257	add	r0, r0, r2
258	cmp	r0, r1
259	blo	loop_ic_inv
260
261	/* Invalidate entire branch predictor array inner shareable */
262	write_bpiallis
263
264	dsb	ishst
265	isb
266
267	bx	lr
268END_FUNC icache_inv_range
269