1/*
2 * Copyright (c) 2017, Linaro Limited. All rights reserved.
3 * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
4 *
5 * SPDX-License-Identifier: BSD-3-Clause
6 */
7
8#include <arm.h>
9#include <asm.S>
10
11.macro  dcache_line_size  reg, tmp
12	mrs     \tmp, ctr_el0
13	ubfx    \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH
14	mov     \reg, #CTR_WORD_SIZE
15	lsl     \reg, \reg, \tmp
16.endm
17
18.macro  icache_line_size  reg, tmp
19	mrs     \tmp, ctr_el0
20	and     \tmp, \tmp, #CTR_IMINLINE_MASK
21	mov     \reg, #CTR_WORD_SIZE
22	lsl     \reg, \reg, \tmp
23.endm
24
25/*
26 * This macro can be used for implementing various data cache operations `op`
27 */
28.macro do_dcache_maintenance_by_mva op
29	dcache_line_size x2, x3
30	add	x1, x0, x1
31	sub	x3, x2, #1
32	bic	x0, x0, x3
33loop_\op:
34	dc	\op, x0
35	add	x0, x0, x2
36	cmp	x0, x1
37	b.lo    loop_\op
38	dsb	sy
39	ret
40.endm
41	/* ------------------------------------------
42	 * Clean+Invalidate from base address till
43	 * size. 'x0' = addr, 'x1' = size
44	 * ------------------------------------------
45	 */
46FUNC dcache_cleaninv_range , :
47	do_dcache_maintenance_by_mva civac
48END_FUNC dcache_cleaninv_range
49
50	/* ------------------------------------------
51	 * Clean from base address till size.
52	 * 'x0' = addr, 'x1' = size
53	 * ------------------------------------------
54	 */
55FUNC dcache_clean_range , :
56	do_dcache_maintenance_by_mva cvac
57END_FUNC dcache_clean_range
58
59	/* ------------------------------------------
60	 * Invalidate from base address till
61	 * size. 'x0' = addr, 'x1' = size
62	 * ------------------------------------------
63	 */
64FUNC dcache_inv_range , :
65	do_dcache_maintenance_by_mva ivac
66END_FUNC dcache_inv_range
67
68	/* ------------------------------------------
69	 * Clean from base address till size to point of unification
70	 * 'x0' = addr, 'x1' = size
71	 * ------------------------------------------
72	 */
73FUNC dcache_clean_range_pou , :
74	do_dcache_maintenance_by_mva cvau
75END_FUNC dcache_clean_range_pou
76
77	/* ---------------------------------------------------------------
78	 * Data cache operations by set/way to the level specified
79	 *
80	 * The main function, do_dcsw_op requires:
81	 * x0: The operation type (0-2), as defined in cache_helpers.h
82	 * x3: The last cache level to operate on
83	 * x9: clidr_el1
84	 * x10: The cache level to begin operation from
85	 * and will carry out the operation on each data cache from level 0
86	 * to the level in x3 in sequence
87	 *
88	 * The dcsw_op macro sets up the x3 and x9 parameters based on
89	 * clidr_el1 cache information before invoking the main function
90	 * ---------------------------------------------------------------
91	 */
92
93	.macro	dcsw_op shift, fw, ls
94	mrs	x9, clidr_el1
95	ubfx	x3, x9, \shift, \fw
96	lsl	x3, x3, \ls
97	mov	x10, xzr
98	b	do_dcsw_op
99	.endm
100
101LOCAL_FUNC do_dcsw_op , :
102	cbz	x3, exit
103	adr	x14, dcsw_loop_table	// compute inner loop address
104	add	x14, x14, x0, lsl #5	// inner loop is 8x32-bit instructions
105BTI(	add	x14, x14, x0, lsl #2)	// inner loop is + "bti j" instruction
106	mov	x0, x9
107	mov	w8, #1
108loop1:
109	add	x2, x10, x10, lsr #1	// work out 3x current cache level
110	lsr	x1, x0, x2		// extract cache type bits from clidr
111	and	x1, x1, #7		// mask the bits for current cache only
112	cmp	x1, #2			// see what cache we have at this level
113	b.lo	level_done		// nothing to do if no cache or icache
114
115	msr	csselr_el1, x10		// select current cache level in csselr
116	isb				// isb to sych the new cssr&csidr
117	mrs	x1, ccsidr_el1		// read the new ccsidr
118	and	x2, x1, #7		// extract the length of the cache lines
119	add	x2, x2, #4		// add 4 (line length offset)
120	ubfx	x4, x1, #3, #10		// maximum way number
121	clz	w5, w4			// bit position of way size increment
122	lsl	w9, w4, w5		// w9 = aligned max way number
123	lsl	w16, w8, w5		// w16 = way number loop decrement
124	orr	w9, w10, w9		// w9 = combine way and cache number
125	ubfx	w6, w1, #13, #15	// w6 = max set number
126	lsl	w17, w8, w2		// w17 = set number loop decrement
127	dsb	sy			// barrier before we start this level
128	br	x14			// jump to DC operation specific loop
129
130	.macro	dcsw_loop _op
131BTI(	bti	j)
132loop2_\_op:
133	lsl	w7, w6, w2		// w7 = aligned max set number
134
135loop3_\_op:
136	orr	w11, w9, w7		// combine cache, way and set number
137	dc	\_op, x11
138	subs	w7, w7, w17		// decrement set number
139	b.hs	loop3_\_op
140
141	subs	x9, x9, x16		// decrement way number
142	b.hs	loop2_\_op
143
144	b	level_done
145	.endm
146
147level_done:
148	add	x10, x10, #2		// increment cache number
149	cmp	x3, x10
150	b.hi    loop1
151	msr	csselr_el1, xzr		// select cache level 0 in csselr
152	dsb	sy			// barrier to complete final cache operation
153	isb
154exit:
155	ret
156
157dcsw_loop_table:
158	dcsw_loop isw
159	dcsw_loop cisw
160	dcsw_loop csw
161END_FUNC do_dcsw_op
162
163FUNC dcache_op_louis , :
164	dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
165END_FUNC dcache_op_louis
166
167FUNC dcache_op_all , :
168	dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
169END_FUNC dcache_op_all
170
171	/* ---------------------------------------------------------------
172	 *  Helper macro for data cache operations by set/way for the
173	 *  level specified
174	 * ---------------------------------------------------------------
175	 */
176	.macro dcsw_op_level level
177	mrs	x9, clidr_el1
178	mov	x3, \level
179	sub	x10, x3, #2
180	b	do_dcsw_op
181	.endm
182
183	/* ---------------------------------------------------------------
184	 * Data cache operations by set/way for level 1 cache
185	 *
186	 * The main function, do_dcsw_op requires:
187	 * x0: The operation type (0-2), as defined in cache_helpers.h
188	 * ---------------------------------------------------------------
189	 */
190FUNC dcache_op_level1 , :
191	dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT)
192END_FUNC dcache_op_level1
193
194	/* ---------------------------------------------------------------
195	 * Data cache operations by set/way for level 2 cache
196	 *
197	 * The main function, do_dcsw_op requires:
198	 * x0: The operation type (0-2), as defined in cache_helpers.h
199	 * ---------------------------------------------------------------
200	 */
201FUNC dcache_op_level2 , :
202	dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT)
203END_FUNC dcache_op_level2
204
205	/* ---------------------------------------------------------------
206	 * Data cache operations by set/way for level 3 cache
207	 *
208	 * The main function, do_dcsw_op requires:
209	 * x0: The operation type (0-2), as defined in cache_helpers.h
210	 * ---------------------------------------------------------------
211	 */
212FUNC dcache_op_level3 , :
213	dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT)
214END_FUNC dcache_op_level3
215
216FUNC icache_inv_all , :
217	/* Invalidate Entire Instruction Cache */
218	ic	ialluis
219
220	dsb	ish	/* ensure that maintenance operations are seen */
221	isb		/* by the instructions rigth after the isb */
222
223	ret
224END_FUNC icache_inv_all
225
226FUNC icache_inv_range , :
227	icache_line_size x2, x3
228	add	x1, x0, x1
229	sub	x3, x2, #1
230	bic	x0, x0, x3
231loop_ic_inv:
232	ic	ivau, x0
233	add	x0, x0, x2
234	cmp	x0, x1
235	b.lo    loop_ic_inv
236	dsb	ish
237	ret
238
239END_FUNC icache_inv_range
240
241BTI(emit_aarch64_feature_1_and     GNU_PROPERTY_AARCH64_FEATURE_1_BTI)
242