1/*
2 * Copyright (c) 2019-2022, ARM Limited. All rights reserved.
3 * Copyright (c) 2021-2022, NVIDIA Corporation. All rights reserved.
4 *
5 * SPDX-License-Identifier: BSD-3-Clause
6 */
7
8#include <arch.h>
9#include <asm_macros.S>
10#include <common/bl_common.h>
11#include <cortex_a78_ae.h>
12#include <cpu_macros.S>
13#include <plat_macros.S>
14#include "wa_cve_2022_23960_bhb_vector.S"
15
16/* Hardware handled coherency */
17#if HW_ASSISTED_COHERENCY == 0
18#error "cortex_a78_ae must be compiled with HW_ASSISTED_COHERENCY enabled"
19#endif
20
21#if WORKAROUND_CVE_2022_23960
22	wa_cve_2022_23960_bhb_vector_table CORTEX_A78_AE_BHB_LOOP_COUNT, cortex_a78_ae
23#endif /* WORKAROUND_CVE_2022_23960 */
24
25/* --------------------------------------------------
26 * Errata Workaround for A78 AE Erratum 1941500.
27 * This applies to revisions r0p0 and r0p1 of A78 AE.
28 * Inputs:
29 * x0: variant[4:7] and revision[0:3] of current cpu.
30 * Shall clobber: x0-x17
31 * --------------------------------------------------
32 */
33func errata_a78_ae_1941500_wa
34	/* Compare x0 against revisions r0p0 - r0p1 */
35	mov	x17, x30
36	bl	check_errata_1941500
37	cbz	x0, 1f
38
39	/* Set bit 8 in ECTLR_EL1 */
40	mrs	x0, CORTEX_A78_AE_CPUECTLR_EL1
41	bic	x0, x0, #CORTEX_A78_AE_CPUECTLR_EL1_BIT_8
42	msr	CORTEX_A78_AE_CPUECTLR_EL1, x0
43	isb
441:
45	ret	x17
46endfunc errata_a78_ae_1941500_wa
47
48func check_errata_1941500
49	/* Applies to revisions r0p0 and r0p1. */
50	mov	x1, #CPU_REV(0, 0)
51	mov	x2, #CPU_REV(0, 1)
52	b	cpu_rev_var_range
53endfunc check_errata_1941500
54
55/* --------------------------------------------------
56 * Errata Workaround for A78 AE Erratum 1951502.
57 * This applies to revisions r0p0 and r0p1 of A78 AE.
58 * Inputs:
59 * x0: variant[4:7] and revision[0:3] of current cpu.
60 * Shall clobber: x0-x17
61 * --------------------------------------------------
62 */
63func errata_a78_ae_1951502_wa
64	/* Compare x0 against revisions r0p0 - r0p1 */
65	mov	x17, x30
66	bl	check_errata_1951502
67	cbz	x0, 1f
68
69	msr	S3_6_c15_c8_0, xzr
70	ldr	x0, =0x10E3900002
71	msr	S3_6_c15_c8_2, x0
72	ldr	x0, =0x10FFF00083
73	msr	S3_6_c15_c8_3, x0
74	ldr	x0, =0x2001003FF
75	msr	S3_6_c15_c8_1, x0
76
77	mov	x0, #1
78	msr	S3_6_c15_c8_0, x0
79	ldr	x0, =0x10E3800082
80	msr	S3_6_c15_c8_2, x0
81	ldr	x0, =0x10FFF00083
82	msr	S3_6_c15_c8_3, x0
83	ldr	x0, =0x2001003FF
84	msr	S3_6_c15_c8_1, x0
85
86	mov	x0, #2
87	msr	S3_6_c15_c8_0, x0
88	ldr	x0, =0x10E3800200
89	msr	S3_6_c15_c8_2, x0
90	ldr	x0, =0x10FFF003E0
91	msr	S3_6_c15_c8_3, x0
92	ldr	x0, =0x2001003FF
93	msr	S3_6_c15_c8_1, x0
94
95	isb
961:
97	ret	x17
98endfunc errata_a78_ae_1951502_wa
99
100func check_errata_1951502
101	/* Applies to revisions r0p0 and r0p1. */
102	mov	x1, #CPU_REV(0, 0)
103	mov	x2, #CPU_REV(0, 1)
104	b	cpu_rev_var_range
105endfunc check_errata_1951502
106
107/* --------------------------------------------------
108 * Errata Workaround for A78 AE Erratum 2376748.
109 * This applies to revisions r0p0 and r0p1 of A78 AE.
110 * Inputs:
111 * x0: variant[4:7] and revision[0:3] of current cpu.
112 * Shall clobber: x0-x17
113 * --------------------------------------------------
114 */
115func errata_a78_ae_2376748_wa
116	/* Compare x0 against revisions r0p0 - r0p1 */
117	mov	x17, x30
118	bl	check_errata_2376748
119	cbz	x0, 1f
120
121	/* -------------------------------------------------------
122	 * Set CPUACTLR2_EL1[0] to 1 to force PLDW/PFRM ST to
123	 * behave like PLD/PRFM LD and not cause invalidations to
124	 * other PE caches. There might be a small performance
125	 * degradation to this workaround for certain workloads
126	 * that share data.
127	 * -------------------------------------------------------
128	 */
129	mrs	x0, CORTEX_A78_AE_ACTLR2_EL1
130	orr	x0, x0, #CORTEX_A78_AE_ACTLR2_EL1_BIT_0
131	msr	CORTEX_A78_AE_ACTLR2_EL1, x0
132	isb
1331:
134	ret	x17
135endfunc errata_a78_ae_2376748_wa
136
137func check_errata_2376748
138	/* Applies to revisions r0p0 and r0p1. */
139	mov	x1, #CPU_REV(0, 0)
140	mov	x2, #CPU_REV(0, 1)
141	b	cpu_rev_var_range
142endfunc check_errata_2376748
143
144/* --------------------------------------------------
145 * Errata Workaround for A78 AE Erratum 2395408.
146 * This applies to revisions r0p0 and r0p1 of A78 AE.
147 * Inputs:
148 * x0: variant[4:7] and revision[0:3] of current cpu.
149 * Shall clobber: x0-x17
150 * --------------------------------------------------
151 */
152func errata_a78_ae_2395408_wa
153	/* Compare x0 against revisions r0p0 - r0p1 */
154	mov	x17, x30
155	bl	check_errata_2395408
156	cbz	x0, 1f
157
158	/* --------------------------------------------------------
159	 * Disable folding of demand requests into older prefetches
160	 * with L2 miss requests outstanding by setting the
161	 * CPUACTLR2_EL1[40] to 1.
162	 * --------------------------------------------------------
163	 */
164	mrs	x0, CORTEX_A78_AE_ACTLR2_EL1
165	orr	x0, x0, #CORTEX_A78_AE_ACTLR2_EL1_BIT_40
166	msr	CORTEX_A78_AE_ACTLR2_EL1, x0
167	isb
1681:
169	ret	x17
170endfunc errata_a78_ae_2395408_wa
171
172func check_errata_2395408
173	/* Applies to revisions r0p0 and r0p1. */
174	mov	x1, #CPU_REV(0, 0)
175	mov	x2, #CPU_REV(0, 1)
176	b	cpu_rev_var_range
177endfunc check_errata_2395408
178
179func check_errata_cve_2022_23960
180#if WORKAROUND_CVE_2022_23960
181	mov	x0, #ERRATA_APPLIES
182#else
183	mov	x0, #ERRATA_MISSING
184#endif
185	ret
186endfunc check_errata_cve_2022_23960
187
188	/* -------------------------------------------------
189	 * The CPU Ops reset function for Cortex-A78-AE
190	 * -------------------------------------------------
191	 */
192func cortex_a78_ae_reset_func
193	mov	x19, x30
194	bl	cpu_get_rev_var
195	mov	x18, x0
196
197#if ERRATA_A78_AE_1941500
198	mov	x0, x18
199	bl	errata_a78_ae_1941500_wa
200#endif
201
202#if ERRATA_A78_AE_1951502
203	mov	x0, x18
204	bl	errata_a78_ae_1951502_wa
205#endif
206
207#if ERRATA_A78_AE_2376748
208	mov	x0, x18
209	bl	errata_a78_ae_2376748_wa
210#endif
211
212#if ERRATA_A78_AE_2395408
213	mov	x0, x18
214	bl	errata_a78_ae_2395408_wa
215#endif
216
217#if ENABLE_AMU
218	/* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */
219	mrs	x0, actlr_el3
220	bic	x0, x0, #CORTEX_A78_ACTLR_TAM_BIT
221	msr	actlr_el3, x0
222
223	/* Make sure accesses from non-secure EL0/EL1 are not trapped to EL2 */
224	mrs	x0, actlr_el2
225	bic	x0, x0, #CORTEX_A78_ACTLR_TAM_BIT
226	msr	actlr_el2, x0
227
228	/* Enable group0 counters */
229	mov	x0, #CORTEX_A78_AMU_GROUP0_MASK
230	msr	CPUAMCNTENSET0_EL0, x0
231
232	/* Enable group1 counters */
233	mov	x0, #CORTEX_A78_AMU_GROUP1_MASK
234	msr	CPUAMCNTENSET1_EL0, x0
235#endif
236
237#if IMAGE_BL31 && WORKAROUND_CVE_2022_23960
238	/*
239	 * The Cortex-A78AE generic vectors are overridden to apply errata
240	 * mitigation on exception entry from lower ELs.
241	 */
242	adr	x0, wa_cve_vbar_cortex_a78_ae
243	msr	vbar_el3, x0
244#endif /* IMAGE_BL31 && WORKAROUND_CVE_2022_23960 */
245
246	isb
247	ret	x19
248endfunc cortex_a78_ae_reset_func
249
250	/* -------------------------------------------------------
251	 * HW will do the cache maintenance while powering down
252	 * -------------------------------------------------------
253	 */
254func cortex_a78_ae_core_pwr_dwn
255	/* -------------------------------------------------------
256	 * Enable CPU power down bit in power control register
257	 * -------------------------------------------------------
258	 */
259	mrs	x0, CORTEX_A78_CPUPWRCTLR_EL1
260	orr	x0, x0, #CORTEX_A78_CPUPWRCTLR_EL1_CORE_PWRDN_EN_BIT
261	msr	CORTEX_A78_CPUPWRCTLR_EL1, x0
262	isb
263	ret
264endfunc cortex_a78_ae_core_pwr_dwn
265
266	/*
267	 * Errata printing function for cortex_a78_ae. Must follow AAPCS.
268	 */
269#if REPORT_ERRATA
270func cortex_a78_ae_errata_report
271	stp	x8, x30, [sp, #-16]!
272
273	bl	cpu_get_rev_var
274	mov	x8, x0
275
276	/*
277	 * Report all errata. The revision-variant information is passed to
278	 * checking functions of each errata.
279	 */
280	report_errata ERRATA_A78_AE_1941500, cortex_a78_ae, 1941500
281	report_errata ERRATA_A78_AE_1951502, cortex_a78_ae, 1951502
282	report_errata ERRATA_A78_AE_2376748, cortex_a78_ae, 2376748
283	report_errata ERRATA_A78_AE_2395408, cortex_a78_ae, 2395408
284	report_errata WORKAROUND_CVE_2022_23960, cortex_a78_ae, cve_2022_23960
285
286	ldp	x8, x30, [sp], #16
287	ret
288endfunc cortex_a78_ae_errata_report
289#endif
290
291	/* -------------------------------------------------------
292	 * This function provides cortex_a78_ae specific
293	 * register information for crash reporting.
294	 * It needs to return with x6 pointing to
295	 * a list of register names in ascii and
296	 * x8 - x15 having values of registers to be
297	 * reported.
298	 * -------------------------------------------------------
299	 */
300.section .rodata.cortex_a78_ae_regs, "aS"
301cortex_a78_ae_regs:  /* The ascii list of register names to be reported */
302	.asciz	"cpuectlr_el1", ""
303
304func cortex_a78_ae_cpu_reg_dump
305	adr	x6, cortex_a78_ae_regs
306	mrs	x8, CORTEX_A78_CPUECTLR_EL1
307	ret
308endfunc cortex_a78_ae_cpu_reg_dump
309
310declare_cpu_ops cortex_a78_ae, CORTEX_A78_AE_MIDR, \
311	cortex_a78_ae_reset_func, \
312	cortex_a78_ae_core_pwr_dwn
313