1/*
2 * Copyright (c) 2017-2022, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6#include <arch.h>
7#include <asm_macros.S>
8#include <assert_macros.S>
9#include <common/debug.h>
10#include <cortex_a57.h>
11#include <cpu_macros.S>
12
13	/* ---------------------------------------------
14	 * Disable intra-cluster coherency
15	 * Clobbers: r0-r1
16	 * ---------------------------------------------
17	 */
18func cortex_a57_disable_smp
19	ldcopr16	r0, r1, CORTEX_A57_ECTLR
20	bic64_imm	r0, r1, CORTEX_A57_ECTLR_SMP_BIT
21	stcopr16	r0, r1, CORTEX_A57_ECTLR
22	bx	lr
23endfunc cortex_a57_disable_smp
24
25	/* ---------------------------------------------
26	 * Disable all types of L2 prefetches.
27	 * Clobbers: r0-r2
28	 * ---------------------------------------------
29	 */
30func cortex_a57_disable_l2_prefetch
31	ldcopr16	r0, r1, CORTEX_A57_ECTLR
32	orr64_imm	r0, r1, CORTEX_A57_ECTLR_DIS_TWD_ACC_PFTCH_BIT
33	bic64_imm	r0, r1, (CORTEX_A57_ECTLR_L2_IPFTCH_DIST_MASK | \
34				 CORTEX_A57_ECTLR_L2_DPFTCH_DIST_MASK)
35	stcopr16	r0, r1, CORTEX_A57_ECTLR
36	isb
37	dsb	ish
38	bx	lr
39endfunc cortex_a57_disable_l2_prefetch
40
41	/* ---------------------------------------------
42	 * Disable debug interfaces
43	 * ---------------------------------------------
44	 */
45func cortex_a57_disable_ext_debug
46	mov	r0, #1
47	stcopr	r0, DBGOSDLR
48	isb
49#if ERRATA_A57_817169
50	/*
51	 * Invalidate any TLB address
52	 */
53	mov	r0, #0
54	stcopr	r0, TLBIMVA
55#endif
56	dsb	sy
57	bx	lr
58endfunc cortex_a57_disable_ext_debug
59
60	/* --------------------------------------------------
61	 * Errata Workaround for Cortex A57 Errata #806969.
62	 * This applies only to revision r0p0 of Cortex A57.
63	 * Inputs:
64	 * r0: variant[4:7] and revision[0:3] of current cpu.
65	 * Shall clobber: r0-r3
66	 * --------------------------------------------------
67	 */
68func errata_a57_806969_wa
69	/*
70	 * Compare r0 against revision r0p0
71	 */
72	mov		r2, lr
73	bl		check_errata_806969
74	mov		lr, r2
75	cmp		r0, #ERRATA_NOT_APPLIES
76	beq		1f
77	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
78	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_NO_ALLOC_WBWA
79	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
801:
81	bx	lr
82endfunc errata_a57_806969_wa
83
84func check_errata_806969
85	mov	r1, #0x00
86	b	cpu_rev_var_ls
87endfunc check_errata_806969
88
89	/* ---------------------------------------------------
90	 * Errata Workaround for Cortex A57 Errata #813419.
91	 * This applies only to revision r0p0 of Cortex A57.
92	 * ---------------------------------------------------
93	 */
94func check_errata_813419
95	/*
96	 * Even though this is only needed for revision r0p0, it
97	 * is always applied due to limitations of the current
98	 * errata framework.
99	 */
100	mov	r0, #ERRATA_APPLIES
101	bx	lr
102endfunc check_errata_813419
103
104	/* ---------------------------------------------------
105	 * Errata Workaround for Cortex A57 Errata #813420.
106	 * This applies only to revision r0p0 of Cortex A57.
107	 * Inputs:
108	 * r0: variant[4:7] and revision[0:3] of current cpu.
109	 * Shall clobber: r0-r3
110	 * ---------------------------------------------------
111	 */
112func errata_a57_813420_wa
113	/*
114	 * Compare r0 against revision r0p0
115	 */
116	mov		r2, lr
117	bl		check_errata_813420
118	mov		lr, r2
119	cmp		r0, #ERRATA_NOT_APPLIES
120	beq		1f
121	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
122	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DCC_AS_DCCI
123	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
1241:
125	bx		lr
126endfunc errata_a57_813420_wa
127
128func check_errata_813420
129	mov	r1, #0x00
130	b	cpu_rev_var_ls
131endfunc check_errata_813420
132
133	/* ---------------------------------------------------
134	 * Errata Workaround for Cortex A57 Errata #814670.
135	 * This applies only to revision r0p0 of Cortex A57.
136	 * Inputs:
137	 * r0: variant[4:7] and revision[0:3] of current cpu.
138	 * Shall clobber: r0-r3
139	 * ---------------------------------------------------
140	 */
141func errata_a57_814670_wa
142	/*
143	 * Compare r0 against revision r0p0
144	 */
145	mov		r2, lr
146	bl		check_errata_814670
147	cmp		r0, #ERRATA_NOT_APPLIES
148	beq		1f
149	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
150	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_DMB_NULLIFICATION
151	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
152	isb
1531:
154	bx		r2
155endfunc errata_a57_814670_wa
156
157func check_errata_814670
158	mov	r1, #0x00
159	b	cpu_rev_var_ls
160endfunc check_errata_814670
161
162	/* ----------------------------------------------------
163	 * Errata Workaround for Cortex A57 Errata #817169.
164	 * This applies only to revision <= r0p1 of Cortex A57.
165	 * ----------------------------------------------------
166	 */
167func check_errata_817169
168	/*
169	 * Even though this is only needed for revision <= r0p1, it
170	 * is always applied because of the low cost of the workaround.
171	 */
172	mov	r0, #ERRATA_APPLIES
173	bx	lr
174endfunc check_errata_817169
175
176	/* --------------------------------------------------------------------
177	 * Disable the over-read from the LDNP instruction.
178	 *
179	 * This applies to all revisions <= r1p2. The performance degradation
180	 * observed with LDNP/STNP has been fixed on r1p3 and onwards.
181	 *
182	 * Inputs:
183	 * r0: variant[4:7] and revision[0:3] of current cpu.
184	 * Shall clobber: r0-r3
185	 * ---------------------------------------------------------------------
186	 */
187func a57_disable_ldnp_overread
188	/*
189	 * Compare r0 against revision r1p2
190	 */
191	mov		r2, lr
192	bl		check_errata_disable_ldnp_overread
193	mov		lr, r2
194	cmp		r0, #ERRATA_NOT_APPLIES
195	beq		1f
196	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
197	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_OVERREAD
198	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
1991:
200	bx		lr
201endfunc a57_disable_ldnp_overread
202
203func check_errata_disable_ldnp_overread
204	mov	r1, #0x12
205	b	cpu_rev_var_ls
206endfunc check_errata_disable_ldnp_overread
207
208	/* ---------------------------------------------------
209	 * Errata Workaround for Cortex A57 Errata #826974.
210	 * This applies only to revision <= r1p1 of Cortex A57.
211	 * Inputs:
212	 * r0: variant[4:7] and revision[0:3] of current cpu.
213	 * Shall clobber: r0-r3
214	 * ---------------------------------------------------
215	 */
216func errata_a57_826974_wa
217	/*
218	 * Compare r0 against revision r1p1
219	 */
220	mov		r2, lr
221	bl		check_errata_826974
222	mov		lr, r2
223	cmp		r0, #ERRATA_NOT_APPLIES
224	beq		1f
225	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
226	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_LOAD_PASS_DMB
227	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
2281:
229	bx		lr
230endfunc errata_a57_826974_wa
231
232func check_errata_826974
233	mov	r1, #0x11
234	b	cpu_rev_var_ls
235endfunc check_errata_826974
236
237	/* ---------------------------------------------------
238	 * Errata Workaround for Cortex A57 Errata #826977.
239	 * This applies only to revision <= r1p1 of Cortex A57.
240	 * Inputs:
241	 * r0: variant[4:7] and revision[0:3] of current cpu.
242	 * Shall clobber: r0-r3
243	 * ---------------------------------------------------
244	 */
245func errata_a57_826977_wa
246	/*
247	 * Compare r0 against revision r1p1
248	 */
249	mov		r2, lr
250	bl		check_errata_826977
251	mov		lr, r2
252	cmp		r0, #ERRATA_NOT_APPLIES
253	beq		1f
254	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
255	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_GRE_NGRE_AS_NGNRE
256	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
2571:
258	bx		lr
259endfunc errata_a57_826977_wa
260
261func check_errata_826977
262	mov	r1, #0x11
263	b	cpu_rev_var_ls
264endfunc check_errata_826977
265
266	/* ---------------------------------------------------
267	 * Errata Workaround for Cortex A57 Errata #828024.
268	 * This applies only to revision <= r1p1 of Cortex A57.
269	 * Inputs:
270	 * r0: variant[4:7] and revision[0:3] of current cpu.
271	 * Shall clobber: r0-r3
272	 * ---------------------------------------------------
273	 */
274func errata_a57_828024_wa
275	/*
276	 * Compare r0 against revision r1p1
277	 */
278	mov		r2, lr
279	bl		check_errata_828024
280	mov		lr, r2
281	cmp		r0, #ERRATA_NOT_APPLIES
282	beq		1f
283	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
284	/*
285	 * Setting the relevant bits in CORTEX_A57_CPUACTLR has to be done in 2
286	 * instructions here because the resulting bitmask doesn't fit in a
287	 * 16-bit value so it cannot be encoded in a single instruction.
288	 */
289	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_NO_ALLOC_WBWA
290	orr64_imm	r0, r1, (CORTEX_A57_CPUACTLR_DIS_L1_STREAMING | CORTEX_A57_CPUACTLR_DIS_STREAMING)
291	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
2921:
293	bx		lr
294endfunc errata_a57_828024_wa
295
296func check_errata_828024
297	mov	r1, #0x11
298	b	cpu_rev_var_ls
299endfunc check_errata_828024
300
301	/* ---------------------------------------------------
302	 * Errata Workaround for Cortex A57 Errata #829520.
303	 * This applies only to revision <= r1p2 of Cortex A57.
304	 * Inputs:
305	 * r0: variant[4:7] and revision[0:3] of current cpu.
306	 * Shall clobber: r0-r3
307	 * ---------------------------------------------------
308	 */
309func errata_a57_829520_wa
310	/*
311	 * Compare r0 against revision r1p2
312	 */
313	mov		r2, lr
314	bl		check_errata_829520
315	mov		lr, r2
316	cmp		r0, #ERRATA_NOT_APPLIES
317	beq		1f
318	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
319	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_INDIRECT_PREDICTOR
320	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
3211:
322	bx		lr
323endfunc errata_a57_829520_wa
324
325func check_errata_829520
326	mov	r1, #0x12
327	b	cpu_rev_var_ls
328endfunc check_errata_829520
329
330	/* ---------------------------------------------------
331	 * Errata Workaround for Cortex A57 Errata #833471.
332	 * This applies only to revision <= r1p2 of Cortex A57.
333	 * Inputs:
334	 * r0: variant[4:7] and revision[0:3] of current cpu.
335	 * Shall clobber: r0-r3
336	 * ---------------------------------------------------
337	 */
338func errata_a57_833471_wa
339	/*
340	 * Compare r0 against revision r1p2
341	 */
342	mov		r2, lr
343	bl		check_errata_833471
344	mov		lr, r2
345	cmp		r0, #ERRATA_NOT_APPLIES
346	beq		1f
347	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
348	orr64_imm	r1, r1, CORTEX_A57_CPUACTLR_FORCE_FPSCR_FLUSH
349	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
3501:
351	bx		lr
352endfunc errata_a57_833471_wa
353
354func check_errata_833471
355	mov	r1, #0x12
356	b	cpu_rev_var_ls
357endfunc check_errata_833471
358
359	/* ---------------------------------------------------
360	 * Errata Workaround for Cortex A57 Errata #859972.
361	 * This applies only to revision <= r1p3 of Cortex A57.
362	 * Inputs:
363	 * r0: variant[4:7] and revision[0:3] of current cpu.
364	 * Shall clobber: r0-r3
365	 * ---------------------------------------------------
366	 */
367func errata_a57_859972_wa
368	mov		r2, lr
369	bl		check_errata_859972
370	mov		lr, r2
371	cmp		r0, #ERRATA_NOT_APPLIES
372	beq		1f
373	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
374	orr64_imm	r1, r1, CORTEX_A57_CPUACTLR_DIS_INSTR_PREFETCH
375	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
3761:
377	bx		lr
378endfunc errata_a57_859972_wa
379
380func check_errata_859972
381	mov	r1, #0x13
382	b	cpu_rev_var_ls
383endfunc check_errata_859972
384
385func check_errata_cve_2017_5715
386	mov	r0, #ERRATA_MISSING
387	bx	lr
388endfunc check_errata_cve_2017_5715
389
390func check_errata_cve_2018_3639
391#if WORKAROUND_CVE_2018_3639
392	mov	r0, #ERRATA_APPLIES
393#else
394	mov	r0, #ERRATA_MISSING
395#endif
396	bx	lr
397endfunc check_errata_cve_2018_3639
398
399func check_errata_cve_2022_23960
400	mov	r0, #ERRATA_MISSING
401	bx	lr
402endfunc check_errata_cve_2022_23960
403
404	/* -------------------------------------------------
405	 * The CPU Ops reset function for Cortex-A57.
406	 * Shall clobber: r0-r6
407	 * -------------------------------------------------
408	 */
409func cortex_a57_reset_func
410	mov	r5, lr
411	bl	cpu_get_rev_var
412	mov	r4, r0
413
414#if ERRATA_A57_806969
415	mov	r0, r4
416	bl	errata_a57_806969_wa
417#endif
418
419#if ERRATA_A57_813420
420	mov	r0, r4
421	bl	errata_a57_813420_wa
422#endif
423
424#if ERRATA_A57_814670
425	mov	r0, r4
426	bl	errata_a57_814670_wa
427#endif
428
429#if A57_DISABLE_NON_TEMPORAL_HINT
430	mov	r0, r4
431	bl	a57_disable_ldnp_overread
432#endif
433
434#if ERRATA_A57_826974
435	mov	r0, r4
436	bl	errata_a57_826974_wa
437#endif
438
439#if ERRATA_A57_826977
440	mov	r0, r4
441	bl	errata_a57_826977_wa
442#endif
443
444#if ERRATA_A57_828024
445	mov	r0, r4
446	bl	errata_a57_828024_wa
447#endif
448
449#if ERRATA_A57_829520
450	mov	r0, r4
451	bl	errata_a57_829520_wa
452#endif
453
454#if ERRATA_A57_833471
455	mov	r0, r4
456	bl	errata_a57_833471_wa
457#endif
458
459#if ERRATA_A57_859972
460	mov	r0, r4
461	bl	errata_a57_859972_wa
462#endif
463
464#if WORKAROUND_CVE_2018_3639
465	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
466	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_LOAD_PASS_STORE
467	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
468	isb
469	dsb	sy
470#endif
471
472	/* ---------------------------------------------
473	 * Enable the SMP bit.
474	 * ---------------------------------------------
475	 */
476	ldcopr16	r0, r1, CORTEX_A57_ECTLR
477	orr64_imm	r0, r1, CORTEX_A57_ECTLR_SMP_BIT
478	stcopr16	r0, r1,	CORTEX_A57_ECTLR
479	isb
480	bx	r5
481endfunc cortex_a57_reset_func
482
483	/* ----------------------------------------------------
484	 * The CPU Ops core power down function for Cortex-A57.
485	 * ----------------------------------------------------
486	 */
487func cortex_a57_core_pwr_dwn
488	push	{r12, lr}
489
490	/* Assert if cache is enabled */
491#if ENABLE_ASSERTIONS
492	ldcopr	r0, SCTLR
493	tst	r0, #SCTLR_C_BIT
494	ASM_ASSERT(eq)
495#endif
496
497	/* ---------------------------------------------
498	 * Disable the L2 prefetches.
499	 * ---------------------------------------------
500	 */
501	bl	cortex_a57_disable_l2_prefetch
502
503	/* ---------------------------------------------
504	 * Flush L1 caches.
505	 * ---------------------------------------------
506	 */
507	mov	r0, #DC_OP_CISW
508	bl	dcsw_op_level1
509
510	/* ---------------------------------------------
511	 * Come out of intra cluster coherency
512	 * ---------------------------------------------
513	 */
514	bl	cortex_a57_disable_smp
515
516	/* ---------------------------------------------
517	 * Force the debug interfaces to be quiescent
518	 * ---------------------------------------------
519	 */
520	pop	{r12, lr}
521	b	cortex_a57_disable_ext_debug
522endfunc cortex_a57_core_pwr_dwn
523
524	/* -------------------------------------------------------
525	 * The CPU Ops cluster power down function for Cortex-A57.
526	 * Clobbers: r0-r3
527	 * -------------------------------------------------------
528	 */
529func cortex_a57_cluster_pwr_dwn
530	push	{r12, lr}
531
532	/* Assert if cache is enabled */
533#if ENABLE_ASSERTIONS
534	ldcopr	r0, SCTLR
535	tst	r0, #SCTLR_C_BIT
536	ASM_ASSERT(eq)
537#endif
538
539	/* ---------------------------------------------
540	 * Disable the L2 prefetches.
541	 * ---------------------------------------------
542	 */
543	bl	cortex_a57_disable_l2_prefetch
544
545	/* ---------------------------------------------
546	 * Flush L1 caches.
547	 * ---------------------------------------------
548	 */
549	mov	r0, #DC_OP_CISW
550	bl	dcsw_op_level1
551
552	/* ---------------------------------------------
553	 * Disable the optional ACP.
554	 * ---------------------------------------------
555	 */
556	bl	plat_disable_acp
557
558	/* ---------------------------------------------
559	 * Flush L2 caches.
560	 * ---------------------------------------------
561	 */
562	mov	r0, #DC_OP_CISW
563	bl	dcsw_op_level2
564
565	/* ---------------------------------------------
566	 * Come out of intra cluster coherency
567	 * ---------------------------------------------
568	 */
569	bl	cortex_a57_disable_smp
570
571	/* ---------------------------------------------
572	 * Force the debug interfaces to be quiescent
573	 * ---------------------------------------------
574	 */
575	pop	{r12, lr}
576	b	cortex_a57_disable_ext_debug
577endfunc cortex_a57_cluster_pwr_dwn
578
579#if REPORT_ERRATA
580/*
581 * Errata printing function for Cortex A57. Must follow AAPCS.
582 */
583func cortex_a57_errata_report
584	push	{r12, lr}
585
586	bl	cpu_get_rev_var
587	mov	r4, r0
588
589	/*
590	 * Report all errata. The revision-variant information is passed to
591	 * checking functions of each errata.
592	 */
593	report_errata ERRATA_A57_806969, cortex_a57, 806969
594	report_errata ERRATA_A57_813419, cortex_a57, 813419
595	report_errata ERRATA_A57_813420, cortex_a57, 813420
596	report_errata ERRATA_A57_814670, cortex_a57, 814670
597	report_errata ERRATA_A57_817169, cortex_a57, 817169
598	report_errata A57_DISABLE_NON_TEMPORAL_HINT, cortex_a57, \
599		disable_ldnp_overread
600	report_errata ERRATA_A57_826974, cortex_a57, 826974
601	report_errata ERRATA_A57_826977, cortex_a57, 826977
602	report_errata ERRATA_A57_828024, cortex_a57, 828024
603	report_errata ERRATA_A57_829520, cortex_a57, 829520
604	report_errata ERRATA_A57_833471, cortex_a57, 833471
605	report_errata ERRATA_A57_859972, cortex_a57, 859972
606	report_errata WORKAROUND_CVE_2017_5715, cortex_a57, cve_2017_5715
607	report_errata WORKAROUND_CVE_2018_3639, cortex_a57, cve_2018_3639
608	report_errata WORKAROUND_CVE_2022_23960, cortex_a57, cve_2022_23960
609
610	pop	{r12, lr}
611	bx	lr
612endfunc cortex_a57_errata_report
613#endif
614
615declare_cpu_ops cortex_a57, CORTEX_A57_MIDR, \
616	cortex_a57_reset_func, \
617	cortex_a57_core_pwr_dwn, \
618	cortex_a57_cluster_pwr_dwn
619