1 /* SPDX-License-Identifier: BSD-2-Clause */
2 /*
3 * Copyright (c) 2015, Linaro Limited
4 */
5 #ifndef ARM64_H
6 #define ARM64_H
7
8 #include <compiler.h>
9 #include <sys/cdefs.h>
10 #include <stdint.h>
11 #include <util.h>
12
13 #define SCTLR_M BIT64(0)
14 #define SCTLR_A BIT64(1)
15 #define SCTLR_C BIT64(2)
16 #define SCTLR_SA BIT64(3)
17 #define SCTLR_I BIT64(12)
18 #define SCTLR_ENDB BIT64(13)
19 #define SCTLR_WXN BIT64(19)
20 #define SCTLR_SPAN BIT64(23)
21 #define SCTLR_ENDA BIT64(27)
22 #define SCTLR_ENIB BIT64(30)
23 #define SCTLR_ENIA BIT64(31)
24 #define SCTLR_BT0 BIT64(35)
25 #define SCTLR_BT1 BIT64(36)
26 #define SCTLR_ITFSB BIT64(37)
27
28 #define SCTLR_TCF_MASK SHIFT_U64(0x3, 40)
29 #define SCTLR_TCF_NONE SHIFT_U64(0x0, 40)
30 #define SCTLR_TCF_SYNC SHIFT_U64(0x1, 40)
31 #define SCTLR_TCF_ASYNC SHIFT_U64(0x2, 40)
32 #define SCTLR_TCF_ASYMM SHIFT_U64(0x3, 40)
33
34 #define SCTLR_TCF0_MASK SHIFT_U64(0x3, 38)
35 #define SCTLR_TCF0_NONE SHIFT_U64(0x0, 38)
36 #define SCTLR_TCF0_SYNC SHIFT_U64(0x1, 38)
37 #define SCTLR_TCF0_ASYNC SHIFT_U64(0x2, 38)
38 #define SCTLR_TCF0_ASYMM SHIFT_U64(0x3, 38)
39
40 #define SCTLR_ATA0 BIT64(42)
41 #define SCTLR_ATA BIT64(43)
42
43 #define TTBR_ASID_MASK U(0xff)
44 #define TTBR_ASID_SHIFT U(48)
45
46 #define CLIDR_LOUIS_SHIFT U(21)
47 #define CLIDR_LOC_SHIFT U(24)
48 #define CLIDR_FIELD_WIDTH U(3)
49
50 #define CSSELR_LEVEL_SHIFT U(1)
51
52 #define DAIFBIT_FIQ BIT32(0)
53 #define DAIFBIT_IRQ BIT32(1)
54 #define DAIFBIT_ABT BIT32(2)
55 #define DAIFBIT_DBG BIT32(3)
56 #define DAIFBIT_ALL (DAIFBIT_FIQ | DAIFBIT_IRQ | \
57 DAIFBIT_ABT | DAIFBIT_DBG)
58
59 #define DAIF_F_SHIFT U(6)
60 #define DAIF_F BIT32(6)
61 #define DAIF_I BIT32(7)
62 #define DAIF_A BIT32(8)
63 #define DAIF_D BIT32(9)
64 #define DAIF_AIF (DAIF_A | DAIF_I | DAIF_F)
65
66 #define SPSR_MODE_RW_SHIFT U(4)
67 #define SPSR_MODE_RW_MASK U(0x1)
68 #define SPSR_MODE_RW_64 U(0x0)
69 #define SPSR_MODE_RW_32 U(0x1)
70
71 #define SPSR_64_MODE_SP_SHIFT U(0)
72 #define SPSR_64_MODE_SP_MASK U(0x1)
73 #define SPSR_64_MODE_SP_EL0 U(0x0)
74 #define SPSR_64_MODE_SP_ELX U(0x1)
75
76 #define SPSR_64_MODE_EL_SHIFT U(2)
77 #define SPSR_64_MODE_EL_MASK U(0x3)
78 #define SPSR_64_MODE_EL1 U(0x1)
79 #define SPSR_64_MODE_EL0 U(0x0)
80
81 #define SPSR_64_DAIF_SHIFT U(6)
82 #define SPSR_64_DAIF_MASK U(0xf)
83
84 #define SPSR_32_AIF_SHIFT U(6)
85 #define SPSR_32_AIF_MASK U(0x7)
86
87 #define SPSR_32_E_SHIFT U(9)
88 #define SPSR_32_E_MASK U(0x1)
89 #define SPSR_32_E_LITTLE U(0x0)
90 #define SPSR_32_E_BIG U(0x1)
91
92 #define SPSR_32_T_SHIFT U(5)
93 #define SPSR_32_T_MASK U(0x1)
94 #define SPSR_32_T_ARM U(0x0)
95 #define SPSR_32_T_THUMB U(0x1)
96
97 #define SPSR_32_MODE_SHIFT U(0)
98 #define SPSR_32_MODE_MASK U(0xf)
99 #define SPSR_32_MODE_USR U(0x0)
100
101
102 #define SPSR_64(el, sp, daif) \
103 (SPSR_MODE_RW_64 << SPSR_MODE_RW_SHIFT | \
104 ((el) & SPSR_64_MODE_EL_MASK) << SPSR_64_MODE_EL_SHIFT | \
105 ((sp) & SPSR_64_MODE_SP_MASK) << SPSR_64_MODE_SP_SHIFT | \
106 ((daif) & SPSR_64_DAIF_MASK) << SPSR_64_DAIF_SHIFT)
107
108 #define SPSR_32(mode, isa, aif) \
109 (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT | \
110 SPSR_32_E_LITTLE << SPSR_32_E_SHIFT | \
111 ((mode) & SPSR_32_MODE_MASK) << SPSR_32_MODE_SHIFT | \
112 ((isa) & SPSR_32_T_MASK) << SPSR_32_T_SHIFT | \
113 ((aif) & SPSR_32_AIF_MASK) << SPSR_32_AIF_SHIFT)
114
115
116 #define TCR_T0SZ_SHIFT U(0)
117 #define TCR_EPD0 BIT64(7)
118 #define TCR_IRGN0_SHIFT U(8)
119 #define TCR_ORGN0_SHIFT U(10)
120 #define TCR_SH0_SHIFT U(12)
121 #define TCR_T1SZ_SHIFT U(16)
122 #define TCR_A1 BIT64(22)
123 #define TCR_EPD1 BIT64(23)
124 #define TCR_IRGN1_SHIFT U(24)
125 #define TCR_ORGN1_SHIFT U(26)
126 #define TCR_SH1_SHIFT U(28)
127 #define TCR_EL1_IPS_SHIFT U(32)
128 #define TCR_EL1_IPS_MASK UINT64_C(0x7)
129 #define TCR_TG1_4KB SHIFT_U64(2, 30)
130 #define TCR_RES1 BIT64(31)
131 #define TCR_TBI0 BIT64(37)
132 #define TCR_TBI1 BIT64(38)
133 #define TCR_TCMA0 BIT64(57)
134 #define TCR_TCMA1 BIT64(58)
135
136
137 /* Normal memory, Inner/Outer Non-cacheable */
138 #define TCR_XRGNX_NC U(0x0)
139 /* Normal memory, Inner/Outer Write-Back Write-Allocate Cacheable */
140 #define TCR_XRGNX_WB U(0x1)
141 /* Normal memory, Inner/Outer Write-Through Cacheable */
142 #define TCR_XRGNX_WT U(0x2)
143 /* Normal memory, Inner/Outer Write-Back no Write-Allocate Cacheable */
144 #define TCR_XRGNX_WBWA U(0x3)
145
146 /* Non-shareable */
147 #define TCR_SHX_NSH U(0x0)
148 /* Outer Shareable */
149 #define TCR_SHX_OSH U(0x2)
150 /* Inner Shareable */
151 #define TCR_SHX_ISH U(0x3)
152
153 #define ESR_EC_SHIFT U(26)
154 #define ESR_EC_MASK U(0x3f)
155
156 #define ESR_EC_UNKNOWN U(0x00)
157 #define ESR_EC_WFI U(0x01)
158 #define ESR_EC_AARCH32_CP15_32 U(0x03)
159 #define ESR_EC_AARCH32_CP15_64 U(0x04)
160 #define ESR_EC_AARCH32_CP14_MR U(0x05)
161 #define ESR_EC_AARCH32_CP14_LS U(0x06)
162 #define ESR_EC_FP_ASIMD U(0x07)
163 #define ESR_EC_AARCH32_CP10_ID U(0x08)
164 #define ESR_EC_PAUTH U(0x09)
165 #define ESR_EC_AARCH32_CP14_64 U(0x0c)
166 #define ESR_EC_BTI U(0x0d)
167 #define ESR_EC_ILLEGAL U(0x0e)
168 #define ESR_EC_AARCH32_SVC U(0x11)
169 #define ESR_EC_AARCH64_SVC U(0x15)
170 #define ESR_EC_AARCH64_SYS U(0x18)
171 #define ESR_EC_ERET U(0x1a)
172 #define ESR_EC_FPAC U(0x1c)
173 #define ESR_EC_IABT_EL0 U(0x20)
174 #define ESR_EC_IABT_EL1 U(0x21)
175 #define ESR_EC_PC_ALIGN U(0x22)
176 #define ESR_EC_DABT_EL0 U(0x24)
177 #define ESR_EC_DABT_EL1 U(0x25)
178 #define ESR_EC_SP_ALIGN U(0x26)
179 #define ESR_EC_AARCH32_FP U(0x28)
180 #define ESR_EC_AARCH64_FP U(0x2c)
181 #define ESR_EC_SERROR U(0x2f)
182 #define ESR_EC_BREAKPT_EL0 U(0x30)
183 #define ESR_EC_BREAKPT_EL1 U(0x31)
184 #define ESR_EC_SOFTSTP_EL0 U(0x32)
185 #define ESR_EC_SOFTSTP_EL1 U(0x33)
186 #define ESR_EC_WATCHPT_EL0 U(0x34)
187 #define ESR_EC_WATCHPT_EL1 U(0x35)
188 #define ESR_EC_AARCH32_BKPT U(0x38)
189 #define ESR_EC_AARCH64_BRK U(0x3c)
190
191 /* Combined defines for DFSC and IFSC */
192 #define ESR_FSC_MASK U(0x3f)
193 #define ESR_FSC_SIZE_L0 U(0x00)
194 #define ESR_FSC_SIZE_L1 U(0x01)
195 #define ESR_FSC_SIZE_L2 U(0x02)
196 #define ESR_FSC_SIZE_L3 U(0x03)
197 #define ESR_FSC_TRANS_L0 U(0x04)
198 #define ESR_FSC_TRANS_L1 U(0x05)
199 #define ESR_FSC_TRANS_L2 U(0x06)
200 #define ESR_FSC_TRANS_L3 U(0x07)
201 #define ESR_FSC_ACCF_L1 U(0x09)
202 #define ESR_FSC_ACCF_L2 U(0x0a)
203 #define ESR_FSC_ACCF_L3 U(0x0b)
204 #define ESR_FSC_PERMF_L1 U(0x0d)
205 #define ESR_FSC_PERMF_L2 U(0x0e)
206 #define ESR_FSC_PERMF_L3 U(0x0f)
207 #define ESR_FSC_TAG_CHECK U(0x11)
208 #define ESR_FSC_ALIGN U(0x21)
209
210 /* WnR for DABT and RES0 for IABT */
211 #define ESR_ABT_WNR BIT32(6)
212
213 #define CPACR_EL1_FPEN_SHIFT U(20)
214 #define CPACR_EL1_FPEN_MASK U(0x3)
215 #define CPACR_EL1_FPEN_NONE U(0x0)
216 #define CPACR_EL1_FPEN_EL1 U(0x1)
217 #define CPACR_EL1_FPEN_EL0EL1 U(0x3)
218 #define CPACR_EL1_FPEN(x) ((x) >> CPACR_EL1_FPEN_SHIFT \
219 & CPACR_EL1_FPEN_MASK)
220
221
222 #define PAR_F BIT32(0)
223 #define PAR_PA_SHIFT U(12)
224 #define PAR_PA_MASK (BIT64(36) - 1)
225
226 #define TLBI_MVA_SHIFT U(12)
227 #define TLBI_ASID_SHIFT U(48)
228 #define TLBI_ASID_MASK U(0xff)
229
230 #define ID_AA64PFR1_EL1_BT_MASK ULL(0xf)
231 #define FEAT_BTI_IMPLEMENTED ULL(0x1)
232
233 #define ID_AA64PFR1_EL1_MTE_MASK UL(0xf)
234 #define ID_AA64PFR1_EL1_MTE_SHIFT U(8)
235 #define FEAT_MTE_NOT_IMPLEMENTED U(0x0)
236 #define FEAT_MTE_IMPLEMENTED U(0x1)
237 #define FEAT_MTE2_IMPLEMENTED U(0x2)
238 #define FEAT_MTE3_IMPLEMENTED U(0x3)
239
240 #define ID_AA64ISAR1_GPI_SHIFT U(28)
241 #define ID_AA64ISAR1_GPI_MASK U(0xf)
242 #define ID_AA64ISAR1_GPI_NI U(0x0)
243 #define ID_AA64ISAR1_GPI_IMP_DEF U(0x1)
244
245 #define ID_AA64ISAR1_GPA_SHIFT U(24)
246 #define ID_AA64ISAR1_GPA_MASK U(0xf)
247 #define ID_AA64ISAR1_GPA_NI U(0x0)
248 #define ID_AA64ISAR1_GPA_ARCHITECTED U(0x1)
249
250 #define ID_AA64ISAR1_API_SHIFT U(8)
251 #define ID_AA64ISAR1_API_MASK U(0xf)
252 #define ID_AA64ISAR1_API_NI U(0x0)
253 #define ID_AA64ISAR1_API_IMP_DEF U(0x1)
254 #define ID_AA64ISAR1_API_IMP_DEF_EPAC U(0x2)
255 #define ID_AA64ISAR1_API_IMP_DEF_EPAC2 U(0x3)
256 #define ID_AA64ISAR1_API_IMP_DEF_EPAC2_FPAC U(0x4)
257 #define ID_AA64ISAR1_API_IMP_DEF_EPAC2_FPAC_CMB U(0x5)
258
259 #define ID_AA64ISAR1_APA_SHIFT U(4)
260 #define ID_AA64ISAR1_APA_MASK U(0xf)
261 #define ID_AA64ISAR1_APA_NI U(0x0)
262 #define ID_AA64ISAR1_APA_ARCHITECTED U(0x1)
263 #define ID_AA64ISAR1_APA_ARCH_EPAC U(0x2)
264 #define ID_AA64ISAR1_APA_ARCH_EPAC2 U(0x3)
265 #define ID_AA64ISAR1_APA_ARCH_EPAC2_FPAC U(0x4)
266 #define ID_AA64ISAR1_APA_ARCH_EPAC2_FPAC_CMB U(0x5)
267
268 #define GCR_EL1_RRND BIT64(16)
269
270 #ifndef __ASSEMBLER__
isb(void)271 static inline __noprof void isb(void)
272 {
273 asm volatile ("isb" : : : "memory");
274 }
275
dsb(void)276 static inline __noprof void dsb(void)
277 {
278 asm volatile ("dsb sy" : : : "memory");
279 }
280
dsb_ish(void)281 static inline __noprof void dsb_ish(void)
282 {
283 asm volatile ("dsb ish" : : : "memory");
284 }
285
dsb_ishst(void)286 static inline __noprof void dsb_ishst(void)
287 {
288 asm volatile ("dsb ishst" : : : "memory");
289 }
290
sev(void)291 static inline __noprof void sev(void)
292 {
293 asm volatile ("sev" : : : "memory");
294 }
295
wfe(void)296 static inline __noprof void wfe(void)
297 {
298 asm volatile ("wfe" : : : "memory");
299 }
300
wfi(void)301 static inline __noprof void wfi(void)
302 {
303 asm volatile ("wfi" : : : "memory");
304 }
305
write_at_s1e1r(uint64_t va)306 static inline __noprof void write_at_s1e1r(uint64_t va)
307 {
308 asm volatile ("at S1E1R, %0" : : "r" (va));
309 }
310
read_pc(void)311 static __always_inline __noprof uint64_t read_pc(void)
312 {
313 uint64_t val;
314
315 asm volatile ("adr %0, ." : "=r" (val));
316 return val;
317 }
318
read_fp(void)319 static __always_inline __noprof uint64_t read_fp(void)
320 {
321 uint64_t val;
322
323 asm volatile ("mov %0, x29" : "=r" (val));
324 return val;
325 }
326
read_pmu_ccnt(void)327 static inline __noprof uint64_t read_pmu_ccnt(void)
328 {
329 uint64_t val;
330
331 asm volatile("mrs %0, PMCCNTR_EL0" : "=r"(val));
332 return val;
333 }
334
tlbi_vaae1is(uint64_t mva)335 static inline __noprof void tlbi_vaae1is(uint64_t mva)
336 {
337 asm volatile ("tlbi vaae1is, %0" : : "r" (mva));
338 }
339
tlbi_vale1is(uint64_t mva)340 static inline __noprof void tlbi_vale1is(uint64_t mva)
341 {
342 asm volatile ("tlbi vale1is, %0" : : "r" (mva));
343 }
344
345 /*
346 * Templates for register read/write functions based on mrs/msr
347 */
348
349 #define DEFINE_REG_READ_FUNC_(reg, type, asmreg) \
350 static inline __noprof type read_##reg(void) \
351 { \
352 uint64_t val64 = 0; \
353 \
354 asm volatile("mrs %0, " #asmreg : "=r" (val64)); \
355 return val64; \
356 }
357
358 #define DEFINE_REG_WRITE_FUNC_(reg, type, asmreg) \
359 static inline __noprof void write_##reg(type val) \
360 { \
361 uint64_t val64 = val; \
362 \
363 asm volatile("msr " #asmreg ", %0" : : "r" (val64)); \
364 }
365
366 #define DEFINE_U32_REG_READ_FUNC(reg) \
367 DEFINE_REG_READ_FUNC_(reg, uint32_t, reg)
368
369 #define DEFINE_U32_REG_WRITE_FUNC(reg) \
370 DEFINE_REG_WRITE_FUNC_(reg, uint32_t, reg)
371
372 #define DEFINE_U32_REG_READWRITE_FUNCS(reg) \
373 DEFINE_U32_REG_READ_FUNC(reg) \
374 DEFINE_U32_REG_WRITE_FUNC(reg)
375
376 #define DEFINE_U64_REG_READ_FUNC(reg) \
377 DEFINE_REG_READ_FUNC_(reg, uint64_t, reg)
378
379 #define DEFINE_U64_REG_WRITE_FUNC(reg) \
380 DEFINE_REG_WRITE_FUNC_(reg, uint64_t, reg)
381
382 #define DEFINE_U64_REG_READWRITE_FUNCS(reg) \
383 DEFINE_U64_REG_READ_FUNC(reg) \
384 DEFINE_U64_REG_WRITE_FUNC(reg)
385
386 /*
387 * Define register access functions
388 */
389
390 DEFINE_U32_REG_READWRITE_FUNCS(cpacr_el1)
391 DEFINE_U32_REG_READWRITE_FUNCS(daif)
392 DEFINE_U32_REG_READWRITE_FUNCS(fpcr)
393 DEFINE_U32_REG_READWRITE_FUNCS(fpsr)
394
395 DEFINE_U32_REG_READ_FUNC(ctr_el0)
396 #define read_ctr() read_ctr_el0()
397 DEFINE_U32_REG_READ_FUNC(contextidr_el1)
398 DEFINE_U64_REG_READ_FUNC(sctlr_el1)
399
400 /* ARM Generic timer functions */
401 DEFINE_REG_READ_FUNC_(cntfrq, uint32_t, cntfrq_el0)
402 DEFINE_REG_READ_FUNC_(cntvct, uint64_t, cntvct_el0)
403 DEFINE_REG_READ_FUNC_(cntpct, uint64_t, cntpct_el0)
404 DEFINE_REG_READ_FUNC_(cntkctl, uint32_t, cntkctl_el1)
405 DEFINE_REG_WRITE_FUNC_(cntkctl, uint32_t, cntkctl_el1)
406 DEFINE_REG_READ_FUNC_(cntps_ctl, uint32_t, cntps_ctl_el1)
407 DEFINE_REG_WRITE_FUNC_(cntps_ctl, uint32_t, cntps_ctl_el1)
408 DEFINE_REG_READ_FUNC_(cntps_tval, uint32_t, cntps_tval_el1)
409 DEFINE_REG_WRITE_FUNC_(cntps_tval, uint32_t, cntps_tval_el1)
410
411 DEFINE_REG_READ_FUNC_(pmccntr, uint64_t, pmccntr_el0)
412
413 DEFINE_U64_REG_READWRITE_FUNCS(ttbr0_el1)
414 DEFINE_U64_REG_READWRITE_FUNCS(ttbr1_el1)
415 DEFINE_U64_REG_READWRITE_FUNCS(tcr_el1)
416
417 DEFINE_U64_REG_READ_FUNC(esr_el1)
418 DEFINE_U64_REG_READ_FUNC(far_el1)
419 DEFINE_U64_REG_READ_FUNC(mpidr_el1)
420 /* Alias for reading this register to avoid ifdefs in code */
421 #define read_mpidr() read_mpidr_el1()
422 DEFINE_U64_REG_READ_FUNC(midr_el1)
423 /* Alias for reading this register to avoid ifdefs in code */
424 #define read_midr() read_midr_el1()
425 DEFINE_U64_REG_READ_FUNC(par_el1)
426
427 DEFINE_U64_REG_WRITE_FUNC(mair_el1)
428
429 DEFINE_U64_REG_READ_FUNC(id_aa64pfr1_el1)
430 DEFINE_U64_REG_READ_FUNC(id_aa64isar1_el1)
431 DEFINE_REG_READ_FUNC_(apiakeylo, uint64_t, S3_0_c2_c1_0)
432 DEFINE_REG_READ_FUNC_(apiakeyhi, uint64_t, S3_0_c2_c1_1)
433
434 DEFINE_REG_WRITE_FUNC_(apibkeylo, uint64_t, S3_0_c2_c1_2)
435 DEFINE_REG_WRITE_FUNC_(apibkeyhi, uint64_t, S3_0_c2_c1_3)
436
437 DEFINE_REG_READ_FUNC_(apdakeylo, uint64_t, S3_0_c2_c2_0)
438 DEFINE_REG_READ_FUNC_(apdakeyhi, uint64_t, S3_0_c2_c2_1)
439
440 DEFINE_REG_WRITE_FUNC_(apdbkeylo, uint64_t, S3_0_c2_c2_2)
441 DEFINE_REG_WRITE_FUNC_(apdbkeyhi, uint64_t, S3_0_c2_c2_3)
442
443 DEFINE_REG_WRITE_FUNC_(apgakeylo, uint64_t, S3_0_c2_c3_0)
444 DEFINE_REG_WRITE_FUNC_(apgakeyhi, uint64_t, S3_0_c2_c3_1)
445
446 /* Register read/write functions for GICC registers by using system interface */
447 DEFINE_REG_READ_FUNC_(icc_ctlr, uint32_t, S3_0_C12_C12_4)
448 DEFINE_REG_WRITE_FUNC_(icc_ctlr, uint32_t, S3_0_C12_C12_4)
449 DEFINE_REG_WRITE_FUNC_(icc_pmr, uint32_t, S3_0_C4_C6_0)
450 DEFINE_REG_READ_FUNC_(icc_iar0, uint32_t, S3_0_c12_c8_0)
451 DEFINE_REG_READ_FUNC_(icc_iar1, uint32_t, S3_0_c12_c12_0)
452 DEFINE_REG_WRITE_FUNC_(icc_eoir0, uint32_t, S3_0_c12_c8_1)
453 DEFINE_REG_WRITE_FUNC_(icc_eoir1, uint32_t, S3_0_c12_c12_1)
454 DEFINE_REG_WRITE_FUNC_(icc_igrpen0, uint32_t, S3_0_C12_C12_6)
455 DEFINE_REG_WRITE_FUNC_(icc_igrpen1, uint32_t, S3_0_C12_C12_7)
456 #endif /*__ASSEMBLER__*/
457
458 #endif /*ARM64_H*/
459
460