1 /*
2 * Copyright 2019, DornerWorks
3 * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230)
4 *
5 * SPDX-License-Identifier: GPL-2.0-only
6 */
7
8 #include <config.h>
9 #include <types.h>
10 #include <arch/machine/gic_v3.h>
11
12 #define IRQ_SET_ALL 0xffffffff
13
14 #define RDIST_BANK_SZ 0x00010000
15 /* One GICR region and one GICR_SGI region */
16 #define GICR_PER_CORE_SIZE (0x20000)
17 /* Assume 8 cores */
18 #define GICR_SIZE (0x100000)
19
20 #define GIC_DEADLINE_MS 2
21 #define GIC_REG_WIDTH 32
22
23 #ifdef CONFIG_ARCH_AARCH64
24 #define ICC_SGI1R_EL1 "S3_0_C12_C11_5"
25 #else
26 #define ICC_SGI1R_EL1 "p15, 0, %Q0, %R0, c12"
27 #endif
28
29 #define ICC_SGI1R_INTID_SHIFT (24)
30 #define ICC_SGI1R_AFF1_SHIFT (16)
31 #define ICC_SGI1R_IRM_BIT (40)
32 #define ICC_SGI1R_CPUTARGETLIST_MASK 0xffff
33
34 volatile struct gic_dist_map *const gic_dist = (volatile struct gic_dist_map *)(GICD_PPTR);
35 volatile void *const gicr_base = (volatile uint8_t *)(GICR_PPTR);
36
37 word_t active_irq[CONFIG_MAX_NUM_NODES] = {IRQ_NONE};
38 volatile struct gic_rdist_map *gic_rdist_map[CONFIG_MAX_NUM_NODES] = { 0 };
39 volatile struct gic_rdist_sgi_ppi_map *gic_rdist_sgi_ppi_map[CONFIG_MAX_NUM_NODES] = { 0 };
40
41 #ifdef CONFIG_ARCH_AARCH64
42 #define MPIDR_AFF0(x) (x & 0xff)
43 #define MPIDR_AFF1(x) ((x >> 8) & 0xff)
44 #define MPIDR_AFF2(x) ((x >> 16) & 0xff)
45 #define MPIDR_AFF3(x) ((x >> 32) & 0xff)
46 #else
47 #define MPIDR_AFF0(x) (x & 0xff)
48 #define MPIDR_AFF1(x) ((x >> 8) & 0xff)
49 #define MPIDR_AFF2(x) ((x >> 16) & 0xff)
50 #define MPIDR_AFF3(x) (0)
51 #endif
52 #define MPIDR_MT(x) (x & BIT(24))
53 #define MPIDR_AFF_MASK(x) (x & 0xff00ffffff)
54
55 static word_t mpidr_map[CONFIG_MAX_NUM_NODES];
56
get_mpidr(word_t core_id)57 static inline word_t get_mpidr(word_t core_id)
58 {
59 return mpidr_map[core_id];
60 }
61
get_current_mpidr(void)62 static inline word_t get_current_mpidr(void)
63 {
64 word_t core_id = CURRENT_CPU_INDEX();
65 return get_mpidr(core_id);
66 }
67
mpidr_to_gic_affinity(void)68 static inline uint64_t mpidr_to_gic_affinity(void)
69 {
70 word_t mpidr = get_current_mpidr();
71 uint64_t affinity = 0;
72 affinity = (uint64_t)MPIDR_AFF3(mpidr) << 32 | MPIDR_AFF2(mpidr) << 16 |
73 MPIDR_AFF1(mpidr) << 8 | MPIDR_AFF0(mpidr);
74 return affinity;
75 }
76
77 /* Wait for completion of a distributor change */
gicv3_do_wait_for_rwp(volatile uint32_t * ctlr_addr)78 static uint32_t gicv3_do_wait_for_rwp(volatile uint32_t *ctlr_addr)
79 {
80 uint32_t val;
81 bool_t waiting = true;
82 uint32_t ret = 0;
83
84 uint32_t gpt_cnt_tval = 0;
85 uint32_t deadline_ms = GIC_DEADLINE_MS;
86 uint32_t gpt_cnt_ciel;
87
88 /* Check the value before reading the generic timer */
89 val = *ctlr_addr;
90 if (!(val & GICD_CTLR_RWP)) {
91 return 0;
92 }
93 SYSTEM_READ_WORD(CNTFRQ, gpt_cnt_tval);
94 gpt_cnt_ciel = gpt_cnt_tval + (deadline_ms * TICKS_PER_MS);
95
96 while (waiting) {
97 SYSTEM_READ_WORD(CNTFRQ, gpt_cnt_tval);
98 val = *ctlr_addr;
99
100 if (gpt_cnt_tval >= gpt_cnt_ciel) {
101 printf("GICV3 RWP Timeout after %u ms\n", deadline_ms);
102 ret = 1;
103 waiting = false;
104
105 } else if (!(val & GICD_CTLR_RWP)) {
106 ret = 0;
107 waiting = false;
108 }
109 }
110 return ret;
111 }
112
gicv3_dist_wait_for_rwp(void)113 static void gicv3_dist_wait_for_rwp(void)
114 {
115 gicv3_do_wait_for_rwp(&gic_dist->ctlr);
116 }
117
gicv3_redist_wait_for_rwp(void)118 static void gicv3_redist_wait_for_rwp(void)
119 {
120 gicv3_do_wait_for_rwp(&gic_rdist_map[CURRENT_CPU_INDEX()]->ctlr);
121 }
122
gicv3_enable_sre(void)123 static void gicv3_enable_sre(void)
124 {
125 uint32_t val = 0;
126
127 /* ICC_SRE_EL1 */
128 SYSTEM_READ_WORD(ICC_SRE_EL1, val);
129 val |= GICC_SRE_EL1_SRE;
130
131 SYSTEM_WRITE_WORD(ICC_SRE_EL1, val);
132 isb();
133 }
134
135
dist_init(void)136 BOOT_CODE static void dist_init(void)
137 {
138 word_t i;
139 uint32_t type;
140 unsigned int nr_lines;
141 uint64_t affinity;
142 uint32_t priority;
143
144 /* Disable GIC Distributor */
145 gic_dist->ctlr = 0;
146 gicv3_dist_wait_for_rwp();
147
148 type = gic_dist->typer;
149
150 nr_lines = GIC_REG_WIDTH * ((type & GICD_TYPE_LINESNR) + 1);
151
152 /* Assume level-triggered */
153 for (i = SPI_START; i < nr_lines; i += 16) {
154 gic_dist->icfgrn[(i / 16)] = 0;
155 }
156
157 /* Default priority for global interrupts */
158 priority = (GIC_PRI_IRQ << 24 | GIC_PRI_IRQ << 16 | GIC_PRI_IRQ << 8 |
159 GIC_PRI_IRQ);
160 for (i = SPI_START; i < nr_lines; i += 4) {
161 gic_dist->ipriorityrn[(i / 4)] = priority;
162 }
163 /* Disable and clear all global interrupts */
164 for (i = SPI_START; i < nr_lines; i += 32) {
165 gic_dist->icenablern[(i / 32)] = IRQ_SET_ALL;
166 gic_dist->icpendrn[(i / 32)] = IRQ_SET_ALL;
167 }
168
169 /* Turn on the distributor */
170 gic_dist->ctlr = GICD_CTL_ENABLE | GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1NS | GICD_CTLR_ENABLE_G0;
171 gicv3_dist_wait_for_rwp();
172
173 /* Route all global IRQs to this CPU */
174 affinity = mpidr_to_gic_affinity();
175 for (i = SPI_START; i < nr_lines; i++) {
176 gic_dist->iroutern[i] = affinity;
177 }
178 }
179
gicr_locate_interface(void)180 BOOT_CODE static void gicr_locate_interface(void)
181 {
182 word_t offset;
183 int core_id = CURRENT_CPU_INDEX();
184 word_t mpidr = get_current_mpidr();
185 uint32_t val;
186
187 /*
188 * Iterate through all redistributor interfaces looking for one that matches
189 * our mpidr.
190 */
191 for (offset = 0; offset < GICR_SIZE; offset += GICR_PER_CORE_SIZE) {
192
193 uint64_t typer = ((struct gic_rdist_map *)((word_t)gicr_base + offset))->typer;
194 if ((typer >> 32) == ((MPIDR_AFF3(mpidr) << 24) |
195 (MPIDR_AFF2(mpidr) << 16) |
196 (MPIDR_AFF1(mpidr) << 8) |
197 MPIDR_AFF0(mpidr))) {
198
199 word_t gicr = (word_t)gicr_base + offset;
200 if (gic_rdist_map[core_id] != NULL || gic_rdist_sgi_ppi_map[core_id] != NULL) {
201 printf("GICv3: %s[%d] %p is not null\n",
202 gic_rdist_map[core_id] == NULL ? "gic_rdist_map" : "gic_rdist_sgi_ppi_map",
203 core_id,
204 gic_rdist_map[core_id] == NULL ? (void *)gic_rdist_map[core_id] : (void *)gic_rdist_sgi_ppi_map[core_id]);
205 halt();
206 }
207 gic_rdist_map[core_id] = (void *)gicr;
208 gic_rdist_sgi_ppi_map[core_id] = (void *)(gicr + RDIST_BANK_SZ);
209
210 /*
211 * GICR_WAKER should be Read-all-zeros in Non-secure world
212 * and we expect redistributors to be alread awoken by an earlier loader.
213 * However if we get a value back then something is probably wrong.
214 */
215 val = gic_rdist_map[core_id]->waker;
216 if (val & GICR_WAKER_ChildrenAsleep) {
217 printf("GICv3: GICR_WAKER returned non-zero %x\n", val);
218 halt();
219 }
220
221 break;
222 }
223 }
224 if (offset >= GICR_SIZE) {
225 printf("GICv3: GICR base for CPU %d %d %d %d (Logic ID %d) not found\n",
226 (int)MPIDR_AFF3(mpidr), (int)MPIDR_AFF2(mpidr),
227 (int)MPIDR_AFF1(mpidr), (int)MPIDR_AFF0(mpidr), core_id);
228 halt();
229 }
230
231
232 }
233
gicr_init(void)234 BOOT_CODE static void gicr_init(void)
235 {
236 int i;
237 uint32_t priority;
238
239 /* Find redistributor for this core. */
240 gicr_locate_interface();
241
242 /* Deactivate SGIs/PPIs */
243 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icactiver0 = ~0;
244
245 /* Set priority on PPI and SGI interrupts */
246 priority = (GIC_PRI_IRQ << 24 | GIC_PRI_IRQ << 16 | GIC_PRI_IRQ << 8 |
247 GIC_PRI_IRQ);
248 for (i = 0; i < SPI_START; i += 4) {
249 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->ipriorityrn[i / 4] = priority;
250 }
251
252 /*
253 * Disable all PPI interrupts, ensure all SGI interrupts are
254 * enabled.
255 */
256 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icenabler0 = 0xffff0000;
257 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->isenabler0 = 0x0000ffff;
258
259 /* Set ICFGR1 for PPIs as level-triggered */
260 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icfgr1 = 0x0;
261
262 gicv3_redist_wait_for_rwp();
263 }
264
cpu_iface_init(void)265 BOOT_CODE static void cpu_iface_init(void)
266 {
267 uint32_t icc_ctlr = 0;
268
269 /* Enable system registers */
270 gicv3_enable_sre();
271
272 /* No priority grouping: ICC_BPR1_EL1 */
273 SYSTEM_WRITE_WORD(ICC_BPR1_EL1, 0);
274
275 /* Set priority mask register: ICC_PMR_EL1 */
276 SYSTEM_WRITE_WORD(ICC_PMR_EL1, DEFAULT_PMR_VALUE);
277
278 /* EOI drops priority and deactivates the interrupt: ICC_CTLR_EL1 */
279 SYSTEM_READ_WORD(ICC_CTLR_EL1, icc_ctlr);
280 icc_ctlr &= ~BIT(GICC_CTLR_EL1_EOImode_drop);
281 SYSTEM_WRITE_WORD(ICC_CTLR_EL1, icc_ctlr);
282
283 /* Enable Group1 interrupts: ICC_IGRPEN1_EL1 */
284 SYSTEM_WRITE_WORD(ICC_IGRPEN1_EL1, 1);
285
286 /* Sync at once at the end of cpu interface configuration */
287 isb();
288 }
289
setIRQTrigger(irq_t irq,bool_t trigger)290 void setIRQTrigger(irq_t irq, bool_t trigger)
291 {
292
293 /* GICv3 has read-only GICR_ICFG0 for SGI with
294 * default value 0xaaaaaaaa, and read-write GICR_ICFG1
295 * for PPI with default 0x00000000.*/
296 word_t hw_irq = IRQT_TO_IRQ(irq);
297 word_t core = IRQT_TO_CORE(irq);
298 if (HW_IRQ_IS_SGI(hw_irq)) {
299 return;
300 }
301 int word = hw_irq >> 4;
302 int bit = ((hw_irq & 0xf) * 2);
303 uint32_t icfgr = 0;
304 if (HW_IRQ_IS_PPI(hw_irq)) {
305 icfgr = gic_rdist_sgi_ppi_map[core]->icfgr1;
306 } else {
307 icfgr = gic_dist->icfgrn[word];
308 }
309
310 if (trigger) {
311 icfgr |= (2 << bit);
312 } else {
313 icfgr &= ~(3 << bit);
314 }
315
316 if (HW_IRQ_IS_PPI(hw_irq)) {
317 gic_rdist_sgi_ppi_map[core]->icfgr1 = icfgr;
318 } else {
319 /* Update GICD_ICFGR<n>. Note that the interrupt should
320 * be disabled before changing the field, and this function
321 * assumes the caller has disabled the interrupt. */
322 gic_dist->icfgrn[word] = icfgr;
323 }
324
325 return;
326 }
327
initIRQController(void)328 BOOT_CODE void initIRQController(void)
329 {
330 dist_init();
331 }
332
cpu_initLocalIRQController(void)333 BOOT_CODE void cpu_initLocalIRQController(void)
334 {
335 word_t mpidr = 0;
336 SYSTEM_READ_WORD(MPIDR, mpidr);
337
338 mpidr_map[CURRENT_CPU_INDEX()] = mpidr;
339
340 gicr_init();
341 cpu_iface_init();
342 }
343
344 #ifdef ENABLE_SMP_SUPPORT
345 #define MPIDR_MT(x) (x & BIT(24))
346
ipi_send_target(irq_t irq,word_t cpuTargetList)347 void ipi_send_target(irq_t irq, word_t cpuTargetList)
348 {
349 uint64_t sgi1r_base = ((word_t) IRQT_TO_IRQ(irq)) << ICC_SGI1R_INTID_SHIFT;
350 word_t sgi1r[CONFIG_MAX_NUM_NODES];
351 word_t last_aff1 = 0;
352
353 for (word_t i = 0; i < CONFIG_MAX_NUM_NODES; i++) {
354 sgi1r[i] = 0;
355 if (cpuTargetList & BIT(i)) {
356 word_t mpidr = mpidr_map[i];
357 word_t aff1 = MPIDR_AFF1(mpidr);
358 word_t aff0 = MPIDR_AFF0(mpidr);
359 // AFF1 is assumed to be contiguous and less than CONFIG_MAX_NUM_NODES.
360 // The targets are grouped by AFF1.
361 assert(aff1 >= 0 && aff1 < CONFIG_MAX_NUM_NODES);
362 sgi1r[aff1] |= sgi1r_base | (aff1 << ICC_SGI1R_AFF1_SHIFT) | (1 << aff0);
363 if (aff1 > last_aff1) {
364 last_aff1 = aff1;
365 }
366 }
367 }
368 for (word_t i = 0; i <= last_aff1; i++) {
369 if (sgi1r[i] != 0) {
370 SYSTEM_WRITE_64(ICC_SGI1R_EL1, sgi1r[i]);
371 }
372 }
373 isb();
374 }
375
setIRQTarget(irq_t irq,seL4_Word target)376 void setIRQTarget(irq_t irq, seL4_Word target)
377 {
378 if (IRQ_IS_PPI(irq)) {
379 fail("PPI can't have designated target core\n");
380 return;
381 }
382
383 word_t hw_irq = IRQT_TO_IRQ(irq);
384 gic_dist->iroutern[hw_irq] = MPIDR_AFF_MASK(mpidr_map[target]);
385 }
386
387 #endif /* ENABLE_SMP_SUPPORT */
388
389 #ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
390
391 unsigned int gic_vcpu_num_list_regs;
392
393 #endif /* End of CONFIG_ARM_HYPERVISOR_SUPPORT */
394