1 /*
2 * Copyright (C) 2018-2022 Intel Corporation.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <types.h>
8 #include <errno.h>
9 #include <irq.h>
10 #include <asm/lib/spinlock.h>
11 #include <asm/ioapic.h>
12 #include <asm/irq.h>
13 #include <asm/pgtable.h>
14 #include <asm/io.h>
15 #include <asm/mmu.h>
16 #include <acpi.h>
17 #include <logmsg.h>
18
19 #define NR_MAX_GSI (CONFIG_MAX_IOAPIC_NUM * CONFIG_MAX_IOAPIC_LINES)
20
21 #define DEFAULT_DEST_MODE IOAPIC_RTE_DESTMODE_LOGICAL
22 #define DEFAULT_DELIVERY_MODE IOAPIC_RTE_DELMODE_LOPRI
23
24 /*
25 * is_valid is by default false when all the
26 * static variables, part of .bss, are initialized to 0s
27 * It is set to true, if the corresponding
28 * gsi falls in ranges identified by IOAPIC data
29 * in ACPI MADT in ioapic_setup_irqs.
30 */
31
32 struct gsi_table {
33 bool is_valid;
34 struct {
35 uint8_t acpi_id;
36 uint8_t index;
37 uint32_t pin;
38 void *base_addr;
39 } ioapic_info;
40 };
41
42 static struct gsi_table gsi_table_data[NR_MAX_GSI];
43 static uint32_t ioapic_max_nr_gsi;
44 static spinlock_t ioapic_lock = { .head = 0U, .tail = 0U, };
45
46 static union ioapic_rte saved_rte[CONFIG_MAX_IOAPIC_NUM][CONFIG_MAX_IOAPIC_LINES];
47
48 static const uint32_t legacy_irq_trigger_mode[NR_LEGACY_PIN] = {
49 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ2*/
50 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ1*/
51 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ0*/
52 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ3*/
53 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ4*/
54 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ5*/
55 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ6*/
56 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ7*/
57 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ8*/
58 IOAPIC_RTE_TRGRMODE_LEVEL, /* IRQ9*/
59 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ10*/
60 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ11*/
61 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ12*/
62 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ13*/
63 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ14*/
64 IOAPIC_RTE_TRGRMODE_EDGE, /* IRQ15*/
65 };
66
67 static const uint32_t pic_ioapic_pin_map[NR_LEGACY_PIN] = {
68 2U, /* pin0*/
69 1U, /* pin1*/
70 0U, /* pin2*/
71 3U, /* pin3*/
72 4U, /* pin4*/
73 5U, /* pin5*/
74 6U, /* pin6*/
75 7U, /* pin7*/
76 8U, /* pin8*/
77 9U, /* pin9*/
78 10U, /* pin10*/
79 11U, /* pin11*/
80 12U, /* pin12*/
81 13U, /* pin13*/
82 14U, /* pin14*/
83 15U, /* pin15*/
84 };
85
86 static struct ioapic_info ioapic_array[CONFIG_MAX_IOAPIC_NUM];
87 static uint8_t ioapic_num;
88
get_pic_pin_from_ioapic_pin(uint32_t pin_index)89 uint32_t get_pic_pin_from_ioapic_pin(uint32_t pin_index)
90 {
91 uint32_t pin_id = INVALID_INTERRUPT_PIN;
92 if (pin_index < NR_LEGACY_PIN) {
93 pin_id = pic_ioapic_pin_map[pin_index];
94 }
95 return pin_id;
96 }
97
get_platform_ioapic_info(struct ioapic_info ** plat_ioapic_info)98 uint8_t get_platform_ioapic_info (struct ioapic_info **plat_ioapic_info)
99 {
100 *plat_ioapic_info = ioapic_array;
101 return ioapic_num;
102 }
103
get_gsi_to_ioapic_index(uint32_t gsi)104 uint8_t get_gsi_to_ioapic_index(uint32_t gsi)
105 {
106 return gsi_table_data[gsi].ioapic_info.index;
107 }
108
109 /*
110 * @pre gsi < NR_MAX_GSI
111 */
gsi_to_ioapic_base(uint32_t gsi)112 void *gsi_to_ioapic_base(uint32_t gsi)
113 {
114
115 return gsi_table_data[gsi].ioapic_info.base_addr;
116 }
117
get_max_nr_gsi(void)118 uint32_t get_max_nr_gsi(void)
119 {
120 return ioapic_max_nr_gsi;
121 }
122
map_ioapic(uint64_t ioapic_paddr)123 static void *map_ioapic(uint64_t ioapic_paddr)
124 {
125 /* At some point we may need to translate this paddr to a vaddr.
126 * 1:1 mapping for now.
127 */
128 return hpa2hva(ioapic_paddr);
129 }
130
131 static inline uint32_t
ioapic_read_reg32(void * ioapic_base,const uint32_t offset)132 ioapic_read_reg32(void *ioapic_base, const uint32_t offset)
133 {
134 uint32_t v;
135 uint64_t rflags;
136
137 spinlock_irqsave_obtain(&ioapic_lock, &rflags);
138
139 /* Write IOREGSEL */
140 mmio_write32(offset, ioapic_base + IOAPIC_REGSEL);
141 /* Read IOWIN */
142 v = mmio_read32(ioapic_base + IOAPIC_WINDOW);
143
144 spinlock_irqrestore_release(&ioapic_lock, rflags);
145 return v;
146 }
147
148 static inline void
ioapic_write_reg32(void * ioapic_base,const uint32_t offset,const uint32_t value)149 ioapic_write_reg32(void *ioapic_base, const uint32_t offset, const uint32_t value)
150 {
151 uint64_t rflags;
152
153 spinlock_irqsave_obtain(&ioapic_lock, &rflags);
154
155 /* Write IOREGSEL */
156 mmio_write32(offset, ioapic_base + IOAPIC_REGSEL);
157 /* Write IOWIN */
158 mmio_write32(value, ioapic_base + IOAPIC_WINDOW);
159
160 spinlock_irqrestore_release(&ioapic_lock, rflags);
161 }
162
163 /**
164 * @pre apic_id < 2
165 */
166 static inline uint64_t
get_ioapic_base(uint8_t apic_id)167 get_ioapic_base(uint8_t apic_id)
168 {
169 /* the ioapic base should be extracted from ACPI MADT table */
170 return ioapic_array[apic_id].addr;
171 }
172
ioapic_get_rte_entry(void * ioapic_base,uint32_t pin,union ioapic_rte * rte)173 void ioapic_get_rte_entry(void *ioapic_base, uint32_t pin, union ioapic_rte *rte)
174 {
175 uint32_t rte_addr = (pin * 2U) + 0x10U;
176 rte->u.lo_32 = ioapic_read_reg32(ioapic_base, rte_addr);
177 rte->u.hi_32 = ioapic_read_reg32(ioapic_base, rte_addr + 1U);
178 }
179
180 static inline void
ioapic_set_rte_entry(void * ioapic_base,uint32_t pin,union ioapic_rte rte)181 ioapic_set_rte_entry(void *ioapic_base,
182 uint32_t pin, union ioapic_rte rte)
183 {
184 uint32_t rte_addr = (pin * 2U) + 0x10U;
185 ioapic_write_reg32(ioapic_base, rte_addr, rte.u.lo_32);
186 ioapic_write_reg32(ioapic_base, rte_addr + 1U, rte.u.hi_32);
187 }
188
189 static inline union ioapic_rte
create_rte_for_legacy_irq(uint32_t irq,uint32_t vr)190 create_rte_for_legacy_irq(uint32_t irq, uint32_t vr)
191 {
192 union ioapic_rte rte;
193
194 /* Legacy IRQ 0-15 setup, default masked
195 * are actually defined in either MPTable or ACPI MADT table
196 * before we have ACPI table parsing in HV we use common hardcode
197 */
198
199 rte.full = 0UL;
200 rte.bits.intr_mask = IOAPIC_RTE_MASK_SET;
201 rte.bits.trigger_mode = legacy_irq_trigger_mode[irq];
202 rte.bits.dest_mode = DEFAULT_DEST_MODE;
203 rte.bits.delivery_mode = DEFAULT_DELIVERY_MODE;
204 rte.bits.vector = vr;
205
206 /* Fixed to active high */
207 rte.bits.intr_polarity = IOAPIC_RTE_INTPOL_AHI;
208
209 /* Dest field: legacy irq fixed to CPU0 */
210 rte.bits.dest_field = 1U;
211
212 return rte;
213 }
214
215 static inline union ioapic_rte
create_rte_for_gsi_irq(uint32_t irq,uint32_t vr)216 create_rte_for_gsi_irq(uint32_t irq, uint32_t vr)
217 {
218 union ioapic_rte rte;
219
220 rte.full = 0UL;
221
222 if (irq < NR_LEGACY_PIN) {
223 rte = create_rte_for_legacy_irq(irq, vr);
224 } else {
225 /* irq default masked, level trig */
226 rte.bits.intr_mask = IOAPIC_RTE_MASK_SET;
227 rte.bits.trigger_mode = IOAPIC_RTE_TRGRMODE_LEVEL;
228 rte.bits.dest_mode = DEFAULT_DEST_MODE;
229 rte.bits.delivery_mode = DEFAULT_DELIVERY_MODE;
230 rte.bits.vector = vr;
231
232 /* Fixed to active high */
233 rte.bits.intr_polarity = IOAPIC_RTE_INTPOL_AHI;
234
235 /* Dest field */
236 rte.bits.dest_field = (uint8_t) ALL_CPUS_MASK;
237 }
238
239 return rte;
240 }
241
ioapic_set_routing(uint32_t gsi,uint32_t vr)242 static void ioapic_set_routing(uint32_t gsi, uint32_t vr)
243 {
244 void *ioapic_base;
245 union ioapic_rte rte;
246
247 ioapic_base = gsi_to_ioapic_base(gsi);
248 rte = create_rte_for_gsi_irq(gsi, vr);
249 ioapic_set_rte_entry(ioapic_base, gsi_table_data[gsi].ioapic_info.pin, rte);
250
251 if (rte.bits.trigger_mode == IOAPIC_RTE_TRGRMODE_LEVEL) {
252 set_irq_trigger_mode(gsi, true);
253 } else {
254 set_irq_trigger_mode(gsi, false);
255 }
256
257 dev_dbg(DBG_LEVEL_IRQ, "GSI: irq:%d pin:%hhu rte:%lx",
258 gsi, gsi_table_data[gsi].ioapic_info.pin,
259 rte.full);
260 }
261
262 /*
263 * @pre rte != NULL
264 * @pre is_ioapic_irq(irq) == true
265 */
ioapic_get_rte(uint32_t irq,union ioapic_rte * rte)266 void ioapic_get_rte(uint32_t irq, union ioapic_rte *rte)
267 {
268 void *addr;
269
270 addr = gsi_to_ioapic_base(irq);
271 ioapic_get_rte_entry(addr, gsi_table_data[irq].ioapic_info.pin, rte);
272 }
273
274 /*
275 * @pre is_ioapic_irq(irq) == true
276 */
ioapic_set_rte(uint32_t irq,union ioapic_rte rte)277 void ioapic_set_rte(uint32_t irq, union ioapic_rte rte)
278 {
279 void *addr;
280
281 addr = gsi_to_ioapic_base(irq);
282 ioapic_set_rte_entry(addr, gsi_table_data[irq].ioapic_info.pin, rte);
283
284 dev_dbg(DBG_LEVEL_IRQ, "GSI: irq:%d pin:%hhu rte:%lx",
285 irq, gsi_table_data[irq].ioapic_info.pin,
286 rte.full);
287 }
288
289 /*
290 * Checks if the gsi is valid
291 * 1) gsi < NR_MAX_GSI
292 * 2) gsi is valid on the platform according to ACPI MADT info
293 */
is_gsi_valid(uint32_t gsi)294 bool is_gsi_valid(uint32_t gsi)
295 {
296
297 return (gsi < NR_MAX_GSI) && (gsi_table_data[gsi].is_valid);
298 }
299
300 /*
301 * IO-APIC gsi and irq are identity mapped in ioapic_setup_irqs
302 * So #gsi = #irq for ACRN
303 */
304
is_ioapic_irq(uint32_t irq)305 bool is_ioapic_irq(uint32_t irq)
306 {
307
308 return is_gsi_valid(irq);
309 }
310
311 /*
312 *@pre gsi < NR_MAX_GSI
313 *@pre is_gsi_valid(gsi) == true
314 */
315
gsi_to_ioapic_pin(uint32_t gsi)316 uint32_t gsi_to_ioapic_pin(uint32_t gsi)
317 {
318 return gsi_table_data[gsi].ioapic_info.pin;
319 }
320
321 /*
322 *@pre is_gsi_valid(gsi) == true
323 */
ioapic_gsi_to_irq(uint32_t gsi)324 uint32_t ioapic_gsi_to_irq(uint32_t gsi)
325 {
326 return gsi;
327 }
328
329 static void
ioapic_irq_gsi_mask_unmask(uint32_t irq,bool mask)330 ioapic_irq_gsi_mask_unmask(uint32_t irq, bool mask)
331 {
332 void *addr = NULL;
333 uint32_t pin;
334 union ioapic_rte rte;
335
336 addr = gsi_to_ioapic_base(irq);
337 pin = gsi_table_data[irq].ioapic_info.pin;
338
339 if (addr != NULL) {
340 ioapic_get_rte_entry(addr, pin, &rte);
341 if (mask) {
342 rte.bits.intr_mask = IOAPIC_RTE_MASK_SET;
343 } else {
344 rte.bits.intr_mask = IOAPIC_RTE_MASK_CLR;
345 }
346 ioapic_set_rte_entry(addr, pin, rte);
347 dev_dbg(DBG_LEVEL_PTIRQ, "update: irq:%d pin:%hhu rte:%lx",
348 irq, pin, rte.full);
349 } else {
350 dev_dbg(DBG_LEVEL_PTIRQ, "NULL Address returned from gsi_table_data");
351 }
352 }
353
ioapic_gsi_mask_irq(uint32_t irq)354 void ioapic_gsi_mask_irq(uint32_t irq)
355 {
356 ioapic_irq_gsi_mask_unmask(irq, true);
357 }
358
ioapic_gsi_unmask_irq(uint32_t irq)359 void ioapic_gsi_unmask_irq(uint32_t irq)
360 {
361 ioapic_irq_gsi_mask_unmask(irq, false);
362 }
363
364 static uint32_t
ioapic_nr_pins(void * ioapic_base)365 ioapic_nr_pins(void *ioapic_base)
366 {
367 uint32_t version;
368 uint32_t nr_pins;
369
370 version = ioapic_read_reg32(ioapic_base, IOAPIC_VER);
371 dev_dbg(DBG_LEVEL_IRQ, "IOAPIC version: %x", version);
372
373 /* The 23:16 bits in the version register is the highest entry in the
374 * I/O redirection table, which is 1 smaller than the number of
375 * interrupt input pins. */
376 nr_pins = (((version & IOAPIC_MAX_RTE_MASK) >> MAX_RTE_SHIFT) + 1U);
377
378
379 return nr_pins;
380 }
381
382 /*
383 * @pre is_ioapic_irq(irq) == true
384 */
ioapic_irq_to_ioapic_id(uint32_t irq)385 uint8_t ioapic_irq_to_ioapic_id(uint32_t irq)
386 {
387
388 return gsi_table_data[irq].ioapic_info.acpi_id;
389 }
390
init_ioapic_id_info(void)391 int32_t init_ioapic_id_info(void)
392 {
393 int32_t ret = 0;
394 uint8_t ioapic_id;
395 void *addr;
396 uint32_t nr_pins, gsi;
397
398 ioapic_num = parse_madt_ioapic(&ioapic_array[0]);
399 if (ioapic_num <= (uint8_t)CONFIG_MAX_IOAPIC_NUM) {
400 /*
401 * Iterate thru all the IO-APICs on the platform
402 * Check the number of pins available on each IOAPIC is less
403 * than the CONFIG_MAX_IOAPIC_LINES
404 */
405
406 gsi = 0U;
407 for (ioapic_id = 0U; ioapic_id < ioapic_num; ioapic_id++) {
408 addr = map_ioapic(ioapic_array[ioapic_id].addr);
409 set_paging_supervisor((uint64_t)addr, PAGE_SIZE);
410
411 nr_pins = ioapic_nr_pins(addr);
412 if (nr_pins <= (uint32_t) CONFIG_MAX_IOAPIC_LINES) {
413 gsi += nr_pins;
414 ioapic_array[ioapic_id].nr_pins = nr_pins;
415 } else {
416 pr_err ("Pin count %x of IOAPIC with %x > CONFIG_MAX_IOAPIC_LINES, bump up CONFIG_MAX_IOAPIC_LINES!",
417 nr_pins, ioapic_array[ioapic_id].id);
418 ret = -EINVAL;
419 break;
420 }
421 }
422
423 /*
424 * Check if total pin count, can be inferred by GSI, is
425 * atleast same as the number of Legacy IRQs, NR_LEGACY_IRQ
426 */
427
428 if (ret == 0) {
429 if (gsi < (uint32_t) NR_LEGACY_PIN) {
430 pr_err ("Total pin count (%x) is less than NR_LEGACY_IRQ!", gsi);
431 ret = -EINVAL;
432 }
433 }
434 } else {
435 pr_err ("Number of IOAPIC on platform %x > CONFIG_MAX_IOAPIC_NUM, try bumping up CONFIG_MAX_IOAPIC_NUM!",
436 ioapic_num);
437 ret = -EINVAL;
438 }
439
440
441 return ret;
442 }
443
ioapic_setup_irqs(void)444 void ioapic_setup_irqs(void)
445 {
446 uint8_t ioapic_id;
447 uint32_t gsi = 0U;
448 uint32_t vr;
449
450 for (ioapic_id = 0U;
451 ioapic_id < ioapic_num; ioapic_id++) {
452 void *addr;
453 uint32_t pin, nr_pins;
454
455 addr = map_ioapic(ioapic_array[ioapic_id].addr);
456
457 nr_pins = ioapic_array[ioapic_id].nr_pins;
458 gsi = ioapic_array[ioapic_id].gsi_base;
459 for (pin = 0U; pin < nr_pins; pin++) {
460 gsi_table_data[gsi].is_valid = true;
461 gsi_table_data[gsi].ioapic_info.acpi_id = ioapic_array[ioapic_id].id;
462 gsi_table_data[gsi].ioapic_info.base_addr = addr;
463 gsi_table_data[gsi].ioapic_info.pin = pin;
464 gsi_table_data[gsi].ioapic_info.index = ioapic_id;
465
466 /* pinned irq before use it */
467 if (reserve_irq_num(gsi) == IRQ_INVALID) {
468 pr_err("failed to alloc IRQ[%d]", gsi);
469 gsi++;
470 continue;
471 }
472
473 /* assign vector for this GSI
474 * for legacy irq, reserved vector and never free
475 */
476 if (gsi < NR_LEGACY_PIN) {
477 vr = alloc_irq_vector(gsi);
478 if (vr == VECTOR_INVALID) {
479 pr_err("failed to alloc VR");
480 gsi++;
481 continue;
482 }
483 } else {
484 vr = 0U; /* not to allocate VR right now */
485 }
486
487 ioapic_set_routing(gsi, vr);
488 gsi++;
489 }
490 }
491
492 /* system max gsi numbers */
493 ioapic_max_nr_gsi = gsi;
494 }
495
suspend_ioapic(void)496 void suspend_ioapic(void)
497 {
498 uint8_t ioapic_id;
499 uint32_t ioapic_pin;
500
501 for (ioapic_id = 0U; ioapic_id < ioapic_num; ioapic_id++) {
502 void *addr;
503
504 addr = map_ioapic(get_ioapic_base(ioapic_id));
505 for (ioapic_pin = 0U; ioapic_pin < ioapic_array[ioapic_id].nr_pins; ioapic_pin++) {
506 ioapic_get_rte_entry(addr, ioapic_pin,
507 &saved_rte[ioapic_id][ioapic_pin]);
508 }
509 }
510 }
511
resume_ioapic(void)512 void resume_ioapic(void)
513 {
514 uint8_t ioapic_id;
515 uint32_t ioapic_pin;
516
517 for (ioapic_id = 0U; ioapic_id < ioapic_num; ioapic_id++) {
518 void *addr;
519
520 addr = map_ioapic(get_ioapic_base(ioapic_id));
521 for (ioapic_pin = 0U; ioapic_pin < ioapic_array[ioapic_id].nr_pins; ioapic_pin++) {
522 ioapic_set_rte_entry(addr, ioapic_pin,
523 saved_rte[ioapic_id][ioapic_pin]);
524 }
525 }
526 }
527