1 /*
2 * based on linux-2.6.17.13/arch/i386/kernel/apic.c
3 *
4 * Local APIC handling, local APIC timers
5 *
6 * (c) 1999, 2000 Ingo Molnar <mingo@redhat.com>
7 *
8 * Fixes
9 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
10 * thanks to Eric Gilmore
11 * and Rolf G. Tews
12 * for testing these extensively.
13 * Maciej W. Rozycki : Various updates and fixes.
14 * Mikael Pettersson : Power Management for UP-APIC.
15 * Pavel Machek and
16 * Mikael Pettersson : PM converted to driver model.
17 */
18
19 #include <xen/perfc.h>
20 #include <xen/errno.h>
21 #include <xen/init.h>
22 #include <xen/mm.h>
23 #include <xen/sched.h>
24 #include <xen/irq.h>
25 #include <xen/delay.h>
26 #include <xen/smp.h>
27 #include <xen/softirq.h>
28 #include <asm/mc146818rtc.h>
29 #include <asm/msr.h>
30 #include <asm/atomic.h>
31 #include <asm/mpspec.h>
32 #include <asm/flushtlb.h>
33 #include <asm/hardirq.h>
34 #include <asm/apic.h>
35 #include <asm/io_apic.h>
36 #include <mach_apic.h>
37 #include <io_ports.h>
38 #include <xen/kexec.h>
39 #include <asm/guest.h>
40 #include <asm/time.h>
41
42 static bool __read_mostly tdt_enabled;
43 static bool __initdata tdt_enable = true;
44 boolean_param("tdt", tdt_enable);
45
46 static struct {
47 int active;
48 /* r/w apic fields */
49 unsigned int apic_id;
50 unsigned int apic_taskpri;
51 unsigned int apic_ldr;
52 unsigned int apic_dfr;
53 unsigned int apic_spiv;
54 unsigned int apic_lvtt;
55 unsigned int apic_lvtpc;
56 unsigned int apic_lvtcmci;
57 unsigned int apic_lvt0;
58 unsigned int apic_lvt1;
59 unsigned int apic_lvterr;
60 unsigned int apic_tmict;
61 unsigned int apic_tdcr;
62 unsigned int apic_thmr;
63 } apic_pm_state;
64
65 /*
66 * Knob to control our willingness to enable the local APIC.
67 */
68 static s8 __initdata enable_local_apic; /* -1=force-disable, +1=force-enable */
69
70 /*
71 * Debug level
72 */
73 u8 __read_mostly apic_verbosity;
74
75 static bool __initdata opt_x2apic = true;
76 boolean_param("x2apic", opt_x2apic);
77
78 /*
79 * Bootstrap processor local APIC boot mode - so we can undo our changes
80 * to the APIC state.
81 */
82 static enum apic_mode apic_boot_mode = APIC_MODE_INVALID;
83
84 bool __read_mostly x2apic_enabled;
85 bool __read_mostly directed_eoi_enabled;
86
modern_apic(void)87 static int modern_apic(void)
88 {
89 unsigned int lvr, version;
90 /* AMD systems use old APIC versions, so check the CPU */
91 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
92 boot_cpu_data.x86 >= 0xf)
93 return 1;
94 lvr = apic_read(APIC_LVR);
95 version = GET_APIC_VERSION(lvr);
96 return version >= 0x14;
97 }
98
99 /*
100 * 'what should we do if we get a hw irq event on an illegal vector'.
101 * each architecture has to answer this themselves.
102 */
ack_bad_irq(unsigned int irq)103 void ack_bad_irq(unsigned int irq)
104 {
105 printk("unexpected IRQ trap at irq %02x\n", irq);
106 /*
107 * Currently unexpected vectors happen only on SMP and APIC.
108 * We _must_ ack these because every local APIC has only N
109 * irq slots per priority level, and a 'hanging, unacked' IRQ
110 * holds up an irq slot - in excessive cases (when multiple
111 * unexpected vectors occur) that might lock up the APIC
112 * completely.
113 * But only ack when the APIC is enabled -AK
114 */
115 if (cpu_has_apic)
116 ack_APIC_irq();
117 }
118
apic_intr_init(void)119 void __init apic_intr_init(void)
120 {
121 smp_intr_init();
122
123 /* self generated IPI for local APIC timer */
124 set_direct_apic_vector(LOCAL_TIMER_VECTOR, apic_timer_interrupt);
125
126 /* IPI vectors for APIC spurious and error interrupts */
127 set_direct_apic_vector(SPURIOUS_APIC_VECTOR, spurious_interrupt);
128 set_direct_apic_vector(ERROR_APIC_VECTOR, error_interrupt);
129
130 /* Performance Counters Interrupt */
131 set_direct_apic_vector(PMU_APIC_VECTOR, pmu_apic_interrupt);
132 }
133
134 /* Using APIC to generate smp_local_timer_interrupt? */
135 static bool __read_mostly using_apic_timer;
136
137 static bool __read_mostly enabled_via_apicbase;
138
get_physical_broadcast(void)139 int get_physical_broadcast(void)
140 {
141 if (modern_apic())
142 return 0xff;
143 else
144 return 0xf;
145 }
146
get_maxlvt(void)147 int get_maxlvt(void)
148 {
149 unsigned int v = apic_read(APIC_LVR);
150
151 return GET_APIC_MAXLVT(v);
152 }
153
clear_local_APIC(void)154 void clear_local_APIC(void)
155 {
156 int maxlvt;
157 unsigned long v;
158
159 maxlvt = get_maxlvt();
160
161 /* Work around AMD Erratum 411. This is a nice thing to do anyway. */
162 apic_write(APIC_TMICT, 0);
163
164 /*
165 * Masking an LVT entry on a P6 can trigger a local APIC error
166 * if the vector is zero. Mask LVTERR first to prevent this.
167 */
168 if (maxlvt >= 3) {
169 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
170 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
171 }
172 /*
173 * Careful: we have to set masks only first to deassert
174 * any level-triggered sources.
175 */
176 v = apic_read(APIC_LVTT);
177 apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
178 v = apic_read(APIC_LVT0);
179 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
180 v = apic_read(APIC_LVT1);
181 apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
182 if (maxlvt >= 4) {
183 v = apic_read(APIC_LVTPC);
184 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
185 }
186
187 /* lets not touch this if we didn't frob it */
188 #ifdef CONFIG_X86_MCE_THERMAL
189 if (maxlvt >= 5) {
190 v = apic_read(APIC_LVTTHMR);
191 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
192 }
193 #endif
194
195 if (maxlvt >= 6) {
196 v = apic_read(APIC_CMCI);
197 apic_write(APIC_CMCI, v | APIC_LVT_MASKED);
198 }
199 /*
200 * Clean APIC state for other OSs:
201 */
202 apic_write(APIC_LVTT, APIC_LVT_MASKED);
203 apic_write(APIC_LVT0, APIC_LVT_MASKED);
204 apic_write(APIC_LVT1, APIC_LVT_MASKED);
205 if (maxlvt >= 3)
206 apic_write(APIC_LVTERR, APIC_LVT_MASKED);
207 if (maxlvt >= 4)
208 apic_write(APIC_LVTPC, APIC_LVT_MASKED);
209
210 #ifdef CONFIG_X86_MCE_THERMAL
211 if (maxlvt >= 5)
212 apic_write(APIC_LVTTHMR, APIC_LVT_MASKED);
213 #endif
214 if (maxlvt >= 6)
215 apic_write(APIC_CMCI, APIC_LVT_MASKED);
216
217 if (maxlvt > 3) /* Due to Pentium errata 3AP and 11AP. */
218 apic_write(APIC_ESR, 0);
219 apic_read(APIC_ESR);
220 }
221
connect_bsp_APIC(void)222 void __init connect_bsp_APIC(void)
223 {
224 if (pic_mode) {
225 /*
226 * Do not trust the local APIC being empty at bootup.
227 */
228 clear_local_APIC();
229 /*
230 * PIC mode, enable APIC mode in the IMCR, i.e.
231 * connect BSP's local APIC to INT and NMI lines.
232 */
233 apic_printk(APIC_VERBOSE, "leaving PIC mode, "
234 "enabling APIC mode.\n");
235 outb(0x70, 0x22);
236 outb(0x01, 0x23);
237 }
238 enable_apic_mode();
239 }
240
disconnect_bsp_APIC(int virt_wire_setup)241 void disconnect_bsp_APIC(int virt_wire_setup)
242 {
243 if (pic_mode) {
244 /*
245 * Put the board back into PIC mode (has an effect
246 * only on certain older boards). Note that APIC
247 * interrupts, including IPIs, won't work beyond
248 * this point! The only exception are INIT IPIs.
249 */
250 apic_printk(APIC_VERBOSE, "disabling APIC mode, "
251 "entering PIC mode.\n");
252 outb(0x70, 0x22);
253 outb(0x00, 0x23);
254 }
255 else {
256 /* Go back to Virtual Wire compatibility mode */
257 unsigned long value;
258
259 /* For the spurious interrupt use vector F, and enable it */
260 value = apic_read(APIC_SPIV);
261 value &= ~APIC_VECTOR_MASK;
262 value |= APIC_SPIV_APIC_ENABLED;
263 value |= 0xf;
264 apic_write(APIC_SPIV, value);
265
266 if (!virt_wire_setup) {
267 /* For LVT0 make it edge triggered, active high, external and enabled */
268 value = apic_read(APIC_LVT0);
269 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
270 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
271 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED );
272 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
273 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
274 apic_write(APIC_LVT0, value);
275 }
276 else {
277 /* Disable LVT0 */
278 apic_write(APIC_LVT0, APIC_LVT_MASKED);
279 }
280
281 /* For LVT1 make it edge triggered, active high, nmi and enabled */
282 value = apic_read(APIC_LVT1);
283 value &= ~(
284 APIC_MODE_MASK | APIC_SEND_PENDING |
285 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
286 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
287 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
288 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
289 apic_write(APIC_LVT1, value);
290 }
291 }
292
disable_local_APIC(void)293 void disable_local_APIC(void)
294 {
295 clear_local_APIC();
296
297 /*
298 * Disable APIC (implies clearing of registers
299 * for 82489DX!).
300 */
301 apic_write(APIC_SPIV, apic_read(APIC_SPIV) & ~APIC_SPIV_APIC_ENABLED);
302
303 if (enabled_via_apicbase) {
304 uint64_t msr_content;
305 rdmsrl(MSR_IA32_APICBASE, msr_content);
306 wrmsrl(MSR_IA32_APICBASE, msr_content &
307 ~(MSR_IA32_APICBASE_ENABLE|MSR_IA32_APICBASE_EXTD));
308 }
309
310 if ( kexecing && (current_local_apic_mode() != apic_boot_mode) )
311 {
312 uint64_t msr_content;
313 rdmsrl(MSR_IA32_APICBASE, msr_content);
314 msr_content &= ~(MSR_IA32_APICBASE_ENABLE|MSR_IA32_APICBASE_EXTD);
315 wrmsrl(MSR_IA32_APICBASE, msr_content);
316
317 switch ( apic_boot_mode )
318 {
319 case APIC_MODE_DISABLED:
320 break; /* Nothing to do - we did this above */
321 case APIC_MODE_XAPIC:
322 msr_content |= MSR_IA32_APICBASE_ENABLE;
323 wrmsrl(MSR_IA32_APICBASE, msr_content);
324 break;
325 case APIC_MODE_X2APIC:
326 msr_content |= MSR_IA32_APICBASE_ENABLE;
327 wrmsrl(MSR_IA32_APICBASE, msr_content);
328 msr_content |= MSR_IA32_APICBASE_EXTD;
329 wrmsrl(MSR_IA32_APICBASE, msr_content);
330 break;
331 default:
332 printk("Default case when reverting #%d lapic to boot state\n",
333 smp_processor_id());
334 break;
335 }
336 }
337
338 }
339
340 /*
341 * This is to verify that we're looking at a real local APIC.
342 * Check these against your board if the CPUs aren't getting
343 * started for no apparent reason.
344 */
verify_local_APIC(void)345 int __init verify_local_APIC(void)
346 {
347 unsigned int reg0, reg1;
348
349 /*
350 * The version register is read-only in a real APIC.
351 */
352 reg0 = apic_read(APIC_LVR);
353 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
354
355 /* We don't try writing LVR in x2APIC mode since that incurs #GP. */
356 if ( !x2apic_enabled )
357 apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
358 reg1 = apic_read(APIC_LVR);
359 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
360
361 /*
362 * The two version reads above should print the same
363 * numbers. If the second one is different, then we
364 * poke at a non-APIC.
365 */
366 if (reg1 != reg0)
367 return 0;
368
369 /*
370 * Check if the version looks reasonably.
371 */
372 reg1 = GET_APIC_VERSION(reg0);
373 if (reg1 == 0x00 || reg1 == 0xff)
374 return 0;
375 reg1 = get_maxlvt();
376 if (reg1 < 0x02 || reg1 == 0xff)
377 return 0;
378
379 /*
380 * Detecting directed EOI on BSP:
381 * If having directed EOI support in lapic, force to use ioapic_ack_old,
382 * and enable the directed EOI for intr handling.
383 */
384 if ( reg0 & APIC_LVR_DIRECTED_EOI )
385 {
386 if ( ioapic_ack_new && ioapic_ack_forced )
387 printk("Not enabling directed EOI because ioapic_ack_new has been "
388 "forced on the command line\n");
389 else
390 {
391 ioapic_ack_new = false;
392 directed_eoi_enabled = true;
393 printk("Enabled directed EOI with ioapic_ack_old on!\n");
394 }
395 }
396
397 /*
398 * The ID register is read/write in a real APIC.
399 */
400 reg0 = apic_read(APIC_ID);
401 apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
402
403 /*
404 * The next two are just to see if we have sane values.
405 * They're only really relevant if we're in Virtual Wire
406 * compatibility mode, but most boxes are anymore.
407 */
408 reg0 = apic_read(APIC_LVT0);
409 apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0);
410 reg1 = apic_read(APIC_LVT1);
411 apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);
412
413 return 1;
414 }
415
sync_Arb_IDs(void)416 void __init sync_Arb_IDs(void)
417 {
418 /* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1
419 And not needed on AMD */
420 if (modern_apic())
421 return;
422 /*
423 * Wait for idle.
424 */
425 apic_wait_icr_idle();
426
427 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
428 apic_write(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG | APIC_DM_INIT);
429 }
430
431 /*
432 * An initial setup of the virtual wire mode.
433 */
init_bsp_APIC(void)434 void __init init_bsp_APIC(void)
435 {
436 unsigned long value;
437
438 /*
439 * Don't do the setup now if we have a SMP BIOS as the
440 * through-I/O-APIC virtual wire mode might be active.
441 */
442 if (smp_found_config || !cpu_has_apic)
443 return;
444
445 /*
446 * Do not trust the local APIC being empty at bootup.
447 */
448 clear_local_APIC();
449
450 /*
451 * Enable APIC.
452 */
453 value = apic_read(APIC_SPIV);
454 value &= ~APIC_VECTOR_MASK;
455 value |= APIC_SPIV_APIC_ENABLED;
456
457 /* This bit is reserved on P4/Xeon and should be cleared */
458 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 15))
459 value &= ~APIC_SPIV_FOCUS_DISABLED;
460 else
461 value |= APIC_SPIV_FOCUS_DISABLED;
462 value |= SPURIOUS_APIC_VECTOR;
463 apic_write(APIC_SPIV, value);
464
465 /*
466 * Set up the virtual wire mode.
467 */
468 apic_write(APIC_LVT0, APIC_DM_EXTINT);
469 apic_write(APIC_LVT1, APIC_DM_NMI);
470 }
471
apic_pm_activate(void)472 static void apic_pm_activate(void)
473 {
474 apic_pm_state.active = 1;
475 }
476
__enable_x2apic(void)477 static void __enable_x2apic(void)
478 {
479 uint64_t msr_content;
480
481 rdmsrl(MSR_IA32_APICBASE, msr_content);
482 if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
483 {
484 msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
485 msr_content = (uint32_t)msr_content;
486 wrmsrl(MSR_IA32_APICBASE, msr_content);
487 }
488 }
489
resume_x2apic(void)490 static void resume_x2apic(void)
491 {
492 struct IO_APIC_route_entry **ioapic_entries = NULL;
493
494 ASSERT(x2apic_enabled);
495
496 ioapic_entries = alloc_ioapic_entries();
497 if ( !ioapic_entries )
498 {
499 printk("Allocate ioapic_entries failed\n");
500 goto out;
501 }
502
503 if ( save_IO_APIC_setup(ioapic_entries) )
504 {
505 printk("Saving IO-APIC state failed\n");
506 goto out;
507 }
508
509 mask_8259A();
510 mask_IO_APIC_setup(ioapic_entries);
511
512 iommu_enable_x2apic_IR();
513 __enable_x2apic();
514
515 restore_IO_APIC_setup(ioapic_entries);
516 unmask_8259A();
517
518 out:
519 if ( ioapic_entries )
520 free_ioapic_entries(ioapic_entries);
521 }
522
setup_local_APIC(void)523 void setup_local_APIC(void)
524 {
525 unsigned long oldvalue, value, maxlvt;
526 int i, j;
527
528 /* Pound the ESR really hard over the head with a big hammer - mbligh */
529 if (esr_disable) {
530 apic_write(APIC_ESR, 0);
531 apic_write(APIC_ESR, 0);
532 apic_write(APIC_ESR, 0);
533 apic_write(APIC_ESR, 0);
534 }
535
536 BUILD_BUG_ON((SPURIOUS_APIC_VECTOR & 0x0f) != 0x0f);
537
538 /*
539 * Double-check whether this APIC is really registered.
540 */
541 if (!apic_id_registered())
542 BUG();
543
544 /*
545 * Intel recommends to set DFR, LDR and TPR before enabling
546 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
547 * document number 292116). So here it goes...
548 */
549 init_apic_ldr();
550
551 /*
552 * Set Task Priority to reject any interrupts below FIRST_DYNAMIC_VECTOR.
553 */
554 apic_write(APIC_TASKPRI, (FIRST_DYNAMIC_VECTOR & 0xF0) - 0x10);
555
556 /*
557 * After a crash, we no longer service the interrupts and a pending
558 * interrupt from previous kernel might still have ISR bit set.
559 *
560 * Most probably by now CPU has serviced that pending interrupt and
561 * it might not have done the ack_APIC_irq() because it thought,
562 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
563 * does not clear the ISR bit and cpu thinks it has already serivced
564 * the interrupt. Hence a vector might get locked. It was noticed
565 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
566 */
567 for (i = APIC_ISR_NR - 1; i >= 0; i--) {
568 value = apic_read(APIC_ISR + i*0x10);
569 for (j = 31; j >= 0; j--) {
570 if (value & (1u << j))
571 ack_APIC_irq();
572 }
573 }
574
575 /*
576 * Now that we are all set up, enable the APIC
577 */
578 value = apic_read(APIC_SPIV);
579 value &= ~APIC_VECTOR_MASK;
580 /*
581 * Enable APIC
582 */
583 value |= APIC_SPIV_APIC_ENABLED;
584
585 /*
586 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
587 * certain networking cards. If high frequency interrupts are
588 * happening on a particular IOAPIC pin, plus the IOAPIC routing
589 * entry is masked/unmasked at a high rate as well then sooner or
590 * later IOAPIC line gets 'stuck', no more interrupts are received
591 * from the device. If focus CPU is disabled then the hang goes
592 * away, oh well :-(
593 *
594 * [ This bug can be reproduced easily with a level-triggered
595 * PCI Ne2000 networking cards and PII/PIII processors, dual
596 * BX chipset. ]
597 */
598 /*
599 * Actually disabling the focus CPU check just makes the hang less
600 * frequent as it makes the interrupt distributon model be more
601 * like LRU than MRU (the short-term load is more even across CPUs).
602 * See also the comment in end_level_ioapic_irq(). --macro
603 */
604 #if 1
605 /* Enable focus processor (bit==0) */
606 value &= ~APIC_SPIV_FOCUS_DISABLED;
607 #else
608 /* Disable focus processor (bit==1) */
609 value |= APIC_SPIV_FOCUS_DISABLED;
610 #endif
611 /*
612 * Set spurious IRQ vector
613 */
614 value |= SPURIOUS_APIC_VECTOR;
615
616 /*
617 * Enable directed EOI
618 */
619 if ( directed_eoi_enabled )
620 {
621 value |= APIC_SPIV_DIRECTED_EOI;
622 apic_printk(APIC_VERBOSE, "Suppress EOI broadcast on CPU#%d\n",
623 smp_processor_id());
624 }
625
626 apic_write(APIC_SPIV, value);
627
628 /*
629 * Set up LVT0, LVT1:
630 *
631 * set up through-local-APIC on the BP's LINT0. This is not
632 * strictly necessery in pure symmetric-IO mode, but sometimes
633 * we delegate interrupts to the 8259A.
634 */
635 /*
636 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
637 */
638 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
639 if (!smp_processor_id() && (pic_mode || !value)) {
640 value = APIC_DM_EXTINT;
641 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n",
642 smp_processor_id());
643 } else {
644 value = APIC_DM_EXTINT | APIC_LVT_MASKED;
645 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n",
646 smp_processor_id());
647 }
648 apic_write(APIC_LVT0, value);
649
650 /*
651 * only the BP should see the LINT1 NMI signal, obviously.
652 */
653 if (!smp_processor_id())
654 value = APIC_DM_NMI;
655 else
656 value = APIC_DM_NMI | APIC_LVT_MASKED;
657 apic_write(APIC_LVT1, value);
658
659 if (!esr_disable) {
660 maxlvt = get_maxlvt();
661 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
662 apic_write(APIC_ESR, 0);
663 oldvalue = apic_read(APIC_ESR);
664
665 value = ERROR_APIC_VECTOR; // enables sending errors
666 apic_write(APIC_LVTERR, value);
667 /*
668 * spec says clear errors after enabling vector.
669 */
670 if (maxlvt > 3)
671 apic_write(APIC_ESR, 0);
672 value = apic_read(APIC_ESR);
673 if (value != oldvalue)
674 apic_printk(APIC_VERBOSE, "ESR value before enabling "
675 "vector: %#lx after: %#lx\n",
676 oldvalue, value);
677 } else {
678 /*
679 * Something untraceble is creating bad interrupts on
680 * secondary quads ... for the moment, just leave the
681 * ESR disabled - we can't do anything useful with the
682 * errors anyway - mbligh
683 */
684 printk("Leaving ESR disabled.\n");
685 }
686
687 if (nmi_watchdog == NMI_LOCAL_APIC)
688 setup_apic_nmi_watchdog();
689 apic_pm_activate();
690 }
691
lapic_suspend(void)692 int lapic_suspend(void)
693 {
694 unsigned long flags;
695 int maxlvt = get_maxlvt();
696 if (!apic_pm_state.active)
697 return 0;
698
699 apic_pm_state.apic_id = apic_read(APIC_ID);
700 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
701 apic_pm_state.apic_ldr = apic_read(APIC_LDR);
702 apic_pm_state.apic_dfr = apic_read(APIC_DFR);
703 apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
704 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
705 if (maxlvt >= 4)
706 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
707
708 if (maxlvt >= 6) {
709 apic_pm_state.apic_lvtcmci = apic_read(APIC_CMCI);
710 }
711
712 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
713 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
714 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
715 apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
716 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
717 if (maxlvt >= 5)
718 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
719
720 local_irq_save(flags);
721 disable_local_APIC();
722 iommu_disable_x2apic_IR();
723 local_irq_restore(flags);
724 return 0;
725 }
726
lapic_resume(void)727 int lapic_resume(void)
728 {
729 uint64_t msr_content;
730 unsigned long flags;
731 int maxlvt;
732
733 if (!apic_pm_state.active)
734 return 0;
735
736 local_irq_save(flags);
737
738 /*
739 * Make sure the APICBASE points to the right address
740 *
741 * FIXME! This will be wrong if we ever support suspend on
742 * SMP! We'll need to do this as part of the CPU restore!
743 */
744 if ( !x2apic_enabled )
745 {
746 rdmsrl(MSR_IA32_APICBASE, msr_content);
747 msr_content &= ~MSR_IA32_APICBASE_BASE;
748 wrmsrl(MSR_IA32_APICBASE,
749 msr_content | MSR_IA32_APICBASE_ENABLE | mp_lapic_addr);
750 }
751 else
752 resume_x2apic();
753
754 maxlvt = get_maxlvt();
755 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
756 apic_write(APIC_ID, apic_pm_state.apic_id);
757 apic_write(APIC_DFR, apic_pm_state.apic_dfr);
758 apic_write(APIC_LDR, apic_pm_state.apic_ldr);
759 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
760 apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
761 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
762 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
763 if (maxlvt >= 5)
764 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
765
766 if (maxlvt >= 6) {
767 apic_write(APIC_CMCI, apic_pm_state.apic_lvtcmci);
768 }
769
770 if (maxlvt >= 4)
771 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
772 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
773 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
774 apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
775 apic_write(APIC_ESR, 0);
776 apic_read(APIC_ESR);
777 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
778 apic_write(APIC_ESR, 0);
779 apic_read(APIC_ESR);
780 local_irq_restore(flags);
781 return 0;
782 }
783
784
785 /*
786 * Detect and enable local APICs on non-SMP boards.
787 * Original code written by Keir Fraser.
788 */
789
lapic_disable(const char * str)790 static int __init lapic_disable(const char *str)
791 {
792 enable_local_apic = -1;
793 setup_clear_cpu_cap(X86_FEATURE_APIC);
794 return 0;
795 }
796 custom_param("nolapic", lapic_disable);
797 boolean_param("lapic", enable_local_apic);
798
apic_set_verbosity(const char * str)799 static int __init apic_set_verbosity(const char *str)
800 {
801 if (strcmp("debug", str) == 0)
802 apic_verbosity = APIC_DEBUG;
803 else if (strcmp("verbose", str) == 0)
804 apic_verbosity = APIC_VERBOSE;
805 else
806 return -EINVAL;
807
808 return 0;
809 }
810 custom_param("apic_verbosity", apic_set_verbosity);
811
detect_init_APIC(void)812 static int __init detect_init_APIC (void)
813 {
814 uint64_t msr_content;
815
816 /* Disabled by kernel option? */
817 if (enable_local_apic < 0)
818 return -1;
819
820 if (rdmsr_safe(MSR_IA32_APICBASE, msr_content)) {
821 printk("No local APIC present\n");
822 return -1;
823 }
824
825 if (!cpu_has_apic) {
826 /*
827 * Over-ride BIOS and try to enable the local
828 * APIC only if "lapic" specified.
829 */
830 if (enable_local_apic <= 0) {
831 printk("Local APIC disabled by BIOS -- "
832 "you can enable it with \"lapic\"\n");
833 return -1;
834 }
835 /*
836 * Some BIOSes disable the local APIC in the
837 * APIC_BASE MSR. This can only be done in
838 * software for Intel P6 or later and AMD K7
839 * (Model > 1) or later.
840 */
841 if (!(msr_content & MSR_IA32_APICBASE_ENABLE)) {
842 printk("Local APIC disabled by BIOS -- reenabling.\n");
843 msr_content &= ~MSR_IA32_APICBASE_BASE;
844 msr_content |= MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE;
845 wrmsrl(MSR_IA32_APICBASE, msr_content);
846 enabled_via_apicbase = true;
847 }
848 }
849 /*
850 * The APIC feature bit should now be enabled
851 * in `cpuid'
852 */
853 if (!(cpuid_edx(1) & cpufeat_mask(X86_FEATURE_APIC))) {
854 printk("Could not enable APIC!\n");
855 return -1;
856 }
857
858 setup_force_cpu_cap(X86_FEATURE_APIC);
859 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
860
861 /* The BIOS may have set up the APIC at some other address */
862 if (msr_content & MSR_IA32_APICBASE_ENABLE)
863 mp_lapic_addr = msr_content & MSR_IA32_APICBASE_BASE;
864
865 if (nmi_watchdog != NMI_NONE)
866 nmi_watchdog = NMI_LOCAL_APIC;
867
868 printk("Found and enabled local APIC!\n");
869
870 apic_pm_activate();
871
872 return 0;
873 }
874
x2apic_ap_setup(void)875 void x2apic_ap_setup(void)
876 {
877 if ( x2apic_enabled )
878 __enable_x2apic();
879 }
880
x2apic_bsp_setup(void)881 void __init x2apic_bsp_setup(void)
882 {
883 struct IO_APIC_route_entry **ioapic_entries = NULL;
884
885 if ( !cpu_has_x2apic )
886 return;
887
888 if ( !opt_x2apic )
889 {
890 if ( !x2apic_enabled )
891 {
892 printk("Not enabling x2APIC: disabled by cmdline.\n");
893 return;
894 }
895 printk("x2APIC: Already enabled by BIOS: Ignoring cmdline disable.\n");
896 }
897
898 if ( !iommu_supports_eim() )
899 {
900 if ( !x2apic_enabled )
901 {
902 printk("Not enabling x2APIC: depends on iommu_supports_eim.\n");
903 return;
904 }
905 panic("x2APIC: already enabled by BIOS, but "
906 "iommu_supports_eim failed");
907 }
908
909 if ( (ioapic_entries = alloc_ioapic_entries()) == NULL )
910 {
911 printk("Allocate ioapic_entries failed\n");
912 goto out;
913 }
914
915 if ( save_IO_APIC_setup(ioapic_entries) )
916 {
917 printk("Saving IO-APIC state failed\n");
918 goto out;
919 }
920
921 mask_8259A();
922 mask_IO_APIC_setup(ioapic_entries);
923
924 switch ( iommu_enable_x2apic_IR() )
925 {
926 case 0:
927 break;
928 case -ENXIO: /* ACPI_DMAR_X2APIC_OPT_OUT set */
929 if ( !x2apic_enabled )
930 {
931 printk("Not enabling x2APIC (upon firmware request)\n");
932 goto restore_out;
933 }
934 /* fall through */
935 default:
936 if ( x2apic_enabled )
937 panic("Interrupt remapping could not be enabled while "
938 "x2APIC is already enabled by BIOS");
939
940 printk(XENLOG_ERR
941 "Failed to enable Interrupt Remapping: Will not enable x2APIC.\n");
942 goto restore_out;
943 }
944
945 force_iommu = 1;
946
947 genapic = apic_x2apic_probe();
948 printk("Switched to APIC driver %s.\n", genapic->name);
949
950 if ( !x2apic_enabled )
951 {
952 x2apic_enabled = true;
953 __enable_x2apic();
954 }
955
956 restore_out:
957 restore_IO_APIC_setup(ioapic_entries);
958 unmask_8259A();
959
960 out:
961 if ( ioapic_entries )
962 free_ioapic_entries(ioapic_entries);
963 }
964
init_apic_mappings(void)965 void __init init_apic_mappings(void)
966 {
967 unsigned long apic_phys;
968
969 if ( x2apic_enabled )
970 goto __next;
971 /*
972 * If no local APIC can be found then set up a fake all
973 * zeroes page to simulate the local APIC and another
974 * one for the IO-APIC.
975 */
976 if (!smp_found_config && detect_init_APIC()) {
977 apic_phys = __pa(alloc_xenheap_page());
978 clear_page(__va(apic_phys));
979 } else
980 apic_phys = mp_lapic_addr;
981
982 set_fixmap_nocache(FIX_APIC_BASE, apic_phys);
983 apic_printk(APIC_VERBOSE, "mapped APIC to %08Lx (%08lx)\n", APIC_BASE,
984 apic_phys);
985
986 __next:
987 /*
988 * Fetch the APIC ID of the BSP in case we have a
989 * default configuration (or the MP table is broken).
990 */
991 if (boot_cpu_physical_apicid == -1U)
992 boot_cpu_physical_apicid = get_apic_id();
993 x86_cpu_to_apicid[0] = get_apic_id();
994
995 init_ioapic_mappings();
996 }
997
998 /*****************************************************************************
999 * APIC calibration
1000 *
1001 * The APIC is programmed in bus cycles.
1002 * Timeout values should specified in real time units.
1003 * The "cheapest" time source is the cyclecounter.
1004 *
1005 * Thus, we need a mappings from: bus cycles <- cycle counter <- system time
1006 *
1007 * The calibration is currently a bit shoddy since it requires the external
1008 * timer chip to generate periodic timer interupts.
1009 *****************************************************************************/
1010
1011 /* used for system time scaling */
1012 static u32 __read_mostly bus_scale; /* scaling factor: ns -> bus cycles */
1013
1014 /*
1015 * The timer chip is already set up at HZ interrupts per second here,
1016 * but we do not accept timer interrupts yet. We only allow the BP
1017 * to calibrate.
1018 */
get_8254_timer_count(void)1019 static unsigned int __init get_8254_timer_count(void)
1020 {
1021 /*extern spinlock_t i8253_lock;*/
1022 /*unsigned long flags;*/
1023
1024 unsigned int count;
1025
1026 /*spin_lock_irqsave(&i8253_lock, flags);*/
1027
1028 outb_p(0x00, PIT_MODE);
1029 count = inb_p(PIT_CH0);
1030 count |= inb_p(PIT_CH0) << 8;
1031
1032 /*spin_unlock_irqrestore(&i8253_lock, flags);*/
1033
1034 return count;
1035 }
1036
1037 /* next tick in 8254 can be caught by catching timer wraparound */
wait_8254_wraparound(void)1038 static void __init wait_8254_wraparound(void)
1039 {
1040 unsigned int curr_count, prev_count;
1041
1042 curr_count = get_8254_timer_count();
1043 do {
1044 prev_count = curr_count;
1045 curr_count = get_8254_timer_count();
1046
1047 /* workaround for broken Mercury/Neptune */
1048 if (prev_count >= curr_count + 0x100)
1049 curr_count = get_8254_timer_count();
1050
1051 } while (prev_count >= curr_count);
1052 }
1053
1054 /*
1055 * This function sets up the local APIC timer, with a timeout of
1056 * 'clocks' APIC bus clock. During calibration we actually call
1057 * this function twice on the boot CPU, once with a bogus timeout
1058 * value, second time for real. The other (noncalibrating) CPUs
1059 * call this function only once, with the real, calibrated value.
1060 *
1061 * We do reads before writes even if unnecessary, to get around the
1062 * P5 APIC double write bug.
1063 */
1064
1065 #define APIC_DIVISOR 1
1066
__setup_APIC_LVTT(unsigned int clocks)1067 static void __setup_APIC_LVTT(unsigned int clocks)
1068 {
1069 unsigned int lvtt_value, tmp_value;
1070
1071 /* NB. Xen uses local APIC timer in one-shot mode. */
1072 lvtt_value = /*APIC_TIMER_MODE_PERIODIC |*/ LOCAL_TIMER_VECTOR;
1073
1074 if ( tdt_enabled )
1075 {
1076 lvtt_value &= (~APIC_TIMER_MODE_MASK);
1077 lvtt_value |= APIC_TIMER_MODE_TSC_DEADLINE;
1078 }
1079
1080 apic_write(APIC_LVTT, lvtt_value);
1081
1082 tmp_value = apic_read(APIC_TDCR);
1083 apic_write(APIC_TDCR, tmp_value | APIC_TDR_DIV_1);
1084
1085 apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
1086 }
1087
setup_APIC_timer(void)1088 static void setup_APIC_timer(void)
1089 {
1090 unsigned long flags;
1091 local_irq_save(flags);
1092 __setup_APIC_LVTT(0);
1093 local_irq_restore(flags);
1094 }
1095
wait_tick_pvh(void)1096 static void wait_tick_pvh(void)
1097 {
1098 u64 lapse_ns = 1000000000ULL / HZ;
1099 s_time_t start, curr_time;
1100
1101 start = NOW();
1102
1103 /* Won't wrap around */
1104 do {
1105 cpu_relax();
1106 curr_time = NOW();
1107 } while ( curr_time - start < lapse_ns );
1108 }
1109
1110 /*
1111 * In this function we calibrate APIC bus clocks to the external
1112 * timer. Unfortunately we cannot use jiffies and the timer irq
1113 * to calibrate, since some later bootup code depends on getting
1114 * the first irq? Ugh.
1115 *
1116 * We want to do the calibration only once since we
1117 * want to have local timer irqs syncron. CPUs connected
1118 * by the same APIC bus have the very same bus frequency.
1119 * And we want to have irqs off anyways, no accidental
1120 * APIC irq that way.
1121 */
1122
calibrate_APIC_clock(void)1123 static int __init calibrate_APIC_clock(void)
1124 {
1125 unsigned long long t1, t2;
1126 long tt1, tt2;
1127 long result;
1128 int i;
1129 unsigned long bus_freq; /* KAF: pointer-size avoids compile warns. */
1130 u32 bus_cycle; /* length of one bus cycle in pico-seconds */
1131 const int LOOPS = HZ/10;
1132
1133 apic_printk(APIC_VERBOSE, "calibrating APIC timer ...\n");
1134
1135 /*
1136 * Put whatever arbitrary (but long enough) timeout
1137 * value into the APIC clock, we just want to get the
1138 * counter running for calibration.
1139 */
1140 __setup_APIC_LVTT(1000000000);
1141
1142 if ( !xen_guest )
1143 /*
1144 * The timer chip counts down to zero. Let's wait
1145 * for a wraparound to start exact measurement:
1146 * (the current tick might have been already half done)
1147 */
1148 wait_8254_wraparound();
1149 else
1150 wait_tick_pvh();
1151
1152 /*
1153 * We wrapped around just now. Let's start:
1154 */
1155 t1 = rdtsc_ordered();
1156 tt1 = apic_read(APIC_TMCCT);
1157
1158 /*
1159 * Let's wait LOOPS ticks:
1160 */
1161 for (i = 0; i < LOOPS; i++)
1162 if ( !xen_guest )
1163 wait_8254_wraparound();
1164 else
1165 wait_tick_pvh();
1166
1167 tt2 = apic_read(APIC_TMCCT);
1168 t2 = rdtsc_ordered();
1169
1170 /*
1171 * The APIC bus clock counter is 32 bits only, it
1172 * might have overflown, but note that we use signed
1173 * longs, thus no extra care needed.
1174 *
1175 * underflown to be exact, as the timer counts down ;)
1176 */
1177
1178 result = (tt1-tt2)*APIC_DIVISOR/LOOPS;
1179
1180 apic_printk(APIC_VERBOSE, "..... CPU clock speed is %ld.%04ld MHz.\n",
1181 ((long)(t2 - t1) / LOOPS) / (1000000 / HZ),
1182 ((long)(t2 - t1) / LOOPS) % (1000000 / HZ));
1183
1184 apic_printk(APIC_VERBOSE, "..... host bus clock speed is %ld.%04ld MHz.\n",
1185 result / (1000000 / HZ), result % (1000000 / HZ));
1186
1187 /* set up multipliers for accurate timer code */
1188 bus_freq = result*HZ;
1189 bus_cycle = (u32) (1000000000000LL/bus_freq); /* in pico seconds */
1190 bus_scale = (1000*262144)/bus_cycle;
1191
1192 apic_printk(APIC_VERBOSE, "..... bus_scale = %#x\n", bus_scale);
1193 /* reset APIC to zero timeout value */
1194 __setup_APIC_LVTT(0);
1195
1196 return result;
1197 }
1198
setup_boot_APIC_clock(void)1199 void __init setup_boot_APIC_clock(void)
1200 {
1201 unsigned long flags;
1202 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n");
1203 using_apic_timer = true;
1204
1205 local_irq_save(flags);
1206
1207 calibrate_APIC_clock();
1208
1209 if ( tdt_enable && boot_cpu_has(X86_FEATURE_TSC_DEADLINE) )
1210 {
1211 printk(KERN_DEBUG "TSC deadline timer enabled\n");
1212 tdt_enabled = true;
1213 }
1214
1215 setup_APIC_timer();
1216
1217 local_irq_restore(flags);
1218 }
1219
setup_secondary_APIC_clock(void)1220 void setup_secondary_APIC_clock(void)
1221 {
1222 setup_APIC_timer();
1223 }
1224
disable_APIC_timer(void)1225 void disable_APIC_timer(void)
1226 {
1227 if (using_apic_timer) {
1228 unsigned long v;
1229
1230 /* Work around AMD Erratum 411. This is a nice thing to do anyway. */
1231 apic_write(APIC_TMICT, 0);
1232
1233 v = apic_read(APIC_LVTT);
1234 apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
1235 }
1236 }
1237
enable_APIC_timer(void)1238 void enable_APIC_timer(void)
1239 {
1240 if (using_apic_timer) {
1241 unsigned long v;
1242
1243 v = apic_read(APIC_LVTT);
1244 apic_write(APIC_LVTT, v & ~APIC_LVT_MASKED);
1245 }
1246 }
1247
1248 #undef APIC_DIVISOR
1249
1250 /*
1251 * reprogram_timer: Reprogram the APIC timer.
1252 * Timeout is a Xen system time (nanoseconds since boot); 0 disables the timer.
1253 * Returns 1 on success; 0 if the timeout is too soon or is in the past.
1254 */
reprogram_timer(s_time_t timeout)1255 int reprogram_timer(s_time_t timeout)
1256 {
1257 s_time_t expire;
1258 u32 apic_tmict = 0;
1259
1260 /* No local APIC: timer list is polled via the PIT interrupt. */
1261 if ( !cpu_has_apic )
1262 return 1;
1263
1264 if ( tdt_enabled )
1265 {
1266 wrmsrl(MSR_IA32_TSC_DEADLINE, timeout ? stime2tsc(timeout) : 0);
1267 return 1;
1268 }
1269
1270 if ( timeout && ((expire = timeout - NOW()) > 0) )
1271 apic_tmict = min_t(u64, (bus_scale * expire) >> 18, UINT_MAX);
1272
1273 apic_write(APIC_TMICT, (unsigned long)apic_tmict);
1274
1275 return apic_tmict || !timeout;
1276 }
1277
apic_timer_interrupt(struct cpu_user_regs * regs)1278 void apic_timer_interrupt(struct cpu_user_regs * regs)
1279 {
1280 ack_APIC_irq();
1281 perfc_incr(apic_timer);
1282 raise_softirq(TIMER_SOFTIRQ);
1283 }
1284
1285 static DEFINE_PER_CPU(bool, state_dump_pending);
1286
smp_send_state_dump(unsigned int cpu)1287 void smp_send_state_dump(unsigned int cpu)
1288 {
1289 /* We overload the spurious interrupt handler to handle the dump. */
1290 per_cpu(state_dump_pending, cpu) = true;
1291 send_IPI_mask(cpumask_of(cpu), SPURIOUS_APIC_VECTOR);
1292 }
1293
1294 /*
1295 * Spurious interrupts should _never_ happen with our APIC/SMP architecture.
1296 */
spurious_interrupt(struct cpu_user_regs * regs)1297 void spurious_interrupt(struct cpu_user_regs *regs)
1298 {
1299 /*
1300 * Check if this is a vectored interrupt (most likely, as this is probably
1301 * a request to dump local CPU state). Vectored interrupts are ACKed;
1302 * spurious interrupts are not.
1303 */
1304 if (apic_isr_read(SPURIOUS_APIC_VECTOR)) {
1305 ack_APIC_irq();
1306 if (this_cpu(state_dump_pending)) {
1307 this_cpu(state_dump_pending) = false;
1308 dump_execstate(regs);
1309 goto out;
1310 }
1311 }
1312
1313 /* see sw-dev-man vol 3, chapter 7.4.13.5 */
1314 printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should "
1315 "never happen.\n", smp_processor_id());
1316
1317 out: ;
1318 }
1319
1320 /*
1321 * This interrupt should never happen with our APIC/SMP architecture
1322 */
1323
error_interrupt(struct cpu_user_regs * regs)1324 void error_interrupt(struct cpu_user_regs *regs)
1325 {
1326 static const char *const esr_fields[] = {
1327 "Send CS error",
1328 "Receive CS error",
1329 "Send accept error",
1330 "Receive accept error",
1331 "Redirectable IPI",
1332 "Send illegal vector",
1333 "Received illegal vector",
1334 "Illegal register address",
1335 };
1336 unsigned int v, v1;
1337 int i;
1338
1339 /* First tickle the hardware, only then report what went on. -- REW */
1340 v = apic_read(APIC_ESR);
1341 apic_write(APIC_ESR, 0);
1342 v1 = apic_read(APIC_ESR);
1343 ack_APIC_irq();
1344
1345 printk(XENLOG_DEBUG "APIC error on CPU%u: %02x(%02x)",
1346 smp_processor_id(), v , v1);
1347 for ( i = 7; i >= 0; --i )
1348 if ( v1 & (1 << i) )
1349 printk(", %s", esr_fields[i]);
1350 printk("\n");
1351 }
1352
1353 /*
1354 * This interrupt handles performance counters interrupt
1355 */
1356
pmu_apic_interrupt(struct cpu_user_regs * regs)1357 void pmu_apic_interrupt(struct cpu_user_regs *regs)
1358 {
1359 ack_APIC_irq();
1360 vpmu_do_interrupt(regs);
1361 }
1362
1363 /*
1364 * This initializes the IO-APIC and APIC hardware if this is
1365 * a UP kernel.
1366 */
APIC_init_uniprocessor(void)1367 int __init APIC_init_uniprocessor (void)
1368 {
1369 if (enable_local_apic < 0)
1370 setup_clear_cpu_cap(X86_FEATURE_APIC);
1371
1372 if (!smp_found_config && !cpu_has_apic) {
1373 skip_ioapic_setup = true;
1374 return -1;
1375 }
1376
1377 /*
1378 * Complain if the BIOS pretends there is one.
1379 */
1380 if (!cpu_has_apic) {
1381 printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
1382 boot_cpu_physical_apicid);
1383 skip_ioapic_setup = true;
1384 return -1;
1385 }
1386
1387 verify_local_APIC();
1388
1389 connect_bsp_APIC();
1390
1391 /*
1392 * Hack: In case of kdump, after a crash, kernel might be booting
1393 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
1394 * might be zero if read from MP tables. Get it from LAPIC.
1395 */
1396 #ifdef CONFIG_CRASH_DUMP
1397 boot_cpu_physical_apicid = get_apic_id();
1398 #endif
1399 physids_clear(phys_cpu_present_map);
1400 physid_set(boot_cpu_physical_apicid, phys_cpu_present_map);
1401
1402 setup_local_APIC();
1403
1404 if (nmi_watchdog == NMI_LOCAL_APIC)
1405 check_nmi_watchdog();
1406
1407 if (smp_found_config)
1408 if (!skip_ioapic_setup && nr_ioapics)
1409 setup_IO_APIC();
1410
1411 setup_boot_APIC_clock();
1412
1413 return 0;
1414 }
1415
apic_mode_to_str(const enum apic_mode mode)1416 static const char * __init apic_mode_to_str(const enum apic_mode mode)
1417 {
1418 switch ( mode )
1419 {
1420 case APIC_MODE_INVALID:
1421 return "invalid";
1422 case APIC_MODE_DISABLED:
1423 return "disabled";
1424 case APIC_MODE_XAPIC:
1425 return "xapic";
1426 case APIC_MODE_X2APIC:
1427 return "x2apic";
1428 default:
1429 return "unrecognised";
1430 }
1431 }
1432
1433 /* Needs to be called during startup. It records the state the BIOS
1434 * leaves the local APIC so we can undo upon kexec.
1435 */
record_boot_APIC_mode(void)1436 void __init record_boot_APIC_mode(void)
1437 {
1438 /* Sanity check - we should only ever run once, but could possibly
1439 * be called several times */
1440 if ( APIC_MODE_INVALID != apic_boot_mode )
1441 return;
1442
1443 apic_boot_mode = current_local_apic_mode();
1444
1445 apic_printk(APIC_DEBUG, "APIC boot state is '%s'\n",
1446 apic_mode_to_str(apic_boot_mode));
1447 }
1448
1449 /* Look at the bits in MSR_IA32_APICBASE and work out which
1450 * APIC mode we are in */
current_local_apic_mode(void)1451 enum apic_mode current_local_apic_mode(void)
1452 {
1453 u64 msr_contents;
1454
1455 rdmsrl(MSR_IA32_APICBASE, msr_contents);
1456
1457 /* Reading EXTD bit from the MSR is only valid if CPUID
1458 * says so, else reserved */
1459 if ( boot_cpu_has(X86_FEATURE_X2APIC)
1460 && (msr_contents & MSR_IA32_APICBASE_EXTD) )
1461 return APIC_MODE_X2APIC;
1462
1463 /* EN bit should always be valid as long as we can read the MSR
1464 */
1465 if ( msr_contents & MSR_IA32_APICBASE_ENABLE )
1466 return APIC_MODE_XAPIC;
1467
1468 return APIC_MODE_DISABLED;
1469 }
1470
1471
check_for_unexpected_msi(unsigned int vector)1472 void check_for_unexpected_msi(unsigned int vector)
1473 {
1474 BUG_ON(apic_isr_read(vector));
1475 }
1476