1 /*
2 * Copyright (c) 2021 Travis Geiselbrecht
3 *
4 * Use of this source code is governed by a MIT-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/MIT
7 */
8 #include "arch/x86/lapic.h"
9
10 #include <sys/types.h>
11 #include <lk/debug.h>
12 #include <lk/err.h>
13 #include <lk/reg.h>
14 #include <lk/trace.h>
15 #include <lk/init.h>
16 #include <lib/fixed_point.h>
17 #include <assert.h>
18 #include <kernel/thread.h>
19 #include <platform/interrupts.h>
20 #include <arch/ops.h>
21 #include <arch/x86.h>
22 #include <arch/x86/feature.h>
23 #include <kernel/spinlock.h>
24 #include <platform/time.h>
25 #include <platform/timer.h>
26 #include <platform/pc/timer.h>
27 #include <kernel/vm.h>
28 #include <kernel/mp.h>
29
30 #define LOCAL_TRACE 0
31
32 static bool lapic_present = false;
33 static bool lapic_x2apic = false;
34 static bool use_tsc_deadline = false;
35 static volatile uint32_t *lapic_mmio;
36 static struct fp_32_64 timebase_to_lapic;
37
38 // TODO: move these callbacks into the shared timer code
39 static platform_timer_callback t_callback;
40 static void *callback_arg;
41
42 static void lapic_init_percpu(uint level);
43
44 // local apic registers
45 enum lapic_regs {
46 LAPIC_ID = 0x20,
47 LAPIC_VERSION = 0x30,
48 LAPIC_TPR = 0x80,
49 LAPIC_APR = 0x90,
50 LAPIC_PPR = 0xa0,
51 LAPIC_EOI = 0xb0,
52 LAPIC_RRD = 0xc0,
53 LAPIC_LDR = 0xd0,
54 LAPIC_DFR = 0xe0,
55 LAPIC_SVR = 0xf0,
56 LAPIC_ISR0 = 0x100,
57
58 LAPIC_TMR0 = 0x180,
59
60 LAPIC_IRR0 = 0x200,
61
62 LAPIC_ESR = 0x280,
63
64 LAPIC_CMCI = 0x2f0,
65 LAPIC_ICRLO = 0x300,
66 LAPIC_ICRHI = 0x310,
67 LAPIC_TIMER = 0x320,
68 LAPIC_THERMAL = 0x330,
69 LAPIC_PERF = 0x340,
70 LAPIC_LINT0 = 0x350,
71 LAPIC_LINT1 = 0x360,
72 LAPIC_ERROR = 0x370,
73 LAPIC_TICR = 0x380,
74 LAPIC_TCCR = 0x390,
75 LAPIC_DIV = 0x3e0,
76
77 // Extended features
78 LAPIC_EXT_FEATURES = 0x400,
79 LAPIC_EXT_CONTROL = 0x410,
80 LAPIC_EXT_SEOI = 0x420,
81 LAPIC_EXT_IER0 = 0x480,
82 LAPIC_EXT_LVT0 = 0x500,
83 };
84
85 enum lapic_interrupts {
86 LAPIC_INT_TIMER = 0xf8,
87 LAPIC_INT_GENERIC,
88 LAPIC_INT_RESCHEDULE,
89
90 LAPIC_INT_SPURIOUS = 0xff, // Bits 0-3 must be 1 for P6 and below compatibility
91 };
92
93 enum lapic_timer_mode {
94 LAPIC_TIMER_MODE_ONESHOT = 0,
95 LAPIC_TIMER_MODE_PERIODIC = 1,
96 LAPIC_TIMER_MODE_TSC_DEADLINE = 2,
97 };
98
lapic_read(enum lapic_regs reg)99 static uint32_t lapic_read(enum lapic_regs reg) {
100 LTRACEF_LEVEL(2, "reg %#x\n", reg);
101 if (lapic_x2apic) {
102 // TODO: do we need barriers here?
103 DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI);
104 return read_msr(X86_MSR_IA32_X2APIC_BASE + reg / 0x10);
105 } else {
106 return mmio_read32(lapic_mmio + reg / 4);
107 }
108 }
109
lapic_write(enum lapic_regs reg,uint32_t val)110 static void lapic_write(enum lapic_regs reg, uint32_t val) {
111 LTRACEF_LEVEL(2, "reg %#x val %#x\n", reg, val);
112 if (lapic_x2apic) {
113 DEBUG_ASSERT(reg != LAPIC_ICRLO && reg != LAPIC_ICRHI);
114 write_msr(X86_MSR_IA32_X2APIC_BASE + reg / 0x10, val);
115 } else {
116 mmio_write32(lapic_mmio + reg / 4, val);
117 }
118 }
119
lapic_wait_for_icr_delivery(void)120 static void lapic_wait_for_icr_delivery(void) {
121 LTRACEF_LEVEL(2, "waiting for icr\n");
122 uint32_t val;
123 do {
124 if (lapic_x2apic) {
125 val = read_msr(X86_MSR_IA32_X2APIC_BASE + 0x30);
126 } else {
127 val = lapic_read(LAPIC_ICRLO);
128 }
129 } while (val & (1u << 12));
130 }
131
132 // special case to write to the ICR register
lapic_write_icr(uint32_t low,uint32_t apic_id)133 static void lapic_write_icr(uint32_t low, uint32_t apic_id) {
134 LTRACEF_LEVEL(2, "%#x apic_id %#x\n", low, apic_id);
135 if (lapic_x2apic) {
136 write_msr(X86_MSR_IA32_X2APIC_BASE + 0x30, ((uint64_t)apic_id << 32) | low);
137 } else {
138 lapic_write(LAPIC_ICRHI, apic_id << 24);
139 lapic_write(LAPIC_ICRLO, low);
140 lapic_wait_for_icr_delivery();
141 }
142 }
143
lapic_set_oneshot_timer(platform_timer_callback callback,void * arg,lk_time_t interval)144 status_t lapic_set_oneshot_timer(platform_timer_callback callback, void *arg, lk_time_t interval) {
145 LTRACEF("cpu %u interval %u\n", arch_curr_cpu_num(), interval);
146
147 DEBUG_ASSERT(arch_ints_disabled());
148
149 t_callback = callback;
150 callback_arg = arg;
151
152 if (use_tsc_deadline) {
153 uint64_t now = __builtin_ia32_rdtsc();
154 uint64_t delta = time_to_tsc_ticks(interval);
155 uint64_t deadline = now + delta;
156 LTRACEF("now %llu delta %llu deadline %llu\n", now, delta, deadline);
157 write_msr(X86_MSR_IA32_TSC_DEADLINE, deadline);
158 } else {
159 // set the initial count, which should trigger the timer
160 uint64_t ticks = u64_mul_u32_fp32_64(interval, timebase_to_lapic);
161 if (ticks > UINT32_MAX) {
162 ticks = UINT32_MAX;
163 }
164
165 lapic_write(LAPIC_TICR, ticks & 0xffffffff);
166 }
167
168 return NO_ERROR;
169 }
170
lapic_cancel_timer(void)171 void lapic_cancel_timer(void) {
172 LTRACE;
173
174 DEBUG_ASSERT(arch_ints_disabled());
175
176 if (use_tsc_deadline) {
177 write_msr(X86_MSR_IA32_TSC_DEADLINE, 0);
178 } else {
179 lapic_write(LAPIC_TICR, 0);
180 }
181 }
182
lapic_timer_handler(void * arg)183 static enum handler_return lapic_timer_handler(void *arg) {
184 LTRACEF("cpu %u\n", arch_curr_cpu_num());
185
186 enum handler_return ret = INT_NO_RESCHEDULE;
187 if (t_callback) {
188 ret = t_callback(callback_arg, current_time());
189 }
190
191 return ret;
192 }
193
lapic_spurious_handler(void * arg)194 static enum handler_return lapic_spurious_handler(void *arg) {
195 LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg);
196
197 return INT_NO_RESCHEDULE;
198 }
199
lapic_generic_handler(void * arg)200 static enum handler_return lapic_generic_handler(void *arg) {
201 LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg);
202
203 return INT_NO_RESCHEDULE;
204 }
205
lapic_reschedule_handler(void * arg)206 static enum handler_return lapic_reschedule_handler(void *arg) {
207 LTRACEF("cpu %u, arg %p\n", arch_curr_cpu_num(), arg);
208
209 return mp_mbx_reschedule_irq();
210 }
211
lapic_init(void)212 void lapic_init(void) {
213 lapic_present = x86_feature_test(X86_FEATURE_APIC);
214 }
215
lapic_init_postvm(uint level)216 static void lapic_init_postvm(uint level) {
217 if (!lapic_present) {
218 return;
219 }
220
221 dprintf(INFO, "X86: local apic detected\n");
222
223 // IA32_APIC_BASE_MSR
224 uint64_t apic_base = read_msr(X86_MSR_IA32_APIC_BASE);
225 LTRACEF("raw apic base msr %#llx\n", apic_base);
226
227 // make sure it's enabled
228 if ((apic_base & (1u<<11)) == 0) {
229 dprintf(INFO, "X86: enabling lapic\n");
230 apic_base |= (1u<<11);
231 write_msr(X86_MSR_IA32_APIC_BASE, apic_base);
232 }
233
234 dprintf(INFO, "X86: lapic physical address %#llx\n", apic_base & ~0xfff);
235
236 // see if x2APIC mode is supported and enable
237 if (x86_feature_test(X86_FEATURE_X2APIC)) {
238 lapic_x2apic = true;
239 dprintf(INFO, "X86: local apic supports x2APIC mode\n");
240
241 write_msr(X86_MSR_IA32_APIC_BASE, apic_base | (1u<<10));
242 }
243
244 // map the lapic into the kernel since it's not guaranteed that the physmap covers it
245 if (!lapic_mmio) {
246 LTRACEF("mapping lapic into kernel\n");
247 status_t err = vmm_alloc_physical(vmm_get_kernel_aspace(), "lapic", PAGE_SIZE, (void **)&lapic_mmio, 0,
248 apic_base & ~0xfff, /* vmm_flags */ 0, ARCH_MMU_FLAG_UNCACHED_DEVICE);
249 ASSERT(err == NO_ERROR);
250 }
251
252 // Read the local apic id and version and features
253 uint32_t id = lapic_read(LAPIC_ID);
254 uint32_t version = lapic_read(LAPIC_VERSION);
255 bool eas = version & (1u<<31);
256 uint32_t max_lvt = (version >> 16) & 0xff;
257 version &= 0xff;
258 dprintf(INFO, "X86: local apic id %#x version %#x\n", id, version);
259 dprintf(INFO, "X86: local apic max lvt entries %u\n", max_lvt);
260 if (eas) {
261 dprintf(INFO, "X86: local apic EAS features %#x\n", lapic_read(LAPIC_EXT_FEATURES));
262 }
263
264 // Finish up some local initialization that all cpus will want to do
265 lapic_init_percpu(0);
266 }
267 LK_INIT_HOOK(lapic_init_postvm, lapic_init_postvm, LK_INIT_LEVEL_VM + 1);
268
lapic_init_percpu(uint level)269 static void lapic_init_percpu(uint level) {
270 // Make sure the apic is enabled and x2apic mode is set (if supported)
271 uint64_t apic_base = read_msr(X86_MSR_IA32_APIC_BASE);
272 apic_base |= (1u<<11);
273 if (lapic_x2apic) {
274 apic_base |= (1u<<10);
275 }
276 write_msr(X86_MSR_IA32_APIC_BASE, apic_base);
277
278 // set the spurious vector register
279 uint32_t svr = (LAPIC_INT_SPURIOUS | (1u<<8)); // enable
280 lapic_write(LAPIC_SVR, svr);
281
282 LTRACEF("lapic svr %#x\n", lapic_read(LAPIC_SVR));
283
284 register_int_handler_msi(LAPIC_INT_SPURIOUS, &lapic_spurious_handler, NULL, false);
285 register_int_handler_msi(LAPIC_INT_GENERIC, &lapic_generic_handler, NULL, false);
286 register_int_handler_msi(LAPIC_INT_RESCHEDULE, &lapic_reschedule_handler, NULL, false);
287 }
288
289 LK_INIT_HOOK_FLAGS(lapic_init_percpu, lapic_init_percpu, LK_INIT_LEVEL_VM, LK_INIT_FLAG_SECONDARY_CPUS);
290
lapic_read_current_tick(void)291 static uint32_t lapic_read_current_tick(void) {
292 if (!lapic_present) {
293 return 0;
294 }
295
296 return lapic_read(LAPIC_TCCR);
297 }
298
lapic_timer_init_percpu(uint level)299 static void lapic_timer_init_percpu(uint level) {
300 // check for deadline mode
301 if (use_tsc_deadline) {
302 // put the timer in TSC deadline and clear the match register
303 uint32_t val = (LAPIC_TIMER_MODE_TSC_DEADLINE << 17) | LAPIC_INT_TIMER;
304 lapic_write(LAPIC_TIMER, val);
305 write_msr(X86_MSR_IA32_TSC_DEADLINE, 0);
306 } else {
307 // configure the local timer and make sure it is not set to fire
308 uint32_t val = (LAPIC_TIMER_MODE_ONESHOT << 17) | LAPIC_INT_TIMER;
309 lapic_write(LAPIC_TIMER, val);
310 lapic_write(LAPIC_TICR, 0);
311 }
312
313 // register the timer interrupt vector
314 register_int_handler_msi(LAPIC_INT_TIMER, &lapic_timer_handler, NULL, false);
315 }
316 LK_INIT_HOOK_FLAGS(lapic_timer_init_percpu, lapic_timer_init_percpu, LK_INIT_LEVEL_VM + 1, LK_INIT_FLAG_SECONDARY_CPUS);
317
lapic_timer_init(bool invariant_tsc_supported)318 status_t lapic_timer_init(bool invariant_tsc_supported) {
319 if (!lapic_present) {
320 return ERR_NOT_FOUND;
321 }
322
323 // check for deadline mode
324 bool tsc_deadline = x86_feature_test(X86_FEATURE_TSC_DEADLINE);
325 if (invariant_tsc_supported && tsc_deadline) {
326 dprintf(INFO, "X86: local apic timer supports TSC deadline mode\n");
327 use_tsc_deadline = true;
328 } else {
329 // configure the local timer and make sure it is not set to fire
330 uint32_t val = (LAPIC_TIMER_MODE_ONESHOT << 17) | LAPIC_INT_TIMER;
331 lapic_write(LAPIC_TIMER, val);
332
333 // calibrate the timer frequency
334 lapic_write(LAPIC_TICR, 0xffffffff); // countdown from the max count
335 uint32_t lapic_hz = pit_calibrate_lapic(&lapic_read_current_tick);
336 lapic_write(LAPIC_TICR, 0);
337 printf("X86: local apic timer frequency %uHz\n", lapic_hz);
338
339 fp_32_64_div_32_32(&timebase_to_lapic, lapic_hz, 1000);
340 dprintf(INFO, "X86: timebase to local apic timer ratio %u.%08u...\n",
341 timebase_to_lapic.l0, timebase_to_lapic.l32);
342 }
343
344 lapic_timer_init_percpu(0);
345
346 return NO_ERROR;
347 }
348
lapic_eoi(unsigned int vector)349 void lapic_eoi(unsigned int vector) {
350 LTRACEF("vector %#x\n", vector);
351 if (!lapic_present) {
352 return;
353 }
354
355 lapic_write(LAPIC_EOI, 0);
356 }
357
lapic_send_init_ipi(uint32_t apic_id,bool level)358 void lapic_send_init_ipi(uint32_t apic_id, bool level) {
359 if (!lapic_present) {
360 return;
361 }
362
363 // Level triggered mode, level according to arg, INIT delivery mode, no shorthand
364 lapic_write_icr((1u << 15) | (level ? (1u << 14) : 0) | (5u << 8), apic_id);
365 }
366
lapic_send_startup_ipi(uint32_t apic_id,uint32_t startup_vector)367 void lapic_send_startup_ipi(uint32_t apic_id, uint32_t startup_vector) {
368 if (!lapic_present) {
369 return;
370 }
371
372 // Startup IPI, no shorthand
373 lapic_write_icr((6u << 8) | (startup_vector >> 12), apic_id);
374 }
375
lapic_send_ipi(uint32_t apic_id,mp_ipi_t ipi)376 void lapic_send_ipi(uint32_t apic_id, mp_ipi_t ipi) {
377 if (!lapic_present) {
378 return;
379 }
380
381 LTRACEF("cpu %u target apic_id %#x, ipi %u\n", arch_curr_cpu_num(), apic_id, ipi);
382
383 uint32_t vector;
384 switch (ipi) {
385 case MP_IPI_GENERIC:
386 vector = LAPIC_INT_GENERIC;
387 break;
388 case MP_IPI_RESCHEDULE:
389 vector = LAPIC_INT_RESCHEDULE;
390 break;
391 default:
392 panic("X86: unknown IPI %u\n", ipi);
393 }
394
395 // send fixed mode, level asserted, no destination shorthand interrupt
396 lapic_write_icr(vector | (1U << 14), apic_id);
397 }