1 #include <xen/init.h>
2 #include <xen/string.h>
3 #include <xen/delay.h>
4 #include <xen/smp.h>
5 #include <asm/current.h>
6 #include <asm/processor.h>
7 #include <asm/xstate.h>
8 #include <asm/msr.h>
9 #include <asm/io.h>
10 #include <asm/mpspec.h>
11 #include <asm/apic.h>
12 #include <mach_apic.h>
13 #include <asm/setup.h>
14 #include <public/sysctl.h> /* for XEN_INVALID_{SOCKET,CORE}_ID */
15
16 #include "cpu.h"
17
18 bool_t opt_arat = 1;
19 boolean_param("arat", opt_arat);
20
21 /* pku: Flag to enable Memory Protection Keys (default on). */
22 static bool_t opt_pku = 1;
23 boolean_param("pku", opt_pku);
24
25 unsigned int opt_cpuid_mask_ecx = ~0u;
26 integer_param("cpuid_mask_ecx", opt_cpuid_mask_ecx);
27 unsigned int opt_cpuid_mask_edx = ~0u;
28 integer_param("cpuid_mask_edx", opt_cpuid_mask_edx);
29
30 unsigned int opt_cpuid_mask_xsave_eax = ~0u;
31 integer_param("cpuid_mask_xsave_eax", opt_cpuid_mask_xsave_eax);
32
33 unsigned int opt_cpuid_mask_ext_ecx = ~0u;
34 integer_param("cpuid_mask_ext_ecx", opt_cpuid_mask_ext_ecx);
35 unsigned int opt_cpuid_mask_ext_edx = ~0u;
36 integer_param("cpuid_mask_ext_edx", opt_cpuid_mask_ext_edx);
37
38 unsigned int __initdata expected_levelling_cap;
39 unsigned int __read_mostly levelling_caps;
40
41 DEFINE_PER_CPU(struct cpuidmasks, cpuidmasks);
42 struct cpuidmasks __read_mostly cpuidmask_defaults;
43
44 const struct cpu_dev *__read_mostly cpu_devs[X86_VENDOR_NUM] = {};
45
46 unsigned int paddr_bits __read_mostly = 36;
47 unsigned int hap_paddr_bits __read_mostly = 36;
48 unsigned int vaddr_bits __read_mostly = VADDR_BITS;
49
50 /*
51 * Default host IA32_CR_PAT value to cover all memory types.
52 * BIOS usually sets it to 0x07040600070406.
53 */
54 u64 host_pat = 0x050100070406;
55
56 static unsigned int cleared_caps[NCAPINTS];
57 static unsigned int forced_caps[NCAPINTS];
58
setup_clear_cpu_cap(unsigned int cap)59 void __init setup_clear_cpu_cap(unsigned int cap)
60 {
61 const uint32_t *dfs;
62 unsigned int i;
63
64 if (__test_and_set_bit(cap, cleared_caps))
65 return;
66
67 if (test_bit(cap, forced_caps))
68 printk("%pS clearing previously forced feature %#x\n",
69 __builtin_return_address(0), cap);
70
71 __clear_bit(cap, boot_cpu_data.x86_capability);
72 dfs = lookup_deep_deps(cap);
73
74 if (!dfs)
75 return;
76
77 for (i = 0; i < FSCAPINTS; ++i) {
78 cleared_caps[i] |= dfs[i];
79 boot_cpu_data.x86_capability[i] &= ~dfs[i];
80 if (!(forced_caps[i] & dfs[i]))
81 continue;
82 printk("%pS implicitly clearing previously forced feature(s) %u:%#x\n",
83 __builtin_return_address(0),
84 i, forced_caps[i] & dfs[i]);
85 }
86 }
87
setup_force_cpu_cap(unsigned int cap)88 void __init setup_force_cpu_cap(unsigned int cap)
89 {
90 if (__test_and_set_bit(cap, forced_caps))
91 return;
92
93 if (test_bit(cap, cleared_caps)) {
94 printk("%pS tries to force previously cleared feature %#x\n",
95 __builtin_return_address(0), cap);
96 return;
97 }
98
99 __set_bit(cap, boot_cpu_data.x86_capability);
100 }
101
default_init(struct cpuinfo_x86 * c)102 static void default_init(struct cpuinfo_x86 * c)
103 {
104 /* Not much we can do here... */
105 /* Check if at least it has cpuid */
106 BUG_ON(c->cpuid_level == -1);
107 __clear_bit(X86_FEATURE_SEP, c->x86_capability);
108 }
109
110 static const struct cpu_dev default_cpu = {
111 .c_init = default_init,
112 .c_vendor = "Unknown",
113 };
114 static const struct cpu_dev *this_cpu = &default_cpu;
115
116 static DEFINE_PER_CPU(uint64_t, msr_misc_features);
117 void (* __read_mostly ctxt_switch_masking)(const struct vcpu *next);
118
probe_cpuid_faulting(void)119 bool __init probe_cpuid_faulting(void)
120 {
121 uint64_t val;
122
123 if (rdmsr_safe(MSR_INTEL_PLATFORM_INFO, val) ||
124 !(val & MSR_PLATFORM_INFO_CPUID_FAULTING) ||
125 rdmsr_safe(MSR_INTEL_MISC_FEATURES_ENABLES,
126 this_cpu(msr_misc_features)))
127 {
128 setup_clear_cpu_cap(X86_FEATURE_CPUID_FAULTING);
129 return false;
130 }
131
132 expected_levelling_cap |= LCAP_faulting;
133 levelling_caps |= LCAP_faulting;
134 setup_force_cpu_cap(X86_FEATURE_CPUID_FAULTING);
135
136 return true;
137 }
138
set_cpuid_faulting(bool enable)139 static void set_cpuid_faulting(bool enable)
140 {
141 uint64_t *this_misc_features = &this_cpu(msr_misc_features);
142 uint64_t val = *this_misc_features;
143
144 if (!!(val & MSR_MISC_FEATURES_CPUID_FAULTING) == enable)
145 return;
146
147 val ^= MSR_MISC_FEATURES_CPUID_FAULTING;
148
149 wrmsrl(MSR_INTEL_MISC_FEATURES_ENABLES, val);
150 *this_misc_features = val;
151 }
152
ctxt_switch_levelling(const struct vcpu * next)153 void ctxt_switch_levelling(const struct vcpu *next)
154 {
155 const struct domain *nextd = next ? next->domain : NULL;
156
157 if (cpu_has_cpuid_faulting) {
158 /*
159 * No need to alter the faulting setting if we are switching
160 * to idle; it won't affect any code running in idle context.
161 */
162 if (nextd && is_idle_domain(nextd))
163 return;
164 /*
165 * We *should* be enabling faulting for the control domain.
166 *
167 * Unfortunately, the domain builder (having only ever been a
168 * PV guest) expects to be able to see host cpuid state in a
169 * native CPUID instruction, to correctly build a CPUID policy
170 * for HVM guests (notably the xstate leaves).
171 *
172 * This logic is fundimentally broken for HVM toolstack
173 * domains, and faulting causes PV guests to behave like HVM
174 * guests from their point of view.
175 *
176 * Future development plans will move responsibility for
177 * generating the maximum full cpuid policy into Xen, at which
178 * this problem will disappear.
179 */
180 set_cpuid_faulting(nextd && !is_control_domain(nextd) &&
181 (is_pv_domain(nextd) ||
182 next->arch.msr->
183 misc_features_enables.cpuid_faulting));
184 return;
185 }
186
187 if (ctxt_switch_masking)
188 ctxt_switch_masking(next);
189 }
190
191 bool_t opt_cpu_info;
192 boolean_param("cpuinfo", opt_cpu_info);
193
get_model_name(struct cpuinfo_x86 * c)194 int get_model_name(struct cpuinfo_x86 *c)
195 {
196 unsigned int *v;
197 char *p, *q;
198
199 if (c->extended_cpuid_level < 0x80000004)
200 return 0;
201
202 v = (unsigned int *) c->x86_model_id;
203 cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
204 cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
205 cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
206 c->x86_model_id[48] = 0;
207
208 /* Intel chips right-justify this string for some dumb reason;
209 undo that brain damage */
210 p = q = &c->x86_model_id[0];
211 while ( *p == ' ' )
212 p++;
213 if ( p != q ) {
214 while ( *p )
215 *q++ = *p++;
216 while ( q <= &c->x86_model_id[48] )
217 *q++ = '\0'; /* Zero-pad the rest */
218 }
219
220 return 1;
221 }
222
223
display_cacheinfo(struct cpuinfo_x86 * c)224 void display_cacheinfo(struct cpuinfo_x86 *c)
225 {
226 unsigned int dummy, ecx, edx, l2size;
227
228 if (c->extended_cpuid_level >= 0x80000005) {
229 cpuid(0x80000005, &dummy, &dummy, &ecx, &edx);
230 if (opt_cpu_info)
231 printk("CPU: L1 I cache %dK (%d bytes/line),"
232 " D cache %dK (%d bytes/line)\n",
233 edx>>24, edx&0xFF, ecx>>24, ecx&0xFF);
234 c->x86_cache_size=(ecx>>24)+(edx>>24);
235 }
236
237 if (c->extended_cpuid_level < 0x80000006) /* Some chips just has a large L1. */
238 return;
239
240 ecx = cpuid_ecx(0x80000006);
241 l2size = ecx >> 16;
242
243 c->x86_cache_size = l2size;
244
245 if (opt_cpu_info)
246 printk("CPU: L2 Cache: %dK (%d bytes/line)\n",
247 l2size, ecx & 0xFF);
248 }
249
get_cpu_vendor(uint32_t b,uint32_t c,uint32_t d,enum get_cpu_vendor mode)250 int get_cpu_vendor(uint32_t b, uint32_t c, uint32_t d, enum get_cpu_vendor mode)
251 {
252 int i;
253 static int printed;
254
255 for (i = 0; i < X86_VENDOR_NUM; i++) {
256 if (cpu_devs[i]) {
257 struct {
258 uint32_t b, d, c;
259 } *ptr = (void *)cpu_devs[i]->c_ident;
260
261 if (ptr->b == b && ptr->c == c && ptr->d == d) {
262 if (mode == gcv_host)
263 this_cpu = cpu_devs[i];
264 return i;
265 }
266 }
267 }
268 if (mode == gcv_guest)
269 return X86_VENDOR_UNKNOWN;
270 if (!printed) {
271 printed++;
272 printk(KERN_ERR "CPU: Vendor unknown, using generic init.\n");
273 printk(KERN_ERR "CPU: Your system may be unstable.\n");
274 }
275 this_cpu = &default_cpu;
276
277 return X86_VENDOR_UNKNOWN;
278 }
279
_phys_pkg_id(u32 cpuid_apic,int index_msb)280 static inline u32 _phys_pkg_id(u32 cpuid_apic, int index_msb)
281 {
282 return cpuid_apic >> index_msb;
283 }
284
285 /*
286 * cpuid returns the value latched in the HW at reset, not the APIC ID
287 * register's value. For any box whose BIOS changes APIC IDs, like
288 * clustered APIC systems, we must use get_apic_id().
289 *
290 * See Intel's IA-32 SW Dev's Manual Vol2 under CPUID.
291 */
phys_pkg_id(u32 cpuid_apic,int index_msb)292 static inline u32 phys_pkg_id(u32 cpuid_apic, int index_msb)
293 {
294 return _phys_pkg_id(get_apic_id(), index_msb);
295 }
296
297 /* Do minimum CPU detection early.
298 Fields really needed: vendor, cpuid_level, family, model, mask, cache alignment.
299 The others are not touched to avoid unwanted side effects.
300
301 WARNING: this function is only called on the BP. Don't add code here
302 that is supposed to run on all CPUs. */
early_cpu_detect(void)303 static void __init early_cpu_detect(void)
304 {
305 struct cpuinfo_x86 *c = &boot_cpu_data;
306 u32 eax, ebx, ecx, edx;
307
308 c->x86_cache_alignment = 32;
309
310 /* Get vendor name */
311 cpuid(0x00000000, &c->cpuid_level, &ebx, &ecx, &edx);
312 *(u32 *)&c->x86_vendor_id[0] = ebx;
313 *(u32 *)&c->x86_vendor_id[8] = ecx;
314 *(u32 *)&c->x86_vendor_id[4] = edx;
315
316 c->x86_vendor = get_cpu_vendor(ebx, ecx, edx, gcv_host);
317
318 cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
319 c->x86 = get_cpu_family(eax, &c->x86_model, &c->x86_mask);
320
321 edx &= ~cleared_caps[cpufeat_word(X86_FEATURE_FPU)];
322 ecx &= ~cleared_caps[cpufeat_word(X86_FEATURE_SSE3)];
323 if (edx & cpufeat_mask(X86_FEATURE_CLFLUSH))
324 c->x86_cache_alignment = ((ebx >> 8) & 0xff) * 8;
325 /* Leaf 0x1 capabilities filled in early for Xen. */
326 c->x86_capability[cpufeat_word(X86_FEATURE_FPU)] = edx;
327 c->x86_capability[cpufeat_word(X86_FEATURE_SSE3)] = ecx;
328
329 printk(XENLOG_INFO
330 "CPU Vendor: %s, Family %u (%#x), Model %u (%#x), Stepping %u (raw %08x)\n",
331 this_cpu->c_vendor, c->x86, c->x86,
332 c->x86_model, c->x86_model, c->x86_mask, eax);
333
334 eax = cpuid_eax(0x80000000);
335 if ((eax >> 16) == 0x8000 && eax >= 0x80000008) {
336 eax = cpuid_eax(0x80000008);
337 paddr_bits = eax & 0xff;
338 if (paddr_bits > PADDR_BITS)
339 paddr_bits = PADDR_BITS;
340 vaddr_bits = (eax >> 8) & 0xff;
341 if (vaddr_bits > VADDR_BITS)
342 vaddr_bits = VADDR_BITS;
343 hap_paddr_bits = ((eax >> 16) & 0xff) ?: paddr_bits;
344 if (hap_paddr_bits > PADDR_BITS)
345 hap_paddr_bits = PADDR_BITS;
346 }
347
348 initialize_cpu_data(0);
349 }
350
generic_identify(struct cpuinfo_x86 * c)351 static void generic_identify(struct cpuinfo_x86 *c)
352 {
353 u32 eax, ebx, ecx, edx, tmp;
354
355 /* Get vendor name */
356 cpuid(0x00000000, &c->cpuid_level, &ebx, &ecx, &edx);
357 *(u32 *)&c->x86_vendor_id[0] = ebx;
358 *(u32 *)&c->x86_vendor_id[8] = ecx;
359 *(u32 *)&c->x86_vendor_id[4] = edx;
360
361 c->x86_vendor = get_cpu_vendor(ebx, ecx, edx, gcv_host);
362 /* Initialize the standard set of capabilities */
363 /* Note that the vendor-specific code below might override */
364
365 /* Model and family information. */
366 cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
367 c->x86 = get_cpu_family(eax, &c->x86_model, &c->x86_mask);
368 c->apicid = phys_pkg_id((ebx >> 24) & 0xFF, 0);
369 c->phys_proc_id = c->apicid;
370
371 if (this_cpu->c_early_init)
372 this_cpu->c_early_init(c);
373
374 /* c_early_init() may have adjusted cpuid levels/features. Reread. */
375 c->cpuid_level = cpuid_eax(0);
376 cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
377 c->x86_capability[cpufeat_word(X86_FEATURE_FPU)] = edx;
378 c->x86_capability[cpufeat_word(X86_FEATURE_SSE3)] = ecx;
379
380 if ( cpu_has(c, X86_FEATURE_CLFLUSH) )
381 c->x86_clflush_size = ((ebx >> 8) & 0xff) * 8;
382
383 if ( (c->cpuid_level >= CPUID_PM_LEAF) &&
384 (cpuid_ecx(CPUID_PM_LEAF) & CPUID6_ECX_APERFMPERF_CAPABILITY) )
385 set_bit(X86_FEATURE_APERFMPERF, c->x86_capability);
386
387 /* AMD-defined flags: level 0x80000001 */
388 c->extended_cpuid_level = cpuid_eax(0x80000000);
389 if ((c->extended_cpuid_level >> 16) != 0x8000)
390 c->extended_cpuid_level = 0;
391 if (c->extended_cpuid_level > 0x80000000)
392 cpuid(0x80000001, &tmp, &tmp,
393 &c->x86_capability[cpufeat_word(X86_FEATURE_LAHF_LM)],
394 &c->x86_capability[cpufeat_word(X86_FEATURE_SYSCALL)]);
395 if (c == &boot_cpu_data)
396 bootsym(cpuid_ext_features) =
397 c->x86_capability[cpufeat_word(X86_FEATURE_NX)];
398
399 if (c->extended_cpuid_level >= 0x80000004)
400 get_model_name(c); /* Default name */
401 if (c->extended_cpuid_level >= 0x80000007)
402 c->x86_capability[cpufeat_word(X86_FEATURE_ITSC)]
403 = cpuid_edx(0x80000007);
404 if (c->extended_cpuid_level >= 0x80000008)
405 c->x86_capability[cpufeat_word(X86_FEATURE_CLZERO)]
406 = cpuid_ebx(0x80000008);
407
408 /* Intel-defined flags: level 0x00000007 */
409 if ( c->cpuid_level >= 0x00000007 )
410 cpuid_count(0x00000007, 0, &tmp,
411 &c->x86_capability[cpufeat_word(X86_FEATURE_FSGSBASE)],
412 &c->x86_capability[cpufeat_word(X86_FEATURE_PKU)],
413 &c->x86_capability[cpufeat_word(X86_FEATURE_AVX512_4VNNIW)]);
414 }
415
416 /*
417 * This does the hard work of actually picking apart the CPU stuff...
418 */
identify_cpu(struct cpuinfo_x86 * c)419 void identify_cpu(struct cpuinfo_x86 *c)
420 {
421 int i;
422
423 c->x86_cache_size = -1;
424 c->x86_vendor = X86_VENDOR_UNKNOWN;
425 c->cpuid_level = -1; /* CPUID not detected */
426 c->x86_model = c->x86_mask = 0; /* So far unknown... */
427 c->x86_vendor_id[0] = '\0'; /* Unset */
428 c->x86_model_id[0] = '\0'; /* Unset */
429 c->x86_max_cores = 1;
430 c->x86_num_siblings = 1;
431 c->x86_clflush_size = 0;
432 c->phys_proc_id = XEN_INVALID_SOCKET_ID;
433 c->cpu_core_id = XEN_INVALID_CORE_ID;
434 c->compute_unit_id = INVALID_CUID;
435 memset(&c->x86_capability, 0, sizeof c->x86_capability);
436
437 generic_identify(c);
438
439 #ifdef NOISY_CAPS
440 printk(KERN_DEBUG "CPU: After vendor identify, caps:");
441 for (i = 0; i < NCAPINTS; i++)
442 printk(" %08x", c->x86_capability[i]);
443 printk("\n");
444 #endif
445
446 /*
447 * Vendor-specific initialization. In this section we
448 * canonicalize the feature flags, meaning if there are
449 * features a certain CPU supports which CPUID doesn't
450 * tell us, CPUID claiming incorrect flags, or other bugs,
451 * we handle them here.
452 *
453 * At the end of this section, c->x86_capability better
454 * indicate the features this CPU genuinely supports!
455 */
456 if (this_cpu->c_init)
457 this_cpu->c_init(c);
458
459
460 if ( !opt_pku )
461 setup_clear_cpu_cap(X86_FEATURE_PKU);
462
463 /*
464 * The vendor-specific functions might have changed features. Now
465 * we do "generic changes."
466 */
467 for (i = 0; i < FSCAPINTS; ++i)
468 c->x86_capability[i] &= known_features[i];
469
470 for (i = 0 ; i < NCAPINTS ; ++i) {
471 c->x86_capability[i] |= forced_caps[i];
472 c->x86_capability[i] &= ~cleared_caps[i];
473 }
474
475 /* If the model name is still unset, do table lookup. */
476 if ( !c->x86_model_id[0] ) {
477 /* Last resort... */
478 snprintf(c->x86_model_id, sizeof(c->x86_model_id),
479 "%02x/%02x", c->x86_vendor, c->x86_model);
480 }
481
482 /* Now the feature flags better reflect actual CPU features! */
483
484 if ( cpu_has_xsave )
485 xstate_init(c);
486
487 #ifdef NOISY_CAPS
488 printk(KERN_DEBUG "CPU: After all inits, caps:");
489 for (i = 0; i < NCAPINTS; i++)
490 printk(" %08x", c->x86_capability[i]);
491 printk("\n");
492 #endif
493
494 /*
495 * On SMP, boot_cpu_data holds the common feature set between
496 * all CPUs; so make sure that we indicate which features are
497 * common between the CPUs. The first time this routine gets
498 * executed, c == &boot_cpu_data.
499 */
500 if ( c != &boot_cpu_data ) {
501 /* AND the already accumulated flags with these */
502 for ( i = 0 ; i < NCAPINTS ; i++ )
503 boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
504
505 mcheck_init(c, false);
506 } else {
507 mcheck_init(c, true);
508
509 mtrr_bp_init();
510 }
511 }
512
513 /* leaf 0xb SMT level */
514 #define SMT_LEVEL 0
515
516 /* leaf 0xb sub-leaf types */
517 #define INVALID_TYPE 0
518 #define SMT_TYPE 1
519 #define CORE_TYPE 2
520
521 #define LEAFB_SUBTYPE(ecx) (((ecx) >> 8) & 0xff)
522 #define BITS_SHIFT_NEXT_LEVEL(eax) ((eax) & 0x1f)
523 #define LEVEL_MAX_SIBLINGS(ebx) ((ebx) & 0xffff)
524
525 /*
526 * Check for extended topology enumeration cpuid leaf 0xb and if it
527 * exists, use it for cpu topology detection.
528 */
detect_extended_topology(struct cpuinfo_x86 * c)529 void detect_extended_topology(struct cpuinfo_x86 *c)
530 {
531 unsigned int eax, ebx, ecx, edx, sub_index;
532 unsigned int ht_mask_width, core_plus_mask_width;
533 unsigned int core_select_mask, core_level_siblings;
534 unsigned int initial_apicid;
535
536 if ( c->cpuid_level < 0xb )
537 return;
538
539 cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx);
540
541 /* Check if the cpuid leaf 0xb is actually implemented */
542 if ( ebx == 0 || (LEAFB_SUBTYPE(ecx) != SMT_TYPE) )
543 return;
544
545 __set_bit(X86_FEATURE_XTOPOLOGY, c->x86_capability);
546
547 initial_apicid = edx;
548
549 /* Populate HT related information from sub-leaf level 0 */
550 core_level_siblings = c->x86_num_siblings = LEVEL_MAX_SIBLINGS(ebx);
551 core_plus_mask_width = ht_mask_width = BITS_SHIFT_NEXT_LEVEL(eax);
552
553 sub_index = 1;
554 do {
555 cpuid_count(0xb, sub_index, &eax, &ebx, &ecx, &edx);
556
557 /* Check for the Core type in the implemented sub leaves */
558 if ( LEAFB_SUBTYPE(ecx) == CORE_TYPE ) {
559 core_level_siblings = LEVEL_MAX_SIBLINGS(ebx);
560 core_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax);
561 break;
562 }
563
564 sub_index++;
565 } while ( LEAFB_SUBTYPE(ecx) != INVALID_TYPE );
566
567 core_select_mask = (~(~0u << core_plus_mask_width)) >> ht_mask_width;
568
569 c->cpu_core_id = phys_pkg_id(initial_apicid, ht_mask_width)
570 & core_select_mask;
571 c->phys_proc_id = phys_pkg_id(initial_apicid, core_plus_mask_width);
572
573 c->apicid = phys_pkg_id(initial_apicid, 0);
574 c->x86_max_cores = (core_level_siblings / c->x86_num_siblings);
575
576 if ( opt_cpu_info )
577 {
578 printk("CPU: Physical Processor ID: %d\n",
579 c->phys_proc_id);
580 if ( c->x86_max_cores > 1 )
581 printk("CPU: Processor Core ID: %d\n",
582 c->cpu_core_id);
583 }
584 }
585
detect_ht(struct cpuinfo_x86 * c)586 void detect_ht(struct cpuinfo_x86 *c)
587 {
588 u32 eax, ebx, ecx, edx;
589 int index_msb, core_bits;
590
591 if (!cpu_has(c, X86_FEATURE_HTT) ||
592 cpu_has(c, X86_FEATURE_CMP_LEGACY) ||
593 cpu_has(c, X86_FEATURE_XTOPOLOGY))
594 return;
595
596 cpuid(1, &eax, &ebx, &ecx, &edx);
597 c->x86_num_siblings = (ebx & 0xff0000) >> 16;
598
599 if (c->x86_num_siblings == 1) {
600 printk(KERN_INFO "CPU: Hyper-Threading is disabled\n");
601 } else if (c->x86_num_siblings > 1 ) {
602 index_msb = get_count_order(c->x86_num_siblings);
603 c->phys_proc_id = phys_pkg_id((ebx >> 24) & 0xFF, index_msb);
604
605 if (opt_cpu_info)
606 printk("CPU: Physical Processor ID: %d\n",
607 c->phys_proc_id);
608
609 c->x86_num_siblings = c->x86_num_siblings / c->x86_max_cores;
610
611 index_msb = get_count_order(c->x86_num_siblings) ;
612
613 core_bits = get_count_order(c->x86_max_cores);
614
615 c->cpu_core_id = phys_pkg_id((ebx >> 24) & 0xFF, index_msb) &
616 ((1 << core_bits) - 1);
617
618 if (opt_cpu_info && c->x86_max_cores > 1)
619 printk("CPU: Processor Core ID: %d\n",
620 c->cpu_core_id);
621 }
622 }
623
apicid_to_socket(unsigned int apicid)624 unsigned int __init apicid_to_socket(unsigned int apicid)
625 {
626 unsigned int dummy;
627
628 if (boot_cpu_has(X86_FEATURE_XTOPOLOGY)) {
629 unsigned int eax, ecx, sub_index = 1, core_plus_mask_width;
630
631 cpuid_count(0xb, SMT_LEVEL, &eax, &dummy, &dummy, &dummy);
632 core_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax);
633 do {
634 cpuid_count(0xb, sub_index, &eax, &dummy, &ecx,
635 &dummy);
636
637 if (LEAFB_SUBTYPE(ecx) == CORE_TYPE) {
638 core_plus_mask_width =
639 BITS_SHIFT_NEXT_LEVEL(eax);
640 break;
641 }
642
643 sub_index++;
644 } while (LEAFB_SUBTYPE(ecx) != INVALID_TYPE);
645
646 return _phys_pkg_id(apicid, core_plus_mask_width);
647 }
648
649 if (boot_cpu_has(X86_FEATURE_HTT) &&
650 !boot_cpu_has(X86_FEATURE_CMP_LEGACY)) {
651 unsigned int num_siblings = (cpuid_ebx(1) & 0xff0000) >> 16;
652
653 if (num_siblings)
654 return _phys_pkg_id(apicid,
655 get_count_order(num_siblings));
656 }
657
658 return apicid;
659 }
660
print_cpu_info(unsigned int cpu)661 void print_cpu_info(unsigned int cpu)
662 {
663 const struct cpuinfo_x86 *c = cpu_data + cpu;
664 const char *vendor = NULL;
665
666 if (!opt_cpu_info)
667 return;
668
669 printk("CPU%u: ", cpu);
670
671 if (c->x86_vendor < X86_VENDOR_NUM)
672 vendor = this_cpu->c_vendor;
673 else
674 vendor = c->x86_vendor_id;
675
676 if (vendor && strncmp(c->x86_model_id, vendor, strlen(vendor)))
677 printk("%s ", vendor);
678
679 if (!c->x86_model_id[0])
680 printk("%d86", c->x86);
681 else
682 printk("%s", c->x86_model_id);
683
684 printk(" stepping %02x\n", c->x86_mask);
685 }
686
687 static cpumask_t cpu_initialized;
688
689 /* This is hacky. :)
690 * We're emulating future behavior.
691 * In the future, the cpu-specific init functions will be called implicitly
692 * via the magic of initcalls.
693 * They will insert themselves into the cpu_devs structure.
694 * Then, when cpu_init() is called, we can just iterate over that array.
695 */
696
early_cpu_init(void)697 void __init early_cpu_init(void)
698 {
699 intel_cpu_init();
700 amd_init_cpu();
701 centaur_init_cpu();
702 early_cpu_detect();
703 }
704
705 /*
706 * Sets up system tables and descriptors.
707 *
708 * - Sets up TSS with stack pointers, including ISTs
709 * - Inserts TSS selector into regular and compat GDTs
710 * - Loads GDT, IDT, TR then null LDT
711 * - Sets up IST references in the IDT
712 */
load_system_tables(void)713 void load_system_tables(void)
714 {
715 unsigned int cpu = smp_processor_id();
716 unsigned long stack_bottom = get_stack_bottom(),
717 stack_top = stack_bottom & ~(STACK_SIZE - 1);
718
719 struct tss_struct *tss = &this_cpu(init_tss);
720 struct desc_struct *gdt =
721 this_cpu(gdt_table) - FIRST_RESERVED_GDT_ENTRY;
722 struct desc_struct *compat_gdt =
723 this_cpu(compat_gdt_table) - FIRST_RESERVED_GDT_ENTRY;
724
725 const struct desc_ptr gdtr = {
726 .base = (unsigned long)gdt,
727 .limit = LAST_RESERVED_GDT_BYTE,
728 };
729 const struct desc_ptr idtr = {
730 .base = (unsigned long)idt_tables[cpu],
731 .limit = (IDT_ENTRIES * sizeof(idt_entry_t)) - 1,
732 };
733
734 *tss = (struct tss_struct){
735 /* Main stack for interrupts/exceptions. */
736 .rsp0 = stack_bottom,
737
738 /* Ring 1 and 2 stacks poisoned. */
739 .rsp1 = 0x8600111111111111ul,
740 .rsp2 = 0x8600111111111111ul,
741
742 /*
743 * MCE, NMI and Double Fault handlers get their own stacks.
744 * All others poisoned.
745 */
746 .ist = {
747 [IST_MCE - 1] = stack_top + IST_MCE * PAGE_SIZE,
748 [IST_DF - 1] = stack_top + IST_DF * PAGE_SIZE,
749 [IST_NMI - 1] = stack_top + IST_NMI * PAGE_SIZE,
750
751 [IST_MAX ... ARRAY_SIZE(tss->ist) - 1] =
752 0x8600111111111111ul,
753 },
754
755 .bitmap = IOBMP_INVALID_OFFSET,
756 };
757
758 _set_tssldt_desc(
759 gdt + TSS_ENTRY,
760 (unsigned long)tss,
761 offsetof(struct tss_struct, __cacheline_filler) - 1,
762 SYS_DESC_tss_avail);
763 _set_tssldt_desc(
764 compat_gdt + TSS_ENTRY,
765 (unsigned long)tss,
766 offsetof(struct tss_struct, __cacheline_filler) - 1,
767 SYS_DESC_tss_busy);
768
769 asm volatile ("lgdt %0" : : "m" (gdtr) );
770 asm volatile ("lidt %0" : : "m" (idtr) );
771 asm volatile ("ltr %w0" : : "rm" (TSS_ENTRY << 3) );
772 asm volatile ("lldt %w0" : : "rm" (0) );
773
774 set_ist(&idt_tables[cpu][TRAP_double_fault], IST_DF);
775 set_ist(&idt_tables[cpu][TRAP_nmi], IST_NMI);
776 set_ist(&idt_tables[cpu][TRAP_machine_check], IST_MCE);
777
778 /*
779 * Bottom-of-stack must be 16-byte aligned!
780 *
781 * Defer checks until exception support is sufficiently set up.
782 */
783 BUILD_BUG_ON((sizeof(struct cpu_info) -
784 offsetof(struct cpu_info, guest_cpu_user_regs.es)) & 0xf);
785 BUG_ON(system_state != SYS_STATE_early_boot && (stack_bottom & 0xf));
786 }
787
788 /*
789 * cpu_init() initializes state that is per-CPU. Some data is already
790 * initialized (naturally) in the bootstrap process, such as the GDT
791 * and IDT. We reload them nevertheless, this function acts as a
792 * 'CPU state barrier', nothing should get across.
793 */
cpu_init(void)794 void cpu_init(void)
795 {
796 int cpu = smp_processor_id();
797
798 if (cpumask_test_and_set_cpu(cpu, &cpu_initialized)) {
799 printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
800 for (;;) local_irq_enable();
801 }
802 if (opt_cpu_info)
803 printk("Initializing CPU#%d\n", cpu);
804
805 if (cpu_has_pat)
806 wrmsrl(MSR_IA32_CR_PAT, host_pat);
807
808 /* Install correct page table. */
809 write_ptbase(current);
810
811 /* Ensure FPU gets initialised for each domain. */
812 stts();
813
814 /* Clear all 6 debug registers: */
815 #define CD(register) asm volatile ( "mov %0,%%db" #register : : "r"(0UL) );
816 CD(0); CD(1); CD(2); CD(3); /* no db4 and db5 */; CD(6); CD(7);
817 #undef CD
818 }
819
cpu_uninit(unsigned int cpu)820 void cpu_uninit(unsigned int cpu)
821 {
822 cpumask_clear_cpu(cpu, &cpu_initialized);
823 }
824
825 /*
826 * x86_match_cpu - match the current CPU against an array of
827 * x86_cpu_ids
828 * @match: Pointer to array of x86_cpu_ids. Last entry terminated with
829 * {}.
830 * Return the entry if the current CPU matches the entries in the
831 * passed x86_cpu_id match table. Otherwise NULL. The match table
832 * contains vendor (X86_VENDOR_*), family, model and feature bits or
833 * respective wildcard entries.
834 *
835 * A typical table entry would be to match a specific CPU
836 * { X86_VENDOR_INTEL, 6, 0x12 }
837 * or to match a specific CPU feature
838 * { X86_FEATURE_MATCH(X86_FEATURE_FOOBAR) }
839 *
840 * This always matches against the boot cpu, assuming models and
841 features are
842 * consistent over all CPUs.
843 */
x86_match_cpu(const struct x86_cpu_id table[])844 const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id table[])
845 {
846 const struct x86_cpu_id *m;
847 const struct cpuinfo_x86 *c = &boot_cpu_data;
848
849 for (m = table; m->vendor | m->family | m->model | m->feature; m++) {
850 if (c->x86_vendor != m->vendor)
851 continue;
852 if (c->x86 != m->family)
853 continue;
854 if (c->x86_model != m->model)
855 continue;
856 if (!cpu_has(c, m->feature))
857 continue;
858 return m;
859 }
860 return NULL;
861 }
862