1
2 /* Portions are: Copyright (c) 1994 Linus Torvalds */
3
4 #ifndef __ASM_X86_PROCESSOR_H
5 #define __ASM_X86_PROCESSOR_H
6
7 #ifndef __ASSEMBLY__
8 #include <xen/cache.h>
9 #include <xen/types.h>
10 #include <xen/smp.h>
11 #include <xen/percpu.h>
12 #include <public/xen.h>
13 #include <asm/types.h>
14 #include <asm/cpufeature.h>
15 #include <asm/desc.h>
16 #include <asm/x86_emulate.h>
17 #endif
18
19 #include <asm/x86-defns.h>
20 #include <asm/x86-vendors.h>
21
22 /*
23 * Trap/fault mnemonics.
24 */
25 #define TRAP_divide_error 0
26 #define TRAP_debug 1
27 #define TRAP_nmi 2
28 #define TRAP_int3 3
29 #define TRAP_overflow 4
30 #define TRAP_bounds 5
31 #define TRAP_invalid_op 6
32 #define TRAP_no_device 7
33 #define TRAP_double_fault 8
34 #define TRAP_copro_seg 9
35 #define TRAP_invalid_tss 10
36 #define TRAP_no_segment 11
37 #define TRAP_stack_error 12
38 #define TRAP_gp_fault 13
39 #define TRAP_page_fault 14
40 #define TRAP_spurious_int 15
41 #define TRAP_copro_error 16
42 #define TRAP_alignment_check 17
43 #define TRAP_machine_check 18
44 #define TRAP_simd_error 19
45 #define TRAP_virtualisation 20
46 #define TRAP_nr 32
47
48 #define TRAP_HAVE_EC \
49 ((1u << TRAP_double_fault) | (1u << TRAP_invalid_tss) | \
50 (1u << TRAP_no_segment) | (1u << TRAP_stack_error) | \
51 (1u << TRAP_gp_fault) | (1u << TRAP_page_fault) | \
52 (1u << TRAP_alignment_check))
53
54 /* Set for entry via SYSCALL. Informs return code to use SYSRETQ not IRETQ. */
55 /* NB. Same as VGCF_in_syscall. No bits in common with any other TRAP_ defn. */
56 #define TRAP_syscall 256
57
58 /* Boolean return code: the reason for a fault has been fixed. */
59 #define EXCRET_fault_fixed 1
60
61 /* 'trap_bounce' flags values */
62 #define TBF_EXCEPTION 1
63 #define TBF_EXCEPTION_ERRCODE 2
64 #define TBF_INTERRUPT 8
65
66 /* 'arch_vcpu' flags values */
67 #define _TF_kernel_mode 0
68 #define TF_kernel_mode (1<<_TF_kernel_mode)
69
70 /* #PF error code values. */
71 #define PFEC_page_present (_AC(1,U) << 0)
72 #define PFEC_write_access (_AC(1,U) << 1)
73 #define PFEC_user_mode (_AC(1,U) << 2)
74 #define PFEC_reserved_bit (_AC(1,U) << 3)
75 #define PFEC_insn_fetch (_AC(1,U) << 4)
76 #define PFEC_prot_key (_AC(1,U) << 5)
77 #define PFEC_arch_mask (_AC(0xffff,U)) /* Architectural PFEC values. */
78 /* Internally used only flags. */
79 #define PFEC_page_paged (1U<<16)
80 #define PFEC_page_shared (1U<<17)
81 #define PFEC_implicit (1U<<18) /* Pagewalk input for ldt/gdt/idt/tr accesses. */
82 #define PFEC_synth_mask (~PFEC_arch_mask) /* Synthetic PFEC values. */
83
84 /* Other exception error code values. */
85 #define X86_XEC_EXT (_AC(1,U) << 0)
86 #define X86_XEC_IDT (_AC(1,U) << 1)
87 #define X86_XEC_TI (_AC(1,U) << 2)
88
89 #define XEN_MINIMAL_CR4 (X86_CR4_PGE | X86_CR4_PAE)
90
91 #define XEN_CR4_PV32_BITS (X86_CR4_SMEP|X86_CR4_SMAP)
92
93 /* Common SYSCALL parameters. */
94 #define XEN_MSR_STAR (((uint64_t)FLAT_RING3_CS32 << 48) | \
95 ((uint64_t)__HYPERVISOR_CS << 32))
96 #define XEN_SYSCALL_MASK (X86_EFLAGS_AC|X86_EFLAGS_VM|X86_EFLAGS_RF| \
97 X86_EFLAGS_NT|X86_EFLAGS_DF|X86_EFLAGS_IF| \
98 X86_EFLAGS_TF)
99
100 #ifndef __ASSEMBLY__
101
102 struct domain;
103 struct vcpu;
104
105 /*
106 * Default implementation of macro that returns current
107 * instruction pointer ("program counter").
108 */
109 #define current_text_addr() ({ \
110 void *pc; \
111 asm ( "leaq 1f(%%rip),%0\n1:" : "=r" (pc) ); \
112 pc; \
113 })
114
115 struct x86_cpu_id {
116 uint16_t vendor;
117 uint16_t family;
118 uint16_t model;
119 uint16_t feature; /* bit index */
120 const void *driver_data;
121 };
122
123 struct cpuinfo_x86 {
124 __u8 x86; /* CPU family */
125 __u8 x86_vendor; /* CPU vendor */
126 __u8 x86_model;
127 __u8 x86_mask;
128 int cpuid_level; /* Maximum supported CPUID level, -1=no CPUID */
129 __u32 extended_cpuid_level; /* Maximum supported CPUID extended level */
130 unsigned int x86_capability[NCAPINTS];
131 char x86_vendor_id[16];
132 char x86_model_id[64];
133 int x86_cache_size; /* in KB - valid for CPUS which support this call */
134 int x86_cache_alignment; /* In bytes */
135 __u32 x86_max_cores; /* cpuid returned max cores value */
136 __u32 booted_cores; /* number of cores as seen by OS */
137 __u32 x86_num_siblings; /* cpuid logical cpus per chip value */
138 __u32 apicid;
139 __u32 phys_proc_id; /* package ID of each logical CPU */
140 __u32 cpu_core_id; /* core ID of each logical CPU*/
141 __u32 compute_unit_id; /* AMD compute unit ID of each logical CPU */
142 unsigned short x86_clflush_size;
143 } __cacheline_aligned;
144
145 /*
146 * capabilities of CPUs
147 */
148
149 extern struct cpuinfo_x86 boot_cpu_data;
150
151 extern struct cpuinfo_x86 cpu_data[];
152 #define current_cpu_data cpu_data[smp_processor_id()]
153
154 extern bool probe_cpuid_faulting(void);
155 extern void ctxt_switch_levelling(const struct vcpu *next);
156 extern void (*ctxt_switch_masking)(const struct vcpu *next);
157
158 extern u64 host_pat;
159 extern bool_t opt_cpu_info;
160 extern u32 cpuid_ext_features;
161 extern u64 trampoline_misc_enable_off;
162
163 /* Maximum width of physical addresses supported by the hardware. */
164 extern unsigned int paddr_bits;
165 /* Max physical address width supported within HAP guests. */
166 extern unsigned int hap_paddr_bits;
167 /* Maximum width of virtual addresses supported by the hardware. */
168 extern unsigned int vaddr_bits;
169
170 extern const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id table[]);
171
172 extern void identify_cpu(struct cpuinfo_x86 *);
173 extern void setup_clear_cpu_cap(unsigned int);
174 extern void setup_force_cpu_cap(unsigned int);
175 extern void print_cpu_info(unsigned int cpu);
176 extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
177
178 extern void detect_extended_topology(struct cpuinfo_x86 *c);
179
180 extern void detect_ht(struct cpuinfo_x86 *c);
181
182 #define cpu_to_core(_cpu) (cpu_data[_cpu].cpu_core_id)
183 #define cpu_to_socket(_cpu) (cpu_data[_cpu].phys_proc_id)
184
185 unsigned int apicid_to_socket(unsigned int);
186
187 /*
188 * Generic CPUID function
189 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
190 * resulting in stale register contents being returned.
191 */
192 #define cpuid(_op,_eax,_ebx,_ecx,_edx) \
193 asm volatile ( "cpuid" \
194 : "=a" (*(int *)(_eax)), \
195 "=b" (*(int *)(_ebx)), \
196 "=c" (*(int *)(_ecx)), \
197 "=d" (*(int *)(_edx)) \
198 : "0" (_op), "2" (0) )
199
200 /* Some CPUID calls want 'count' to be placed in ecx */
cpuid_count(unsigned int op,unsigned int count,unsigned int * eax,unsigned int * ebx,unsigned int * ecx,unsigned int * edx)201 static inline void cpuid_count(
202 unsigned int op,
203 unsigned int count,
204 unsigned int *eax,
205 unsigned int *ebx,
206 unsigned int *ecx,
207 unsigned int *edx)
208 {
209 asm volatile ( "cpuid"
210 : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx)
211 : "0" (op), "c" (count) );
212 }
213
214 /*
215 * CPUID functions returning a single datum
216 */
cpuid_eax(unsigned int op)217 static always_inline unsigned int cpuid_eax(unsigned int op)
218 {
219 unsigned int eax;
220
221 asm volatile ( "cpuid"
222 : "=a" (eax)
223 : "0" (op)
224 : "bx", "cx", "dx" );
225 return eax;
226 }
227
cpuid_ebx(unsigned int op)228 static always_inline unsigned int cpuid_ebx(unsigned int op)
229 {
230 unsigned int eax, ebx;
231
232 asm volatile ( "cpuid"
233 : "=a" (eax), "=b" (ebx)
234 : "0" (op)
235 : "cx", "dx" );
236 return ebx;
237 }
238
cpuid_ecx(unsigned int op)239 static always_inline unsigned int cpuid_ecx(unsigned int op)
240 {
241 unsigned int eax, ecx;
242
243 asm volatile ( "cpuid"
244 : "=a" (eax), "=c" (ecx)
245 : "0" (op)
246 : "bx", "dx" );
247 return ecx;
248 }
249
cpuid_edx(unsigned int op)250 static always_inline unsigned int cpuid_edx(unsigned int op)
251 {
252 unsigned int eax, edx;
253
254 asm volatile ( "cpuid"
255 : "=a" (eax), "=d" (edx)
256 : "0" (op)
257 : "bx", "cx" );
258 return edx;
259 }
260
cpuid_count_ebx(unsigned int leaf,unsigned int subleaf)261 static always_inline unsigned int cpuid_count_ebx(
262 unsigned int leaf, unsigned int subleaf)
263 {
264 unsigned int ebx, tmp;
265
266 cpuid_count(leaf, subleaf, &tmp, &ebx, &tmp, &tmp);
267
268 return ebx;
269 }
270
cpuid_count_leaf(uint32_t leaf,uint32_t subleaf,struct cpuid_leaf * data)271 static always_inline void cpuid_count_leaf(uint32_t leaf, uint32_t subleaf,
272 struct cpuid_leaf *data)
273 {
274 cpuid_count(leaf, subleaf, &data->a, &data->b, &data->c, &data->d);
275 }
276
read_cr0(void)277 static inline unsigned long read_cr0(void)
278 {
279 unsigned long cr0;
280 asm volatile ( "mov %%cr0,%0\n\t" : "=r" (cr0) );
281 return cr0;
282 }
283
write_cr0(unsigned long val)284 static inline void write_cr0(unsigned long val)
285 {
286 asm volatile ( "mov %0,%%cr0" : : "r" ((unsigned long)val) );
287 }
288
read_cr2(void)289 static inline unsigned long read_cr2(void)
290 {
291 unsigned long cr2;
292 asm volatile ( "mov %%cr2,%0\n\t" : "=r" (cr2) );
293 return cr2;
294 }
295
read_cr4(void)296 static inline unsigned long read_cr4(void)
297 {
298 return get_cpu_info()->cr4;
299 }
300
write_cr4(unsigned long val)301 static inline void write_cr4(unsigned long val)
302 {
303 get_cpu_info()->cr4 = val;
304 asm volatile ( "mov %0,%%cr4" : : "r" (val) );
305 }
306
307 /* Clear and set 'TS' bit respectively */
clts(void)308 static inline void clts(void)
309 {
310 asm volatile ( "clts" );
311 }
312
stts(void)313 static inline void stts(void)
314 {
315 write_cr0(X86_CR0_TS|read_cr0());
316 }
317
318 /*
319 * Save the cr4 feature set we're using (ie
320 * Pentium 4MB enable and PPro Global page
321 * enable), so that any CPU's that boot up
322 * after us can get the correct flags.
323 */
324 extern unsigned long mmu_cr4_features;
325
set_in_cr4(unsigned long mask)326 static always_inline void set_in_cr4 (unsigned long mask)
327 {
328 mmu_cr4_features |= mask;
329 write_cr4(read_cr4() | mask);
330 }
331
clear_in_cr4(unsigned long mask)332 static always_inline void clear_in_cr4 (unsigned long mask)
333 {
334 mmu_cr4_features &= ~mask;
335 write_cr4(read_cr4() & ~mask);
336 }
337
read_pkru(void)338 static inline unsigned int read_pkru(void)
339 {
340 unsigned int pkru;
341 unsigned long cr4 = read_cr4();
342
343 /*
344 * _PAGE_PKEY_BITS have a conflict with _PAGE_GNTTAB used by PV guests,
345 * so that X86_CR4_PKE is disabled on hypervisor. To use RDPKRU, CR4.PKE
346 * gets temporarily enabled.
347 */
348 write_cr4(cr4 | X86_CR4_PKE);
349 asm volatile (".byte 0x0f,0x01,0xee"
350 : "=a" (pkru) : "c" (0) : "dx");
351 write_cr4(cr4);
352
353 return pkru;
354 }
355
356 /* Macros for PKRU domain */
357 #define PKRU_READ (0)
358 #define PKRU_WRITE (1)
359 #define PKRU_ATTRS (2)
360
361 /*
362 * PKRU defines 32 bits, there are 16 domains and 2 attribute bits per
363 * domain in pkru, pkeys is index to a defined domain, so the value of
364 * pte_pkeys * PKRU_ATTRS + R/W is offset of a defined domain attribute.
365 */
read_pkru_ad(uint32_t pkru,unsigned int pkey)366 static inline bool_t read_pkru_ad(uint32_t pkru, unsigned int pkey)
367 {
368 ASSERT(pkey < 16);
369 return (pkru >> (pkey * PKRU_ATTRS + PKRU_READ)) & 1;
370 }
371
read_pkru_wd(uint32_t pkru,unsigned int pkey)372 static inline bool_t read_pkru_wd(uint32_t pkru, unsigned int pkey)
373 {
374 ASSERT(pkey < 16);
375 return (pkru >> (pkey * PKRU_ATTRS + PKRU_WRITE)) & 1;
376 }
377
378 /*
379 * NSC/Cyrix CPU configuration register indexes
380 */
381
382 #define CX86_PCR0 0x20
383 #define CX86_GCR 0xb8
384 #define CX86_CCR0 0xc0
385 #define CX86_CCR1 0xc1
386 #define CX86_CCR2 0xc2
387 #define CX86_CCR3 0xc3
388 #define CX86_CCR4 0xe8
389 #define CX86_CCR5 0xe9
390 #define CX86_CCR6 0xea
391 #define CX86_CCR7 0xeb
392 #define CX86_PCR1 0xf0
393 #define CX86_DIR0 0xfe
394 #define CX86_DIR1 0xff
395 #define CX86_ARR_BASE 0xc4
396 #define CX86_RCR_BASE 0xdc
397
398 /*
399 * NSC/Cyrix CPU indexed register access macros
400 */
401
402 #define getCx86(reg) ({ outb((reg), 0x22); inb(0x23); })
403
404 #define setCx86(reg, data) do { \
405 outb((reg), 0x22); \
406 outb((data), 0x23); \
407 } while (0)
408
__monitor(const void * eax,unsigned long ecx,unsigned long edx)409 static always_inline void __monitor(const void *eax, unsigned long ecx,
410 unsigned long edx)
411 {
412 /* "monitor %eax,%ecx,%edx;" */
413 asm volatile (
414 ".byte 0x0f,0x01,0xc8;"
415 : : "a" (eax), "c" (ecx), "d"(edx) );
416 }
417
__mwait(unsigned long eax,unsigned long ecx)418 static always_inline void __mwait(unsigned long eax, unsigned long ecx)
419 {
420 /* "mwait %eax,%ecx;" */
421 asm volatile (
422 ".byte 0x0f,0x01,0xc9;"
423 : : "a" (eax), "c" (ecx) );
424 }
425
426 #define IOBMP_BYTES 8192
427 #define IOBMP_INVALID_OFFSET 0x8000
428
429 struct __packed __cacheline_aligned tss_struct {
430 uint32_t :32;
431 uint64_t rsp0, rsp1, rsp2;
432 uint64_t :64;
433 /*
434 * Interrupt Stack Table is 1-based so tss->ist[0] corresponds to an IST
435 * value of 1 in an Interrupt Descriptor.
436 */
437 uint64_t ist[7];
438 uint64_t :64;
439 uint16_t :16, bitmap;
440 /* Pads the TSS to be cacheline-aligned (total size is 0x80). */
441 uint8_t __cacheline_filler[24];
442 };
443
444 #define IST_NONE 0UL
445 #define IST_DF 1UL
446 #define IST_NMI 2UL
447 #define IST_MCE 3UL
448 #define IST_MAX 3UL
449
450 /* Set the interrupt stack table used by a particular interrupt
451 * descriptor table entry. */
set_ist(idt_entry_t * idt,unsigned long ist)452 static always_inline void set_ist(idt_entry_t *idt, unsigned long ist)
453 {
454 idt_entry_t new = *idt;
455
456 /* IST is a 3 bit field, 32 bits into the IDT entry. */
457 ASSERT(ist <= IST_MAX);
458 new.a = (idt->a & ~(7UL << 32)) | (ist << 32);
459 _write_gate_lower(idt, &new);
460 }
461
462 #define IDT_ENTRIES 256
463 extern idt_entry_t idt_table[];
464 extern idt_entry_t *idt_tables[];
465
466 DECLARE_PER_CPU(struct tss_struct, init_tss);
467
468 extern void init_int80_direct_trap(struct vcpu *v);
469
470 extern void write_ptbase(struct vcpu *v);
471
472 /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
rep_nop(void)473 static always_inline void rep_nop(void)
474 {
475 asm volatile ( "rep;nop" : : : "memory" );
476 }
477
478 #define cpu_relax() rep_nop()
479
480 void show_stack(const struct cpu_user_regs *regs);
481 void show_stack_overflow(unsigned int cpu, const struct cpu_user_regs *regs);
482 void show_registers(const struct cpu_user_regs *regs);
483 void show_execution_state(const struct cpu_user_regs *regs);
484 #define dump_execution_state() run_in_exception_handler(show_execution_state)
485 void show_page_walk(unsigned long addr);
486 void noreturn fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote);
487
488 extern void mtrr_ap_init(void);
489 extern void mtrr_bp_init(void);
490
491 void mcheck_init(struct cpuinfo_x86 *c, bool_t bsp);
492
493 /* Dispatch table for exceptions */
494 extern void (* const exception_table[TRAP_nr])(struct cpu_user_regs *regs);
495
496 #define DECLARE_TRAP_HANDLER(_name) \
497 void _name(void); \
498 void do_ ## _name(struct cpu_user_regs *regs)
499 #define DECLARE_TRAP_HANDLER_CONST(_name) \
500 void _name(void); \
501 void do_ ## _name(const struct cpu_user_regs *regs)
502
503 DECLARE_TRAP_HANDLER(divide_error);
504 DECLARE_TRAP_HANDLER(debug);
505 DECLARE_TRAP_HANDLER_CONST(nmi);
506 DECLARE_TRAP_HANDLER(int3);
507 DECLARE_TRAP_HANDLER(overflow);
508 DECLARE_TRAP_HANDLER(bounds);
509 DECLARE_TRAP_HANDLER(invalid_op);
510 DECLARE_TRAP_HANDLER(device_not_available);
511 DECLARE_TRAP_HANDLER(double_fault);
512 DECLARE_TRAP_HANDLER(invalid_TSS);
513 DECLARE_TRAP_HANDLER(segment_not_present);
514 DECLARE_TRAP_HANDLER(stack_segment);
515 DECLARE_TRAP_HANDLER(general_protection);
516 DECLARE_TRAP_HANDLER(page_fault);
517 DECLARE_TRAP_HANDLER(early_page_fault);
518 DECLARE_TRAP_HANDLER(coprocessor_error);
519 DECLARE_TRAP_HANDLER(simd_coprocessor_error);
520 DECLARE_TRAP_HANDLER_CONST(machine_check);
521 DECLARE_TRAP_HANDLER(alignment_check);
522
523 DECLARE_TRAP_HANDLER(entry_int82);
524
525 #undef DECLARE_TRAP_HANDLER_CONST
526 #undef DECLARE_TRAP_HANDLER
527
528 void trap_nop(void);
529 void enable_nmis(void);
530 void do_reserved_trap(struct cpu_user_regs *regs);
531
532 void sysenter_entry(void);
533 void sysenter_eflags_saved(void);
534 void int80_direct_trap(void);
535
536 #define STUBS_PER_PAGE (PAGE_SIZE / STUB_BUF_SIZE)
537
538 struct stubs {
539 union {
540 void(*func)(void);
541 unsigned long addr;
542 };
543 unsigned long mfn;
544 };
545
546 DECLARE_PER_CPU(struct stubs, stubs);
547 unsigned long alloc_stub_page(unsigned int cpu, unsigned long *mfn);
548
549 void cpuid_hypervisor_leaves(const struct vcpu *v, uint32_t leaf,
550 uint32_t subleaf, struct cpuid_leaf *res);
551 int rdmsr_hypervisor_regs(uint32_t idx, uint64_t *val);
552 int wrmsr_hypervisor_regs(uint32_t idx, uint64_t val);
553
554 void microcode_set_module(unsigned int);
555 int microcode_update(XEN_GUEST_HANDLE_PARAM(const_void), unsigned long len);
556 int microcode_resume_cpu(unsigned int cpu);
557 int early_microcode_update_cpu(bool start_update);
558 int early_microcode_init(void);
559 int microcode_init_intel(void);
560 int microcode_init_amd(void);
561
562 enum get_cpu_vendor {
563 gcv_host,
564 gcv_guest,
565 };
566
567 int get_cpu_vendor(uint32_t b, uint32_t c, uint32_t d, enum get_cpu_vendor mode);
568
get_cpu_family(uint32_t raw,uint8_t * model,uint8_t * stepping)569 static inline uint8_t get_cpu_family(uint32_t raw, uint8_t *model,
570 uint8_t *stepping)
571 {
572 uint8_t fam = (raw >> 8) & 0xf;
573
574 if ( fam == 0xf )
575 fam += (raw >> 20) & 0xff;
576
577 if ( model )
578 {
579 uint8_t mod = (raw >> 4) & 0xf;
580
581 if ( fam >= 0x6 )
582 mod |= (raw >> 12) & 0xf0;
583
584 *model = mod;
585 }
586 if ( stepping )
587 *stepping = raw & 0xf;
588 return fam;
589 }
590
591 #endif /* !__ASSEMBLY__ */
592
593 #endif /* __ASM_X86_PROCESSOR_H */
594
595 /*
596 * Local variables:
597 * mode: C
598 * c-file-style: "BSD"
599 * c-basic-offset: 4
600 * tab-width: 4
601 * indent-tabs-mode: nil
602 * End:
603 */
604