1
2 /* Portions are: Copyright (c) 1994 Linus Torvalds */
3
4 #ifndef __ASM_X86_PROCESSOR_H
5 #define __ASM_X86_PROCESSOR_H
6
7 #ifndef __ASSEMBLY__
8 #include <xen/types.h>
9 #include <xen/smp.h>
10 #include <xen/percpu.h>
11 #include <asm/cpufeature.h>
12 #include <asm/desc.h>
13 #endif
14
15 #include <asm/x86-defns.h>
16 #include <asm/x86-vendors.h>
17
18 /* Set for entry via SYSCALL. Informs return code to use SYSRETQ not IRETQ. */
19 /* NB. Same as VGCF_in_syscall. No bits in common with any other TRAP_ defn. */
20 #define TRAP_syscall 256
21
22 /* Boolean return code: the reason for a fault has been fixed. */
23 #define EXCRET_fault_fixed 1
24
25 /* 'trap_bounce' flags values */
26 #define TBF_EXCEPTION 1
27 #define TBF_EXCEPTION_ERRCODE 2
28 #define TBF_INTERRUPT 8
29
30 /* 'arch_vcpu' flags values */
31 #define _TF_kernel_mode 0
32 #define TF_kernel_mode (1<<_TF_kernel_mode)
33
34 /* #PF error code values. */
35 #define PFEC_page_present (_AC(1,U) << 0)
36 #define PFEC_write_access (_AC(1,U) << 1)
37 #define PFEC_user_mode (_AC(1,U) << 2)
38 #define PFEC_reserved_bit (_AC(1,U) << 3)
39 #define PFEC_insn_fetch (_AC(1,U) << 4)
40 #define PFEC_prot_key (_AC(1,U) << 5)
41 #define PFEC_shstk (_AC(1,U) << 6)
42 #define PFEC_arch_mask (_AC(0xffff,U)) /* Architectural PFEC values. */
43 /* Internally used only flags. */
44 #define PFEC_page_paged (1U<<16)
45 #define PFEC_page_shared (1U<<17)
46 #define PFEC_implicit (1U<<18) /* Pagewalk input for ldt/gdt/idt/tr accesses. */
47 #define PFEC_synth_mask (~PFEC_arch_mask) /* Synthetic PFEC values. */
48
49 /* Other exception error code values. */
50 #define X86_XEC_EXT (_AC(1,U) << 0)
51 #define X86_XEC_IDT (_AC(1,U) << 1)
52 #define X86_XEC_TI (_AC(1,U) << 2)
53
54 #define XEN_MINIMAL_CR4 (X86_CR4_PGE | X86_CR4_PAE)
55
56 #define XEN_CR4_PV32_BITS (X86_CR4_SMEP|X86_CR4_SMAP)
57
58 /* Common SYSCALL parameters. */
59 #define XEN_MSR_STAR (((uint64_t)FLAT_RING3_CS32 << 48) | \
60 ((uint64_t)__HYPERVISOR_CS << 32))
61 #define XEN_SYSCALL_MASK (X86_EFLAGS_AC|X86_EFLAGS_VM|X86_EFLAGS_RF| \
62 X86_EFLAGS_NT|X86_EFLAGS_DF|X86_EFLAGS_IF| \
63 X86_EFLAGS_TF)
64
65 /*
66 * Host IA32_CR_PAT value to cover all memory types. This is not the default
67 * MSR_PAT value, and is an ABI with PV guests.
68 */
69 #define XEN_MSR_PAT ((_AC(X86_MT_WB, ULL) << 0x00) | \
70 (_AC(X86_MT_WT, ULL) << 0x08) | \
71 (_AC(X86_MT_UCM, ULL) << 0x10) | \
72 (_AC(X86_MT_UC, ULL) << 0x18) | \
73 (_AC(X86_MT_WC, ULL) << 0x20) | \
74 (_AC(X86_MT_WP, ULL) << 0x28) | \
75 (_AC(X86_MT_UC, ULL) << 0x30) | \
76 (_AC(X86_MT_UC, ULL) << 0x38))
77
78 #ifndef __ASSEMBLY__
79
80 struct domain;
81 struct vcpu;
82
83 extern struct cpuinfo_x86 cpu_data[];
84 #define current_cpu_data cpu_data[smp_processor_id()]
85
86 extern bool probe_cpuid_faulting(void);
87 extern void ctxt_switch_levelling(const struct vcpu *next);
88 extern void (*ctxt_switch_masking)(const struct vcpu *next);
89
90 extern bool opt_cpu_info;
91
92 /* Maximum width of physical addresses supported by the hardware. */
93 extern unsigned int paddr_bits;
94 /* Max physical address width supported within HAP guests. */
95 extern unsigned int hap_paddr_bits;
96 /* Maximum width of virtual addresses supported by the hardware. */
97 extern unsigned int vaddr_bits;
98
99 extern void identify_cpu(struct cpuinfo_x86 *c);
100 extern void setup_clear_cpu_cap(unsigned int cap);
101 extern void setup_force_cpu_cap(unsigned int cap);
102 extern bool is_forced_cpu_cap(unsigned int cap);
103 extern void print_cpu_info(unsigned int cpu);
104 extern void init_intel_cacheinfo(struct cpuinfo_x86 *c);
105
106 #define cpu_to_core(_cpu) (cpu_data[_cpu].cpu_core_id)
107 #define cpu_to_socket(_cpu) (cpu_data[_cpu].phys_proc_id)
108
109 unsigned int apicid_to_socket(unsigned int apicid);
110
111 /* Some CPUID calls want 'count' to be placed in ecx */
cpuid_count(unsigned int op,unsigned int count,unsigned int * eax,unsigned int * ebx,unsigned int * ecx,unsigned int * edx)112 static inline void cpuid_count(
113 unsigned int op,
114 unsigned int count,
115 unsigned int *eax,
116 unsigned int *ebx,
117 unsigned int *ecx,
118 unsigned int *edx)
119 {
120 asm volatile ( "cpuid"
121 : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx)
122 : "0" (op), "c" (count) );
123 }
124
125 /*
126 * Generic CPUID function
127 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
128 * resulting in stale register contents being returned.
129 */
cpuid(unsigned int leaf,unsigned int * eax,unsigned int * ebx,unsigned int * ecx,unsigned int * edx)130 static inline void cpuid(
131 unsigned int leaf,
132 unsigned int *eax,
133 unsigned int *ebx,
134 unsigned int *ecx,
135 unsigned int *edx)
136 {
137 cpuid_count(leaf, 0, eax, ebx, ecx, edx);
138 }
139
140 /*
141 * CPUID functions returning a single datum
142 */
cpuid_eax(unsigned int op)143 static always_inline unsigned int cpuid_eax(unsigned int op)
144 {
145 unsigned int eax;
146
147 asm volatile ( "cpuid"
148 : "=a" (eax)
149 : "0" (op)
150 : "bx", "cx", "dx" );
151 return eax;
152 }
153
cpuid_ebx(unsigned int op)154 static always_inline unsigned int cpuid_ebx(unsigned int op)
155 {
156 unsigned int eax, ebx;
157
158 asm volatile ( "cpuid"
159 : "=a" (eax), "=b" (ebx)
160 : "0" (op)
161 : "cx", "dx" );
162 return ebx;
163 }
164
cpuid_ecx(unsigned int op)165 static always_inline unsigned int cpuid_ecx(unsigned int op)
166 {
167 unsigned int eax, ecx;
168
169 asm volatile ( "cpuid"
170 : "=a" (eax), "=c" (ecx)
171 : "0" (op)
172 : "bx", "dx" );
173 return ecx;
174 }
175
cpuid_edx(unsigned int op)176 static always_inline unsigned int cpuid_edx(unsigned int op)
177 {
178 unsigned int eax, edx;
179
180 asm volatile ( "cpuid"
181 : "=a" (eax), "=d" (edx)
182 : "0" (op)
183 : "bx", "cx" );
184 return edx;
185 }
186
cpuid_count_ebx(unsigned int leaf,unsigned int subleaf)187 static always_inline unsigned int cpuid_count_ebx(
188 unsigned int leaf, unsigned int subleaf)
189 {
190 unsigned int ebx, tmp;
191
192 cpuid_count(leaf, subleaf, &tmp, &ebx, &tmp, &tmp);
193
194 return ebx;
195 }
196
cpuid_count_edx(unsigned int leaf,unsigned int subleaf)197 static always_inline unsigned int cpuid_count_edx(
198 unsigned int leaf, unsigned int subleaf)
199 {
200 unsigned int edx, tmp;
201
202 cpuid_count(leaf, subleaf, &tmp, &tmp, &tmp, &edx);
203
204 return edx;
205 }
206
read_cr0(void)207 static inline unsigned long read_cr0(void)
208 {
209 unsigned long cr0;
210 asm volatile ( "mov %%cr0,%0\n\t" : "=r" (cr0) );
211 return cr0;
212 }
213
write_cr0(unsigned long val)214 static inline void write_cr0(unsigned long val)
215 {
216 asm volatile ( "mov %0,%%cr0" : : "r" ((unsigned long)val) );
217 }
218
read_cr2(void)219 static inline unsigned long read_cr2(void)
220 {
221 unsigned long cr2;
222 asm volatile ( "mov %%cr2,%0\n\t" : "=r" (cr2) );
223 return cr2;
224 }
225
write_cr3(unsigned long val)226 static inline void write_cr3(unsigned long val)
227 {
228 asm volatile ( "mov %0, %%cr3" : : "r" (val) : "memory" );
229 }
230
cr3_pa(unsigned long cr3)231 static inline unsigned long cr3_pa(unsigned long cr3)
232 {
233 return cr3 & X86_CR3_ADDR_MASK;
234 }
235
cr3_pcid(unsigned long cr3)236 static inline unsigned int cr3_pcid(unsigned long cr3)
237 {
238 return IS_ENABLED(CONFIG_PV) ? cr3 & X86_CR3_PCID_MASK : 0;
239 }
240
read_cr4(void)241 static inline unsigned long read_cr4(void)
242 {
243 return get_cpu_info()->cr4;
244 }
245
write_cr4(unsigned long val)246 static inline void write_cr4(unsigned long val)
247 {
248 struct cpu_info *info = get_cpu_info();
249
250 #ifdef CONFIG_PV
251 /* No global pages in case of PCIDs enabled! */
252 ASSERT(!(val & X86_CR4_PGE) || !(val & X86_CR4_PCIDE));
253 #else
254 ASSERT(!(val & X86_CR4_PCIDE));
255 #endif
256
257 /*
258 * On hardware supporting FSGSBASE, the value in %cr4 is the kernel's
259 * choice for 64bit PV guests, which impacts whether Xen can use the
260 * instructions.
261 *
262 * The {rd,wr}{fs,gs}base() helpers use info->cr4 to work out whether it
263 * is safe to execute the {RD,WR}{FS,GS}BASE instruction, falling back to
264 * the MSR path if not. Some users require interrupt safety.
265 *
266 * If FSGSBASE is currently or about to become clear, reflect this in
267 * info->cr4 before updating %cr4, so an interrupt which hits in the
268 * middle won't observe FSGSBASE set in info->cr4 but clear in %cr4.
269 */
270 info->cr4 = val & (info->cr4 | ~X86_CR4_FSGSBASE);
271
272 asm volatile ( "mov %[val], %%cr4"
273 : "+m" (info->cr4) /* Force ordering without a barrier. */
274 : [val] "r" (val) );
275
276 info->cr4 = val;
277 }
278
279 /* Clear and set 'TS' bit respectively */
clts(void)280 static inline void clts(void)
281 {
282 asm volatile ( "clts" );
283 }
284
stts(void)285 static inline void stts(void)
286 {
287 write_cr0(X86_CR0_TS|read_cr0());
288 }
289
290 /*
291 * Save the cr4 feature set we're using (ie
292 * Pentium 4MB enable and PPro Global page
293 * enable), so that any CPU's that boot up
294 * after us can get the correct flags.
295 */
296 extern unsigned long mmu_cr4_features;
297 extern unsigned long cr4_pv32_mask;
298
set_in_cr4(unsigned long mask)299 static always_inline void set_in_cr4 (unsigned long mask)
300 {
301 mmu_cr4_features |= mask;
302 write_cr4(read_cr4() | mask);
303
304 if ( IS_ENABLED(CONFIG_PV32) && (mask & XEN_CR4_PV32_BITS) )
305 cr4_pv32_mask |= (mask & XEN_CR4_PV32_BITS);
306 }
307
308 #define IOBMP_BYTES 8192
309 #define IOBMP_INVALID_OFFSET 0x8000
310
311 struct __packed tss64 {
312 uint32_t :32;
313 uint64_t rsp0, rsp1, rsp2;
314 uint64_t :64;
315 /*
316 * Interrupt Stack Table is 1-based so tss->ist[0] corresponds to an IST
317 * value of 1 in an Interrupt Descriptor.
318 */
319 uint64_t ist[7];
320 uint64_t :64;
321 uint16_t :16, bitmap;
322 };
323 struct tss_page {
324 uint64_t __aligned(PAGE_SIZE) ist_ssp[8];
325 struct tss64 tss;
326 };
327 DECLARE_PER_CPU(struct tss_page, tss_page);
328
329 DECLARE_PER_CPU(root_pgentry_t *, root_pgt);
330
331 extern void write_ptbase(struct vcpu *v);
332
333 /* PAUSE (encoding: REP NOP) is a good thing to insert into busy-wait loops. */
334 #define cpu_relax() asm volatile ( "pause" ::: "memory" )
335
336 void show_registers(const struct cpu_user_regs *regs);
337 #define dump_execution_state() run_in_exception_handler(show_execution_state)
338 void show_page_walk(unsigned long addr);
339 void noreturn fatal_trap(const struct cpu_user_regs *regs, bool show_remote);
340 void show_execution_state_nmi(const cpumask_t *mask, bool show_all);
341
342 extern void mtrr_ap_init(void);
343 extern void mtrr_bp_init(void);
344
345 void mcheck_init(struct cpuinfo_x86 *c, bool bsp);
346
347 void do_nmi(const struct cpu_user_regs *regs);
348 void do_machine_check(const struct cpu_user_regs *regs);
349
350 void trap_nop(void);
351
enable_nmis(void)352 static inline void enable_nmis(void)
353 {
354 unsigned long tmp;
355
356 asm volatile ( "mov %%rsp, %[rsp] \n\t"
357 "lea .Ldone(%%rip), %[rip] \n\t"
358 #ifdef CONFIG_XEN_SHSTK
359 /* Check for CET-SS being active. */
360 "mov $1, %k[ssp] \n\t"
361 "rdsspq %[ssp] \n\t"
362 "cmp $1, %k[ssp] \n\t"
363 "je .Lshstk_done \n\t"
364
365 /* Push 3 words on the shadow stack */
366 ".rept 3 \n\t"
367 "call 1f; nop; 1: \n\t"
368 ".endr \n\t"
369
370 /* Fixup to be an IRET shadow stack frame */
371 "wrssq %q[cs], -1*8(%[ssp]) \n\t"
372 "wrssq %[rip], -2*8(%[ssp]) \n\t"
373 "wrssq %[ssp], -3*8(%[ssp]) \n\t"
374
375 ".Lshstk_done:"
376 #endif
377 /* Write an IRET regular frame */
378 "push %[ss] \n\t"
379 "push %[rsp] \n\t"
380 "pushf \n\t"
381 "push %q[cs] \n\t"
382 "push %[rip] \n\t"
383 "iretq \n\t"
384 ".Ldone: \n\t"
385 : [rip] "=&r" (tmp),
386 [rsp] "=&r" (tmp),
387 [ssp] "=&r" (tmp)
388 : [ss] "i" (__HYPERVISOR_DS),
389 [cs] "r" (__HYPERVISOR_CS) );
390 }
391
392 void nocall sysenter_entry(void);
393
get_cpu_family(uint32_t raw,uint8_t * model,uint8_t * stepping)394 static inline uint8_t get_cpu_family(uint32_t raw, uint8_t *model,
395 uint8_t *stepping)
396 {
397 uint8_t fam = (raw >> 8) & 0xf;
398
399 if ( fam == 0xf )
400 fam += (raw >> 20) & 0xff;
401
402 if ( model )
403 {
404 uint8_t mod = (raw >> 4) & 0xf;
405
406 if ( fam >= 0x6 )
407 mod |= (raw >> 12) & 0xf0;
408
409 *model = mod;
410 }
411 if ( stepping )
412 *stepping = raw & 0xf;
413 return fam;
414 }
415
416 #ifdef CONFIG_INTEL
417 extern int8_t opt_tsx;
418 extern bool rtm_disabled;
419 void tsx_init(void);
420 #else
421 #define opt_tsx 0 /* explicitly indicate TSX is off */
422 #define rtm_disabled false /* RTM was not force-disabled */
tsx_init(void)423 static inline void tsx_init(void) {}
424 #endif
425
426 void update_mcu_opt_ctrl(void);
427 void set_in_mcu_opt_ctrl(uint32_t mask, uint32_t val);
428
429 void update_pb_opt_ctrl(void);
430 void set_in_pb_opt_ctrl(uint32_t mask, uint32_t val);
431
432 enum ap_boot_method {
433 AP_BOOT_NORMAL,
434 AP_BOOT_SKINIT,
435 };
436 extern enum ap_boot_method ap_boot_method;
437
438 void amd_check_zenbleed(void);
439
440 #endif /* !__ASSEMBLY__ */
441
442 #endif /* __ASM_X86_PROCESSOR_H */
443
444 /*
445 * Local variables:
446 * mode: C
447 * c-file-style: "BSD"
448 * c-basic-offset: 4
449 * tab-width: 4
450 * indent-tabs-mode: nil
451 * End:
452 */
453