1 /*-
2 * Copyright (c) 1989, 1990 William F. Jolitz
3 * Copyright (c) 1990 The Regents of the University of California.
4 * Copyright (c) 2017-2022 Intel Corporation.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * William Jolitz.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * from: @(#)segments.h 7.1 (Berkeley) 5/9/91
34 * $FreeBSD$
35 */
36
37 #ifndef CPU_H
38 #define CPU_H
39 #include <types.h>
40 #include <util.h>
41 #include <acrn_common.h>
42 #include <asm/msr.h>
43 #include <errno.h>
44
45 /* Define CPU stack alignment */
46 #define CPU_STACK_ALIGN 16UL
47
48 /* CR0 register definitions */
49 #define CR0_PG (1UL<<31U) /* paging enable */
50 #define CR0_CD (1UL<<30U) /* cache disable */
51 #define CR0_NW (1UL<<29U) /* not write through */
52 #define CR0_AM (1UL<<18U) /* alignment mask */
53 #define CR0_WP (1UL<<16U) /* write protect */
54 #define CR0_NE (1UL<<5U) /* numeric error */
55 #define CR0_ET (1UL<<4U) /* extension type */
56 #define CR0_TS (1UL<<3U) /* task switched */
57 #define CR0_EM (1UL<<2U) /* emulation */
58 #define CR0_MP (1UL<<1U) /* monitor coprocessor */
59 #define CR0_PE (1UL<<0U) /* protected mode enabled */
60
61 /* CR3 register definitions */
62 #define CR3_PWT (1UL<<3U) /* page-level write through */
63 #define CR3_PCD (1UL<<4U) /* page-level cache disable */
64
65 /* CR4 register definitions */
66 #define CR4_VME (1UL<<0U) /* virtual 8086 mode extensions */
67 #define CR4_PVI (1UL<<1U) /* protected mode virtual interrupts */
68 #define CR4_TSD (1UL<<2U) /* time stamp disable */
69 #define CR4_DE (1UL<<3U) /* debugging extensions */
70 #define CR4_PSE (1UL<<4U) /* page size extensions */
71 #define CR4_PAE (1UL<<5U) /* physical address extensions */
72 #define CR4_MCE (1UL<<6U) /* machine check enable */
73 #define CR4_PGE (1UL<<7U) /* page global enable */
74 #define CR4_PCE (1UL<<8U)
75 /* performance monitoring counter enable */
76 #define CR4_OSFXSR (1UL<<9U) /* OS support for FXSAVE/FXRSTOR */
77 #define CR4_OSXMMEXCPT (1UL<<10U)
78 /* OS support for unmasked SIMD floating point exceptions */
79 #define CR4_UMIP (1UL<<11U) /* User-Mode Inst prevention */
80 #define CR4_LA57 (1UL<<12U) /* 57-bit linear address */
81 #define CR4_VMXE (1UL<<13U) /* VMX enable */
82 #define CR4_SMXE (1UL<<14U) /* SMX enable */
83 #define CR4_FSGSBASE (1UL<<16U) /* RD(FS|GS|FS)BASE inst */
84 #define CR4_PCIDE (1UL<<17U) /* PCID enable */
85 /* XSAVE and Processor Extended States enable bit */
86 #define CR4_OSXSAVE (1UL<<18U)
87 #define CR4_KL (1UL<<19U) /* KeyLocker enable */
88 #define CR4_SMEP (1UL<<20U)
89 #define CR4_SMAP (1UL<<21U)
90 #define CR4_PKE (1UL<<22U) /* Protect-key-enable */
91 #define CR4_CET (1UL<<23U) /* Control-flow Enforcement Technology enable */
92 #define CR4_PKS (1UL<<24U) /* Enable protection keys for supervisor-mode pages */
93
94 /* XCR0_SSE */
95 #define XCR0_SSE (1UL<<1U)
96 /* XCR0_AVX */
97 #define XCR0_AVX (1UL<<2U)
98 /* XCR0_BNDREGS */
99 #define XCR0_BNDREGS (1UL<<3U)
100 /* XCR0_BNDCSR */
101 #define XCR0_BNDCSR (1UL<<4U)
102 /* According to SDM Vol1 13.3:
103 * XCR0[63:10] and XCR0[8] are reserved. Executing the XSETBV instruction causes
104 * a general-protection fault if ECX = 0 and any corresponding bit in EDX:EAX
105 * is not 0.
106 */
107 #define XCR0_RESERVED_BITS ((~((1UL << 10U) - 1UL)) | (1UL << 8U))
108
109
110 /*
111 * Entries in the Interrupt Descriptor Table (IDT)
112 */
113 #define IDT_DE 0U /* #DE: Divide Error */
114 #define IDT_DB 1U /* #DB: Debug */
115 #define IDT_NMI 2U /* Nonmaskable External Interrupt */
116 #define IDT_BP 3U /* #BP: Breakpoint */
117 #define IDT_OF 4U /* #OF: Overflow */
118 #define IDT_BR 5U /* #BR: Bound Range Exceeded */
119 #define IDT_UD 6U /* #UD: Undefined/Invalid Opcode */
120 #define IDT_NM 7U /* #NM: No Math Coprocessor */
121 #define IDT_DF 8U /* #DF: Double Fault */
122 #define IDT_FPUGP 9U /* Coprocessor Segment Overrun */
123 #define IDT_TS 10U /* #TS: Invalid TSS */
124 #define IDT_NP 11U /* #NP: Segment Not Present */
125 #define IDT_SS 12U /* #SS: Stack Segment Fault */
126 #define IDT_GP 13U /* #GP: General Protection Fault */
127 #define IDT_PF 14U /* #PF: Page Fault */
128 #define IDT_MF 16U /* #MF: FPU Floating-Point Error */
129 #define IDT_AC 17U /* #AC: Alignment Check */
130 #define IDT_MC 18U /* #MC: Machine Check */
131 #define IDT_XF 19U /* #XF: SIMD Floating-Point Exception */
132 #define IDT_VE 20U /* #VE: Virtualization Exception */
133
134 /*Bits in EFER special registers */
135 #define EFER_LMA 0x00000400U /* Long mode active (R) */
136
137 #define RFLAGS_C (1U<<0U)
138 #define RFLAGS_P (1U<<2U)
139 #define RFLAGS_A (1U<<4U)
140 #define RFLAGS_Z (1U<<6U)
141 #define RFLAGS_S (1U<<7U)
142 #define RFLAGS_O (1U<<11U)
143 #define RFLAGS_VM (1U<<17U)
144 #define RFLAGS_AC (1U<<18U)
145
146 /* CPU clock frequencies (FSB) */
147 #define CPU_FSB_83KHZ 83200
148 #define CPU_FSB_100KHZ 99840
149 #define CPU_FSB_133KHZ 133200
150 #define CPU_FSB_166KHZ 166400
151
152 /* Time conversions */
153 #define CPU_GHZ_TO_HZ 1000000000
154 #define CPU_GHZ_TO_KHZ 1000000
155 #define CPU_GHZ_TO_MHZ 1000
156 #define CPU_MHZ_TO_HZ 1000000
157 #define CPU_MHZ_TO_KHZ 1000
158
159
160 /* Number of GPRs saved / restored for guest in VCPU structure */
161 #define NUM_GPRS 16U
162
163 #define XSAVE_STATE_AREA_SIZE 4096U
164 #define XSAVE_LEGACY_AREA_SIZE 512U
165 #define XSAVE_HEADER_AREA_SIZE 64U
166 #define XSAVE_EXTEND_AREA_SIZE (XSAVE_STATE_AREA_SIZE - \
167 XSAVE_HEADER_AREA_SIZE - \
168 XSAVE_LEGACY_AREA_SIZE)
169 #define XSAVE_COMPACTED_FORMAT (1UL << 63U)
170
171 #define XSAVE_FPU (1UL << 0U)
172 #define XSAVE_SSE (1UL << 1U)
173
174 #define CPU_CONTEXT_OFFSET_RAX 0U
175 #define CPU_CONTEXT_OFFSET_RCX 8U
176 #define CPU_CONTEXT_OFFSET_RDX 16U
177 #define CPU_CONTEXT_OFFSET_RBX 24U
178 #define CPU_CONTEXT_OFFSET_RSP 32U
179 #define CPU_CONTEXT_OFFSET_RBP 40U
180 #define CPU_CONTEXT_OFFSET_RSI 48U
181 #define CPU_CONTEXT_OFFSET_RDI 56U
182 #define CPU_CONTEXT_OFFSET_R8 64U
183 #define CPU_CONTEXT_OFFSET_R9 72U
184 #define CPU_CONTEXT_OFFSET_R10 80U
185 #define CPU_CONTEXT_OFFSET_R11 88U
186 #define CPU_CONTEXT_OFFSET_R12 96U
187 #define CPU_CONTEXT_OFFSET_R13 104U
188 #define CPU_CONTEXT_OFFSET_R14 112U
189 #define CPU_CONTEXT_OFFSET_R15 120U
190 #define CPU_CONTEXT_OFFSET_CR0 128U
191 #define CPU_CONTEXT_OFFSET_CR2 136U
192 #define CPU_CONTEXT_OFFSET_CR4 144U
193 #define CPU_CONTEXT_OFFSET_RIP 152U
194 #define CPU_CONTEXT_OFFSET_RFLAGS 160U
195 #define CPU_CONTEXT_OFFSET_IA32_SPEC_CTRL 168U
196 #define CPU_CONTEXT_OFFSET_IA32_EFER 176U
197 #define CPU_CONTEXT_OFFSET_EXTCTX_START 184U
198 #define CPU_CONTEXT_OFFSET_CR3 184U
199 #define CPU_CONTEXT_OFFSET_IDTR 192U
200 #define CPU_CONTEXT_OFFSET_LDTR 216U
201
202 #ifndef ASSEMBLER
203
204 #define ALL_CPUS_MASK ((1UL << get_pcpu_nums()) - 1UL)
205 #define AP_MASK (ALL_CPUS_MASK & ~(1UL << BSP_CPU_ID))
206
207 /**
208 *
209 * Identifiers for architecturally defined registers.
210 *
211 * These register names is used in condition statement.
212 * Within the following groups,register name need to be
213 * kept in order:
214 * General register names group (CPU_REG_RAX~CPU_REG_R15);
215 * Non general register names group (CPU_REG_CR0~CPU_REG_GDTR);
216 * Segement register names group (CPU_REG_ES~CPU_REG_GS).
217 */
218 enum cpu_reg_name {
219 /* General purpose register layout should align with
220 * struct acrn_gp_regs
221 */
222 CPU_REG_RAX,
223 CPU_REG_RCX,
224 CPU_REG_RDX,
225 CPU_REG_RBX,
226 CPU_REG_RSP,
227 CPU_REG_RBP,
228 CPU_REG_RSI,
229 CPU_REG_RDI,
230 CPU_REG_R8,
231 CPU_REG_R9,
232 CPU_REG_R10,
233 CPU_REG_R11,
234 CPU_REG_R12,
235 CPU_REG_R13,
236 CPU_REG_R14,
237 CPU_REG_R15,
238
239 CPU_REG_CR0,
240 CPU_REG_CR2,
241 CPU_REG_CR3,
242 CPU_REG_CR4,
243 CPU_REG_DR7,
244 CPU_REG_RIP,
245 CPU_REG_RFLAGS,
246 /*CPU_REG_NATURAL_LAST*/
247 CPU_REG_EFER,
248 CPU_REG_PDPTE0,
249 CPU_REG_PDPTE1,
250 CPU_REG_PDPTE2,
251 CPU_REG_PDPTE3,
252 /*CPU_REG_64BIT_LAST,*/
253 CPU_REG_ES,
254 CPU_REG_CS,
255 CPU_REG_SS,
256 CPU_REG_DS,
257 CPU_REG_FS,
258 CPU_REG_GS,
259 CPU_REG_LDTR,
260 CPU_REG_TR,
261 CPU_REG_IDTR,
262 CPU_REG_GDTR
263 /*CPU_REG_LAST*/
264 };
265
266 /**********************************/
267 /* EXTERNAL VARIABLES */
268 /**********************************/
269
270 /* In trampoline range, hold the jump target which trampline will jump to */
271 extern uint64_t main_entry[1];
272 extern uint64_t secondary_cpu_stack[1];
273
274 /*
275 * To support per_cpu access, we use a special struct "per_cpu_region" to hold
276 * the pattern of per CPU data. And we allocate memory for per CPU data
277 * according to multiple this struct size and pcpu number.
278 *
279 * +-------------------+------------------+---+------------------+
280 * | percpu for pcpu0 | percpu for pcpu1 |...| percpu for pcpuX |
281 * +-------------------+------------------+---+------------------+
282 * ^ ^
283 * | |
284 * <per_cpu_region size>
285 *
286 * To access per cpu data, we use:
287 * per_cpu_base_ptr + sizeof(struct per_cpu_region) * curr_pcpu_id
288 * + offset_of_member_per_cpu_region
289 * to locate the per cpu data.
290 */
291
292 /* Boot CPU ID */
293 #define BSP_CPU_ID 0U
294
295 /**
296 *The invalid cpu_id (INVALID_CPU_ID) is error
297 *code for error handling, this means that
298 *caller can't find a valid physical cpu
299 *or virtual cpu.
300 */
301 #define INVALID_CPU_ID 0xffffU
302 /**
303 *The broadcast id (BROADCAST_CPU_ID)
304 *used to notify all valid phyiscal cpu
305 *or virtual cpu.
306 */
307 #define BROADCAST_CPU_ID 0xfffeU
308
309 struct descriptor_table {
310 uint16_t limit;
311 uint64_t base;
312 } __packed;
313
314 /* CPU states defined */
315 enum pcpu_boot_state {
316 PCPU_STATE_RESET = 0U,
317 PCPU_STATE_INITIALIZING,
318 PCPU_STATE_RUNNING,
319 PCPU_STATE_HALTED,
320 PCPU_STATE_DEAD,
321 };
322
323 #define NEED_OFFLINE (1U)
324 #define NEED_SHUTDOWN_VM (2U)
325 void make_pcpu_offline(uint16_t pcpu_id);
326 bool need_offline(uint16_t pcpu_id);
327
328 struct segment_sel {
329 uint16_t selector;
330 uint64_t base;
331 uint32_t limit;
332 uint32_t attr;
333 };
334
335 /**
336 * @brief registers info saved for vcpu running context
337 */
338 struct run_context {
339 /* Contains the guest register set.
340 * NOTE: This must be the first element in the structure, so that the offsets
341 * in vmx_asm.S match
342 */
343 union cpu_regs_t {
344 struct acrn_gp_regs regs;
345 uint64_t longs[NUM_GPRS];
346 } cpu_regs;
347
348 /** The guests CR registers 0, 2, 3 and 4. */
349 uint64_t cr0;
350
351 /* CPU_CONTEXT_OFFSET_CR2 =
352 * offsetof(struct run_context, cr2) = 136
353 */
354 uint64_t cr2;
355 uint64_t cr4;
356
357 uint64_t rip;
358 uint64_t rflags;
359
360 /* CPU_CONTEXT_OFFSET_IA32_SPEC_CTRL =
361 * offsetof(struct run_context, ia32_spec_ctrl) = 168
362 */
363 uint64_t ia32_spec_ctrl;
364 uint64_t ia32_efer;
365 };
366
367 union xsave_header {
368 uint64_t value[XSAVE_HEADER_AREA_SIZE / sizeof(uint64_t)];
369 struct {
370 /* bytes 7:0 */
371 uint64_t xstate_bv;
372 /* bytes 15:8 */
373 uint64_t xcomp_bv;
374 } hdr;
375 };
376
377 struct xsave_area {
378 uint64_t legacy_region[XSAVE_LEGACY_AREA_SIZE / sizeof(uint64_t)];
379 union xsave_header xsave_hdr;
380 uint64_t extend_region[XSAVE_EXTEND_AREA_SIZE / sizeof(uint64_t)];
381 } __aligned(64);
382 /*
383 * extended context does not save/restore during vm exit/entry, it's mainly
384 * used in trusty world switch
385 */
386 struct ext_context {
387 uint64_t cr3;
388
389 /* segment registers */
390 struct segment_sel idtr;
391 struct segment_sel ldtr;
392 struct segment_sel gdtr;
393 struct segment_sel tr;
394 struct segment_sel cs;
395 struct segment_sel ss;
396 struct segment_sel ds;
397 struct segment_sel es;
398 struct segment_sel fs;
399 struct segment_sel gs;
400
401 uint64_t ia32_star;
402 uint64_t ia32_cstar;
403 uint64_t ia32_lstar;
404 uint64_t ia32_fmask;
405 uint64_t ia32_kernel_gs_base;
406
407 uint64_t ia32_pat;
408 uint32_t ia32_sysenter_cs;
409 uint64_t ia32_sysenter_esp;
410 uint64_t ia32_sysenter_eip;
411 uint64_t ia32_debugctl;
412
413 uint64_t dr7;
414 uint64_t tsc_offset;
415 uint64_t tsc_aux;
416
417 struct xsave_area xs_area;
418 uint64_t xcr0;
419 };
420
421 struct cpu_context {
422 struct run_context run_ctx;
423 struct ext_context ext_ctx;
424 };
425
426 /*
427 * Definition of the stack frame layout
428 */
429 struct intr_excp_ctx {
430 struct acrn_gp_regs gp_regs;
431 uint64_t vector;
432 uint64_t error_code;
433 uint64_t rip;
434 uint64_t cs;
435 uint64_t rflags;
436 uint64_t rsp;
437 uint64_t ss;
438 };
439
440 void dispatch_exception(struct intr_excp_ctx *ctx);
441
442 /**
443 * @brief Handle NMI
444 *
445 * To handle an NMI
446 *
447 * @param ctx Pointer to interrupt exception context
448 */
449 void handle_nmi(__unused struct intr_excp_ctx *ctx);
450
451 /* Function prototypes */
452 void cpu_do_idle(void);
453 void cpu_dead(void);
454 void trampoline_start16(void);
455 void load_pcpu_state_data(void);
456 void init_pcpu_pre(bool is_bsp);
457 /* The function should be called on the same CPU core as specified by pcpu_id,
458 * hereby, pcpu_id is actually the current physcial cpu id.
459 */
460 void init_pcpu_post(uint16_t pcpu_id);
461 bool start_pcpus(uint64_t mask);
462 void wait_pcpus_offline(uint64_t mask);
463 void stop_pcpus(void);
464 void wait_sync_change(volatile const uint64_t *sync, uint64_t wake_sync);
465
466 #define CPU_SEG_READ(seg, result_ptr) \
467 { \
468 asm volatile ("mov %%" STRINGIFY(seg) ", %0": "=r" (*(result_ptr))); \
469 }
470
471 /* Read control register */
472 #define CPU_CR_READ(cr, result_ptr) \
473 { \
474 asm volatile ("mov %%" STRINGIFY(cr) ", %0" \
475 : "=r"(*(result_ptr))); \
476 }
477
478 /* Write control register */
479 #define CPU_CR_WRITE(cr, value) \
480 { \
481 asm volatile ("mov %0, %%" STRINGIFY(cr) \
482 : /* No output */ \
483 : "r"(value)); \
484 }
485
486 #define CPU_XMM_READ(xmm, result_m128i_p) \
487 { \
488 asm volatile ("movdqu %%" STRINGIFY(xmm) ", %0": "=m" (*(result_m128i_p))); \
489 }
490
491 #define CPU_XMM_WRITE(xmm, input_m128i_p) \
492 { \
493 asm volatile ("movdqu %0, %%" STRINGIFY(xmm) \
494 : /* No output */ \
495 : "m"(*(input_m128i_p))); \
496 }
497
sgdt(void)498 static inline uint64_t sgdt(void)
499 {
500 struct descriptor_table gdtb = {0U, 0UL};
501 asm volatile ("sgdt %0":"=m"(gdtb)::"memory");
502 return gdtb.base;
503 }
504
sidt(void)505 static inline uint64_t sidt(void)
506 {
507 struct descriptor_table idtb = {0U, 0UL};
508 asm volatile ("sidt %0":"=m"(idtb)::"memory");
509 return idtb.base;
510 }
511
512 /* Read MSR */
cpu_msr_read(uint32_t reg)513 static inline uint64_t cpu_msr_read(uint32_t reg)
514 {
515 uint32_t msrl, msrh;
516
517 asm volatile (" rdmsr ":"=a"(msrl), "=d"(msrh) : "c" (reg));
518 return (((uint64_t)msrh << 32U) | msrl);
519 }
520
521 /* Write MSR */
cpu_msr_write(uint32_t reg,uint64_t msr_val)522 static inline void cpu_msr_write(uint32_t reg, uint64_t msr_val)
523 {
524 asm volatile (" wrmsr " : : "c" (reg), "a" ((uint32_t)msr_val), "d" ((uint32_t)(msr_val >> 32U)));
525 }
526
asm_pause(void)527 static inline void asm_pause(void)
528 {
529 asm volatile ("pause" ::: "memory");
530 }
531
asm_hlt(void)532 static inline void asm_hlt(void)
533 {
534 asm volatile ("hlt");
535 }
536
537 /* interrupts remain inhibited on the instruction boundary following
538 * an execution of STI. This will make sure pending interrupts will
539 * wake hlt.
540 */
asm_safe_hlt(void)541 static inline void asm_safe_hlt(void)
542 {
543 asm volatile ("sti; hlt; cli" : : : "cc");
544 }
545
546 /* Disables interrupts on the current CPU */
547 #ifdef CONFIG_KEEP_IRQ_DISABLED
548 #define CPU_IRQ_DISABLE_ON_CONFIG() do { } while (0)
549 #else
550 #define CPU_IRQ_DISABLE_ON_CONFIG() \
551 { \
552 asm volatile ("cli\n" : : : "cc"); \
553 }
554 #endif
555
556 /* Enables interrupts on the current CPU
557 * If CONFIG_KEEP_IRQ_DISABLED is 'y', all interrupts
558 * received in root mode will be handled in external interrupt
559 * vmexit after next VM entry. The postpone latency is negligible.
560 * Permanently turning off interrupts in root mode can be useful in
561 * many scenarios (e.g., x86_tee).
562 */
563 #ifdef CONFIG_KEEP_IRQ_DISABLED
564 #define CPU_IRQ_ENABLE_ON_CONFIG() do { } while (0)
565 #else
566 #define CPU_IRQ_ENABLE_ON_CONFIG() \
567 { \
568 asm volatile ("sti\n" : : : "cc"); \
569 }
570 #endif
571
572 /* This macro writes the stack pointer. */
cpu_sp_write(uint64_t * stack_ptr)573 static inline void cpu_sp_write(uint64_t *stack_ptr)
574 {
575 uint64_t rsp = (uint64_t)stack_ptr & ~(CPU_STACK_ALIGN - 1UL);
576
577 asm volatile ("movq %0, %%rsp" : : "r"(rsp));
578 }
579
580 /* Synchronizes all write accesses to memory */
cpu_write_memory_barrier(void)581 static inline void cpu_write_memory_barrier(void)
582 {
583 asm volatile ("sfence\n" : : : "memory");
584 }
585
586 /* Synchronizes all read and write accesses to/from memory */
cpu_memory_barrier(void)587 static inline void cpu_memory_barrier(void)
588 {
589 asm volatile ("mfence\n" : : : "memory");
590 }
591
592 /* Prevents compilers from reordering read/write access across this barrier */
cpu_compiler_barrier(void)593 static inline void cpu_compiler_barrier(void)
594 {
595 asm volatile ("" : : : "memory");
596 }
597
invlpg(unsigned long addr)598 static inline void invlpg(unsigned long addr)
599 {
600 asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
601 }
602
wbinvd(void)603 static inline void wbinvd(void)
604 {
605 asm volatile (" wbinvd\n" : : : "memory");
606 }
607
clflush(const volatile void * p)608 static inline void clflush(const volatile void *p)
609 {
610 asm volatile ("clflush (%0)" :: "r"(p));
611 }
612
clflushopt(const volatile void * p)613 static inline void clflushopt(const volatile void *p)
614 {
615 asm volatile ("clflushopt (%0)" :: "r"(p));
616 }
617
618 /* Write the task register */
619 #define CPU_LTR_EXECUTE(ltr_ptr) \
620 { \
621 asm volatile ("ltr %%ax\n" : : "a"(ltr_ptr)); \
622 }
623
624 /* Read time-stamp counter / processor ID */
625 static inline void
cpu_rdtscp_execute(uint64_t * timestamp_ptr,uint32_t * cpu_id_ptr)626 cpu_rdtscp_execute(uint64_t *timestamp_ptr, uint32_t *cpu_id_ptr)
627 {
628 uint32_t tsl, tsh;
629
630 asm volatile ("rdtscp":"=a"(tsl), "=d"(tsh), "=c"(*cpu_id_ptr));
631 *timestamp_ptr = ((uint64_t)tsh << 32U) | tsl;
632 }
633
634 /* Macro to save rflags register */
635 #define CPU_RFLAGS_SAVE(rflags_ptr) \
636 { \
637 asm volatile (" pushf"); \
638 asm volatile (" pop %0" \
639 : "=r" (*(rflags_ptr)) \
640 : /* No inputs */); \
641 }
642
643 /* Macro to restore rflags register */
644 #define CPU_RFLAGS_RESTORE(rflags) \
645 { \
646 asm volatile (" push %0\n\t" \
647 "popf \n\t": : "r" (rflags) \
648 :"cc"); \
649 }
650
651 /* This macro locks out interrupts and saves the current architecture status
652 * register / state register to the specified address. This function does not
653 * attempt to mask any bits in the return register value and can be used as a
654 * quick method to guard a critical section.
655 * NOTE: This macro is used in conjunction with CPU_INT_ALL_RESTORE
656 * defined below and CPU_INT_CONTROL_VARS defined above.
657 */
658
659 #define CPU_INT_ALL_DISABLE(p_rflags) \
660 { \
661 CPU_RFLAGS_SAVE(p_rflags); \
662 CPU_IRQ_DISABLE_ON_CONFIG(); \
663 }
664
665 /* This macro restores the architecture status / state register used to lockout
666 * interrupts to the value provided. The intent of this function is to be a
667 * fast mechanism to restore the interrupt level at the end of a critical
668 * section to its original level.
669 * NOTE: This macro is used in conjunction with CPU_INT_ALL_DISABLE
670 * and CPU_INT_CONTROL_VARS defined above.
671 */
672 #define CPU_INT_ALL_RESTORE(rflags) \
673 { \
674 CPU_RFLAGS_RESTORE(rflags); \
675 }
676
677 #define ACRN_PSEUDO_PCPUID_MSR MSR_IA32_SYSENTER_CS
678
679 /*
680 * Macro to get CPU ID
681 * @pre: the return CPU ID would never equal or large than phys_cpu_num.
682 */
get_pcpu_id(void)683 static inline uint16_t get_pcpu_id(void)
684 {
685 return (uint16_t)cpu_msr_read(ACRN_PSEUDO_PCPUID_MSR);
686 }
687
cpu_rsp_get(void)688 static inline uint64_t cpu_rsp_get(void)
689 {
690 uint64_t ret;
691
692 asm volatile("movq %%rsp, %0" : "=r"(ret));
693 return ret;
694 }
695
cpu_rbp_get(void)696 static inline uint64_t cpu_rbp_get(void)
697 {
698 uint64_t ret;
699
700 asm volatile("movq %%rbp, %0" : "=r"(ret));
701 return ret;
702 }
703
msr_read(uint32_t reg_num)704 static inline uint64_t msr_read(uint32_t reg_num)
705 {
706 return cpu_msr_read(reg_num);
707 }
708
msr_write(uint32_t reg_num,uint64_t value64)709 static inline void msr_write(uint32_t reg_num, uint64_t value64)
710 {
711 cpu_msr_write(reg_num, value64);
712 }
713
msr_write_safe(uint32_t reg_num,uint64_t value64,uint64_t rsvd)714 static inline int32_t msr_write_safe(uint32_t reg_num, uint64_t value64, uint64_t rsvd)
715 {
716 int32_t err = 0;
717
718 if ((value64 & rsvd) == 0) {
719 msr_write(reg_num, value64);
720 } else {
721 err = -EACCES;
722 }
723 return err;
724 }
725
726 /* wrmsr/rdmsr smp call data */
727 struct msr_data_struct {
728 uint32_t msr_index;
729 uint64_t read_val;
730 uint64_t write_val;
731 };
732
733 void msr_write_pcpu(uint32_t msr_index, uint64_t value64, uint16_t pcpu_id);
734 uint64_t msr_read_pcpu(uint32_t msr_index, uint16_t pcpu_id);
735
write_xcr(int32_t reg,uint64_t val)736 static inline void write_xcr(int32_t reg, uint64_t val)
737 {
738 asm volatile("xsetbv" : : "c" (reg), "a" ((uint32_t)val), "d" ((uint32_t)(val >> 32U)));
739 }
740
read_xcr(int32_t reg)741 static inline uint64_t read_xcr(int32_t reg)
742 {
743 uint32_t xcrl, xcrh;
744
745 asm volatile ("xgetbv ": "=a"(xcrl), "=d"(xcrh) : "c" (reg));
746 return (((uint64_t)xcrh << 32U) | xcrl);
747 }
748
xsaves(struct xsave_area * region_addr,uint64_t mask)749 static inline void xsaves(struct xsave_area *region_addr, uint64_t mask)
750 {
751 asm volatile("xsaves %0"
752 : : "m" (*(region_addr)),
753 "d" ((uint32_t)(mask >> 32U)),
754 "a" ((uint32_t)mask):
755 "memory");
756 }
757
xrstors(const struct xsave_area * region_addr,uint64_t mask)758 static inline void xrstors(const struct xsave_area *region_addr, uint64_t mask)
759 {
760 asm volatile("xrstors %0"
761 : : "m" (*(region_addr)),
762 "d" ((uint32_t)(mask >> 32U)),
763 "a" ((uint32_t)mask):
764 "memory");
765 }
766
asm_loadiwkey(uint32_t eax)767 static inline void asm_loadiwkey(uint32_t eax)
768 {
769 asm volatile(".byte 0xf3, 0x0f, 0x38, 0xdc, 0xd1;": : "a" (eax));
770 }
771
read_xmm_0_2(uint64_t * xmm0_addr,uint64_t * xmm1_addr,uint64_t * xmm2_addr)772 static inline void read_xmm_0_2(uint64_t *xmm0_addr, uint64_t *xmm1_addr, uint64_t *xmm2_addr)
773 {
774 CPU_XMM_READ(xmm0, xmm0_addr);
775 CPU_XMM_READ(xmm1, xmm1_addr);
776 CPU_XMM_READ(xmm2, xmm2_addr);
777 }
778
write_xmm_0_2(uint64_t * xmm0_addr,uint64_t * xmm1_addr,uint64_t * xmm2_addr)779 static inline void write_xmm_0_2(uint64_t *xmm0_addr, uint64_t *xmm1_addr, uint64_t *xmm2_addr)
780 {
781 CPU_XMM_WRITE(xmm0, xmm0_addr);
782 CPU_XMM_WRITE(xmm1, xmm1_addr);
783 CPU_XMM_WRITE(xmm2, xmm2_addr);
784 }
785
786 /*
787 * stac/clac pair is used to access guest's memory protected by SMAP,
788 * following below flow:
789 *
790 * stac();
791 * #access guest's memory.
792 * clac();
793 *
794 * Notes:Avoid inserting another stac/clac pair between stac and clac,
795 * As once clac after multiple stac will invalidate SMAP protection
796 * and hence Page Fault crash.
797 * Logging message to memory buffer will induce this case,
798 * please disable SMAP temporlly or don't log messages to shared
799 * memory buffer, if it is evitable for you for debug purpose.
800 */
stac(void)801 static inline void stac(void)
802 {
803 asm volatile ("stac" : : : "memory");
804 }
805
clac(void)806 static inline void clac(void)
807 {
808 asm volatile ("clac" : : : "memory");
809 }
810
811 /*
812 * @post return <= MAX_PCPU_NUM
813 */
814 uint16_t get_pcpu_nums(void);
815 bool is_pcpu_active(uint16_t pcpu_id);
816 uint64_t get_active_pcpu_bitmap(void);
817 #else /* ASSEMBLER defined */
818
819 #endif /* ASSEMBLER defined */
820
821 #endif /* CPU_H */
822