1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_SYNC_CORE_H
3 #define _ASM_X86_SYNC_CORE_H
4 
5 #include <linux/preempt.h>
6 #include <asm/processor.h>
7 #include <asm/cpufeature.h>
8 #include <asm/special_insns.h>
9 
10 #ifdef CONFIG_X86_32
iret_to_self(void)11 static inline void iret_to_self(void)
12 {
13 	asm volatile (
14 		"pushfl\n\t"
15 		"pushl %%cs\n\t"
16 		"pushl $1f\n\t"
17 		"iret\n\t"
18 		"1:"
19 		: ASM_CALL_CONSTRAINT : : "memory");
20 }
21 #else
iret_to_self(void)22 static inline void iret_to_self(void)
23 {
24 	unsigned int tmp;
25 
26 	asm volatile (
27 		"mov %%ss, %0\n\t"
28 		"pushq %q0\n\t"
29 		"pushq %%rsp\n\t"
30 		"addq $8, (%%rsp)\n\t"
31 		"pushfq\n\t"
32 		"mov %%cs, %0\n\t"
33 		"pushq %q0\n\t"
34 		"pushq $1f\n\t"
35 		"iretq\n\t"
36 		"1:"
37 		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
38 }
39 #endif /* CONFIG_X86_32 */
40 
41 /*
42  * This function forces the icache and prefetched instruction stream to
43  * catch up with reality in two very specific cases:
44  *
45  *  a) Text was modified using one virtual address and is about to be executed
46  *     from the same physical page at a different virtual address.
47  *
48  *  b) Text was modified on a different CPU, may subsequently be
49  *     executed on this CPU, and you want to make sure the new version
50  *     gets executed.  This generally means you're calling this in an IPI.
51  *
52  * If you're calling this for a different reason, you're probably doing
53  * it wrong.
54  *
55  * Like all of Linux's memory ordering operations, this is a
56  * compiler barrier as well.
57  */
sync_core(void)58 static inline void sync_core(void)
59 {
60 	/*
61 	 * The SERIALIZE instruction is the most straightforward way to
62 	 * do this, but it is not universally available.
63 	 */
64 	if (static_cpu_has(X86_FEATURE_SERIALIZE)) {
65 		serialize();
66 		return;
67 	}
68 
69 	/*
70 	 * For all other processors, there are quite a few ways to do this.
71 	 * IRET-to-self is nice because it works on every CPU, at any CPL
72 	 * (so it's compatible with paravirtualization), and it never exits
73 	 * to a hypervisor.  The only downsides are that it's a bit slow
74 	 * (it seems to be a bit more than 2x slower than the fastest
75 	 * options) and that it unmasks NMIs.  The "push %cs" is needed,
76 	 * because in paravirtual environments __KERNEL_CS may not be a
77 	 * valid CS value when we do IRET directly.
78 	 *
79 	 * In case NMI unmasking or performance ever becomes a problem,
80 	 * the next best option appears to be MOV-to-CR2 and an
81 	 * unconditional jump.  That sequence also works on all CPUs,
82 	 * but it will fault at CPL3 (i.e. Xen PV).
83 	 *
84 	 * CPUID is the conventional way, but it's nasty: it doesn't
85 	 * exist on some 486-like CPUs, and it usually exits to a
86 	 * hypervisor.
87 	 */
88 	iret_to_self();
89 }
90 
91 /*
92  * Ensure that a core serializing instruction is issued before returning
93  * to user-mode. x86 implements return to user-space through sysexit,
94  * sysrel, and sysretq, which are not core serializing.
95  */
sync_core_before_usermode(void)96 static inline void sync_core_before_usermode(void)
97 {
98 	/* With PTI, we unconditionally serialize before running user code. */
99 	if (static_cpu_has(X86_FEATURE_PTI))
100 		return;
101 
102 	/*
103 	 * Even if we're in an interrupt, we might reschedule before returning,
104 	 * in which case we could switch to a different thread in the same mm
105 	 * and return using SYSRET or SYSEXIT.  Instead of trying to keep
106 	 * track of our need to sync the core, just sync right away.
107 	 */
108 	sync_core();
109 }
110 
111 #endif /* _ASM_X86_SYNC_CORE_H */
112