1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /******************************************************************************
3 * include/asm-x86/spec_ctrl.h
4 *
5 * Copyright (c) 2017-2018 Citrix Systems Ltd.
6 */
7
8 #ifndef __X86_SPEC_CTRL_H__
9 #define __X86_SPEC_CTRL_H__
10
11 /*
12 * Encoding of Xen's speculation control flags in:
13 * cpuinfo.scf
14 * default_scf
15 * domain.scf
16 *
17 * Live settings are in the top-of-stack block, because they need to be
18 * accessable when XPTI is active. Some settings are fixed from boot, some
19 * context switched per domain, and some inhibited in the S3 path.
20 */
21 #define SCF_use_shadow (1 << 0)
22 #define SCF_ist_sc_msr (1 << 1)
23 #define SCF_ist_rsb (1 << 2)
24 #define SCF_verw (1 << 3)
25 #define SCF_ist_ibpb (1 << 4)
26 #define SCF_entry_ibpb (1 << 5)
27 #define SCF_entry_bhb (1 << 6)
28
29 /*
30 * The IST paths (NMI/#MC) can interrupt any arbitrary context. Some
31 * functionality requires updated microcode to work.
32 *
33 * On boot, this is easy; we load microcode before figuring out which
34 * speculative protections to apply. However, on the S3 resume path, we must
35 * be able to disable the configured mitigations until microcode is reloaded.
36 *
37 * These are the controls to inhibit on the S3 resume path until microcode has
38 * been reloaded.
39 */
40 #define SCF_IST_MASK (SCF_ist_sc_msr | SCF_ist_ibpb)
41
42 /*
43 * Some speculative protections are per-domain. These settings are merged
44 * into the top-of-stack block in the context switch path.
45 */
46 #define SCF_DOM_MASK (SCF_verw | SCF_entry_ibpb | SCF_entry_bhb)
47
48 #ifndef __ASSEMBLY__
49
50 #include <asm/alternative.h>
51 #include <asm/current.h>
52 #include <asm/msr.h>
53
54 void init_speculation_mitigations(void);
55 void spec_ctrl_init_domain(struct domain *d);
56
57 /*
58 * Switch to a new guest prediction context.
59 *
60 * This flushes all indirect branch predictors (BTB, RSB/RAS), so guest code
61 * which has previously run on this CPU can't attack subsequent guest code.
62 *
63 * As this flushes the RSB/RAS, it destroys the predictions of the calling
64 * context. For best performace, arrange for this to be used when we're going
65 * to jump out of the current context, e.g. with reset_stack_and_jump().
66 *
67 * For hardware which mis-implements IBPB, fix up by flushing the RSB/RAS
68 * manually.
69 */
spec_ctrl_new_guest_context(void)70 static always_inline void spec_ctrl_new_guest_context(void)
71 {
72 wrmsrl(MSR_PRED_CMD, PRED_CMD_IBPB);
73
74 /* (ab)use alternative_input() to specify clobbers. */
75 alternative_input("", "DO_OVERWRITE_RSB xu=%=", X86_BUG_IBPB_NO_RET,
76 : "rax", "rcx");
77 }
78
79 extern int8_t opt_ibpb_ctxt_switch;
80 extern bool opt_ssbd;
81 extern int8_t opt_bhi_dis_s;
82 extern int8_t opt_eager_fpu;
83 extern int8_t opt_l1d_flush;
84
85 extern bool bsp_delay_spec_ctrl;
86 extern unsigned int default_xen_spec_ctrl;
87 extern uint8_t default_scf;
88
89 extern int8_t opt_xpti_hwdom, opt_xpti_domu;
90
91 extern bool cpu_has_bug_l1tf;
92 extern int8_t opt_pv_l1tf_hwdom, opt_pv_l1tf_domu;
93
94 /*
95 * The L1D address mask, which might be wider than reported in CPUID, and the
96 * system physical address above which there are believed to be no cacheable
97 * memory regions, thus unable to leak data via the L1TF vulnerability.
98 */
99 extern paddr_t l1tf_addr_mask, l1tf_safe_maddr;
100
init_shadow_spec_ctrl_state(void)101 static inline void init_shadow_spec_ctrl_state(void)
102 {
103 struct cpu_info *info = get_cpu_info();
104
105 info->shadow_spec_ctrl = 0;
106 info->xen_spec_ctrl = default_xen_spec_ctrl;
107 info->scf = default_scf;
108
109 /*
110 * For least latency, the VERW selector should be a writeable data
111 * descriptor resident in the cache. __HYPERVISOR_DS32 shares a cache
112 * line with __HYPERVISOR_CS, so is expected to be very cache-hot.
113 */
114 info->verw_sel = __HYPERVISOR_DS32;
115 }
116
117 /* WARNING! `ret`, `call *`, `jmp *` not safe after this call. */
spec_ctrl_enter_idle(struct cpu_info * info)118 static always_inline void spec_ctrl_enter_idle(struct cpu_info *info)
119 {
120 uint32_t val = 0;
121
122 /*
123 * It is recommended in some cases to clear MSR_SPEC_CTRL when going idle,
124 * to avoid impacting sibling threads.
125 *
126 * Latch the new shadow value, then enable shadowing, then update the MSR.
127 * There are no SMP issues here; only local processor ordering concerns.
128 */
129 info->shadow_spec_ctrl = val;
130 barrier();
131 info->scf |= SCF_use_shadow;
132 barrier();
133 alternative_input("", "wrmsr", X86_FEATURE_SC_MSR_IDLE,
134 "a" (val), "c" (MSR_SPEC_CTRL), "d" (0));
135 barrier();
136
137 /*
138 * Microarchitectural Store Buffer Data Sampling:
139 *
140 * On vulnerable systems, store buffer entries are statically partitioned
141 * between active threads. When entering idle, our store buffer entries
142 * are re-partitioned to allow the other threads to use them.
143 *
144 * Flush the buffers to ensure that no sensitive data of ours can be
145 * leaked by a sibling after it gets our store buffer entries.
146 *
147 * Note: VERW must be encoded with a memory operand, as it is only that
148 * form which causes a flush.
149 */
150 alternative_input("", "verw %[sel]", X86_FEATURE_SC_VERW_IDLE,
151 [sel] "m" (info->verw_sel));
152
153 /*
154 * Cross-Thread Return Address Predictions:
155 *
156 * On vulnerable systems, the return predictions (RSB/RAS) are statically
157 * partitioned between active threads. When entering idle, our entries
158 * are re-partitioned to allow the other threads to use them.
159 *
160 * In some cases, we might still have guest entries in the RAS, so flush
161 * them before injecting them sideways to our sibling thread.
162 *
163 * (ab)use alternative_input() to specify clobbers.
164 */
165 alternative_input("", "DO_OVERWRITE_RSB xu=%=", X86_FEATURE_SC_RSB_IDLE,
166 : "rax", "rcx");
167 }
168
169 /* WARNING! `ret`, `call *`, `jmp *` not safe before this call. */
spec_ctrl_exit_idle(struct cpu_info * info)170 static always_inline void spec_ctrl_exit_idle(struct cpu_info *info)
171 {
172 uint32_t val = info->xen_spec_ctrl;
173
174 /*
175 * Restore MSR_SPEC_CTRL on exit from idle.
176 *
177 * Disable shadowing before updating the MSR. There are no SMP issues
178 * here; only local processor ordering concerns.
179 */
180 info->scf &= ~SCF_use_shadow;
181 barrier();
182 alternative_input("", "wrmsr", X86_FEATURE_SC_MSR_IDLE,
183 "a" (val), "c" (MSR_SPEC_CTRL), "d" (0));
184 barrier();
185
186 /*
187 * Microarchitectural Store Buffer Data Sampling:
188 *
189 * On vulnerable systems, store buffer entries are statically partitioned
190 * between active threads. When exiting idle, the other threads store
191 * buffer entries are re-partitioned to give us some.
192 *
193 * We now have store buffer entries with stale data from sibling threads.
194 * A flush if necessary will be performed on the return to guest path.
195 */
196 }
197
198 #endif /* __ASSEMBLY__ */
199 #endif /* !__X86_SPEC_CTRL_H__ */
200
201 /*
202 * Local variables:
203 * mode: C
204 * c-file-style: "BSD"
205 * c-basic-offset: 4
206 * tab-width: 4
207 * indent-tabs-mode: nil
208 * End:
209 */
210