1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /******************************************************************************
3  * include/asm-x86/spec_ctrl.h
4  *
5  * Copyright (c) 2017-2018 Citrix Systems Ltd.
6  */
7 
8 #ifndef __X86_SPEC_CTRL_H__
9 #define __X86_SPEC_CTRL_H__
10 
11 /*
12  * Encoding of Xen's speculation control flags in:
13  *   cpuinfo.scf
14  *   default_scf
15  *   domain.scf
16  *
17  * Live settings are in the top-of-stack block, because they need to be
18  * accessable when XPTI is active.  Some settings are fixed from boot, some
19  * context switched per domain, and some inhibited in the S3 path.
20  */
21 #define SCF_use_shadow (1 << 0)
22 #define SCF_ist_sc_msr (1 << 1)
23 #define SCF_ist_rsb    (1 << 2)
24 #define SCF_verw       (1 << 3)
25 #define SCF_ist_ibpb   (1 << 4)
26 #define SCF_entry_ibpb (1 << 5)
27 #define SCF_entry_bhb  (1 << 6)
28 
29 /*
30  * The IST paths (NMI/#MC) can interrupt any arbitrary context.  Some
31  * functionality requires updated microcode to work.
32  *
33  * On boot, this is easy; we load microcode before figuring out which
34  * speculative protections to apply.  However, on the S3 resume path, we must
35  * be able to disable the configured mitigations until microcode is reloaded.
36  *
37  * These are the controls to inhibit on the S3 resume path until microcode has
38  * been reloaded.
39  */
40 #define SCF_IST_MASK (SCF_ist_sc_msr | SCF_ist_ibpb)
41 
42 /*
43  * Some speculative protections are per-domain.  These settings are merged
44  * into the top-of-stack block in the context switch path.
45  */
46 #define SCF_DOM_MASK (SCF_verw | SCF_entry_ibpb | SCF_entry_bhb)
47 
48 #ifndef __ASSEMBLY__
49 
50 #include <asm/alternative.h>
51 #include <asm/current.h>
52 #include <asm/msr.h>
53 
54 void init_speculation_mitigations(void);
55 void spec_ctrl_init_domain(struct domain *d);
56 
57 /*
58  * Switch to a new guest prediction context.
59  *
60  * This flushes all indirect branch predictors (BTB, RSB/RAS), so guest code
61  * which has previously run on this CPU can't attack subsequent guest code.
62  *
63  * As this flushes the RSB/RAS, it destroys the predictions of the calling
64  * context.  For best performace, arrange for this to be used when we're going
65  * to jump out of the current context, e.g. with reset_stack_and_jump().
66  *
67  * For hardware which mis-implements IBPB, fix up by flushing the RSB/RAS
68  * manually.
69  */
spec_ctrl_new_guest_context(void)70 static always_inline void spec_ctrl_new_guest_context(void)
71 {
72     wrmsrl(MSR_PRED_CMD, PRED_CMD_IBPB);
73 
74     /* (ab)use alternative_input() to specify clobbers. */
75     alternative_input("", "DO_OVERWRITE_RSB xu=%=", X86_BUG_IBPB_NO_RET,
76                       : "rax", "rcx");
77 }
78 
79 extern int8_t opt_ibpb_ctxt_switch;
80 extern bool opt_ssbd;
81 extern int8_t opt_bhi_dis_s;
82 extern int8_t opt_eager_fpu;
83 extern int8_t opt_l1d_flush;
84 
85 extern bool bsp_delay_spec_ctrl;
86 extern unsigned int default_xen_spec_ctrl;
87 extern uint8_t default_scf;
88 
89 extern int8_t opt_xpti_hwdom, opt_xpti_domu;
90 
91 extern bool cpu_has_bug_l1tf;
92 extern int8_t opt_pv_l1tf_hwdom, opt_pv_l1tf_domu;
93 extern bool opt_bp_spec_reduce;
94 
95 /*
96  * The L1D address mask, which might be wider than reported in CPUID, and the
97  * system physical address above which there are believed to be no cacheable
98  * memory regions, thus unable to leak data via the L1TF vulnerability.
99  */
100 extern paddr_t l1tf_addr_mask, l1tf_safe_maddr;
101 
init_shadow_spec_ctrl_state(struct cpu_info * info)102 static inline void init_shadow_spec_ctrl_state(struct cpu_info *info)
103 {
104     info->shadow_spec_ctrl = 0;
105     info->xen_spec_ctrl = default_xen_spec_ctrl;
106     info->scf = default_scf;
107 
108     /*
109      * For least latency, the VERW selector should be a writeable data
110      * descriptor resident in the cache.  __HYPERVISOR_DS32 shares a cache
111      * line with __HYPERVISOR_CS, so is expected to be very cache-hot.
112      */
113     info->verw_sel = __HYPERVISOR_DS32;
114 }
115 
__spec_ctrl_enter_idle_verw(struct cpu_info * info)116 static always_inline void __spec_ctrl_enter_idle_verw(struct cpu_info *info)
117 {
118     /*
119      * Flush/scrub structures which are statically partitioned between active
120      * threads.  Otherwise data of ours (of unknown sensitivity) will become
121      * available to our sibling when we go idle.
122      *
123      * Note: VERW must be encoded with a memory operand, as it is only that
124      * form with side effects.
125      */
126     alternative_input("", "verw %[sel]", X86_FEATURE_SC_VERW_IDLE,
127                       [sel] "m" (info->verw_sel));
128 }
129 
130 /* WARNING! `ret`, `call *`, `jmp *` not safe after this call. */
__spec_ctrl_enter_idle(struct cpu_info * info,bool verw)131 static always_inline void __spec_ctrl_enter_idle(struct cpu_info *info, bool verw)
132 {
133     uint32_t val = 0;
134 
135     /*
136      * It is recommended in some cases to clear MSR_SPEC_CTRL when going idle,
137      * to avoid impacting sibling threads.
138      *
139      * Latch the new shadow value, then enable shadowing, then update the MSR.
140      * There are no SMP issues here; only local processor ordering concerns.
141      */
142     info->shadow_spec_ctrl = val;
143     barrier();
144     info->scf |= SCF_use_shadow;
145     barrier();
146     alternative_input("", "wrmsr", X86_FEATURE_SC_MSR_IDLE,
147                       "a" (val), "c" (MSR_SPEC_CTRL), "d" (0));
148     barrier();
149 
150     if ( verw ) /* Expected to be const-propagated. */
151         __spec_ctrl_enter_idle_verw(info);
152 
153     /*
154      * Cross-Thread Return Address Predictions:
155      *
156      * On vulnerable systems, the return predictions (RSB/RAS) are statically
157      * partitioned between active threads.  When entering idle, our entries
158      * are re-partitioned to allow the other threads to use them.
159      *
160      * In some cases, we might still have guest entries in the RAS, so flush
161      * them before injecting them sideways to our sibling thread.
162      *
163      * (ab)use alternative_input() to specify clobbers.
164      */
165     alternative_input("", "DO_OVERWRITE_RSB xu=%=", X86_FEATURE_SC_RSB_IDLE,
166                       : "rax", "rcx");
167 }
168 
169 /* WARNING! `ret`, `call *`, `jmp *` not safe after this call. */
spec_ctrl_enter_idle(struct cpu_info * info)170 static always_inline void spec_ctrl_enter_idle(struct cpu_info *info)
171 {
172     __spec_ctrl_enter_idle(info, true /* VERW */);
173 }
174 
175 /* WARNING! `ret`, `call *`, `jmp *` not safe before this call. */
spec_ctrl_exit_idle(struct cpu_info * info)176 static always_inline void spec_ctrl_exit_idle(struct cpu_info *info)
177 {
178     uint32_t val = info->xen_spec_ctrl;
179 
180     /*
181      * Restore MSR_SPEC_CTRL on exit from idle.
182      *
183      * Disable shadowing before updating the MSR.  There are no SMP issues
184      * here; only local processor ordering concerns.
185      */
186     info->scf &= ~SCF_use_shadow;
187     barrier();
188     alternative_input("", "wrmsr", X86_FEATURE_SC_MSR_IDLE,
189                       "a" (val), "c" (MSR_SPEC_CTRL), "d" (0));
190     barrier();
191 
192     /*
193      * Microarchitectural Store Buffer Data Sampling:
194      *
195      * On vulnerable systems, store buffer entries are statically partitioned
196      * between active threads.  When exiting idle, the other threads store
197      * buffer entries are re-partitioned to give us some.
198      *
199      * We now have store buffer entries with stale data from sibling threads.
200      * A flush if necessary will be performed on the return to guest path.
201      */
202 }
203 
204 #endif /* __ASSEMBLY__ */
205 #endif /* !__X86_SPEC_CTRL_H__ */
206 
207 /*
208  * Local variables:
209  * mode: C
210  * c-file-style: "BSD"
211  * c-basic-offset: 4
212  * tab-width: 4
213  * indent-tabs-mode: nil
214  * End:
215  */
216