1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Paravirtualization interfaces
3 Copyright (C) 2006 Rusty Russell IBM Corporation
4
5
6 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
7 */
8
9 #include <linux/errno.h>
10 #include <linux/init.h>
11 #include <linux/export.h>
12 #include <linux/efi.h>
13 #include <linux/bcd.h>
14 #include <linux/highmem.h>
15 #include <linux/kprobes.h>
16 #include <linux/pgtable.h>
17 #include <linux/static_call.h>
18
19 #include <asm/bug.h>
20 #include <asm/paravirt.h>
21 #include <asm/debugreg.h>
22 #include <asm/desc.h>
23 #include <asm/setup.h>
24 #include <asm/time.h>
25 #include <asm/pgalloc.h>
26 #include <asm/irq.h>
27 #include <asm/delay.h>
28 #include <asm/fixmap.h>
29 #include <asm/apic.h>
30 #include <asm/tlbflush.h>
31 #include <asm/timer.h>
32 #include <asm/special_insns.h>
33 #include <asm/tlb.h>
34 #include <asm/io_bitmap.h>
35 #include <asm/gsseg.h>
36
37 /*
38 * nop stub, which must not clobber anything *including the stack* to
39 * avoid confusing the entry prologues.
40 */
41 DEFINE_PARAVIRT_ASM(_paravirt_nop, "", .entry.text);
42
43 /* stub always returning 0. */
44 DEFINE_PARAVIRT_ASM(paravirt_ret0, "xor %eax,%eax", .entry.text);
45
default_banner(void)46 void __init default_banner(void)
47 {
48 printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
49 pv_info.name);
50 }
51
52 /* Undefined instruction for dealing with missing ops pointers. */
paravirt_BUG(void)53 noinstr void paravirt_BUG(void)
54 {
55 BUG();
56 }
57
paravirt_patch_call(void * insn_buff,const void * target,unsigned long addr,unsigned len)58 static unsigned paravirt_patch_call(void *insn_buff, const void *target,
59 unsigned long addr, unsigned len)
60 {
61 __text_gen_insn(insn_buff, CALL_INSN_OPCODE,
62 (void *)addr, target, CALL_INSN_SIZE);
63 return CALL_INSN_SIZE;
64 }
65
66 #ifdef CONFIG_PARAVIRT_XXL
67 /* identity function, which can be inlined */
_paravirt_ident_64(u64 x)68 u64 notrace _paravirt_ident_64(u64 x)
69 {
70 return x;
71 }
72 #endif
73
74 DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
75
native_pv_lock_init(void)76 void __init native_pv_lock_init(void)
77 {
78 if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
79 static_branch_disable(&virt_spin_lock_key);
80 }
81
paravirt_patch(u8 type,void * insn_buff,unsigned long addr,unsigned int len)82 unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr,
83 unsigned int len)
84 {
85 /*
86 * Neat trick to map patch type back to the call within the
87 * corresponding structure.
88 */
89 void *opfunc = *((void **)&pv_ops + type);
90 unsigned ret;
91
92 if (opfunc == NULL)
93 /* If there's no function, patch it with paravirt_BUG() */
94 ret = paravirt_patch_call(insn_buff, paravirt_BUG, addr, len);
95 else if (opfunc == _paravirt_nop)
96 ret = 0;
97 else
98 /* Otherwise call the function. */
99 ret = paravirt_patch_call(insn_buff, opfunc, addr, len);
100
101 return ret;
102 }
103
104 struct static_key paravirt_steal_enabled;
105 struct static_key paravirt_steal_rq_enabled;
106
native_steal_clock(int cpu)107 static u64 native_steal_clock(int cpu)
108 {
109 return 0;
110 }
111
112 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
113 DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock);
114
paravirt_set_sched_clock(u64 (* func)(void))115 void paravirt_set_sched_clock(u64 (*func)(void))
116 {
117 static_call_update(pv_sched_clock, func);
118 }
119
120 /* These are in entry.S */
121 static struct resource reserve_ioports = {
122 .start = 0,
123 .end = IO_SPACE_LIMIT,
124 .name = "paravirt-ioport",
125 .flags = IORESOURCE_IO | IORESOURCE_BUSY,
126 };
127
128 /*
129 * Reserve the whole legacy IO space to prevent any legacy drivers
130 * from wasting time probing for their hardware. This is a fairly
131 * brute-force approach to disabling all non-virtual drivers.
132 *
133 * Note that this must be called very early to have any effect.
134 */
paravirt_disable_iospace(void)135 int paravirt_disable_iospace(void)
136 {
137 return request_resource(&ioport_resource, &reserve_ioports);
138 }
139
140 static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
141
enter_lazy(enum paravirt_lazy_mode mode)142 static inline void enter_lazy(enum paravirt_lazy_mode mode)
143 {
144 BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
145
146 this_cpu_write(paravirt_lazy_mode, mode);
147 }
148
leave_lazy(enum paravirt_lazy_mode mode)149 static void leave_lazy(enum paravirt_lazy_mode mode)
150 {
151 BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode);
152
153 this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
154 }
155
paravirt_enter_lazy_mmu(void)156 void paravirt_enter_lazy_mmu(void)
157 {
158 enter_lazy(PARAVIRT_LAZY_MMU);
159 }
160
paravirt_leave_lazy_mmu(void)161 void paravirt_leave_lazy_mmu(void)
162 {
163 leave_lazy(PARAVIRT_LAZY_MMU);
164 }
165
paravirt_flush_lazy_mmu(void)166 void paravirt_flush_lazy_mmu(void)
167 {
168 preempt_disable();
169
170 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
171 arch_leave_lazy_mmu_mode();
172 arch_enter_lazy_mmu_mode();
173 }
174
175 preempt_enable();
176 }
177
178 #ifdef CONFIG_PARAVIRT_XXL
paravirt_start_context_switch(struct task_struct * prev)179 void paravirt_start_context_switch(struct task_struct *prev)
180 {
181 BUG_ON(preemptible());
182
183 if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
184 arch_leave_lazy_mmu_mode();
185 set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
186 }
187 enter_lazy(PARAVIRT_LAZY_CPU);
188 }
189
paravirt_end_context_switch(struct task_struct * next)190 void paravirt_end_context_switch(struct task_struct *next)
191 {
192 BUG_ON(preemptible());
193
194 leave_lazy(PARAVIRT_LAZY_CPU);
195
196 if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES))
197 arch_enter_lazy_mmu_mode();
198 }
199
pv_native_read_cr2(void)200 static noinstr unsigned long pv_native_read_cr2(void)
201 {
202 return native_read_cr2();
203 }
204
pv_native_write_cr2(unsigned long val)205 static noinstr void pv_native_write_cr2(unsigned long val)
206 {
207 native_write_cr2(val);
208 }
209
pv_native_get_debugreg(int regno)210 static noinstr unsigned long pv_native_get_debugreg(int regno)
211 {
212 return native_get_debugreg(regno);
213 }
214
pv_native_set_debugreg(int regno,unsigned long val)215 static noinstr void pv_native_set_debugreg(int regno, unsigned long val)
216 {
217 native_set_debugreg(regno, val);
218 }
219
pv_native_wbinvd(void)220 noinstr void pv_native_wbinvd(void)
221 {
222 native_wbinvd();
223 }
224
pv_native_irq_enable(void)225 static noinstr void pv_native_irq_enable(void)
226 {
227 native_irq_enable();
228 }
229
pv_native_irq_disable(void)230 static noinstr void pv_native_irq_disable(void)
231 {
232 native_irq_disable();
233 }
234
pv_native_safe_halt(void)235 static noinstr void pv_native_safe_halt(void)
236 {
237 native_safe_halt();
238 }
239 #endif
240
paravirt_get_lazy_mode(void)241 enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
242 {
243 if (in_interrupt())
244 return PARAVIRT_LAZY_NONE;
245
246 return this_cpu_read(paravirt_lazy_mode);
247 }
248
249 struct pv_info pv_info = {
250 .name = "bare hardware",
251 #ifdef CONFIG_PARAVIRT_XXL
252 .extra_user_64bit_cs = __USER_CS,
253 #endif
254 };
255
256 /* 64-bit pagetable entries */
257 #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
258
259 struct paravirt_patch_template pv_ops = {
260 /* Cpu ops. */
261 .cpu.io_delay = native_io_delay,
262
263 #ifdef CONFIG_PARAVIRT_XXL
264 .cpu.cpuid = native_cpuid,
265 .cpu.get_debugreg = pv_native_get_debugreg,
266 .cpu.set_debugreg = pv_native_set_debugreg,
267 .cpu.read_cr0 = native_read_cr0,
268 .cpu.write_cr0 = native_write_cr0,
269 .cpu.write_cr4 = native_write_cr4,
270 .cpu.wbinvd = pv_native_wbinvd,
271 .cpu.read_msr = native_read_msr,
272 .cpu.write_msr = native_write_msr,
273 .cpu.read_msr_safe = native_read_msr_safe,
274 .cpu.write_msr_safe = native_write_msr_safe,
275 .cpu.read_pmc = native_read_pmc,
276 .cpu.load_tr_desc = native_load_tr_desc,
277 .cpu.set_ldt = native_set_ldt,
278 .cpu.load_gdt = native_load_gdt,
279 .cpu.load_idt = native_load_idt,
280 .cpu.store_tr = native_store_tr,
281 .cpu.load_tls = native_load_tls,
282 .cpu.load_gs_index = native_load_gs_index,
283 .cpu.write_ldt_entry = native_write_ldt_entry,
284 .cpu.write_gdt_entry = native_write_gdt_entry,
285 .cpu.write_idt_entry = native_write_idt_entry,
286
287 .cpu.alloc_ldt = paravirt_nop,
288 .cpu.free_ldt = paravirt_nop,
289
290 .cpu.load_sp0 = native_load_sp0,
291
292 #ifdef CONFIG_X86_IOPL_IOPERM
293 .cpu.invalidate_io_bitmap = native_tss_invalidate_io_bitmap,
294 .cpu.update_io_bitmap = native_tss_update_io_bitmap,
295 #endif
296
297 .cpu.start_context_switch = paravirt_nop,
298 .cpu.end_context_switch = paravirt_nop,
299
300 /* Irq ops. */
301 .irq.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
302 .irq.irq_disable = __PV_IS_CALLEE_SAVE(pv_native_irq_disable),
303 .irq.irq_enable = __PV_IS_CALLEE_SAVE(pv_native_irq_enable),
304 .irq.safe_halt = pv_native_safe_halt,
305 .irq.halt = native_halt,
306 #endif /* CONFIG_PARAVIRT_XXL */
307
308 /* Mmu ops. */
309 .mmu.flush_tlb_user = native_flush_tlb_local,
310 .mmu.flush_tlb_kernel = native_flush_tlb_global,
311 .mmu.flush_tlb_one_user = native_flush_tlb_one_user,
312 .mmu.flush_tlb_multi = native_flush_tlb_multi,
313 .mmu.tlb_remove_table =
314 (void (*)(struct mmu_gather *, void *))tlb_remove_page,
315
316 .mmu.exit_mmap = paravirt_nop,
317 .mmu.notify_page_enc_status_changed = paravirt_nop,
318
319 #ifdef CONFIG_PARAVIRT_XXL
320 .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2),
321 .mmu.write_cr2 = pv_native_write_cr2,
322 .mmu.read_cr3 = __native_read_cr3,
323 .mmu.write_cr3 = native_write_cr3,
324
325 .mmu.pgd_alloc = __paravirt_pgd_alloc,
326 .mmu.pgd_free = paravirt_nop,
327
328 .mmu.alloc_pte = paravirt_nop,
329 .mmu.alloc_pmd = paravirt_nop,
330 .mmu.alloc_pud = paravirt_nop,
331 .mmu.alloc_p4d = paravirt_nop,
332 .mmu.release_pte = paravirt_nop,
333 .mmu.release_pmd = paravirt_nop,
334 .mmu.release_pud = paravirt_nop,
335 .mmu.release_p4d = paravirt_nop,
336
337 .mmu.set_pte = native_set_pte,
338 .mmu.set_pmd = native_set_pmd,
339
340 .mmu.ptep_modify_prot_start = __ptep_modify_prot_start,
341 .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit,
342
343 .mmu.set_pud = native_set_pud,
344
345 .mmu.pmd_val = PTE_IDENT,
346 .mmu.make_pmd = PTE_IDENT,
347
348 .mmu.pud_val = PTE_IDENT,
349 .mmu.make_pud = PTE_IDENT,
350
351 .mmu.set_p4d = native_set_p4d,
352
353 #if CONFIG_PGTABLE_LEVELS >= 5
354 .mmu.p4d_val = PTE_IDENT,
355 .mmu.make_p4d = PTE_IDENT,
356
357 .mmu.set_pgd = native_set_pgd,
358 #endif /* CONFIG_PGTABLE_LEVELS >= 5 */
359
360 .mmu.pte_val = PTE_IDENT,
361 .mmu.pgd_val = PTE_IDENT,
362
363 .mmu.make_pte = PTE_IDENT,
364 .mmu.make_pgd = PTE_IDENT,
365
366 .mmu.dup_mmap = paravirt_nop,
367 .mmu.activate_mm = paravirt_nop,
368
369 .mmu.lazy_mode = {
370 .enter = paravirt_nop,
371 .leave = paravirt_nop,
372 .flush = paravirt_nop,
373 },
374
375 .mmu.set_fixmap = native_set_fixmap,
376 #endif /* CONFIG_PARAVIRT_XXL */
377
378 #if defined(CONFIG_PARAVIRT_SPINLOCKS)
379 /* Lock ops. */
380 #ifdef CONFIG_SMP
381 .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
382 .lock.queued_spin_unlock =
383 PV_CALLEE_SAVE(__native_queued_spin_unlock),
384 .lock.wait = paravirt_nop,
385 .lock.kick = paravirt_nop,
386 .lock.vcpu_is_preempted =
387 PV_CALLEE_SAVE(__native_vcpu_is_preempted),
388 #endif /* SMP */
389 #endif
390 };
391
392 #ifdef CONFIG_PARAVIRT_XXL
393 NOKPROBE_SYMBOL(native_load_idt);
394 #endif
395
396 EXPORT_SYMBOL(pv_ops);
397 EXPORT_SYMBOL_GPL(pv_info);
398