1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2021 Google LLC
4 * Author: Fuad Tabba <tabba@google.com>
5 */
6
7 #include <linux/irqchip/arm-gic-v3.h>
8
9 #include <asm/kvm_asm.h>
10 #include <asm/kvm_mmu.h>
11
12 #include <hyp/adjust_pc.h>
13
14 #include <nvhe/fixed_config.h>
15
16 #include "../../sys_regs.h"
17
18 /*
19 * Copies of the host's CPU features registers holding sanitized values at hyp.
20 */
21 u64 id_aa64pfr0_el1_sys_val;
22 u64 id_aa64pfr1_el1_sys_val;
23 u64 id_aa64isar0_el1_sys_val;
24 u64 id_aa64isar1_el1_sys_val;
25 u64 id_aa64isar2_el1_sys_val;
26 u64 id_aa64mmfr0_el1_sys_val;
27 u64 id_aa64mmfr1_el1_sys_val;
28 u64 id_aa64mmfr2_el1_sys_val;
29 u64 id_aa64smfr0_el1_sys_val;
30
31 /*
32 * Inject an unknown/undefined exception to an AArch64 guest while most of its
33 * sysregs are live.
34 */
inject_undef64(struct kvm_vcpu * vcpu)35 static void inject_undef64(struct kvm_vcpu *vcpu)
36 {
37 u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
38
39 *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
40 *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR);
41
42 kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
43
44 __kvm_adjust_pc(vcpu);
45
46 write_sysreg_el1(esr, SYS_ESR);
47 write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR);
48 write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
49 write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
50 }
51
52 /*
53 * Returns the restricted features values of the feature register based on the
54 * limitations in restrict_fields.
55 * A feature id field value of 0b0000 does not impose any restrictions.
56 * Note: Use only for unsigned feature field values.
57 */
get_restricted_features_unsigned(u64 sys_reg_val,u64 restrict_fields)58 static u64 get_restricted_features_unsigned(u64 sys_reg_val,
59 u64 restrict_fields)
60 {
61 u64 value = 0UL;
62 u64 mask = GENMASK_ULL(ARM64_FEATURE_FIELD_BITS - 1, 0);
63
64 /*
65 * According to the Arm Architecture Reference Manual, feature fields
66 * use increasing values to indicate increases in functionality.
67 * Iterate over the restricted feature fields and calculate the minimum
68 * unsigned value between the one supported by the system, and what the
69 * value is being restricted to.
70 */
71 while (sys_reg_val && restrict_fields) {
72 value |= min(sys_reg_val & mask, restrict_fields & mask);
73 sys_reg_val &= ~mask;
74 restrict_fields &= ~mask;
75 mask <<= ARM64_FEATURE_FIELD_BITS;
76 }
77
78 return value;
79 }
80
81 /*
82 * Functions that return the value of feature id registers for protected VMs
83 * based on allowed features, system features, and KVM support.
84 */
85
get_pvm_id_aa64pfr0(const struct kvm_vcpu * vcpu)86 static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu)
87 {
88 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm);
89 u64 set_mask = 0;
90 u64 allow_mask = PVM_ID_AA64PFR0_ALLOW;
91
92 set_mask |= get_restricted_features_unsigned(id_aa64pfr0_el1_sys_val,
93 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED);
94
95 /* Spectre and Meltdown mitigation in KVM */
96 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2),
97 (u64)kvm->arch.pfr0_csv2);
98 set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3),
99 (u64)kvm->arch.pfr0_csv3);
100
101 return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask;
102 }
103
get_pvm_id_aa64pfr1(const struct kvm_vcpu * vcpu)104 static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu)
105 {
106 const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm);
107 u64 allow_mask = PVM_ID_AA64PFR1_ALLOW;
108
109 if (!kvm_has_mte(kvm))
110 allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE);
111
112 return id_aa64pfr1_el1_sys_val & allow_mask;
113 }
114
get_pvm_id_aa64zfr0(const struct kvm_vcpu * vcpu)115 static u64 get_pvm_id_aa64zfr0(const struct kvm_vcpu *vcpu)
116 {
117 /*
118 * No support for Scalable Vectors, therefore, hyp has no sanitized
119 * copy of the feature id register.
120 */
121 BUILD_BUG_ON(PVM_ID_AA64ZFR0_ALLOW != 0ULL);
122 return 0;
123 }
124
get_pvm_id_aa64dfr0(const struct kvm_vcpu * vcpu)125 static u64 get_pvm_id_aa64dfr0(const struct kvm_vcpu *vcpu)
126 {
127 /*
128 * No support for debug, including breakpoints, and watchpoints,
129 * therefore, pKVM has no sanitized copy of the feature id register.
130 */
131 BUILD_BUG_ON(PVM_ID_AA64DFR0_ALLOW != 0ULL);
132 return 0;
133 }
134
get_pvm_id_aa64dfr1(const struct kvm_vcpu * vcpu)135 static u64 get_pvm_id_aa64dfr1(const struct kvm_vcpu *vcpu)
136 {
137 /*
138 * No support for debug, therefore, hyp has no sanitized copy of the
139 * feature id register.
140 */
141 BUILD_BUG_ON(PVM_ID_AA64DFR1_ALLOW != 0ULL);
142 return 0;
143 }
144
get_pvm_id_aa64afr0(const struct kvm_vcpu * vcpu)145 static u64 get_pvm_id_aa64afr0(const struct kvm_vcpu *vcpu)
146 {
147 /*
148 * No support for implementation defined features, therefore, hyp has no
149 * sanitized copy of the feature id register.
150 */
151 BUILD_BUG_ON(PVM_ID_AA64AFR0_ALLOW != 0ULL);
152 return 0;
153 }
154
get_pvm_id_aa64afr1(const struct kvm_vcpu * vcpu)155 static u64 get_pvm_id_aa64afr1(const struct kvm_vcpu *vcpu)
156 {
157 /*
158 * No support for implementation defined features, therefore, hyp has no
159 * sanitized copy of the feature id register.
160 */
161 BUILD_BUG_ON(PVM_ID_AA64AFR1_ALLOW != 0ULL);
162 return 0;
163 }
164
get_pvm_id_aa64isar0(const struct kvm_vcpu * vcpu)165 static u64 get_pvm_id_aa64isar0(const struct kvm_vcpu *vcpu)
166 {
167 return id_aa64isar0_el1_sys_val & PVM_ID_AA64ISAR0_ALLOW;
168 }
169
get_pvm_id_aa64isar1(const struct kvm_vcpu * vcpu)170 static u64 get_pvm_id_aa64isar1(const struct kvm_vcpu *vcpu)
171 {
172 u64 allow_mask = PVM_ID_AA64ISAR1_ALLOW;
173
174 if (!vcpu_has_ptrauth(vcpu))
175 allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) |
176 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) |
177 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) |
178 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI));
179
180 return id_aa64isar1_el1_sys_val & allow_mask;
181 }
182
get_pvm_id_aa64isar2(const struct kvm_vcpu * vcpu)183 static u64 get_pvm_id_aa64isar2(const struct kvm_vcpu *vcpu)
184 {
185 u64 allow_mask = PVM_ID_AA64ISAR2_ALLOW;
186
187 if (!vcpu_has_ptrauth(vcpu))
188 allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) |
189 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
190
191 return id_aa64isar2_el1_sys_val & allow_mask;
192 }
193
get_pvm_id_aa64mmfr0(const struct kvm_vcpu * vcpu)194 static u64 get_pvm_id_aa64mmfr0(const struct kvm_vcpu *vcpu)
195 {
196 u64 set_mask;
197
198 set_mask = get_restricted_features_unsigned(id_aa64mmfr0_el1_sys_val,
199 PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED);
200
201 return (id_aa64mmfr0_el1_sys_val & PVM_ID_AA64MMFR0_ALLOW) | set_mask;
202 }
203
get_pvm_id_aa64mmfr1(const struct kvm_vcpu * vcpu)204 static u64 get_pvm_id_aa64mmfr1(const struct kvm_vcpu *vcpu)
205 {
206 return id_aa64mmfr1_el1_sys_val & PVM_ID_AA64MMFR1_ALLOW;
207 }
208
get_pvm_id_aa64mmfr2(const struct kvm_vcpu * vcpu)209 static u64 get_pvm_id_aa64mmfr2(const struct kvm_vcpu *vcpu)
210 {
211 return id_aa64mmfr2_el1_sys_val & PVM_ID_AA64MMFR2_ALLOW;
212 }
213
214 /* Read a sanitized cpufeature ID register by its encoding */
pvm_read_id_reg(const struct kvm_vcpu * vcpu,u32 id)215 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id)
216 {
217 switch (id) {
218 case SYS_ID_AA64PFR0_EL1:
219 return get_pvm_id_aa64pfr0(vcpu);
220 case SYS_ID_AA64PFR1_EL1:
221 return get_pvm_id_aa64pfr1(vcpu);
222 case SYS_ID_AA64ZFR0_EL1:
223 return get_pvm_id_aa64zfr0(vcpu);
224 case SYS_ID_AA64DFR0_EL1:
225 return get_pvm_id_aa64dfr0(vcpu);
226 case SYS_ID_AA64DFR1_EL1:
227 return get_pvm_id_aa64dfr1(vcpu);
228 case SYS_ID_AA64AFR0_EL1:
229 return get_pvm_id_aa64afr0(vcpu);
230 case SYS_ID_AA64AFR1_EL1:
231 return get_pvm_id_aa64afr1(vcpu);
232 case SYS_ID_AA64ISAR0_EL1:
233 return get_pvm_id_aa64isar0(vcpu);
234 case SYS_ID_AA64ISAR1_EL1:
235 return get_pvm_id_aa64isar1(vcpu);
236 case SYS_ID_AA64ISAR2_EL1:
237 return get_pvm_id_aa64isar2(vcpu);
238 case SYS_ID_AA64MMFR0_EL1:
239 return get_pvm_id_aa64mmfr0(vcpu);
240 case SYS_ID_AA64MMFR1_EL1:
241 return get_pvm_id_aa64mmfr1(vcpu);
242 case SYS_ID_AA64MMFR2_EL1:
243 return get_pvm_id_aa64mmfr2(vcpu);
244 default:
245 /* Unhandled ID register, RAZ */
246 return 0;
247 }
248 }
249
read_id_reg(const struct kvm_vcpu * vcpu,struct sys_reg_desc const * r)250 static u64 read_id_reg(const struct kvm_vcpu *vcpu,
251 struct sys_reg_desc const *r)
252 {
253 return pvm_read_id_reg(vcpu, reg_to_encoding(r));
254 }
255
256 /* Handler to RAZ/WI sysregs */
pvm_access_raz_wi(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)257 static bool pvm_access_raz_wi(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
258 const struct sys_reg_desc *r)
259 {
260 if (!p->is_write)
261 p->regval = 0;
262
263 return true;
264 }
265
266 /*
267 * Accessor for AArch32 feature id registers.
268 *
269 * The value of these registers is "unknown" according to the spec if AArch32
270 * isn't supported.
271 */
pvm_access_id_aarch32(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)272 static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu,
273 struct sys_reg_params *p,
274 const struct sys_reg_desc *r)
275 {
276 if (p->is_write) {
277 inject_undef64(vcpu);
278 return false;
279 }
280
281 /*
282 * No support for AArch32 guests, therefore, pKVM has no sanitized copy
283 * of AArch32 feature id registers.
284 */
285 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1),
286 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
287
288 return pvm_access_raz_wi(vcpu, p, r);
289 }
290
291 /*
292 * Accessor for AArch64 feature id registers.
293 *
294 * If access is allowed, set the regval to the protected VM's view of the
295 * register and return true.
296 * Otherwise, inject an undefined exception and return false.
297 */
pvm_access_id_aarch64(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)298 static bool pvm_access_id_aarch64(struct kvm_vcpu *vcpu,
299 struct sys_reg_params *p,
300 const struct sys_reg_desc *r)
301 {
302 if (p->is_write) {
303 inject_undef64(vcpu);
304 return false;
305 }
306
307 p->regval = read_id_reg(vcpu, r);
308 return true;
309 }
310
pvm_gic_read_sre(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)311 static bool pvm_gic_read_sre(struct kvm_vcpu *vcpu,
312 struct sys_reg_params *p,
313 const struct sys_reg_desc *r)
314 {
315 /* pVMs only support GICv3. 'nuf said. */
316 if (!p->is_write)
317 p->regval = ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB | ICC_SRE_EL1_SRE;
318
319 return true;
320 }
321
322 /* Mark the specified system register as an AArch32 feature id register. */
323 #define AARCH32(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch32 }
324
325 /* Mark the specified system register as an AArch64 feature id register. */
326 #define AARCH64(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch64 }
327
328 /*
329 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
330 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
331 * (1 <= crm < 8, 0 <= Op2 < 8).
332 */
333 #define ID_UNALLOCATED(crm, op2) { \
334 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
335 .access = pvm_access_id_aarch64, \
336 }
337
338 /* Mark the specified system register as Read-As-Zero/Write-Ignored */
339 #define RAZ_WI(REG) { SYS_DESC(REG), .access = pvm_access_raz_wi }
340
341 /* Mark the specified system register as not being handled in hyp. */
342 #define HOST_HANDLED(REG) { SYS_DESC(REG), .access = NULL }
343
344 /*
345 * Architected system registers.
346 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
347 *
348 * NOTE: Anything not explicitly listed here is *restricted by default*, i.e.,
349 * it will lead to injecting an exception into the guest.
350 */
351 static const struct sys_reg_desc pvm_sys_reg_descs[] = {
352 /* Cache maintenance by set/way operations are restricted. */
353
354 /* Debug and Trace Registers are restricted. */
355
356 /* AArch64 mappings of the AArch32 ID registers */
357 /* CRm=1 */
358 AARCH32(SYS_ID_PFR0_EL1),
359 AARCH32(SYS_ID_PFR1_EL1),
360 AARCH32(SYS_ID_DFR0_EL1),
361 AARCH32(SYS_ID_AFR0_EL1),
362 AARCH32(SYS_ID_MMFR0_EL1),
363 AARCH32(SYS_ID_MMFR1_EL1),
364 AARCH32(SYS_ID_MMFR2_EL1),
365 AARCH32(SYS_ID_MMFR3_EL1),
366
367 /* CRm=2 */
368 AARCH32(SYS_ID_ISAR0_EL1),
369 AARCH32(SYS_ID_ISAR1_EL1),
370 AARCH32(SYS_ID_ISAR2_EL1),
371 AARCH32(SYS_ID_ISAR3_EL1),
372 AARCH32(SYS_ID_ISAR4_EL1),
373 AARCH32(SYS_ID_ISAR5_EL1),
374 AARCH32(SYS_ID_MMFR4_EL1),
375 AARCH32(SYS_ID_ISAR6_EL1),
376
377 /* CRm=3 */
378 AARCH32(SYS_MVFR0_EL1),
379 AARCH32(SYS_MVFR1_EL1),
380 AARCH32(SYS_MVFR2_EL1),
381 ID_UNALLOCATED(3,3),
382 AARCH32(SYS_ID_PFR2_EL1),
383 AARCH32(SYS_ID_DFR1_EL1),
384 AARCH32(SYS_ID_MMFR5_EL1),
385 ID_UNALLOCATED(3,7),
386
387 /* AArch64 ID registers */
388 /* CRm=4 */
389 AARCH64(SYS_ID_AA64PFR0_EL1),
390 AARCH64(SYS_ID_AA64PFR1_EL1),
391 ID_UNALLOCATED(4,2),
392 ID_UNALLOCATED(4,3),
393 AARCH64(SYS_ID_AA64ZFR0_EL1),
394 ID_UNALLOCATED(4,5),
395 ID_UNALLOCATED(4,6),
396 ID_UNALLOCATED(4,7),
397 AARCH64(SYS_ID_AA64DFR0_EL1),
398 AARCH64(SYS_ID_AA64DFR1_EL1),
399 ID_UNALLOCATED(5,2),
400 ID_UNALLOCATED(5,3),
401 AARCH64(SYS_ID_AA64AFR0_EL1),
402 AARCH64(SYS_ID_AA64AFR1_EL1),
403 ID_UNALLOCATED(5,6),
404 ID_UNALLOCATED(5,7),
405 AARCH64(SYS_ID_AA64ISAR0_EL1),
406 AARCH64(SYS_ID_AA64ISAR1_EL1),
407 AARCH64(SYS_ID_AA64ISAR2_EL1),
408 ID_UNALLOCATED(6,3),
409 ID_UNALLOCATED(6,4),
410 ID_UNALLOCATED(6,5),
411 ID_UNALLOCATED(6,6),
412 ID_UNALLOCATED(6,7),
413 AARCH64(SYS_ID_AA64MMFR0_EL1),
414 AARCH64(SYS_ID_AA64MMFR1_EL1),
415 AARCH64(SYS_ID_AA64MMFR2_EL1),
416 ID_UNALLOCATED(7,3),
417 ID_UNALLOCATED(7,4),
418 ID_UNALLOCATED(7,5),
419 ID_UNALLOCATED(7,6),
420 ID_UNALLOCATED(7,7),
421
422 /* Scalable Vector Registers are restricted. */
423
424 RAZ_WI(SYS_ERRIDR_EL1),
425 RAZ_WI(SYS_ERRSELR_EL1),
426 RAZ_WI(SYS_ERXFR_EL1),
427 RAZ_WI(SYS_ERXCTLR_EL1),
428 RAZ_WI(SYS_ERXSTATUS_EL1),
429 RAZ_WI(SYS_ERXADDR_EL1),
430 RAZ_WI(SYS_ERXMISC0_EL1),
431 RAZ_WI(SYS_ERXMISC1_EL1),
432
433 /* Performance Monitoring Registers are restricted. */
434
435 /* Limited Ordering Regions Registers are restricted. */
436
437 HOST_HANDLED(SYS_ICC_SGI1R_EL1),
438 HOST_HANDLED(SYS_ICC_ASGI1R_EL1),
439 HOST_HANDLED(SYS_ICC_SGI0R_EL1),
440 { SYS_DESC(SYS_ICC_SRE_EL1), .access = pvm_gic_read_sre, },
441
442 HOST_HANDLED(SYS_CCSIDR_EL1),
443 HOST_HANDLED(SYS_CLIDR_EL1),
444 HOST_HANDLED(SYS_CSSELR_EL1),
445 HOST_HANDLED(SYS_CTR_EL0),
446
447 /* Performance Monitoring Registers are restricted. */
448
449 /* Activity Monitoring Registers are restricted. */
450
451 HOST_HANDLED(SYS_CNTP_TVAL_EL0),
452 HOST_HANDLED(SYS_CNTP_CTL_EL0),
453 HOST_HANDLED(SYS_CNTP_CVAL_EL0),
454
455 /* Performance Monitoring Registers are restricted. */
456 };
457
458 /*
459 * Checks that the sysreg table is unique and in-order.
460 *
461 * Returns 0 if the table is consistent, or 1 otherwise.
462 */
kvm_check_pvm_sysreg_table(void)463 int kvm_check_pvm_sysreg_table(void)
464 {
465 unsigned int i;
466
467 for (i = 1; i < ARRAY_SIZE(pvm_sys_reg_descs); i++) {
468 if (cmp_sys_reg(&pvm_sys_reg_descs[i-1], &pvm_sys_reg_descs[i]) >= 0)
469 return 1;
470 }
471
472 return 0;
473 }
474
475 /*
476 * Handler for protected VM MSR, MRS or System instruction execution.
477 *
478 * Returns true if the hypervisor has handled the exit, and control should go
479 * back to the guest, or false if it hasn't, to be handled by the host.
480 */
kvm_handle_pvm_sysreg(struct kvm_vcpu * vcpu,u64 * exit_code)481 bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
482 {
483 const struct sys_reg_desc *r;
484 struct sys_reg_params params;
485 unsigned long esr = kvm_vcpu_get_esr(vcpu);
486 int Rt = kvm_vcpu_sys_get_rt(vcpu);
487
488 params = esr_sys64_to_params(esr);
489 params.regval = vcpu_get_reg(vcpu, Rt);
490
491 r = find_reg(¶ms, pvm_sys_reg_descs, ARRAY_SIZE(pvm_sys_reg_descs));
492
493 /* Undefined (RESTRICTED). */
494 if (r == NULL) {
495 inject_undef64(vcpu);
496 return true;
497 }
498
499 /* Handled by the host (HOST_HANDLED) */
500 if (r->access == NULL)
501 return false;
502
503 /* Handled by hyp: skip instruction if instructed to do so. */
504 if (r->access(vcpu, ¶ms, r))
505 __kvm_skip_instr(vcpu);
506
507 if (!params.is_write)
508 vcpu_set_reg(vcpu, Rt, params.regval);
509
510 return true;
511 }
512
513 /*
514 * Handler for protected VM restricted exceptions.
515 *
516 * Inject an undefined exception into the guest and return true to indicate that
517 * the hypervisor has handled the exit, and control should go back to the guest.
518 */
kvm_handle_pvm_restricted(struct kvm_vcpu * vcpu,u64 * exit_code)519 bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code)
520 {
521 inject_undef64(vcpu);
522 return true;
523 }
524