1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * AMD Encrypted Register State Support
4  *
5  * Author: Joerg Roedel <jroedel@suse.de>
6  *
7  * This file is not compiled stand-alone. It contains code shared
8  * between the pre-decompression boot code and the running Linux kernel
9  * and is included directly into both code-bases.
10  */
11 
12 #ifndef __BOOT_COMPRESSED
13 #define error(v)	pr_err(v)
14 #define has_cpuflag(f)	boot_cpu_has(f)
15 #endif
16 
17 /* I/O parameters for CPUID-related helpers */
18 struct cpuid_leaf {
19 	u32 fn;
20 	u32 subfn;
21 	u32 eax;
22 	u32 ebx;
23 	u32 ecx;
24 	u32 edx;
25 };
26 
27 /*
28  * Individual entries of the SNP CPUID table, as defined by the SNP
29  * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
30  */
31 struct snp_cpuid_fn {
32 	u32 eax_in;
33 	u32 ecx_in;
34 	u64 xcr0_in;
35 	u64 xss_in;
36 	u32 eax;
37 	u32 ebx;
38 	u32 ecx;
39 	u32 edx;
40 	u64 __reserved;
41 } __packed;
42 
43 /*
44  * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
45  * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
46  * of 64 entries per CPUID table.
47  */
48 #define SNP_CPUID_COUNT_MAX 64
49 
50 struct snp_cpuid_table {
51 	u32 count;
52 	u32 __reserved1;
53 	u64 __reserved2;
54 	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
55 } __packed;
56 
57 /*
58  * Since feature negotiation related variables are set early in the boot
59  * process they must reside in the .data section so as not to be zeroed
60  * out when the .bss section is later cleared.
61  *
62  * GHCB protocol version negotiated with the hypervisor.
63  */
64 static u16 ghcb_version __ro_after_init;
65 
66 /* Copy of the SNP firmware's CPUID page. */
67 static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
68 
69 /*
70  * These will be initialized based on CPUID table so that non-present
71  * all-zero leaves (for sparse tables) can be differentiated from
72  * invalid/out-of-range leaves. This is needed since all-zero leaves
73  * still need to be post-processed.
74  */
75 static u32 cpuid_std_range_max __ro_after_init;
76 static u32 cpuid_hyp_range_max __ro_after_init;
77 static u32 cpuid_ext_range_max __ro_after_init;
78 
sev_es_check_cpu_features(void)79 static bool __init sev_es_check_cpu_features(void)
80 {
81 	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
82 		error("RDRAND instruction not supported - no trusted source of randomness available\n");
83 		return false;
84 	}
85 
86 	return true;
87 }
88 
sev_es_terminate(unsigned int set,unsigned int reason)89 static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason)
90 {
91 	u64 val = GHCB_MSR_TERM_REQ;
92 
93 	/* Tell the hypervisor what went wrong. */
94 	val |= GHCB_SEV_TERM_REASON(set, reason);
95 
96 	/* Request Guest Termination from Hypvervisor */
97 	sev_es_wr_ghcb_msr(val);
98 	VMGEXIT();
99 
100 	while (true)
101 		asm volatile("hlt\n" : : : "memory");
102 }
103 
104 /*
105  * The hypervisor features are available from GHCB version 2 onward.
106  */
get_hv_features(void)107 static u64 get_hv_features(void)
108 {
109 	u64 val;
110 
111 	if (ghcb_version < 2)
112 		return 0;
113 
114 	sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
115 	VMGEXIT();
116 
117 	val = sev_es_rd_ghcb_msr();
118 	if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
119 		return 0;
120 
121 	return GHCB_MSR_HV_FT_RESP_VAL(val);
122 }
123 
snp_register_ghcb_early(unsigned long paddr)124 static void snp_register_ghcb_early(unsigned long paddr)
125 {
126 	unsigned long pfn = paddr >> PAGE_SHIFT;
127 	u64 val;
128 
129 	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
130 	VMGEXIT();
131 
132 	val = sev_es_rd_ghcb_msr();
133 
134 	/* If the response GPA is not ours then abort the guest */
135 	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
136 	    (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
137 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
138 }
139 
sev_es_negotiate_protocol(void)140 static bool sev_es_negotiate_protocol(void)
141 {
142 	u64 val;
143 
144 	/* Do the GHCB protocol version negotiation */
145 	sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
146 	VMGEXIT();
147 	val = sev_es_rd_ghcb_msr();
148 
149 	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
150 		return false;
151 
152 	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
153 	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
154 		return false;
155 
156 	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
157 
158 	return true;
159 }
160 
vc_ghcb_invalidate(struct ghcb * ghcb)161 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
162 {
163 	ghcb->save.sw_exit_code = 0;
164 	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
165 }
166 
vc_decoding_needed(unsigned long exit_code)167 static bool vc_decoding_needed(unsigned long exit_code)
168 {
169 	/* Exceptions don't require to decode the instruction */
170 	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
171 		 exit_code <= SVM_EXIT_LAST_EXCP);
172 }
173 
vc_init_em_ctxt(struct es_em_ctxt * ctxt,struct pt_regs * regs,unsigned long exit_code)174 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
175 				      struct pt_regs *regs,
176 				      unsigned long exit_code)
177 {
178 	enum es_result ret = ES_OK;
179 
180 	memset(ctxt, 0, sizeof(*ctxt));
181 	ctxt->regs = regs;
182 
183 	if (vc_decoding_needed(exit_code))
184 		ret = vc_decode_insn(ctxt);
185 
186 	return ret;
187 }
188 
vc_finish_insn(struct es_em_ctxt * ctxt)189 static void vc_finish_insn(struct es_em_ctxt *ctxt)
190 {
191 	ctxt->regs->ip += ctxt->insn.length;
192 }
193 
verify_exception_info(struct ghcb * ghcb,struct es_em_ctxt * ctxt)194 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
195 {
196 	u32 ret;
197 
198 	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
199 	if (!ret)
200 		return ES_OK;
201 
202 	if (ret == 1) {
203 		u64 info = ghcb->save.sw_exit_info_2;
204 		unsigned long v = info & SVM_EVTINJ_VEC_MASK;
205 
206 		/* Check if exception information from hypervisor is sane. */
207 		if ((info & SVM_EVTINJ_VALID) &&
208 		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
209 		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
210 			ctxt->fi.vector = v;
211 
212 			if (info & SVM_EVTINJ_VALID_ERR)
213 				ctxt->fi.error_code = info >> 32;
214 
215 			return ES_EXCEPTION;
216 		}
217 	}
218 
219 	return ES_VMM_ERROR;
220 }
221 
sev_es_ghcb_hv_call(struct ghcb * ghcb,struct es_em_ctxt * ctxt,u64 exit_code,u64 exit_info_1,u64 exit_info_2)222 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
223 					  struct es_em_ctxt *ctxt,
224 					  u64 exit_code, u64 exit_info_1,
225 					  u64 exit_info_2)
226 {
227 	/* Fill in protocol and format specifiers */
228 	ghcb->protocol_version = ghcb_version;
229 	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
230 
231 	ghcb_set_sw_exit_code(ghcb, exit_code);
232 	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
233 	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
234 
235 	sev_es_wr_ghcb_msr(__pa(ghcb));
236 	VMGEXIT();
237 
238 	return verify_exception_info(ghcb, ctxt);
239 }
240 
__sev_cpuid_hv(u32 fn,int reg_idx,u32 * reg)241 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
242 {
243 	u64 val;
244 
245 	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
246 	VMGEXIT();
247 	val = sev_es_rd_ghcb_msr();
248 	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
249 		return -EIO;
250 
251 	*reg = (val >> 32);
252 
253 	return 0;
254 }
255 
sev_cpuid_hv(struct cpuid_leaf * leaf)256 static int sev_cpuid_hv(struct cpuid_leaf *leaf)
257 {
258 	int ret;
259 
260 	/*
261 	 * MSR protocol does not support fetching non-zero subfunctions, but is
262 	 * sufficient to handle current early-boot cases. Should that change,
263 	 * make sure to report an error rather than ignoring the index and
264 	 * grabbing random values. If this issue arises in the future, handling
265 	 * can be added here to use GHCB-page protocol for cases that occur late
266 	 * enough in boot that GHCB page is available.
267 	 */
268 	if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
269 		return -EINVAL;
270 
271 	ret =         __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
272 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
273 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
274 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
275 
276 	return ret;
277 }
278 
279 /*
280  * This may be called early while still running on the initial identity
281  * mapping. Use RIP-relative addressing to obtain the correct address
282  * while running with the initial identity mapping as well as the
283  * switch-over to kernel virtual addresses later.
284  */
snp_cpuid_get_table(void)285 static const struct snp_cpuid_table *snp_cpuid_get_table(void)
286 {
287 	void *ptr;
288 
289 	asm ("lea cpuid_table_copy(%%rip), %0"
290 	     : "=r" (ptr)
291 	     : "p" (&cpuid_table_copy));
292 
293 	return ptr;
294 }
295 
296 /*
297  * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
298  * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
299  * and 1 based on the corresponding features enabled by a particular
300  * combination of XCR0 and XSS registers so that a guest can look up the
301  * version corresponding to the features currently enabled in its XCR0/XSS
302  * registers. The only values that differ between these versions/table
303  * entries is the enabled XSAVE area size advertised via EBX.
304  *
305  * While hypervisors may choose to make use of this support, it is more
306  * robust/secure for a guest to simply find the entry corresponding to the
307  * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
308  * XSAVE area size using subfunctions 2 through 64, as documented in APM
309  * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
310  *
311  * Since base/legacy XSAVE area size is documented as 0x240, use that value
312  * directly rather than relying on the base size in the CPUID table.
313  *
314  * Return: XSAVE area size on success, 0 otherwise.
315  */
snp_cpuid_calc_xsave_size(u64 xfeatures_en,bool compacted)316 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
317 {
318 	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
319 	u64 xfeatures_found = 0;
320 	u32 xsave_size = 0x240;
321 	int i;
322 
323 	for (i = 0; i < cpuid_table->count; i++) {
324 		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
325 
326 		if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
327 			continue;
328 		if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
329 			continue;
330 		if (xfeatures_found & (BIT_ULL(e->ecx_in)))
331 			continue;
332 
333 		xfeatures_found |= (BIT_ULL(e->ecx_in));
334 
335 		if (compacted)
336 			xsave_size += e->eax;
337 		else
338 			xsave_size = max(xsave_size, e->eax + e->ebx);
339 	}
340 
341 	/*
342 	 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
343 	 * entries in the CPUID table were not present. This is not a valid
344 	 * state to be in.
345 	 */
346 	if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
347 		return 0;
348 
349 	return xsave_size;
350 }
351 
352 static bool
snp_cpuid_get_validated_func(struct cpuid_leaf * leaf)353 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
354 {
355 	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
356 	int i;
357 
358 	for (i = 0; i < cpuid_table->count; i++) {
359 		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
360 
361 		if (e->eax_in != leaf->fn)
362 			continue;
363 
364 		if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
365 			continue;
366 
367 		/*
368 		 * For 0xD subfunctions 0 and 1, only use the entry corresponding
369 		 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
370 		 * See the comments above snp_cpuid_calc_xsave_size() for more
371 		 * details.
372 		 */
373 		if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
374 			if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
375 				continue;
376 
377 		leaf->eax = e->eax;
378 		leaf->ebx = e->ebx;
379 		leaf->ecx = e->ecx;
380 		leaf->edx = e->edx;
381 
382 		return true;
383 	}
384 
385 	return false;
386 }
387 
snp_cpuid_hv(struct cpuid_leaf * leaf)388 static void snp_cpuid_hv(struct cpuid_leaf *leaf)
389 {
390 	if (sev_cpuid_hv(leaf))
391 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
392 }
393 
snp_cpuid_postprocess(struct cpuid_leaf * leaf)394 static int snp_cpuid_postprocess(struct cpuid_leaf *leaf)
395 {
396 	struct cpuid_leaf leaf_hv = *leaf;
397 
398 	switch (leaf->fn) {
399 	case 0x1:
400 		snp_cpuid_hv(&leaf_hv);
401 
402 		/* initial APIC ID */
403 		leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
404 		/* APIC enabled bit */
405 		leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
406 
407 		/* OSXSAVE enabled bit */
408 		if (native_read_cr4() & X86_CR4_OSXSAVE)
409 			leaf->ecx |= BIT(27);
410 		break;
411 	case 0x7:
412 		/* OSPKE enabled bit */
413 		leaf->ecx &= ~BIT(4);
414 		if (native_read_cr4() & X86_CR4_PKE)
415 			leaf->ecx |= BIT(4);
416 		break;
417 	case 0xB:
418 		leaf_hv.subfn = 0;
419 		snp_cpuid_hv(&leaf_hv);
420 
421 		/* extended APIC ID */
422 		leaf->edx = leaf_hv.edx;
423 		break;
424 	case 0xD: {
425 		bool compacted = false;
426 		u64 xcr0 = 1, xss = 0;
427 		u32 xsave_size;
428 
429 		if (leaf->subfn != 0 && leaf->subfn != 1)
430 			return 0;
431 
432 		if (native_read_cr4() & X86_CR4_OSXSAVE)
433 			xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
434 		if (leaf->subfn == 1) {
435 			/* Get XSS value if XSAVES is enabled. */
436 			if (leaf->eax & BIT(3)) {
437 				unsigned long lo, hi;
438 
439 				asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
440 						     : "c" (MSR_IA32_XSS));
441 				xss = (hi << 32) | lo;
442 			}
443 
444 			/*
445 			 * The PPR and APM aren't clear on what size should be
446 			 * encoded in 0xD:0x1:EBX when compaction is not enabled
447 			 * by either XSAVEC (feature bit 1) or XSAVES (feature
448 			 * bit 3) since SNP-capable hardware has these feature
449 			 * bits fixed as 1. KVM sets it to 0 in this case, but
450 			 * to avoid this becoming an issue it's safer to simply
451 			 * treat this as unsupported for SNP guests.
452 			 */
453 			if (!(leaf->eax & (BIT(1) | BIT(3))))
454 				return -EINVAL;
455 
456 			compacted = true;
457 		}
458 
459 		xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
460 		if (!xsave_size)
461 			return -EINVAL;
462 
463 		leaf->ebx = xsave_size;
464 		}
465 		break;
466 	case 0x8000001E:
467 		snp_cpuid_hv(&leaf_hv);
468 
469 		/* extended APIC ID */
470 		leaf->eax = leaf_hv.eax;
471 		/* compute ID */
472 		leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
473 		/* node ID */
474 		leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
475 		break;
476 	default:
477 		/* No fix-ups needed, use values as-is. */
478 		break;
479 	}
480 
481 	return 0;
482 }
483 
484 /*
485  * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
486  * should be treated as fatal by caller.
487  */
snp_cpuid(struct cpuid_leaf * leaf)488 static int snp_cpuid(struct cpuid_leaf *leaf)
489 {
490 	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
491 
492 	if (!cpuid_table->count)
493 		return -EOPNOTSUPP;
494 
495 	if (!snp_cpuid_get_validated_func(leaf)) {
496 		/*
497 		 * Some hypervisors will avoid keeping track of CPUID entries
498 		 * where all values are zero, since they can be handled the
499 		 * same as out-of-range values (all-zero). This is useful here
500 		 * as well as it allows virtually all guest configurations to
501 		 * work using a single SNP CPUID table.
502 		 *
503 		 * To allow for this, there is a need to distinguish between
504 		 * out-of-range entries and in-range zero entries, since the
505 		 * CPUID table entries are only a template that may need to be
506 		 * augmented with additional values for things like
507 		 * CPU-specific information during post-processing. So if it's
508 		 * not in the table, set the values to zero. Then, if they are
509 		 * within a valid CPUID range, proceed with post-processing
510 		 * using zeros as the initial values. Otherwise, skip
511 		 * post-processing and just return zeros immediately.
512 		 */
513 		leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
514 
515 		/* Skip post-processing for out-of-range zero leafs. */
516 		if (!(leaf->fn <= cpuid_std_range_max ||
517 		      (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||
518 		      (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
519 			return 0;
520 	}
521 
522 	return snp_cpuid_postprocess(leaf);
523 }
524 
525 /*
526  * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
527  * page yet, so it only supports the MSR based communication with the
528  * hypervisor and only the CPUID exit-code.
529  */
do_vc_no_ghcb(struct pt_regs * regs,unsigned long exit_code)530 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
531 {
532 	unsigned int subfn = lower_bits(regs->cx, 32);
533 	unsigned int fn = lower_bits(regs->ax, 32);
534 	struct cpuid_leaf leaf;
535 	int ret;
536 
537 	/* Only CPUID is supported via MSR protocol */
538 	if (exit_code != SVM_EXIT_CPUID)
539 		goto fail;
540 
541 	leaf.fn = fn;
542 	leaf.subfn = subfn;
543 
544 	ret = snp_cpuid(&leaf);
545 	if (!ret)
546 		goto cpuid_done;
547 
548 	if (ret != -EOPNOTSUPP)
549 		goto fail;
550 
551 	if (sev_cpuid_hv(&leaf))
552 		goto fail;
553 
554 cpuid_done:
555 	regs->ax = leaf.eax;
556 	regs->bx = leaf.ebx;
557 	regs->cx = leaf.ecx;
558 	regs->dx = leaf.edx;
559 
560 	/*
561 	 * This is a VC handler and the #VC is only raised when SEV-ES is
562 	 * active, which means SEV must be active too. Do sanity checks on the
563 	 * CPUID results to make sure the hypervisor does not trick the kernel
564 	 * into the no-sev path. This could map sensitive data unencrypted and
565 	 * make it accessible to the hypervisor.
566 	 *
567 	 * In particular, check for:
568 	 *	- Availability of CPUID leaf 0x8000001f
569 	 *	- SEV CPUID bit.
570 	 *
571 	 * The hypervisor might still report the wrong C-bit position, but this
572 	 * can't be checked here.
573 	 */
574 
575 	if (fn == 0x80000000 && (regs->ax < 0x8000001f))
576 		/* SEV leaf check */
577 		goto fail;
578 	else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
579 		/* SEV bit */
580 		goto fail;
581 
582 	/* Skip over the CPUID two-byte opcode */
583 	regs->ip += 2;
584 
585 	return;
586 
587 fail:
588 	/* Terminate the guest */
589 	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
590 }
591 
vc_insn_string_read(struct es_em_ctxt * ctxt,void * src,char * buf,unsigned int data_size,unsigned int count,bool backwards)592 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
593 					  void *src, char *buf,
594 					  unsigned int data_size,
595 					  unsigned int count,
596 					  bool backwards)
597 {
598 	int i, b = backwards ? -1 : 1;
599 	enum es_result ret = ES_OK;
600 
601 	for (i = 0; i < count; i++) {
602 		void *s = src + (i * data_size * b);
603 		char *d = buf + (i * data_size);
604 
605 		ret = vc_read_mem(ctxt, s, d, data_size);
606 		if (ret != ES_OK)
607 			break;
608 	}
609 
610 	return ret;
611 }
612 
vc_insn_string_write(struct es_em_ctxt * ctxt,void * dst,char * buf,unsigned int data_size,unsigned int count,bool backwards)613 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
614 					   void *dst, char *buf,
615 					   unsigned int data_size,
616 					   unsigned int count,
617 					   bool backwards)
618 {
619 	int i, s = backwards ? -1 : 1;
620 	enum es_result ret = ES_OK;
621 
622 	for (i = 0; i < count; i++) {
623 		void *d = dst + (i * data_size * s);
624 		char *b = buf + (i * data_size);
625 
626 		ret = vc_write_mem(ctxt, d, b, data_size);
627 		if (ret != ES_OK)
628 			break;
629 	}
630 
631 	return ret;
632 }
633 
634 #define IOIO_TYPE_STR  BIT(2)
635 #define IOIO_TYPE_IN   1
636 #define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
637 #define IOIO_TYPE_OUT  0
638 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
639 
640 #define IOIO_REP       BIT(3)
641 
642 #define IOIO_ADDR_64   BIT(9)
643 #define IOIO_ADDR_32   BIT(8)
644 #define IOIO_ADDR_16   BIT(7)
645 
646 #define IOIO_DATA_32   BIT(6)
647 #define IOIO_DATA_16   BIT(5)
648 #define IOIO_DATA_8    BIT(4)
649 
650 #define IOIO_SEG_ES    (0 << 10)
651 #define IOIO_SEG_DS    (3 << 10)
652 
vc_ioio_exitinfo(struct es_em_ctxt * ctxt,u64 * exitinfo)653 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
654 {
655 	struct insn *insn = &ctxt->insn;
656 	*exitinfo = 0;
657 
658 	switch (insn->opcode.bytes[0]) {
659 	/* INS opcodes */
660 	case 0x6c:
661 	case 0x6d:
662 		*exitinfo |= IOIO_TYPE_INS;
663 		*exitinfo |= IOIO_SEG_ES;
664 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
665 		break;
666 
667 	/* OUTS opcodes */
668 	case 0x6e:
669 	case 0x6f:
670 		*exitinfo |= IOIO_TYPE_OUTS;
671 		*exitinfo |= IOIO_SEG_DS;
672 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
673 		break;
674 
675 	/* IN immediate opcodes */
676 	case 0xe4:
677 	case 0xe5:
678 		*exitinfo |= IOIO_TYPE_IN;
679 		*exitinfo |= (u8)insn->immediate.value << 16;
680 		break;
681 
682 	/* OUT immediate opcodes */
683 	case 0xe6:
684 	case 0xe7:
685 		*exitinfo |= IOIO_TYPE_OUT;
686 		*exitinfo |= (u8)insn->immediate.value << 16;
687 		break;
688 
689 	/* IN register opcodes */
690 	case 0xec:
691 	case 0xed:
692 		*exitinfo |= IOIO_TYPE_IN;
693 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
694 		break;
695 
696 	/* OUT register opcodes */
697 	case 0xee:
698 	case 0xef:
699 		*exitinfo |= IOIO_TYPE_OUT;
700 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
701 		break;
702 
703 	default:
704 		return ES_DECODE_FAILED;
705 	}
706 
707 	switch (insn->opcode.bytes[0]) {
708 	case 0x6c:
709 	case 0x6e:
710 	case 0xe4:
711 	case 0xe6:
712 	case 0xec:
713 	case 0xee:
714 		/* Single byte opcodes */
715 		*exitinfo |= IOIO_DATA_8;
716 		break;
717 	default:
718 		/* Length determined by instruction parsing */
719 		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
720 						     : IOIO_DATA_32;
721 	}
722 	switch (insn->addr_bytes) {
723 	case 2:
724 		*exitinfo |= IOIO_ADDR_16;
725 		break;
726 	case 4:
727 		*exitinfo |= IOIO_ADDR_32;
728 		break;
729 	case 8:
730 		*exitinfo |= IOIO_ADDR_64;
731 		break;
732 	}
733 
734 	if (insn_has_rep_prefix(insn))
735 		*exitinfo |= IOIO_REP;
736 
737 	return ES_OK;
738 }
739 
vc_handle_ioio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)740 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
741 {
742 	struct pt_regs *regs = ctxt->regs;
743 	u64 exit_info_1, exit_info_2;
744 	enum es_result ret;
745 
746 	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
747 	if (ret != ES_OK)
748 		return ret;
749 
750 	if (exit_info_1 & IOIO_TYPE_STR) {
751 
752 		/* (REP) INS/OUTS */
753 
754 		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
755 		unsigned int io_bytes, exit_bytes;
756 		unsigned int ghcb_count, op_count;
757 		unsigned long es_base;
758 		u64 sw_scratch;
759 
760 		/*
761 		 * For the string variants with rep prefix the amount of in/out
762 		 * operations per #VC exception is limited so that the kernel
763 		 * has a chance to take interrupts and re-schedule while the
764 		 * instruction is emulated.
765 		 */
766 		io_bytes   = (exit_info_1 >> 4) & 0x7;
767 		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
768 
769 		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
770 		exit_info_2 = min(op_count, ghcb_count);
771 		exit_bytes  = exit_info_2 * io_bytes;
772 
773 		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
774 
775 		/* Read bytes of OUTS into the shared buffer */
776 		if (!(exit_info_1 & IOIO_TYPE_IN)) {
777 			ret = vc_insn_string_read(ctxt,
778 					       (void *)(es_base + regs->si),
779 					       ghcb->shared_buffer, io_bytes,
780 					       exit_info_2, df);
781 			if (ret)
782 				return ret;
783 		}
784 
785 		/*
786 		 * Issue an VMGEXIT to the HV to consume the bytes from the
787 		 * shared buffer or to have it write them into the shared buffer
788 		 * depending on the instruction: OUTS or INS.
789 		 */
790 		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
791 		ghcb_set_sw_scratch(ghcb, sw_scratch);
792 		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
793 					  exit_info_1, exit_info_2);
794 		if (ret != ES_OK)
795 			return ret;
796 
797 		/* Read bytes from shared buffer into the guest's destination. */
798 		if (exit_info_1 & IOIO_TYPE_IN) {
799 			ret = vc_insn_string_write(ctxt,
800 						   (void *)(es_base + regs->di),
801 						   ghcb->shared_buffer, io_bytes,
802 						   exit_info_2, df);
803 			if (ret)
804 				return ret;
805 
806 			if (df)
807 				regs->di -= exit_bytes;
808 			else
809 				regs->di += exit_bytes;
810 		} else {
811 			if (df)
812 				regs->si -= exit_bytes;
813 			else
814 				regs->si += exit_bytes;
815 		}
816 
817 		if (exit_info_1 & IOIO_REP)
818 			regs->cx -= exit_info_2;
819 
820 		ret = regs->cx ? ES_RETRY : ES_OK;
821 
822 	} else {
823 
824 		/* IN/OUT into/from rAX */
825 
826 		int bits = (exit_info_1 & 0x70) >> 1;
827 		u64 rax = 0;
828 
829 		if (!(exit_info_1 & IOIO_TYPE_IN))
830 			rax = lower_bits(regs->ax, bits);
831 
832 		ghcb_set_rax(ghcb, rax);
833 
834 		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
835 		if (ret != ES_OK)
836 			return ret;
837 
838 		if (exit_info_1 & IOIO_TYPE_IN) {
839 			if (!ghcb_rax_is_valid(ghcb))
840 				return ES_VMM_ERROR;
841 			regs->ax = lower_bits(ghcb->save.rax, bits);
842 		}
843 	}
844 
845 	return ret;
846 }
847 
vc_handle_cpuid_snp(struct pt_regs * regs)848 static int vc_handle_cpuid_snp(struct pt_regs *regs)
849 {
850 	struct cpuid_leaf leaf;
851 	int ret;
852 
853 	leaf.fn = regs->ax;
854 	leaf.subfn = regs->cx;
855 	ret = snp_cpuid(&leaf);
856 	if (!ret) {
857 		regs->ax = leaf.eax;
858 		regs->bx = leaf.ebx;
859 		regs->cx = leaf.ecx;
860 		regs->dx = leaf.edx;
861 	}
862 
863 	return ret;
864 }
865 
vc_handle_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt)866 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
867 				      struct es_em_ctxt *ctxt)
868 {
869 	struct pt_regs *regs = ctxt->regs;
870 	u32 cr4 = native_read_cr4();
871 	enum es_result ret;
872 	int snp_cpuid_ret;
873 
874 	snp_cpuid_ret = vc_handle_cpuid_snp(regs);
875 	if (!snp_cpuid_ret)
876 		return ES_OK;
877 	if (snp_cpuid_ret != -EOPNOTSUPP)
878 		return ES_VMM_ERROR;
879 
880 	ghcb_set_rax(ghcb, regs->ax);
881 	ghcb_set_rcx(ghcb, regs->cx);
882 
883 	if (cr4 & X86_CR4_OSXSAVE)
884 		/* Safe to read xcr0 */
885 		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
886 	else
887 		/* xgetbv will cause #GP - use reset value for xcr0 */
888 		ghcb_set_xcr0(ghcb, 1);
889 
890 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
891 	if (ret != ES_OK)
892 		return ret;
893 
894 	if (!(ghcb_rax_is_valid(ghcb) &&
895 	      ghcb_rbx_is_valid(ghcb) &&
896 	      ghcb_rcx_is_valid(ghcb) &&
897 	      ghcb_rdx_is_valid(ghcb)))
898 		return ES_VMM_ERROR;
899 
900 	regs->ax = ghcb->save.rax;
901 	regs->bx = ghcb->save.rbx;
902 	regs->cx = ghcb->save.rcx;
903 	regs->dx = ghcb->save.rdx;
904 
905 	return ES_OK;
906 }
907 
vc_handle_rdtsc(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long exit_code)908 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
909 				      struct es_em_ctxt *ctxt,
910 				      unsigned long exit_code)
911 {
912 	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
913 	enum es_result ret;
914 
915 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
916 	if (ret != ES_OK)
917 		return ret;
918 
919 	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
920 	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
921 		return ES_VMM_ERROR;
922 
923 	ctxt->regs->ax = ghcb->save.rax;
924 	ctxt->regs->dx = ghcb->save.rdx;
925 	if (rdtscp)
926 		ctxt->regs->cx = ghcb->save.rcx;
927 
928 	return ES_OK;
929 }
930 
931 struct cc_setup_data {
932 	struct setup_data header;
933 	u32 cc_blob_address;
934 };
935 
936 /*
937  * Search for a Confidential Computing blob passed in as a setup_data entry
938  * via the Linux Boot Protocol.
939  */
find_cc_blob_setup_data(struct boot_params * bp)940 static struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
941 {
942 	struct cc_setup_data *sd = NULL;
943 	struct setup_data *hdr;
944 
945 	hdr = (struct setup_data *)bp->hdr.setup_data;
946 
947 	while (hdr) {
948 		if (hdr->type == SETUP_CC_BLOB) {
949 			sd = (struct cc_setup_data *)hdr;
950 			return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
951 		}
952 		hdr = (struct setup_data *)hdr->next;
953 	}
954 
955 	return NULL;
956 }
957 
958 /*
959  * Initialize the kernel's copy of the SNP CPUID table, and set up the
960  * pointer that will be used to access it.
961  *
962  * Maintaining a direct mapping of the SNP CPUID table used by firmware would
963  * be possible as an alternative, but the approach is brittle since the
964  * mapping needs to be updated in sync with all the changes to virtual memory
965  * layout and related mapping facilities throughout the boot process.
966  */
setup_cpuid_table(const struct cc_blob_sev_info * cc_info)967 static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
968 {
969 	const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
970 	int i;
971 
972 	if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
973 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
974 
975 	cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
976 	if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
977 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
978 
979 	cpuid_table = snp_cpuid_get_table();
980 	memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
981 
982 	/* Initialize CPUID ranges for range-checking. */
983 	for (i = 0; i < cpuid_table->count; i++) {
984 		const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
985 
986 		if (fn->eax_in == 0x0)
987 			cpuid_std_range_max = fn->eax;
988 		else if (fn->eax_in == 0x40000000)
989 			cpuid_hyp_range_max = fn->eax;
990 		else if (fn->eax_in == 0x80000000)
991 			cpuid_ext_range_max = fn->eax;
992 	}
993 }
994