1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * AMD Memory Encryption Support
4 *
5 * Copyright (C) 2017 Advanced Micro Devices, Inc.
6 *
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 */
9
10#include <linux/linkage.h>
11
12#include <asm/processor-flags.h>
13#include <asm/msr.h>
14#include <asm/asm-offsets.h>
15#include <asm/segment.h>
16#include <asm/trapnr.h>
17
18	.text
19	.code32
20SYM_FUNC_START(get_sev_encryption_bit)
21	push	%ebx
22
23	movl	$0x80000000, %eax	/* CPUID to check the highest leaf */
24	cpuid
25	cmpl	$0x8000001f, %eax	/* See if 0x8000001f is available */
26	jb	.Lno_sev
27
28	/*
29	 * Check for the SEV feature:
30	 *   CPUID Fn8000_001F[EAX] - Bit 1
31	 *   CPUID Fn8000_001F[EBX] - Bits 5:0
32	 *     Pagetable bit position used to indicate encryption
33	 */
34	movl	$0x8000001f, %eax
35	cpuid
36	bt	$1, %eax		/* Check if SEV is available */
37	jnc	.Lno_sev
38
39	movl	$MSR_AMD64_SEV, %ecx	/* Read the SEV MSR */
40	rdmsr
41	bt	$MSR_AMD64_SEV_ENABLED_BIT, %eax	/* Check if SEV is active */
42	jnc	.Lno_sev
43
44	movl	%ebx, %eax
45	andl	$0x3f, %eax		/* Return the encryption bit location */
46	jmp	.Lsev_exit
47
48.Lno_sev:
49	xor	%eax, %eax
50
51.Lsev_exit:
52	pop	%ebx
53	RET
54SYM_FUNC_END(get_sev_encryption_bit)
55
56/**
57 * sev_es_req_cpuid - Request a CPUID value from the Hypervisor using
58 *		      the GHCB MSR protocol
59 *
60 * @%eax:	Register to request (0=EAX, 1=EBX, 2=ECX, 3=EDX)
61 * @%edx:	CPUID Function
62 *
63 * Returns 0 in %eax on success, non-zero on failure
64 * %edx returns CPUID value on success
65 */
66SYM_CODE_START_LOCAL(sev_es_req_cpuid)
67	shll	$30, %eax
68	orl     $0x00000004, %eax
69	movl    $MSR_AMD64_SEV_ES_GHCB, %ecx
70	wrmsr
71	rep; vmmcall		# VMGEXIT
72	rdmsr
73
74	/* Check response */
75	movl	%eax, %ecx
76	andl	$0x3ffff000, %ecx	# Bits [12-29] MBZ
77	jnz	2f
78
79	/* Check return code */
80	andl    $0xfff, %eax
81	cmpl    $5, %eax
82	jne	2f
83
84	/* All good - return success */
85	xorl	%eax, %eax
861:
87	RET
882:
89	movl	$-1, %eax
90	jmp	1b
91SYM_CODE_END(sev_es_req_cpuid)
92
93SYM_CODE_START_LOCAL(startup32_vc_handler)
94	pushl	%eax
95	pushl	%ebx
96	pushl	%ecx
97	pushl	%edx
98
99	/* Keep CPUID function in %ebx */
100	movl	%eax, %ebx
101
102	/* Check if error-code == SVM_EXIT_CPUID */
103	cmpl	$0x72, 16(%esp)
104	jne	.Lfail
105
106	movl	$0, %eax		# Request CPUID[fn].EAX
107	movl	%ebx, %edx		# CPUID fn
108	call	sev_es_req_cpuid	# Call helper
109	testl	%eax, %eax		# Check return code
110	jnz	.Lfail
111	movl	%edx, 12(%esp)		# Store result
112
113	movl	$1, %eax		# Request CPUID[fn].EBX
114	movl	%ebx, %edx		# CPUID fn
115	call	sev_es_req_cpuid	# Call helper
116	testl	%eax, %eax		# Check return code
117	jnz	.Lfail
118	movl	%edx, 8(%esp)		# Store result
119
120	movl	$2, %eax		# Request CPUID[fn].ECX
121	movl	%ebx, %edx		# CPUID fn
122	call	sev_es_req_cpuid	# Call helper
123	testl	%eax, %eax		# Check return code
124	jnz	.Lfail
125	movl	%edx, 4(%esp)		# Store result
126
127	movl	$3, %eax		# Request CPUID[fn].EDX
128	movl	%ebx, %edx		# CPUID fn
129	call	sev_es_req_cpuid	# Call helper
130	testl	%eax, %eax		# Check return code
131	jnz	.Lfail
132	movl	%edx, 0(%esp)		# Store result
133
134	/*
135	 * Sanity check CPUID results from the Hypervisor. See comment in
136	 * do_vc_no_ghcb() for more details on why this is necessary.
137	 */
138
139	/* Fail if SEV leaf not available in CPUID[0x80000000].EAX */
140	cmpl    $0x80000000, %ebx
141	jne     .Lcheck_sev
142	cmpl    $0x8000001f, 12(%esp)
143	jb      .Lfail
144	jmp     .Ldone
145
146.Lcheck_sev:
147	/* Fail if SEV bit not set in CPUID[0x8000001f].EAX[1] */
148	cmpl    $0x8000001f, %ebx
149	jne     .Ldone
150	btl     $1, 12(%esp)
151	jnc     .Lfail
152
153.Ldone:
154	popl	%edx
155	popl	%ecx
156	popl	%ebx
157	popl	%eax
158
159	/* Remove error code */
160	addl	$4, %esp
161
162	/* Jump over CPUID instruction */
163	addl	$2, (%esp)
164
165	iret
166.Lfail:
167	/* Send terminate request to Hypervisor */
168	movl    $0x100, %eax
169	xorl    %edx, %edx
170	movl    $MSR_AMD64_SEV_ES_GHCB, %ecx
171	wrmsr
172	rep; vmmcall
173
174	/* If request fails, go to hlt loop */
175	hlt
176	jmp .Lfail
177SYM_CODE_END(startup32_vc_handler)
178
179/*
180 * Write an IDT entry into boot32_idt
181 *
182 * Parameters:
183 *
184 * %eax:	Handler address
185 * %edx:	Vector number
186 * %ecx:	IDT address
187 */
188SYM_FUNC_START_LOCAL(startup32_set_idt_entry)
189	/* IDT entry address to %ecx */
190	leal	(%ecx, %edx, 8), %ecx
191
192	/* Build IDT entry, lower 4 bytes */
193	movl    %eax, %edx
194	andl    $0x0000ffff, %edx		# Target code segment offset [15:0]
195	orl	$(__KERNEL32_CS << 16), %edx	# Target code segment selector
196
197	/* Store lower 4 bytes to IDT */
198	movl    %edx, (%ecx)
199
200	/* Build IDT entry, upper 4 bytes */
201	movl    %eax, %edx
202	andl    $0xffff0000, %edx	# Target code segment offset [31:16]
203	orl     $0x00008e00, %edx	# Present, Type 32-bit Interrupt Gate
204
205	/* Store upper 4 bytes to IDT */
206	movl    %edx, 4(%ecx)
207
208	RET
209SYM_FUNC_END(startup32_set_idt_entry)
210
211SYM_FUNC_START(startup32_load_idt)
212	push	%ebp
213	push	%ebx
214
215	call	1f
2161:	pop	%ebp
217
218	leal    (boot32_idt - 1b)(%ebp), %ebx
219
220	/* #VC handler */
221	leal    (startup32_vc_handler - 1b)(%ebp), %eax
222	movl    $X86_TRAP_VC, %edx
223	movl	%ebx, %ecx
224	call    startup32_set_idt_entry
225
226	/* Load IDT */
227	leal	(boot32_idt_desc - 1b)(%ebp), %ecx
228	movl	%ebx, 2(%ecx)
229	lidt    (%ecx)
230
231	pop	%ebx
232	pop	%ebp
233	RET
234SYM_FUNC_END(startup32_load_idt)
235
236/*
237 * Check for the correct C-bit position when the startup_32 boot-path is used.
238 *
239 * The check makes use of the fact that all memory is encrypted when paging is
240 * disabled. The function creates 64 bits of random data using the RDRAND
241 * instruction. RDRAND is mandatory for SEV guests, so always available. If the
242 * hypervisor violates that the kernel will crash right here.
243 *
244 * The 64 bits of random data are stored to a memory location and at the same
245 * time kept in the %eax and %ebx registers. Since encryption is always active
246 * when paging is off the random data will be stored encrypted in main memory.
247 *
248 * Then paging is enabled. When the C-bit position is correct all memory is
249 * still mapped encrypted and comparing the register values with memory will
250 * succeed. An incorrect C-bit position will map all memory unencrypted, so that
251 * the compare will use the encrypted random data and fail.
252 */
253SYM_FUNC_START(startup32_check_sev_cbit)
254	pushl	%ebx
255	pushl	%ebp
256
257	call	0f
2580:	popl	%ebp
259
260	/* Check for non-zero sev_status */
261	movl	(sev_status - 0b)(%ebp), %eax
262	testl	%eax, %eax
263	jz	4f
264
265	/*
266	 * Get two 32-bit random values - Don't bail out if RDRAND fails
267	 * because it is better to prevent forward progress if no random value
268	 * can be gathered.
269	 */
2701:	rdrand	%eax
271	jnc	1b
2722:	rdrand	%ebx
273	jnc	2b
274
275	/* Store to memory and keep it in the registers */
276	leal	(sev_check_data - 0b)(%ebp), %ebp
277	movl	%eax, 0(%ebp)
278	movl	%ebx, 4(%ebp)
279
280	/* Enable paging to see if encryption is active */
281	movl	%cr0, %edx			 /* Backup %cr0 in %edx */
282	movl	$(X86_CR0_PG | X86_CR0_PE), %ecx /* Enable Paging and Protected mode */
283	movl	%ecx, %cr0
284
285	cmpl	%eax, 0(%ebp)
286	jne	3f
287	cmpl	%ebx, 4(%ebp)
288	jne	3f
289
290	movl	%edx, %cr0	/* Restore previous %cr0 */
291
292	jmp	4f
293
2943:	/* Check failed - hlt the machine */
295	hlt
296	jmp	3b
297
2984:
299	popl	%ebp
300	popl	%ebx
301	RET
302SYM_FUNC_END(startup32_check_sev_cbit)
303
304	.code64
305
306#include "../../kernel/sev_verify_cbit.S"
307
308	.data
309
310	.balign	8
311SYM_DATA(sme_me_mask,		.quad 0)
312SYM_DATA(sev_status,		.quad 0)
313SYM_DATA(sev_check_data,	.quad 0)
314
315SYM_DATA_START_LOCAL(boot32_idt)
316	.rept	32
317	.quad	0
318	.endr
319SYM_DATA_END(boot32_idt)
320
321SYM_DATA_START_LOCAL(boot32_idt_desc)
322	.word	. - boot32_idt - 1
323	.long	0
324SYM_DATA_END(boot32_idt_desc)
325