1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * amx tests
4 *
5 * Copyright (C) 2021, Intel, Inc.
6 *
7 * Tests for amx #NM exception and save/restore.
8 */
9
10 #define _GNU_SOURCE /* for program_invocation_short_name */
11 #include <fcntl.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <sys/ioctl.h>
16 #include <sys/syscall.h>
17
18 #include "test_util.h"
19
20 #include "kvm_util.h"
21 #include "processor.h"
22 #include "vmx.h"
23
24 #ifndef __x86_64__
25 # error This test is 64-bit only
26 #endif
27
28 #define NUM_TILES 8
29 #define TILE_SIZE 1024
30 #define XSAVE_SIZE ((NUM_TILES * TILE_SIZE) + PAGE_SIZE)
31
32 /* Tile configuration associated: */
33 #define MAX_TILES 16
34 #define RESERVED_BYTES 14
35
36 #define XFEATURE_XTILECFG 17
37 #define XFEATURE_XTILEDATA 18
38 #define XFEATURE_MASK_XTILECFG (1 << XFEATURE_XTILECFG)
39 #define XFEATURE_MASK_XTILEDATA (1 << XFEATURE_XTILEDATA)
40 #define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILECFG | XFEATURE_MASK_XTILEDATA)
41
42 #define XSAVE_HDR_OFFSET 512
43
44 struct xsave_data {
45 u8 area[XSAVE_SIZE];
46 } __aligned(64);
47
48 struct tile_config {
49 u8 palette_id;
50 u8 start_row;
51 u8 reserved[RESERVED_BYTES];
52 u16 colsb[MAX_TILES];
53 u8 rows[MAX_TILES];
54 };
55
56 struct tile_data {
57 u8 data[NUM_TILES * TILE_SIZE];
58 };
59
60 struct xtile_info {
61 u16 bytes_per_tile;
62 u16 bytes_per_row;
63 u16 max_names;
64 u16 max_rows;
65 u32 xsave_offset;
66 u32 xsave_size;
67 };
68
69 static struct xtile_info xtile;
70
__xgetbv(u32 index)71 static inline u64 __xgetbv(u32 index)
72 {
73 u32 eax, edx;
74
75 asm volatile("xgetbv;"
76 : "=a" (eax), "=d" (edx)
77 : "c" (index));
78 return eax + ((u64)edx << 32);
79 }
80
__xsetbv(u32 index,u64 value)81 static inline void __xsetbv(u32 index, u64 value)
82 {
83 u32 eax = value;
84 u32 edx = value >> 32;
85
86 asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index));
87 }
88
__ldtilecfg(void * cfg)89 static inline void __ldtilecfg(void *cfg)
90 {
91 asm volatile(".byte 0xc4,0xe2,0x78,0x49,0x00"
92 : : "a"(cfg));
93 }
94
__tileloadd(void * tile)95 static inline void __tileloadd(void *tile)
96 {
97 asm volatile(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10"
98 : : "a"(tile), "d"(0));
99 }
100
__tilerelease(void)101 static inline void __tilerelease(void)
102 {
103 asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::);
104 }
105
__xsavec(struct xsave_data * data,uint64_t rfbm)106 static inline void __xsavec(struct xsave_data *data, uint64_t rfbm)
107 {
108 uint32_t rfbm_lo = rfbm;
109 uint32_t rfbm_hi = rfbm >> 32;
110
111 asm volatile("xsavec (%%rdi)"
112 : : "D" (data), "a" (rfbm_lo), "d" (rfbm_hi)
113 : "memory");
114 }
115
check_cpuid_xsave(void)116 static inline void check_cpuid_xsave(void)
117 {
118 GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE));
119 GUEST_ASSERT(this_cpu_has(X86_FEATURE_OSXSAVE));
120 }
121
check_xsave_supports_xtile(void)122 static bool check_xsave_supports_xtile(void)
123 {
124 return __xgetbv(0) & XFEATURE_MASK_XTILE;
125 }
126
check_xtile_info(void)127 static void check_xtile_info(void)
128 {
129 GUEST_ASSERT(this_cpu_has_p(X86_PROPERTY_XSTATE_MAX_SIZE_XCR0));
130 GUEST_ASSERT(this_cpu_property(X86_PROPERTY_XSTATE_MAX_SIZE_XCR0) <= XSAVE_SIZE);
131
132 xtile.xsave_offset = this_cpu_property(X86_PROPERTY_XSTATE_TILE_OFFSET);
133 GUEST_ASSERT(xtile.xsave_offset == 2816);
134 xtile.xsave_size = this_cpu_property(X86_PROPERTY_XSTATE_TILE_SIZE);
135 GUEST_ASSERT(xtile.xsave_size == 8192);
136 GUEST_ASSERT(sizeof(struct tile_data) >= xtile.xsave_size);
137
138 GUEST_ASSERT(this_cpu_has_p(X86_PROPERTY_AMX_NR_TILE_REGS));
139 xtile.max_names = this_cpu_property(X86_PROPERTY_AMX_NR_TILE_REGS);
140 GUEST_ASSERT(xtile.max_names == 8);
141 xtile.bytes_per_tile = this_cpu_property(X86_PROPERTY_AMX_BYTES_PER_TILE);
142 GUEST_ASSERT(xtile.bytes_per_tile == 1024);
143 xtile.bytes_per_row = this_cpu_property(X86_PROPERTY_AMX_BYTES_PER_ROW);
144 GUEST_ASSERT(xtile.bytes_per_row == 64);
145 xtile.max_rows = this_cpu_property(X86_PROPERTY_AMX_MAX_ROWS);
146 GUEST_ASSERT(xtile.max_rows == 16);
147 }
148
set_tilecfg(struct tile_config * cfg)149 static void set_tilecfg(struct tile_config *cfg)
150 {
151 int i;
152
153 /* Only palette id 1 */
154 cfg->palette_id = 1;
155 for (i = 0; i < xtile.max_names; i++) {
156 cfg->colsb[i] = xtile.bytes_per_row;
157 cfg->rows[i] = xtile.max_rows;
158 }
159 }
160
set_xstatebv(void * data,uint64_t bv)161 static void set_xstatebv(void *data, uint64_t bv)
162 {
163 *(uint64_t *)(data + XSAVE_HDR_OFFSET) = bv;
164 }
165
get_xstatebv(void * data)166 static u64 get_xstatebv(void *data)
167 {
168 return *(u64 *)(data + XSAVE_HDR_OFFSET);
169 }
170
init_regs(void)171 static void init_regs(void)
172 {
173 uint64_t cr4, xcr0;
174
175 /* turn on CR4.OSXSAVE */
176 cr4 = get_cr4();
177 cr4 |= X86_CR4_OSXSAVE;
178 set_cr4(cr4);
179
180 xcr0 = __xgetbv(0);
181 xcr0 |= XFEATURE_MASK_XTILE;
182 __xsetbv(0x0, xcr0);
183 }
184
guest_code(struct tile_config * amx_cfg,struct tile_data * tiledata,struct xsave_data * xsave_data)185 static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg,
186 struct tile_data *tiledata,
187 struct xsave_data *xsave_data)
188 {
189 init_regs();
190 check_cpuid_xsave();
191 check_xsave_supports_xtile();
192 check_xtile_info();
193 GUEST_SYNC(1);
194
195 /* xfd=0, enable amx */
196 wrmsr(MSR_IA32_XFD, 0);
197 GUEST_SYNC(2);
198 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0);
199 set_tilecfg(amx_cfg);
200 __ldtilecfg(amx_cfg);
201 GUEST_SYNC(3);
202 /* Check save/restore when trap to userspace */
203 __tileloadd(tiledata);
204 GUEST_SYNC(4);
205 __tilerelease();
206 GUEST_SYNC(5);
207 /* bit 18 not in the XCOMP_BV after xsavec() */
208 set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA);
209 __xsavec(xsave_data, XFEATURE_MASK_XTILEDATA);
210 GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0);
211
212 /* xfd=0x40000, disable amx tiledata */
213 wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA);
214 GUEST_SYNC(6);
215 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA);
216 set_tilecfg(amx_cfg);
217 __ldtilecfg(amx_cfg);
218 /* Trigger #NM exception */
219 __tileloadd(tiledata);
220 GUEST_SYNC(10);
221
222 GUEST_DONE();
223 }
224
guest_nm_handler(struct ex_regs * regs)225 void guest_nm_handler(struct ex_regs *regs)
226 {
227 /* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */
228 GUEST_SYNC(7);
229 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
230 GUEST_SYNC(8);
231 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
232 /* Clear xfd_err */
233 wrmsr(MSR_IA32_XFD_ERR, 0);
234 /* xfd=0, enable amx */
235 wrmsr(MSR_IA32_XFD, 0);
236 GUEST_SYNC(9);
237 }
238
main(int argc,char * argv[])239 int main(int argc, char *argv[])
240 {
241 struct kvm_regs regs1, regs2;
242 struct kvm_vcpu *vcpu;
243 struct kvm_vm *vm;
244 struct kvm_run *run;
245 struct kvm_x86_state *state;
246 int xsave_restore_size;
247 vm_vaddr_t amx_cfg, tiledata, xsavedata;
248 struct ucall uc;
249 u32 amx_offset;
250 int stage, ret;
251
252 /*
253 * Note, all off-by-default features must be enabled before anything
254 * caches KVM_GET_SUPPORTED_CPUID, e.g. before using kvm_cpu_has().
255 */
256 vm_xsave_require_permission(XSTATE_XTILE_DATA_BIT);
257
258 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XFD));
259 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XSAVE));
260 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_AMX_TILE));
261 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILECFG));
262 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILEDATA));
263
264 /* Create VM */
265 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
266
267 TEST_ASSERT(kvm_cpu_has_p(X86_PROPERTY_XSTATE_MAX_SIZE),
268 "KVM should enumerate max XSAVE size when XSAVE is supported");
269 xsave_restore_size = kvm_cpu_property(X86_PROPERTY_XSTATE_MAX_SIZE);
270
271 run = vcpu->run;
272 vcpu_regs_get(vcpu, ®s1);
273
274 /* Register #NM handler */
275 vm_init_descriptor_tables(vm);
276 vcpu_init_descriptor_tables(vcpu);
277 vm_install_exception_handler(vm, NM_VECTOR, guest_nm_handler);
278
279 /* amx cfg for guest_code */
280 amx_cfg = vm_vaddr_alloc_page(vm);
281 memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize());
282
283 /* amx tiledata for guest_code */
284 tiledata = vm_vaddr_alloc_pages(vm, 2);
285 memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize());
286
287 /* xsave data for guest_code */
288 xsavedata = vm_vaddr_alloc_pages(vm, 3);
289 memset(addr_gva2hva(vm, xsavedata), 0, 3 * getpagesize());
290 vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xsavedata);
291
292 for (stage = 1; ; stage++) {
293 vcpu_run(vcpu);
294 TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
295 "Stage %d: unexpected exit reason: %u (%s),\n",
296 stage, run->exit_reason,
297 exit_reason_str(run->exit_reason));
298
299 switch (get_ucall(vcpu, &uc)) {
300 case UCALL_ABORT:
301 REPORT_GUEST_ASSERT(uc);
302 /* NOT REACHED */
303 case UCALL_SYNC:
304 switch (uc.args[1]) {
305 case 1:
306 case 2:
307 case 3:
308 case 5:
309 case 6:
310 case 7:
311 case 8:
312 fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]);
313 break;
314 case 4:
315 case 10:
316 fprintf(stderr,
317 "GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]);
318
319 /* Compacted mode, get amx offset by xsave area
320 * size subtract 8K amx size.
321 */
322 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
323 state = vcpu_save_state(vcpu);
324 void *amx_start = (void *)state->xsave + amx_offset;
325 void *tiles_data = (void *)addr_gva2hva(vm, tiledata);
326 /* Only check TMM0 register, 1 tile */
327 ret = memcmp(amx_start, tiles_data, TILE_SIZE);
328 TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret);
329 kvm_x86_state_cleanup(state);
330 break;
331 case 9:
332 fprintf(stderr,
333 "GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]);
334 break;
335 }
336 break;
337 case UCALL_DONE:
338 fprintf(stderr, "UCALL_DONE\n");
339 goto done;
340 default:
341 TEST_FAIL("Unknown ucall %lu", uc.cmd);
342 }
343
344 state = vcpu_save_state(vcpu);
345 memset(®s1, 0, sizeof(regs1));
346 vcpu_regs_get(vcpu, ®s1);
347
348 kvm_vm_release(vm);
349
350 /* Restore state in a new VM. */
351 vcpu = vm_recreate_with_one_vcpu(vm);
352 vcpu_load_state(vcpu, state);
353 run = vcpu->run;
354 kvm_x86_state_cleanup(state);
355
356 memset(®s2, 0, sizeof(regs2));
357 vcpu_regs_get(vcpu, ®s2);
358 TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)),
359 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
360 (ulong) regs2.rdi, (ulong) regs2.rsi);
361 }
362 done:
363 kvm_vm_free(vm);
364 }
365