1 /*
2 * Copyright 2024 The Hafnium Authors.
3 *
4 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/BSD-3-Clause.
7 */
8
9 #include "hf/arch/fpu.h"
10 #include "hf/arch/sme.h"
11 #include "hf/arch/sve.h"
12
13 #include "hf/types.h"
14 #include "hf/vcpu.h"
15 #include "hf/vm.h"
16
17 #include "smc.h"
18 #include "sysregs.h"
19
20 /**
21 * Other world SIMD SVE/SME context.
22 * By design the VM struct stores FPU/Adv. SIMD contexts but no SIMD context
23 * for newer extensions. The structure below supplements the VM struct with
24 * SVE/SME contexts just for the other world VM usage.
25 * Restricting to the other world VM limits the required memory footprint when
26 * compared to a design where the VM struct holds the full FPU/Adv. SIMD/SVE/SME
27 * contexts. There is no immediate requirement to extend the VM struct provided
28 * SVE/SME are not enabled for secure partitions (or secondary VMs).
29 */
30 static struct {
31 /** SMCCCv1.3 FID[16] hint bit state recorded on SPMC entry from NWd. */
32 bool hint;
33
34 /** SVE context. */
35 uint64_t zcr_el2;
36 struct sve_context sve_context;
37
38 /** SME context. */
39 uint64_t svcr;
40 uint64_t smcr_el2;
41 } ns_simd_ctx[MAX_CPUS];
42
43 /**
44 * Restore FPU/Adv. SIMD/SVE/SME 'Other world' context when exiting the SPMC.
45 * Called from exceptions.S: other_world_loop.
46 */
plat_restore_ns_simd_context(struct vcpu * vcpu)47 void plat_restore_ns_simd_context(struct vcpu *vcpu)
48 {
49 bool sve = is_arch_feat_sve_supported();
50 bool sme = is_arch_feat_sme_supported();
51 bool fa64 = is_arch_feat_sme_fa64_supported();
52 bool streaming_mode = false;
53 bool hint;
54 uint32_t cpu_id;
55
56 assert(vcpu->vm->id == HF_HYPERVISOR_VM_ID);
57 cpu_id = cpu_index(vcpu->cpu);
58 hint = ns_simd_ctx[cpu_id].hint;
59
60 if (sme) {
61 /* Disable SME EL2 and lower traps. */
62 arch_sme_disable_traps();
63
64 /* Assert ZA array did not change state. */
65 assert((arch_sme_svcr_get() & MSR_SVCR_ZA) ==
66 (ns_simd_ctx[cpu_id].svcr & MSR_SVCR_ZA));
67
68 /*
69 * Restore SVCR, in particular (re)enable SSVE if it was enabled
70 * at entry.
71 * NOTE: a PSTATE.SM transition resets Z0-Z31, P0-P15,
72 * FFR and FPSR registers to an architecturally defined
73 * constant.
74 */
75 arch_sme_svcr_set(ns_simd_ctx[cpu_id].svcr);
76
77 streaming_mode =
78 (ns_simd_ctx[cpu_id].svcr & MSR_SVCR_SM) == MSR_SVCR_SM;
79
80 /*
81 * Streaming SVE vector length is determined by SMCR_EL2.LEN
82 * that was set earlier during the save operation.
83 */
84 }
85
86 if (sve) {
87 /* Disable SVE EL2 and lower traps. */
88 arch_sve_disable_traps();
89
90 /*
91 * SVE vector length is determined by ZCR_EL2.LEN
92 * that was set earlier during the save operation.
93 */
94 }
95
96 /* Restore FPCR/FPSR common to FPU/Adv. SIMD./SVE/SME. */
97 arch_fpu_state_restore_from_vcpu(vcpu);
98
99 /*
100 * If SVE or SME is implemented and SVE hint is false as it was
101 * passed by the normal world caller when entering the SPMC, then
102 * restore the SVE (or Streaming SVE) state.
103 * Omit restoring the SVE state, if only SME is implemented (and
104 * SVE is not implemented) and Streaming SVE is disabled.
105 */
106 if ((sve || sme) && !hint && !(!sve && sme && !streaming_mode)) {
107 /*
108 * Restore FFR register before predicates,
109 * if SVE only is implemented, or both SVE and SME are
110 * implemented and Streaming SVE is disabled,
111 * or both SME and FEAT_SME_FA64 are implemented and
112 * Streaming SVE is enabled.
113 */
114 if ((sve && !sme) || (sve && sme && !streaming_mode) ||
115 (sme && fa64 && streaming_mode)) {
116 __asm__ volatile(
117 ".arch_extension sve;"
118 "ldr p0, [%0];"
119 "wrffr p0.b;"
120 ".arch_extension nosve"
121 :
122 : "r"(&ns_simd_ctx[cpu_id].sve_context.ffr));
123 }
124
125 /*
126 * Restore predicates if SVE only is implemented,
127 * or both SME and SVE are implemented and Streaming SVE
128 * is disabled,
129 * or SME is implemented and Streaming SVE is enabled.
130 */
131 if ((sve && !sme) || (sve && sme && !streaming_mode) ||
132 (sme && streaming_mode)) {
133 /* Restore predicate registers. */
134 __asm__ volatile(
135 ".arch_extension sve;"
136 "ldr p0, [%0, #0, MUL VL];"
137 "ldr p1, [%0, #1, MUL VL];"
138 "ldr p2, [%0, #2, MUL VL];"
139 "ldr p3, [%0, #3, MUL VL];"
140 "ldr p4, [%0, #4, MUL VL];"
141 "ldr p5, [%0, #5, MUL VL];"
142 "ldr p6, [%0, #6, MUL VL];"
143 "ldr p7, [%0, #7, MUL VL];"
144 "ldr p8, [%0, #8, MUL VL];"
145 "ldr p9, [%0, #9, MUL VL];"
146 "ldr p10, [%0, #10, MUL VL];"
147 "ldr p11, [%0, #11, MUL VL];"
148 "ldr p12, [%0, #12, MUL VL];"
149 "ldr p13, [%0, #13, MUL VL];"
150 "ldr p14, [%0, #14, MUL VL];"
151 "ldr p15, [%0, #15, MUL VL];"
152 ".arch_extension nosve"
153 :
154 : "r"(&ns_simd_ctx[cpu_id]
155 .sve_context.predicates));
156
157 /* Restore SVE/Streaming SVE vectors. */
158 __asm__ volatile(
159 ".arch_extension sve;"
160 "ldr z0, [%0, #0, MUL VL];"
161 "ldr z1, [%0, #1, MUL VL];"
162 "ldr z2, [%0, #2, MUL VL];"
163 "ldr z3, [%0, #3, MUL VL];"
164 "ldr z4, [%0, #4, MUL VL];"
165 "ldr z5, [%0, #5, MUL VL];"
166 "ldr z6, [%0, #6, MUL VL];"
167 "ldr z7, [%0, #7, MUL VL];"
168 "ldr z8, [%0, #8, MUL VL];"
169 "ldr z9, [%0, #9, MUL VL];"
170 "ldr z10, [%0, #10, MUL VL];"
171 "ldr z11, [%0, #11, MUL VL];"
172 "ldr z12, [%0, #12, MUL VL];"
173 "ldr z13, [%0, #13, MUL VL];"
174 "ldr z14, [%0, #14, MUL VL];"
175 "ldr z15, [%0, #15, MUL VL];"
176 "ldr z16, [%0, #16, MUL VL];"
177 "ldr z17, [%0, #17, MUL VL];"
178 "ldr z18, [%0, #18, MUL VL];"
179 "ldr z19, [%0, #19, MUL VL];"
180 "ldr z20, [%0, #20, MUL VL];"
181 "ldr z21, [%0, #21, MUL VL];"
182 "ldr z22, [%0, #22, MUL VL];"
183 "ldr z23, [%0, #23, MUL VL];"
184 "ldr z24, [%0, #24, MUL VL];"
185 "ldr z25, [%0, #25, MUL VL];"
186 "ldr z26, [%0, #26, MUL VL];"
187 "ldr z27, [%0, #27, MUL VL];"
188 "ldr z28, [%0, #28, MUL VL];"
189 "ldr z29, [%0, #29, MUL VL];"
190 "ldr z30, [%0, #30, MUL VL];"
191 "ldr z31, [%0, #31, MUL VL];"
192 ".arch_extension nosve"
193 :
194 : "r"(&ns_simd_ctx[cpu_id]
195 .sve_context.vectors));
196 }
197 } else {
198 /* Restore FPU/Adv. SIMD vectors. */
199 arch_fpu_regs_restore_from_vcpu(vcpu);
200
201 if ((sve || sme) && hint) {
202 /* TODO: clear predicates and ffr */
203 }
204 }
205
206 if (sve) {
207 /*
208 * Restore normal world ZCR_EL2.
209 * ZCR_EL1 is untouched as SVE is not enabled for SPs.
210 */
211 write_msr(MSR_ZCR_EL2, ns_simd_ctx[cpu_id].zcr_el2);
212 isb();
213
214 arch_sve_enable_traps();
215 }
216
217 if (sme) {
218 /* Restore SSVE vector length if enabled. */
219 write_msr(MSR_SMCR_EL2, ns_simd_ctx[cpu_id].smcr_el2);
220 isb();
221
222 arch_sme_enable_traps();
223 }
224 }
225
226 /**
227 * Save FPU/Adv SIMD/SVE/SME 'Other world' context when entering the SPMC.
228 * Called from handler.c: smc_handler_from_nwd.
229 */
plat_save_ns_simd_context(struct vcpu * vcpu)230 void plat_save_ns_simd_context(struct vcpu *vcpu)
231 {
232 uint32_t cpu_id;
233 uint64_t smc_fid;
234 bool sve = is_arch_feat_sve_supported();
235 bool sme = is_arch_feat_sme_supported();
236 bool fa64 = is_arch_feat_sme_fa64_supported();
237 bool streaming_mode = false;
238 bool hint;
239
240 assert(vcpu->vm->id == HF_HYPERVISOR_VM_ID);
241 cpu_id = cpu_index(vcpu->cpu);
242
243 /* Get SMCCCv1.3 SMC FID[16] SVE hint, and clear it from vCPU r0. */
244 smc_fid = vcpu->regs.r[0];
245 hint = ns_simd_ctx[cpu_id].hint = (smc_fid & SMCCC_SVE_HINT_MASK) != 0;
246
247 if (sme) {
248 /* Disable SME EL2 and lower traps. */
249 arch_sme_disable_traps();
250
251 /*
252 * Save current SMCR_EL2 value, in particular to preserve
253 * NS context SSVE vector length.
254 */
255 ns_simd_ctx[cpu_id].smcr_el2 = read_msr(MSR_SMCR_EL2);
256
257 /* Configure EL2 SSVE vector length. */
258 arch_sme_configure_svl();
259
260 /* Save ZA array and SSVE enable state. */
261 ns_simd_ctx[cpu_id].svcr = arch_sme_svcr_get();
262
263 streaming_mode =
264 (ns_simd_ctx[cpu_id].svcr & MSR_SVCR_SM) == MSR_SVCR_SM;
265 }
266
267 if (sve) {
268 /* Disable SVE EL2 and lower traps. */
269 arch_sve_disable_traps();
270
271 /*
272 * Save current ZCR_EL2 value, in particular to preserve
273 * NS context SVE Vector Length.
274 */
275 ns_simd_ctx[cpu_id].zcr_el2 = read_msr(MSR_ZCR_EL2);
276
277 /* Configure EL2 SVE Vector Length. */
278 arch_sve_configure_vector_length();
279 }
280
281 /* Save FPCR/FPSR common to FPU/Adv. SIMD/SVE/SME. */
282 arch_fpu_state_save_to_vcpu(vcpu);
283
284 /*
285 * If SVE or SME is implemented and SVE hint is false as passed by
286 * the normal world caller, then save the SVE (or Streaming SVE) state.
287 * Omit saving the SVE state, if only SME is implemented (and SVE is not
288 * implemented) and Streaming SVE is disabled.
289 */
290 if ((sve || sme) && !hint && !(!sve && sme && !streaming_mode)) {
291 /*
292 * Save predicates if SVE only is implemented,
293 * or both SME and SVE are implemented and Streaming SVE
294 * is disabled,
295 * or SME is implemented and Streaming SVE is enabled.
296 */
297 if ((sve && !sme) || (sve && sme && !streaming_mode) ||
298 (sme && streaming_mode)) {
299 /* Save predicate registers. */
300 __asm__ volatile(
301 ".arch_extension sve;"
302 "str p0, [%0, #0, MUL VL];"
303 "str p1, [%0, #1, MUL VL];"
304 "str p2, [%0, #2, MUL VL];"
305 "str p3, [%0, #3, MUL VL];"
306 "str p4, [%0, #4, MUL VL];"
307 "str p5, [%0, #5, MUL VL];"
308 "str p6, [%0, #6, MUL VL];"
309 "str p7, [%0, #7, MUL VL];"
310 "str p8, [%0, #8, MUL VL];"
311 "str p9, [%0, #9, MUL VL];"
312 "str p10, [%0, #10, MUL VL];"
313 "str p11, [%0, #11, MUL VL];"
314 "str p12, [%0, #12, MUL VL];"
315 "str p13, [%0, #13, MUL VL];"
316 "str p14, [%0, #14, MUL VL];"
317 "str p15, [%0, #15, MUL VL];"
318 ".arch_extension nosve"
319 :
320 : "r"(&ns_simd_ctx[cpu_id]
321 .sve_context.predicates));
322 }
323
324 /*
325 * Save FFR register if SVE only is implemented,
326 * or both SVE and SME are implemented and Streaming SVE
327 * is disabled, or both SME and FEAT_SME_FA64 are implemented
328 * and Streaming SVE is enabled.
329 */
330 if ((sve && !sme) || (sve && sme && !streaming_mode) ||
331 (sme && fa64 && streaming_mode)) {
332 __asm__ volatile(
333 ".arch_extension sve;"
334 "rdffr p0.b;"
335 "str p0, [%0];"
336 ".arch_extension nosve"
337 :
338 : "r"(&ns_simd_ctx[cpu_id].sve_context.ffr));
339 }
340
341 /*
342 * Save SVE/Streaming SVE vectors (similar conditions as
343 * predicates above).
344 */
345 if ((sve && !sme) || (sve && sme && !streaming_mode) ||
346 (sme && streaming_mode)) {
347 __asm__ volatile(
348 ".arch_extension sve;"
349 "str z0, [%0, #0, MUL VL];"
350 "str z1, [%0, #1, MUL VL];"
351 "str z2, [%0, #2, MUL VL];"
352 "str z3, [%0, #3, MUL VL];"
353 "str z4, [%0, #4, MUL VL];"
354 "str z5, [%0, #5, MUL VL];"
355 "str z6, [%0, #6, MUL VL];"
356 "str z7, [%0, #7, MUL VL];"
357 "str z8, [%0, #8, MUL VL];"
358 "str z9, [%0, #9, MUL VL];"
359 "str z10, [%0, #10, MUL VL];"
360 "str z11, [%0, #11, MUL VL];"
361 "str z12, [%0, #12, MUL VL];"
362 "str z13, [%0, #13, MUL VL];"
363 "str z14, [%0, #14, MUL VL];"
364 "str z15, [%0, #15, MUL VL];"
365 "str z16, [%0, #16, MUL VL];"
366 "str z17, [%0, #17, MUL VL];"
367 "str z18, [%0, #18, MUL VL];"
368 "str z19, [%0, #19, MUL VL];"
369 "str z20, [%0, #20, MUL VL];"
370 "str z21, [%0, #21, MUL VL];"
371 "str z22, [%0, #22, MUL VL];"
372 "str z23, [%0, #23, MUL VL];"
373 "str z24, [%0, #24, MUL VL];"
374 "str z25, [%0, #25, MUL VL];"
375 "str z26, [%0, #26, MUL VL];"
376 "str z27, [%0, #27, MUL VL];"
377 "str z28, [%0, #28, MUL VL];"
378 "str z29, [%0, #29, MUL VL];"
379 "str z30, [%0, #30, MUL VL];"
380 "str z31, [%0, #31, MUL VL];"
381 ".arch_extension nosve"
382 :
383 : "r"(&ns_simd_ctx[cpu_id]
384 .sve_context.vectors));
385 }
386 } else {
387 /* Save FPU/Adv. SIMD vectors. */
388 arch_fpu_regs_save_to_vcpu(vcpu);
389 }
390
391 if (sve) {
392 arch_sve_enable_traps();
393 }
394
395 /*
396 * SVCR.ZA=1 indicates the ZA array is live.
397 * We deliberately choose to leave the ZA array enabled, knowing
398 * that S-EL2 and lower won't make use of SME.
399 * S-EL1 and lower are prevented SME registers access. There is
400 * a probable performance impact but this avoids us saving/restoring
401 * the ZA array contents. SME2 ZT0 isn't touched by EL2 and lower ELs
402 * hence no need to save/restore it.
403 */
404
405 if (sme) {
406 if (streaming_mode) {
407 /*
408 * SVCR.SM=1 indicates active Streaming SVE mode.
409 * It is preferable to disable it to save power.
410 * The overall NS SIMD state has been saved above.
411 * Disabling SSVE destroys the live state. Change
412 * to this field doesn't impact the ZA storage.
413 */
414 arch_sme_svcr_set(ns_simd_ctx[cpu_id].svcr &
415 ~MSR_SVCR_SM);
416 }
417
418 arch_sme_enable_traps();
419 }
420 }
421