1 /*
2 * xen/arch/arm/traps.c
3 *
4 * ARM Trap handlers
5 *
6 * Copyright (c) 2011 Citrix Systems.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 */
18
19 #include <xen/domain_page.h>
20 #include <xen/errno.h>
21 #include <xen/hypercall.h>
22 #include <xen/init.h>
23 #include <xen/iocap.h>
24 #include <xen/irq.h>
25 #include <xen/lib.h>
26 #include <xen/livepatch.h>
27 #include <xen/mem_access.h>
28 #include <xen/mm.h>
29 #include <xen/perfc.h>
30 #include <xen/smp.h>
31 #include <xen/softirq.h>
32 #include <xen/string.h>
33 #include <xen/symbols.h>
34 #include <xen/version.h>
35 #include <xen/virtual_region.h>
36
37 #include <public/sched.h>
38 #include <public/xen.h>
39
40 #include <asm/acpi.h>
41 #include <asm/cpuerrata.h>
42 #include <asm/cpufeature.h>
43 #include <asm/debugger.h>
44 #include <asm/event.h>
45 #include <asm/flushtlb.h>
46 #include <asm/gic.h>
47 #include <asm/mmio.h>
48 #include <asm/monitor.h>
49 #include <asm/psci.h>
50 #include <asm/regs.h>
51 #include <asm/traps.h>
52 #include <asm/vgic.h>
53 #include <asm/vtimer.h>
54
55 #include "decode.h"
56
57 /* The base of the stack must always be double-word aligned, which means
58 * that both the kernel half of struct cpu_user_regs (which is pushed in
59 * entry.S) and struct cpu_info (which lives at the bottom of a Xen
60 * stack) must be doubleword-aligned in size. */
check_stack_alignment_constraints(void)61 static inline void check_stack_alignment_constraints(void) {
62 #ifdef CONFIG_ARM_64
63 BUILD_BUG_ON((sizeof (struct cpu_user_regs)) & 0xf);
64 BUILD_BUG_ON((offsetof(struct cpu_user_regs, spsr_el1)) & 0xf);
65 BUILD_BUG_ON((offsetof(struct cpu_user_regs, lr)) & 0xf);
66 BUILD_BUG_ON((sizeof (struct cpu_info)) & 0xf);
67 #else
68 BUILD_BUG_ON((sizeof (struct cpu_user_regs)) & 0x7);
69 BUILD_BUG_ON((offsetof(struct cpu_user_regs, sp_usr)) & 0x7);
70 BUILD_BUG_ON((sizeof (struct cpu_info)) & 0x7);
71 #endif
72 }
73
74 /*
75 * GUEST_BUG_ON is intended for checking that the guest state has not been
76 * corrupted in hardware and/or that the hardware behaves as we
77 * believe it should (i.e. that certain traps can only occur when the
78 * guest is in a particular mode).
79 *
80 * The intention is to limit the damage such h/w bugs (or spec
81 * misunderstandings) can do by turning them into Denial of Service
82 * attacks instead of e.g. information leaks or privilege escalations.
83 *
84 * GUEST_BUG_ON *MUST* *NOT* be used to check for guest controllable state!
85 *
86 * Compared with regular BUG_ON it dumps the guest vcpu state instead
87 * of Xen's state.
88 */
89 #define guest_bug_on_failed(p) \
90 do { \
91 show_execution_state(guest_cpu_user_regs()); \
92 panic("Guest Bug: %pv: '%s', line %d, file %s\n", \
93 current, p, __LINE__, __FILE__); \
94 } while (0)
95 #define GUEST_BUG_ON(p) \
96 do { if ( unlikely(p) ) guest_bug_on_failed(#p); } while (0)
97
98 #ifdef CONFIG_ARM_32
99 static int debug_stack_lines = 20;
100 #define stack_words_per_line 8
101 #else
102 static int debug_stack_lines = 40;
103 #define stack_words_per_line 4
104 #endif
105
106 integer_param("debug_stack_lines", debug_stack_lines);
107
108 static enum {
109 TRAP,
110 NATIVE,
111 } vwfi;
112
parse_vwfi(const char * s)113 static int __init parse_vwfi(const char *s)
114 {
115 if ( !strcmp(s, "native") )
116 vwfi = NATIVE;
117 else
118 vwfi = TRAP;
119
120 return 0;
121 }
122 custom_param("vwfi", parse_vwfi);
123
get_default_hcr_flags(void)124 register_t get_default_hcr_flags(void)
125 {
126 return (HCR_PTW|HCR_BSU_INNER|HCR_AMO|HCR_IMO|HCR_FMO|HCR_VM|
127 (vwfi != NATIVE ? (HCR_TWI|HCR_TWE) : 0) |
128 HCR_TSC|HCR_TAC|HCR_SWIO|HCR_TIDCP|HCR_FB);
129 }
130
131 static enum {
132 SERRORS_DIVERSE,
133 SERRORS_FORWARD,
134 SERRORS_PANIC,
135 } serrors_op;
136
parse_serrors_behavior(const char * str)137 static int __init parse_serrors_behavior(const char *str)
138 {
139 if ( !strcmp(str, "forward") )
140 serrors_op = SERRORS_FORWARD;
141 else if ( !strcmp(str, "panic") )
142 serrors_op = SERRORS_PANIC;
143 else
144 serrors_op = SERRORS_DIVERSE;
145
146 return 0;
147 }
148 custom_param("serrors", parse_serrors_behavior);
149
update_serrors_cpu_caps(void)150 static int __init update_serrors_cpu_caps(void)
151 {
152 if ( serrors_op != SERRORS_DIVERSE )
153 cpus_set_cap(SKIP_SYNCHRONIZE_SERROR_ENTRY_EXIT);
154
155 if ( serrors_op != SERRORS_FORWARD )
156 cpus_set_cap(SKIP_CTXT_SWITCH_SERROR_SYNC);
157
158 return 0;
159 }
160 __initcall(update_serrors_cpu_caps);
161
init_traps(void)162 void init_traps(void)
163 {
164 /* Setup Hyp vector base */
165 WRITE_SYSREG((vaddr_t)hyp_traps_vector, VBAR_EL2);
166
167 /* Trap Debug and Performance Monitor accesses */
168 WRITE_SYSREG(HDCR_TDRA|HDCR_TDOSA|HDCR_TDA|HDCR_TPM|HDCR_TPMCR,
169 MDCR_EL2);
170
171 /* Trap CP15 c15 used for implementation defined registers */
172 WRITE_SYSREG(HSTR_T(15), HSTR_EL2);
173
174 /* Trap all coprocessor registers (0-13) except cp10 and
175 * cp11 for VFP.
176 *
177 * /!\ All coprocessors except cp10 and cp11 cannot be used in Xen.
178 *
179 * On ARM64 the TCPx bits which we set here (0..9,12,13) are all
180 * RES1, i.e. they would trap whether we did this write or not.
181 */
182 WRITE_SYSREG((HCPTR_CP_MASK & ~(HCPTR_CP(10) | HCPTR_CP(11))) | HCPTR_TTA,
183 CPTR_EL2);
184
185 /* Setup hypervisor traps */
186 WRITE_SYSREG(get_default_hcr_flags(), HCR_EL2);
187 isb();
188 }
189
__div0(void)190 void __div0(void)
191 {
192 printk("Division by zero in hypervisor.\n");
193 BUG();
194 }
195
196 /* XXX could/should be common code */
print_xen_info(void)197 static void print_xen_info(void)
198 {
199 char taint_str[TAINT_STRING_MAX_LEN];
200
201 printk("----[ Xen-%d.%d%s %s debug=%c " gcov_string " %s ]----\n",
202 xen_major_version(), xen_minor_version(), xen_extra_version(),
203 #ifdef CONFIG_ARM_32
204 "arm32",
205 #else
206 "arm64",
207 #endif
208 debug_build() ? 'y' : 'n', print_tainted(taint_str));
209 }
210
211 #ifdef CONFIG_ARM_32
is_zero_register(int reg)212 static inline bool is_zero_register(int reg)
213 {
214 /* There is no zero register for ARM32 */
215 return false;
216 }
217 #else
is_zero_register(int reg)218 static inline bool is_zero_register(int reg)
219 {
220 /*
221 * For store/load and sysreg instruction, the encoding 31 always
222 * corresponds to {w,x}zr which is the zero register.
223 */
224 return (reg == 31);
225 }
226 #endif
227
228 /*
229 * Returns a pointer to the given register value in regs, taking the
230 * processor mode (CPSR) into account.
231 *
232 * Note that this function should not be used directly but via
233 * {get,set}_user_reg.
234 */
select_user_reg(struct cpu_user_regs * regs,int reg)235 static register_t *select_user_reg(struct cpu_user_regs *regs, int reg)
236 {
237 BUG_ON( !guest_mode(regs) );
238
239 #ifdef CONFIG_ARM_32
240 /*
241 * We rely heavily on the layout of cpu_user_regs to avoid having
242 * to handle all of the registers individually. Use BUILD_BUG_ON to
243 * ensure that things which expect are contiguous actually are.
244 */
245 #define REGOFFS(R) offsetof(struct cpu_user_regs, R)
246
247 switch ( reg ) {
248 case 0 ... 7: /* Unbanked registers */
249 BUILD_BUG_ON(REGOFFS(r0) + 7*sizeof(register_t) != REGOFFS(r7));
250 return ®s->r0 + reg;
251 case 8 ... 12: /* Register banked in FIQ mode */
252 BUILD_BUG_ON(REGOFFS(r8_fiq) + 4*sizeof(register_t) != REGOFFS(r12_fiq));
253 if ( fiq_mode(regs) )
254 return ®s->r8_fiq + reg - 8;
255 else
256 return ®s->r8 + reg - 8;
257 case 13 ... 14: /* Banked SP + LR registers */
258 BUILD_BUG_ON(REGOFFS(sp_fiq) + 1*sizeof(register_t) != REGOFFS(lr_fiq));
259 BUILD_BUG_ON(REGOFFS(sp_irq) + 1*sizeof(register_t) != REGOFFS(lr_irq));
260 BUILD_BUG_ON(REGOFFS(sp_svc) + 1*sizeof(register_t) != REGOFFS(lr_svc));
261 BUILD_BUG_ON(REGOFFS(sp_abt) + 1*sizeof(register_t) != REGOFFS(lr_abt));
262 BUILD_BUG_ON(REGOFFS(sp_und) + 1*sizeof(register_t) != REGOFFS(lr_und));
263 switch ( regs->cpsr & PSR_MODE_MASK )
264 {
265 case PSR_MODE_USR:
266 case PSR_MODE_SYS: /* Sys regs are the usr regs */
267 if ( reg == 13 )
268 return ®s->sp_usr;
269 else /* lr_usr == lr in a user frame */
270 return ®s->lr;
271 case PSR_MODE_FIQ:
272 return ®s->sp_fiq + reg - 13;
273 case PSR_MODE_IRQ:
274 return ®s->sp_irq + reg - 13;
275 case PSR_MODE_SVC:
276 return ®s->sp_svc + reg - 13;
277 case PSR_MODE_ABT:
278 return ®s->sp_abt + reg - 13;
279 case PSR_MODE_UND:
280 return ®s->sp_und + reg - 13;
281 case PSR_MODE_MON:
282 case PSR_MODE_HYP:
283 default:
284 BUG();
285 }
286 case 15: /* PC */
287 return ®s->pc;
288 default:
289 BUG();
290 }
291 #undef REGOFFS
292 #else
293 /*
294 * On 64-bit the syndrome register contains the register index as
295 * viewed in AArch64 state even if the trap was from AArch32 mode.
296 */
297 BUG_ON(is_zero_register(reg)); /* Cannot be {w,x}zr */
298 return ®s->x0 + reg;
299 #endif
300 }
301
get_user_reg(struct cpu_user_regs * regs,int reg)302 register_t get_user_reg(struct cpu_user_regs *regs, int reg)
303 {
304 if ( is_zero_register(reg) )
305 return 0;
306
307 return *select_user_reg(regs, reg);
308 }
309
set_user_reg(struct cpu_user_regs * regs,int reg,register_t value)310 void set_user_reg(struct cpu_user_regs *regs, int reg, register_t value)
311 {
312 if ( is_zero_register(reg) )
313 return;
314
315 *select_user_reg(regs, reg) = value;
316 }
317
decode_fsc(uint32_t fsc,int * level)318 static const char *decode_fsc(uint32_t fsc, int *level)
319 {
320 const char *msg = NULL;
321
322 switch ( fsc & 0x3f )
323 {
324 case FSC_FLT_TRANS ... FSC_FLT_TRANS + 3:
325 msg = "Translation fault";
326 *level = fsc & FSC_LL_MASK;
327 break;
328 case FSC_FLT_ACCESS ... FSC_FLT_ACCESS + 3:
329 msg = "Access fault";
330 *level = fsc & FSC_LL_MASK;
331 break;
332 case FSC_FLT_PERM ... FSC_FLT_PERM + 3:
333 msg = "Permission fault";
334 *level = fsc & FSC_LL_MASK;
335 break;
336
337 case FSC_SEA:
338 msg = "Synchronous External Abort";
339 break;
340 case FSC_SPE:
341 msg = "Memory Access Synchronous Parity Error";
342 break;
343 case FSC_APE:
344 msg = "Memory Access Asynchronous Parity Error";
345 break;
346 case FSC_SEATT ... FSC_SEATT + 3:
347 msg = "Sync. Ext. Abort Translation Table";
348 *level = fsc & FSC_LL_MASK;
349 break;
350 case FSC_SPETT ... FSC_SPETT + 3:
351 msg = "Sync. Parity. Error Translation Table";
352 *level = fsc & FSC_LL_MASK;
353 break;
354 case FSC_AF:
355 msg = "Alignment Fault";
356 break;
357 case FSC_DE:
358 msg = "Debug Event";
359 break;
360
361 case FSC_LKD:
362 msg = "Implementation Fault: Lockdown Abort";
363 break;
364 case FSC_CPR:
365 msg = "Implementation Fault: Coprocossor Abort";
366 break;
367
368 default:
369 msg = "Unknown Failure";
370 break;
371 }
372 return msg;
373 }
374
fsc_level_str(int level)375 static const char *fsc_level_str(int level)
376 {
377 switch ( level )
378 {
379 case -1: return "";
380 case 1: return " at level 1";
381 case 2: return " at level 2";
382 case 3: return " at level 3";
383 default: return " (level invalid)";
384 }
385 }
386
panic_PAR(uint64_t par)387 void panic_PAR(uint64_t par)
388 {
389 const char *msg;
390 int level = -1;
391 int stage = par & PAR_STAGE2 ? 2 : 1;
392 int second_in_first = !!(par & PAR_STAGE21);
393
394 msg = decode_fsc( (par&PAR_FSC_MASK) >> PAR_FSC_SHIFT, &level);
395
396 printk("PAR: %016"PRIx64": %s stage %d%s%s\n",
397 par, msg,
398 stage,
399 second_in_first ? " during second stage lookup" : "",
400 fsc_level_str(level));
401
402 panic("Error during Hypervisor-to-physical address translation");
403 }
404
cpsr_switch_mode(struct cpu_user_regs * regs,int mode)405 static void cpsr_switch_mode(struct cpu_user_regs *regs, int mode)
406 {
407 uint32_t sctlr = READ_SYSREG32(SCTLR_EL1);
408
409 regs->cpsr &= ~(PSR_MODE_MASK|PSR_IT_MASK|PSR_JAZELLE|PSR_BIG_ENDIAN|PSR_THUMB);
410
411 regs->cpsr |= mode;
412 regs->cpsr |= PSR_IRQ_MASK;
413 if ( mode == PSR_MODE_ABT )
414 regs->cpsr |= PSR_ABT_MASK;
415 if ( sctlr & SCTLR_TE )
416 regs->cpsr |= PSR_THUMB;
417 if ( sctlr & SCTLR_EE )
418 regs->cpsr |= PSR_BIG_ENDIAN;
419 }
420
exception_handler32(vaddr_t offset)421 static vaddr_t exception_handler32(vaddr_t offset)
422 {
423 uint32_t sctlr = READ_SYSREG32(SCTLR_EL1);
424
425 if (sctlr & SCTLR_V)
426 return 0xffff0000 + offset;
427 else /* always have security exceptions */
428 return READ_SYSREG(VBAR_EL1) + offset;
429 }
430
431 /* Injects an Undefined Instruction exception into the current vcpu,
432 * PC is the exact address of the faulting instruction (without
433 * pipeline adjustments). See TakeUndefInstrException pseudocode in
434 * ARM ARM.
435 */
inject_undef32_exception(struct cpu_user_regs * regs)436 static void inject_undef32_exception(struct cpu_user_regs *regs)
437 {
438 uint32_t spsr = regs->cpsr;
439 int is_thumb = (regs->cpsr & PSR_THUMB);
440 /* Saved PC points to the instruction past the faulting instruction. */
441 uint32_t return_offset = is_thumb ? 2 : 4;
442
443 BUG_ON( !is_32bit_domain(current->domain) );
444
445 /* Update processor mode */
446 cpsr_switch_mode(regs, PSR_MODE_UND);
447
448 /* Update banked registers */
449 regs->spsr_und = spsr;
450 regs->lr_und = regs->pc32 + return_offset;
451
452 /* Branch to exception vector */
453 regs->pc32 = exception_handler32(VECTOR32_UND);
454 }
455
456 /* Injects an Abort exception into the current vcpu, PC is the exact
457 * address of the faulting instruction (without pipeline
458 * adjustments). See TakePrefetchAbortException and
459 * TakeDataAbortException pseudocode in ARM ARM.
460 */
inject_abt32_exception(struct cpu_user_regs * regs,int prefetch,register_t addr)461 static void inject_abt32_exception(struct cpu_user_regs *regs,
462 int prefetch,
463 register_t addr)
464 {
465 uint32_t spsr = regs->cpsr;
466 int is_thumb = (regs->cpsr & PSR_THUMB);
467 /* Saved PC points to the instruction past the faulting instruction. */
468 uint32_t return_offset = is_thumb ? 4 : 0;
469 register_t fsr;
470
471 BUG_ON( !is_32bit_domain(current->domain) );
472
473 cpsr_switch_mode(regs, PSR_MODE_ABT);
474
475 /* Update banked registers */
476 regs->spsr_abt = spsr;
477 regs->lr_abt = regs->pc32 + return_offset;
478
479 regs->pc32 = exception_handler32(prefetch ? VECTOR32_PABT : VECTOR32_DABT);
480
481 /* Inject a debug fault, best we can do right now */
482 if ( READ_SYSREG(TCR_EL1) & TTBCR_EAE )
483 fsr = FSR_LPAE | FSRL_STATUS_DEBUG;
484 else
485 fsr = FSRS_FS_DEBUG;
486
487 if ( prefetch )
488 {
489 /* Set IFAR and IFSR */
490 #ifdef CONFIG_ARM_32
491 WRITE_SYSREG(addr, IFAR);
492 WRITE_SYSREG(fsr, IFSR);
493 #else
494 /* FAR_EL1[63:32] is AArch32 register IFAR */
495 register_t far = READ_SYSREG(FAR_EL1) & 0xffffffffUL;
496 far |= addr << 32;
497 WRITE_SYSREG(far, FAR_EL1);
498 WRITE_SYSREG(fsr, IFSR32_EL2);
499 #endif
500 }
501 else
502 {
503 #ifdef CONFIG_ARM_32
504 /* Set DFAR and DFSR */
505 WRITE_SYSREG(addr, DFAR);
506 WRITE_SYSREG(fsr, DFSR);
507 #else
508 /* FAR_EL1[31:0] is AArch32 register DFAR */
509 register_t far = READ_SYSREG(FAR_EL1) & ~0xffffffffUL;
510 far |= addr;
511 WRITE_SYSREG(far, FAR_EL1);
512 /* ESR_EL1 is AArch32 register DFSR */
513 WRITE_SYSREG(fsr, ESR_EL1);
514 #endif
515 }
516 }
517
inject_dabt32_exception(struct cpu_user_regs * regs,register_t addr)518 static void inject_dabt32_exception(struct cpu_user_regs *regs,
519 register_t addr)
520 {
521 inject_abt32_exception(regs, 0, addr);
522 }
523
inject_pabt32_exception(struct cpu_user_regs * regs,register_t addr)524 static void inject_pabt32_exception(struct cpu_user_regs *regs,
525 register_t addr)
526 {
527 inject_abt32_exception(regs, 1, addr);
528 }
529
530 #ifdef CONFIG_ARM_64
531 /*
532 * Take care to call this while regs contains the original faulting
533 * state and not the (partially constructed) exception state.
534 */
exception_handler64(struct cpu_user_regs * regs,vaddr_t offset)535 static vaddr_t exception_handler64(struct cpu_user_regs *regs, vaddr_t offset)
536 {
537 vaddr_t base = READ_SYSREG(VBAR_EL1);
538
539 if ( usr_mode(regs) )
540 base += VECTOR64_LOWER32_BASE;
541 else if ( psr_mode(regs->cpsr,PSR_MODE_EL0t) )
542 base += VECTOR64_LOWER64_BASE;
543 else /* Otherwise must be from kernel mode */
544 base += VECTOR64_CURRENT_SPx_BASE;
545
546 return base + offset;
547 }
548
549 /* Inject an undefined exception into a 64 bit guest */
inject_undef64_exception(struct cpu_user_regs * regs,int instr_len)550 void inject_undef64_exception(struct cpu_user_regs *regs, int instr_len)
551 {
552 vaddr_t handler;
553 const union hsr esr = {
554 .iss = 0,
555 .len = instr_len,
556 .ec = HSR_EC_UNKNOWN,
557 };
558
559 BUG_ON( is_32bit_domain(current->domain) );
560
561 handler = exception_handler64(regs, VECTOR64_SYNC_OFFSET);
562
563 regs->spsr_el1 = regs->cpsr;
564 regs->elr_el1 = regs->pc;
565
566 regs->cpsr = PSR_MODE_EL1h | PSR_ABT_MASK | PSR_FIQ_MASK | \
567 PSR_IRQ_MASK | PSR_DBG_MASK;
568 regs->pc = handler;
569
570 WRITE_SYSREG32(esr.bits, ESR_EL1);
571 }
572
573 /* Inject an abort exception into a 64 bit guest */
inject_abt64_exception(struct cpu_user_regs * regs,int prefetch,register_t addr,int instr_len)574 static void inject_abt64_exception(struct cpu_user_regs *regs,
575 int prefetch,
576 register_t addr,
577 int instr_len)
578 {
579 vaddr_t handler;
580 union hsr esr = {
581 .iss = 0,
582 .len = instr_len,
583 };
584
585 if ( psr_mode_is_user(regs) )
586 esr.ec = prefetch
587 ? HSR_EC_INSTR_ABORT_LOWER_EL : HSR_EC_DATA_ABORT_LOWER_EL;
588 else
589 esr.ec = prefetch
590 ? HSR_EC_INSTR_ABORT_CURR_EL : HSR_EC_DATA_ABORT_CURR_EL;
591
592 BUG_ON( is_32bit_domain(current->domain) );
593
594 handler = exception_handler64(regs, VECTOR64_SYNC_OFFSET);
595
596 regs->spsr_el1 = regs->cpsr;
597 regs->elr_el1 = regs->pc;
598
599 regs->cpsr = PSR_MODE_EL1h | PSR_ABT_MASK | PSR_FIQ_MASK | \
600 PSR_IRQ_MASK | PSR_DBG_MASK;
601 regs->pc = handler;
602
603 WRITE_SYSREG(addr, FAR_EL1);
604 WRITE_SYSREG32(esr.bits, ESR_EL1);
605 }
606
inject_dabt64_exception(struct cpu_user_regs * regs,register_t addr,int instr_len)607 static void inject_dabt64_exception(struct cpu_user_regs *regs,
608 register_t addr,
609 int instr_len)
610 {
611 inject_abt64_exception(regs, 0, addr, instr_len);
612 }
613
inject_iabt64_exception(struct cpu_user_regs * regs,register_t addr,int instr_len)614 static void inject_iabt64_exception(struct cpu_user_regs *regs,
615 register_t addr,
616 int instr_len)
617 {
618 inject_abt64_exception(regs, 1, addr, instr_len);
619 }
620
621 #endif
622
inject_undef_exception(struct cpu_user_regs * regs,const union hsr hsr)623 void inject_undef_exception(struct cpu_user_regs *regs, const union hsr hsr)
624 {
625 if ( is_32bit_domain(current->domain) )
626 inject_undef32_exception(regs);
627 #ifdef CONFIG_ARM_64
628 else
629 inject_undef64_exception(regs, hsr.len);
630 #endif
631 }
632
inject_iabt_exception(struct cpu_user_regs * regs,register_t addr,int instr_len)633 static void inject_iabt_exception(struct cpu_user_regs *regs,
634 register_t addr,
635 int instr_len)
636 {
637 if ( is_32bit_domain(current->domain) )
638 inject_pabt32_exception(regs, addr);
639 #ifdef CONFIG_ARM_64
640 else
641 inject_iabt64_exception(regs, addr, instr_len);
642 #endif
643 }
644
inject_dabt_exception(struct cpu_user_regs * regs,register_t addr,int instr_len)645 static void inject_dabt_exception(struct cpu_user_regs *regs,
646 register_t addr,
647 int instr_len)
648 {
649 if ( is_32bit_domain(current->domain) )
650 inject_dabt32_exception(regs, addr);
651 #ifdef CONFIG_ARM_64
652 else
653 inject_dabt64_exception(regs, addr, instr_len);
654 #endif
655 }
656
657 /* Inject a virtual Abort/SError into the guest. */
inject_vabt_exception(struct cpu_user_regs * regs)658 static void inject_vabt_exception(struct cpu_user_regs *regs)
659 {
660 const union hsr hsr = { .bits = regs->hsr };
661
662 /*
663 * SVC/HVC/SMC already have an adjusted PC (See ARM ARM DDI 0487A.j
664 * D1.10.1 for more details), which we need to correct in order to
665 * return to after having injected the SError.
666 */
667 switch ( hsr.ec )
668 {
669 case HSR_EC_SVC32:
670 case HSR_EC_HVC32:
671 case HSR_EC_SMC32:
672 #ifdef CONFIG_ARM_64
673 case HSR_EC_SVC64:
674 case HSR_EC_HVC64:
675 case HSR_EC_SMC64:
676 #endif
677 regs->pc -= hsr.len ? 4 : 2;
678 break;
679
680 default:
681 break;
682 }
683
684 current->arch.hcr_el2 |= HCR_VA;
685 WRITE_SYSREG(current->arch.hcr_el2, HCR_EL2);
686 }
687
688 /*
689 * SError exception handler. We only handle the following 3 types of SErrors:
690 * 1) Guest-generated SError and had been delivered in EL1 and then
691 * been forwarded to EL2.
692 * 2) Guest-generated SError but hadn't been delivered in EL1 before
693 * trapping to EL2. This SError would be caught in EL2 as soon as
694 * we just unmasked the PSTATE.A bit.
695 * 3) Hypervisor generated native SError, that would be a bug.
696 *
697 * A true parameter "guest" means that the SError is type#1 or type#2.
698 */
__do_trap_serror(struct cpu_user_regs * regs,bool guest)699 static void __do_trap_serror(struct cpu_user_regs *regs, bool guest)
700 {
701 /*
702 * Only "DIVERSE" option needs to distinguish the guest-generated SErrors
703 * from hypervisor SErrors.
704 */
705 if ( serrors_op == SERRORS_DIVERSE )
706 {
707 /* Forward the type#1 and type#2 SErrors to guests. */
708 if ( guest )
709 return inject_vabt_exception(regs);
710
711 /* Type#3 SErrors will panic the whole system */
712 goto crash_system;
713 }
714
715 /*
716 * The "FORWARD" option will forward all SErrors to the guests, except
717 * idle domain generated SErrors.
718 */
719 if ( serrors_op == SERRORS_FORWARD )
720 {
721 /*
722 * Because the idle domain doesn't have the ability to handle the
723 * SErrors, we have to crash the whole system while we get a SError
724 * generated by idle domain.
725 */
726 if ( is_idle_vcpu(current) )
727 goto crash_system;
728
729 return inject_vabt_exception(regs);
730 }
731
732 crash_system:
733 /* Three possibilities to crash the whole system:
734 * 1) "DIVERSE" option with Hypervisor generated SErrors.
735 * 2) "FORWARD" option with Idle Domain generated SErrors.
736 * 3) "PANIC" option with all SErrors.
737 */
738 do_unexpected_trap("SError", regs);
739 }
740
741 struct reg_ctxt {
742 /* Guest-side state */
743 uint32_t sctlr_el1;
744 register_t tcr_el1;
745 uint64_t ttbr0_el1, ttbr1_el1;
746 #ifdef CONFIG_ARM_32
747 uint32_t dfsr, ifsr;
748 uint32_t dfar, ifar;
749 #else
750 uint32_t esr_el1;
751 uint64_t far;
752 uint32_t ifsr32_el2;
753 #endif
754
755 /* Hypervisor-side state */
756 uint64_t vttbr_el2;
757 };
758
mode_string(uint32_t cpsr)759 static const char *mode_string(uint32_t cpsr)
760 {
761 uint32_t mode;
762 static const char *mode_strings[] = {
763 [PSR_MODE_USR] = "32-bit Guest USR",
764 [PSR_MODE_FIQ] = "32-bit Guest FIQ",
765 [PSR_MODE_IRQ] = "32-bit Guest IRQ",
766 [PSR_MODE_SVC] = "32-bit Guest SVC",
767 [PSR_MODE_MON] = "32-bit Monitor",
768 [PSR_MODE_ABT] = "32-bit Guest ABT",
769 [PSR_MODE_HYP] = "Hypervisor",
770 [PSR_MODE_UND] = "32-bit Guest UND",
771 [PSR_MODE_SYS] = "32-bit Guest SYS",
772 #ifdef CONFIG_ARM_64
773 [PSR_MODE_EL3h] = "64-bit EL3h (Monitor, handler)",
774 [PSR_MODE_EL3t] = "64-bit EL3t (Monitor, thread)",
775 [PSR_MODE_EL2h] = "64-bit EL2h (Hypervisor, handler)",
776 [PSR_MODE_EL2t] = "64-bit EL2t (Hypervisor, thread)",
777 [PSR_MODE_EL1h] = "64-bit EL1h (Guest Kernel, handler)",
778 [PSR_MODE_EL1t] = "64-bit EL1t (Guest Kernel, thread)",
779 [PSR_MODE_EL0t] = "64-bit EL0t (Guest User)",
780 #endif
781 };
782 mode = cpsr & PSR_MODE_MASK;
783
784 if ( mode >= ARRAY_SIZE(mode_strings) )
785 return "Unknown";
786 return mode_strings[mode] ? : "Unknown";
787 }
788
show_registers_32(struct cpu_user_regs * regs,struct reg_ctxt * ctxt,int guest_mode,const struct vcpu * v)789 static void show_registers_32(struct cpu_user_regs *regs,
790 struct reg_ctxt *ctxt,
791 int guest_mode,
792 const struct vcpu *v)
793 {
794
795 #ifdef CONFIG_ARM_64
796 BUG_ON( ! (regs->cpsr & PSR_MODE_BIT) );
797 printk("PC: %08"PRIx32"\n", regs->pc32);
798 #else
799 printk("PC: %08"PRIx32, regs->pc);
800 if ( !guest_mode )
801 printk(" %pS", _p(regs->pc));
802 printk("\n");
803 #endif
804 printk("CPSR: %08"PRIx32" MODE:%s\n", regs->cpsr,
805 mode_string(regs->cpsr));
806 printk(" R0: %08"PRIx32" R1: %08"PRIx32" R2: %08"PRIx32" R3: %08"PRIx32"\n",
807 regs->r0, regs->r1, regs->r2, regs->r3);
808 printk(" R4: %08"PRIx32" R5: %08"PRIx32" R6: %08"PRIx32" R7: %08"PRIx32"\n",
809 regs->r4, regs->r5, regs->r6, regs->r7);
810 printk(" R8: %08"PRIx32" R9: %08"PRIx32" R10:%08"PRIx32" R11:%08"PRIx32" R12:%08"PRIx32"\n",
811 regs->r8, regs->r9, regs->r10,
812 #ifdef CONFIG_ARM_64
813 regs->r11,
814 #else
815 regs->fp,
816 #endif
817 regs->r12);
818
819 if ( guest_mode )
820 {
821 printk("USR: SP: %08"PRIx32" LR: %08"PRIregister"\n",
822 regs->sp_usr, regs->lr);
823 printk("SVC: SP: %08"PRIx32" LR: %08"PRIx32" SPSR:%08"PRIx32"\n",
824 regs->sp_svc, regs->lr_svc, regs->spsr_svc);
825 printk("ABT: SP: %08"PRIx32" LR: %08"PRIx32" SPSR:%08"PRIx32"\n",
826 regs->sp_abt, regs->lr_abt, regs->spsr_abt);
827 printk("UND: SP: %08"PRIx32" LR: %08"PRIx32" SPSR:%08"PRIx32"\n",
828 regs->sp_und, regs->lr_und, regs->spsr_und);
829 printk("IRQ: SP: %08"PRIx32" LR: %08"PRIx32" SPSR:%08"PRIx32"\n",
830 regs->sp_irq, regs->lr_irq, regs->spsr_irq);
831 printk("FIQ: SP: %08"PRIx32" LR: %08"PRIx32" SPSR:%08"PRIx32"\n",
832 regs->sp_fiq, regs->lr_fiq, regs->spsr_fiq);
833 printk("FIQ: R8: %08"PRIx32" R9: %08"PRIx32" R10:%08"PRIx32" R11:%08"PRIx32" R12:%08"PRIx32"\n",
834 regs->r8_fiq, regs->r9_fiq, regs->r10_fiq, regs->r11_fiq, regs->r11_fiq);
835 }
836 #ifndef CONFIG_ARM_64
837 else
838 {
839 printk("HYP: SP: %08"PRIx32" LR: %08"PRIregister"\n", regs->sp, regs->lr);
840 }
841 #endif
842 printk("\n");
843
844 if ( guest_mode )
845 {
846 printk(" SCTLR: %08"PRIx32"\n", ctxt->sctlr_el1);
847 printk(" TCR: %08"PRIregister"\n", ctxt->tcr_el1);
848 printk(" TTBR0: %016"PRIx64"\n", ctxt->ttbr0_el1);
849 printk(" TTBR1: %016"PRIx64"\n", ctxt->ttbr1_el1);
850 printk(" IFAR: %08"PRIx32", IFSR: %08"PRIx32"\n"
851 " DFAR: %08"PRIx32", DFSR: %08"PRIx32"\n",
852 #ifdef CONFIG_ARM_64
853 (uint32_t)(ctxt->far >> 32),
854 ctxt->ifsr32_el2,
855 (uint32_t)(ctxt->far & 0xffffffff),
856 ctxt->esr_el1
857 #else
858 ctxt->ifar, ctxt->ifsr, ctxt->dfar, ctxt->dfsr
859 #endif
860 );
861 printk("\n");
862 }
863 }
864
865 #ifdef CONFIG_ARM_64
show_registers_64(struct cpu_user_regs * regs,struct reg_ctxt * ctxt,int guest_mode,const struct vcpu * v)866 static void show_registers_64(struct cpu_user_regs *regs,
867 struct reg_ctxt *ctxt,
868 int guest_mode,
869 const struct vcpu *v)
870 {
871
872 BUG_ON( (regs->cpsr & PSR_MODE_BIT) );
873
874 printk("PC: %016"PRIx64, regs->pc);
875 if ( !guest_mode )
876 printk(" %pS", _p(regs->pc));
877 printk("\n");
878 printk("LR: %016"PRIx64"\n", regs->lr);
879 if ( guest_mode )
880 {
881 printk("SP_EL0: %016"PRIx64"\n", regs->sp_el0);
882 printk("SP_EL1: %016"PRIx64"\n", regs->sp_el1);
883 }
884 else
885 {
886 printk("SP: %016"PRIx64"\n", regs->sp);
887 }
888 printk("CPSR: %08"PRIx32" MODE:%s\n", regs->cpsr,
889 mode_string(regs->cpsr));
890 printk(" X0: %016"PRIx64" X1: %016"PRIx64" X2: %016"PRIx64"\n",
891 regs->x0, regs->x1, regs->x2);
892 printk(" X3: %016"PRIx64" X4: %016"PRIx64" X5: %016"PRIx64"\n",
893 regs->x3, regs->x4, regs->x5);
894 printk(" X6: %016"PRIx64" X7: %016"PRIx64" X8: %016"PRIx64"\n",
895 regs->x6, regs->x7, regs->x8);
896 printk(" X9: %016"PRIx64" X10: %016"PRIx64" X11: %016"PRIx64"\n",
897 regs->x9, regs->x10, regs->x11);
898 printk(" X12: %016"PRIx64" X13: %016"PRIx64" X14: %016"PRIx64"\n",
899 regs->x12, regs->x13, regs->x14);
900 printk(" X15: %016"PRIx64" X16: %016"PRIx64" X17: %016"PRIx64"\n",
901 regs->x15, regs->x16, regs->x17);
902 printk(" X18: %016"PRIx64" X19: %016"PRIx64" X20: %016"PRIx64"\n",
903 regs->x18, regs->x19, regs->x20);
904 printk(" X21: %016"PRIx64" X22: %016"PRIx64" X23: %016"PRIx64"\n",
905 regs->x21, regs->x22, regs->x23);
906 printk(" X24: %016"PRIx64" X25: %016"PRIx64" X26: %016"PRIx64"\n",
907 regs->x24, regs->x25, regs->x26);
908 printk(" X27: %016"PRIx64" X28: %016"PRIx64" FP: %016"PRIx64"\n",
909 regs->x27, regs->x28, regs->fp);
910 printk("\n");
911
912 if ( guest_mode )
913 {
914 printk(" ELR_EL1: %016"PRIx64"\n", regs->elr_el1);
915 printk(" ESR_EL1: %08"PRIx32"\n", ctxt->esr_el1);
916 printk(" FAR_EL1: %016"PRIx64"\n", ctxt->far);
917 printk("\n");
918 printk(" SCTLR_EL1: %08"PRIx32"\n", ctxt->sctlr_el1);
919 printk(" TCR_EL1: %08"PRIregister"\n", ctxt->tcr_el1);
920 printk(" TTBR0_EL1: %016"PRIx64"\n", ctxt->ttbr0_el1);
921 printk(" TTBR1_EL1: %016"PRIx64"\n", ctxt->ttbr1_el1);
922 printk("\n");
923 }
924 }
925 #endif
926
_show_registers(struct cpu_user_regs * regs,struct reg_ctxt * ctxt,int guest_mode,const struct vcpu * v)927 static void _show_registers(struct cpu_user_regs *regs,
928 struct reg_ctxt *ctxt,
929 int guest_mode,
930 const struct vcpu *v)
931 {
932 print_xen_info();
933
934 printk("CPU: %d\n", smp_processor_id());
935
936 if ( guest_mode )
937 {
938 if ( is_32bit_domain(v->domain) )
939 show_registers_32(regs, ctxt, guest_mode, v);
940 #ifdef CONFIG_ARM_64
941 else if ( is_64bit_domain(v->domain) )
942 {
943 if ( psr_mode_is_32bit(regs->cpsr) )
944 {
945 BUG_ON(!usr_mode(regs));
946 show_registers_32(regs, ctxt, guest_mode, v);
947 }
948 else
949 {
950 show_registers_64(regs, ctxt, guest_mode, v);
951 }
952 }
953 #endif
954 }
955 else
956 {
957 #ifdef CONFIG_ARM_64
958 show_registers_64(regs, ctxt, guest_mode, v);
959 #else
960 show_registers_32(regs, ctxt, guest_mode, v);
961 #endif
962 }
963 printk(" VTCR_EL2: %08"PRIx32"\n", READ_SYSREG32(VTCR_EL2));
964 printk(" VTTBR_EL2: %016"PRIx64"\n", ctxt->vttbr_el2);
965 printk("\n");
966
967 printk(" SCTLR_EL2: %08"PRIx32"\n", READ_SYSREG32(SCTLR_EL2));
968 printk(" HCR_EL2: %016"PRIregister"\n", READ_SYSREG(HCR_EL2));
969 printk(" TTBR0_EL2: %016"PRIx64"\n", READ_SYSREG64(TTBR0_EL2));
970 printk("\n");
971 printk(" ESR_EL2: %08"PRIx32"\n", regs->hsr);
972 printk(" HPFAR_EL2: %016"PRIregister"\n", READ_SYSREG(HPFAR_EL2));
973
974 #ifdef CONFIG_ARM_32
975 printk(" HDFAR: %08"PRIx32"\n", READ_CP32(HDFAR));
976 printk(" HIFAR: %08"PRIx32"\n", READ_CP32(HIFAR));
977 #else
978 printk(" FAR_EL2: %016"PRIx64"\n", READ_SYSREG64(FAR_EL2));
979 #endif
980 printk("\n");
981 }
982
show_registers(struct cpu_user_regs * regs)983 void show_registers(struct cpu_user_regs *regs)
984 {
985 struct reg_ctxt ctxt;
986 ctxt.sctlr_el1 = READ_SYSREG(SCTLR_EL1);
987 ctxt.tcr_el1 = READ_SYSREG(TCR_EL1);
988 ctxt.ttbr0_el1 = READ_SYSREG64(TTBR0_EL1);
989 ctxt.ttbr1_el1 = READ_SYSREG64(TTBR1_EL1);
990 #ifdef CONFIG_ARM_32
991 ctxt.dfar = READ_CP32(DFAR);
992 ctxt.ifar = READ_CP32(IFAR);
993 ctxt.dfsr = READ_CP32(DFSR);
994 ctxt.ifsr = READ_CP32(IFSR);
995 #else
996 ctxt.far = READ_SYSREG(FAR_EL1);
997 ctxt.esr_el1 = READ_SYSREG(ESR_EL1);
998 if ( guest_mode(regs) && is_32bit_domain(current->domain) )
999 ctxt.ifsr32_el2 = READ_SYSREG(IFSR32_EL2);
1000 #endif
1001 ctxt.vttbr_el2 = READ_SYSREG64(VTTBR_EL2);
1002
1003 _show_registers(regs, &ctxt, guest_mode(regs), current);
1004 }
1005
vcpu_show_registers(const struct vcpu * v)1006 void vcpu_show_registers(const struct vcpu *v)
1007 {
1008 struct reg_ctxt ctxt;
1009 ctxt.sctlr_el1 = v->arch.sctlr;
1010 ctxt.tcr_el1 = v->arch.ttbcr;
1011 ctxt.ttbr0_el1 = v->arch.ttbr0;
1012 ctxt.ttbr1_el1 = v->arch.ttbr1;
1013 #ifdef CONFIG_ARM_32
1014 ctxt.dfar = v->arch.dfar;
1015 ctxt.ifar = v->arch.ifar;
1016 ctxt.dfsr = v->arch.dfsr;
1017 ctxt.ifsr = v->arch.ifsr;
1018 #else
1019 ctxt.far = v->arch.far;
1020 ctxt.esr_el1 = v->arch.esr;
1021 ctxt.ifsr32_el2 = v->arch.ifsr;
1022 #endif
1023
1024 ctxt.vttbr_el2 = v->domain->arch.p2m.vttbr;
1025
1026 _show_registers(&v->arch.cpu_info->guest_cpu_user_regs, &ctxt, 1, v);
1027 }
1028
show_guest_stack(struct vcpu * v,struct cpu_user_regs * regs)1029 static void show_guest_stack(struct vcpu *v, struct cpu_user_regs *regs)
1030 {
1031 int i;
1032 vaddr_t sp;
1033 struct page_info *page;
1034 void *mapped;
1035 unsigned long *stack, addr;
1036
1037 if ( test_bit(_VPF_down, &v->pause_flags) )
1038 {
1039 printk("No stack trace, VCPU offline\n");
1040 return;
1041 }
1042
1043 switch ( regs->cpsr & PSR_MODE_MASK )
1044 {
1045 case PSR_MODE_USR:
1046 case PSR_MODE_SYS:
1047 #ifdef CONFIG_ARM_64
1048 case PSR_MODE_EL0t:
1049 #endif
1050 printk("No stack trace for guest user-mode\n");
1051 return;
1052
1053 case PSR_MODE_FIQ:
1054 sp = regs->sp_fiq;
1055 break;
1056 case PSR_MODE_IRQ:
1057 sp = regs->sp_irq;
1058 break;
1059 case PSR_MODE_SVC:
1060 sp = regs->sp_svc;
1061 break;
1062 case PSR_MODE_ABT:
1063 sp = regs->sp_abt;
1064 break;
1065 case PSR_MODE_UND:
1066 sp = regs->sp_und;
1067 break;
1068
1069 #ifdef CONFIG_ARM_64
1070 case PSR_MODE_EL1t:
1071 sp = regs->sp_el0;
1072 break;
1073 case PSR_MODE_EL1h:
1074 sp = regs->sp_el1;
1075 break;
1076 #endif
1077
1078 case PSR_MODE_HYP:
1079 case PSR_MODE_MON:
1080 #ifdef CONFIG_ARM_64
1081 case PSR_MODE_EL3h:
1082 case PSR_MODE_EL3t:
1083 case PSR_MODE_EL2h:
1084 case PSR_MODE_EL2t:
1085 #endif
1086 default:
1087 BUG();
1088 return;
1089 }
1090
1091 printk("Guest stack trace from sp=%"PRIvaddr":\n ", sp);
1092
1093 if ( sp & ( sizeof(long) - 1 ) )
1094 {
1095 printk("Stack is misaligned\n");
1096 return;
1097 }
1098
1099 page = get_page_from_gva(v, sp, GV2M_READ);
1100 if ( page == NULL )
1101 {
1102 printk("Failed to convert stack to physical address\n");
1103 return;
1104 }
1105
1106 mapped = __map_domain_page(page);
1107
1108 stack = mapped + (sp & ~PAGE_MASK);
1109
1110 for ( i = 0; i < (debug_stack_lines*stack_words_per_line); i++ )
1111 {
1112 if ( (((long)stack - 1) ^ ((long)(stack + 1) - 1)) & PAGE_SIZE )
1113 break;
1114 addr = *stack;
1115 if ( (i != 0) && ((i % stack_words_per_line) == 0) )
1116 printk("\n ");
1117 printk(" %p", _p(addr));
1118 stack++;
1119 }
1120 if ( i == 0 )
1121 printk("Stack empty.");
1122 printk("\n");
1123 unmap_domain_page(mapped);
1124 put_page(page);
1125 }
1126
1127 #define STACK_BEFORE_EXCEPTION(regs) ((register_t*)(regs)->sp)
1128 #ifdef CONFIG_ARM_32
1129 /* Frame pointer points to the return address:
1130 * (largest address)
1131 * | cpu_info
1132 * | [...] |
1133 * | return addr <-----------------, |
1134 * | fp --------------------------------+----'
1135 * | [...] |
1136 * | return addr <------------, |
1137 * | fp ---------------------------+----'
1138 * | [...] |
1139 * | return addr <- regs->fp |
1140 * | fp ---------------------------'
1141 * |
1142 * v (smallest address, sp)
1143 */
1144 #define STACK_FRAME_BASE(fp) ((register_t*)(fp) - 1)
1145 #else
1146 /* Frame pointer points to the next frame:
1147 * (largest address)
1148 * | cpu_info
1149 * | [...] |
1150 * | return addr |
1151 * | fp <-------------------------------, >--'
1152 * | [...] |
1153 * | return addr |
1154 * | fp <--------------------------, >--'
1155 * | [...] |
1156 * | return addr <- regs->fp |
1157 * | fp ---------------------------'
1158 * |
1159 * v (smallest address, sp)
1160 */
1161 #define STACK_FRAME_BASE(fp) ((register_t*)(fp))
1162 #endif
show_trace(struct cpu_user_regs * regs)1163 static void show_trace(struct cpu_user_regs *regs)
1164 {
1165 register_t *frame, next, addr, low, high;
1166
1167 printk("Xen call trace:\n");
1168
1169 printk(" [<%p>] %pS (PC)\n", _p(regs->pc), _p(regs->pc));
1170 printk(" [<%p>] %pS (LR)\n", _p(regs->lr), _p(regs->lr));
1171
1172 /* Bounds for range of valid frame pointer. */
1173 low = (register_t)(STACK_BEFORE_EXCEPTION(regs));
1174 high = (low & ~(STACK_SIZE - 1)) +
1175 (STACK_SIZE - sizeof(struct cpu_info));
1176
1177 /* The initial frame pointer. */
1178 next = regs->fp;
1179
1180 for ( ; ; )
1181 {
1182 if ( (next < low) || (next >= high) )
1183 break;
1184
1185 /* Ordinary stack frame. */
1186 frame = STACK_FRAME_BASE(next);
1187 next = frame[0];
1188 addr = frame[1];
1189
1190 printk(" [<%p>] %pS\n", _p(addr), _p(addr));
1191
1192 low = (register_t)&frame[1];
1193 }
1194
1195 printk("\n");
1196 }
1197
show_stack(struct cpu_user_regs * regs)1198 void show_stack(struct cpu_user_regs *regs)
1199 {
1200 register_t *stack = STACK_BEFORE_EXCEPTION(regs), addr;
1201 int i;
1202
1203 if ( guest_mode(regs) )
1204 return show_guest_stack(current, regs);
1205
1206 printk("Xen stack trace from sp=%p:\n ", stack);
1207
1208 for ( i = 0; i < (debug_stack_lines*stack_words_per_line); i++ )
1209 {
1210 if ( ((long)stack & (STACK_SIZE-BYTES_PER_LONG)) == 0 )
1211 break;
1212 if ( (i != 0) && ((i % stack_words_per_line) == 0) )
1213 printk("\n ");
1214
1215 addr = *stack++;
1216 printk(" %p", _p(addr));
1217 }
1218 if ( i == 0 )
1219 printk("Stack empty.");
1220 printk("\n");
1221
1222 show_trace(regs);
1223 }
1224
show_execution_state(struct cpu_user_regs * regs)1225 void show_execution_state(struct cpu_user_regs *regs)
1226 {
1227 show_registers(regs);
1228 show_stack(regs);
1229 }
1230
vcpu_show_execution_state(struct vcpu * v)1231 void vcpu_show_execution_state(struct vcpu *v)
1232 {
1233 printk("*** Dumping Dom%d vcpu#%d state: ***\n",
1234 v->domain->domain_id, v->vcpu_id);
1235
1236 if ( v == current )
1237 {
1238 show_execution_state(guest_cpu_user_regs());
1239 return;
1240 }
1241
1242 vcpu_pause(v); /* acceptably dangerous */
1243
1244 vcpu_show_registers(v);
1245 if ( !psr_mode_is_user(&v->arch.cpu_info->guest_cpu_user_regs) )
1246 show_guest_stack(v, &v->arch.cpu_info->guest_cpu_user_regs);
1247
1248 vcpu_unpause(v);
1249 }
1250
do_unexpected_trap(const char * msg,struct cpu_user_regs * regs)1251 void do_unexpected_trap(const char *msg, struct cpu_user_regs *regs)
1252 {
1253 printk("CPU%d: Unexpected Trap: %s\n", smp_processor_id(), msg);
1254 show_execution_state(regs);
1255 panic("CPU%d: Unexpected Trap: %s\n", smp_processor_id(), msg);
1256 }
1257
do_bug_frame(struct cpu_user_regs * regs,vaddr_t pc)1258 int do_bug_frame(struct cpu_user_regs *regs, vaddr_t pc)
1259 {
1260 const struct bug_frame *bug = NULL;
1261 const char *prefix = "", *filename, *predicate;
1262 unsigned long fixup;
1263 int id = -1, lineno;
1264 const struct virtual_region *region;
1265
1266 region = find_text_region(pc);
1267 if ( region )
1268 {
1269 for ( id = 0; id < BUGFRAME_NR; id++ )
1270 {
1271 const struct bug_frame *b;
1272 unsigned int i;
1273
1274 for ( i = 0, b = region->frame[id].bugs;
1275 i < region->frame[id].n_bugs; b++, i++ )
1276 {
1277 if ( ((vaddr_t)bug_loc(b)) == pc )
1278 {
1279 bug = b;
1280 goto found;
1281 }
1282 }
1283 }
1284 }
1285 found:
1286 if ( !bug )
1287 return -ENOENT;
1288
1289 /* WARN, BUG or ASSERT: decode the filename pointer and line number. */
1290 filename = bug_file(bug);
1291 if ( !is_kernel(filename) )
1292 return -EINVAL;
1293 fixup = strlen(filename);
1294 if ( fixup > 50 )
1295 {
1296 filename += fixup - 47;
1297 prefix = "...";
1298 }
1299 lineno = bug_line(bug);
1300
1301 switch ( id )
1302 {
1303 case BUGFRAME_warn:
1304 printk("Xen WARN at %s%s:%d\n", prefix, filename, lineno);
1305 show_execution_state(regs);
1306 return 0;
1307
1308 case BUGFRAME_bug:
1309 printk("Xen BUG at %s%s:%d\n", prefix, filename, lineno);
1310
1311 if ( debugger_trap_fatal(TRAP_invalid_op, regs) )
1312 return 0;
1313
1314 show_execution_state(regs);
1315 panic("Xen BUG at %s%s:%d", prefix, filename, lineno);
1316
1317 case BUGFRAME_assert:
1318 /* ASSERT: decode the predicate string pointer. */
1319 predicate = bug_msg(bug);
1320 if ( !is_kernel(predicate) )
1321 predicate = "<unknown>";
1322
1323 printk("Assertion '%s' failed at %s%s:%d\n",
1324 predicate, prefix, filename, lineno);
1325 if ( debugger_trap_fatal(TRAP_invalid_op, regs) )
1326 return 0;
1327 show_execution_state(regs);
1328 panic("Assertion '%s' failed at %s%s:%d",
1329 predicate, prefix, filename, lineno);
1330 }
1331
1332 return -EINVAL;
1333 }
1334
1335 #ifdef CONFIG_ARM_64
do_trap_brk(struct cpu_user_regs * regs,const union hsr hsr)1336 static void do_trap_brk(struct cpu_user_regs *regs, const union hsr hsr)
1337 {
1338 /* HCR_EL2.TGE and MDCR_EL2.TDE are not set so we never receive
1339 * software breakpoint exception for EL1 and EL0 here.
1340 */
1341 BUG_ON(!hyp_mode(regs));
1342
1343 switch (hsr.brk.comment)
1344 {
1345 case BRK_BUG_FRAME_IMM:
1346 if ( do_bug_frame(regs, regs->pc) )
1347 goto die;
1348
1349 regs->pc += 4;
1350
1351 break;
1352
1353 default:
1354 die:
1355 do_unexpected_trap("Undefined Breakpoint Value", regs);
1356 }
1357 }
1358 #endif
1359
do_deprecated_hypercall(void)1360 static register_t do_deprecated_hypercall(void)
1361 {
1362 struct cpu_user_regs *regs = guest_cpu_user_regs();
1363 const register_t op =
1364 #ifdef CONFIG_ARM_64
1365 !is_32bit_domain(current->domain) ?
1366 regs->x16
1367 :
1368 #endif
1369 regs->r12;
1370
1371 gdprintk(XENLOG_DEBUG, "%pv: deprecated hypercall %lu\n",
1372 current, (unsigned long)op);
1373 return -ENOSYS;
1374 }
1375
1376 typedef register_t (*arm_hypercall_fn_t)(
1377 register_t, register_t, register_t, register_t, register_t);
1378
1379 typedef struct {
1380 arm_hypercall_fn_t fn;
1381 int nr_args;
1382 } arm_hypercall_t;
1383
1384 #define HYPERCALL(_name, _nr_args) \
1385 [ __HYPERVISOR_ ## _name ] = { \
1386 .fn = (arm_hypercall_fn_t) &do_ ## _name, \
1387 .nr_args = _nr_args, \
1388 }
1389
1390 #define HYPERCALL_ARM(_name, _nr_args) \
1391 [ __HYPERVISOR_ ## _name ] = { \
1392 .fn = (arm_hypercall_fn_t) &do_arm_ ## _name, \
1393 .nr_args = _nr_args, \
1394 }
1395 /*
1396 * Only use this for hypercalls which were deprecated (i.e. replaced
1397 * by something else) before Xen on ARM was created, i.e. *not* for
1398 * hypercalls which are simply not yet used on ARM.
1399 */
1400 #define HYPERCALL_DEPRECATED(_name, _nr_args) \
1401 [ __HYPERVISOR_##_name ] = { \
1402 .fn = (arm_hypercall_fn_t) &do_deprecated_hypercall, \
1403 .nr_args = _nr_args, \
1404 }
1405
1406 static arm_hypercall_t arm_hypercall_table[] = {
1407 HYPERCALL(memory_op, 2),
1408 HYPERCALL(domctl, 1),
1409 HYPERCALL(sched_op, 2),
1410 HYPERCALL_DEPRECATED(sched_op_compat, 2),
1411 HYPERCALL(console_io, 3),
1412 HYPERCALL(xen_version, 2),
1413 HYPERCALL(xsm_op, 1),
1414 HYPERCALL(event_channel_op, 2),
1415 HYPERCALL_DEPRECATED(event_channel_op_compat, 1),
1416 HYPERCALL(physdev_op, 2),
1417 HYPERCALL_DEPRECATED(physdev_op_compat, 1),
1418 HYPERCALL(sysctl, 2),
1419 HYPERCALL(hvm_op, 2),
1420 HYPERCALL(grant_table_op, 3),
1421 HYPERCALL(multicall, 2),
1422 HYPERCALL(platform_op, 1),
1423 HYPERCALL_ARM(vcpu_op, 3),
1424 HYPERCALL(vm_assist, 2),
1425 };
1426
1427 #ifndef NDEBUG
do_debug_trap(struct cpu_user_regs * regs,unsigned int code)1428 static void do_debug_trap(struct cpu_user_regs *regs, unsigned int code)
1429 {
1430 uint32_t reg;
1431 uint32_t domid = current->domain->domain_id;
1432 switch ( code ) {
1433 case 0xe0 ... 0xef:
1434 reg = code - 0xe0;
1435 printk("DOM%d: R%d = 0x%"PRIregister" at 0x%"PRIvaddr"\n",
1436 domid, reg, get_user_reg(regs, reg), regs->pc);
1437 break;
1438 case 0xfd:
1439 printk("DOM%d: Reached %"PRIvaddr"\n", domid, regs->pc);
1440 break;
1441 case 0xfe:
1442 printk("%c", (char)(get_user_reg(regs, 0) & 0xff));
1443 break;
1444 case 0xff:
1445 printk("DOM%d: DEBUG\n", domid);
1446 show_execution_state(regs);
1447 break;
1448 default:
1449 panic("DOM%d: Unhandled debug trap %#x", domid, code);
1450 break;
1451 }
1452 }
1453 #endif
1454
1455 #ifdef CONFIG_ARM_64
1456 #define HYPERCALL_RESULT_REG(r) (r)->x0
1457 #define HYPERCALL_ARG1(r) (r)->x0
1458 #define HYPERCALL_ARG2(r) (r)->x1
1459 #define HYPERCALL_ARG3(r) (r)->x2
1460 #define HYPERCALL_ARG4(r) (r)->x3
1461 #define HYPERCALL_ARG5(r) (r)->x4
1462 #define HYPERCALL_ARGS(r) (r)->x0, (r)->x1, (r)->x2, (r)->x3, (r)->x4
1463 #else
1464 #define HYPERCALL_RESULT_REG(r) (r)->r0
1465 #define HYPERCALL_ARG1(r) (r)->r0
1466 #define HYPERCALL_ARG2(r) (r)->r1
1467 #define HYPERCALL_ARG3(r) (r)->r2
1468 #define HYPERCALL_ARG4(r) (r)->r3
1469 #define HYPERCALL_ARG5(r) (r)->r4
1470 #define HYPERCALL_ARGS(r) (r)->r0, (r)->r1, (r)->r2, (r)->r3, (r)->r4
1471 #endif
1472
do_trap_hypercall(struct cpu_user_regs * regs,register_t * nr,unsigned long iss)1473 static void do_trap_hypercall(struct cpu_user_regs *regs, register_t *nr,
1474 unsigned long iss)
1475 {
1476 arm_hypercall_fn_t call = NULL;
1477
1478 BUILD_BUG_ON(NR_hypercalls < ARRAY_SIZE(arm_hypercall_table) );
1479
1480 if ( iss != XEN_HYPERCALL_TAG )
1481 domain_crash_synchronous();
1482
1483 if ( *nr >= ARRAY_SIZE(arm_hypercall_table) )
1484 {
1485 perfc_incr(invalid_hypercalls);
1486 HYPERCALL_RESULT_REG(regs) = -ENOSYS;
1487 return;
1488 }
1489
1490 current->hcall_preempted = false;
1491
1492 perfc_incra(hypercalls, *nr);
1493 call = arm_hypercall_table[*nr].fn;
1494 if ( call == NULL )
1495 {
1496 HYPERCALL_RESULT_REG(regs) = -ENOSYS;
1497 return;
1498 }
1499
1500 HYPERCALL_RESULT_REG(regs) = call(HYPERCALL_ARGS(regs));
1501
1502 #ifndef NDEBUG
1503 if ( !current->hcall_preempted )
1504 {
1505 /* Deliberately corrupt parameter regs used by this hypercall. */
1506 switch ( arm_hypercall_table[*nr].nr_args ) {
1507 case 5: HYPERCALL_ARG5(regs) = 0xDEADBEEF;
1508 case 4: HYPERCALL_ARG4(regs) = 0xDEADBEEF;
1509 case 3: HYPERCALL_ARG3(regs) = 0xDEADBEEF;
1510 case 2: HYPERCALL_ARG2(regs) = 0xDEADBEEF;
1511 case 1: /* Don't clobber x0/r0 -- it's the return value */
1512 break;
1513 default: BUG();
1514 }
1515 *nr = 0xDEADBEEF;
1516 }
1517 #endif
1518
1519 /* Ensure the hypercall trap instruction is re-executed. */
1520 if ( current->hcall_preempted )
1521 regs->pc -= 4; /* re-execute 'hvc #XEN_HYPERCALL_TAG' */
1522 }
1523
check_multicall_32bit_clean(struct multicall_entry * multi)1524 static bool check_multicall_32bit_clean(struct multicall_entry *multi)
1525 {
1526 int i;
1527
1528 for ( i = 0; i < arm_hypercall_table[multi->op].nr_args; i++ )
1529 {
1530 if ( unlikely(multi->args[i] & 0xffffffff00000000ULL) )
1531 {
1532 printk("%pv: multicall argument %d is not 32-bit clean %"PRIx64"\n",
1533 current, i, multi->args[i]);
1534 domain_crash(current->domain);
1535 return false;
1536 }
1537 }
1538
1539 return true;
1540 }
1541
arch_do_multicall_call(struct mc_state * state)1542 enum mc_disposition arch_do_multicall_call(struct mc_state *state)
1543 {
1544 struct multicall_entry *multi = &state->call;
1545 arm_hypercall_fn_t call = NULL;
1546
1547 if ( multi->op >= ARRAY_SIZE(arm_hypercall_table) )
1548 {
1549 multi->result = -ENOSYS;
1550 return mc_continue;
1551 }
1552
1553 call = arm_hypercall_table[multi->op].fn;
1554 if ( call == NULL )
1555 {
1556 multi->result = -ENOSYS;
1557 return mc_continue;
1558 }
1559
1560 if ( is_32bit_domain(current->domain) &&
1561 !check_multicall_32bit_clean(multi) )
1562 return mc_continue;
1563
1564 multi->result = call(multi->args[0], multi->args[1],
1565 multi->args[2], multi->args[3],
1566 multi->args[4]);
1567
1568 return likely(!psr_mode_is_user(guest_cpu_user_regs()))
1569 ? mc_continue : mc_preempt;
1570 }
1571
1572 /*
1573 * stolen from arch/arm/kernel/opcodes.c
1574 *
1575 * condition code lookup table
1576 * index into the table is test code: EQ, NE, ... LT, GT, AL, NV
1577 *
1578 * bit position in short is condition code: NZCV
1579 */
1580 static const unsigned short cc_map[16] = {
1581 0xF0F0, /* EQ == Z set */
1582 0x0F0F, /* NE */
1583 0xCCCC, /* CS == C set */
1584 0x3333, /* CC */
1585 0xFF00, /* MI == N set */
1586 0x00FF, /* PL */
1587 0xAAAA, /* VS == V set */
1588 0x5555, /* VC */
1589 0x0C0C, /* HI == C set && Z clear */
1590 0xF3F3, /* LS == C clear || Z set */
1591 0xAA55, /* GE == (N==V) */
1592 0x55AA, /* LT == (N!=V) */
1593 0x0A05, /* GT == (!Z && (N==V)) */
1594 0xF5FA, /* LE == (Z || (N!=V)) */
1595 0xFFFF, /* AL always */
1596 0 /* NV */
1597 };
1598
check_conditional_instr(struct cpu_user_regs * regs,const union hsr hsr)1599 int check_conditional_instr(struct cpu_user_regs *regs, const union hsr hsr)
1600 {
1601 unsigned long cpsr, cpsr_cond;
1602 int cond;
1603
1604 /*
1605 * SMC32 instruction case is special. Under SMC32 we mean SMC
1606 * instruction on ARMv7 or SMC instruction originating from
1607 * AArch32 state on ARMv8.
1608 * On ARMv7 it will be trapped only if it passed condition check
1609 * (ARM DDI 0406C.c page B3-1431), but we need to check condition
1610 * flags on ARMv8 (ARM DDI 0487B.a page D7-2271).
1611 * Encoding for HSR.ISS on ARMv8 is backwards compatible with ARMv7:
1612 * HSR.ISS is defined as UNK/SBZP on ARMv7 which means, that it
1613 * will be read as 0. This includes CCKNOWNPASS field.
1614 * If CCKNOWNPASS == 0 then this was an unconditional instruction or
1615 * it has passed conditional check (ARM DDI 0487B.a page D7-2272).
1616 */
1617 if ( hsr.ec == HSR_EC_SMC32 && hsr.smc32.ccknownpass == 0 )
1618 return 1;
1619
1620 /* Unconditional Exception classes */
1621 if ( hsr.ec == HSR_EC_UNKNOWN ||
1622 (hsr.ec >= 0x10 && hsr.ec != HSR_EC_SMC32) )
1623 return 1;
1624
1625 /* Check for valid condition in hsr */
1626 cond = hsr.cond.ccvalid ? hsr.cond.cc : -1;
1627
1628 /* Unconditional instruction */
1629 if ( cond == 0xe )
1630 return 1;
1631
1632 cpsr = regs->cpsr;
1633
1634 /* If cc is not valid then we need to examine the IT state */
1635 if ( cond < 0 )
1636 {
1637 unsigned long it;
1638
1639 BUG_ON( !psr_mode_is_32bit(regs->cpsr) || !(cpsr&PSR_THUMB) );
1640
1641 it = ( (cpsr >> (10-2)) & 0xfc) | ((cpsr >> 25) & 0x3 );
1642
1643 /* it == 0 => unconditional. */
1644 if ( it == 0 )
1645 return 1;
1646
1647 /* The cond for this instruction works out as the top 4 bits. */
1648 cond = ( it >> 4 );
1649 }
1650
1651 cpsr_cond = cpsr >> 28;
1652
1653 if ( !((cc_map[cond] >> cpsr_cond) & 1) )
1654 {
1655 perfc_incr(trap_uncond);
1656 return 0;
1657 }
1658 return 1;
1659 }
1660
advance_pc(struct cpu_user_regs * regs,const union hsr hsr)1661 void advance_pc(struct cpu_user_regs *regs, const union hsr hsr)
1662 {
1663 unsigned long itbits, cond, cpsr = regs->cpsr;
1664
1665 /* PSR_IT_MASK bits can only be set for 32-bit processors in Thumb mode. */
1666 BUG_ON( (!psr_mode_is_32bit(cpsr)||!(cpsr&PSR_THUMB))
1667 && (cpsr&PSR_IT_MASK) );
1668
1669 if ( cpsr&PSR_IT_MASK )
1670 {
1671 /* The ITSTATE[7:0] block is contained in CPSR[15:10],CPSR[26:25]
1672 *
1673 * ITSTATE[7:5] are the condition code
1674 * ITSTATE[4:0] are the IT bits
1675 *
1676 * If the condition is non-zero then the IT state machine is
1677 * advanced by shifting the IT bits left.
1678 *
1679 * See A2-51 and B1-1148 of DDI 0406C.b.
1680 */
1681 cond = (cpsr & 0xe000) >> 13;
1682 itbits = (cpsr & 0x1c00) >> (10 - 2);
1683 itbits |= (cpsr & (0x3 << 25)) >> 25;
1684
1685 if ( (itbits & 0x7) == 0 )
1686 itbits = cond = 0;
1687 else
1688 itbits = (itbits << 1) & 0x1f;
1689
1690 cpsr &= ~PSR_IT_MASK;
1691 cpsr |= cond << 13;
1692 cpsr |= (itbits & 0x1c) << (10 - 2);
1693 cpsr |= (itbits & 0x3) << 25;
1694
1695 regs->cpsr = cpsr;
1696 }
1697
1698 regs->pc += hsr.len ? 4 : 2;
1699 }
1700
1701 /* Read as zero and write ignore */
handle_raz_wi(struct cpu_user_regs * regs,int regidx,bool read,const union hsr hsr,int min_el)1702 void handle_raz_wi(struct cpu_user_regs *regs,
1703 int regidx,
1704 bool read,
1705 const union hsr hsr,
1706 int min_el)
1707 {
1708 ASSERT((min_el == 0) || (min_el == 1));
1709
1710 if ( min_el > 0 && psr_mode_is_user(regs) )
1711 return inject_undef_exception(regs, hsr);
1712
1713 if ( read )
1714 set_user_reg(regs, regidx, 0);
1715 /* else: write ignored */
1716
1717 advance_pc(regs, hsr);
1718 }
1719
1720 /* write only as write ignore */
handle_wo_wi(struct cpu_user_regs * regs,int regidx,bool read,const union hsr hsr,int min_el)1721 void handle_wo_wi(struct cpu_user_regs *regs,
1722 int regidx,
1723 bool read,
1724 const union hsr hsr,
1725 int min_el)
1726 {
1727 ASSERT((min_el == 0) || (min_el == 1));
1728
1729 if ( min_el > 0 && psr_mode_is_user(regs) )
1730 return inject_undef_exception(regs, hsr);
1731
1732 if ( read )
1733 return inject_undef_exception(regs, hsr);
1734 /* else: ignore */
1735
1736 advance_pc(regs, hsr);
1737 }
1738
1739 /* Read only as read as zero */
handle_ro_raz(struct cpu_user_regs * regs,int regidx,bool read,const union hsr hsr,int min_el)1740 void handle_ro_raz(struct cpu_user_regs *regs,
1741 int regidx,
1742 bool read,
1743 const union hsr hsr,
1744 int min_el)
1745 {
1746 ASSERT((min_el == 0) || (min_el == 1));
1747
1748 if ( min_el > 0 && psr_mode_is_user(regs) )
1749 return inject_undef_exception(regs, hsr);
1750
1751 if ( !read )
1752 return inject_undef_exception(regs, hsr);
1753 /* else: raz */
1754
1755 set_user_reg(regs, regidx, 0);
1756
1757 advance_pc(regs, hsr);
1758 }
1759
dump_guest_s1_walk(struct domain * d,vaddr_t addr)1760 void dump_guest_s1_walk(struct domain *d, vaddr_t addr)
1761 {
1762 register_t ttbcr = READ_SYSREG(TCR_EL1);
1763 uint64_t ttbr0 = READ_SYSREG64(TTBR0_EL1);
1764 uint32_t offset;
1765 uint32_t *first = NULL, *second = NULL;
1766 mfn_t mfn;
1767
1768 mfn = gfn_to_mfn(d, gaddr_to_gfn(ttbr0));
1769
1770 printk("dom%d VA 0x%08"PRIvaddr"\n", d->domain_id, addr);
1771 printk(" TTBCR: 0x%08"PRIregister"\n", ttbcr);
1772 printk(" TTBR0: 0x%016"PRIx64" = 0x%"PRIpaddr"\n",
1773 ttbr0, mfn_to_maddr(mfn));
1774
1775 if ( ttbcr & TTBCR_EAE )
1776 {
1777 printk("Cannot handle LPAE guest PT walk\n");
1778 return;
1779 }
1780 if ( (ttbcr & TTBCR_N_MASK) != 0 )
1781 {
1782 printk("Cannot handle TTBR1 guest walks\n");
1783 return;
1784 }
1785
1786 if ( mfn_eq(mfn, INVALID_MFN) )
1787 {
1788 printk("Failed TTBR0 maddr lookup\n");
1789 goto done;
1790 }
1791 first = map_domain_page(mfn);
1792
1793 offset = addr >> (12+8);
1794 printk("1ST[0x%"PRIx32"] (0x%"PRIpaddr") = 0x%08"PRIx32"\n",
1795 offset, mfn_to_maddr(mfn), first[offset]);
1796 if ( !(first[offset] & 0x1) ||
1797 (first[offset] & 0x2) )
1798 goto done;
1799
1800 mfn = gfn_to_mfn(d, gaddr_to_gfn(first[offset]));
1801
1802 if ( mfn_eq(mfn, INVALID_MFN) )
1803 {
1804 printk("Failed L1 entry maddr lookup\n");
1805 goto done;
1806 }
1807 second = map_domain_page(mfn);
1808 offset = (addr >> 12) & 0x3FF;
1809 printk("2ND[0x%"PRIx32"] (0x%"PRIpaddr") = 0x%08"PRIx32"\n",
1810 offset, mfn_to_maddr(mfn), second[offset]);
1811
1812 done:
1813 if (second) unmap_domain_page(second);
1814 if (first) unmap_domain_page(first);
1815 }
1816
1817 /*
1818 * Return the value of the hypervisor fault address register.
1819 *
1820 * On ARM32, the register will be different depending whether the
1821 * fault is a prefetch abort or data abort.
1822 */
get_hfar(bool is_data)1823 static inline vaddr_t get_hfar(bool is_data)
1824 {
1825 vaddr_t gva;
1826
1827 #ifdef CONFIG_ARM_32
1828 if ( is_data )
1829 gva = READ_CP32(HDFAR);
1830 else
1831 gva = READ_CP32(HIFAR);
1832 #else
1833 gva = READ_SYSREG(FAR_EL2);
1834 #endif
1835
1836 return gva;
1837 }
1838
get_faulting_ipa(vaddr_t gva)1839 static inline paddr_t get_faulting_ipa(vaddr_t gva)
1840 {
1841 register_t hpfar = READ_SYSREG(HPFAR_EL2);
1842 paddr_t ipa;
1843
1844 ipa = (paddr_t)(hpfar & HPFAR_MASK) << (12 - 4);
1845 ipa |= gva & ~PAGE_MASK;
1846
1847 return ipa;
1848 }
1849
hpfar_is_valid(bool s1ptw,uint8_t fsc)1850 static inline bool hpfar_is_valid(bool s1ptw, uint8_t fsc)
1851 {
1852 /*
1853 * HPFAR is valid if one of the following cases are true:
1854 * 1. the stage 2 fault happen during a stage 1 page table walk
1855 * (the bit ESR_EL2.S1PTW is set)
1856 * 2. the fault was due to a translation fault and the processor
1857 * does not carry erratum #8342220
1858 *
1859 * Note that technically HPFAR is valid for other cases, but they
1860 * are currently not supported by Xen.
1861 */
1862 return s1ptw || (fsc == FSC_FLT_TRANS && !check_workaround_834220());
1863 }
1864
do_trap_instr_abort_guest(struct cpu_user_regs * regs,const union hsr hsr)1865 static void do_trap_instr_abort_guest(struct cpu_user_regs *regs,
1866 const union hsr hsr)
1867 {
1868 int rc;
1869 register_t gva;
1870 uint8_t fsc = hsr.iabt.ifsc & ~FSC_LL_MASK;
1871 paddr_t gpa;
1872 mfn_t mfn;
1873
1874 gva = get_hfar(false /* is_data */);
1875
1876 /*
1877 * If this bit has been set, it means that this instruction abort is caused
1878 * by a guest external abort. We can handle this instruction abort as guest
1879 * SError.
1880 */
1881 if ( hsr.iabt.eat )
1882 return __do_trap_serror(regs, true);
1883
1884
1885 if ( hpfar_is_valid(hsr.iabt.s1ptw, fsc) )
1886 gpa = get_faulting_ipa(gva);
1887 else
1888 {
1889 /*
1890 * Flush the TLB to make sure the DTLB is clear before
1891 * doing GVA->IPA translation. If we got here because of
1892 * an entry only present in the ITLB, this translation may
1893 * still be inaccurate.
1894 */
1895 flush_tlb_local();
1896
1897 /*
1898 * We may not be able to translate because someone is
1899 * playing with the Stage-2 page table of the domain.
1900 * Return to the guest.
1901 */
1902 rc = gva_to_ipa(gva, &gpa, GV2M_READ);
1903 if ( rc == -EFAULT )
1904 return; /* Try again */
1905 }
1906
1907 switch ( fsc )
1908 {
1909 case FSC_FLT_PERM:
1910 {
1911 const struct npfec npfec = {
1912 .insn_fetch = 1,
1913 .gla_valid = 1,
1914 .kind = hsr.iabt.s1ptw ? npfec_kind_in_gpt : npfec_kind_with_gla
1915 };
1916
1917 p2m_mem_access_check(gpa, gva, npfec);
1918 /*
1919 * The only way to get here right now is because of mem_access,
1920 * thus reinjecting the exception to the guest is never required.
1921 */
1922 return;
1923 }
1924 case FSC_FLT_TRANS:
1925 /*
1926 * The PT walk may have failed because someone was playing
1927 * with the Stage-2 page table. Walk the Stage-2 PT to check
1928 * if the entry exists. If it's the case, return to the guest
1929 */
1930 mfn = gfn_to_mfn(current->domain, _gfn(paddr_to_pfn(gpa)));
1931 if ( !mfn_eq(mfn, INVALID_MFN) )
1932 return;
1933 }
1934
1935 inject_iabt_exception(regs, gva, hsr.len);
1936 }
1937
try_handle_mmio(struct cpu_user_regs * regs,mmio_info_t * info)1938 static bool try_handle_mmio(struct cpu_user_regs *regs,
1939 mmio_info_t *info)
1940 {
1941 const struct hsr_dabt dabt = info->dabt;
1942 int rc;
1943
1944 /* stage-1 page table should never live in an emulated MMIO region */
1945 if ( dabt.s1ptw )
1946 return false;
1947
1948 /* All the instructions used on emulated MMIO region should be valid */
1949 if ( !dabt.valid )
1950 return false;
1951
1952 /*
1953 * Erratum 766422: Thumb store translation fault to Hypervisor may
1954 * not have correct HSR Rt value.
1955 */
1956 if ( check_workaround_766422() && (regs->cpsr & PSR_THUMB) &&
1957 dabt.write )
1958 {
1959 rc = decode_instruction(regs, &info->dabt);
1960 if ( rc )
1961 {
1962 gprintk(XENLOG_DEBUG, "Unable to decode instruction\n");
1963 return false;
1964 }
1965 }
1966
1967 return !!handle_mmio(info);
1968 }
1969
1970 /*
1971 * When using ACPI, most of the MMIO regions will be mapped on-demand
1972 * in stage-2 page tables for the hardware domain because Xen is not
1973 * able to know from the EFI memory map the MMIO regions.
1974 */
try_map_mmio(gfn_t gfn)1975 static bool try_map_mmio(gfn_t gfn)
1976 {
1977 struct domain *d = current->domain;
1978
1979 /* For the hardware domain, all MMIOs are mapped with GFN == MFN */
1980 mfn_t mfn = _mfn(gfn_x(gfn));
1981
1982 /*
1983 * Device-Tree should already have everything mapped when building
1984 * the hardware domain.
1985 */
1986 if ( acpi_disabled )
1987 return false;
1988
1989 if ( !is_hardware_domain(d) )
1990 return false;
1991
1992 /* The hardware domain can only map permitted MMIO regions */
1993 if ( !iomem_access_permitted(d, mfn_x(mfn), mfn_x(mfn) + 1) )
1994 return false;
1995
1996 return !map_regions_p2mt(d, gfn, 1, mfn, p2m_mmio_direct_c);
1997 }
1998
do_trap_data_abort_guest(struct cpu_user_regs * regs,const union hsr hsr)1999 static void do_trap_data_abort_guest(struct cpu_user_regs *regs,
2000 const union hsr hsr)
2001 {
2002 const struct hsr_dabt dabt = hsr.dabt;
2003 int rc;
2004 mmio_info_t info;
2005 uint8_t fsc = hsr.dabt.dfsc & ~FSC_LL_MASK;
2006 mfn_t mfn;
2007
2008 /*
2009 * If this bit has been set, it means that this data abort is caused
2010 * by a guest external abort. We treat this data abort as guest SError.
2011 */
2012 if ( dabt.eat )
2013 return __do_trap_serror(regs, true);
2014
2015 info.dabt = dabt;
2016
2017 info.gva = get_hfar(true /* is_data */);
2018
2019 if ( hpfar_is_valid(dabt.s1ptw, fsc) )
2020 info.gpa = get_faulting_ipa(info.gva);
2021 else
2022 {
2023 rc = gva_to_ipa(info.gva, &info.gpa, GV2M_READ);
2024 /*
2025 * We may not be able to translate because someone is
2026 * playing with the Stage-2 page table of the domain.
2027 * Return to the guest.
2028 */
2029 if ( rc == -EFAULT )
2030 return; /* Try again */
2031 }
2032
2033 switch ( fsc )
2034 {
2035 case FSC_FLT_PERM:
2036 {
2037 const struct npfec npfec = {
2038 .read_access = !dabt.write,
2039 .write_access = dabt.write,
2040 .gla_valid = 1,
2041 .kind = dabt.s1ptw ? npfec_kind_in_gpt : npfec_kind_with_gla
2042 };
2043
2044 p2m_mem_access_check(info.gpa, info.gva, npfec);
2045 /*
2046 * The only way to get here right now is because of mem_access,
2047 * thus reinjecting the exception to the guest is never required.
2048 */
2049 return;
2050 }
2051 case FSC_FLT_TRANS:
2052 /*
2053 * Attempt first to emulate the MMIO as the data abort will
2054 * likely happen in an emulated region.
2055 */
2056 if ( try_handle_mmio(regs, &info) )
2057 {
2058 advance_pc(regs, hsr);
2059 return;
2060 }
2061
2062 /*
2063 * The PT walk may have failed because someone was playing
2064 * with the Stage-2 page table. Walk the Stage-2 PT to check
2065 * if the entry exists. If it's the case, return to the guest
2066 */
2067 mfn = gfn_to_mfn(current->domain, gaddr_to_gfn(info.gpa));
2068 if ( !mfn_eq(mfn, INVALID_MFN) )
2069 return;
2070
2071 if ( try_map_mmio(gaddr_to_gfn(info.gpa)) )
2072 return;
2073
2074 break;
2075 default:
2076 gprintk(XENLOG_WARNING, "Unsupported DFSC: HSR=%#x DFSC=%#x\n",
2077 hsr.bits, dabt.dfsc);
2078 }
2079
2080 gdprintk(XENLOG_DEBUG, "HSR=0x%x pc=%#"PRIregister" gva=%#"PRIvaddr
2081 " gpa=%#"PRIpaddr"\n", hsr.bits, regs->pc, info.gva, info.gpa);
2082 inject_dabt_exception(regs, info.gva, hsr.len);
2083 }
2084
enter_hypervisor_head(struct cpu_user_regs * regs)2085 static void enter_hypervisor_head(struct cpu_user_regs *regs)
2086 {
2087 if ( guest_mode(regs) )
2088 {
2089 /*
2090 * If we pended a virtual abort, preserve it until it gets cleared.
2091 * See ARM ARM DDI 0487A.j D1.14.3 (Virtual Interrupts) for details,
2092 * but the crucial bit is "On taking a vSError interrupt, HCR_EL2.VSE
2093 * (alias of HCR.VA) is cleared to 0."
2094 */
2095 if ( current->arch.hcr_el2 & HCR_VA )
2096 current->arch.hcr_el2 = READ_SYSREG(HCR_EL2);
2097
2098 gic_clear_lrs(current);
2099 }
2100 }
2101
do_trap_guest_sync(struct cpu_user_regs * regs)2102 void do_trap_guest_sync(struct cpu_user_regs *regs)
2103 {
2104 const union hsr hsr = { .bits = regs->hsr };
2105
2106 enter_hypervisor_head(regs);
2107
2108 switch (hsr.ec) {
2109 case HSR_EC_WFI_WFE:
2110 /*
2111 * HCR_EL2.TWI, HCR_EL2.TWE
2112 *
2113 * ARMv7 (DDI 0406C.b): B1.14.9
2114 * ARMv8 (DDI 0487A.d): D1-1505 Table D1-51
2115 */
2116 if ( !check_conditional_instr(regs, hsr) )
2117 {
2118 advance_pc(regs, hsr);
2119 return;
2120 }
2121 if ( hsr.wfi_wfe.ti ) {
2122 /* Yield the VCPU for WFE */
2123 perfc_incr(trap_wfe);
2124 vcpu_yield();
2125 } else {
2126 /* Block the VCPU for WFI */
2127 perfc_incr(trap_wfi);
2128 vcpu_block_unless_event_pending(current);
2129 }
2130 advance_pc(regs, hsr);
2131 break;
2132 case HSR_EC_CP15_32:
2133 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2134 perfc_incr(trap_cp15_32);
2135 do_cp15_32(regs, hsr);
2136 break;
2137 case HSR_EC_CP15_64:
2138 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2139 perfc_incr(trap_cp15_64);
2140 do_cp15_64(regs, hsr);
2141 break;
2142 case HSR_EC_CP14_32:
2143 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2144 perfc_incr(trap_cp14_32);
2145 do_cp14_32(regs, hsr);
2146 break;
2147 case HSR_EC_CP14_64:
2148 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2149 perfc_incr(trap_cp14_64);
2150 do_cp14_64(regs, hsr);
2151 break;
2152 case HSR_EC_CP14_DBG:
2153 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2154 perfc_incr(trap_cp14_dbg);
2155 do_cp14_dbg(regs, hsr);
2156 break;
2157 case HSR_EC_CP:
2158 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2159 perfc_incr(trap_cp);
2160 do_cp(regs, hsr);
2161 break;
2162 case HSR_EC_SMC32:
2163 /*
2164 * HCR_EL2.TSC
2165 *
2166 * ARMv7 (DDI 0406C.b): B1.14.8
2167 * ARMv8 (DDI 0487A.d): D1-1501 Table D1-44
2168 */
2169 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2170 perfc_incr(trap_smc32);
2171 do_trap_smc(regs, hsr);
2172 break;
2173 case HSR_EC_HVC32:
2174 GUEST_BUG_ON(!psr_mode_is_32bit(regs->cpsr));
2175 perfc_incr(trap_hvc32);
2176 #ifndef NDEBUG
2177 if ( (hsr.iss & 0xff00) == 0xff00 )
2178 return do_debug_trap(regs, hsr.iss & 0x00ff);
2179 #endif
2180 if ( hsr.iss == 0 )
2181 return do_trap_hvc_smccc(regs);
2182 do_trap_hypercall(regs, (register_t *)®s->r12, hsr.iss);
2183 break;
2184 #ifdef CONFIG_ARM_64
2185 case HSR_EC_HVC64:
2186 GUEST_BUG_ON(psr_mode_is_32bit(regs->cpsr));
2187 perfc_incr(trap_hvc64);
2188 #ifndef NDEBUG
2189 if ( (hsr.iss & 0xff00) == 0xff00 )
2190 return do_debug_trap(regs, hsr.iss & 0x00ff);
2191 #endif
2192 if ( hsr.iss == 0 )
2193 return do_trap_hvc_smccc(regs);
2194 do_trap_hypercall(regs, ®s->x16, hsr.iss);
2195 break;
2196 case HSR_EC_SMC64:
2197 /*
2198 * HCR_EL2.TSC
2199 *
2200 * ARMv8 (DDI 0487A.d): D1-1501 Table D1-44
2201 */
2202 GUEST_BUG_ON(psr_mode_is_32bit(regs->cpsr));
2203 perfc_incr(trap_smc64);
2204 do_trap_smc(regs, hsr);
2205 break;
2206 case HSR_EC_SYSREG:
2207 GUEST_BUG_ON(psr_mode_is_32bit(regs->cpsr));
2208 perfc_incr(trap_sysreg);
2209 do_sysreg(regs, hsr);
2210 break;
2211 #endif
2212
2213 case HSR_EC_INSTR_ABORT_LOWER_EL:
2214 perfc_incr(trap_iabt);
2215 do_trap_instr_abort_guest(regs, hsr);
2216 break;
2217 case HSR_EC_DATA_ABORT_LOWER_EL:
2218 perfc_incr(trap_dabt);
2219 do_trap_data_abort_guest(regs, hsr);
2220 break;
2221
2222 default:
2223 gprintk(XENLOG_WARNING,
2224 "Unknown Guest Trap. HSR=0x%x EC=0x%x IL=%x Syndrome=0x%"PRIx32"\n",
2225 hsr.bits, hsr.ec, hsr.len, hsr.iss);
2226 inject_undef_exception(regs, hsr);
2227 }
2228 }
2229
do_trap_hyp_sync(struct cpu_user_regs * regs)2230 void do_trap_hyp_sync(struct cpu_user_regs *regs)
2231 {
2232 const union hsr hsr = { .bits = regs->hsr };
2233
2234 enter_hypervisor_head(regs);
2235
2236 switch ( hsr.ec )
2237 {
2238 #ifdef CONFIG_ARM_64
2239 case HSR_EC_BRK:
2240 do_trap_brk(regs, hsr);
2241 break;
2242 #endif
2243 case HSR_EC_DATA_ABORT_CURR_EL:
2244 case HSR_EC_INSTR_ABORT_CURR_EL:
2245 {
2246 bool is_data = (hsr.ec == HSR_EC_DATA_ABORT_CURR_EL);
2247 const char *fault = (is_data) ? "Data Abort" : "Instruction Abort";
2248
2249 printk("%s Trap. Syndrome=%#x\n", fault, hsr.iss);
2250 /*
2251 * FAR may not be valid for a Synchronous External abort other
2252 * than translation table walk.
2253 */
2254 if ( hsr.xabt.fsc == FSC_SEA && hsr.xabt.fnv )
2255 printk("Invalid FAR, not walking the hypervisor tables\n");
2256 else
2257 dump_hyp_walk(get_hfar(is_data));
2258
2259 do_unexpected_trap(fault, regs);
2260
2261 break;
2262 }
2263 default:
2264 printk("Hypervisor Trap. HSR=0x%x EC=0x%x IL=%x Syndrome=0x%"PRIx32"\n",
2265 hsr.bits, hsr.ec, hsr.len, hsr.iss);
2266 do_unexpected_trap("Hypervisor", regs);
2267 }
2268 }
2269
do_trap_hyp_serror(struct cpu_user_regs * regs)2270 void do_trap_hyp_serror(struct cpu_user_regs *regs)
2271 {
2272 enter_hypervisor_head(regs);
2273
2274 __do_trap_serror(regs, VABORT_GEN_BY_GUEST(regs));
2275 }
2276
do_trap_guest_serror(struct cpu_user_regs * regs)2277 void do_trap_guest_serror(struct cpu_user_regs *regs)
2278 {
2279 enter_hypervisor_head(regs);
2280
2281 __do_trap_serror(regs, true);
2282 }
2283
do_trap_irq(struct cpu_user_regs * regs)2284 void do_trap_irq(struct cpu_user_regs *regs)
2285 {
2286 enter_hypervisor_head(regs);
2287 gic_interrupt(regs, 0);
2288 }
2289
do_trap_fiq(struct cpu_user_regs * regs)2290 void do_trap_fiq(struct cpu_user_regs *regs)
2291 {
2292 enter_hypervisor_head(regs);
2293 gic_interrupt(regs, 1);
2294 }
2295
leave_hypervisor_tail(void)2296 void leave_hypervisor_tail(void)
2297 {
2298 while (1)
2299 {
2300 local_irq_disable();
2301 if (!softirq_pending(smp_processor_id())) {
2302 gic_inject();
2303
2304 /*
2305 * If the SErrors handle option is "DIVERSE", we have to prevent
2306 * slipping the hypervisor SError to guest. In this option, before
2307 * returning from trap, we have to synchronize SErrors to guarantee
2308 * that the pending SError would be caught in hypervisor.
2309 *
2310 * If option is NOT "DIVERSE", SKIP_SYNCHRONIZE_SERROR_ENTRY_EXIT
2311 * will be set to cpu_hwcaps. This means we can use the alternative
2312 * to skip synchronizing SErrors for other SErrors handle options.
2313 */
2314 SYNCHRONIZE_SERROR(SKIP_SYNCHRONIZE_SERROR_ENTRY_EXIT);
2315
2316 return;
2317 }
2318 local_irq_enable();
2319 do_softirq();
2320 /*
2321 * Must be the last one - as the IPI will trigger us to come here
2322 * and we want to patch the hypervisor with almost no stack.
2323 */
2324 check_for_livepatch_work();
2325 }
2326 }
2327
2328 /*
2329 * Local variables:
2330 * mode: C
2331 * c-file-style: "BSD"
2332 * c-basic-offset: 4
2333 * indent-tabs-mode: nil
2334 * End:
2335 */
2336