Lines Matching refs:regs
37 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs) in __enter_from_kernel_mode() argument
39 regs->exit_rcu = false; in __enter_from_kernel_mode()
46 regs->exit_rcu = true; in __enter_from_kernel_mode()
55 static void noinstr enter_from_kernel_mode(struct pt_regs *regs) in enter_from_kernel_mode() argument
57 __enter_from_kernel_mode(regs); in enter_from_kernel_mode()
69 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs) in __exit_to_kernel_mode() argument
73 if (interrupts_enabled(regs)) { in __exit_to_kernel_mode()
74 if (regs->exit_rcu) { in __exit_to_kernel_mode()
84 if (regs->exit_rcu) in __exit_to_kernel_mode()
89 static void noinstr exit_to_kernel_mode(struct pt_regs *regs) in exit_to_kernel_mode() argument
92 __exit_to_kernel_mode(regs); in exit_to_kernel_mode()
108 static __always_inline void enter_from_user_mode(struct pt_regs *regs) in enter_from_user_mode() argument
126 static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs) in prepare_exit_to_user_mode() argument
134 do_notify_resume(regs, flags); in prepare_exit_to_user_mode()
137 static __always_inline void exit_to_user_mode(struct pt_regs *regs) in exit_to_user_mode() argument
139 prepare_exit_to_user_mode(regs); in exit_to_user_mode()
144 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) in asm_exit_to_user_mode() argument
146 exit_to_user_mode(regs); in asm_exit_to_user_mode()
154 static void noinstr arm64_enter_nmi(struct pt_regs *regs) in arm64_enter_nmi() argument
156 regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); in arm64_enter_nmi()
172 static void noinstr arm64_exit_nmi(struct pt_regs *regs) in arm64_exit_nmi() argument
174 bool restore = regs->lockdep_hardirqs; in arm64_exit_nmi()
194 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) in arm64_enter_el1_dbg() argument
196 regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); in arm64_enter_el1_dbg()
209 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) in arm64_exit_el1_dbg() argument
211 bool restore = regs->lockdep_hardirqs; in arm64_exit_el1_dbg()
248 static void do_interrupt_handler(struct pt_regs *regs, in do_interrupt_handler() argument
251 struct pt_regs *old_regs = set_irq_regs(regs); in do_interrupt_handler()
254 call_on_irq_stack(regs, handler); in do_interrupt_handler()
256 handler(regs); in do_interrupt_handler()
264 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, in __panic_unhandled() argument
267 arm64_enter_nmi(regs); in __panic_unhandled()
275 __show_regs(regs); in __panic_unhandled()
280 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
283 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
312 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) in cortex_a76_erratum_1463225_debug_handler() argument
324 regs->pstate |= PSR_D_BIT; in cortex_a76_erratum_1463225_debug_handler()
329 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) in cortex_a76_erratum_1463225_debug_handler() argument
340 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) in el1_abort() argument
344 enter_from_kernel_mode(regs); in el1_abort()
345 local_daif_inherit(regs); in el1_abort()
346 do_mem_abort(far, esr, regs); in el1_abort()
348 exit_to_kernel_mode(regs); in el1_abort()
351 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) in el1_pc() argument
355 enter_from_kernel_mode(regs); in el1_pc()
356 local_daif_inherit(regs); in el1_pc()
357 do_sp_pc_abort(far, esr, regs); in el1_pc()
359 exit_to_kernel_mode(regs); in el1_pc()
362 static void noinstr el1_undef(struct pt_regs *regs) in el1_undef() argument
364 enter_from_kernel_mode(regs); in el1_undef()
365 local_daif_inherit(regs); in el1_undef()
366 do_undefinstr(regs); in el1_undef()
368 exit_to_kernel_mode(regs); in el1_undef()
371 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr) in el1_dbg() argument
375 arm64_enter_el1_dbg(regs); in el1_dbg()
376 if (!cortex_a76_erratum_1463225_debug_handler(regs)) in el1_dbg()
377 do_debug_exception(far, esr, regs); in el1_dbg()
378 arm64_exit_el1_dbg(regs); in el1_dbg()
381 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) in el1_fpac() argument
383 enter_from_kernel_mode(regs); in el1_fpac()
384 local_daif_inherit(regs); in el1_fpac()
385 do_ptrauth_fault(regs, esr); in el1_fpac()
387 exit_to_kernel_mode(regs); in el1_fpac()
390 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) in el1h_64_sync_handler() argument
397 el1_abort(regs, esr); in el1h_64_sync_handler()
404 el1_pc(regs, esr); in el1h_64_sync_handler()
408 el1_undef(regs); in el1h_64_sync_handler()
414 el1_dbg(regs, esr); in el1h_64_sync_handler()
417 el1_fpac(regs, esr); in el1h_64_sync_handler()
420 __panic_unhandled(regs, "64-bit el1h sync", esr); in el1h_64_sync_handler()
424 static __always_inline void __el1_pnmi(struct pt_regs *regs, in __el1_pnmi() argument
427 arm64_enter_nmi(regs); in __el1_pnmi()
428 do_interrupt_handler(regs, handler); in __el1_pnmi()
429 arm64_exit_nmi(regs); in __el1_pnmi()
432 static __always_inline void __el1_irq(struct pt_regs *regs, in __el1_irq() argument
435 enter_from_kernel_mode(regs); in __el1_irq()
438 do_interrupt_handler(regs, handler); in __el1_irq()
450 exit_to_kernel_mode(regs); in __el1_irq()
452 static void noinstr el1_interrupt(struct pt_regs *regs, in el1_interrupt() argument
457 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) in el1_interrupt()
458 __el1_pnmi(regs, handler); in el1_interrupt()
460 __el1_irq(regs, handler); in el1_interrupt()
463 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) in el1h_64_irq_handler() argument
465 el1_interrupt(regs, handle_arch_irq); in el1h_64_irq_handler()
468 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) in el1h_64_fiq_handler() argument
470 el1_interrupt(regs, handle_arch_fiq); in el1h_64_fiq_handler()
473 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) in el1h_64_error_handler() argument
478 arm64_enter_nmi(regs); in el1h_64_error_handler()
479 do_serror(regs, esr); in el1h_64_error_handler()
480 arm64_exit_nmi(regs); in el1h_64_error_handler()
483 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) in el0_da() argument
487 enter_from_user_mode(regs); in el0_da()
489 do_mem_abort(far, esr, regs); in el0_da()
490 exit_to_user_mode(regs); in el0_da()
493 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) in el0_ia() argument
505 enter_from_user_mode(regs); in el0_ia()
507 do_mem_abort(far, esr, regs); in el0_ia()
508 exit_to_user_mode(regs); in el0_ia()
511 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) in el0_fpsimd_acc() argument
513 enter_from_user_mode(regs); in el0_fpsimd_acc()
515 do_fpsimd_acc(esr, regs); in el0_fpsimd_acc()
516 exit_to_user_mode(regs); in el0_fpsimd_acc()
519 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) in el0_sve_acc() argument
521 enter_from_user_mode(regs); in el0_sve_acc()
523 do_sve_acc(esr, regs); in el0_sve_acc()
524 exit_to_user_mode(regs); in el0_sve_acc()
527 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) in el0_fpsimd_exc() argument
529 enter_from_user_mode(regs); in el0_fpsimd_exc()
531 do_fpsimd_exc(esr, regs); in el0_fpsimd_exc()
532 exit_to_user_mode(regs); in el0_fpsimd_exc()
535 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) in el0_sys() argument
537 enter_from_user_mode(regs); in el0_sys()
539 do_sysinstr(esr, regs); in el0_sys()
540 exit_to_user_mode(regs); in el0_sys()
543 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) in el0_pc() argument
547 if (!is_ttbr0_addr(instruction_pointer(regs))) in el0_pc()
550 enter_from_user_mode(regs); in el0_pc()
552 do_sp_pc_abort(far, esr, regs); in el0_pc()
553 exit_to_user_mode(regs); in el0_pc()
556 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) in el0_sp() argument
558 enter_from_user_mode(regs); in el0_sp()
560 do_sp_pc_abort(regs->sp, esr, regs); in el0_sp()
561 exit_to_user_mode(regs); in el0_sp()
564 static void noinstr el0_undef(struct pt_regs *regs) in el0_undef() argument
566 enter_from_user_mode(regs); in el0_undef()
568 do_undefinstr(regs); in el0_undef()
569 exit_to_user_mode(regs); in el0_undef()
572 static void noinstr el0_bti(struct pt_regs *regs) in el0_bti() argument
574 enter_from_user_mode(regs); in el0_bti()
576 do_bti(regs); in el0_bti()
577 exit_to_user_mode(regs); in el0_bti()
580 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) in el0_inv() argument
582 enter_from_user_mode(regs); in el0_inv()
584 bad_el0_sync(regs, 0, esr); in el0_inv()
585 exit_to_user_mode(regs); in el0_inv()
588 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) in el0_dbg() argument
593 enter_from_user_mode(regs); in el0_dbg()
594 do_debug_exception(far, esr, regs); in el0_dbg()
596 exit_to_user_mode(regs); in el0_dbg()
599 static void noinstr el0_svc(struct pt_regs *regs) in el0_svc() argument
601 enter_from_user_mode(regs); in el0_svc()
603 do_el0_svc(regs); in el0_svc()
604 exit_to_user_mode(regs); in el0_svc()
607 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) in el0_fpac() argument
609 enter_from_user_mode(regs); in el0_fpac()
611 do_ptrauth_fault(regs, esr); in el0_fpac()
612 exit_to_user_mode(regs); in el0_fpac()
615 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) in el0t_64_sync_handler() argument
621 el0_svc(regs); in el0t_64_sync_handler()
624 el0_da(regs, esr); in el0t_64_sync_handler()
627 el0_ia(regs, esr); in el0t_64_sync_handler()
630 el0_fpsimd_acc(regs, esr); in el0t_64_sync_handler()
633 el0_sve_acc(regs, esr); in el0t_64_sync_handler()
636 el0_fpsimd_exc(regs, esr); in el0t_64_sync_handler()
640 el0_sys(regs, esr); in el0t_64_sync_handler()
643 el0_sp(regs, esr); in el0t_64_sync_handler()
646 el0_pc(regs, esr); in el0t_64_sync_handler()
649 el0_undef(regs); in el0t_64_sync_handler()
652 el0_bti(regs); in el0t_64_sync_handler()
658 el0_dbg(regs, esr); in el0t_64_sync_handler()
661 el0_fpac(regs, esr); in el0t_64_sync_handler()
664 el0_inv(regs, esr); in el0t_64_sync_handler()
668 static void noinstr el0_interrupt(struct pt_regs *regs, in el0_interrupt() argument
671 enter_from_user_mode(regs); in el0_interrupt()
675 if (regs->pc & BIT(55)) in el0_interrupt()
679 do_interrupt_handler(regs, handler); in el0_interrupt()
682 exit_to_user_mode(regs); in el0_interrupt()
685 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) in __el0_irq_handler_common() argument
687 el0_interrupt(regs, handle_arch_irq); in __el0_irq_handler_common()
690 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) in el0t_64_irq_handler() argument
692 __el0_irq_handler_common(regs); in el0t_64_irq_handler()
695 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) in __el0_fiq_handler_common() argument
697 el0_interrupt(regs, handle_arch_fiq); in __el0_fiq_handler_common()
700 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) in el0t_64_fiq_handler() argument
702 __el0_fiq_handler_common(regs); in el0t_64_fiq_handler()
705 static void noinstr __el0_error_handler_common(struct pt_regs *regs) in __el0_error_handler_common() argument
709 enter_from_user_mode(regs); in __el0_error_handler_common()
711 arm64_enter_nmi(regs); in __el0_error_handler_common()
712 do_serror(regs, esr); in __el0_error_handler_common()
713 arm64_exit_nmi(regs); in __el0_error_handler_common()
715 exit_to_user_mode(regs); in __el0_error_handler_common()
718 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) in el0t_64_error_handler() argument
720 __el0_error_handler_common(regs); in el0t_64_error_handler()
724 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) in el0_cp15() argument
726 enter_from_user_mode(regs); in el0_cp15()
728 do_cp15instr(esr, regs); in el0_cp15()
729 exit_to_user_mode(regs); in el0_cp15()
732 static void noinstr el0_svc_compat(struct pt_regs *regs) in el0_svc_compat() argument
734 enter_from_user_mode(regs); in el0_svc_compat()
736 do_el0_svc_compat(regs); in el0_svc_compat()
737 exit_to_user_mode(regs); in el0_svc_compat()
740 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) in el0t_32_sync_handler() argument
746 el0_svc_compat(regs); in el0t_32_sync_handler()
749 el0_da(regs, esr); in el0t_32_sync_handler()
752 el0_ia(regs, esr); in el0t_32_sync_handler()
755 el0_fpsimd_acc(regs, esr); in el0t_32_sync_handler()
758 el0_fpsimd_exc(regs, esr); in el0t_32_sync_handler()
761 el0_pc(regs, esr); in el0t_32_sync_handler()
767 el0_undef(regs); in el0t_32_sync_handler()
771 el0_cp15(regs, esr); in el0t_32_sync_handler()
777 el0_dbg(regs, esr); in el0t_32_sync_handler()
780 el0_inv(regs, esr); in el0t_32_sync_handler()
784 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) in el0t_32_irq_handler() argument
786 __el0_irq_handler_common(regs); in el0t_32_irq_handler()
789 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) in el0t_32_fiq_handler() argument
791 __el0_fiq_handler_common(regs); in el0t_32_fiq_handler()
794 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) in el0t_32_error_handler() argument
796 __el0_error_handler_common(regs); in el0t_32_error_handler()
806 asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs) in handle_bad_stack() argument
811 arm64_enter_nmi(regs); in handle_bad_stack()
812 panic_bad_stack(regs, esr, far); in handle_bad_stack()
818 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) in __sdei_handler() argument
844 arm64_enter_nmi(regs); in __sdei_handler()
845 ret = do_sdei_event(regs, arg); in __sdei_handler()
846 arm64_exit_nmi(regs); in __sdei_handler()