39 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)  in __enter_from_kernel_mode()  argument
41 	regs->exit_rcu = false;  in __enter_from_kernel_mode()
48 		regs->exit_rcu = true;  in __enter_from_kernel_mode()
57 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)  in enter_from_kernel_mode()  argument
59 	__enter_from_kernel_mode(regs);  in enter_from_kernel_mode()
72 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)  in __exit_to_kernel_mode()  argument
76 	if (interrupts_enabled(regs)) {  in __exit_to_kernel_mode()
77 		if (regs->exit_rcu) {  in __exit_to_kernel_mode()
87 		if (regs->exit_rcu)  in __exit_to_kernel_mode()
92 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)  in exit_to_kernel_mode()  argument
95 	__exit_to_kernel_mode(regs);  in exit_to_kernel_mode()
112 static __always_inline void enter_from_user_mode(struct pt_regs *regs)  in enter_from_user_mode()  argument
130 static void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)  in do_notify_resume()  argument
139 			uprobe_notify_resume(regs);  in do_notify_resume()
148 			do_signal(regs);  in do_notify_resume()
151 			resume_user_mode_work(regs);  in do_notify_resume()
161 static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)  in exit_to_user_mode_prepare()  argument
169 		do_notify_resume(regs, flags);  in exit_to_user_mode_prepare()
176 static __always_inline void exit_to_user_mode(struct pt_regs *regs)  in exit_to_user_mode()  argument
178 	exit_to_user_mode_prepare(regs);  in exit_to_user_mode()
183 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)  in asm_exit_to_user_mode()  argument
185 	exit_to_user_mode(regs);  in asm_exit_to_user_mode()
193 static void noinstr arm64_enter_nmi(struct pt_regs *regs)  in arm64_enter_nmi()  argument
195 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_nmi()
211 static void noinstr arm64_exit_nmi(struct pt_regs *regs)  in arm64_exit_nmi()  argument
213 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_nmi()
233 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)  in arm64_enter_el1_dbg()  argument
235 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_el1_dbg()
248 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)  in arm64_exit_el1_dbg()  argument
250 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_el1_dbg()
304 static void do_interrupt_handler(struct pt_regs *regs,  in do_interrupt_handler()  argument
307 	struct pt_regs *old_regs = set_irq_regs(regs);  in do_interrupt_handler()
310 		call_on_irq_stack(regs, handler);  in do_interrupt_handler()
312 		handler(regs);  in do_interrupt_handler()
320 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,  in __panic_unhandled()  argument
323 	arm64_enter_nmi(regs);  in __panic_unhandled()
331 	__show_regs(regs);  in __panic_unhandled()
336 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs)	\
339 	__panic_unhandled(regs, desc, read_sysreg(esr_el1));				\
369 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
381 	regs->pstate |= PSR_D_BIT;  in cortex_a76_erratum_1463225_debug_handler()
386 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
426 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)  in el1_abort()  argument
430 	enter_from_kernel_mode(regs);  in el1_abort()
431 	local_daif_inherit(regs);  in el1_abort()
432 	do_mem_abort(far, esr, regs);  in el1_abort()
434 	exit_to_kernel_mode(regs);  in el1_abort()
437 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)  in el1_pc()  argument
441 	enter_from_kernel_mode(regs);  in el1_pc()
442 	local_daif_inherit(regs);  in el1_pc()
443 	do_sp_pc_abort(far, esr, regs);  in el1_pc()
445 	exit_to_kernel_mode(regs);  in el1_pc()
448 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)  in el1_undef()  argument
450 	enter_from_kernel_mode(regs);  in el1_undef()
451 	local_daif_inherit(regs);  in el1_undef()
452 	do_el1_undef(regs, esr);  in el1_undef()
454 	exit_to_kernel_mode(regs);  in el1_undef()
457 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)  in el1_bti()  argument
459 	enter_from_kernel_mode(regs);  in el1_bti()
460 	local_daif_inherit(regs);  in el1_bti()
461 	do_el1_bti(regs, esr);  in el1_bti()
463 	exit_to_kernel_mode(regs);  in el1_bti()
466 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)  in el1_dbg()  argument
470 	arm64_enter_el1_dbg(regs);  in el1_dbg()
471 	if (!cortex_a76_erratum_1463225_debug_handler(regs))  in el1_dbg()
472 		do_debug_exception(far, esr, regs);  in el1_dbg()
473 	arm64_exit_el1_dbg(regs);  in el1_dbg()
476 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)  in el1_fpac()  argument
478 	enter_from_kernel_mode(regs);  in el1_fpac()
479 	local_daif_inherit(regs);  in el1_fpac()
480 	do_el1_fpac(regs, esr);  in el1_fpac()
482 	exit_to_kernel_mode(regs);  in el1_fpac()
485 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)  in el1h_64_sync_handler()  argument
492 		el1_abort(regs, esr);  in el1h_64_sync_handler()
499 		el1_pc(regs, esr);  in el1h_64_sync_handler()
503 		el1_undef(regs, esr);  in el1h_64_sync_handler()
506 		el1_bti(regs, esr);  in el1h_64_sync_handler()
512 		el1_dbg(regs, esr);  in el1h_64_sync_handler()
515 		el1_fpac(regs, esr);  in el1h_64_sync_handler()
518 		__panic_unhandled(regs, "64-bit el1h sync", esr);  in el1h_64_sync_handler()
522 static __always_inline void __el1_pnmi(struct pt_regs *regs,  in __el1_pnmi()  argument
525 	arm64_enter_nmi(regs);  in __el1_pnmi()
526 	do_interrupt_handler(regs, handler);  in __el1_pnmi()
527 	arm64_exit_nmi(regs);  in __el1_pnmi()
530 static __always_inline void __el1_irq(struct pt_regs *regs,  in __el1_irq()  argument
533 	enter_from_kernel_mode(regs);  in __el1_irq()
536 	do_interrupt_handler(regs, handler);  in __el1_irq()
541 	exit_to_kernel_mode(regs);  in __el1_irq()
543 static void noinstr el1_interrupt(struct pt_regs *regs,  in el1_interrupt()  argument
548 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in el1_interrupt()
549 		__el1_pnmi(regs, handler);  in el1_interrupt()
551 		__el1_irq(regs, handler);  in el1_interrupt()
554 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)  in el1h_64_irq_handler()  argument
556 	el1_interrupt(regs, handle_arch_irq);  in el1h_64_irq_handler()
559 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)  in el1h_64_fiq_handler()  argument
561 	el1_interrupt(regs, handle_arch_fiq);  in el1h_64_fiq_handler()
564 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)  in el1h_64_error_handler()  argument
569 	arm64_enter_nmi(regs);  in el1h_64_error_handler()
570 	do_serror(regs, esr);  in el1h_64_error_handler()
571 	arm64_exit_nmi(regs);  in el1h_64_error_handler()
574 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)  in el0_da()  argument
578 	enter_from_user_mode(regs);  in el0_da()
580 	do_mem_abort(far, esr, regs);  in el0_da()
581 	exit_to_user_mode(regs);  in el0_da()
584 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)  in el0_ia()  argument
596 	enter_from_user_mode(regs);  in el0_ia()
598 	do_mem_abort(far, esr, regs);  in el0_ia()
599 	exit_to_user_mode(regs);  in el0_ia()
602 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_acc()  argument
604 	enter_from_user_mode(regs);  in el0_fpsimd_acc()
606 	do_fpsimd_acc(esr, regs);  in el0_fpsimd_acc()
607 	exit_to_user_mode(regs);  in el0_fpsimd_acc()
610 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)  in el0_sve_acc()  argument
612 	enter_from_user_mode(regs);  in el0_sve_acc()
614 	do_sve_acc(esr, regs);  in el0_sve_acc()
615 	exit_to_user_mode(regs);  in el0_sve_acc()
618 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)  in el0_sme_acc()  argument
620 	enter_from_user_mode(regs);  in el0_sme_acc()
622 	do_sme_acc(esr, regs);  in el0_sme_acc()
623 	exit_to_user_mode(regs);  in el0_sme_acc()
626 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_exc()  argument
628 	enter_from_user_mode(regs);  in el0_fpsimd_exc()
630 	do_fpsimd_exc(esr, regs);  in el0_fpsimd_exc()
631 	exit_to_user_mode(regs);  in el0_fpsimd_exc()
634 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)  in el0_sys()  argument
636 	enter_from_user_mode(regs);  in el0_sys()
638 	do_el0_sys(esr, regs);  in el0_sys()
639 	exit_to_user_mode(regs);  in el0_sys()
642 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)  in el0_pc()  argument
646 	if (!is_ttbr0_addr(instruction_pointer(regs)))  in el0_pc()
649 	enter_from_user_mode(regs);  in el0_pc()
651 	do_sp_pc_abort(far, esr, regs);  in el0_pc()
652 	exit_to_user_mode(regs);  in el0_pc()
655 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)  in el0_sp()  argument
657 	enter_from_user_mode(regs);  in el0_sp()
659 	do_sp_pc_abort(regs->sp, esr, regs);  in el0_sp()
660 	exit_to_user_mode(regs);  in el0_sp()
663 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)  in el0_undef()  argument
665 	enter_from_user_mode(regs);  in el0_undef()
667 	do_el0_undef(regs, esr);  in el0_undef()
668 	exit_to_user_mode(regs);  in el0_undef()
671 static void noinstr el0_bti(struct pt_regs *regs)  in el0_bti()  argument
673 	enter_from_user_mode(regs);  in el0_bti()
675 	do_el0_bti(regs);  in el0_bti()
676 	exit_to_user_mode(regs);  in el0_bti()
679 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)  in el0_mops()  argument
681 	enter_from_user_mode(regs);  in el0_mops()
683 	do_el0_mops(regs, esr);  in el0_mops()
684 	exit_to_user_mode(regs);  in el0_mops()
687 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)  in el0_inv()  argument
689 	enter_from_user_mode(regs);  in el0_inv()
691 	bad_el0_sync(regs, 0, esr);  in el0_inv()
692 	exit_to_user_mode(regs);  in el0_inv()
695 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)  in el0_dbg()  argument
700 	enter_from_user_mode(regs);  in el0_dbg()
701 	do_debug_exception(far, esr, regs);  in el0_dbg()
703 	exit_to_user_mode(regs);  in el0_dbg()
706 static void noinstr el0_svc(struct pt_regs *regs)  in el0_svc()  argument
708 	enter_from_user_mode(regs);  in el0_svc()
712 	do_el0_svc(regs);  in el0_svc()
713 	exit_to_user_mode(regs);  in el0_svc()
716 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)  in el0_fpac()  argument
718 	enter_from_user_mode(regs);  in el0_fpac()
720 	do_el0_fpac(regs, esr);  in el0_fpac()
721 	exit_to_user_mode(regs);  in el0_fpac()
724 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)  in el0t_64_sync_handler()  argument
730 		el0_svc(regs);  in el0t_64_sync_handler()
733 		el0_da(regs, esr);  in el0t_64_sync_handler()
736 		el0_ia(regs, esr);  in el0t_64_sync_handler()
739 		el0_fpsimd_acc(regs, esr);  in el0t_64_sync_handler()
742 		el0_sve_acc(regs, esr);  in el0t_64_sync_handler()
745 		el0_sme_acc(regs, esr);  in el0t_64_sync_handler()
748 		el0_fpsimd_exc(regs, esr);  in el0t_64_sync_handler()
752 		el0_sys(regs, esr);  in el0t_64_sync_handler()
755 		el0_sp(regs, esr);  in el0t_64_sync_handler()
758 		el0_pc(regs, esr);  in el0t_64_sync_handler()
761 		el0_undef(regs, esr);  in el0t_64_sync_handler()
764 		el0_bti(regs);  in el0t_64_sync_handler()
767 		el0_mops(regs, esr);  in el0t_64_sync_handler()
773 		el0_dbg(regs, esr);  in el0t_64_sync_handler()
776 		el0_fpac(regs, esr);  in el0t_64_sync_handler()
779 		el0_inv(regs, esr);  in el0t_64_sync_handler()
783 static void noinstr el0_interrupt(struct pt_regs *regs,  in el0_interrupt()  argument
786 	enter_from_user_mode(regs);  in el0_interrupt()
790 	if (regs->pc & BIT(55))  in el0_interrupt()
794 	do_interrupt_handler(regs, handler);  in el0_interrupt()
797 	exit_to_user_mode(regs);  in el0_interrupt()
800 static void noinstr __el0_irq_handler_common(struct pt_regs *regs)  in __el0_irq_handler_common()  argument
802 	el0_interrupt(regs, handle_arch_irq);  in __el0_irq_handler_common()
805 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)  in el0t_64_irq_handler()  argument
807 	__el0_irq_handler_common(regs);  in el0t_64_irq_handler()
810 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)  in __el0_fiq_handler_common()  argument
812 	el0_interrupt(regs, handle_arch_fiq);  in __el0_fiq_handler_common()
815 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)  in el0t_64_fiq_handler()  argument
817 	__el0_fiq_handler_common(regs);  in el0t_64_fiq_handler()
820 static void noinstr __el0_error_handler_common(struct pt_regs *regs)  in __el0_error_handler_common()  argument
824 	enter_from_user_mode(regs);  in __el0_error_handler_common()
826 	arm64_enter_nmi(regs);  in __el0_error_handler_common()
827 	do_serror(regs, esr);  in __el0_error_handler_common()
828 	arm64_exit_nmi(regs);  in __el0_error_handler_common()
830 	exit_to_user_mode(regs);  in __el0_error_handler_common()
833 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)  in el0t_64_error_handler()  argument
835 	__el0_error_handler_common(regs);  in el0t_64_error_handler()
839 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)  in el0_cp15()  argument
841 	enter_from_user_mode(regs);  in el0_cp15()
843 	do_el0_cp15(esr, regs);  in el0_cp15()
844 	exit_to_user_mode(regs);  in el0_cp15()
847 static void noinstr el0_svc_compat(struct pt_regs *regs)  in el0_svc_compat()  argument
849 	enter_from_user_mode(regs);  in el0_svc_compat()
852 	do_el0_svc_compat(regs);  in el0_svc_compat()
853 	exit_to_user_mode(regs);  in el0_svc_compat()
856 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)  in el0t_32_sync_handler()  argument
862 		el0_svc_compat(regs);  in el0t_32_sync_handler()
865 		el0_da(regs, esr);  in el0t_32_sync_handler()
868 		el0_ia(regs, esr);  in el0t_32_sync_handler()
871 		el0_fpsimd_acc(regs, esr);  in el0t_32_sync_handler()
874 		el0_fpsimd_exc(regs, esr);  in el0t_32_sync_handler()
877 		el0_pc(regs, esr);  in el0t_32_sync_handler()
883 		el0_undef(regs, esr);  in el0t_32_sync_handler()
887 		el0_cp15(regs, esr);  in el0t_32_sync_handler()
893 		el0_dbg(regs, esr);  in el0t_32_sync_handler()
896 		el0_inv(regs, esr);  in el0t_32_sync_handler()
900 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)  in el0t_32_irq_handler()  argument
902 	__el0_irq_handler_common(regs);  in el0t_32_irq_handler()
905 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)  in el0t_32_fiq_handler()  argument
907 	__el0_fiq_handler_common(regs);  in el0t_32_fiq_handler()
910 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)  in el0t_32_error_handler()  argument
912 	__el0_error_handler_common(regs);  in el0t_32_error_handler()
922 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)  in handle_bad_stack()  argument
927 	arm64_enter_nmi(regs);  in handle_bad_stack()
928 	panic_bad_stack(regs, esr, far);  in handle_bad_stack()
934 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)  in __sdei_handler()  argument
960 	arm64_enter_nmi(regs);  in __sdei_handler()
961 	ret = do_sdei_event(regs, arg);  in __sdei_handler()
962 	arm64_exit_nmi(regs);  in __sdei_handler()