arm64/kernel/entry.S

1 /* SPDX-License-Identifier: GPL-2.0-only */
3  * Low-level exception handling code
10 #include <linux/arm-smccc.h>
16 #include <asm/asm-offsets.h>
29 #include <asm/asm-uaccess.h>
64 	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp
66 	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0
67 	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp
89 	 * after panic() re-enables interrupts.
93 	tst	x0, #~(OVERFLOW_STACK_SIZE - 1)	// within range?
94 	b.ne	__bad_stack			// no? -> bad stack pointer
105 	.set	.Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text
112 	 * This macro corrupts x0-x3. It is the caller's duty  to save/restore
241 	 * Enable IA for in-kernel PAC if the task had it disabled. Although
243 	 * a load, this was measured to be slower on Cortex-A75 and Cortex-A76.
265 	 * Any non-self-synchronizing system register updates required for
328 	 * x20 - ICC_PMR_EL1
329 	 * x21 - aborted SP
330 	 * x22 - aborted PC
331 	 * x23 - aborted PSTATE
390 	 * IA was enabled for in-kernel PAC. Disable it now if needed, or
391 	 * alternatively install the user's IA. All other per-task keys and
457 	/* Ensure any device/NC reads complete */
490 	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
501 	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
522 	kernel_ventry	1, t, 64, fiq		// FIQ EL1t
527 	kernel_ventry	1, h, 64, fiq		// FIQ EL1h
530 	kernel_ventry	0, t, 64, sync		// Synchronous 64-bit EL0
531 	kernel_ventry	0, t, 64, irq		// IRQ 64-bit EL0
532 	kernel_ventry	0, t, 64, fiq		// FIQ 64-bit EL0
533 	kernel_ventry	0, t, 64, error		// Error 64-bit EL0
535 	kernel_ventry	0, t, 32, sync		// Synchronous 32-bit EL0
536 	kernel_ventry	0, t, 32, irq		// IRQ 32-bit EL0
537 	kernel_ventry	0, t, 32, fiq		// FIQ 32-bit EL0
538 	kernel_ventry	0, t, 32, error		// Error 32-bit EL0
590 	entry_handler	1, t, 64, fiq
595 	entry_handler	1, h, 64, fiq
600 	entry_handler	0, t, 64, fiq
605 	entry_handler	0, t, 32, fiq
613 	ldr	x19, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
635 	movk	\tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12)
672 	 * 47 bits of the address space we can sign-extend bit 47 and avoid an
710 	 * enter the full-fat kernel vectors.
719 	prfm	plil1strm, [x30, #(1b - \vector_start)]
737 	add	x30, x30, #(1b - \vector_start + 4)
788 	kernel_ventry	1, t, 64, fiq		// FIQ EL1h
793 	kernel_ventry	1, h, 64, fiq		// FIQ EL1h
818  * Register switch for AArch64. The callee-saved registers need to be saved
829 	stp	x19, x20, [x8], #16		// store callee-saved registers
837 	ldp	x19, x20, [x8], #16		// restore callee-saved registers
882 	stp	x29, x30, [sp, #-16]!
972  * Firmware has preserved x0->x17 for us, we must save/restore the rest to
973  * follow SMC-CC. We save (or retrieve) all the registers as the handler may
996 	/* Store the registered-event for crash_smp_send_stop() */
1029 	 * We may have interrupted userspace, or a guest, or exit-from or
1030 	 * return-to either of these. We can't trust sp_el0, restore it.
1043 	stp	x29, x4, [sp, #-16]!
1069 	/* Clear the registered-event seen by crash_smp_send_stop() */
1094 	// Exit will stomp x0-x17, PSTATE, ELR_ELx, and SPSR_ELx.