arm64/kernel/entry.S

1 /* SPDX-License-Identifier: GPL-2.0-only */
3  * Low-level exception handling code
10 #include <linux/arm-smccc.h>
16 #include <asm/asm-offsets.h>
29 #include <asm/asm-uaccess.h>
44 	 * skipped by the trampoline vectors, to trigger the cleanup.
64 	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp
66 	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0
67 	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp
89 	 * after panic() re-enables interrupts.
93 	tst	x0, #~(OVERFLOW_STACK_SIZE - 1)	// within range?
94 	b.ne	__bad_stack			// no? -> bad stack pointer
105 	.set	.Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text
112 	 * This macro corrupts x0-x3. It is the caller's duty  to save/restore
241 	 * Enable IA for in-kernel PAC if the task had it disabled. Although
243 	 * a load, this was measured to be slower on Cortex-A75 and Cortex-A76.
265 	 * Any non-self-synchronizing system register updates required for
328 	 * x20 - ICC_PMR_EL1
329 	 * x21 - aborted SP
330 	 * x22 - aborted PC
331 	 * x23 - aborted PSTATE
390 	 * IA was enabled for in-kernel PAC. Disable it now if needed, or
391 	 * alternatively install the user's IA. All other per-task keys and
476 	tst	x21, #TTBR_ASID_MASK		// Check for the reserved ASID
490 	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
501 	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
514  * Exception vectors.
519 SYM_CODE_START(vectors)
530 	kernel_ventry	0, t, 64, sync		// Synchronous 64-bit EL0
531 	kernel_ventry	0, t, 64, irq		// IRQ 64-bit EL0
532 	kernel_ventry	0, t, 64, fiq		// FIQ 64-bit EL0
533 	kernel_ventry	0, t, 64, error		// Error 64-bit EL0
535 	kernel_ventry	0, t, 32, sync		// Synchronous 32-bit EL0
536 	kernel_ventry	0, t, 32, irq		// IRQ 32-bit EL0
537 	kernel_ventry	0, t, 32, fiq		// FIQ 32-bit EL0
538 	kernel_ventry	0, t, 32, error		// Error 32-bit EL0
539 SYM_CODE_END(vectors)
613 	ldr	x19, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
634 	/* 2MB boundary containing the vectors, so we nobble the walk cache */
635 	movk	\tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12)
672 	 * 47 bits of the address space we can sign-extend bit 47 and avoid an
710 	 * enter the full-fat kernel vectors.
717 	tramp_data_read_var	x30, vectors
719 	prfm	plil1strm, [x30, #(1b - \vector_start)]
725 	adr_l	x30, vectors
737 	add	x30, x30, #(1b - \vector_start + 4)
756  * Exception vectors trampoline.
781  * Exception vectors for spectre mitigations on entry from EL1 when
818  * Register switch for AArch64. The callee-saved registers need to be saved
829 	stp	x19, x20, [x8], #16		// store callee-saved registers
837 	ldp	x19, x20, [x8], #16		// restore callee-saved registers
872  * Calls func(regs) using this CPU's irq stack and shadow irq stack.
882 	stp	x29, x30, [sp, #-16]!
972  * Firmware has preserved x0->x17 for us, we must save/restore the rest to
973  * follow SMC-CC. We save (or retrieve) all the registers as the handler may
996 	/* Store the registered-event for crash_smp_send_stop() */
1008 	 * stack for this CPU.
1029 	 * We may have interrupted userspace, or a guest, or exit-from or
1030 	 * return-to either of these. We can't trust sp_el0, restore it.
1043 	stp	x29, x4, [sp, #-16]!
1069 	/* Clear the registered-event seen by crash_smp_send_stop() */
1094 	// Exit will stomp x0-x17, PSTATE, ELR_ELx, and SPSR_ELx.