1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <linux/arm-smccc.h>
8#include <linux/cfi_types.h>
9#include <linux/linkage.h>
10
11#include <asm/alternative.h>
12#include <asm/assembler.h>
13#include <asm/el2_setup.h>
14#include <asm/kvm_arm.h>
15#include <asm/kvm_asm.h>
16#include <asm/kvm_mmu.h>
17#include <asm/pgtable-hwdef.h>
18#include <asm/sysreg.h>
19#include <asm/virt.h>
20
21	.text
22	.pushsection	.idmap.text, "ax"
23
24	.align	11
25
26SYM_CODE_START(__kvm_hyp_init)
27	ventry	.			// Synchronous EL2t
28	ventry	.			// IRQ EL2t
29	ventry	.			// FIQ EL2t
30	ventry	.			// Error EL2t
31
32	ventry	.			// Synchronous EL2h
33	ventry	.			// IRQ EL2h
34	ventry	.			// FIQ EL2h
35	ventry	.			// Error EL2h
36
37	ventry	__do_hyp_init		// Synchronous 64-bit EL1
38	ventry	.			// IRQ 64-bit EL1
39	ventry	.			// FIQ 64-bit EL1
40	ventry	.			// Error 64-bit EL1
41
42	ventry	.			// Synchronous 32-bit EL1
43	ventry	.			// IRQ 32-bit EL1
44	ventry	.			// FIQ 32-bit EL1
45	ventry	.			// Error 32-bit EL1
46
47	/*
48	 * Only uses x0..x3 so as to not clobber callee-saved SMCCC registers.
49	 *
50	 * x0: SMCCC function ID
51	 * x1: struct kvm_nvhe_init_params PA
52	 */
53__do_hyp_init:
54	/* Check for a stub HVC call */
55	cmp	x0, #HVC_STUB_HCALL_NR
56	b.lo	__kvm_handle_stub_hvc
57
58	bic	x0, x0, #ARM_SMCCC_CALL_HINTS
59	mov	x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)
60	cmp	x0, x3
61	b.eq	1f
62
63	mov	x0, #SMCCC_RET_NOT_SUPPORTED
64	eret
65
661:	mov	x0, x1
67	mov	x3, lr
68	bl	___kvm_hyp_init			// Clobbers x0..x2
69	mov	lr, x3
70
71	/* Hello, World! */
72	mov	x0, #SMCCC_RET_SUCCESS
73	eret
74SYM_CODE_END(__kvm_hyp_init)
75
76SYM_CODE_START_LOCAL(__kvm_init_el2_state)
77	/* Initialize EL2 CPU state to sane values. */
78	init_el2_state				// Clobbers x0..x2
79	finalise_el2_state
80	ret
81SYM_CODE_END(__kvm_init_el2_state)
82
83/*
84 * Initialize the hypervisor in EL2.
85 *
86 * Only uses x0..x2 so as to not clobber callee-saved SMCCC registers
87 * and leave x3 for the caller.
88 *
89 * x0: struct kvm_nvhe_init_params PA
90 */
91SYM_CODE_START_LOCAL(___kvm_hyp_init)
92	ldr	x1, [x0, #NVHE_INIT_STACK_HYP_VA]
93	mov	sp, x1
94
95	ldr	x1, [x0, #NVHE_INIT_MAIR_EL2]
96	msr	mair_el2, x1
97
98	ldr	x1, [x0, #NVHE_INIT_HCR_EL2]
99	msr	hcr_el2, x1
100
101	mov	x2, #HCR_E2H
102	and	x2, x1, x2
103	cbz	x2, 1f
104
105	// hVHE: Replay the EL2 setup to account for the E2H bit
106	// TPIDR_EL2 is used to preserve x0 across the macro maze...
107	isb
108	msr	tpidr_el2, x0
109	str	lr, [x0, #NVHE_INIT_TMP]
110
111	bl	__kvm_init_el2_state
112
113	mrs	x0, tpidr_el2
114	ldr	lr, [x0, #NVHE_INIT_TMP]
115
1161:
117	ldr	x1, [x0, #NVHE_INIT_TPIDR_EL2]
118	msr	tpidr_el2, x1
119
120	ldr	x1, [x0, #NVHE_INIT_VTTBR]
121	msr	vttbr_el2, x1
122
123	ldr	x1, [x0, #NVHE_INIT_VTCR]
124	msr	vtcr_el2, x1
125
126	ldr	x1, [x0, #NVHE_INIT_PGD_PA]
127	phys_to_ttbr x2, x1
128alternative_if ARM64_HAS_CNP
129	orr	x2, x2, #TTBR_CNP_BIT
130alternative_else_nop_endif
131	msr	ttbr0_el2, x2
132
133	ldr	x0, [x0, #NVHE_INIT_TCR_EL2]
134	msr	tcr_el2, x0
135
136	isb
137
138	/* Invalidate the stale TLBs from Bootloader */
139	tlbi	alle2
140	tlbi	alle1
141	dsb	sy
142
143	mov_q	x0, INIT_SCTLR_EL2_MMU_ON
144alternative_if ARM64_HAS_ADDRESS_AUTH
145	mov_q	x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \
146		     SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)
147	orr	x0, x0, x1
148alternative_else_nop_endif
149
150#ifdef CONFIG_ARM64_BTI_KERNEL
151alternative_if ARM64_BTI
152	orr	x0, x0, #SCTLR_EL2_BT
153alternative_else_nop_endif
154#endif /* CONFIG_ARM64_BTI_KERNEL */
155
156	msr	sctlr_el2, x0
157	isb
158
159	/* Set the host vector */
160	ldr	x0, =__kvm_hyp_host_vector
161	msr	vbar_el2, x0
162
163	ret
164SYM_CODE_END(___kvm_hyp_init)
165
166/*
167 * PSCI CPU_ON entry point
168 *
169 * x0: struct kvm_nvhe_init_params PA
170 */
171SYM_CODE_START(kvm_hyp_cpu_entry)
172	mov	x1, #1				// is_cpu_on = true
173	b	__kvm_hyp_init_cpu
174SYM_CODE_END(kvm_hyp_cpu_entry)
175
176/*
177 * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point
178 *
179 * x0: struct kvm_nvhe_init_params PA
180 */
181SYM_CODE_START(kvm_hyp_cpu_resume)
182	mov	x1, #0				// is_cpu_on = false
183	b	__kvm_hyp_init_cpu
184SYM_CODE_END(kvm_hyp_cpu_resume)
185
186/*
187 * Common code for CPU entry points. Initializes EL2 state and
188 * installs the hypervisor before handing over to a C handler.
189 *
190 * x0: struct kvm_nvhe_init_params PA
191 * x1: bool is_cpu_on
192 */
193SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu)
194	mov	x28, x0				// Stash arguments
195	mov	x29, x1
196
197	/* Check that the core was booted in EL2. */
198	mrs	x0, CurrentEL
199	cmp	x0, #CurrentEL_EL2
200	b.eq	2f
201
202	/* The core booted in EL1. KVM cannot be initialized on it. */
2031:	wfe
204	wfi
205	b	1b
206
2072:	msr	SPsel, #1			// We want to use SP_EL{1,2}
208
209	bl	__kvm_init_el2_state
210
211	__init_el2_nvhe_prepare_eret
212
213	/* Enable MMU, set vectors and stack. */
214	mov	x0, x28
215	bl	___kvm_hyp_init			// Clobbers x0..x2
216
217	/* Leave idmap. */
218	mov	x0, x29
219	ldr	x1, =kvm_host_psci_cpu_entry
220	br	x1
221SYM_CODE_END(__kvm_hyp_init_cpu)
222
223SYM_CODE_START(__kvm_handle_stub_hvc)
224	/*
225	 * __kvm_handle_stub_hvc called from __host_hvc through branch instruction(br) so
226	 * we need bti j at beginning.
227	 */
228	bti j
229	cmp	x0, #HVC_SOFT_RESTART
230	b.ne	1f
231
232	/* This is where we're about to jump, staying at EL2 */
233	msr	elr_el2, x1
234	mov	x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT | PSR_MODE_EL2h)
235	msr	spsr_el2, x0
236
237	/* Shuffle the arguments, and don't come back */
238	mov	x0, x2
239	mov	x1, x3
240	mov	x2, x4
241	b	reset
242
2431:	cmp	x0, #HVC_RESET_VECTORS
244	b.ne	1f
245
246	/*
247	 * Set the HVC_RESET_VECTORS return code before entering the common
248	 * path so that we do not clobber x0-x2 in case we are coming via
249	 * HVC_SOFT_RESTART.
250	 */
251	mov	x0, xzr
252reset:
253	/* Reset kvm back to the hyp stub. */
254	mov_q	x5, INIT_SCTLR_EL2_MMU_OFF
255	pre_disable_mmu_workaround
256	msr	sctlr_el2, x5
257	isb
258
259alternative_if ARM64_KVM_PROTECTED_MODE
260	mov_q	x5, HCR_HOST_NVHE_FLAGS
261	msr	hcr_el2, x5
262alternative_else_nop_endif
263
264	/* Install stub vectors */
265	adr_l	x5, __hyp_stub_vectors
266	msr	vbar_el2, x5
267	eret
268
2691:	/* Bad stub call */
270	mov_q	x0, HVC_STUB_ERR
271	eret
272
273SYM_CODE_END(__kvm_handle_stub_hvc)
274
275/*
276 * void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
277 *                             void (*fn)(void));
278 *
279 * SYM_TYPED_FUNC_START() allows C to call this ID-mapped function indirectly
280 * using a physical pointer without triggering a kCFI failure.
281 */
282SYM_TYPED_FUNC_START(__pkvm_init_switch_pgd)
283	/* Turn the MMU off */
284	pre_disable_mmu_workaround
285	mrs	x3, sctlr_el2
286	bic	x4, x3, #SCTLR_ELx_M
287	msr	sctlr_el2, x4
288	isb
289
290	tlbi	alle2
291
292	/* Install the new pgtables */
293	phys_to_ttbr x5, x0
294alternative_if ARM64_HAS_CNP
295	orr	x5, x5, #TTBR_CNP_BIT
296alternative_else_nop_endif
297	msr	ttbr0_el2, x5
298
299	/* Set the new stack pointer */
300	mov	sp, x1
301
302	/* And turn the MMU back on! */
303	dsb	nsh
304	isb
305	set_sctlr_el2	x3
306	ret	x2
307SYM_FUNC_END(__pkvm_init_switch_pgd)
308
309	.popsection
310