50 static int create_rstor_token(unsigned long ssp, unsigned long *token_addr)  in create_rstor_token()  argument
55 	if (!IS_ALIGNED(ssp, 8))  in create_rstor_token()
58 	addr = ssp - SS_FRAME_SIZE;  in create_rstor_token()
61 	 * SSP is aligned, so reserved bits and mode bit are a zero, just mark  in create_rstor_token()
64 	ssp |= BIT(0);  in create_rstor_token()
66 	if (write_user_shstk_64((u64 __user *)addr, (u64)ssp))  in create_rstor_token()
79  * The shadow stack pointer(SSP) is moved by CALL, RET, and INCSSPQ. The
86  * and INCSSP. In addition to modifying SSP, INCSSP also reads from the
90  * READ_ONCE(ssp);       // read+discard top element on stack
91  * ssp += nr_to_pop * 8; // move the shadow stack
92  * READ_ONCE(ssp-8);     // read+discard last popped stack element
94  * The maximum distance INCSSP can move the SSP is 2040 bytes, before
96  * to prevent any operation from shifting the SSP to an adjacent stack,
238 	unsigned long long ssp;  in get_user_shstk_addr()  local
242 	rdmsrl(MSR_IA32_PL3_SSP, ssp);  in get_user_shstk_addr()
246 	return ssp;  in get_user_shstk_addr()
280 static int shstk_push_sigframe(unsigned long *ssp)  in shstk_push_sigframe()  argument
282 	unsigned long target_ssp = *ssp;  in shstk_push_sigframe()
288 	*ssp -= SS_FRAME_SIZE;  in shstk_push_sigframe()
289 	if (put_shstk_data((void __user *)*ssp, target_ssp))  in shstk_push_sigframe()
295 static int shstk_pop_sigframe(unsigned long *ssp)  in shstk_pop_sigframe()  argument
303 	 * It is possible for the SSP to be off the end of a shadow stack by 4  in shstk_pop_sigframe()
308 	if (!IS_ALIGNED(*ssp, 8))  in shstk_pop_sigframe()
311 	need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp;  in shstk_pop_sigframe()
316 	err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp);  in shstk_pop_sigframe()
321 		vma = find_vma(current->mm, *ssp);  in shstk_pop_sigframe()
330 	/* Restore SSP aligned? */  in shstk_pop_sigframe()
334 	/* SSP in userspace? */  in shstk_pop_sigframe()
338 	*ssp = token_addr;  in shstk_pop_sigframe()
350 	unsigned long ssp;  in setup_signal_shadow_stack()  local
360 	ssp = get_user_shstk_addr();  in setup_signal_shadow_stack()
361 	if (unlikely(!ssp))  in setup_signal_shadow_stack()
364 	err = shstk_push_sigframe(&ssp);  in setup_signal_shadow_stack()
369 	ssp -= SS_FRAME_SIZE;  in setup_signal_shadow_stack()
370 	err = write_user_shstk_64((u64 __user *)ssp, (u64)restorer);  in setup_signal_shadow_stack()
375 	wrmsrl(MSR_IA32_PL3_SSP, ssp);  in setup_signal_shadow_stack()
383 	unsigned long ssp;  in restore_signal_shadow_stack()  local
390 	ssp = get_user_shstk_addr();  in restore_signal_shadow_stack()
391 	if (unlikely(!ssp))  in restore_signal_shadow_stack()
394 	err = shstk_pop_sigframe(&ssp);  in restore_signal_shadow_stack()
399 	wrmsrl(MSR_IA32_PL3_SSP, ssp);  in restore_signal_shadow_stack()
583 	unsigned long ssp;  in shstk_update_last_frame()  local
588 	ssp = get_user_shstk_addr();  in shstk_update_last_frame()
589 	return write_user_shstk_64((u64 __user *)ssp, (u64)val);  in shstk_update_last_frame()