42 static __always_inline bool do_syscall_x64(struct pt_regs *regs, int nr)  in do_syscall_x64()  argument
52 		regs->ax = x64_sys_call(regs, unr);  in do_syscall_x64()
58 static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)  in do_syscall_x32()  argument
69 		regs->ax = x32_sys_call(regs, xnr);  in do_syscall_x32()
76 __visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)  in do_syscall_64()  argument
79 	nr = syscall_enter_from_user_mode(regs, nr);  in do_syscall_64()
83 	if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {  in do_syscall_64()
85 		regs->ax = __x64_sys_ni_syscall(regs);  in do_syscall_64()
89 	syscall_exit_to_user_mode(regs);  in do_syscall_64()
102 	if (unlikely(regs->cx != regs->ip || regs->r11 != regs->flags))  in do_syscall_64()
106 	if (unlikely(regs->cs != __USER_CS || regs->ss != __USER_DS))  in do_syscall_64()
117 	if (unlikely(regs->ip >= TASK_SIZE_MAX))  in do_syscall_64()
125 	if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)))  in do_syscall_64()
134 static __always_inline int syscall_32_enter(struct pt_regs *regs)  in syscall_32_enter()  argument
139 	return (int)regs->orig_ax;  in syscall_32_enter()
155 static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, int nr)  in do_syscall_32_irqs_on()  argument
165 		regs->ax = ia32_sys_call(regs, unr);  in do_syscall_32_irqs_on()
167 		regs->ax = __ia32_sys_ni_syscall(regs);  in do_syscall_32_irqs_on()
210 __visible noinstr void do_int80_emulation(struct pt_regs *regs)  in do_int80_emulation()  argument
215 	if (unlikely(!user_mode(regs))) {  in do_int80_emulation()
216 		irqentry_enter(regs);  in do_int80_emulation()
226 	enter_from_user_mode(regs);  in do_int80_emulation()
246 	regs->orig_ax = regs->ax & GENMASK(31, 0);  in do_int80_emulation()
247 	regs->ax = -ENOSYS;  in do_int80_emulation()
249 	nr = syscall_32_enter(regs);  in do_int80_emulation()
252 	nr = syscall_enter_from_user_mode_work(regs, nr);  in do_int80_emulation()
253 	do_syscall_32_irqs_on(regs, nr);  in do_int80_emulation()
256 	syscall_exit_to_user_mode(regs);  in do_int80_emulation()
294 	enter_from_user_mode(regs);  in DEFINE_FREDENTRY_RAW()
310 	regs->orig_ax = regs->ax & GENMASK(31, 0);  in DEFINE_FREDENTRY_RAW()
311 	regs->ax = -ENOSYS;  in DEFINE_FREDENTRY_RAW()
313 	nr = syscall_32_enter(regs);  in DEFINE_FREDENTRY_RAW()
316 	nr = syscall_enter_from_user_mode_work(regs, nr);  in DEFINE_FREDENTRY_RAW()
317 	do_syscall_32_irqs_on(regs, nr);  in DEFINE_FREDENTRY_RAW()
320 	syscall_exit_to_user_mode(regs);  in DEFINE_FREDENTRY_RAW()
326 __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)  in do_int80_syscall_32()  argument
328 	int nr = syscall_32_enter(regs);  in do_int80_syscall_32()
336 	nr = syscall_enter_from_user_mode(regs, nr);  in do_int80_syscall_32()
339 	do_syscall_32_irqs_on(regs, nr);  in do_int80_syscall_32()
342 	syscall_exit_to_user_mode(regs);  in do_int80_syscall_32()
346 static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)  in __do_fast_syscall_32()  argument
348 	int nr = syscall_32_enter(regs);  in __do_fast_syscall_32()
357 	syscall_enter_from_user_mode_prepare(regs);  in __do_fast_syscall_32()
366 		res = __get_user(*(u32 *)&regs->bp,  in __do_fast_syscall_32()
367 			 (u32 __user __force *)(unsigned long)(u32)regs->sp);  in __do_fast_syscall_32()
369 		res = get_user(*(u32 *)&regs->bp,  in __do_fast_syscall_32()
370 		       (u32 __user __force *)(unsigned long)(u32)regs->sp);  in __do_fast_syscall_32()
375 		regs->ax = -EFAULT;  in __do_fast_syscall_32()
379 		irqentry_exit_to_user_mode(regs);  in __do_fast_syscall_32()
383 	nr = syscall_enter_from_user_mode_work(regs, nr);  in __do_fast_syscall_32()
386 	do_syscall_32_irqs_on(regs, nr);  in __do_fast_syscall_32()
389 	syscall_exit_to_user_mode(regs);  in __do_fast_syscall_32()
394 __visible noinstr bool do_fast_syscall_32(struct pt_regs *regs)  in do_fast_syscall_32()  argument
408 	regs->ip = landing_pad;  in do_fast_syscall_32()
411 	if (!__do_fast_syscall_32(regs))  in do_fast_syscall_32()
425 	if (unlikely(regs->ip != landing_pad))  in do_fast_syscall_32()
429 	if (unlikely(regs->cs != __USER32_CS || regs->ss != __USER_DS))  in do_fast_syscall_32()
433 	if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)))  in do_fast_syscall_32()
441 __visible noinstr bool do_SYSENTER_32(struct pt_regs *regs)  in do_SYSENTER_32()  argument
444 	regs->sp = regs->bp;  in do_SYSENTER_32()
447 	regs->flags |= X86_EFLAGS_IF;  in do_SYSENTER_32()
449 	return do_fast_syscall_32(regs);  in do_SYSENTER_32()
494 static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs)  in __xen_pv_evtchn_do_upcall()  argument
496 	struct pt_regs *old_regs = set_irq_regs(regs);  in __xen_pv_evtchn_do_upcall()
505 __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)  in xen_pv_evtchn_do_upcall()  argument
507 	irqentry_state_t state = irqentry_enter(regs);  in xen_pv_evtchn_do_upcall()
511 	run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs);  in xen_pv_evtchn_do_upcall()
520 		irqentry_exit(regs, state);  in xen_pv_evtchn_do_upcall()