6  *  General FPU state handling cleanups
9 #include <asm/fpu/api.h>
10 #include <asm/fpu/regset.h>
11 #include <asm/fpu/sched.h>
12 #include <asm/fpu/signal.h>
13 #include <asm/fpu/types.h>
29 #include <asm/trace/fpu.h>
36 /* The FPU state configuration data for kernel and user space */
41  * Represents the initial FPU state. It's mostly (but not completely) zeroes,
42  * depending on the FPU hardware format:
46 /* Track in-kernel FPU usage */
50  * Track which context is using the FPU on the CPU:
52 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
55  * Can we use the FPU in kernel mode with the
63 	/* In kernel FPU usage already active? */  in irq_fpu_usable()
68 	 * When not in NMI or hard interrupt context, FPU can be used in:  in irq_fpu_usable()
92 static void update_avx_timestamp(struct fpu *fpu)  in update_avx_timestamp()  argument
97 	if (fpu->fpstate->regs.xsave.header.xfeatures & AVX512_TRACKING_MASK)  in update_avx_timestamp()
98 		fpu->avx512_timestamp = jiffies;  in update_avx_timestamp()
102  * Save the FPU register state in fpu->fpstate->regs. The register state is
107  * The legacy FNSAVE instruction clears all FPU state unconditionally, so
109  * when the FPU is going to be used by another task right after that. But
113  * FXSAVE and all XSAVE variants preserve the FPU register state.
115 void save_fpregs_to_fpstate(struct fpu *fpu)  in save_fpregs_to_fpstate()  argument
118 		os_xsave(fpu->fpstate);  in save_fpregs_to_fpstate()
119 		update_avx_timestamp(fpu);  in save_fpregs_to_fpstate()
124 		fxsave(&fpu->fpstate->regs.fxsave);  in save_fpregs_to_fpstate()
129 	 * Legacy FPU register saving, FNSAVE always clears FPU registers,  in save_fpregs_to_fpstate()
132 	asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->fpstate->regs.fsave));  in save_fpregs_to_fpstate()
133 	frstor(&fpu->fpstate->regs.fsave);  in save_fpregs_to_fpstate()
158 		 * allocation of the larger FPU buffer lazy from #NM or if  in restore_fpregs_from_fpstate()
207 	fpuperm = &current->group_leader->thread.fpu.guest_perm;  in fpu_init_guest_permissions()
239 	 * KVM sets the FP+SSE bits in the XSAVE header when copying FPU state  in fpu_alloc_guest_fpstate()
240 	 * to userspace, even when XSAVE is unsupported, so that restoring FPU  in fpu_alloc_guest_fpstate()
274   * @guest_fpu:         Pointer to the guest FPU container
319 	struct fpstate *fps = current->thread.fpu.fpstate;  in fpu_sync_guest_vmexit_xfd_state()
333 	struct fpu *fpu = &current->thread.fpu;  in fpu_swap_kvm_fpstate()  local
334 	struct fpstate *cur_fps = fpu->fpstate;  in fpu_swap_kvm_fpstate()
338 		save_fpregs_to_fpstate(fpu);  in fpu_swap_kvm_fpstate()
342 		fpu->__task_fpstate = cur_fps;  in fpu_swap_kvm_fpstate()
343 		fpu->fpstate = guest_fps;  in fpu_swap_kvm_fpstate()
347 		fpu->fpstate = fpu->__task_fpstate;  in fpu_swap_kvm_fpstate()
348 		fpu->__task_fpstate = NULL;  in fpu_swap_kvm_fpstate()
351 	cur_fps = fpu->fpstate;  in fpu_swap_kvm_fpstate()
433 		save_fpregs_to_fpstate(&current->thread.fpu);  in kernel_fpu_begin_mask()
456  * Sync the FPU register state to current's memory register state when the
457  * current task owns the FPU. The hardware register state is preserved.
459 void fpu_sync_fpstate(struct fpu *fpu)  in fpu_sync_fpstate()  argument
461 	WARN_ON_FPU(fpu != &current->thread.fpu);  in fpu_sync_fpstate()
464 	trace_x86_fpu_before_save(fpu);  in fpu_sync_fpstate()
467 		save_fpregs_to_fpstate(fpu);  in fpu_sync_fpstate()
469 	trace_x86_fpu_after_save(fpu);  in fpu_sync_fpstate()
529 void fpstate_reset(struct fpu *fpu)  in fpstate_reset()  argument
532 	fpu->fpstate = &fpu->__fpstate;  in fpstate_reset()
533 	__fpstate_reset(fpu->fpstate, init_fpstate.xfd);  in fpstate_reset()
535 	/* Initialize the permission related info in fpu */  in fpstate_reset()
536 	fpu->perm.__state_perm		= fpu_kernel_cfg.default_features;  in fpstate_reset()
537 	fpu->perm.__state_size		= fpu_kernel_cfg.default_size;  in fpstate_reset()
538 	fpu->perm.__user_state_size	= fpu_user_cfg.default_size;  in fpstate_reset()
540 	fpu->guest_perm = fpu->perm;  in fpstate_reset()
543 static inline void fpu_inherit_perms(struct fpu *dst_fpu)  in fpu_inherit_perms()
546 		struct fpu *src_fpu = &current->group_leader->thread.fpu;  in fpu_inherit_perms()
566 	xstate = get_xsave_addr(&dst->thread.fpu.fpstate->regs.xsave,  in update_fpu_shstk()
571 	 * stack and the fpu state should be up to date since it was just copied  in update_fpu_shstk()
583 /* Clone current's FPU state on fork */
587 	struct fpu *src_fpu = &current->thread.fpu;  in fpu_clone()
588 	struct fpu *dst_fpu = &dst->thread.fpu;  in fpu_clone()
590 	/* The new task's FPU state cannot be valid in the hardware. */  in fpu_clone()
600 	 * from trying to save the FPU registers on context switch.  in fpu_clone()
605 	 * No FPU state inheritance for kernel threads and IO  in fpu_clone()
622 	 * Save the default portion of the current FPU state into the  in fpu_clone()
658  * Whitelist the FPU register state embedded into task_struct for hardened
663 	*offset = offsetof(struct thread_struct, fpu.__fpstate.regs);  in fpu_thread_struct_whitelist()
668  * Drops current FPU state: deactivates the fpregs and
670  * in the fpregs in the eager-FPU case.
676 void fpu__drop(struct fpu *fpu)  in fpu__drop()  argument
680 	if (fpu == &current->thread.fpu) {  in fpu__drop()
685 		fpregs_deactivate(fpu);  in fpu__drop()
688 	trace_x86_fpu_dropped(fpu);  in fpu__drop()
694  * Clear FPU registers by setting them up from the init fpstate.
710  * Reset current->fpu memory state to the init values.
714 	struct fpu *fpu = &current->thread.fpu;  in fpu_reset_fpregs()  local
717 	__fpu_invalidate_fpregs_state(fpu);  in fpu_reset_fpregs()
731 	memcpy(&fpu->fpstate->regs, &init_fpstate.regs, init_fpstate_copy_size());  in fpu_reset_fpregs()
737  * Reset current's user FPU states to the init states.  current's
741 void fpu__clear_user_states(struct fpu *fpu)  in fpu__clear_user_states()  argument
743 	WARN_ON_FPU(fpu != &current->thread.fpu);  in fpu__clear_user_states()
757 	    !fpregs_state_valid(fpu, smp_processor_id()))  in fpu__clear_user_states()
758 		os_xrstor_supervisor(fpu->fpstate);  in fpu__clear_user_states()
764 	 * Now all FPU registers have their desired values.  Inform the FPU  in fpu__clear_user_states()
765 	 * state machine that current's FPU registers are in the hardware  in fpu__clear_user_states()
768 	 * current's FPU is marked active.  in fpu__clear_user_states()
776 	fpstate_reset(&current->thread.fpu);  in fpu_flush_thread()
780  * Load FPU context before returning to userspace.
811  * If current FPU state according to its tracking (loaded FPU context on this
817 	struct fpu *fpu = &current->thread.fpu;  in fpregs_assert_state_consistent()  local
822 	WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id()));  in fpregs_assert_state_consistent()
829 	struct fpu *fpu = &current->thread.fpu;  in fpregs_mark_activate()  local
831 	fpregs_activate(fpu);  in fpregs_mark_activate()
832 	fpu->last_cpu = smp_processor_id();  in fpregs_mark_activate()
840 int fpu__exception_code(struct fpu *fpu, int trap_nr)  in fpu__exception_code()  argument
852 		 * then we have a bad program that isn't synchronizing its FPU usage  in fpu__exception_code()
857 			cwd = fpu->fpstate->regs.fxsave.cwd;  in fpu__exception_code()
858 			swd = fpu->fpstate->regs.fxsave.swd;  in fpu__exception_code()
860 			cwd = (unsigned short)fpu->fpstate->regs.fsave.cwd;  in fpu__exception_code()
861 			swd = (unsigned short)fpu->fpstate->regs.fsave.swd;  in fpu__exception_code()
867 		 * The SIMD FPU exceptions are handled a little differently, as there  in fpu__exception_code()
875 			mxcsr = fpu->fpstate->regs.fxsave.mxcsr;  in fpu__exception_code()