1 // SPDX-License-Identifier: GPL-2.0-only
6  *   https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability
12  * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
20 #include <linux/arm-smccc.h>
22 #include <linux/cpu.h>
28 #include <asm/debug-monitors.h>
32 #include <asm/vectors.h>
37  * onlining a late CPU.
70  * This one sucks. A CPU is either:
72  * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2.
73  * - Mitigated in hardware and listed in our "safe list".
74  * - Mitigated in software by firmware.
75  * - Mitigated in software by a CPU-specific dance in the kernel and a
77  * - Vulnerable.
97 		pr_info_once("spectre-v2 mitigation disabled by command line option\n");  in spectre_v2_mitigations_off()
137 		 * Platforms affected by Spectre-BHB can't report  in cpu_show_spectre_v2()
138 		 * "Not affected" for Spectre-v2.  in cpu_show_spectre_v2()
169 	/* If the CPU has CSV2 set, we're safe */  in spectre_v2_get_cpu_hw_mitigation_state()
228 	 * the door when we're a guest. Skip the hyp-vectors work.  in install_bp_hardening_cb()
297 	 * Prefer a CPU-specific workaround if it exists. Note that we  in spectre_v2_enable_fw_mitigation()
319  * Spectre-v3a.
322  * an indirect trampoline for the hyp vectors so that guests can't read
342 		data->slot += HYP_VECTOR_INDIRECT;  in spectre_v3a_enable_mitigation()
348  * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is
351  * - Mitigated in hardware and listed in our "safe list".
352  * - Mitigated in hardware via PSTATE.SSBS.
353  * - Mitigated in software by firmware (sometimes referred to as SSBD).
358  * on a per-task basis, but can also be forced on for the kernel, necessitating
359  * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs
361  * being stale when re-entering the kernel. The usual big.LITTLE caveats apply,
365  * be opt-in for userspace. Yes, really, the cure is worse than the disease.
369  * vulnerable CPU if one of the boot CPUs is using the firmware mitigation.
371  * Give me a VAX-11/780 any day of the week...
375 /* This is the per-cpu state tracking whether we need to talk to firmware */
390 	{ "force-on",	SPECTRE_V4_POLICY_MITIGATION_ENABLED, },
391 	{ "force-off",	SPECTRE_V4_POLICY_MITIGATION_DISABLED, },
399 		return -EINVAL;  in parse_spectre_v4_param()
404 		if (strncmp(str, param->str, strlen(param->str)))  in parse_spectre_v4_param()
407 		__spectre_v4_policy = param->policy;  in parse_spectre_v4_param()
411 	return -EINVAL;  in parse_spectre_v4_param()
428 		pr_info_once("spectre-v4 mitigation disabled by command-line option\n");  in spectre_v4_mitigations_off()
481 	/* CPU features are detected first */  in spectre_v4_get_cpu_hw_mitigation_state()
533 		regs->pstate |= PSR_SSBS_BIT;  in try_emulate_el1_ssbs()
535 		regs->pstate &= ~PSR_SSBS_BIT;  in try_emulate_el1_ssbs()
546 	 * If the system is mitigated but this CPU doesn't have SSBS, then  in spectre_v4_enable_hw_mitigation()
563 	 * SSBS is self-synchronizing and is intended to affect subsequent  in spectre_v4_enable_hw_mitigation()
568 	 * could mis-speculate branches and bypass a conditional barrier.  in spectre_v4_enable_hw_mitigation()
577  * Patch a branch over the Spectre-v4 mitigation code with a NOP so that
578  * we fallthrough and check whether firmware needs to be called on this CPU.
584 	BUG_ON(nr_inst != 1); /* Branch -> NOP */  in spectre_v4_patch_fw_mitigation_enable()
597  * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
606 	BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */  in smccc_patch_fw_mitigation_conduit()
661 		regs->pstate |= bit;  in __update_pstate_ssbs()
663 		regs->pstate &= ~bit;  in __update_pstate_ssbs()
669 	bool ssbs = false, kthread = tsk->flags & PF_KTHREAD;  in spectre_v4_enable_task_mitigation()
680  * The Spectre-v4 mitigation can be controlled via a prctl() from userspace.
707 		 * re-enabled.  in ssbd_prctl_set()
710 			return -EPERM;  in ssbd_prctl_set()
714 		 * off and we again prevent it from being re-enabled.  in ssbd_prctl_set()
717 			return -EPERM;  in ssbd_prctl_set()
728 			return -EPERM;  in ssbd_prctl_set()
736 			return -EPERM;  in ssbd_prctl_set()
749 			return -EPERM;  in ssbd_prctl_set()
756 		return -ERANGE;  in ssbd_prctl_set()
770 		return -ENODEV;  in arch_prctl_spec_ctrl_set()
813 		return -ENODEV;  in arch_prctl_spec_ctrl_get()
820  * A CPU is either:
821  * - Mitigated by a branchy loop a CPU specific number of times, and listed
823  * - Mitigated in software by the firmware Spectre v2 call.
824  * - Has the ClearBHB instruction to perform the mitigation.
825  * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
826  *   software mitigation in the vectors is needed.
827  * - Has CSV2.3, so is unaffected.
845  * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
984 	 * When KPTI is in use, the vectors are switched when exiting to  in this_cpu_set_vectors()
985 	 * user-space.  in this_cpu_set_vectors()
1012 		/* No point mitigating Spectre-BHB alone. */  in spectre_bhb_enable_mitigation()
1014 		pr_info_once("spectre-bhb mitigation disabled by compile time option\n");  in spectre_bhb_enable_mitigation()
1016 		pr_info_once("spectre-bhb mitigation disabled by command line option\n");  in spectre_bhb_enable_mitigation()
1025 		if (!data->slot)  in spectre_bhb_enable_mitigation()
1026 			data->slot = HYP_VECTOR_INDIRECT;  in spectre_bhb_enable_mitigation()
1034 		 * branchy-loop added. A57/A72-r0 will already have selected  in spectre_bhb_enable_mitigation()
1035 		 * the spectre-indirect vector, which is sufficient for BHB  in spectre_bhb_enable_mitigation()
1038 		if (!data->slot)  in spectre_bhb_enable_mitigation()
1039 			data->slot = HYP_VECTOR_INDIRECT;  in spectre_bhb_enable_mitigation()
1049 			 * vectors. The indirect vector doesn't include the EL3  in spectre_bhb_enable_mitigation()
1053 			if (!data->slot || data->slot == HYP_VECTOR_INDIRECT)  in spectre_bhb_enable_mitigation()
1054 				data->slot += 1;  in spectre_bhb_enable_mitigation()
1059 			 * The WA3 call in the vectors supersedes the WA1 call  in spectre_bhb_enable_mitigation()
1060 			 * made during context-switch. Uninstall any firmware  in spectre_bhb_enable_mitigation()
1105 	BUG_ON(nr_inst != 1); /* MOV -> MOV */  in spectre_bhb_patch_loop_iter()
1125 	BUG_ON(nr_inst != 1); /* MOV -> MOV */  in spectre_bhb_patch_wa3()