1 // SPDX-License-Identifier: GPL-2.0-only
23 #define __ALT_PTR(a, f)		((void *)&(a)->f + (a)->f)
27 #define ALT_CAP(a)		((a)->cpucap & ~ARM64_CB_BIT)
28 #define ALT_HAS_CB(a)		((a)->cpucap & ARM64_CB_BIT)
51 static __always_inline bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)  in branch_insn_requires_update()  argument
53 	unsigned long replptr = (unsigned long)ALT_REPL_PTR(alt);  in branch_insn_requires_update()
54 	return !(pc >= replptr && pc <= (replptr + alt->alt_len));  in branch_insn_requires_update()
57 #define align_down(x, a)	((unsigned long)(x) & ~(((unsigned long)(a)) - 1))
59 static __always_inline u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr)  in get_alt_insn()  argument
76 		if (branch_insn_requires_update(alt, target)) {  in get_alt_insn()
77 			offset = target - (unsigned long)insnptr;  in get_alt_insn()
85 		 * If we're replacing an adrp instruction, which uses PC-relative  in get_alt_insn()
91 		new_offset = target - align_down(insnptr, SZ_4K);  in get_alt_insn()
104 static noinstr void patch_alternative(struct alt_instr *alt,  in patch_alternative()  argument
110 	replptr = ALT_REPL_PTR(alt);  in patch_alternative()
114 		insn = get_alt_insn(alt, origptr + i, replptr + i);  in patch_alternative()
120  * We provide our own, private D-cache cleaning function so that we don't
131 	cur = start & ~(d_size - 1);  in clean_dcache_range_nopatch()
135 		 * Cortex-A53 errata 826319, 827319, 824069 and 819472  in clean_dcache_range_nopatch()
146 	struct alt_instr *alt;  in __apply_alternatives()  local
150 	for (alt = region->begin; alt < region->end; alt++) {  in __apply_alternatives()
152 		int cap = ALT_CAP(alt);  in __apply_alternatives()
160 		if (ALT_HAS_CB(alt))  in __apply_alternatives()
161 			BUG_ON(alt->alt_len != 0);  in __apply_alternatives()
163 			BUG_ON(alt->alt_len != alt->orig_len);  in __apply_alternatives()
165 		origptr = ALT_ORIG_PTR(alt);  in __apply_alternatives()
167 		nr_inst = alt->orig_len / AARCH64_INSN_SIZE;  in __apply_alternatives()
169 		if (ALT_HAS_CB(alt))  in __apply_alternatives()
170 			alt_cb  = ALT_REPL_PTR(alt);  in __apply_alternatives()
174 		alt_cb(alt, origptr, updptr, nr_inst);  in __apply_alternatives()
203 	const struct elf64_shdr *alt;  in apply_alternatives_vdso()  local
209 	shdr = (void *)hdr + hdr->e_shoff;  in apply_alternatives_vdso()
210 	alt = find_section(hdr, shdr, ".altinstructions");  in apply_alternatives_vdso()
211 	if (!alt)  in apply_alternatives_vdso()
215 		.begin	= (void *)hdr + alt->sh_offset,  in apply_alternatives_vdso()
216 		.end	= (void *)hdr + alt->sh_offset + alt->sh_size,  in apply_alternatives_vdso()
256 	pr_info("applying system-wide alternatives\n");  in apply_alternatives_all()
270 	/* If called on non-boot cpu things could go wrong */  in apply_boot_alternatives()
294 noinstr void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr,  in alt_cb_patch_nops()  argument