44 int raw_patch_instruction(u32 *addr, ppc_inst_t instr)  in raw_patch_instruction()  argument
47 		return __patch_mem(addr, ppc_inst_as_ulong(instr), addr, true);  in raw_patch_instruction()
49 		return __patch_mem(addr, ppc_inst_val(instr), addr, false);  in raw_patch_instruction()
57 	unsigned long addr;  member
63 static int map_patch_area(void *addr, unsigned long text_poke_addr);
64 static void unmap_patch_area(unsigned long addr);
108 	unsigned long addr;  in text_area_cpu_up()  local
119 	addr = (unsigned long)area->addr;  in text_area_cpu_up()
120 	err = map_patch_area(empty_zero_page, addr);  in text_area_cpu_up()
124 	unmap_patch_area(addr);  in text_area_cpu_up()
127 	this_cpu_write(cpu_patching_context.addr, addr);  in text_area_cpu_up()
128 	this_cpu_write(cpu_patching_context.pte, virt_to_kpte(addr));  in text_area_cpu_up()
137 	this_cpu_write(cpu_patching_context.addr, 0);  in text_area_cpu_down()
154 	unsigned long addr;  in text_area_cpu_up_mm()  local
167 	addr = (1 + (get_random_long() % (DEFAULT_MAP_WINDOW / PAGE_SIZE - 2))) << PAGE_SHIFT;  in text_area_cpu_up_mm()
177 	pte = get_locked_pte(mm, addr, &ptl);  in text_area_cpu_up_mm()
183 	this_cpu_write(cpu_patching_context.addr, addr);  in text_area_cpu_up_mm()
188 	put_patching_mm(mm, addr);  in text_area_cpu_up_mm()
196 			this_cpu_read(cpu_patching_context.addr));  in text_area_cpu_down_mm()
199 	this_cpu_write(cpu_patching_context.addr, 0);  in text_area_cpu_down_mm()
228 static unsigned long get_patch_pfn(void *addr)  in get_patch_pfn()  argument
230 	if (IS_ENABLED(CONFIG_EXECMEM) && is_vmalloc_or_module_addr(addr))  in get_patch_pfn()
231 		return vmalloc_to_pfn(addr);  in get_patch_pfn()
233 		return __pa_symbol(addr) >> PAGE_SHIFT;  in get_patch_pfn()
239 static int map_patch_area(void *addr, unsigned long text_poke_addr)  in map_patch_area()  argument
241 	unsigned long pfn = get_patch_pfn(addr);  in map_patch_area()
246 static void unmap_patch_area(unsigned long addr)  in unmap_patch_area()  argument
254 	pgdp = pgd_offset_k(addr);  in unmap_patch_area()
258 	p4dp = p4d_offset(pgdp, addr);  in unmap_patch_area()
262 	pudp = pud_offset(p4dp, addr);  in unmap_patch_area()
266 	pmdp = pmd_offset(pudp, addr);  in unmap_patch_area()
270 	ptep = pte_offset_kernel(pmdp, addr);  in unmap_patch_area()
277 	pte_clear(&init_mm, addr, ptep);  in unmap_patch_area()
278 	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);  in unmap_patch_area()
281 static int __do_patch_mem_mm(void *addr, unsigned long val, bool is_dword)  in __do_patch_mem_mm()  argument
287 	unsigned long pfn = get_patch_pfn(addr);  in __do_patch_mem_mm()
293 	text_poke_addr = __this_cpu_read(cpu_patching_context.addr);  in __do_patch_mem_mm()
294 	patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));  in __do_patch_mem_mm()
310 	err = __patch_mem(addr, val, patch_addr, is_dword);  in __do_patch_mem_mm()
327 static int __do_patch_mem(void *addr, unsigned long val, bool is_dword)  in __do_patch_mem()  argument
333 	unsigned long pfn = get_patch_pfn(addr);  in __do_patch_mem()
335 	text_poke_addr = (unsigned long)__this_cpu_read(cpu_patching_context.addr) & PAGE_MASK;  in __do_patch_mem()
336 	patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));  in __do_patch_mem()
344 	err = __patch_mem(addr, val, patch_addr, is_dword);  in __do_patch_mem()
352 static int patch_mem(void *addr, unsigned long val, bool is_dword)  in patch_mem()  argument
364 		return __patch_mem(addr, val, addr, is_dword);  in patch_mem()
368 		err = __do_patch_mem_mm(addr, val, is_dword);  in patch_mem()
370 		err = __do_patch_mem(addr, val, is_dword);  in patch_mem()
378 int patch_instruction(u32 *addr, ppc_inst_t instr)  in patch_instruction()  argument
381 		return patch_mem(addr, ppc_inst_as_ulong(instr), true);  in patch_instruction()
383 		return patch_mem(addr, ppc_inst_val(instr), false);  in patch_instruction()
387 int patch_uint(void *addr, unsigned int val)  in patch_uint()  argument
389 	if (!IS_ALIGNED((unsigned long)addr, sizeof(unsigned int)))  in patch_uint()
392 	return patch_mem(addr, val, false);  in patch_uint()
396 int patch_ulong(void *addr, unsigned long val)  in patch_ulong()  argument
398 	if (!IS_ALIGNED((unsigned long)addr, sizeof(unsigned long)))  in patch_ulong()
401 	return patch_mem(addr, val, true);  in patch_ulong()
407 int patch_instruction(u32 *addr, ppc_inst_t instr)  in patch_instruction()  argument
409 	return patch_mem(addr, ppc_inst_val(instr), false);  in patch_instruction()
415 static int patch_memset64(u64 *addr, u64 val, size_t count)  in NOKPROBE_SYMBOL()
417 	for (u64 *end = addr + count; addr < end; addr++)  in NOKPROBE_SYMBOL()
418 		__put_kernel_nofault(addr, &val, u64, failed);  in NOKPROBE_SYMBOL()
426 static int patch_memset32(u32 *addr, u32 val, size_t count)  in patch_memset32()  argument
428 	for (u32 *end = addr + count; addr < end; addr++)  in patch_memset32()
429 		__put_kernel_nofault(addr, &val, u32, failed);  in patch_memset32()
466  * Assumes 'len' to be (PAGE_SIZE - offset_in_page(addr)) or below.
468 static int __do_patch_instructions_mm(u32 *addr, u32 *code, size_t len, bool repeat_instr)  in __do_patch_instructions_mm()  argument
471 	unsigned long pfn = get_patch_pfn(addr);  in __do_patch_instructions_mm()
479 	text_poke_addr = __this_cpu_read(cpu_patching_context.addr);  in __do_patch_instructions_mm()
480 	patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));  in __do_patch_instructions_mm()
515  * Assumes 'len' to be (PAGE_SIZE - offset_in_page(addr)) or below.
517 static int __do_patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr)  in __do_patch_instructions()  argument
519 	unsigned long pfn = get_patch_pfn(addr);  in __do_patch_instructions()
525 	text_poke_addr = (unsigned long)__this_cpu_read(cpu_patching_context.addr) & PAGE_MASK;  in __do_patch_instructions()
526 	patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));  in __do_patch_instructions()
543  * Patch 'addr' with 'len' bytes of instructions from 'code'.
548 int patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr)  in patch_instructions()  argument
555 		plen = min_t(size_t, PAGE_SIZE - offset_in_page(addr), len);  in patch_instructions()
559 			err = __do_patch_instructions_mm(addr, code, plen, repeat_instr);  in patch_instructions()
561 			err = __do_patch_instructions(addr, code, plen, repeat_instr);  in patch_instructions()
567 		addr = (u32 *)((unsigned long)addr + plen);  in patch_instructions()
576 int patch_branch(u32 *addr, unsigned long target, int flags)  in patch_branch()  argument
580 	if (create_branch(&instr, addr, target, flags))  in patch_branch()
583 	return patch_instruction(addr, instr);  in patch_branch()
608 int create_cond_branch(ppc_inst_t *instr, const u32 *addr,  in create_cond_branch()  argument
615 		offset = offset - (unsigned long)addr;  in create_cond_branch()