1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * transition.c - Kernel Live Patching transition functions
5  * Copyright (C) 2015-2016 Josh Poimboeuf <jpoimboe@redhat.com>
14 #include "patch.h"
32  * cond_resched().  This helps CPU-bound kthreads get patched.
74  * We allow to patch also functions where RCU is not watching,
87  * The transition to the target patch state is complete.  Clean up the data
98 		 klp_transition_patch->mod->name,  in klp_complete_transition()
101 	if (klp_transition_patch->replace && klp_target_state == KLP_TRANSITION_PATCHED) {  in klp_complete_transition()
115 		 * from this patch on the ops->func_stack.  Otherwise, after  in klp_complete_transition()
116 		 * func->transition gets cleared, the handler may choose a  in klp_complete_transition()
124 			func->transition = false;  in klp_complete_transition()
133 		task->patch_state = KLP_TRANSITION_IDLE;  in klp_complete_transition()
140 		task->patch_state = KLP_TRANSITION_IDLE;  in klp_complete_transition()
152 	pr_notice("'%s': %s complete\n", klp_transition_patch->mod->name,  in klp_complete_transition()
171 		 klp_transition_patch->mod->name);  in klp_cancel_transition()
179  * patch state.
201 	 *    of func->transition, if klp_ftrace_handler() is called later on  in klp_update_patch_state()
205 		task->patch_state = READ_ONCE(klp_target_state);  in klp_update_patch_state()
212  * to-be-patched or to-be-unpatched function.
217 	unsigned long func_addr, func_size, address;  in klp_check_stack_func()  local
223 		  * Check for the to-be-unpatched function  in klp_check_stack_func()
226 		func_addr = (unsigned long)func->new_func;  in klp_check_stack_func()
227 		func_size = func->new_size;  in klp_check_stack_func()
230 		 * Check for the to-be-patched function  in klp_check_stack_func()
233 		ops = klp_find_ops(func->old_func);  in klp_check_stack_func()
235 		if (list_is_singular(&ops->func_stack)) {  in klp_check_stack_func()
237 			func_addr = (unsigned long)func->old_func;  in klp_check_stack_func()
238 			func_size = func->old_size;  in klp_check_stack_func()
244 			func_addr = (unsigned long)prev->new_func;  in klp_check_stack_func()
245 			func_size = prev->new_size;  in klp_check_stack_func()
250 		address = entries[i];  in klp_check_stack_func()
252 		if (address >= func_addr && address < func_addr + func_size)  in klp_check_stack_func()
253 			return -EAGAIN;  in klp_check_stack_func()
260  * Determine whether it's safe to transition the task to the target patch state
261  * by looking for any to-be-patched or to-be-unpatched functions on its stack.
275 		return -EINVAL;  in klp_check_stack()
279 		if (!obj->patched)  in klp_check_stack()
284 				*oldname = func->old_name;  in klp_check_stack()
285 				return -EADDRINUSE;  in klp_check_stack()
298 		return -EBUSY;  in klp_check_and_switch_task()
305 	task->patch_state = klp_target_state;  in klp_check_and_switch_task()
310  * Try to safely switch a task to the target patch state.  If it's currently
311  * running, or it's sleeping on a to-be-patched or to-be-unpatched function, or
320 	if (task->patch_state == klp_target_state)  in klp_try_switch_task()
331 	 * Now try to check the stack for any to-be-patched or to-be-unpatched  in klp_try_switch_task()
332 	 * functions.  If all goes well, switch the task to the target patch  in klp_try_switch_task()
344 	case -EBUSY:	/* klp_check_and_switch_task() */  in klp_try_switch_task()
346 			 __func__, task->comm, task->pid);  in klp_try_switch_task()
348 	case -EINVAL:	/* klp_check_and_switch_task() */  in klp_try_switch_task()
350 			 __func__, task->comm, task->pid);  in klp_try_switch_task()
352 	case -EADDRINUSE: /* klp_check_and_switch_task() */  in klp_try_switch_task()
354 			 __func__, task->comm, task->pid, old_name);  in klp_try_switch_task()
359 			 __func__, ret, task->comm, task->pid);  in klp_try_switch_task()
405  * Sends a fake signal to all non-kthread tasks with TIF_PATCH_PENDING set.
426 		if (task->flags & PF_KTHREAD) {  in klp_send_signals()
434 			 * Send fake signal to all non-kthread tasks which are  in klp_send_signals()
444  * Try to switch all remaining tasks to the target patch state by walking the
445  * stacks of sleeping tasks and looking for any to-be-patched or
446  * to-be-unpatched functions.  If such functions are found, the task can't be
449  * If any tasks are still stuck in the initial patch state, schedule a retry.
455 	struct klp_patch *patch;  in klp_try_complete_transition()  local
461 	 * Try to switch the tasks to the target patch state by walking their  in klp_try_complete_transition()
462 	 * stacks and looking for any to-be-patched or to-be-unpatched  in klp_try_complete_transition()
467 	 * unless the patch includes changes to a very common function.  in klp_try_complete_transition()
487 		} else if (task->patch_state != klp_target_state) {  in klp_try_complete_transition()
490 			task->patch_state = klp_target_state;  in klp_try_complete_transition()
512 	patch = klp_transition_patch;  in klp_try_complete_transition()
520 	if (!patch->enabled)  in klp_try_complete_transition()
521 		klp_free_patch_async(patch);  in klp_try_complete_transition()
522 	else if (patch->replace)  in klp_try_complete_transition()
523 		klp_free_replaced_patches_async(patch);  in klp_try_complete_transition()
527  * Start the transition to the specified target patch state so tasks can begin
538 		  klp_transition_patch->mod->name,  in klp_start_transition()
542 	 * Mark all normal tasks as needing a patch state update.  They'll  in klp_start_transition()
548 		if (task->patch_state != klp_target_state)  in klp_start_transition()
553 	 * Mark all idle tasks as needing a patch state update.  They'll switch  in klp_start_transition()
559 		if (task->patch_state != klp_target_state)  in klp_start_transition()
569  * Initialize the global target patch state and all tasks to the initial patch
573 void klp_init_transition(struct klp_patch *patch, int state)  in klp_init_transition()  argument
583 	klp_transition_patch = patch;  in klp_init_transition()
586 	 * Set the global target patch state which tasks will switch to.  This  in klp_init_transition()
591 	pr_debug("'%s': initializing %s transition\n", patch->mod->name,  in klp_init_transition()
595 	 * Initialize all tasks to the initial patch state to prepare them for  in klp_init_transition()
600 		WARN_ON_ONCE(task->patch_state != KLP_TRANSITION_IDLE);  in klp_init_transition()
601 		task->patch_state = initial_state;  in klp_init_transition()
610 		WARN_ON_ONCE(task->patch_state != KLP_TRANSITION_IDLE);  in klp_init_transition()
611 		task->patch_state = initial_state;  in klp_init_transition()
615 	 * Enforce the order of the task->patch_state initializations and the  in klp_init_transition()
616 	 * func->transition updates to ensure that klp_ftrace_handler() doesn't  in klp_init_transition()
617 	 * see a func in transition with a task->patch_state of KLP_TRANSITION_IDLE.  in klp_init_transition()
621 	 * __klp_sched_try_switch() don't set a task->patch_state to  in klp_init_transition()
637 	klp_for_each_object(patch, obj)  in klp_init_transition()
639 			func->transition = true;  in klp_init_transition()
644  * reverse the direction of the target patch state.  This can be done to
646  * tasks which are stuck in the initial patch state.
654 		 klp_transition_patch->mod->name,  in klp_reverse_transition()
679 	 * All patching has stopped, now re-initialize the global variables to  in klp_reverse_transition()
682 	klp_transition_patch->enabled = !klp_transition_patch->enabled;  in klp_reverse_transition()
689 	 * task->patch_state to the wrong value.  in klp_reverse_transition()
715 	child->patch_state = current->patch_state;  in klp_copy_process()
729 	struct klp_patch *patch;  in klp_force_transition()  local
745 		klp_transition_patch->forced = true;  in klp_force_transition()
746 	else if (klp_transition_patch->replace) {  in klp_force_transition()
747 		klp_for_each_patch(patch) {  in klp_force_transition()
748 			if (patch != klp_transition_patch)  in klp_force_transition()
749 				patch->forced = true;  in klp_force_transition()