Lines Matching +full:patch +full:- +full:address
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * core.c - Kernel Live Patching Core
25 #include "patch.h"
31 * accesses to klp-related variables and structures must have mutex protection,
34 * - klp_ftrace_handler()
35 * - klp_update_patch_state()
36 * - __klp_sched_try_switch()
51 return obj->name; in klp_is_module()
54 /* sets obj->mod if object is not vmlinux and module is found */
68 mod = find_module(obj->name); in klp_find_object_module()
71 * Note that the patch might still be needed before klp_module_going() in klp_find_object_module()
73 * until mod->exit() finishes. This is especially important for in klp_find_object_module()
76 if (mod && mod->klp_alive) in klp_find_object_module()
77 obj->mod = mod; in klp_find_object_module()
93 if ((strcmp(old_func->old_name, func->old_name) == 0) && in klp_find_func()
94 (old_func->old_sympos == func->old_sympos)) { in klp_find_func()
102 static struct klp_object *klp_find_object(struct klp_patch *patch, in klp_find_object() argument
107 klp_for_each_object(patch, obj) { in klp_find_object()
110 strcmp(old_obj->name, obj->name) == 0) { in klp_find_object()
132 args->addr = addr; in klp_match_callback()
133 args->count++; in klp_match_callback()
137 * or the position is not defined for a non-unique symbol. in klp_match_callback()
139 if ((args->pos && (args->count == args->pos)) || in klp_match_callback()
140 (!args->pos && (args->count > 1))) in klp_match_callback()
150 if (strcmp(args->name, name)) in klp_find_callback()
172 * Ensure an address was found. If sympos is 0, ensure symbol is unique; in klp_find_object_symbol()
189 return -EINVAL; in klp_find_object_symbol()
207 * call are hard-coded and correspond to MODULE_NAME_LEN and in klp_resolve_symbols()
217 relas = (Elf_Rela *) relasec->sh_addr; in klp_resolve_symbols()
219 for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) { in klp_resolve_symbols()
221 if (sym->st_shndx != SHN_LIVEPATCH) { in klp_resolve_symbols()
223 strtab + sym->st_name); in klp_resolve_symbols()
224 return -EINVAL; in klp_resolve_symbols()
228 cnt = sscanf(strtab + sym->st_name, in klp_resolve_symbols()
233 strtab + sym->st_name); in klp_resolve_symbols()
234 return -EINVAL; in klp_resolve_symbols()
240 * Prevent module-specific KLP rela sections from referencing in klp_resolve_symbols()
246 …pr_err("invalid access to vmlinux symbol '%s' from module-specific livepatch relocation section\n", in klp_resolve_symbols()
248 return -EINVAL; in klp_resolve_symbols()
257 sym->st_value = addr; in klp_resolve_symbols()
272 * At a high-level, there are two types of klp relocation sections: those which
277 * write vmlinux-specific klp relocations (.klp.rela.vmlinux.* sections).
284 * 2) When a to-be-patched module loads -- or is already loaded when a
285 * corresponding klp module loads -- klp code calls this function to write
286 * module-specific klp relocations (.klp.rela.{module}.* sections). These
288 * reference symbols which live in the to-be-patched module or one of its
291 * the to-be-patched module to be loaded and patched sometime *after* the
308 cnt = sscanf(shstrtab + sec->sh_name, ".klp.rela.%55[^.]", in klp_write_section_relocs()
312 shstrtab + sec->sh_name); in klp_write_section_relocs()
313 return -EINVAL; in klp_write_section_relocs()
345 * /sys/kernel/livepatch/<patch>
346 * /sys/kernel/livepatch/<patch>/enabled
347 * /sys/kernel/livepatch/<patch>/transition
348 * /sys/kernel/livepatch/<patch>/force
349 * /sys/kernel/livepatch/<patch>/replace
350 * /sys/kernel/livepatch/<patch>/<object>
351 * /sys/kernel/livepatch/<patch>/<object>/patched
352 * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
354 static int __klp_disable_patch(struct klp_patch *patch);
359 struct klp_patch *patch; in enabled_store() local
367 patch = container_of(kobj, struct klp_patch, kobj); in enabled_store()
371 if (patch->enabled == enabled) { in enabled_store()
373 ret = -EINVAL; in enabled_store()
382 * Do not allow to re-enable a disabled patch. in enabled_store()
384 if (patch == klp_transition_patch) in enabled_store()
387 ret = __klp_disable_patch(patch); in enabled_store()
389 ret = -EINVAL; in enabled_store()
402 struct klp_patch *patch; in enabled_show() local
404 patch = container_of(kobj, struct klp_patch, kobj); in enabled_show()
405 return sysfs_emit(buf, "%d\n", patch->enabled); in enabled_show()
411 struct klp_patch *patch; in transition_show() local
413 patch = container_of(kobj, struct klp_patch, kobj); in transition_show()
414 return sysfs_emit(buf, "%d\n", patch == klp_transition_patch); in transition_show()
420 struct klp_patch *patch; in force_store() local
433 patch = container_of(kobj, struct klp_patch, kobj); in force_store()
434 if (patch != klp_transition_patch) { in force_store()
436 return -EINVAL; in force_store()
449 struct klp_patch *patch; in replace_show() local
451 patch = container_of(kobj, struct klp_patch, kobj); in replace_show()
452 return sysfs_emit(buf, "%d\n", patch->replace); in replace_show()
474 return sysfs_emit(buf, "%d\n", obj->patched); in patched_show()
486 kfree(obj->name); in klp_free_object_dynamic()
492 static void klp_init_object_early(struct klp_patch *patch,
496 struct klp_patch *patch) in klp_alloc_object_dynamic() argument
505 obj->name = kstrdup(name, GFP_KERNEL); in klp_alloc_object_dynamic()
506 if (!obj->name) { in klp_alloc_object_dynamic()
512 klp_init_object_early(patch, obj); in klp_alloc_object_dynamic()
513 obj->dynamic = true; in klp_alloc_object_dynamic()
520 kfree(func->old_name); in klp_free_func_nop()
533 if (old_func->old_name) { in klp_alloc_func_nop()
534 func->old_name = kstrdup(old_func->old_name, GFP_KERNEL); in klp_alloc_func_nop()
535 if (!func->old_name) { in klp_alloc_func_nop()
543 * func->new_func is same as func->old_func. These addresses are in klp_alloc_func_nop()
546 func->old_sympos = old_func->old_sympos; in klp_alloc_func_nop()
547 func->nop = true; in klp_alloc_func_nop()
552 static int klp_add_object_nops(struct klp_patch *patch, in klp_add_object_nops() argument
558 obj = klp_find_object(patch, old_obj); in klp_add_object_nops()
561 obj = klp_alloc_object_dynamic(old_obj->name, patch); in klp_add_object_nops()
563 return -ENOMEM; in klp_add_object_nops()
573 return -ENOMEM; in klp_add_object_nops()
582 * patch to facilitate a 'replace' mode.
584 static int klp_add_nops(struct klp_patch *patch) in klp_add_nops() argument
593 err = klp_add_object_nops(patch, old_obj); in klp_add_nops()
604 struct klp_patch *patch; in klp_kobj_release_patch() local
606 patch = container_of(kobj, struct klp_patch, kobj); in klp_kobj_release_patch()
607 complete(&patch->finish); in klp_kobj_release_patch()
622 if (obj->dynamic) in klp_kobj_release_object()
638 if (func->nop) in klp_kobj_release_func()
652 if (nops_only && !func->nop) in __klp_free_funcs()
655 list_del(&func->node); in __klp_free_funcs()
656 kobject_put(&func->kobj); in __klp_free_funcs()
665 obj->mod = NULL; in klp_free_object_loaded()
668 func->old_func = NULL; in klp_free_object_loaded()
670 if (func->nop) in klp_free_object_loaded()
671 func->new_func = NULL; in klp_free_object_loaded()
675 static void __klp_free_objects(struct klp_patch *patch, bool nops_only) in __klp_free_objects() argument
679 klp_for_each_object_safe(patch, obj, tmp_obj) { in __klp_free_objects()
682 if (nops_only && !obj->dynamic) in __klp_free_objects()
685 list_del(&obj->node); in __klp_free_objects()
686 kobject_put(&obj->kobj); in __klp_free_objects()
690 static void klp_free_objects(struct klp_patch *patch) in klp_free_objects() argument
692 __klp_free_objects(patch, false); in klp_free_objects()
695 static void klp_free_objects_dynamic(struct klp_patch *patch) in klp_free_objects_dynamic() argument
697 __klp_free_objects(patch, true); in klp_free_objects_dynamic()
707 static void klp_free_patch_start(struct klp_patch *patch) in klp_free_patch_start() argument
709 if (!list_empty(&patch->list)) in klp_free_patch_start()
710 list_del(&patch->list); in klp_free_patch_start()
712 klp_free_objects(patch); in klp_free_patch_start()
720 * the last function accessing the livepatch structures when the patch
723 static void klp_free_patch_finish(struct klp_patch *patch) in klp_free_patch_finish() argument
728 * this is called when the patch gets disabled and it in klp_free_patch_finish()
731 kobject_put(&patch->kobj); in klp_free_patch_finish()
732 wait_for_completion(&patch->finish); in klp_free_patch_finish()
735 if (!patch->forced) in klp_free_patch_finish()
736 module_put(patch->mod); in klp_free_patch_finish()
740 * The livepatch might be freed from sysfs interface created by the patch.
746 struct klp_patch *patch = in klp_free_patch_work_fn() local
749 klp_free_patch_finish(patch); in klp_free_patch_work_fn()
752 void klp_free_patch_async(struct klp_patch *patch) in klp_free_patch_async() argument
754 klp_free_patch_start(patch); in klp_free_patch_async()
755 schedule_work(&patch->free_work); in klp_free_patch_async()
771 if (!func->old_name) in klp_init_func()
772 return -EINVAL; in klp_init_func()
775 * NOPs get the address later. The patched module must be loaded, in klp_init_func()
778 if (!func->new_func && !func->nop) in klp_init_func()
779 return -EINVAL; in klp_init_func()
781 if (strlen(func->old_name) >= KSYM_NAME_LEN) in klp_init_func()
782 return -EINVAL; in klp_init_func()
784 INIT_LIST_HEAD(&func->stack_node); in klp_init_func()
785 func->patched = false; in klp_init_func()
786 func->transition = false; in klp_init_func()
793 return kobject_add(&func->kobj, &obj->kobj, "%s,%lu", in klp_init_func()
794 func->old_name, in klp_init_func()
795 func->old_sympos ? func->old_sympos : 1); in klp_init_func()
798 static int klp_write_object_relocs(struct klp_patch *patch, in klp_write_object_relocs() argument
803 struct klp_modinfo *info = patch->mod->klp_info; in klp_write_object_relocs()
805 for (i = 1; i < info->hdr.e_shnum; i++) { in klp_write_object_relocs()
806 Elf_Shdr *sec = info->sechdrs + i; in klp_write_object_relocs()
808 if (!(sec->sh_flags & SHF_RELA_LIVEPATCH)) in klp_write_object_relocs()
811 ret = klp_write_section_relocs(patch->mod, info->sechdrs, in klp_write_object_relocs()
812 info->secstrings, in klp_write_object_relocs()
813 patch->mod->core_kallsyms.strtab, in klp_write_object_relocs()
814 info->symndx, i, obj->name, apply); in klp_write_object_relocs()
822 static int klp_apply_object_relocs(struct klp_patch *patch, in klp_apply_object_relocs() argument
825 return klp_write_object_relocs(patch, obj, true); in klp_apply_object_relocs()
828 static void klp_clear_object_relocs(struct klp_patch *patch, in klp_clear_object_relocs() argument
831 klp_write_object_relocs(patch, obj, false); in klp_clear_object_relocs()
835 static int klp_init_object_loaded(struct klp_patch *patch, in klp_init_object_loaded() argument
843 * Only write module-specific relocations here in klp_init_object_loaded()
844 * (.klp.rela.{module}.*). vmlinux-specific relocations were in klp_init_object_loaded()
848 ret = klp_apply_object_relocs(patch, obj); in klp_init_object_loaded()
854 ret = klp_find_object_symbol(obj->name, func->old_name, in klp_init_object_loaded()
855 func->old_sympos, in klp_init_object_loaded()
856 (unsigned long *)&func->old_func); in klp_init_object_loaded()
860 ret = kallsyms_lookup_size_offset((unsigned long)func->old_func, in klp_init_object_loaded()
861 &func->old_size, NULL); in klp_init_object_loaded()
864 func->old_name); in klp_init_object_loaded()
865 return -ENOENT; in klp_init_object_loaded()
868 if (func->nop) in klp_init_object_loaded()
869 func->new_func = func->old_func; in klp_init_object_loaded()
871 ret = kallsyms_lookup_size_offset((unsigned long)func->new_func, in klp_init_object_loaded()
872 &func->new_size, NULL); in klp_init_object_loaded()
875 func->old_name); in klp_init_object_loaded()
876 return -ENOENT; in klp_init_object_loaded()
883 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) in klp_init_object() argument
889 if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN) in klp_init_object()
890 return -EINVAL; in klp_init_object()
892 obj->patched = false; in klp_init_object()
893 obj->mod = NULL; in klp_init_object()
897 name = klp_is_module(obj) ? obj->name : "vmlinux"; in klp_init_object()
898 ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name); in klp_init_object()
909 ret = klp_init_object_loaded(patch, obj); in klp_init_object()
917 kobject_init(&func->kobj, &klp_ktype_func); in klp_init_func_early()
918 list_add_tail(&func->node, &obj->func_list); in klp_init_func_early()
921 static void klp_init_object_early(struct klp_patch *patch, in klp_init_object_early() argument
924 INIT_LIST_HEAD(&obj->func_list); in klp_init_object_early()
925 kobject_init(&obj->kobj, &klp_ktype_object); in klp_init_object_early()
926 list_add_tail(&obj->node, &patch->obj_list); in klp_init_object_early()
929 static void klp_init_patch_early(struct klp_patch *patch) in klp_init_patch_early() argument
934 INIT_LIST_HEAD(&patch->list); in klp_init_patch_early()
935 INIT_LIST_HEAD(&patch->obj_list); in klp_init_patch_early()
936 kobject_init(&patch->kobj, &klp_ktype_patch); in klp_init_patch_early()
937 patch->enabled = false; in klp_init_patch_early()
938 patch->forced = false; in klp_init_patch_early()
939 INIT_WORK(&patch->free_work, klp_free_patch_work_fn); in klp_init_patch_early()
940 init_completion(&patch->finish); in klp_init_patch_early()
942 klp_for_each_object_static(patch, obj) { in klp_init_patch_early()
943 klp_init_object_early(patch, obj); in klp_init_patch_early()
951 static int klp_init_patch(struct klp_patch *patch) in klp_init_patch() argument
956 ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name); in klp_init_patch()
960 if (patch->replace) { in klp_init_patch()
961 ret = klp_add_nops(patch); in klp_init_patch()
966 klp_for_each_object(patch, obj) { in klp_init_patch()
967 ret = klp_init_object(patch, obj); in klp_init_patch()
972 list_add_tail(&patch->list, &klp_patches); in klp_init_patch()
977 static int __klp_disable_patch(struct klp_patch *patch) in __klp_disable_patch() argument
981 if (WARN_ON(!patch->enabled)) in __klp_disable_patch()
982 return -EINVAL; in __klp_disable_patch()
985 return -EBUSY; in __klp_disable_patch()
987 klp_init_transition(patch, KLP_TRANSITION_UNPATCHED); in __klp_disable_patch()
989 klp_for_each_object(patch, obj) in __klp_disable_patch()
990 if (obj->patched) in __klp_disable_patch()
994 * Enforce the order of the func->transition writes in in __klp_disable_patch()
998 * this ensures the handler sees that func->transition is set. in __klp_disable_patch()
1003 patch->enabled = false; in __klp_disable_patch()
1009 static int __klp_enable_patch(struct klp_patch *patch) in __klp_enable_patch() argument
1015 return -EBUSY; in __klp_enable_patch()
1017 if (WARN_ON(patch->enabled)) in __klp_enable_patch()
1018 return -EINVAL; in __klp_enable_patch()
1020 pr_notice("enabling patch '%s'\n", patch->mod->name); in __klp_enable_patch()
1022 klp_init_transition(patch, KLP_TRANSITION_PATCHED); in __klp_enable_patch()
1025 * Enforce the order of the func->transition writes in in __klp_enable_patch()
1026 * klp_init_transition() and the ops->func_stack writes in in __klp_enable_patch()
1028 * func->transition updates before the handler is registered and the in __klp_enable_patch()
1033 klp_for_each_object(patch, obj) { in __klp_enable_patch()
1039 pr_warn("pre-patch callback failed for object '%s'\n", in __klp_enable_patch()
1040 klp_is_module(obj) ? obj->name : "vmlinux"); in __klp_enable_patch()
1046 pr_warn("failed to patch object '%s'\n", in __klp_enable_patch()
1047 klp_is_module(obj) ? obj->name : "vmlinux"); in __klp_enable_patch()
1053 patch->enabled = true; in __klp_enable_patch()
1058 pr_warn("failed to enable patch '%s'\n", patch->mod->name); in __klp_enable_patch()
1065 * klp_enable_patch() - enable the livepatch
1066 * @patch: patch to be enabled
1068 * Initializes the data structure associated with the patch, creates the sysfs
1077 int klp_enable_patch(struct klp_patch *patch) in klp_enable_patch() argument
1082 if (!patch || !patch->mod || !patch->objs) in klp_enable_patch()
1083 return -EINVAL; in klp_enable_patch()
1085 klp_for_each_object_static(patch, obj) { in klp_enable_patch()
1086 if (!obj->funcs) in klp_enable_patch()
1087 return -EINVAL; in klp_enable_patch()
1091 if (!is_livepatch_module(patch->mod)) { in klp_enable_patch()
1093 patch->mod->name); in klp_enable_patch()
1094 return -EINVAL; in klp_enable_patch()
1098 return -ENODEV; in klp_enable_patch()
1107 if (!klp_is_patch_compatible(patch)) { in klp_enable_patch()
1108 pr_err("Livepatch patch (%s) is not compatible with the already installed livepatches.\n", in klp_enable_patch()
1109 patch->mod->name); in klp_enable_patch()
1111 return -EINVAL; in klp_enable_patch()
1114 if (!try_module_get(patch->mod)) { in klp_enable_patch()
1116 return -ENODEV; in klp_enable_patch()
1119 klp_init_patch_early(patch); in klp_enable_patch()
1121 ret = klp_init_patch(patch); in klp_enable_patch()
1125 ret = __klp_enable_patch(patch); in klp_enable_patch()
1134 klp_free_patch_start(patch); in klp_enable_patch()
1138 klp_free_patch_finish(patch); in klp_enable_patch()
1154 * klp_ftrace_handler() might still see the replaced patch on the stack.
1157 * this is handled transparently by patch->module_put.
1167 old_patch->enabled = false; in klp_unpatch_replaced_patches()
1179 * will see a valid ops->func_stack entry thanks to RCU.
1182 * in ops->func_stack. Therefore unregister_ftrace_function() is called.
1202 struct klp_patch *patch; in klp_cleanup_module_patches_limited() local
1205 klp_for_each_patch(patch) { in klp_cleanup_module_patches_limited()
1206 if (patch == limit) in klp_cleanup_module_patches_limited()
1209 klp_for_each_object(patch, obj) { in klp_cleanup_module_patches_limited()
1210 if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) in klp_cleanup_module_patches_limited()
1213 if (patch != klp_transition_patch) in klp_cleanup_module_patches_limited()
1216 pr_notice("reverting patch '%s' on unloading module '%s'\n", in klp_cleanup_module_patches_limited()
1217 patch->mod->name, obj->mod->name); in klp_cleanup_module_patches_limited()
1221 klp_clear_object_relocs(patch, obj); in klp_cleanup_module_patches_limited()
1231 struct klp_patch *patch; in klp_module_coming() local
1234 if (WARN_ON(mod->state != MODULE_STATE_COMING)) in klp_module_coming()
1235 return -EINVAL; in klp_module_coming()
1237 if (!strcmp(mod->name, "vmlinux")) { in klp_module_coming()
1239 return -EINVAL; in klp_module_coming()
1246 * get patched by a new patch. in klp_module_coming()
1248 mod->klp_alive = true; in klp_module_coming()
1250 klp_for_each_patch(patch) { in klp_module_coming()
1251 klp_for_each_object(patch, obj) { in klp_module_coming()
1252 if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) in klp_module_coming()
1255 obj->mod = mod; in klp_module_coming()
1257 ret = klp_init_object_loaded(patch, obj); in klp_module_coming()
1259 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n", in klp_module_coming()
1260 patch->mod->name, obj->mod->name, ret); in klp_module_coming()
1264 pr_notice("applying patch '%s' to loading module '%s'\n", in klp_module_coming()
1265 patch->mod->name, obj->mod->name); in klp_module_coming()
1269 pr_warn("pre-patch callback failed for object '%s'\n", in klp_module_coming()
1270 obj->name); in klp_module_coming()
1276 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n", in klp_module_coming()
1277 patch->mod->name, obj->mod->name, ret); in klp_module_coming()
1283 if (patch != klp_transition_patch) in klp_module_coming()
1296 * If a patch is unsuccessfully applied, return in klp_module_coming()
1299 pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n", in klp_module_coming()
1300 patch->mod->name, obj->mod->name, obj->mod->name); in klp_module_coming()
1301 mod->klp_alive = false; in klp_module_coming()
1302 obj->mod = NULL; in klp_module_coming()
1303 klp_cleanup_module_patches_limited(mod, patch); in klp_module_coming()
1311 if (WARN_ON(mod->state != MODULE_STATE_GOING && in klp_module_going()
1312 mod->state != MODULE_STATE_COMING)) in klp_module_going()
1319 * get patched by a new patch. in klp_module_going()
1321 mod->klp_alive = false; in klp_module_going()
1332 return -ENOMEM; in klp_init()