1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Functions used by the KMSAN runtime. 4 * 5 * Copyright (C) 2017-2022 Google LLC 6 * Author: Alexander Potapenko <glider@google.com> 7 * 8 */ 9 10 #ifndef __MM_KMSAN_KMSAN_H 11 #define __MM_KMSAN_KMSAN_H 12 13 #include <linux/irqflags.h> 14 #include <linux/kmsan.h> 15 #include <linux/mm.h> 16 #include <linux/nmi.h> 17 #include <linux/pgtable.h> 18 #include <linux/printk.h> 19 #include <linux/sched.h> 20 #include <linux/stackdepot.h> 21 #include <linux/stacktrace.h> 22 23 #define KMSAN_ALLOCA_MAGIC_ORIGIN 0xabcd0100 24 #define KMSAN_CHAIN_MAGIC_ORIGIN 0xabcd0200 25 26 #define KMSAN_POISON_NOCHECK 0x0 27 #define KMSAN_POISON_CHECK 0x1 28 #define KMSAN_POISON_FREE 0x2 29 30 #define KMSAN_ORIGIN_SIZE 4 31 #define KMSAN_MAX_ORIGIN_DEPTH 7 32 33 #define KMSAN_STACK_DEPTH 64 34 35 #define KMSAN_META_SHADOW (false) 36 #define KMSAN_META_ORIGIN (true) 37 38 /* 39 * A pair of metadata pointers to be returned by the instrumentation functions. 40 */ 41 struct shadow_origin_ptr { 42 void *shadow, *origin; 43 }; 44 45 struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *addr, u64 size, 46 bool store); 47 void __init kmsan_init_alloc_meta_for_range(void *start, void *end); 48 49 enum kmsan_bug_reason { 50 REASON_ANY, 51 REASON_COPY_TO_USER, 52 REASON_SUBMIT_URB, 53 }; 54 55 void kmsan_print_origin(depot_stack_handle_t origin); 56 57 /** 58 * kmsan_report() - Report a use of uninitialized value. 59 * @origin: Stack ID of the uninitialized value. 60 * @address: Address at which the memory access happens. 61 * @size: Memory access size. 62 * @off_first: Offset (from @address) of the first byte to be reported. 63 * @off_last: Offset (from @address) of the last byte to be reported. 64 * @user_addr: When non-NULL, denotes the userspace address to which the kernel 65 * is leaking data. 66 * @reason: Error type from enum kmsan_bug_reason. 67 * 68 * kmsan_report() prints an error message for a consequent group of bytes 69 * sharing the same origin. If an uninitialized value is used in a comparison, 70 * this function is called once without specifying the addresses. When checking 71 * a memory range, KMSAN may call kmsan_report() multiple times with the same 72 * @address, @size, @user_addr and @reason, but different @off_first and 73 * @off_last corresponding to different @origin values. 74 */ 75 void kmsan_report(depot_stack_handle_t origin, void *address, int size, 76 int off_first, int off_last, const void __user *user_addr, 77 enum kmsan_bug_reason reason); 78 79 DECLARE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx); 80 kmsan_get_context(void)81 static __always_inline struct kmsan_ctx *kmsan_get_context(void) 82 { 83 return in_task() ? ¤t->kmsan_ctx : raw_cpu_ptr(&kmsan_percpu_ctx); 84 } 85 86 /* 87 * When a compiler hook or KMSAN runtime function is invoked, it may make a 88 * call to instrumented code and eventually call itself recursively. To avoid 89 * that, we guard the runtime entry regions with 90 * kmsan_enter_runtime()/kmsan_leave_runtime() and exit the hook if 91 * kmsan_in_runtime() is true. 92 * 93 * Non-runtime code may occasionally get executed in nested IRQs from the 94 * runtime code (e.g. when called via smp_call_function_single()). Because some 95 * KMSAN routines may take locks (e.g. for memory allocation), we conservatively 96 * bail out instead of calling them. To minimize the effect of this (potentially 97 * missing initialization events) kmsan_in_runtime() is not checked in 98 * non-blocking runtime functions. 99 */ kmsan_in_runtime(void)100 static __always_inline bool kmsan_in_runtime(void) 101 { 102 if ((hardirq_count() >> HARDIRQ_SHIFT) > 1) 103 return true; 104 if (in_nmi()) 105 return true; 106 return kmsan_get_context()->kmsan_in_runtime; 107 } 108 kmsan_enter_runtime(void)109 static __always_inline void kmsan_enter_runtime(void) 110 { 111 struct kmsan_ctx *ctx; 112 113 ctx = kmsan_get_context(); 114 KMSAN_WARN_ON(ctx->kmsan_in_runtime++); 115 } 116 kmsan_leave_runtime(void)117 static __always_inline void kmsan_leave_runtime(void) 118 { 119 struct kmsan_ctx *ctx = kmsan_get_context(); 120 121 KMSAN_WARN_ON(--ctx->kmsan_in_runtime); 122 } 123 124 depot_stack_handle_t kmsan_save_stack(void); 125 depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags, 126 unsigned int extra_bits); 127 128 /* 129 * Pack and unpack the origin chain depth and UAF flag to/from the extra bits 130 * provided by the stack depot. 131 * The UAF flag is stored in the lowest bit, followed by the depth in the upper 132 * bits. 133 * set_dsh_extra_bits() is responsible for clamping the value. 134 */ kmsan_extra_bits(unsigned int depth,bool uaf)135 static __always_inline unsigned int kmsan_extra_bits(unsigned int depth, 136 bool uaf) 137 { 138 return (depth << 1) | uaf; 139 } 140 kmsan_uaf_from_eb(unsigned int extra_bits)141 static __always_inline bool kmsan_uaf_from_eb(unsigned int extra_bits) 142 { 143 return extra_bits & 1; 144 } 145 kmsan_depth_from_eb(unsigned int extra_bits)146 static __always_inline unsigned int kmsan_depth_from_eb(unsigned int extra_bits) 147 { 148 return extra_bits >> 1; 149 } 150 151 /* 152 * kmsan_internal_ functions are supposed to be very simple and not require the 153 * kmsan_in_runtime() checks. 154 */ 155 void kmsan_internal_memmove_metadata(void *dst, void *src, size_t n); 156 void kmsan_internal_poison_memory(void *address, size_t size, gfp_t flags, 157 unsigned int poison_flags); 158 void kmsan_internal_unpoison_memory(void *address, size_t size, bool checked); 159 void kmsan_internal_set_shadow_origin(void *address, size_t size, int b, 160 u32 origin, bool checked); 161 depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id); 162 163 void kmsan_internal_task_create(struct task_struct *task); 164 165 bool kmsan_metadata_is_contiguous(void *addr, size_t size); 166 void kmsan_internal_check_memory(void *addr, size_t size, 167 const void __user *user_addr, int reason); 168 169 struct page *kmsan_vmalloc_to_page_or_null(void *vaddr); 170 void kmsan_setup_meta(struct page *page, struct page *shadow, 171 struct page *origin, int order); 172 173 /* 174 * kmsan_internal_is_module_addr() and kmsan_internal_is_vmalloc_addr() are 175 * non-instrumented versions of is_module_address() and is_vmalloc_addr() that 176 * are safe to call from KMSAN runtime without recursion. 177 */ kmsan_internal_is_module_addr(void * vaddr)178 static inline bool kmsan_internal_is_module_addr(void *vaddr) 179 { 180 return ((u64)vaddr >= MODULES_VADDR) && ((u64)vaddr < MODULES_END); 181 } 182 kmsan_internal_is_vmalloc_addr(void * addr)183 static inline bool kmsan_internal_is_vmalloc_addr(void *addr) 184 { 185 return ((u64)addr >= VMALLOC_START) && ((u64)addr < VMALLOC_END); 186 } 187 188 #endif /* __MM_KMSAN_KMSAN_H */ 189