1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 /* 4 * This header provides generic wrappers for memory access instrumentation that 5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN. 6 */ 7 #ifndef _LINUX_INSTRUMENTED_H 8 #define _LINUX_INSTRUMENTED_H 9 10 #include <linux/compiler.h> 11 #include <linux/kasan-checks.h> 12 #include <linux/kcsan-checks.h> 13 #include <linux/kmsan-checks.h> 14 #include <linux/types.h> 15 16 /** 17 * instrument_read - instrument regular read access 18 * @v: address of access 19 * @size: size of access 20 * 21 * Instrument a regular read access. The instrumentation should be inserted 22 * before the actual read happens. 23 */ instrument_read(const volatile void * v,size_t size)24 static __always_inline void instrument_read(const volatile void *v, size_t size) 25 { 26 kasan_check_read(v, size); 27 kcsan_check_read(v, size); 28 } 29 30 /** 31 * instrument_write - instrument regular write access 32 * @v: address of access 33 * @size: size of access 34 * 35 * Instrument a regular write access. The instrumentation should be inserted 36 * before the actual write happens. 37 */ instrument_write(const volatile void * v,size_t size)38 static __always_inline void instrument_write(const volatile void *v, size_t size) 39 { 40 kasan_check_write(v, size); 41 kcsan_check_write(v, size); 42 } 43 44 /** 45 * instrument_read_write - instrument regular read-write access 46 * @v: address of access 47 * @size: size of access 48 * 49 * Instrument a regular write access. The instrumentation should be inserted 50 * before the actual write happens. 51 */ instrument_read_write(const volatile void * v,size_t size)52 static __always_inline void instrument_read_write(const volatile void *v, size_t size) 53 { 54 kasan_check_write(v, size); 55 kcsan_check_read_write(v, size); 56 } 57 58 /** 59 * instrument_atomic_read - instrument atomic read access 60 * @v: address of access 61 * @size: size of access 62 * 63 * Instrument an atomic read access. The instrumentation should be inserted 64 * before the actual read happens. 65 */ instrument_atomic_read(const volatile void * v,size_t size)66 static __always_inline void instrument_atomic_read(const volatile void *v, size_t size) 67 { 68 kasan_check_read(v, size); 69 kcsan_check_atomic_read(v, size); 70 } 71 72 /** 73 * instrument_atomic_write - instrument atomic write access 74 * @v: address of access 75 * @size: size of access 76 * 77 * Instrument an atomic write access. The instrumentation should be inserted 78 * before the actual write happens. 79 */ instrument_atomic_write(const volatile void * v,size_t size)80 static __always_inline void instrument_atomic_write(const volatile void *v, size_t size) 81 { 82 kasan_check_write(v, size); 83 kcsan_check_atomic_write(v, size); 84 } 85 86 /** 87 * instrument_atomic_read_write - instrument atomic read-write access 88 * @v: address of access 89 * @size: size of access 90 * 91 * Instrument an atomic read-write access. The instrumentation should be 92 * inserted before the actual write happens. 93 */ instrument_atomic_read_write(const volatile void * v,size_t size)94 static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size) 95 { 96 kasan_check_write(v, size); 97 kcsan_check_atomic_read_write(v, size); 98 } 99 100 /** 101 * instrument_copy_to_user - instrument reads of copy_to_user 102 * @to: destination address 103 * @from: source address 104 * @n: number of bytes to copy 105 * 106 * Instrument reads from kernel memory, that are due to copy_to_user (and 107 * variants). The instrumentation must be inserted before the accesses. 108 */ 109 static __always_inline void instrument_copy_to_user(void __user * to,const void * from,unsigned long n)110 instrument_copy_to_user(void __user *to, const void *from, unsigned long n) 111 { 112 kasan_check_read(from, n); 113 kcsan_check_read(from, n); 114 kmsan_copy_to_user(to, from, n, 0); 115 } 116 117 /** 118 * instrument_copy_from_user_before - add instrumentation before copy_from_user 119 * @to: destination address 120 * @from: source address 121 * @n: number of bytes to copy 122 * 123 * Instrument writes to kernel memory, that are due to copy_from_user (and 124 * variants). The instrumentation should be inserted before the accesses. 125 */ 126 static __always_inline void instrument_copy_from_user_before(const void * to,const void __user * from,unsigned long n)127 instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n) 128 { 129 kasan_check_write(to, n); 130 kcsan_check_write(to, n); 131 } 132 133 /** 134 * instrument_copy_from_user_after - add instrumentation after copy_from_user 135 * @to: destination address 136 * @from: source address 137 * @n: number of bytes to copy 138 * @left: number of bytes not copied (as returned by copy_from_user) 139 * 140 * Instrument writes to kernel memory, that are due to copy_from_user (and 141 * variants). The instrumentation should be inserted after the accesses. 142 */ 143 static __always_inline void instrument_copy_from_user_after(const void * to,const void __user * from,unsigned long n,unsigned long left)144 instrument_copy_from_user_after(const void *to, const void __user *from, 145 unsigned long n, unsigned long left) 146 { 147 kmsan_unpoison_memory(to, n - left); 148 } 149 150 /** 151 * instrument_memcpy_before - add instrumentation before non-instrumented memcpy 152 * @to: destination address 153 * @from: source address 154 * @n: number of bytes to copy 155 * 156 * Instrument memory accesses that happen in custom memcpy implementations. The 157 * instrumentation should be inserted before the memcpy call. 158 */ instrument_memcpy_before(void * to,const void * from,unsigned long n)159 static __always_inline void instrument_memcpy_before(void *to, const void *from, 160 unsigned long n) 161 { 162 kasan_check_write(to, n); 163 kasan_check_read(from, n); 164 kcsan_check_write(to, n); 165 kcsan_check_read(from, n); 166 } 167 168 /** 169 * instrument_memcpy_after - add instrumentation after non-instrumented memcpy 170 * @to: destination address 171 * @from: source address 172 * @n: number of bytes to copy 173 * @left: number of bytes not copied (if known) 174 * 175 * Instrument memory accesses that happen in custom memcpy implementations. The 176 * instrumentation should be inserted after the memcpy call. 177 */ instrument_memcpy_after(void * to,const void * from,unsigned long n,unsigned long left)178 static __always_inline void instrument_memcpy_after(void *to, const void *from, 179 unsigned long n, 180 unsigned long left) 181 { 182 kmsan_memmove(to, from, n - left); 183 } 184 185 /** 186 * instrument_get_user() - add instrumentation to get_user()-like macros 187 * @to: destination variable, may not be address-taken 188 * 189 * get_user() and friends are fragile, so it may depend on the implementation 190 * whether the instrumentation happens before or after the data is copied from 191 * the userspace. 192 */ 193 #define instrument_get_user(to) \ 194 ({ \ 195 u64 __tmp = (u64)(to); \ 196 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \ 197 to = __tmp; \ 198 }) 199 200 201 /** 202 * instrument_put_user() - add instrumentation to put_user()-like macros 203 * @from: source address 204 * @ptr: userspace pointer to copy to 205 * @size: number of bytes to copy 206 * 207 * put_user() and friends are fragile, so it may depend on the implementation 208 * whether the instrumentation happens before or after the data is copied from 209 * the userspace. 210 */ 211 #define instrument_put_user(from, ptr, size) \ 212 ({ \ 213 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \ 214 }) 215 216 #endif /* _LINUX_INSTRUMENTED_H */ 217