1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2017-2022 Google LLC
22 #define shadow_page_for(page) ((page)->kmsan_shadow)  argument
24 #define origin_page_for(page) ((page)->kmsan_origin)  argument
26 static void *shadow_ptr_for(struct page *page)  in shadow_ptr_for()  argument
28 	return page_address(shadow_page_for(page));  in shadow_ptr_for()
31 static void *origin_ptr_for(struct page *page)  in origin_ptr_for()  argument
33 	return page_address(origin_page_for(page));  in origin_ptr_for()
36 static bool page_has_metadata(struct page *page)  in page_has_metadata()  argument
38 	return shadow_page_for(page) && origin_page_for(page);  in page_has_metadata()
41 static void set_no_shadow_origin_page(struct page *page)  in set_no_shadow_origin_page()  argument
43 	shadow_page_for(page) = NULL;  in set_no_shadow_origin_page()
44 	origin_page_for(page) = NULL;  in set_no_shadow_origin_page()
61 		off = addr64 - VMALLOC_START;  in vmalloc_meta()
66 		off = addr64 - MODULES_VADDR;  in vmalloc_meta()
73 static struct page *virt_to_page_or_null(void *vaddr)  in virt_to_page_or_null()
81 struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,  in kmsan_get_shadow_origin_ptr()  argument
88 	 * Even if we redirect this memory access to the dummy page, it will  in kmsan_get_shadow_origin_ptr()
91 	KMSAN_WARN_ON(size > PAGE_SIZE);  in kmsan_get_shadow_origin_ptr()
96 	KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(address, size));  in kmsan_get_shadow_origin_ptr()
120  * none. The caller must check the return value for being non-NULL if needed.
127 	struct page *page;  in kmsan_get_metadata()  local
141 	page = virt_to_page_or_null(address);  in kmsan_get_metadata()
142 	if (!page)  in kmsan_get_metadata()
144 	if (!page_has_metadata(page))  in kmsan_get_metadata()
148 	return (is_origin ? origin_ptr_for(page) : shadow_ptr_for(page)) + off;  in kmsan_get_metadata()
151 void kmsan_copy_page_meta(struct page *dst, struct page *src)  in kmsan_copy_page_meta()
170 void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)  in kmsan_alloc_page()  argument
173 	struct page *shadow, *origin;  in kmsan_alloc_page()
177 	if (!page)  in kmsan_alloc_page()
180 	shadow = shadow_page_for(page);  in kmsan_alloc_page()
181 	origin = origin_page_for(page);  in kmsan_alloc_page()
193 	__memset(page_address(shadow), -1, PAGE_SIZE * pages);  in kmsan_alloc_page()
198 	 * Addresses are page-aligned, pages are contiguous, so it's ok  in kmsan_alloc_page()
205 void kmsan_free_page(struct page *page, unsigned int order)  in kmsan_free_page()  argument
210 	kmsan_internal_poison_memory(page_address(page),  in kmsan_free_page()
211 				     page_size(page),  in kmsan_free_page()
218 				   pgprot_t prot, struct page **pages,  in kmsan_vmap_pages_range_noflush()
222 	struct page **s_pages, **o_pages;  in kmsan_vmap_pages_range_noflush()
233 	nr = (end - start) / PAGE_SIZE;  in kmsan_vmap_pages_range_noflush()
237 		err = -ENOMEM;  in kmsan_vmap_pages_range_noflush()
276 	struct page *shadow_p, *origin_p;  in kmsan_init_alloc_meta_for_range()
278 	struct page *page;  in kmsan_init_alloc_meta_for_range()  local
279 	u64 size;  in kmsan_init_alloc_meta_for_range()  local
282 	size = PAGE_ALIGN((u64)end - (u64)start);  in kmsan_init_alloc_meta_for_range()
283 	shadow = memblock_alloc(size, PAGE_SIZE);  in kmsan_init_alloc_meta_for_range()
284 	origin = memblock_alloc(size, PAGE_SIZE);  in kmsan_init_alloc_meta_for_range()
287 		panic("%s: Failed to allocate metadata memory for early boot range of size %llu",  in kmsan_init_alloc_meta_for_range()
288 		      __func__, size);  in kmsan_init_alloc_meta_for_range()
290 	for (u64 addr = 0; addr < size; addr += PAGE_SIZE) {  in kmsan_init_alloc_meta_for_range()
291 		page = virt_to_page_or_null((char *)start + addr);  in kmsan_init_alloc_meta_for_range()
294 		shadow_page_for(page) = shadow_p;  in kmsan_init_alloc_meta_for_range()
297 		origin_page_for(page) = origin_p;  in kmsan_init_alloc_meta_for_range()
301 void kmsan_setup_meta(struct page *page, struct page *shadow,  in kmsan_setup_meta()  argument
302 		      struct page *origin, int order)  in kmsan_setup_meta()
307 		shadow_page_for(&page[i]) = &shadow[i];  in kmsan_setup_meta()
308 		origin_page_for(&page[i]) = &origin[i];  in kmsan_setup_meta()