1 // SPDX-License-Identifier: GPL-2.0
11  * struct wp_walk - Private struct for pagetable walk callbacks
25  * wp_pte - Write-protect a pte
31  * The function write-protects a pte and records the range in
37 	struct wp_walk *wpwalk = walk->private;  in wp_pte()
41 		pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte);  in wp_pte()
44 		ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent);  in wp_pte()
45 		wpwalk->total++;  in wp_pte()
46 		wpwalk->tlbflush_start = min(wpwalk->tlbflush_start, addr);  in wp_pte()
47 		wpwalk->tlbflush_end = max(wpwalk->tlbflush_end,  in wp_pte()
55  * struct clean_walk - Private struct for the clean_record_pte function.
76  * clean_record_pte - Clean a pte and record its address space offset in a
92 	struct wp_walk *wpwalk = walk->private;  in clean_record_pte()
97 		pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) +  in clean_record_pte()
98 			walk->vma->vm_pgoff - cwalk->bitmap_pgoff;  in clean_record_pte()
99 		pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte);  in clean_record_pte()
102 		ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent);  in clean_record_pte()
104 		wpwalk->total++;  in clean_record_pte()
105 		wpwalk->tlbflush_start = min(wpwalk->tlbflush_start, addr);  in clean_record_pte()
106 		wpwalk->tlbflush_end = max(wpwalk->tlbflush_end,  in clean_record_pte()
109 		__set_bit(pgoff, cwalk->bitmap);  in clean_record_pte()
110 		cwalk->start = min(cwalk->start, pgoff);  in clean_record_pte()
111 		cwalk->end = max(cwalk->end, pgoff + 1);  in clean_record_pte()
118  * wp_clean_pmd_entry - The pagewalk pmd callback.
120  * Dirty-tracking should take place on the PTE level, so
134 		walk->action = ACTION_CONTINUE;  in wp_clean_pmd_entry()
140  * wp_clean_pud_entry - The pagewalk pud callback.
142  * Dirty-tracking should take place on the PTE level, so
157 		walk->action = ACTION_CONTINUE;  in wp_clean_pud_entry()
164  * wp_clean_pre_vma - The pagewalk pre_vma callback.
172 	struct wp_walk *wpwalk = walk->private;  in wp_clean_pre_vma()
174 	wpwalk->tlbflush_start = end;  in wp_clean_pre_vma()
175 	wpwalk->tlbflush_end = start;  in wp_clean_pre_vma()
177 	mmu_notifier_range_init(&wpwalk->range, MMU_NOTIFY_PROTECTION_PAGE, 0,  in wp_clean_pre_vma()
178 				walk->mm, start, end);  in wp_clean_pre_vma()
179 	mmu_notifier_invalidate_range_start(&wpwalk->range);  in wp_clean_pre_vma()
180 	flush_cache_range(walk->vma, start, end);  in wp_clean_pre_vma()
187 	inc_tlb_flush_pending(walk->mm);  in wp_clean_pre_vma()
193  * wp_clean_post_vma - The pagewalk post_vma callback.
200 	struct wp_walk *wpwalk = walk->private;  in wp_clean_post_vma()
202 	if (mm_tlb_flush_nested(walk->mm))  in wp_clean_post_vma()
203 		flush_tlb_range(walk->vma, wpwalk->range.start,  in wp_clean_post_vma()
204 				wpwalk->range.end);  in wp_clean_post_vma()
205 	else if (wpwalk->tlbflush_end > wpwalk->tlbflush_start)  in wp_clean_post_vma()
206 		flush_tlb_range(walk->vma, wpwalk->tlbflush_start,  in wp_clean_post_vma()
207 				wpwalk->tlbflush_end);  in wp_clean_post_vma()
209 	mmu_notifier_invalidate_range_end(&wpwalk->range);  in wp_clean_post_vma()
210 	dec_tlb_flush_pending(walk->mm);  in wp_clean_post_vma()
214  * wp_clean_test_walk - The pagewalk test_walk callback.
216  * Won't perform dirty-tracking on COW, read-only or HUGETLB vmas.
221 	unsigned long vm_flags = READ_ONCE(walk->vma->vm_flags);  in wp_clean_test_walk()
223 	/* Skip non-applicable VMAs */  in wp_clean_test_walk()
250  * wp_shared_mapping_range - Write-protect all ptes in an address space range
251  * @mapping: The address_space we want to write protect
255  * Note: This function currently skips transhuge page-table entries, since
256  * it's intended for dirty-tracking on the PTE level. It will warn on
257  * encountering transhuge write-enabled entries, though, and can easily be
260  * Return: The number of ptes actually write-protected. Note that
261  * already write-protected ptes are not counted.
278  * clean_record_shared_mapping_range - Clean and record all ptes in an
301  * additional are added, it first needs to write-protect the address-space
303  * pfn_mkwrite(). And then after a TLB flush following the write-protection
306  * This function currently skips transhuge page-table entries, since
307  * it's intended for dirty-tracking on the PTE level. It will warn on