| /linux-6.12.1/mm/ |
| D | vma.h | 33 struct vma_iterator *vmi; member
75 struct vma_iterator *vmi; member
107 .vmi = vmi_, \
119 .vmi = vmi_, \
149 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
152 static inline int vma_iter_store_gfp(struct vma_iterator *vmi, in vma_iter_store_gfp() argument
156 if (vmi->mas.status != ma_start && in vma_iter_store_gfp()
157 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start))) in vma_iter_store_gfp()
158 vma_iter_invalidate(vmi); in vma_iter_store_gfp()
160 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1); in vma_iter_store_gfp()
[all …]
|
| D | vma.c | 205 static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi, in vma_complete() argument
226 vma_iter_store(vmi, vp->insert); in vma_complete()
367 static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, in __split_vma() argument
395 vma_iter_config(vmi, new->vm_start, new->vm_end); in __split_vma()
396 if (vma_iter_prealloc(vmi, new)) in __split_vma()
429 vma_complete(&vp, vmi, vma->vm_mm); in __split_vma()
434 vma_next(vmi); in __split_vma()
436 vma_prev(vmi); in __split_vma()
443 vma_iter_free(vmi); in __split_vma()
453 static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, in split_vma() argument
[all …]
|
| D | mmap.c | 113 static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
123 struct vma_iterator vmi; in SYSCALL_DEFINE1() local
166 vma_iter_init(&vmi, mm, newbrk); in SYSCALL_DEFINE1()
167 brkvma = vma_find(&vmi, oldbrk); in SYSCALL_DEFINE1()
176 if (do_vmi_align_munmap(&vmi, brkvma, mm, newbrk, oldbrk, &uf, in SYSCALL_DEFINE1()
190 vma_iter_init(&vmi, mm, oldbrk); in SYSCALL_DEFINE1()
191 next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); in SYSCALL_DEFINE1()
195 brkvma = vma_prev_limit(&vmi, mm->start_brk); in SYSCALL_DEFINE1()
197 if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) in SYSCALL_DEFINE1()
611 VMA_ITERATOR(vmi, current->mm, 0); in unmapped_area()
[all …]
|
| D | mseal.c | 71 static int mseal_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, in mseal_fixup() argument
81 vma = vma_modify_flags(vmi, *prev, vma, start, end, newflags); in mseal_fixup()
105 VMA_ITERATOR(vmi, current->mm, start); in check_mm_seal()
108 for_each_vma_range(vmi, vma, end) { in check_mm_seal()
130 VMA_ITERATOR(vmi, current->mm, start); in apply_mm_seal()
132 vma = vma_iter_load(&vmi); in apply_mm_seal()
137 prev = vma_prev(&vmi); in apply_mm_seal()
142 for_each_vma_range(vmi, vma, end) { in apply_mm_seal()
151 error = mseal_fixup(&vmi, vma, &prev, nstart, tmp, newflags); in apply_mm_seal()
154 nstart = vma_iter_end(&vmi); in apply_mm_seal()
|
| D | nommu.c | 573 VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_start); in delete_vma_from_mm()
575 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); in delete_vma_from_mm()
576 if (vma_iter_prealloc(&vmi, NULL)) { in delete_vma_from_mm()
584 vma_iter_clear(&vmi); in delete_vma_from_mm()
616 VMA_ITERATOR(vmi, mm, addr); in find_vma()
618 return vma_iter_load(&vmi); in find_vma()
663 VMA_ITERATOR(vmi, mm, addr); in find_vma_exact()
665 vma = vma_iter_load(&vmi); in find_vma_exact()
1017 VMA_ITERATOR(vmi, current->mm, 0); in do_mmap()
1187 vma_iter_config(&vmi, vma->vm_start, vma->vm_end); in do_mmap()
[all …]
|
| D | mlock.c | 466 static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, in mlock_fixup() argument
481 vma = vma_modify_flags(vmi, *prev, vma, start, end, newflags); in mlock_fixup()
519 VMA_ITERATOR(vmi, current->mm, start); in apply_vma_lock_flags()
528 vma = vma_iter_load(&vmi); in apply_vma_lock_flags()
532 prev = vma_prev(&vmi); in apply_vma_lock_flags()
538 for_each_vma_range(vmi, vma, end) { in apply_vma_lock_flags()
551 error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags); in apply_vma_lock_flags()
554 tmp = vma_iter_end(&vmi); in apply_vma_lock_flags()
577 VMA_ITERATOR(vmi, mm, start); in count_mm_mlocked_page_nr()
585 for_each_vma_range(vmi, vma, end) { in count_mm_mlocked_page_nr()
[all …]
|
| D | mprotect.c | 603 mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, in mprotect_fixup() argument
664 vma = vma_modify_flags(vmi, *pprev, vma, start, end, newflags); in mprotect_fixup()
719 struct vma_iterator vmi; in do_mprotect_pkey() local
751 vma_iter_init(&vmi, current->mm, start); in do_mprotect_pkey()
752 vma = vma_find(&vmi, end); in do_mprotect_pkey()
775 prev = vma_prev(&vmi); in do_mprotect_pkey()
782 for_each_vma_range(vmi, vma, end) { in do_mprotect_pkey()
838 error = mprotect_fixup(&vmi, &tlb, vma, &prev, nstart, tmp, newflags); in do_mprotect_pkey()
842 tmp = vma_iter_end(&vmi); in do_mprotect_pkey()
|
| D | mremap.c | 674 struct vma_iterator vmi; in move_vma() local
791 vma_iter_init(&vmi, mm, old_addr); in move_vma()
792 if (do_vmi_munmap(&vmi, mm, old_addr, old_len, uf_unmap, false) < 0) { in move_vma()
808 vma = vma_prev(&vmi); in move_vma()
813 vma = vma_next(&vmi); in move_vma()
1089 VMA_ITERATOR(vmi, mm, addr + new_len); in SYSCALL_DEFINE5()
1096 ret = do_vmi_munmap(&vmi, mm, addr + new_len, old_len - new_len, in SYSCALL_DEFINE5()
1122 VMA_ITERATOR(vmi, mm, vma->vm_end); in SYSCALL_DEFINE5()
1142 vma = vma_merge_extend(&vmi, vma, delta); in SYSCALL_DEFINE5()
|
| D | debug.c | 294 void vma_iter_dump_tree(const struct vma_iterator *vmi) in vma_iter_dump_tree() argument
297 mas_dump(&vmi->mas); in vma_iter_dump_tree()
298 mt_dump(vmi->mas.tree, mt_dump_hex); in vma_iter_dump_tree()
|
| D | userfaultfd.c | 941 VMA_ITERATOR(vmi, dst_mm, start); in mwriteprotect_range()
965 for_each_vma_range(vmi, dst_vma, end) { in mwriteprotect_range()
1797 struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, in userfaultfd_clear_vma() argument
1809 ret = vma_modify_flags_uffd(vmi, prev, vma, start, end, in userfaultfd_clear_vma()
1831 VMA_ITERATOR(vmi, ctx->mm, start); in userfaultfd_register_range()
1832 struct vm_area_struct *prev = vma_prev(&vmi); in userfaultfd_register_range()
1839 for_each_vma_range(vmi, vma, end) { in userfaultfd_register_range()
1860 vma = vma_modify_flags_uffd(&vmi, prev, vma, start, vma_end, in userfaultfd_register_range()
1888 VMA_ITERATOR(vmi, mm, 0); in userfaultfd_release_new()
1892 for_each_vma(vmi, vma) { in userfaultfd_release_new()
[all …]
|
| D | mempolicy.c | 418 VMA_ITERATOR(vmi, mm, 0); in mpol_rebind_mm()
421 for_each_vma(vmi, vma) { in mpol_rebind_mm()
819 static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma, in mbind_range() argument
838 vma = vma_modify_policy(vmi, *prev, vma, vmstart, vmend, new_pol); in mbind_range()
1268 struct vma_iterator vmi; in do_mbind() local
1333 vma_iter_init(&vmi, mm, start); in do_mbind()
1334 prev = vma_prev(&vmi); in do_mbind()
1335 for_each_vma_range(vmi, vma, end) { in do_mbind()
1336 err = mbind_range(&vmi, vma, &prev, start, end, new); in do_mbind()
1368 vma_iter_init(&vmi, mm, start); in do_mbind()
[all …]
|
| /linux-6.12.1/tools/testing/vma/ |
| D | vma.c | 16 #define vma_iter_prealloc(vmi, vma) \ argument
17 (fail_prealloc ? -ENOMEM : mas_preallocate(&(vmi)->mas, (vma), GFP_KERNEL))
102 vmg->next = vma_next(vmg->vmi); in merge_new()
103 vmg->prev = vma_prev(vmg->vmi); in merge_new()
104 vma_iter_next_range(vmg->vmi); in merge_new()
134 vma_iter_set(vmg->vmi, start); in vmg_set_range()
190 static int cleanup_mm(struct mm_struct *mm, struct vma_iterator *vmi) in cleanup_mm() argument
198 vma_iter_set(vmi, 0); in cleanup_mm()
199 for_each_vma(*vmi, vma) { in cleanup_mm()
233 VMA_ITERATOR(vmi, &mm, 0x1000); in test_simple_merge()
[all …]
|
| D | vma_internal.h | 366 static inline void vma_iter_invalidate(struct vma_iterator *vmi) in vma_iter_invalidate() argument
368 mas_pause(&vmi->mas); in vma_iter_invalidate()
392 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) in vma_next() argument
398 return mas_find(&vmi->mas, ULONG_MAX); in vma_next()
521 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) in vma_find() argument
523 return mas_find(&vmi->mas, max - 1); in vma_find()
526 static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, in vma_iter_clear_gfp() argument
529 __mas_set_range(&vmi->mas, start, end - 1); in vma_iter_clear_gfp()
530 mas_store_gfp(&vmi->mas, NULL, gfp); in vma_iter_clear_gfp()
531 if (unlikely(mas_is_err(&vmi->mas))) in vma_iter_clear_gfp()
[all …]
|
| /linux-6.12.1/arch/xtensa/kernel/ |
| D | syscall.c | 62 struct vma_iterator vmi; in arch_get_unmapped_area() local
84 vma_iter_init(&vmi, current->mm, addr); in arch_get_unmapped_area()
85 for_each_vma(vmi, vmm) { in arch_get_unmapped_area()
|
| /linux-6.12.1/fs/proc/ |
| D | task_nommu.c | 23 VMA_ITERATOR(vmi, mm, 0); in task_mem()
29 for_each_vma(vmi, vma) { in task_mem()
83 VMA_ITERATOR(vmi, mm, 0); in task_vsize()
88 for_each_vma(vmi, vma) in task_vsize()
98 VMA_ITERATOR(vmi, mm, 0); in task_statm()
104 for_each_vma(vmi, vma) { in task_statm()
|
| /linux-6.12.1/arch/powerpc/mm/book3s32/ |
| D | tlb.c | 84 VMA_ITERATOR(vmi, mm, 0); in hash__flush_tlb_mm()
92 for_each_vma(vmi, mp) in hash__flush_tlb_mm()
|
| /linux-6.12.1/arch/x86/entry/vdso/ |
| D | vma.c | 118 VMA_ITERATOR(vmi, mm, 0); in vdso_join_timens()
121 for_each_vma(vmi, vma) { in vdso_join_timens()
284 VMA_ITERATOR(vmi, mm, 0); in map_vdso_once()
294 for_each_vma(vmi, vma) { in map_vdso_once()
|
| /linux-6.12.1/include/linux/ |
| D | mm.h | 996 struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) in vma_find() argument
998 return mas_find(&vmi->mas, max - 1); in vma_find()
1001 static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) in vma_next() argument
1007 return mas_find(&vmi->mas, ULONG_MAX); in vma_next()
1011 struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) in vma_iter_next_range() argument
1013 return mas_next_range(&vmi->mas, ULONG_MAX); in vma_iter_next_range()
1017 static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) in vma_prev() argument
1019 return mas_prev(&vmi->mas, 0); in vma_prev()
1022 static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, in vma_iter_clear_gfp() argument
1025 __mas_set_range(&vmi->mas, start, end - 1); in vma_iter_clear_gfp()
[all …]
|
| /linux-6.12.1/arch/loongarch/kernel/ |
| D | vdso.c | 137 VMA_ITERATOR(vmi, mm, 0); in vdso_join_timens()
140 for_each_vma(vmi, vma) { in vdso_join_timens()
|
| /linux-6.12.1/arch/riscv/kernel/ |
| D | vdso.c | 106 VMA_ITERATOR(vmi, mm, 0); in vdso_join_timens()
110 for_each_vma(vmi, vma) { in vdso_join_timens()
|
| /linux-6.12.1/fs/ |
| D | userfaultfd.c | 1251 struct vma_iterator vmi; in userfaultfd_register() local
1296 vma_iter_init(&vmi, mm, start); in userfaultfd_register()
1297 vma = vma_find(&vmi, end); in userfaultfd_register()
1375 } for_each_vma_range(vmi, cur, end); in userfaultfd_register()
1423 struct vma_iterator vmi; in userfaultfd_unregister() local
1444 vma_iter_init(&vmi, mm, start); in userfaultfd_unregister()
1445 vma = vma_find(&vmi, end); in userfaultfd_unregister()
1482 } for_each_vma_range(vmi, cur, end); in userfaultfd_unregister()
1485 vma_iter_set(&vmi, start); in userfaultfd_unregister()
1486 prev = vma_prev(&vmi); in userfaultfd_unregister()
[all …]
|
| /linux-6.12.1/arch/um/kernel/ |
| D | tlb.c | 228 VMA_ITERATOR(vmi, mm, 0); in flush_tlb_mm()
230 for_each_vma(vmi, vma) in flush_tlb_mm()
|
| /linux-6.12.1/arch/s390/kernel/ |
| D | vdso.c | 51 VMA_ITERATOR(vmi, mm, 0); in vdso_join_timens()
55 for_each_vma(vmi, vma) { in vdso_join_timens()
|
| /linux-6.12.1/arch/powerpc/mm/book3s64/ |
| D | subpage_prot.c | 155 VMA_ITERATOR(vmi, mm, addr); in subpage_mark_vma_nohuge()
161 for_each_vma_range(vmi, vma, addr + len) { in subpage_mark_vma_nohuge()
|
| /linux-6.12.1/arch/parisc/kernel/ |
| D | cache.c | 717 VMA_ITERATOR(vmi, mm, 0); in mm_total_size()
719 for_each_vma(vmi, vma) { in mm_total_size()
730 VMA_ITERATOR(vmi, mm, 0); in flush_cache_mm()
750 for_each_vma(vmi, vma) in flush_cache_mm()
|