1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * mm/mmap.c
4 *
5 * Written by obz.
6 *
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 */
9
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
14 #include <linux/backing-dev.h>
15 #include <linux/mm.h>
16 #include <linux/mm_inline.h>
17 #include <linux/shm.h>
18 #include <linux/mman.h>
19 #include <linux/pagemap.h>
20 #include <linux/swap.h>
21 #include <linux/syscalls.h>
22 #include <linux/capability.h>
23 #include <linux/init.h>
24 #include <linux/file.h>
25 #include <linux/fs.h>
26 #include <linux/personality.h>
27 #include <linux/security.h>
28 #include <linux/hugetlb.h>
29 #include <linux/shmem_fs.h>
30 #include <linux/profile.h>
31 #include <linux/export.h>
32 #include <linux/mount.h>
33 #include <linux/mempolicy.h>
34 #include <linux/rmap.h>
35 #include <linux/mmu_notifier.h>
36 #include <linux/mmdebug.h>
37 #include <linux/perf_event.h>
38 #include <linux/audit.h>
39 #include <linux/khugepaged.h>
40 #include <linux/uprobes.h>
41 #include <linux/notifier.h>
42 #include <linux/memory.h>
43 #include <linux/printk.h>
44 #include <linux/userfaultfd_k.h>
45 #include <linux/moduleparam.h>
46 #include <linux/pkeys.h>
47 #include <linux/oom.h>
48 #include <linux/sched/mm.h>
49 #include <linux/ksm.h>
50
51 #include <linux/uaccess.h>
52 #include <asm/cacheflush.h>
53 #include <asm/tlb.h>
54 #include <asm/mmu_context.h>
55
56 #define CREATE_TRACE_POINTS
57 #include <trace/events/mmap.h>
58
59 #include "internal.h"
60
61 #ifndef arch_mmap_check
62 #define arch_mmap_check(addr, len, flags) (0)
63 #endif
64
65 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
66 const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
67 int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX;
68 int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
69 #endif
70 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
71 const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
72 const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
73 int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
74 #endif
75
76 static bool ignore_rlimit_data;
77 core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
78
79 /* Update vma->vm_page_prot to reflect vma->vm_flags. */
vma_set_page_prot(struct vm_area_struct * vma)80 void vma_set_page_prot(struct vm_area_struct *vma)
81 {
82 unsigned long vm_flags = vma->vm_flags;
83 pgprot_t vm_page_prot;
84
85 vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
86 if (vma_wants_writenotify(vma, vm_page_prot)) {
87 vm_flags &= ~VM_SHARED;
88 vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
89 }
90 /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */
91 WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
92 }
93
94 /*
95 * check_brk_limits() - Use platform specific check of range & verify mlock
96 * limits.
97 * @addr: The address to check
98 * @len: The size of increase.
99 *
100 * Return: 0 on success.
101 */
check_brk_limits(unsigned long addr,unsigned long len)102 static int check_brk_limits(unsigned long addr, unsigned long len)
103 {
104 unsigned long mapped_addr;
105
106 mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
107 if (IS_ERR_VALUE(mapped_addr))
108 return mapped_addr;
109
110 return mlock_future_ok(current->mm, current->mm->def_flags, len)
111 ? 0 : -EAGAIN;
112 }
113 static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
114 unsigned long addr, unsigned long request, unsigned long flags);
SYSCALL_DEFINE1(brk,unsigned long,brk)115 SYSCALL_DEFINE1(brk, unsigned long, brk)
116 {
117 unsigned long newbrk, oldbrk, origbrk;
118 struct mm_struct *mm = current->mm;
119 struct vm_area_struct *brkvma, *next = NULL;
120 unsigned long min_brk;
121 bool populate = false;
122 LIST_HEAD(uf);
123 struct vma_iterator vmi;
124
125 if (mmap_write_lock_killable(mm))
126 return -EINTR;
127
128 origbrk = mm->brk;
129
130 #ifdef CONFIG_COMPAT_BRK
131 /*
132 * CONFIG_COMPAT_BRK can still be overridden by setting
133 * randomize_va_space to 2, which will still cause mm->start_brk
134 * to be arbitrarily shifted
135 */
136 if (current->brk_randomized)
137 min_brk = mm->start_brk;
138 else
139 min_brk = mm->end_data;
140 #else
141 min_brk = mm->start_brk;
142 #endif
143 if (brk < min_brk)
144 goto out;
145
146 /*
147 * Check against rlimit here. If this check is done later after the test
148 * of oldbrk with newbrk then it can escape the test and let the data
149 * segment grow beyond its set limit the in case where the limit is
150 * not page aligned -Ram Gupta
151 */
152 if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
153 mm->end_data, mm->start_data))
154 goto out;
155
156 newbrk = PAGE_ALIGN(brk);
157 oldbrk = PAGE_ALIGN(mm->brk);
158 if (oldbrk == newbrk) {
159 mm->brk = brk;
160 goto success;
161 }
162
163 /* Always allow shrinking brk. */
164 if (brk <= mm->brk) {
165 /* Search one past newbrk */
166 vma_iter_init(&vmi, mm, newbrk);
167 brkvma = vma_find(&vmi, oldbrk);
168 if (!brkvma || brkvma->vm_start >= oldbrk)
169 goto out; /* mapping intersects with an existing non-brk vma. */
170 /*
171 * mm->brk must be protected by write mmap_lock.
172 * do_vmi_align_munmap() will drop the lock on success, so
173 * update it before calling do_vma_munmap().
174 */
175 mm->brk = brk;
176 if (do_vmi_align_munmap(&vmi, brkvma, mm, newbrk, oldbrk, &uf,
177 /* unlock = */ true))
178 goto out;
179
180 goto success_unlocked;
181 }
182
183 if (check_brk_limits(oldbrk, newbrk - oldbrk))
184 goto out;
185
186 /*
187 * Only check if the next VMA is within the stack_guard_gap of the
188 * expansion area
189 */
190 vma_iter_init(&vmi, mm, oldbrk);
191 next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap);
192 if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
193 goto out;
194
195 brkvma = vma_prev_limit(&vmi, mm->start_brk);
196 /* Ok, looks good - let it rip. */
197 if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
198 goto out;
199
200 mm->brk = brk;
201 if (mm->def_flags & VM_LOCKED)
202 populate = true;
203
204 success:
205 mmap_write_unlock(mm);
206 success_unlocked:
207 userfaultfd_unmap_complete(mm, &uf);
208 if (populate)
209 mm_populate(oldbrk, newbrk - oldbrk);
210 return brk;
211
212 out:
213 mm->brk = origbrk;
214 mmap_write_unlock(mm);
215 return origbrk;
216 }
217
218 /*
219 * If a hint addr is less than mmap_min_addr change hint to be as
220 * low as possible but still greater than mmap_min_addr
221 */
round_hint_to_min(unsigned long hint)222 static inline unsigned long round_hint_to_min(unsigned long hint)
223 {
224 hint &= PAGE_MASK;
225 if (((void *)hint != NULL) &&
226 (hint < mmap_min_addr))
227 return PAGE_ALIGN(mmap_min_addr);
228 return hint;
229 }
230
mlock_future_ok(struct mm_struct * mm,unsigned long flags,unsigned long bytes)231 bool mlock_future_ok(struct mm_struct *mm, unsigned long flags,
232 unsigned long bytes)
233 {
234 unsigned long locked_pages, limit_pages;
235
236 if (!(flags & VM_LOCKED) || capable(CAP_IPC_LOCK))
237 return true;
238
239 locked_pages = bytes >> PAGE_SHIFT;
240 locked_pages += mm->locked_vm;
241
242 limit_pages = rlimit(RLIMIT_MEMLOCK);
243 limit_pages >>= PAGE_SHIFT;
244
245 return locked_pages <= limit_pages;
246 }
247
file_mmap_size_max(struct file * file,struct inode * inode)248 static inline u64 file_mmap_size_max(struct file *file, struct inode *inode)
249 {
250 if (S_ISREG(inode->i_mode))
251 return MAX_LFS_FILESIZE;
252
253 if (S_ISBLK(inode->i_mode))
254 return MAX_LFS_FILESIZE;
255
256 if (S_ISSOCK(inode->i_mode))
257 return MAX_LFS_FILESIZE;
258
259 /* Special "we do even unsigned file positions" case */
260 if (file->f_op->fop_flags & FOP_UNSIGNED_OFFSET)
261 return 0;
262
263 /* Yes, random drivers might want more. But I'm tired of buggy drivers */
264 return ULONG_MAX;
265 }
266
file_mmap_ok(struct file * file,struct inode * inode,unsigned long pgoff,unsigned long len)267 static inline bool file_mmap_ok(struct file *file, struct inode *inode,
268 unsigned long pgoff, unsigned long len)
269 {
270 u64 maxsize = file_mmap_size_max(file, inode);
271
272 if (maxsize && len > maxsize)
273 return false;
274 maxsize -= len;
275 if (pgoff > maxsize >> PAGE_SHIFT)
276 return false;
277 return true;
278 }
279
280 /*
281 * The caller must write-lock current->mm->mmap_lock.
282 */
do_mmap(struct file * file,unsigned long addr,unsigned long len,unsigned long prot,unsigned long flags,vm_flags_t vm_flags,unsigned long pgoff,unsigned long * populate,struct list_head * uf)283 unsigned long do_mmap(struct file *file, unsigned long addr,
284 unsigned long len, unsigned long prot,
285 unsigned long flags, vm_flags_t vm_flags,
286 unsigned long pgoff, unsigned long *populate,
287 struct list_head *uf)
288 {
289 struct mm_struct *mm = current->mm;
290 int pkey = 0;
291
292 *populate = 0;
293
294 if (!len)
295 return -EINVAL;
296
297 /*
298 * Does the application expect PROT_READ to imply PROT_EXEC?
299 *
300 * (the exception is when the underlying filesystem is noexec
301 * mounted, in which case we don't add PROT_EXEC.)
302 */
303 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
304 if (!(file && path_noexec(&file->f_path)))
305 prot |= PROT_EXEC;
306
307 /* force arch specific MAP_FIXED handling in get_unmapped_area */
308 if (flags & MAP_FIXED_NOREPLACE)
309 flags |= MAP_FIXED;
310
311 if (!(flags & MAP_FIXED))
312 addr = round_hint_to_min(addr);
313
314 /* Careful about overflows.. */
315 len = PAGE_ALIGN(len);
316 if (!len)
317 return -ENOMEM;
318
319 /* offset overflow? */
320 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
321 return -EOVERFLOW;
322
323 /* Too many mappings? */
324 if (mm->map_count > sysctl_max_map_count)
325 return -ENOMEM;
326
327 /*
328 * addr is returned from get_unmapped_area,
329 * There are two cases:
330 * 1> MAP_FIXED == false
331 * unallocated memory, no need to check sealing.
332 * 1> MAP_FIXED == true
333 * sealing is checked inside mmap_region when
334 * do_vmi_munmap is called.
335 */
336
337 if (prot == PROT_EXEC) {
338 pkey = execute_only_pkey(mm);
339 if (pkey < 0)
340 pkey = 0;
341 }
342
343 /* Do simple checking here so the lower-level routines won't have
344 * to. we assume access permissions have been handled by the open
345 * of the memory object, so we don't do any here.
346 */
347 vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) |
348 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
349
350 /* Obtain the address to map to. we verify (or select) it and ensure
351 * that it represents a valid section of the address space.
352 */
353 addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags);
354 if (IS_ERR_VALUE(addr))
355 return addr;
356
357 if (flags & MAP_FIXED_NOREPLACE) {
358 if (find_vma_intersection(mm, addr, addr + len))
359 return -EEXIST;
360 }
361
362 if (flags & MAP_LOCKED)
363 if (!can_do_mlock())
364 return -EPERM;
365
366 if (!mlock_future_ok(mm, vm_flags, len))
367 return -EAGAIN;
368
369 if (file) {
370 struct inode *inode = file_inode(file);
371 unsigned long flags_mask;
372
373 if (!file_mmap_ok(file, inode, pgoff, len))
374 return -EOVERFLOW;
375
376 flags_mask = LEGACY_MAP_MASK;
377 if (file->f_op->fop_flags & FOP_MMAP_SYNC)
378 flags_mask |= MAP_SYNC;
379
380 switch (flags & MAP_TYPE) {
381 case MAP_SHARED:
382 /*
383 * Force use of MAP_SHARED_VALIDATE with non-legacy
384 * flags. E.g. MAP_SYNC is dangerous to use with
385 * MAP_SHARED as you don't know which consistency model
386 * you will get. We silently ignore unsupported flags
387 * with MAP_SHARED to preserve backward compatibility.
388 */
389 flags &= LEGACY_MAP_MASK;
390 fallthrough;
391 case MAP_SHARED_VALIDATE:
392 if (flags & ~flags_mask)
393 return -EOPNOTSUPP;
394 if (prot & PROT_WRITE) {
395 if (!(file->f_mode & FMODE_WRITE))
396 return -EACCES;
397 if (IS_SWAPFILE(file->f_mapping->host))
398 return -ETXTBSY;
399 }
400
401 /*
402 * Make sure we don't allow writing to an append-only
403 * file..
404 */
405 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
406 return -EACCES;
407
408 vm_flags |= VM_SHARED | VM_MAYSHARE;
409 if (!(file->f_mode & FMODE_WRITE))
410 vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
411 fallthrough;
412 case MAP_PRIVATE:
413 if (!(file->f_mode & FMODE_READ))
414 return -EACCES;
415 if (path_noexec(&file->f_path)) {
416 if (vm_flags & VM_EXEC)
417 return -EPERM;
418 vm_flags &= ~VM_MAYEXEC;
419 }
420
421 if (!file->f_op->mmap)
422 return -ENODEV;
423 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
424 return -EINVAL;
425 break;
426
427 default:
428 return -EINVAL;
429 }
430 } else {
431 switch (flags & MAP_TYPE) {
432 case MAP_SHARED:
433 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
434 return -EINVAL;
435 /*
436 * Ignore pgoff.
437 */
438 pgoff = 0;
439 vm_flags |= VM_SHARED | VM_MAYSHARE;
440 break;
441 case MAP_DROPPABLE:
442 if (VM_DROPPABLE == VM_NONE)
443 return -ENOTSUPP;
444 /*
445 * A locked or stack area makes no sense to be droppable.
446 *
447 * Also, since droppable pages can just go away at any time
448 * it makes no sense to copy them on fork or dump them.
449 *
450 * And don't attempt to combine with hugetlb for now.
451 */
452 if (flags & (MAP_LOCKED | MAP_HUGETLB))
453 return -EINVAL;
454 if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP))
455 return -EINVAL;
456
457 vm_flags |= VM_DROPPABLE;
458
459 /*
460 * If the pages can be dropped, then it doesn't make
461 * sense to reserve them.
462 */
463 vm_flags |= VM_NORESERVE;
464
465 /*
466 * Likewise, they're volatile enough that they
467 * shouldn't survive forks or coredumps.
468 */
469 vm_flags |= VM_WIPEONFORK | VM_DONTDUMP;
470 fallthrough;
471 case MAP_PRIVATE:
472 /*
473 * Set pgoff according to addr for anon_vma.
474 */
475 pgoff = addr >> PAGE_SHIFT;
476 break;
477 default:
478 return -EINVAL;
479 }
480 }
481
482 /*
483 * Set 'VM_NORESERVE' if we should not account for the
484 * memory use of this mapping.
485 */
486 if (flags & MAP_NORESERVE) {
487 /* We honor MAP_NORESERVE if allowed to overcommit */
488 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
489 vm_flags |= VM_NORESERVE;
490
491 /* hugetlb applies strict overcommit unless MAP_NORESERVE */
492 if (file && is_file_hugepages(file))
493 vm_flags |= VM_NORESERVE;
494 }
495
496 addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
497 if (!IS_ERR_VALUE(addr) &&
498 ((vm_flags & VM_LOCKED) ||
499 (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
500 *populate = len;
501 return addr;
502 }
503
ksys_mmap_pgoff(unsigned long addr,unsigned long len,unsigned long prot,unsigned long flags,unsigned long fd,unsigned long pgoff)504 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
505 unsigned long prot, unsigned long flags,
506 unsigned long fd, unsigned long pgoff)
507 {
508 struct file *file = NULL;
509 unsigned long retval;
510
511 if (!(flags & MAP_ANONYMOUS)) {
512 audit_mmap_fd(fd, flags);
513 file = fget(fd);
514 if (!file)
515 return -EBADF;
516 if (is_file_hugepages(file)) {
517 len = ALIGN(len, huge_page_size(hstate_file(file)));
518 } else if (unlikely(flags & MAP_HUGETLB)) {
519 retval = -EINVAL;
520 goto out_fput;
521 }
522 } else if (flags & MAP_HUGETLB) {
523 struct hstate *hs;
524
525 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
526 if (!hs)
527 return -EINVAL;
528
529 len = ALIGN(len, huge_page_size(hs));
530 /*
531 * VM_NORESERVE is used because the reservations will be
532 * taken when vm_ops->mmap() is called
533 */
534 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
535 VM_NORESERVE,
536 HUGETLB_ANONHUGE_INODE,
537 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
538 if (IS_ERR(file))
539 return PTR_ERR(file);
540 }
541
542 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
543 out_fput:
544 if (file)
545 fput(file);
546 return retval;
547 }
548
SYSCALL_DEFINE6(mmap_pgoff,unsigned long,addr,unsigned long,len,unsigned long,prot,unsigned long,flags,unsigned long,fd,unsigned long,pgoff)549 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
550 unsigned long, prot, unsigned long, flags,
551 unsigned long, fd, unsigned long, pgoff)
552 {
553 return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
554 }
555
556 #ifdef __ARCH_WANT_SYS_OLD_MMAP
557 struct mmap_arg_struct {
558 unsigned long addr;
559 unsigned long len;
560 unsigned long prot;
561 unsigned long flags;
562 unsigned long fd;
563 unsigned long offset;
564 };
565
SYSCALL_DEFINE1(old_mmap,struct mmap_arg_struct __user *,arg)566 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
567 {
568 struct mmap_arg_struct a;
569
570 if (copy_from_user(&a, arg, sizeof(a)))
571 return -EFAULT;
572 if (offset_in_page(a.offset))
573 return -EINVAL;
574
575 return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
576 a.offset >> PAGE_SHIFT);
577 }
578 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
579
580 /*
581 * We account for memory if it's a private writeable mapping,
582 * not hugepages and VM_NORESERVE wasn't set.
583 */
accountable_mapping(struct file * file,vm_flags_t vm_flags)584 static inline bool accountable_mapping(struct file *file, vm_flags_t vm_flags)
585 {
586 /*
587 * hugetlb has its own accounting separate from the core VM
588 * VM_HUGETLB may not be set yet so we cannot check for that flag.
589 */
590 if (file && is_file_hugepages(file))
591 return false;
592
593 return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
594 }
595
596 /**
597 * unmapped_area() - Find an area between the low_limit and the high_limit with
598 * the correct alignment and offset, all from @info. Note: current->mm is used
599 * for the search.
600 *
601 * @info: The unmapped area information including the range [low_limit -
602 * high_limit), the alignment offset and mask.
603 *
604 * Return: A memory address or -ENOMEM.
605 */
unmapped_area(struct vm_unmapped_area_info * info)606 static unsigned long unmapped_area(struct vm_unmapped_area_info *info)
607 {
608 unsigned long length, gap;
609 unsigned long low_limit, high_limit;
610 struct vm_area_struct *tmp;
611 VMA_ITERATOR(vmi, current->mm, 0);
612
613 /* Adjust search length to account for worst case alignment overhead */
614 length = info->length + info->align_mask + info->start_gap;
615 if (length < info->length)
616 return -ENOMEM;
617
618 low_limit = info->low_limit;
619 if (low_limit < mmap_min_addr)
620 low_limit = mmap_min_addr;
621 high_limit = info->high_limit;
622 retry:
623 if (vma_iter_area_lowest(&vmi, low_limit, high_limit, length))
624 return -ENOMEM;
625
626 /*
627 * Adjust for the gap first so it doesn't interfere with the
628 * later alignment. The first step is the minimum needed to
629 * fulill the start gap, the next steps is the minimum to align
630 * that. It is the minimum needed to fulill both.
631 */
632 gap = vma_iter_addr(&vmi) + info->start_gap;
633 gap += (info->align_offset - gap) & info->align_mask;
634 tmp = vma_next(&vmi);
635 if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
636 if (vm_start_gap(tmp) < gap + length - 1) {
637 low_limit = tmp->vm_end;
638 vma_iter_reset(&vmi);
639 goto retry;
640 }
641 } else {
642 tmp = vma_prev(&vmi);
643 if (tmp && vm_end_gap(tmp) > gap) {
644 low_limit = vm_end_gap(tmp);
645 vma_iter_reset(&vmi);
646 goto retry;
647 }
648 }
649
650 return gap;
651 }
652
653 /**
654 * unmapped_area_topdown() - Find an area between the low_limit and the
655 * high_limit with the correct alignment and offset at the highest available
656 * address, all from @info. Note: current->mm is used for the search.
657 *
658 * @info: The unmapped area information including the range [low_limit -
659 * high_limit), the alignment offset and mask.
660 *
661 * Return: A memory address or -ENOMEM.
662 */
unmapped_area_topdown(struct vm_unmapped_area_info * info)663 static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
664 {
665 unsigned long length, gap, gap_end;
666 unsigned long low_limit, high_limit;
667 struct vm_area_struct *tmp;
668 VMA_ITERATOR(vmi, current->mm, 0);
669
670 /* Adjust search length to account for worst case alignment overhead */
671 length = info->length + info->align_mask + info->start_gap;
672 if (length < info->length)
673 return -ENOMEM;
674
675 low_limit = info->low_limit;
676 if (low_limit < mmap_min_addr)
677 low_limit = mmap_min_addr;
678 high_limit = info->high_limit;
679 retry:
680 if (vma_iter_area_highest(&vmi, low_limit, high_limit, length))
681 return -ENOMEM;
682
683 gap = vma_iter_end(&vmi) - info->length;
684 gap -= (gap - info->align_offset) & info->align_mask;
685 gap_end = vma_iter_end(&vmi);
686 tmp = vma_next(&vmi);
687 if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
688 if (vm_start_gap(tmp) < gap_end) {
689 high_limit = vm_start_gap(tmp);
690 vma_iter_reset(&vmi);
691 goto retry;
692 }
693 } else {
694 tmp = vma_prev(&vmi);
695 if (tmp && vm_end_gap(tmp) > gap) {
696 high_limit = tmp->vm_start;
697 vma_iter_reset(&vmi);
698 goto retry;
699 }
700 }
701
702 return gap;
703 }
704
705 /*
706 * Determine if the allocation needs to ensure that there is no
707 * existing mapping within it's guard gaps, for use as start_gap.
708 */
stack_guard_placement(vm_flags_t vm_flags)709 static inline unsigned long stack_guard_placement(vm_flags_t vm_flags)
710 {
711 if (vm_flags & VM_SHADOW_STACK)
712 return PAGE_SIZE;
713
714 return 0;
715 }
716
717 /*
718 * Search for an unmapped address range.
719 *
720 * We are looking for a range that:
721 * - does not intersect with any VMA;
722 * - is contained within the [low_limit, high_limit) interval;
723 * - is at least the desired size.
724 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
725 */
vm_unmapped_area(struct vm_unmapped_area_info * info)726 unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
727 {
728 unsigned long addr;
729
730 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
731 addr = unmapped_area_topdown(info);
732 else
733 addr = unmapped_area(info);
734
735 trace_vm_unmapped_area(addr, info);
736 return addr;
737 }
738
739 /* Get an address range which is currently unmapped.
740 * For shmat() with addr=0.
741 *
742 * Ugly calling convention alert:
743 * Return value with the low bits set means error value,
744 * ie
745 * if (ret & ~PAGE_MASK)
746 * error = ret;
747 *
748 * This function "knows" that -ENOMEM has the bits set.
749 */
750 unsigned long
generic_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)751 generic_get_unmapped_area(struct file *filp, unsigned long addr,
752 unsigned long len, unsigned long pgoff,
753 unsigned long flags, vm_flags_t vm_flags)
754 {
755 struct mm_struct *mm = current->mm;
756 struct vm_area_struct *vma, *prev;
757 struct vm_unmapped_area_info info = {};
758 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
759
760 if (len > mmap_end - mmap_min_addr)
761 return -ENOMEM;
762
763 if (flags & MAP_FIXED)
764 return addr;
765
766 if (addr) {
767 addr = PAGE_ALIGN(addr);
768 vma = find_vma_prev(mm, addr, &prev);
769 if (mmap_end - len >= addr && addr >= mmap_min_addr &&
770 (!vma || addr + len <= vm_start_gap(vma)) &&
771 (!prev || addr >= vm_end_gap(prev)))
772 return addr;
773 }
774
775 info.length = len;
776 info.low_limit = mm->mmap_base;
777 info.high_limit = mmap_end;
778 info.start_gap = stack_guard_placement(vm_flags);
779 return vm_unmapped_area(&info);
780 }
781
782 #ifndef HAVE_ARCH_UNMAPPED_AREA
783 unsigned long
arch_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)784 arch_get_unmapped_area(struct file *filp, unsigned long addr,
785 unsigned long len, unsigned long pgoff,
786 unsigned long flags, vm_flags_t vm_flags)
787 {
788 return generic_get_unmapped_area(filp, addr, len, pgoff, flags,
789 vm_flags);
790 }
791 #endif
792
793 /*
794 * This mmap-allocator allocates new areas top-down from below the
795 * stack's low limit (the base):
796 */
797 unsigned long
generic_get_unmapped_area_topdown(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)798 generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
799 unsigned long len, unsigned long pgoff,
800 unsigned long flags, vm_flags_t vm_flags)
801 {
802 struct vm_area_struct *vma, *prev;
803 struct mm_struct *mm = current->mm;
804 struct vm_unmapped_area_info info = {};
805 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
806
807 /* requested length too big for entire address space */
808 if (len > mmap_end - mmap_min_addr)
809 return -ENOMEM;
810
811 if (flags & MAP_FIXED)
812 return addr;
813
814 /* requesting a specific address */
815 if (addr) {
816 addr = PAGE_ALIGN(addr);
817 vma = find_vma_prev(mm, addr, &prev);
818 if (mmap_end - len >= addr && addr >= mmap_min_addr &&
819 (!vma || addr + len <= vm_start_gap(vma)) &&
820 (!prev || addr >= vm_end_gap(prev)))
821 return addr;
822 }
823
824 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
825 info.length = len;
826 info.low_limit = PAGE_SIZE;
827 info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
828 info.start_gap = stack_guard_placement(vm_flags);
829 addr = vm_unmapped_area(&info);
830
831 /*
832 * A failed mmap() very likely causes application failure,
833 * so fall back to the bottom-up function here. This scenario
834 * can happen with large stack limits and large mmap()
835 * allocations.
836 */
837 if (offset_in_page(addr)) {
838 VM_BUG_ON(addr != -ENOMEM);
839 info.flags = 0;
840 info.low_limit = TASK_UNMAPPED_BASE;
841 info.high_limit = mmap_end;
842 addr = vm_unmapped_area(&info);
843 }
844
845 return addr;
846 }
847
848 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
849 unsigned long
arch_get_unmapped_area_topdown(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)850 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
851 unsigned long len, unsigned long pgoff,
852 unsigned long flags, vm_flags_t vm_flags)
853 {
854 return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags,
855 vm_flags);
856 }
857 #endif
858
mm_get_unmapped_area_vmflags(struct mm_struct * mm,struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)859 unsigned long mm_get_unmapped_area_vmflags(struct mm_struct *mm, struct file *filp,
860 unsigned long addr, unsigned long len,
861 unsigned long pgoff, unsigned long flags,
862 vm_flags_t vm_flags)
863 {
864 if (test_bit(MMF_TOPDOWN, &mm->flags))
865 return arch_get_unmapped_area_topdown(filp, addr, len, pgoff,
866 flags, vm_flags);
867 return arch_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags);
868 }
869
870 unsigned long
__get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags,vm_flags_t vm_flags)871 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
872 unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags)
873 {
874 unsigned long (*get_area)(struct file *, unsigned long,
875 unsigned long, unsigned long, unsigned long)
876 = NULL;
877
878 unsigned long error = arch_mmap_check(addr, len, flags);
879 if (error)
880 return error;
881
882 /* Careful about overflows.. */
883 if (len > TASK_SIZE)
884 return -ENOMEM;
885
886 if (file) {
887 if (file->f_op->get_unmapped_area)
888 get_area = file->f_op->get_unmapped_area;
889 } else if (flags & MAP_SHARED) {
890 /*
891 * mmap_region() will call shmem_zero_setup() to create a file,
892 * so use shmem's get_unmapped_area in case it can be huge.
893 */
894 get_area = shmem_get_unmapped_area;
895 }
896
897 /* Always treat pgoff as zero for anonymous memory. */
898 if (!file)
899 pgoff = 0;
900
901 if (get_area) {
902 addr = get_area(file, addr, len, pgoff, flags);
903 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)
904 && IS_ALIGNED(len, PMD_SIZE)) {
905 /* Ensures that larger anonymous mappings are THP aligned. */
906 addr = thp_get_unmapped_area_vmflags(file, addr, len,
907 pgoff, flags, vm_flags);
908 } else {
909 addr = mm_get_unmapped_area_vmflags(current->mm, file, addr, len,
910 pgoff, flags, vm_flags);
911 }
912 if (IS_ERR_VALUE(addr))
913 return addr;
914
915 if (addr > TASK_SIZE - len)
916 return -ENOMEM;
917 if (offset_in_page(addr))
918 return -EINVAL;
919
920 error = security_mmap_addr(addr);
921 return error ? error : addr;
922 }
923
924 unsigned long
mm_get_unmapped_area(struct mm_struct * mm,struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)925 mm_get_unmapped_area(struct mm_struct *mm, struct file *file,
926 unsigned long addr, unsigned long len,
927 unsigned long pgoff, unsigned long flags)
928 {
929 if (test_bit(MMF_TOPDOWN, &mm->flags))
930 return arch_get_unmapped_area_topdown(file, addr, len, pgoff, flags, 0);
931 return arch_get_unmapped_area(file, addr, len, pgoff, flags, 0);
932 }
933 EXPORT_SYMBOL(mm_get_unmapped_area);
934
935 /**
936 * find_vma_intersection() - Look up the first VMA which intersects the interval
937 * @mm: The process address space.
938 * @start_addr: The inclusive start user address.
939 * @end_addr: The exclusive end user address.
940 *
941 * Returns: The first VMA within the provided range, %NULL otherwise. Assumes
942 * start_addr < end_addr.
943 */
find_vma_intersection(struct mm_struct * mm,unsigned long start_addr,unsigned long end_addr)944 struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
945 unsigned long start_addr,
946 unsigned long end_addr)
947 {
948 unsigned long index = start_addr;
949
950 mmap_assert_locked(mm);
951 return mt_find(&mm->mm_mt, &index, end_addr - 1);
952 }
953 EXPORT_SYMBOL(find_vma_intersection);
954
955 /**
956 * find_vma() - Find the VMA for a given address, or the next VMA.
957 * @mm: The mm_struct to check
958 * @addr: The address
959 *
960 * Returns: The VMA associated with addr, or the next VMA.
961 * May return %NULL in the case of no VMA at addr or above.
962 */
find_vma(struct mm_struct * mm,unsigned long addr)963 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
964 {
965 unsigned long index = addr;
966
967 mmap_assert_locked(mm);
968 return mt_find(&mm->mm_mt, &index, ULONG_MAX);
969 }
970 EXPORT_SYMBOL(find_vma);
971
972 /**
973 * find_vma_prev() - Find the VMA for a given address, or the next vma and
974 * set %pprev to the previous VMA, if any.
975 * @mm: The mm_struct to check
976 * @addr: The address
977 * @pprev: The pointer to set to the previous VMA
978 *
979 * Note that RCU lock is missing here since the external mmap_lock() is used
980 * instead.
981 *
982 * Returns: The VMA associated with @addr, or the next vma.
983 * May return %NULL in the case of no vma at addr or above.
984 */
985 struct vm_area_struct *
find_vma_prev(struct mm_struct * mm,unsigned long addr,struct vm_area_struct ** pprev)986 find_vma_prev(struct mm_struct *mm, unsigned long addr,
987 struct vm_area_struct **pprev)
988 {
989 struct vm_area_struct *vma;
990 VMA_ITERATOR(vmi, mm, addr);
991
992 vma = vma_iter_load(&vmi);
993 *pprev = vma_prev(&vmi);
994 if (!vma)
995 vma = vma_next(&vmi);
996 return vma;
997 }
998
999 /*
1000 * Verify that the stack growth is acceptable and
1001 * update accounting. This is shared with both the
1002 * grow-up and grow-down cases.
1003 */
acct_stack_growth(struct vm_area_struct * vma,unsigned long size,unsigned long grow)1004 static int acct_stack_growth(struct vm_area_struct *vma,
1005 unsigned long size, unsigned long grow)
1006 {
1007 struct mm_struct *mm = vma->vm_mm;
1008 unsigned long new_start;
1009
1010 /* address space limit tests */
1011 if (!may_expand_vm(mm, vma->vm_flags, grow))
1012 return -ENOMEM;
1013
1014 /* Stack limit test */
1015 if (size > rlimit(RLIMIT_STACK))
1016 return -ENOMEM;
1017
1018 /* mlock limit tests */
1019 if (!mlock_future_ok(mm, vma->vm_flags, grow << PAGE_SHIFT))
1020 return -ENOMEM;
1021
1022 /* Check to ensure the stack will not grow into a hugetlb-only region */
1023 new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
1024 vma->vm_end - size;
1025 if (is_hugepage_only_range(vma->vm_mm, new_start, size))
1026 return -EFAULT;
1027
1028 /*
1029 * Overcommit.. This must be the final test, as it will
1030 * update security statistics.
1031 */
1032 if (security_vm_enough_memory_mm(mm, grow))
1033 return -ENOMEM;
1034
1035 return 0;
1036 }
1037
1038 #if defined(CONFIG_STACK_GROWSUP)
1039 /*
1040 * PA-RISC uses this for its stack.
1041 * vma is the last one with address > vma->vm_end. Have to extend vma.
1042 */
expand_upwards(struct vm_area_struct * vma,unsigned long address)1043 static int expand_upwards(struct vm_area_struct *vma, unsigned long address)
1044 {
1045 struct mm_struct *mm = vma->vm_mm;
1046 struct vm_area_struct *next;
1047 unsigned long gap_addr;
1048 int error = 0;
1049 VMA_ITERATOR(vmi, mm, vma->vm_start);
1050
1051 if (!(vma->vm_flags & VM_GROWSUP))
1052 return -EFAULT;
1053
1054 /* Guard against exceeding limits of the address space. */
1055 address &= PAGE_MASK;
1056 if (address >= (TASK_SIZE & PAGE_MASK))
1057 return -ENOMEM;
1058 address += PAGE_SIZE;
1059
1060 /* Enforce stack_guard_gap */
1061 gap_addr = address + stack_guard_gap;
1062
1063 /* Guard against overflow */
1064 if (gap_addr < address || gap_addr > TASK_SIZE)
1065 gap_addr = TASK_SIZE;
1066
1067 next = find_vma_intersection(mm, vma->vm_end, gap_addr);
1068 if (next && vma_is_accessible(next)) {
1069 if (!(next->vm_flags & VM_GROWSUP))
1070 return -ENOMEM;
1071 /* Check that both stack segments have the same anon_vma? */
1072 }
1073
1074 if (next)
1075 vma_iter_prev_range_limit(&vmi, address);
1076
1077 vma_iter_config(&vmi, vma->vm_start, address);
1078 if (vma_iter_prealloc(&vmi, vma))
1079 return -ENOMEM;
1080
1081 /* We must make sure the anon_vma is allocated. */
1082 if (unlikely(anon_vma_prepare(vma))) {
1083 vma_iter_free(&vmi);
1084 return -ENOMEM;
1085 }
1086
1087 /* Lock the VMA before expanding to prevent concurrent page faults */
1088 vma_start_write(vma);
1089 /*
1090 * vma->vm_start/vm_end cannot change under us because the caller
1091 * is required to hold the mmap_lock in read mode. We need the
1092 * anon_vma lock to serialize against concurrent expand_stacks.
1093 */
1094 anon_vma_lock_write(vma->anon_vma);
1095
1096 /* Somebody else might have raced and expanded it already */
1097 if (address > vma->vm_end) {
1098 unsigned long size, grow;
1099
1100 size = address - vma->vm_start;
1101 grow = (address - vma->vm_end) >> PAGE_SHIFT;
1102
1103 error = -ENOMEM;
1104 if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
1105 error = acct_stack_growth(vma, size, grow);
1106 if (!error) {
1107 /*
1108 * We only hold a shared mmap_lock lock here, so
1109 * we need to protect against concurrent vma
1110 * expansions. anon_vma_lock_write() doesn't
1111 * help here, as we don't guarantee that all
1112 * growable vmas in a mm share the same root
1113 * anon vma. So, we reuse mm->page_table_lock
1114 * to guard against concurrent vma expansions.
1115 */
1116 spin_lock(&mm->page_table_lock);
1117 if (vma->vm_flags & VM_LOCKED)
1118 mm->locked_vm += grow;
1119 vm_stat_account(mm, vma->vm_flags, grow);
1120 anon_vma_interval_tree_pre_update_vma(vma);
1121 vma->vm_end = address;
1122 /* Overwrite old entry in mtree. */
1123 vma_iter_store(&vmi, vma);
1124 anon_vma_interval_tree_post_update_vma(vma);
1125 spin_unlock(&mm->page_table_lock);
1126
1127 perf_event_mmap(vma);
1128 }
1129 }
1130 }
1131 anon_vma_unlock_write(vma->anon_vma);
1132 vma_iter_free(&vmi);
1133 validate_mm(mm);
1134 return error;
1135 }
1136 #endif /* CONFIG_STACK_GROWSUP */
1137
1138 /*
1139 * vma is the first one with address < vma->vm_start. Have to extend vma.
1140 * mmap_lock held for writing.
1141 */
expand_downwards(struct vm_area_struct * vma,unsigned long address)1142 int expand_downwards(struct vm_area_struct *vma, unsigned long address)
1143 {
1144 struct mm_struct *mm = vma->vm_mm;
1145 struct vm_area_struct *prev;
1146 int error = 0;
1147 VMA_ITERATOR(vmi, mm, vma->vm_start);
1148
1149 if (!(vma->vm_flags & VM_GROWSDOWN))
1150 return -EFAULT;
1151
1152 address &= PAGE_MASK;
1153 if (address < mmap_min_addr || address < FIRST_USER_ADDRESS)
1154 return -EPERM;
1155
1156 /* Enforce stack_guard_gap */
1157 prev = vma_prev(&vmi);
1158 /* Check that both stack segments have the same anon_vma? */
1159 if (prev) {
1160 if (!(prev->vm_flags & VM_GROWSDOWN) &&
1161 vma_is_accessible(prev) &&
1162 (address - prev->vm_end < stack_guard_gap))
1163 return -ENOMEM;
1164 }
1165
1166 if (prev)
1167 vma_iter_next_range_limit(&vmi, vma->vm_start);
1168
1169 vma_iter_config(&vmi, address, vma->vm_end);
1170 if (vma_iter_prealloc(&vmi, vma))
1171 return -ENOMEM;
1172
1173 /* We must make sure the anon_vma is allocated. */
1174 if (unlikely(anon_vma_prepare(vma))) {
1175 vma_iter_free(&vmi);
1176 return -ENOMEM;
1177 }
1178
1179 /* Lock the VMA before expanding to prevent concurrent page faults */
1180 vma_start_write(vma);
1181 /*
1182 * vma->vm_start/vm_end cannot change under us because the caller
1183 * is required to hold the mmap_lock in read mode. We need the
1184 * anon_vma lock to serialize against concurrent expand_stacks.
1185 */
1186 anon_vma_lock_write(vma->anon_vma);
1187
1188 /* Somebody else might have raced and expanded it already */
1189 if (address < vma->vm_start) {
1190 unsigned long size, grow;
1191
1192 size = vma->vm_end - address;
1193 grow = (vma->vm_start - address) >> PAGE_SHIFT;
1194
1195 error = -ENOMEM;
1196 if (grow <= vma->vm_pgoff) {
1197 error = acct_stack_growth(vma, size, grow);
1198 if (!error) {
1199 /*
1200 * We only hold a shared mmap_lock lock here, so
1201 * we need to protect against concurrent vma
1202 * expansions. anon_vma_lock_write() doesn't
1203 * help here, as we don't guarantee that all
1204 * growable vmas in a mm share the same root
1205 * anon vma. So, we reuse mm->page_table_lock
1206 * to guard against concurrent vma expansions.
1207 */
1208 spin_lock(&mm->page_table_lock);
1209 if (vma->vm_flags & VM_LOCKED)
1210 mm->locked_vm += grow;
1211 vm_stat_account(mm, vma->vm_flags, grow);
1212 anon_vma_interval_tree_pre_update_vma(vma);
1213 vma->vm_start = address;
1214 vma->vm_pgoff -= grow;
1215 /* Overwrite old entry in mtree. */
1216 vma_iter_store(&vmi, vma);
1217 anon_vma_interval_tree_post_update_vma(vma);
1218 spin_unlock(&mm->page_table_lock);
1219
1220 perf_event_mmap(vma);
1221 }
1222 }
1223 }
1224 anon_vma_unlock_write(vma->anon_vma);
1225 vma_iter_free(&vmi);
1226 validate_mm(mm);
1227 return error;
1228 }
1229
1230 /* enforced gap between the expanding stack and other mappings. */
1231 unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
1232
cmdline_parse_stack_guard_gap(char * p)1233 static int __init cmdline_parse_stack_guard_gap(char *p)
1234 {
1235 unsigned long val;
1236 char *endptr;
1237
1238 val = simple_strtoul(p, &endptr, 10);
1239 if (!*endptr)
1240 stack_guard_gap = val << PAGE_SHIFT;
1241
1242 return 1;
1243 }
1244 __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
1245
1246 #ifdef CONFIG_STACK_GROWSUP
expand_stack_locked(struct vm_area_struct * vma,unsigned long address)1247 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
1248 {
1249 return expand_upwards(vma, address);
1250 }
1251
find_extend_vma_locked(struct mm_struct * mm,unsigned long addr)1252 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
1253 {
1254 struct vm_area_struct *vma, *prev;
1255
1256 addr &= PAGE_MASK;
1257 vma = find_vma_prev(mm, addr, &prev);
1258 if (vma && (vma->vm_start <= addr))
1259 return vma;
1260 if (!prev)
1261 return NULL;
1262 if (expand_stack_locked(prev, addr))
1263 return NULL;
1264 if (prev->vm_flags & VM_LOCKED)
1265 populate_vma_page_range(prev, addr, prev->vm_end, NULL);
1266 return prev;
1267 }
1268 #else
expand_stack_locked(struct vm_area_struct * vma,unsigned long address)1269 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
1270 {
1271 return expand_downwards(vma, address);
1272 }
1273
find_extend_vma_locked(struct mm_struct * mm,unsigned long addr)1274 struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
1275 {
1276 struct vm_area_struct *vma;
1277 unsigned long start;
1278
1279 addr &= PAGE_MASK;
1280 vma = find_vma(mm, addr);
1281 if (!vma)
1282 return NULL;
1283 if (vma->vm_start <= addr)
1284 return vma;
1285 start = vma->vm_start;
1286 if (expand_stack_locked(vma, addr))
1287 return NULL;
1288 if (vma->vm_flags & VM_LOCKED)
1289 populate_vma_page_range(vma, addr, start, NULL);
1290 return vma;
1291 }
1292 #endif
1293
1294 #if defined(CONFIG_STACK_GROWSUP)
1295
1296 #define vma_expand_up(vma,addr) expand_upwards(vma, addr)
1297 #define vma_expand_down(vma, addr) (-EFAULT)
1298
1299 #else
1300
1301 #define vma_expand_up(vma,addr) (-EFAULT)
1302 #define vma_expand_down(vma, addr) expand_downwards(vma, addr)
1303
1304 #endif
1305
1306 /*
1307 * expand_stack(): legacy interface for page faulting. Don't use unless
1308 * you have to.
1309 *
1310 * This is called with the mm locked for reading, drops the lock, takes
1311 * the lock for writing, tries to look up a vma again, expands it if
1312 * necessary, and downgrades the lock to reading again.
1313 *
1314 * If no vma is found or it can't be expanded, it returns NULL and has
1315 * dropped the lock.
1316 */
expand_stack(struct mm_struct * mm,unsigned long addr)1317 struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
1318 {
1319 struct vm_area_struct *vma, *prev;
1320
1321 mmap_read_unlock(mm);
1322 if (mmap_write_lock_killable(mm))
1323 return NULL;
1324
1325 vma = find_vma_prev(mm, addr, &prev);
1326 if (vma && vma->vm_start <= addr)
1327 goto success;
1328
1329 if (prev && !vma_expand_up(prev, addr)) {
1330 vma = prev;
1331 goto success;
1332 }
1333
1334 if (vma && !vma_expand_down(vma, addr))
1335 goto success;
1336
1337 mmap_write_unlock(mm);
1338 return NULL;
1339
1340 success:
1341 mmap_write_downgrade(mm);
1342 return vma;
1343 }
1344
1345 /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
1346 * @mm: The mm_struct
1347 * @start: The start address to munmap
1348 * @len: The length to be munmapped.
1349 * @uf: The userfaultfd list_head
1350 *
1351 * Return: 0 on success, error otherwise.
1352 */
do_munmap(struct mm_struct * mm,unsigned long start,size_t len,struct list_head * uf)1353 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
1354 struct list_head *uf)
1355 {
1356 VMA_ITERATOR(vmi, mm, start);
1357
1358 return do_vmi_munmap(&vmi, mm, start, len, uf, false);
1359 }
1360
__mmap_region(struct file * file,unsigned long addr,unsigned long len,vm_flags_t vm_flags,unsigned long pgoff,struct list_head * uf)1361 static unsigned long __mmap_region(struct file *file, unsigned long addr,
1362 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
1363 struct list_head *uf)
1364 {
1365 struct mm_struct *mm = current->mm;
1366 struct vm_area_struct *vma = NULL;
1367 pgoff_t pglen = PHYS_PFN(len);
1368 unsigned long charged = 0;
1369 struct vma_munmap_struct vms;
1370 struct ma_state mas_detach;
1371 struct maple_tree mt_detach;
1372 unsigned long end = addr + len;
1373 int error;
1374 VMA_ITERATOR(vmi, mm, addr);
1375 VMG_STATE(vmg, mm, &vmi, addr, end, vm_flags, pgoff);
1376
1377 vmg.file = file;
1378 /* Find the first overlapping VMA */
1379 vma = vma_find(&vmi, end);
1380 init_vma_munmap(&vms, &vmi, vma, addr, end, uf, /* unlock = */ false);
1381 if (vma) {
1382 mt_init_flags(&mt_detach, vmi.mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
1383 mt_on_stack(mt_detach);
1384 mas_init(&mas_detach, &mt_detach, /* addr = */ 0);
1385 /* Prepare to unmap any existing mapping in the area */
1386 error = vms_gather_munmap_vmas(&vms, &mas_detach);
1387 if (error)
1388 goto gather_failed;
1389
1390 vmg.next = vms.next;
1391 vmg.prev = vms.prev;
1392 vma = NULL;
1393 } else {
1394 vmg.next = vma_iter_next_rewind(&vmi, &vmg.prev);
1395 }
1396
1397 /* Check against address space limit. */
1398 if (!may_expand_vm(mm, vm_flags, pglen - vms.nr_pages)) {
1399 error = -ENOMEM;
1400 goto abort_munmap;
1401 }
1402
1403 /*
1404 * Private writable mapping: check memory availability
1405 */
1406 if (accountable_mapping(file, vm_flags)) {
1407 charged = pglen;
1408 charged -= vms.nr_accounted;
1409 if (charged) {
1410 error = security_vm_enough_memory_mm(mm, charged);
1411 if (error)
1412 goto abort_munmap;
1413 }
1414
1415 vms.nr_accounted = 0;
1416 vm_flags |= VM_ACCOUNT;
1417 vmg.flags = vm_flags;
1418 }
1419
1420 /*
1421 * clear PTEs while the vma is still in the tree so that rmap
1422 * cannot race with the freeing later in the truncate scenario.
1423 * This is also needed for mmap_file(), which is why vm_ops
1424 * close function is called.
1425 */
1426 vms_clean_up_area(&vms, &mas_detach);
1427 vma = vma_merge_new_range(&vmg);
1428 if (vma)
1429 goto expanded;
1430 /*
1431 * Determine the object being mapped and call the appropriate
1432 * specific mapper. the address has already been validated, but
1433 * not unmapped, but the maps are removed from the list.
1434 */
1435 vma = vm_area_alloc(mm);
1436 if (!vma) {
1437 error = -ENOMEM;
1438 goto unacct_error;
1439 }
1440
1441 vma_iter_config(&vmi, addr, end);
1442 vma_set_range(vma, addr, end, pgoff);
1443 vm_flags_init(vma, vm_flags);
1444 vma->vm_page_prot = vm_get_page_prot(vm_flags);
1445
1446 if (vma_iter_prealloc(&vmi, vma)) {
1447 error = -ENOMEM;
1448 goto free_vma;
1449 }
1450
1451 if (file) {
1452 vma->vm_file = get_file(file);
1453 error = mmap_file(file, vma);
1454 if (error)
1455 goto unmap_and_free_file_vma;
1456
1457 /* Drivers cannot alter the address of the VMA. */
1458 WARN_ON_ONCE(addr != vma->vm_start);
1459 /*
1460 * Drivers should not permit writability when previously it was
1461 * disallowed.
1462 */
1463 VM_WARN_ON_ONCE(vm_flags != vma->vm_flags &&
1464 !(vm_flags & VM_MAYWRITE) &&
1465 (vma->vm_flags & VM_MAYWRITE));
1466
1467 vma_iter_config(&vmi, addr, end);
1468 /*
1469 * If vm_flags changed after mmap_file(), we should try merge
1470 * vma again as we may succeed this time.
1471 */
1472 if (unlikely(vm_flags != vma->vm_flags && vmg.prev)) {
1473 struct vm_area_struct *merge;
1474
1475 vmg.flags = vma->vm_flags;
1476 /* If this fails, state is reset ready for a reattempt. */
1477 merge = vma_merge_new_range(&vmg);
1478
1479 if (merge) {
1480 /*
1481 * ->mmap() can change vma->vm_file and fput
1482 * the original file. So fput the vma->vm_file
1483 * here or we would add an extra fput for file
1484 * and cause general protection fault
1485 * ultimately.
1486 */
1487 fput(vma->vm_file);
1488 vm_area_free(vma);
1489 vma = merge;
1490 /* Update vm_flags to pick up the change. */
1491 vm_flags = vma->vm_flags;
1492 goto file_expanded;
1493 }
1494
1495 /*
1496 * In the unlikely even that more memory was needed, but
1497 * not available for the vma merge, the vma iterator
1498 * will have no memory reserved for the write we told
1499 * the driver was happening. To keep up the ruse,
1500 * ensure the allocation for the store succeeds.
1501 */
1502 if (vmg_nomem(&vmg)) {
1503 mas_preallocate(&vmi.mas, vma,
1504 GFP_KERNEL|__GFP_NOFAIL);
1505 }
1506 }
1507
1508 vm_flags = vma->vm_flags;
1509 } else if (vm_flags & VM_SHARED) {
1510 error = shmem_zero_setup(vma);
1511 if (error)
1512 goto free_iter_vma;
1513 } else {
1514 vma_set_anonymous(vma);
1515 }
1516
1517 #ifdef CONFIG_SPARC64
1518 /* TODO: Fix SPARC ADI! */
1519 WARN_ON_ONCE(!arch_validate_flags(vm_flags));
1520 #endif
1521
1522 /* Lock the VMA since it is modified after insertion into VMA tree */
1523 vma_start_write(vma);
1524 vma_iter_store(&vmi, vma);
1525 mm->map_count++;
1526 vma_link_file(vma);
1527
1528 /*
1529 * vma_merge_new_range() calls khugepaged_enter_vma() too, the below
1530 * call covers the non-merge case.
1531 */
1532 khugepaged_enter_vma(vma, vma->vm_flags);
1533
1534 file_expanded:
1535 file = vma->vm_file;
1536 ksm_add_vma(vma);
1537 expanded:
1538 perf_event_mmap(vma);
1539
1540 /* Unmap any existing mapping in the area */
1541 vms_complete_munmap_vmas(&vms, &mas_detach);
1542
1543 vm_stat_account(mm, vm_flags, pglen);
1544 if (vm_flags & VM_LOCKED) {
1545 if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
1546 is_vm_hugetlb_page(vma) ||
1547 vma == get_gate_vma(current->mm))
1548 vm_flags_clear(vma, VM_LOCKED_MASK);
1549 else
1550 mm->locked_vm += pglen;
1551 }
1552
1553 if (file)
1554 uprobe_mmap(vma);
1555
1556 /*
1557 * New (or expanded) vma always get soft dirty status.
1558 * Otherwise user-space soft-dirty page tracker won't
1559 * be able to distinguish situation when vma area unmapped,
1560 * then new mapped in-place (which must be aimed as
1561 * a completely new data area).
1562 */
1563 vm_flags_set(vma, VM_SOFTDIRTY);
1564
1565 vma_set_page_prot(vma);
1566
1567 return addr;
1568
1569 unmap_and_free_file_vma:
1570 fput(vma->vm_file);
1571 vma->vm_file = NULL;
1572
1573 vma_iter_set(&vmi, vma->vm_end);
1574 /* Undo any partial mapping done by a device driver. */
1575 unmap_region(&vmi.mas, vma, vmg.prev, vmg.next);
1576 free_iter_vma:
1577 vma_iter_free(&vmi);
1578 free_vma:
1579 vm_area_free(vma);
1580 unacct_error:
1581 if (charged)
1582 vm_unacct_memory(charged);
1583
1584 abort_munmap:
1585 vms_abort_munmap_vmas(&vms, &mas_detach);
1586 gather_failed:
1587 return error;
1588 }
1589
mmap_region(struct file * file,unsigned long addr,unsigned long len,vm_flags_t vm_flags,unsigned long pgoff,struct list_head * uf)1590 unsigned long mmap_region(struct file *file, unsigned long addr,
1591 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
1592 struct list_head *uf)
1593 {
1594 unsigned long ret;
1595 bool writable_file_mapping = false;
1596
1597 /* Check to see if MDWE is applicable. */
1598 if (map_deny_write_exec(vm_flags, vm_flags))
1599 return -EACCES;
1600
1601 /* Allow architectures to sanity-check the vm_flags. */
1602 if (!arch_validate_flags(vm_flags))
1603 return -EINVAL;
1604
1605 /* Map writable and ensure this isn't a sealed memfd. */
1606 if (file && is_shared_maywrite(vm_flags)) {
1607 int error = mapping_map_writable(file->f_mapping);
1608
1609 if (error)
1610 return error;
1611 writable_file_mapping = true;
1612 }
1613
1614 ret = __mmap_region(file, addr, len, vm_flags, pgoff, uf);
1615
1616 /* Clear our write mapping regardless of error. */
1617 if (writable_file_mapping)
1618 mapping_unmap_writable(file->f_mapping);
1619
1620 validate_mm(current->mm);
1621 return ret;
1622 }
1623
__vm_munmap(unsigned long start,size_t len,bool unlock)1624 static int __vm_munmap(unsigned long start, size_t len, bool unlock)
1625 {
1626 int ret;
1627 struct mm_struct *mm = current->mm;
1628 LIST_HEAD(uf);
1629 VMA_ITERATOR(vmi, mm, start);
1630
1631 if (mmap_write_lock_killable(mm))
1632 return -EINTR;
1633
1634 ret = do_vmi_munmap(&vmi, mm, start, len, &uf, unlock);
1635 if (ret || !unlock)
1636 mmap_write_unlock(mm);
1637
1638 userfaultfd_unmap_complete(mm, &uf);
1639 return ret;
1640 }
1641
vm_munmap(unsigned long start,size_t len)1642 int vm_munmap(unsigned long start, size_t len)
1643 {
1644 return __vm_munmap(start, len, false);
1645 }
1646 EXPORT_SYMBOL(vm_munmap);
1647
SYSCALL_DEFINE2(munmap,unsigned long,addr,size_t,len)1648 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1649 {
1650 addr = untagged_addr(addr);
1651 return __vm_munmap(addr, len, true);
1652 }
1653
1654
1655 /*
1656 * Emulation of deprecated remap_file_pages() syscall.
1657 */
SYSCALL_DEFINE5(remap_file_pages,unsigned long,start,unsigned long,size,unsigned long,prot,unsigned long,pgoff,unsigned long,flags)1658 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
1659 unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
1660 {
1661
1662 struct mm_struct *mm = current->mm;
1663 struct vm_area_struct *vma;
1664 unsigned long populate = 0;
1665 unsigned long ret = -EINVAL;
1666 struct file *file;
1667 vm_flags_t vm_flags;
1668
1669 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n",
1670 current->comm, current->pid);
1671
1672 if (prot)
1673 return ret;
1674 start = start & PAGE_MASK;
1675 size = size & PAGE_MASK;
1676
1677 if (start + size <= start)
1678 return ret;
1679
1680 /* Does pgoff wrap? */
1681 if (pgoff + (size >> PAGE_SHIFT) < pgoff)
1682 return ret;
1683
1684 if (mmap_read_lock_killable(mm))
1685 return -EINTR;
1686
1687 /*
1688 * Look up VMA under read lock first so we can perform the security
1689 * without holding locks (which can be problematic). We reacquire a
1690 * write lock later and check nothing changed underneath us.
1691 */
1692 vma = vma_lookup(mm, start);
1693
1694 if (!vma || !(vma->vm_flags & VM_SHARED)) {
1695 mmap_read_unlock(mm);
1696 return -EINVAL;
1697 }
1698
1699 prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
1700 prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
1701 prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
1702
1703 flags &= MAP_NONBLOCK;
1704 flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
1705 if (vma->vm_flags & VM_LOCKED)
1706 flags |= MAP_LOCKED;
1707
1708 /* Save vm_flags used to calculate prot and flags, and recheck later. */
1709 vm_flags = vma->vm_flags;
1710 file = get_file(vma->vm_file);
1711
1712 mmap_read_unlock(mm);
1713
1714 /* Call outside mmap_lock to be consistent with other callers. */
1715 ret = security_mmap_file(file, prot, flags);
1716 if (ret) {
1717 fput(file);
1718 return ret;
1719 }
1720
1721 ret = -EINVAL;
1722
1723 /* OK security check passed, take write lock + let it rip. */
1724 if (mmap_write_lock_killable(mm)) {
1725 fput(file);
1726 return -EINTR;
1727 }
1728
1729 vma = vma_lookup(mm, start);
1730
1731 if (!vma)
1732 goto out;
1733
1734 /* Make sure things didn't change under us. */
1735 if (vma->vm_flags != vm_flags)
1736 goto out;
1737 if (vma->vm_file != file)
1738 goto out;
1739
1740 if (start + size > vma->vm_end) {
1741 VMA_ITERATOR(vmi, mm, vma->vm_end);
1742 struct vm_area_struct *next, *prev = vma;
1743
1744 for_each_vma_range(vmi, next, start + size) {
1745 /* hole between vmas ? */
1746 if (next->vm_start != prev->vm_end)
1747 goto out;
1748
1749 if (next->vm_file != vma->vm_file)
1750 goto out;
1751
1752 if (next->vm_flags != vma->vm_flags)
1753 goto out;
1754
1755 if (start + size <= next->vm_end)
1756 break;
1757
1758 prev = next;
1759 }
1760
1761 if (!next)
1762 goto out;
1763 }
1764
1765 ret = do_mmap(vma->vm_file, start, size,
1766 prot, flags, 0, pgoff, &populate, NULL);
1767 out:
1768 mmap_write_unlock(mm);
1769 fput(file);
1770 if (populate)
1771 mm_populate(ret, populate);
1772 if (!IS_ERR_VALUE(ret))
1773 ret = 0;
1774 return ret;
1775 }
1776
1777 /*
1778 * do_brk_flags() - Increase the brk vma if the flags match.
1779 * @vmi: The vma iterator
1780 * @addr: The start address
1781 * @len: The length of the increase
1782 * @vma: The vma,
1783 * @flags: The VMA Flags
1784 *
1785 * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags
1786 * do not match then create a new anonymous VMA. Eventually we may be able to
1787 * do some brk-specific accounting here.
1788 */
do_brk_flags(struct vma_iterator * vmi,struct vm_area_struct * vma,unsigned long addr,unsigned long len,unsigned long flags)1789 static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma,
1790 unsigned long addr, unsigned long len, unsigned long flags)
1791 {
1792 struct mm_struct *mm = current->mm;
1793
1794 /*
1795 * Check against address space limits by the changed size
1796 * Note: This happens *after* clearing old mappings in some code paths.
1797 */
1798 flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
1799 if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
1800 return -ENOMEM;
1801
1802 if (mm->map_count > sysctl_max_map_count)
1803 return -ENOMEM;
1804
1805 if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
1806 return -ENOMEM;
1807
1808 /*
1809 * Expand the existing vma if possible; Note that singular lists do not
1810 * occur after forking, so the expand will only happen on new VMAs.
1811 */
1812 if (vma && vma->vm_end == addr) {
1813 VMG_STATE(vmg, mm, vmi, addr, addr + len, flags, PHYS_PFN(addr));
1814
1815 vmg.prev = vma;
1816 /* vmi is positioned at prev, which this mode expects. */
1817 vmg.merge_flags = VMG_FLAG_JUST_EXPAND;
1818
1819 if (vma_merge_new_range(&vmg))
1820 goto out;
1821 else if (vmg_nomem(&vmg))
1822 goto unacct_fail;
1823 }
1824
1825 if (vma)
1826 vma_iter_next_range(vmi);
1827 /* create a vma struct for an anonymous mapping */
1828 vma = vm_area_alloc(mm);
1829 if (!vma)
1830 goto unacct_fail;
1831
1832 vma_set_anonymous(vma);
1833 vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT);
1834 vm_flags_init(vma, flags);
1835 vma->vm_page_prot = vm_get_page_prot(flags);
1836 vma_start_write(vma);
1837 if (vma_iter_store_gfp(vmi, vma, GFP_KERNEL))
1838 goto mas_store_fail;
1839
1840 mm->map_count++;
1841 validate_mm(mm);
1842 ksm_add_vma(vma);
1843 out:
1844 perf_event_mmap(vma);
1845 mm->total_vm += len >> PAGE_SHIFT;
1846 mm->data_vm += len >> PAGE_SHIFT;
1847 if (flags & VM_LOCKED)
1848 mm->locked_vm += (len >> PAGE_SHIFT);
1849 vm_flags_set(vma, VM_SOFTDIRTY);
1850 return 0;
1851
1852 mas_store_fail:
1853 vm_area_free(vma);
1854 unacct_fail:
1855 vm_unacct_memory(len >> PAGE_SHIFT);
1856 return -ENOMEM;
1857 }
1858
vm_brk_flags(unsigned long addr,unsigned long request,unsigned long flags)1859 int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
1860 {
1861 struct mm_struct *mm = current->mm;
1862 struct vm_area_struct *vma = NULL;
1863 unsigned long len;
1864 int ret;
1865 bool populate;
1866 LIST_HEAD(uf);
1867 VMA_ITERATOR(vmi, mm, addr);
1868
1869 len = PAGE_ALIGN(request);
1870 if (len < request)
1871 return -ENOMEM;
1872 if (!len)
1873 return 0;
1874
1875 /* Until we need other flags, refuse anything except VM_EXEC. */
1876 if ((flags & (~VM_EXEC)) != 0)
1877 return -EINVAL;
1878
1879 if (mmap_write_lock_killable(mm))
1880 return -EINTR;
1881
1882 ret = check_brk_limits(addr, len);
1883 if (ret)
1884 goto limits_failed;
1885
1886 ret = do_vmi_munmap(&vmi, mm, addr, len, &uf, 0);
1887 if (ret)
1888 goto munmap_failed;
1889
1890 vma = vma_prev(&vmi);
1891 ret = do_brk_flags(&vmi, vma, addr, len, flags);
1892 populate = ((mm->def_flags & VM_LOCKED) != 0);
1893 mmap_write_unlock(mm);
1894 userfaultfd_unmap_complete(mm, &uf);
1895 if (populate && !ret)
1896 mm_populate(addr, len);
1897 return ret;
1898
1899 munmap_failed:
1900 limits_failed:
1901 mmap_write_unlock(mm);
1902 return ret;
1903 }
1904 EXPORT_SYMBOL(vm_brk_flags);
1905
1906 /* Release all mmaps. */
exit_mmap(struct mm_struct * mm)1907 void exit_mmap(struct mm_struct *mm)
1908 {
1909 struct mmu_gather tlb;
1910 struct vm_area_struct *vma;
1911 unsigned long nr_accounted = 0;
1912 VMA_ITERATOR(vmi, mm, 0);
1913 int count = 0;
1914
1915 /* mm's last user has gone, and its about to be pulled down */
1916 mmu_notifier_release(mm);
1917
1918 mmap_read_lock(mm);
1919 arch_exit_mmap(mm);
1920
1921 vma = vma_next(&vmi);
1922 if (!vma || unlikely(xa_is_zero(vma))) {
1923 /* Can happen if dup_mmap() received an OOM */
1924 mmap_read_unlock(mm);
1925 mmap_write_lock(mm);
1926 goto destroy;
1927 }
1928
1929 lru_add_drain();
1930 flush_cache_mm(mm);
1931 tlb_gather_mmu_fullmm(&tlb, mm);
1932 /* update_hiwater_rss(mm) here? but nobody should be looking */
1933 /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */
1934 unmap_vmas(&tlb, &vmi.mas, vma, 0, ULONG_MAX, ULONG_MAX, false);
1935 mmap_read_unlock(mm);
1936
1937 /*
1938 * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper
1939 * because the memory has been already freed.
1940 */
1941 set_bit(MMF_OOM_SKIP, &mm->flags);
1942 mmap_write_lock(mm);
1943 mt_clear_in_rcu(&mm->mm_mt);
1944 vma_iter_set(&vmi, vma->vm_end);
1945 free_pgtables(&tlb, &vmi.mas, vma, FIRST_USER_ADDRESS,
1946 USER_PGTABLES_CEILING, true);
1947 tlb_finish_mmu(&tlb);
1948
1949 /*
1950 * Walk the list again, actually closing and freeing it, with preemption
1951 * enabled, without holding any MM locks besides the unreachable
1952 * mmap_write_lock.
1953 */
1954 vma_iter_set(&vmi, vma->vm_end);
1955 do {
1956 if (vma->vm_flags & VM_ACCOUNT)
1957 nr_accounted += vma_pages(vma);
1958 remove_vma(vma, /* unreachable = */ true);
1959 count++;
1960 cond_resched();
1961 vma = vma_next(&vmi);
1962 } while (vma && likely(!xa_is_zero(vma)));
1963
1964 BUG_ON(count != mm->map_count);
1965
1966 trace_exit_mmap(mm);
1967 destroy:
1968 __mt_destroy(&mm->mm_mt);
1969 mmap_write_unlock(mm);
1970 vm_unacct_memory(nr_accounted);
1971 }
1972
1973 /* Insert vm structure into process list sorted by address
1974 * and into the inode's i_mmap tree. If vm_file is non-NULL
1975 * then i_mmap_rwsem is taken here.
1976 */
insert_vm_struct(struct mm_struct * mm,struct vm_area_struct * vma)1977 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
1978 {
1979 unsigned long charged = vma_pages(vma);
1980
1981
1982 if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
1983 return -ENOMEM;
1984
1985 if ((vma->vm_flags & VM_ACCOUNT) &&
1986 security_vm_enough_memory_mm(mm, charged))
1987 return -ENOMEM;
1988
1989 /*
1990 * The vm_pgoff of a purely anonymous vma should be irrelevant
1991 * until its first write fault, when page's anon_vma and index
1992 * are set. But now set the vm_pgoff it will almost certainly
1993 * end up with (unless mremap moves it elsewhere before that
1994 * first wfault), so /proc/pid/maps tells a consistent story.
1995 *
1996 * By setting it to reflect the virtual start address of the
1997 * vma, merges and splits can happen in a seamless way, just
1998 * using the existing file pgoff checks and manipulations.
1999 * Similarly in do_mmap and in do_brk_flags.
2000 */
2001 if (vma_is_anonymous(vma)) {
2002 BUG_ON(vma->anon_vma);
2003 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2004 }
2005
2006 if (vma_link(mm, vma)) {
2007 if (vma->vm_flags & VM_ACCOUNT)
2008 vm_unacct_memory(charged);
2009 return -ENOMEM;
2010 }
2011
2012 return 0;
2013 }
2014
2015 /*
2016 * Return true if the calling process may expand its vm space by the passed
2017 * number of pages
2018 */
may_expand_vm(struct mm_struct * mm,vm_flags_t flags,unsigned long npages)2019 bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
2020 {
2021 if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
2022 return false;
2023
2024 if (is_data_mapping(flags) &&
2025 mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
2026 /* Workaround for Valgrind */
2027 if (rlimit(RLIMIT_DATA) == 0 &&
2028 mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
2029 return true;
2030
2031 pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n",
2032 current->comm, current->pid,
2033 (mm->data_vm + npages) << PAGE_SHIFT,
2034 rlimit(RLIMIT_DATA),
2035 ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data");
2036
2037 if (!ignore_rlimit_data)
2038 return false;
2039 }
2040
2041 return true;
2042 }
2043
vm_stat_account(struct mm_struct * mm,vm_flags_t flags,long npages)2044 void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
2045 {
2046 WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages);
2047
2048 if (is_exec_mapping(flags))
2049 mm->exec_vm += npages;
2050 else if (is_stack_mapping(flags))
2051 mm->stack_vm += npages;
2052 else if (is_data_mapping(flags))
2053 mm->data_vm += npages;
2054 }
2055
2056 static vm_fault_t special_mapping_fault(struct vm_fault *vmf);
2057
2058 /*
2059 * Close hook, called for unmap() and on the old vma for mremap().
2060 *
2061 * Having a close hook prevents vma merging regardless of flags.
2062 */
special_mapping_close(struct vm_area_struct * vma)2063 static void special_mapping_close(struct vm_area_struct *vma)
2064 {
2065 const struct vm_special_mapping *sm = vma->vm_private_data;
2066
2067 if (sm->close)
2068 sm->close(sm, vma);
2069 }
2070
special_mapping_name(struct vm_area_struct * vma)2071 static const char *special_mapping_name(struct vm_area_struct *vma)
2072 {
2073 return ((struct vm_special_mapping *)vma->vm_private_data)->name;
2074 }
2075
special_mapping_mremap(struct vm_area_struct * new_vma)2076 static int special_mapping_mremap(struct vm_area_struct *new_vma)
2077 {
2078 struct vm_special_mapping *sm = new_vma->vm_private_data;
2079
2080 if (WARN_ON_ONCE(current->mm != new_vma->vm_mm))
2081 return -EFAULT;
2082
2083 if (sm->mremap)
2084 return sm->mremap(sm, new_vma);
2085
2086 return 0;
2087 }
2088
special_mapping_split(struct vm_area_struct * vma,unsigned long addr)2089 static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr)
2090 {
2091 /*
2092 * Forbid splitting special mappings - kernel has expectations over
2093 * the number of pages in mapping. Together with VM_DONTEXPAND
2094 * the size of vma should stay the same over the special mapping's
2095 * lifetime.
2096 */
2097 return -EINVAL;
2098 }
2099
2100 static const struct vm_operations_struct special_mapping_vmops = {
2101 .close = special_mapping_close,
2102 .fault = special_mapping_fault,
2103 .mremap = special_mapping_mremap,
2104 .name = special_mapping_name,
2105 /* vDSO code relies that VVAR can't be accessed remotely */
2106 .access = NULL,
2107 .may_split = special_mapping_split,
2108 };
2109
special_mapping_fault(struct vm_fault * vmf)2110 static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
2111 {
2112 struct vm_area_struct *vma = vmf->vma;
2113 pgoff_t pgoff;
2114 struct page **pages;
2115 struct vm_special_mapping *sm = vma->vm_private_data;
2116
2117 if (sm->fault)
2118 return sm->fault(sm, vmf->vma, vmf);
2119
2120 pages = sm->pages;
2121
2122 for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
2123 pgoff--;
2124
2125 if (*pages) {
2126 struct page *page = *pages;
2127 get_page(page);
2128 vmf->page = page;
2129 return 0;
2130 }
2131
2132 return VM_FAULT_SIGBUS;
2133 }
2134
__install_special_mapping(struct mm_struct * mm,unsigned long addr,unsigned long len,unsigned long vm_flags,void * priv,const struct vm_operations_struct * ops)2135 static struct vm_area_struct *__install_special_mapping(
2136 struct mm_struct *mm,
2137 unsigned long addr, unsigned long len,
2138 unsigned long vm_flags, void *priv,
2139 const struct vm_operations_struct *ops)
2140 {
2141 int ret;
2142 struct vm_area_struct *vma;
2143
2144 vma = vm_area_alloc(mm);
2145 if (unlikely(vma == NULL))
2146 return ERR_PTR(-ENOMEM);
2147
2148 vma_set_range(vma, addr, addr + len, 0);
2149 vm_flags_init(vma, (vm_flags | mm->def_flags |
2150 VM_DONTEXPAND | VM_SOFTDIRTY) & ~VM_LOCKED_MASK);
2151 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2152
2153 vma->vm_ops = ops;
2154 vma->vm_private_data = priv;
2155
2156 ret = insert_vm_struct(mm, vma);
2157 if (ret)
2158 goto out;
2159
2160 vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
2161
2162 perf_event_mmap(vma);
2163
2164 return vma;
2165
2166 out:
2167 vm_area_free(vma);
2168 return ERR_PTR(ret);
2169 }
2170
vma_is_special_mapping(const struct vm_area_struct * vma,const struct vm_special_mapping * sm)2171 bool vma_is_special_mapping(const struct vm_area_struct *vma,
2172 const struct vm_special_mapping *sm)
2173 {
2174 return vma->vm_private_data == sm &&
2175 vma->vm_ops == &special_mapping_vmops;
2176 }
2177
2178 /*
2179 * Called with mm->mmap_lock held for writing.
2180 * Insert a new vma covering the given region, with the given flags.
2181 * Its pages are supplied by the given array of struct page *.
2182 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
2183 * The region past the last page supplied will always produce SIGBUS.
2184 * The array pointer and the pages it points to are assumed to stay alive
2185 * for as long as this mapping might exist.
2186 */
_install_special_mapping(struct mm_struct * mm,unsigned long addr,unsigned long len,unsigned long vm_flags,const struct vm_special_mapping * spec)2187 struct vm_area_struct *_install_special_mapping(
2188 struct mm_struct *mm,
2189 unsigned long addr, unsigned long len,
2190 unsigned long vm_flags, const struct vm_special_mapping *spec)
2191 {
2192 return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
2193 &special_mapping_vmops);
2194 }
2195
2196 /*
2197 * initialise the percpu counter for VM
2198 */
mmap_init(void)2199 void __init mmap_init(void)
2200 {
2201 int ret;
2202
2203 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
2204 VM_BUG_ON(ret);
2205 }
2206
2207 /*
2208 * Initialise sysctl_user_reserve_kbytes.
2209 *
2210 * This is intended to prevent a user from starting a single memory hogging
2211 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
2212 * mode.
2213 *
2214 * The default value is min(3% of free memory, 128MB)
2215 * 128MB is enough to recover with sshd/login, bash, and top/kill.
2216 */
init_user_reserve(void)2217 static int init_user_reserve(void)
2218 {
2219 unsigned long free_kbytes;
2220
2221 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
2222
2223 sysctl_user_reserve_kbytes = min(free_kbytes / 32, SZ_128K);
2224 return 0;
2225 }
2226 subsys_initcall(init_user_reserve);
2227
2228 /*
2229 * Initialise sysctl_admin_reserve_kbytes.
2230 *
2231 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
2232 * to log in and kill a memory hogging process.
2233 *
2234 * Systems with more than 256MB will reserve 8MB, enough to recover
2235 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
2236 * only reserve 3% of free pages by default.
2237 */
init_admin_reserve(void)2238 static int init_admin_reserve(void)
2239 {
2240 unsigned long free_kbytes;
2241
2242 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
2243
2244 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, SZ_8K);
2245 return 0;
2246 }
2247 subsys_initcall(init_admin_reserve);
2248
2249 /*
2250 * Reinititalise user and admin reserves if memory is added or removed.
2251 *
2252 * The default user reserve max is 128MB, and the default max for the
2253 * admin reserve is 8MB. These are usually, but not always, enough to
2254 * enable recovery from a memory hogging process using login/sshd, a shell,
2255 * and tools like top. It may make sense to increase or even disable the
2256 * reserve depending on the existence of swap or variations in the recovery
2257 * tools. So, the admin may have changed them.
2258 *
2259 * If memory is added and the reserves have been eliminated or increased above
2260 * the default max, then we'll trust the admin.
2261 *
2262 * If memory is removed and there isn't enough free memory, then we
2263 * need to reset the reserves.
2264 *
2265 * Otherwise keep the reserve set by the admin.
2266 */
reserve_mem_notifier(struct notifier_block * nb,unsigned long action,void * data)2267 static int reserve_mem_notifier(struct notifier_block *nb,
2268 unsigned long action, void *data)
2269 {
2270 unsigned long tmp, free_kbytes;
2271
2272 switch (action) {
2273 case MEM_ONLINE:
2274 /* Default max is 128MB. Leave alone if modified by operator. */
2275 tmp = sysctl_user_reserve_kbytes;
2276 if (tmp > 0 && tmp < SZ_128K)
2277 init_user_reserve();
2278
2279 /* Default max is 8MB. Leave alone if modified by operator. */
2280 tmp = sysctl_admin_reserve_kbytes;
2281 if (tmp > 0 && tmp < SZ_8K)
2282 init_admin_reserve();
2283
2284 break;
2285 case MEM_OFFLINE:
2286 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
2287
2288 if (sysctl_user_reserve_kbytes > free_kbytes) {
2289 init_user_reserve();
2290 pr_info("vm.user_reserve_kbytes reset to %lu\n",
2291 sysctl_user_reserve_kbytes);
2292 }
2293
2294 if (sysctl_admin_reserve_kbytes > free_kbytes) {
2295 init_admin_reserve();
2296 pr_info("vm.admin_reserve_kbytes reset to %lu\n",
2297 sysctl_admin_reserve_kbytes);
2298 }
2299 break;
2300 default:
2301 break;
2302 }
2303 return NOTIFY_OK;
2304 }
2305
init_reserve_notifier(void)2306 static int __meminit init_reserve_notifier(void)
2307 {
2308 if (hotplug_memory_notifier(reserve_mem_notifier, DEFAULT_CALLBACK_PRI))
2309 pr_err("Failed registering memory add/remove notifier for admin reserve\n");
2310
2311 return 0;
2312 }
2313 subsys_initcall(init_reserve_notifier);
2314
2315 /*
2316 * Relocate a VMA downwards by shift bytes. There cannot be any VMAs between
2317 * this VMA and its relocated range, which will now reside at [vma->vm_start -
2318 * shift, vma->vm_end - shift).
2319 *
2320 * This function is almost certainly NOT what you want for anything other than
2321 * early executable temporary stack relocation.
2322 */
relocate_vma_down(struct vm_area_struct * vma,unsigned long shift)2323 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift)
2324 {
2325 /*
2326 * The process proceeds as follows:
2327 *
2328 * 1) Use shift to calculate the new vma endpoints.
2329 * 2) Extend vma to cover both the old and new ranges. This ensures the
2330 * arguments passed to subsequent functions are consistent.
2331 * 3) Move vma's page tables to the new range.
2332 * 4) Free up any cleared pgd range.
2333 * 5) Shrink the vma to cover only the new range.
2334 */
2335
2336 struct mm_struct *mm = vma->vm_mm;
2337 unsigned long old_start = vma->vm_start;
2338 unsigned long old_end = vma->vm_end;
2339 unsigned long length = old_end - old_start;
2340 unsigned long new_start = old_start - shift;
2341 unsigned long new_end = old_end - shift;
2342 VMA_ITERATOR(vmi, mm, new_start);
2343 VMG_STATE(vmg, mm, &vmi, new_start, old_end, 0, vma->vm_pgoff);
2344 struct vm_area_struct *next;
2345 struct mmu_gather tlb;
2346
2347 BUG_ON(new_start > new_end);
2348
2349 /*
2350 * ensure there are no vmas between where we want to go
2351 * and where we are
2352 */
2353 if (vma != vma_next(&vmi))
2354 return -EFAULT;
2355
2356 vma_iter_prev_range(&vmi);
2357 /*
2358 * cover the whole range: [new_start, old_end)
2359 */
2360 vmg.vma = vma;
2361 if (vma_expand(&vmg))
2362 return -ENOMEM;
2363
2364 /*
2365 * move the page tables downwards, on failure we rely on
2366 * process cleanup to remove whatever mess we made.
2367 */
2368 if (length != move_page_tables(vma, old_start,
2369 vma, new_start, length, false, true))
2370 return -ENOMEM;
2371
2372 lru_add_drain();
2373 tlb_gather_mmu(&tlb, mm);
2374 next = vma_next(&vmi);
2375 if (new_end > old_start) {
2376 /*
2377 * when the old and new regions overlap clear from new_end.
2378 */
2379 free_pgd_range(&tlb, new_end, old_end, new_end,
2380 next ? next->vm_start : USER_PGTABLES_CEILING);
2381 } else {
2382 /*
2383 * otherwise, clean from old_start; this is done to not touch
2384 * the address space in [new_end, old_start) some architectures
2385 * have constraints on va-space that make this illegal (IA64) -
2386 * for the others its just a little faster.
2387 */
2388 free_pgd_range(&tlb, old_start, old_end, new_end,
2389 next ? next->vm_start : USER_PGTABLES_CEILING);
2390 }
2391 tlb_finish_mmu(&tlb);
2392
2393 vma_prev(&vmi);
2394 /* Shrink the vma to just the new range */
2395 return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff);
2396 }
2397