27 							unsigned long addr,  in hugetlb_get_unmapped_area_bottomup()  argument
43 	addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_bottomup()
45 	if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {  in hugetlb_get_unmapped_area_bottomup()
46 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_bottomup()
49 		addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_bottomup()
52 	return addr;  in hugetlb_get_unmapped_area_bottomup()
63 	unsigned long addr = addr0;  in hugetlb_get_unmapped_area_topdown()  local
74 	addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_topdown()
82 	if (addr & ~PAGE_MASK) {  in hugetlb_get_unmapped_area_topdown()
83 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_topdown()
87 		addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_topdown()
90 	return addr;  in hugetlb_get_unmapped_area_topdown()
94 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,  in hugetlb_get_unmapped_area()  argument
111 		if (prepare_hugepage_range(file, addr, len))  in hugetlb_get_unmapped_area()
113 		return addr;  in hugetlb_get_unmapped_area()
116 	if (addr) {  in hugetlb_get_unmapped_area()
117 		addr = ALIGN(addr, huge_page_size(h));  in hugetlb_get_unmapped_area()
118 		vma = find_vma(mm, addr);  in hugetlb_get_unmapped_area()
119 		if (task_size - len >= addr &&  in hugetlb_get_unmapped_area()
120 		    (!vma || addr + len <= vm_start_gap(vma)))  in hugetlb_get_unmapped_area()
121 			return addr;  in hugetlb_get_unmapped_area()
124 		return hugetlb_get_unmapped_area_bottomup(file, addr, len,  in hugetlb_get_unmapped_area()
127 		return hugetlb_get_unmapped_area_topdown(file, addr, len,  in hugetlb_get_unmapped_area()
279 			unsigned long addr, unsigned long sz)  in huge_pte_alloc()  argument
286 	pgd = pgd_offset(mm, addr);  in huge_pte_alloc()
287 	p4d = p4d_offset(pgd, addr);  in huge_pte_alloc()
288 	pud = pud_alloc(mm, p4d, addr);  in huge_pte_alloc()
293 	pmd = pmd_alloc(mm, pud, addr);  in huge_pte_alloc()
298 	return pte_alloc_huge(mm, pmd, addr);  in huge_pte_alloc()
302 		       unsigned long addr, unsigned long sz)  in huge_pte_offset()  argument
309 	pgd = pgd_offset(mm, addr);  in huge_pte_offset()
312 	p4d = p4d_offset(pgd, addr);  in huge_pte_offset()
315 	pud = pud_offset(p4d, addr);  in huge_pte_offset()
320 	pmd = pmd_offset(pud, addr);  in huge_pte_offset()
325 	return pte_offset_huge(pmd, addr);  in huge_pte_offset()
328 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,  in __set_huge_pte_at()  argument
350 	addr &= ~(size - 1);  in __set_huge_pte_at()
357 	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);  in __set_huge_pte_at()
360 		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,  in __set_huge_pte_at()
364 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,  in set_huge_pte_at()  argument
367 	__set_huge_pte_at(mm, addr, ptep, entry);  in set_huge_pte_at()
370 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,  in huge_ptep_get_and_clear()  argument
394 	addr &= ~(size - 1);  in huge_ptep_get_and_clear()
398 	maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);  in huge_ptep_get_and_clear()
401 		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,  in huge_ptep_get_and_clear()
408 			   unsigned long addr)  in hugetlb_free_pte_range()  argument
413 	pte_free_tlb(tlb, token, addr);  in hugetlb_free_pte_range()
418 				   unsigned long addr, unsigned long end,  in hugetlb_free_pmd_range()  argument
425 	start = addr;  in hugetlb_free_pmd_range()
426 	pmd = pmd_offset(pud, addr);  in hugetlb_free_pmd_range()
428 		next = pmd_addr_end(addr, end);  in hugetlb_free_pmd_range()
434 			hugetlb_free_pte_range(tlb, pmd, addr);  in hugetlb_free_pmd_range()
435 	} while (pmd++, addr = next, addr != end);  in hugetlb_free_pmd_range()
455 				   unsigned long addr, unsigned long end,  in hugetlb_free_pud_range()  argument
462 	start = addr;  in hugetlb_free_pud_range()
463 	pud = pud_offset(p4d, addr);  in hugetlb_free_pud_range()
465 		next = pud_addr_end(addr, end);  in hugetlb_free_pud_range()
471 			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,  in hugetlb_free_pud_range()
473 	} while (pud++, addr = next, addr != end);  in hugetlb_free_pud_range()
493 			    unsigned long addr, unsigned long end,  in hugetlb_free_pgd_range()  argument
500 	addr &= PMD_MASK;  in hugetlb_free_pgd_range()
501 	if (addr < floor) {  in hugetlb_free_pgd_range()
502 		addr += PMD_SIZE;  in hugetlb_free_pgd_range()
503 		if (!addr)  in hugetlb_free_pgd_range()
513 	if (addr > end - 1)  in hugetlb_free_pgd_range()
516 	pgd = pgd_offset(tlb->mm, addr);  in hugetlb_free_pgd_range()
517 	p4d = p4d_offset(pgd, addr);  in hugetlb_free_pgd_range()
519 		next = p4d_addr_end(addr, end);  in hugetlb_free_pgd_range()
522 		hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);  in hugetlb_free_pgd_range()
523 	} while (p4d++, addr = next, addr != end);  in hugetlb_free_pgd_range()