53 u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena)  in bpf_arena_get_kern_vm_start()  argument
55 	return arena ? (u64) (long) arena->kern_vm->addr + GUARD_SZ / 2 : 0;  in bpf_arena_get_kern_vm_start()
58 u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)  in bpf_arena_get_user_vm_start()  argument
60 	return arena ? arena->user_vm_start : 0;  in bpf_arena_get_user_vm_start()
88 static long compute_pgoff(struct bpf_arena *arena, long uaddr)  in compute_pgoff()  argument
90 	return (u32)(uaddr - (u32)arena->user_vm_start) >> PAGE_SHIFT;  in compute_pgoff()
97 	struct bpf_arena *arena;  in arena_map_alloc()  local
124 	arena = bpf_map_area_alloc(sizeof(*arena), numa_node);  in arena_map_alloc()
125 	if (!arena)  in arena_map_alloc()
128 	arena->kern_vm = kern_vm;  in arena_map_alloc()
129 	arena->user_vm_start = attr->map_extra;  in arena_map_alloc()
130 	if (arena->user_vm_start)  in arena_map_alloc()
131 		arena->user_vm_end = arena->user_vm_start + vm_range;  in arena_map_alloc()
133 	INIT_LIST_HEAD(&arena->vma_list);  in arena_map_alloc()
134 	bpf_map_init_from_attr(&arena->map, attr);  in arena_map_alloc()
135 	mt_init_flags(&arena->mt, MT_FLAGS_ALLOC_RANGE);  in arena_map_alloc()
136 	mutex_init(&arena->lock);  in arena_map_alloc()
138 	return &arena->map;  in arena_map_alloc()
166 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in arena_map_free()  local
174 	if (WARN_ON_ONCE(!list_empty(&arena->vma_list)))  in arena_map_free()
183 	apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena),  in arena_map_free()
185 	free_vm_area(arena->kern_vm);  in arena_map_free()
186 	mtree_destroy(&arena->mt);  in arena_map_free()
187 	bpf_map_area_free(arena);  in arena_map_free()
218 static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma)  in remember_vma()  argument
228 	list_add(&vml->head, &arena->vma_list);  in remember_vma()
242 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in arena_vm_close()  local
247 	guard(mutex)(&arena->lock);  in arena_vm_close()
259 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in arena_vm_fault()  local
264 	kbase = bpf_arena_get_kern_vm_start(arena);  in arena_vm_fault()
267 	guard(mutex)(&arena->lock);  in arena_vm_fault()
273 	if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT)  in arena_vm_fault()
277 	ret = mtree_insert(&arena->mt, vmf->pgoff, MT_ENTRY, GFP_KERNEL);  in arena_vm_fault()
284 		mtree_erase(&arena->mt, vmf->pgoff);  in arena_vm_fault()
288 	ret = vm_area_map_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE, &page);  in arena_vm_fault()
290 		mtree_erase(&arena->mt, vmf->pgoff);  in arena_vm_fault()
311 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in arena_get_unmapped_area()  local
320 	if (arena->user_vm_start) {  in arena_get_unmapped_area()
321 		if (len > arena->user_vm_end - arena->user_vm_start)  in arena_get_unmapped_area()
323 		if (len != arena->user_vm_end - arena->user_vm_start)  in arena_get_unmapped_area()
325 		if (addr != arena->user_vm_start)  in arena_get_unmapped_area()
334 	if (WARN_ON_ONCE(arena->user_vm_start))  in arena_get_unmapped_area()
342 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in arena_map_mmap()  local
344 	guard(mutex)(&arena->lock);  in arena_map_mmap()
345 	if (arena->user_vm_start && arena->user_vm_start != vma->vm_start)  in arena_map_mmap()
356 	if (arena->user_vm_end && arena->user_vm_end != vma->vm_end)  in arena_map_mmap()
364 	if (remember_vma(arena, vma))  in arena_map_mmap()
367 	arena->user_vm_start = vma->vm_start;  in arena_map_mmap()
368 	arena->user_vm_end = vma->vm_end;  in arena_map_mmap()
381 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in arena_map_direct_value_addr()  local
383 	if ((u64)off > arena->user_vm_end - arena->user_vm_start)  in arena_map_direct_value_addr()
385 	*imm = (unsigned long)arena->user_vm_start;  in arena_map_direct_value_addr()
418 static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt, int node_id)  in arena_alloc_pages()  argument
421 	long page_cnt_max = (arena->user_vm_end - arena->user_vm_start) >> PAGE_SHIFT;  in arena_alloc_pages()
422 	u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena);  in arena_alloc_pages()
434 		pgoff = compute_pgoff(arena, uaddr);  in arena_alloc_pages()
445 	guard(mutex)(&arena->lock);  in arena_alloc_pages()
448 		ret = mtree_insert_range(&arena->mt, pgoff, pgoff + page_cnt - 1,  in arena_alloc_pages()
451 		ret = mtree_alloc_range(&arena->mt, &pgoff, MT_ENTRY,  in arena_alloc_pages()
456 	ret = bpf_map_alloc_pages(&arena->map, GFP_KERNEL | __GFP_ZERO,  in arena_alloc_pages()
461 	uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE);  in arena_alloc_pages()
469 	ret = vm_area_map_pages(arena->kern_vm, kern_vm_start + uaddr32,  in arena_alloc_pages()
477 	return clear_lo32(arena->user_vm_start) + uaddr32;  in arena_alloc_pages()
479 	mtree_erase(&arena->mt, pgoff);  in arena_alloc_pages()
490 static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)  in zap_pages()  argument
494 	list_for_each_entry(vml, &arena->vma_list, head)  in zap_pages()
499 static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt)  in arena_free_pages()  argument
508 	full_uaddr = clear_lo32(arena->user_vm_start) + uaddr;  in arena_free_pages()
509 	uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT));  in arena_free_pages()
515 	guard(mutex)(&arena->lock);  in arena_free_pages()
517 	pgoff = compute_pgoff(arena, uaddr);  in arena_free_pages()
519 	mtree_store_range(&arena->mt, pgoff, pgoff + page_cnt - 1, NULL, GFP_KERNEL);  in arena_free_pages()
523 		zap_pages(arena, full_uaddr, page_cnt);  in arena_free_pages()
525 	kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr;  in arena_free_pages()
536 			zap_pages(arena, full_uaddr, 1);  in arena_free_pages()
537 		vm_area_unmap_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE);  in arena_free_pages()
548 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in bpf_arena_alloc_pages()  local
553 	return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id);  in bpf_arena_alloc_pages()
559 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);  in bpf_arena_free_pages()  local
563 	arena_free_pages(arena, (long)ptr__ign, page_cnt);  in bpf_arena_free_pages()